From 452f17b75181d62d1d92016edecd38e1d70fecab Mon Sep 17 00:00:00 2001 From: james Date: Tue, 10 Nov 2026 19:15:39 +0000 Subject: [PATCH] chore(import): snapshot james.dad.town/core at f70c71c7696a --- .air/deliberi.toml | 10 + .air/spindle.toml | 10 + .cargo/config.toml | 3 + .config/nextest.toml | 4 + .dockerignore | 49 + .editorconfig | 29 + .gitattributes | 7 + .gitignore | 50 + .prettierrc.json | 12 + .tangled/hooks/commit-msg | 14 + .tangled/hooks/commit-msg-lint.sh | 262 + .tangled/hooks/install.sh | 29 + .tangled/workflows/blog.yml | 28 + .tangled/workflows/cargo-nix.yml | 21 + .tangled/workflows/commit-lint.yml | 40 + .tangled/workflows/deploy-blog.yml | 27 + .tangled/workflows/deploy-docs.yml | 24 + .tangled/workflows/deploy-gcp-dev.yml | 122 + .tangled/workflows/fmt.yml | 13 + .tangled/workflows/preview-web-cleanup.yml | 19 + .tangled/workflows/preview-web.yml | 27 + .tangled/workflows/rust-checks.yml | 58 + .tangled/workflows/rust-test-bobbin.yml | 57 + .tangled/workflows/rust-test-knot.yml | 83 + .tangled/workflows/rust-test-knotmirror.yml | 66 + .tangled/workflows/test.yml | 40 + .tangled/workflows/web.yml | 67 + Cargo.lock | 11223 ++++ Cargo.toml | 308 + _typos.toml | 23 + api/org_tangled/actordefs.go | 39 + api/org_tangled/actorgetProfile.go | 38 + api/org_tangled/actorgetProfileActivity.go | 82 + api/org_tangled/actorgetProfiles.go | 43 + api/org_tangled/actorgetTrending.go | 31 + api/org_tangled/actorprofile.go | 39 + api/org_tangled/actorsearchActorsTypeahead.go | 46 + api/org_tangled/bobbinawaitRecord.go | 46 + api/org_tangled/cbor_gen.go | 10326 ++++ api/org_tangled/ciapprovePipeline.go | 32 + api/org_tangled/cicancelPipeline.go | 34 + api/org_tangled/cidefs.go | 182 + api/org_tangled/cidefs_ext.go | 10 + api/org_tangled/cidefs_test.go | 71 + api/org_tangled/cigetPipeline.go | 32 + api/org_tangled/cigetWorkflowDefinition.go | 34 + api/org_tangled/cilistWorkflowDefinitions.go | 36 + api/org_tangled/ciqueryPipelines.go | 54 + api/org_tangled/ciretryPipeline.go | 41 + api/org_tangled/cisubscribePipelineLogs.go | 38 + .../cisubscribePipelineLogs_ext.go | 99 + .../cisubscribePipelineLogs_ext_test.go | 39 + api/org_tangled/citriggermanual.go | 35 + api/org_tangled/citriggerschedule.go | 30 + api/org_tangled/embedcommit.go | 33 + api/org_tangled/embedunspeccedfile.go | 16 + api/org_tangled/eventnotifyEvent.go | 84 + api/org_tangled/eventpullRequest.go | 33 + api/org_tangled/eventpush.go | 31 + api/org_tangled/feedcomment.go | 195 + api/org_tangled/feedcountComments.go | 38 + api/org_tangled/feedcountCommentsBy.go | 38 + api/org_tangled/feedcountReactions.go | 38 + api/org_tangled/feedcountReactionsBy.go | 38 + api/org_tangled/feedcountStars.go | 38 + api/org_tangled/feedcountStarsBy.go | 38 + api/org_tangled/feeddefs.go | 82 + api/org_tangled/feedgetStar.go | 38 + api/org_tangled/feedgetTimeline.go | 140 + api/org_tangled/feedlistComments.go | 63 + api/org_tangled/feedlistCommentsBy.go | 53 + api/org_tangled/feedlistReactions.go | 61 + api/org_tangled/feedlistReactionsBy.go | 53 + api/org_tangled/feedlistStars.go | 61 + api/org_tangled/feedlistStarsBy.go | 53 + api/org_tangled/feedreaction.go | 25 + api/org_tangled/feedstar.go | 114 + api/org_tangled/feedsubscription.go | 115 + api/org_tangled/gitdefs.go | 92 + api/org_tangled/gitdeleteBranch.go | 31 + api/org_tangled/gitgetArchive.go | 39 + api/org_tangled/gitgetBlame.go | 71 + api/org_tangled/gitgetBlob.go | 37 + api/org_tangled/gitgetBranch.go | 45 + api/org_tangled/gitgetDiff.go | 44 + api/org_tangled/gitgetEntry.go | 49 + api/org_tangled/gitgetTag.go | 45 + api/org_tangled/gitgetTree.go | 55 + api/org_tangled/gitlistBranches.go | 60 + api/org_tangled/gitlistCommits.go | 51 + api/org_tangled/gitlistLanguages.go | 61 + api/org_tangled/gitlistRefs.go | 62 + api/org_tangled/gitlistTags.go | 66 + api/org_tangled/gitoid.go | 22 + api/org_tangled/gitpushEvent.go | 44 + api/org_tangled/gitreceivePack.go | 24 + api/org_tangled/gitrefs.go | 25 + api/org_tangled/gitsetDefaultBranch.go | 41 + api/org_tangled/graphattestation.go | 26 + api/org_tangled/graphcollaborator.go | 22 + api/org_tangled/graphcountFollows.go | 38 + api/org_tangled/graphcountFollowsBy.go | 38 + api/org_tangled/graphfollow.go | 22 + api/org_tangled/graphgetFollow.go | 38 + api/org_tangled/graphlistFollows.go | 63 + api/org_tangled/graphlistFollowsBy.go | 53 + api/org_tangled/graphlistNetworkVouches.go | 54 + api/org_tangled/graphvouch.go | 29 + api/org_tangled/keysshKey.go | 28 + api/org_tangled/keyunspeccedgpgKey.go | 26 + api/org_tangled/knotblock.go | 22 + api/org_tangled/knotdescribeKnot.go | 48 + api/org_tangled/knotmember.go | 22 + api/org_tangled/labelcountDefinitions.go | 38 + api/org_tangled/labeldefinition.go | 26 + api/org_tangled/labeldefs.go | 16 + api/org_tangled/labellistDefinitions.go | 61 + api/org_tangled/markupembedimage.go | 22 + api/org_tangled/markupembedvideo.go | 22 + api/org_tangled/markupmarkdown.go | 112 + api/org_tangled/markupplaintext.go | 14 + api/org_tangled/publicKeycountKeys.go | 38 + api/org_tangled/publicKeygetPublicKey.go | 38 + api/org_tangled/publicKeylistKeys.go | 61 + api/org_tangled/queryenrichResponse.go | 89 + api/org_tangled/repoartifact.go | 88 + api/org_tangled/repocountArtifacts.go | 38 + api/org_tangled/repocountArtifactsBy.go | 38 + api/org_tangled/repocountCollaborators.go | 38 + api/org_tangled/repocountCollaboratorsBy.go | 38 + api/org_tangled/repocountForks.go | 38 + api/org_tangled/repocountIssues.go | 46 + api/org_tangled/repocountPulls.go | 46 + api/org_tangled/repocountRepos.go | 38 + api/org_tangled/repocreateRepo.go | 89 + api/org_tangled/repodeclaration.go | 24 + api/org_tangled/repodefs.go | 57 + api/org_tangled/repodeleteRepo.go | 31 + api/org_tangled/repoforkSync.go | 32 + api/org_tangled/repogetImportStatus.go | 38 + api/org_tangled/repogetRepo.go | 40 + api/org_tangled/repogetRepoByName.go | 42 + api/org_tangled/repogetRepoByRepoDid.go | 40 + api/org_tangled/repogetRepos.go | 43 + api/org_tangled/repogetReposByRepoDids.go | 43 + api/org_tangled/repolistArtifacts.go | 61 + api/org_tangled/repolistArtifactsBy.go | 53 + .../repolistCollaboratorInvitesBy.go | 55 + api/org_tangled/repolistCollaborators.go | 71 + api/org_tangled/repolistCollaboratorsBy.go | 53 + api/org_tangled/repolistForks.go | 61 + api/org_tangled/repolistIssues.go | 77 + api/org_tangled/repolistIssuesBy.go | 57 + api/org_tangled/repolistPulls.go | 77 + api/org_tangled/repolistPullsBy.go | 57 + api/org_tangled/repolistRecordVersions.go | 61 + api/org_tangled/repolistRepos.go | 61 + api/org_tangled/repomanifest.go | 115 + api/org_tangled/repomintToken.go | 43 + api/org_tangled/reporeserveKey.go | 38 + api/org_tangled/reporeviewunspeccedround.go | 31 + api/org_tangled/reviewpatch.go | 33 + api/org_tangled/reviewsubmitPatch.go | 51 + api/org_tangled/reviewunspecceddefs.go | 7 + api/org_tangled/searchquery.go | 110 + api/org_tangled/secretaddSecret.go | 31 + api/org_tangled/secretlistSecrets.go | 40 + api/org_tangled/secretremoveSecret.go | 30 + api/org_tangled/snippetstring.go | 94 + api/org_tangled/spindledescribeSpindle.go | 48 + api/org_tangled/stringcountStrings.go | 38 + api/org_tangled/stringgetString.go | 38 + api/org_tangled/stringlistStrings.go | 61 + api/org_tangled/tempaccountaddEmail.go | 30 + api/org_tangled/tempaccountbeginSignup.go | 32 + api/org_tangled/tempaccountcompleteSignup.go | 56 + api/org_tangled/tempaccountdeleteEmail.go | 30 + .../tempaccountdismissNewsletter.go | 24 + .../tempaccountgetNewsletterPreference.go | 30 + api/org_tangled/tempaccountgetSignup.go | 38 + api/org_tangled/tempaccountlistEmails.go | 41 + api/org_tangled/tempaccountresendSignup.go | 30 + api/org_tangled/tempaccountsetPrimaryEmail.go | 30 + .../tempaccountsubscribeNewsletter.go | 24 + api/org_tangled/tempaccountverifyEmail.go | 37 + api/org_tangled/tempactordefs.go | 122 + api/org_tangled/tempfocusbeginSession.go | 37 + api/org_tangled/tempfocusendSession.go | 24 + api/org_tangled/tempfocusnextItem.go | 43 + api/org_tangled/tempgitdeleteBranch.go | 31 + api/org_tangled/tempgitformatPatch.go | 35 + api/org_tangled/tempgitgetArchive.go | 39 + api/org_tangled/tempgitgetBlob.go | 41 + api/org_tangled/tempgitgetBranch.go | 45 + api/org_tangled/tempgitgetEntry.go | 49 + api/org_tangled/tempgitgetMergeBase.go | 39 + api/org_tangled/tempgitgetRef.go | 47 + api/org_tangled/tempgitgetTag.go | 45 + api/org_tangled/tempgitgetTree.go | 55 + api/org_tangled/tempgitlistBranches.go | 60 + api/org_tangled/tempgitlistCommits.go | 110 + api/org_tangled/tempgitlistCommitsByRef.go | 50 + api/org_tangled/tempgitlistTags.go | 70 + api/org_tangled/tempgitmergeCommit.go | 59 + api/org_tangled/tempgitsyncFork.go | 33 + api/org_tangled/tempmigratorcreateTask.go | 34 + api/org_tangled/tempmigratordefs.go | 55 + api/org_tangled/tempmigratordescribeSource.go | 38 + api/org_tangled/tempmigratorgetTask.go | 30 + api/org_tangled/tempmigratorlistTasks.go | 37 + api/org_tangled/tempmigratorretryJob.go | 35 + api/org_tangled/tempnewslettersignup.go | 30 + .../tempnotificationdeleteNotification.go | 30 + .../tempnotificationgetPreferences.go | 39 + .../tempnotificationgetUnreadCount.go | 30 + .../tempnotificationlistNotifications.go | 71 + .../tempnotificationlistRecipients.go | 39 + .../tempnotificationmarkAllRead.go | 24 + .../tempnotificationmarkEntityRead.go | 30 + .../tempnotificationupdatePreferences.go | 38 + api/org_tangled/tempnotificationupdateSeen.go | 31 + api/org_tangled/temprepodeploySite.go | 30 + api/org_tangled/temprepodisableSite.go | 30 + api/org_tangled/temprepogetDeployHistory.go | 54 + api/org_tangled/temprepogetRepoByName.go | 53 + api/org_tangled/temprepogetSiteConfig.go | 46 + api/org_tangled/temprepolistRecordOrigins.go | 59 + api/org_tangled/temprepoupdateSiteConfig.go | 42 + api/org_tangled/tempreviewlistPatches.go | 61 + api/org_tangled/tempsearchsearchCode.go | 79 + api/org_tangled/tempserverackHandover.go | 33 + api/org_tangled/tempservercancelHandover.go | 35 + api/org_tangled/tempservercreateInviteCode.go | 36 + api/org_tangled/tempserverhandoverRepo.go | 37 + api/org_tangled/tempserverredeemInviteCode.go | 29 + api/org_tangled/tempserverseedRepo.go | 63 + api/org_tangled/tempserveruploadBlob.go | 36 + api/org_tangled/tempsiteclaimDomain.go | 30 + api/org_tangled/tempsitegetDomainClaim.go | 31 + api/org_tangled/tempsitereleaseDomain.go | 30 + api/org_tangled/tempspindleallowListadd.go | 30 + api/org_tangled/tempspindleallowListlist.go | 31 + api/org_tangled/tempspindleallowListremove.go | 30 + api/org_tangled/tempspindlemoderationban.go | 30 + api/org_tangled/tempspindlemoderationdefs.go | 17 + .../tempspindlemoderationgetBan.go | 36 + .../tempspindlemoderationlistBans.go | 31 + api/org_tangled/tempspindlemoderationunban.go | 30 + api/org_tangled/tempspindlequotadefs.go | 33 + api/org_tangled/tempspindlequotaget.go | 38 + api/org_tangled/tempspindlequotalist.go | 31 + .../tempspindlequotareleaseNars.go | 41 + api/org_tangled/tempspindlequotaset.go | 36 + api/org_tangled/tempspindlequotaunset.go | 32 + api/org_tangled/tempspindlequotausage.go | 42 + api/org_tangled/trackdefs.go | 112 + api/org_tangled/trackgetTicketView.go | 28 + api/org_tangled/trackticket.go | 200 + api/org_tangled/trackunspeccedmilestone.go | 21 + api/org_tangled/webhookcreateWebhook.go | 45 + api/org_tangled/webhookdefs.go | 41 + api/org_tangled/webhookdeleteWebhook.go | 32 + .../webhooklistWebhookDeliveries.go | 39 + api/org_tangled/webhooklistWebhooks.go | 35 + .../webhookretryWebhookDelivery.go | 34 + api/org_tangled/webhookupdateWebhook.go | 36 + api/tangled/gitdefs.go | 71 + api/tangled/gitmergeCommit.go | 59 + api/tangled/gittemp2getBlame.go | 62 + api/tangled/gittemp2getCommitStats.go | 46 + api/tangled/gittemp2getDiff.go | 36 + api/tangled/gittemp2getInterdiff.go | 38 + api/tangled/gittemp2mergeCheck.go | 51 + api/tangled/gittemplistIndexedLanguages.go | 52 + api/tangled/gittemplistLanguages.go | 55 + api/tangled/identityresolveCommitters.go | 35 + api/tangled/repolistCobVersions.go | 63 + avatar/.gitignore | 172 + avatar/package.json | 22 + avatar/playground.html | 316 + avatar/pnpm-lock.yaml | 2711 + avatar/pnpm-workspace.yaml | 4 + avatar/readme.md | 31 + avatar/src/gradient.gold.json | 1 + avatar/src/gradient.js | 28 + avatar/src/gradient.test.js | 9 + avatar/src/index.js | 290 + avatar/src/index.test.js | 157 + avatar/src/size.js | 6 + avatar/wrangler.dev.jsonc | 15 + avatar/wrangler.jsonc | 15 + blog/.gitignore | 6 + blog/astro.config.mjs | 73 + blog/blog.go | 97 + blog/lychee.toml | 14 + blog/package.json | 30 + blog/pnpm-lock.yaml | 3742 ++ blog/pnpm-workspace.yaml | 3 + blog/posts/6-months.md | 161 + blog/posts/bobbin.md | 202 + blog/posts/ci.md | 216 + blog/posts/docs.md | 240 + blog/posts/federation.md | 53 + blog/posts/forge.mdx | 164 + blog/posts/intro.md | 66 + blog/posts/newsletter-01.md | 91 + blog/posts/newsletter-02.md | 115 + blog/posts/newsletter-03.md | 83 + blog/posts/pty.md | 58 + blog/posts/pulls.md | 196 + blog/posts/seed.md | 74 + blog/posts/selfhost-ci.md | 38 + blog/posts/spindle-microvm.md | 377 + blog/posts/stacking.md | 353 + blog/posts/vouching.md | 91 + blog/public/avatars/anirudh.fi.jpg | Bin 0 -> 2664 bytes blog/public/avatars/oppi.li.jpg | Bin 0 -> 5760 bytes blog/public/avatars/oyster.cafe.jpg | Bin 0 -> 4364 bytes blog/public/avatars/ptr.pet.jpg | Bin 0 -> 4676 bytes blog/public/favicon.svg | 21 + blog/scripts/check-links.sh | 31 + blog/src/components/FeedButton.svelte | 16 + blog/src/components/Footer.astro | 43 + blog/src/components/ForgeTimeline.svelte | 205 + blog/src/components/MarkupLightbox.svelte | 17 + blog/src/components/Ripple.astro | 57 + blog/src/components/SignupBanner.svelte | 34 + .../embeds/OrganizationsEmbed.svelte | 132 + .../components/embeds/PullStackEmbed.svelte | 153 + blog/src/components/embeds/Reveal.svelte | 67 + .../src/components/embeds/TicketsEmbed.svelte | 175 + .../components/embeds/TimelineEmbed.svelte | 274 + blog/src/content.config.ts | 39 + blog/src/dashes.mjs | 14 + blog/src/layouts/Base.astro | 85 + blog/src/lib/posts.ts | 89 + .../.well-known/site.standard.publication.ts | 6 + blog/src/pages/404.astro | 11 + blog/src/pages/[slug].astro | 64 + blog/src/pages/[slug].md.ts | 15 + blog/src/pages/feed.json.ts | 35 + blog/src/pages/feed.xml.ts | 42 + blog/src/pages/index.astro | 136 + blog/src/pages/llms-full.txt.ts | 9 + blog/src/pages/llms.txt.ts | 30 + blog/src/stubs/app-env.ts | 6 + blog/src/stubs/app-paths.ts | 8 + blog/src/styles/global.css | 130 + blog/src/tangled.theme.mjs | 58 + blog/standard.go | 162 + blog/standard_test.go | 151 + blog/svelte.config.js | 5 + blog/tsconfig.json | 14 + bobbin/containerfiles/bobbin.Containerfile | 77 + bobbin/containerfiles/hydrant.Containerfile | 29 + bobbin/containerfiles/slingshot.Containerfile | 22 + bobbin/crates/bobbin-sim/Cargo.toml | 39 + bobbin/crates/bobbin-sim/src/determinism.rs | 249 + bobbin/crates/bobbin-sim/src/lib.rs | 13 + bobbin/crates/bobbin-sim/src/main.rs | 382 + bobbin/crates/bobbin-sim/src/report.rs | 37 + bobbin/crates/bobbin-sim/src/runtime.rs | 201 + bobbin/crates/bobbin-sim/src/trace_capture.rs | 98 + bobbin/crates/bobbin-sim/src/workload.rs | 35 + .../src/workloads/cancel_mid_hydration.rs | 291 + .../workloads/cold_start_under_live_load.rs | 235 + .../concurrent_reads_during_replay.rs | 274 + .../bobbin-sim/src/workloads/frame_burst.rs | 188 + .../workloads/hydrant_disconnect_barrage.rs | 255 + bobbin/crates/bobbin-sim/src/workloads/mod.rs | 16 + .../src/workloads/slingshot_flap.rs | 420 + .../crates/bobbin-sim/src/workloads/util.rs | 80 + .../bobbin-sim/tests/determinism_leak.rs | 236 + .../bobbin-sim/tests/reconnect_replay.rs | 71 + .../crates/bobbin-sim/tests/warming_buffer.rs | 248 + .../crates/bobbin-sim/tests/warming_shadow.rs | 68 + bobbin/crates/bobbin/Cargo.toml | 53 + bobbin/crates/bobbin/src/config.rs | 809 + bobbin/crates/bobbin/src/main.rs | 750 + bobbin/crates/bobbin/src/mem/cgroup.rs | 181 + bobbin/crates/bobbin/src/mem/mod.rs | 9 + bobbin/crates/bobbin/src/mem/report.rs | 195 + bobbin/crates/bobbin/src/mem/sizing.rs | 100 + bobbin/crates/bobbin/src/mem/watcher.rs | 103 + bobbin/crates/bobbin/src/metrics.rs | 297 + bobbin/crates/bobbin/src/metrics/process.rs | 140 + bobbin/crates/bobbin/src/otlp.rs | 249 + bobbin/crates/codesearch/Cargo.toml | 18 + bobbin/crates/codesearch/src/lib.rs | 308 + bobbin/crates/edge-index/Cargo.toml | 33 + .../edge-index/benches/distinct_authors.rs | 154 + .../edge-index/benches/profile_activity.rs | 136 + bobbin/crates/edge-index/src/avatars.rs | 141 + bobbin/crates/edge-index/src/claims.rs | 344 + bobbin/crates/edge-index/src/coverage.rs | 267 + bobbin/crates/edge-index/src/invites.rs | 204 + bobbin/crates/edge-index/src/labels.rs | 439 + bobbin/crates/edge-index/src/lib.rs | 3566 ++ bobbin/crates/edge-index/src/reactions.rs | 184 + bobbin/crates/edge-index/src/settle.rs | 548 + bobbin/crates/edge-index/src/sorted_blocks.rs | 483 + bobbin/crates/edge-index/src/state_index.rs | 33 + bobbin/crates/edge-index/src/state_view.rs | 250 + bobbin/crates/edge-index/src/stream.rs | 147 + .../crates/edge-index/tests/bucket_scaling.rs | 227 + bobbin/crates/ingest/Cargo.toml | 51 + bobbin/crates/ingest/benches/json_decode.rs | 237 + bobbin/crates/ingest/examples/chunk_parity.rs | 239 + bobbin/crates/ingest/examples/smoke.rs | 97 + bobbin/crates/ingest/src/analytics.rs | 753 + bobbin/crates/ingest/src/authorship.rs | 205 + bobbin/crates/ingest/src/chunks.rs | 369 + bobbin/crates/ingest/src/frame.rs | 102 + bobbin/crates/ingest/src/lib.rs | 5977 +++ bobbin/crates/ingest/src/metrics.rs | 216 + bobbin/crates/ingest/src/moderation/frame.rs | 58 + bobbin/crates/ingest/src/moderation/mod.rs | 351 + bobbin/crates/ingest/src/resolver.rs | 1 + bobbin/crates/ingest/src/shadow.rs | 172 + bobbin/crates/ingest/src/wait.rs | 136 + bobbin/crates/ingest/src/warming.rs | 532 + bobbin/crates/knot-proxy/Cargo.toml | 25 + bobbin/crates/knot-proxy/src/breaker.rs | 272 + bobbin/crates/knot-proxy/src/dns.rs | 109 + bobbin/crates/knot-proxy/src/host.rs | 349 + bobbin/crates/knot-proxy/src/lib.rs | 796 + bobbin/crates/knot-proxy/src/mirror.rs | 156 + bobbin/crates/knot-proxy/src/origins.rs | 450 + bobbin/crates/languages/Cargo.toml | 27 + bobbin/crates/languages/src/fake.rs | 69 + bobbin/crates/languages/src/heads.rs | 100 + bobbin/crates/languages/src/index.rs | 253 + bobbin/crates/languages/src/lib.rs | 14 + bobbin/crates/languages/src/live.rs | 259 + bobbin/crates/languages/src/mirror.rs | 166 + bobbin/crates/languages/src/source.rs | 38 + bobbin/crates/languages/src/sync.rs | 161 + bobbin/crates/query/Cargo.toml | 10 + bobbin/crates/query/src/ast.rs | 209 + bobbin/crates/query/src/error.rs | 84 + bobbin/crates/query/src/lex.rs | 329 + bobbin/crates/query/src/lib.rs | 10 + bobbin/crates/query/src/lower.rs | 477 + bobbin/crates/record-lru/Cargo.toml | 16 + bobbin/crates/record-lru/src/lib.rs | 319 + bobbin/crates/record-lru/src/record.dict | Bin 0 -> 32768 bytes bobbin/crates/resolver/Cargo.toml | 30 + bobbin/crates/resolver/src/decode.rs | 180 + bobbin/crates/resolver/src/hydrant.rs | 372 + bobbin/crates/resolver/src/identity.rs | 2009 + bobbin/crates/resolver/src/lib.rs | 1389 + bobbin/crates/resolver/src/normalize.rs | 64 + bobbin/crates/runtime/Cargo.toml | 24 + bobbin/crates/runtime/src/clock.rs | 241 + bobbin/crates/runtime/src/entropy.rs | 116 + bobbin/crates/runtime/src/hasher.rs | 101 + bobbin/crates/runtime/src/lib.rs | 20 + bobbin/crates/runtime/src/mem.rs | 12 + bobbin/crates/runtime/src/mem_network.rs | 359 + bobbin/crates/runtime/src/network.rs | 359 + bobbin/crates/search/Cargo.toml | 24 + bobbin/crates/search/src/actor.rs | 975 + bobbin/crates/search/src/cursor.rs | 170 + bobbin/crates/search/src/lib.rs | 1860 + bobbin/crates/slingshot-client/Cargo.toml | 25 + bobbin/crates/slingshot-client/src/lib.rs | 661 + bobbin/crates/types/Cargo.toml | 24 + bobbin/crates/types/src/edges.rs | 1309 + bobbin/crates/types/src/identity.rs | 34 + bobbin/crates/types/src/ids.rs | 145 + bobbin/crates/types/src/lib.rs | 10 + bobbin/crates/types/src/record.rs | 21 + bobbin/crates/types/src/search.rs | 518 + bobbin/crates/xrpc/Cargo.toml | 50 + bobbin/crates/xrpc/src/actor.rs | 176 + bobbin/crates/xrpc/src/authors.rs | 33 + bobbin/crates/xrpc/src/backpressure.rs | 238 + bobbin/crates/xrpc/src/change_ids.rs | 85 + bobbin/crates/xrpc/src/client_address.rs | 150 + bobbin/crates/xrpc/src/codesearch.rs | 513 + bobbin/crates/xrpc/src/commit_stats.rs | 668 + bobbin/crates/xrpc/src/enrich.rs | 977 + bobbin/crates/xrpc/src/feed.rs | 410 + bobbin/crates/xrpc/src/filter.rs | 128 + bobbin/crates/xrpc/src/labels.rs | 67 + bobbin/crates/xrpc/src/lib.rs | 3921 ++ bobbin/crates/xrpc/src/pocket.rs | 106 + bobbin/crates/xrpc/src/profile_activity.rs | 469 + bobbin/crates/xrpc/src/recordpath.rs | 310 + bobbin/crates/xrpc/src/repo.rs | 160 + bobbin/crates/xrpc/src/search.rs | 483 + bobbin/crates/xrpc/src/view.rs | 178 + bobbin/crates/xrpc/tests/aggregation.rs | 3070 ++ bobbin/crates/xrpc/tests/avatar.rs | 202 + bobbin/crates/xrpc/tests/await_record.rs | 287 + bobbin/crates/xrpc/tests/await_record_e2e.rs | 234 + bobbin/crates/xrpc/tests/bulk.rs | 645 + bobbin/crates/xrpc/tests/code_search.rs | 96 + bobbin/crates/xrpc/tests/cold_start.rs | 868 + bobbin/crates/xrpc/tests/coverage.rs | 351 + bobbin/crates/xrpc/tests/enrich.rs | 1128 + bobbin/crates/xrpc/tests/extended.rs | 375 + bobbin/crates/xrpc/tests/mirror_proxy.rs | 199 + bobbin/crates/xrpc/tests/pocket_proxy.rs | 153 + bobbin/crates/xrpc/tests/search.rs | 1485 + bobbin/crates/xrpc/tests/trending.rs | 272 + bobbin/default.nix | 34 + bobbin/example.toml | 309 + bobbin/worker/package.json | 20 + bobbin/worker/pnpm-lock.yaml | 912 + bobbin/worker/pnpm-workspace.yaml | 4 + bobbin/worker/src/index.ts | 201 + bobbin/worker/tsconfig.json | 16 + bobbin/worker/wrangler.dev.jsonc | 62 + bobbin/worker/wrangler.jsonc | 59 + buf.gen.yaml | 14 + buf.lock | 6 + buf.yaml | 6 + camo/.gitignore | 174 + camo/package.json | 17 + camo/pnpm-lock.yaml | 1845 + camo/pnpm-workspace.yaml | 4 + camo/readme.md | 17 + camo/src/index.js | 146 + camo/src/index.test.js | 60 + camo/src/mimetypes.json | 49 + camo/wrangler.dev.jsonc | 19 + camo/wrangler.jsonc | 20 + cilogs/cihelpers.go | 50 + cilogs/cilogs_test.go | 233 + cilogs/error.go | 31 + cilogs/logstream.go | 47 + cilogs/server.go | 71 + cilogs/session.go | 64 + cilogs/spindles.go | 69 + cilogs/tui.go | 500 + cmd/blog/main.go | 50 + cmd/cborgen/cborgen.go | 65 + cmd/cilogs/main.go | 38 + cmd/claimer/emit/emit.go | 38 + cmd/claimer/main.go | 364 + cmd/claimer/main_test.go | 53 + cmd/deliberi/main.go | 53 + cmd/delicurl.sh | 95 + cmd/dolly/main.go | 259 + cmd/dolly/templates/logo.html | 22 + cmd/dolly/templates/logotype.html | 25 + cmd/lexpostgen/main.go | 57 + cmd/migrator/main.go | 79 + cmd/prefill-zoekt/main.go | 174 + cmd/sitesd/main.go | 52 + cmd/spindle-microvm-run/main.go | 26 + cmd/spindle-microvm-run/main_linux.go | 381 + cmd/spindle-microvm-run/unsupported.go | 13 + cmd/spindle/ban_cli_test.go | 55 + cmd/spindle/main.go | 736 + cmd/spindle/main_test.go | 310 + cmd/verifysig/main.go | 103 + cmd/zoekt-tngl-indexserver/index.go | 170 + cmd/zoekt-tngl-indexserver/main.go | 315 + cmd/zoekt-tngl-indexserver/metrics.go | 17 + cmd/zoekt-tngl-indexserver/queue.go | 72 + cmd/zoekt-tngl-indexserver/server.go | 212 + cmd/zoekt-tngl-indexserver/server_test.go | 84 + crates/gix-chunk/Cargo.toml | 121 + crates/gix-chunk/LICENSE-APACHE | 177 + crates/gix-chunk/LICENSE-MIT | 17 + crates/gix-chunk/src/file/decode.rs | 93 + crates/gix-chunk/src/file/index.rs | 88 + crates/gix-chunk/src/file/mod.rs | 20 + crates/gix-chunk/src/file/write.rs | 139 + crates/gix-chunk/src/lib.rs | 62 + crates/gix-commitgraph/Cargo.toml | 194 + crates/gix-commitgraph/LICENSE-APACHE | 177 + crates/gix-commitgraph/LICENSE-MIT | 17 + crates/gix-commitgraph/src/access.rs | 112 + crates/gix-commitgraph/src/bloom.rs | 374 + crates/gix-commitgraph/src/file/access.rs | 165 + crates/gix-commitgraph/src/file/commit.rs | 329 + crates/gix-commitgraph/src/file/init.rs | 268 + crates/gix-commitgraph/src/file/mod.rs | 52 + crates/gix-commitgraph/src/file/verify.rs | 155 + crates/gix-commitgraph/src/init.rs | 147 + crates/gix-commitgraph/src/lib.rs | 114 + crates/gix-commitgraph/src/verify.rs | 157 + crates/gix-commitgraph/src/write/info_dir.rs | 183 + crates/gix-commitgraph/src/write/mod.rs | 1142 + crates/gix-rebase/Cargo.toml | 35 + crates/gix-rebase/README.md | 4 + crates/gix-rebase/src/lib.rs | 6 + crates/gix-rebase/src/replay/function.rs | 126 + crates/gix-rebase/src/replay/mod.rs | 150 + .../tests/fixtures/rebase-baseline.sh | 192 + crates/gix-rebase/tests/rebase/baseline.rs | 351 + crates/gix-rebase/tests/rebase/main.rs | 193 + crates/gix-receive-pack/Cargo.toml | 30 + crates/gix-receive-pack/src/advertisement.rs | 194 + crates/gix-receive-pack/src/command.rs | 360 + crates/gix-receive-pack/src/error.rs | 19 + crates/gix-receive-pack/src/lib.rs | 12 + crates/gix-receive-pack/src/report.rs | 185 + crates/gix-receive-pack/src/session.rs | 191 + crates/gix-receive-pack/src/types.rs | 43 + crates/knot-capability/Cargo.toml | 9 + crates/knot-capability/src/lib.rs | 116 + crates/lexicons/Cargo.toml | 36 + crates/lexicons/build.rs | 105 + crates/lexicons/src/lib.rs | 22 + crates/tangled-axum/Cargo.toml | 16 + crates/tangled-axum/src/atproto.rs | 179 + crates/tangled-axum/src/lib.rs | 1 + crates/tranquil-gate/Cargo.toml | 31 + crates/tranquil-gate/config.example.toml | 13 + crates/tranquil-gate/src/config.rs | 111 + crates/tranquil-gate/src/gate.rs | 144 + crates/tranquil-gate/src/lib.rs | 337 + crates/tranquil-gate/src/main.rs | 50 + crates/tranquil-gate/src/telemetry.rs | 26 + crates/tranquil-gate/src/tests.rs | 381 + crates/tranquil-gate/tests/config.rs | 136 + crates/trusted-proxies/Cargo.toml | 10 + crates/trusted-proxies/src/lib.rs | 308 + crypto/verify.go | 42 + crypto/verify_test.go | 134 + default.nix | 16 + deliberi/account_ingest_test.go | 65 + deliberi/analytics/analytics.go | 67 + deliberi/bobbin.go | 104 + deliberi/bobbin_test.go | 132 + deliberi/config/config.go | 89 + deliberi/config/config_test.go | 44 + deliberi/db/cache.go | 64 + deliberi/db/cursor.go | 31 + deliberi/db/db.go | 335 + deliberi/db/email.go | 351 + deliberi/db/ingest.go | 180 + deliberi/db/newsletter.go | 52 + deliberi/db/notifications.go | 260 + deliberi/db/notifications_test.go | 187 + deliberi/db/signup.go | 51 + deliberi/db/signup_test.go | 111 + deliberi/deliberi.go | 104 + deliberi/digest.go | 330 + deliberi/digest_test.go | 114 + deliberi/email/email.go | 102 + deliberi/email/email_test.go | 53 + deliberi/emailtemplate/template.go | 45 + deliberi/ingest.go | 660 + deliberi/ingest_test.go | 1135 + deliberi/invites.go | 160 + deliberi/invites_test.go | 254 + deliberi/kv/kv.go | 108 + deliberi/kv/kv_test.go | 190 + deliberi/mailer/mailer.go | 77 + deliberi/markup/atlink.go | 121 + deliberi/mentions.go | 61 + deliberi/mentions_test.go | 34 + deliberi/models/email.go | 14 + deliberi/models/notifications.go | 229 + deliberi/models/notifications_test.go | 98 + deliberi/models/signup.go | 13 + deliberi/readme.md | 2 + deliberi/records.go | 122 + deliberi/xrpc/account.go | 343 + deliberi/xrpc/account_test.go | 529 + deliberi/xrpc/ingested.go | 30 + deliberi/xrpc/ingested_test.go | 75 + deliberi/xrpc/internal_committers.go | 44 + deliberi/xrpc/internal_committers_test.go | 77 + deliberi/xrpc/newsletter.go | 114 + deliberi/xrpc/newsletter_test.go | 186 + deliberi/xrpc/notifications.go | 262 + deliberi/xrpc/pds_email.go | 68 + deliberi/xrpc/pds_email_test.go | 244 + deliberi/xrpc/signup.go | 555 + deliberi/xrpc/signup_test.go | 879 + deliberi/xrpc/xrpc.go | 148 + deliberi/xrpc/xrpc_test.go | 398 + docs/.gitignore | 3 + docs/astro.config.mjs | 229 + docs/package-lock.json | 8435 ++++ docs/package.json | 39 + docs/pnpm-lock.yaml | 5454 ++ docs/pnpm-workspace.yaml | 3 + docs/public/apple-touch-icon.png | Bin 0 -> 4666 bytes docs/public/favicon.svg | 21 + docs/scripts/xrpc-api.mjs | 220 + docs/src/assets/diagrams/bobbin.svg | 70 + docs/src/assets/diagrams/spindle.svg | 73 + docs/src/components/Diagram.astro | 15 + docs/src/components/Header.astro | 16 + docs/src/components/Hero.astro | 158 + docs/src/components/MarkdownContent.astro | 7 + docs/src/components/MobileMenuFooter.astro | 3 + docs/src/components/PageTitle.astro | 5 + docs/src/components/PullSelectors.svelte | 142 + docs/src/components/RepoCreationName.svelte | 92 + docs/src/components/Search.astro | 142 + docs/src/components/Sidebar.astro | 79 + docs/src/components/SiteTitle.astro | 10 + docs/src/components/ThemeProvider.astro | 10 + docs/src/components/VouchCardDemo.svelte | 38 + docs/src/content.config.ts | 7 + docs/src/content/docs/api/bobbin.md | 946 + docs/src/content/docs/api/knot.md | 510 + docs/src/content/docs/api/spindle.md | 236 + .../content/docs/comparison-with-github.md | 57 + docs/src/content/docs/contributing/guide.md | 137 + docs/src/content/docs/contributing/hacking.md | 15 + docs/src/content/docs/guides/access.md | 102 + docs/src/content/docs/guides/importing.md | 37 + docs/src/content/docs/guides/notifications.md | 42 + .../src/content/docs/guides/organizations.mdx | 50 + docs/src/content/docs/guides/profile.md | 55 + .../src/content/docs/guides/pull-requests.mdx | 105 + docs/src/content/docs/guides/repositories.md | 102 + .../content/docs/guides/signing-commits.md | 48 + docs/src/content/docs/guides/vouching.mdx | 61 + docs/src/content/docs/index.mdx | 112 + .../content/docs/integrations/contributing.md | 43 + docs/src/content/docs/integrations/discord.md | 70 + docs/src/content/docs/introduction.md | 62 + docs/src/content/docs/knots/index.md | 198 + docs/src/content/docs/knots/repositories.md | 58 + docs/src/content/docs/knots/self-hosting.md | 229 + .../src/content/docs/migrating-from-github.md | 107 + docs/src/content/docs/quick-start.md | 112 + .../src/content/docs/reference/identifiers.md | 170 + .../src/content/docs/reference/knot-config.md | 352 + docs/src/content/docs/reference/records.md | 99 + .../content/docs/reference/spindle-config.md | 226 + .../src/content/docs/self-hosting/frontend.md | 6 + docs/src/content/docs/self-hosting/index.mdx | 12 + docs/src/content/docs/self-hosting/indexer.md | 38 + docs/src/content/docs/self-hosting/knot.md | 150 + docs/src/content/docs/self-hosting/spindle.md | 86 + docs/src/content/docs/sites.md | 47 + .../content/docs/spindles/architecture.mdx | 178 + docs/src/content/docs/spindles/index.md | 94 + docs/src/content/docs/spindles/secrets.md | 82 + .../src/content/docs/spindles/self-hosting.md | 196 + docs/src/content/docs/spindles/webhooks.md | 291 + docs/src/content/docs/spindles/workflows.md | 710 + .../docs/spindles/writing-workflows.md | 535 + docs/src/content/docs/troubleshooting.md | 100 + docs/src/content/docs/upgrading/knot-2.md | 384 + .../docs/upgrading/knots-and-spindles.md | 188 + docs/src/content/docs/why-atproto.md | 30 + docs/src/pages/404.astro | 17 + docs/src/starlightRouteData.ts | 22 + docs/src/stubs/app-env.ts | 6 + docs/src/stubs/app-paths.ts | 7 + docs/src/styles/custom.css | 117 + docs/src/styles/diagrams.css | 132 + docs/src/styles/global.css | 57 + docs/src/styles/tangled.css | 649 + docs/tsconfig.json | 5 + docs/wrangler.jsonc | 21 + e2e/.gitignore | 4 + e2e/.prettierrc.json | 6 + e2e/README.md | 83 + e2e/harness/api.ts | 491 + e2e/harness/attestations.ts | 50 + e2e/harness/auth.setup.ts | 19 + e2e/harness/chores.ts | 229 + e2e/harness/cobgit.ts | 273 + e2e/harness/config.ts | 76 + e2e/harness/firehose.ts | 208 + e2e/harness/flow.ts | 388 + e2e/harness/frames.ts | 43 + e2e/harness/git.ts | 263 + e2e/harness/global.ts | 19 + e2e/harness/lexicons.ts | 17 + e2e/harness/net.ts | 145 + e2e/harness/processes.ts | 91 + e2e/harness/record.ts | 163 + e2e/harness/report/ExchangeRow.svelte | 214 + e2e/harness/report/Report.svelte | 755 + e2e/harness/report/data.ts | 50 + e2e/harness/report/render.ts | 148 + e2e/harness/report/search.js | 366 + e2e/harness/report/write.ts | 89 + e2e/harness/reporter.ts | 271 + e2e/harness/run.ts | 349 + e2e/harness/services.ts | 663 + e2e/harness/shared.ts | 580 + e2e/harness/signin.ts | 371 + e2e/harness/stream.ts | 203 + e2e/harness/tabs.ts | 267 + e2e/harness/versions.ts | 80 + e2e/harness/vocabulary.ts | 269 + e2e/harness/web-hash.ts | 33 + e2e/harness/webhooks.ts | 261 + e2e/outages/knot-outage.spec.ts | 69 + e2e/outages/spindle-outage.spec.ts | 120 + e2e/package.json | 26 + e2e/playwright.config.ts | 36 + e2e/pnpm-lock.yaml | 346 + e2e/run | 77 + e2e/specs/access.spec.ts | 413 + e2e/specs/account.spec.ts | 307 + e2e/specs/chores.spec.ts | 79 + e2e/specs/ci-logs.spec.ts | 51 + e2e/specs/comments.spec.ts | 376 + e2e/specs/faults.spec.ts | 347 + e2e/specs/fork-pulls.spec.ts | 197 + e2e/specs/forks.spec.ts | 113 + e2e/specs/git.spec.ts | 420 + e2e/specs/issues.spec.ts | 411 + e2e/specs/knots.spec.ts | 279 + e2e/specs/labels.spec.ts | 208 + e2e/specs/markup.spec.ts | 106 + e2e/specs/merges.spec.ts | 338 + e2e/specs/opengraph.spec.ts | 271 + e2e/specs/permissions.spec.ts | 200 + e2e/specs/pipelines.spec.ts | 475 + e2e/specs/profile-settings.spec.ts | 315 + e2e/specs/profile.spec.ts | 420 + e2e/specs/pull-pipelines.spec.ts | 123 + e2e/specs/pulls.spec.ts | 386 + e2e/specs/quota.spec.ts | 91 + e2e/specs/recents.spec.ts | 78 + e2e/specs/refusals.spec.ts | 333 + e2e/specs/repo-settings.spec.ts | 428 + e2e/specs/repos.spec.ts | 490 + e2e/specs/routes.spec.ts | 371 + e2e/specs/search.spec.ts | 106 + e2e/specs/session.spec.ts | 293 + e2e/specs/signup.spec.ts | 140 + e2e/specs/social.spec.ts | 786 + e2e/specs/strings.spec.ts | 179 + e2e/specs/webhooks.spec.ts | 257 + e2e/tools/sweep.spec.ts | 63 + e2e/tools/tidy.spec.ts | 19 + e2e/tsconfig.json | 24 + e2e/up | 179 + email-did/package.json | 18 + email-did/pnpm-lock.yaml | 1868 + email-did/pnpm-workspace.yaml | 6 + email-did/src/index.ts | 43 + email-did/test/index.test.ts | 67 + email-did/test/setup.ts | 10 + email-did/tsconfig.json | 17 + email-did/vitest.config.ts | 3 + email-did/wrangler.jsonc | 17 + filetree/filetree.go | 65 + flake.lock | 305 + flake.nix | 859 + gitclient/gitclient.go | 444 + gitclient/gitclient_test.go | 198 + gitclient/pack.go | 365 + gitclient/pkt.go | 148 + gitutil/archive.go | 278 + gitutil/archive_test.go | 281 + gitutil/command.go | 77 + gitutil/command_test.go | 48 + gitutil/keyed_mutex_test.go | 52 + gitutil/readme.go | 22 + gitutil/readme_test.go | 48 + gitutil/sync.go | 208 + gitutil/sync_test.go | 310 + go.mod | 283 + go.sum | 988 + hostutil/hostutil.go | 80 + hostutil/safedial.go | 76 + ico/ico.go | 88 + icons/bluesky.svg | 4 + icons/user-round-arrow-forward.svg | 17 + idresolver/mock.go | 35 + idresolver/resolver.go | 167 + input.css | 1699 + jetstream/jetstream.go | 297 + jetstream/jetstream_test.go | 478 + jetstream/stream.go | 220 + jetstream/stream_test.go | 321 + jetstream/watermark.go | 96 + jetstream/watermark_test.go | 120 + jetstream/zstd_dictionary | Bin 0 -> 112640 bytes jobs/.gitignore | 4 + jobs/package-lock.json | 1552 + jobs/package.json | 20 + jobs/src/index.ts | 617 + .../src/postings/archive/software-engineer.md | 16 + jobs/src/woff2.d.ts | 4 + jobs/tsconfig.json | 14 + jobs/wrangler.toml | 28 + knot2/.gitignore | 9 + knot2/Containerfile | 23 + knot2/README.md | 24 + knot2/_typos.toml | 4 + knot2/crates/knot-acl/Cargo.toml | 19 + knot2/crates/knot-acl/src/lib.rs | 923 + knot2/crates/knot-atproto/Cargo.toml | 30 + knot2/crates/knot-atproto/fuzz/.gitignore | 4 + knot2/crates/knot-atproto/fuzz/Cargo.lock | 4225 ++ knot2/crates/knot-atproto/fuzz/Cargo.toml | 30 + .../fuzz/fuzz_targets/did_document.rs | 7 + .../knot-atproto/fuzz/fuzz_targets/pubkey.rs | 7 + knot2/crates/knot-atproto/src/auth.rs | 60 + knot2/crates/knot-atproto/src/identity.rs | 454 + knot2/crates/knot-atproto/src/jwt.rs | 669 + knot2/crates/knot-atproto/src/lib.rs | 2609 + knot2/crates/knot-atproto/src/plc.rs | 307 + knot2/crates/knot-atproto/src/pointer.rs | 166 + knot2/crates/knot-atproto/src/pubkeys.rs | 245 + knot2/crates/knot-atproto/src/resolve.rs | 720 + knot2/crates/knot-atproto/src/test_support.rs | 197 + knot2/crates/knot-atproto/tests/fuzz_smoke.rs | 15 + knot2/crates/knot-atproto/tests/security.rs | 62 + knot2/crates/knot-bench/Cargo.toml | 50 + knot2/crates/knot-bench/benches/advert.rs | 30 + knot2/crates/knot-bench/benches/blame.rs | 289 + knot2/crates/knot-bench/benches/cob.rs | 27 + knot2/crates/knot-bench/benches/coldstart.rs | 66 + knot2/crates/knot-bench/benches/pack.rs | 297 + knot2/crates/knot-bench/benches/projection.rs | 73 + knot2/crates/knot-bench/maint-bench.sh | 67 + .../crates/knot-bench/src/bin/maint-bench.rs | 154 + knot2/crates/knot-bench/src/fixtures.rs | 808 + knot2/crates/knot-bench/src/latency.rs | 32 + knot2/crates/knot-bench/src/lib.rs | 12 + knot2/crates/knot-bench/tests/boot.rs | 26 + knot2/crates/knot-bench/tests/gate.rs | 86 + knot2/crates/knot-bench/tests/members_gate.rs | 44 + .../crates/knot-bench/tests/registry_gate.rs | 44 + knot2/crates/knot-cache/Cargo.toml | 11 + knot2/crates/knot-cache/src/expiring.rs | 388 + knot2/crates/knot-cache/src/lib.rs | 647 + knot2/crates/knot-cob/Cargo.toml | 26 + knot2/crates/knot-cob/src/backend.rs | 541 + knot2/crates/knot-cob/src/change.rs | 523 + knot2/crates/knot-cob/src/error.rs | 130 + knot2/crates/knot-cob/src/graph.rs | 141 + knot2/crates/knot-cob/src/instrument.rs | 8 + knot2/crates/knot-cob/src/lib.rs | 3543 ++ knot2/crates/knot-cob/src/object.rs | 159 + knot2/crates/knot-cobs/Cargo.toml | 23 + knot2/crates/knot-cobs/fuzz/.gitignore | 4 + knot2/crates/knot-cobs/fuzz/Cargo.lock | 5398 ++ knot2/crates/knot-cobs/fuzz/Cargo.toml | 33 + .../knot-cobs/fuzz/fuzz_targets/cob_change.rs | 7 + .../knot-cobs/fuzz/fuzz_targets/cob_ref.rs | 7 + knot2/crates/knot-cobs/src/artifact.rs | 44 + knot2/crates/knot-cobs/src/blocklist.rs | 9 + knot2/crates/knot-cobs/src/collaborators.rs | 9 + knot2/crates/knot-cobs/src/comment.rs | 161 + knot2/crates/knot-cobs/src/grant.rs | 432 + knot2/crates/knot-cobs/src/import.rs | 54 + knot2/crates/knot-cobs/src/label.rs | 49 + knot2/crates/knot-cobs/src/lib.rs | 66 + knot2/crates/knot-cobs/src/manifest.rs | 49 + knot2/crates/knot-cobs/src/members.rs | 9 + knot2/crates/knot-cobs/src/pull.rs | 53 + knot2/crates/knot-cobs/src/reaction.rs | 98 + knot2/crates/knot-cobs/src/registry.rs | 895 + knot2/crates/knot-cobs/src/social.rs | 1241 + knot2/crates/knot-cobs/src/ticket.rs | 164 + knot2/crates/knot-cobs/tests/cobs.rs | 363 + knot2/crates/knot-cobs/tests/common/mod.rs | 217 + knot2/crates/knot-cobs/tests/fuzz_smoke.rs | 15 + knot2/crates/knot-cobs/tests/invariants.rs | 504 + knot2/crates/knot-cobs/tests/social.rs | 332 + knot2/crates/knot-config/Cargo.toml | 16 + knot2/crates/knot-config/src/lib.rs | 2130 + knot2/crates/knot-edge/Cargo.toml | 41 + knot2/crates/knot-edge/fuzz/.gitignore | 4 + knot2/crates/knot-edge/fuzz/Cargo.lock | 4654 ++ knot2/crates/knot-edge/fuzz/Cargo.toml | 30 + .../knot-edge/fuzz/fuzz_targets/spki.rs | 7 + .../knot-edge/fuzz/fuzz_targets/spki_pin.rs | 7 + knot2/crates/knot-edge/src/acme.rs | 276 + knot2/crates/knot-edge/src/altsvc.rs | 103 + knot2/crates/knot-edge/src/compression.rs | 228 + knot2/crates/knot-edge/src/cors.rs | 30 + knot2/crates/knot-edge/src/lib.rs | 381 + knot2/crates/knot-edge/src/limits.rs | 113 + knot2/crates/knot-edge/src/peer.rs | 58 + knot2/crates/knot-edge/src/protocol.rs | 78 + knot2/crates/knot-edge/src/quic.rs | 609 + knot2/crates/knot-edge/src/robustness.rs | 763 + knot2/crates/knot-edge/src/tcp.rs | 521 + knot2/crates/knot-edge/src/tls.rs | 679 + knot2/crates/knot-edge/src/zerortt.rs | 194 + knot2/crates/knot-edge/tests/fuzz_smoke.rs | 11 + knot2/crates/knot-events/Cargo.toml | 12 + knot2/crates/knot-events/src/lib.rs | 756 + knot2/crates/knot-fixtures/Cargo.toml | 10 + knot2/crates/knot-fixtures/src/lib.rs | 142 + knot2/crates/knot-git/Cargo.toml | 31 + knot2/crates/knot-git/fuzz/.gitignore | 4 + knot2/crates/knot-git/fuzz/Cargo.lock | 5247 ++ knot2/crates/knot-git/fuzz/Cargo.toml | 26 + .../knot-git/fuzz/fuzz_targets/patch.rs | 7 + knot2/crates/knot-git/src/archive.rs | 332 + knot2/crates/knot-git/src/base85.rs | 154 + knot2/crates/knot-git/src/bitmap/bitset.rs | 188 + knot2/crates/knot-git/src/bitmap/count.rs | 66 + knot2/crates/knot-git/src/bitmap/ewah.rs | 170 + knot2/crates/knot-git/src/bitmap/midx.rs | 77 + knot2/crates/knot-git/src/bitmap/mod.rs | 180 + knot2/crates/knot-git/src/bitmap/reach.rs | 179 + knot2/crates/knot-git/src/bitmap/reader.rs | 141 + knot2/crates/knot-git/src/bitmap/revindex.rs | 246 + knot2/crates/knot-git/src/bitmap/writer.rs | 435 + knot2/crates/knot-git/src/error.rs | 92 + knot2/crates/knot-git/src/instrument.rs | 8 + knot2/crates/knot-git/src/lib.rs | 62 + knot2/crates/knot-git/src/maintenance.rs | 355 + knot2/crates/knot-git/src/objects.rs | 1555 + knot2/crates/knot-git/src/patch.rs | 596 + knot2/crates/knot-git/src/patch_apply.rs | 735 + knot2/crates/knot-git/src/patch_parse.rs | 1189 + knot2/crates/knot-git/src/reads.rs | 817 + knot2/crates/knot-git/src/repo.rs | 1969 + knot2/crates/knot-git/src/staging.rs | 143 + knot2/crates/knot-git/tests/apply.rs | 432 + knot2/crates/knot-git/tests/bitmap.rs | 377 + knot2/crates/knot-git/tests/common/mod.rs | 18 + .../crates/knot-git/tests/config_isolation.rs | 142 + knot2/crates/knot-git/tests/fuzz_smoke.rs | 10 + knot2/crates/knot-git/tests/reads.rs | 851 + knot2/crates/knot-index/Cargo.toml | 29 + knot2/crates/knot-index/src/coverage.rs | 70 + knot2/crates/knot-index/src/error.rs | 25 + knot2/crates/knot-index/src/intern.rs | 102 + knot2/crates/knot-index/src/lib.rs | 959 + knot2/crates/knot-index/src/projections.rs | 1412 + knot2/crates/knot-index/src/social.rs | 1008 + knot2/crates/knot-index/tests/common/mod.rs | 714 + knot2/crates/knot-index/tests/lifecycle.rs | 448 + knot2/crates/knot-index/tests/meta_repo.rs | 174 + knot2/crates/knot-index/tests/projections.rs | 2711 + knot2/crates/knot-index/tests/properties.rs | 262 + knot2/crates/knot-keyfill/Cargo.toml | 27 + knot2/crates/knot-keyfill/src/lib.rs | 620 + knot2/crates/knot-keyfill/tests/fill.rs | 445 + knot2/crates/knot-langs/Cargo.toml | 12 + knot2/crates/knot-langs/src/langs.rs | 127 + knot2/crates/knot-langs/src/lib.rs | 4 + knot2/crates/knot-langs/src/linguist.rs | 196 + knot2/crates/knot-lexicons/Cargo.toml | 33 + knot2/crates/knot-lexicons/build.rs | 103 + knot2/crates/knot-lexicons/src/lib.rs | 57 + knot2/crates/knot-lfs/Cargo.toml | 24 + knot2/crates/knot-lfs/fuzz/.gitignore | 4 + knot2/crates/knot-lfs/fuzz/Cargo.lock | 5414 ++ knot2/crates/knot-lfs/fuzz/Cargo.toml | 40 + .../knot-lfs/fuzz/fuzz_targets/batch.rs | 7 + .../knot-lfs/fuzz/fuzz_targets/pointer.rs | 7 + .../knot-lfs/fuzz/fuzz_targets/transfer.rs | 7 + knot2/crates/knot-lfs/src/admission.rs | 177 + knot2/crates/knot-lfs/src/batch.rs | 222 + knot2/crates/knot-lfs/src/error.rs | 51 + knot2/crates/knot-lfs/src/gc.rs | 338 + knot2/crates/knot-lfs/src/lib.rs | 77 + knot2/crates/knot-lfs/src/pointer.rs | 88 + knot2/crates/knot-lfs/src/scan.rs | 32 + knot2/crates/knot-lfs/src/store.rs | 1151 + knot2/crates/knot-lfs/src/transfer.rs | 1191 + knot2/crates/knot-lfs/src/types.rs | 315 + knot2/crates/knot-lfs/tests/chaos.rs | 320 + knot2/crates/knot-lfs/tests/common/mod.rs | 98 + knot2/crates/knot-lfs/tests/fuzz_smoke.rs | 20 + knot2/crates/knot-lfs/tests/properties.rs | 124 + knot2/crates/knot-lfs/tests/race.rs | 132 + knot2/crates/knot-lfs/tests/soak.rs | 161 + knot2/crates/knot-maintenance/Cargo.toml | 27 + knot2/crates/knot-maintenance/src/bitmap.rs | 85 + .../knot-maintenance/src/commitgraph.rs | 825 + knot2/crates/knot-maintenance/src/cruft.rs | 531 + knot2/crates/knot-maintenance/src/fsio.rs | 206 + knot2/crates/knot-maintenance/src/lib.rs | 477 + knot2/crates/knot-maintenance/src/midx.rs | 279 + knot2/crates/knot-maintenance/src/policy.rs | 60 + knot2/crates/knot-maintenance/src/prune.rs | 31 + knot2/crates/knot-maintenance/src/repack.rs | 324 + .../crates/knot-maintenance/src/scheduler.rs | 789 + .../knot-maintenance/src/test_support.rs | 57 + knot2/crates/knot-maintenance/tests/chaos.rs | 237 + .../knot-maintenance/tests/commitgraph.rs | 325 + .../knot-maintenance/tests/common/mod.rs | 227 + knot2/crates/knot-maintenance/tests/cruft.rs | 134 + knot2/crates/knot-maintenance/tests/engine.rs | 240 + .../knot-maintenance/tests/geometric.rs | 216 + knot2/crates/knot-messages/Cargo.toml | 10 + knot2/crates/knot-messages/src/lib.rs | 455 + knot2/crates/knot-messages/src/template.rs | 188 + knot2/crates/knot-migrate/Cargo.toml | 35 + knot2/crates/knot-migrate/src/adopt.rs | 378 + knot2/crates/knot-migrate/src/casbin.rs | 301 + knot2/crates/knot-migrate/src/emit.rs | 774 + knot2/crates/knot-migrate/src/envfile.rs | 114 + knot2/crates/knot-migrate/src/lib.rs | 8 + knot2/crates/knot-migrate/src/main.rs | 700 + knot2/crates/knot-migrate/src/mapping.rs | 743 + knot2/crates/knot-migrate/src/rehearse.rs | 237 + knot2/crates/knot-migrate/src/report.rs | 254 + knot2/crates/knot-migrate/src/source.rs | 317 + knot2/crates/knot-migrate/tests/migrate.rs | 1907 + knot2/crates/knot-pack/Cargo.toml | 57 + knot2/crates/knot-pack/examples/ingest_mem.rs | 86 + .../knot-pack/examples/knot_http_backend.rs | 205 + .../crates/knot-pack/examples/receive_mem.rs | 129 + knot2/crates/knot-pack/fuzz/.gitignore | 4 + knot2/crates/knot-pack/fuzz/Cargo.lock | 6275 +++ knot2/crates/knot-pack/fuzz/Cargo.toml | 47 + .../knot-pack/fuzz/fuzz_targets/pack.rs | 7 + .../crates/knot-pack/fuzz/fuzz_targets/pkt.rs | 7 + .../fuzz/fuzz_targets/receive_commands.rs | 7 + .../fuzz/fuzz_targets/upload_args.rs | 7 + knot2/crates/knot-pack/src/archive.rs | 149 + knot2/crates/knot-pack/src/cache.rs | 512 + knot2/crates/knot-pack/src/error.rs | 71 + knot2/crates/knot-pack/src/fetch.rs | 763 + knot2/crates/knot-pack/src/frame.rs | 457 + knot2/crates/knot-pack/src/guard.rs | 62 + knot2/crates/knot-pack/src/ids.rs | 51 + knot2/crates/knot-pack/src/idxwrite.rs | 218 + knot2/crates/knot-pack/src/lib.rs | 920 + knot2/crates/knot-pack/src/meter.rs | 234 + knot2/crates/knot-pack/src/objects.rs | 1032 + knot2/crates/knot-pack/src/pkt.rs | 317 + knot2/crates/knot-pack/src/quarantine.rs | 72 + knot2/crates/knot-pack/src/receive.rs | 858 + knot2/crates/knot-pack/src/receiver.rs | 162 + knot2/crates/knot-pack/src/resolve.rs | 338 + knot2/crates/knot-pack/src/upload.rs | 1131 + knot2/crates/knot-pack/tests/chaos.rs | 174 + knot2/crates/knot-pack/tests/common/mod.rs | 332 + knot2/crates/knot-pack/tests/differential.rs | 692 + knot2/crates/knot-pack/tests/fetch.rs | 425 + knot2/crates/knot-pack/tests/fuzz_smoke.rs | 76 + knot2/crates/knot-pack/tests/git_client.rs | 990 + .../crates/knot-pack/tests/h3_conformance.rs | 574 + knot2/crates/knot-pack/tests/handle_owner.rs | 122 + knot2/crates/knot-pack/tests/hardening.rs | 1478 + knot2/crates/knot-pack/tests/quarantine.rs | 651 + knot2/crates/knot-pack/tests/serving.rs | 473 + knot2/crates/knot-pack/tests/soak.rs | 273 + knot2/crates/knot-postreceive/Cargo.toml | 20 + knot2/crates/knot-postreceive/src/lib.rs | 317 + .../knot-postreceive/tests/post_receive.rs | 318 + knot2/crates/knot-receive/Cargo.toml | 21 + knot2/crates/knot-receive/src/lib.rs | 380 + knot2/crates/knot-record/Cargo.toml | 31 + knot2/crates/knot-record/src/artifact.rs | 159 + knot2/crates/knot-record/src/attachment.rs | 561 + knot2/crates/knot-record/src/blocks.rs | 375 + knot2/crates/knot-record/src/chain.rs | 2533 + knot2/crates/knot-record/src/comment.rs | 308 + knot2/crates/knot-record/src/git_refs.rs | 82 + knot2/crates/knot-record/src/grant.rs | 169 + knot2/crates/knot-record/src/label.rs | 200 + knot2/crates/knot-record/src/lib.rs | 294 + knot2/crates/knot-record/src/manifest.rs | 270 + knot2/crates/knot-record/src/mst.rs | 119 + knot2/crates/knot-record/src/proof.rs | 171 + knot2/crates/knot-record/src/pull.rs | 173 + knot2/crates/knot-record/src/reaction.rs | 175 + knot2/crates/knot-record/src/ticket.rs | 343 + knot2/crates/knot-record/tests/car_exports.rs | 132 + .../knot-record/tests/proof_fixtures.rs | 158 + knot2/crates/knot-resource/Cargo.toml | 16 + knot2/crates/knot-resource/src/admission.rs | 1121 + knot2/crates/knot-resource/src/cpu.rs | 258 + knot2/crates/knot-resource/src/disk.rs | 143 + knot2/crates/knot-resource/src/fsio.rs | 347 + knot2/crates/knot-resource/src/lib.rs | 65 + knot2/crates/knot-resource/src/mem.rs | 833 + knot2/crates/knot-resource/src/slots.rs | 160 + knot2/crates/knot-runtime/Cargo.toml | 24 + knot2/crates/knot-runtime/src/clock.rs | 84 + knot2/crates/knot-runtime/src/dns.rs | 86 + knot2/crates/knot-runtime/src/entropy.rs | 146 + knot2/crates/knot-runtime/src/http.rs | 561 + knot2/crates/knot-runtime/src/lib.rs | 92 + knot2/crates/knot-runtime/src/signer.rs | 141 + knot2/crates/knot-secrets/Cargo.toml | 23 + knot2/crates/knot-secrets/src/lib.rs | 790 + knot2/crates/knot-server/Cargo.toml | 49 + knot2/crates/knot-server/src/admin.rs | 127 + knot2/crates/knot-server/src/allocator.rs | 148 + knot2/crates/knot-server/src/doctor.rs | 59 + knot2/crates/knot-server/src/homepage.html | 21 + knot2/crates/knot-server/src/main.rs | 1063 + knot2/crates/knot-server/tests/invariants.rs | 246 + knot2/crates/knot-sim/Cargo.toml | 54 + knot2/crates/knot-sim/src/harness.rs | 974 + knot2/crates/knot-sim/src/lib.rs | 26 + knot2/crates/knot-sim/src/main.rs | 57 + knot2/crates/knot-sim/src/realdata.rs | 668 + knot2/crates/knot-sim/src/trace.rs | 92 + knot2/crates/knot-sim/src/workload.rs | 1021 + knot2/crates/knot-sim/tests/common/mod.rs | 324 + knot2/crates/knot-sim/tests/h3.rs | 249 + knot2/crates/knot-sim/tests/lfs_chaos.rs | 229 + knot2/crates/knot-sim/tests/lfs_roundtrip.rs | 1567 + knot2/crates/knot-sim/tests/reproducible.rs | 294 + knot2/crates/knot-sim/tests/social_chaos.rs | 249 + knot2/crates/knot-sim/tests/ssh.rs | 302 + knot2/crates/knot-ssh/Cargo.toml | 46 + .../knot-ssh/examples/ephemeral_knot.rs | 255 + knot2/crates/knot-ssh/src/exec.rs | 1100 + knot2/crates/knot-ssh/src/identity.rs | 187 + knot2/crates/knot-ssh/src/lib.rs | 435 + knot2/crates/knot-ssh/src/proxy.rs | 118 + knot2/crates/knot-ssh/src/server.rs | 226 + knot2/crates/knot-ssh/tests/ssh_push.rs | 2403 + knot2/crates/knot-storage/Cargo.toml | 14 + knot2/crates/knot-storage/src/error.rs | 41 + knot2/crates/knot-storage/src/id.rs | 200 + knot2/crates/knot-storage/src/kv.rs | 86 + knot2/crates/knot-storage/src/lib.rs | 20 + knot2/crates/knot-storage/src/objects.rs | 131 + knot2/crates/knot-storage/src/pack.rs | 424 + knot2/crates/knot-storage/src/refs.rs | 307 + knot2/crates/knot-storage/src/store.rs | 40 + knot2/crates/knot-types/Cargo.toml | 23 + knot2/crates/knot-types/src/changes.rs | 156 + knot2/crates/knot-types/src/hex.rs | 65 + knot2/crates/knot-types/src/ids.rs | 1583 + knot2/crates/knot-types/src/lib.rs | 42 + knot2/crates/knot-types/src/net.rs | 309 + knot2/crates/knot-types/src/newtype.rs | 241 + knot2/crates/knot-types/src/policy.rs | 300 + knot2/crates/knot-types/src/record.rs | 610 + knot2/crates/knot-workflow/Cargo.toml | 15 + knot2/crates/knot-workflow/src/lib.rs | 524 + knot2/crates/knot-xrpc/Cargo.toml | 67 + knot2/crates/knot-xrpc/src/atproto.rs | 884 + knot2/crates/knot-xrpc/src/blobs.rs | 707 + knot2/crates/knot-xrpc/src/blocklist.rs | 80 + knot2/crates/knot-xrpc/src/body.rs | 55 + knot2/crates/knot-xrpc/src/branches.rs | 125 + knot2/crates/knot-xrpc/src/cob.rs | 49 + knot2/crates/knot-xrpc/src/collaborators.rs | 136 + knot2/crates/knot-xrpc/src/error.rs | 568 + knot2/crates/knot-xrpc/src/firehose.rs | 1632 + knot2/crates/knot-xrpc/src/forks.rs | 365 + knot2/crates/knot-xrpc/src/foster.rs | 1534 + knot2/crates/knot-xrpc/src/git.rs | 99 + knot2/crates/knot-xrpc/src/grants.rs | 143 + knot2/crates/knot-xrpc/src/handover.rs | 1424 + knot2/crates/knot-xrpc/src/imports.rs | 505 + knot2/crates/knot-xrpc/src/invite_codes.rs | 175 + knot2/crates/knot-xrpc/src/keep_sweep.rs | 241 + knot2/crates/knot-xrpc/src/legacy_admin.rs | 219 + knot2/crates/knot-xrpc/src/lfs.rs | 1129 + knot2/crates/knot-xrpc/src/lib.rs | 888 + knot2/crates/knot-xrpc/src/locks.rs | 38 + knot2/crates/knot-xrpc/src/materialize.rs | 225 + knot2/crates/knot-xrpc/src/members.rs | 125 + knot2/crates/knot-xrpc/src/migrate.rs | 184 + knot2/crates/knot-xrpc/src/mint.rs | 88 + knot2/crates/knot-xrpc/src/out.rs | 96 + knot2/crates/knot-xrpc/src/outbox.rs | 1514 + knot2/crates/knot-xrpc/src/plc.rs | 557 + knot2/crates/knot-xrpc/src/query.rs | 256 + knot2/crates/knot-xrpc/src/reads.rs | 1823 + knot2/crates/knot-xrpc/src/receive.rs | 297 + knot2/crates/knot-xrpc/src/refrecords.rs | 338 + knot2/crates/knot-xrpc/src/repos.rs | 865 + knot2/crates/knot-xrpc/src/reservations.rs | 198 + knot2/crates/knot-xrpc/src/service.rs | 93 + knot2/crates/knot-xrpc/src/social.rs | 3289 ++ knot2/crates/knot-xrpc/src/tests.rs | 7857 +++ knot2/crates/knot-xrpc/tests/atproto.rs | 673 + .../crates/knot-xrpc/tests/atproto_interop.rs | 526 + .../blob_properties.proptest-regressions | 7 + .../crates/knot-xrpc/tests/blob_properties.rs | 381 + knot2/crates/knot-xrpc/tests/blobs.rs | 661 + knot2/crates/knot-xrpc/tests/common/mod.rs | 1546 + knot2/crates/knot-xrpc/tests/grant_records.rs | 224 + knot2/crates/knot-xrpc/tests/handover.rs | 1521 + knot2/crates/knot-xrpc/tests/lfs.rs | 303 + knot2/crates/knot-xrpc/tests/lfs_soak.rs | 159 + knot2/crates/knot-xrpc/tests/mint.rs | 217 + .../knot-xrpc/tests/policy_reconcile.rs | 96 + knot2/crates/knot-xrpc/tests/projection.rs | 357 + knot2/crates/knot-xrpc/tests/reads.rs | 1989 + knot2/crates/knot-xrpc/tests/ref_records.rs | 613 + knot2/crates/knot-xrpc/tests/retire.rs | 503 + .../crates/knot-xrpc/tests/spindle_outbox.rs | 90 + knot2/example.toml | 845 + knot2/interop/atproto_firehose_test.go | 219 + knot2/interop/atproto_plc_attestation_test.go | 68 + knot2/interop/atproto_reads_test.go | 180 + knot2/interop/atproto_ticket_test.go | 175 + knot2/interop/foster_test.go | 347 + knot2/interop/knotfeed_consumer_test.go | 104 + knot2/justfile | 169 + .../gix-imara-diff/.cargo_vcs_info.json | 6 + knot2/third_party/gix-imara-diff/Cargo.toml | 82 + knot2/third_party/gix-imara-diff/LICENSE | 201 + knot2/third_party/gix-imara-diff/README.md | 114 + .../gix-imara-diff/src/histogram.rs | 109 + .../gix-imara-diff/src/histogram/lcs.rs | 139 + .../gix-imara-diff/src/histogram/list_pool.rs | 259 + .../third_party/gix-imara-diff/src/intern.rs | 258 + knot2/third_party/gix-imara-diff/src/lib.rs | 525 + knot2/third_party/gix-imara-diff/src/myers.rs | 304 + .../gix-imara-diff/src/myers/middle_snake.rs | 278 + .../gix-imara-diff/src/myers/preprocess.rs | 231 + .../gix-imara-diff/src/myers/slice.rs | 80 + .../gix-imara-diff/src/postprocess.rs | 162 + .../gix-imara-diff/src/slider_heuristic.rs | 396 + .../third_party/gix-imara-diff/src/sources.rs | 232 + .../gix-imara-diff/src/unified_diff.rs | 281 + knot2/third_party/gix-imara-diff/src/util.rs | 135 + .../third_party/gix-pack/.cargo_vcs_info.json | 6 + knot2/third_party/gix-pack/Cargo.toml | 238 + knot2/third_party/gix-pack/LICENSE-APACHE | 177 + knot2/third_party/gix-pack/LICENSE-MIT | 17 + knot2/third_party/gix-pack/src/bundle/find.rs | 75 + knot2/third_party/gix-pack/src/bundle/init.rs | 46 + knot2/third_party/gix-pack/src/bundle/mod.rs | 59 + .../gix-pack/src/bundle/write/error.rs | 17 + .../gix-pack/src/bundle/write/mod.rs | 444 + .../gix-pack/src/bundle/write/types.rs | 97 + .../gix-pack/src/cache/delta/from_offsets.rs | 169 + .../gix-pack/src/cache/delta/mod.rs | 25 + .../gix-pack/src/cache/delta/traverse/mod.rs | 295 + .../src/cache/delta/traverse/resolve.rs | 601 + .../gix-pack/src/cache/delta/tree.rs | 352 + knot2/third_party/gix-pack/src/cache/lru.rs | 292 + knot2/third_party/gix-pack/src/cache/mod.rs | 97 + .../third_party/gix-pack/src/cache/object.rs | 116 + knot2/third_party/gix-pack/src/data/delta.rs | 127 + .../gix-pack/src/data/entry/decode.rs | 228 + .../gix-pack/src/data/entry/header.rs | 164 + .../gix-pack/src/data/entry/mod.rs | 65 + .../gix-pack/src/data/file/decode/entry.rs | 540 + .../gix-pack/src/data/file/decode/header.rs | 164 + .../gix-pack/src/data/file/decode/mod.rs | 31 + .../gix-pack/src/data/file/init.rs | 82 + .../third_party/gix-pack/src/data/file/mod.rs | 9 + .../gix-pack/src/data/file/verify.rs | 62 + knot2/third_party/gix-pack/src/data/header.rs | 57 + .../src/data/input/bytes_to_entries.rs | 332 + .../src/data/input/entries_to_bytes.rs | 162 + .../gix-pack/src/data/input/entry.rs | 68 + .../data/input/lookup_ref_delta_objects.rs | 236 + .../gix-pack/src/data/input/mod.rs | 41 + .../gix-pack/src/data/input/types.rs | 70 + knot2/third_party/gix-pack/src/data/mod.rs | 175 + .../gix-pack/src/data/output/bytes.rs | 171 + .../gix-pack/src/data/output/count/mod.rs | 57 + .../src/data/output/count/objects/mod.rs | 501 + .../data/output/count/objects/reachable.rs | 216 + .../src/data/output/count/objects/reduce.rs | 40 + .../src/data/output/count/objects/tree.rs | 129 + .../src/data/output/count/objects/types.rs | 96 + .../src/data/output/count/objects/util.rs | 24 + .../src/data/output/entry/iter_from_counts.rs | 458 + .../gix-pack/src/data/output/entry/mod.rs | 195 + .../gix-pack/src/data/output/mod.rs | 41 + knot2/third_party/gix-pack/src/find.rs | 12 + knot2/third_party/gix-pack/src/find_traits.rs | 343 + .../third_party/gix-pack/src/index/access.rs | 312 + .../third_party/gix-pack/src/index/encode.rs | 170 + knot2/third_party/gix-pack/src/index/init.rs | 211 + knot2/third_party/gix-pack/src/index/mod.rs | 163 + .../gix-pack/src/index/traverse/error.rs | 44 + .../gix-pack/src/index/traverse/mod.rs | 242 + .../gix-pack/src/index/traverse/reduce.rs | 135 + .../gix-pack/src/index/traverse/types.rs | 116 + .../gix-pack/src/index/traverse/with_index.rs | 258 + .../src/index/traverse/with_lookup.rs | 208 + knot2/third_party/gix-pack/src/index/util.rs | 23 + .../third_party/gix-pack/src/index/verify.rs | 285 + .../gix-pack/src/index/write/error.rs | 27 + .../gix-pack/src/index/write/mod.rs | 385 + knot2/third_party/gix-pack/src/lib.rs | 104 + .../gix-pack/src/multi_index/access.rs | 149 + .../gix-pack/src/multi_index/chunk.rs | 318 + .../gix-pack/src/multi_index/init.rs | 189 + .../gix-pack/src/multi_index/mod.rs | 53 + .../gix-pack/src/multi_index/verify.rs | 343 + .../gix-pack/src/multi_index/write.rs | 289 + knot2/third_party/gix-pack/src/verify.rs | 65 + knotfeed/frame.go | 330 + knotfeed/frame_test.go | 236 + knotfeed/legacy.go | 140 + knotfeed/legacy_test.go | 273 + knotfeed/record.go | 288 + knotfeed/record_test.go | 171 + knotfeed/subscribe.go | 397 + knotfeed/subscribe_test.go | 376 + ...bede293bb1e45b7a6a1aec0af40a729cf89d0.json | 14 + ...1c474657fa4a1f7899135a690b98cc59a6f97.json | 17 + ...b9d9751d1ffec1101bf42196c52fff6047a5a.json | 17 + ...38feedf8c3f714eabe06dc1e2aba3dddb0c62.json | 18 + ...1f5c01b8576aa4b8782b97c20aeed78009328.json | 26 + ...8d5665152b7431776b565fa9ed9324090a78d.json | 16 + ...20a0367e03cd610a742b878b416eeb9a3f561.json | 65 + ...5f20cb2a40f8a2f3c89b0a07115caf920c876.json | 38 + ...97238098885401886cc6aab8d386405c1370c.json | 14 + ...9e61a7c74c77ae3418ea43a9bc929d38f324e.json | 14 + ...120962bc21fe3a3303e88bcb7d071d817c253.json | 16 + ...ba285f3a15b97646b005ef28a2975217ba20a.json | 24 + ...0dac22678ede2b8949cd557b6ce704d79b0b7.json | 26 + ...6b42d7b656676e2465608ff0618b702735270.json | 17 + ...95227d765603eb5299393e1ba39af09222fa7.json | 15 + ...e291b13885004da7d0c98201c98e98e805649.json | 14 + ...4f120f79ba4de8b635035ea61883ee1ff4595.json | 16 + ...43c3d663f73924e887886e15aa971c79f5053.json | 25 + ...b6b0a7a33ecf56143fa25e76b2e27df7de3c2.json | 89 + ...e6af8677050b0984519f7150e3b755511768f.json | 40 + ...a3997759e6abe4478ea7020bbe9cf133168bc.json | 65 + ...90746f85e8fc0d88ed641ec9f901ad3fcacfa.json | 14 + ...5f0ea4a5d3323784cd4f261efb99aa02679fc.json | 38 + ...0891856550ef9da693a41d18cd62189aad307.json | 15 + ...ab230c726baf267f68f625aa1cdbcba5bd312.json | 17 + ...6f9266ffce82420c0e8b00c8e8b1e863aa938.json | 55 + knotmirror/crates/knotmirror-db/Cargo.toml | 20 + .../migrations/20260916000000_initial.sql | 29 + .../migrations/20260926000000_index.sql | 18 + .../migrations/20261019000000_atproto_rev.sql | 1 + .../20261029000000_commit_activity.sql | 27 + .../20261030000000_commit_details.sql | 13 + knotmirror/crates/knotmirror-db/src/lib.rs | 736 + .../knotmirror-db/tests/commit_activity.rs | 182 + knotmirror/crates/knotmirror-git/Cargo.toml | 18 + .../crates/knotmirror-git/src/layout.rs | 73 + knotmirror/crates/knotmirror-git/src/lib.rs | 8 + .../crates/knotmirror-git/src/repo_ext.rs | 113 + .../crates/knotmirror-git/src/scratch.rs | 30 + knotmirror/crates/knotmirror-index/Cargo.toml | 32 + .../crates/knotmirror-index/src/activity.rs | 332 + knotmirror/crates/knotmirror-index/src/lib.rs | 204 + .../crates/knotmirror-ingest/Cargo.toml | 23 + .../crates/knotmirror-ingest/src/lib.rs | 328 + .../crates/knotmirror-maintenance/Cargo.toml | 22 + .../crates/knotmirror-maintenance/src/lib.rs | 118 + knotmirror/crates/knotmirror-sync/Cargo.toml | 39 + .../crates/knotmirror-sync/src/admission.rs | 35 + knotmirror/crates/knotmirror-sync/src/lib.rs | 328 + knotmirror/crates/knotmirror-sync/src/task.rs | 433 + knotmirror/crates/knotmirror-types/Cargo.toml | 16 + .../knotmirror-types/src/jacquard_ext.rs | 29 + knotmirror/crates/knotmirror-types/src/lib.rs | 135 + knotmirror/crates/knotmirror-xrpc/Cargo.toml | 63 + knotmirror/crates/knotmirror-xrpc/clippy.toml | 3 + .../crates/knotmirror-xrpc/src/archive.rs | 170 + .../crates/knotmirror-xrpc/src/committers.rs | 113 + .../crates/knotmirror-xrpc/src/did_ext.rs | 29 + knotmirror/crates/knotmirror-xrpc/src/diff.rs | 418 + .../crates/knotmirror-xrpc/src/error.rs | 174 + .../knotmirror-xrpc/src/format_patch.rs | 2461 + .../knotmirror-xrpc/src/git_transport.rs | 446 + .../crates/knotmirror-xrpc/src/last_commit.rs | 556 + knotmirror/crates/knotmirror-xrpc/src/lib.rs | 141 + .../crates/knotmirror-xrpc/src/merge.rs | 293 + .../crates/knotmirror-xrpc/src/metrics.rs | 352 + .../knotmirror-xrpc/src/routes/activity.rs | 338 + .../crates/knotmirror-xrpc/src/routes/cob.rs | 581 + .../crates/knotmirror-xrpc/src/routes/git.rs | 2288 + .../crates/knotmirror-xrpc/src/routes/mod.rs | 4 + .../knotmirror-xrpc/src/routes/reads.rs | 1467 + .../crates/knotmirror-xrpc/src/signer.rs | 70 + .../crates/knotmirror-xrpc/src/sniff.rs | 267 + knotmirror/crates/knotmirror/Cargo.toml | 51 + knotmirror/crates/knotmirror/src/config.rs | 272 + knotmirror/crates/knotmirror/src/main.rs | 528 + lexicon-build-config.json | 22 + lexicons/com/atproto/label/defs.json | 152 + .../com/atproto/label/subscribeLabels.json | 50 + lexicons/com/atproto/repo/strongRef.json | 15 + lexicons/com/atproto/sync/subscribeRepos.json | 222 + lexicons/com/bad-example/pocket/defs.json | 18 + .../bad-example/pocket/getPreferences.json | 30 + .../bad-example/pocket/putPreferences.json | 32 + lexicons/git/defs.json | 98 + lexicons/git/mergeCommit.json | 85 + lexicons/git/temp/listIndexedLanguages.json | 74 + lexicons/git/temp/listLanguages.json | 86 + lexicons/git/temp2/getBlame.json | 136 + lexicons/git/temp2/getCommitStats.json | 73 + lexicons/git/temp2/getDiff.json | 51 + lexicons/git/temp2/getInterdiff.json | 59 + lexicons/git/temp2/mergeCheck.json | 75 + lexicons/identity/resolveCommitters.json | 53 + lexicons/org/tangled/actor/defs.json | 65 + lexicons/org/tangled/actor/getProfile.json | 41 + .../org/tangled/actor/getProfileActivity.json | 205 + lexicons/org/tangled/actor/getProfiles.json | 58 + lexicons/org/tangled/actor/getTrending.json | 35 + lexicons/org/tangled/actor/profile.json | 86 + .../tangled/actor/searchActorsTypeahead.json | 63 + lexicons/org/tangled/bobbin/awaitRecord.json | 71 + lexicons/org/tangled/ci/approvePipeline.json | 43 + lexicons/org/tangled/ci/cancelPipeline.json | 53 + lexicons/org/tangled/ci/defs.json | 219 + lexicons/org/tangled/ci/getPipeline.json | 44 + .../org/tangled/ci/getWorkflowDefinition.json | 56 + .../tangled/ci/listWorkflowDefinitions.json | 58 + lexicons/org/tangled/ci/queryPipelines.json | 92 + lexicons/org/tangled/ci/retryPipeline.json | 64 + .../org/tangled/ci/subscribePipelineLogs.json | 147 + lexicons/org/tangled/ci/trigger/manual.json | 55 + lexicons/org/tangled/ci/trigger/schedule.json | 34 + lexicons/org/tangled/embed/commit.json | 29 + .../org/tangled/embed/unspecced/file.json | 19 + lexicons/org/tangled/event/notifyEvent.json | 64 + lexicons/org/tangled/event/pullRequest.json | 47 + lexicons/org/tangled/event/push.json | 46 + lexicons/org/tangled/feed/comment.json | 42 + lexicons/org/tangled/feed/countComments.json | 39 + .../org/tangled/feed/countCommentsBy.json | 39 + lexicons/org/tangled/feed/countReactions.json | 39 + .../org/tangled/feed/countReactionsBy.json | 39 + lexicons/org/tangled/feed/countStars.json | 38 + lexicons/org/tangled/feed/countStarsBy.json | 39 + lexicons/org/tangled/feed/defs.json | 37 + lexicons/org/tangled/feed/getStar.json | 38 + lexicons/org/tangled/feed/getTimeline.json | 98 + lexicons/org/tangled/feed/listComments.json | 72 + lexicons/org/tangled/feed/listCommentsBy.json | 59 + lexicons/org/tangled/feed/listReactions.json | 71 + .../org/tangled/feed/listReactionsBy.json | 59 + lexicons/org/tangled/feed/listStars.json | 70 + lexicons/org/tangled/feed/listStarsBy.json | 59 + lexicons/org/tangled/feed/reaction.json | 33 + lexicons/org/tangled/feed/star.json | 42 + lexicons/org/tangled/feed/subscription.json | 46 + lexicons/org/tangled/git/defs.json | 235 + lexicons/org/tangled/git/deleteBranch.json | 30 + lexicons/org/tangled/git/getArchive.json | 63 + lexicons/org/tangled/git/getBlame.json | 139 + lexicons/org/tangled/git/getBlob.json | 36 + lexicons/org/tangled/git/getBranch.json | 76 + lexicons/org/tangled/git/getDiff.json | 67 + lexicons/org/tangled/git/getEntry.json | 95 + lexicons/org/tangled/git/getTag.json | 76 + lexicons/org/tangled/git/getTree.json | 105 + lexicons/org/tangled/git/listBranches.json | 105 + lexicons/org/tangled/git/listCommits.json | 79 + lexicons/org/tangled/git/listLanguages.json | 107 + lexicons/org/tangled/git/listRefs.json | 100 + lexicons/org/tangled/git/listTags.json | 117 + lexicons/org/tangled/git/oid.json | 18 + lexicons/org/tangled/git/pushEvent.json | 73 + lexicons/org/tangled/git/receivePack.json | 10 + lexicons/org/tangled/git/refs.json | 30 + .../org/tangled/git/setDefaultBranch.json | 51 + lexicons/org/tangled/graph/attestation.json | 29 + lexicons/org/tangled/graph/collaborator.json | 23 + lexicons/org/tangled/graph/countFollows.json | 39 + .../org/tangled/graph/countFollowsBy.json | 39 + lexicons/org/tangled/graph/follow.json | 23 + lexicons/org/tangled/graph/getFollow.json | 39 + lexicons/org/tangled/graph/listFollows.json | 76 + lexicons/org/tangled/graph/listFollowsBy.json | 59 + .../org/tangled/graph/listNetworkVouches.json | 68 + lexicons/org/tangled/graph/vouch.json | 45 + lexicons/org/tangled/key/sshKey.json | 44 + .../org/tangled/key/unspecced/gpgKey.json | 35 + lexicons/org/tangled/knot/block.json | 23 + lexicons/org/tangled/knot/describeKnot.json | 61 + lexicons/org/tangled/knot/member.json | 23 + .../org/tangled/label/countDefinitions.json | 38 + lexicons/org/tangled/label/definition.json | 30 + lexicons/org/tangled/label/defs.json | 17 + .../org/tangled/label/listDefinitions.json | 70 + lexicons/org/tangled/markup/embed/image.json | 17 + lexicons/org/tangled/markup/embed/video.json | 17 + lexicons/org/tangled/markup/markdown.json | 43 + lexicons/org/tangled/markup/plaintext.json | 16 + lexicons/org/tangled/publicKey/countKeys.json | 39 + .../org/tangled/publicKey/getPublicKey.json | 41 + lexicons/org/tangled/publicKey/listKeys.json | 71 + .../org/tangled/query/enrichResponse.json | 156 + lexicons/org/tangled/repo/artifact.json | 43 + lexicons/org/tangled/repo/countArtifacts.json | 38 + .../org/tangled/repo/countArtifactsBy.json | 39 + .../org/tangled/repo/countCollaborators.json | 39 + .../tangled/repo/countCollaboratorsBy.json | 39 + lexicons/org/tangled/repo/countForks.json | 39 + lexicons/org/tangled/repo/countIssues.json | 49 + lexicons/org/tangled/repo/countPulls.json | 49 + lexicons/org/tangled/repo/countRepos.json | 39 + lexicons/org/tangled/repo/createRepo.json | 84 + lexicons/org/tangled/repo/declaration.json | 26 + lexicons/org/tangled/repo/defs.json | 54 + lexicons/org/tangled/repo/deleteRepo.json | 27 + lexicons/org/tangled/repo/forkSync.json | 28 + .../org/tangled/repo/getImportStatus.json | 48 + lexicons/org/tangled/repo/getRepo.json | 45 + lexicons/org/tangled/repo/getRepoByName.json | 49 + .../org/tangled/repo/getRepoByRepoDid.json | 45 + lexicons/org/tangled/repo/getRepos.json | 58 + .../org/tangled/repo/getReposByRepoDids.json | 46 + lexicons/org/tangled/repo/listArtifacts.json | 70 + .../org/tangled/repo/listArtifactsBy.json | 59 + .../repo/listCollaboratorInvitesBy.json | 75 + .../org/tangled/repo/listCollaborators.json | 100 + .../org/tangled/repo/listCollaboratorsBy.json | 59 + lexicons/org/tangled/repo/listForks.json | 71 + lexicons/org/tangled/repo/listIssues.json | 112 + lexicons/org/tangled/repo/listIssuesBy.json | 69 + lexicons/org/tangled/repo/listPulls.json | 112 + lexicons/org/tangled/repo/listPullsBy.json | 69 + .../org/tangled/repo/listRecordVersions.json | 100 + lexicons/org/tangled/repo/listRepos.json | 71 + lexicons/org/tangled/repo/manifest.json | 76 + lexicons/org/tangled/repo/mintToken.json | 59 + lexicons/org/tangled/repo/reserveKey.json | 41 + lexicons/org/tangled/review/patch.json | 39 + lexicons/org/tangled/review/submitPatch.json | 59 + .../org/tangled/review/unspecced/defs.json | 15 + .../org/tangled/review/unspecced/round.json | 43 + lexicons/org/tangled/search/query.json | 175 + lexicons/org/tangled/secret/addSecret.json | 51 + lexicons/org/tangled/secret/listSecrets.json | 72 + lexicons/org/tangled/secret/removeSecret.json | 45 + lexicons/org/tangled/snippet/string.json | 44 + .../org/tangled/spindle/describeSpindle.json | 61 + lexicons/org/tangled/string/countStrings.json | 38 + lexicons/org/tangled/string/getString.json | 41 + lexicons/org/tangled/string/listStrings.json | 70 + lexicons/org/tangled/temp/actor/defs.json | 171 + .../org/tangled/temp/git/deleteBranch.json | 30 + .../org/tangled/temp/git/formatPatch.json | 51 + lexicons/org/tangled/temp/git/getArchive.json | 56 + lexicons/org/tangled/temp/git/getBlob.json | 38 + lexicons/org/tangled/temp/git/getBranch.json | 67 + lexicons/org/tangled/temp/git/getEntry.json | 81 + .../org/tangled/temp/git/getMergeBase.json | 60 + lexicons/org/tangled/temp/git/getRef.json | 75 + lexicons/org/tangled/temp/git/getTag.json | 71 + lexicons/org/tangled/temp/git/getTree.json | 97 + .../org/tangled/temp/git/listBranches.json | 92 + .../org/tangled/temp/git/listCommits.json | 120 + .../tangled/temp/git/listCommitsByRef.json | 69 + lexicons/org/tangled/temp/git/listTags.json | 112 + .../org/tangled/temp/git/mergeCommit.json | 85 + lexicons/org/tangled/temp/git/syncFork.json | 36 + .../org/tangled/temp/repo/getRepoByName.json | 75 + .../tangled/temp/repo/listRecordOrigins.json | 110 + .../org/tangled/temp/server/ackHandover.json | 33 + .../tangled/temp/server/cancelHandover.json | 36 + .../tangled/temp/server/createInviteCode.json | 31 + .../org/tangled/temp/server/handoverRepo.json | 44 + .../tangled/temp/server/redeemInviteCode.json | 20 + .../org/tangled/temp/server/seedRepo.json | 106 + .../org/tangled/temp/server/uploadBlob.json | 36 + .../tangled/temp/spindle/allowList/add.json | 24 + .../tangled/temp/spindle/allowList/list.json | 24 + .../temp/spindle/allowList/remove.json | 24 + lexicons/org/tangled/track/defs.json | 80 + lexicons/org/tangled/track/getTicketView.json | 26 + lexicons/org/tangled/track/ticket.json | 58 + .../tangled/track/unspecced/milestone.json | 15 + .../org/tangled/webhook/createWebhook.json | 62 + lexicons/org/tangled/webhook/defs.json | 73 + .../org/tangled/webhook/deleteWebhook.json | 31 + .../webhook/listWebhookDeliveries.json | 48 + .../org/tangled/webhook/listWebhooks.json | 37 + .../tangled/webhook/retryWebhookDelivery.json | 36 + .../org/tangled/webhook/updateWebhook.json | 48 + lexicons/repo/listCobVersions.json | 112 + lexicons/temp/account/addEmail.json | 26 + lexicons/temp/account/beginSignup.json | 34 + lexicons/temp/account/completeSignup.json | 76 + lexicons/temp/account/deleteEmail.json | 26 + lexicons/temp/account/dismissNewsletter.json | 10 + .../temp/account/getNewsletterPreference.json | 22 + lexicons/temp/account/getSignup.json | 42 + lexicons/temp/account/listEmails.json | 48 + lexicons/temp/account/resendSignup.json | 29 + lexicons/temp/account/setPrimaryEmail.json | 27 + .../temp/account/subscribeNewsletter.json | 10 + lexicons/temp/account/verifyEmail.json | 39 + lexicons/temp/focus/beginSession.json | 43 + lexicons/temp/focus/endSession.json | 10 + lexicons/temp/focus/nextItem.json | 50 + lexicons/temp/migrator/createTask.json | 62 + lexicons/temp/migrator/defs.json | 137 + lexicons/temp/migrator/describeSource.json | 48 + lexicons/temp/migrator/getTask.json | 35 + lexicons/temp/migrator/listTasks.json | 40 + lexicons/temp/migrator/retryJob.json | 51 + lexicons/temp/newsletter/signup.json | 26 + .../temp/notification/deleteNotification.json | 23 + .../temp/notification/getPreferences.json | 44 + .../temp/notification/getUnreadCount.json | 22 + .../temp/notification/listNotifications.json | 102 + .../temp/notification/listRecipients.json | 38 + lexicons/temp/notification/markAllRead.json | 10 + .../temp/notification/markEntityRead.json | 24 + .../temp/notification/updatePreferences.json | 28 + lexicons/temp/notification/updateSeen.json | 26 + lexicons/temp/repo/deploySite.json | 33 + lexicons/temp/repo/disableSite.json | 31 + lexicons/temp/repo/getDeployHistory.json | 80 + lexicons/temp/repo/getSiteConfig.json | 57 + lexicons/temp/repo/updateSiteConfig.json | 61 + lexicons/temp/review/listPatches.json | 71 + lexicons/temp/search/searchCode.json | 127 + lexicons/temp/site/claimDomain.json | 31 + lexicons/temp/site/getDomainClaim.json | 25 + lexicons/temp/site/releaseDomain.json | 27 + lexicons/temp/spindle/moderation/ban.json | 31 + lexicons/temp/spindle/moderation/defs.json | 26 + lexicons/temp/spindle/moderation/getBan.json | 39 + .../temp/spindle/moderation/listBans.json | 29 + lexicons/temp/spindle/moderation/unban.json | 34 + lexicons/temp/spindle/quota/defs.json | 63 + lexicons/temp/spindle/quota/get.json | 44 + lexicons/temp/spindle/quota/list.json | 29 + lexicons/temp/spindle/quota/releaseNars.json | 59 + lexicons/temp/spindle/quota/set.json | 39 + lexicons/temp/spindle/quota/unset.json | 31 + lexicons/temp/spindle/quota/usage.json | 47 + lexutil/client.go | 243 + lexutil/client_test.go | 700 + lexutil/lexutil.go | 49 + license | 22 + localinfra/cmd/deliberi-bootstrap/main.go | 128 + .../cmd/deliberi-bootstrap/main_test.go | 125 + localinfra/cmd/seed/main.go | 443 + ...mrj4iuxiceb6srr2boltkbbf7yizwtgwurqv42sltm | 16 + ...krquhibb3csu6i3kogd76uk6ktptc7s5exim6rfu6u | 22 + ...7dw35cfylzmb3htqg7kpksf76pvascxlv3s56rb7o4 | 4 + ...ngvjho6rx7dbvmlrbuzfybf5ygdxxqhpjdef4.json | 7 + ...3egr3yutd26fsmq2kfdaibiccx7sq57qwvuty.json | 9 + ...r35tg7fslxikhjfffulmovin3q6ngfbsyqkle.json | 13 + ...2u2fvh7dwfxanrpu2t5a6iut2pf2h6tov5z7i.json | 17 + ...js2kbcotijs6qhu2rsbm3rleskancgxb5mk6y.json | 20 + ...inhabpmw4i4n7bev7wadlaoahfwnivogujupy.json | 24 + ...bqflygph5msxghmuw7buyph3dbq6btlkwoi3y.json | 20 + ...lysndsvhum5x3d57wllrxbelmrhgvp6lfvnli.json | 20 + ...ek4sltctxu4ub4v6dtglfjwdlyfvnjc6ulmpi.json | 13 + ...m5j7ybz2ufvgnszzgnwtd66d3qtoenla7ohma.json | 10 + ...wrzp7wcnndbp46g7acfhhque5i2y53ndek4rm.json | 10 + ...r2kldwcjutvsvmdh7hptnx7bmwhprvoeb75fi.json | 10 + ...x552ivpes3epirw6dpsvzo546kc4agy7r4gtm.json | 7 + ...ma3gqqnr3ldjgpspdkfqfu3feolsrftnxznne.json | 6 + ...osezrcmdxgutjctoresjsv3r4pfbwvlnqyre4.json | 6 + ...cnvfhqt6thqx76ol5sqmanzhkiru24kdgple4.json | 6 + ...m6hnid3i7ius6kdkkyph7afjc7jjp2fb7bz6e.json | 6 + ...373tkf4hoimhfc5t2jdrmsgqkblalrcl52y7a.json | 13 + ...vssd4pnbyqjtzl5n4nc5gqbgpbsdekdf3mfwu.json | 12 + ...xtgqfj3o72mlgdp3ncfvvolfwce535kitqwni.json | 12 + ...beog2h2g7vg5g67shjkk2hukto76jo4gjgb3u.json | 12 + ...c5wc4a3ikmg5ytard3b6g5plli3t3p5rkbuf4.json | 12 + ...djoqzlim6a32vc7jfpa7pxub5ctv2c7eveavm.json | 12 + ...6yy2evebgr6yv5ajgdztxoodaihsoddfn3g4u.json | 12 + ...nen3s7cdmdyczm6t37y6eof36aoobmeclv2tq.json | 12 + ...cmulqkyvcdrrbakd5amj2vswxcoukgsl7bo3a.json | 12 + ...yd26ju4uadsdq3ikpff2hofr5cgnah356mnhu.json | 20 + ...ex4yepqgq5kez64vm56trqlhsnrup37am6r2q.json | 14 + ...imydbxyvini2xvmuux4ambboqyrv24nnuvcee.json | 20 + ...sqkxtncjxoe324kpklmdd4qnttvuptadfjh2y.json | 15 + ...5gt4haea6qxekd4lbu2ant2c3m5yqucqyz2wi.json | 21 + ...pqrlgt4govlivo6iy6id3vshljqgt4awnof4u.json | 25 + ...6aulj6hixoojfmifrdjbic2coo7musy6gutla.json | 15 + ...xfep5ztun3tjfkhpm54odte3gjh2j6oiwko4y.json | 15 + ...qiw6kfsgov7rh4h4fdzx4xqhk5q3wosuk7gbi.json | 16 + ...gmhhdxqdolkwxycugm3hhppjh5d3ht4ww3umi.json | 9 + ...ox4etvcptekjj7aa5d5jmpniljh3hfqe5zt24.json | 8 + ...uevdxhapoginh2snmu4vuyuwzkvjvnmfnckm4.json | 8 + ...unztrrxqhr3qkbvqwsscrvnukpjqedszx7tnq.json | 8 + ...put3prbhbkyyz7r4pf3r6ejwu6nqoj4u5ehmq.json | 8 + ...pmt4toeiq4u46fbqtdhos54dmvcuqckjqkhkq.json | 8 + ...gohfym2fp7oanx4cbvkjubatnxrdkpl7n62zm.json | 8 + ...ycyxy3eomtcgheevpsnd6vevshtihb65cwqwe.json | 8 + ...2juei6wcmh2fpjxqp23urpivj4kp6xnccl6pa.json | 4 + ...l5umvrrewoatpia5hgjuckybf3tfk4y5bnm3i.json | 4 + ...kw45gol7z3w67ktf27dlvluu2h4a6m54fx3ga.json | 12 + ...dnag57vntcoc7ye2oq22fgsqirkjgwbgv6gx4.json | 5 + ...zbei2thp7wdxepgzeitnmv2aj7ww7sc5wq3ne.json | 17 + ...xrcqisv6ctlza4mfkhxdcs5iuba3t6kfagh3y.json | 17 + ...zi2t7rxlvovw3cz6ivk5l2d7omuptcfje7a6i.json | 4 + ...houevcgx7kbay4b4nj2og5qlpfjh6sugrdux4.json | 6 + ...x552ivpes3epirw6dpsvzo546kc4agy7r4gtm.json | 7 + ...unl4v5ki6nyazawttswpas3giafc7dgd7uuza.json | 13 + ...4cl6mshfgdicv45ple4hrxrrgd7b3t4masruy.json | 4 + ...2sbm4xavv7jqzvec6hbwqypinudwonkoukvti.json | 4 + ...q2och6fomjpne7kleyfvt75s5si3px76gdcoy.json | 6 + ...cu2x56ptgvynt4eb3g535fuf5azyfranlxok4.json | 12 + ...6cky7p7kjgl5eb3xxwye3kk6ajbtvb2lpxc5m.json | 8 + ...ymftts44zumyz5m6qwccqrpt4gihmnu652qae.json | 4 + ...mppnrnmicfzcty5d42kjz24igyktw6ekxfhpa.json | 6 + ...5xdda7b66ycxnt6ct6w5gyfzb6mlozgbqxlg4.json | 6 + ...u4zyheqhyk24dbxtlcqxowgfpz5g2wpeclkgm.json | 5 + ...zxofcfagfld3dg2qslubaewelbpkoz5qsepcy.json | 5 + ...rm35hxpdm2h2jyuffg2fatpxpo7iyxfnuk5ka.json | 16 + localinfra/github-stub/__init__.py | 1 + localinfra/github-stub/__main__.py | 4 + localinfra/github-stub/server.py | 673 + localinfra/ncps-secret-key | 1 + localinfra/ncps.yaml | 16 + .../provisioning/dashboards/bobbin.json | 1777 + .../provisioning/dashboards/spindle.json | 2547 + .../provisioning/dashboards/spindle.yml | 11 + .../provisioning/datasources/prometheus.yml | 44 + localinfra/observability/loki/loki.yml | 32 + .../observability/prometheus/prometheus.yml | 9 + localinfra/observability/tempo/tempo.yml | 23 + localinfra/ozone.env | 14 + localinfra/pc/Caddyfile | 59 + localinfra/pc/README.md | 114 + localinfra/pc/airgap.sb | 5 + localinfra/pc/bin/doctor | 67 + localinfra/pc/bin/env.sh | 25 + localinfra/pc/bin/linux-ns | 51 + localinfra/pc/bin/ns-inner | 31 + localinfra/pc/bin/offline | 7 + localinfra/pc/bin/patch-workerd | 9 + localinfra/pc/bin/pc | 4 + localinfra/pc/bin/profile-on | 6 + localinfra/pc/bin/selftest | 31 + localinfra/pc/bin/services | 22 + localinfra/pc/bin/setup | 40 + localinfra/pc/bin/svc/caddy | 11 + localinfra/pc/bin/svc/dns | 7 + localinfra/pc/bin/svc/hydrant | 21 + localinfra/pc/bin/svc/init-data | 63 + localinfra/pc/bin/svc/knot2 | 40 + localinfra/pc/bin/svc/pds | 16 + localinfra/pc/bin/svc/pg-init | 6 + localinfra/pc/bin/svc/plc | 4 + localinfra/pc/bin/svc/postgres | 8 + localinfra/pc/bin/tcp-up | 4 + localinfra/pc/bin/up | 36 + localinfra/pc/commands.env | 7 + localinfra/pc/ports.env | 16 + localinfra/pc/process-compose.yaml | 106 + localinfra/pc/services/README.md | 28 + localinfra/pc/services/avatar/Caddyfile | 4 + localinfra/pc/services/avatar/commands.env | 2 + localinfra/pc/services/avatar/doctor | 3 + localinfra/pc/services/avatar/packages | 2 + localinfra/pc/services/avatar/ports.env | 2 + localinfra/pc/services/avatar/process.yaml | 19 + localinfra/pc/services/avatar/run | 11 + localinfra/pc/services/avatar/setup | 6 + localinfra/pc/services/bobbin/Caddyfile | 8 + localinfra/pc/services/bobbin/commands.env | 2 + localinfra/pc/services/bobbin/doctor | 3 + localinfra/pc/services/bobbin/ports.env | 1 + localinfra/pc/services/bobbin/process.yaml | 19 + localinfra/pc/services/bobbin/run | 23 + localinfra/pc/services/bobbin/setup | 5 + localinfra/pc/services/camo/Caddyfile | 4 + localinfra/pc/services/camo/commands.env | 2 + localinfra/pc/services/camo/doctor | 3 + localinfra/pc/services/camo/packages | 2 + localinfra/pc/services/camo/ports.env | 2 + localinfra/pc/services/camo/process.yaml | 17 + localinfra/pc/services/camo/run | 10 + localinfra/pc/services/camo/setup | 6 + localinfra/pc/services/deliberi/Caddyfile | 10 + localinfra/pc/services/deliberi/commands.env | 4 + localinfra/pc/services/deliberi/doctor | 4 + localinfra/pc/services/deliberi/packages | 2 + localinfra/pc/services/deliberi/ports.env | 1 + localinfra/pc/services/deliberi/process.yaml | 23 + localinfra/pc/services/deliberi/run | 28 + localinfra/pc/services/deliberi/setup | 8 + localinfra/pc/services/github-stub/Caddyfile | 4 + .../pc/services/github-stub/commands.env | 1 + localinfra/pc/services/github-stub/doctor | 3 + localinfra/pc/services/github-stub/packages | 2 + localinfra/pc/services/github-stub/ports.env | 1 + .../pc/services/github-stub/process.yaml | 15 + localinfra/pc/services/github-stub/run | 16 + localinfra/pc/services/github-stub/setup | 9 + localinfra/pc/services/grafana/commands.env | 1 + localinfra/pc/services/grafana/doctor | 1 + localinfra/pc/services/grafana/packages | 1 + localinfra/pc/services/grafana/ports.env | 1 + localinfra/pc/services/grafana/process.yaml | 19 + localinfra/pc/services/grafana/profile | 1 + localinfra/pc/services/grafana/run | 39 + localinfra/pc/services/knotmirror/Caddyfile | 4 + .../pc/services/knotmirror/commands.env | 2 + localinfra/pc/services/knotmirror/doctor | 3 + localinfra/pc/services/knotmirror/packages | 1 + localinfra/pc/services/knotmirror/ports.env | 2 + .../pc/services/knotmirror/process.yaml | 21 + localinfra/pc/services/knotmirror/run | 24 + localinfra/pc/services/knotmirror/setup | 5 + localinfra/pc/services/loki/commands.env | 1 + localinfra/pc/services/loki/doctor | 1 + localinfra/pc/services/loki/packages | 1 + localinfra/pc/services/loki/ports.env | 2 + localinfra/pc/services/loki/process.yaml | 12 + localinfra/pc/services/loki/profile | 1 + localinfra/pc/services/loki/run | 49 + localinfra/pc/services/migrator/Caddyfile | 4 + localinfra/pc/services/migrator/commands.env | 2 + localinfra/pc/services/migrator/doctor | 3 + localinfra/pc/services/migrator/packages | 2 + localinfra/pc/services/migrator/ports.env | 1 + localinfra/pc/services/migrator/process.yaml | 19 + localinfra/pc/services/migrator/run | 30 + localinfra/pc/services/migrator/setup | 27 + .../pc/services/ncps-migrate/commands.env | 1 + localinfra/pc/services/ncps-migrate/doctor | 1 + localinfra/pc/services/ncps-migrate/packages | 1 + .../pc/services/ncps-migrate/process.yaml | 6 + localinfra/pc/services/ncps-migrate/profile | 2 + localinfra/pc/services/ncps-migrate/run | 6 + localinfra/pc/services/ncps/commands.env | 1 + localinfra/pc/services/ncps/doctor | 1 + localinfra/pc/services/ncps/packages | 1 + localinfra/pc/services/ncps/ports.env | 1 + localinfra/pc/services/ncps/process.yaml | 15 + localinfra/pc/services/ncps/profile | 2 + localinfra/pc/services/ncps/run | 27 + localinfra/pc/services/ogre/Caddyfile | 4 + localinfra/pc/services/ogre/commands.env | 1 + localinfra/pc/services/ogre/doctor | 1 + localinfra/pc/services/ogre/packages | 2 + localinfra/pc/services/ogre/ports.env | 2 + localinfra/pc/services/ogre/process.yaml | 14 + localinfra/pc/services/ogre/run | 9 + localinfra/pc/services/ogre/setup | 9 + .../pc/services/otlp-gateway/commands.env | 1 + localinfra/pc/services/otlp-gateway/doctor | 1 + localinfra/pc/services/otlp-gateway/packages | 1 + localinfra/pc/services/otlp-gateway/ports.env | 1 + .../pc/services/otlp-gateway/process.yaml | 17 + localinfra/pc/services/otlp-gateway/profile | 1 + localinfra/pc/services/otlp-gateway/run | 37 + localinfra/pc/services/ozone/Caddyfile | 4 + localinfra/pc/services/ozone/commands.env | 1 + localinfra/pc/services/ozone/doctor | 1 + localinfra/pc/services/ozone/packages | 1 + localinfra/pc/services/ozone/ports.env | 1 + localinfra/pc/services/ozone/process.yaml | 19 + localinfra/pc/services/ozone/run | 17 + localinfra/pc/services/ozone/setup | 26 + localinfra/pc/services/pdsls/Caddyfile | 4 + localinfra/pc/services/pdsls/commands.env | 1 + localinfra/pc/services/pdsls/doctor | 1 + localinfra/pc/services/pdsls/packages | 3 + localinfra/pc/services/pdsls/ports.env | 1 + localinfra/pc/services/pdsls/process.yaml | 18 + localinfra/pc/services/pdsls/profile | 1 + localinfra/pc/services/pdsls/run | 21 + localinfra/pc/services/pdsls/setup | 12 + localinfra/pc/services/pocket/Caddyfile | 4 + localinfra/pc/services/pocket/bind.patch | 10 + localinfra/pc/services/pocket/commands.env | 2 + localinfra/pc/services/pocket/doctor | 3 + localinfra/pc/services/pocket/packages | 1 + localinfra/pc/services/pocket/ports.env | 1 + localinfra/pc/services/pocket/process.yaml | 12 + localinfra/pc/services/pocket/run | 27 + localinfra/pc/services/pocket/setup | 16 + .../pc/services/prometheus/commands.env | 1 + localinfra/pc/services/prometheus/doctor | 1 + localinfra/pc/services/prometheus/packages | 1 + localinfra/pc/services/prometheus/ports.env | 1 + .../pc/services/prometheus/process.yaml | 12 + localinfra/pc/services/prometheus/profile | 1 + localinfra/pc/services/prometheus/run | 19 + localinfra/pc/services/redis/commands.env | 1 + localinfra/pc/services/redis/doctor | 1 + localinfra/pc/services/redis/packages | 1 + localinfra/pc/services/redis/ports.env | 1 + localinfra/pc/services/redis/process.yaml | 9 + localinfra/pc/services/redis/run | 5 + localinfra/pc/services/sites/Caddyfile | 4 + localinfra/pc/services/sites/commands.env | 3 + localinfra/pc/services/sites/doctor | 4 + localinfra/pc/services/sites/packages | 1 + localinfra/pc/services/sites/ports.env | 2 + localinfra/pc/services/sites/process.yaml | 16 + localinfra/pc/services/sites/profile | 1 + localinfra/pc/services/sites/run | 24 + localinfra/pc/services/sites/setup | 8 + localinfra/pc/services/sitesd/Caddyfile | 4 + localinfra/pc/services/sitesd/commands.env | 2 + localinfra/pc/services/sitesd/doctor | 3 + localinfra/pc/services/sitesd/ports.env | 2 + localinfra/pc/services/sitesd/process.yaml | 19 + localinfra/pc/services/sitesd/profile | 1 + localinfra/pc/services/sitesd/run | 19 + localinfra/pc/services/sitesd/setup | 6 + localinfra/pc/services/spindle/Caddyfile | 5 + localinfra/pc/services/spindle/commands.env | 3 + localinfra/pc/services/spindle/doctor | 13 + localinfra/pc/services/spindle/ports.env | 2 + localinfra/pc/services/spindle/process.yaml | 21 + localinfra/pc/services/spindle/profile | 2 + localinfra/pc/services/spindle/run | 54 + localinfra/pc/services/spindle/setup | 36 + localinfra/pc/services/tempo/commands.env | 1 + localinfra/pc/services/tempo/doctor | 1 + localinfra/pc/services/tempo/packages | 1 + localinfra/pc/services/tempo/ports.env | 3 + localinfra/pc/services/tempo/process.yaml | 12 + localinfra/pc/services/tempo/profile | 1 + localinfra/pc/services/tempo/run | 33 + .../pc/services/tranquil-gate/Caddyfile | 4 + .../pc/services/tranquil-gate/commands.env | 2 + localinfra/pc/services/tranquil-gate/doctor | 3 + localinfra/pc/services/tranquil-gate/packages | 0 .../pc/services/tranquil-gate/ports.env | 1 + .../pc/services/tranquil-gate/process.yaml | 23 + localinfra/pc/services/tranquil-gate/run | 18 + localinfra/pc/services/tranquil-gate/setup | 5 + localinfra/pc/services/tranquil/Caddyfile | 4 + localinfra/pc/services/tranquil/commands.env | 2 + localinfra/pc/services/tranquil/doctor | 3 + localinfra/pc/services/tranquil/packages | 2 + localinfra/pc/services/tranquil/ports.env | 1 + localinfra/pc/services/tranquil/process.yaml | 29 + localinfra/pc/services/tranquil/run | 29 + localinfra/pc/services/tranquil/setup | 15 + localinfra/pc/services/web/Caddyfile | 9 + localinfra/pc/services/web/commands.env | 2 + localinfra/pc/services/web/doctor | 3 + localinfra/pc/services/web/packages | 1 + localinfra/pc/services/web/ports.env | 1 + localinfra/pc/services/web/process.yaml | 19 + localinfra/pc/services/web/run | 8 + localinfra/pc/services/web/setup | 33 + localinfra/pc/services/web/web.env | 34 + .../zoekt-tngl-indexserver/commands.env | 1 + .../pc/services/zoekt-tngl-indexserver/doctor | 4 + .../services/zoekt-tngl-indexserver/packages | 1 + .../services/zoekt-tngl-indexserver/ports.env | 1 + .../zoekt-tngl-indexserver/process.yaml | 23 + .../pc/services/zoekt-tngl-indexserver/run | 16 + .../pc/services/zoekt-tngl-indexserver/setup | 16 + .../pc/services/zoekt-webserver/Caddyfile | 11 + .../pc/services/zoekt-webserver/commands.env | 1 + localinfra/pc/services/zoekt-webserver/doctor | 4 + .../pc/services/zoekt-webserver/packages | 1 + .../pc/services/zoekt-webserver/ports.env | 1 + .../pc/services/zoekt-webserver/process.yaml | 17 + localinfra/pc/services/zoekt-webserver/run | 6 + localinfra/pc/services/zoekt-webserver/setup | 16 + localinfra/pds-patch.mjs | 103 + localinfra/pds-theme.css | 384 + localinfra/pds.env | 37 + localinfra/pdsls.patch | 219 + localinfra/pocket.patch | 32 + localinfra/postgres-init.sql | 4 + localinfra/scripts/add-accounts.sh | 35 + localinfra/scripts/init-data.sh | 397 + localinfra/scripts/lib.sh | 104 + localinfra/sites-r2-bridge/index.js | 52 + localinfra/sites-r2-bridge/wrangler.toml | 11 + log/log.go | 378 + log/log_test.go | 383 + migrator/api/describe.go | 72 + migrator/api/describe_test.go | 134 + migrator/api/grant_test.go | 266 + migrator/api/server.go | 576 + migrator/api/server_test.go | 695 + migrator/api/validation.go | 116 + migrator/config/config.go | 189 + migrator/config/config_test.go | 145 + migrator/crypto/crypto.go | 122 + migrator/crypto/crypto_test.go | 84 + migrator/db/db.go | 876 + migrator/db/db_test.go | 595 + migrator/db/models.go | 71 + migrator/git/describe.go | 44 + migrator/git/describe_test.go | 105 + migrator/git/runner.go | 336 + migrator/git/runner_test.go | 143 + migrator/gitserver/server.go | 233 + migrator/gitserver/server_test.go | 243 + migrator/migrator.go | 134 + migrator/oauth/client.go | 490 + migrator/oauth/client_test.go | 310 + migrator/oauth/redirect_test.go | 231 + migrator/oauth/session_test.go | 354 + migrator/oauth/store.go | 119 + migrator/oauth/store_test.go | 135 + migrator/worker/worker.go | 610 + migrator/worker/worker_test.go | 896 + netutil/connectproxy.go | 278 + netutil/connectproxy_test.go | 310 + netutil/ssrf.go | 52 + netutil/ssrf_test.go | 43 + nix/Cargo.nix | 42151 ++++++++++++++++ nix/cargo-nix.nix | 32 + nix/crate-hashes.json | 12 + nix/go-src.nix | 46 + nix/gomod2nix.toml | 801 + nix/microvm/base.nix | 354 + nix/microvm/qemu.nix | 26 + nix/microvm/spindle-vm.nix | 73 + nix/microvm/user-config.nix | 135 + nix/modules/cilogs.nix | 55 + nix/modules/deliberi.nix | 100 + nix/modules/knot-rs.nix | 471 + nix/modules/knotmirror.nix | 129 + nix/modules/migrator.nix | 140 + nix/modules/shuttle.nix | 59 + nix/modules/sitesd.nix | 130 + nix/modules/spindle.nix | 1255 + nix/modules/tranquil-gate.nix | 134 + nix/modules/zoekt-tnglserver.nix | 109 + nix/pkgs/attic-server.nix | 4 + nix/pkgs/attic-server/delete-paths.patch | 171 + nix/pkgs/blog.nix | 62 + nix/pkgs/cilogs.nix | 19 + nix/pkgs/deliberi.nix | 24 + nix/pkgs/docs.nix | 33 + nix/pkgs/dolly.nix | 20 + nix/pkgs/goat.nix | 12 + nix/pkgs/hydrant-bin.nix | 10 + nix/pkgs/hydrant-ffi.nix | 19 + nix/pkgs/lexgen.nix | 18 + nix/pkgs/migrator.nix | 24 + nix/pkgs/ogre-fonts.nix | 10 + nix/pkgs/sitesd.nix | 24 + nix/pkgs/spindle-almalinux-image.nix | 286 + nix/pkgs/spindle-alpine-image.nix | 145 + .../distrust-roots.py | 227 + .../roots/aaa-certificate-services.crt | 25 + .../roots/emsign-ecc-root-ca-g3.crt | 15 + .../roots/emsign-root-cs-ca-g3.crt | 14 + .../roots/emsign-root-tls-ca-g3.crt | 14 + nix/pkgs/spindle-image-helpers.nix | 155 + nix/pkgs/spindle-nixos-image.nix | 62 + nix/pkgs/spindle-qemu-runner.nix | 32 + nix/pkgs/spindle-static-git.nix | 14 + nix/pkgs/spindle.nix | 21 + nix/pkgs/sqlite-lib.nix | 20 + nix/pkgs/web-static-files.nix | 18 + nix/pkgs/zoekt-tngl-indexserver.nix | 18 + nix/pkgs/zoekt-webserver.nix | 28 + notifier/notifier.go | 46 + ogre/.gitignore | 5 + ogre/bun.lock | 443 + ogre/knip.json | 4 + ogre/package.json | 34 + ogre/packages/runtime/index.ts | 56 + ogre/packages/runtime/package.json | 15 + ogre/packages/runtime/types.ts | 10 + ogre/packages/runtime/workerd.ts | 55 + ogre/readme.md | 3 + ogre/src/__tests__/assets/avatar.jpg | Bin 0 -> 31537 bytes ogre/src/__tests__/contrast.test.ts | 119 + ogre/src/__tests__/fixtures.ts | 204 + ogre/src/__tests__/get.test.ts | 91 + ogre/src/__tests__/render.test.ts | 349 + ogre/src/components/cards/commit.tsx | 104 + ogre/src/components/cards/file.tsx | 68 + ogre/src/components/cards/issue.tsx | 51 + ogre/src/components/cards/pipeline.tsx | 109 + ogre/src/components/cards/profile.tsx | 109 + ogre/src/components/cards/pull-request.tsx | 113 + ogre/src/components/cards/repository.tsx | 57 + ogre/src/components/cards/string.tsx | 65 + ogre/src/components/cards/tag.tsx | 82 + ogre/src/components/shared/avatar.tsx | 80 + ogre/src/components/shared/card-footer.tsx | 25 + ogre/src/components/shared/card-header.tsx | 40 + ogre/src/components/shared/chip.tsx | 66 + ogre/src/components/shared/code-panel.tsx | 74 + ogre/src/components/shared/constants.ts | 76 + ogre/src/components/shared/footer-stats.tsx | 55 + ogre/src/components/shared/label-pill.tsx | 41 + .../components/shared/language-circles.tsx | 94 + ogre/src/components/shared/layout.tsx | 48 + ogre/src/components/shared/logo.tsx | 20 + ogre/src/components/shared/metrics.tsx | 48 + ogre/src/components/shared/stat-item.tsx | 19 + ogre/src/components/shared/status-badge.tsx | 56 + ogre/src/icons/lucide.tsx | 64 + ogre/src/index.tsx | 150 + ogre/src/lib/fit.ts | 20 + ogre/src/lib/format.ts | 17 + ogre/src/lib/pluralize.ts | 1 + ogre/src/lib/render.ts | 46 + ogre/src/types.d.ts | 14 + ogre/src/validation.ts | 176 + ogre/tsconfig.json | 19 + ogre/wrangler.dev.jsonc | 29 + ogre/wrangler.jsonc | 54 + orm/orm.go | 165 + orm/run_migration_test.go | 47 + package.json | 1 + pds/preauthorize.mjs | 144 + pds/preauthorize.test.mjs | 410 + pnpm-lock.yaml | 9 + rbac/rbac.go | 353 + rbac/rbac_test.go | 502 + rbac/txn.go | 41 + rbac/util.go | 133 + readme.md | 43 + repoident/knoturl.go | 150 + repoident/knoturl_test.go | 193 + repoident/repoident.go | 40 + repoident/repoident_test.go | 61 + rust-toolchain.toml | 5 + scratch-example.toml | 0 seo-reporter/.gitignore | 1 + seo-reporter/package.json | 22 + seo-reporter/pnpm-lock.yaml | 2151 + seo-reporter/pnpm-workspace.yaml | 6 + seo-reporter/readme.md | 6 + seo-reporter/scripts/dry-report.ts | 4 + seo-reporter/src/gsc.ts | 204 + seo-reporter/src/index.ts | 11 + seo-reporter/src/report.ts | 195 + seo-reporter/test/report.test.ts | 227 + seo-reporter/tsconfig.json | 15 + seo-reporter/wrangler.dev.jsonc | 17 + seo-reporter/wrangler.jsonc | 17 + sets/gen.go | 31 + sets/readme.txt | 35 + sets/set.go | 174 + sets/set_test.go | 411 + shuttle/Cargo.toml | 24 + shuttle/README.md | 1 + shuttle/src/activation.rs | 326 + shuttle/src/cache/mod.rs | 724 + shuttle/src/cache/read_proxy.rs | 27 + shuttle/src/cache/write_proxy.rs | 22 + shuttle/src/command.rs | 447 + shuttle/src/dns_proxy.rs | 255 + shuttle/src/exec.rs | 295 + shuttle/src/gen/file_descriptor_set.bin | Bin 0 -> 323498 bytes .../gen/spindle/agent/v1/spindle.agent.v1.rs | 180 + .../gen/spindle/mill/v1/spindle.mill.v1.rs | 380 + shuttle/src/host_proxy.rs | 92 + shuttle/src/logging.rs | 49 + shuttle/src/main.rs | 64 + shuttle/src/nix_config.rs | 277 + shuttle/src/protocol.rs | 216 + shuttle/src/pty.rs | 264 + shuttle/src/session.rs | 244 + sitemap/migrations/0001_init.sql | 30 + sitemap/migrations/0002_handles.sql | 16 + sitemap/migrations/0003_accounts.sql | 10 + sitemap/migrations/0004_claims.sql | 9 + sitemap/package.json | 23 + sitemap/pnpm-lock.yaml | 2029 + sitemap/pnpm-workspace.yaml | 6 + sitemap/src/db.ts | 413 + sitemap/src/hydrant.ts | 104 + sitemap/src/identity.ts | 49 + sitemap/src/index.ts | 301 + sitemap/src/jetstream.ts | 173 + sitemap/src/sitemap.ts | 77 + sitemap/src/types.ts | 29 + sitemap/src/xml.ts | 72 + sitemap/test/cloudflare-workers.ts | 2 + sitemap/test/d1.ts | 77 + sitemap/test/events.ts | 89 + sitemap/test/jetstream.test.ts | 63 + sitemap/test/node.d.ts | 22 + sitemap/test/sitemap.test.ts | 254 + sitemap/test/sync.test.ts | 692 + sitemap/tsconfig.json | 17 + sitemap/vitest.config.ts | 8 + sitemap/wrangler.dev.jsonc | 47 + sitemap/wrangler.jsonc | 47 + sites/.gitignore | 4 + sites/Cargo.lock | 1401 + sites/Cargo.toml | 32 + sites/migrations/0001_init.sql | 39 + sites/migrations/0002_site_deploys_manual.sql | 23 + sites/src/claim.rs | 1850 + sites/src/kvmap.rs | 125 + sites/src/lib.rs | 327 + sites/src/serviceauth.rs | 846 + sites/src/sites.rs | 414 + sites/src/xrpc.rs | 1838 + sites/wrangler.dev.toml | 24 + sites/wrangler.toml | 44 + sitesd/README.md | 34 + sitesd/cache/cache.go | 11 + sitesd/cloudflare/r2.go | 247 + sitesd/cloudflare/r2_test.go | 86 + sitesd/config/config.go | 29 + sitesd/models/models.go | 53 + sitesd/reposync/reposync.go | 102 + sitesd/reposync/reposync_test.go | 118 + sitesd/sitefeed/feed.go | 339 + sitesd/sitefeed/feed_test.go | 566 + sitesd/sites/sites.go | 94 + sitesd/sites/sites_test.go | 153 + sitesd/sitesd.go | 385 + sitesd/sitesd_test.go | 662 + sitesd/worker/worker.go | 182 + sitesd/worker/worker_test.go | 209 + spindle/TORTURE.md | 120 + spindle/account_job_test.go | 91 + spindle/admission_pause.go | 17 + spindle/agentproto/gen/agent.pb.go | 1289 + spindle/agentproto/protocol.go | 105 + spindle/agentproto/protocol_test.go | 66 + .../agentproto/spindle/agent/v1/agent.proto | 147 + spindle/allowlist_test.go | 73 + spindle/artifactstore/artifactstore.go | 186 + spindle/artifactstore/artifactstore_test.go | 140 + spindle/backfill_later_test.go | 52 + spindle/backfill_marks_test.go | 57 + spindle/backfill_skip_test.go | 65 + spindle/banned_actor_test.go | 63 + spindle/casbin_copy.go | 87 + spindle/collaborator_marks_test.go | 103 + spindle/collaborator_sync_test.go | 32 + spindle/collaborators.go | 79 + spindle/config/config.go | 492 + spindle/config/config_test.go | 711 + spindle/db/allowlist.go | 97 + spindle/db/approve_test.go | 80 + spindle/db/artifacts.go | 51 + spindle/db/artifacts_test.go | 163 + spindle/db/backfill.go | 124 + spindle/db/bans.go | 79 + spindle/db/bans_test.go | 84 + spindle/db/cache.go | 652 + spindle/db/cache_test.go | 672 + spindle/db/collaborators.go | 117 + spindle/db/collaborators_test.go | 94 + spindle/db/db.go | 1627 + spindle/db/delete_test.go | 202 + spindle/db/durable_test.go | 239 + spindle/db/events.go | 286 + spindle/db/events_metrics_test.go | 86 + spindle/db/fd_leak_test.go | 64 + spindle/db/identity_migration_test.go | 270 + spindle/db/invites.go | 73 + spindle/db/jobs.go | 372 + spindle/db/jobs_depth_test.go | 55 + spindle/db/jobs_test.go | 191 + spindle/db/jobs_unreadable_test.go | 35 + spindle/db/log_migration.go | 218 + spindle/db/log_migration_wait_test.go | 37 + spindle/db/mill_state.go | 579 + spindle/db/mill_state_test.go | 673 + spindle/db/mill_tokens.go | 208 + spindle/db/mill_tokens_test.go | 357 + spindle/db/nix_charges_test.go | 55 + spindle/db/nix_grants_test.go | 59 + spindle/db/nix_objects.go | 267 + spindle/db/nix_objects_test.go | 548 + spindle/db/nixos_toplevel_cache.go | 41 + spindle/db/orphans.go | 52 + spindle/db/orphans_test.go | 72 + spindle/db/pipelines.go | 239 + spindle/db/pipelines_kind_test.go | 60 + spindle/db/pipelines_test.go | 293 + spindle/db/pipelines_total_test.go | 35 + spindle/db/pull_events_limit_test.go | 31 + spindle/db/pull_tickets_test.go | 54 + spindle/db/pulls.go | 272 + spindle/db/quota.go | 1027 + spindle/db/quota_charge_test.go | 71 + spindle/db/quota_minutes.go | 150 + spindle/db/quota_minutes_test.go | 330 + spindle/db/quota_release_test.go | 33 + spindle/db/quota_test.go | 488 + spindle/db/quota_workflow_test.go | 189 + spindle/db/repo_pushes.go | 28 + spindle/db/repo_pushes_test.go | 52 + spindle/db/repo_retries.go | 244 + spindle/db/repo_retries_test.go | 284 + spindle/db/repo_turns.go | 40 + spindle/db/repos.go | 173 + spindle/db/repos_test.go | 17 + spindle/db/retry.go | 52 + spindle/db/retry_test.go | 75 + spindle/db/schedule_runs.go | 52 + spindle/db/schedules.go | 123 + spindle/db/schedules_test.go | 188 + spindle/db/webhooks.go | 332 + spindle/db/webhooks_smoke_test.go | 94 + spindle/db/wipes.go | 106 + spindle/debug_keys.go | 156 + spindle/debug_keys_test.go | 195 + spindle/engine/archive_cleanup_test.go | 43 + spindle/engine/archive_inplace_test.go | 78 + spindle/engine/archive_partial_test.go | 38 + spindle/engine/cache.go | 115 + spindle/engine/cache_controller.go | 627 + spindle/engine/cache_evict_test.go | 126 + spindle/engine/cache_prune.go | 202 + spindle/engine/cache_prune_test.go | 219 + spindle/engine/cache_test.go | 865 + spindle/engine/engine.go | 874 + spindle/engine/engine_minutes_test.go | 218 + spindle/engine/engine_queue_timeout_test.go | 195 + spindle/engine/engine_quota_test.go | 257 + spindle/engine/engine_refusal_test.go | 59 + spindle/engine/engine_slot_log_test.go | 60 + spindle/engine/engine_test.go | 1215 + spindle/engine/failure_metrics_test.go | 65 + spindle/engine/manifest.go | 242 + spindle/engine/manifest_test.go | 120 + spindle/engine/mask_test.go | 549 + spindle/engine/placement.go | 10 + spindle/engine/scheduler.go | 185 + spindle/engine/scheduler_test.go | 253 + spindle/engine/slot.go | 79 + spindle/engine/slot_test.go | 149 + spindle/engine/usage.go | 23 + spindle/engine_microvm_linux.go | 24 + spindle/engine_microvm_other.go | 18 + spindle/engine_microvm_other_test.go | 26 + spindle/engines/dummy/engine.go | 127 + spindle/engines/microvm/README.md | 295 + spindle/engines/microvm/agent.go | 472 + spindle/engines/microvm/agent_test.go | 64 + spindle/engines/microvm/args.go | 71 + spindle/engines/microvm/attic.go | 69 + spindle/engines/microvm/attic_test.go | 309 + spindle/engines/microvm/bench-boot.sh | 38 + spindle/engines/microvm/budget.go | 108 + spindle/engines/microvm/budget_test.go | 28 + spindle/engines/microvm/cache.go | 132 + spindle/engines/microvm/cgroup.go | 389 + spindle/engines/microvm/cgroup_oom_test.go | 132 + spindle/engines/microvm/cgroup_other.go | 71 + .../engines/microvm/cgroup_resources_test.go | 20 + spindle/engines/microvm/cgroup_test.go | 167 + spindle/engines/microvm/cid_test.go | 32 + spindle/engines/microvm/debug.go | 401 + spindle/engines/microvm/debug_e2e_test.go | 290 + spindle/engines/microvm/debug_test.go | 140 + spindle/engines/microvm/debugssh.go | 207 + spindle/engines/microvm/debugssh_test.go | 88 + spindle/engines/microvm/dns_proxy.go | 382 + spindle/engines/microvm/dns_proxy_test.go | 75 + spindle/engines/microvm/engine.go | 884 + spindle/engines/microvm/engine_test.go | 195 + spindle/engines/microvm/image.go | 315 + spindle/engines/microvm/image_test.go | 195 + spindle/engines/microvm/models.go | 53 + spindle/engines/microvm/models_test.go | 43 + spindle/engines/microvm/narinfo.go | 140 + spindle/engines/microvm/netns_wrapper.sh.tmpl | 25 + spindle/engines/microvm/networking.go | 121 + spindle/engines/microvm/nixos_toplevel.go | 105 + .../engines/microvm/placement/placement.go | 15 + .../microvm/placement/placement_test.go | 26 + spindle/engines/microvm/placement_linux.go | 61 + spindle/engines/microvm/qemu.go | 824 + spindle/engines/microvm/qemu_lifetime_test.go | 150 + spindle/engines/microvm/runner.go | 26 + .../engines/microvm/start-test-substituter.sh | 86 + spindle/engines/microvm/substituter_proxy.go | 471 + .../engines/microvm/substituter_proxy_test.go | 288 + .../microvm/substituter_upload_attic.go | 81 + .../microvm/substituter_upload_attic_test.go | 407 + .../microvm/substituter_upload_http.go | 139 + .../microvm/substituter_upload_nix_store.go | 541 + .../substituter_upload_nix_store_test.go | 536 + .../microvm/substituter_upload_proxy.go | 206 + .../microvm/substituter_upload_proxy_test.go | 130 + .../engines/microvm/test-spindle-microvm.sh | 1133 + spindle/engines/microvm/upload_quota_test.go | 1196 + spindle/engines/microvm/upload_staging.go | 771 + .../engines/microvm/upload_store_path_test.go | 56 + spindle/engines/microvm/vm.go | 562 + spindle/engines/microvm/vm_test.go | 202 + spindle/engines/nixery/ansi_stripper.go | 21 + spindle/engines/nixery/cache.go | 167 + spindle/engines/nixery/engine.go | 682 + spindle/engines/nixery/engine_test.go | 265 + spindle/engines/nixery/envs.go | 26 + spindle/engines/nixery/envs_test.go | 48 + spindle/engines/nixery/errors.go | 7 + spindle/engines/nixery/seats.go | 138 + spindle/engines/nixery/seats_test.go | 164 + spindle/engines/nixery/setup_steps.go | 57 + spindle/fakes_test.go | 784 + spindle/faults_test.go | 579 + spindle/filter_workflows_test.go | 58 + spindle/http_server_test.go | 73 + spindle/hydrant.go | 1109 + spindle/hydrant_pending_test.go | 248 + spindle/hydrant_test.go | 970 + spindle/identity/identity.go | 62 + spindle/identity/identity_test.go | 95 + spindle/identity_route_test.go | 72 + spindle/ingest_shards.go | 166 + spindle/ingest_shards_test.go | 298 + spindle/ingester.go | 303 + spindle/ingester_enrollment_test.go | 48 + spindle/ingester_test.go | 310 + spindle/instance_lock_other.go | 5 + spindle/instance_lock_test.go | 38 + spindle/instance_lock_unix.go | 29 + spindle/job_workers_test.go | 45 + spindle/knotclient/knotclient.go | 316 + spindle/knotclient/knotclient_test.go | 224 + spindle/load_pipeline_test.go | 115 + spindle/logview/archive_retry_test.go | 76 + spindle/logview/follow_longline_test.go | 75 + spindle/logview/follow_stop_test.go | 67 + spindle/logview/logview.go | 135 + spindle/logview/logview_test.go | 82 + spindle/main_test.go | 22 + spindle/middleware.go | 35 + spindle/mill/README.md | 161 + spindle/mill/auth_test.go | 529 + spindle/mill/cache.go | 64 + spindle/mill/debug_key.go | 60 + spindle/mill/debug_key_test.go | 122 + spindle/mill/engine.go | 109 + spindle/mill/executor/cache.go | 80 + spindle/mill/executor/cache_test.go | 66 + spindle/mill/executor/capability_test.go | 201 + spindle/mill/executor/debug_key_client.go | 108 + .../mill/executor/debug_key_client_test.go | 68 + spindle/mill/executor/executor.go | 1018 + spindle/mill/executor/observe.go | 325 + spindle/mill/executor/observe_test.go | 90 + spindle/mill/executor/outbox.go | 468 + spindle/mill/executor/quota_client.go | 209 + spindle/mill/executor/quota_client_test.go | 759 + spindle/mill/executor/reserved.go | 81 + spindle/mill/executor/reserved_test.go | 1827 + spindle/mill/fair.go | 119 + spindle/mill/fair_test.go | 194 + spindle/mill/handler.go | 198 + spindle/mill/integration_test.go | 379 + spindle/mill/jump.go | 192 + spindle/mill/jump_test.go | 158 + spindle/mill/lease.go | 230 + spindle/mill/lost_executor_test.go | 241 + spindle/mill/mill.go | 2480 + spindle/mill/mill_test.go | 2151 + spindle/mill/nix_cache.go | 87 + spindle/mill/nix_cache_test.go | 238 + spindle/mill/proto/gen/mill.pb.go | 2964 ++ spindle/mill/proto/protocol.go | 113 + spindle/mill/proto/protocol_test.go | 435 + spindle/mill/proto/spindle/mill/v1/mill.proto | 354 + spindle/mill/proto/ws.go | 58 + spindle/mill/quota_admission_test.go | 1275 + spindle/mill/quota_test_helper_test.go | 46 + spindle/mill/restore.go | 289 + spindle/mill/restore_test.go | 495 + spindle/mill/session.go | 182 + spindle/mill/token.go | 23 + spindle/mill/token_to_executor_test.go | 181 + spindle/mill/workflow_limit_test.go | 49 + spindle/models/cache.go | 132 + spindle/models/cache_test.go | 78 + spindle/models/clone.go | 171 + spindle/models/clone_test.go | 496 + spindle/models/engine.go | 23 + spindle/models/logger.go | 261 + spindle/models/logger_bound_test.go | 90 + spindle/models/logger_cap_test.go | 92 + spindle/models/logger_test.go | 223 + spindle/models/models.go | 136 + spindle/models/pipeline.go | 88 + spindle/models/pipeline_env.go | 181 + spindle/models/pipeline_env_test.go | 407 + spindle/models/pipeline_record.go | 92 + spindle/models/pipeline_record_test.go | 21 + spindle/models/pipeline_test.go | 59 + spindle/models/secret_mask.go | 85 + spindle/models/secret_mask_test.go | 148 + spindle/models/webhook.go | 151 + spindle/motd | 25 + spindle/netguard/check_host_test.go | 19 + spindle/netguard/netguard.go | 102 + spindle/nix_cache_grant_test.go | 61 + spindle/nixcache/atticd_test.go | 293 + spindle/nixcache/client.go | 242 + spindle/nixcache/client_test.go | 265 + spindle/nixcache/issuer.go | 140 + spindle/nixcache/issuer_test.go | 293 + spindle/nixcache/name.go | 27 + spindle/nixcache/name_test.go | 37 + spindle/nixcache/prune.go | 305 + spindle/nixcache/prune_index_test.go | 246 + spindle/nixcache/prune_test.go | 717 + spindle/nixcache/publishing.go | 95 + spindle/nixcache/publishing_index_test.go | 105 + spindle/nixcache/publishing_test.go | 176 + spindle/nixcache/staged.go | 54 + spindle/nixcache/staged_test.go | 176 + spindle/nixcache/token.go | 72 + spindle/nixcache/token_test.go | 112 + spindle/nixcache/upstream.go | 24 + spindle/nixcache/upstream_test.go | 30 + spindle/nixcache/usage.go | 116 + spindle/nixcache/usage_test.go | 108 + spindle/notify_admission_test.go | 79 + spindle/notify_event.go | 529 + spindle/notify_event_test.go | 532 + spindle/notify_shutdown_test.go | 30 + spindle/notify_slots_test.go | 267 + spindle/notify_status_test.go | 77 + spindle/observability/logging.go | 60 + spindle/observability/logging_test.go | 83 + spindle/observability/metrics.go | 1464 + spindle/observability/metrics_test.go | 941 + spindle/observability/tracing.go | 162 + spindle/observability/tracing_test.go | 100 + spindle/oidc/issuer.go | 294 + spindle/oidc/issuer_test.go | 378 + spindle/pause_atomic_test.go | 71 + spindle/pause_test.go | 132 + spindle/pause_webhook_test.go | 117 + spindle/pemkey/pemkey.go | 51 + spindle/pemkey/pemkey_test.go | 76 + spindle/pemkey/pemkeytest/pemkeytest.go | 58 + spindle/permissions_test.go | 273 + spindle/pipeline/codec.go | 160 + spindle/pipeline/codec_test.go | 62 + spindle/pull_backlog_test.go | 61 + spindle/pull_pipelines_test.go | 484 + spindle/pull_trust_test.go | 282 + spindle/pull_versions_test.go | 109 + spindle/pulls.go | 530 + spindle/pulls_test.go | 415 + spindle/quota/denial.go | 64 + spindle/quota/evict.go | 171 + spindle/quota/evict_test.go | 140 + spindle/quota/idle_waiters_test.go | 107 + spindle/quota/manager.go | 876 + spindle/quota/manager_test.go | 1180 + spindle/quota/minutes.go | 97 + spindle/quota/minutes_test.go | 117 + spindle/quota/nix_object_test.go | 35 + spindle/quota/quota.go | 293 + spindle/quota/workflow_limit_test.go | 138 + spindle/record_marks_test.go | 123 + spindle/repo_job_backoff_test.go | 47 + spindle/repo_job_limits_test.go | 263 + spindle/repo_jobs.go | 472 + spindle/repo_jobs_test.go | 259 + spindle/repo_locks.go | 91 + spindle/repo_locks_cleanup_test.go | 41 + spindle/repo_locks_test.go | 452 + spindle/repo_retry.go | 166 + spindle/repo_retry_cap_test.go | 55 + spindle/repo_retry_test.go | 119 + spindle/repos.go | 390 + spindle/repos_test.go | 855 + spindle/retry_approval_test.go | 94 + spindle/run_env_test.go | 382 + spindle/run_pipeline_test.go | 52 + spindle/schedule.go | 806 + spindle/schedule_test.go | 658 + spindle/secret_copy.go | 98 + spindle/secrets/manager.go | 71 + spindle/secrets/openbao.go | 359 + spindle/secrets/openbao_test.go | 622 + spindle/secrets/policy.hcl | 22 + spindle/secrets/sqlite.go | 186 + spindle/secrets/sqlite_key_test.go | 24 + spindle/secrets/sqlite_test.go | 630 + spindle/server.go | 1604 + spindle/server_test.go | 290 + spindle/source_repo_test.go | 31 + spindle/startup_migrations.go | 29 + spindle/startup_migrations_test.go | 759 + spindle/storage/disk.go | 137 + spindle/storage/s3.go | 140 + spindle/storage/storage.go | 43 + spindle/storage/storage_test.go | 171 + spindle/storage/unversioned_s3.go | 30 + spindle/torture_admission_test.go | 521 + spindle/torture_engine_test.go | 188 + spindle/torture_ingest_test.go | 704 + spindle/torture_notify_abuse_test.go | 567 + spindle/torture_notify_test.go | 292 + spindle/torture_pool_test.go | 274 + spindle/torture_queue_test.go | 887 + spindle/torture_soak_test.go | 98 + spindle/torture_test.go | 487 + spindle/torture_world_test.go | 698 + spindle/webhook/webhook.go | 456 + spindle/webhook/webhook_detach_test.go | 62 + spindle/webhook/webhook_fairness_test.go | 59 + spindle/webhook/webhook_lane_test.go | 22 + spindle/webhook/webhook_test.go | 99 + spindle/wipe.go | 335 + spindle/wipe_claim_test.go | 130 + spindle/wipe_test.go | 362 + spindle/wipes_test.go | 146 + spindle/workflow_defs.go | 65 + spindle/workflow_defs_test.go | 60 + spindle/workflow_limits.go | 8 + spindle/workflow_limits_test.go | 39 + spindle/workflow_read.go | 134 + spindle/workflow_read_test.go | 427 + spindle/xrpc/allowlist.go | 96 + spindle/xrpc/allowlist_test.go | 138 + spindle/xrpc/bad_input_test.go | 64 + spindle/xrpc/ban.go | 191 + spindle/xrpc/ban_test.go | 363 + spindle/xrpc/banned_did_test.go | 167 + spindle/xrpc/body_limit_test.go | 66 + spindle/xrpc/ci_pipeline_subscribe_logs.go | 322 + spindle/xrpc/ci_subscribe_logs_cap_test.go | 45 + spindle/xrpc/ci_subscribe_logs_drain_test.go | 44 + spindle/xrpc/ci_subscribe_logs_limit_test.go | 49 + spindle/xrpc/ci_subscribe_logs_origin_test.go | 63 + spindle/xrpc/ci_subscribe_logs_stall_test.go | 47 + .../xrpc/ci_subscribe_logs_workflows_test.go | 33 + spindle/xrpc/cors_test.go | 86 + spindle/xrpc/definitions_busy_test.go | 35 + spindle/xrpc/invites.go | 73 + spindle/xrpc/invites_test.go | 268 + spindle/xrpc/legacy_routes_test.go | 43 + spindle/xrpc/notify_status_test.go | 32 + .../xrpc/org_tangled_ci_approve_pipeline.go | 78 + .../org_tangled_ci_approve_pipeline_test.go | 54 + .../xrpc/org_tangled_ci_cancel_pipeline.go | 108 + spindle/xrpc/org_tangled_ci_get_pipeline.go | 38 + .../org_tangled_ci_get_workflow_definition.go | 83 + spindle/xrpc/org_tangled_ci_pipeline.go | 160 + .../xrpc/org_tangled_ci_query_pipelines.go | 43 + spindle/xrpc/org_tangled_ci_retry_pipeline.go | 87 + .../org_tangled_ci_retry_pipeline_test.go | 57 + .../org_tangled_ci_subscribe_pipeline_logs.go | 36 + .../xrpc/org_tangled_event_notify_event.go | 217 + .../org_tangled_event_notify_event_test.go | 342 + spindle/xrpc/org_tangled_secret_add_secret.go | 53 + spindle/xrpc/org_tangled_secret_common.go | 40 + spindle/xrpc/org_tangled_secret_key_test.go | 34 + .../xrpc/org_tangled_secret_list_secrets.go | 29 + .../xrpc/org_tangled_secret_remove_secret.go | 39 + .../org_tangled_spindle_describe_spindle.go | 27 + spindle/xrpc/org_tangled_test.go | 531 + spindle/xrpc/quota.go | 371 + spindle/xrpc/quota_release_test.go | 110 + spindle/xrpc/quota_test.go | 571 + spindle/xrpc/repo_auth.go | 105 + spindle/xrpc/repo_auth_test.go | 354 + spindle/xrpc/repo_lookup.go | 112 + spindle/xrpc/repo_lookup_test.go | 161 + spindle/xrpc/scheduled_pipeline_test.go | 89 + spindle/xrpc/secret_access.go | 37 + spindle/xrpc/trigger_paused_test.go | 30 + spindle/xrpc/trigger_sha_test.go | 39 + spindle/xrpc/validation_test.go | 29 + spindle/xrpc/webhook_paused_test.go | 77 + spindle/xrpc/webhooks.go | 419 + spindle/xrpc/xrpc.go | 238 + spindle/xrpc/xrpc_test.go | 420 + sqlite/sqlite.go | 22 + status/.gitignore | 4 + status/package-lock.json | 2659 + status/package.json | 25 + status/src/access.test.ts | 88 + status/src/access.ts | 98 + status/src/css.d.ts | 9 + status/src/fonts.ts | 40 + status/src/footer.ts | 57 + status/src/index.ts | 1170 + status/src/logotype.ts | 9 + status/static/.assetsignore | 5 + status/static/favicon.svg | 21 + status/static/logos/dolly.svg | 21 + status/tailwind.config.js | 3 + status/tsconfig.json | 16 + status/tsconfig.test.json | 6 + status/wrangler.jsonc | 46 + systemd/knot.service | 52 + tailwind.config.js | 92 + tapc/readme.md | 3 + tapc/simple_indexer.go | 35 + tapc/tap.go | 176 + tapc/tap_test.go | 65 + tapc/types.go | 62 + tid/tid.go | 9 + web/.dockerignore | 8 + web/.gitignore | 37 + web/.npmrc | 1 + web/.prettierignore | 9 + web/.storybook/BoneCaptureDecorator.svelte | 16 + web/.storybook/DatasetToggle.svelte | 68 + web/.storybook/MockAuth.svelte | 111 + web/.storybook/MockNotifications.svelte | 42 + web/.storybook/SimulatedRepoSettings.svelte | 208 + web/.storybook/VouchArrivalReplay.svelte | 23 + web/.storybook/WorkflowRunReplay.svelte | 60 + web/.storybook/arrival.ts | 45 + web/.storybook/clipboard.ts | 6 + web/.storybook/holdErrorReset.ts | 8 + web/.storybook/installFetch.ts | 15 + web/.storybook/main.ts | 51 + web/.storybook/mocks/navigation.ts | 23 + web/.storybook/moderation.ts | 5 + web/.storybook/pendingOnboarding.ts | 16 + web/.storybook/play.ts | 5 + web/.storybook/preview.ts | 65 + web/.storybook/respond.ts | 21 + web/Containerfile | 41 + web/README.md | 43 + web/_headers | 5 + web/boneyard.config.json | 152 + web/eslint.config.js | 127 + web/knip.json | 6 + web/lex.config.ts | 66 + web/package.json | 112 + web/patches/@pierre__diffs@1.6.1.patch | 81 + .../@sveltejs__adapter-bun@1.0.0.patch | 51 + web/patches/boneyard-js@1.9.0.patch | 457 + web/patches/cobe@2.0.1.patch | 8 + web/patches/svelte@5.57.2.patch | 38 + web/pnpm-lock.yaml | 6753 +++ web/pnpm-workspace.yaml | 16 + web/prettier.config.js | 13 + web/scripts/bones.js | 111 + web/scripts/deploy-cloudflare-preview.sh | 37 + web/scripts/preview-worker-name.sh | 22 + web/scripts/staleAwait.test.ts | 44 + web/scripts/staleAwait.ts | 9 + web/scripts/testids-check.ts | 46 + web/scripts/testids.test.ts | 131 + web/scripts/testids.ts | 371 + web/scripts/tsconfig.json | 5 + web/src/app.css | 48 + web/src/app.d.ts | 49 + web/src/app.html | 46 + web/src/env.test.ts | 11 + web/src/env.ts | 56 + web/src/fixtures/blobHeaderWorstCase.ts | 101 + web/src/fixtures/comments.ts | 156 + web/src/fixtures/commits.ts | 28 + web/src/fixtures/diff.ts | 378 + web/src/fixtures/emails.ts | 50 + web/src/fixtures/githubImportWorstCase.ts | 169 + web/src/fixtures/identities.ts | 2 + web/src/fixtures/logFeed.svelte.ts | 21 + web/src/fixtures/orgCreate.ts | 55 + web/src/fixtures/pipelines.ts | 438 + web/src/fixtures/profileWorstCase.ts | 111 + web/src/fixtures/pull.ts | 429 + web/src/fixtures/pullCommitRows.ts | 44 + web/src/fixtures/pullWorstCase.ts | 400 + web/src/fixtures/reactions.ts | 46 + web/src/fixtures/repo.ts | 12 + web/src/fixtures/search.ts | 2046 + web/src/fixtures/timeline.ts | 102 + web/src/fonts.css | 110 + web/src/fontsBase.mjs | 20 + web/src/hooks.server.test.ts | 379 + web/src/hooks.server.ts | 214 + web/src/hooks.ts | 7 + web/src/icon.d.ts | 11 + web/src/lib/action.svelte.ts | 80 + web/src/lib/action.test.ts | 57 + web/src/lib/api/_request.test.ts | 107 + web/src/lib/api/_request.ts | 153 + web/src/lib/api/access.test.ts | 66 + web/src/lib/api/access.ts | 51 + web/src/lib/api/account.ts | 114 + web/src/lib/api/actAs.test.ts | 227 + web/src/lib/api/actAs.ts | 207 + web/src/lib/api/appview.ts | 82 + web/src/lib/api/archive.test.ts | 145 + web/src/lib/api/archive.ts | 80 + web/src/lib/api/artifact.ts | 109 + web/src/lib/api/attestation.test.ts | 282 + web/src/lib/api/attestation.ts | 262 + web/src/lib/api/awaitRecord.test.ts | 54 + web/src/lib/api/awaitRecord.ts | 65 + web/src/lib/api/blame.test.ts | 184 + web/src/lib/api/blame.ts | 124 + web/src/lib/api/blob.test.ts | 101 + web/src/lib/api/blob.ts | 125 + web/src/lib/api/cache.test.ts | 102 + web/src/lib/api/cache.ts | 27 + web/src/lib/api/client.test.ts | 153 + web/src/lib/api/client.ts | 113 + web/src/lib/api/cob.test.ts | 302 + web/src/lib/api/cob.ts | 282 + web/src/lib/api/collaborators.test.ts | 65 + web/src/lib/api/collaborators.ts | 55 + web/src/lib/api/comment.ts | 126 + web/src/lib/api/commitStatuses.ts | 17 + web/src/lib/api/count.ts | 44 + web/src/lib/api/coverage.ts | 11 + web/src/lib/api/delegation.test.ts | 194 + web/src/lib/api/delegation.ts | 113 + web/src/lib/api/deliberi.ts | 81 + web/src/lib/api/descriptors.ts | 94 + web/src/lib/api/diff.ts | 11 + web/src/lib/api/emailDid.test.ts | 50 + web/src/lib/api/emailDid.ts | 25 + web/src/lib/api/emails.test.ts | 123 + web/src/lib/api/emails.ts | 91 + web/src/lib/api/enrich.ts | 153 + web/src/lib/api/fileHistory.test.ts | 64 + web/src/lib/api/fileHistory.ts | 128 + web/src/lib/api/gitclient.test.ts | 131 + web/src/lib/api/gitclient.ts | 86 + web/src/lib/api/githubImport.test.ts | 806 + web/src/lib/api/githubImport.ts | 429 + web/src/lib/api/graph.test.ts | 184 + web/src/lib/api/graph.ts | 301 + web/src/lib/api/handle.ts | 40 + web/src/lib/api/identity.test.ts | 124 + web/src/lib/api/identity.ts | 74 + web/src/lib/api/index.ts | 12 + web/src/lib/api/indexnow.ts | 16 + web/src/lib/api/invite.test.ts | 83 + web/src/lib/api/invite.ts | 34 + web/src/lib/api/issue.ts | 37 + web/src/lib/api/knot.test.ts | 127 + web/src/lib/api/knot.ts | 72 + web/src/lib/api/knotmirror.test.ts | 436 + web/src/lib/api/knotmirror.ts | 422 + web/src/lib/api/labels.ts | 147 + web/src/lib/api/lexicons/index.ts | 234 + .../types/com/bad-example/pocket/defs.ts | 20 + .../com/bad-example/pocket/getPreferences.ts | 38 + .../com/bad-example/pocket/putPreferences.ts | 41 + .../lexicons/types/org/tangled/actor/defs.ts | 64 + .../types/org/tangled/actor/getProfile.ts | 38 + .../org/tangled/actor/getProfileActivity.ts | 150 + .../types/org/tangled/actor/getProfiles.ts | 56 + .../types/org/tangled/actor/getTrending.ts | 47 + .../types/org/tangled/actor/profile.ts | 117 + .../tangled/actor/searchActorsTypeahead.ts | 69 + .../types/org/tangled/bobbin/awaitRecord.ts | 68 + .../types/org/tangled/ci/approvePipeline.ts | 39 + .../types/org/tangled/ci/cancelPipeline.ts | 42 + .../api/lexicons/types/org/tangled/ci/defs.ts | 214 + .../types/org/tangled/ci/getPipeline.ts | 38 + .../org/tangled/ci/getWorkflowDefinition.ts | 43 + .../org/tangled/ci/listWorkflowDefinitions.ts | 46 + .../types/org/tangled/ci/queryPipelines.ts | 81 + .../types/org/tangled/ci/retryPipeline.ts | 51 + .../org/tangled/ci/subscribePipelineLogs.ts | 105 + .../types/org/tangled/ci/trigger/manual.ts | 49 + .../types/org/tangled/ci/trigger/schedule.ts | 33 + .../types/org/tangled/embed/commit.ts | 45 + .../types/org/tangled/event/notifyEvent.ts | 76 + .../types/org/tangled/event/pullRequest.ts | 48 + .../lexicons/types/org/tangled/event/push.ts | 33 + .../types/org/tangled/feed/comment.ts | 42 + .../types/org/tangled/feed/countComments.ts | 42 + .../types/org/tangled/feed/countCommentsBy.ts | 42 + .../types/org/tangled/feed/countReactions.ts | 42 + .../org/tangled/feed/countReactionsBy.ts | 42 + .../types/org/tangled/feed/countStars.ts | 42 + .../types/org/tangled/feed/countStarsBy.ts | 42 + .../lexicons/types/org/tangled/feed/defs.ts | 40 + .../types/org/tangled/feed/getStar.ts | 40 + .../types/org/tangled/feed/getTimeline.ts | 129 + .../types/org/tangled/feed/listComments.ts | 85 + .../types/org/tangled/feed/listCommentsBy.ts | 69 + .../types/org/tangled/feed/listReactions.ts | 84 + .../types/org/tangled/feed/listReactionsBy.ts | 71 + .../types/org/tangled/feed/listStars.ts | 84 + .../types/org/tangled/feed/listStarsBy.ts | 69 + .../types/org/tangled/feed/reaction.ts | 42 + .../lexicons/types/org/tangled/feed/star.ts | 51 + .../types/org/tangled/feed/subscription.ts | 52 + .../lexicons/types/org/tangled/git/defs.ts | 197 + .../types/org/tangled/git/deleteBranch.ts | 33 + .../types/org/tangled/git/getArchive.ts | 46 + .../types/org/tangled/git/getBlame.ts | 119 + .../lexicons/types/org/tangled/git/getBlob.ts | 39 + .../types/org/tangled/git/getBranch.ts | 56 + .../lexicons/types/org/tangled/git/getDiff.ts | 51 + .../types/org/tangled/git/getEntry.ts | 67 + .../lexicons/types/org/tangled/git/getTag.ts | 59 + .../lexicons/types/org/tangled/git/getTree.ts | 80 + .../types/org/tangled/git/listBranches.ts | 89 + .../types/org/tangled/git/listCommits.ts | 65 + .../types/org/tangled/git/listLanguages.ts | 96 + .../types/org/tangled/git/listRefs.ts | 120 + .../types/org/tangled/git/listTags.ts | 104 + .../api/lexicons/types/org/tangled/git/oid.ts | 23 + .../types/org/tangled/git/pushEvent.ts | 79 + .../types/org/tangled/git/receivePack.ts | 23 + .../lexicons/types/org/tangled/git/refs.ts | 38 + .../types/org/tangled/git/setDefaultBranch.ts | 55 + .../types/org/tangled/graph/attestation.ts | 33 + .../types/org/tangled/graph/collaborator.ts | 25 + .../types/org/tangled/graph/countFollows.ts | 42 + .../types/org/tangled/graph/countFollowsBy.ts | 42 + .../types/org/tangled/graph/follow.ts | 25 + .../types/org/tangled/graph/getFollow.ts | 40 + .../types/org/tangled/graph/listFollows.ts | 88 + .../types/org/tangled/graph/listFollowsBy.ts | 69 + .../org/tangled/graph/listNetworkVouches.ts | 80 + .../lexicons/types/org/tangled/graph/vouch.ts | 57 + .../lexicons/types/org/tangled/key/sshKey.ts | 51 + .../lexicons/types/org/tangled/knot/block.ts | 25 + .../types/org/tangled/knot/describeKnot.ts | 71 + .../lexicons/types/org/tangled/knot/member.ts | 25 + .../org/tangled/label/countDefinitions.ts | 45 + .../types/org/tangled/label/definition.ts | 38 + .../lexicons/types/org/tangled/label/defs.ts | 20 + .../org/tangled/label/listDefinitions.ts | 84 + .../types/org/tangled/markup/embed/image.ts | 24 + .../types/org/tangled/markup/embed/video.ts | 24 + .../types/org/tangled/markup/markdown.ts | 58 + .../types/org/tangled/markup/plaintext.ts | 22 + .../types/org/tangled/publicKey/countKeys.ts | 42 + .../org/tangled/publicKey/getPublicKey.ts | 41 + .../types/org/tangled/publicKey/listKeys.ts | 84 + .../types/org/tangled/query/enrichResponse.ts | 178 + .../types/org/tangled/repo/artifact.ts | 46 + .../types/org/tangled/repo/countArtifacts.ts | 42 + .../org/tangled/repo/countArtifactsBy.ts | 42 + .../org/tangled/repo/countCollaborators.ts | 45 + .../org/tangled/repo/countCollaboratorsBy.ts | 45 + .../types/org/tangled/repo/countForks.ts | 42 + .../types/org/tangled/repo/countIssues.ts | 52 + .../types/org/tangled/repo/countPulls.ts | 52 + .../types/org/tangled/repo/countRepos.ts | 42 + .../types/org/tangled/repo/createRepo.ts | 91 + .../types/org/tangled/repo/declaration.ts | 29 + .../lexicons/types/org/tangled/repo/defs.ts | 94 + .../types/org/tangled/repo/deleteRepo.ts | 33 + .../types/org/tangled/repo/forkSync.ts | 36 + .../types/org/tangled/repo/getImportStatus.ts | 44 + .../types/org/tangled/repo/getRepo.ts | 42 + .../types/org/tangled/repo/getRepoByName.ts | 46 + .../org/tangled/repo/getRepoByRepoDid.ts | 42 + .../types/org/tangled/repo/getRepos.ts | 56 + .../org/tangled/repo/getReposByRepoDids.ts | 59 + .../types/org/tangled/repo/listArtifacts.ts | 84 + .../types/org/tangled/repo/listArtifactsBy.ts | 71 + .../tangled/repo/listCollaboratorInvitesBy.ts | 78 + .../org/tangled/repo/listCollaborators.ts | 109 + .../org/tangled/repo/listCollaboratorsBy.ts | 74 + .../types/org/tangled/repo/listForks.ts | 84 + .../types/org/tangled/repo/listIssues.ts | 113 + .../types/org/tangled/repo/listIssuesBy.ts | 77 + .../types/org/tangled/repo/listPulls.ts | 111 + .../types/org/tangled/repo/listPullsBy.ts | 77 + .../org/tangled/repo/listRecordVersions.ts | 92 + .../types/org/tangled/repo/listRepos.ts | 84 + .../types/org/tangled/repo/manifest.ts | 90 + .../types/org/tangled/repo/mintToken.ts | 53 + .../types/org/tangled/repo/reserveKey.ts | 42 + .../types/org/tangled/review/patch.ts | 55 + .../types/org/tangled/review/submitPatch.ts | 84 + .../types/org/tangled/search/query.ts | 181 + .../types/org/tangled/secret/addSecret.ts | 43 + .../types/org/tangled/secret/listSecrets.ts | 51 + .../types/org/tangled/secret/removeSecret.ts | 39 + .../types/org/tangled/snippet/string.ts | 53 + .../types/org/tangled/string/countStrings.ts | 42 + .../types/org/tangled/string/getString.ts | 38 + .../types/org/tangled/string/listStrings.ts | 84 + .../org/tangled/temp/account/addEmail.ts | 35 + .../org/tangled/temp/account/beginSignup.ts | 39 + .../tangled/temp/account/completeSignup.ts | 83 + .../org/tangled/temp/account/deleteEmail.ts | 35 + .../tangled/temp/account/dismissNewsletter.ts | 26 + .../temp/account/getNewsletterPreference.ts | 33 + .../org/tangled/temp/account/getSignup.ts | 43 + .../org/tangled/temp/account/listEmails.ts | 56 + .../org/tangled/temp/account/resendSignup.ts | 35 + .../tangled/temp/account/setPrimaryEmail.ts | 35 + .../temp/account/subscribeNewsletter.ts | 26 + .../org/tangled/temp/account/verifyEmail.ts | 44 + .../types/org/tangled/temp/actor/defs.ts | 171 + .../org/tangled/temp/focus/beginSession.ts | 47 + .../org/tangled/temp/focus/endSession.ts | 26 + .../types/org/tangled/temp/focus/nextItem.ts | 56 + .../org/tangled/temp/git/deleteBranch.ts | 36 + .../types/org/tangled/temp/git/formatPatch.ts | 38 + .../types/org/tangled/temp/git/getArchive.ts | 46 + .../types/org/tangled/temp/git/getBlob.ts | 44 + .../types/org/tangled/temp/git/getBranch.ts | 56 + .../types/org/tangled/temp/git/getEntry.ts | 68 + .../org/tangled/temp/git/getMergeBase.ts | 46 + .../types/org/tangled/temp/git/getRef.ts | 65 + .../types/org/tangled/temp/git/getTag.ts | 61 + .../types/org/tangled/temp/git/getTree.ts | 81 + .../org/tangled/temp/git/listBranches.ts | 89 + .../types/org/tangled/temp/git/listCommits.ts | 128 + .../org/tangled/temp/git/listCommitsByRef.ts | 67 + .../types/org/tangled/temp/git/listTags.ts | 117 + .../types/org/tangled/temp/git/mergeCommit.ts | 116 + .../types/org/tangled/temp/git/syncFork.ts | 37 + .../org/tangled/temp/migrator/createTask.ts | 60 + .../types/org/tangled/temp/migrator/defs.ts | 111 + .../tangled/temp/migrator/describeSource.ts | 46 + .../org/tangled/temp/migrator/getTask.ts | 34 + .../org/tangled/temp/migrator/listTasks.ts | 47 + .../org/tangled/temp/migrator/retryJob.ts | 50 + .../org/tangled/temp/newsletter/signup.ts | 35 + .../temp/notification/deleteNotification.ts | 35 + .../temp/notification/getPreferences.ts | 53 + .../temp/notification/getUnreadCount.ts | 31 + .../temp/notification/listNotifications.ts | 98 + .../temp/notification/listRecipients.ts | 40 + .../tangled/temp/notification/markAllRead.ts | 26 + .../temp/notification/markEntityRead.ts | 35 + .../temp/notification/updatePreferences.ts | 41 + .../tangled/temp/notification/updateSeen.ts | 36 + .../types/org/tangled/temp/repo/deploySite.ts | 35 + .../org/tangled/temp/repo/disableSite.ts | 35 + .../org/tangled/temp/repo/getDeployHistory.ts | 85 + .../org/tangled/temp/repo/getSiteConfig.ts | 63 + .../org/tangled/temp/repo/updateSiteConfig.ts | 59 + .../org/tangled/temp/review/listPatches.ts | 87 + .../org/tangled/temp/search/searchCode.ts | 131 + .../org/tangled/temp/server/ackHandover.ts | 38 + .../org/tangled/temp/server/cancelHandover.ts | 38 + .../tangled/temp/server/createInviteCode.ts | 39 + .../org/tangled/temp/server/handoverRepo.ts | 40 + .../tangled/temp/server/redeemInviteCode.ts | 32 + .../types/org/tangled/temp/server/seedRepo.ts | 101 + .../org/tangled/temp/server/uploadBlob.ts | 40 + .../org/tangled/temp/site/claimDomain.ts | 35 + .../org/tangled/temp/site/getDomainClaim.ts | 34 + .../org/tangled/temp/site/releaseDomain.ts | 35 + .../tangled/temp/spindle/moderation/ban.ts | 35 + .../tangled/temp/spindle/moderation/defs.ts | 24 + .../tangled/temp/spindle/moderation/getBan.ts | 42 + .../temp/spindle/moderation/listBans.ts | 39 + .../tangled/temp/spindle/moderation/unban.ts | 35 + .../org/tangled/temp/spindle/quota/defs.ts | 53 + .../org/tangled/temp/spindle/quota/get.ts | 46 + .../org/tangled/temp/spindle/quota/list.ts | 39 + .../tangled/temp/spindle/quota/releaseNars.ts | 61 + .../org/tangled/temp/spindle/quota/set.ts | 47 + .../org/tangled/temp/spindle/quota/unset.ts | 39 + .../org/tangled/temp/spindle/quota/usage.ts | 50 + .../lexicons/types/org/tangled/track/defs.ts | 69 + .../types/org/tangled/track/getTicketView.ts | 31 + .../types/org/tangled/track/ticket.ts | 65 + .../org/tangled/webhook/createWebhook.ts | 64 + .../types/org/tangled/webhook/defs.ts | 80 + .../org/tangled/webhook/deleteWebhook.ts | 39 + .../tangled/webhook/listWebhookDeliveries.ts | 58 + .../types/org/tangled/webhook/listWebhooks.ts | 36 + .../tangled/webhook/retryWebhookDelivery.ts | 43 + .../org/tangled/webhook/updateWebhook.ts | 48 + .../api/lexicons/types/sh/tangled/git/defs.ts | 152 + .../types/sh/tangled/git/mergeCommit.ts | 111 + .../tangled/git/temp/listIndexedLanguages.ts | 80 + .../sh/tangled/git/temp/listLanguages.ts | 71 + .../types/sh/tangled/git/temp2/getBlame.ts | 102 + .../sh/tangled/git/temp2/getCommitStats.ts | 73 + .../types/sh/tangled/git/temp2/getDiff.ts | 36 + .../sh/tangled/git/temp2/getInterdiff.ts | 38 + .../types/sh/tangled/git/temp2/mergeCheck.ts | 67 + .../sh/tangled/identity/resolveCommitters.ts | 44 + .../types/sh/tangled/repo/listCobVersions.ts | 93 + web/src/lib/api/liveCommitStatuses.svelte.ts | 68 + web/src/lib/api/load.test.ts | 212 + web/src/lib/api/load.ts | 158 + web/src/lib/api/lookupCache.test.ts | 118 + web/src/lib/api/lookupCache.ts | 116 + web/src/lib/api/markupBlobs.test.ts | 78 + web/src/lib/api/markupBlobs.ts | 99 + web/src/lib/api/migrator.test.ts | 394 + web/src/lib/api/migrator.ts | 221 + web/src/lib/api/moderation.ts | 67 + web/src/lib/api/moderationLabels.ts | 110 + web/src/lib/api/newsletter.test.ts | 43 + web/src/lib/api/newsletter.ts | 43 + web/src/lib/api/notifications.test.ts | 124 + web/src/lib/api/notifications.ts | 193 + web/src/lib/api/orgCreate.test.ts | 147 + web/src/lib/api/orgCreate.ts | 88 + web/src/lib/api/orgMembers.test.ts | 142 + web/src/lib/api/orgMembers.ts | 117 + web/src/lib/api/orgProfile.test.ts | 113 + web/src/lib/api/orgProfile.ts | 45 + web/src/lib/api/organizations.test.ts | 195 + web/src/lib/api/organizations.ts | 102 + web/src/lib/api/pagination.test.ts | 120 + web/src/lib/api/pagination.ts | 122 + web/src/lib/api/patchStream.test.ts | 97 + web/src/lib/api/patchStream.ts | 103 + web/src/lib/api/pinning.test.ts | 101 + web/src/lib/api/pinning.ts | 69 + web/src/lib/api/pipelineActions.test.ts | 72 + web/src/lib/api/pipelineActions.ts | 32 + web/src/lib/api/preferences.test.ts | 435 + web/src/lib/api/preferences.ts | 184 + web/src/lib/api/profile.test.ts | 107 + web/src/lib/api/profile.ts | 90 + web/src/lib/api/profileIdentifier.test.ts | 28 + web/src/lib/api/profileIdentifier.ts | 9 + web/src/lib/api/profileSettings.test.ts | 262 + web/src/lib/api/profileSettings.ts | 131 + web/src/lib/api/pull.test.ts | 132 + web/src/lib/api/pull.ts | 135 + web/src/lib/api/pullCompose.test.ts | 237 + web/src/lib/api/pullCompose.ts | 241 + web/src/lib/api/pullRoute.test.ts | 103 + web/src/lib/api/pullRoute.ts | 111 + web/src/lib/api/rawCommit.ts | 14 + web/src/lib/api/reaction.ts | 65 + web/src/lib/api/records.ts | 181 + web/src/lib/api/repo.test.ts | 410 + web/src/lib/api/repo.ts | 407 + web/src/lib/api/repoCompare.ts | 127 + web/src/lib/api/repoCounts.ts | 29 + web/src/lib/api/repoCreate.test.ts | 332 + web/src/lib/api/repoCreate.ts | 247 + web/src/lib/api/repoDiff.test.ts | 519 + web/src/lib/api/repoDiff.ts | 335 + web/src/lib/api/repoIndex.test.ts | 243 + web/src/lib/api/repoIndex.ts | 169 + web/src/lib/api/repoSettings.test.ts | 316 + web/src/lib/api/repoSettings.ts | 189 + web/src/lib/api/search.test.ts | 167 + web/src/lib/api/search.ts | 76 + web/src/lib/api/secrets.test.ts | 65 + web/src/lib/api/secrets.ts | 51 + web/src/lib/api/settings.test.ts | 51 + web/src/lib/api/settings.ts | 147 + web/src/lib/api/signup.test.ts | 328 + web/src/lib/api/signup.ts | 143 + web/src/lib/api/sites.test.ts | 258 + web/src/lib/api/sites.ts | 135 + web/src/lib/api/spindle.test.ts | 208 + web/src/lib/api/spindle.ts | 99 + web/src/lib/api/spindleLogs.test.ts | 89 + web/src/lib/api/spindleLogs.ts | 101 + web/src/lib/api/spindlewrite.test.ts | 97 + web/src/lib/api/spindlewrite.ts | 100 + web/src/lib/api/strings.test.ts | 304 + web/src/lib/api/strings.ts | 115 + web/src/lib/api/syntax.test.ts | 68 + web/src/lib/api/syntax.ts | 119 + web/src/lib/api/ticket.test.ts | 130 + web/src/lib/api/ticket.ts | 108 + web/src/lib/api/ticketSearch.ts | 110 + web/src/lib/api/tranquil.test.ts | 128 + web/src/lib/api/tranquil.ts | 231 + web/src/lib/api/uri.test.ts | 58 + web/src/lib/api/uri.ts | 41 + web/src/lib/api/webhooks.test.ts | 162 + web/src/lib/api/webhooks.ts | 193 + web/src/lib/api/write.test.ts | 153 + web/src/lib/api/write.ts | 105 + web/src/lib/auth.svelte.test.ts | 125 + web/src/lib/auth.svelte.ts | 703 + web/src/lib/auth.test.ts | 277 + web/src/lib/auth/accounts.ts | 180 + web/src/lib/auth/agent.test.ts | 17 + web/src/lib/auth/agent.ts | 51 + web/src/lib/auth/guards.ts | 32 + web/src/lib/auth/handleResolver.test.ts | 138 + web/src/lib/auth/handleResolver.ts | 81 + web/src/lib/auth/onboarding.test.ts | 69 + web/src/lib/auth/onboarding.ts | 61 + web/src/lib/auth/orgSessions.test.ts | 82 + web/src/lib/auth/orgSessions.ts | 58 + web/src/lib/auth/pending.ts | 81 + web/src/lib/auth/returnTo.test.ts | 45 + web/src/lib/auth/returnTo.ts | 21 + web/src/lib/auth/scopes.test.ts | 77 + web/src/lib/auth/scopes.ts | 79 + web/src/lib/auth/session.ts | 9 + web/src/lib/avatar.test.ts | 28 + web/src/lib/avatar.ts | 69 + web/src/lib/avatarGradient.test.ts | 50 + web/src/lib/avatarGradient.ts | 83 + web/src/lib/avatarWidths.ts | 2 + web/src/lib/blog.ts | 39 + ...ones-forklist--skeleton-fixture.bones.json | 637 + .../bones-loadingprimitives--count.bones.json | 33 + ...s-loadingprimitives--short-text.bones.json | 33 + .../bones/bones-radio--loading-row.bones.json | 45 + ...nes-starredby--skeleton-fixture.bones.json | 837 + ...ones-treeview--skeleton-fixture.bones.json | 335 + ...ificationlist--skeleton-fixture.bones.json | 129 + ...onlist--social-skeleton-fixture.bones.json | 121 + .../profile-followcard--bordered.bones.json | 65 + ...le-networkvouches--many-vouches.bones.json | 49 + ...e-overviewtab--skeleton-fixture.bones.json | 283 + ...profilechrome--skeleton-fixture.bones.json | 271 + .../profile-stringcard--default.bones.json | 53 + .../bones/repo-blobview--code-file.bones.json | 438 + ...view--markdown-skeleton-fixture.bones.json | 197 + ...o-branchtable--skeleton-fixture.bones.json | 1856 + ...commitlogview--skeleton-fixture.bones.json | 1585 + .../repo-commitview--full-commit.bones.json | 261 + ...iew--skeleton-fixture-collapsed.bones.json | 739 + ...repo-diffview--skeleton-fixture.bones.json | 793 + ...lehistoryview--skeleton-fixture.bones.json | 315 + .../repo-issues-issueform--create.bones.json | 114 + ...suethreadview--skeleton-fixture.bones.json | 312 + ...pipelines-pipelinelist--default.bones.json | 150 + ...-pipelines-pipelinetoolbar--all.bones.json | 69 + ...rkflowrunview--skeleton-fixture.bones.json | 85 + ...repo-pulls-pullcompose--default.bones.json | 768 + ...-pullviewpage--skeleton-fixture.bones.json | 593 + ...readme--rendered-sanitized-html.bones.json | 62 + ...repo-repocard--with-star-button.bones.json | 63 + ...po-repochrome--skeleton-fixture.bones.json | 97 + ...-repoforkform--skeleton-fixture.bones.json | 159 + .../repo-repoheader--full-metadata.bones.json | 85 + ...dexview--panel-skeleton-fixture.bones.json | 196 + ...repoindexview--skeleton-fixture.bones.json | 220 + .../bones/repo-tagcard--annotated.bones.json | 63 + .../repo-taglist--skeleton-fixture.bones.json | 707 + ...repo-tickets-ticketlist--issues.bones.json | 129 + .../repo-tickets-ticketlist--pulls.bones.json | 1633 + ...kets-tickettoolbar--issues-open.bones.json | 65 + ...ckets-tickettoolbar--pulls-open.bones.json | 69 + ...panel--content-skeleton-fixture.bones.json | 105 + .../strings-stringview--code-file.bones.json | 379 + ...view--markdown-skeleton-fixture.bones.json | 121 + ...ationspreview--skeleton-fixture.bones.json | 125 + ...ecentspreview--skeleton-fixture.bones.json | 133 + ...line-timeline--skeleton-fixture.bones.json | 154 + ...dingreposlist--skeleton-fixture.bones.json | 88 + .../lib/bones/ui-toggle--checked.bones.json | 45 + web/src/lib/cloneUrl.test.ts | 35 + web/src/lib/cloneUrl.ts | 24 + web/src/lib/components/StatusDot.svelte | 70 + .../attestations/AttestationBanner.svelte | 131 + .../auth/AccountSelector.stories.svelte | 48 + .../components/auth/AccountSelector.svelte | 90 + .../components/comment/Comment.stories.svelte | 38 + web/src/lib/components/comment/Comment.svelte | 191 + .../comment/CommentArrivalFixture.svelte | 46 + .../comment/CommentBox.stories.svelte | 42 + .../lib/components/comment/CommentBox.svelte | 66 + .../comment/CommentCard.stories.svelte | 299 + .../lib/components/comment/CommentCard.svelte | 297 + .../comment/CommentEditor.stories.svelte | 109 + .../components/comment/CommentEditor.svelte | 239 + .../comment/CommentList.stories.svelte | 112 + .../lib/components/comment/CommentList.svelte | 40 + .../components/comment/CommentSection.svelte | 84 + .../components/comment/OlderComments.svelte | 46 + .../lib/components/comment/comments.test.ts | 265 + web/src/lib/components/comment/comments.ts | 274 + web/src/lib/components/comment/reveal.ts | 15 + .../lib/components/editor/PierreEditor.svelte | 134 + web/src/lib/components/icons/Bluesky.svelte | 21 + .../components/icons/BookArrowRight.svelte | 23 + .../lib/components/icons/DollyCheck.svelte | 14 + web/src/lib/components/icons/EyeFilled.svelte | 15 + .../lib/components/icons/FolderOpenDuo.svelte | 29 + .../icons/ShieldDirectDenounce.svelte | 15 + .../components/icons/ShieldDirectVouch.svelte | 15 + .../icons/ShieldIndirectDenounce.svelte | 25 + .../icons/ShieldIndirectVouch.svelte | 22 + .../lib/components/icons/ShieldMixed.svelte | 15 + .../icons/UserRoundArrowForward.svelte | 29 + .../components/import/ConnectAccount.svelte | 47 + .../import/GitHubAuthorizeStep.svelte | 43 + .../import/GitHubConnectStep.svelte | 28 + .../import/GitHubImporter.stories.svelte | 96 + .../components/import/GitHubImporter.svelte | 88 + .../import/GitHubImporterFixture.svelte | 55 + .../import/GitHubRepoSelectStep.svelte | 300 + .../lib/components/import/MethodPicker.svelte | 52 + .../lib/components/import/MigrationRow.svelte | 109 + .../components/import/MigrationStep.svelte | 194 + .../lib/components/import/RepoConfig.svelte | 75 + web/src/lib/components/import/RepoRow.svelte | 207 + .../components/import/StartedBanner.svelte | 27 + .../components/import/UrlRepoImporter.svelte | 361 + .../import/gitHubImportState.svelte.ts | 818 + .../import/gitHubImportState.test.ts | 233 + .../components/import/grantEvidence.test.ts | 27 + .../lib/components/import/grantEvidence.ts | 17 + web/src/lib/components/import/jobMatcher.ts | 33 + .../components/import/migrationRows.test.ts | 170 + .../lib/components/import/migrationRows.ts | 83 + .../lib/components/import/stepGating.test.ts | 109 + web/src/lib/components/import/stepGating.ts | 28 + .../components/landing/EmbossFilter.svelte | 54 + .../components/landing/Landing.stories.svelte | 17 + web/src/lib/components/landing/Landing.svelte | 78 + .../lib/components/landing/LiveLanding.svelte | 49 + .../landing/NotFound.stories.svelte | 15 + .../lib/components/landing/NotFound.svelte | 207 + .../assets/cards/ci-cable-after-light.svg | 1 + .../landing/assets/cards/ci-cable-after.svg | 1 + .../assets/cards/ci-cable-before-light.svg | 1 + .../landing/assets/cards/ci-cable-before.svg | 1 + .../landing/assets/cards/ci-vps-cap-l.svg | 1 + .../landing/assets/cards/ci-vps-cap-r.svg | 1 + .../landing/assets/cards/ci-vps-grill.svg | 1 + .../landing/assets/knot/loop-left-edge.svg | 1 + .../landing/assets/knot/loop-left.svg | 1 + .../landing/assets/knot/loop-right-edge.svg | 1 + .../landing/assets/knot/loop-right.svg | 1 + .../landing/assets/knot/strand-bottom.svg | 1 + .../landing/assets/knot/strand-top.svg | 1 + .../assets/not-found/strand-bottom.svg | 1 + .../landing/assets/not-found/strand-end.svg | 1 + .../landing/assets/not-found/strand-top.svg | 1 + .../components/landing/cta/Cta.stories.svelte | 15 + web/src/lib/components/landing/cta/Cta.svelte | 473 + .../lib/components/landing/cta/cta-knot.ts | 179 + .../landing/features/AgentCard.svelte | 220 + .../components/landing/features/Beat.svelte | 29 + .../landing/features/CiDevice.stories.svelte | 15 + .../landing/features/CiDevice.svelte | 252 + .../landing/features/CiStep.stories.svelte | 29 + .../components/landing/features/CiStep.svelte | 32 + .../landing/features/Features.stories.svelte | 16 + .../landing/features/Features.svelte | 81 + .../landing/features/GlobeCard.stories.svelte | 15 + .../landing/features/GlobeCard.svelte | 183 + .../landing/features/LearnMore.stories.svelte | 17 + .../landing/features/LearnMore.svelte | 17 + .../landing/features/PrCard.stories.svelte | 16 + .../components/landing/features/PrCard.svelte | 209 + .../landing/features/VouchCard.stories.svelte | 15 + .../landing/features/VouchCard.svelte | 186 + .../components/landing/features/agents.css | 329 + .../lib/components/landing/features/ci.css | 403 + .../components/landing/features/ci.svelte.ts | 37 + .../components/landing/features/features.css | 129 + .../lib/components/landing/features/globe.css | 154 + .../components/landing/features/pr-stack.ts | 119 + .../lib/components/landing/features/pr.css | 440 + .../lib/components/landing/features/vouch.css | 317 + .../lib/components/landing/guest.svelte.ts | 56 + .../landing/hackable/Corners.svelte | 6 + .../hackable/DollyCanvas.stories.svelte | 17 + .../landing/hackable/DollyCanvas.svelte | 323 + .../landing/hackable/Hackable.stories.svelte | 15 + .../landing/hackable/Hackable.svelte | 127 + .../components/landing/hackable/dolly.test.ts | 54 + .../lib/components/landing/hackable/dolly.ts | 99 + .../components/landing/hackable/hackable.css | 37 + .../landing/hero/Hero.stories.svelte | 19 + .../lib/components/landing/hero/Hero.svelte | 356 + .../landing/hero/IsoStack.stories.svelte | 28 + .../components/landing/hero/IsoStack.svelte | 222 + .../components/landing/hero/activity.test.ts | 142 + .../lib/components/landing/hero/activity.ts | 123 + web/src/lib/components/landing/hero/mock.ts | 235 + .../components/landing/hero/notifications.ts | 19 + web/src/lib/components/landing/iso-stack.css | 138 + web/src/lib/components/landing/landing.css | 255 + .../components/landing/legal/LegalPage.svelte | 318 + web/src/lib/components/landing/live.svelte.ts | 63 + web/src/lib/components/landing/material.css | 254 + .../landing/motion/spinning-globe.svelte.ts | 179 + .../landing/selfhost/SelfHost.stories.svelte | 15 + .../landing/selfhost/SelfHost.svelte | 180 + .../electronics/FloppyComputer.stories.svelte | 19 + .../electronics/FloppyComputer.svelte | 261 + .../landing/selfhost/electronics/Guide.svelte | 23 + .../landing/selfhost/electronics/Logo.svelte | 13 + .../electronics/PiCartridge.stories.svelte | 19 + .../selfhost/electronics/PiCartridge.svelte | 192 + .../landing/selfhost/electronics/Plate.svelte | 72 + .../electronics/SnesCartridge.stories.svelte | 19 + .../selfhost/electronics/SnesCartridge.svelte | 180 + .../landing/selfhost/electronics/Solid.svelte | 36 + .../selfhost/electronics/insert.svelte.ts | 52 + .../landing/selfhost/electronics/iso.test.ts | 122 + .../landing/selfhost/electronics/iso.ts | 248 + .../selfhost/electronics/loop.svelte.ts | 39 + .../landing/selfhost/electronics/pi.test.ts | 25 + .../landing/selfhost/electronics/pi.ts | 273 + .../components/landing/selfhost/selfhost.css | 247 + .../landing/shell/CopyLink.stories.svelte | 20 + .../components/landing/shell/CopyLink.svelte | 147 + .../landing/shell/Footer.stories.svelte | 16 + .../components/landing/shell/Footer.svelte | 348 + .../landing/shell/MobileMenu.stories.svelte | 17 + .../landing/shell/MobileMenu.svelte | 175 + .../landing/shell/Navbar.stories.svelte | 16 + .../components/landing/shell/Navbar.svelte | 127 + .../components/landing/shell/SiteLink.svelte | 32 + web/src/lib/components/landing/shell/links.ts | 77 + web/src/lib/components/landing/sound.ts | 16 + .../landing/stories/LandingFrame.svelte | 68 + .../components/landing/stories/fixtures.ts | 51 + .../lib/components/landing/stories/frame.ts | 15 + .../landing/thread/Knot.stories.svelte | 28 + .../lib/components/landing/thread/Knot.svelte | 262 + .../landing/thread/Ribbon.stories.svelte | 83 + .../components/landing/thread/Ribbon.svelte | 151 + .../landing/thread/ThreadRail.svelte | 512 + .../landing/thread/geometry.test.ts | 195 + .../lib/components/landing/thread/geometry.ts | 312 + .../landing/thread/thread.svelte.ts | 107 + .../components/landing/ui/Container.svelte | 45 + .../components/landing/ui/EmailForm.svelte | 137 + .../lib/components/landing/ui/Heading.svelte | 37 + .../lib/components/landing/ui/Section.svelte | 25 + .../updates/RecentUpdates.stories.svelte | 18 + .../landing/updates/RecentUpdates.svelte | 41 + .../landing/updates/UpdateCard.stories.svelte | 17 + .../landing/updates/UpdateCard.svelte | 89 + .../components/landing/updates/updates.css | 114 + .../landing/updates/updates.test.ts | 91 + .../lib/components/landing/updates/updates.ts | 101 + .../lib/components/lines/LineSearch.svelte | 86 + web/src/lib/components/lines/diff.test.ts | 78 + web/src/lib/components/lines/diff.ts | 125 + web/src/lib/components/lines/finder.svelte.ts | 64 + web/src/lib/components/lines/highlight.ts | 42 + web/src/lib/components/lines/search.test.ts | 29 + web/src/lib/components/lines/search.ts | 33 + .../migration/MigrationLanding.stories.svelte | 66 + .../migration/MigrationLanding.svelte | 232 + .../ModerationNotice.stories.svelte | 45 + .../moderation/ModerationNotice.svelte | 22 + .../moderation/ReportDialog.stories.svelte | 77 + .../components/moderation/ReportDialog.svelte | 126 + .../moderation/ReportItems.stories.svelte | 84 + .../components/moderation/ReportItems.svelte | 76 + .../lib/components/moderation/gate.svelte.ts | 80 + .../components/moderation/reportReasons.ts | 60 + .../components/moderation/reporting.svelte.ts | 22 + .../NotificationBell.stories.svelte | 39 + .../notifications/NotificationBell.svelte | 143 + .../NotificationItem.stories.svelte | 50 + .../notifications/NotificationItem.svelte | 195 + .../notifications/NotificationItem.test.ts | 38 + .../NotificationList.stories.svelte | 81 + .../notifications/NotificationList.svelte | 89 + .../NotificationToolbar.stories.svelte | 16 + .../notifications/NotificationToolbar.svelte | 92 + web/src/lib/components/notifications/mock.ts | 264 + .../notifications/notification.test.ts | 77 + .../components/notifications/notification.ts | 154 + web/src/lib/components/notifications/types.ts | 53 + .../OrganizationCreationForm.stories.svelte | 194 + .../OrganizationCreationForm.svelte | 286 + .../welcome/OrganizationFinishStep.svelte | 109 + .../OrganizationMembersStep.stories.svelte | 20 + .../welcome/OrganizationMembersStep.svelte | 10 + .../welcome/OrganizationProfileStep.svelte | 126 + .../profile/AvatarList.stories.svelte | 46 + .../lib/components/profile/AvatarList.svelte | 60 + .../profile/FollowButton.stories.svelte | 143 + .../components/profile/FollowButton.svelte | 136 + .../profile/FollowCard.stories.svelte | 71 + .../lib/components/profile/FollowCard.svelte | 24 + .../profile/FollowCardContent.svelte | 108 + .../profile/FollowerFollowing.svelte | 32 + .../components/profile/ListViewSwitch.svelte | 26 + .../profile/NetworkVouches.stories.svelte | 44 + .../components/profile/NetworkVouches.svelte | 70 + .../profile/NotJoinedProfile.svelte | 39 + .../lib/components/profile/PagedList.svelte | 122 + .../profile/PagedListSkeleton.svelte | 34 + .../components/profile/PinSuggestions.svelte | 112 + .../profile/PinnedReposEditor.svelte | 320 + .../profile/PinnedRepositories.svelte | 62 + .../profile/ProfileActivityChart.svelte | 267 + .../lib/components/profile/ProfileBio.svelte | 28 + .../profile/ProfileChrome.stories.svelte | 776 + .../components/profile/ProfileChrome.svelte | 121 + .../components/profile/ProfileEditForm.svelte | 165 + .../components/profile/ProfileMoreMenu.svelte | 22 + .../profile/ProfilePopover.stories.svelte | 111 + .../components/profile/ProfilePopover.svelte | 211 + .../components/profile/ProfileShell.svelte | 60 + .../profile/ProfileSocialLinks.svelte | 170 + .../lib/components/profile/ProfileTabs.svelte | 101 + .../components/profile/RepoSearchForm.svelte | 44 + .../profile/StringCard.stories.svelte | 33 + .../lib/components/profile/StringCard.svelte | 21 + .../profile/StringCardContent.svelte | 48 + .../profile/UserProfileChrome.svelte | 327 + .../profile/VouchAvatar.stories.svelte | 57 + .../lib/components/profile/VouchAvatar.svelte | 86 + .../profile/VouchButton.stories.svelte | 145 + .../lib/components/profile/VouchButton.svelte | 291 + .../lib/components/profile/VouchCard.svelte | 54 + web/src/lib/components/profile/bio.test.ts | 24 + web/src/lib/components/profile/bio.ts | 17 + .../lib/components/profile/counts.svelte.ts | 53 + .../lib/components/profile/listView.svelte.ts | 49 + .../organization/OrganizationChrome.svelte | 196 + .../OrganizationOverviewTab.svelte | 68 + web/src/lib/components/profile/pages.test.ts | 782 + web/src/lib/components/profile/pages.ts | 683 + .../components/profile/pagesPinned.test.ts | 55 + .../components/profile/pagination.svelte.ts | 60 + .../lib/components/profile/pagination.test.ts | 70 + .../components/profile/pinnedEdit.svelte.ts | 45 + .../components/profile/pinnedReorder.test.ts | 42 + .../lib/components/profile/pinnedReorder.ts | 30 + .../profile/profileActivityChart.test.ts | 121 + .../profile/profileActivityChart.ts | 157 + .../components/profile/profileLinks.test.ts | 35 + .../lib/components/profile/profileLinks.ts | 31 + .../profile/tabs/OverviewTab.stories.svelte | 113 + .../profile/tabs/OverviewTab.svelte | 22 + .../components/profile/tabs/PeopleTab.svelte | 54 + .../profile/tabs/RepoListTab.stories.svelte | 181 + .../profile/tabs/RepoListTab.svelte | 78 + .../components/profile/tabs/StarredTab.svelte | 73 + .../profile/tabs/StringListTab.svelte | 49 + .../profile/tabs/VouchTab.stories.svelte | 85 + .../components/profile/tabs/VouchTab.svelte | 70 + web/src/lib/components/profile/types.ts | 148 + .../components/profile/vouchBadge.svelte.ts | 122 + .../lib/components/profile/vouchBadge.test.ts | 95 + .../profile/vouchRelationship.test.ts | 100 + .../components/profile/vouchRelationship.ts | 71 + .../lib/components/reaction/Reaction.svelte | 70 + .../reaction/ReactionPicker.stories.svelte | 29 + .../components/reaction/ReactionPicker.svelte | 113 + .../reaction/ReactionSummary.svelte | 203 + .../reaction/Reactions.stories.svelte | 223 + .../lib/components/reaction/Reactions.svelte | 165 + .../components/reaction/ReactorList.svelte | 29 + .../components/reaction/reactions.svelte.ts | 51 + .../lib/components/reaction/reactions.test.ts | 266 + web/src/lib/components/reaction/reactions.ts | 287 + .../components/repo/BlobHeader.stories.svelte | 167 + web/src/lib/components/repo/BlobHeader.svelte | 244 + .../components/repo/BlobView.stories.svelte | 359 + web/src/lib/components/repo/BlobView.svelte | 304 + .../components/repo/BranchList.stories.svelte | 37 + web/src/lib/components/repo/BranchList.svelte | 64 + .../repo/BranchTable.stories.svelte | 60 + .../lib/components/repo/BranchTable.svelte | 117 + .../repo/CloneDropdown.stories.svelte | 67 + .../lib/components/repo/CloneDropdown.svelte | 141 + .../lib/components/repo/CommitAuthor.svelte | 27 + .../components/repo/CommitBodyToggle.svelte | 22 + .../repo/CommitHeader.stories.svelte | 134 + .../lib/components/repo/CommitHeader.svelte | 206 + .../components/repo/CommitList.stories.svelte | 77 + web/src/lib/components/repo/CommitList.svelte | 106 + .../repo/CommitLogView.stories.svelte | 116 + .../lib/components/repo/CommitLogView.svelte | 75 + .../components/repo/CommitRow.stories.svelte | 143 + web/src/lib/components/repo/CommitRow.svelte | 188 + web/src/lib/components/repo/CommitRows.svelte | 25 + .../components/repo/CommitShaButton.svelte | 67 + .../lib/components/repo/CommitSubject.svelte | 54 + .../components/repo/CommitView.stories.svelte | 146 + web/src/lib/components/repo/CommitView.svelte | 154 + .../lib/components/repo/CompareView.svelte | 222 + .../lib/components/repo/DiffCodeView.svelte | 660 + .../lib/components/repo/DiffFileHeader.svelte | 81 + .../repo/DiffFileTree.stories.svelte | 57 + .../lib/components/repo/DiffFileTree.svelte | 139 + web/src/lib/components/repo/DiffFiles.svelte | 231 + .../repo/DiffStatPill.stories.svelte | 60 + .../lib/components/repo/DiffStatPill.svelte | 63 + .../components/repo/DiffView.stories.svelte | 52 + web/src/lib/components/repo/DiffView.svelte | 427 + .../components/repo/EmptyRepo.stories.svelte | 49 + web/src/lib/components/repo/EmptyRepo.svelte | 142 + .../repo/EntitySubscribeButton.stories.svelte | 176 + .../repo/EntitySubscribeButton.svelte | 153 + .../repo/FileCardHeader.stories.svelte | 26 + .../lib/components/repo/FileCardHeader.svelte | 67 + .../repo/FileHistoryView.stories.svelte | 83 + .../components/repo/FileHistoryView.svelte | 97 + .../lib/components/repo/FilePathChip.svelte | 24 + .../components/repo/FileTree.stories.svelte | 44 + web/src/lib/components/repo/FileTree.svelte | 192 + .../repo/ForkList.bones.stories.svelte | 28 + web/src/lib/components/repo/ForkList.svelte | 12 + .../repo/KnotUpgradeTag.stories.svelte | 20 + .../lib/components/repo/KnotUpgradeTag.svelte | 45 + .../components/repo/LabelPanel.stories.svelte | 137 + web/src/lib/components/repo/LabelPanel.svelte | 269 + .../lib/components/repo/LabelSummary.svelte | 122 + .../repo/LanguageList.stories.svelte | 45 + .../lib/components/repo/LanguageList.svelte | 61 + .../repo/LanguageStats.stories.svelte | 33 + .../lib/components/repo/LanguageStats.svelte | 107 + .../repo/LastCommitPanel.stories.svelte | 20 + .../components/repo/LastCommitPanel.svelte | 44 + .../lib/components/repo/LineHandles.svelte | 291 + web/src/lib/components/repo/OwnerTitle.svelte | 45 + .../repo/PanelHeader.stories.svelte | 16 + .../lib/components/repo/PanelHeader.svelte | 46 + .../components/repo/PierreFile.stories.svelte | 116 + web/src/lib/components/repo/PierreFile.svelte | 180 + .../lib/components/repo/Readme.stories.svelte | 50 + web/src/lib/components/repo/Readme.svelte | 41 + .../components/repo/RefChip.stories.svelte | 49 + web/src/lib/components/repo/RefChip.svelte | 38 + .../components/repo/RefLists.stories.svelte | 246 + .../repo/RefSelector.stories.svelte | 23 + .../lib/components/repo/RefSelector.svelte | 91 + .../components/repo/RepoCard.stories.svelte | 95 + web/src/lib/components/repo/RepoCard.svelte | 59 + .../components/repo/RepoCardContent.svelte | 152 + .../components/repo/RepoChrome.stories.svelte | 73 + web/src/lib/components/repo/RepoChrome.svelte | 38 + .../components/repo/RepoCreationForm.svelte | 407 + .../components/repo/RepoCrumbs.stories.svelte | 58 + web/src/lib/components/repo/RepoCrumbs.svelte | 45 + .../repo/RepoForkForm.stories.svelte | 28 + .../lib/components/repo/RepoForkForm.svelte | 57 + .../components/repo/RepoHeader.stories.svelte | 109 + web/src/lib/components/repo/RepoHeader.svelte | 147 + .../repo/RepoIndexPanelSkeleton.svelte | 6 + .../repo/RepoIndexView.stories.svelte | 294 + .../lib/components/repo/RepoIndexView.svelte | 218 + .../components/repo/RepoReadmeSkeleton.svelte | 6 + .../lib/components/repo/RepoSection.svelte | 54 + .../components/repo/RepoTabs.stories.svelte | 24 + web/src/lib/components/repo/RepoTabs.svelte | 66 + .../repo/RepoToolbar.stories.svelte | 27 + .../lib/components/repo/RepoToolbar.svelte | 71 + web/src/lib/components/repo/RevId.svelte | 33 + .../repo/ServicePicker.stories.svelte | 32 + .../lib/components/repo/ServicePicker.svelte | 91 + .../lib/components/repo/SourceSelector.svelte | 207 + .../components/repo/StarButton.stories.svelte | 142 + web/src/lib/components/repo/StarButton.svelte | 215 + .../repo/StarredBy.bones.stories.svelte | 33 + web/src/lib/components/repo/StarredBy.svelte | 17 + .../components/repo/StarredByContent.svelte | 31 + .../components/repo/SubjectLabelPanel.svelte | 114 + .../repo/SubscribeButton.stories.svelte | 138 + .../components/repo/SubscribeButton.svelte | 114 + .../lib/components/repo/TagArtifacts.svelte | 213 + .../components/repo/TagCard.stories.svelte | 51 + web/src/lib/components/repo/TagCard.svelte | 76 + .../components/repo/TagList.stories.svelte | 61 + web/src/lib/components/repo/TagList.svelte | 97 + .../components/repo/TopicList.stories.svelte | 36 + web/src/lib/components/repo/TopicList.svelte | 106 + .../components/repo/TreeHeader.stories.svelte | 27 + web/src/lib/components/repo/TreeHeader.svelte | 44 + .../repo/TreeView.bones.stories.svelte | 48 + .../components/repo/blame/BlameChip.svelte | 78 + .../components/repo/blame/BlameOverlay.svelte | 45 + .../lib/components/repo/blame/hover.svelte.ts | 54 + .../repo/blame/request.stories.svelte | 64 + .../components/repo/blame/request.svelte.ts | 78 + .../lib/components/repo/blame/types.test.ts | 30 + web/src/lib/components/repo/blame/types.ts | 35 + web/src/lib/components/repo/branchOptions.ts | 12 + .../lib/components/repo/commitSignature.ts | 17 + web/src/lib/components/repo/diffQueue.test.ts | 277 + web/src/lib/components/repo/diffQueue.ts | 206 + web/src/lib/components/repo/diffRpc.ts | 152 + web/src/lib/components/repo/diffWorker.ts | 132 + .../lib/components/repo/fileAnchor.svelte.ts | 82 + web/src/lib/components/repo/fileDiff.test.ts | 62 + web/src/lib/components/repo/fileDiff.ts | 47 + .../lib/components/repo/importPoll.test.ts | 62 + web/src/lib/components/repo/importPoll.ts | 44 + .../repo/issues/IssueForm.stories.svelte | 96 + .../components/repo/issues/IssueForm.svelte | 197 + .../issues/IssueThreadView.stories.svelte | 67 + .../repo/issues/IssueThreadView.svelte | 376 + web/src/lib/components/repo/languageColors.ts | 742 + .../lib/components/repo/lineAnchor.test.ts | 110 + web/src/lib/components/repo/lineAnchor.ts | 67 + web/src/lib/components/repo/pierre.ts | 179 + web/src/lib/components/repo/pierreDiff.css | 258 + web/src/lib/components/repo/pierreFile.css | 9 + .../repo/pipelines/Elapsed.stories.svelte | 26 + .../components/repo/pipelines/Elapsed.svelte | 33 + .../pipelines/PipelineCard.stories.svelte | 45 + .../repo/pipelines/PipelineCard.svelte | 36 + .../repo/pipelines/PipelineCardContent.svelte | 164 + .../repo/pipelines/PipelineEmptyState.svelte | 82 + .../pipelines/PipelineList.stories.svelte | 78 + .../repo/pipelines/PipelineList.svelte | 97 + .../PipelineRetryButton.stories.svelte | 50 + .../repo/pipelines/PipelineRetryButton.svelte | 45 + .../pipelines/PipelineSearch.stories.svelte | 16 + .../repo/pipelines/PipelineSearch.svelte | 74 + .../repo/pipelines/PipelineStatusIcon.svelte | 68 + .../repo/pipelines/PipelineSymbol.svelte | 86 + .../pipelines/PipelineToolbar.stories.svelte | 22 + .../repo/pipelines/PipelineToolbar.svelte | 76 + .../PipelineWorkflows.stories.svelte | 33 + .../repo/pipelines/PipelineWorkflows.svelte | 171 + .../pipelines/StepStatusIcon.stories.svelte | 38 + .../repo/pipelines/StepStatusIcon.svelte | 110 + .../pipelines/WorkflowLogs.stories.svelte | 192 + .../repo/pipelines/WorkflowLogs.svelte | 672 + .../pipelines/WorkflowRunView.stories.svelte | 42 + .../repo/pipelines/WorkflowRunView.svelte | 100 + .../pipelines/WorkflowSidebar.stories.svelte | 61 + .../repo/pipelines/WorkflowSidebar.svelte | 147 + .../pipelines/WorkflowSteps.stories.svelte | 128 + .../repo/pipelines/WorkflowSteps.svelte | 129 + .../pipelines/WorkflowView.stories.svelte | 91 + .../repo/pipelines/WorkflowView.svelte | 96 + .../repo/pipelines/livePipelines.test.ts | 89 + .../repo/pipelines/livePipelines.ts | 90 + .../repo/pipelines/logStream.svelte.ts | 100 + .../components/repo/pipelines/logs.test.ts | 318 + web/src/lib/components/repo/pipelines/logs.ts | 188 + .../repo/pipelines/pipeline.test.ts | 27 + .../lib/components/repo/pipelines/pipeline.ts | 104 + .../repo/pulls/PullActions.stories.svelte | 92 + .../components/repo/pulls/PullActions.svelte | 163 + .../repo/pulls/PullApproveCi.svelte | 88 + .../repo/pulls/PullCommitList.stories.svelte | 207 + .../repo/pulls/PullCommitList.svelte | 140 + .../pulls/PullCommitListHeader.stories.svelte | 89 + .../repo/pulls/PullCommitListHeader.svelte | 57 + .../repo/pulls/PullCommitRow.svelte | 136 + .../repo/pulls/PullCompose.stories.svelte | 154 + .../components/repo/pulls/PullCompose.svelte | 449 + .../repo/pulls/PullDiscussion.stories.svelte | 217 + .../repo/pulls/PullDiscussion.svelte | 132 + .../pulls/PullDiscussionArrivalFixture.svelte | 29 + .../pulls/PullReviewComment.stories.svelte | 100 + .../repo/pulls/PullReviewComment.svelte | 245 + .../repo/pulls/PullReviewCommentEditor.svelte | 81 + .../PullReviewCommentForm.stories.svelte | 114 + .../repo/pulls/PullReviewCommentForm.svelte | 217 + .../repo/pulls/PullVersionList.stories.svelte | 70 + .../repo/pulls/PullVersionList.svelte | 91 + .../repo/pulls/PullVersionSheet.svelte | 92 + .../components/repo/pulls/RevIdButton.svelte | 27 + .../components/repo/pulls/discussionWidth.ts | 16 + .../repo/pulls/mirrorBehind.test.ts | 117 + .../lib/components/repo/pulls/mirrorBehind.ts | 55 + .../repo/tickets/AssigneeField.svelte | 135 + .../repo/tickets/TargetBranchField.svelte | 45 + .../repo/tickets/Ticket.stories.svelte | 193 + .../lib/components/repo/tickets/Ticket.svelte | 267 + .../repo/tickets/TicketBody.stories.svelte | 30 + .../components/repo/tickets/TicketBody.svelte | 20 + .../repo/tickets/TicketCard.stories.svelte | 77 + .../components/repo/tickets/TicketCard.svelte | 41 + .../repo/tickets/TicketCardContent.svelte | 56 + .../repo/tickets/TicketForm.stories.svelte | 82 + .../components/repo/tickets/TicketForm.svelte | 158 + .../repo/tickets/TicketInfoBar.svelte | 194 + .../repo/tickets/TicketLabels.svelte | 112 + .../repo/tickets/TicketList.stories.svelte | 87 + .../components/repo/tickets/TicketList.svelte | 45 + .../tickets/TicketQueryInput.stories.svelte | 370 + .../repo/tickets/TicketQueryInput.svelte | 373 + .../repo/tickets/TicketSearch.stories.svelte | 115 + .../repo/tickets/TicketSearch.svelte | 38 + .../tickets/TicketStatePill.stories.svelte | 26 + .../repo/tickets/TicketStatePill.svelte | 78 + .../repo/tickets/TicketToolbar.stories.svelte | 89 + .../repo/tickets/TicketToolbar.svelte | 112 + web/src/lib/components/repo/types.ts | 125 + web/src/lib/components/repo/urls.test.ts | 16 + web/src/lib/components/repo/urls.ts | 27 + .../search/CodeResultCard.stories.svelte | 26 + .../components/search/CodeResultCard.svelte | 59 + .../components/search/CodeSearchView.svelte | 162 + .../search/FileMatch.stories.svelte | 50 + .../lib/components/search/FileMatch.svelte | 211 + .../lib/components/search/QueryToken.svelte | 9 + .../search/RepoSearchResults.svelte | 67 + .../lib/components/search/SearchBar.svelte | 27 + .../lib/components/search/SearchLayout.svelte | 33 + .../search/SearchPage.stories.svelte | 169 + .../components/search/SearchSidebar.svelte | 61 + .../lib/components/search/SearchTabs.svelte | 24 + web/src/lib/components/search/chunks.test.ts | 148 + web/src/lib/components/search/chunks.ts | 132 + .../components/search/codeSearch.svelte.ts | 150 + web/src/lib/components/search/highlight.ts | 32 + web/src/lib/components/search/repoSearch.ts | 46 + .../search/tabs/CodeSearchTab.svelte | 11 + .../search/tabs/RepoSearchTab.svelte | 69 + web/src/lib/components/seo/Meta.svelte | 78 + .../settings/AccountFarewell.svelte | 55 + .../settings/AvatarForm.stories.svelte | 37 + .../lib/components/settings/AvatarForm.svelte | 114 + .../settings/ChangeHandleForm.svelte | 198 + .../lib/components/settings/CodeChip.svelte | 17 + .../lib/components/settings/DocsButton.svelte | 20 + .../lib/components/settings/DrillDown.svelte | 45 + .../components/settings/FormActions.svelte | 17 + .../lib/components/settings/FormRow.svelte | 43 + .../ProfileSettingsForm.stories.svelte | 133 + .../settings/ProfileSettingsForm.svelte | 216 + .../settings/RepoGeneralForm.svelte | 348 + .../components/settings/SettingsBlock.svelte | 65 + .../components/settings/SettingsEmpty.svelte | 14 + .../components/settings/SettingsHeader.svelte | 30 + .../components/settings/SettingsList.svelte | 36 + .../settings/SettingsNav.stories.svelte | 58 + .../components/settings/SettingsNav.svelte | 42 + .../settings/SettingsPanel.stories.svelte | 154 + .../components/settings/SettingsPanel.svelte | 30 + .../components/settings/SettingsRow.svelte | 112 + .../settings/SettingsSaveBar.svelte | 35 + .../settings/SettingsSection.svelte | 36 + .../settings/SettingsToolbar.svelte | 24 + .../settings/ThemeChip.stories.svelte | 30 + .../lib/components/settings/ThemeChip.svelte | 23 + .../settings/ThemeOption.stories.svelte | 36 + .../components/settings/ThemeOption.svelte | 24 + .../settings/ThemePicker.stories.svelte | 19 + .../components/settings/ThemePicker.svelte | 31 + .../settings/VerifyEmailCard.stories.svelte | 34 + .../settings/VerifyEmailCard.svelte | 52 + .../components/settings/descriptionLines.ts | 4 + .../settings/tabs/EmailsTab.stories.svelte | 132 + .../components/settings/tabs/EmailsTab.svelte | 325 + .../components/settings/tabs/KeysTab.svelte | 141 + .../components/settings/tabs/KnotsTab.svelte | 80 + .../settings/tabs/NotificationsTab.svelte | 203 + .../tabs/OrganizationsTab.stories.svelte | 116 + .../settings/tabs/OrganizationsTab.svelte | 159 + .../settings/tabs/ProfileTab.svelte | 73 + .../components/settings/tabs/SitesTab.svelte | 170 + .../settings/tabs/SpindlesTab.svelte | 79 + .../settings/tabs/emailRecheck.test.ts | 245 + .../components/settings/tabs/emailRecheck.ts | 121 + .../components/shell/Topbar.stories.svelte | 115 + web/src/lib/components/shell/Topbar.svelte | 416 + .../lib/components/shell/TopbarSearch.svelte | 229 + web/src/lib/components/shell/siteLinks.ts | 59 + .../lib/components/shell/topbarSearch.test.ts | 26 + web/src/lib/components/shell/topbarSearch.ts | 8 + .../strings/StringForm.stories.svelte | 50 + .../lib/components/strings/StringForm.svelte | 183 + .../components/strings/StringHeader.svelte | 101 + .../strings/StringView.stories.svelte | 136 + .../lib/components/strings/StringView.svelte | 68 + web/src/lib/components/strings/urls.ts | 7 + .../timeline/Announcements.stories.svelte | 30 + .../components/timeline/Announcements.svelte | 52 + .../components/timeline/NewsletterCard.svelte | 100 + .../NotificationsPreview.stories.svelte | 46 + .../timeline/NotificationsPreview.svelte | 103 + .../components/timeline/PreviewCard.svelte | 50 + .../components/timeline/PreviewEmpty.svelte | 16 + .../timeline/RecentItem.stories.svelte | 55 + .../lib/components/timeline/RecentItem.svelte | 102 + .../timeline/RecentsPreview.stories.svelte | 85 + .../components/timeline/RecentsPreview.svelte | 46 + .../timeline/Timeline.stories.svelte | 42 + .../lib/components/timeline/Timeline.svelte | 57 + .../timeline/TimelineActorHeader.svelte | 36 + .../components/timeline/TimelineCard.svelte | 23 + .../TimelineFollowEvent.stories.svelte | 82 + .../timeline/TimelineFollowEvent.svelte | 43 + .../timeline/TimelineFollowers.svelte | 38 + .../timeline/TimelineForkers.svelte | 36 + .../components/timeline/TimelineGroup.svelte | 50 + .../timeline/TimelineRepoEvent.stories.svelte | 78 + .../timeline/TimelineRepoEvent.svelte | 68 + .../timeline/TimelineSkeleton.svelte | 6 + .../timeline/TimelineStarEvent.stories.svelte | 56 + .../timeline/TimelineStarEvent.svelte | 41 + .../timeline/TimelineStargazers.svelte | 36 + .../timeline/TimelineTargetHeader.svelte | 29 + .../timeline/TimelineTargets.stories.svelte | 115 + .../timeline/TimelineView.stories.svelte | 81 + .../components/timeline/TimelineView.svelte | 183 + .../timeline/TrendingRepos.stories.svelte | 16 + .../components/timeline/TrendingRepos.svelte | 29 + .../timeline/TrendingReposList.stories.svelte | 18 + .../timeline/TrendingReposList.svelte | 62 + .../timeline/TrendingSkeleton.svelte | 12 + web/src/lib/components/timeline/group.test.ts | 274 + web/src/lib/components/timeline/group.ts | 140 + web/src/lib/components/timeline/load.test.ts | 281 + web/src/lib/components/timeline/load.ts | 125 + web/src/lib/components/timeline/mappers.ts | 35 + .../lib/components/timeline/recents.svelte.ts | 64 + .../lib/components/timeline/recents.test.ts | 384 + web/src/lib/components/timeline/recents.ts | 129 + web/src/lib/components/timeline/types.ts | 33 + web/src/lib/components/ui/ActorInput.svelte | 290 + web/src/lib/components/ui/AnimatedLogo.svelte | 335 + .../lib/components/ui/Avatar.stories.svelte | 56 + web/src/lib/components/ui/Avatar.svelte | 123 + web/src/lib/components/ui/Avatar.test.ts | 25 + web/src/lib/components/ui/Bones.svelte | 172 + .../lib/components/ui/Button.stories.svelte | 220 + web/src/lib/components/ui/Button.svelte | 574 + .../components/ui/ButtonCard.stories.svelte | 35 + web/src/lib/components/ui/ButtonCard.svelte | 118 + .../components/ui/ButtonGroup.stories.svelte | 35 + web/src/lib/components/ui/ButtonGroup.svelte | 64 + web/src/lib/components/ui/Card.stories.svelte | 26 + web/src/lib/components/ui/Card.svelte | 66 + .../lib/components/ui/Checkbox.stories.svelte | 26 + web/src/lib/components/ui/Checkbox.svelte | 74 + .../lib/components/ui/CodeBlockToolbar.svelte | 72 + .../components/ui/ColorBall.stories.svelte | 25 + web/src/lib/components/ui/ColorBall.svelte | 36 + web/src/lib/components/ui/Combobox.svelte | 494 + .../components/ui/ConfirmModal.stories.svelte | 32 + web/src/lib/components/ui/ConfirmModal.svelte | 39 + .../components/ui/CopyButton.stories.svelte | 83 + web/src/lib/components/ui/CopyButton.svelte | 213 + .../lib/components/ui/Dropdown.stories.svelte | 151 + web/src/lib/components/ui/Dropdown.svelte | 393 + .../lib/components/ui/DropdownGroup.svelte | 33 + web/src/lib/components/ui/DropdownItem.svelte | 90 + .../lib/components/ui/DropdownSubmenu.svelte | 362 + .../components/ui/EmptyState.stories.svelte | 24 + web/src/lib/components/ui/EmptyState.svelte | 59 + .../lib/components/ui/Error.stories.svelte | 24 + web/src/lib/components/ui/Error.svelte | 91 + .../lib/components/ui/Field.stories.svelte | 48 + web/src/lib/components/ui/Field.svelte | 81 + .../lib/components/ui/FormStep.stories.svelte | 39 + web/src/lib/components/ui/FormStep.svelte | 81 + .../components/ui/FormStepDisclosure.svelte | 40 + .../components/ui/GraphCell.stories.svelte | 125 + web/src/lib/components/ui/GraphCell.svelte | 105 + .../ui/HiddenContent.stories.svelte | 27 + .../lib/components/ui/HiddenContent.svelte | 52 + .../components/ui/Hovercard.stories.svelte | 49 + web/src/lib/components/ui/Hovercard.svelte | 325 + .../lib/components/ui/InlineMarkdown.svelte | 17 + .../lib/components/ui/Input.stories.svelte | 224 + web/src/lib/components/ui/Input.svelte | 168 + web/src/lib/components/ui/Kbd.stories.svelte | 53 + web/src/lib/components/ui/Kbd.svelte | 32 + .../components/ui/LayerCard.stories.svelte | 38 + web/src/lib/components/ui/LayerCard.svelte | 24 + .../lib/components/ui/LayerCardPrimary.svelte | 36 + .../components/ui/LayerCardSecondary.svelte | 23 + .../lib/components/ui/Lightbox.stories.svelte | 79 + web/src/lib/components/ui/Lightbox.svelte | 555 + web/src/lib/components/ui/Link.stories.svelte | 40 + web/src/lib/components/ui/Link.svelte | 113 + web/src/lib/components/ui/ListPanel.svelte | 25 + web/src/lib/components/ui/ListToolbar.svelte | 46 + .../ui/LoadingPrimitives.bones.stories.svelte | 16 + web/src/lib/components/ui/Logo.stories.svelte | 44 + web/src/lib/components/ui/Logo.svelte | 38 + .../ui/MarkdownEditor.stories.svelte | 88 + .../lib/components/ui/MarkdownEditor.svelte | 628 + .../components/ui/MarkdownTypeahead.svelte | 308 + web/src/lib/components/ui/Markup.svelte | 105 + .../lib/components/ui/Modal.stories.svelte | 52 + web/src/lib/components/ui/Modal.svelte | 366 + .../lib/components/ui/MoreMenu.stories.svelte | 38 + web/src/lib/components/ui/MoreMenu.svelte | 24 + web/src/lib/components/ui/NumberFlip.svelte | 112 + .../components/ui/Pagination.stories.svelte | 33 + web/src/lib/components/ui/Pagination.svelte | 218 + web/src/lib/components/ui/Pending.svelte | 40 + .../components/ui/Radio.bones.stories.svelte | 13 + .../lib/components/ui/Radio.stories.svelte | 25 + web/src/lib/components/ui/Radio.svelte | 57 + .../ui/RangeSelector.stories.svelte | 93 + .../lib/components/ui/RangeSelector.svelte | 365 + .../lib/components/ui/RangeSelectorRow.svelte | 55 + .../components/ui/ResizeHandle.stories.svelte | 54 + web/src/lib/components/ui/ResizeHandle.svelte | 65 + .../ui/RoutePagination.stories.svelte | 50 + .../lib/components/ui/RoutePagination.svelte | 24 + .../lib/components/ui/Section.stories.svelte | 65 + web/src/lib/components/ui/Section.svelte | 48 + web/src/lib/components/ui/Segment.svelte | 81 + .../ui/SegmentedControl.stories.svelte | 59 + .../lib/components/ui/SegmentedControl.svelte | 161 + .../lib/components/ui/Select.stories.svelte | 107 + web/src/lib/components/ui/Select.svelte | 143 + .../components/ui/Separator.stories.svelte | 64 + web/src/lib/components/ui/Separator.svelte | 22 + .../components/ui/ShimmerText.stories.svelte | 47 + web/src/lib/components/ui/ShimmerText.svelte | 98 + .../components/ui/SignupPrompt.stories.svelte | 13 + web/src/lib/components/ui/SignupPrompt.svelte | 22 + .../lib/components/ui/Spinner.stories.svelte | 24 + web/src/lib/components/ui/Spinner.svelte | 12 + .../components/ui/SplitButton.stories.svelte | 45 + web/src/lib/components/ui/SplitButton.svelte | 103 + web/src/lib/components/ui/StaleReload.svelte | 17 + .../components/ui/StepMarker.stories.svelte | 34 + web/src/lib/components/ui/StepMarker.svelte | 22 + web/src/lib/components/ui/Tabs.stories.svelte | 70 + web/src/lib/components/ui/Tabs.svelte | 314 + web/src/lib/components/ui/Tag.stories.svelte | 29 + web/src/lib/components/ui/Tag.svelte | 78 + web/src/lib/components/ui/TagInput.svelte | 674 + .../lib/components/ui/TextSwap.stories.svelte | 58 + web/src/lib/components/ui/TextSwap.svelte | 36 + .../lib/components/ui/Textarea.stories.svelte | 39 + web/src/lib/components/ui/Textarea.svelte | 131 + .../lib/components/ui/TimeAgo.stories.svelte | 35 + web/src/lib/components/ui/TimeAgo.svelte | 18 + .../lib/components/ui/Toggle.stories.svelte | 43 + web/src/lib/components/ui/Toggle.svelte | 82 + web/src/lib/components/ui/ToggleFaces.svelte | 60 + .../lib/components/ui/Tooltip.stories.svelte | 153 + web/src/lib/components/ui/Tooltip.svelte | 465 + web/src/lib/components/ui/Turnstile.svelte | 104 + web/src/lib/components/ui/User.stories.svelte | 45 + web/src/lib/components/ui/User.svelte | 144 + .../lib/components/ui/codeblockToolbars.ts | 36 + web/src/lib/components/ui/inputField.ts | 62 + web/src/lib/components/ui/keyLabels.test.ts | 11 + web/src/lib/components/ui/keyLabels.ts | 8 + web/src/lib/components/ui/logotype.ts | 12 + .../lib/components/ui/markdownFormat.test.ts | 237 + web/src/lib/components/ui/markdownFormat.ts | 263 + .../components/ui/markdownSuggestions.test.ts | 73 + .../lib/components/ui/markdownSuggestions.ts | 129 + web/src/lib/components/ui/mentionPeople.ts | 14 + web/src/lib/components/ui/menuItems.ts | 39 + web/src/lib/components/ui/mermaidDiagrams.ts | 54 + web/src/lib/components/ui/popover.svelte.ts | 115 + web/src/lib/components/ui/rangeMode.test.ts | 64 + web/src/lib/components/ui/rangeMode.ts | 46 + web/src/lib/components/ui/selectField.ts | 56 + .../welcome/EmailsStep.stories.svelte | 75 + .../lib/components/welcome/EmailsStep.svelte | 21 + .../welcome/FinishStep.stories.svelte | 14 + .../lib/components/welcome/FinishStep.svelte | 213 + .../welcome/KeysStep.stories.svelte | 35 + .../lib/components/welcome/KeysStep.svelte | 191 + .../components/welcome/OnboardingNudge.svelte | 77 + .../lib/components/welcome/ProfileStep.svelte | 155 + .../lib/components/welcome/SocialStep.svelte | 58 + .../lib/components/welcome/StepHeader.svelte | 31 + .../components/welcome/Steps.stories.svelte | 39 + web/src/lib/components/welcome/Steps.svelte | 169 + .../lib/components/welcome/WelcomeCard.svelte | 220 + web/src/lib/components/welcome/steps.ts | 14 + web/src/lib/fetch.ts | 1 + web/src/lib/fetchJson.ts | 15 + web/src/lib/format.test.ts | 141 + web/src/lib/format.ts | 135 + web/src/lib/github.ts | 66 + web/src/lib/handle.test.ts | 20 + web/src/lib/handle.ts | 15 + web/src/lib/images.test.ts | 18 + web/src/lib/images.ts | 86 + web/src/lib/lightbox.svelte.ts | 74 + web/src/lib/markup/collapse.test.ts | 65 + web/src/lib/markup/collapse.ts | 20 + web/src/lib/markup/format.ts | 3 + web/src/lib/markup/highlight.ts | 97 + web/src/lib/markup/index.ts | 3 + web/src/lib/markup/inline.test.ts | 30 + web/src/lib/markup/inline.ts | 18 + web/src/lib/markup/markdown.test.ts | 336 + web/src/lib/markup/markdown.ts | 155 + web/src/lib/markup/mermaid.ts | 45 + web/src/lib/markup/pageContext.ts | 10 + web/src/lib/markup/paths.ts | 90 + web/src/lib/markup/preview.ts | 8 + web/src/lib/markup/render.test.ts | 13 + web/src/lib/markup/render.ts | 5 + web/src/lib/markup/sanitize.ts | 161 + web/src/lib/markup/text.test.ts | 60 + web/src/lib/markup/text.ts | 62 + web/src/lib/migration/import.test.ts | 427 + web/src/lib/migration/import.ts | 367 + web/src/lib/migration/liar.test.ts | 53 + web/src/lib/migration/liar.ts | 84 + web/src/lib/migration/mock.ts | 70 + web/src/lib/migration/repoCar.ts | 54 + web/src/lib/migration/status.test.ts | 167 + web/src/lib/migration/status.ts | 71 + web/src/lib/mobile.svelte.ts | 3 + web/src/lib/motion/reveal.ts | 50 + web/src/lib/motion/spring.ts | 54 + web/src/lib/motion/value.svelte.ts | 90 + web/src/lib/navPulse.test.ts | 35 + web/src/lib/navPulse.ts | 8 + web/src/lib/notifications.svelte.ts | 178 + web/src/lib/notifications.test.ts | 99 + web/src/lib/oauth-client-metadata.test.ts | 76 + web/src/lib/oauth-client-metadata.ts | 121 + web/src/lib/optimistic.svelte.ts | 181 + web/src/lib/pagination.svelte.ts | 25 + web/src/lib/pagination.test.ts | 55 + web/src/lib/pagination.ts | 14 + web/src/lib/popoverExit.ts | 107 + web/src/lib/publicFetch.test.ts | 124 + web/src/lib/publicFetch.ts | 94 + web/src/lib/rem.ts | 4 + web/src/lib/repoQuery.test.ts | 47 + web/src/lib/repoQuery.ts | 54 + web/src/lib/revIdHighlight.svelte.ts | 20 + web/src/lib/safeHref.test.ts | 38 + web/src/lib/safeHref.ts | 17 + web/src/lib/seo/jsonLd.test.ts | 17 + web/src/lib/seo/jsonLd.ts | 13 + web/src/lib/seo/og.test.ts | 246 + web/src/lib/seo/og.ts | 463 + web/src/lib/server/avatar.test.ts | 40 + web/src/lib/server/avatar.ts | 32 + web/src/lib/server/blob.ts | 135 + web/src/lib/server/camo.ts | 23 + web/src/lib/server/camotarget.test.ts | 36 + web/src/lib/server/camotarget.ts | 7 + web/src/lib/server/config.test.ts | 24 + web/src/lib/server/config.ts | 245 + web/src/lib/server/gitHttp.test.ts | 184 + web/src/lib/server/gitHttp.ts | 70 + web/src/lib/server/github.test.ts | 264 + web/src/lib/server/github/api.ts | 245 + web/src/lib/server/github/cookies.ts | 38 + web/src/lib/server/github/env.ts | 36 + web/src/lib/server/github/http.ts | 20 + web/src/lib/server/github/index.ts | 30 + web/src/lib/server/github/oauth.ts | 90 + web/src/lib/server/github/session.ts | 52 + web/src/lib/server/indexnow.test.ts | 97 + web/src/lib/server/indexnow.ts | 121 + web/src/lib/server/lookupEvents.test.ts | 173 + web/src/lib/server/lookupEvents.ts | 144 + web/src/lib/server/posthog.ts | 26 + web/src/lib/server/repo.test.ts | 51 + web/src/lib/server/repo.ts | 56 + web/src/lib/server/repoIndex.ts | 203 + web/src/lib/server/sealed.ts | 54 + web/src/lib/shiftKey.svelte.ts | 29 + web/src/lib/ssr/pool.test.ts | 116 + web/src/lib/ssr/pool.ts | 255 + web/src/lib/ssr/render.test.ts | 31 + web/src/lib/ssr/render.ts | 63 + web/src/lib/ssr/worker.ts | 44 + web/src/lib/status.test.ts | 24 + web/src/lib/status.ts | 31 + web/src/lib/stuck.ts | 12 + web/src/lib/subdomain.test.ts | 19 + web/src/lib/subdomain.ts | 8 + web/src/lib/syntax.test.ts | 48 + web/src/lib/syntax.ts | 12 + web/src/lib/theme.test.ts | 32 + web/src/lib/theme.ts | 96 + web/src/lib/ticketQuery.test.ts | 217 + web/src/lib/ticketQuery.ts | 387 + web/src/lib/twMergeConfig.ts | 25 + web/src/lib/unsavedChanges.ts | 13 + web/src/lib/uri.ts | 7 + web/src/listView.css | 105 + web/src/logs.css | 111 + web/src/markup.css | 283 + web/src/params.ts | 10 + web/src/pierre.css | 84 + web/src/routes/+error.svelte | 80 + web/src/routes/+layout.server.ts | 27 + web/src/routes/+layout.svelte | 171 + web/src/routes/+page.svelte | 45 + web/src/routes/+page.ts | 39 + web/src/routes/[handle]/+layout.svelte | 157 + web/src/routes/[handle]/+layout.ts | 146 + web/src/routes/[handle]/+page.svelte | 168 + web/src/routes/[handle]/+page.ts | 124 + .../routes/[handle]/[repo]/+layout.server.ts | 174 + web/src/routes/[handle]/[repo]/+layout.ts | 24 + .../routes/[handle]/[repo]/+layout@.svelte | 326 + .../routes/[handle]/[repo]/+page.server.ts | 15 + web/src/routes/[handle]/[repo]/+page.svelte | 76 + .../[repo]/archive/[...ref]/+server.ts | 45 + .../[handle]/[repo]/archive/archive.test.ts | 116 + .../[repo]/artifacts/[rkey]/+server.ts | 52 + .../blob/[ref]/[...path]/+page.server.ts | 30 + .../[repo]/blob/[ref]/[...path]/+page.svelte | 72 + .../[handle]/[repo]/branches/+page.svelte | 24 + .../routes/[handle]/[repo]/branches/+page.ts | 31 + .../[repo]/commit-statuses/+server.ts | 39 + .../[handle]/[repo]/commit/[ref]/+page.svelte | 97 + .../[handle]/[repo]/commit/[ref]/+page.ts | 142 + .../[repo]/commit/[spec=rawCommit]/+server.ts | 39 + .../[repo]/commits/[ref]/+page.svelte | 37 + .../[handle]/[repo]/commits/[ref]/+page.ts | 74 + .../[repo]/compare/[...spec]/+page.svelte | 11 + .../[repo]/compare/[...spec]/+page.ts | 36 + .../[handle]/[repo]/feed.atom/+server.test.ts | 75 + .../[handle]/[repo]/feed.atom/+server.ts | 130 + .../[handle]/[repo]/fork/+page.server.ts | 8 + .../routes/[handle]/[repo]/fork/+page.svelte | 16 + web/src/routes/[handle]/[repo]/fork/+page.ts | 9 + .../routes/[handle]/[repo]/forks/+page.svelte | 29 + web/src/routes/[handle]/[repo]/forks/+page.ts | 32 + .../[handle]/[repo]/forks/forks.test.ts | 86 + .../[repo]/git-receive-pack/+server.ts | 4 + .../[repo]/git-upload-archive/+server.ts | 4 + .../[repo]/git-upload-pack/+server.ts | 4 + .../history/[ref]/[...path]/+page.svelte | 26 + .../[repo]/history/[ref]/[...path]/+page.ts | 52 + .../[handle]/[repo]/info/refs/+server.ts | 4 + .../[handle]/[repo]/issues/+page.server.ts | 95 + .../[handle]/[repo]/issues/+page.svelte | 57 + .../[repo]/issues/[rkey]/+page.server.ts | 173 + .../[repo]/issues/[rkey]/+page.svelte | 110 + .../[repo]/issues/[rkey]/issue.test.ts | 166 + .../[handle]/[repo]/issues/issues.test.ts | 183 + .../[repo]/issues/new/+page.server.ts | 6 + .../[handle]/[repo]/issues/new/+page.svelte | 42 + .../[handle]/[repo]/issues/new/+page.ts | 9 + .../[handle]/[repo]/languages/+page.svelte | 10 + .../routes/[handle]/[repo]/languages/+page.ts | 16 + .../[repo]/languages/languages.test.ts | 68 + web/src/routes/[handle]/[repo]/layout.test.ts | 55 + .../[handle]/[repo]/pipelines/+page.svelte | 62 + .../routes/[handle]/[repo]/pipelines/+page.ts | 127 + .../workflow/[workflow]/+page.svelte | 53 + .../[pipeline]/workflow/[workflow]/+page.ts | 38 + .../[repo]/pipelines/pipelines.test.ts | 294 + .../[handle]/[repo]/pulls/+page.server.ts | 83 + .../routes/[handle]/[repo]/pulls/+page.svelte | 59 + .../[repo]/pulls/[rkey]/+layout.server.ts | 156 + .../[handle]/[repo]/pulls/[rkey]/+page.svelte | 1 + .../[handle]/[repo]/pulls/[rkey]/+page.ts | 12 + .../pulls/[rkey]/[version].patch/+server.ts | 42 + .../pulls/[rkey]/[version]/+layout.server.ts | 65 + .../[rkey]/[version]/[[range]]/+page.svelte | 51 + .../pulls/[rkey]/[version]/[[range]]/+page.ts | 95 + .../[[range]]/PullViewPage.stories.svelte | 766 + .../[version]/[[range]]/PullViewPage.svelte | 1369 + .../[handle]/[repo]/pulls/new/+page.server.ts | 6 + .../[handle]/[repo]/pulls/new/+page.svelte | 27 + .../routes/[handle]/[repo]/pulls/new/+page.ts | 25 + .../[handle]/[repo]/pulls/pulls.test.ts | 120 + .../[repo]/raw/[ref]/[...path]/+server.ts | 26 + .../routes/[handle]/[repo]/raw/raw.test.ts | 37 + .../[handle]/[repo]/search/+page.svelte | 18 + .../routes/[handle]/[repo]/search/+page.ts | 8 + .../[handle]/[repo]/settings/+layout.svelte | 45 + .../[handle]/[repo]/settings/+layout.ts | 67 + .../[handle]/[repo]/settings/+page.svelte | 28 + .../routes/[handle]/[repo]/settings/+page.ts | 28 + .../[repo]/settings/access/+page.svelte | 103 + .../[handle]/[repo]/settings/access/+page.ts | 15 + .../[repo]/settings/access/new/+page.svelte | 65 + .../[repo]/settings/access/new/+page.ts | 6 + .../[repo]/settings/hooks/+layout.svelte | 45 + .../[handle]/[repo]/settings/hooks/+layout.ts | 6 + .../[repo]/settings/hooks/+page.svelte | 152 + .../[repo]/settings/hooks/[id]/+layout.ts | 3 + .../[repo]/settings/hooks/[id]/+page.svelte | 193 + .../hooks/[id]/deliveries/+page.svelte | 156 + .../[repo]/settings/hooks/new/+page.svelte | 143 + .../[repo]/settings/labels/new/+page.svelte | 127 + .../[repo]/settings/labels/new/+page.ts | 6 + .../[repo]/settings/pipelines/+page.svelte | 256 + .../pipelines/secrets/new/+page.svelte | 76 + .../[repo]/settings/rename/+page.svelte | 98 + .../[handle]/[repo]/settings/rename/+page.ts | 6 + .../[repo]/settings/sites/+page.svelte | 431 + .../[handle]/[repo]/settings/sites/+page.ts | 21 + .../[repo]/settings/sites/sites.test.ts | 87 + .../routes/[handle]/[repo]/stars/+page.svelte | 33 + web/src/routes/[handle]/[repo]/stars/+page.ts | 56 + .../routes/[handle]/[repo]/tags/+page.svelte | 37 + web/src/routes/[handle]/[repo]/tags/+page.ts | 31 + .../[handle]/[repo]/tags/[tag]/+page.svelte | 41 + .../[handle]/[repo]/tags/[tag]/+page.ts | 45 + .../tags/[tag]/download/[file]/+server.ts | 45 + .../[repo]/tree/[ref]/+page.server.ts | 11 + .../[handle]/[repo]/tree/[ref]/+page.svelte | 24 + .../tree/[ref]/[...path]/+page.server.ts | 21 + .../[repo]/tree/[ref]/[...path]/+page.svelte | 57 + web/src/routes/[handle]/handle.test.ts | 192 + web/src/routes/[handle]/info/refs/+server.ts | 5 + .../settings/OrganizationAvatar.svelte | 35 + .../settings/OrganizationHandle.svelte | 41 + .../OrganizationProfileTab.stories.svelte | 97 + .../settings/OrganizationProfileTab.svelte | 48 + .../settings/OrganizationSettingsTab.svelte | 99 + .../members/InviteMembers.stories.svelte | 79 + .../settings/members/InviteMembers.svelte | 150 + .../members/MembersTab.stories.svelte | 24 + .../settings/members/MembersTab.svelte | 262 + .../_internal/analytics/signin/+server.ts | 19 + web/src/routes/_internal/email-did/+server.ts | 21 + .../email-did/email-did.server.test.ts | 68 + web/src/routes/_internal/github/+server.ts | 35 + .../routes/_internal/github/avatar/+server.ts | 67 + .../_internal/github/callback/+server.ts | 69 + .../_internal/github/connect/+server.ts | 43 + .../routes/_internal/github/github.test.ts | 467 + .../github/install/callback/+server.ts | 40 + .../github/migrator/createTask/+server.ts | 63 + web/src/routes/_internal/indexnow/+server.ts | 68 + web/src/routes/_internal/status/+server.ts | 26 + web/src/routes/avatar/[did]/+server.ts | 23 + web/src/routes/blob/[did]/[cid]/+server.ts | 35 + web/src/routes/camo/[hex]/+server.ts | 40 + web/src/routes/company/+page.svelte | 123 + web/src/routes/company/CompanyArt.svelte | 63 + web/src/routes/compare/+page.svelte | 518 + web/src/routes/compare/MoveArt.svelte | 294 + web/src/routes/core/+server.ts | 14 + web/src/routes/core/core.test.ts | 28 + web/src/routes/dev/components/+page.svelte | 775 + web/src/routes/dev/components/+page.ts | 8 + web/src/routes/dev/components/Shell.svelte | 224 + web/src/routes/dev/components/Tile.svelte | 26 + web/src/routes/favicon.svg/+server.ts | 20 + web/src/routes/home/+page.svelte | 11 + web/src/routes/invite/+page.server.ts | 6 + web/src/routes/invite/+page.svelte | 115 + web/src/routes/login/+page.svelte | 134 + web/src/routes/notifications/+page.svelte | 90 + web/src/routes/notifications/+page.ts | 17 + .../oauth-client-metadata.json/+server.ts | 14 + web/src/routes/oauth/callback/+page.svelte | 69 + .../[handle]/welcome/+page.svelte | 86 + .../routes/organizations/new/+page.server.ts | 6 + web/src/routes/organizations/new/+page.svelte | 22 + web/src/routes/privacy/+page.svelte | 253 + web/src/routes/repo/migrate/+page.server.ts | 6 + web/src/routes/repo/migrate/+page.svelte | 111 + web/src/routes/repo/new/+page.server.ts | 6 + web/src/routes/repo/new/+page.svelte | 55 + web/src/routes/repo/new/+page.ts | 6 + web/src/routes/search/+page.svelte | 13 + web/src/routes/search/+page.ts | 27 + web/src/routes/settings/+layout.server.ts | 6 + web/src/routes/settings/+layout.svelte | 50 + web/src/routes/settings/+page.svelte | 5 + web/src/routes/settings/emails/+page.svelte | 5 + .../routes/settings/emails/new/+page.svelte | 82 + .../emails/new/AddEmailPage.stories.svelte | 90 + web/src/routes/settings/keys/+page.svelte | 5 + web/src/routes/settings/keys/new/+page.svelte | 77 + web/src/routes/settings/knots/+page.svelte | 5 + .../routes/settings/knots/new/+page.svelte | 62 + .../settings/notifications/+page.svelte | 5 + .../settings/organizations/+page.svelte | 7 + .../settings/organizations/new/+page.svelte | 13 + .../settings/profile/avatar/+page.svelte | 28 + .../settings/profile/delete/+page.svelte | 149 + .../settings/profile/handle/+page.svelte | 19 + .../handle/ChangeHandlePage.stories.svelte | 126 + .../settings/profile/password/+page.svelte | 158 + web/src/routes/settings/sites/+page.svelte | 5 + web/src/routes/settings/spindles/+page.svelte | 5 + .../routes/settings/spindles/new/+page.svelte | 62 + web/src/routes/signup/+page.svelte | 235 + web/src/routes/signup/verify/+page.svelte | 381 + .../strings/[handle]/[rkey]/+page.server.ts | 92 + .../strings/[handle]/[rkey]/+page.svelte | 29 + .../[handle]/[rkey]/StringPageContent.svelte | 154 + .../strings/[handle]/[rkey]/raw/+server.ts | 40 + .../strings/[handle]/[rkey]/skeleton.test.ts | 14 + .../[handle]/[rkey]/stringPage.test.ts | 132 + web/src/routes/strings/new/+page.server.ts | 6 + web/src/routes/strings/new/+page.svelte | 31 + web/src/routes/terms/+page.svelte | 208 + web/src/routes/timeline/+page.server.ts | 24 + web/src/routes/timeline/+page.svelte | 39 + web/src/routes/verify/email/+page.svelte | 49 + .../email/VerifyEmailPage.stories.svelte | 90 + web/src/routes/welcome/+page.svelte | 487 + web/src/routes/welcome/+page.ts | 59 + web/src/theme.css | 1071 + web/static/.gitignore | 7 + .../d76fae270607bfc761b35c14f3182c9e.txt | 1 + web/static/llms.txt | 50 + web/static/og-image.png | Bin 0 -> 79633 bytes web/static/robots.txt | 33 + web/static/tangled_og.png | Bin 0 -> 79633 bytes web/tsconfig.json | 9 + web/vite.config.ts | 176 + web/vitest.shims.d.ts | 1 + web/wrangler.dev.jsonc | 38 + workflow/compile.go | 174 + workflow/compile_test.go | 374 + workflow/def.go | 650 + workflow/def_test.go | 949 + workflow/spec_types.go | 78 + xrpc/errors/errors.go | 135 + xrpc/serviceauth/lookup_failure_test.go | 53 + xrpc/serviceauth/service_auth.go | 209 + xrpc/serviceauth/service_auth_resolve_test.go | 87 + xrpc/serviceauth/service_auth_test.go | 161 + xrpc/serviceauth/signer.go | 65 + xrpc/serviceauth/signer_test.go | 154 + xrpc/xrpcclient/xrpc.go | 51 + xrpc/xrpcclient/xrpc_test.go | 43 + 4462 files changed, 813996 insertions(+) create mode 100644 .air/deliberi.toml create mode 100644 .air/spindle.toml create mode 100644 .cargo/config.toml create mode 100644 .config/nextest.toml create mode 100644 .dockerignore create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 .prettierrc.json create mode 100755 .tangled/hooks/commit-msg create mode 100755 .tangled/hooks/commit-msg-lint.sh create mode 100755 .tangled/hooks/install.sh create mode 100644 .tangled/workflows/blog.yml create mode 100644 .tangled/workflows/cargo-nix.yml create mode 100644 .tangled/workflows/commit-lint.yml create mode 100644 .tangled/workflows/deploy-blog.yml create mode 100644 .tangled/workflows/deploy-docs.yml create mode 100644 .tangled/workflows/deploy-gcp-dev.yml create mode 100644 .tangled/workflows/fmt.yml create mode 100644 .tangled/workflows/preview-web-cleanup.yml create mode 100644 .tangled/workflows/preview-web.yml create mode 100644 .tangled/workflows/rust-checks.yml create mode 100644 .tangled/workflows/rust-test-bobbin.yml create mode 100644 .tangled/workflows/rust-test-knot.yml create mode 100644 .tangled/workflows/rust-test-knotmirror.yml create mode 100644 .tangled/workflows/test.yml create mode 100644 .tangled/workflows/web.yml create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 _typos.toml create mode 100644 api/org_tangled/actordefs.go create mode 100644 api/org_tangled/actorgetProfile.go create mode 100644 api/org_tangled/actorgetProfileActivity.go create mode 100644 api/org_tangled/actorgetProfiles.go create mode 100644 api/org_tangled/actorgetTrending.go create mode 100644 api/org_tangled/actorprofile.go create mode 100644 api/org_tangled/actorsearchActorsTypeahead.go create mode 100644 api/org_tangled/bobbinawaitRecord.go create mode 100644 api/org_tangled/cbor_gen.go create mode 100644 api/org_tangled/ciapprovePipeline.go create mode 100644 api/org_tangled/cicancelPipeline.go create mode 100644 api/org_tangled/cidefs.go create mode 100644 api/org_tangled/cidefs_ext.go create mode 100644 api/org_tangled/cidefs_test.go create mode 100644 api/org_tangled/cigetPipeline.go create mode 100644 api/org_tangled/cigetWorkflowDefinition.go create mode 100644 api/org_tangled/cilistWorkflowDefinitions.go create mode 100644 api/org_tangled/ciqueryPipelines.go create mode 100644 api/org_tangled/ciretryPipeline.go create mode 100644 api/org_tangled/cisubscribePipelineLogs.go create mode 100644 api/org_tangled/cisubscribePipelineLogs_ext.go create mode 100644 api/org_tangled/cisubscribePipelineLogs_ext_test.go create mode 100644 api/org_tangled/citriggermanual.go create mode 100644 api/org_tangled/citriggerschedule.go create mode 100644 api/org_tangled/embedcommit.go create mode 100644 api/org_tangled/embedunspeccedfile.go create mode 100644 api/org_tangled/eventnotifyEvent.go create mode 100644 api/org_tangled/eventpullRequest.go create mode 100644 api/org_tangled/eventpush.go create mode 100644 api/org_tangled/feedcomment.go create mode 100644 api/org_tangled/feedcountComments.go create mode 100644 api/org_tangled/feedcountCommentsBy.go create mode 100644 api/org_tangled/feedcountReactions.go create mode 100644 api/org_tangled/feedcountReactionsBy.go create mode 100644 api/org_tangled/feedcountStars.go create mode 100644 api/org_tangled/feedcountStarsBy.go create mode 100644 api/org_tangled/feeddefs.go create mode 100644 api/org_tangled/feedgetStar.go create mode 100644 api/org_tangled/feedgetTimeline.go create mode 100644 api/org_tangled/feedlistComments.go create mode 100644 api/org_tangled/feedlistCommentsBy.go create mode 100644 api/org_tangled/feedlistReactions.go create mode 100644 api/org_tangled/feedlistReactionsBy.go create mode 100644 api/org_tangled/feedlistStars.go create mode 100644 api/org_tangled/feedlistStarsBy.go create mode 100644 api/org_tangled/feedreaction.go create mode 100644 api/org_tangled/feedstar.go create mode 100644 api/org_tangled/feedsubscription.go create mode 100644 api/org_tangled/gitdefs.go create mode 100644 api/org_tangled/gitdeleteBranch.go create mode 100644 api/org_tangled/gitgetArchive.go create mode 100644 api/org_tangled/gitgetBlame.go create mode 100644 api/org_tangled/gitgetBlob.go create mode 100644 api/org_tangled/gitgetBranch.go create mode 100644 api/org_tangled/gitgetDiff.go create mode 100644 api/org_tangled/gitgetEntry.go create mode 100644 api/org_tangled/gitgetTag.go create mode 100644 api/org_tangled/gitgetTree.go create mode 100644 api/org_tangled/gitlistBranches.go create mode 100644 api/org_tangled/gitlistCommits.go create mode 100644 api/org_tangled/gitlistLanguages.go create mode 100644 api/org_tangled/gitlistRefs.go create mode 100644 api/org_tangled/gitlistTags.go create mode 100644 api/org_tangled/gitoid.go create mode 100644 api/org_tangled/gitpushEvent.go create mode 100644 api/org_tangled/gitreceivePack.go create mode 100644 api/org_tangled/gitrefs.go create mode 100644 api/org_tangled/gitsetDefaultBranch.go create mode 100644 api/org_tangled/graphattestation.go create mode 100644 api/org_tangled/graphcollaborator.go create mode 100644 api/org_tangled/graphcountFollows.go create mode 100644 api/org_tangled/graphcountFollowsBy.go create mode 100644 api/org_tangled/graphfollow.go create mode 100644 api/org_tangled/graphgetFollow.go create mode 100644 api/org_tangled/graphlistFollows.go create mode 100644 api/org_tangled/graphlistFollowsBy.go create mode 100644 api/org_tangled/graphlistNetworkVouches.go create mode 100644 api/org_tangled/graphvouch.go create mode 100644 api/org_tangled/keysshKey.go create mode 100644 api/org_tangled/keyunspeccedgpgKey.go create mode 100644 api/org_tangled/knotblock.go create mode 100644 api/org_tangled/knotdescribeKnot.go create mode 100644 api/org_tangled/knotmember.go create mode 100644 api/org_tangled/labelcountDefinitions.go create mode 100644 api/org_tangled/labeldefinition.go create mode 100644 api/org_tangled/labeldefs.go create mode 100644 api/org_tangled/labellistDefinitions.go create mode 100644 api/org_tangled/markupembedimage.go create mode 100644 api/org_tangled/markupembedvideo.go create mode 100644 api/org_tangled/markupmarkdown.go create mode 100644 api/org_tangled/markupplaintext.go create mode 100644 api/org_tangled/publicKeycountKeys.go create mode 100644 api/org_tangled/publicKeygetPublicKey.go create mode 100644 api/org_tangled/publicKeylistKeys.go create mode 100644 api/org_tangled/queryenrichResponse.go create mode 100644 api/org_tangled/repoartifact.go create mode 100644 api/org_tangled/repocountArtifacts.go create mode 100644 api/org_tangled/repocountArtifactsBy.go create mode 100644 api/org_tangled/repocountCollaborators.go create mode 100644 api/org_tangled/repocountCollaboratorsBy.go create mode 100644 api/org_tangled/repocountForks.go create mode 100644 api/org_tangled/repocountIssues.go create mode 100644 api/org_tangled/repocountPulls.go create mode 100644 api/org_tangled/repocountRepos.go create mode 100644 api/org_tangled/repocreateRepo.go create mode 100644 api/org_tangled/repodeclaration.go create mode 100644 api/org_tangled/repodefs.go create mode 100644 api/org_tangled/repodeleteRepo.go create mode 100644 api/org_tangled/repoforkSync.go create mode 100644 api/org_tangled/repogetImportStatus.go create mode 100644 api/org_tangled/repogetRepo.go create mode 100644 api/org_tangled/repogetRepoByName.go create mode 100644 api/org_tangled/repogetRepoByRepoDid.go create mode 100644 api/org_tangled/repogetRepos.go create mode 100644 api/org_tangled/repogetReposByRepoDids.go create mode 100644 api/org_tangled/repolistArtifacts.go create mode 100644 api/org_tangled/repolistArtifactsBy.go create mode 100644 api/org_tangled/repolistCollaboratorInvitesBy.go create mode 100644 api/org_tangled/repolistCollaborators.go create mode 100644 api/org_tangled/repolistCollaboratorsBy.go create mode 100644 api/org_tangled/repolistForks.go create mode 100644 api/org_tangled/repolistIssues.go create mode 100644 api/org_tangled/repolistIssuesBy.go create mode 100644 api/org_tangled/repolistPulls.go create mode 100644 api/org_tangled/repolistPullsBy.go create mode 100644 api/org_tangled/repolistRecordVersions.go create mode 100644 api/org_tangled/repolistRepos.go create mode 100644 api/org_tangled/repomanifest.go create mode 100644 api/org_tangled/repomintToken.go create mode 100644 api/org_tangled/reporeserveKey.go create mode 100644 api/org_tangled/reporeviewunspeccedround.go create mode 100644 api/org_tangled/reviewpatch.go create mode 100644 api/org_tangled/reviewsubmitPatch.go create mode 100644 api/org_tangled/reviewunspecceddefs.go create mode 100644 api/org_tangled/searchquery.go create mode 100644 api/org_tangled/secretaddSecret.go create mode 100644 api/org_tangled/secretlistSecrets.go create mode 100644 api/org_tangled/secretremoveSecret.go create mode 100644 api/org_tangled/snippetstring.go create mode 100644 api/org_tangled/spindledescribeSpindle.go create mode 100644 api/org_tangled/stringcountStrings.go create mode 100644 api/org_tangled/stringgetString.go create mode 100644 api/org_tangled/stringlistStrings.go create mode 100644 api/org_tangled/tempaccountaddEmail.go create mode 100644 api/org_tangled/tempaccountbeginSignup.go create mode 100644 api/org_tangled/tempaccountcompleteSignup.go create mode 100644 api/org_tangled/tempaccountdeleteEmail.go create mode 100644 api/org_tangled/tempaccountdismissNewsletter.go create mode 100644 api/org_tangled/tempaccountgetNewsletterPreference.go create mode 100644 api/org_tangled/tempaccountgetSignup.go create mode 100644 api/org_tangled/tempaccountlistEmails.go create mode 100644 api/org_tangled/tempaccountresendSignup.go create mode 100644 api/org_tangled/tempaccountsetPrimaryEmail.go create mode 100644 api/org_tangled/tempaccountsubscribeNewsletter.go create mode 100644 api/org_tangled/tempaccountverifyEmail.go create mode 100644 api/org_tangled/tempactordefs.go create mode 100644 api/org_tangled/tempfocusbeginSession.go create mode 100644 api/org_tangled/tempfocusendSession.go create mode 100644 api/org_tangled/tempfocusnextItem.go create mode 100644 api/org_tangled/tempgitdeleteBranch.go create mode 100644 api/org_tangled/tempgitformatPatch.go create mode 100644 api/org_tangled/tempgitgetArchive.go create mode 100644 api/org_tangled/tempgitgetBlob.go create mode 100644 api/org_tangled/tempgitgetBranch.go create mode 100644 api/org_tangled/tempgitgetEntry.go create mode 100644 api/org_tangled/tempgitgetMergeBase.go create mode 100644 api/org_tangled/tempgitgetRef.go create mode 100644 api/org_tangled/tempgitgetTag.go create mode 100644 api/org_tangled/tempgitgetTree.go create mode 100644 api/org_tangled/tempgitlistBranches.go create mode 100644 api/org_tangled/tempgitlistCommits.go create mode 100644 api/org_tangled/tempgitlistCommitsByRef.go create mode 100644 api/org_tangled/tempgitlistTags.go create mode 100644 api/org_tangled/tempgitmergeCommit.go create mode 100644 api/org_tangled/tempgitsyncFork.go create mode 100644 api/org_tangled/tempmigratorcreateTask.go create mode 100644 api/org_tangled/tempmigratordefs.go create mode 100644 api/org_tangled/tempmigratordescribeSource.go create mode 100644 api/org_tangled/tempmigratorgetTask.go create mode 100644 api/org_tangled/tempmigratorlistTasks.go create mode 100644 api/org_tangled/tempmigratorretryJob.go create mode 100644 api/org_tangled/tempnewslettersignup.go create mode 100644 api/org_tangled/tempnotificationdeleteNotification.go create mode 100644 api/org_tangled/tempnotificationgetPreferences.go create mode 100644 api/org_tangled/tempnotificationgetUnreadCount.go create mode 100644 api/org_tangled/tempnotificationlistNotifications.go create mode 100644 api/org_tangled/tempnotificationlistRecipients.go create mode 100644 api/org_tangled/tempnotificationmarkAllRead.go create mode 100644 api/org_tangled/tempnotificationmarkEntityRead.go create mode 100644 api/org_tangled/tempnotificationupdatePreferences.go create mode 100644 api/org_tangled/tempnotificationupdateSeen.go create mode 100644 api/org_tangled/temprepodeploySite.go create mode 100644 api/org_tangled/temprepodisableSite.go create mode 100644 api/org_tangled/temprepogetDeployHistory.go create mode 100644 api/org_tangled/temprepogetRepoByName.go create mode 100644 api/org_tangled/temprepogetSiteConfig.go create mode 100644 api/org_tangled/temprepolistRecordOrigins.go create mode 100644 api/org_tangled/temprepoupdateSiteConfig.go create mode 100644 api/org_tangled/tempreviewlistPatches.go create mode 100644 api/org_tangled/tempsearchsearchCode.go create mode 100644 api/org_tangled/tempserverackHandover.go create mode 100644 api/org_tangled/tempservercancelHandover.go create mode 100644 api/org_tangled/tempservercreateInviteCode.go create mode 100644 api/org_tangled/tempserverhandoverRepo.go create mode 100644 api/org_tangled/tempserverredeemInviteCode.go create mode 100644 api/org_tangled/tempserverseedRepo.go create mode 100644 api/org_tangled/tempserveruploadBlob.go create mode 100644 api/org_tangled/tempsiteclaimDomain.go create mode 100644 api/org_tangled/tempsitegetDomainClaim.go create mode 100644 api/org_tangled/tempsitereleaseDomain.go create mode 100644 api/org_tangled/tempspindleallowListadd.go create mode 100644 api/org_tangled/tempspindleallowListlist.go create mode 100644 api/org_tangled/tempspindleallowListremove.go create mode 100644 api/org_tangled/tempspindlemoderationban.go create mode 100644 api/org_tangled/tempspindlemoderationdefs.go create mode 100644 api/org_tangled/tempspindlemoderationgetBan.go create mode 100644 api/org_tangled/tempspindlemoderationlistBans.go create mode 100644 api/org_tangled/tempspindlemoderationunban.go create mode 100644 api/org_tangled/tempspindlequotadefs.go create mode 100644 api/org_tangled/tempspindlequotaget.go create mode 100644 api/org_tangled/tempspindlequotalist.go create mode 100644 api/org_tangled/tempspindlequotareleaseNars.go create mode 100644 api/org_tangled/tempspindlequotaset.go create mode 100644 api/org_tangled/tempspindlequotaunset.go create mode 100644 api/org_tangled/tempspindlequotausage.go create mode 100644 api/org_tangled/trackdefs.go create mode 100644 api/org_tangled/trackgetTicketView.go create mode 100644 api/org_tangled/trackticket.go create mode 100644 api/org_tangled/trackunspeccedmilestone.go create mode 100644 api/org_tangled/webhookcreateWebhook.go create mode 100644 api/org_tangled/webhookdefs.go create mode 100644 api/org_tangled/webhookdeleteWebhook.go create mode 100644 api/org_tangled/webhooklistWebhookDeliveries.go create mode 100644 api/org_tangled/webhooklistWebhooks.go create mode 100644 api/org_tangled/webhookretryWebhookDelivery.go create mode 100644 api/org_tangled/webhookupdateWebhook.go create mode 100644 api/tangled/gitdefs.go create mode 100644 api/tangled/gitmergeCommit.go create mode 100644 api/tangled/gittemp2getBlame.go create mode 100644 api/tangled/gittemp2getCommitStats.go create mode 100644 api/tangled/gittemp2getDiff.go create mode 100644 api/tangled/gittemp2getInterdiff.go create mode 100644 api/tangled/gittemp2mergeCheck.go create mode 100644 api/tangled/gittemplistIndexedLanguages.go create mode 100644 api/tangled/gittemplistLanguages.go create mode 100644 api/tangled/identityresolveCommitters.go create mode 100644 api/tangled/repolistCobVersions.go create mode 100644 avatar/.gitignore create mode 100644 avatar/package.json create mode 100644 avatar/playground.html create mode 100644 avatar/pnpm-lock.yaml create mode 100644 avatar/pnpm-workspace.yaml create mode 100644 avatar/readme.md create mode 100644 avatar/src/gradient.gold.json create mode 100644 avatar/src/gradient.js create mode 100644 avatar/src/gradient.test.js create mode 100644 avatar/src/index.js create mode 100644 avatar/src/index.test.js create mode 100644 avatar/src/size.js create mode 100644 avatar/wrangler.dev.jsonc create mode 100644 avatar/wrangler.jsonc create mode 100644 blog/.gitignore create mode 100644 blog/astro.config.mjs create mode 100644 blog/blog.go create mode 100644 blog/lychee.toml create mode 100644 blog/package.json create mode 100644 blog/pnpm-lock.yaml create mode 100644 blog/pnpm-workspace.yaml create mode 100644 blog/posts/6-months.md create mode 100644 blog/posts/bobbin.md create mode 100644 blog/posts/ci.md create mode 100644 blog/posts/docs.md create mode 100644 blog/posts/federation.md create mode 100644 blog/posts/forge.mdx create mode 100644 blog/posts/intro.md create mode 100644 blog/posts/newsletter-01.md create mode 100644 blog/posts/newsletter-02.md create mode 100644 blog/posts/newsletter-03.md create mode 100644 blog/posts/pty.md create mode 100644 blog/posts/pulls.md create mode 100644 blog/posts/seed.md create mode 100644 blog/posts/selfhost-ci.md create mode 100644 blog/posts/spindle-microvm.md create mode 100644 blog/posts/stacking.md create mode 100644 blog/posts/vouching.md create mode 100644 blog/public/avatars/anirudh.fi.jpg create mode 100644 blog/public/avatars/oppi.li.jpg create mode 100644 blog/public/avatars/oyster.cafe.jpg create mode 100644 blog/public/avatars/ptr.pet.jpg create mode 100644 blog/public/favicon.svg create mode 100755 blog/scripts/check-links.sh create mode 100644 blog/src/components/FeedButton.svelte create mode 100644 blog/src/components/Footer.astro create mode 100644 blog/src/components/ForgeTimeline.svelte create mode 100644 blog/src/components/MarkupLightbox.svelte create mode 100644 blog/src/components/Ripple.astro create mode 100644 blog/src/components/SignupBanner.svelte create mode 100644 blog/src/components/embeds/OrganizationsEmbed.svelte create mode 100644 blog/src/components/embeds/PullStackEmbed.svelte create mode 100644 blog/src/components/embeds/Reveal.svelte create mode 100644 blog/src/components/embeds/TicketsEmbed.svelte create mode 100644 blog/src/components/embeds/TimelineEmbed.svelte create mode 100644 blog/src/content.config.ts create mode 100644 blog/src/dashes.mjs create mode 100644 blog/src/layouts/Base.astro create mode 100644 blog/src/lib/posts.ts create mode 100644 blog/src/pages/.well-known/site.standard.publication.ts create mode 100644 blog/src/pages/404.astro create mode 100644 blog/src/pages/[slug].astro create mode 100644 blog/src/pages/[slug].md.ts create mode 100644 blog/src/pages/feed.json.ts create mode 100644 blog/src/pages/feed.xml.ts create mode 100644 blog/src/pages/index.astro create mode 100644 blog/src/pages/llms-full.txt.ts create mode 100644 blog/src/pages/llms.txt.ts create mode 100644 blog/src/stubs/app-env.ts create mode 100644 blog/src/stubs/app-paths.ts create mode 100644 blog/src/styles/global.css create mode 100644 blog/src/tangled.theme.mjs create mode 100644 blog/standard.go create mode 100644 blog/standard_test.go create mode 100644 blog/svelte.config.js create mode 100644 blog/tsconfig.json create mode 100644 bobbin/containerfiles/bobbin.Containerfile create mode 100644 bobbin/containerfiles/hydrant.Containerfile create mode 100644 bobbin/containerfiles/slingshot.Containerfile create mode 100644 bobbin/crates/bobbin-sim/Cargo.toml create mode 100644 bobbin/crates/bobbin-sim/src/determinism.rs create mode 100644 bobbin/crates/bobbin-sim/src/lib.rs create mode 100644 bobbin/crates/bobbin-sim/src/main.rs create mode 100644 bobbin/crates/bobbin-sim/src/report.rs create mode 100644 bobbin/crates/bobbin-sim/src/runtime.rs create mode 100644 bobbin/crates/bobbin-sim/src/trace_capture.rs create mode 100644 bobbin/crates/bobbin-sim/src/workload.rs create mode 100644 bobbin/crates/bobbin-sim/src/workloads/cancel_mid_hydration.rs create mode 100644 bobbin/crates/bobbin-sim/src/workloads/cold_start_under_live_load.rs create mode 100644 bobbin/crates/bobbin-sim/src/workloads/concurrent_reads_during_replay.rs create mode 100644 bobbin/crates/bobbin-sim/src/workloads/frame_burst.rs create mode 100644 bobbin/crates/bobbin-sim/src/workloads/hydrant_disconnect_barrage.rs create mode 100644 bobbin/crates/bobbin-sim/src/workloads/mod.rs create mode 100644 bobbin/crates/bobbin-sim/src/workloads/slingshot_flap.rs create mode 100644 bobbin/crates/bobbin-sim/src/workloads/util.rs create mode 100644 bobbin/crates/bobbin-sim/tests/determinism_leak.rs create mode 100644 bobbin/crates/bobbin-sim/tests/reconnect_replay.rs create mode 100644 bobbin/crates/bobbin-sim/tests/warming_buffer.rs create mode 100644 bobbin/crates/bobbin-sim/tests/warming_shadow.rs create mode 100644 bobbin/crates/bobbin/Cargo.toml create mode 100644 bobbin/crates/bobbin/src/config.rs create mode 100644 bobbin/crates/bobbin/src/main.rs create mode 100644 bobbin/crates/bobbin/src/mem/cgroup.rs create mode 100644 bobbin/crates/bobbin/src/mem/mod.rs create mode 100644 bobbin/crates/bobbin/src/mem/report.rs create mode 100644 bobbin/crates/bobbin/src/mem/sizing.rs create mode 100644 bobbin/crates/bobbin/src/mem/watcher.rs create mode 100644 bobbin/crates/bobbin/src/metrics.rs create mode 100644 bobbin/crates/bobbin/src/metrics/process.rs create mode 100644 bobbin/crates/bobbin/src/otlp.rs create mode 100644 bobbin/crates/codesearch/Cargo.toml create mode 100644 bobbin/crates/codesearch/src/lib.rs create mode 100644 bobbin/crates/edge-index/Cargo.toml create mode 100644 bobbin/crates/edge-index/benches/distinct_authors.rs create mode 100644 bobbin/crates/edge-index/benches/profile_activity.rs create mode 100644 bobbin/crates/edge-index/src/avatars.rs create mode 100644 bobbin/crates/edge-index/src/claims.rs create mode 100644 bobbin/crates/edge-index/src/coverage.rs create mode 100644 bobbin/crates/edge-index/src/invites.rs create mode 100644 bobbin/crates/edge-index/src/labels.rs create mode 100644 bobbin/crates/edge-index/src/lib.rs create mode 100644 bobbin/crates/edge-index/src/reactions.rs create mode 100644 bobbin/crates/edge-index/src/settle.rs create mode 100644 bobbin/crates/edge-index/src/sorted_blocks.rs create mode 100644 bobbin/crates/edge-index/src/state_index.rs create mode 100644 bobbin/crates/edge-index/src/state_view.rs create mode 100644 bobbin/crates/edge-index/src/stream.rs create mode 100644 bobbin/crates/edge-index/tests/bucket_scaling.rs create mode 100644 bobbin/crates/ingest/Cargo.toml create mode 100644 bobbin/crates/ingest/benches/json_decode.rs create mode 100644 bobbin/crates/ingest/examples/chunk_parity.rs create mode 100644 bobbin/crates/ingest/examples/smoke.rs create mode 100644 bobbin/crates/ingest/src/analytics.rs create mode 100644 bobbin/crates/ingest/src/authorship.rs create mode 100644 bobbin/crates/ingest/src/chunks.rs create mode 100644 bobbin/crates/ingest/src/frame.rs create mode 100644 bobbin/crates/ingest/src/lib.rs create mode 100644 bobbin/crates/ingest/src/metrics.rs create mode 100644 bobbin/crates/ingest/src/moderation/frame.rs create mode 100644 bobbin/crates/ingest/src/moderation/mod.rs create mode 100644 bobbin/crates/ingest/src/resolver.rs create mode 100644 bobbin/crates/ingest/src/shadow.rs create mode 100644 bobbin/crates/ingest/src/wait.rs create mode 100644 bobbin/crates/ingest/src/warming.rs create mode 100644 bobbin/crates/knot-proxy/Cargo.toml create mode 100644 bobbin/crates/knot-proxy/src/breaker.rs create mode 100644 bobbin/crates/knot-proxy/src/dns.rs create mode 100644 bobbin/crates/knot-proxy/src/host.rs create mode 100644 bobbin/crates/knot-proxy/src/lib.rs create mode 100644 bobbin/crates/knot-proxy/src/mirror.rs create mode 100644 bobbin/crates/knot-proxy/src/origins.rs create mode 100644 bobbin/crates/languages/Cargo.toml create mode 100644 bobbin/crates/languages/src/fake.rs create mode 100644 bobbin/crates/languages/src/heads.rs create mode 100644 bobbin/crates/languages/src/index.rs create mode 100644 bobbin/crates/languages/src/lib.rs create mode 100644 bobbin/crates/languages/src/live.rs create mode 100644 bobbin/crates/languages/src/mirror.rs create mode 100644 bobbin/crates/languages/src/source.rs create mode 100644 bobbin/crates/languages/src/sync.rs create mode 100644 bobbin/crates/query/Cargo.toml create mode 100644 bobbin/crates/query/src/ast.rs create mode 100644 bobbin/crates/query/src/error.rs create mode 100644 bobbin/crates/query/src/lex.rs create mode 100644 bobbin/crates/query/src/lib.rs create mode 100644 bobbin/crates/query/src/lower.rs create mode 100644 bobbin/crates/record-lru/Cargo.toml create mode 100644 bobbin/crates/record-lru/src/lib.rs create mode 100644 bobbin/crates/record-lru/src/record.dict create mode 100644 bobbin/crates/resolver/Cargo.toml create mode 100644 bobbin/crates/resolver/src/decode.rs create mode 100644 bobbin/crates/resolver/src/hydrant.rs create mode 100644 bobbin/crates/resolver/src/identity.rs create mode 100644 bobbin/crates/resolver/src/lib.rs create mode 100644 bobbin/crates/resolver/src/normalize.rs create mode 100644 bobbin/crates/runtime/Cargo.toml create mode 100644 bobbin/crates/runtime/src/clock.rs create mode 100644 bobbin/crates/runtime/src/entropy.rs create mode 100644 bobbin/crates/runtime/src/hasher.rs create mode 100644 bobbin/crates/runtime/src/lib.rs create mode 100644 bobbin/crates/runtime/src/mem.rs create mode 100644 bobbin/crates/runtime/src/mem_network.rs create mode 100644 bobbin/crates/runtime/src/network.rs create mode 100644 bobbin/crates/search/Cargo.toml create mode 100644 bobbin/crates/search/src/actor.rs create mode 100644 bobbin/crates/search/src/cursor.rs create mode 100644 bobbin/crates/search/src/lib.rs create mode 100644 bobbin/crates/slingshot-client/Cargo.toml create mode 100644 bobbin/crates/slingshot-client/src/lib.rs create mode 100644 bobbin/crates/types/Cargo.toml create mode 100644 bobbin/crates/types/src/edges.rs create mode 100644 bobbin/crates/types/src/identity.rs create mode 100644 bobbin/crates/types/src/ids.rs create mode 100644 bobbin/crates/types/src/lib.rs create mode 100644 bobbin/crates/types/src/record.rs create mode 100644 bobbin/crates/types/src/search.rs create mode 100644 bobbin/crates/xrpc/Cargo.toml create mode 100644 bobbin/crates/xrpc/src/actor.rs create mode 100644 bobbin/crates/xrpc/src/authors.rs create mode 100644 bobbin/crates/xrpc/src/backpressure.rs create mode 100644 bobbin/crates/xrpc/src/change_ids.rs create mode 100644 bobbin/crates/xrpc/src/client_address.rs create mode 100644 bobbin/crates/xrpc/src/codesearch.rs create mode 100644 bobbin/crates/xrpc/src/commit_stats.rs create mode 100644 bobbin/crates/xrpc/src/enrich.rs create mode 100644 bobbin/crates/xrpc/src/feed.rs create mode 100644 bobbin/crates/xrpc/src/filter.rs create mode 100644 bobbin/crates/xrpc/src/labels.rs create mode 100644 bobbin/crates/xrpc/src/lib.rs create mode 100644 bobbin/crates/xrpc/src/pocket.rs create mode 100644 bobbin/crates/xrpc/src/profile_activity.rs create mode 100644 bobbin/crates/xrpc/src/recordpath.rs create mode 100644 bobbin/crates/xrpc/src/repo.rs create mode 100644 bobbin/crates/xrpc/src/search.rs create mode 100644 bobbin/crates/xrpc/src/view.rs create mode 100644 bobbin/crates/xrpc/tests/aggregation.rs create mode 100644 bobbin/crates/xrpc/tests/avatar.rs create mode 100644 bobbin/crates/xrpc/tests/await_record.rs create mode 100644 bobbin/crates/xrpc/tests/await_record_e2e.rs create mode 100644 bobbin/crates/xrpc/tests/bulk.rs create mode 100644 bobbin/crates/xrpc/tests/code_search.rs create mode 100644 bobbin/crates/xrpc/tests/cold_start.rs create mode 100644 bobbin/crates/xrpc/tests/coverage.rs create mode 100644 bobbin/crates/xrpc/tests/enrich.rs create mode 100644 bobbin/crates/xrpc/tests/extended.rs create mode 100644 bobbin/crates/xrpc/tests/mirror_proxy.rs create mode 100644 bobbin/crates/xrpc/tests/pocket_proxy.rs create mode 100644 bobbin/crates/xrpc/tests/search.rs create mode 100644 bobbin/crates/xrpc/tests/trending.rs create mode 100644 bobbin/default.nix create mode 100644 bobbin/example.toml create mode 100644 bobbin/worker/package.json create mode 100644 bobbin/worker/pnpm-lock.yaml create mode 100644 bobbin/worker/pnpm-workspace.yaml create mode 100644 bobbin/worker/src/index.ts create mode 100644 bobbin/worker/tsconfig.json create mode 100644 bobbin/worker/wrangler.dev.jsonc create mode 100644 bobbin/worker/wrangler.jsonc create mode 100644 buf.gen.yaml create mode 100644 buf.lock create mode 100644 buf.yaml create mode 100644 camo/.gitignore create mode 100644 camo/package.json create mode 100644 camo/pnpm-lock.yaml create mode 100644 camo/pnpm-workspace.yaml create mode 100644 camo/readme.md create mode 100644 camo/src/index.js create mode 100644 camo/src/index.test.js create mode 100644 camo/src/mimetypes.json create mode 100644 camo/wrangler.dev.jsonc create mode 100644 camo/wrangler.jsonc create mode 100644 cilogs/cihelpers.go create mode 100644 cilogs/cilogs_test.go create mode 100644 cilogs/error.go create mode 100644 cilogs/logstream.go create mode 100644 cilogs/server.go create mode 100644 cilogs/session.go create mode 100644 cilogs/spindles.go create mode 100644 cilogs/tui.go create mode 100644 cmd/blog/main.go create mode 100644 cmd/cborgen/cborgen.go create mode 100644 cmd/cilogs/main.go create mode 100644 cmd/claimer/emit/emit.go create mode 100644 cmd/claimer/main.go create mode 100644 cmd/claimer/main_test.go create mode 100644 cmd/deliberi/main.go create mode 100755 cmd/delicurl.sh create mode 100644 cmd/dolly/main.go create mode 100644 cmd/dolly/templates/logo.html create mode 100644 cmd/dolly/templates/logotype.html create mode 100644 cmd/lexpostgen/main.go create mode 100644 cmd/migrator/main.go create mode 100644 cmd/prefill-zoekt/main.go create mode 100644 cmd/sitesd/main.go create mode 100644 cmd/spindle-microvm-run/main.go create mode 100644 cmd/spindle-microvm-run/main_linux.go create mode 100644 cmd/spindle-microvm-run/unsupported.go create mode 100644 cmd/spindle/ban_cli_test.go create mode 100644 cmd/spindle/main.go create mode 100644 cmd/spindle/main_test.go create mode 100644 cmd/verifysig/main.go create mode 100644 cmd/zoekt-tngl-indexserver/index.go create mode 100644 cmd/zoekt-tngl-indexserver/main.go create mode 100644 cmd/zoekt-tngl-indexserver/metrics.go create mode 100644 cmd/zoekt-tngl-indexserver/queue.go create mode 100644 cmd/zoekt-tngl-indexserver/server.go create mode 100644 cmd/zoekt-tngl-indexserver/server_test.go create mode 100644 crates/gix-chunk/Cargo.toml create mode 100644 crates/gix-chunk/LICENSE-APACHE create mode 100644 crates/gix-chunk/LICENSE-MIT create mode 100644 crates/gix-chunk/src/file/decode.rs create mode 100644 crates/gix-chunk/src/file/index.rs create mode 100644 crates/gix-chunk/src/file/mod.rs create mode 100644 crates/gix-chunk/src/file/write.rs create mode 100644 crates/gix-chunk/src/lib.rs create mode 100644 crates/gix-commitgraph/Cargo.toml create mode 100644 crates/gix-commitgraph/LICENSE-APACHE create mode 100644 crates/gix-commitgraph/LICENSE-MIT create mode 100644 crates/gix-commitgraph/src/access.rs create mode 100644 crates/gix-commitgraph/src/bloom.rs create mode 100644 crates/gix-commitgraph/src/file/access.rs create mode 100644 crates/gix-commitgraph/src/file/commit.rs create mode 100644 crates/gix-commitgraph/src/file/init.rs create mode 100644 crates/gix-commitgraph/src/file/mod.rs create mode 100644 crates/gix-commitgraph/src/file/verify.rs create mode 100644 crates/gix-commitgraph/src/init.rs create mode 100644 crates/gix-commitgraph/src/lib.rs create mode 100644 crates/gix-commitgraph/src/verify.rs create mode 100644 crates/gix-commitgraph/src/write/info_dir.rs create mode 100644 crates/gix-commitgraph/src/write/mod.rs create mode 100644 crates/gix-rebase/Cargo.toml create mode 100644 crates/gix-rebase/README.md create mode 100644 crates/gix-rebase/src/lib.rs create mode 100644 crates/gix-rebase/src/replay/function.rs create mode 100644 crates/gix-rebase/src/replay/mod.rs create mode 100755 crates/gix-rebase/tests/fixtures/rebase-baseline.sh create mode 100644 crates/gix-rebase/tests/rebase/baseline.rs create mode 100644 crates/gix-rebase/tests/rebase/main.rs create mode 100644 crates/gix-receive-pack/Cargo.toml create mode 100644 crates/gix-receive-pack/src/advertisement.rs create mode 100644 crates/gix-receive-pack/src/command.rs create mode 100644 crates/gix-receive-pack/src/error.rs create mode 100644 crates/gix-receive-pack/src/lib.rs create mode 100644 crates/gix-receive-pack/src/report.rs create mode 100644 crates/gix-receive-pack/src/session.rs create mode 100644 crates/gix-receive-pack/src/types.rs create mode 100644 crates/knot-capability/Cargo.toml create mode 100644 crates/knot-capability/src/lib.rs create mode 100644 crates/lexicons/Cargo.toml create mode 100644 crates/lexicons/build.rs create mode 100644 crates/lexicons/src/lib.rs create mode 100644 crates/tangled-axum/Cargo.toml create mode 100644 crates/tangled-axum/src/atproto.rs create mode 100644 crates/tangled-axum/src/lib.rs create mode 100644 crates/tranquil-gate/Cargo.toml create mode 100644 crates/tranquil-gate/config.example.toml create mode 100644 crates/tranquil-gate/src/config.rs create mode 100644 crates/tranquil-gate/src/gate.rs create mode 100644 crates/tranquil-gate/src/lib.rs create mode 100644 crates/tranquil-gate/src/main.rs create mode 100644 crates/tranquil-gate/src/telemetry.rs create mode 100644 crates/tranquil-gate/src/tests.rs create mode 100644 crates/tranquil-gate/tests/config.rs create mode 100644 crates/trusted-proxies/Cargo.toml create mode 100644 crates/trusted-proxies/src/lib.rs create mode 100644 crypto/verify.go create mode 100644 crypto/verify_test.go create mode 100644 default.nix create mode 100644 deliberi/account_ingest_test.go create mode 100644 deliberi/analytics/analytics.go create mode 100644 deliberi/bobbin.go create mode 100644 deliberi/bobbin_test.go create mode 100644 deliberi/config/config.go create mode 100644 deliberi/config/config_test.go create mode 100644 deliberi/db/cache.go create mode 100644 deliberi/db/cursor.go create mode 100644 deliberi/db/db.go create mode 100644 deliberi/db/email.go create mode 100644 deliberi/db/ingest.go create mode 100644 deliberi/db/newsletter.go create mode 100644 deliberi/db/notifications.go create mode 100644 deliberi/db/notifications_test.go create mode 100644 deliberi/db/signup.go create mode 100644 deliberi/db/signup_test.go create mode 100644 deliberi/deliberi.go create mode 100644 deliberi/digest.go create mode 100644 deliberi/digest_test.go create mode 100644 deliberi/email/email.go create mode 100644 deliberi/email/email_test.go create mode 100644 deliberi/emailtemplate/template.go create mode 100644 deliberi/ingest.go create mode 100644 deliberi/ingest_test.go create mode 100644 deliberi/invites.go create mode 100644 deliberi/invites_test.go create mode 100644 deliberi/kv/kv.go create mode 100644 deliberi/kv/kv_test.go create mode 100644 deliberi/mailer/mailer.go create mode 100644 deliberi/markup/atlink.go create mode 100644 deliberi/mentions.go create mode 100644 deliberi/mentions_test.go create mode 100644 deliberi/models/email.go create mode 100644 deliberi/models/notifications.go create mode 100644 deliberi/models/notifications_test.go create mode 100644 deliberi/models/signup.go create mode 100644 deliberi/readme.md create mode 100644 deliberi/records.go create mode 100644 deliberi/xrpc/account.go create mode 100644 deliberi/xrpc/account_test.go create mode 100644 deliberi/xrpc/ingested.go create mode 100644 deliberi/xrpc/ingested_test.go create mode 100644 deliberi/xrpc/internal_committers.go create mode 100644 deliberi/xrpc/internal_committers_test.go create mode 100644 deliberi/xrpc/newsletter.go create mode 100644 deliberi/xrpc/newsletter_test.go create mode 100644 deliberi/xrpc/notifications.go create mode 100644 deliberi/xrpc/pds_email.go create mode 100644 deliberi/xrpc/pds_email_test.go create mode 100644 deliberi/xrpc/signup.go create mode 100644 deliberi/xrpc/signup_test.go create mode 100644 deliberi/xrpc/xrpc.go create mode 100644 deliberi/xrpc/xrpc_test.go create mode 100644 docs/.gitignore create mode 100644 docs/astro.config.mjs create mode 100644 docs/package-lock.json create mode 100644 docs/package.json create mode 100644 docs/pnpm-lock.yaml create mode 100644 docs/pnpm-workspace.yaml create mode 100644 docs/public/apple-touch-icon.png create mode 100644 docs/public/favicon.svg create mode 100644 docs/scripts/xrpc-api.mjs create mode 100644 docs/src/assets/diagrams/bobbin.svg create mode 100644 docs/src/assets/diagrams/spindle.svg create mode 100644 docs/src/components/Diagram.astro create mode 100644 docs/src/components/Header.astro create mode 100644 docs/src/components/Hero.astro create mode 100644 docs/src/components/MarkdownContent.astro create mode 100644 docs/src/components/MobileMenuFooter.astro create mode 100644 docs/src/components/PageTitle.astro create mode 100644 docs/src/components/PullSelectors.svelte create mode 100644 docs/src/components/RepoCreationName.svelte create mode 100644 docs/src/components/Search.astro create mode 100644 docs/src/components/Sidebar.astro create mode 100644 docs/src/components/SiteTitle.astro create mode 100644 docs/src/components/ThemeProvider.astro create mode 100644 docs/src/components/VouchCardDemo.svelte create mode 100644 docs/src/content.config.ts create mode 100644 docs/src/content/docs/api/bobbin.md create mode 100644 docs/src/content/docs/api/knot.md create mode 100644 docs/src/content/docs/api/spindle.md create mode 100644 docs/src/content/docs/comparison-with-github.md create mode 100644 docs/src/content/docs/contributing/guide.md create mode 100644 docs/src/content/docs/contributing/hacking.md create mode 100644 docs/src/content/docs/guides/access.md create mode 100644 docs/src/content/docs/guides/importing.md create mode 100644 docs/src/content/docs/guides/notifications.md create mode 100644 docs/src/content/docs/guides/organizations.mdx create mode 100644 docs/src/content/docs/guides/profile.md create mode 100644 docs/src/content/docs/guides/pull-requests.mdx create mode 100644 docs/src/content/docs/guides/repositories.md create mode 100644 docs/src/content/docs/guides/signing-commits.md create mode 100644 docs/src/content/docs/guides/vouching.mdx create mode 100644 docs/src/content/docs/index.mdx create mode 100644 docs/src/content/docs/integrations/contributing.md create mode 100644 docs/src/content/docs/integrations/discord.md create mode 100644 docs/src/content/docs/introduction.md create mode 100644 docs/src/content/docs/knots/index.md create mode 100644 docs/src/content/docs/knots/repositories.md create mode 100644 docs/src/content/docs/knots/self-hosting.md create mode 100644 docs/src/content/docs/migrating-from-github.md create mode 100644 docs/src/content/docs/quick-start.md create mode 100644 docs/src/content/docs/reference/identifiers.md create mode 100644 docs/src/content/docs/reference/knot-config.md create mode 100644 docs/src/content/docs/reference/records.md create mode 100644 docs/src/content/docs/reference/spindle-config.md create mode 100644 docs/src/content/docs/self-hosting/frontend.md create mode 100644 docs/src/content/docs/self-hosting/index.mdx create mode 100644 docs/src/content/docs/self-hosting/indexer.md create mode 100644 docs/src/content/docs/self-hosting/knot.md create mode 100644 docs/src/content/docs/self-hosting/spindle.md create mode 100644 docs/src/content/docs/sites.md create mode 100644 docs/src/content/docs/spindles/architecture.mdx create mode 100644 docs/src/content/docs/spindles/index.md create mode 100644 docs/src/content/docs/spindles/secrets.md create mode 100644 docs/src/content/docs/spindles/self-hosting.md create mode 100644 docs/src/content/docs/spindles/webhooks.md create mode 100644 docs/src/content/docs/spindles/workflows.md create mode 100644 docs/src/content/docs/spindles/writing-workflows.md create mode 100644 docs/src/content/docs/troubleshooting.md create mode 100644 docs/src/content/docs/upgrading/knot-2.md create mode 100644 docs/src/content/docs/upgrading/knots-and-spindles.md create mode 100644 docs/src/content/docs/why-atproto.md create mode 100644 docs/src/pages/404.astro create mode 100644 docs/src/starlightRouteData.ts create mode 100644 docs/src/stubs/app-env.ts create mode 100644 docs/src/stubs/app-paths.ts create mode 100644 docs/src/styles/custom.css create mode 100644 docs/src/styles/diagrams.css create mode 100644 docs/src/styles/global.css create mode 100644 docs/src/styles/tangled.css create mode 100644 docs/tsconfig.json create mode 100644 docs/wrangler.jsonc create mode 100644 e2e/.gitignore create mode 100644 e2e/.prettierrc.json create mode 100644 e2e/README.md create mode 100644 e2e/harness/api.ts create mode 100644 e2e/harness/attestations.ts create mode 100644 e2e/harness/auth.setup.ts create mode 100644 e2e/harness/chores.ts create mode 100644 e2e/harness/cobgit.ts create mode 100644 e2e/harness/config.ts create mode 100644 e2e/harness/firehose.ts create mode 100644 e2e/harness/flow.ts create mode 100644 e2e/harness/frames.ts create mode 100644 e2e/harness/git.ts create mode 100644 e2e/harness/global.ts create mode 100644 e2e/harness/lexicons.ts create mode 100644 e2e/harness/net.ts create mode 100644 e2e/harness/processes.ts create mode 100644 e2e/harness/record.ts create mode 100644 e2e/harness/report/ExchangeRow.svelte create mode 100644 e2e/harness/report/Report.svelte create mode 100644 e2e/harness/report/data.ts create mode 100644 e2e/harness/report/render.ts create mode 100644 e2e/harness/report/search.js create mode 100644 e2e/harness/report/write.ts create mode 100644 e2e/harness/reporter.ts create mode 100644 e2e/harness/run.ts create mode 100644 e2e/harness/services.ts create mode 100644 e2e/harness/shared.ts create mode 100644 e2e/harness/signin.ts create mode 100644 e2e/harness/stream.ts create mode 100644 e2e/harness/tabs.ts create mode 100644 e2e/harness/versions.ts create mode 100644 e2e/harness/vocabulary.ts create mode 100644 e2e/harness/web-hash.ts create mode 100644 e2e/harness/webhooks.ts create mode 100644 e2e/outages/knot-outage.spec.ts create mode 100644 e2e/outages/spindle-outage.spec.ts create mode 100644 e2e/package.json create mode 100644 e2e/playwright.config.ts create mode 100644 e2e/pnpm-lock.yaml create mode 100755 e2e/run create mode 100644 e2e/specs/access.spec.ts create mode 100644 e2e/specs/account.spec.ts create mode 100644 e2e/specs/chores.spec.ts create mode 100644 e2e/specs/ci-logs.spec.ts create mode 100644 e2e/specs/comments.spec.ts create mode 100644 e2e/specs/faults.spec.ts create mode 100644 e2e/specs/fork-pulls.spec.ts create mode 100644 e2e/specs/forks.spec.ts create mode 100644 e2e/specs/git.spec.ts create mode 100644 e2e/specs/issues.spec.ts create mode 100644 e2e/specs/knots.spec.ts create mode 100644 e2e/specs/labels.spec.ts create mode 100644 e2e/specs/markup.spec.ts create mode 100644 e2e/specs/merges.spec.ts create mode 100644 e2e/specs/opengraph.spec.ts create mode 100644 e2e/specs/permissions.spec.ts create mode 100644 e2e/specs/pipelines.spec.ts create mode 100644 e2e/specs/profile-settings.spec.ts create mode 100644 e2e/specs/profile.spec.ts create mode 100644 e2e/specs/pull-pipelines.spec.ts create mode 100644 e2e/specs/pulls.spec.ts create mode 100644 e2e/specs/quota.spec.ts create mode 100644 e2e/specs/recents.spec.ts create mode 100644 e2e/specs/refusals.spec.ts create mode 100644 e2e/specs/repo-settings.spec.ts create mode 100644 e2e/specs/repos.spec.ts create mode 100644 e2e/specs/routes.spec.ts create mode 100644 e2e/specs/search.spec.ts create mode 100644 e2e/specs/session.spec.ts create mode 100644 e2e/specs/signup.spec.ts create mode 100644 e2e/specs/social.spec.ts create mode 100644 e2e/specs/strings.spec.ts create mode 100644 e2e/specs/webhooks.spec.ts create mode 100644 e2e/tools/sweep.spec.ts create mode 100644 e2e/tools/tidy.spec.ts create mode 100644 e2e/tsconfig.json create mode 100755 e2e/up create mode 100644 email-did/package.json create mode 100644 email-did/pnpm-lock.yaml create mode 100644 email-did/pnpm-workspace.yaml create mode 100644 email-did/src/index.ts create mode 100644 email-did/test/index.test.ts create mode 100644 email-did/test/setup.ts create mode 100644 email-did/tsconfig.json create mode 100644 email-did/vitest.config.ts create mode 100644 email-did/wrangler.jsonc create mode 100644 filetree/filetree.go create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 gitclient/gitclient.go create mode 100644 gitclient/gitclient_test.go create mode 100644 gitclient/pack.go create mode 100644 gitclient/pkt.go create mode 100644 gitutil/archive.go create mode 100644 gitutil/archive_test.go create mode 100644 gitutil/command.go create mode 100644 gitutil/command_test.go create mode 100644 gitutil/keyed_mutex_test.go create mode 100644 gitutil/readme.go create mode 100644 gitutil/readme_test.go create mode 100644 gitutil/sync.go create mode 100644 gitutil/sync_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hostutil/hostutil.go create mode 100644 hostutil/safedial.go create mode 100644 ico/ico.go create mode 100644 icons/bluesky.svg create mode 100644 icons/user-round-arrow-forward.svg create mode 100644 idresolver/mock.go create mode 100644 idresolver/resolver.go create mode 100644 input.css create mode 100644 jetstream/jetstream.go create mode 100644 jetstream/jetstream_test.go create mode 100644 jetstream/stream.go create mode 100644 jetstream/stream_test.go create mode 100644 jetstream/watermark.go create mode 100644 jetstream/watermark_test.go create mode 100644 jetstream/zstd_dictionary create mode 100644 jobs/.gitignore create mode 100644 jobs/package-lock.json create mode 100644 jobs/package.json create mode 100644 jobs/src/index.ts create mode 100644 jobs/src/postings/archive/software-engineer.md create mode 100644 jobs/src/woff2.d.ts create mode 100644 jobs/tsconfig.json create mode 100644 jobs/wrangler.toml create mode 100644 knot2/.gitignore create mode 100644 knot2/Containerfile create mode 100644 knot2/README.md create mode 100644 knot2/_typos.toml create mode 100644 knot2/crates/knot-acl/Cargo.toml create mode 100644 knot2/crates/knot-acl/src/lib.rs create mode 100644 knot2/crates/knot-atproto/Cargo.toml create mode 100644 knot2/crates/knot-atproto/fuzz/.gitignore create mode 100644 knot2/crates/knot-atproto/fuzz/Cargo.lock create mode 100644 knot2/crates/knot-atproto/fuzz/Cargo.toml create mode 100644 knot2/crates/knot-atproto/fuzz/fuzz_targets/did_document.rs create mode 100644 knot2/crates/knot-atproto/fuzz/fuzz_targets/pubkey.rs create mode 100644 knot2/crates/knot-atproto/src/auth.rs create mode 100644 knot2/crates/knot-atproto/src/identity.rs create mode 100644 knot2/crates/knot-atproto/src/jwt.rs create mode 100644 knot2/crates/knot-atproto/src/lib.rs create mode 100644 knot2/crates/knot-atproto/src/plc.rs create mode 100644 knot2/crates/knot-atproto/src/pointer.rs create mode 100644 knot2/crates/knot-atproto/src/pubkeys.rs create mode 100644 knot2/crates/knot-atproto/src/resolve.rs create mode 100644 knot2/crates/knot-atproto/src/test_support.rs create mode 100644 knot2/crates/knot-atproto/tests/fuzz_smoke.rs create mode 100644 knot2/crates/knot-atproto/tests/security.rs create mode 100644 knot2/crates/knot-bench/Cargo.toml create mode 100644 knot2/crates/knot-bench/benches/advert.rs create mode 100644 knot2/crates/knot-bench/benches/blame.rs create mode 100644 knot2/crates/knot-bench/benches/cob.rs create mode 100644 knot2/crates/knot-bench/benches/coldstart.rs create mode 100644 knot2/crates/knot-bench/benches/pack.rs create mode 100644 knot2/crates/knot-bench/benches/projection.rs create mode 100755 knot2/crates/knot-bench/maint-bench.sh create mode 100644 knot2/crates/knot-bench/src/bin/maint-bench.rs create mode 100644 knot2/crates/knot-bench/src/fixtures.rs create mode 100644 knot2/crates/knot-bench/src/latency.rs create mode 100644 knot2/crates/knot-bench/src/lib.rs create mode 100644 knot2/crates/knot-bench/tests/boot.rs create mode 100644 knot2/crates/knot-bench/tests/gate.rs create mode 100644 knot2/crates/knot-bench/tests/members_gate.rs create mode 100644 knot2/crates/knot-bench/tests/registry_gate.rs create mode 100644 knot2/crates/knot-cache/Cargo.toml create mode 100644 knot2/crates/knot-cache/src/expiring.rs create mode 100644 knot2/crates/knot-cache/src/lib.rs create mode 100644 knot2/crates/knot-cob/Cargo.toml create mode 100644 knot2/crates/knot-cob/src/backend.rs create mode 100644 knot2/crates/knot-cob/src/change.rs create mode 100644 knot2/crates/knot-cob/src/error.rs create mode 100644 knot2/crates/knot-cob/src/graph.rs create mode 100644 knot2/crates/knot-cob/src/instrument.rs create mode 100644 knot2/crates/knot-cob/src/lib.rs create mode 100644 knot2/crates/knot-cob/src/object.rs create mode 100644 knot2/crates/knot-cobs/Cargo.toml create mode 100644 knot2/crates/knot-cobs/fuzz/.gitignore create mode 100644 knot2/crates/knot-cobs/fuzz/Cargo.lock create mode 100644 knot2/crates/knot-cobs/fuzz/Cargo.toml create mode 100644 knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_change.rs create mode 100644 knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_ref.rs create mode 100644 knot2/crates/knot-cobs/src/artifact.rs create mode 100644 knot2/crates/knot-cobs/src/blocklist.rs create mode 100644 knot2/crates/knot-cobs/src/collaborators.rs create mode 100644 knot2/crates/knot-cobs/src/comment.rs create mode 100644 knot2/crates/knot-cobs/src/grant.rs create mode 100644 knot2/crates/knot-cobs/src/import.rs create mode 100644 knot2/crates/knot-cobs/src/label.rs create mode 100644 knot2/crates/knot-cobs/src/lib.rs create mode 100644 knot2/crates/knot-cobs/src/manifest.rs create mode 100644 knot2/crates/knot-cobs/src/members.rs create mode 100644 knot2/crates/knot-cobs/src/pull.rs create mode 100644 knot2/crates/knot-cobs/src/reaction.rs create mode 100644 knot2/crates/knot-cobs/src/registry.rs create mode 100644 knot2/crates/knot-cobs/src/social.rs create mode 100644 knot2/crates/knot-cobs/src/ticket.rs create mode 100644 knot2/crates/knot-cobs/tests/cobs.rs create mode 100644 knot2/crates/knot-cobs/tests/common/mod.rs create mode 100644 knot2/crates/knot-cobs/tests/fuzz_smoke.rs create mode 100644 knot2/crates/knot-cobs/tests/invariants.rs create mode 100644 knot2/crates/knot-cobs/tests/social.rs create mode 100644 knot2/crates/knot-config/Cargo.toml create mode 100644 knot2/crates/knot-config/src/lib.rs create mode 100644 knot2/crates/knot-edge/Cargo.toml create mode 100644 knot2/crates/knot-edge/fuzz/.gitignore create mode 100644 knot2/crates/knot-edge/fuzz/Cargo.lock create mode 100644 knot2/crates/knot-edge/fuzz/Cargo.toml create mode 100644 knot2/crates/knot-edge/fuzz/fuzz_targets/spki.rs create mode 100644 knot2/crates/knot-edge/fuzz/fuzz_targets/spki_pin.rs create mode 100644 knot2/crates/knot-edge/src/acme.rs create mode 100644 knot2/crates/knot-edge/src/altsvc.rs create mode 100644 knot2/crates/knot-edge/src/compression.rs create mode 100644 knot2/crates/knot-edge/src/cors.rs create mode 100644 knot2/crates/knot-edge/src/lib.rs create mode 100644 knot2/crates/knot-edge/src/limits.rs create mode 100644 knot2/crates/knot-edge/src/peer.rs create mode 100644 knot2/crates/knot-edge/src/protocol.rs create mode 100644 knot2/crates/knot-edge/src/quic.rs create mode 100644 knot2/crates/knot-edge/src/robustness.rs create mode 100644 knot2/crates/knot-edge/src/tcp.rs create mode 100644 knot2/crates/knot-edge/src/tls.rs create mode 100644 knot2/crates/knot-edge/src/zerortt.rs create mode 100644 knot2/crates/knot-edge/tests/fuzz_smoke.rs create mode 100644 knot2/crates/knot-events/Cargo.toml create mode 100644 knot2/crates/knot-events/src/lib.rs create mode 100644 knot2/crates/knot-fixtures/Cargo.toml create mode 100644 knot2/crates/knot-fixtures/src/lib.rs create mode 100644 knot2/crates/knot-git/Cargo.toml create mode 100644 knot2/crates/knot-git/fuzz/.gitignore create mode 100644 knot2/crates/knot-git/fuzz/Cargo.lock create mode 100644 knot2/crates/knot-git/fuzz/Cargo.toml create mode 100644 knot2/crates/knot-git/fuzz/fuzz_targets/patch.rs create mode 100644 knot2/crates/knot-git/src/archive.rs create mode 100644 knot2/crates/knot-git/src/base85.rs create mode 100644 knot2/crates/knot-git/src/bitmap/bitset.rs create mode 100644 knot2/crates/knot-git/src/bitmap/count.rs create mode 100644 knot2/crates/knot-git/src/bitmap/ewah.rs create mode 100644 knot2/crates/knot-git/src/bitmap/midx.rs create mode 100644 knot2/crates/knot-git/src/bitmap/mod.rs create mode 100644 knot2/crates/knot-git/src/bitmap/reach.rs create mode 100644 knot2/crates/knot-git/src/bitmap/reader.rs create mode 100644 knot2/crates/knot-git/src/bitmap/revindex.rs create mode 100644 knot2/crates/knot-git/src/bitmap/writer.rs create mode 100644 knot2/crates/knot-git/src/error.rs create mode 100644 knot2/crates/knot-git/src/instrument.rs create mode 100644 knot2/crates/knot-git/src/lib.rs create mode 100644 knot2/crates/knot-git/src/maintenance.rs create mode 100644 knot2/crates/knot-git/src/objects.rs create mode 100644 knot2/crates/knot-git/src/patch.rs create mode 100644 knot2/crates/knot-git/src/patch_apply.rs create mode 100644 knot2/crates/knot-git/src/patch_parse.rs create mode 100644 knot2/crates/knot-git/src/reads.rs create mode 100644 knot2/crates/knot-git/src/repo.rs create mode 100644 knot2/crates/knot-git/src/staging.rs create mode 100644 knot2/crates/knot-git/tests/apply.rs create mode 100644 knot2/crates/knot-git/tests/bitmap.rs create mode 100644 knot2/crates/knot-git/tests/common/mod.rs create mode 100644 knot2/crates/knot-git/tests/config_isolation.rs create mode 100644 knot2/crates/knot-git/tests/fuzz_smoke.rs create mode 100644 knot2/crates/knot-git/tests/reads.rs create mode 100644 knot2/crates/knot-index/Cargo.toml create mode 100644 knot2/crates/knot-index/src/coverage.rs create mode 100644 knot2/crates/knot-index/src/error.rs create mode 100644 knot2/crates/knot-index/src/intern.rs create mode 100644 knot2/crates/knot-index/src/lib.rs create mode 100644 knot2/crates/knot-index/src/projections.rs create mode 100644 knot2/crates/knot-index/src/social.rs create mode 100644 knot2/crates/knot-index/tests/common/mod.rs create mode 100644 knot2/crates/knot-index/tests/lifecycle.rs create mode 100644 knot2/crates/knot-index/tests/meta_repo.rs create mode 100644 knot2/crates/knot-index/tests/projections.rs create mode 100644 knot2/crates/knot-index/tests/properties.rs create mode 100644 knot2/crates/knot-keyfill/Cargo.toml create mode 100644 knot2/crates/knot-keyfill/src/lib.rs create mode 100644 knot2/crates/knot-keyfill/tests/fill.rs create mode 100644 knot2/crates/knot-langs/Cargo.toml create mode 100644 knot2/crates/knot-langs/src/langs.rs create mode 100644 knot2/crates/knot-langs/src/lib.rs create mode 100644 knot2/crates/knot-langs/src/linguist.rs create mode 100644 knot2/crates/knot-lexicons/Cargo.toml create mode 100644 knot2/crates/knot-lexicons/build.rs create mode 100644 knot2/crates/knot-lexicons/src/lib.rs create mode 100644 knot2/crates/knot-lfs/Cargo.toml create mode 100644 knot2/crates/knot-lfs/fuzz/.gitignore create mode 100644 knot2/crates/knot-lfs/fuzz/Cargo.lock create mode 100644 knot2/crates/knot-lfs/fuzz/Cargo.toml create mode 100644 knot2/crates/knot-lfs/fuzz/fuzz_targets/batch.rs create mode 100644 knot2/crates/knot-lfs/fuzz/fuzz_targets/pointer.rs create mode 100644 knot2/crates/knot-lfs/fuzz/fuzz_targets/transfer.rs create mode 100644 knot2/crates/knot-lfs/src/admission.rs create mode 100644 knot2/crates/knot-lfs/src/batch.rs create mode 100644 knot2/crates/knot-lfs/src/error.rs create mode 100644 knot2/crates/knot-lfs/src/gc.rs create mode 100644 knot2/crates/knot-lfs/src/lib.rs create mode 100644 knot2/crates/knot-lfs/src/pointer.rs create mode 100644 knot2/crates/knot-lfs/src/scan.rs create mode 100644 knot2/crates/knot-lfs/src/store.rs create mode 100644 knot2/crates/knot-lfs/src/transfer.rs create mode 100644 knot2/crates/knot-lfs/src/types.rs create mode 100644 knot2/crates/knot-lfs/tests/chaos.rs create mode 100644 knot2/crates/knot-lfs/tests/common/mod.rs create mode 100644 knot2/crates/knot-lfs/tests/fuzz_smoke.rs create mode 100644 knot2/crates/knot-lfs/tests/properties.rs create mode 100644 knot2/crates/knot-lfs/tests/race.rs create mode 100644 knot2/crates/knot-lfs/tests/soak.rs create mode 100644 knot2/crates/knot-maintenance/Cargo.toml create mode 100644 knot2/crates/knot-maintenance/src/bitmap.rs create mode 100644 knot2/crates/knot-maintenance/src/commitgraph.rs create mode 100644 knot2/crates/knot-maintenance/src/cruft.rs create mode 100644 knot2/crates/knot-maintenance/src/fsio.rs create mode 100644 knot2/crates/knot-maintenance/src/lib.rs create mode 100644 knot2/crates/knot-maintenance/src/midx.rs create mode 100644 knot2/crates/knot-maintenance/src/policy.rs create mode 100644 knot2/crates/knot-maintenance/src/prune.rs create mode 100644 knot2/crates/knot-maintenance/src/repack.rs create mode 100644 knot2/crates/knot-maintenance/src/scheduler.rs create mode 100644 knot2/crates/knot-maintenance/src/test_support.rs create mode 100644 knot2/crates/knot-maintenance/tests/chaos.rs create mode 100644 knot2/crates/knot-maintenance/tests/commitgraph.rs create mode 100644 knot2/crates/knot-maintenance/tests/common/mod.rs create mode 100644 knot2/crates/knot-maintenance/tests/cruft.rs create mode 100644 knot2/crates/knot-maintenance/tests/engine.rs create mode 100644 knot2/crates/knot-maintenance/tests/geometric.rs create mode 100644 knot2/crates/knot-messages/Cargo.toml create mode 100644 knot2/crates/knot-messages/src/lib.rs create mode 100644 knot2/crates/knot-messages/src/template.rs create mode 100644 knot2/crates/knot-migrate/Cargo.toml create mode 100644 knot2/crates/knot-migrate/src/adopt.rs create mode 100644 knot2/crates/knot-migrate/src/casbin.rs create mode 100644 knot2/crates/knot-migrate/src/emit.rs create mode 100644 knot2/crates/knot-migrate/src/envfile.rs create mode 100644 knot2/crates/knot-migrate/src/lib.rs create mode 100644 knot2/crates/knot-migrate/src/main.rs create mode 100644 knot2/crates/knot-migrate/src/mapping.rs create mode 100644 knot2/crates/knot-migrate/src/rehearse.rs create mode 100644 knot2/crates/knot-migrate/src/report.rs create mode 100644 knot2/crates/knot-migrate/src/source.rs create mode 100644 knot2/crates/knot-migrate/tests/migrate.rs create mode 100644 knot2/crates/knot-pack/Cargo.toml create mode 100644 knot2/crates/knot-pack/examples/ingest_mem.rs create mode 100644 knot2/crates/knot-pack/examples/knot_http_backend.rs create mode 100644 knot2/crates/knot-pack/examples/receive_mem.rs create mode 100644 knot2/crates/knot-pack/fuzz/.gitignore create mode 100644 knot2/crates/knot-pack/fuzz/Cargo.lock create mode 100644 knot2/crates/knot-pack/fuzz/Cargo.toml create mode 100644 knot2/crates/knot-pack/fuzz/fuzz_targets/pack.rs create mode 100644 knot2/crates/knot-pack/fuzz/fuzz_targets/pkt.rs create mode 100644 knot2/crates/knot-pack/fuzz/fuzz_targets/receive_commands.rs create mode 100644 knot2/crates/knot-pack/fuzz/fuzz_targets/upload_args.rs create mode 100644 knot2/crates/knot-pack/src/archive.rs create mode 100644 knot2/crates/knot-pack/src/cache.rs create mode 100644 knot2/crates/knot-pack/src/error.rs create mode 100644 knot2/crates/knot-pack/src/fetch.rs create mode 100644 knot2/crates/knot-pack/src/frame.rs create mode 100644 knot2/crates/knot-pack/src/guard.rs create mode 100644 knot2/crates/knot-pack/src/ids.rs create mode 100644 knot2/crates/knot-pack/src/idxwrite.rs create mode 100644 knot2/crates/knot-pack/src/lib.rs create mode 100644 knot2/crates/knot-pack/src/meter.rs create mode 100644 knot2/crates/knot-pack/src/objects.rs create mode 100644 knot2/crates/knot-pack/src/pkt.rs create mode 100644 knot2/crates/knot-pack/src/quarantine.rs create mode 100644 knot2/crates/knot-pack/src/receive.rs create mode 100644 knot2/crates/knot-pack/src/receiver.rs create mode 100644 knot2/crates/knot-pack/src/resolve.rs create mode 100644 knot2/crates/knot-pack/src/upload.rs create mode 100644 knot2/crates/knot-pack/tests/chaos.rs create mode 100644 knot2/crates/knot-pack/tests/common/mod.rs create mode 100644 knot2/crates/knot-pack/tests/differential.rs create mode 100644 knot2/crates/knot-pack/tests/fetch.rs create mode 100644 knot2/crates/knot-pack/tests/fuzz_smoke.rs create mode 100644 knot2/crates/knot-pack/tests/git_client.rs create mode 100644 knot2/crates/knot-pack/tests/h3_conformance.rs create mode 100644 knot2/crates/knot-pack/tests/handle_owner.rs create mode 100644 knot2/crates/knot-pack/tests/hardening.rs create mode 100644 knot2/crates/knot-pack/tests/quarantine.rs create mode 100644 knot2/crates/knot-pack/tests/serving.rs create mode 100644 knot2/crates/knot-pack/tests/soak.rs create mode 100644 knot2/crates/knot-postreceive/Cargo.toml create mode 100644 knot2/crates/knot-postreceive/src/lib.rs create mode 100644 knot2/crates/knot-postreceive/tests/post_receive.rs create mode 100644 knot2/crates/knot-receive/Cargo.toml create mode 100644 knot2/crates/knot-receive/src/lib.rs create mode 100644 knot2/crates/knot-record/Cargo.toml create mode 100644 knot2/crates/knot-record/src/artifact.rs create mode 100644 knot2/crates/knot-record/src/attachment.rs create mode 100644 knot2/crates/knot-record/src/blocks.rs create mode 100644 knot2/crates/knot-record/src/chain.rs create mode 100644 knot2/crates/knot-record/src/comment.rs create mode 100644 knot2/crates/knot-record/src/git_refs.rs create mode 100644 knot2/crates/knot-record/src/grant.rs create mode 100644 knot2/crates/knot-record/src/label.rs create mode 100644 knot2/crates/knot-record/src/lib.rs create mode 100644 knot2/crates/knot-record/src/manifest.rs create mode 100644 knot2/crates/knot-record/src/mst.rs create mode 100644 knot2/crates/knot-record/src/proof.rs create mode 100644 knot2/crates/knot-record/src/pull.rs create mode 100644 knot2/crates/knot-record/src/reaction.rs create mode 100644 knot2/crates/knot-record/src/ticket.rs create mode 100644 knot2/crates/knot-record/tests/car_exports.rs create mode 100644 knot2/crates/knot-record/tests/proof_fixtures.rs create mode 100644 knot2/crates/knot-resource/Cargo.toml create mode 100644 knot2/crates/knot-resource/src/admission.rs create mode 100644 knot2/crates/knot-resource/src/cpu.rs create mode 100644 knot2/crates/knot-resource/src/disk.rs create mode 100644 knot2/crates/knot-resource/src/fsio.rs create mode 100644 knot2/crates/knot-resource/src/lib.rs create mode 100644 knot2/crates/knot-resource/src/mem.rs create mode 100644 knot2/crates/knot-resource/src/slots.rs create mode 100644 knot2/crates/knot-runtime/Cargo.toml create mode 100644 knot2/crates/knot-runtime/src/clock.rs create mode 100644 knot2/crates/knot-runtime/src/dns.rs create mode 100644 knot2/crates/knot-runtime/src/entropy.rs create mode 100644 knot2/crates/knot-runtime/src/http.rs create mode 100644 knot2/crates/knot-runtime/src/lib.rs create mode 100644 knot2/crates/knot-runtime/src/signer.rs create mode 100644 knot2/crates/knot-secrets/Cargo.toml create mode 100644 knot2/crates/knot-secrets/src/lib.rs create mode 100644 knot2/crates/knot-server/Cargo.toml create mode 100644 knot2/crates/knot-server/src/admin.rs create mode 100644 knot2/crates/knot-server/src/allocator.rs create mode 100644 knot2/crates/knot-server/src/doctor.rs create mode 100644 knot2/crates/knot-server/src/homepage.html create mode 100644 knot2/crates/knot-server/src/main.rs create mode 100644 knot2/crates/knot-server/tests/invariants.rs create mode 100644 knot2/crates/knot-sim/Cargo.toml create mode 100644 knot2/crates/knot-sim/src/harness.rs create mode 100644 knot2/crates/knot-sim/src/lib.rs create mode 100644 knot2/crates/knot-sim/src/main.rs create mode 100644 knot2/crates/knot-sim/src/realdata.rs create mode 100644 knot2/crates/knot-sim/src/trace.rs create mode 100644 knot2/crates/knot-sim/src/workload.rs create mode 100644 knot2/crates/knot-sim/tests/common/mod.rs create mode 100644 knot2/crates/knot-sim/tests/h3.rs create mode 100644 knot2/crates/knot-sim/tests/lfs_chaos.rs create mode 100644 knot2/crates/knot-sim/tests/lfs_roundtrip.rs create mode 100644 knot2/crates/knot-sim/tests/reproducible.rs create mode 100644 knot2/crates/knot-sim/tests/social_chaos.rs create mode 100644 knot2/crates/knot-sim/tests/ssh.rs create mode 100644 knot2/crates/knot-ssh/Cargo.toml create mode 100644 knot2/crates/knot-ssh/examples/ephemeral_knot.rs create mode 100644 knot2/crates/knot-ssh/src/exec.rs create mode 100644 knot2/crates/knot-ssh/src/identity.rs create mode 100644 knot2/crates/knot-ssh/src/lib.rs create mode 100644 knot2/crates/knot-ssh/src/proxy.rs create mode 100644 knot2/crates/knot-ssh/src/server.rs create mode 100644 knot2/crates/knot-ssh/tests/ssh_push.rs create mode 100644 knot2/crates/knot-storage/Cargo.toml create mode 100644 knot2/crates/knot-storage/src/error.rs create mode 100644 knot2/crates/knot-storage/src/id.rs create mode 100644 knot2/crates/knot-storage/src/kv.rs create mode 100644 knot2/crates/knot-storage/src/lib.rs create mode 100644 knot2/crates/knot-storage/src/objects.rs create mode 100644 knot2/crates/knot-storage/src/pack.rs create mode 100644 knot2/crates/knot-storage/src/refs.rs create mode 100644 knot2/crates/knot-storage/src/store.rs create mode 100644 knot2/crates/knot-types/Cargo.toml create mode 100644 knot2/crates/knot-types/src/changes.rs create mode 100644 knot2/crates/knot-types/src/hex.rs create mode 100644 knot2/crates/knot-types/src/ids.rs create mode 100644 knot2/crates/knot-types/src/lib.rs create mode 100644 knot2/crates/knot-types/src/net.rs create mode 100644 knot2/crates/knot-types/src/newtype.rs create mode 100644 knot2/crates/knot-types/src/policy.rs create mode 100644 knot2/crates/knot-types/src/record.rs create mode 100644 knot2/crates/knot-workflow/Cargo.toml create mode 100644 knot2/crates/knot-workflow/src/lib.rs create mode 100644 knot2/crates/knot-xrpc/Cargo.toml create mode 100644 knot2/crates/knot-xrpc/src/atproto.rs create mode 100644 knot2/crates/knot-xrpc/src/blobs.rs create mode 100644 knot2/crates/knot-xrpc/src/blocklist.rs create mode 100644 knot2/crates/knot-xrpc/src/body.rs create mode 100644 knot2/crates/knot-xrpc/src/branches.rs create mode 100644 knot2/crates/knot-xrpc/src/cob.rs create mode 100644 knot2/crates/knot-xrpc/src/collaborators.rs create mode 100644 knot2/crates/knot-xrpc/src/error.rs create mode 100644 knot2/crates/knot-xrpc/src/firehose.rs create mode 100644 knot2/crates/knot-xrpc/src/forks.rs create mode 100644 knot2/crates/knot-xrpc/src/foster.rs create mode 100644 knot2/crates/knot-xrpc/src/git.rs create mode 100644 knot2/crates/knot-xrpc/src/grants.rs create mode 100644 knot2/crates/knot-xrpc/src/handover.rs create mode 100644 knot2/crates/knot-xrpc/src/imports.rs create mode 100644 knot2/crates/knot-xrpc/src/invite_codes.rs create mode 100644 knot2/crates/knot-xrpc/src/keep_sweep.rs create mode 100644 knot2/crates/knot-xrpc/src/legacy_admin.rs create mode 100644 knot2/crates/knot-xrpc/src/lfs.rs create mode 100644 knot2/crates/knot-xrpc/src/lib.rs create mode 100644 knot2/crates/knot-xrpc/src/locks.rs create mode 100644 knot2/crates/knot-xrpc/src/materialize.rs create mode 100644 knot2/crates/knot-xrpc/src/members.rs create mode 100644 knot2/crates/knot-xrpc/src/migrate.rs create mode 100644 knot2/crates/knot-xrpc/src/mint.rs create mode 100644 knot2/crates/knot-xrpc/src/out.rs create mode 100644 knot2/crates/knot-xrpc/src/outbox.rs create mode 100644 knot2/crates/knot-xrpc/src/plc.rs create mode 100644 knot2/crates/knot-xrpc/src/query.rs create mode 100644 knot2/crates/knot-xrpc/src/reads.rs create mode 100644 knot2/crates/knot-xrpc/src/receive.rs create mode 100644 knot2/crates/knot-xrpc/src/refrecords.rs create mode 100644 knot2/crates/knot-xrpc/src/repos.rs create mode 100644 knot2/crates/knot-xrpc/src/reservations.rs create mode 100644 knot2/crates/knot-xrpc/src/service.rs create mode 100644 knot2/crates/knot-xrpc/src/social.rs create mode 100644 knot2/crates/knot-xrpc/src/tests.rs create mode 100644 knot2/crates/knot-xrpc/tests/atproto.rs create mode 100644 knot2/crates/knot-xrpc/tests/atproto_interop.rs create mode 100644 knot2/crates/knot-xrpc/tests/blob_properties.proptest-regressions create mode 100644 knot2/crates/knot-xrpc/tests/blob_properties.rs create mode 100644 knot2/crates/knot-xrpc/tests/blobs.rs create mode 100644 knot2/crates/knot-xrpc/tests/common/mod.rs create mode 100644 knot2/crates/knot-xrpc/tests/grant_records.rs create mode 100644 knot2/crates/knot-xrpc/tests/handover.rs create mode 100644 knot2/crates/knot-xrpc/tests/lfs.rs create mode 100644 knot2/crates/knot-xrpc/tests/lfs_soak.rs create mode 100644 knot2/crates/knot-xrpc/tests/mint.rs create mode 100644 knot2/crates/knot-xrpc/tests/policy_reconcile.rs create mode 100644 knot2/crates/knot-xrpc/tests/projection.rs create mode 100644 knot2/crates/knot-xrpc/tests/reads.rs create mode 100644 knot2/crates/knot-xrpc/tests/ref_records.rs create mode 100644 knot2/crates/knot-xrpc/tests/retire.rs create mode 100644 knot2/crates/knot-xrpc/tests/spindle_outbox.rs create mode 100644 knot2/example.toml create mode 100644 knot2/interop/atproto_firehose_test.go create mode 100644 knot2/interop/atproto_plc_attestation_test.go create mode 100644 knot2/interop/atproto_reads_test.go create mode 100644 knot2/interop/atproto_ticket_test.go create mode 100644 knot2/interop/foster_test.go create mode 100644 knot2/interop/knotfeed_consumer_test.go create mode 100644 knot2/justfile create mode 100644 knot2/third_party/gix-imara-diff/.cargo_vcs_info.json create mode 100644 knot2/third_party/gix-imara-diff/Cargo.toml create mode 100644 knot2/third_party/gix-imara-diff/LICENSE create mode 100644 knot2/third_party/gix-imara-diff/README.md create mode 100644 knot2/third_party/gix-imara-diff/src/histogram.rs create mode 100644 knot2/third_party/gix-imara-diff/src/histogram/lcs.rs create mode 100644 knot2/third_party/gix-imara-diff/src/histogram/list_pool.rs create mode 100644 knot2/third_party/gix-imara-diff/src/intern.rs create mode 100644 knot2/third_party/gix-imara-diff/src/lib.rs create mode 100644 knot2/third_party/gix-imara-diff/src/myers.rs create mode 100644 knot2/third_party/gix-imara-diff/src/myers/middle_snake.rs create mode 100644 knot2/third_party/gix-imara-diff/src/myers/preprocess.rs create mode 100644 knot2/third_party/gix-imara-diff/src/myers/slice.rs create mode 100644 knot2/third_party/gix-imara-diff/src/postprocess.rs create mode 100644 knot2/third_party/gix-imara-diff/src/slider_heuristic.rs create mode 100644 knot2/third_party/gix-imara-diff/src/sources.rs create mode 100644 knot2/third_party/gix-imara-diff/src/unified_diff.rs create mode 100644 knot2/third_party/gix-imara-diff/src/util.rs create mode 100644 knot2/third_party/gix-pack/.cargo_vcs_info.json create mode 100644 knot2/third_party/gix-pack/Cargo.toml create mode 100644 knot2/third_party/gix-pack/LICENSE-APACHE create mode 100644 knot2/third_party/gix-pack/LICENSE-MIT create mode 100644 knot2/third_party/gix-pack/src/bundle/find.rs create mode 100644 knot2/third_party/gix-pack/src/bundle/init.rs create mode 100644 knot2/third_party/gix-pack/src/bundle/mod.rs create mode 100644 knot2/third_party/gix-pack/src/bundle/write/error.rs create mode 100644 knot2/third_party/gix-pack/src/bundle/write/mod.rs create mode 100644 knot2/third_party/gix-pack/src/bundle/write/types.rs create mode 100644 knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs create mode 100644 knot2/third_party/gix-pack/src/cache/delta/mod.rs create mode 100644 knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs create mode 100644 knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs create mode 100644 knot2/third_party/gix-pack/src/cache/delta/tree.rs create mode 100644 knot2/third_party/gix-pack/src/cache/lru.rs create mode 100644 knot2/third_party/gix-pack/src/cache/mod.rs create mode 100644 knot2/third_party/gix-pack/src/cache/object.rs create mode 100644 knot2/third_party/gix-pack/src/data/delta.rs create mode 100644 knot2/third_party/gix-pack/src/data/entry/decode.rs create mode 100644 knot2/third_party/gix-pack/src/data/entry/header.rs create mode 100644 knot2/third_party/gix-pack/src/data/entry/mod.rs create mode 100644 knot2/third_party/gix-pack/src/data/file/decode/entry.rs create mode 100644 knot2/third_party/gix-pack/src/data/file/decode/header.rs create mode 100644 knot2/third_party/gix-pack/src/data/file/decode/mod.rs create mode 100644 knot2/third_party/gix-pack/src/data/file/init.rs create mode 100644 knot2/third_party/gix-pack/src/data/file/mod.rs create mode 100644 knot2/third_party/gix-pack/src/data/file/verify.rs create mode 100644 knot2/third_party/gix-pack/src/data/header.rs create mode 100644 knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs create mode 100644 knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs create mode 100644 knot2/third_party/gix-pack/src/data/input/entry.rs create mode 100644 knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs create mode 100644 knot2/third_party/gix-pack/src/data/input/mod.rs create mode 100644 knot2/third_party/gix-pack/src/data/input/types.rs create mode 100644 knot2/third_party/gix-pack/src/data/mod.rs create mode 100644 knot2/third_party/gix-pack/src/data/output/bytes.rs create mode 100644 knot2/third_party/gix-pack/src/data/output/count/mod.rs create mode 100644 knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs create mode 100644 knot2/third_party/gix-pack/src/data/output/count/objects/reachable.rs create mode 100644 knot2/third_party/gix-pack/src/data/output/count/objects/reduce.rs create mode 100644 knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs create mode 100644 knot2/third_party/gix-pack/src/data/output/count/objects/types.rs create mode 100644 knot2/third_party/gix-pack/src/data/output/count/objects/util.rs create mode 100644 knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs create mode 100644 knot2/third_party/gix-pack/src/data/output/entry/mod.rs create mode 100644 knot2/third_party/gix-pack/src/data/output/mod.rs create mode 100644 knot2/third_party/gix-pack/src/find.rs create mode 100644 knot2/third_party/gix-pack/src/find_traits.rs create mode 100644 knot2/third_party/gix-pack/src/index/access.rs create mode 100644 knot2/third_party/gix-pack/src/index/encode.rs create mode 100644 knot2/third_party/gix-pack/src/index/init.rs create mode 100644 knot2/third_party/gix-pack/src/index/mod.rs create mode 100644 knot2/third_party/gix-pack/src/index/traverse/error.rs create mode 100644 knot2/third_party/gix-pack/src/index/traverse/mod.rs create mode 100644 knot2/third_party/gix-pack/src/index/traverse/reduce.rs create mode 100644 knot2/third_party/gix-pack/src/index/traverse/types.rs create mode 100644 knot2/third_party/gix-pack/src/index/traverse/with_index.rs create mode 100644 knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs create mode 100644 knot2/third_party/gix-pack/src/index/util.rs create mode 100644 knot2/third_party/gix-pack/src/index/verify.rs create mode 100644 knot2/third_party/gix-pack/src/index/write/error.rs create mode 100644 knot2/third_party/gix-pack/src/index/write/mod.rs create mode 100755 knot2/third_party/gix-pack/src/lib.rs create mode 100644 knot2/third_party/gix-pack/src/multi_index/access.rs create mode 100644 knot2/third_party/gix-pack/src/multi_index/chunk.rs create mode 100644 knot2/third_party/gix-pack/src/multi_index/init.rs create mode 100644 knot2/third_party/gix-pack/src/multi_index/mod.rs create mode 100644 knot2/third_party/gix-pack/src/multi_index/verify.rs create mode 100644 knot2/third_party/gix-pack/src/multi_index/write.rs create mode 100644 knot2/third_party/gix-pack/src/verify.rs create mode 100644 knotfeed/frame.go create mode 100644 knotfeed/frame_test.go create mode 100644 knotfeed/legacy.go create mode 100644 knotfeed/legacy_test.go create mode 100644 knotfeed/record.go create mode 100644 knotfeed/record_test.go create mode 100644 knotfeed/subscribe.go create mode 100644 knotfeed/subscribe_test.go create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-27286d65426e3f215744767b774bede293bb1e45b7a6a1aec0af40a729cf89d0.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-27487b1b800decd26cb5a1500711c474657fa4a1f7899135a690b98cc59a6f97.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-4639c2d4a3d29f03f9145d7352eb9d9751d1ffec1101bf42196c52fff6047a5a.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-48bc0c0d369583d5cce9f45760338feedf8c3f714eabe06dc1e2aba3dddb0c62.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-593e82adb20973a2ddd9e853e731f5c01b8576aa4b8782b97c20aeed78009328.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-63518fc08ee86cdaa6a881183578d5665152b7431776b565fa9ed9324090a78d.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-66df0c0bb00794d4fabfc89d3ca20a0367e03cd610a742b878b416eeb9a3f561.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-6a8ebbb538b3254515d113c27135f20cb2a40f8a2f3c89b0a07115caf920c876.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-77c15f179c796a3a8d1d6e3049797238098885401886cc6aab8d386405c1370c.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-7ff739bd0a1b556db369e2948df9e61a7c74c77ae3418ea43a9bc929d38f324e.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-886c1448ba85e0aba97c0017c24120962bc21fe3a3303e88bcb7d071d817c253.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-93000db396c7146e57bd7d7e60eba285f3a15b97646b005ef28a2975217ba20a.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-ab795867f17041560dff32d76690dac22678ede2b8949cd557b6ce704d79b0b7.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-b2e6317ee0c6d9ef282e9d02c5b6b42d7b656676e2465608ff0618b702735270.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-bdcad83ba694785812ced85914595227d765603eb5299393e1ba39af09222fa7.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-bebe7c55b9010d9e1f1aa5daef9e291b13885004da7d0c98201c98e98e805649.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-c0899b71414aaa2fb7885289cb34f120f79ba4de8b635035ea61883ee1ff4595.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-c15b02adc2ecf9a6f65e30af9ac43c3d663f73924e887886e15aa971c79f5053.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-c467065187d676ef4fb8e9d3bdbb6b0a7a33ecf56143fa25e76b2e27df7de3c2.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-c95a6e06127c35f49b322512d19e6af8677050b0984519f7150e3b755511768f.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-d2272d8970ee58876b35266ab2ba3997759e6abe4478ea7020bbe9cf133168bc.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-d947bb488a5ec5f7f30a17d4faf90746f85e8fc0d88ed641ec9f901ad3fcacfa.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-e680cd589b5830abdbf1eaad8be5f0ea4a5d3323784cd4f261efb99aa02679fc.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-ee3e927e80d0f3051ed3d2de99c0891856550ef9da693a41d18cd62189aad307.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-ef69831ad84fc5dc6989cd661cfab230c726baf267f68f625aa1cdbcba5bd312.json create mode 100644 knotmirror/crates/knotmirror-db/.sqlx/query-fba3815aee0953d64337931438e6f9266ffce82420c0e8b00c8e8b1e863aa938.json create mode 100644 knotmirror/crates/knotmirror-db/Cargo.toml create mode 100644 knotmirror/crates/knotmirror-db/migrations/20260916000000_initial.sql create mode 100644 knotmirror/crates/knotmirror-db/migrations/20260926000000_index.sql create mode 100644 knotmirror/crates/knotmirror-db/migrations/20261019000000_atproto_rev.sql create mode 100644 knotmirror/crates/knotmirror-db/migrations/20261029000000_commit_activity.sql create mode 100644 knotmirror/crates/knotmirror-db/migrations/20261030000000_commit_details.sql create mode 100644 knotmirror/crates/knotmirror-db/src/lib.rs create mode 100644 knotmirror/crates/knotmirror-db/tests/commit_activity.rs create mode 100644 knotmirror/crates/knotmirror-git/Cargo.toml create mode 100644 knotmirror/crates/knotmirror-git/src/layout.rs create mode 100644 knotmirror/crates/knotmirror-git/src/lib.rs create mode 100644 knotmirror/crates/knotmirror-git/src/repo_ext.rs create mode 100644 knotmirror/crates/knotmirror-git/src/scratch.rs create mode 100644 knotmirror/crates/knotmirror-index/Cargo.toml create mode 100644 knotmirror/crates/knotmirror-index/src/activity.rs create mode 100644 knotmirror/crates/knotmirror-index/src/lib.rs create mode 100644 knotmirror/crates/knotmirror-ingest/Cargo.toml create mode 100644 knotmirror/crates/knotmirror-ingest/src/lib.rs create mode 100644 knotmirror/crates/knotmirror-maintenance/Cargo.toml create mode 100644 knotmirror/crates/knotmirror-maintenance/src/lib.rs create mode 100644 knotmirror/crates/knotmirror-sync/Cargo.toml create mode 100644 knotmirror/crates/knotmirror-sync/src/admission.rs create mode 100644 knotmirror/crates/knotmirror-sync/src/lib.rs create mode 100644 knotmirror/crates/knotmirror-sync/src/task.rs create mode 100644 knotmirror/crates/knotmirror-types/Cargo.toml create mode 100644 knotmirror/crates/knotmirror-types/src/jacquard_ext.rs create mode 100644 knotmirror/crates/knotmirror-types/src/lib.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/Cargo.toml create mode 100644 knotmirror/crates/knotmirror-xrpc/clippy.toml create mode 100644 knotmirror/crates/knotmirror-xrpc/src/archive.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/committers.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/did_ext.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/diff.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/error.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/format_patch.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/git_transport.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/last_commit.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/lib.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/merge.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/metrics.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/routes/activity.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/routes/cob.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/routes/git.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/routes/mod.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/routes/reads.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/signer.rs create mode 100644 knotmirror/crates/knotmirror-xrpc/src/sniff.rs create mode 100644 knotmirror/crates/knotmirror/Cargo.toml create mode 100644 knotmirror/crates/knotmirror/src/config.rs create mode 100644 knotmirror/crates/knotmirror/src/main.rs create mode 100644 lexicon-build-config.json create mode 100644 lexicons/com/atproto/label/defs.json create mode 100644 lexicons/com/atproto/label/subscribeLabels.json create mode 100644 lexicons/com/atproto/repo/strongRef.json create mode 100644 lexicons/com/atproto/sync/subscribeRepos.json create mode 100644 lexicons/com/bad-example/pocket/defs.json create mode 100644 lexicons/com/bad-example/pocket/getPreferences.json create mode 100644 lexicons/com/bad-example/pocket/putPreferences.json create mode 100644 lexicons/git/defs.json create mode 100644 lexicons/git/mergeCommit.json create mode 100644 lexicons/git/temp/listIndexedLanguages.json create mode 100644 lexicons/git/temp/listLanguages.json create mode 100644 lexicons/git/temp2/getBlame.json create mode 100644 lexicons/git/temp2/getCommitStats.json create mode 100644 lexicons/git/temp2/getDiff.json create mode 100644 lexicons/git/temp2/getInterdiff.json create mode 100644 lexicons/git/temp2/mergeCheck.json create mode 100644 lexicons/identity/resolveCommitters.json create mode 100644 lexicons/org/tangled/actor/defs.json create mode 100644 lexicons/org/tangled/actor/getProfile.json create mode 100644 lexicons/org/tangled/actor/getProfileActivity.json create mode 100644 lexicons/org/tangled/actor/getProfiles.json create mode 100644 lexicons/org/tangled/actor/getTrending.json create mode 100644 lexicons/org/tangled/actor/profile.json create mode 100644 lexicons/org/tangled/actor/searchActorsTypeahead.json create mode 100644 lexicons/org/tangled/bobbin/awaitRecord.json create mode 100644 lexicons/org/tangled/ci/approvePipeline.json create mode 100644 lexicons/org/tangled/ci/cancelPipeline.json create mode 100644 lexicons/org/tangled/ci/defs.json create mode 100644 lexicons/org/tangled/ci/getPipeline.json create mode 100644 lexicons/org/tangled/ci/getWorkflowDefinition.json create mode 100644 lexicons/org/tangled/ci/listWorkflowDefinitions.json create mode 100644 lexicons/org/tangled/ci/queryPipelines.json create mode 100644 lexicons/org/tangled/ci/retryPipeline.json create mode 100644 lexicons/org/tangled/ci/subscribePipelineLogs.json create mode 100644 lexicons/org/tangled/ci/trigger/manual.json create mode 100644 lexicons/org/tangled/ci/trigger/schedule.json create mode 100644 lexicons/org/tangled/embed/commit.json create mode 100644 lexicons/org/tangled/embed/unspecced/file.json create mode 100644 lexicons/org/tangled/event/notifyEvent.json create mode 100644 lexicons/org/tangled/event/pullRequest.json create mode 100644 lexicons/org/tangled/event/push.json create mode 100644 lexicons/org/tangled/feed/comment.json create mode 100644 lexicons/org/tangled/feed/countComments.json create mode 100644 lexicons/org/tangled/feed/countCommentsBy.json create mode 100644 lexicons/org/tangled/feed/countReactions.json create mode 100644 lexicons/org/tangled/feed/countReactionsBy.json create mode 100644 lexicons/org/tangled/feed/countStars.json create mode 100644 lexicons/org/tangled/feed/countStarsBy.json create mode 100644 lexicons/org/tangled/feed/defs.json create mode 100644 lexicons/org/tangled/feed/getStar.json create mode 100644 lexicons/org/tangled/feed/getTimeline.json create mode 100644 lexicons/org/tangled/feed/listComments.json create mode 100644 lexicons/org/tangled/feed/listCommentsBy.json create mode 100644 lexicons/org/tangled/feed/listReactions.json create mode 100644 lexicons/org/tangled/feed/listReactionsBy.json create mode 100644 lexicons/org/tangled/feed/listStars.json create mode 100644 lexicons/org/tangled/feed/listStarsBy.json create mode 100644 lexicons/org/tangled/feed/reaction.json create mode 100644 lexicons/org/tangled/feed/star.json create mode 100644 lexicons/org/tangled/feed/subscription.json create mode 100644 lexicons/org/tangled/git/defs.json create mode 100644 lexicons/org/tangled/git/deleteBranch.json create mode 100644 lexicons/org/tangled/git/getArchive.json create mode 100644 lexicons/org/tangled/git/getBlame.json create mode 100644 lexicons/org/tangled/git/getBlob.json create mode 100644 lexicons/org/tangled/git/getBranch.json create mode 100644 lexicons/org/tangled/git/getDiff.json create mode 100644 lexicons/org/tangled/git/getEntry.json create mode 100644 lexicons/org/tangled/git/getTag.json create mode 100644 lexicons/org/tangled/git/getTree.json create mode 100644 lexicons/org/tangled/git/listBranches.json create mode 100644 lexicons/org/tangled/git/listCommits.json create mode 100644 lexicons/org/tangled/git/listLanguages.json create mode 100644 lexicons/org/tangled/git/listRefs.json create mode 100644 lexicons/org/tangled/git/listTags.json create mode 100644 lexicons/org/tangled/git/oid.json create mode 100644 lexicons/org/tangled/git/pushEvent.json create mode 100644 lexicons/org/tangled/git/receivePack.json create mode 100644 lexicons/org/tangled/git/refs.json create mode 100644 lexicons/org/tangled/git/setDefaultBranch.json create mode 100644 lexicons/org/tangled/graph/attestation.json create mode 100644 lexicons/org/tangled/graph/collaborator.json create mode 100644 lexicons/org/tangled/graph/countFollows.json create mode 100644 lexicons/org/tangled/graph/countFollowsBy.json create mode 100644 lexicons/org/tangled/graph/follow.json create mode 100644 lexicons/org/tangled/graph/getFollow.json create mode 100644 lexicons/org/tangled/graph/listFollows.json create mode 100644 lexicons/org/tangled/graph/listFollowsBy.json create mode 100644 lexicons/org/tangled/graph/listNetworkVouches.json create mode 100644 lexicons/org/tangled/graph/vouch.json create mode 100644 lexicons/org/tangled/key/sshKey.json create mode 100644 lexicons/org/tangled/key/unspecced/gpgKey.json create mode 100644 lexicons/org/tangled/knot/block.json create mode 100644 lexicons/org/tangled/knot/describeKnot.json create mode 100644 lexicons/org/tangled/knot/member.json create mode 100644 lexicons/org/tangled/label/countDefinitions.json create mode 100644 lexicons/org/tangled/label/definition.json create mode 100644 lexicons/org/tangled/label/defs.json create mode 100644 lexicons/org/tangled/label/listDefinitions.json create mode 100644 lexicons/org/tangled/markup/embed/image.json create mode 100644 lexicons/org/tangled/markup/embed/video.json create mode 100644 lexicons/org/tangled/markup/markdown.json create mode 100644 lexicons/org/tangled/markup/plaintext.json create mode 100644 lexicons/org/tangled/publicKey/countKeys.json create mode 100644 lexicons/org/tangled/publicKey/getPublicKey.json create mode 100644 lexicons/org/tangled/publicKey/listKeys.json create mode 100644 lexicons/org/tangled/query/enrichResponse.json create mode 100644 lexicons/org/tangled/repo/artifact.json create mode 100644 lexicons/org/tangled/repo/countArtifacts.json create mode 100644 lexicons/org/tangled/repo/countArtifactsBy.json create mode 100644 lexicons/org/tangled/repo/countCollaborators.json create mode 100644 lexicons/org/tangled/repo/countCollaboratorsBy.json create mode 100644 lexicons/org/tangled/repo/countForks.json create mode 100644 lexicons/org/tangled/repo/countIssues.json create mode 100644 lexicons/org/tangled/repo/countPulls.json create mode 100644 lexicons/org/tangled/repo/countRepos.json create mode 100644 lexicons/org/tangled/repo/createRepo.json create mode 100644 lexicons/org/tangled/repo/declaration.json create mode 100644 lexicons/org/tangled/repo/defs.json create mode 100644 lexicons/org/tangled/repo/deleteRepo.json create mode 100644 lexicons/org/tangled/repo/forkSync.json create mode 100644 lexicons/org/tangled/repo/getImportStatus.json create mode 100644 lexicons/org/tangled/repo/getRepo.json create mode 100644 lexicons/org/tangled/repo/getRepoByName.json create mode 100644 lexicons/org/tangled/repo/getRepoByRepoDid.json create mode 100644 lexicons/org/tangled/repo/getRepos.json create mode 100644 lexicons/org/tangled/repo/getReposByRepoDids.json create mode 100644 lexicons/org/tangled/repo/listArtifacts.json create mode 100644 lexicons/org/tangled/repo/listArtifactsBy.json create mode 100644 lexicons/org/tangled/repo/listCollaboratorInvitesBy.json create mode 100644 lexicons/org/tangled/repo/listCollaborators.json create mode 100644 lexicons/org/tangled/repo/listCollaboratorsBy.json create mode 100644 lexicons/org/tangled/repo/listForks.json create mode 100644 lexicons/org/tangled/repo/listIssues.json create mode 100644 lexicons/org/tangled/repo/listIssuesBy.json create mode 100644 lexicons/org/tangled/repo/listPulls.json create mode 100644 lexicons/org/tangled/repo/listPullsBy.json create mode 100644 lexicons/org/tangled/repo/listRecordVersions.json create mode 100644 lexicons/org/tangled/repo/listRepos.json create mode 100644 lexicons/org/tangled/repo/manifest.json create mode 100644 lexicons/org/tangled/repo/mintToken.json create mode 100644 lexicons/org/tangled/repo/reserveKey.json create mode 100644 lexicons/org/tangled/review/patch.json create mode 100644 lexicons/org/tangled/review/submitPatch.json create mode 100644 lexicons/org/tangled/review/unspecced/defs.json create mode 100644 lexicons/org/tangled/review/unspecced/round.json create mode 100644 lexicons/org/tangled/search/query.json create mode 100644 lexicons/org/tangled/secret/addSecret.json create mode 100644 lexicons/org/tangled/secret/listSecrets.json create mode 100644 lexicons/org/tangled/secret/removeSecret.json create mode 100644 lexicons/org/tangled/snippet/string.json create mode 100644 lexicons/org/tangled/spindle/describeSpindle.json create mode 100644 lexicons/org/tangled/string/countStrings.json create mode 100644 lexicons/org/tangled/string/getString.json create mode 100644 lexicons/org/tangled/string/listStrings.json create mode 100644 lexicons/org/tangled/temp/actor/defs.json create mode 100644 lexicons/org/tangled/temp/git/deleteBranch.json create mode 100644 lexicons/org/tangled/temp/git/formatPatch.json create mode 100644 lexicons/org/tangled/temp/git/getArchive.json create mode 100644 lexicons/org/tangled/temp/git/getBlob.json create mode 100644 lexicons/org/tangled/temp/git/getBranch.json create mode 100644 lexicons/org/tangled/temp/git/getEntry.json create mode 100644 lexicons/org/tangled/temp/git/getMergeBase.json create mode 100644 lexicons/org/tangled/temp/git/getRef.json create mode 100644 lexicons/org/tangled/temp/git/getTag.json create mode 100644 lexicons/org/tangled/temp/git/getTree.json create mode 100644 lexicons/org/tangled/temp/git/listBranches.json create mode 100644 lexicons/org/tangled/temp/git/listCommits.json create mode 100644 lexicons/org/tangled/temp/git/listCommitsByRef.json create mode 100644 lexicons/org/tangled/temp/git/listTags.json create mode 100644 lexicons/org/tangled/temp/git/mergeCommit.json create mode 100644 lexicons/org/tangled/temp/git/syncFork.json create mode 100644 lexicons/org/tangled/temp/repo/getRepoByName.json create mode 100644 lexicons/org/tangled/temp/repo/listRecordOrigins.json create mode 100644 lexicons/org/tangled/temp/server/ackHandover.json create mode 100644 lexicons/org/tangled/temp/server/cancelHandover.json create mode 100644 lexicons/org/tangled/temp/server/createInviteCode.json create mode 100644 lexicons/org/tangled/temp/server/handoverRepo.json create mode 100644 lexicons/org/tangled/temp/server/redeemInviteCode.json create mode 100644 lexicons/org/tangled/temp/server/seedRepo.json create mode 100644 lexicons/org/tangled/temp/server/uploadBlob.json create mode 100644 lexicons/org/tangled/temp/spindle/allowList/add.json create mode 100644 lexicons/org/tangled/temp/spindle/allowList/list.json create mode 100644 lexicons/org/tangled/temp/spindle/allowList/remove.json create mode 100644 lexicons/org/tangled/track/defs.json create mode 100644 lexicons/org/tangled/track/getTicketView.json create mode 100644 lexicons/org/tangled/track/ticket.json create mode 100644 lexicons/org/tangled/track/unspecced/milestone.json create mode 100644 lexicons/org/tangled/webhook/createWebhook.json create mode 100644 lexicons/org/tangled/webhook/defs.json create mode 100644 lexicons/org/tangled/webhook/deleteWebhook.json create mode 100644 lexicons/org/tangled/webhook/listWebhookDeliveries.json create mode 100644 lexicons/org/tangled/webhook/listWebhooks.json create mode 100644 lexicons/org/tangled/webhook/retryWebhookDelivery.json create mode 100644 lexicons/org/tangled/webhook/updateWebhook.json create mode 100644 lexicons/repo/listCobVersions.json create mode 100644 lexicons/temp/account/addEmail.json create mode 100644 lexicons/temp/account/beginSignup.json create mode 100644 lexicons/temp/account/completeSignup.json create mode 100644 lexicons/temp/account/deleteEmail.json create mode 100644 lexicons/temp/account/dismissNewsletter.json create mode 100644 lexicons/temp/account/getNewsletterPreference.json create mode 100644 lexicons/temp/account/getSignup.json create mode 100644 lexicons/temp/account/listEmails.json create mode 100644 lexicons/temp/account/resendSignup.json create mode 100644 lexicons/temp/account/setPrimaryEmail.json create mode 100644 lexicons/temp/account/subscribeNewsletter.json create mode 100644 lexicons/temp/account/verifyEmail.json create mode 100644 lexicons/temp/focus/beginSession.json create mode 100644 lexicons/temp/focus/endSession.json create mode 100644 lexicons/temp/focus/nextItem.json create mode 100644 lexicons/temp/migrator/createTask.json create mode 100644 lexicons/temp/migrator/defs.json create mode 100644 lexicons/temp/migrator/describeSource.json create mode 100644 lexicons/temp/migrator/getTask.json create mode 100644 lexicons/temp/migrator/listTasks.json create mode 100644 lexicons/temp/migrator/retryJob.json create mode 100644 lexicons/temp/newsletter/signup.json create mode 100644 lexicons/temp/notification/deleteNotification.json create mode 100644 lexicons/temp/notification/getPreferences.json create mode 100644 lexicons/temp/notification/getUnreadCount.json create mode 100644 lexicons/temp/notification/listNotifications.json create mode 100644 lexicons/temp/notification/listRecipients.json create mode 100644 lexicons/temp/notification/markAllRead.json create mode 100644 lexicons/temp/notification/markEntityRead.json create mode 100644 lexicons/temp/notification/updatePreferences.json create mode 100644 lexicons/temp/notification/updateSeen.json create mode 100644 lexicons/temp/repo/deploySite.json create mode 100644 lexicons/temp/repo/disableSite.json create mode 100644 lexicons/temp/repo/getDeployHistory.json create mode 100644 lexicons/temp/repo/getSiteConfig.json create mode 100644 lexicons/temp/repo/updateSiteConfig.json create mode 100644 lexicons/temp/review/listPatches.json create mode 100644 lexicons/temp/search/searchCode.json create mode 100644 lexicons/temp/site/claimDomain.json create mode 100644 lexicons/temp/site/getDomainClaim.json create mode 100644 lexicons/temp/site/releaseDomain.json create mode 100644 lexicons/temp/spindle/moderation/ban.json create mode 100644 lexicons/temp/spindle/moderation/defs.json create mode 100644 lexicons/temp/spindle/moderation/getBan.json create mode 100644 lexicons/temp/spindle/moderation/listBans.json create mode 100644 lexicons/temp/spindle/moderation/unban.json create mode 100644 lexicons/temp/spindle/quota/defs.json create mode 100644 lexicons/temp/spindle/quota/get.json create mode 100644 lexicons/temp/spindle/quota/list.json create mode 100644 lexicons/temp/spindle/quota/releaseNars.json create mode 100644 lexicons/temp/spindle/quota/set.json create mode 100644 lexicons/temp/spindle/quota/unset.json create mode 100644 lexicons/temp/spindle/quota/usage.json create mode 100644 lexutil/client.go create mode 100644 lexutil/client_test.go create mode 100644 lexutil/lexutil.go create mode 100644 license create mode 100644 localinfra/cmd/deliberi-bootstrap/main.go create mode 100644 localinfra/cmd/deliberi-bootstrap/main_test.go create mode 100644 localinfra/cmd/seed/main.go create mode 100644 localinfra/fixtures/blobs/did:plc:igzuueb63dytyy3tpaweofuu/bafkreieify5bix4cmrj4iuxiceb6srr2boltkbbf7yizwtgwurqv42sltm create mode 100644 localinfra/fixtures/blobs/did:plc:igzuueb63dytyy3tpaweofuu/bafkreifqrrwn3ioqkrquhibb3csu6i3kogd76uk6ktptc7s5exim6rfu6u create mode 100644 localinfra/fixtures/blobs/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafkreiaovmar5rtz7dw35cfylzmb3htqg7kpksf76pvascxlv3s56rb7o4 create mode 100644 localinfra/fixtures/records/did:plc:77uzwanh7ixsvttf6cvkoszj/org.tangled.git.refs/self/bafyreig6c2mmv2wctety2ngvjho6rx7dbvmlrbuzfybf5ygdxxqhpjdef4.json create mode 100644 localinfra/fixtures/records/did:plc:77uzwanh7ixsvttf6cvkoszj/org.tangled.repo.manifest/self/bafyreid3zlshhjzryrysr3egr3yutd26fsmq2kfdaibiccx7sq57qwvuty.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3lzg6rlrui222/bafyreietwvvhovqaxq2llr35tg7fslxikhjfffulmovin3q6ngfbsyqkle.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3lzg6teywa222/bafyreigsfex4ezvor5tig2u2fvh7dwfxanrpu2t5a6iut2pf2h6tov5z7i.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqk2a4c22/bafyreibxtmildrcsuhjvijs2kbcotijs6qhu2rsbm3rleskancgxb5mk6y.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqm7b6d22/bafyreih3mcgo3rtzqxmf2inhabpmw4i4n7bev7wadlaoahfwnivogujupy.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqp3c5e22/bafyreicbbu3wdenayqq4ubqflygph5msxghmuw7buyph3dbq6btlkwoi3y.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqr5d2f22/bafyreic2qpppocpszrr7nlysndsvhum5x3d57wllrxbelmrhgvp6lfvnli.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mwkb2qu3gr22/bafyreiagzsjco3xpkp7wrek4sltctxu4ub4v6dtglfjwdlyfvnjc6ulmpi.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg6yqo3i222/bafyreibmpb4deiyux2bm2m5j7ybz2ufvgnszzgnwtd66d3qtoenla7ohma.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg72jv5a222/bafyreib3rqmzxclxnqnkdwrzp7wcnndbp46g7acfhhque5i2y53ndek4rm.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg74d46y222/bafyreiau7g54kz5ftdos3r2kldwcjutvsvmdh7hptnx7bmwhprvoeb75fi.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.git.refs/self/bafyreigkwkkfyqmuwvl5zx552ivpes3epirw6dpsvzo546kc4agy7r4gtm.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/documentation/bafyreif5mbys4b7mn4rquma3gqqnr3ldjgpspdkfqfu3feolsrftnxznne.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/duplicate/bafyreie224tuyp3mrrp5cosezrcmdxgutjctoresjsv3r4pfbwvlnqyre4.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/good-first-issue/bafyreiguf6otpa3jlctujcnvfhqt6thqx76ol5sqmanzhkiru24kdgple4.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/wontfix/bafyreihhxjm2coiw242usm6hnid3i7ius6kdkkyph7afjc7jjp2fb7bz6e.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.repo.manifest/self/bafyreie6djkwfaeyrpulc373tkf4hoimhfc5t2jdrmsgqkblalrcl52y7a.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreiarut2cqseyjfwefvssd4pnbyqjtzl5n4nc5gqbgpbsdekdf3mfwu/bafyreiarut2cqseyjfwefvssd4pnbyqjtzl5n4nc5gqbgpbsdekdf3mfwu.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreicbcuncwl73zhwlwxtgqfj3o72mlgdp3ncfvvolfwce535kitqwni/bafyreicbcuncwl73zhwlwxtgqfj3o72mlgdp3ncfvvolfwce535kitqwni.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreidyg53s7fyjbh63abeog2h2g7vg5g67shjkk2hukto76jo4gjgb3u/bafyreidyg53s7fyjbh63abeog2h2g7vg5g67shjkk2hukto76jo4gjgb3u.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreifhetuhjxfajktrkc5wc4a3ikmg5ytard3b6g5plli3t3p5rkbuf4/bafyreifhetuhjxfajktrkc5wc4a3ikmg5ytard3b6g5plli3t3p5rkbuf4.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreig4rj2sbb3jjo6l2djoqzlim6a32vc7jfpa7pxub5ctv2c7eveavm/bafyreig4rj2sbb3jjo6l2djoqzlim6a32vc7jfpa7pxub5ctv2c7eveavm.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreiglruqe4czsrzewu6yy2evebgr6yv5ajgdztxoodaihsoddfn3g4u/bafyreiglruqe4czsrzewu6yy2evebgr6yv5ajgdztxoodaihsoddfn3g4u.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreignfuruuizasc4cinen3s7cdmdyczm6t37y6eof36aoobmeclv2tq/bafyreignfuruuizasc4cinen3s7cdmdyczm6t37y6eof36aoobmeclv2tq.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreigsqm243ma2cjtfwcmulqkyvcdrrbakd5amj2vswxcoukgsl7bo3a/bafyreigsqm243ma2cjtfwcmulqkyvcdrrbakd5amj2vswxcoukgsl7bo3a.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/1/bafyreidbxu22o4wwg2io5yd26ju4uadsdq3ikpff2hofr5cgnah356mnhu.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/2/bafyreig33fgtw2h2lwz7uex4yepqgq5kez64vm56trqlhsnrup37am6r2q.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/3/bafyreie3gbwyh7d5ctgicimydbxyvini2xvmuux4ambboqyrv24nnuvcee.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/4/bafyreiedbneftlnpu47xjsqkxtncjxoe324kpklmdd4qnttvuptadfjh2y.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/5/bafyreidjm7iv7dstbkm5q5gt4haea6qxekd4lbu2ant2c3m5yqucqyz2wi.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/6/bafyreieopiuug56i4hcivpqrlgt4govlivo6iy6id3vshljqgt4awnof4u.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/7/bafyreign52t3km5rd6l2x6aulj6hixoojfmifrdjbic2coo7musy6gutla.json create mode 100644 localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/8/bafyreidv3qgzq3mm53gfcxfep5ztun3tjfkhpm54odte3gjh2j6oiwko4y.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.feed.comment/3lzg6wxgzq222/bafyreihkqylhqxdmt2je5qiw6kfsgov7rh4h4fdzx4xqhk5q3wosuk7gbi.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.feed.reaction/3lzg764daq222/bafyreifgmnafntxfnfdx4gmhhdxqdolkwxycugm3hhppjh5d3ht4ww3umi.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7splszs22/bafyreib76tnkofhqjhrcgox4etvcptekjj7aa5d5jmpniljh3hfqe5zt24.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7spnnms22/bafyreichv6dkuk3fmgpcwuevdxhapoginh2snmu4vuyuwzkvjvnmfnckm4.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sppi7s22/bafyreihz5nhffleuard4bunztrrxqhr3qkbvqwsscrvnukpjqedszx7tnq.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sr3ghs22/bafyreiaencwsxfzy3o7uxput3prbhbkyyz7r4pf3r6ejwu6nqoj4u5ehmq.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sr73ns22/bafyreifbgyq24ub4mphl3pmt4toeiq4u46fbqtdhos54dmvcuqckjqkhkq.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7srcqts22/bafyreif7u22hndb4el5xkgohfym2fp7oanx4cbvkjubatnxrdkpl7n62zm.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223nzzsbaeu22/bafyreigtl4u36d3uy5ymeycyxy3eomtcgheevpsnd6vevshtihb65cwqwe.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.follow/did:plc:pgkgoz6rpvmgynswviohltpx/bafyreibzfpdma3syj27j42juei6wcmh2fpjxqp23urpivj4kp6xnccl6pa.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.follow/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreifl7dsgyvu7fvkl4l5umvrrewoatpia5hgjuckybf3tfk4y5bnm3i.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.vouch/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreifm5ul6hhiesuuhukw45gol7z3w67ktf27dlvluu2h4a6m54fx3ga.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.repo.declaration/core/bafyreigpl5grnco2z7nkadnag57vntcoc7ye2oq22fgsqirkjgwbgv6gx4.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.snippet.string/3lzg6ma4pa222/bafyreiaskfifni2inl42xzbei2thp7wdxepgzeitnmv2aj7ww7sc5wq3ne.json create mode 100644 localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.snippet.string/3lzg6nzdqy222/bafyreifweqc5cgslk56emxrcqisv6ctlza4mfkhxdcs5iuba3t6kfagh3y.json create mode 100644 localinfra/fixtures/records/did:plc:pgkgoz6rpvmgynswviohltpx/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreifuy2yp7w7c6z3frzi2t7rxlvovw3cz6ivk5l2d7omuptcfje7a6i.json create mode 100644 localinfra/fixtures/records/did:plc:pgkgoz6rpvmgynswviohltpx/org.tangled.graph.vouch/did:plc:ryxq2u2b6sadhhjflpwiefg4/bafyreig2c6t2yhkdxuupyhouevcgx7kbay4b4nj2og5qlpfjh6sugrdux4.json create mode 100644 localinfra/fixtures/records/did:plc:qbqdycrgcna6n7w5lskwod6y/org.tangled.git.refs/self/bafyreigkwkkfyqmuwvl5zx552ivpes3epirw6dpsvzo546kc4agy7r4gtm.json create mode 100644 localinfra/fixtures/records/did:plc:qbqdycrgcna6n7w5lskwod6y/org.tangled.repo.manifest/self/bafyreic6d75ygy5fdaoonunl4v5ki6nyazawttswpas3giafc7dgd7uuza.json create mode 100644 localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreicpsiuzccktcxgcp4cl6mshfgdicv45ple4hrxrrgd7b3t4masruy.json create mode 100644 localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.follow/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreiewemtnmq2h2nugt2sbm4xavv7jqzvec6hbwqypinudwonkoukvti.json create mode 100644 localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.vouch/did:plc:igzuueb63dytyy3tpaweofuu/bafyreiemubhyfd7wu4louq2och6fomjpne7kleyfvt75s5si3px76gdcoy.json create mode 100644 localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.feed.comment/3lzg6v67xy222/bafyreic35k7ygx5jg5mxqcu2x56ptgvynt4eb3g535fuf5azyfranlxok4.json create mode 100644 localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.attestation/223n7srelgs22/bafyreiegqxgcgfs3vlusn6cky7p7kjgl5eb3xxwye3kk6ajbtvb2lpxc5m.json create mode 100644 localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreiec5mggcrtwhdlkpymftts44zumyz5m6qwccqrpt4gihmnu652qae.json create mode 100644 localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.vouch/did:plc:pgkgoz6rpvmgynswviohltpx/bafyreif7tywomyuzgtcawmppnrnmicfzcty5d42kjz24igyktw6ekxfhpa.json create mode 100644 localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.vouch/did:plc:ryxq2u2b6sadhhjflpwiefg4/bafyreic7neuxz42d4373k5xdda7b66ycxnt6ct6w5gyfzb6mlozgbqxlg4.json create mode 100644 localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.repo.declaration/core-fork/bafyreiacqdhafvv7x4aefu4zyheqhyk24dbxtlcqxowgfpz5g2wpeclkgm.json create mode 100644 localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.repo.declaration/empty-repo/bafyreic2ngxwckzpfk7tlzxofcfagfld3dg2qslubaewelbpkoz5qsepcy.json create mode 100644 localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.snippet.string/3lzg6psksq222/bafyreieohwiytxwzkuczdrm35hxpdm2h2jyuffg2fatpxpo7iyxfnuk5ka.json create mode 100644 localinfra/github-stub/__init__.py create mode 100644 localinfra/github-stub/__main__.py create mode 100644 localinfra/github-stub/server.py create mode 100644 localinfra/ncps-secret-key create mode 100644 localinfra/ncps.yaml create mode 100644 localinfra/observability/grafana/provisioning/dashboards/bobbin.json create mode 100644 localinfra/observability/grafana/provisioning/dashboards/spindle.json create mode 100644 localinfra/observability/grafana/provisioning/dashboards/spindle.yml create mode 100644 localinfra/observability/grafana/provisioning/datasources/prometheus.yml create mode 100644 localinfra/observability/loki/loki.yml create mode 100644 localinfra/observability/prometheus/prometheus.yml create mode 100644 localinfra/observability/tempo/tempo.yml create mode 100644 localinfra/ozone.env create mode 100644 localinfra/pc/Caddyfile create mode 100644 localinfra/pc/README.md create mode 100644 localinfra/pc/airgap.sb create mode 100755 localinfra/pc/bin/doctor create mode 100644 localinfra/pc/bin/env.sh create mode 100755 localinfra/pc/bin/linux-ns create mode 100755 localinfra/pc/bin/ns-inner create mode 100755 localinfra/pc/bin/offline create mode 100755 localinfra/pc/bin/patch-workerd create mode 100755 localinfra/pc/bin/pc create mode 100755 localinfra/pc/bin/profile-on create mode 100755 localinfra/pc/bin/selftest create mode 100755 localinfra/pc/bin/services create mode 100755 localinfra/pc/bin/setup create mode 100755 localinfra/pc/bin/svc/caddy create mode 100755 localinfra/pc/bin/svc/dns create mode 100755 localinfra/pc/bin/svc/hydrant create mode 100755 localinfra/pc/bin/svc/init-data create mode 100755 localinfra/pc/bin/svc/knot2 create mode 100755 localinfra/pc/bin/svc/pds create mode 100755 localinfra/pc/bin/svc/pg-init create mode 100755 localinfra/pc/bin/svc/plc create mode 100755 localinfra/pc/bin/svc/postgres create mode 100755 localinfra/pc/bin/tcp-up create mode 100755 localinfra/pc/bin/up create mode 100644 localinfra/pc/commands.env create mode 100644 localinfra/pc/ports.env create mode 100644 localinfra/pc/process-compose.yaml create mode 100644 localinfra/pc/services/README.md create mode 100644 localinfra/pc/services/avatar/Caddyfile create mode 100644 localinfra/pc/services/avatar/commands.env create mode 100644 localinfra/pc/services/avatar/doctor create mode 100644 localinfra/pc/services/avatar/packages create mode 100644 localinfra/pc/services/avatar/ports.env create mode 100644 localinfra/pc/services/avatar/process.yaml create mode 100755 localinfra/pc/services/avatar/run create mode 100755 localinfra/pc/services/avatar/setup create mode 100644 localinfra/pc/services/bobbin/Caddyfile create mode 100644 localinfra/pc/services/bobbin/commands.env create mode 100644 localinfra/pc/services/bobbin/doctor create mode 100644 localinfra/pc/services/bobbin/ports.env create mode 100644 localinfra/pc/services/bobbin/process.yaml create mode 100755 localinfra/pc/services/bobbin/run create mode 100755 localinfra/pc/services/bobbin/setup create mode 100644 localinfra/pc/services/camo/Caddyfile create mode 100644 localinfra/pc/services/camo/commands.env create mode 100644 localinfra/pc/services/camo/doctor create mode 100644 localinfra/pc/services/camo/packages create mode 100644 localinfra/pc/services/camo/ports.env create mode 100644 localinfra/pc/services/camo/process.yaml create mode 100755 localinfra/pc/services/camo/run create mode 100755 localinfra/pc/services/camo/setup create mode 100644 localinfra/pc/services/deliberi/Caddyfile create mode 100644 localinfra/pc/services/deliberi/commands.env create mode 100644 localinfra/pc/services/deliberi/doctor create mode 100644 localinfra/pc/services/deliberi/packages create mode 100644 localinfra/pc/services/deliberi/ports.env create mode 100644 localinfra/pc/services/deliberi/process.yaml create mode 100755 localinfra/pc/services/deliberi/run create mode 100755 localinfra/pc/services/deliberi/setup create mode 100644 localinfra/pc/services/github-stub/Caddyfile create mode 100644 localinfra/pc/services/github-stub/commands.env create mode 100644 localinfra/pc/services/github-stub/doctor create mode 100644 localinfra/pc/services/github-stub/packages create mode 100644 localinfra/pc/services/github-stub/ports.env create mode 100644 localinfra/pc/services/github-stub/process.yaml create mode 100755 localinfra/pc/services/github-stub/run create mode 100755 localinfra/pc/services/github-stub/setup create mode 100644 localinfra/pc/services/grafana/commands.env create mode 100644 localinfra/pc/services/grafana/doctor create mode 100644 localinfra/pc/services/grafana/packages create mode 100644 localinfra/pc/services/grafana/ports.env create mode 100644 localinfra/pc/services/grafana/process.yaml create mode 100644 localinfra/pc/services/grafana/profile create mode 100755 localinfra/pc/services/grafana/run create mode 100644 localinfra/pc/services/knotmirror/Caddyfile create mode 100644 localinfra/pc/services/knotmirror/commands.env create mode 100644 localinfra/pc/services/knotmirror/doctor create mode 100644 localinfra/pc/services/knotmirror/packages create mode 100644 localinfra/pc/services/knotmirror/ports.env create mode 100644 localinfra/pc/services/knotmirror/process.yaml create mode 100755 localinfra/pc/services/knotmirror/run create mode 100755 localinfra/pc/services/knotmirror/setup create mode 100644 localinfra/pc/services/loki/commands.env create mode 100644 localinfra/pc/services/loki/doctor create mode 100644 localinfra/pc/services/loki/packages create mode 100644 localinfra/pc/services/loki/ports.env create mode 100644 localinfra/pc/services/loki/process.yaml create mode 100644 localinfra/pc/services/loki/profile create mode 100755 localinfra/pc/services/loki/run create mode 100644 localinfra/pc/services/migrator/Caddyfile create mode 100644 localinfra/pc/services/migrator/commands.env create mode 100644 localinfra/pc/services/migrator/doctor create mode 100644 localinfra/pc/services/migrator/packages create mode 100644 localinfra/pc/services/migrator/ports.env create mode 100644 localinfra/pc/services/migrator/process.yaml create mode 100755 localinfra/pc/services/migrator/run create mode 100755 localinfra/pc/services/migrator/setup create mode 100644 localinfra/pc/services/ncps-migrate/commands.env create mode 100644 localinfra/pc/services/ncps-migrate/doctor create mode 100644 localinfra/pc/services/ncps-migrate/packages create mode 100644 localinfra/pc/services/ncps-migrate/process.yaml create mode 100644 localinfra/pc/services/ncps-migrate/profile create mode 100755 localinfra/pc/services/ncps-migrate/run create mode 100644 localinfra/pc/services/ncps/commands.env create mode 100644 localinfra/pc/services/ncps/doctor create mode 100644 localinfra/pc/services/ncps/packages create mode 100644 localinfra/pc/services/ncps/ports.env create mode 100644 localinfra/pc/services/ncps/process.yaml create mode 100644 localinfra/pc/services/ncps/profile create mode 100755 localinfra/pc/services/ncps/run create mode 100644 localinfra/pc/services/ogre/Caddyfile create mode 100644 localinfra/pc/services/ogre/commands.env create mode 100644 localinfra/pc/services/ogre/doctor create mode 100644 localinfra/pc/services/ogre/packages create mode 100644 localinfra/pc/services/ogre/ports.env create mode 100644 localinfra/pc/services/ogre/process.yaml create mode 100755 localinfra/pc/services/ogre/run create mode 100755 localinfra/pc/services/ogre/setup create mode 100644 localinfra/pc/services/otlp-gateway/commands.env create mode 100644 localinfra/pc/services/otlp-gateway/doctor create mode 100644 localinfra/pc/services/otlp-gateway/packages create mode 100644 localinfra/pc/services/otlp-gateway/ports.env create mode 100644 localinfra/pc/services/otlp-gateway/process.yaml create mode 100644 localinfra/pc/services/otlp-gateway/profile create mode 100755 localinfra/pc/services/otlp-gateway/run create mode 100644 localinfra/pc/services/ozone/Caddyfile create mode 100644 localinfra/pc/services/ozone/commands.env create mode 100644 localinfra/pc/services/ozone/doctor create mode 100644 localinfra/pc/services/ozone/packages create mode 100644 localinfra/pc/services/ozone/ports.env create mode 100644 localinfra/pc/services/ozone/process.yaml create mode 100755 localinfra/pc/services/ozone/run create mode 100755 localinfra/pc/services/ozone/setup create mode 100644 localinfra/pc/services/pdsls/Caddyfile create mode 100644 localinfra/pc/services/pdsls/commands.env create mode 100644 localinfra/pc/services/pdsls/doctor create mode 100644 localinfra/pc/services/pdsls/packages create mode 100644 localinfra/pc/services/pdsls/ports.env create mode 100644 localinfra/pc/services/pdsls/process.yaml create mode 100644 localinfra/pc/services/pdsls/profile create mode 100755 localinfra/pc/services/pdsls/run create mode 100755 localinfra/pc/services/pdsls/setup create mode 100644 localinfra/pc/services/pocket/Caddyfile create mode 100644 localinfra/pc/services/pocket/bind.patch create mode 100644 localinfra/pc/services/pocket/commands.env create mode 100644 localinfra/pc/services/pocket/doctor create mode 100644 localinfra/pc/services/pocket/packages create mode 100644 localinfra/pc/services/pocket/ports.env create mode 100644 localinfra/pc/services/pocket/process.yaml create mode 100755 localinfra/pc/services/pocket/run create mode 100755 localinfra/pc/services/pocket/setup create mode 100644 localinfra/pc/services/prometheus/commands.env create mode 100644 localinfra/pc/services/prometheus/doctor create mode 100644 localinfra/pc/services/prometheus/packages create mode 100644 localinfra/pc/services/prometheus/ports.env create mode 100644 localinfra/pc/services/prometheus/process.yaml create mode 100644 localinfra/pc/services/prometheus/profile create mode 100755 localinfra/pc/services/prometheus/run create mode 100644 localinfra/pc/services/redis/commands.env create mode 100644 localinfra/pc/services/redis/doctor create mode 100644 localinfra/pc/services/redis/packages create mode 100644 localinfra/pc/services/redis/ports.env create mode 100644 localinfra/pc/services/redis/process.yaml create mode 100755 localinfra/pc/services/redis/run create mode 100644 localinfra/pc/services/sites/Caddyfile create mode 100644 localinfra/pc/services/sites/commands.env create mode 100644 localinfra/pc/services/sites/doctor create mode 100644 localinfra/pc/services/sites/packages create mode 100644 localinfra/pc/services/sites/ports.env create mode 100644 localinfra/pc/services/sites/process.yaml create mode 100644 localinfra/pc/services/sites/profile create mode 100755 localinfra/pc/services/sites/run create mode 100755 localinfra/pc/services/sites/setup create mode 100644 localinfra/pc/services/sitesd/Caddyfile create mode 100644 localinfra/pc/services/sitesd/commands.env create mode 100644 localinfra/pc/services/sitesd/doctor create mode 100644 localinfra/pc/services/sitesd/ports.env create mode 100644 localinfra/pc/services/sitesd/process.yaml create mode 100644 localinfra/pc/services/sitesd/profile create mode 100755 localinfra/pc/services/sitesd/run create mode 100755 localinfra/pc/services/sitesd/setup create mode 100644 localinfra/pc/services/spindle/Caddyfile create mode 100644 localinfra/pc/services/spindle/commands.env create mode 100644 localinfra/pc/services/spindle/doctor create mode 100644 localinfra/pc/services/spindle/ports.env create mode 100644 localinfra/pc/services/spindle/process.yaml create mode 100644 localinfra/pc/services/spindle/profile create mode 100755 localinfra/pc/services/spindle/run create mode 100644 localinfra/pc/services/spindle/setup create mode 100644 localinfra/pc/services/tempo/commands.env create mode 100644 localinfra/pc/services/tempo/doctor create mode 100644 localinfra/pc/services/tempo/packages create mode 100644 localinfra/pc/services/tempo/ports.env create mode 100644 localinfra/pc/services/tempo/process.yaml create mode 100644 localinfra/pc/services/tempo/profile create mode 100755 localinfra/pc/services/tempo/run create mode 100644 localinfra/pc/services/tranquil-gate/Caddyfile create mode 100644 localinfra/pc/services/tranquil-gate/commands.env create mode 100644 localinfra/pc/services/tranquil-gate/doctor create mode 100644 localinfra/pc/services/tranquil-gate/packages create mode 100644 localinfra/pc/services/tranquil-gate/ports.env create mode 100644 localinfra/pc/services/tranquil-gate/process.yaml create mode 100755 localinfra/pc/services/tranquil-gate/run create mode 100755 localinfra/pc/services/tranquil-gate/setup create mode 100644 localinfra/pc/services/tranquil/Caddyfile create mode 100644 localinfra/pc/services/tranquil/commands.env create mode 100644 localinfra/pc/services/tranquil/doctor create mode 100644 localinfra/pc/services/tranquil/packages create mode 100644 localinfra/pc/services/tranquil/ports.env create mode 100644 localinfra/pc/services/tranquil/process.yaml create mode 100755 localinfra/pc/services/tranquil/run create mode 100755 localinfra/pc/services/tranquil/setup create mode 100644 localinfra/pc/services/web/Caddyfile create mode 100644 localinfra/pc/services/web/commands.env create mode 100644 localinfra/pc/services/web/doctor create mode 100644 localinfra/pc/services/web/packages create mode 100644 localinfra/pc/services/web/ports.env create mode 100644 localinfra/pc/services/web/process.yaml create mode 100755 localinfra/pc/services/web/run create mode 100755 localinfra/pc/services/web/setup create mode 100644 localinfra/pc/services/web/web.env create mode 100644 localinfra/pc/services/zoekt-tngl-indexserver/commands.env create mode 100644 localinfra/pc/services/zoekt-tngl-indexserver/doctor create mode 100644 localinfra/pc/services/zoekt-tngl-indexserver/packages create mode 100644 localinfra/pc/services/zoekt-tngl-indexserver/ports.env create mode 100644 localinfra/pc/services/zoekt-tngl-indexserver/process.yaml create mode 100755 localinfra/pc/services/zoekt-tngl-indexserver/run create mode 100755 localinfra/pc/services/zoekt-tngl-indexserver/setup create mode 100644 localinfra/pc/services/zoekt-webserver/Caddyfile create mode 100644 localinfra/pc/services/zoekt-webserver/commands.env create mode 100644 localinfra/pc/services/zoekt-webserver/doctor create mode 100644 localinfra/pc/services/zoekt-webserver/packages create mode 100644 localinfra/pc/services/zoekt-webserver/ports.env create mode 100644 localinfra/pc/services/zoekt-webserver/process.yaml create mode 100755 localinfra/pc/services/zoekt-webserver/run create mode 100755 localinfra/pc/services/zoekt-webserver/setup create mode 100644 localinfra/pds-patch.mjs create mode 100644 localinfra/pds-theme.css create mode 100644 localinfra/pds.env create mode 100644 localinfra/pdsls.patch create mode 100644 localinfra/pocket.patch create mode 100644 localinfra/postgres-init.sql create mode 100644 localinfra/scripts/add-accounts.sh create mode 100644 localinfra/scripts/init-data.sh create mode 100644 localinfra/scripts/lib.sh create mode 100644 localinfra/sites-r2-bridge/index.js create mode 100644 localinfra/sites-r2-bridge/wrangler.toml create mode 100644 log/log.go create mode 100644 log/log_test.go create mode 100644 migrator/api/describe.go create mode 100644 migrator/api/describe_test.go create mode 100644 migrator/api/grant_test.go create mode 100644 migrator/api/server.go create mode 100644 migrator/api/server_test.go create mode 100644 migrator/api/validation.go create mode 100644 migrator/config/config.go create mode 100644 migrator/config/config_test.go create mode 100644 migrator/crypto/crypto.go create mode 100644 migrator/crypto/crypto_test.go create mode 100644 migrator/db/db.go create mode 100644 migrator/db/db_test.go create mode 100644 migrator/db/models.go create mode 100644 migrator/git/describe.go create mode 100644 migrator/git/describe_test.go create mode 100644 migrator/git/runner.go create mode 100644 migrator/git/runner_test.go create mode 100644 migrator/gitserver/server.go create mode 100644 migrator/gitserver/server_test.go create mode 100644 migrator/migrator.go create mode 100644 migrator/oauth/client.go create mode 100644 migrator/oauth/client_test.go create mode 100644 migrator/oauth/redirect_test.go create mode 100644 migrator/oauth/session_test.go create mode 100644 migrator/oauth/store.go create mode 100644 migrator/oauth/store_test.go create mode 100644 migrator/worker/worker.go create mode 100644 migrator/worker/worker_test.go create mode 100644 netutil/connectproxy.go create mode 100644 netutil/connectproxy_test.go create mode 100644 netutil/ssrf.go create mode 100644 netutil/ssrf_test.go create mode 100644 nix/Cargo.nix create mode 100644 nix/cargo-nix.nix create mode 100644 nix/crate-hashes.json create mode 100644 nix/go-src.nix create mode 100644 nix/gomod2nix.toml create mode 100644 nix/microvm/base.nix create mode 100644 nix/microvm/qemu.nix create mode 100644 nix/microvm/spindle-vm.nix create mode 100644 nix/microvm/user-config.nix create mode 100644 nix/modules/cilogs.nix create mode 100644 nix/modules/deliberi.nix create mode 100644 nix/modules/knot-rs.nix create mode 100644 nix/modules/knotmirror.nix create mode 100644 nix/modules/migrator.nix create mode 100644 nix/modules/shuttle.nix create mode 100644 nix/modules/sitesd.nix create mode 100644 nix/modules/spindle.nix create mode 100644 nix/modules/tranquil-gate.nix create mode 100644 nix/modules/zoekt-tnglserver.nix create mode 100644 nix/pkgs/attic-server.nix create mode 100644 nix/pkgs/attic-server/delete-paths.patch create mode 100644 nix/pkgs/blog.nix create mode 100644 nix/pkgs/cilogs.nix create mode 100644 nix/pkgs/deliberi.nix create mode 100644 nix/pkgs/docs.nix create mode 100644 nix/pkgs/dolly.nix create mode 100644 nix/pkgs/goat.nix create mode 100644 nix/pkgs/hydrant-bin.nix create mode 100644 nix/pkgs/hydrant-ffi.nix create mode 100644 nix/pkgs/lexgen.nix create mode 100644 nix/pkgs/migrator.nix create mode 100644 nix/pkgs/ogre-fonts.nix create mode 100644 nix/pkgs/sitesd.nix create mode 100644 nix/pkgs/spindle-almalinux-image.nix create mode 100644 nix/pkgs/spindle-alpine-image.nix create mode 100644 nix/pkgs/spindle-distrusted-roots/distrust-roots.py create mode 100644 nix/pkgs/spindle-distrusted-roots/roots/aaa-certificate-services.crt create mode 100644 nix/pkgs/spindle-distrusted-roots/roots/emsign-ecc-root-ca-g3.crt create mode 100644 nix/pkgs/spindle-distrusted-roots/roots/emsign-root-cs-ca-g3.crt create mode 100644 nix/pkgs/spindle-distrusted-roots/roots/emsign-root-tls-ca-g3.crt create mode 100644 nix/pkgs/spindle-image-helpers.nix create mode 100644 nix/pkgs/spindle-nixos-image.nix create mode 100644 nix/pkgs/spindle-qemu-runner.nix create mode 100644 nix/pkgs/spindle-static-git.nix create mode 100644 nix/pkgs/spindle.nix create mode 100644 nix/pkgs/sqlite-lib.nix create mode 100644 nix/pkgs/web-static-files.nix create mode 100644 nix/pkgs/zoekt-tngl-indexserver.nix create mode 100644 nix/pkgs/zoekt-webserver.nix create mode 100644 notifier/notifier.go create mode 100644 ogre/.gitignore create mode 100644 ogre/bun.lock create mode 100644 ogre/knip.json create mode 100644 ogre/package.json create mode 100644 ogre/packages/runtime/index.ts create mode 100644 ogre/packages/runtime/package.json create mode 100644 ogre/packages/runtime/types.ts create mode 100644 ogre/packages/runtime/workerd.ts create mode 100644 ogre/readme.md create mode 100644 ogre/src/__tests__/assets/avatar.jpg create mode 100644 ogre/src/__tests__/contrast.test.ts create mode 100644 ogre/src/__tests__/fixtures.ts create mode 100644 ogre/src/__tests__/get.test.ts create mode 100644 ogre/src/__tests__/render.test.ts create mode 100644 ogre/src/components/cards/commit.tsx create mode 100644 ogre/src/components/cards/file.tsx create mode 100644 ogre/src/components/cards/issue.tsx create mode 100644 ogre/src/components/cards/pipeline.tsx create mode 100644 ogre/src/components/cards/profile.tsx create mode 100644 ogre/src/components/cards/pull-request.tsx create mode 100644 ogre/src/components/cards/repository.tsx create mode 100644 ogre/src/components/cards/string.tsx create mode 100644 ogre/src/components/cards/tag.tsx create mode 100644 ogre/src/components/shared/avatar.tsx create mode 100644 ogre/src/components/shared/card-footer.tsx create mode 100644 ogre/src/components/shared/card-header.tsx create mode 100644 ogre/src/components/shared/chip.tsx create mode 100644 ogre/src/components/shared/code-panel.tsx create mode 100644 ogre/src/components/shared/constants.ts create mode 100644 ogre/src/components/shared/footer-stats.tsx create mode 100644 ogre/src/components/shared/label-pill.tsx create mode 100644 ogre/src/components/shared/language-circles.tsx create mode 100644 ogre/src/components/shared/layout.tsx create mode 100644 ogre/src/components/shared/logo.tsx create mode 100644 ogre/src/components/shared/metrics.tsx create mode 100644 ogre/src/components/shared/stat-item.tsx create mode 100644 ogre/src/components/shared/status-badge.tsx create mode 100644 ogre/src/icons/lucide.tsx create mode 100644 ogre/src/index.tsx create mode 100644 ogre/src/lib/fit.ts create mode 100644 ogre/src/lib/format.ts create mode 100644 ogre/src/lib/pluralize.ts create mode 100644 ogre/src/lib/render.ts create mode 100644 ogre/src/types.d.ts create mode 100644 ogre/src/validation.ts create mode 100644 ogre/tsconfig.json create mode 100644 ogre/wrangler.dev.jsonc create mode 100644 ogre/wrangler.jsonc create mode 100644 orm/orm.go create mode 100644 orm/run_migration_test.go create mode 100644 package.json create mode 100644 pds/preauthorize.mjs create mode 100644 pds/preauthorize.test.mjs create mode 100644 pnpm-lock.yaml create mode 100644 rbac/rbac.go create mode 100644 rbac/rbac_test.go create mode 100644 rbac/txn.go create mode 100644 rbac/util.go create mode 100644 readme.md create mode 100644 repoident/knoturl.go create mode 100644 repoident/knoturl_test.go create mode 100644 repoident/repoident.go create mode 100644 repoident/repoident_test.go create mode 100644 rust-toolchain.toml create mode 100644 scratch-example.toml create mode 100644 seo-reporter/.gitignore create mode 100644 seo-reporter/package.json create mode 100644 seo-reporter/pnpm-lock.yaml create mode 100644 seo-reporter/pnpm-workspace.yaml create mode 100644 seo-reporter/readme.md create mode 100644 seo-reporter/scripts/dry-report.ts create mode 100644 seo-reporter/src/gsc.ts create mode 100644 seo-reporter/src/index.ts create mode 100644 seo-reporter/src/report.ts create mode 100644 seo-reporter/test/report.test.ts create mode 100644 seo-reporter/tsconfig.json create mode 100644 seo-reporter/wrangler.dev.jsonc create mode 100644 seo-reporter/wrangler.jsonc create mode 100644 sets/gen.go create mode 100644 sets/readme.txt create mode 100644 sets/set.go create mode 100644 sets/set_test.go create mode 100644 shuttle/Cargo.toml create mode 100644 shuttle/README.md create mode 100644 shuttle/src/activation.rs create mode 100644 shuttle/src/cache/mod.rs create mode 100644 shuttle/src/cache/read_proxy.rs create mode 100644 shuttle/src/cache/write_proxy.rs create mode 100644 shuttle/src/command.rs create mode 100644 shuttle/src/dns_proxy.rs create mode 100644 shuttle/src/exec.rs create mode 100644 shuttle/src/gen/file_descriptor_set.bin create mode 100644 shuttle/src/gen/spindle/agent/v1/spindle.agent.v1.rs create mode 100644 shuttle/src/gen/spindle/mill/v1/spindle.mill.v1.rs create mode 100644 shuttle/src/host_proxy.rs create mode 100644 shuttle/src/logging.rs create mode 100644 shuttle/src/main.rs create mode 100644 shuttle/src/nix_config.rs create mode 100644 shuttle/src/protocol.rs create mode 100644 shuttle/src/pty.rs create mode 100644 shuttle/src/session.rs create mode 100644 sitemap/migrations/0001_init.sql create mode 100644 sitemap/migrations/0002_handles.sql create mode 100644 sitemap/migrations/0003_accounts.sql create mode 100644 sitemap/migrations/0004_claims.sql create mode 100644 sitemap/package.json create mode 100644 sitemap/pnpm-lock.yaml create mode 100644 sitemap/pnpm-workspace.yaml create mode 100644 sitemap/src/db.ts create mode 100644 sitemap/src/hydrant.ts create mode 100644 sitemap/src/identity.ts create mode 100644 sitemap/src/index.ts create mode 100644 sitemap/src/jetstream.ts create mode 100644 sitemap/src/sitemap.ts create mode 100644 sitemap/src/types.ts create mode 100644 sitemap/src/xml.ts create mode 100644 sitemap/test/cloudflare-workers.ts create mode 100644 sitemap/test/d1.ts create mode 100644 sitemap/test/events.ts create mode 100644 sitemap/test/jetstream.test.ts create mode 100644 sitemap/test/node.d.ts create mode 100644 sitemap/test/sitemap.test.ts create mode 100644 sitemap/test/sync.test.ts create mode 100644 sitemap/tsconfig.json create mode 100644 sitemap/vitest.config.ts create mode 100644 sitemap/wrangler.dev.jsonc create mode 100644 sitemap/wrangler.jsonc create mode 100644 sites/.gitignore create mode 100644 sites/Cargo.lock create mode 100644 sites/Cargo.toml create mode 100644 sites/migrations/0001_init.sql create mode 100644 sites/migrations/0002_site_deploys_manual.sql create mode 100644 sites/src/claim.rs create mode 100644 sites/src/kvmap.rs create mode 100644 sites/src/lib.rs create mode 100644 sites/src/serviceauth.rs create mode 100644 sites/src/sites.rs create mode 100644 sites/src/xrpc.rs create mode 100644 sites/wrangler.dev.toml create mode 100644 sites/wrangler.toml create mode 100644 sitesd/README.md create mode 100644 sitesd/cache/cache.go create mode 100644 sitesd/cloudflare/r2.go create mode 100644 sitesd/cloudflare/r2_test.go create mode 100644 sitesd/config/config.go create mode 100644 sitesd/models/models.go create mode 100644 sitesd/reposync/reposync.go create mode 100644 sitesd/reposync/reposync_test.go create mode 100644 sitesd/sitefeed/feed.go create mode 100644 sitesd/sitefeed/feed_test.go create mode 100644 sitesd/sites/sites.go create mode 100644 sitesd/sites/sites_test.go create mode 100644 sitesd/sitesd.go create mode 100644 sitesd/sitesd_test.go create mode 100644 sitesd/worker/worker.go create mode 100644 sitesd/worker/worker_test.go create mode 100644 spindle/TORTURE.md create mode 100644 spindle/account_job_test.go create mode 100644 spindle/admission_pause.go create mode 100644 spindle/agentproto/gen/agent.pb.go create mode 100644 spindle/agentproto/protocol.go create mode 100644 spindle/agentproto/protocol_test.go create mode 100644 spindle/agentproto/spindle/agent/v1/agent.proto create mode 100644 spindle/allowlist_test.go create mode 100644 spindle/artifactstore/artifactstore.go create mode 100644 spindle/artifactstore/artifactstore_test.go create mode 100644 spindle/backfill_later_test.go create mode 100644 spindle/backfill_marks_test.go create mode 100644 spindle/backfill_skip_test.go create mode 100644 spindle/banned_actor_test.go create mode 100644 spindle/casbin_copy.go create mode 100644 spindle/collaborator_marks_test.go create mode 100644 spindle/collaborator_sync_test.go create mode 100644 spindle/collaborators.go create mode 100644 spindle/config/config.go create mode 100644 spindle/config/config_test.go create mode 100644 spindle/db/allowlist.go create mode 100644 spindle/db/approve_test.go create mode 100644 spindle/db/artifacts.go create mode 100644 spindle/db/artifacts_test.go create mode 100644 spindle/db/backfill.go create mode 100644 spindle/db/bans.go create mode 100644 spindle/db/bans_test.go create mode 100644 spindle/db/cache.go create mode 100644 spindle/db/cache_test.go create mode 100644 spindle/db/collaborators.go create mode 100644 spindle/db/collaborators_test.go create mode 100644 spindle/db/db.go create mode 100644 spindle/db/delete_test.go create mode 100644 spindle/db/durable_test.go create mode 100644 spindle/db/events.go create mode 100644 spindle/db/events_metrics_test.go create mode 100644 spindle/db/fd_leak_test.go create mode 100644 spindle/db/identity_migration_test.go create mode 100644 spindle/db/invites.go create mode 100644 spindle/db/jobs.go create mode 100644 spindle/db/jobs_depth_test.go create mode 100644 spindle/db/jobs_test.go create mode 100644 spindle/db/jobs_unreadable_test.go create mode 100644 spindle/db/log_migration.go create mode 100644 spindle/db/log_migration_wait_test.go create mode 100644 spindle/db/mill_state.go create mode 100644 spindle/db/mill_state_test.go create mode 100644 spindle/db/mill_tokens.go create mode 100644 spindle/db/mill_tokens_test.go create mode 100644 spindle/db/nix_charges_test.go create mode 100644 spindle/db/nix_grants_test.go create mode 100644 spindle/db/nix_objects.go create mode 100644 spindle/db/nix_objects_test.go create mode 100644 spindle/db/nixos_toplevel_cache.go create mode 100644 spindle/db/orphans.go create mode 100644 spindle/db/orphans_test.go create mode 100644 spindle/db/pipelines.go create mode 100644 spindle/db/pipelines_kind_test.go create mode 100644 spindle/db/pipelines_test.go create mode 100644 spindle/db/pipelines_total_test.go create mode 100644 spindle/db/pull_events_limit_test.go create mode 100644 spindle/db/pull_tickets_test.go create mode 100644 spindle/db/pulls.go create mode 100644 spindle/db/quota.go create mode 100644 spindle/db/quota_charge_test.go create mode 100644 spindle/db/quota_minutes.go create mode 100644 spindle/db/quota_minutes_test.go create mode 100644 spindle/db/quota_release_test.go create mode 100644 spindle/db/quota_test.go create mode 100644 spindle/db/quota_workflow_test.go create mode 100644 spindle/db/repo_pushes.go create mode 100644 spindle/db/repo_pushes_test.go create mode 100644 spindle/db/repo_retries.go create mode 100644 spindle/db/repo_retries_test.go create mode 100644 spindle/db/repo_turns.go create mode 100644 spindle/db/repos.go create mode 100644 spindle/db/repos_test.go create mode 100644 spindle/db/retry.go create mode 100644 spindle/db/retry_test.go create mode 100644 spindle/db/schedule_runs.go create mode 100644 spindle/db/schedules.go create mode 100644 spindle/db/schedules_test.go create mode 100644 spindle/db/webhooks.go create mode 100644 spindle/db/webhooks_smoke_test.go create mode 100644 spindle/db/wipes.go create mode 100644 spindle/debug_keys.go create mode 100644 spindle/debug_keys_test.go create mode 100644 spindle/engine/archive_cleanup_test.go create mode 100644 spindle/engine/archive_inplace_test.go create mode 100644 spindle/engine/archive_partial_test.go create mode 100644 spindle/engine/cache.go create mode 100644 spindle/engine/cache_controller.go create mode 100644 spindle/engine/cache_evict_test.go create mode 100644 spindle/engine/cache_prune.go create mode 100644 spindle/engine/cache_prune_test.go create mode 100644 spindle/engine/cache_test.go create mode 100644 spindle/engine/engine.go create mode 100644 spindle/engine/engine_minutes_test.go create mode 100644 spindle/engine/engine_queue_timeout_test.go create mode 100644 spindle/engine/engine_quota_test.go create mode 100644 spindle/engine/engine_refusal_test.go create mode 100644 spindle/engine/engine_slot_log_test.go create mode 100644 spindle/engine/engine_test.go create mode 100644 spindle/engine/failure_metrics_test.go create mode 100644 spindle/engine/manifest.go create mode 100644 spindle/engine/manifest_test.go create mode 100644 spindle/engine/mask_test.go create mode 100644 spindle/engine/placement.go create mode 100644 spindle/engine/scheduler.go create mode 100644 spindle/engine/scheduler_test.go create mode 100644 spindle/engine/slot.go create mode 100644 spindle/engine/slot_test.go create mode 100644 spindle/engine/usage.go create mode 100644 spindle/engine_microvm_linux.go create mode 100644 spindle/engine_microvm_other.go create mode 100644 spindle/engine_microvm_other_test.go create mode 100644 spindle/engines/dummy/engine.go create mode 100644 spindle/engines/microvm/README.md create mode 100644 spindle/engines/microvm/agent.go create mode 100644 spindle/engines/microvm/agent_test.go create mode 100644 spindle/engines/microvm/args.go create mode 100644 spindle/engines/microvm/attic.go create mode 100644 spindle/engines/microvm/attic_test.go create mode 100755 spindle/engines/microvm/bench-boot.sh create mode 100644 spindle/engines/microvm/budget.go create mode 100644 spindle/engines/microvm/budget_test.go create mode 100644 spindle/engines/microvm/cache.go create mode 100644 spindle/engines/microvm/cgroup.go create mode 100644 spindle/engines/microvm/cgroup_oom_test.go create mode 100644 spindle/engines/microvm/cgroup_other.go create mode 100644 spindle/engines/microvm/cgroup_resources_test.go create mode 100644 spindle/engines/microvm/cgroup_test.go create mode 100644 spindle/engines/microvm/cid_test.go create mode 100644 spindle/engines/microvm/debug.go create mode 100644 spindle/engines/microvm/debug_e2e_test.go create mode 100644 spindle/engines/microvm/debug_test.go create mode 100644 spindle/engines/microvm/debugssh.go create mode 100644 spindle/engines/microvm/debugssh_test.go create mode 100644 spindle/engines/microvm/dns_proxy.go create mode 100644 spindle/engines/microvm/dns_proxy_test.go create mode 100644 spindle/engines/microvm/engine.go create mode 100644 spindle/engines/microvm/engine_test.go create mode 100644 spindle/engines/microvm/image.go create mode 100644 spindle/engines/microvm/image_test.go create mode 100644 spindle/engines/microvm/models.go create mode 100644 spindle/engines/microvm/models_test.go create mode 100644 spindle/engines/microvm/narinfo.go create mode 100644 spindle/engines/microvm/netns_wrapper.sh.tmpl create mode 100644 spindle/engines/microvm/networking.go create mode 100644 spindle/engines/microvm/nixos_toplevel.go create mode 100644 spindle/engines/microvm/placement/placement.go create mode 100644 spindle/engines/microvm/placement/placement_test.go create mode 100644 spindle/engines/microvm/placement_linux.go create mode 100644 spindle/engines/microvm/qemu.go create mode 100644 spindle/engines/microvm/qemu_lifetime_test.go create mode 100644 spindle/engines/microvm/runner.go create mode 100755 spindle/engines/microvm/start-test-substituter.sh create mode 100644 spindle/engines/microvm/substituter_proxy.go create mode 100644 spindle/engines/microvm/substituter_proxy_test.go create mode 100644 spindle/engines/microvm/substituter_upload_attic.go create mode 100644 spindle/engines/microvm/substituter_upload_attic_test.go create mode 100644 spindle/engines/microvm/substituter_upload_http.go create mode 100644 spindle/engines/microvm/substituter_upload_nix_store.go create mode 100644 spindle/engines/microvm/substituter_upload_nix_store_test.go create mode 100644 spindle/engines/microvm/substituter_upload_proxy.go create mode 100644 spindle/engines/microvm/substituter_upload_proxy_test.go create mode 100755 spindle/engines/microvm/test-spindle-microvm.sh create mode 100644 spindle/engines/microvm/upload_quota_test.go create mode 100644 spindle/engines/microvm/upload_staging.go create mode 100644 spindle/engines/microvm/upload_store_path_test.go create mode 100644 spindle/engines/microvm/vm.go create mode 100644 spindle/engines/microvm/vm_test.go create mode 100644 spindle/engines/nixery/ansi_stripper.go create mode 100644 spindle/engines/nixery/cache.go create mode 100644 spindle/engines/nixery/engine.go create mode 100644 spindle/engines/nixery/engine_test.go create mode 100644 spindle/engines/nixery/envs.go create mode 100644 spindle/engines/nixery/envs_test.go create mode 100644 spindle/engines/nixery/errors.go create mode 100644 spindle/engines/nixery/seats.go create mode 100644 spindle/engines/nixery/seats_test.go create mode 100644 spindle/engines/nixery/setup_steps.go create mode 100644 spindle/fakes_test.go create mode 100644 spindle/faults_test.go create mode 100644 spindle/filter_workflows_test.go create mode 100644 spindle/http_server_test.go create mode 100644 spindle/hydrant.go create mode 100644 spindle/hydrant_pending_test.go create mode 100644 spindle/hydrant_test.go create mode 100644 spindle/identity/identity.go create mode 100644 spindle/identity/identity_test.go create mode 100644 spindle/identity_route_test.go create mode 100644 spindle/ingest_shards.go create mode 100644 spindle/ingest_shards_test.go create mode 100644 spindle/ingester.go create mode 100644 spindle/ingester_enrollment_test.go create mode 100644 spindle/ingester_test.go create mode 100644 spindle/instance_lock_other.go create mode 100644 spindle/instance_lock_test.go create mode 100644 spindle/instance_lock_unix.go create mode 100644 spindle/job_workers_test.go create mode 100644 spindle/knotclient/knotclient.go create mode 100644 spindle/knotclient/knotclient_test.go create mode 100644 spindle/load_pipeline_test.go create mode 100644 spindle/logview/archive_retry_test.go create mode 100644 spindle/logview/follow_longline_test.go create mode 100644 spindle/logview/follow_stop_test.go create mode 100644 spindle/logview/logview.go create mode 100644 spindle/logview/logview_test.go create mode 100644 spindle/main_test.go create mode 100644 spindle/middleware.go create mode 100644 spindle/mill/README.md create mode 100644 spindle/mill/auth_test.go create mode 100644 spindle/mill/cache.go create mode 100644 spindle/mill/debug_key.go create mode 100644 spindle/mill/debug_key_test.go create mode 100644 spindle/mill/engine.go create mode 100644 spindle/mill/executor/cache.go create mode 100644 spindle/mill/executor/cache_test.go create mode 100644 spindle/mill/executor/capability_test.go create mode 100644 spindle/mill/executor/debug_key_client.go create mode 100644 spindle/mill/executor/debug_key_client_test.go create mode 100644 spindle/mill/executor/executor.go create mode 100644 spindle/mill/executor/observe.go create mode 100644 spindle/mill/executor/observe_test.go create mode 100644 spindle/mill/executor/outbox.go create mode 100644 spindle/mill/executor/quota_client.go create mode 100644 spindle/mill/executor/quota_client_test.go create mode 100644 spindle/mill/executor/reserved.go create mode 100644 spindle/mill/executor/reserved_test.go create mode 100644 spindle/mill/fair.go create mode 100644 spindle/mill/fair_test.go create mode 100644 spindle/mill/handler.go create mode 100644 spindle/mill/integration_test.go create mode 100644 spindle/mill/jump.go create mode 100644 spindle/mill/jump_test.go create mode 100644 spindle/mill/lease.go create mode 100644 spindle/mill/lost_executor_test.go create mode 100644 spindle/mill/mill.go create mode 100644 spindle/mill/mill_test.go create mode 100644 spindle/mill/nix_cache.go create mode 100644 spindle/mill/nix_cache_test.go create mode 100644 spindle/mill/proto/gen/mill.pb.go create mode 100644 spindle/mill/proto/protocol.go create mode 100644 spindle/mill/proto/protocol_test.go create mode 100644 spindle/mill/proto/spindle/mill/v1/mill.proto create mode 100644 spindle/mill/proto/ws.go create mode 100644 spindle/mill/quota_admission_test.go create mode 100644 spindle/mill/quota_test_helper_test.go create mode 100644 spindle/mill/restore.go create mode 100644 spindle/mill/restore_test.go create mode 100644 spindle/mill/session.go create mode 100644 spindle/mill/token.go create mode 100644 spindle/mill/token_to_executor_test.go create mode 100644 spindle/mill/workflow_limit_test.go create mode 100644 spindle/models/cache.go create mode 100644 spindle/models/cache_test.go create mode 100644 spindle/models/clone.go create mode 100644 spindle/models/clone_test.go create mode 100644 spindle/models/engine.go create mode 100644 spindle/models/logger.go create mode 100644 spindle/models/logger_bound_test.go create mode 100644 spindle/models/logger_cap_test.go create mode 100644 spindle/models/logger_test.go create mode 100644 spindle/models/models.go create mode 100644 spindle/models/pipeline.go create mode 100644 spindle/models/pipeline_env.go create mode 100644 spindle/models/pipeline_env_test.go create mode 100644 spindle/models/pipeline_record.go create mode 100644 spindle/models/pipeline_record_test.go create mode 100644 spindle/models/pipeline_test.go create mode 100644 spindle/models/secret_mask.go create mode 100644 spindle/models/secret_mask_test.go create mode 100644 spindle/models/webhook.go create mode 100644 spindle/motd create mode 100644 spindle/netguard/check_host_test.go create mode 100644 spindle/netguard/netguard.go create mode 100644 spindle/nix_cache_grant_test.go create mode 100644 spindle/nixcache/atticd_test.go create mode 100644 spindle/nixcache/client.go create mode 100644 spindle/nixcache/client_test.go create mode 100644 spindle/nixcache/issuer.go create mode 100644 spindle/nixcache/issuer_test.go create mode 100644 spindle/nixcache/name.go create mode 100644 spindle/nixcache/name_test.go create mode 100644 spindle/nixcache/prune.go create mode 100644 spindle/nixcache/prune_index_test.go create mode 100644 spindle/nixcache/prune_test.go create mode 100644 spindle/nixcache/publishing.go create mode 100644 spindle/nixcache/publishing_index_test.go create mode 100644 spindle/nixcache/publishing_test.go create mode 100644 spindle/nixcache/staged.go create mode 100644 spindle/nixcache/staged_test.go create mode 100644 spindle/nixcache/token.go create mode 100644 spindle/nixcache/token_test.go create mode 100644 spindle/nixcache/upstream.go create mode 100644 spindle/nixcache/upstream_test.go create mode 100644 spindle/nixcache/usage.go create mode 100644 spindle/nixcache/usage_test.go create mode 100644 spindle/notify_admission_test.go create mode 100644 spindle/notify_event.go create mode 100644 spindle/notify_event_test.go create mode 100644 spindle/notify_shutdown_test.go create mode 100644 spindle/notify_slots_test.go create mode 100644 spindle/notify_status_test.go create mode 100644 spindle/observability/logging.go create mode 100644 spindle/observability/logging_test.go create mode 100644 spindle/observability/metrics.go create mode 100644 spindle/observability/metrics_test.go create mode 100644 spindle/observability/tracing.go create mode 100644 spindle/observability/tracing_test.go create mode 100644 spindle/oidc/issuer.go create mode 100644 spindle/oidc/issuer_test.go create mode 100644 spindle/pause_atomic_test.go create mode 100644 spindle/pause_test.go create mode 100644 spindle/pause_webhook_test.go create mode 100644 spindle/pemkey/pemkey.go create mode 100644 spindle/pemkey/pemkey_test.go create mode 100644 spindle/pemkey/pemkeytest/pemkeytest.go create mode 100644 spindle/permissions_test.go create mode 100644 spindle/pipeline/codec.go create mode 100644 spindle/pipeline/codec_test.go create mode 100644 spindle/pull_backlog_test.go create mode 100644 spindle/pull_pipelines_test.go create mode 100644 spindle/pull_trust_test.go create mode 100644 spindle/pull_versions_test.go create mode 100644 spindle/pulls.go create mode 100644 spindle/pulls_test.go create mode 100644 spindle/quota/denial.go create mode 100644 spindle/quota/evict.go create mode 100644 spindle/quota/evict_test.go create mode 100644 spindle/quota/idle_waiters_test.go create mode 100644 spindle/quota/manager.go create mode 100644 spindle/quota/manager_test.go create mode 100644 spindle/quota/minutes.go create mode 100644 spindle/quota/minutes_test.go create mode 100644 spindle/quota/nix_object_test.go create mode 100644 spindle/quota/quota.go create mode 100644 spindle/quota/workflow_limit_test.go create mode 100644 spindle/record_marks_test.go create mode 100644 spindle/repo_job_backoff_test.go create mode 100644 spindle/repo_job_limits_test.go create mode 100644 spindle/repo_jobs.go create mode 100644 spindle/repo_jobs_test.go create mode 100644 spindle/repo_locks.go create mode 100644 spindle/repo_locks_cleanup_test.go create mode 100644 spindle/repo_locks_test.go create mode 100644 spindle/repo_retry.go create mode 100644 spindle/repo_retry_cap_test.go create mode 100644 spindle/repo_retry_test.go create mode 100644 spindle/repos.go create mode 100644 spindle/repos_test.go create mode 100644 spindle/retry_approval_test.go create mode 100644 spindle/run_env_test.go create mode 100644 spindle/run_pipeline_test.go create mode 100644 spindle/schedule.go create mode 100644 spindle/schedule_test.go create mode 100644 spindle/secret_copy.go create mode 100644 spindle/secrets/manager.go create mode 100644 spindle/secrets/openbao.go create mode 100644 spindle/secrets/openbao_test.go create mode 100644 spindle/secrets/policy.hcl create mode 100644 spindle/secrets/sqlite.go create mode 100644 spindle/secrets/sqlite_key_test.go create mode 100644 spindle/secrets/sqlite_test.go create mode 100644 spindle/server.go create mode 100644 spindle/server_test.go create mode 100644 spindle/source_repo_test.go create mode 100644 spindle/startup_migrations.go create mode 100644 spindle/startup_migrations_test.go create mode 100644 spindle/storage/disk.go create mode 100644 spindle/storage/s3.go create mode 100644 spindle/storage/storage.go create mode 100644 spindle/storage/storage_test.go create mode 100644 spindle/storage/unversioned_s3.go create mode 100644 spindle/torture_admission_test.go create mode 100644 spindle/torture_engine_test.go create mode 100644 spindle/torture_ingest_test.go create mode 100644 spindle/torture_notify_abuse_test.go create mode 100644 spindle/torture_notify_test.go create mode 100644 spindle/torture_pool_test.go create mode 100644 spindle/torture_queue_test.go create mode 100644 spindle/torture_soak_test.go create mode 100644 spindle/torture_test.go create mode 100644 spindle/torture_world_test.go create mode 100644 spindle/webhook/webhook.go create mode 100644 spindle/webhook/webhook_detach_test.go create mode 100644 spindle/webhook/webhook_fairness_test.go create mode 100644 spindle/webhook/webhook_lane_test.go create mode 100644 spindle/webhook/webhook_test.go create mode 100644 spindle/wipe.go create mode 100644 spindle/wipe_claim_test.go create mode 100644 spindle/wipe_test.go create mode 100644 spindle/wipes_test.go create mode 100644 spindle/workflow_defs.go create mode 100644 spindle/workflow_defs_test.go create mode 100644 spindle/workflow_limits.go create mode 100644 spindle/workflow_limits_test.go create mode 100644 spindle/workflow_read.go create mode 100644 spindle/workflow_read_test.go create mode 100644 spindle/xrpc/allowlist.go create mode 100644 spindle/xrpc/allowlist_test.go create mode 100644 spindle/xrpc/bad_input_test.go create mode 100644 spindle/xrpc/ban.go create mode 100644 spindle/xrpc/ban_test.go create mode 100644 spindle/xrpc/banned_did_test.go create mode 100644 spindle/xrpc/body_limit_test.go create mode 100644 spindle/xrpc/ci_pipeline_subscribe_logs.go create mode 100644 spindle/xrpc/ci_subscribe_logs_cap_test.go create mode 100644 spindle/xrpc/ci_subscribe_logs_drain_test.go create mode 100644 spindle/xrpc/ci_subscribe_logs_limit_test.go create mode 100644 spindle/xrpc/ci_subscribe_logs_origin_test.go create mode 100644 spindle/xrpc/ci_subscribe_logs_stall_test.go create mode 100644 spindle/xrpc/ci_subscribe_logs_workflows_test.go create mode 100644 spindle/xrpc/cors_test.go create mode 100644 spindle/xrpc/definitions_busy_test.go create mode 100644 spindle/xrpc/invites.go create mode 100644 spindle/xrpc/invites_test.go create mode 100644 spindle/xrpc/legacy_routes_test.go create mode 100644 spindle/xrpc/notify_status_test.go create mode 100644 spindle/xrpc/org_tangled_ci_approve_pipeline.go create mode 100644 spindle/xrpc/org_tangled_ci_approve_pipeline_test.go create mode 100644 spindle/xrpc/org_tangled_ci_cancel_pipeline.go create mode 100644 spindle/xrpc/org_tangled_ci_get_pipeline.go create mode 100644 spindle/xrpc/org_tangled_ci_get_workflow_definition.go create mode 100644 spindle/xrpc/org_tangled_ci_pipeline.go create mode 100644 spindle/xrpc/org_tangled_ci_query_pipelines.go create mode 100644 spindle/xrpc/org_tangled_ci_retry_pipeline.go create mode 100644 spindle/xrpc/org_tangled_ci_retry_pipeline_test.go create mode 100644 spindle/xrpc/org_tangled_ci_subscribe_pipeline_logs.go create mode 100644 spindle/xrpc/org_tangled_event_notify_event.go create mode 100644 spindle/xrpc/org_tangled_event_notify_event_test.go create mode 100644 spindle/xrpc/org_tangled_secret_add_secret.go create mode 100644 spindle/xrpc/org_tangled_secret_common.go create mode 100644 spindle/xrpc/org_tangled_secret_key_test.go create mode 100644 spindle/xrpc/org_tangled_secret_list_secrets.go create mode 100644 spindle/xrpc/org_tangled_secret_remove_secret.go create mode 100644 spindle/xrpc/org_tangled_spindle_describe_spindle.go create mode 100644 spindle/xrpc/org_tangled_test.go create mode 100644 spindle/xrpc/quota.go create mode 100644 spindle/xrpc/quota_release_test.go create mode 100644 spindle/xrpc/quota_test.go create mode 100644 spindle/xrpc/repo_auth.go create mode 100644 spindle/xrpc/repo_auth_test.go create mode 100644 spindle/xrpc/repo_lookup.go create mode 100644 spindle/xrpc/repo_lookup_test.go create mode 100644 spindle/xrpc/scheduled_pipeline_test.go create mode 100644 spindle/xrpc/secret_access.go create mode 100644 spindle/xrpc/trigger_paused_test.go create mode 100644 spindle/xrpc/trigger_sha_test.go create mode 100644 spindle/xrpc/validation_test.go create mode 100644 spindle/xrpc/webhook_paused_test.go create mode 100644 spindle/xrpc/webhooks.go create mode 100644 spindle/xrpc/xrpc.go create mode 100644 spindle/xrpc/xrpc_test.go create mode 100644 sqlite/sqlite.go create mode 100644 status/.gitignore create mode 100644 status/package-lock.json create mode 100644 status/package.json create mode 100644 status/src/access.test.ts create mode 100644 status/src/access.ts create mode 100644 status/src/css.d.ts create mode 100644 status/src/fonts.ts create mode 100644 status/src/footer.ts create mode 100644 status/src/index.ts create mode 100644 status/src/logotype.ts create mode 100644 status/static/.assetsignore create mode 100644 status/static/favicon.svg create mode 100644 status/static/logos/dolly.svg create mode 100644 status/tailwind.config.js create mode 100644 status/tsconfig.json create mode 100644 status/tsconfig.test.json create mode 100644 status/wrangler.jsonc create mode 100644 systemd/knot.service create mode 100644 tailwind.config.js create mode 100644 tapc/readme.md create mode 100644 tapc/simple_indexer.go create mode 100644 tapc/tap.go create mode 100644 tapc/tap_test.go create mode 100644 tapc/types.go create mode 100644 tid/tid.go create mode 100644 web/.dockerignore create mode 100644 web/.gitignore create mode 100644 web/.npmrc create mode 100644 web/.prettierignore create mode 100644 web/.storybook/BoneCaptureDecorator.svelte create mode 100644 web/.storybook/DatasetToggle.svelte create mode 100644 web/.storybook/MockAuth.svelte create mode 100644 web/.storybook/MockNotifications.svelte create mode 100644 web/.storybook/SimulatedRepoSettings.svelte create mode 100644 web/.storybook/VouchArrivalReplay.svelte create mode 100644 web/.storybook/WorkflowRunReplay.svelte create mode 100644 web/.storybook/arrival.ts create mode 100644 web/.storybook/clipboard.ts create mode 100644 web/.storybook/holdErrorReset.ts create mode 100644 web/.storybook/installFetch.ts create mode 100644 web/.storybook/main.ts create mode 100644 web/.storybook/mocks/navigation.ts create mode 100644 web/.storybook/moderation.ts create mode 100644 web/.storybook/pendingOnboarding.ts create mode 100644 web/.storybook/play.ts create mode 100644 web/.storybook/preview.ts create mode 100644 web/.storybook/respond.ts create mode 100644 web/Containerfile create mode 100644 web/README.md create mode 100644 web/_headers create mode 100644 web/boneyard.config.json create mode 100644 web/eslint.config.js create mode 100644 web/knip.json create mode 100644 web/lex.config.ts create mode 100644 web/package.json create mode 100644 web/patches/@pierre__diffs@1.6.1.patch create mode 100644 web/patches/@sveltejs__adapter-bun@1.0.0.patch create mode 100644 web/patches/boneyard-js@1.9.0.patch create mode 100644 web/patches/cobe@2.0.1.patch create mode 100644 web/patches/svelte@5.57.2.patch create mode 100644 web/pnpm-lock.yaml create mode 100644 web/pnpm-workspace.yaml create mode 100644 web/prettier.config.js create mode 100755 web/scripts/bones.js create mode 100755 web/scripts/deploy-cloudflare-preview.sh create mode 100755 web/scripts/preview-worker-name.sh create mode 100644 web/scripts/staleAwait.test.ts create mode 100644 web/scripts/staleAwait.ts create mode 100644 web/scripts/testids-check.ts create mode 100644 web/scripts/testids.test.ts create mode 100644 web/scripts/testids.ts create mode 100644 web/scripts/tsconfig.json create mode 100644 web/src/app.css create mode 100644 web/src/app.d.ts create mode 100644 web/src/app.html create mode 100644 web/src/env.test.ts create mode 100644 web/src/env.ts create mode 100644 web/src/fixtures/blobHeaderWorstCase.ts create mode 100644 web/src/fixtures/comments.ts create mode 100644 web/src/fixtures/commits.ts create mode 100644 web/src/fixtures/diff.ts create mode 100644 web/src/fixtures/emails.ts create mode 100644 web/src/fixtures/githubImportWorstCase.ts create mode 100644 web/src/fixtures/identities.ts create mode 100644 web/src/fixtures/logFeed.svelte.ts create mode 100644 web/src/fixtures/orgCreate.ts create mode 100644 web/src/fixtures/pipelines.ts create mode 100644 web/src/fixtures/profileWorstCase.ts create mode 100644 web/src/fixtures/pull.ts create mode 100644 web/src/fixtures/pullCommitRows.ts create mode 100644 web/src/fixtures/pullWorstCase.ts create mode 100644 web/src/fixtures/reactions.ts create mode 100644 web/src/fixtures/repo.ts create mode 100644 web/src/fixtures/search.ts create mode 100644 web/src/fixtures/timeline.ts create mode 100644 web/src/fonts.css create mode 100644 web/src/fontsBase.mjs create mode 100644 web/src/hooks.server.test.ts create mode 100644 web/src/hooks.server.ts create mode 100644 web/src/hooks.ts create mode 100644 web/src/icon.d.ts create mode 100644 web/src/lib/action.svelte.ts create mode 100644 web/src/lib/action.test.ts create mode 100644 web/src/lib/api/_request.test.ts create mode 100644 web/src/lib/api/_request.ts create mode 100644 web/src/lib/api/access.test.ts create mode 100644 web/src/lib/api/access.ts create mode 100644 web/src/lib/api/account.ts create mode 100644 web/src/lib/api/actAs.test.ts create mode 100644 web/src/lib/api/actAs.ts create mode 100644 web/src/lib/api/appview.ts create mode 100644 web/src/lib/api/archive.test.ts create mode 100644 web/src/lib/api/archive.ts create mode 100644 web/src/lib/api/artifact.ts create mode 100644 web/src/lib/api/attestation.test.ts create mode 100644 web/src/lib/api/attestation.ts create mode 100644 web/src/lib/api/awaitRecord.test.ts create mode 100644 web/src/lib/api/awaitRecord.ts create mode 100644 web/src/lib/api/blame.test.ts create mode 100644 web/src/lib/api/blame.ts create mode 100644 web/src/lib/api/blob.test.ts create mode 100644 web/src/lib/api/blob.ts create mode 100644 web/src/lib/api/cache.test.ts create mode 100644 web/src/lib/api/cache.ts create mode 100644 web/src/lib/api/client.test.ts create mode 100644 web/src/lib/api/client.ts create mode 100644 web/src/lib/api/cob.test.ts create mode 100644 web/src/lib/api/cob.ts create mode 100644 web/src/lib/api/collaborators.test.ts create mode 100644 web/src/lib/api/collaborators.ts create mode 100644 web/src/lib/api/comment.ts create mode 100644 web/src/lib/api/commitStatuses.ts create mode 100644 web/src/lib/api/count.ts create mode 100644 web/src/lib/api/coverage.ts create mode 100644 web/src/lib/api/delegation.test.ts create mode 100644 web/src/lib/api/delegation.ts create mode 100644 web/src/lib/api/deliberi.ts create mode 100644 web/src/lib/api/descriptors.ts create mode 100644 web/src/lib/api/diff.ts create mode 100644 web/src/lib/api/emailDid.test.ts create mode 100644 web/src/lib/api/emailDid.ts create mode 100644 web/src/lib/api/emails.test.ts create mode 100644 web/src/lib/api/emails.ts create mode 100644 web/src/lib/api/enrich.ts create mode 100644 web/src/lib/api/fileHistory.test.ts create mode 100644 web/src/lib/api/fileHistory.ts create mode 100644 web/src/lib/api/gitclient.test.ts create mode 100644 web/src/lib/api/gitclient.ts create mode 100644 web/src/lib/api/githubImport.test.ts create mode 100644 web/src/lib/api/githubImport.ts create mode 100644 web/src/lib/api/graph.test.ts create mode 100644 web/src/lib/api/graph.ts create mode 100644 web/src/lib/api/handle.ts create mode 100644 web/src/lib/api/identity.test.ts create mode 100644 web/src/lib/api/identity.ts create mode 100644 web/src/lib/api/index.ts create mode 100644 web/src/lib/api/indexnow.ts create mode 100644 web/src/lib/api/invite.test.ts create mode 100644 web/src/lib/api/invite.ts create mode 100644 web/src/lib/api/issue.ts create mode 100644 web/src/lib/api/knot.test.ts create mode 100644 web/src/lib/api/knot.ts create mode 100644 web/src/lib/api/knotmirror.test.ts create mode 100644 web/src/lib/api/knotmirror.ts create mode 100644 web/src/lib/api/labels.ts create mode 100644 web/src/lib/api/lexicons/index.ts create mode 100644 web/src/lib/api/lexicons/types/com/bad-example/pocket/defs.ts create mode 100644 web/src/lib/api/lexicons/types/com/bad-example/pocket/getPreferences.ts create mode 100644 web/src/lib/api/lexicons/types/com/bad-example/pocket/putPreferences.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/actor/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/actor/getProfile.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/actor/getProfileActivity.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/actor/getProfiles.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/actor/getTrending.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/actor/profile.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/actor/searchActorsTypeahead.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/bobbin/awaitRecord.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/ci/approvePipeline.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/ci/cancelPipeline.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/ci/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/ci/getPipeline.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/ci/getWorkflowDefinition.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/ci/listWorkflowDefinitions.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/ci/queryPipelines.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/ci/retryPipeline.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/ci/subscribePipelineLogs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/ci/trigger/manual.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/ci/trigger/schedule.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/embed/commit.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/event/notifyEvent.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/event/pullRequest.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/event/push.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/comment.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/countComments.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/countCommentsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/countReactions.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/countReactionsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/countStars.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/countStarsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/getStar.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/getTimeline.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/listComments.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/listCommentsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/listReactions.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/listReactionsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/listStars.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/listStarsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/reaction.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/star.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/feed/subscription.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/deleteBranch.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/getArchive.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/getBlame.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/getBlob.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/getBranch.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/getDiff.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/getEntry.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/getTag.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/getTree.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/listBranches.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/listCommits.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/listLanguages.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/listRefs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/listTags.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/oid.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/pushEvent.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/receivePack.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/refs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/git/setDefaultBranch.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/graph/attestation.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/graph/collaborator.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/graph/countFollows.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/graph/countFollowsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/graph/follow.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/graph/getFollow.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/graph/listFollows.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/graph/listFollowsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/graph/listNetworkVouches.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/graph/vouch.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/key/sshKey.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/knot/block.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/knot/describeKnot.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/knot/member.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/label/countDefinitions.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/label/definition.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/label/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/label/listDefinitions.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/markup/embed/image.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/markup/embed/video.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/markup/markdown.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/markup/plaintext.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/publicKey/countKeys.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/publicKey/getPublicKey.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/publicKey/listKeys.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/query/enrichResponse.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/artifact.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/countArtifacts.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/countArtifactsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/countCollaborators.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/countCollaboratorsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/countForks.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/countIssues.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/countPulls.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/countRepos.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/createRepo.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/declaration.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/deleteRepo.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/forkSync.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/getImportStatus.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/getRepo.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/getRepoByName.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/getRepoByRepoDid.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/getRepos.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/getReposByRepoDids.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listArtifacts.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listArtifactsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listCollaboratorInvitesBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listCollaborators.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listCollaboratorsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listForks.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listIssues.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listIssuesBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listPulls.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listPullsBy.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listRecordVersions.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/listRepos.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/manifest.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/mintToken.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/repo/reserveKey.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/review/patch.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/review/submitPatch.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/search/query.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/secret/addSecret.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/secret/listSecrets.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/secret/removeSecret.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/snippet/string.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/string/countStrings.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/string/getString.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/string/listStrings.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/addEmail.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/beginSignup.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/completeSignup.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/deleteEmail.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/dismissNewsletter.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/getNewsletterPreference.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/getSignup.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/listEmails.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/resendSignup.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/setPrimaryEmail.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/subscribeNewsletter.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/account/verifyEmail.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/actor/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/focus/beginSession.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/focus/endSession.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/focus/nextItem.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/deleteBranch.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/formatPatch.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/getArchive.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/getBlob.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/getBranch.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/getEntry.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/getMergeBase.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/getRef.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/getTag.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/getTree.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/listBranches.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/listCommits.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/listCommitsByRef.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/listTags.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/mergeCommit.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/git/syncFork.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/migrator/createTask.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/migrator/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/migrator/describeSource.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/migrator/getTask.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/migrator/listTasks.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/migrator/retryJob.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/newsletter/signup.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/notification/deleteNotification.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/notification/getPreferences.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/notification/getUnreadCount.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/notification/listNotifications.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/notification/listRecipients.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/notification/markAllRead.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/notification/markEntityRead.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/notification/updatePreferences.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/notification/updateSeen.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/repo/deploySite.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/repo/disableSite.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/repo/getDeployHistory.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/repo/getSiteConfig.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/repo/updateSiteConfig.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/review/listPatches.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/search/searchCode.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/server/ackHandover.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/server/cancelHandover.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/server/createInviteCode.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/server/handoverRepo.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/server/redeemInviteCode.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/server/seedRepo.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/server/uploadBlob.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/site/claimDomain.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/site/getDomainClaim.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/site/releaseDomain.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/moderation/ban.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/moderation/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/moderation/getBan.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/moderation/listBans.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/moderation/unban.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/quota/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/quota/get.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/quota/list.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/quota/releaseNars.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/quota/set.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/quota/unset.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/temp/spindle/quota/usage.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/track/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/track/getTicketView.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/track/ticket.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/webhook/createWebhook.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/webhook/defs.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/webhook/deleteWebhook.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/webhook/listWebhookDeliveries.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/webhook/listWebhooks.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/webhook/retryWebhookDelivery.ts create mode 100644 web/src/lib/api/lexicons/types/org/tangled/webhook/updateWebhook.ts create mode 100644 web/src/lib/api/lexicons/types/sh/tangled/git/defs.ts create mode 100644 web/src/lib/api/lexicons/types/sh/tangled/git/mergeCommit.ts create mode 100644 web/src/lib/api/lexicons/types/sh/tangled/git/temp/listIndexedLanguages.ts create mode 100644 web/src/lib/api/lexicons/types/sh/tangled/git/temp/listLanguages.ts create mode 100644 web/src/lib/api/lexicons/types/sh/tangled/git/temp2/getBlame.ts create mode 100644 web/src/lib/api/lexicons/types/sh/tangled/git/temp2/getCommitStats.ts create mode 100644 web/src/lib/api/lexicons/types/sh/tangled/git/temp2/getDiff.ts create mode 100644 web/src/lib/api/lexicons/types/sh/tangled/git/temp2/getInterdiff.ts create mode 100644 web/src/lib/api/lexicons/types/sh/tangled/git/temp2/mergeCheck.ts create mode 100644 web/src/lib/api/lexicons/types/sh/tangled/identity/resolveCommitters.ts create mode 100644 web/src/lib/api/lexicons/types/sh/tangled/repo/listCobVersions.ts create mode 100644 web/src/lib/api/liveCommitStatuses.svelte.ts create mode 100644 web/src/lib/api/load.test.ts create mode 100644 web/src/lib/api/load.ts create mode 100644 web/src/lib/api/lookupCache.test.ts create mode 100644 web/src/lib/api/lookupCache.ts create mode 100644 web/src/lib/api/markupBlobs.test.ts create mode 100644 web/src/lib/api/markupBlobs.ts create mode 100644 web/src/lib/api/migrator.test.ts create mode 100644 web/src/lib/api/migrator.ts create mode 100644 web/src/lib/api/moderation.ts create mode 100644 web/src/lib/api/moderationLabels.ts create mode 100644 web/src/lib/api/newsletter.test.ts create mode 100644 web/src/lib/api/newsletter.ts create mode 100644 web/src/lib/api/notifications.test.ts create mode 100644 web/src/lib/api/notifications.ts create mode 100644 web/src/lib/api/orgCreate.test.ts create mode 100644 web/src/lib/api/orgCreate.ts create mode 100644 web/src/lib/api/orgMembers.test.ts create mode 100644 web/src/lib/api/orgMembers.ts create mode 100644 web/src/lib/api/orgProfile.test.ts create mode 100644 web/src/lib/api/orgProfile.ts create mode 100644 web/src/lib/api/organizations.test.ts create mode 100644 web/src/lib/api/organizations.ts create mode 100644 web/src/lib/api/pagination.test.ts create mode 100644 web/src/lib/api/pagination.ts create mode 100644 web/src/lib/api/patchStream.test.ts create mode 100644 web/src/lib/api/patchStream.ts create mode 100644 web/src/lib/api/pinning.test.ts create mode 100644 web/src/lib/api/pinning.ts create mode 100644 web/src/lib/api/pipelineActions.test.ts create mode 100644 web/src/lib/api/pipelineActions.ts create mode 100644 web/src/lib/api/preferences.test.ts create mode 100644 web/src/lib/api/preferences.ts create mode 100644 web/src/lib/api/profile.test.ts create mode 100644 web/src/lib/api/profile.ts create mode 100644 web/src/lib/api/profileIdentifier.test.ts create mode 100644 web/src/lib/api/profileIdentifier.ts create mode 100644 web/src/lib/api/profileSettings.test.ts create mode 100644 web/src/lib/api/profileSettings.ts create mode 100644 web/src/lib/api/pull.test.ts create mode 100644 web/src/lib/api/pull.ts create mode 100644 web/src/lib/api/pullCompose.test.ts create mode 100644 web/src/lib/api/pullCompose.ts create mode 100644 web/src/lib/api/pullRoute.test.ts create mode 100644 web/src/lib/api/pullRoute.ts create mode 100644 web/src/lib/api/rawCommit.ts create mode 100644 web/src/lib/api/reaction.ts create mode 100644 web/src/lib/api/records.ts create mode 100644 web/src/lib/api/repo.test.ts create mode 100644 web/src/lib/api/repo.ts create mode 100644 web/src/lib/api/repoCompare.ts create mode 100644 web/src/lib/api/repoCounts.ts create mode 100644 web/src/lib/api/repoCreate.test.ts create mode 100644 web/src/lib/api/repoCreate.ts create mode 100644 web/src/lib/api/repoDiff.test.ts create mode 100644 web/src/lib/api/repoDiff.ts create mode 100644 web/src/lib/api/repoIndex.test.ts create mode 100644 web/src/lib/api/repoIndex.ts create mode 100644 web/src/lib/api/repoSettings.test.ts create mode 100644 web/src/lib/api/repoSettings.ts create mode 100644 web/src/lib/api/search.test.ts create mode 100644 web/src/lib/api/search.ts create mode 100644 web/src/lib/api/secrets.test.ts create mode 100644 web/src/lib/api/secrets.ts create mode 100644 web/src/lib/api/settings.test.ts create mode 100644 web/src/lib/api/settings.ts create mode 100644 web/src/lib/api/signup.test.ts create mode 100644 web/src/lib/api/signup.ts create mode 100644 web/src/lib/api/sites.test.ts create mode 100644 web/src/lib/api/sites.ts create mode 100644 web/src/lib/api/spindle.test.ts create mode 100644 web/src/lib/api/spindle.ts create mode 100644 web/src/lib/api/spindleLogs.test.ts create mode 100644 web/src/lib/api/spindleLogs.ts create mode 100644 web/src/lib/api/spindlewrite.test.ts create mode 100644 web/src/lib/api/spindlewrite.ts create mode 100644 web/src/lib/api/strings.test.ts create mode 100644 web/src/lib/api/strings.ts create mode 100644 web/src/lib/api/syntax.test.ts create mode 100644 web/src/lib/api/syntax.ts create mode 100644 web/src/lib/api/ticket.test.ts create mode 100644 web/src/lib/api/ticket.ts create mode 100644 web/src/lib/api/ticketSearch.ts create mode 100644 web/src/lib/api/tranquil.test.ts create mode 100644 web/src/lib/api/tranquil.ts create mode 100644 web/src/lib/api/uri.test.ts create mode 100644 web/src/lib/api/uri.ts create mode 100644 web/src/lib/api/webhooks.test.ts create mode 100644 web/src/lib/api/webhooks.ts create mode 100644 web/src/lib/api/write.test.ts create mode 100644 web/src/lib/api/write.ts create mode 100644 web/src/lib/auth.svelte.test.ts create mode 100644 web/src/lib/auth.svelte.ts create mode 100644 web/src/lib/auth.test.ts create mode 100644 web/src/lib/auth/accounts.ts create mode 100644 web/src/lib/auth/agent.test.ts create mode 100644 web/src/lib/auth/agent.ts create mode 100644 web/src/lib/auth/guards.ts create mode 100644 web/src/lib/auth/handleResolver.test.ts create mode 100644 web/src/lib/auth/handleResolver.ts create mode 100644 web/src/lib/auth/onboarding.test.ts create mode 100644 web/src/lib/auth/onboarding.ts create mode 100644 web/src/lib/auth/orgSessions.test.ts create mode 100644 web/src/lib/auth/orgSessions.ts create mode 100644 web/src/lib/auth/pending.ts create mode 100644 web/src/lib/auth/returnTo.test.ts create mode 100644 web/src/lib/auth/returnTo.ts create mode 100644 web/src/lib/auth/scopes.test.ts create mode 100644 web/src/lib/auth/scopes.ts create mode 100644 web/src/lib/auth/session.ts create mode 100644 web/src/lib/avatar.test.ts create mode 100644 web/src/lib/avatar.ts create mode 100644 web/src/lib/avatarGradient.test.ts create mode 100644 web/src/lib/avatarGradient.ts create mode 100644 web/src/lib/avatarWidths.ts create mode 100644 web/src/lib/blog.ts create mode 100644 web/src/lib/bones/bones-forklist--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/bones-loadingprimitives--count.bones.json create mode 100644 web/src/lib/bones/bones-loadingprimitives--short-text.bones.json create mode 100644 web/src/lib/bones/bones-radio--loading-row.bones.json create mode 100644 web/src/lib/bones/bones-starredby--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/bones-treeview--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/notifications-notificationlist--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/notifications-notificationlist--social-skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/profile-followcard--bordered.bones.json create mode 100644 web/src/lib/bones/profile-networkvouches--many-vouches.bones.json create mode 100644 web/src/lib/bones/profile-overviewtab--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/profile-profilechrome--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/profile-stringcard--default.bones.json create mode 100644 web/src/lib/bones/repo-blobview--code-file.bones.json create mode 100644 web/src/lib/bones/repo-blobview--markdown-skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-branchtable--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-commitlogview--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-commitview--full-commit.bones.json create mode 100644 web/src/lib/bones/repo-diffview--skeleton-fixture-collapsed.bones.json create mode 100644 web/src/lib/bones/repo-diffview--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-filehistoryview--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-issues-issueform--create.bones.json create mode 100644 web/src/lib/bones/repo-issues-issuethreadview--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-pipelines-pipelinelist--default.bones.json create mode 100644 web/src/lib/bones/repo-pipelines-pipelinetoolbar--all.bones.json create mode 100644 web/src/lib/bones/repo-pipelines-workflowrunview--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-pulls-pullcompose--default.bones.json create mode 100644 web/src/lib/bones/repo-pulls-pullviewpage--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-readme--rendered-sanitized-html.bones.json create mode 100644 web/src/lib/bones/repo-repocard--with-star-button.bones.json create mode 100644 web/src/lib/bones/repo-repochrome--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-repoforkform--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-repoheader--full-metadata.bones.json create mode 100644 web/src/lib/bones/repo-repoindexview--panel-skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-repoindexview--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-tagcard--annotated.bones.json create mode 100644 web/src/lib/bones/repo-taglist--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/repo-tickets-ticketlist--issues.bones.json create mode 100644 web/src/lib/bones/repo-tickets-ticketlist--pulls.bones.json create mode 100644 web/src/lib/bones/repo-tickets-tickettoolbar--issues-open.bones.json create mode 100644 web/src/lib/bones/repo-tickets-tickettoolbar--pulls-open.bones.json create mode 100644 web/src/lib/bones/settings-settingspanel--content-skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/strings-stringview--code-file.bones.json create mode 100644 web/src/lib/bones/strings-stringview--markdown-skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/timeline-notificationspreview--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/timeline-recentspreview--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/timeline-timeline--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/timeline-trendingreposlist--skeleton-fixture.bones.json create mode 100644 web/src/lib/bones/ui-toggle--checked.bones.json create mode 100644 web/src/lib/cloneUrl.test.ts create mode 100644 web/src/lib/cloneUrl.ts create mode 100644 web/src/lib/components/StatusDot.svelte create mode 100644 web/src/lib/components/attestations/AttestationBanner.svelte create mode 100644 web/src/lib/components/auth/AccountSelector.stories.svelte create mode 100644 web/src/lib/components/auth/AccountSelector.svelte create mode 100644 web/src/lib/components/comment/Comment.stories.svelte create mode 100644 web/src/lib/components/comment/Comment.svelte create mode 100644 web/src/lib/components/comment/CommentArrivalFixture.svelte create mode 100644 web/src/lib/components/comment/CommentBox.stories.svelte create mode 100644 web/src/lib/components/comment/CommentBox.svelte create mode 100644 web/src/lib/components/comment/CommentCard.stories.svelte create mode 100644 web/src/lib/components/comment/CommentCard.svelte create mode 100644 web/src/lib/components/comment/CommentEditor.stories.svelte create mode 100644 web/src/lib/components/comment/CommentEditor.svelte create mode 100644 web/src/lib/components/comment/CommentList.stories.svelte create mode 100644 web/src/lib/components/comment/CommentList.svelte create mode 100644 web/src/lib/components/comment/CommentSection.svelte create mode 100644 web/src/lib/components/comment/OlderComments.svelte create mode 100644 web/src/lib/components/comment/comments.test.ts create mode 100644 web/src/lib/components/comment/comments.ts create mode 100644 web/src/lib/components/comment/reveal.ts create mode 100644 web/src/lib/components/editor/PierreEditor.svelte create mode 100644 web/src/lib/components/icons/Bluesky.svelte create mode 100644 web/src/lib/components/icons/BookArrowRight.svelte create mode 100644 web/src/lib/components/icons/DollyCheck.svelte create mode 100644 web/src/lib/components/icons/EyeFilled.svelte create mode 100644 web/src/lib/components/icons/FolderOpenDuo.svelte create mode 100644 web/src/lib/components/icons/ShieldDirectDenounce.svelte create mode 100644 web/src/lib/components/icons/ShieldDirectVouch.svelte create mode 100644 web/src/lib/components/icons/ShieldIndirectDenounce.svelte create mode 100644 web/src/lib/components/icons/ShieldIndirectVouch.svelte create mode 100644 web/src/lib/components/icons/ShieldMixed.svelte create mode 100644 web/src/lib/components/icons/UserRoundArrowForward.svelte create mode 100644 web/src/lib/components/import/ConnectAccount.svelte create mode 100644 web/src/lib/components/import/GitHubAuthorizeStep.svelte create mode 100644 web/src/lib/components/import/GitHubConnectStep.svelte create mode 100644 web/src/lib/components/import/GitHubImporter.stories.svelte create mode 100644 web/src/lib/components/import/GitHubImporter.svelte create mode 100644 web/src/lib/components/import/GitHubImporterFixture.svelte create mode 100644 web/src/lib/components/import/GitHubRepoSelectStep.svelte create mode 100644 web/src/lib/components/import/MethodPicker.svelte create mode 100644 web/src/lib/components/import/MigrationRow.svelte create mode 100644 web/src/lib/components/import/MigrationStep.svelte create mode 100644 web/src/lib/components/import/RepoConfig.svelte create mode 100644 web/src/lib/components/import/RepoRow.svelte create mode 100644 web/src/lib/components/import/StartedBanner.svelte create mode 100644 web/src/lib/components/import/UrlRepoImporter.svelte create mode 100644 web/src/lib/components/import/gitHubImportState.svelte.ts create mode 100644 web/src/lib/components/import/gitHubImportState.test.ts create mode 100644 web/src/lib/components/import/grantEvidence.test.ts create mode 100644 web/src/lib/components/import/grantEvidence.ts create mode 100644 web/src/lib/components/import/jobMatcher.ts create mode 100644 web/src/lib/components/import/migrationRows.test.ts create mode 100644 web/src/lib/components/import/migrationRows.ts create mode 100644 web/src/lib/components/import/stepGating.test.ts create mode 100644 web/src/lib/components/import/stepGating.ts create mode 100644 web/src/lib/components/landing/EmbossFilter.svelte create mode 100644 web/src/lib/components/landing/Landing.stories.svelte create mode 100644 web/src/lib/components/landing/Landing.svelte create mode 100644 web/src/lib/components/landing/LiveLanding.svelte create mode 100644 web/src/lib/components/landing/NotFound.stories.svelte create mode 100644 web/src/lib/components/landing/NotFound.svelte create mode 100644 web/src/lib/components/landing/assets/cards/ci-cable-after-light.svg create mode 100644 web/src/lib/components/landing/assets/cards/ci-cable-after.svg create mode 100644 web/src/lib/components/landing/assets/cards/ci-cable-before-light.svg create mode 100644 web/src/lib/components/landing/assets/cards/ci-cable-before.svg create mode 100644 web/src/lib/components/landing/assets/cards/ci-vps-cap-l.svg create mode 100644 web/src/lib/components/landing/assets/cards/ci-vps-cap-r.svg create mode 100644 web/src/lib/components/landing/assets/cards/ci-vps-grill.svg create mode 100644 web/src/lib/components/landing/assets/knot/loop-left-edge.svg create mode 100644 web/src/lib/components/landing/assets/knot/loop-left.svg create mode 100644 web/src/lib/components/landing/assets/knot/loop-right-edge.svg create mode 100644 web/src/lib/components/landing/assets/knot/loop-right.svg create mode 100644 web/src/lib/components/landing/assets/knot/strand-bottom.svg create mode 100644 web/src/lib/components/landing/assets/knot/strand-top.svg create mode 100644 web/src/lib/components/landing/assets/not-found/strand-bottom.svg create mode 100644 web/src/lib/components/landing/assets/not-found/strand-end.svg create mode 100644 web/src/lib/components/landing/assets/not-found/strand-top.svg create mode 100644 web/src/lib/components/landing/cta/Cta.stories.svelte create mode 100644 web/src/lib/components/landing/cta/Cta.svelte create mode 100644 web/src/lib/components/landing/cta/cta-knot.ts create mode 100644 web/src/lib/components/landing/features/AgentCard.svelte create mode 100644 web/src/lib/components/landing/features/Beat.svelte create mode 100644 web/src/lib/components/landing/features/CiDevice.stories.svelte create mode 100644 web/src/lib/components/landing/features/CiDevice.svelte create mode 100644 web/src/lib/components/landing/features/CiStep.stories.svelte create mode 100644 web/src/lib/components/landing/features/CiStep.svelte create mode 100644 web/src/lib/components/landing/features/Features.stories.svelte create mode 100644 web/src/lib/components/landing/features/Features.svelte create mode 100644 web/src/lib/components/landing/features/GlobeCard.stories.svelte create mode 100644 web/src/lib/components/landing/features/GlobeCard.svelte create mode 100644 web/src/lib/components/landing/features/LearnMore.stories.svelte create mode 100644 web/src/lib/components/landing/features/LearnMore.svelte create mode 100644 web/src/lib/components/landing/features/PrCard.stories.svelte create mode 100644 web/src/lib/components/landing/features/PrCard.svelte create mode 100644 web/src/lib/components/landing/features/VouchCard.stories.svelte create mode 100644 web/src/lib/components/landing/features/VouchCard.svelte create mode 100644 web/src/lib/components/landing/features/agents.css create mode 100644 web/src/lib/components/landing/features/ci.css create mode 100644 web/src/lib/components/landing/features/ci.svelte.ts create mode 100644 web/src/lib/components/landing/features/features.css create mode 100644 web/src/lib/components/landing/features/globe.css create mode 100644 web/src/lib/components/landing/features/pr-stack.ts create mode 100644 web/src/lib/components/landing/features/pr.css create mode 100644 web/src/lib/components/landing/features/vouch.css create mode 100644 web/src/lib/components/landing/guest.svelte.ts create mode 100644 web/src/lib/components/landing/hackable/Corners.svelte create mode 100644 web/src/lib/components/landing/hackable/DollyCanvas.stories.svelte create mode 100644 web/src/lib/components/landing/hackable/DollyCanvas.svelte create mode 100644 web/src/lib/components/landing/hackable/Hackable.stories.svelte create mode 100644 web/src/lib/components/landing/hackable/Hackable.svelte create mode 100644 web/src/lib/components/landing/hackable/dolly.test.ts create mode 100644 web/src/lib/components/landing/hackable/dolly.ts create mode 100644 web/src/lib/components/landing/hackable/hackable.css create mode 100644 web/src/lib/components/landing/hero/Hero.stories.svelte create mode 100644 web/src/lib/components/landing/hero/Hero.svelte create mode 100644 web/src/lib/components/landing/hero/IsoStack.stories.svelte create mode 100644 web/src/lib/components/landing/hero/IsoStack.svelte create mode 100644 web/src/lib/components/landing/hero/activity.test.ts create mode 100644 web/src/lib/components/landing/hero/activity.ts create mode 100644 web/src/lib/components/landing/hero/mock.ts create mode 100644 web/src/lib/components/landing/hero/notifications.ts create mode 100644 web/src/lib/components/landing/iso-stack.css create mode 100644 web/src/lib/components/landing/landing.css create mode 100644 web/src/lib/components/landing/legal/LegalPage.svelte create mode 100644 web/src/lib/components/landing/live.svelte.ts create mode 100644 web/src/lib/components/landing/material.css create mode 100644 web/src/lib/components/landing/motion/spinning-globe.svelte.ts create mode 100644 web/src/lib/components/landing/selfhost/SelfHost.stories.svelte create mode 100644 web/src/lib/components/landing/selfhost/SelfHost.svelte create mode 100644 web/src/lib/components/landing/selfhost/electronics/FloppyComputer.stories.svelte create mode 100644 web/src/lib/components/landing/selfhost/electronics/FloppyComputer.svelte create mode 100644 web/src/lib/components/landing/selfhost/electronics/Guide.svelte create mode 100644 web/src/lib/components/landing/selfhost/electronics/Logo.svelte create mode 100644 web/src/lib/components/landing/selfhost/electronics/PiCartridge.stories.svelte create mode 100644 web/src/lib/components/landing/selfhost/electronics/PiCartridge.svelte create mode 100644 web/src/lib/components/landing/selfhost/electronics/Plate.svelte create mode 100644 web/src/lib/components/landing/selfhost/electronics/SnesCartridge.stories.svelte create mode 100644 web/src/lib/components/landing/selfhost/electronics/SnesCartridge.svelte create mode 100644 web/src/lib/components/landing/selfhost/electronics/Solid.svelte create mode 100644 web/src/lib/components/landing/selfhost/electronics/insert.svelte.ts create mode 100644 web/src/lib/components/landing/selfhost/electronics/iso.test.ts create mode 100644 web/src/lib/components/landing/selfhost/electronics/iso.ts create mode 100644 web/src/lib/components/landing/selfhost/electronics/loop.svelte.ts create mode 100644 web/src/lib/components/landing/selfhost/electronics/pi.test.ts create mode 100644 web/src/lib/components/landing/selfhost/electronics/pi.ts create mode 100644 web/src/lib/components/landing/selfhost/selfhost.css create mode 100644 web/src/lib/components/landing/shell/CopyLink.stories.svelte create mode 100644 web/src/lib/components/landing/shell/CopyLink.svelte create mode 100644 web/src/lib/components/landing/shell/Footer.stories.svelte create mode 100644 web/src/lib/components/landing/shell/Footer.svelte create mode 100644 web/src/lib/components/landing/shell/MobileMenu.stories.svelte create mode 100644 web/src/lib/components/landing/shell/MobileMenu.svelte create mode 100644 web/src/lib/components/landing/shell/Navbar.stories.svelte create mode 100644 web/src/lib/components/landing/shell/Navbar.svelte create mode 100644 web/src/lib/components/landing/shell/SiteLink.svelte create mode 100644 web/src/lib/components/landing/shell/links.ts create mode 100644 web/src/lib/components/landing/sound.ts create mode 100644 web/src/lib/components/landing/stories/LandingFrame.svelte create mode 100644 web/src/lib/components/landing/stories/fixtures.ts create mode 100644 web/src/lib/components/landing/stories/frame.ts create mode 100644 web/src/lib/components/landing/thread/Knot.stories.svelte create mode 100644 web/src/lib/components/landing/thread/Knot.svelte create mode 100644 web/src/lib/components/landing/thread/Ribbon.stories.svelte create mode 100644 web/src/lib/components/landing/thread/Ribbon.svelte create mode 100644 web/src/lib/components/landing/thread/ThreadRail.svelte create mode 100644 web/src/lib/components/landing/thread/geometry.test.ts create mode 100644 web/src/lib/components/landing/thread/geometry.ts create mode 100644 web/src/lib/components/landing/thread/thread.svelte.ts create mode 100644 web/src/lib/components/landing/ui/Container.svelte create mode 100644 web/src/lib/components/landing/ui/EmailForm.svelte create mode 100644 web/src/lib/components/landing/ui/Heading.svelte create mode 100644 web/src/lib/components/landing/ui/Section.svelte create mode 100644 web/src/lib/components/landing/updates/RecentUpdates.stories.svelte create mode 100644 web/src/lib/components/landing/updates/RecentUpdates.svelte create mode 100644 web/src/lib/components/landing/updates/UpdateCard.stories.svelte create mode 100644 web/src/lib/components/landing/updates/UpdateCard.svelte create mode 100644 web/src/lib/components/landing/updates/updates.css create mode 100644 web/src/lib/components/landing/updates/updates.test.ts create mode 100644 web/src/lib/components/landing/updates/updates.ts create mode 100644 web/src/lib/components/lines/LineSearch.svelte create mode 100644 web/src/lib/components/lines/diff.test.ts create mode 100644 web/src/lib/components/lines/diff.ts create mode 100644 web/src/lib/components/lines/finder.svelte.ts create mode 100644 web/src/lib/components/lines/highlight.ts create mode 100644 web/src/lib/components/lines/search.test.ts create mode 100644 web/src/lib/components/lines/search.ts create mode 100644 web/src/lib/components/migration/MigrationLanding.stories.svelte create mode 100644 web/src/lib/components/migration/MigrationLanding.svelte create mode 100644 web/src/lib/components/moderation/ModerationNotice.stories.svelte create mode 100644 web/src/lib/components/moderation/ModerationNotice.svelte create mode 100644 web/src/lib/components/moderation/ReportDialog.stories.svelte create mode 100644 web/src/lib/components/moderation/ReportDialog.svelte create mode 100644 web/src/lib/components/moderation/ReportItems.stories.svelte create mode 100644 web/src/lib/components/moderation/ReportItems.svelte create mode 100644 web/src/lib/components/moderation/gate.svelte.ts create mode 100644 web/src/lib/components/moderation/reportReasons.ts create mode 100644 web/src/lib/components/moderation/reporting.svelte.ts create mode 100644 web/src/lib/components/notifications/NotificationBell.stories.svelte create mode 100644 web/src/lib/components/notifications/NotificationBell.svelte create mode 100644 web/src/lib/components/notifications/NotificationItem.stories.svelte create mode 100644 web/src/lib/components/notifications/NotificationItem.svelte create mode 100644 web/src/lib/components/notifications/NotificationItem.test.ts create mode 100644 web/src/lib/components/notifications/NotificationList.stories.svelte create mode 100644 web/src/lib/components/notifications/NotificationList.svelte create mode 100644 web/src/lib/components/notifications/NotificationToolbar.stories.svelte create mode 100644 web/src/lib/components/notifications/NotificationToolbar.svelte create mode 100644 web/src/lib/components/notifications/mock.ts create mode 100644 web/src/lib/components/notifications/notification.test.ts create mode 100644 web/src/lib/components/notifications/notification.ts create mode 100644 web/src/lib/components/notifications/types.ts create mode 100644 web/src/lib/components/organization/OrganizationCreationForm.stories.svelte create mode 100644 web/src/lib/components/organization/OrganizationCreationForm.svelte create mode 100644 web/src/lib/components/organization/welcome/OrganizationFinishStep.svelte create mode 100644 web/src/lib/components/organization/welcome/OrganizationMembersStep.stories.svelte create mode 100644 web/src/lib/components/organization/welcome/OrganizationMembersStep.svelte create mode 100644 web/src/lib/components/organization/welcome/OrganizationProfileStep.svelte create mode 100644 web/src/lib/components/profile/AvatarList.stories.svelte create mode 100644 web/src/lib/components/profile/AvatarList.svelte create mode 100644 web/src/lib/components/profile/FollowButton.stories.svelte create mode 100644 web/src/lib/components/profile/FollowButton.svelte create mode 100644 web/src/lib/components/profile/FollowCard.stories.svelte create mode 100644 web/src/lib/components/profile/FollowCard.svelte create mode 100644 web/src/lib/components/profile/FollowCardContent.svelte create mode 100644 web/src/lib/components/profile/FollowerFollowing.svelte create mode 100644 web/src/lib/components/profile/ListViewSwitch.svelte create mode 100644 web/src/lib/components/profile/NetworkVouches.stories.svelte create mode 100644 web/src/lib/components/profile/NetworkVouches.svelte create mode 100644 web/src/lib/components/profile/NotJoinedProfile.svelte create mode 100644 web/src/lib/components/profile/PagedList.svelte create mode 100644 web/src/lib/components/profile/PagedListSkeleton.svelte create mode 100644 web/src/lib/components/profile/PinSuggestions.svelte create mode 100644 web/src/lib/components/profile/PinnedReposEditor.svelte create mode 100644 web/src/lib/components/profile/PinnedRepositories.svelte create mode 100644 web/src/lib/components/profile/ProfileActivityChart.svelte create mode 100644 web/src/lib/components/profile/ProfileBio.svelte create mode 100644 web/src/lib/components/profile/ProfileChrome.stories.svelte create mode 100644 web/src/lib/components/profile/ProfileChrome.svelte create mode 100644 web/src/lib/components/profile/ProfileEditForm.svelte create mode 100644 web/src/lib/components/profile/ProfileMoreMenu.svelte create mode 100644 web/src/lib/components/profile/ProfilePopover.stories.svelte create mode 100644 web/src/lib/components/profile/ProfilePopover.svelte create mode 100644 web/src/lib/components/profile/ProfileShell.svelte create mode 100644 web/src/lib/components/profile/ProfileSocialLinks.svelte create mode 100644 web/src/lib/components/profile/ProfileTabs.svelte create mode 100644 web/src/lib/components/profile/RepoSearchForm.svelte create mode 100644 web/src/lib/components/profile/StringCard.stories.svelte create mode 100644 web/src/lib/components/profile/StringCard.svelte create mode 100644 web/src/lib/components/profile/StringCardContent.svelte create mode 100644 web/src/lib/components/profile/UserProfileChrome.svelte create mode 100644 web/src/lib/components/profile/VouchAvatar.stories.svelte create mode 100644 web/src/lib/components/profile/VouchAvatar.svelte create mode 100644 web/src/lib/components/profile/VouchButton.stories.svelte create mode 100644 web/src/lib/components/profile/VouchButton.svelte create mode 100644 web/src/lib/components/profile/VouchCard.svelte create mode 100644 web/src/lib/components/profile/bio.test.ts create mode 100644 web/src/lib/components/profile/bio.ts create mode 100644 web/src/lib/components/profile/counts.svelte.ts create mode 100644 web/src/lib/components/profile/listView.svelte.ts create mode 100644 web/src/lib/components/profile/organization/OrganizationChrome.svelte create mode 100644 web/src/lib/components/profile/organization/OrganizationOverviewTab.svelte create mode 100644 web/src/lib/components/profile/pages.test.ts create mode 100644 web/src/lib/components/profile/pages.ts create mode 100644 web/src/lib/components/profile/pagesPinned.test.ts create mode 100644 web/src/lib/components/profile/pagination.svelte.ts create mode 100644 web/src/lib/components/profile/pagination.test.ts create mode 100644 web/src/lib/components/profile/pinnedEdit.svelte.ts create mode 100644 web/src/lib/components/profile/pinnedReorder.test.ts create mode 100644 web/src/lib/components/profile/pinnedReorder.ts create mode 100644 web/src/lib/components/profile/profileActivityChart.test.ts create mode 100644 web/src/lib/components/profile/profileActivityChart.ts create mode 100644 web/src/lib/components/profile/profileLinks.test.ts create mode 100644 web/src/lib/components/profile/profileLinks.ts create mode 100644 web/src/lib/components/profile/tabs/OverviewTab.stories.svelte create mode 100644 web/src/lib/components/profile/tabs/OverviewTab.svelte create mode 100644 web/src/lib/components/profile/tabs/PeopleTab.svelte create mode 100644 web/src/lib/components/profile/tabs/RepoListTab.stories.svelte create mode 100644 web/src/lib/components/profile/tabs/RepoListTab.svelte create mode 100644 web/src/lib/components/profile/tabs/StarredTab.svelte create mode 100644 web/src/lib/components/profile/tabs/StringListTab.svelte create mode 100644 web/src/lib/components/profile/tabs/VouchTab.stories.svelte create mode 100644 web/src/lib/components/profile/tabs/VouchTab.svelte create mode 100644 web/src/lib/components/profile/types.ts create mode 100644 web/src/lib/components/profile/vouchBadge.svelte.ts create mode 100644 web/src/lib/components/profile/vouchBadge.test.ts create mode 100644 web/src/lib/components/profile/vouchRelationship.test.ts create mode 100644 web/src/lib/components/profile/vouchRelationship.ts create mode 100644 web/src/lib/components/reaction/Reaction.svelte create mode 100644 web/src/lib/components/reaction/ReactionPicker.stories.svelte create mode 100644 web/src/lib/components/reaction/ReactionPicker.svelte create mode 100644 web/src/lib/components/reaction/ReactionSummary.svelte create mode 100644 web/src/lib/components/reaction/Reactions.stories.svelte create mode 100644 web/src/lib/components/reaction/Reactions.svelte create mode 100644 web/src/lib/components/reaction/ReactorList.svelte create mode 100644 web/src/lib/components/reaction/reactions.svelte.ts create mode 100644 web/src/lib/components/reaction/reactions.test.ts create mode 100644 web/src/lib/components/reaction/reactions.ts create mode 100644 web/src/lib/components/repo/BlobHeader.stories.svelte create mode 100644 web/src/lib/components/repo/BlobHeader.svelte create mode 100644 web/src/lib/components/repo/BlobView.stories.svelte create mode 100644 web/src/lib/components/repo/BlobView.svelte create mode 100644 web/src/lib/components/repo/BranchList.stories.svelte create mode 100644 web/src/lib/components/repo/BranchList.svelte create mode 100644 web/src/lib/components/repo/BranchTable.stories.svelte create mode 100644 web/src/lib/components/repo/BranchTable.svelte create mode 100644 web/src/lib/components/repo/CloneDropdown.stories.svelte create mode 100644 web/src/lib/components/repo/CloneDropdown.svelte create mode 100644 web/src/lib/components/repo/CommitAuthor.svelte create mode 100644 web/src/lib/components/repo/CommitBodyToggle.svelte create mode 100644 web/src/lib/components/repo/CommitHeader.stories.svelte create mode 100644 web/src/lib/components/repo/CommitHeader.svelte create mode 100644 web/src/lib/components/repo/CommitList.stories.svelte create mode 100644 web/src/lib/components/repo/CommitList.svelte create mode 100644 web/src/lib/components/repo/CommitLogView.stories.svelte create mode 100644 web/src/lib/components/repo/CommitLogView.svelte create mode 100644 web/src/lib/components/repo/CommitRow.stories.svelte create mode 100644 web/src/lib/components/repo/CommitRow.svelte create mode 100644 web/src/lib/components/repo/CommitRows.svelte create mode 100644 web/src/lib/components/repo/CommitShaButton.svelte create mode 100644 web/src/lib/components/repo/CommitSubject.svelte create mode 100644 web/src/lib/components/repo/CommitView.stories.svelte create mode 100644 web/src/lib/components/repo/CommitView.svelte create mode 100644 web/src/lib/components/repo/CompareView.svelte create mode 100644 web/src/lib/components/repo/DiffCodeView.svelte create mode 100644 web/src/lib/components/repo/DiffFileHeader.svelte create mode 100644 web/src/lib/components/repo/DiffFileTree.stories.svelte create mode 100644 web/src/lib/components/repo/DiffFileTree.svelte create mode 100644 web/src/lib/components/repo/DiffFiles.svelte create mode 100644 web/src/lib/components/repo/DiffStatPill.stories.svelte create mode 100644 web/src/lib/components/repo/DiffStatPill.svelte create mode 100644 web/src/lib/components/repo/DiffView.stories.svelte create mode 100644 web/src/lib/components/repo/DiffView.svelte create mode 100644 web/src/lib/components/repo/EmptyRepo.stories.svelte create mode 100644 web/src/lib/components/repo/EmptyRepo.svelte create mode 100644 web/src/lib/components/repo/EntitySubscribeButton.stories.svelte create mode 100644 web/src/lib/components/repo/EntitySubscribeButton.svelte create mode 100644 web/src/lib/components/repo/FileCardHeader.stories.svelte create mode 100644 web/src/lib/components/repo/FileCardHeader.svelte create mode 100644 web/src/lib/components/repo/FileHistoryView.stories.svelte create mode 100644 web/src/lib/components/repo/FileHistoryView.svelte create mode 100644 web/src/lib/components/repo/FilePathChip.svelte create mode 100644 web/src/lib/components/repo/FileTree.stories.svelte create mode 100644 web/src/lib/components/repo/FileTree.svelte create mode 100644 web/src/lib/components/repo/ForkList.bones.stories.svelte create mode 100644 web/src/lib/components/repo/ForkList.svelte create mode 100644 web/src/lib/components/repo/KnotUpgradeTag.stories.svelte create mode 100644 web/src/lib/components/repo/KnotUpgradeTag.svelte create mode 100644 web/src/lib/components/repo/LabelPanel.stories.svelte create mode 100644 web/src/lib/components/repo/LabelPanel.svelte create mode 100644 web/src/lib/components/repo/LabelSummary.svelte create mode 100644 web/src/lib/components/repo/LanguageList.stories.svelte create mode 100644 web/src/lib/components/repo/LanguageList.svelte create mode 100644 web/src/lib/components/repo/LanguageStats.stories.svelte create mode 100644 web/src/lib/components/repo/LanguageStats.svelte create mode 100644 web/src/lib/components/repo/LastCommitPanel.stories.svelte create mode 100644 web/src/lib/components/repo/LastCommitPanel.svelte create mode 100644 web/src/lib/components/repo/LineHandles.svelte create mode 100644 web/src/lib/components/repo/OwnerTitle.svelte create mode 100644 web/src/lib/components/repo/PanelHeader.stories.svelte create mode 100644 web/src/lib/components/repo/PanelHeader.svelte create mode 100644 web/src/lib/components/repo/PierreFile.stories.svelte create mode 100644 web/src/lib/components/repo/PierreFile.svelte create mode 100644 web/src/lib/components/repo/Readme.stories.svelte create mode 100644 web/src/lib/components/repo/Readme.svelte create mode 100644 web/src/lib/components/repo/RefChip.stories.svelte create mode 100644 web/src/lib/components/repo/RefChip.svelte create mode 100644 web/src/lib/components/repo/RefLists.stories.svelte create mode 100644 web/src/lib/components/repo/RefSelector.stories.svelte create mode 100644 web/src/lib/components/repo/RefSelector.svelte create mode 100644 web/src/lib/components/repo/RepoCard.stories.svelte create mode 100644 web/src/lib/components/repo/RepoCard.svelte create mode 100644 web/src/lib/components/repo/RepoCardContent.svelte create mode 100644 web/src/lib/components/repo/RepoChrome.stories.svelte create mode 100644 web/src/lib/components/repo/RepoChrome.svelte create mode 100644 web/src/lib/components/repo/RepoCreationForm.svelte create mode 100644 web/src/lib/components/repo/RepoCrumbs.stories.svelte create mode 100644 web/src/lib/components/repo/RepoCrumbs.svelte create mode 100644 web/src/lib/components/repo/RepoForkForm.stories.svelte create mode 100644 web/src/lib/components/repo/RepoForkForm.svelte create mode 100644 web/src/lib/components/repo/RepoHeader.stories.svelte create mode 100644 web/src/lib/components/repo/RepoHeader.svelte create mode 100644 web/src/lib/components/repo/RepoIndexPanelSkeleton.svelte create mode 100644 web/src/lib/components/repo/RepoIndexView.stories.svelte create mode 100644 web/src/lib/components/repo/RepoIndexView.svelte create mode 100644 web/src/lib/components/repo/RepoReadmeSkeleton.svelte create mode 100644 web/src/lib/components/repo/RepoSection.svelte create mode 100644 web/src/lib/components/repo/RepoTabs.stories.svelte create mode 100644 web/src/lib/components/repo/RepoTabs.svelte create mode 100644 web/src/lib/components/repo/RepoToolbar.stories.svelte create mode 100644 web/src/lib/components/repo/RepoToolbar.svelte create mode 100644 web/src/lib/components/repo/RevId.svelte create mode 100644 web/src/lib/components/repo/ServicePicker.stories.svelte create mode 100644 web/src/lib/components/repo/ServicePicker.svelte create mode 100644 web/src/lib/components/repo/SourceSelector.svelte create mode 100644 web/src/lib/components/repo/StarButton.stories.svelte create mode 100644 web/src/lib/components/repo/StarButton.svelte create mode 100644 web/src/lib/components/repo/StarredBy.bones.stories.svelte create mode 100644 web/src/lib/components/repo/StarredBy.svelte create mode 100644 web/src/lib/components/repo/StarredByContent.svelte create mode 100644 web/src/lib/components/repo/SubjectLabelPanel.svelte create mode 100644 web/src/lib/components/repo/SubscribeButton.stories.svelte create mode 100644 web/src/lib/components/repo/SubscribeButton.svelte create mode 100644 web/src/lib/components/repo/TagArtifacts.svelte create mode 100644 web/src/lib/components/repo/TagCard.stories.svelte create mode 100644 web/src/lib/components/repo/TagCard.svelte create mode 100644 web/src/lib/components/repo/TagList.stories.svelte create mode 100644 web/src/lib/components/repo/TagList.svelte create mode 100644 web/src/lib/components/repo/TopicList.stories.svelte create mode 100644 web/src/lib/components/repo/TopicList.svelte create mode 100644 web/src/lib/components/repo/TreeHeader.stories.svelte create mode 100644 web/src/lib/components/repo/TreeHeader.svelte create mode 100644 web/src/lib/components/repo/TreeView.bones.stories.svelte create mode 100644 web/src/lib/components/repo/blame/BlameChip.svelte create mode 100644 web/src/lib/components/repo/blame/BlameOverlay.svelte create mode 100644 web/src/lib/components/repo/blame/hover.svelte.ts create mode 100644 web/src/lib/components/repo/blame/request.stories.svelte create mode 100644 web/src/lib/components/repo/blame/request.svelte.ts create mode 100644 web/src/lib/components/repo/blame/types.test.ts create mode 100644 web/src/lib/components/repo/blame/types.ts create mode 100644 web/src/lib/components/repo/branchOptions.ts create mode 100644 web/src/lib/components/repo/commitSignature.ts create mode 100644 web/src/lib/components/repo/diffQueue.test.ts create mode 100644 web/src/lib/components/repo/diffQueue.ts create mode 100644 web/src/lib/components/repo/diffRpc.ts create mode 100644 web/src/lib/components/repo/diffWorker.ts create mode 100644 web/src/lib/components/repo/fileAnchor.svelte.ts create mode 100644 web/src/lib/components/repo/fileDiff.test.ts create mode 100644 web/src/lib/components/repo/fileDiff.ts create mode 100644 web/src/lib/components/repo/importPoll.test.ts create mode 100644 web/src/lib/components/repo/importPoll.ts create mode 100644 web/src/lib/components/repo/issues/IssueForm.stories.svelte create mode 100644 web/src/lib/components/repo/issues/IssueForm.svelte create mode 100644 web/src/lib/components/repo/issues/IssueThreadView.stories.svelte create mode 100644 web/src/lib/components/repo/issues/IssueThreadView.svelte create mode 100644 web/src/lib/components/repo/languageColors.ts create mode 100644 web/src/lib/components/repo/lineAnchor.test.ts create mode 100644 web/src/lib/components/repo/lineAnchor.ts create mode 100644 web/src/lib/components/repo/pierre.ts create mode 100644 web/src/lib/components/repo/pierreDiff.css create mode 100644 web/src/lib/components/repo/pierreFile.css create mode 100644 web/src/lib/components/repo/pipelines/Elapsed.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/Elapsed.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineCard.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineCard.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineCardContent.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineEmptyState.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineList.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineList.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineRetryButton.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineRetryButton.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineSearch.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineSearch.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineStatusIcon.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineSymbol.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineToolbar.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineToolbar.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineWorkflows.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/PipelineWorkflows.svelte create mode 100644 web/src/lib/components/repo/pipelines/StepStatusIcon.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/StepStatusIcon.svelte create mode 100644 web/src/lib/components/repo/pipelines/WorkflowLogs.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/WorkflowLogs.svelte create mode 100644 web/src/lib/components/repo/pipelines/WorkflowRunView.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/WorkflowRunView.svelte create mode 100644 web/src/lib/components/repo/pipelines/WorkflowSidebar.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/WorkflowSidebar.svelte create mode 100644 web/src/lib/components/repo/pipelines/WorkflowSteps.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/WorkflowSteps.svelte create mode 100644 web/src/lib/components/repo/pipelines/WorkflowView.stories.svelte create mode 100644 web/src/lib/components/repo/pipelines/WorkflowView.svelte create mode 100644 web/src/lib/components/repo/pipelines/livePipelines.test.ts create mode 100644 web/src/lib/components/repo/pipelines/livePipelines.ts create mode 100644 web/src/lib/components/repo/pipelines/logStream.svelte.ts create mode 100644 web/src/lib/components/repo/pipelines/logs.test.ts create mode 100644 web/src/lib/components/repo/pipelines/logs.ts create mode 100644 web/src/lib/components/repo/pipelines/pipeline.test.ts create mode 100644 web/src/lib/components/repo/pipelines/pipeline.ts create mode 100644 web/src/lib/components/repo/pulls/PullActions.stories.svelte create mode 100644 web/src/lib/components/repo/pulls/PullActions.svelte create mode 100644 web/src/lib/components/repo/pulls/PullApproveCi.svelte create mode 100644 web/src/lib/components/repo/pulls/PullCommitList.stories.svelte create mode 100644 web/src/lib/components/repo/pulls/PullCommitList.svelte create mode 100644 web/src/lib/components/repo/pulls/PullCommitListHeader.stories.svelte create mode 100644 web/src/lib/components/repo/pulls/PullCommitListHeader.svelte create mode 100644 web/src/lib/components/repo/pulls/PullCommitRow.svelte create mode 100644 web/src/lib/components/repo/pulls/PullCompose.stories.svelte create mode 100644 web/src/lib/components/repo/pulls/PullCompose.svelte create mode 100644 web/src/lib/components/repo/pulls/PullDiscussion.stories.svelte create mode 100644 web/src/lib/components/repo/pulls/PullDiscussion.svelte create mode 100644 web/src/lib/components/repo/pulls/PullDiscussionArrivalFixture.svelte create mode 100644 web/src/lib/components/repo/pulls/PullReviewComment.stories.svelte create mode 100644 web/src/lib/components/repo/pulls/PullReviewComment.svelte create mode 100644 web/src/lib/components/repo/pulls/PullReviewCommentEditor.svelte create mode 100644 web/src/lib/components/repo/pulls/PullReviewCommentForm.stories.svelte create mode 100644 web/src/lib/components/repo/pulls/PullReviewCommentForm.svelte create mode 100644 web/src/lib/components/repo/pulls/PullVersionList.stories.svelte create mode 100644 web/src/lib/components/repo/pulls/PullVersionList.svelte create mode 100644 web/src/lib/components/repo/pulls/PullVersionSheet.svelte create mode 100644 web/src/lib/components/repo/pulls/RevIdButton.svelte create mode 100644 web/src/lib/components/repo/pulls/discussionWidth.ts create mode 100644 web/src/lib/components/repo/pulls/mirrorBehind.test.ts create mode 100644 web/src/lib/components/repo/pulls/mirrorBehind.ts create mode 100644 web/src/lib/components/repo/tickets/AssigneeField.svelte create mode 100644 web/src/lib/components/repo/tickets/TargetBranchField.svelte create mode 100644 web/src/lib/components/repo/tickets/Ticket.stories.svelte create mode 100644 web/src/lib/components/repo/tickets/Ticket.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketBody.stories.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketBody.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketCard.stories.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketCard.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketCardContent.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketForm.stories.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketForm.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketInfoBar.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketLabels.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketList.stories.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketList.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketQueryInput.stories.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketQueryInput.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketSearch.stories.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketSearch.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketStatePill.stories.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketStatePill.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketToolbar.stories.svelte create mode 100644 web/src/lib/components/repo/tickets/TicketToolbar.svelte create mode 100644 web/src/lib/components/repo/types.ts create mode 100644 web/src/lib/components/repo/urls.test.ts create mode 100644 web/src/lib/components/repo/urls.ts create mode 100644 web/src/lib/components/search/CodeResultCard.stories.svelte create mode 100644 web/src/lib/components/search/CodeResultCard.svelte create mode 100644 web/src/lib/components/search/CodeSearchView.svelte create mode 100644 web/src/lib/components/search/FileMatch.stories.svelte create mode 100644 web/src/lib/components/search/FileMatch.svelte create mode 100644 web/src/lib/components/search/QueryToken.svelte create mode 100644 web/src/lib/components/search/RepoSearchResults.svelte create mode 100644 web/src/lib/components/search/SearchBar.svelte create mode 100644 web/src/lib/components/search/SearchLayout.svelte create mode 100644 web/src/lib/components/search/SearchPage.stories.svelte create mode 100644 web/src/lib/components/search/SearchSidebar.svelte create mode 100644 web/src/lib/components/search/SearchTabs.svelte create mode 100644 web/src/lib/components/search/chunks.test.ts create mode 100644 web/src/lib/components/search/chunks.ts create mode 100644 web/src/lib/components/search/codeSearch.svelte.ts create mode 100644 web/src/lib/components/search/highlight.ts create mode 100644 web/src/lib/components/search/repoSearch.ts create mode 100644 web/src/lib/components/search/tabs/CodeSearchTab.svelte create mode 100644 web/src/lib/components/search/tabs/RepoSearchTab.svelte create mode 100644 web/src/lib/components/seo/Meta.svelte create mode 100644 web/src/lib/components/settings/AccountFarewell.svelte create mode 100644 web/src/lib/components/settings/AvatarForm.stories.svelte create mode 100644 web/src/lib/components/settings/AvatarForm.svelte create mode 100644 web/src/lib/components/settings/ChangeHandleForm.svelte create mode 100644 web/src/lib/components/settings/CodeChip.svelte create mode 100644 web/src/lib/components/settings/DocsButton.svelte create mode 100644 web/src/lib/components/settings/DrillDown.svelte create mode 100644 web/src/lib/components/settings/FormActions.svelte create mode 100644 web/src/lib/components/settings/FormRow.svelte create mode 100644 web/src/lib/components/settings/ProfileSettingsForm.stories.svelte create mode 100644 web/src/lib/components/settings/ProfileSettingsForm.svelte create mode 100644 web/src/lib/components/settings/RepoGeneralForm.svelte create mode 100644 web/src/lib/components/settings/SettingsBlock.svelte create mode 100644 web/src/lib/components/settings/SettingsEmpty.svelte create mode 100644 web/src/lib/components/settings/SettingsHeader.svelte create mode 100644 web/src/lib/components/settings/SettingsList.svelte create mode 100644 web/src/lib/components/settings/SettingsNav.stories.svelte create mode 100644 web/src/lib/components/settings/SettingsNav.svelte create mode 100644 web/src/lib/components/settings/SettingsPanel.stories.svelte create mode 100644 web/src/lib/components/settings/SettingsPanel.svelte create mode 100644 web/src/lib/components/settings/SettingsRow.svelte create mode 100644 web/src/lib/components/settings/SettingsSaveBar.svelte create mode 100644 web/src/lib/components/settings/SettingsSection.svelte create mode 100644 web/src/lib/components/settings/SettingsToolbar.svelte create mode 100644 web/src/lib/components/settings/ThemeChip.stories.svelte create mode 100644 web/src/lib/components/settings/ThemeChip.svelte create mode 100644 web/src/lib/components/settings/ThemeOption.stories.svelte create mode 100644 web/src/lib/components/settings/ThemeOption.svelte create mode 100644 web/src/lib/components/settings/ThemePicker.stories.svelte create mode 100644 web/src/lib/components/settings/ThemePicker.svelte create mode 100644 web/src/lib/components/settings/VerifyEmailCard.stories.svelte create mode 100644 web/src/lib/components/settings/VerifyEmailCard.svelte create mode 100644 web/src/lib/components/settings/descriptionLines.ts create mode 100644 web/src/lib/components/settings/tabs/EmailsTab.stories.svelte create mode 100644 web/src/lib/components/settings/tabs/EmailsTab.svelte create mode 100644 web/src/lib/components/settings/tabs/KeysTab.svelte create mode 100644 web/src/lib/components/settings/tabs/KnotsTab.svelte create mode 100644 web/src/lib/components/settings/tabs/NotificationsTab.svelte create mode 100644 web/src/lib/components/settings/tabs/OrganizationsTab.stories.svelte create mode 100644 web/src/lib/components/settings/tabs/OrganizationsTab.svelte create mode 100644 web/src/lib/components/settings/tabs/ProfileTab.svelte create mode 100644 web/src/lib/components/settings/tabs/SitesTab.svelte create mode 100644 web/src/lib/components/settings/tabs/SpindlesTab.svelte create mode 100644 web/src/lib/components/settings/tabs/emailRecheck.test.ts create mode 100644 web/src/lib/components/settings/tabs/emailRecheck.ts create mode 100644 web/src/lib/components/shell/Topbar.stories.svelte create mode 100644 web/src/lib/components/shell/Topbar.svelte create mode 100644 web/src/lib/components/shell/TopbarSearch.svelte create mode 100644 web/src/lib/components/shell/siteLinks.ts create mode 100644 web/src/lib/components/shell/topbarSearch.test.ts create mode 100644 web/src/lib/components/shell/topbarSearch.ts create mode 100644 web/src/lib/components/strings/StringForm.stories.svelte create mode 100644 web/src/lib/components/strings/StringForm.svelte create mode 100644 web/src/lib/components/strings/StringHeader.svelte create mode 100644 web/src/lib/components/strings/StringView.stories.svelte create mode 100644 web/src/lib/components/strings/StringView.svelte create mode 100644 web/src/lib/components/strings/urls.ts create mode 100644 web/src/lib/components/timeline/Announcements.stories.svelte create mode 100644 web/src/lib/components/timeline/Announcements.svelte create mode 100644 web/src/lib/components/timeline/NewsletterCard.svelte create mode 100644 web/src/lib/components/timeline/NotificationsPreview.stories.svelte create mode 100644 web/src/lib/components/timeline/NotificationsPreview.svelte create mode 100644 web/src/lib/components/timeline/PreviewCard.svelte create mode 100644 web/src/lib/components/timeline/PreviewEmpty.svelte create mode 100644 web/src/lib/components/timeline/RecentItem.stories.svelte create mode 100644 web/src/lib/components/timeline/RecentItem.svelte create mode 100644 web/src/lib/components/timeline/RecentsPreview.stories.svelte create mode 100644 web/src/lib/components/timeline/RecentsPreview.svelte create mode 100644 web/src/lib/components/timeline/Timeline.stories.svelte create mode 100644 web/src/lib/components/timeline/Timeline.svelte create mode 100644 web/src/lib/components/timeline/TimelineActorHeader.svelte create mode 100644 web/src/lib/components/timeline/TimelineCard.svelte create mode 100644 web/src/lib/components/timeline/TimelineFollowEvent.stories.svelte create mode 100644 web/src/lib/components/timeline/TimelineFollowEvent.svelte create mode 100644 web/src/lib/components/timeline/TimelineFollowers.svelte create mode 100644 web/src/lib/components/timeline/TimelineForkers.svelte create mode 100644 web/src/lib/components/timeline/TimelineGroup.svelte create mode 100644 web/src/lib/components/timeline/TimelineRepoEvent.stories.svelte create mode 100644 web/src/lib/components/timeline/TimelineRepoEvent.svelte create mode 100644 web/src/lib/components/timeline/TimelineSkeleton.svelte create mode 100644 web/src/lib/components/timeline/TimelineStarEvent.stories.svelte create mode 100644 web/src/lib/components/timeline/TimelineStarEvent.svelte create mode 100644 web/src/lib/components/timeline/TimelineStargazers.svelte create mode 100644 web/src/lib/components/timeline/TimelineTargetHeader.svelte create mode 100644 web/src/lib/components/timeline/TimelineTargets.stories.svelte create mode 100644 web/src/lib/components/timeline/TimelineView.stories.svelte create mode 100644 web/src/lib/components/timeline/TimelineView.svelte create mode 100644 web/src/lib/components/timeline/TrendingRepos.stories.svelte create mode 100644 web/src/lib/components/timeline/TrendingRepos.svelte create mode 100644 web/src/lib/components/timeline/TrendingReposList.stories.svelte create mode 100644 web/src/lib/components/timeline/TrendingReposList.svelte create mode 100644 web/src/lib/components/timeline/TrendingSkeleton.svelte create mode 100644 web/src/lib/components/timeline/group.test.ts create mode 100644 web/src/lib/components/timeline/group.ts create mode 100644 web/src/lib/components/timeline/load.test.ts create mode 100644 web/src/lib/components/timeline/load.ts create mode 100644 web/src/lib/components/timeline/mappers.ts create mode 100644 web/src/lib/components/timeline/recents.svelte.ts create mode 100644 web/src/lib/components/timeline/recents.test.ts create mode 100644 web/src/lib/components/timeline/recents.ts create mode 100644 web/src/lib/components/timeline/types.ts create mode 100644 web/src/lib/components/ui/ActorInput.svelte create mode 100644 web/src/lib/components/ui/AnimatedLogo.svelte create mode 100644 web/src/lib/components/ui/Avatar.stories.svelte create mode 100644 web/src/lib/components/ui/Avatar.svelte create mode 100644 web/src/lib/components/ui/Avatar.test.ts create mode 100644 web/src/lib/components/ui/Bones.svelte create mode 100644 web/src/lib/components/ui/Button.stories.svelte create mode 100644 web/src/lib/components/ui/Button.svelte create mode 100644 web/src/lib/components/ui/ButtonCard.stories.svelte create mode 100644 web/src/lib/components/ui/ButtonCard.svelte create mode 100644 web/src/lib/components/ui/ButtonGroup.stories.svelte create mode 100644 web/src/lib/components/ui/ButtonGroup.svelte create mode 100644 web/src/lib/components/ui/Card.stories.svelte create mode 100644 web/src/lib/components/ui/Card.svelte create mode 100644 web/src/lib/components/ui/Checkbox.stories.svelte create mode 100644 web/src/lib/components/ui/Checkbox.svelte create mode 100644 web/src/lib/components/ui/CodeBlockToolbar.svelte create mode 100644 web/src/lib/components/ui/ColorBall.stories.svelte create mode 100644 web/src/lib/components/ui/ColorBall.svelte create mode 100644 web/src/lib/components/ui/Combobox.svelte create mode 100644 web/src/lib/components/ui/ConfirmModal.stories.svelte create mode 100644 web/src/lib/components/ui/ConfirmModal.svelte create mode 100644 web/src/lib/components/ui/CopyButton.stories.svelte create mode 100644 web/src/lib/components/ui/CopyButton.svelte create mode 100644 web/src/lib/components/ui/Dropdown.stories.svelte create mode 100644 web/src/lib/components/ui/Dropdown.svelte create mode 100644 web/src/lib/components/ui/DropdownGroup.svelte create mode 100644 web/src/lib/components/ui/DropdownItem.svelte create mode 100644 web/src/lib/components/ui/DropdownSubmenu.svelte create mode 100644 web/src/lib/components/ui/EmptyState.stories.svelte create mode 100644 web/src/lib/components/ui/EmptyState.svelte create mode 100644 web/src/lib/components/ui/Error.stories.svelte create mode 100644 web/src/lib/components/ui/Error.svelte create mode 100644 web/src/lib/components/ui/Field.stories.svelte create mode 100644 web/src/lib/components/ui/Field.svelte create mode 100644 web/src/lib/components/ui/FormStep.stories.svelte create mode 100644 web/src/lib/components/ui/FormStep.svelte create mode 100644 web/src/lib/components/ui/FormStepDisclosure.svelte create mode 100644 web/src/lib/components/ui/GraphCell.stories.svelte create mode 100644 web/src/lib/components/ui/GraphCell.svelte create mode 100644 web/src/lib/components/ui/HiddenContent.stories.svelte create mode 100644 web/src/lib/components/ui/HiddenContent.svelte create mode 100644 web/src/lib/components/ui/Hovercard.stories.svelte create mode 100644 web/src/lib/components/ui/Hovercard.svelte create mode 100644 web/src/lib/components/ui/InlineMarkdown.svelte create mode 100644 web/src/lib/components/ui/Input.stories.svelte create mode 100644 web/src/lib/components/ui/Input.svelte create mode 100644 web/src/lib/components/ui/Kbd.stories.svelte create mode 100644 web/src/lib/components/ui/Kbd.svelte create mode 100644 web/src/lib/components/ui/LayerCard.stories.svelte create mode 100644 web/src/lib/components/ui/LayerCard.svelte create mode 100644 web/src/lib/components/ui/LayerCardPrimary.svelte create mode 100644 web/src/lib/components/ui/LayerCardSecondary.svelte create mode 100644 web/src/lib/components/ui/Lightbox.stories.svelte create mode 100644 web/src/lib/components/ui/Lightbox.svelte create mode 100644 web/src/lib/components/ui/Link.stories.svelte create mode 100644 web/src/lib/components/ui/Link.svelte create mode 100644 web/src/lib/components/ui/ListPanel.svelte create mode 100644 web/src/lib/components/ui/ListToolbar.svelte create mode 100644 web/src/lib/components/ui/LoadingPrimitives.bones.stories.svelte create mode 100644 web/src/lib/components/ui/Logo.stories.svelte create mode 100644 web/src/lib/components/ui/Logo.svelte create mode 100644 web/src/lib/components/ui/MarkdownEditor.stories.svelte create mode 100644 web/src/lib/components/ui/MarkdownEditor.svelte create mode 100644 web/src/lib/components/ui/MarkdownTypeahead.svelte create mode 100644 web/src/lib/components/ui/Markup.svelte create mode 100644 web/src/lib/components/ui/Modal.stories.svelte create mode 100644 web/src/lib/components/ui/Modal.svelte create mode 100644 web/src/lib/components/ui/MoreMenu.stories.svelte create mode 100644 web/src/lib/components/ui/MoreMenu.svelte create mode 100644 web/src/lib/components/ui/NumberFlip.svelte create mode 100644 web/src/lib/components/ui/Pagination.stories.svelte create mode 100644 web/src/lib/components/ui/Pagination.svelte create mode 100644 web/src/lib/components/ui/Pending.svelte create mode 100644 web/src/lib/components/ui/Radio.bones.stories.svelte create mode 100644 web/src/lib/components/ui/Radio.stories.svelte create mode 100644 web/src/lib/components/ui/Radio.svelte create mode 100644 web/src/lib/components/ui/RangeSelector.stories.svelte create mode 100644 web/src/lib/components/ui/RangeSelector.svelte create mode 100644 web/src/lib/components/ui/RangeSelectorRow.svelte create mode 100644 web/src/lib/components/ui/ResizeHandle.stories.svelte create mode 100644 web/src/lib/components/ui/ResizeHandle.svelte create mode 100644 web/src/lib/components/ui/RoutePagination.stories.svelte create mode 100644 web/src/lib/components/ui/RoutePagination.svelte create mode 100644 web/src/lib/components/ui/Section.stories.svelte create mode 100644 web/src/lib/components/ui/Section.svelte create mode 100644 web/src/lib/components/ui/Segment.svelte create mode 100644 web/src/lib/components/ui/SegmentedControl.stories.svelte create mode 100644 web/src/lib/components/ui/SegmentedControl.svelte create mode 100644 web/src/lib/components/ui/Select.stories.svelte create mode 100644 web/src/lib/components/ui/Select.svelte create mode 100644 web/src/lib/components/ui/Separator.stories.svelte create mode 100644 web/src/lib/components/ui/Separator.svelte create mode 100644 web/src/lib/components/ui/ShimmerText.stories.svelte create mode 100644 web/src/lib/components/ui/ShimmerText.svelte create mode 100644 web/src/lib/components/ui/SignupPrompt.stories.svelte create mode 100644 web/src/lib/components/ui/SignupPrompt.svelte create mode 100644 web/src/lib/components/ui/Spinner.stories.svelte create mode 100644 web/src/lib/components/ui/Spinner.svelte create mode 100644 web/src/lib/components/ui/SplitButton.stories.svelte create mode 100644 web/src/lib/components/ui/SplitButton.svelte create mode 100644 web/src/lib/components/ui/StaleReload.svelte create mode 100644 web/src/lib/components/ui/StepMarker.stories.svelte create mode 100644 web/src/lib/components/ui/StepMarker.svelte create mode 100644 web/src/lib/components/ui/Tabs.stories.svelte create mode 100644 web/src/lib/components/ui/Tabs.svelte create mode 100644 web/src/lib/components/ui/Tag.stories.svelte create mode 100644 web/src/lib/components/ui/Tag.svelte create mode 100644 web/src/lib/components/ui/TagInput.svelte create mode 100644 web/src/lib/components/ui/TextSwap.stories.svelte create mode 100644 web/src/lib/components/ui/TextSwap.svelte create mode 100644 web/src/lib/components/ui/Textarea.stories.svelte create mode 100644 web/src/lib/components/ui/Textarea.svelte create mode 100644 web/src/lib/components/ui/TimeAgo.stories.svelte create mode 100644 web/src/lib/components/ui/TimeAgo.svelte create mode 100644 web/src/lib/components/ui/Toggle.stories.svelte create mode 100644 web/src/lib/components/ui/Toggle.svelte create mode 100644 web/src/lib/components/ui/ToggleFaces.svelte create mode 100644 web/src/lib/components/ui/Tooltip.stories.svelte create mode 100644 web/src/lib/components/ui/Tooltip.svelte create mode 100644 web/src/lib/components/ui/Turnstile.svelte create mode 100644 web/src/lib/components/ui/User.stories.svelte create mode 100644 web/src/lib/components/ui/User.svelte create mode 100644 web/src/lib/components/ui/codeblockToolbars.ts create mode 100644 web/src/lib/components/ui/inputField.ts create mode 100644 web/src/lib/components/ui/keyLabels.test.ts create mode 100644 web/src/lib/components/ui/keyLabels.ts create mode 100644 web/src/lib/components/ui/logotype.ts create mode 100644 web/src/lib/components/ui/markdownFormat.test.ts create mode 100644 web/src/lib/components/ui/markdownFormat.ts create mode 100644 web/src/lib/components/ui/markdownSuggestions.test.ts create mode 100644 web/src/lib/components/ui/markdownSuggestions.ts create mode 100644 web/src/lib/components/ui/mentionPeople.ts create mode 100644 web/src/lib/components/ui/menuItems.ts create mode 100644 web/src/lib/components/ui/mermaidDiagrams.ts create mode 100644 web/src/lib/components/ui/popover.svelte.ts create mode 100644 web/src/lib/components/ui/rangeMode.test.ts create mode 100644 web/src/lib/components/ui/rangeMode.ts create mode 100644 web/src/lib/components/ui/selectField.ts create mode 100644 web/src/lib/components/welcome/EmailsStep.stories.svelte create mode 100644 web/src/lib/components/welcome/EmailsStep.svelte create mode 100644 web/src/lib/components/welcome/FinishStep.stories.svelte create mode 100644 web/src/lib/components/welcome/FinishStep.svelte create mode 100644 web/src/lib/components/welcome/KeysStep.stories.svelte create mode 100644 web/src/lib/components/welcome/KeysStep.svelte create mode 100644 web/src/lib/components/welcome/OnboardingNudge.svelte create mode 100644 web/src/lib/components/welcome/ProfileStep.svelte create mode 100644 web/src/lib/components/welcome/SocialStep.svelte create mode 100644 web/src/lib/components/welcome/StepHeader.svelte create mode 100644 web/src/lib/components/welcome/Steps.stories.svelte create mode 100644 web/src/lib/components/welcome/Steps.svelte create mode 100644 web/src/lib/components/welcome/WelcomeCard.svelte create mode 100644 web/src/lib/components/welcome/steps.ts create mode 100644 web/src/lib/fetch.ts create mode 100644 web/src/lib/fetchJson.ts create mode 100644 web/src/lib/format.test.ts create mode 100644 web/src/lib/format.ts create mode 100644 web/src/lib/github.ts create mode 100644 web/src/lib/handle.test.ts create mode 100644 web/src/lib/handle.ts create mode 100644 web/src/lib/images.test.ts create mode 100644 web/src/lib/images.ts create mode 100644 web/src/lib/lightbox.svelte.ts create mode 100644 web/src/lib/markup/collapse.test.ts create mode 100644 web/src/lib/markup/collapse.ts create mode 100644 web/src/lib/markup/format.ts create mode 100644 web/src/lib/markup/highlight.ts create mode 100644 web/src/lib/markup/index.ts create mode 100644 web/src/lib/markup/inline.test.ts create mode 100644 web/src/lib/markup/inline.ts create mode 100644 web/src/lib/markup/markdown.test.ts create mode 100644 web/src/lib/markup/markdown.ts create mode 100644 web/src/lib/markup/mermaid.ts create mode 100644 web/src/lib/markup/pageContext.ts create mode 100644 web/src/lib/markup/paths.ts create mode 100644 web/src/lib/markup/preview.ts create mode 100644 web/src/lib/markup/render.test.ts create mode 100644 web/src/lib/markup/render.ts create mode 100644 web/src/lib/markup/sanitize.ts create mode 100644 web/src/lib/markup/text.test.ts create mode 100644 web/src/lib/markup/text.ts create mode 100644 web/src/lib/migration/import.test.ts create mode 100644 web/src/lib/migration/import.ts create mode 100644 web/src/lib/migration/liar.test.ts create mode 100644 web/src/lib/migration/liar.ts create mode 100644 web/src/lib/migration/mock.ts create mode 100644 web/src/lib/migration/repoCar.ts create mode 100644 web/src/lib/migration/status.test.ts create mode 100644 web/src/lib/migration/status.ts create mode 100644 web/src/lib/mobile.svelte.ts create mode 100644 web/src/lib/motion/reveal.ts create mode 100644 web/src/lib/motion/spring.ts create mode 100644 web/src/lib/motion/value.svelte.ts create mode 100644 web/src/lib/navPulse.test.ts create mode 100644 web/src/lib/navPulse.ts create mode 100644 web/src/lib/notifications.svelte.ts create mode 100644 web/src/lib/notifications.test.ts create mode 100644 web/src/lib/oauth-client-metadata.test.ts create mode 100644 web/src/lib/oauth-client-metadata.ts create mode 100644 web/src/lib/optimistic.svelte.ts create mode 100644 web/src/lib/pagination.svelte.ts create mode 100644 web/src/lib/pagination.test.ts create mode 100644 web/src/lib/pagination.ts create mode 100644 web/src/lib/popoverExit.ts create mode 100644 web/src/lib/publicFetch.test.ts create mode 100644 web/src/lib/publicFetch.ts create mode 100644 web/src/lib/rem.ts create mode 100644 web/src/lib/repoQuery.test.ts create mode 100644 web/src/lib/repoQuery.ts create mode 100644 web/src/lib/revIdHighlight.svelte.ts create mode 100644 web/src/lib/safeHref.test.ts create mode 100644 web/src/lib/safeHref.ts create mode 100644 web/src/lib/seo/jsonLd.test.ts create mode 100644 web/src/lib/seo/jsonLd.ts create mode 100644 web/src/lib/seo/og.test.ts create mode 100644 web/src/lib/seo/og.ts create mode 100644 web/src/lib/server/avatar.test.ts create mode 100644 web/src/lib/server/avatar.ts create mode 100644 web/src/lib/server/blob.ts create mode 100644 web/src/lib/server/camo.ts create mode 100644 web/src/lib/server/camotarget.test.ts create mode 100644 web/src/lib/server/camotarget.ts create mode 100644 web/src/lib/server/config.test.ts create mode 100644 web/src/lib/server/config.ts create mode 100644 web/src/lib/server/gitHttp.test.ts create mode 100644 web/src/lib/server/gitHttp.ts create mode 100644 web/src/lib/server/github.test.ts create mode 100644 web/src/lib/server/github/api.ts create mode 100644 web/src/lib/server/github/cookies.ts create mode 100644 web/src/lib/server/github/env.ts create mode 100644 web/src/lib/server/github/http.ts create mode 100644 web/src/lib/server/github/index.ts create mode 100644 web/src/lib/server/github/oauth.ts create mode 100644 web/src/lib/server/github/session.ts create mode 100644 web/src/lib/server/indexnow.test.ts create mode 100644 web/src/lib/server/indexnow.ts create mode 100644 web/src/lib/server/lookupEvents.test.ts create mode 100644 web/src/lib/server/lookupEvents.ts create mode 100644 web/src/lib/server/posthog.ts create mode 100644 web/src/lib/server/repo.test.ts create mode 100644 web/src/lib/server/repo.ts create mode 100644 web/src/lib/server/repoIndex.ts create mode 100644 web/src/lib/server/sealed.ts create mode 100644 web/src/lib/shiftKey.svelte.ts create mode 100644 web/src/lib/ssr/pool.test.ts create mode 100644 web/src/lib/ssr/pool.ts create mode 100644 web/src/lib/ssr/render.test.ts create mode 100644 web/src/lib/ssr/render.ts create mode 100644 web/src/lib/ssr/worker.ts create mode 100644 web/src/lib/status.test.ts create mode 100644 web/src/lib/status.ts create mode 100644 web/src/lib/stuck.ts create mode 100644 web/src/lib/subdomain.test.ts create mode 100644 web/src/lib/subdomain.ts create mode 100644 web/src/lib/syntax.test.ts create mode 100644 web/src/lib/syntax.ts create mode 100644 web/src/lib/theme.test.ts create mode 100644 web/src/lib/theme.ts create mode 100644 web/src/lib/ticketQuery.test.ts create mode 100644 web/src/lib/ticketQuery.ts create mode 100644 web/src/lib/twMergeConfig.ts create mode 100644 web/src/lib/unsavedChanges.ts create mode 100644 web/src/lib/uri.ts create mode 100644 web/src/listView.css create mode 100644 web/src/logs.css create mode 100644 web/src/markup.css create mode 100644 web/src/params.ts create mode 100644 web/src/pierre.css create mode 100644 web/src/routes/+error.svelte create mode 100644 web/src/routes/+layout.server.ts create mode 100644 web/src/routes/+layout.svelte create mode 100644 web/src/routes/+page.svelte create mode 100644 web/src/routes/+page.ts create mode 100644 web/src/routes/[handle]/+layout.svelte create mode 100644 web/src/routes/[handle]/+layout.ts create mode 100644 web/src/routes/[handle]/+page.svelte create mode 100644 web/src/routes/[handle]/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/+layout.server.ts create mode 100644 web/src/routes/[handle]/[repo]/+layout.ts create mode 100644 web/src/routes/[handle]/[repo]/+layout@.svelte create mode 100644 web/src/routes/[handle]/[repo]/+page.server.ts create mode 100644 web/src/routes/[handle]/[repo]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/archive/[...ref]/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/archive/archive.test.ts create mode 100644 web/src/routes/[handle]/[repo]/artifacts/[rkey]/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/blob/[ref]/[...path]/+page.server.ts create mode 100644 web/src/routes/[handle]/[repo]/blob/[ref]/[...path]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/branches/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/branches/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/commit-statuses/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/commit/[ref]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/commit/[ref]/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/commit/[spec=rawCommit]/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/commits/[ref]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/commits/[ref]/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/compare/[...spec]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/compare/[...spec]/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/feed.atom/+server.test.ts create mode 100644 web/src/routes/[handle]/[repo]/feed.atom/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/fork/+page.server.ts create mode 100644 web/src/routes/[handle]/[repo]/fork/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/fork/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/forks/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/forks/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/forks/forks.test.ts create mode 100644 web/src/routes/[handle]/[repo]/git-receive-pack/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/git-upload-archive/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/git-upload-pack/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/history/[ref]/[...path]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/history/[ref]/[...path]/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/info/refs/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/issues/+page.server.ts create mode 100644 web/src/routes/[handle]/[repo]/issues/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/issues/[rkey]/+page.server.ts create mode 100644 web/src/routes/[handle]/[repo]/issues/[rkey]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/issues/[rkey]/issue.test.ts create mode 100644 web/src/routes/[handle]/[repo]/issues/issues.test.ts create mode 100644 web/src/routes/[handle]/[repo]/issues/new/+page.server.ts create mode 100644 web/src/routes/[handle]/[repo]/issues/new/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/issues/new/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/languages/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/languages/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/languages/languages.test.ts create mode 100644 web/src/routes/[handle]/[repo]/layout.test.ts create mode 100644 web/src/routes/[handle]/[repo]/pipelines/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/pipelines/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/pipelines/[pipeline]/workflow/[workflow]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/pipelines/[pipeline]/workflow/[workflow]/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/pipelines/pipelines.test.ts create mode 100644 web/src/routes/[handle]/[repo]/pulls/+page.server.ts create mode 100644 web/src/routes/[handle]/[repo]/pulls/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/pulls/[rkey]/+layout.server.ts create mode 100644 web/src/routes/[handle]/[repo]/pulls/[rkey]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/pulls/[rkey]/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/pulls/[rkey]/[version].patch/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/pulls/[rkey]/[version]/+layout.server.ts create mode 100644 web/src/routes/[handle]/[repo]/pulls/[rkey]/[version]/[[range]]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/pulls/[rkey]/[version]/[[range]]/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/pulls/[rkey]/[version]/[[range]]/PullViewPage.stories.svelte create mode 100644 web/src/routes/[handle]/[repo]/pulls/[rkey]/[version]/[[range]]/PullViewPage.svelte create mode 100644 web/src/routes/[handle]/[repo]/pulls/new/+page.server.ts create mode 100644 web/src/routes/[handle]/[repo]/pulls/new/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/pulls/new/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/pulls/pulls.test.ts create mode 100644 web/src/routes/[handle]/[repo]/raw/[ref]/[...path]/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/raw/raw.test.ts create mode 100644 web/src/routes/[handle]/[repo]/search/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/search/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/settings/+layout.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/+layout.ts create mode 100644 web/src/routes/[handle]/[repo]/settings/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/settings/access/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/access/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/settings/access/new/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/access/new/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/settings/hooks/+layout.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/hooks/+layout.ts create mode 100644 web/src/routes/[handle]/[repo]/settings/hooks/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/hooks/[id]/+layout.ts create mode 100644 web/src/routes/[handle]/[repo]/settings/hooks/[id]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/hooks/[id]/deliveries/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/hooks/new/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/labels/new/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/labels/new/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/settings/pipelines/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/pipelines/secrets/new/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/rename/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/rename/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/settings/sites/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/settings/sites/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/settings/sites/sites.test.ts create mode 100644 web/src/routes/[handle]/[repo]/stars/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/stars/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/tags/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/tags/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/tags/[tag]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/tags/[tag]/+page.ts create mode 100644 web/src/routes/[handle]/[repo]/tags/[tag]/download/[file]/+server.ts create mode 100644 web/src/routes/[handle]/[repo]/tree/[ref]/+page.server.ts create mode 100644 web/src/routes/[handle]/[repo]/tree/[ref]/+page.svelte create mode 100644 web/src/routes/[handle]/[repo]/tree/[ref]/[...path]/+page.server.ts create mode 100644 web/src/routes/[handle]/[repo]/tree/[ref]/[...path]/+page.svelte create mode 100644 web/src/routes/[handle]/handle.test.ts create mode 100644 web/src/routes/[handle]/info/refs/+server.ts create mode 100644 web/src/routes/[handle]/settings/OrganizationAvatar.svelte create mode 100644 web/src/routes/[handle]/settings/OrganizationHandle.svelte create mode 100644 web/src/routes/[handle]/settings/OrganizationProfileTab.stories.svelte create mode 100644 web/src/routes/[handle]/settings/OrganizationProfileTab.svelte create mode 100644 web/src/routes/[handle]/settings/OrganizationSettingsTab.svelte create mode 100644 web/src/routes/[handle]/settings/members/InviteMembers.stories.svelte create mode 100644 web/src/routes/[handle]/settings/members/InviteMembers.svelte create mode 100644 web/src/routes/[handle]/settings/members/MembersTab.stories.svelte create mode 100644 web/src/routes/[handle]/settings/members/MembersTab.svelte create mode 100644 web/src/routes/_internal/analytics/signin/+server.ts create mode 100644 web/src/routes/_internal/email-did/+server.ts create mode 100644 web/src/routes/_internal/email-did/email-did.server.test.ts create mode 100644 web/src/routes/_internal/github/+server.ts create mode 100644 web/src/routes/_internal/github/avatar/+server.ts create mode 100644 web/src/routes/_internal/github/callback/+server.ts create mode 100644 web/src/routes/_internal/github/connect/+server.ts create mode 100644 web/src/routes/_internal/github/github.test.ts create mode 100644 web/src/routes/_internal/github/install/callback/+server.ts create mode 100644 web/src/routes/_internal/github/migrator/createTask/+server.ts create mode 100644 web/src/routes/_internal/indexnow/+server.ts create mode 100644 web/src/routes/_internal/status/+server.ts create mode 100644 web/src/routes/avatar/[did]/+server.ts create mode 100644 web/src/routes/blob/[did]/[cid]/+server.ts create mode 100644 web/src/routes/camo/[hex]/+server.ts create mode 100644 web/src/routes/company/+page.svelte create mode 100644 web/src/routes/company/CompanyArt.svelte create mode 100644 web/src/routes/compare/+page.svelte create mode 100644 web/src/routes/compare/MoveArt.svelte create mode 100644 web/src/routes/core/+server.ts create mode 100644 web/src/routes/core/core.test.ts create mode 100644 web/src/routes/dev/components/+page.svelte create mode 100644 web/src/routes/dev/components/+page.ts create mode 100644 web/src/routes/dev/components/Shell.svelte create mode 100644 web/src/routes/dev/components/Tile.svelte create mode 100644 web/src/routes/favicon.svg/+server.ts create mode 100644 web/src/routes/home/+page.svelte create mode 100644 web/src/routes/invite/+page.server.ts create mode 100644 web/src/routes/invite/+page.svelte create mode 100644 web/src/routes/login/+page.svelte create mode 100644 web/src/routes/notifications/+page.svelte create mode 100644 web/src/routes/notifications/+page.ts create mode 100644 web/src/routes/oauth-client-metadata.json/+server.ts create mode 100644 web/src/routes/oauth/callback/+page.svelte create mode 100644 web/src/routes/organizations/[handle]/welcome/+page.svelte create mode 100644 web/src/routes/organizations/new/+page.server.ts create mode 100644 web/src/routes/organizations/new/+page.svelte create mode 100644 web/src/routes/privacy/+page.svelte create mode 100644 web/src/routes/repo/migrate/+page.server.ts create mode 100644 web/src/routes/repo/migrate/+page.svelte create mode 100644 web/src/routes/repo/new/+page.server.ts create mode 100644 web/src/routes/repo/new/+page.svelte create mode 100644 web/src/routes/repo/new/+page.ts create mode 100644 web/src/routes/search/+page.svelte create mode 100644 web/src/routes/search/+page.ts create mode 100644 web/src/routes/settings/+layout.server.ts create mode 100644 web/src/routes/settings/+layout.svelte create mode 100644 web/src/routes/settings/+page.svelte create mode 100644 web/src/routes/settings/emails/+page.svelte create mode 100644 web/src/routes/settings/emails/new/+page.svelte create mode 100644 web/src/routes/settings/emails/new/AddEmailPage.stories.svelte create mode 100644 web/src/routes/settings/keys/+page.svelte create mode 100644 web/src/routes/settings/keys/new/+page.svelte create mode 100644 web/src/routes/settings/knots/+page.svelte create mode 100644 web/src/routes/settings/knots/new/+page.svelte create mode 100644 web/src/routes/settings/notifications/+page.svelte create mode 100644 web/src/routes/settings/organizations/+page.svelte create mode 100644 web/src/routes/settings/organizations/new/+page.svelte create mode 100644 web/src/routes/settings/profile/avatar/+page.svelte create mode 100644 web/src/routes/settings/profile/delete/+page.svelte create mode 100644 web/src/routes/settings/profile/handle/+page.svelte create mode 100644 web/src/routes/settings/profile/handle/ChangeHandlePage.stories.svelte create mode 100644 web/src/routes/settings/profile/password/+page.svelte create mode 100644 web/src/routes/settings/sites/+page.svelte create mode 100644 web/src/routes/settings/spindles/+page.svelte create mode 100644 web/src/routes/settings/spindles/new/+page.svelte create mode 100644 web/src/routes/signup/+page.svelte create mode 100644 web/src/routes/signup/verify/+page.svelte create mode 100644 web/src/routes/strings/[handle]/[rkey]/+page.server.ts create mode 100644 web/src/routes/strings/[handle]/[rkey]/+page.svelte create mode 100644 web/src/routes/strings/[handle]/[rkey]/StringPageContent.svelte create mode 100644 web/src/routes/strings/[handle]/[rkey]/raw/+server.ts create mode 100644 web/src/routes/strings/[handle]/[rkey]/skeleton.test.ts create mode 100644 web/src/routes/strings/[handle]/[rkey]/stringPage.test.ts create mode 100644 web/src/routes/strings/new/+page.server.ts create mode 100644 web/src/routes/strings/new/+page.svelte create mode 100644 web/src/routes/terms/+page.svelte create mode 100644 web/src/routes/timeline/+page.server.ts create mode 100644 web/src/routes/timeline/+page.svelte create mode 100644 web/src/routes/verify/email/+page.svelte create mode 100644 web/src/routes/verify/email/VerifyEmailPage.stories.svelte create mode 100644 web/src/routes/welcome/+page.svelte create mode 100644 web/src/routes/welcome/+page.ts create mode 100644 web/src/theme.css create mode 100644 web/static/.gitignore create mode 100644 web/static/d76fae270607bfc761b35c14f3182c9e.txt create mode 100644 web/static/llms.txt create mode 100644 web/static/og-image.png create mode 100644 web/static/robots.txt create mode 100644 web/static/tangled_og.png create mode 100644 web/tsconfig.json create mode 100644 web/vite.config.ts create mode 100644 web/vitest.shims.d.ts create mode 100644 web/wrangler.dev.jsonc create mode 100644 workflow/compile.go create mode 100644 workflow/compile_test.go create mode 100644 workflow/def.go create mode 100644 workflow/def_test.go create mode 100644 workflow/spec_types.go create mode 100644 xrpc/errors/errors.go create mode 100644 xrpc/serviceauth/lookup_failure_test.go create mode 100644 xrpc/serviceauth/service_auth.go create mode 100644 xrpc/serviceauth/service_auth_resolve_test.go create mode 100644 xrpc/serviceauth/service_auth_test.go create mode 100644 xrpc/serviceauth/signer.go create mode 100644 xrpc/serviceauth/signer_test.go create mode 100644 xrpc/xrpcclient/xrpc.go create mode 100644 xrpc/xrpcclient/xrpc_test.go diff --git a/.air/deliberi.toml b/.air/deliberi.toml new file mode 100644 index 0000000000000000000000000000000000000000..2564ad7ecf9becce2548b310c02374f11b5fb79c --- /dev/null +++ b/.air/deliberi.toml @@ -0,0 +1,10 @@ +root = "." +tmp_dir = "out" + +[build] +cmd = "go build -tags libsqlite3 -o out/deliberi.out cmd/deliberi/main.go" +bin = "out/deliberi.out" + +include_ext = ["go"] +exclude_dir = ["avatar", "camo", "sites", "indexes", "nix", "tmp"] +stop_on_error = true diff --git a/.air/spindle.toml b/.air/spindle.toml new file mode 100644 index 0000000000000000000000000000000000000000..d224a7bab617dbdd5f1a4209ade95d1aad9f290e --- /dev/null +++ b/.air/spindle.toml @@ -0,0 +1,10 @@ +root = "." +tmp_dir = "out" + +[build] +cmd = "go build -o out/spindle.out cmd/spindle/main.go" +bin = "out/spindle.out" + +include_ext = ["go"] +exclude_dir = ["avatar", "camo", "sites", "indexes", "nix", "tmp"] +stop_on_error = true diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000000000000000000000000000000000000..59cfbcd3dcfeaeeb43ac2cb7422dae55fbe936f5 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,3 @@ +[target.aarch64-apple-darwin] +linker = "arm64-apple-darwin-cc" + diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 0000000000000000000000000000000000000000..5ce8d5294131d3ccf0f1532d67f30d4471af6422 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,4 @@ +[profile.ci] +fail-fast = false +# a test sleeping through its budget fails on its own instead of timing out the whole workflow +slow-timeout = { period = "60s", terminate-after = 3 } diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..23dfc49c8621509003565bc00fb956087873f9b5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,49 @@ +.git +.jj +.claude/ +.DS_Store +._.DS_Store +.direnv/ +.env* +*.qcow2 +genjwks.out +/nix/vm-data +blog/build +build/ +.wrangler/ +localinfra/certs/*.key +localinfra/ncps-secret-key + +sites/target +sites/.wrangler +target + +result +result-* +build +.bin +out/ +node_modules/ +**/node_modules/ +*.db +*.db-* +*.rdb +*.bleve +**/*.bleve + +tmp +patches + +docker-compose.yml +docker-compose.*.yml +**/Dockerfile +**/*.Dockerfile + +knot2/pack-spike +**/src/_lex +!bobbin/crates/types/src/ +!bobbin/crates/types/src/_lex/ +!bobbin/crates/types/src/_lex/** +!crates/lexicons/src/ +!crates/lexicons/src/_lex/ +!crates/lexicons/src/_lex/** diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000000000000000000000000000000000..30ba88374e85e353fc81e2aa3698264e908dedf7 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,29 @@ +root = true + +[*] +insert_final_newline = true + +[*.html] +indent_size = 2 + +[*.json] +indent_size = 2 + +[*.nix] +indent_style = space +indent_size = 2 + +[*.yml] +indent_size = 2 + +[*.rs] +indent_style = space +indent_size = 4 + +[*.ts] +indent_style = tab +indent_size = 4 + +[*.svelte] +indent_style = tab +indent_size = 4 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..cd718a389ec12a6578ae4a10d4b52dbf4d1c0585 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +api/tangled/** linguist-generated -diff +api/tangled/*_ext.go -linguist-generated diff +flake.lock -diff +web/src/lib/api/lexicons/** linguist-generated -diff +spindle/mill/proto/gen/** linguist-generated -diff +Cargo.lock -diff +nix/Cargo.nix linguist-generated -diff diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..23b068123b62caf30ac8a6d958755d05ba3617bb --- /dev/null +++ b/.gitignore @@ -0,0 +1,50 @@ +.DS_Store +._.DS_Store +.direnv/ +.envrc +.env +.claude/ +.playwright-mcp/ +.browser-artifacts + +id_rsa +id_dsa +id_ecdsa +id_ed25519 +*.pem +*.key + +node_modules/ +__pycache__/ +*.pyc +.astro/ +.wrangler/ +build/ +out/ +target/ +result +.bin/ +tmp + +*.db +*.db-* +*.rdb +*.bleve +*.qcow2 + +/patches +/prefill-zoekt +/zoekt-tngl-indexserver +/.localinfra/ +/nix/vm-data + +# Created if following hacking.md +genjwks.out + +spindle/spindle +spindle/spindle-microvm-run +localinfra/certs/* +localinfra/vendor/ +localinfra/pc/process-compose.override.yaml + +!.gitkeep diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000000000000000000000000000000000000..0e3236ceb5d8ba6d6ff99c9960cbe76e0e175bb4 --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,12 @@ +{ + "overrides": [ + { + "files": ["*.html"], + "options": { + "parser": "go-template" + } + } + ], + "bracketSameLine": true, + "htmlWhitespaceSensitivity": "ignore" +} diff --git a/.tangled/hooks/commit-msg b/.tangled/hooks/commit-msg new file mode 100755 index 0000000000000000000000000000000000000000..f7968fec421b17408659bc10a861668d2224e126 --- /dev/null +++ b/.tangled/hooks/commit-msg @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# git commit-msg hook — lint the message against the Go commit conventions. +# +# Installed by pointing core.hooksPath at this directory: +# ./.tangled/hooks/install.sh +# +# Bypass for a single commit with `git commit --no-verify`. +# +# Note: jj (jujutsu) does not run git hooks. jj users are covered by the +# `commit-lint` CI workflow instead, and can lint locally on demand with: +# .tangled/hooks/commit-msg-lint.sh --rev @ + +exec "$(dirname "$0")/commit-msg-lint.sh" --file "$1" diff --git a/.tangled/hooks/commit-msg-lint.sh b/.tangled/hooks/commit-msg-lint.sh new file mode 100755 index 0000000000000000000000000000000000000000..9a573823577f40813e8d15b1bf629824f607daaa --- /dev/null +++ b/.tangled/hooks/commit-msg-lint.sh @@ -0,0 +1,262 @@ +#!/usr/bin/env bash +# +# commit-msg-lint.sh — enforce Go-style commit messages. +# +# Reference: https://go.dev/wiki/CommitMessage +# +# pkg/path: short summary in the imperative mood +# +# Optional body explaining what changed and why, wrapped at ~76 columns. +# The blank line between the summary and the body is required. +# +# Fixes #123 +# +# Usage: +# commit-msg-lint.sh --file # lint a single message file (git commit-msg hook) +# commit-msg-lint.sh --rev # lint the message of one commit +# commit-msg-lint.sh --range # lint every commit in a..b (exclusive of a) +# ... | commit-msg-lint.sh # lint a message on stdin +# +# Flags: +# --strict treat warnings as errors (exit non-zero on warnings too) +# +# Exit status: 0 = clean, 1 = at least one error (or warning under --strict). + +set -euo pipefail + +# Maximum length of the summary line. Go's guideline is "less than 76". +MAX_SUMMARY_LEN=76 + +# Bare prefixes that look like Conventional Commits rather than a Go package +# path. These are rejected because the summary should name the package/path +# affected, e.g. `ogre: ...` instead of `fix: ...`. +CONVENTIONAL_TYPES="feat fix chore refactor perf style ci revert" + +# Special prefixes that are not repo paths but are conventionally allowed, +# following the Go project (e.g. `all:` for tree-wide changes). +PATH_ALLOWLIST="all" + +# Space-separated list of real top-level entries in the repo (dirs and files), +# used to validate that a prefix names an actual path. Populated per run from +# the tree being linted; empty means "couldn't determine, skip path checks". +TOPLEVEL="" + +# is_toplevel — true if is a real top-level entry or allowlisted. +is_toplevel() { + case " $TOPLEVEL " in *" $1 "*) return 0 ;; esac + case " $PATH_ALLOWLIST " in *" $1 "*) return 0 ;; esac + return 1 +} + +# Non-imperative openers (past tense / gerund) that read as "described the +# change" rather than "make the change". Warning only. +NON_IMPERATIVE="added adds adding fixed fixes fixing updated updates updating \ +removed removes removing changed changes changing implemented implements \ +implementing created creates creating deleted refactored refactoring \ +improved improves improving bumped bumps" + +STRICT=0 +errors=0 +warnings=0 + +err() { printf ' \033[31merror\033[0m %s\n' "$1" >&2; errors=$((errors + 1)); } +warn() { printf ' \033[33mwarn\033[0m %s\n' "$1" >&2; warnings=$((warnings + 1)); } + +# lint_message + + +## notifications + +In-app notifications now exist! You get notifications for a variety of events now: + +* new issues/pulls on your repos (also for collaborators) +* comments on your issues/pulls (also for collaborators) +* close/reopen (or merge) of issues/pulls +* new stars +* new follows + +All of this can be fine-tuned in [/settings/notifications](https://tangled.org/settings/notifications). + +![notifications](https://assets.tangled.network/blog/notifications.png) + + +## the future + +We're working on a *lot* of exciting new things and possibly some big +announcements to come. Be on the lookout for: + +* email notifications +* preliminary support for issue and PR search +* total "atprotation" [^1] -- the last two holdouts here are repo and pull records +* total federation -- i.e. supporting third-party appviews by making it + reproducible +* achieve complete independence from Bluesky PBC by hosting our own relay + +That's all for now; we'll see you in the atmosphere! Meanwhile, if you'd like to contribute to projects on Tangled, look for "good first issue" labels on the repositories you use. + +[^1]: atprotation implies a two-way sync between the PDS and appview. Currently, pull requests and repositories are not ingested -- so writing/updating either records on your PDS will not show up on the appview. diff --git a/blog/posts/bobbin.md b/blog/posts/bobbin.md new file mode 100644 index 0000000000000000000000000000000000000000..a7d135664532f1a5cd64056d0bbf1e591e2a3409 --- /dev/null +++ b/blog/posts/bobbin.md @@ -0,0 +1,202 @@ +--- +atroot: true +template: +slug: bobbin +rkey: 3mqif4rzc22uh +title: Introducing Bobbin +subtitle: A diskless, API-only AppView for Tangled +image: https://assets.tangled.network/blog/bobbin.png +date: 2026-07-13 +announce: true +authors: + - name: Lewis + email: lewis@tangled.org + handle: oyster.cafe +--- + +Hello! It is I, Lewis, with my first Tangled post. +At Tangled my job involves a lot of Research & Development, by +which I mean fucking around and finding out. Let me tell you about one +such finding out, which turned into this new Tangled API. + +Bobbin is a read-only, API-only +[AppView](https://atproto.com/guides/glossary#app-view) +for Tangled. It serves the read side of the `sh.tangled.*` +[lexicons](https://atproto.com/specs/lexicon) +over [XRPC](https://atproto.com/specs/xrpc), so a +client can render git repositories, issues, pulls, +comments, stars, & follows, without having to ping N +[PDSes](https://atproto.com/guides/glossary#pds-personal-data-server) +for their data. It has no permanent storage. Everything it knows is only +ever in RAM and gets backfilled from upstream every restart. + +> I, along with many others, find the term "AppView" to be too +> abstract or perhaps even unfortunately-named. In atproto, datasets +> are distributed across Personal Data Servers (PDSes) that users each +> own, so an AppView is a single API through which one can query +> a dataset, in contrast to all downstream clients each maintaining a +> connection with every PDS. An AppView is one of the many programs in +> use to reduce the strain on PDSes, so that a PDS is relatively cheap and +> resource-efficient to run. + +Let me list some problems that I find Tangled's AppView to have, such +that the journey of discovery that led me to Bobbin makes sense: + +1. Tangled's AppView has no API; it doubles as a web client via SSR. +2. It is brittle in the sense that it's not able to backfill +[atproto](https://atproto.com) data easily. +3. Its architecture from the ground up is inherently built for the kind +of web program that assumes *it* is the source of truth of data. +4. Because of all of the above, and the fact that there *is* some +private data such as email -> user [DID](https://atproto.com/specs/did) +ties, there ought to only be one AppView for Tangled. This means that +latency to Tangled is vaguely a function of distance from the server +in Stockholm. +5. It is hungry for RAM during a RAM crisis. + +Flipping to the inverse of all of these items would yield me my ideal +AppView. An API-only, easily-backfillable, easily-distributable, +easy-on-RAM program. + +Why "API-only"? Once it's easy to retrieve Tangled data via generic +XRPC API, more programs can be built on top of it; programs such +as alternative frontends, third-party clients, one's own targeted +dashboards, in my case it was to enable a good CLI (though I suppose +that just comes under alternative frontends). The existing Tangled +AppView being its own web client means there's nothing extendable to +build on, unless one is a scraping wizard; one has to figure out their +own way to go grab the Tangled dataset from the world of PDSes in the +wild. + +While I was mulling how to go about creating such a program for fun, I +have also been having a crisis about where to store state, after I've +woken up in cold sweats wondering if Tranquil PDS' homegrown embedded +DB could fail in new mystical ways. How could I avoid this next program +having to write any durable, fallible state? Besides, AppViews are +downstream of PDSes, which means the more we try and hold onto state in +an AppView, the more chance there is that it'll diverge from reality. +All it takes to have an AppView be wildly incorrect is missing a single +record, so it is of upmost importance to have an AppView be able to +reconcile what it thinks it knows with what's actually true. + +Being the simpleton that I am, I think that there is no better way +to make a program whose only job is to cache and serve records than +to make it never touch disk in the first place. I would want it to +simply backfill all of its data from scratch on every restart. In that +case, reconciliation with reality is just a restart away and happens +automatically! Was that feasible? What if such a backfill took a whole +day every time? That would sure make deployments slow, would it anyhow +be worth it? + +My first tests of this concept gave promising results, I +threw together a little server that could in fact backfill +all of Tangled's atproto dataset in around 15 minutes, using +[Hydrant](https://tangled.org/did:plc:6v3ul2ptnqctyxwkz5ti4amn) as its +aggregator. I decided that an acceptable backfill time would be under +5 minutes or less, so that it would be feasible to switch out running +instances on a blue/green deploy within a reasonable warming timeframe. + +> Hydrant is an aggregator for arbitrary atproto datasets that works +> by tailing the live [firehose](https://atproto.com/specs/event-stream), +> backfilling every [repo](https://atproto.com/guides/glossary#data-repo) +> on the network by pulling their +> [CARs](https://atproto.com/specs/repository#car-file-serialization), +> and serving it back over a websocket. All Bobbin has to do is open a +> connection, ask for cursor 0, and (re)build its index off the resulting +> barrage. +> +> Massive gratitude for +> [Dawn](https://tangled.org/did:plc:dfl62fgb7wtjj3fcbb72naae) for having +> made Hydrant! Then for hearing my struggles and going and optimizing it +> also for fun! + +Once I reached backfill in 5 minutes, I moved the goalpost for fun, that +the acceptable backfill time should now be 90 seconds or less. This is +around the number we are approaching today, depending on how warm the +upstream Hydrant instance is; ranging from 30 seconds in best-case +scenario to 20 minutes in a disaster where Hydrant is also pulling +records from PDSes from scratch. + +> Yes, you too can spin these up & order the entirety of Tangled in 20 +> minutes or less, or you get a refund. (Of €0.) + +There's a little conundrum that I encountered: what to have the +program serve while the backfill is still warming? Do I bother +having the program serve requests while aggregate/linked data is +possibly incorrect? I think not. So do I have a program serving +nothing in the backfill time that is possibly 20 minutes at +worst? I probably could have done that, but instead we connect to +[Slingshot](https://tangled.org/did:plc:c7mc2fn47ihdihul4vjwsuy3/tree/main/slingshot) +for point-lookups at least and immediately get accurate +data in the meantime, even if the program itself hasn't technically seen +that data yet. + +> Slingshot is a wonderful edge cache for atproto records +> & identities. Bobbin asks it for single record and identity +> lookups, so that a freshly-born Bobbin can answer individual +> queries accurately before its own index has warmed up. Slingshot +> is part of the [microcosm](https://www.microcosm.blue/) project, +> a bundle of community-run atproto infrastructure. Thanks +> [Fig](https://tangled.org/did:plc:hdhoaan3xa3jiuq4fg4mefid) for making +> it! + +The program that was turning into Bobbin was able to hold all of +Tangled's atproto dataset in around 200MB of RAM, which was good, but +some compression led to 100MB. There is likely more I could compress +that is yet to be done. "Wow, such a small amount of data for the entire +social layer of a whole platform!" you remark, but on the contrary +remember that this is the *smallest* the dataset will likely ever be +from today onwards. Space-saving is of the essence. + +Bobbin uses Slingshot as a record/identity resolver in real time for +anything it doesn't already hold in RAM or hasn't necessarily received +from Hydrant. I love Fig's work, don't get me wrong, but my lean on +Slingshot instead of just writing the pings myself was a cop-out to save +time; if there's one major architectural change yet to come to Bobbin, +it's inlining that away such that Bobbin never has to ping Slingshot and +can figure out faster ways of getting the same data. + +One would be tempted to throw in an embedded instance of Hydrant to +their AppView program itself, and have the serving of a particular +atproto dataset be just a thin wrapper over the Hydrant data; but +I want it to be easy to spin up possibly hundreds of instances of my +program. Having hundreds of Hydrants all pinging PDSes all the time +would be quite wasteful. One Hydrant instance can serve hundreds of +Bobbins without a sweat. + +Since we don't have a Hydrant/relay equivalent for Knot-related data, +Bobbin proxies Knot-related requests straight through to a given knot. I +think a knotstream aggregator is in order, perhaps even just a plugin to +Hydrant that has it serving both. I think Bobbin and Knot Mirror should +merge into one service, but that's just my opinion. + +With a "stateless" AppView, we don't have to worry about databases and +their woes: migrations, production massaging when things are slightly +off compared to the "true" dataset, et cetera. Any time we want to +change/add functionality, all we have to do is simply restart +Bobbin and the operation is done. Another benefit is that Bobbin lends +itself to "cloud-native deployments" as the corpos say. Tangled's own +flagship instance runs on Cloudflare Containers. + +All this being said, I'm not necessarily married to the idea that Bobbin +never touches disk, I think that one cute improvement could be to simply +dump RAM-stored-data contents as a snapshot once every hour and backfill +the difference at startup; but for now I think there's headroom to +continue down this path, considering Bobbin can healthily run with 200MB +given to it at time of writing. Additionally, my original requirement of +being easy on RAM is still fulfilled for now, but the moment it becomes +too heavy is another point at which Bobbin will offload parts of itself +to disk dynamically during runtime. How do I define "too heavy", you +ask? I look inside my heart, which tells me that 1GB of RAM is wasteful, +10GB is unacceptable. Bobbin also must serve users, whose each API +request puts more pressure on the RAM beyond just storing the Tangled +dataset. + +There you have it! Now there's this fun AppView with no disk storage, +sub-90-second backfill, that serves API requests as a little engine +that could. With it, anyone can build programs using the Tangled dataset +more easily. + +There is an instance of Bobbin on [api.tangled.org](https://api.tangled.org), +and hopefully you'd find it easy to host your own. Bobbin's source lives in +[our monorepo](https://tangled.org/tangled.org/core/tree/master/bobbin). diff --git a/blog/posts/ci.md b/blog/posts/ci.md new file mode 100644 index 0000000000000000000000000000000000000000..80e4abb3404c3677a7beeff285415a4e2064c66c --- /dev/null +++ b/blog/posts/ci.md @@ -0,0 +1,216 @@ +--- +atroot: true +template: +slug: ci +rkey: 3lvow4ttk22b3 +title: Introducing spindle +subtitle: Tangled's new CI runner is now generally available +date: 2025-08-06 +announce: true +authors: + - name: Anirudh + email: anirudh@tangled.org + handle: anirudh.fi + - name: Akshay + email: akshay@tangled.org + handle: oppi.li +--- + +Since launching Tangled, continuous integration has +consistently topped our feature request list. Today, CI is +no longer a wishlist item, but a fully-featured reality. + +Meet **spindle**: Tangled's new CI runner built atop Nix and +AT Protocol. In typical Tangled fashion we've been +dogfooding spindle for a while now; this very blog post +you're reading was [built and published using +spindle](https://tangled.org/tangled.org/core/pipelines). + +Tangled is a new social-enabled Git collaboration platform, +[read our intro](/blog/intro) for more about the project. + +![spindle architecture](https://assets.tangled.network/blog/spindle-arch.png) + +## how spindle works + +Spindle is designed around simplicity and the decentralized +nature of the AT Protocol. In ingests "pipeline" records and +emits job status updates. + +When you push code or open a pull request, the knot hosting +your repository emits a pipeline event +(`sh.tangled.pipeline`). Running as a dedicated service, +spindle subscribes to these events via websocket connections +to your knot. + +Once triggered, spindle reads your pipeline manifest, spins +up the necessary execution environment (covered below), and +runs your defined workflow steps. Throughout execution, it +streams real-time logs and status updates +(`sh.tangled.pipeline.status`) back through websockets, +which the Tangled appview subscribes to for live updates. + +Over at the appview, these updates are ingested and stored, +and logs are streamed live. + +## spindle pipelines + +The pipeline manifest is defined in YAML, and should be +relatively familiar to those that have used other CI +solutions. Here's a minimal example: + +```yaml +# test.yaml + +when: + - event: ["push", "pull_request"] + branch: ["master"] + +dependencies: + nixpkgs: + - go + +steps: + - name: run all tests + environment: + CGO_ENABLED: 1 + command: | + go test -v ./... +``` + +You can read the [full manifest spec +here](https://docs.tangled.org/spindles#pipelines), +but the `dependencies` block is the real interesting bit. +Dependencies for your workflow, like Go, Node.js, Python +etc. can be pulled in from nixpkgs. +[Nixpkgs](https://github.com/nixos/nixpkgs/) -- for the +uninitiated -- is a vast collection of packages for the Nix +package manager. Fortunately, you needn't know nor care +about Nix to use it! Just head to https://search.nixos.org +to find your package of choice (I'll bet 1€ that it's +there[^1]), toss it in the list and run your build. The +Nix-savvy of you lot will be happy to know that you can use +custom registries too. + +[^1]: I mean, if it isn't there, it's nowhere. + +Workflow manifests are intentionally simple. We do not want +to include a "marketplace" of workflows or complex job +orchestration. The bulk of the work should be offloaded to a +build system, and CI should be used simply for finishing +touches. That being said, this is still the first revision +for CI, there is a lot more on the roadmap! + +Let's take a look at how spindle executes workflow steps. + +## workflow execution + +At present, the spindle "engine" supports just the Docker +backend[^2]. Podman is known to work with the Docker socket +feature enabled. Each step is run in a separate container, +with the `/tangled/workspace` and `/nix` volumes persisted +across steps. + +[^2]: Support for additional backends like Firecracker are + planned. Contributions welcome! + +The container image is built using +[Nixery](https://nixery.dev). Nixery is a nifty little tool +that takes a path-separated set of Nix packages and returns +an OCI image with each package in a separate layer. Try this +in your terminal if you've got Docker installed: + +``` +docker run nixery.dev/bash/hello-go hello-go +``` + +This should output `Hello, world!`. This is running the +[hello-go](https://search.nixos.org/packages?channel=25.05&show=hello-go) +package from nixpkgs. + +Nixery is super handy since we can construct these images +for CI environments on the fly, with all dependencies baked +in, and the best part: caching for commonly used packages is +free thanks to Docker (pre-existing layers get reused). We +run a Nixery instance of our own at +https://nixery.tangled.sh but you may override that if you +choose to. + +## debugging CI + +We understand that debugging CI can be the worst. There are +two parts to this problem: + +- CI services often bring their own workflow definition + formats and it can sometimes be difficult to know why the + workflow won't run or why the workflow definition is + incorrect +- The CI job itself fails, but this has more to do with the + build system of choice + +To mend the first problem: we are making use of git +[push-options](https://git-scm.com/docs/git-push#Documentation/git-push.txt--ooption). +When you push to a repository with an option like so: + +``` +git push origin master -o verbose-ci +``` + +The server runs a basic set of analysis rules on your +workflow file, and reports any errors: + +``` +λ git push origin main -o verbose-ci + . + . + . + . +remote: error: failed to parse workflow(s): +remote: - at .tangled/workflows/fmt.yml: yaml: line 14: did not find expected key +remote: +remote: warning(s) on pipeline: +remote: - at build.yml: workflow skipped: did not match trigger push +``` + +The analysis performed at the moment is quite basic (expect +it to get better over time), but it is already quite useful +to help debug workflows that don't trigger! + +## pipeline secrets + +Secrets are a bit tricky since AT Protocol has no notion of +private data. Secrets are instead written directly from the +appview to the spindle instance using [service +auth](https://docs.bsky.app/docs/api/com-atproto-server-get-service-auth). +In essence, the appview makes a signed request using the +logged-in user's DID key; spindle verifies this signature by +fetching the public key from the DID document. + +![pipeline secrets](https://assets.tangled.network/blog/pipeline-secrets.png) + +The secrets themselves are stored in a secret manager. By +default, this is the same sqlite database that spindle uses. +This is *fine* for self-hosters. The hosted, flagship +instance at https://spindle.tangled.network however uses +[OpenBao](https://openbao.org), an OSS fork of HashiCorp +Vault. + +## get started now + +You can run your own spindle instance pretty easily: the +[spindle self-hosting +guide](https://docs.tangled.org/spindles#self-hosting-guide) +should have you covered. Once done, head to your +repository's settings tab and set it up! Doesn't work? Feel +free to pop into [Discord](https://chat.tangled.org) to get +help -- we have a nice little crew that's always around to +help. + +All Tangled users have access to our hosted spindle +instance, free of charge[^3]. You don't have any more +excuses to not migrate to Tangled now -- [get +started](https://tangled.org/login) with your AT Protocol +account today. + +[^3]: We can't promise we won't charge for it at some point + but there will always be a free tier. diff --git a/blog/posts/docs.md b/blog/posts/docs.md new file mode 100644 index 0000000000000000000000000000000000000000..dfe0eafbbe734fadec5d3ee179a4a2466bed741d --- /dev/null +++ b/blog/posts/docs.md @@ -0,0 +1,240 @@ +--- +atroot: true +template: +slug: docs +rkey: 3mc6qba4s22am +title: We rolled our own documentation site +subtitle: You don't need Mintlify +date: 2026-01-12 +authors: + - name: Akshay + email: akshay@tangled.org + handle: oppi.li +draft: false +--- + +We recently organized our documentation and put it up on +https://docs.tangled.org, using just pandoc. For several +reasons, using pandoc to roll your own static sites is more +than sufficient for small projects. + +![docs.tangled.org](https://assets.tangled.network/blog/docs_homepage.png) + +## requirements + +- Lives in [our + monorepo](https://tangled.org/tangled.org/core). +- No JS: a collection of pages containing just text + should not require JS to view! +- Searchability: in practice, documentation engines that + come bundled with a search-engine have always been lack + lustre. I tend to Ctrl+F or use an actual search engine in + most scenarios. +- Low complexity: building, testing, deploying should be + easy. +- Easy to style + +## evaluating the ecosystem + +I took the time to evaluate several documentation engine +solutions: + +- [Mintlify](https://www.mintlify.com/): It is quite obvious + from their homepage that Mintlify is performing an AI + pivot for the sake of doing so. +- [Docusaurus](https://docusaurus.io/): The generated + documentation site is quite nice, but the value of pages + being served as a full-blown React SPA is questionable. +- [MkDocs](https://www.mkdocs.org/): Works great with JS + disabled, however the table of contents needs to be + maintained via `mkdocs.yml`, which can be quite tedious. +- [MdBook](https://rust-lang.github.io/mdBook/index.html): + As above, you need a `SUMMARY.md` file to control the + table-of-contents. + +MkDocs and MdBook are still on my radar however, in case we +need a bigger feature set. + +## using pandoc + +[pandoc](https://pandoc.org/) is a wonderfully customizable +markup converter. It provides a "chunkedhtml" output format, +which is perfect for generating documentation sites. Without +any customization, +[this](https://pandoc.org/demo/example33/) is the generated +output, for this [markdown file +input](https://pandoc.org/demo/MANUAL.txt). + +- You get an autogenerated TOC based on the document layout +- Each section is turned into a page of its own + +Massaging pandoc to work for us was quite straightforward: + +- I first combined all our individual markdown files into + one big `DOCS.md` file. +- Modified the [default + template](https://github.com/jgm/pandoc-templates/blob/master/default.chunkedhtml) + to put the TOC on every page, to form a "sidebar", see + `docs/template.html` +- Inserted tailwind `prose` classes where necessary, such + that markdown content is rendered the same way between + `tangled.org` and `docs.tangled.org` + +Generating the docs is done with one pandoc command: + +```bash +pandoc docs/DOCS.md \ + -o out/ \ + -t chunkedhtml \ + --variable toc \ + --toc-depth=2 \ + --css=docs/stylesheet.css \ + --chunk-template="%i.html" \ + --highlight-style=docs/highlight.theme \ + --template=docs/template.html +``` + +## avoiding javascript + +The "sidebar" style table-of-contents needs to be collapsed +on mobile displays. Most of the engines I evaluated seem to +require JS to collapse and expand the sidebar, with MkDocs +being the outlier, it uses a checkbox with the +[`:checked`](https://developer.mozilla.org/en-US/docs/Web/CSS/Reference/Selectors/:checked) +pseudo-class trick to avoid JS. + +The other ways to do this are: + +- Use ``: this is definitely a + "hack", clicking outside the sidebar does not collapse it. + Using Ctrl+F or "Find in page" still works through the + details tag though. +- Use the new `popover` API: this seems like the perfect fit + for a "sidebar" component. + +The bar at the top includes a button to trigger the popover: + +```html + +``` + +And a `fixed` position div includes the TOC itself: + +```html +
+
    + Quick Start +
  • ...
    • +
  • ...
    • +
  • ...
    • +
+
+``` + +The TOC is scrollable independently and can be collapsed by +clicking anywhere on the screen outside the sidebar. +Searching for content in the page via "Find in page" does +not show any results that are present in the popover +however. The collapsible TOC is only available on smaller +viewports, the TOC is not hidden on larger viewports. + +## search + +There is no native search on the site for now. Taking +inspiration from [https://htmx.org](https://htmx.org)'s search bar, our search +bar also simply redirects to Google: + +```html +
+ + ... +
+``` + +I mentioned earlier that Ctrl+F has typically worked better +for me than, say, the search engine provided by Docusaurus. +To that end, the same docs have been exported to a "single +page" format, by +just removing the `chunkedhtml` related options: + +```diff + pandoc docs/DOCS.md \ + -o out/ \ +- -t chunkedhtml \ + --variable toc \ + --toc-depth=2 \ + --css=docs/stylesheet.css \ +- --chunk-template="%i.html" \ + --highlight-style=docs/highlight.theme \ + --template=docs/template.html +``` + +With all the content on a single page, it is trivial to +search through the entire site with the browser. If the docs +do outgrow this, I will consider other options! + +## building and deploying + +We use [nix](https://nixos.org) and +[colmena](https://colmena.cli.rs/) to build and deploy all +Tangled services. A nix derivation to [build the +documentation](https://tangled.org/tangled.org/core/blob/master/nix/pkgs/docs.nix) +site is written very easily with the `runCommandLocal` +helper: + +```nix +runCommandLocal "docs" {} '' + . + . + . + ${pandoc}/bin/pandoc ${src}/docs/DOCS.md ... + . + . + . +'' +``` + +The NixOS machine is configured to serve the site [via +nginx](https://tangled.org/tangled.org/infra/blob/master/hosts/nixery/services/nginx.nix#L7): + +```nix +services.nginx = { + enable = true; + virtualHosts = { + "docs.tangled.org" = { + root = "${tangled-pkgs.docs}"; + locations."/" = { + tryFiles = "$uri $uri/ =404"; + index = "index.html"; + }; + }; + }; +}; +``` + +And deployed using `colmena`: + +```bash +nix run nixpkgs#colmena -- apply +``` + +To update the site, I first run: + +```bash +nix flake update tangled +``` + +Which bumps the `tangled` flake input, and thus +`tangled-pkgs.docs`. The above `colmena` invocation applies +the changes to the machine serving the site. + +## notes + +Going homegrown has made it a lot easier to style the +documentation site to match the main site. Unfortunately +there are still a few discrepancies between pandoc's +markdown rendering and +[goldmark's](https://pkg.go.dev/github.com/yuin/goldmark/) +markdown rendering (which is what we use in Tangled). We may +yet roll our own SSG, +[TigerStyle](https://tigerbeetle.com/blog/2025-02-27-why-we-designed-tigerbeetles-docs-from-scratch/)! diff --git a/blog/posts/federation.md b/blog/posts/federation.md new file mode 100644 index 0000000000000000000000000000000000000000..36c63708bd45b60c62e1434065180c8415d36339 --- /dev/null +++ b/blog/posts/federation.md @@ -0,0 +1,53 @@ +--- +atroot: true +template: +slug: federation +rkey: 3mkls5qh222ng +title: We need a federation of forges +subtitle: Git is decentralized, but what of the rest? +image: https://assets.tangled.network/blog/federation/Light-OG.png +date: 2026-04-29 +authors: + - name: Akshay + email: akshay@tangled.org + handle: oppi.li +--- + +GitHub seems to be crumbling the past couple of weeks. +Whatever the reason, ultimately its not great for 90% of the +world's OSS to depend on one provider. Centralized systems +always crumble; it's the emails, gits, and IRCs that stand +the test of time. Tangled aims to fit in this space, allow +me to explain. + +Code collaboration has always made use of two protocols, one +for code transfer and one for communication: + +- It began with the email flow: git (code transfer) + email + (comms) +- Then there was GitHub: git (code transfer) + GitHub the + website (comms) +- There is the ForgeFed project: git (code transfer) + + [maybe + ActivityPub](https://forgefed.org/blog/actor-programming/) + (comms) +- We are building Tangled: git (code transfer) + [AT + Protocol](https://atproto.com/) (comms) + +Tangled federates events among git servers (called "knots"). +You can collaborate on repositories on any server and you +can fork across servers. You can even push to a repository +on your own server, and open a pull-request on a repo hosted +on a completely different server. In a lot of ways, this is +quite like hosting your own cgit instance, and sending out +patches via email. + +Tangled uses AT to facilitate the Authenticated Transfer of +events surrounding code: like issues and pull-requests, and +it also enables a few social bits: a timeline of events, +follows, stars (and vouches very soon). AT is used to share +collaborator invites and ssh pubkeys, but the rest is just +good ol' git. + +OSS needs to break free from monocultures like GitHub, but +code collaboration should still be fun and social. diff --git a/blog/posts/forge.mdx b/blog/posts/forge.mdx new file mode 100644 index 0000000000000000000000000000000000000000..3605d5a775f5789b595c223f868218c8f83f07de --- /dev/null +++ b/blog/posts/forge.mdx @@ -0,0 +1,164 @@ +--- +atroot: true +template: +slug: forge +rkey: 3myvay62j2u22 +title: The federated code forge +subtitle: +date: 2026-10-30 +announce: true +image: https://assets.tangled.network/blog/forge/og.png +video: https://assets.tangled.network/blog/forge/og.mp4 +authors: + - name: Anirudh + email: anirudh@tangled.org + handle: anirudh.fi +draft: false +--- + +import ForgeTimeline from "../src/components/ForgeTimeline.svelte"; +import Ripple from "../src/components/Ripple.astro"; +import OrganizationsEmbed from "../src/components/embeds/OrganizationsEmbed.svelte"; +import PullStackEmbed from "../src/components/embeds/PullStackEmbed.svelte"; +import TicketsEmbed from "../src/components/embeds/TicketsEmbed.svelte"; +import TimelineEmbed from "../src/components/embeds/TimelineEmbed.svelte"; + +For most of the past two decades, the code forge has been where projects live: +where code is hosted, where changes are reviewed, and where contributors find +each other. Forges offered convenience in exchange for centralisation. + +But the centralised forge is crumbling. Relying on someone else for uptime, for +features, and to be a good steward of your code hasn't played out well. What +*has* stood the test of time is email: Git to move the code, and email -- an +open protocol -- to talk about it. At the cost of user experience, however. + + + +Tangled aims to give you the best of both: the longevity of an open +protocol, with the convenience and polish of a proper forge. + +It's the first truly federated code forge -- you can host code on your own +servers and still collaborate seamlessly with others. Our approach is +pragmatic: self-host just the parts you want, like Git hosting or CI runners, +or all of it, including the API server and UI. Whatever [you +choose](https://tangled.org/docs/self-hosting), you can still talk to everyone +else. + +Owning your infrastructure also means you can hack on, extend +and customise your forge as you see fit. Because Tangled is +built on the AT Protocol, you can build your own extensions, +subscribe to the firehose, or even write entirely custom knots, +like [this one](https://tangled.org/oppi.li/naut) that runs on +Cloudflare Workers and uses their Artifacts product for git +storage. Or an entirely custom spindle such as +[mitchellh.com/tack](https://tangled.org/mitchellh.com/tack). Go +even further and host our [ultralight API +server](https://tangled.org/docs/self-hosting/indexer) and build +your own custom forge. + +Agents have made building such hyper-personalized software trivial, and +especially so when software is designed with hackability in mind from the +get-go. Tangled is incredibly [agent](https://tangled.org/docs/api/bobbin/) +[friendly](https://atproto.com/guides/identity). + +The future of software engineering may be a moving target, but our bets +on open protocols and developer agency give us the agility to move with +it, and to become . + +# Product updates + +## A fresh look! + +The obvious big one: with this release, Tangled has a whole new look. +We've redone the UI on top of our new architecture (described further +below), in Svelte. Things should now be quicker, visually consistent and +prettier! The keen eyed of you might've also spotted that we've swapped +out Inter for [Timeless Sans](https://timeless.co/type). + + + +## A bobbin near you + +Until now, Tangled's web UI and backend were a single Go monolith called the +`appview`. The appview service handled AT Protocol indexing, caching, +off-protocol data and UI rendering using `html/template` + htmx. It ran on a +single SQLite database on one medium-sized server in Sweden. + +This was great as long as you were ~50–100 ms away from Sweden, but most +of our users aren't! We rearchitected: we built a new standalone +frontend app in Svelte, deployed at the edge. It is populated by our new +diskless, entirely in-memory indexer service written in Rust: [bobbin](https://tangled.org/docs/api/bobbin/). +Bobbins are deployed globally in over 11+ regions, ensuring users around +the world get the best API and UI performance. + +The astute reader might've noticed: bobbin is purely read-only and only +deals with AT Protocol data. What of the rest? That functionality (and +more) has been delegated to a new set of supporting services. + +## The new data model and repo-owned data + +First, the data model itself: we have a new set of lexicons, and everything now +lives under `org.tangled.*`, replacing the old `sh.tangled.*` records. + +Knot2 is our aptly named, second implementation of the knot server, this time +rewritten in Rust. Knot2 bakes a fully custom Git server, and also bundles in +an SSH server in the same binary. + +With knot2, we've also changed where some of that data lives. Issues, pull +requests, comments, reactions and labels are now stored on the knot, right next +to the code, as collaborative objects (COBs). A COB is just a chain of Git +objects in the repo itself. Collaborators, knot members and the blocklist are +also stored the same way. + +In that sense, knots now behave a lot like a PDS: they hold signed records, +full history included, for everything that belongs to a repo and much like a +PDS, these records are synced via the standard AT Protocol firehose. + +But why? A project's issues and pull requests are as much a part of it as its code, +so they should live with it, wherever you choose to host it. This gives your +project maximum portability. + +If the knot writes your issues and comments, what stops it from putting words +in your mouth? Attestations. Every time you create or edit a record on a knot, +the web app also writes an `org.tangled.graph.attestation` record to your own +PDS, pointing at that exact version of the record and the COB change behind it. +Two-way verification! + + + +Self-hosters will have to upgrade their knots to the v1.20.0 by [following the +upgrade guide]( https://tangled.org/docs/upgrading/knots-and-spindles/). + +## Organizations + +Organization accounts are here! An Organization on Tangled is a real AT +Protocol account, with its own DID and handle, hosted on a [Tranquil +PDS](https://tangled.org/tranquil.farm/tranquil-pds) instance. They're +passwordless, delegated accounts: you create one from the [New +Organization](https://tangled.org/organizations/new) page, and you get +`.tngl.sh` with yourself as its owner. Owners can invite members, and +anyone in the Organization can act as it when creating or forking repositories. +Like any other account, you can swap the handle out for a domain you own. If +you've been running a regular account as a stand-in "org", hang tight -- we're +working on a way to move those over. Read more in the +[docs](https://tangled.org/docs/guides/organizations). + + + +## Stacked PRs for everyone + +Our PR flow has been overhauled. The new flow gives everyone, regardless +of whether you use Jujutsu or just vanilla Git, the ability to submit +and review stacked PRs. If you're using Git, you now get the ability to +interdiff between versions (previously called rounds). JJ users get to +interdiff between individual changes as well. + + + +## Launch weeks ahead + +This isn't all. We're staggering this launch across the next few weeks. Up next +are private repos, a CLI, SDKs for different languages and skills for your +agent. + +Until then, enjoy every part of this launch equally. diff --git a/blog/posts/intro.md b/blog/posts/intro.md new file mode 100644 index 0000000000000000000000000000000000000000..877246da93a1105eaf491a877305fb3c537f11c2 --- /dev/null +++ b/blog/posts/intro.md @@ -0,0 +1,66 @@ +--- +atroot: true +template: +slug: intro +rkey: 3lje4wchs227l +title: Introducing Tangled +subtitle: A Git collaboration platform, built on AT Protocol +date: 2025-03-02 +announce: true +authors: + - name: Anirudh + email: anirudh@tangled.org + handle: anirudh.fi +--- + + +[Tangled](https://tangled.org) is a new social-enabled Git collaboration +platform, built on top of the [AT Protocol](https://atproto.com). We +envision a place where developers have complete ownership of their code, +open source communities can freely self-govern and most importantly, +coding can be social and fun again. + +There are several models for decentralized code collaboration platforms, +ranging from ActivityPub's (Forgejo) federated model, to Radicle's +entirely P2P model. Our approach attempts to be the best of both worlds +by adopting AT Protocol -- a protocol for building decentralized social +applications with a central identity. + +![tangled architecture](https://assets.tangled.network/blog/arch.svg) + +Our approach to this is the idea of "knots". Knots are lightweight, +headless servers that enable users to host Git repositories with ease. +Knots are designed for either single or multi-tenant use which is +perfect for self-hosting on a Raspberry Pi at home, or larger +"community" servers. By default, Tangled provides managed knots where +you can host your repositories for free. + +The [App View][appview] at [tangled.org](https://tangled.org) acts as a +consolidated "view" into the whole network, allowing users to access, +clone and contribute to repositories hosted across different knots -- +completely seamlessly. + +Tangled is still in its infancy, and we're building out several of its +core features as we [dogfood it ourselves][dogfood]. We developed these +three tenets to guide our decisions: + +1. Ownership of data +2. Low barrier to entry +3. No compromise on user-experience + +Collaborating on code isn't easy, and the tools and workflows we use +should feel natural and stay out of the way. Tangled's architecture +enables common workflows to work as you'd expect, all while remaining +decentralized. + +We believe that AT Protocol has greatly simplified one of the hardest parts +of social media: having your friends on it. Today, we're rolling out +invite-only access to Tangled -- join us on IRC at `#tangled` on +[libera.chat](https://libera.chat) and we'll get you set up. + +**Update**: Tangled is open to public, simply login at +[tangled.org/login](https://tangled.org/login)! Have fun! + +[pds]: https://atproto.com/guides/glossary#pds-personal-data-server +[appview]: https://docs.bsky.app/docs/advanced-guides/federation-architecture#app-views +[dogfood]: https://tangled.org/core diff --git a/blog/posts/newsletter-01.md b/blog/posts/newsletter-01.md new file mode 100644 index 0000000000000000000000000000000000000000..d58187dcb0dec5235c72b068cf5994bb2d898a0d --- /dev/null +++ b/blog/posts/newsletter-01.md @@ -0,0 +1,91 @@ +--- +atroot: true +template: +slug: newsletter-01 +rkey: 3mjl43rhc22oe +title: Newsletter 01 — Hello! +subtitle: Kicking off our newsletter! +image: https://assets.tangled.network/blog/newsletter-01.png +date: 2026-04-16 +authors: + - name: Anirudh + email: anirudh@tangled.org + handle: anirudh.fi +--- + +*This was originally sent as an email.* + +Hello Tanglers! We're excited to kick off the first ever issue of our +newsletter. We haven't settled on a cutesy name for it yet -- +"untangled"? Or simply "tangled weekly"? Let us know! + +If you're new here (how'd you end up on our list?!), Tangled is a new +social coding platform. You can host Git repositories, collaborate with +friends, and self-host your git remotes +([knots](https://tangled.org/docs/knots/self-hosting)) and CI +runners (spindles). We recently raised €3,8M ($4.5M) in [seed +financing](/blog/seed) and have been heads-down building. + +With that out of the way, here's what we shipped recently. + +## Repository search is live + +![search +graphic](https://assets.tangled.network/blog/search-graphic.png) + +We finally have it. You can now search for your favourite slopware at +[tangled.org/search](https://tangled.org/search) (or from the 🔍 icon in +the top nav). + +## Fresh new OpenGraph images + +![og graphic](https://assets.tangled.network/blog/og-graphic.png) + +[Étienne](https://tangled.org/eti.tf) helped us ship `ogre` -- the new +OpenGraph Rendering Engine. It's what's behind our lovely link previews +that you may have seen elsewhere already. + +## PR review improvements + +![review flow](https://assets.tangled.network/blog/reviewflow.gif) + +[Lewis](https://tangled.org/oyster.cafe), our most recent addition to +the team (based out of Helsinki!), shipped these handy PR review +improvements! You can multi-line select to reference relevant code in +review comments, and mark files that you've already looked at as +"reviewed". Nifty! + +## knotmirror and other planned features + +[Seongmin](https://tangled.org/boltless.me), our engineer from South +Korea 🇰🇷, built our new repository indexer service: knotmirror. + +Knotmirror is our first of many performance optimizations for repo +viewing & fetching on tangled. It attempts to fetch and index all known +git repos across knots, allowing the appview (the main service at +tangled.org) to serve repos much faster, regardless of where your knot +lives! + +Up next, we're working hard towards: + +* First implementation of **VM-based CI** to enable more advanced + workflows. This will be a new spindle engine likely backed by QEMU. At + least initially. We'll let you know when we ship it! +* **Vouching**: Tangled aims to be foundational infrastructure for the + future of open source dev -- and having first-class tooling for the + slopocalypse is paramount. The web-of-trust model has proven to be + rather powerful here and that's coming to Tangled very soon. +* Fully achieve full **AT Protocol compliance**. One of our internal + goals is to have a Tangled appview that anyone can run, with complete + backfill and ingestion -- dubbed the "ephemeral appview". We're not quite + there yet: repos are the tough part. We recently swapped to using DIDs + to identify repos, which gets us one step closer to being able to + rename them, transfer them between knots, and finally ingest them + through the firehose. + +That's all for now! We'd love to hear your thoughts on this newsletter +-- you can reply to this email or join our +[Discord](https://chat.tangled.org) to let us know. We'll also publish +this on our blog for those that prefer RSS. + +-- Anirudh diff --git a/blog/posts/newsletter-02.md b/blog/posts/newsletter-02.md new file mode 100644 index 0000000000000000000000000000000000000000..026499386154053f333a462dd70088ee8a62dc6f --- /dev/null +++ b/blog/posts/newsletter-02.md @@ -0,0 +1,115 @@ +--- +atroot: true +template: +slug: newsletter-02 +rkey: 3mnvfqfsk22br +title: Newsletter 02 +subtitle: Vouching, CI logs over SSH, and more +image: https://assets.tangled.network/blog/newsletter/02/og.png +date: 2026-06-10 +authors: + - name: Anirudh + email: anirudh@tangled.org + handle: anirudh.fi +--- + +*This was originally sent as an email.* + +Hello again, Tanglers! It's been a busy few weeks here at Tangled; so +busy in fact that we missed last month's newsletter. :D We've shipped a +lot, so let's get straight to it. + +## Vouching and web-of-trust + +![vouching graphic](https://assets.tangled.network/vouch.png) + +You can now vouch for (or denounce) anyone you interact with on Tangled. +Vouched users show a green shield beside their profile picture; +denounced users get a red one. These "hats" appear at the points where +it matters most -- in issues, PRs, and comments -- so you can make an +informed call before spending time on a review. Vouching is web-of-trust +scoped: you only see decisions made by you and the people you trust. + +As always, vouch records are public AT Protocol records stored on your +PDS. + +[Read more](https://blog.tangled.org/vouching) + +## SSH log tailing + +![astral +projection](https://assets.tangled.network/blog/pty/astral-projection.webp) + +You can now tail CI logs right from your terminal! Push to a branch, and +the remote now hands you an SSH command right in your terminal: + +```bash +λ jj git push -c @- + -- snip -- +remote: → Browse CI logs in your terminal: +remote: ssh -t -p 3333 tangled.org did:plc:j5hmlfdrwkvtxm7cjmu7j2is 796ecc5b0ce5381ed5b5021e7cc28b4b05e03c92 +``` + +Paste that into a new shell and you're dropped into a full-blown TUI +served over SSH -- no installation required. It's built on +[bubbletea](https://github.com/charmbracelet/bubbletea) and +[wish](https://github.com/charmbracelet/wish), so it themes with your +terminal colors and handles ANSI escape codes correctly because it's +writing to a real PTY. + +[Read more](https://blog.tangled.org/ssh) + +## New PR page + +![pr page +graphic](https://assets.tangled.network/blog/newsletter/02/pr.png) + +The pull request creation page got a significant refresh. Go check it out! + +## New timeline and notification pages + +![timeline +graphic](https://assets.tangled.network/blog/newsletter/02/timeline.png) + +We've rebuilt the timeline and notification pages from the ground up. +The timeline page now splits Global and Following feeds, and includes +new Notifications and Recents columns. + +![notifs +graphic](https://assets.tangled.network/blog/newsletter/02/notifications.png) + +The dedicated Notifications page now splits "work" and "social" +notifications into two separate feeds, helping declutter your inbox. + +## AT Protocol progress + +We've made meaningful strides on our "ephemeral appview" goal -- the +milestone where anyone can run a fully backfilled Tangled appview with +minimal compute. + +- **Unified feed lexicon.** We consolidated comment and +reaction records into a pair of new lexicon types: +`sh.tangled.feed.comment` and `sh.tangled.feed.reaction`. + +- **PR record ingestion.** Pull request records (`sh.tangled.repo.pull`) +are now fully ingested through the firehose. Your agents can create pull +requests by simply writing records to your (or their!?) PDS! + +- **Repository renames.** We teased this in our last newsletter: after +switching to DIDs for repo identity, we needed to wire up the rest of +the plumbing. That work is done. You can now rename a repository from +the settings page. Repository migrations between knots are next on this +list. + +## VM-based CI is in internal testing + +We're currently testing the new spindle engine backed by QEMU micro VMs, +with special niceties for Nix. :^) We're not quite ready to open this up +yet, but expect it to land for everyone in the coming weeks. + +--- + +That's it for this issue! As always, you can reply to this email, or +find us on [Discord](https://chat.tangled.org). + +-- Anirudh diff --git a/blog/posts/newsletter-03.md b/blog/posts/newsletter-03.md new file mode 100644 index 0000000000000000000000000000000000000000..91ee3a16352d2eee760abad7f15bbfd5d8b469c7 --- /dev/null +++ b/blog/posts/newsletter-03.md @@ -0,0 +1,83 @@ +--- +atroot: true +template: +slug: newsletter-03 +rkey: 3mqatpzn222v6 +title: Newsletter 03 +subtitle: Code search, focus mode, microVM CI and a new home +image: https://assets.tangled.network/blog/newsletter/03/og.png +date: 2026-07-10 +authors: + - name: Anirudh + email: anirudh@tangled.org + handle: anirudh.fi +--- + +*This was originally sent as an email.* + +Happy summer, Tanglers! While the rest of Europe vacations, we've been busy +shipping. Quality of life improvements like Focus Mode, a whole new CI engine +powered by microVMs, code search and a revamped timeline page. Lots to cover, +let's get into it! + +## Code search + +![Code Search feature +card](https://assets.tangled.network/blog/newsletter/03/CodeSearch.png) + +Full-text code search is here. You can now search for code across the +*entire* Tangled network, and yes, regex is supported. :) + +Try it here [in our monorepo](https://tangled.org/tangled.org/core/search)! + +## Focus mode + +![Focus Mode feature +card](https://assets.tangled.network/blog/newsletter/03/Focus.png) + +Focus Mode is a new way to clear notifications: lock in and knock out your list +one item at a time, with a task bar that nudges you to stay on track until the +queue is empty. + +## Experimental XRPC API + +![Experimental API feature +card](https://assets.tangled.network/blog/newsletter/03/API.png) + +The Tangled XRPC API is live. An experimental API (using the [AT Protocol XRPC +format](https://atproto.com/specs/xrpc)) to query Tangled is now available at +[api.tangled.org](https://api.tangled.org). Give it a spin: + +``` +curl https://api.tangled.org/xrpc/sh.tangled.repo.listRepos?subject=did:plc:wshs7t2adsemcrrd4snkeqli | jq +``` + +You can browse the available lexicons in the monorepo, more docs to follow. + +## A new home + +![New Home feature +card](https://assets.tangled.network/blog/newsletter/03/Home.png) + + The homepage now uses a brand-new three-column layout: Notifications and + Recents on the left, your Timeline (Global / Following) in the middle, and + Trending, Announcements, and more on the right. We hope you like the improved + information density just as much as we do! + +## MicroVM-powered CI + +![MicroVM CI feature +card](https://assets.tangled.network/blog/newsletter/03/MicroVM.png) + +Spindles can now run jobs inside NixOS/Alpine QEMU microVMs. That means you can +build Docker containers or run services like Postgres directly in your CI +workflows, each job isolated by default. + +[Read more →](https://blog.tangled.org/spindle-microvm) + +--- + +That's it for this issue! As always, you can reply to this email, or +find us on [Discord](https://chat.tangled.org). + +— Anirudh diff --git a/blog/posts/pty.md b/blog/posts/pty.md new file mode 100644 index 0000000000000000000000000000000000000000..d95c2ef9baa7fc753291ae55880475c8f1b8e41b --- /dev/null +++ b/blog/posts/pty.md @@ -0,0 +1,58 @@ +--- +atroot: true +template: +slug: ssh +rkey: 3mmupogss22ul +title: Tail CI logs over SSH +subtitle: Never leave your terminal to check on CI! +image: https://assets.tangled.network/blog/pty/astral-projection.webp +date: 2026-05-28 +announce: true +authors: + - name: Akshay + email: akshay@tangled.org + handle: oppi.li +--- + +If you push to a new branch on a remote, if CI was +triggered, Tangled now helpfully supplies an `ssh` command +for you: + +```bash +λ jj git push -c @- + -- snip -- +remote: → Browse CI logs in your terminal: +remote: ssh -t -p 3333 tangled.org did:plc:j5hmlfdrwkvtxm7cjmu7j2is 796ecc5b0ce5381ed5b5021e7cc28b4b05e03c92 +``` + +Paste that command in a new shell, and you are dropped into +a fullblown TUI, served over SSH: + + + +Try it for yourself: + +``` +ssh -t -p 3333 tangled.org did:plc:j5hmlfdrwkvtxm7cjmu7j2is 796ecc5b0ce5381ed5b5021e7cc28b4b05e03c92 +``` + +That SSH commmand runs a "screen based" program on the +remote machine. In this case, the program is a TUI built +using +[bubbletea](https://github.com/charmbracelet/bubbletea) and +[wish](https://github.com/charmbracelet/wish). + +SSH powered interfaces are wonderful for several reasons: + +- They require zero installation +- They work great for interactive bits like menus +- They have all the upsides of running a TUI: you can + persist them over tmux, theme them with terminal colors + etc. + +But they are doubly wonderful when streaming CI logs, the +screen-program is totally unaware of ANSI escape codes, but +they are handled correctly by virtue of writing to a PTY. +Both ends (the commands running in the CI job and the PTY) +are speaking the same language. diff --git a/blog/posts/pulls.md b/blog/posts/pulls.md new file mode 100644 index 0000000000000000000000000000000000000000..628648802c4c52a12d5d2cc7e88d77b339534309 --- /dev/null +++ b/blog/posts/pulls.md @@ -0,0 +1,196 @@ +--- +atroot: true +template: +slug: pulls +rkey: 3lmvbvq7k225r +title: The lifecycle of a pull request +subtitle: We shipped a bunch of PR features recently; here's how we built it +date: 2025-04-16 +image: https://assets.tangled.network/blog/hidden-ref.png +authors: + - name: Anirudh + email: anirudh@tangled.sh + handle: anirudh.fi + - name: Akshay + email: akshay@tangled.sh + handle: oppi.li +draft: false +--- + +We've spent the last couple of weeks building out a pull +request system for Tangled, and today we want to lift the +hood and show you how it works. + +If you're new to Tangled, [read our intro](/blog/intro) for the +full story! + +You have three options to contribute to a repository: + +- Paste a patch on the web UI +- Compare two local branches (you'll see this only if you're a +collaborator on the repo) +- Compare across forks + +Whatever you choose, at the core of every PR is the patch. +First, you write some code. Then, you run `git diff` to +produce a patch and make everyone's lives easier, or push to +a branch, and we generate it ourselves by comparing against +the target. + +## patch generation + +When you create a PR from a branch, we create a "patch" by +calculating the difference between your branch and the +target branch. Consider this scenario: + +
+ +
A is the merge-base for +feature and main.
+
+ +Your `feature` branch has advanced 2 commits since you first +branched out, but in the meanwhile, `main` has also advanced +2 commits. Doing a trivial `git diff feature main` will +produce a confusing patch: + +- the patch will apply the changes from `X` and `Y` +- the patch will **revert** the changes from `B` and `C` + +We obviously do not want the second part! To only show the +changes added by `feature`, we have to identify the +"merge-base": the nearest common ancestor of `feature` and +`main`. + + +In this case, `A` is the nearest common ancestor, and +subsequently, the patch calculated will contain just `X` and +`Y`. + +### ref comparisons across forks + +The plumbing described above is easy to do across two +branches, but what about forks? And what if they live on +different servers altogether (as they can in Tangled!)? + +Here's the concept: since we already have all the necessary +components to compare two local refs, why not simply +"localize" the remote ref? + +In simpler terms, we instruct Git to fetch the target branch +from the original repository and store it in your fork under +a special name. This approach allows us to compare your +changes against the most current version of the branch +you're trying to contribute to, all while remaining within +your fork. + +
+ +
Hidden tracking ref.
+
+ +We call this a "hidden tracking ref." When you create a pull +request from a fork, we establish a refspec that tracks the +remote branch, which we then use to generate a diff. A +refspec is essentially a rule that tells Git how to map +references between a remote and your local repository during +fetch or push operations. + +For example, if your fork has a feature branch called +`feature-1`, and you want to make a pull request to the +`main` branch of the original repository, we fetch the +remote `main` into a local hidden ref using a refspec like +this: + +``` ++refs/heads/main:refs/hidden/feature-1/main +``` + +Since we already have a remote (`origin`, by default) to the +original repository (remember, we cloned it earlier), we can +use `fetch` with this refspec to bring the remote `main` +branch into our local hidden ref. Each pull request gets its +own hidden ref, hence the `refs/hidden/:localRef/:remoteRef` +format. We keep this ref updated whenever you push new +commits to your feature branch, ensuring that comparisons -- +and any potential merge conflicts -- are always based on the +latest state of the target branch. + +And just like earlier, we produce the patch by diffing your +feature branch with the hidden tracking ref. Also, the entire pull +request is stored as [an AT Protocol record][atproto-record] and updated +each time the patch changes. + +[atproto-record]: https://pdsls.dev/at://did:plc:qfpnj4og54vl56wngdriaxug/sh.tangled.repo.pull/3lmwniim2i722 + +Neat, now that we have a patch; we can move on the hard +part: code review. + + +## your patch does the rounds + +Tangled uses a "round-based" review format. Your initial +submission starts "round 0". Once your submission receives +scrutiny, you can address reviews and resubmit your patch. +This resubmission starts "round 1". You keep whittling on +your patch till it is good enough, and eventually merged (or +closed if you are unlucky). + +
+ +
A new pull request with a couple +rounds of reviews.
+
+ +Rounds are a far superior to standard branch-based +approaches: + +- Submissions are immutable: how many times have your + reviews gone out-of-date because the author pushed commits + _during_ your review? +- Reviews are attached to submissions: at a glance, it is + easy to tell which comment applies to which "version" of + the pull-request +- The author can choose when to resubmit! They can commit as + much as they want to their branch, but a new round begins + when they choose to hit "resubmit" +- It is possible to "interdiff" and observe changes made + across submissions (this is coming very soon to Tangled!) + +This [post by Mitchell +Hashimoto](https://mitchellh.com/writing/github-changesets) +goes into further detail on what can be achieved with +round-based reviews. + +## future plans + +To close off this post, we wanted to share some of our +future plans for pull requests: + +* `format-patch` support: both for pasting in the UI and + internally. This allows us to show commits in the PR page, + and offer different merge strategies to choose from + (squash, rebase, ...). + **Update 2025-08-12**: We have format-patch support! + +* Gerrit-style `refs/for/main`: we're still hashing out the + details but being able to push commits to a ref to + "auto-create" a PR would be super handy! + +* Change ID support: This will allow us to group changes + together and track them across multiple commits, and to + provide "history" for each change. This works great with [Jujutsu][jj]. + **Update 2025-08-12**: This has now landed: https://blog.tangled.org/stacking + +Join us on [Discord](https://chat.tangled.sh) or +`#tangled` on libera.chat (the two are bridged, so we will +never miss a message!). We are always available to help +setup knots, listen to feedback on features, or even +shepherd contributions! + +**Update 2025-08-12**: We move fast, and we now have jujutsu support, and an +early in-house CI: https://blog.tangled.org/ci. You no longer need a Bluesky +account to sign-up; head to https://tangled.sh/signup and sign up with your +email! + +[jj]: https://jj-vcs.github.io/jj/latest/ diff --git a/blog/posts/seed.md b/blog/posts/seed.md new file mode 100644 index 0000000000000000000000000000000000000000..a2878d10334d16d5674a2a67b2d60220985844f4 --- /dev/null +++ b/blog/posts/seed.md @@ -0,0 +1,74 @@ +--- +atroot: true +template: +slug: seed +rkey: 3mfzx4dpk227g +title: Announcing our €3,8M seed round +subtitle: And more on what's next +date: 2026-03-02 +announce: true +image: https://assets.tangled.network/blog/seed.png +authors: + - name: Anirudh + email: anirudh@tangled.org + handle: anirudh.fi +--- + +Today, we're announcing our €3,8M ($4.5M) financing round led by +[byFounders](https://byfounders.vc), with participation from [Bain +Capital Crypto](https://baincapitalcrypto.com/), +[Antler](https://antler.co), Thomas Dohmke (former GitHub CEO), Avery +Pennarun (CEO of Tailscale), among other incredible angels. + +For the past year, we've been building Tangled from the ground up -- +starting from first principles and asking ourselves what code +collaboration should really look like. We made deliberate, +[future-facing technology](https://anirudh.fi/future) choices. We chose +to build on top of the AT Protocol as it helped us realize a federated, +open network where users can own their code and social data. We shipped +stacked PRs to enable more efficient contribution and review workflows. +What started off as a side project, grew to over 7k+ users, who've +created over 5k+ repositories. + +Our vision for Tangled has always been big: we want to build the best +code forge ever, and become foundational infrastructure for the next +generation of open source. Whatever that looks like: hundreds of devs +building artisanal libraries, or one dev and a hundred agents building a +micro-SaaS. + +And finding the right investors to help us achieve this vision wasn't +something we took lightly. We spent months getting to know potential +partners -- among which, byFounders stood out immediately. Like us, +they're community-driven at their core, and their commitment to +transparency runs deep -- you can see the very term sheet we signed on +their website! With these shared fundamental values, we knew byFounders +were the right people to have in our corner and we're incredibly excited +to work with them. + +## what's next + +We're heads down building. For 2026, expect to see: + +* a fully revamped CI (spindle v2!) built on micro VMs to allow for + faster builds and more choice of build environments. Oh, and a proper + Nix CI -- we know you want it. +* protocol-level improvements across the board that'll unlock nifty + things like repo migrations across knots, organizations, and more! +* a customizable "mission control" dashboard for your active PRs, + issues, and anything else you might want to track. +* a migration tool to help you move off GitHub +* all things search: code search, repo search, etc. +* platform and infrastructure performance improvements & more global +presence +* Tangled CLI! + +If all this sounds exciting to you: we're growing our team! Shoot us +[an email](mailto:team@tangled.org) telling us a bit about yourself and +any past work that might be relevant, and what part of the roadmap +interests you most. We can hire from (almost) anywhere. + +New to Tangled? [Get started here](https://docs.tangled.org/). Oh, and +come hang on [Discord](https://chat.tangled.org)! + +A sincere thank you to everyone that helped us get here -- we're giddy +about what's to come. diff --git a/blog/posts/selfhost-ci.md b/blog/posts/selfhost-ci.md new file mode 100644 index 0000000000000000000000000000000000000000..a98fb57e0e33d7e2038a632f59d20913619bed7e --- /dev/null +++ b/blog/posts/selfhost-ci.md @@ -0,0 +1,38 @@ +--- +atroot: true +template: +slug: selfhost-ci +rkey: 3mtafgz5k22x6 +title: Run your own CI +subtitle: It is worth it +image: https://assets.tangled.network/blog/rare.png +date: 2026-08-17 +authors: + - name: Akshay + email: akshay@tangled.org + handle: oppi.li +--- + +If the outage of a service prevents you from building and +testing your software, it's time for you to take things into +your own hands. + +Hosting your own CI will be worth it. Agents are always new +to codebases and need to constantly test their work to avoid +regressions. They need CI even more than you do, arguably. + +Tangled makes self-hosting CI trivial. You can run one by +simply [setting up a +machine](https://docs.tangled.org/spindles#self-hosting-guide) +with our CI runner module. Alternatively, you can also write +your own runner by satisfying the +[interface](https://tangled.org/tangled.org/core/tree/74ce1a0f0c21749bdf15198a4bc7dc457696bf97/lexicons/ci): +[mitchellh.com/tack](https://tangled.org/mitchellh.com/tack) +can proxy builds to Buildkite, Tekton and sourcehut CI. Our +CI engine is infinitely extensible. + +Tangled's approach to self-hosted CI is different from the +likes of GitHub. If the app server at Tangled goes down, +your CI jobs continue to schedule and run. As they should! + +Unicorns are supposed to be rare. diff --git a/blog/posts/spindle-microvm.md b/blog/posts/spindle-microvm.md new file mode 100644 index 0000000000000000000000000000000000000000..223bb7437380922096d72ed30118d9e169699d01 --- /dev/null +++ b/blog/posts/spindle-microvm.md @@ -0,0 +1,377 @@ +--- +atroot: true +template: +slug: spindle-microvm +rkey: 3moeijwl222kc +title: Tangled CI runs on microVMs +subtitle: How we built spindle's new QEMU-based microVM engine +date: 2026-06-16 +announce: true +image: https://assets.tangled.network/blog/microvm.png +authors: + - name: dawn + email: dawn@tangled.org + handle: ptr.pet +--- + +Spindles are the self-hostable CI runners. It now supports a +new mode of execution using QEMU MicroVMs. With the new +microVM engine, each workflow gets its own little virtual +machine, a whole real environment you can do anything +inside. + +The interesting part is NixOS images: you configure the +machine directly from the workflow file. A few things you +can do: + +You can bring services up: + +```yaml +services: + postgresql: + enable: true + ensureDatabases: ["spindle-workflow"] + ensureUsers: + - name: spindle-workflow + ensureDBOwnership: true +``` + +You can build Docker containers: + +```yaml +virtualisation: + docker: true +steps: + - name: "do the thing!" + command: docker build ... +``` + +And you can use non-NixOS images: + +```yaml +image: alpine +steps: + - name: install golang + command: apk add go +``` + +It's an upgrade from the existing Nixery engine while +staying fully compatible with it, so if you already have a +working Nixery workflow, just change `nixery` to `microvm` +and it will work! + +It's quick on the second run, too, because it caches aggressively: your +dependencies, your services, and any other Nix derivation built inside the +microVM get pushed to spindle's Nix cache, so the next workflow that needs them +doesn't rebuild those. More on that [below](#nix-cache-both-ways). + +And like everything else in Tangled, the whole thing is self-hostable, so you +can run your own spindle with the microVM engine on your own hardware (see the +[self-hosting +guide](https://docs.tangled.org/spindles.html#self-hosting-guide)). If you want +fuller examples, there are [recipes in the +docs](https://docs.tangled.org/spindles.html#recipes) too. + +## What's in a microVM + +A microVM is just a VM with most of the boring parts removed. There's no BIOS, +no PCI bus to probe, no emulated graphics card, none of the slow legacy stuff a +normal QEMU machine drags along. You get virtio devices and not much +else, which means it boots very quickly and uses very little memory. Right now +QEMU is the only runner we support, but the engine is written so that other +runners (firecracker for example) can slot in later. + +Inside the guest there's a small piece of software we call the agent. Spindle +never SSHes in or runs commands "from the outside"; instead the agent dials back +to spindle over vsock the moment it boots, says hello, and from then on every +step of your workflow is sent to it as a message. The agent runs the command as +an unprivileged user, streams stdout and stderr back, and reports the exit code. +The host side of this lives in +[`spindle`](https://tangled.org/tangled.org/core/tree/master/spindle/engines/microvm/agent.go) +and the guest side is a little Rust binary called +[`shuttle`](https://tangled.org/tangled.org/core/tree/master/shuttle). +(`shuttle` implements +[`agentproto`](https://tangled.org/tangled.org/core/tree/master/spindle/) which +is the protocol used by `spindle`. Technically speaking anyone could implement +this and, assuming side effects hold, you could have your own agent!) + +![](https://assets.tangled.network/blog/microvm/diagram1.png) + +## Two kinds of images + +There are two "flavours" of image you can boot, and they're aimed at fairly +different people. + +The first is **NixOS images**. These are the interesting ones: because the whole +guest is built with Nix, you can configure it from your workflow file directly. +Things like `dependencies`, `services`, `virtualisation` (e.g. Docker), +`registry` and `caches` are all written right there in the YAML, and the guest +agent builds and activates that config before any of your steps run. If we've +built that exact base plus config before, spindle can just hand the guest a +store path to realize (fetching from whatever cache `spindle` has configured) +instead of rebuilding it, so the second run is quick. + +The second is **non-NixOS images**, which today just means Alpine, but can be +anything. You don't get the workflow-level NixOS config here (there's no NixOS +to configure), but if Nix happens to exist inside the image, like it does in our +Alpine one, it can still talk to the spindle Nix cache just fine. + +## An example NixOS workflow + +If you've used spindle before, this will look familiar: it's the same manifest +you already know, just with a few extra keys that the NixOS image understands. +Here's a workflow that needs Postgres to test against and Docker to build an +image: + +```yaml +# .tangled/workflows/test.yaml +engine: microvm + +when: + - event: ["push", "pull_request"] + branch: ["master"] + +image: nixos + +dependencies: + - go + - github:nixos/nixpkgs#hello + +registry: + nixpkgs: github:nixos/nixpkgs/nixos-unstable + +caches: + https://nix-community.cachix.org: "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + +services: + postgresql: + enable: true + ensureDatabases: ["spindle-workflow"] + ensureUsers: + - name: spindle-workflow + ensureDBOwnership: true + +virtualisation: + docker: true + +steps: + - name: run tests + environment: + PGHOST: /run/postgresql + command: | + docker build -t app . + psql -c "select 1" + go test ./... +``` + +The new keys each do one job: + +- **`dependencies`** are the packages your steps get to use. They go into a + `mkShellNoCC` devshell that every step sources before it runs, so you get the + whole stdenv environment (setup hooks like `pkg-config` wiring up + `PKG_CONFIG_PATH`, etc.) and not just the bare binaries. That means you can + use a dependency like `openssl` and compile the `openssl-sys` Rust crate + without pain! A bare name like `go` is looked up in nixpkgs (same as Nixery), + but you can also point at any flake with the `flakeref#attr` syntax, so + `github:nixos/nixpkgs#hello` pulls `hello` straight out of that flake. +- **`registry`** is how you remap the global refs. Here we pin `nixpkgs` to + `nixos-unstable`, so now the bare `go` above resolves from unstable. You can + alias your own flakes the same way (`myflake: github:me/x`, then + `myflake#tool` in `dependencies`). +- **`caches`** is a map of binary cache URL to its trusted public key. They get + wired into the read proxy (more on that just below), so the guest can + substitute prebuilt paths from them instead of building everything from + scratch. + +`services` and `virtualisation` are the interesting parts: they're passed +straight through to NixOS, so anything you could write in a NixOS config you can +write here. `services.postgresql.enable` brings Postgres up before any of your +steps run. + +Since steps run as the `spindle-workflow` user, naming a database after that +user with `ensureDBOwnership` is the easy path to a working DB -- Postgres peer +auth maps the unix user straight to the matching role, so `psql` connects over +the socket with no password and no extra setup (this name-matching is a NixOS +requirement for `ensureDBOwnership`, if you want a differently named DB you'd +grant access yourself). + +`virtualisation.docker: true` is shorthand for `virtualisation.docker.enable = +true`, which gets you a real Docker daemon inside the VM. By the time your first +step runs, Postgres is listening and the Docker socket is there, no sidecar +dance, it's just part of the machine. + +(`true` works as shorthand for `.enable = true` anywhere an `enable` option +exists, so most "just turn this on" services are a one-liner!) + +## The architecture + +![](https://assets.tangled.network/blog/microvm/diagram3.png) + +### Nix cache, both ways + +Spindle talks to its Nix cache through two proxies that run on the host, so the +guest never needs credentials or direct network access to reach it. Like the +agent, they use vsock to talk to spindle. + +The read proxy fans out to the configured substituters plus any caches you +listed in your workflow, so when the guest needs to realize a store path it asks +the proxy and the proxy fetches it. The request is sent concurrently to the read +caches, so the one that answers it first wins. + +The upload proxy goes the other way: any path built inside the guest gets pushed +back out to spindle's Nix cache (if one is configured), so the next workflow +that needs it doesn't have to build it again. Any paths that already exist on +any of the configured read caches won't be uploaded. As the agent reports +built paths, they're queued and uploaded in the background while the rest of the +workflow keeps running, so uploads overlap with work instead of blocking it. If +any are still in flight when we reach VM teardown, the workflow waits until +everything has drained. + +Spindle can be configured to use `http`, `ssh-ng` or `ssh` URLs as a binary +cache to upload to, so for example, `ssh-ng://localhost` would just upload to +the local Nix store on the machine that the spindle runs on! `ssh-ng` and `ssh` +require Nix to be present in PATH so that the spindle can use `nix copy` to +upload to them, but if you are using a binary cache that supports `http` (for +example, [ncps](https://github.com/kalbasit/ncps)) Nix does not need to be +present. + +### Building the images + +Image builds are done with Nix. For NixOS we lean on +[microvm.nix](https://github.com/microvm-nix/microvm.nix) and layer our own bits +on top (stripping down kernel modules, configuring users, etc.). For Alpine +there's a smallish Nix definition that fetches the kernel, the initrd and the +kernel modules, sets up an init script that configures the machine on boot, +copies in the dependencies we want (`nix`, `git`, etc.) and compresses the whole +rootfs into a squashfs. + +None of this *has* to be Nix, though. As far as spindle is concerned an image is +valid as long as a few things hold: a guest agent (that implements `agentproto`) +is present and gets started on boot, a `spindle-workflow` user exists, and the +work directory is set up at `/workspace`. That can be built however you like. + +### Finding an image + +Every built image ships a `spec.json` next to its artifacts. The spec is the +whole contract: where the kernel and initrd and read-only store disk live, the +boot args, how much memory and how many vCPUs to give it, the shell to run steps +in, the writable volumes, the network interfaces, and the runner-specific knobs +(machine type, CPU, extra QEMU args). NixOS images also carry a `baseConfigHash` +identifying the base config baked in (this is the hash of +`nixosSystem.config.system.build.toplevel.outPath`). + +A workflow picks an image with the `image` key at the top level. The name is +matched literally against what's on disk, we look for a directory called +`` with a `spec.json` in it, then fall back to a flat `.json`. The +nice property here is that resolution depends *only* on the name and what's on +disk, never on the host doing the resolving, so the same workflow resolves to +the same image on every spindle. If an operator keeps multiple arches side by +side they can name them `nixos-x86_64`, `alpine-aarch64` and so on (that suffix +is just part of the name, it's not handled specially). If you want, for example, +`nixos` to work, you can just symlink `nixos` to `nixos-x86_64`. + +Right before launch we double-check the referenced files actually exist +and that the host has the tools we need: `mkfs.ext4` for the volumes, the +QEMU binary for the spec's arch, `/dev/kvm` and `/dev/vhost-vsock`, plus +the `ip` / `mount` / `slirp4netns` / `unshare` toolchain if the image +wants networking. + +### The life of a workflow + +A workflow moves through a handful of stages: it gets parsed and its +image resolved, it waits for a slot, it gets set up, its steps run, and +then everything is torn down. + +The waiting bit matters a lot. Each image declares how much memory, how many +vCPUs and how much disk it needs, and a workflow has to acquire a slot from a +resource scheduler before anything boots. The scheduler is work-conserving with +aging and per-user fairness, so one person submitting a hundred jobs won't +starve everyone else, and slots don't sit idle if there's work that fits in the +budget. + +Once a slot is acquired, we do the setup. Spindle allocates a random vsock CID +for the guest and registers it with the agent hub. It creates the per-workflow +work directory, starts the two cache proxies (described earlier), a DNS proxy +that resolves through the host and filters out private/special-use addresses, +then creates the VM: writable volumes become sparse files formatted ext4, the +store disk is attached read-only, and QEMU is started with `-sandbox on`, +`-nodefaults`, no display, no monitor, etc. with serial (on boot) / +`virtio_console` output to a log file and a QMP socket for control. + +Then we wait for the machine. We poll QMP until QEMU says the guest is running, +then wait for the agent's handshake to arrive over vsock from the CID we expect. +The agent tells us its protocol and versions, and spindle sends back the job id, +the trusted cache public keys, and the cache and DNS proxy ports. From there +steps run one at a time as `$shell -lc `, as the unprivileged workflow +user in `/workspace/repo`, with the right environment and any unlocked secrets. +If the workflow activates a NixOS config and we've already built that exact base +plus config, the activation step can realize a cached toplevel store path instead +of rebuilding. Either way, whether it's building the config fresh or pulling a +cached toplevel down, that output streams straight into the activation step's log +as it happens, so you can watch the closure come in instead of staring at a blank +screen wondering if anything's happening. + +Timeouts are cooperative: we work out a deadline from the workflow timeout +and send it to the guest, with a little grace on our side so the guest +gets a chance to report the timeout itself rather than us just yanking the +machine out from under it. And if the VM crashes mid-step we tail the +serial and QEMU logs into the step's stderr, because "guest agent +connection lost: EOF" is a genuinely useless thing to read at 2am... + +Teardown is the same whether the workflow passed, failed or timed out: +drain any pending Nix cache uploads, ask the agent to power off, wait for +QEMU to exit (falling back to a QMP `system_powerdown`, and finally a +kill if it's being stubborn), then close the proxies and remove the work +directory. + +### Locking down the network + +A VM that can reach the host's local network is a VM that can reach things it +has no business reaching. So QEMU doesn't run in the host's network namespace at +all. We `unshare` into fresh user, net and mount namespaces first. Inside that +namespace a small wrapper bind-mounts a resolv.conf pointing at `127.0.0.1` so +that QEMU's built-in slirp DNS isn't used, then installs blackhole routes for +every special-use IP range (RFC 6890, so private networks, link-local, loopback, +etc.) before it execs QEMU. `slirp4netns` then provides the namespace's outbound +internet connection, with `--disable-host-loopback`, sandbox and seccomp all on. +QEMU runs *inside* that namespace, and the guest's network card is attached to +QEMU's own built-in user-mode networking. So every packet from the guest takes +two hops: guest → QEMU's slirp → the namespace's `slirp4netns` → the internet. +The guest never sees the host's network and the host's network never sees the +guest. All of this is done without needing any privileges! + +Guest DNS doesn't use either slirp layer. The guest's `/etc/resolv.conf` points +at shuttle on `127.0.0.1:53`, and shuttle forwards DNS packets over vsock to +the host-side DNS proxy. That proxy resolves through the host's real resolver +and strips any answers that point at private or special-use addresses, so guest +traffic can only ever reach the outside world, never the host or anything on its +local networks. + +### Budgets and cgroups + +The scheduler's budget is bookkeeping on its own, it tracks what it's handed +out, and the runner (QEMU) will ensure that a workflow only gets those. But +optionally the whole thing (QEMU and slirp4netns both) gets placed in a +per-workflow cgroup with memory, swap etc. limits, which is an extra enforcement +layer on top, considering QEMU and slirp4netns themselves also use resources. A +nice side effect is that when the cgroup OOM-kills the VM we can see that it was +an OOM and report it as such, instead of surfacing it as a generic crash and +leaving you guessing. + +The spindle itself also gets a cgroup with `memory.min` set, which means that in +a host OOM situation, it should be the workflows that die first, not the spindle +itself. + +## On the roadmap + +A few things that are coming next: + +- [firecracker](https://github.com/firecracker-microvm/firecracker) runner + support. QEMU microVMs are good and all, but firecracker VMs are more + efficient to run concurrently and are leaner overall. +- ssh-on-fail: when a workflow fails, you should be able to ssh in to debug why. + This can be really useful in situations where you need just *a little* bit + more info if something unexpected fails so you don't sit around there running + the workflow 10 times over. + +Feel free to come and ask any questions you might have on https://chat.tangled.org! diff --git a/blog/posts/stacking.md b/blog/posts/stacking.md new file mode 100644 index 0000000000000000000000000000000000000000..66d3f902da011b0dec7456dccba6342cc41b211b --- /dev/null +++ b/blog/posts/stacking.md @@ -0,0 +1,353 @@ +--- +atroot: true +template: +slug: stacking +rkey: 3lqlhsyus22bt +title: Jujutsu on Tangled +subtitle: Tangled now supports Jujutsu change-ids! +date: 2025-06-02 +announce: true +image: https://assets.tangled.network/blog/interdiff_difference.jpeg +authors: + - name: Akshay + email: akshay@tangled.org + handle: oppi.li +draft: false +--- + +Jujutsu is built around structuring your work into +meaningful commits. Naturally, during code-review, you'd +expect reviewers to be able to comment on individual +commits, and also see the evolution of a commit over time, +as reviews are addressed. We set out to natively support +this model of code-review on Tangled. + +Tangled is a new social-enabled Git collaboration platform, +[read our intro](/blog/intro) for more about the project. + +For starters, I would like to contrast the two schools of +code-review, the "diff-soup" model and the interdiff model. + +## the diff-soup model + +When you create a PR on traditional code forges (GitHub +specifically), the UX implicitly encourages you to address +your code review by *adding commits* on top of your original +set of changes: + +- GitHub's "Apply Suggestion" button directly commits the + suggestion into your PR +- GitHub only shows you the diff of all files at once by + default +- It is difficult to know what changed across force pushes + +Consider a hypothetical PR that adds 3 commits: + +``` +[c] implement new feature across the board (HEAD) + | +[b] introduce new feature + | +[a] some small refactor +``` + +And when only newly added commits are easy to review, this +is what ends up happening: + +``` +[f] formatting & linting (HEAD) + | +[e] update name of new feature + | +[d] fix bug in refactor + | +[c] implement new feature across the board + | +[b] introduce new feature + | +[a] some small refactor +``` + +It is impossible to tell what addresses what at a glance, +there is an implicit relation between each change: + +``` +[f] formatting & linting + | +[e] update name of new feature -------------. + | | +[d] fix bug in refactor -----------. | + | | | +[c] implement new feature across the board | + | | | +[b] introduce new feature <-----------------' + | | +[a] some small refactor <----------' +``` + +This has the downside of clobbering the output of `git +blame` (if there is a bug in the new feature, you will first +land on `e`, and upon digging further, you will land on +`b`). This becomes incredibly tricky to navigate if reviews +go on through multiple cycles. + + +## the interdiff model + +With jujutsu however, you have the tools at hand to +fearlessly edit, split, squash and rework old commits (you +can absolutely achieve this with git and interactive +rebasing, but it is certainly not trivial). + +Let's try that again: + +``` +[c] implement new feature across the board (HEAD) + | +[b] introduce new feature + | +[a] some small refactor +``` + +To fix the bug in the refactor: + +``` +$ jj edit a +Working copy (@) now at: [a] some small refactor + +$ # hack hack hack + +$ jj log -r a:: +Rebased 2 descendant commits onto updated working copy +[c] implement new feature across the board (HEAD) + | +[b] introduce new feature + | +[a] some small refactor +``` + +Jujutsu automatically rebases the descendants without having +to lift a finger. Brilliant! You can repeat the same +exercise for all review comments, and effectively, your +PR will have evolved like so: + +``` + a -> b -> c initial attempt + | | | + v v v + a' -> b' -> c' after first cycle of reviews +``` + +## the catch + +If you use `git rebase`, you will know that it modifies +history and therefore changes the commit SHA. How then, +should one tell the difference between the "old" and "new" +state of affairs? + +Tools like `git-range-diff` make use of a variety of +text-based heuristics to roughly match `a` to `a'` and `b` +to `b'` etc. + +Jujutsu however, works around this by assigning stable +"change id"s to each change (which internally point to a git +commit, if you use the git backing). If you edit a commit, +its SHA changes, but its change-id remains the same. + +And this is the essence of our new stacked PRs feature! + +## interdiff code review on tangled + +To really explain how this works, let's start with a [new +codebase](https://tangled.org/oppi.li/stacking-demo/): + +``` +$ jj git init --colocate + +# -- initialize codebase -- + +$ jj log +@ n set: introduce Set type main HEAD 1h +``` + +I have kicked things off by creating a new go module that +adds a `HashSet` data structure. My first changeset +introduces some basic set operations: + +``` +$ jj log +@ so set: introduce set difference HEAD +├ sq set: introduce set intersection +├ mk set: introduce set union +├ my set: introduce basic set operations +~ + +$ jj git push -c @ +Changes to push to origin: + Add bookmark push-soqmukrvport to fc06362295bd +``` + +When submitting a pull request, select "Submit as stacked PRs": + +
+ + + +
Submitting Stacked PRs
+
+ +This submits each change as an individual pull request: + +
+ + + +
The "stack" is similar to Gerrit's relation chain
+
+ +After a while, I receive a couple of review comments, not on +my entire submission, but rather, on each *individual +change*. Additionally, the reviewer is happy with my first +change, and has gone ahead and merged that: + +
+
+ + The first change has been merged + +
The first change has been merged
+
+ +
+ + A review on the set union implementation + +
A review on the set union implementation
+
+ +
+ + A review on the set difference implementation + +
A review on the set difference implementation
+
+
+ +Let us address the first review: + +> can you use the new `maps.Copy` api here? + +``` +$ jj log +@ so set: introduce set difference push-soqmukrvport +├ sq set: introduce set intersection +├ mk set: introduce set union +├ my set: introduce basic set operations +~ + +# let's edit the implementation of `Union` +$ jj edit mk + +# hack, hack, hack + +$ jj log +Rebased 2 descendant commits onto updated working copy +├ so set: introduce set difference push-soqmukrvport* +├ sq set: introduce set intersection +@ mk set: introduce set union +├ my set: introduce basic set operations +~ +``` + +Next, let us address the bug: + +> there is a logic bug here, the condition should be negated. + +``` +# let's edit the implementation of `Difference` +$ jj edit so + +# hack, hack, hack +``` + +We are done addressing reviews: +``` +$ jj git push +Changes to push to origin: + Move sideways bookmark push-soqmukrvport from fc06362295bd to dfe2750f6d40 +``` + +Upon resubmitting the PR for review, Tangled is able to +accurately trace the commit across rewrites, using jujutsu +change-ids, and map it to the corresponding PR: + +
+
+ + PR #2 advances to the next round + +
PR #2 advances to the next round
+
+ +
+ + PR #4 advances to the next round + +
PR #4 advances to the next round
+
+
+ +Of note here are a few things: + +- The initial submission is still visible under `round #0` +- By resubmitting, the round has simply advanced to `round + #1` +- There is a helpful "interdiff" button to look at the + difference between the two submissions + +The individual diffs are still available, but most +importantly, the reviewer can view the *evolution* of a +change by hitting the interdiff button: + +
+
+ + Diff from round #0 + +
Diff from round #0
+
+ +
+ + Diff from round #1 + +
Diff from round #1
+
+
+ +
+ + Interdiff between round #0 and #1 + +
Interdiff between round #1 and #0
+
+ +Indeed, the logic bug has been addressed! + +## start stacking today + +If you are a jujutsu user, you can enable this flag on more +recent versions of jujutsu: + +``` +λ jj --version +jj 0.29.0-8c7ca30074767257d75e3842581b61e764d022cf + +# -- in your config.toml file -- +[git] +write-change-id-header = true +``` + +This feature writes `change-id` headers directly into the +git commit object, and is visible to code forges upon push, +and allows you to stack your PRs on Tangled. diff --git a/blog/posts/vouching.md b/blog/posts/vouching.md new file mode 100644 index 0000000000000000000000000000000000000000..f9f43cac6ad329b4dd1c260c6bb3e79aff2a920e --- /dev/null +++ b/blog/posts/vouching.md @@ -0,0 +1,91 @@ +--- +atroot: true +template: +slug: vouching +rkey: 3mkqt3lek2234 +title: Combat LLM spam by building a web of trust +subtitle: Vouching on Tangled! +image: https://assets.tangled.network/vouch.png +date: 2026-05-01 +announce: true +authors: + - name: Akshay + email: akshay@tangled.org + handle: oppi.li +draft: false +--- + +Tangled now has native support for +[vouching](https://github.com/mitchellh/vouch/)! You can +vouch or denounce users that you interact with. Vouched +users will have a green shield icon beside their profile +pictures, and denounced users will have a red one. You can +use this to inform decisions about an interaction. You can +also see the vouch/denounce decisions made by your circle. + +## why vouch? + +Vouching serves as a signal of trust to your circle. + +The bar to submit code to a project has never been lower +thanks to LLM based tooling. LLM tools are really good at +creating "uncanny valley" submissions. Code that looks +correct but is subtly wrong. The onus is on maintainers to +now take the time to review such submissions. To ease this +burden, maintainers from across the Tangled network can now +vouch for or denounce contributors that misuse these tools +and create a maintenance burden. + +## mindful design + +Such systems need careful consideration. Vouching on Tangled +includes the following to begin with: + +- vouching/denouncing with a text-based reason field +- attenuation: you can only view decisions made by you and + your circle +- no consequences to being denounced: at present, denounced + users aren't blocked from the project, but simply have a + red warning label in parts of the UI + +Some additions that I want to put in down the line: + +- decay of vouches: maintainers and contributors tend to + move on from projects over time, so vouches should decay + as time passes, and be renewed every now and then +- evidence trails: vouching for a user right after merging a + PR should add the PR to the vouch record as a piece of + evidence + + + + + +## how it works + +When you vouch for or denounce somebody on Tangled, you +create a **public** record on your +[PDS](https://atproto.com/guides/glossary#pds-personal-data-server). +The record includes: + +- whether you vouched for or denounced somebody +- an optional reason for doing so + +The Tangled appview then aggregates vouch data from across +the network, and displays vouch "hats" over profiles at +points of interaction: + +- in issues and issue comments +- in pull-requests and pull-request comments + +A hat appears over a user only if you have directly +vouched/denounced them, or if somebody you have vouched for +has vouched/denounced them. + +Additionally, there are no consequences for a denounced +user. Only a hat. You can click on the hat to see who +vouched/denounced this user in your circle. The consequences +may change eventually, but for now you can use the hat to +inform a decision. + +Start building your web of trust on Tangled today. diff --git a/blog/public/avatars/anirudh.fi.jpg b/blog/public/avatars/anirudh.fi.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1dd2131cc9ca49a961ab5feab65785c7d2f7b1ad GIT binary patch literal 2664 zcmV-u3YYa#Nk&Fs3IG6CMM6+kP&il$000080001g004gg09H^qAiN0x0B|h;odGI< z0Du5KZ7h>Tq#~h~ix!|j2E?EYm|{Jz=bmnPHi3`ez89r!zufH_YrcRTFjkiBmDjCW(e^dd!!48j9!gG!B}*dx4e-dL!V>l->Dw1}8}y)NY7YZv3Cpxr z;}JsOSPi_ewmUFSe*PYa%IaLF>gYdpn3@XtZq^I0zmEO}CPDBG`g_&QA}=VbsdDoh zr`cguv;zx|9pgIjb7|X{1i74}ey*6y3(0tZ408*SzpTABwa3#EJaYd!LD`iPYE&*S zG_gE@w!wK-4!tJbvfG?~r>K88)Sd_^$Nfz6)_2aqxy-rXSLz3KvyIs3!t@T~p%N?4 zWb9cf`H@d2(D2N4L0}gQ2Ob%|sB#DH!xmW}?{F3(EzwbLbm*$DWebvleqPJ^jPeuvWW{_LF^ zhBSKBgUIgP=QHi{wHP)kIckWbLC(mWTd!=MocbUY0n-DW`YH-LOJO>onJ$Sc@n?vX z+6^b*CWzbMn@7A+aHPwql1!iQ{5Wr`A>Sqv@*__*kv(v|_|Ujhsf2@nYtP)JbzovE zs0`sTV!I#6hO6jou@7khlcm5_=w1qbd)@9(|$=D`i%odMh}flo_J(L3Miq^ z^6v1ka(fibKYNO-CoZy*RtE2=1r(kI6#cv-6mVHrG5?zpObAhUA4BX{;w;5)2UEqu z!xnpCQRM}6c#0F<;M+S=vs#QXlQBNm2XcbZw5KdkfE zMtfF9j2+-bicVp&OLx4jBi}C$ECt1iKG~;F|w2e5g z8tVj^uks*qxKYqgCC?R(`>lc#Hd_?J|hCbJk*ZvD$ePgbN%aw|DUxlhFJTy1Z2M^7bFd@}`W;&7-cVVu9jRiv=oW6En zdF2l!yY&uFETxURlvOAZK?GC`kKAP~^L*(?Xae_UHqJ;pd1;Pwd2qp>={$0!iUbFr zEs&vNy@|&ql`O-`0M4TJWY=KXI%sMMxIPiX1)QNE$Yu7j3r1>e&zNgETb7{g zY!u&);qV`}7rL`XBEb1`IJTV@5qWGqxNzXj=628I;Z}*^Sfob!sPI#8jEazlN9V6+ zFFY`HoqyK+ye%@j9jCWTF$#kKc`cBi-WCGrR4?01^~JCN41+*>+27|V-xnlx_gcKyFDGqIar2fyC2rBw08x2ClI8&k8eL<{@c`Fe0`%njMP|A^UBuIKzOmsxkl znH^;&JQCdh&45+G$Mme2008SHZltkN%B=c-XM9ns&y~fkyXi$y3HXc~{c!wKTm^!N zAwwppMVG}l3F{(Q;RW=s-GaQ%J4Mwg4nUM1*j*~|wSX41=!b(cDnZOk4moUg{S~qL zLlrejb)izmDy80i;GP6EyK%{n5Px%H(1M>^!P-p^JN;N_rTJS$chZC^ZMKJ|H7Igl zmOYstLbo030F4k1K&ptW?k@B_D6jy@OnR*k9@K#{l_tFFg;UHZqS54Ap9Vf&HD)g^ zW)@#2hh)f^RqB&;AeHw9*Jx8%2r;Z8CA?idI3cVfGqGijkST}Hq_Mp#5~(QxtXFd-%Y}lWJrrd6#v*IunXvms!I(awV4d#vw~^|E`Bi- zcyR#iv;w>XK~c8Rfy|?8Hd;br+w^Ce7uA9g!n)bTC_hXZ704fuxFBXpPr4Nt8}!+v~0Z$EOh(D9rZk}YUN(=}C>EB0=}3at5m z7EsYgJmM(&5PlR8_VYF|_FfEhFJf@n<9!l_-xZ8jh5u_>s01JeuSJCht&cXGI{MH< zkT0a1huQVvj3HTf!dA6Xjj3I78>Z*F@fQX6vO77Gtbx;0p8qn7ECY{R4iZ)O&rwK} zUgI*i7HXUAJz`JJ?XW>6&Cq+SSDo@nqSn%eza2tA7Xy1NaJijw48#)DgsHuyFv#>` zRs(sGp)OCeA@ye~Wy{0IN?ayXp}ZAz1Nn0M66=L7UM_Vm$RyWscL=m;)gUMSq`Q;_ zOGcV4r}Rz<_IsrkFw~$l&zs`rc?Fjd+IB~rF{rh#)nO_-|7jio07Y0yMt%SQ07ZCd zW&i+9O#mtY000O81pyKO0{{R30RRC2009~R1poj50RR91N&y=H1poj50RR91QUNFc z0{{R30RRF30EcM+1ONa40RR91S^xk500000NB{r;0RR91NB{r;0RRG^0ssU600031 W004lX0{{d700031004jh0001;S>gcz literal 0 HcmV?d00001 diff --git a/blog/public/avatars/oppi.li.jpg b/blog/public/avatars/oppi.li.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a3783865b7f62db6c432332b6c39c8a745810954 GIT binary patch literal 5760 zcmYjVWl$VSw_G5|;_hT|4Z$sFg1ZweK+xbG*aa2|mf#TF-F0yZiv;)J?(QxD9`}CV ztM{hnoYT|Q(|_j2RJDe@jEpBS0Ps=zgPM+-pe{N906=>w-+%WDL!Cqf0F221(h*qw z0Db^53;GD@JX#vchx)$~C{ne6&ol&fI6}{|cglyoR=%GfTLN|G7R;WQidX(DfA`@Q z8F*TP^xvC21(fG^70*Z_j-XJ=#{tAbZxr+|p{& z7UgxxoP3G*Ba8mnyL82TU=?F(%BH?sIeY5nuzg9 zIY(DP^j-cZ+WN9xD)f%)^7V&DlL*4u_jiulHlf2sB}T+r8x!AMO^drZw|4Z}NT+6$ zJT>SusB&*kxy4lIBu2#b&FSRgO7BcY@L76j6I=!_W3Bzg1C;8F7l2E7nat#Z*-`V6 zTbcYFXL7r3-p@cp&dLCFRSxGm&_M=27P6w4;FlX}6D0FqT%NR=4(iqicN~(fUj|y6C*A|QE0u<)1h^362n;J&GuU%r3tL@ z?_*`yL+CU?@sIQUL-g5k!8*k`oG2kU-T=#o!>REkNl1bgCQ;p$&}yUkojO01*e^SO zP$iTkzpyArGOvQcH_;}@)F?-$AhqNEBu^RUzn);2!&!>TZ(BMU8(KQARkde4B@~p{ z%M_Sbzq~EMs%nD&ZaZntJk&96C7j9V=2UFaQkVC;qk2*;LUHZ2`E}KG@Z|k#0O0BB zUe#wK%7c#}J#8dNUC)#6INm`$h~`$v!Mf>_?oqm4OvN^|6w|~wmnrcsGWOsq3tI${ zB6nF5wg}V$+WREu@p^i`A5GJ~Tx$0Y`0cPJY9S}xP>oXe72$4lC|`NgK;x2`aGLPM zPy`$~9JY}i(#hKuvTj&R=jA;h5oh~l|0zX4AenuYM;(*P(|#Hc;G>Dl%^$=-HR&3L zyste+Q+5^jd5AgKl$1;-uyvOuqTD+wEf(U?_iJ6O1^u)mraFq0ujxxJ z2){XH@HAWB0GUcJDk^Z;&<6^q$HO^P@N|r0Y;Dp5_-B5Q0MfV0qcnZV{xbe-G@W2OqCO@s@A55MDvE5z+E-oPuV&xm zC_@fdq32aj!Gg2`R7BhlOKd83Z(_%%Yf2SttcC_cPdY{Qi4Hi$YuX)EowS;F2sTc{ zE50q?$y2_n^T`wV%frrtoCJ{ccFLtDwEIzF!~hQx6+V9Z_|4Kb3)I1n zH;o?g7Qabelk2&987fgi4kcG~#=OYrBh4eQsV-nCw-`W?n(vDGWn{xs0LPKIx zzKibHx4YKK$$X3`5NzKU_I=J~O0A0P&tfWT;Ehi5z2Ht)!fSu_3Lxp^l@+4t=bZSCbnFbEI3K;Ys%p@Q5(r~;=v?M$A4h`uW+bQqjXCu1(MT7k9 zL+>SSmMRBN<8(F4C^quC7Dw~F=s;kbR-$yz%`;AM8f2nT>=k?=oo_PqPqlo zq71x34kc-t*l@i}BuoLJAGB;f*o^1x{Wxwdo#V!xtcuk?tkDoRR)Bs9J={k5DLtNK`v!7d5h zo7xBEa!!?KjVYG%Je>*7EDdWuHf{>FcWmM#m`DUsLvv(hPAE7oS0K4TI~7_gz$EpJ zqbMRp!Hmfp?V7<_XfumQeAn{&!>-y%4K^mBz@p|JwWKC3jCAa*Ag#=uP)S{4)5rST z+#1coL%cuco)G!(o-m0V1zFu&(QZEeHP%yEBbT-DsQTfS_-74+D|ZZZL<+m`7hiHbIWKEAvR)y~-z0 zWaIk1UJr}a8%jSxWku&)GrKHm+Iobd!N^zXJaZG2P~}37v-_|0lPRP4#$Ej=>dJha z(s1I>ts{S1O(Gb^sC?$mt;vKA#S*upQ6O@+0fdehYQZ&?841|@PGlYX=O~s^qzB}y z(o)`40jgA^;fND$iu0a7fCjd9)y-SKJ5=)^Uw5RTK4XikP$pDN%>=FXy4$wmKf5_jIs3JnKea4H*uZhAk`TbTI@yb? z_CC6#`{60`k3_!77bh{=+H1X&H*|jzI~57T-h2{w?(uLYs&Hy5YjBNyKs!cEe(coo zzrok*sU&$-YxwtAq=RI1+9g5nDURzeJ}eD_qD5OFU`m|qlWO|_aXt_rGNs5V-N_ z(Ektymv)ALP1D5lM>#R16tt(I?~WiMqm6#s*h(=7oN|68MIRgqeJMvpVAtfKmAEsZvY_i@_gk|`l7Z)d~s!iVx!uONc z0pn}(X=H1wkWEkyBmzwE+|7IE`~CQg8hNM*BeSle&A}jpvimk3b8PQ`M><8YaaD*+ zX?NiS-X7~US%~k3@3O1r1`r*dJpd`DS0S{W8)@F(xlVsLDOgS&lHSzRW$Ed-i9nH< zW;d!^e|;E7g~2kl;3O$pXd}s7vCrWlXJ&ra_eO@4SFV4JpP@$Vohmvs5l4PXW={3w zgm-SDCl+N?&-eU{DeG7Kxu=ec#X4bVcv%X#T#VvledV36qGD2fI&T#&VfltOHN3uR zVhdms|1_m|^6p2Q^)@VWo;0%~s~KmgrDvmLg!fl9+0r2$eYCj%r$SkBczR^T#*DQq zqsgUIIV0r?+I=8Jvin+Pi2)2HfUHyKEKA0VElVP2=gCUF4OmCJKgXN=5Fk!q;tWM@4lxV&(s8Wh z8O=_1R{1#=_+AfB*_l< zhocAe;H;1Pd?p0f@9XhdH%nHl0n}~}5D}odtg9*NC&qoW65eP0+Gjpfr;ibvQSV#u z2E`k+<1INZ;2V=gi`XW10n3@qM<^J`yNd-NAw|2v{@*eO!tLnyWOvz}Z?Pn5(=hEZ zHXQww@sEvrd^-U{bm7_UsZ16#=z)Vi<;pzo1h0+B#Pq(VMjlXd!}&-0H+@(WfG2OoiU*6 z`E7mZF>tgpSnHI(iQ(K!`E#etX?Uui3VvZH@Z=jK(`pw>c^6O zjH_!1`gJv<`mbd)qY0y-4dzg|5SaJ)Jz=q|S2g$^wt+gl`Fwim_Mm!!_LdIntV`5i ztKpVU8C+3@T=&g{k(SfVR%8b>@@MIMW39O$-nh3}@UX1+=kK;YLjj( z3>S>b^M_I6A^(pnE{TqFYyu}yQ-cp%%^WJ#%37k>$eNjet3N+Re~c^!7GfR1igDJY z3`oy)85(h;f(EI*efE7D>i~XPb~`WyFd0{c_=fy0&$onu@q>q<8R)cE;CIE7`bWi8 z{A*C`_c!MK-G>%Z1X8P=WT`mUKpkhf?;G=nP)7T?t;!@g%l6y70I}VnPh+#f;7DHi zqbyk~l{Fcij8Q&kNySPAes}4if)4ZF0@p0o_5@z_kvNFJUz^F?VqTksGPbS)`LaZq z<*;EdY=ZpXJQ!4h=sn-ClPf>7dE3)KM{S2)ZfThZS!#hBxeaWmj4cq%m=v}ip?LV` z;7U$*`B#0QXh+(L1Nd5rM0HN++btCYw=nr`)nIF85Xc9{58o^xVbkcvt;x3sP>3k#~dE1$0t`+umd}fkPyOk zYuv%n$ZVb;?&Cv8u)RzNRzx?*R%}nD&wF;*u_%LJS$_$|i8)SyKYyodmf?-&hJP3w z#JU18k!e5$zW*|L)?+bCetXrLcHeIGiqEAQrgTLmFdU)6^3E|8N3>=MKhqSW*K1?j zui|T-W$FjJzIUnd%yP#H+D!JQS!Z=k{R!K*YVm&KA`D#`FHnJ zDr~A+J3=l?$Xk1;4j1Kd`JHd~^N=a+yH~?X!B2se40YnyE3`o97ELlfa$+nl7i{0n3 zgZZM7{){))oefz3DWDL4^!%M|qyA=tS&?tQVTw2ZB2T&drn1)e38wLv>ddWWmS-`l z(K}yI0;{;ew}}#2g)$L7`DjB=v#^Z3Yj!Uv`p!jFn{TlW2)Q0#;s@u+!b37H^6MzC z?SohrVYoqBGgV`qfz{SRmuh4>`8QP+heGby?OGJ{zwq*Aer5`FVQSS3kH|I2=qHtr zOzIo1vJq!w289efLBOyHgo4y~N`PImvCVw!cInNQky)FZO?ym=1(&VH$_k{lwdBV* zf7A>osrlaJjm^s!PtMY0M)Cv37q`sd`6>RQr6{9v_S#6EJ#QOx61|z6isO1Hy5dKW zg8W=$Ex4=fWO5h1X=g0kuW=Bx4`LH^D@O@dT)G1u9*4J0ew+;9mN6{j@BTdvBm4_$ zDLN;?CnUC2k0E35d~}7Qs`AuqB~j+Je(}k^s>AYKJ!3GBqtX|PxBuDUUmi?_3qT6Y zbYCOR4nJ1llsR4YY*syP3I0)m;bB<|M~!I5nZ$}a*@dRbiO9*bAN&of2uJ-pWx;Cj zyuF>hovYi^sL^LQTdwD7QiRZn<4XCS`VnvKi=f4PxgDlwgT}=tGzj^{oF8@oOfH?I z^ln*?O*Ck@hEi1P+P2-2dlMTgTLObv&fHF5Wq1$lRqYs6LwJR$Kh;ZLr}k$U#!lJ4 zzSUnrs2p6FH_IWCg+mh?MoY80x30cjI8~px;_mzN*%v>}cBxWBLvu9V`GWD*zMR#= z1XOsRIn?Zoz_0MeV?OTaOM~Ey(S!LEzJ3L(!YHQB8@BeY%7w!L=&#tOWsq<7T}bdy z4gv;(Qd;oqB{WOjt3T?N+%h8nBpy~e8`mV!95!<)=@#`3nBtAeYwMNvxl5_lI8_9Q zYyudSA7D}PpbGO{+mzs>%zE`mH7{x2o}I6~EoXXs;R{0xH*>(d2FyJJ?92NBt8o^i zo2}ysyiOcAkVHHV+#msk7sc^kRQ4)EOXWNes0`53>yw5@M21`S^jqSnlXP3?{szlJ zulIP@x0E}eRohl~tl-=yy@K3aDYJ=aegm?8z3VI!Q6yJsszdO_D{}Yibg}YCba-{p z6cHo& z+DeA!)oo}uzIR~NR5G%kXW1Yc%t3!U{?s9<6k+}z`#Cuc$5cJPqr8clfAVBx*L9xa!K>w zcaQl><&Z};IODo}jH~8Bp8W{^V2u`F76XM7GtmdM^Kq<;n^0vO8e;{WNbLtg!>e=+=b$-VS{eExT$007abfLAXJCpQ--Ld<_K!ixhS>%Tk~ cCnrMIe;l8m0RY6T|2X=++(G((_~i)r9~l-A1ONa4 literal 0 HcmV?d00001 diff --git a/blog/public/avatars/oyster.cafe.jpg b/blog/public/avatars/oyster.cafe.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7fca67fb1ad9767f26237821af360525d206ea3b GIT binary patch literal 4364 zcmV+n5%ca+Nk&El5dZ*JMM6+kP&il$0000O0001g004gg06|PpNP_|Z009@!|A+|C zaU@BSB#8r_ycZBT@9_b;_?YYepQ((<+>?#yKf<(aW3y&WKZ#18HiYQk(8Xb-t*H+y z$CYtm_(*kSZ+zx(+XnJV%LUod19ZS>Z28<99G$3J`-BQMw-9jUy=5K_pAM*cQ||dj zkr3qW&UaF}VO{GNeh*?te$t&#^pYkn>Ec&?QrjA8;K=33>!m=FmwEu>!q-eP^$U9O z%?qPZ4s+2Z#D(1uyc=B@D+3iJFOmX6oaK@bHDYb&B-vusjvA%M2AY^^=VLwY;nJ{cUC?%2z& zw}e<-A@+<*Q|@_KTVd>&ra~hXa~tO{go=`0^s$W}QkY2CuM!Lp4tvYe%DNHpNeaez zg~JWQ!gtLbW3jx@LG2FnlkhUZ+*GryOvIS6q`?~%E>0WQ-#HWKa!iOT-8J=}Ka1lm z)eaJr<7IxEKjPLEDd~ghrd8;k%m@e&BpvcAzh}oFz0c}XBBi5&S?NrU`5whEia?XJ zNz$C4G!ie!E>}nC_;EBbTryA)hCC$j@2lTeX}lryH6wU9upuW z>5*yl!=eiF0RtE-M#S*G=I~`By z=Og^x=3g-C@k{@+p8S5q42*}#d_p{}^XcoyIl8oe=zaY)W8ISWZ$4l-209H^qAk+;20I)d#odGI<0Du5KZ7z{UBqE`qEtOorfDMUa0L0ct zTfVD)NAgqKZsPuj@b^8*_{;<0m#l9UuAe?)zh-}IecpX7`Un4b>R0|!?{`dCZ z+5-dohrgllpUnTvXBL&(raXPj|O3aC2g+%Cia&l`lxQ^UJ+zx*4O`%+y^LhMSf z?U`bpHxm*o`Z5!16A;rh#pJS|RKDrrS^^ts}bE$}) zyyHHmtgP6*eS@eHEbD|%0@pl!)?F{c5OPV4~n zeZuMJ1?ZQYe!o{wxM3lReGg8yD$+vX)hp4l_W{fd>ZIF$(t;?K6@IHdvgI}~-+8 z1R%z0ujs=q#$>}XmXTYI3EXLsY7-ITaH;rd0{@gwih81K_}ZAFRo7y5e+l?BYFG@>_Cx!Zde6wX~G7KL>1$l?)?|5TUBI z`YQs<7Aq;e%Dvf*jU@G(qn3c|vf+piT-!UHL5uL%T`S`<2s*Ih-2&Lg0Gk6E6fes& z*dQG3qap2lS72b`Xs3ww&Z>e~YhX5a#_yh`@|fxN5#*6DF_9q6L5j{bE1jr%Zyd~W z8NJ@y)$w?k;(bPnw0CNuAFcg`oq?Y*3#FR2&{fw20{QQVZc}0hfjMJi9t&$&E%hfa zZzmZz6m5wnylLFtk+elP9eaV$Zi4KnTVf9E9@2ep61OER9n1e|XuH?&$hgJaNoN(4 z^>(##T>AU*H9R6Csm~`21aKNuLEtH>g>jH=6T2ee@s_mHu9b{vwGe0*e1yQ3@mu4F zgL%f!^%r>G$z(cViO}^zzEZFKe09clAS<({U)O8(w0;U36Wyx#732~Qod*NabNFj{ z>n#$iLAp<6eTQK8_f-%Z&k`5yr2k+ua+MwOUay5^z{-&ruJezjF%NVg*g!sOa>9z-x%MvLsfxj1f@}(7quV?>h#yRU~obMT&^8^~y)g^^!YWFltyGx!vq;l^oVkdOApV}f7b{#k@ zXp+1&$FH`(1N#^*p8+G=Kya1Y8dP=3#gT-p(EChU{w;}HLMJw@ja1AK?$`Ry3ReaZ z9`5rsnp&V;xcnIIoKT&$2w6fjNYHJw%omj4kG5kdH%h{dTQ6y?TZtC2E2-!cV<~l8 zWuFZG8v05rvQz`iVvcP7E;B_nrkp2*EKIG7F6%Wv7)>O= zNsZOTle6AGGd;+()kC9-`ooWQM>)Mv<&XlUcoE>P=O}2r2x+Be$l^*YRHf7t3Ulaq zwR9i~KM|?XwCCCLfB*qtr%%{kA%q1Bim*;@sH=sQCo(BRz+yXT{1VP^oE}hlt|Hrd zBfah&Yf3_VKuGtW!+#B$$>+VIYgy4;8wzqP4&>5CE2XgB+#&AdYm0)pkGA;~YC zRN~@&qjixfE{}aahs>DtHt$%eBtg#!F(l7*23}d?cyO#<&ONzji7a6k?T8u0f%m%c zGOPL(Mst`34rQ#g02qFGR0J4ELDXxO?Lax^NB;E$Ty}V=Zf&4*u~2kFZ+zNn&B4W^ zzgCw|Q4o4m0Bj(8j(8sahVg}hUPEP=5|{=Qdu3g=Ar0qQRTOp8Z_bpy6F|8m!$wty zNaZhO8x2(v^)E$_9^)m7rz06~FKp{l_%B<$S*?Wx`PWM?o*lY zUD66e_+oW4niV#$e3K`Z(BVWvC4Qz_t(#er9u$3B42#zON#+CT2`I&=<4uTYdJ zlw>Ew)h&OZctpX)6M`Act<+)q^$fAbn5|dzfHmvKSLSj}x?fYNF}0ghK)Age#lN1} z*L8t6%Y#ji+rS4Qj9BcFwf0ib`_mb|vN-i`QzF1`mg0k)(YAfQ7%+C~K;Ma_h8Gek zzylVfTu`utZoDuN%^^hLw^s#)=2p(MOqNyi!kC{0VlHpnW^L*kLYBd)h_6N{=&~(@+L)}G zFGW=1+-d(B*c;0qM2>L2E_Gy~0Scmvyk;|cpHIB+9>hv$NHpeB)?F|K$G-~5kTC!M z#?Q=L#(@F}ID*@#3eEQ8rFhFW0*t|oSSTS{U``|q?v-LNd_mm4PksX)=HFoCE|@7} z@RmMIQcMcUDuGORAl+erx{4uFp6AguK%j`_C1^=jOaRv^(qL&VcRc(0Qm;9j-@DRc zCq!r6J+!cFE;^96vqb;P9@@|7Q%W04X=%C5QY#bh*lBq;vm9vo8J0(((A&av134WTa$y z`N4}gq>iDC8mB9Yl9C3;f%+wB)zfD>hbX!LyLiZuIl2YVr5*?(qbW)#Ui$YLrs9C@ zP$LQC%R*P@nb#t54DeuU%$4G!Be&6_nv(nBdfwaUQT!wX?`N9 zcR%=Mx*UArSt1u$joq!6s3E>bI#Sj26XjpW*I-1^!kc^uoxF&ClMbM1zIwY zFEg;bQjsu7iWW8%AY5c5okS!q2xP6uxhottisV99xb~&&twTZ~{qN=RDcae58G%RX`4iQTyg#bSuOjKxA(ZpUk0`#K>x8+)g#_)p0gf?ZtuYQYWGsY$*&( z>Cp@nn*XspOtE&$=Y{Loz&?vQWZ`&iHPXofZ7F5e6UU?pm;u4e^c=z@4K(11ykMS~ z#ReXL3xRl}v+zQZT8e=3XQ4$W?+A$j`T*X;$$G(A=r~ws5(WIRTA$&HhmlbaivWt1 zMPqNct;sC)aq6U3;tKAGD4TST|Id{Y7&^{bfIZ?aFWOwC7=$_-tCB8X%XRc7_!*^? zmMh4Lb#ad?GS}shrOuE59CZF!tLWAM07Y0yMt%SQ07ZCdW&i+9O#mtY000O81pyKO z0{{R30RRC2009~R1poj50RR91N&y=H1poj50RR91QUNFc0{{R30RRF30EcM+1ONa4 z0RR91S^xk500000NB{r;0RR91NB{r;0RRG^0ssU600031004lX0{{d700031004jh G0000ibzC0RMAUbPct*?>Zo8 z5*tUv>&}VaS*mZ0gONr-IO2D)c)t{AEJ8cczSRP2R+j{K`>aBg06?QM;2n@Z91spr zbm33a_`qJ4cVj-UOc2!qfB*(kUahWzZ!1sK7Aq!hd6J^gZ&Ob<&*XcREKqva;TY(7 z=-CrY;;b|ZZe&nOr~^N|pn_kK&%g%4v{9hnBV#s=VN0kZ?9=s;kFZ4)1KM-{`>xXG z+x53cxA0zUI(Xr2M3@q~3q7&T6JO#>4?v%yc2KF~pc7~^Yz6h;LVO`0#}oq#W~lW07w#BC3j1iT|VUB%#V;;>+w1u_=lCqunHeh_13HT3hhgPF}CV~5a zJS{JN>*_^V86%X>+U#{;*m*+uOOXrT+1bp74!Iv5W;c`_-fQ9R(n!$s4RWXA-rQUv z6UXDmrU`a4!kKcDh4UCpoo(e8_yRKedhZ&Go;~GFZrw#JM%#iiJL1WDm73@uer2-@ z?|GJcWfMZhA^S+iME8oD-4j0`9r$GOJfVC{mPRRWeid=OC8FkUWhmD7HeK$;pFOmf zKNzxI{ozZT&FyGH`ZtLWerVHN(Kkkvk*Jfhkr3sQ7wO6pvDP5Wc#vV}gJw;zlAMnY z5B;}qU=>SG1_?d=YHkbYi05wTNxVc0YE6AP)rjGcHl%qPR-k$bybrj&zQJ*9b6&*$ zX+zh+1Uwzqs)_lnAFAW*gq*(Xgyuan+fnyH_J%6!biKGY3Wnv7oj=O9~aD>}mN85&&;JOX*caS%+ z=6r66FhuXC4;%C$tvu|za;;TP!0#{-zhbH2w{n#AkXr@;g6*sQ*;RPc`)Qw=h-2D@ zlo^AXMa72r9z8A0#<;H)NB$IkPSzTzP|T2it(8~k^f610>p|+}DkUQG7N4zsSO)=4 zXL=td$bFl$)Zt}z1e{VzEaCnwZ=}z7fxooQ>o23K9R#yxCf5FzArmQ!nF$f$4qo{< z3_bnJH`J7j_%^7q*vJ=0MM3|p-rnnA*XJN-xp0;)@T0sMWxbNoZ%r^g3Fol_TluFzgmg7VCy;`3!+W)dY)Geg-mizjBlj!>CJu;wtVy?_;A@870UAtd95Qo7$@%M zBF|SbcR@02yM)1Ws}7m&>%V ziB+$%-e}NydR~t@%e-f!Aerp^L$Nti_GLWDzxR3FLjN&0addnQez?~u);uO8jRFfS`V-r?DVttIG&vv1z77C|jOo(`gfGLKg!bm0sEvU68o zD{bRWbki(~nsP3qxTGz9NWB(Kz^zXOkw1bwTp*p>ViDkFJTA70#)DyyU41@RN zBgHt4GKV61Er+eW>JAlVzblFmvZf>ALJr8gG3DvC1u?j980;?<+M^@sroGEJUejhw z4M@4#qLHWXQ$yp?SNgP=nd*ARqQhRU6MyEsgO>c|i0+&(#UB0|_VW%P8I&jGIj#>J zgJk1Gp3Xh|D|DNdMHkw>Y=UPbDm{gO|3X_PWSFWg4WrJwwiliT`um7}un5bxU|u1N z72O;-c&Y;-4S4Tzk{X_$2NvzrDh)BnY0ul=)XX0xRxv+`pDszn3;vv7+#Jm+rIgFi zom|=ME^orCW*esrwq%<`@!<0V>_0H~2i@1E7@Uo6-PmcAh>AY|&99eqTQrY$@5=mq zPdZ^292^7XDDc%(njI|n8dX-Y9uWJZj|Z$t+*VBftgMEI6l_tKo*W8^uc2-4^sr?g zoA@RySdVfnW4*7Ts5<5_s>1z zwAxZkQpN(b>kg`*R{RvRrH5m|I==+5<2nfkW5zuxW3 zX$eWHcg_|D^;0y5e2vD|>D>O6Z-m@Ln-DW-2z~ti8Q;fLG&$hBueRqF6So-6dG)Mu zp(HL?cYm>#;d555nx) za6ahfnliAwy%Xfz)!1+X=Tf{cNW_2|4y9!ouI%GV_>=TThUpeO=1al-L4&I{;o}Ek z(=I&8H0cj{0g=Sd1!$eKWRS9 z1PU;?8n3-@)F3Z%7su`2@<~6g4t}!WQR1D*^?Vj`l`by$hELm4=9%~=@sL=3|BN@K zN7Sm-IaNK)oTS~Du9k!s!Q7RZdL~zjWS{A0rhvw+m<%cB`j<&=Y1Z44JjsDDcny(> zpvT)Q3%#i%U0NcB_jxgLM7a7SSH_)H>lD+k6Fa4iU~}OR_Vi&78BIfp5))S8ZNIN1 zFZtLTOc<9!Q}u~9N}hO5rl;ize)J~twtm|Glhbpolk(9np<68MyaeN-jxoY@h{6&N z8crx*g=ee^;>iok3W=Ysj<(^FFlxQL6g-=(*-9MfXDY1pn3`j2sCd~&ClHq1N=2mD zwn{2142-^2Pp&${8N5MgfO4^8ubI&Z1}gX7p=d@-jbUn zMA+qepmr@*wmmI#E5?h+KEJ-HgeoILT7u;QbLviIZWJAx0;dul)zdv(jpvtguVfb3 zLnj99fd=m3mJziH473>>mMN>1*)@DZToN-HOE|-5cSwXO}KnaLj9F?4)Sp4LM@Y!tNehF5Y(VY3Jk>~b> zU|xT;?Te7w2K6&TS*e{4%f|KjA798^i~05F3a28%^4*k|4<-N*T18JwU7>Z3ndima%njDDi8m?Dn6 ze6Yi3i8@yAPRMvMeaJo4u{p1wfkc*!^lq4Xd}sUomWnMWHB_>x;VdAxYC~3E8-^7_ zH~c#B345;kXL(iyx)azt(C~?6L@CxKB`WrTlliL=uTW$czmIDt+vDB5&^TMjB-qL3 zdj&z=_7{$lkrJ9|V;GOl3p%q@XgzJY7-D#a$z&SndX*)5-1>QiC0@l=eEQX|>G4aN zXhm-9Sn9~09Hl~Eum1z)dM69m)mu3Nv|T1@EjPvJd5!($tRrPU=TVA!RlCC%a+i_K z9$hBfzn-dH!eK8S^X9`&_w1+iryaPeszd5O)1@*!)X}HeBTgD;FSXgW$l0}HBKyQQ zW|R%emYiwxqjjXM2>6z1&= zeVkx;@})EhJJjFh8{!@x|ISvn`n@zUO1VpkSDWxzFoKVY+EJ4=cU3&_@ybS$ZoyQI z`3k|&JT%;x_b85oJcY=52=|uC{r=d)2!g|g-%S1tzVD3asisoC4aa<^w z>-nqF<-_j-mR4w?y(9S&uP7J90^Oz!8z=QHP2J6iI6Q0_ulHkX?fLqWb&ZR$;SOnG z;!6SNhkjM^L29u~x)8E~axCGxxTMK1j#e;dP3CRz>0w@h#g%_v<0)eIv@T z3xq9k)SWQV(-P@S|D-`MqI + + + + + diff --git a/blog/scripts/check-links.sh b/blog/scripts/check-links.sh new file mode 100755 index 0000000000000000000000000000000000000000..b2acd95748f7086a7aede0a78c06918b5e4580e9 --- /dev/null +++ b/blog/scripts/check-links.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# checks the links in a built blog: every link between pages and every +# #fragment, offline, and with --external the links out to other sites too. +# +# scripts/check-links.sh [--external] [dir] (dir defaults to dist) +# +# lychee can't parse the at:// uris in the standard.site s (the did's +# colons read as a port) and fails on them before any exclude applies, so they +# come out of a copy of the site first. +set -euo pipefail + +here=$(cd "$(dirname "$0")/.." && pwd) +external=false +if [ "${1:-}" = "--external" ]; then + external=true + shift +fi +site=$(readlink -f "${1:-$here/dist}") + +copy=$(mktemp -d) +trap 'rm -rf "$copy"' EXIT +cp -R "$site/." "$copy" +chmod -R u+w "$copy" +find "$copy" -name '*.html' -exec sed -i.bak -E 's###g' {} + +find "$copy" -name '*.bak' -delete + +if $external; then + lychee --config "$here/lychee.toml" --no-progress --root-dir "$copy" "$copy/**/*.html" +else + lychee --config "$here/lychee.toml" --offline --include-fragments --no-progress --root-dir "$copy" "$copy/**/*.html" +fi diff --git a/blog/src/components/FeedButton.svelte b/blog/src/components/FeedButton.svelte new file mode 100644 index 0000000000000000000000000000000000000000..2748bf980eaf676baf15e1d2df57db0e2203a3fd --- /dev/null +++ b/blog/src/components/FeedButton.svelte @@ -0,0 +1,16 @@ + + + diff --git a/blog/src/components/Footer.astro b/blog/src/components/Footer.astro new file mode 100644 index 0000000000000000000000000000000000000000..50cf8fde856d4d1060cd1f3bdc2d74258d8246f8 --- /dev/null +++ b/blog/src/components/Footer.astro @@ -0,0 +1,43 @@ +--- +import { withBase } from "../lib/posts"; + +// the app's minimal footer, which the blog shared until the web app dropped it for an +// about menu; its links point at tangled.org, the blog being a separate site +const links = [ + { label: "Tangled", href: "https://tangled.org" }, + { label: "Docs", href: "https://docs.tangled.org" }, + { label: "Source", href: "https://tangled.org/@tangled.org/core" }, + { label: "Discord", href: "https://chat.tangled.org", newTab: true }, + { label: "Bluesky", href: "https://bsky.app/profile/tangled.org", newTab: true }, + { label: "X/Twitter", href: "https://x.com/tangled_org", newTab: true }, + { label: "Terms", href: "https://tangled.org/terms" }, + { label: "Privacy", href: "https://tangled.org/privacy" }, +]; +--- + + diff --git a/blog/src/components/ForgeTimeline.svelte b/blog/src/components/ForgeTimeline.svelte new file mode 100644 index 0000000000000000000000000000000000000000..241d22a533c90f3cd9f655789feac7b58f3927f1 --- /dev/null +++ b/blog/src/components/ForgeTimeline.svelte @@ -0,0 +1,205 @@ + + +
+ +
+ + diff --git a/blog/src/components/MarkupLightbox.svelte b/blog/src/components/MarkupLightbox.svelte new file mode 100644 index 0000000000000000000000000000000000000000..a9025ed3da1676b6f18ddf663ae931eb954dcf8b --- /dev/null +++ b/blog/src/components/MarkupLightbox.svelte @@ -0,0 +1,17 @@ + + +{#if lightbox.images.length > 0} + lightbox.close()} + onExited={() => lightbox.clear()} + /> +{/if} diff --git a/blog/src/components/Ripple.astro b/blog/src/components/Ripple.astro new file mode 100644 index 0000000000000000000000000000000000000000..56b363e83f2af0d94499d026b1805c186f461fec --- /dev/null +++ b/blog/src/components/Ripple.astro @@ -0,0 +1,57 @@ +--- +// a phrase whose letters ripple when you point at it: each lifts and settles a beat +// after the one before, left to right. css only; screen readers get the phrase whole +interface Props { + text: string; +} + +const { text } = Astro.props; +// words stay whole so the line still breaks between them, never inside one +const words = text.split(" "); +let index = 0; +--- + +{ + words.map((word, w) => ( + <> + + {w < words.length - 1 && " "} + + )) + } + + diff --git a/blog/src/components/SignupBanner.svelte b/blog/src/components/SignupBanner.svelte new file mode 100644 index 0000000000000000000000000000000000000000..efe7d42c89cbfe22bb4b84dcf606b26034084ea4 --- /dev/null +++ b/blog/src/components/SignupBanner.svelte @@ -0,0 +1,34 @@ + + + + +

Your code, your network.
Come build with us on Tangled.

+ + + + diff --git a/blog/src/components/embeds/OrganizationsEmbed.svelte b/blog/src/components/embeds/OrganizationsEmbed.svelte new file mode 100644 index 0000000000000000000000000000000000000000..3e41bf57fd97de1a50bd21c228a2c9183616eaa1 --- /dev/null +++ b/blog/src/components/embeds/OrganizationsEmbed.svelte @@ -0,0 +1,132 @@ + + + + {#each organizations as organization, i (i)} +
+ + + + + + + {#each { length: organization.members } as _, m (m)} + + {/each} + {#if organization.joins}{/if} + + +
+ {/each} + + + diff --git a/blog/src/components/embeds/PullStackEmbed.svelte b/blog/src/components/embeds/PullStackEmbed.svelte new file mode 100644 index 0000000000000000000000000000000000000000..f042c542dc543a08b3a0e42aafb4bae80edbf60f --- /dev/null +++ b/blog/src/components/embeds/PullStackEmbed.svelte @@ -0,0 +1,153 @@ + + + +
+ + + +
+
+ + {#each commits as commit, i (i)} +
+ + + + + + {#if commit.del}{/if} + +
+ {/each} +
+ + + diff --git a/blog/src/components/embeds/Reveal.svelte b/blog/src/components/embeds/Reveal.svelte new file mode 100644 index 0000000000000000000000000000000000000000..bfebedbca0709f14d41f287e6ab94f1ee074b9be --- /dev/null +++ b/blog/src/components/embeds/Reveal.svelte @@ -0,0 +1,67 @@ + + +
+ {@render children()} +
+ + diff --git a/blog/src/components/embeds/TicketsEmbed.svelte b/blog/src/components/embeds/TicketsEmbed.svelte new file mode 100644 index 0000000000000000000000000000000000000000..83f826aa4baedc91c94c6db3e724f3110e96be2c --- /dev/null +++ b/blog/src/components/embeds/TicketsEmbed.svelte @@ -0,0 +1,175 @@ + + + + {#each tickets as ticket, i (i)} +
+ + + + + + {#if ticket.kind === "issue"}{:else}{/if} + + {#if ticket.merges}{/if} + + + + + + + + +
+ {/each} + + + diff --git a/blog/src/components/embeds/TimelineEmbed.svelte b/blog/src/components/embeds/TimelineEmbed.svelte new file mode 100644 index 0000000000000000000000000000000000000000..cf84417b1e0e74ddffb7109964565c240b6a6502 --- /dev/null +++ b/blog/src/components/embeds/TimelineEmbed.svelte @@ -0,0 +1,274 @@ + + +{#snippet rail(tiles: Tile[])} +
+ {#each tiles as tile (tile.kind)} +
+ + + + + + {#each tile.rows as row, r (r)} + + + + + {/each} + +
+ {/each} +
+{/snippet} + + + {@render rail(left)} +
+ {#each events as event, e (e)} +
+ + + + + + + +
+ {/each} +
+ {@render rail(right)} + + + diff --git a/blog/src/content.config.ts b/blog/src/content.config.ts new file mode 100644 index 0000000000000000000000000000000000000000..0f10b179280576005c79475d61a5979dc51fb5ba --- /dev/null +++ b/blog/src/content.config.ts @@ -0,0 +1,39 @@ +import { defineCollection, z } from "astro:content"; +import { glob } from "astro/loaders"; + +// the posts stay in blog/posts: the appview embeds the same files +// (blog.PostsFS) for the announcements on its home page, and `blog publish` +// (cmd/blog) mirrors them to the pds as standard.site documents +const posts = defineCollection({ + // .mdx for posts that embed a component (a diagram, say) + loader: glob({ pattern: "*.{md,mdx}", base: "./posts" }), + schema: z.object({ + slug: z.string(), + // the standard.site document's record key, a TID; `blog publish` requires + // it on every published post + rkey: z.string().optional(), + title: z.string(), + // a bare `subtitle:` in yaml is null: read it as none, as a wip post often has + subtitle: z + .string() + .nullish() + .transform((subtitle) => subtitle ?? ""), + // tagged "announcement" in feed.json, which the web app's timeline shows + announce: z.boolean().default(false), + date: z.coerce.date(), + image: z.string().url().optional(), + video: z.string().url().optional(), + draft: z.boolean().default(false), + authors: z + .array( + z.object({ + name: z.string(), + email: z.string().optional(), + handle: z.string(), + }), + ) + .min(1), + }), +}); + +export const collections = { posts }; diff --git a/blog/src/dashes.mjs b/blog/src/dashes.mjs new file mode 100644 index 0000000000000000000000000000000000000000..d812be79f49e888e0e5cd5039b19b255886aa882 --- /dev/null +++ b/blog/src/dashes.mjs @@ -0,0 +1,14 @@ +// @ts-check +import { defineMdastPlugin } from "satteri"; + +// the go renderer's dashes (appview/pages/markup/extension/dashes.go), which the +// posts are written for: `--` is an em dash, and a hyphen between two digits +// (50-100) an en dash. satteri's own smart dashes make `--` an en dash instead, so +// they're off (astro.config.mjs) and this does the dashes; code is left alone +export const dashes = defineMdastPlugin({ + name: "tangled-dashes", + text(node, ctx) { + const value = node.value.replace(/--/g, "\u2014").replace(/(\d)-(?=\d)/g, "$1\u2013"); + if (value !== node.value) ctx.replaceNode(node, { type: "text", value }); + }, +}); diff --git a/blog/src/layouts/Base.astro b/blog/src/layouts/Base.astro new file mode 100644 index 0000000000000000000000000000000000000000..12b8f7251ae8616d26f12e5c6669c54ba80803a2 --- /dev/null +++ b/blog/src/layouts/Base.astro @@ -0,0 +1,85 @@ +--- +import "../styles/global.css"; +import Logo from "$lib/components/ui/Logo.svelte"; +import Footer from "../components/Footer.astro"; +import { withBase } from "../lib/posts"; + +interface Props { + title: string; + description: string; + image?: string; + video?: string; + type?: "website" | "article"; + // a post's publication date and authors, for the article meta + published?: Date; + authors?: { name: string; url: string }[]; + // the page as plain markdown, advertised for llms + markdown?: string; + // schema.org data, rendered as json-ld + jsonLd?: Record; + noindex?: boolean; + // the standard.site record this page is: the publication on the index, a + // document on each post + standard?: { rel: "site.standard.publication" | "site.standard.document"; uri: string }; +} + +const { title, description, image, video, type = "website", published, authors, markdown, jsonLd, noindex, standard } = + Astro.props; +const ogType = video ? "video.other" : type; +const canonical = new URL(Astro.url.pathname, Astro.site).href; +const ogImage = image ?? "https://assets.tangled.network/tangled_og.png"; +// `<` escaped so a title can't close the script tag +const ldJson = jsonLd && JSON.stringify({ "@context": "https://schema.org", ...jsonLd }).replace(/ + + + + + {title} + + + {noindex && } + + + {standard && } + {markdown && } + + + + + + + + + + + + + {video && } + {video && } + + + + + {published && } + {authors?.map((author) => )} + {ldJson && '; + const fresh = await serve( + "https://ebilupstream.test/logo.svg", + vi.fn(async () => + new Response(svg, { headers: { "Content-Type": "image/svg+xml" } }), + ), + ); + expect(fresh.status).toBe(200); + expect(fresh.headers.get("Content-Type")).toBe("image/svg+xml"); + expect(fresh.headers.get("Content-Security-Policy")).toBe(SANDBOX_CSP); + expect(fresh.headers.get("X-Content-Type-Options")).toBe("nosniff"); + + const stored = new Response(svg, { + headers: { + "Content-Type": "image/svg+xml", + "Cache-Control": "public, max-age=86400, immutable", + }, + }); + const cached = await serve( + "https://ebilupstream.test/logo.svg", + vi.fn(), + async () => stored, + ); + expect(cached.status).toBe(200); + expect(cached.headers.get("Content-Security-Policy")).toBe(SANDBOX_CSP); + expect(cached.headers.get("X-Content-Type-Options")).toBe("nosniff"); + }); + + it("returns 415 for a type off the allowlist", async () => { + const res = await serve( + "https://ebilupstream.test/page", + vi.fn(async () => + new Response("", { + headers: { "Content-Type": "text/html" }, + }), + ), + ); + expect(res.status).toBe(415); + }); +}); diff --git a/camo/src/mimetypes.json b/camo/src/mimetypes.json new file mode 100644 index 0000000000000000000000000000000000000000..46f106b5daf944a8ff5d8ab9f4bc568097bcce14 --- /dev/null +++ b/camo/src/mimetypes.json @@ -0,0 +1,49 @@ +[ + "image/avif", + "image/bmp", + "image/cgm", + "image/g3fax", + "image/gif", + "image/heif", + "image/ief", + "image/jp2", + "image/jxl", + "image/jpeg", + "image/jpg", + "image/pict", + "image/png", + "image/prs.btif", + "image/svg+xml", + "image/tiff", + "image/vnd.adobe.photoshop", + "image/vnd.djvu", + "image/vnd.dwg", + "image/vnd.dxf", + "image/vnd.fastbidsheet", + "image/vnd.fpx", + "image/vnd.fst", + "image/vnd.fujixerox.edmics-mmr", + "image/vnd.fujixerox.edmics-rlc", + "image/vnd.microsoft.icon", + "image/vnd.ms-modi", + "image/vnd.net-fpx", + "image/vnd.wap.wbmp", + "image/vnd.xiff", + "image/webp", + "image/x-cmu-raster", + "image/x-cmx", + "image/x-icon", + "image/x-macpaint", + "image/x-pcx", + "image/x-pict", + "image/x-portable-anymap", + "image/x-portable-bitmap", + "image/x-portable-graymap", + "image/x-portable-pixmap", + "image/x-quicktime", + "image/x-rgb", + "image/x-xbitmap", + "image/x-xpixmap", + "image/x-xwindowdump", + "video/mp4" +] diff --git a/camo/wrangler.dev.jsonc b/camo/wrangler.dev.jsonc new file mode 100644 index 0000000000000000000000000000000000000000..a6cdf0cf38abe8e8354c39b683a021e6ef51d8d4 --- /dev/null +++ b/camo/wrangler.dev.jsonc @@ -0,0 +1,19 @@ +/** + * Development worker for the sv-fe preview. + */ +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "camo-svfe-dev", + "main": "src/index.js", + "compatibility_date": "2025-04-30", + "observability": { + "enabled": true, + }, + + "routes": [ + { + "pattern": "camo-dev.tangled.org", + "custom_domain": true, + }, + ], +} diff --git a/camo/wrangler.jsonc b/camo/wrangler.jsonc new file mode 100644 index 0000000000000000000000000000000000000000..bd7957e0d1ba177de3ba84ee1b429a6f0342d628 --- /dev/null +++ b/camo/wrangler.jsonc @@ -0,0 +1,20 @@ +/** + * For more details on how to configure Wrangler, refer to: + * https://developers.cloudflare.com/workers/wrangler/configuration/ + */ +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "camo", + "main": "src/index.js", + "compatibility_date": "2025-04-30", + "observability": { + "enabled": true, + }, + + "routes": [ + { + "pattern": "camo.tangled.sh", + "custom_domain": true, + }, + ], +} diff --git a/cilogs/cihelpers.go b/cilogs/cihelpers.go new file mode 100644 index 0000000000000000000000000000000000000000..8d3df6027b1c66d6b77e4065b793530c29abd25f --- /dev/null +++ b/cilogs/cihelpers.go @@ -0,0 +1,50 @@ +package cilogs + +import ( + "time" + + "tangled.org/core/api/org_tangled" +) + +// helper functions against generated code + +func workflowElapsed(wf *org_tangled.CiDefs_WorkflowRun, now time.Time) time.Duration { + if wf.StartedAt == nil { + return 0 + } + started, err := time.Parse(time.RFC3339, *wf.StartedAt) + if err != nil { + return 0 + } + if wf.FinishedAt == nil { + return now.Sub(started) + } + finished, err := time.Parse(time.RFC3339, *wf.FinishedAt) + if err != nil { + return 0 + } + return finished.Sub(started) +} + +func workflowName(wf *org_tangled.CiDefs_WorkflowRun) string { + if wf.Name != nil { + return *wf.Name + } + return wf.Id +} + +var finishedStatuses = map[string]bool{ + "failed": true, + "timeout": true, + "cancelled": true, + "success": true, +} + +func pipelineFinished(p *org_tangled.CiDefs_Pipeline) bool { + for _, wf := range p.Workflows { + if !finishedStatuses[wf.Status] { + return false + } + } + return true +} diff --git a/cilogs/cilogs_test.go b/cilogs/cilogs_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f517024f6ba2ae5b84fa4c419eb91df8c47fd43d --- /dev/null +++ b/cilogs/cilogs_test.go @@ -0,0 +1,233 @@ +package cilogs + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/gorilla/websocket" + gossh "golang.org/x/crypto/ssh" + "tangled.org/core/api/org_tangled" + "tangled.org/core/repoident" +) + +const ( + testRepo = "did:plc:cilogsrepo" + testSha = "4f1c2b3a4f1c2b3a4f1c2b3a4f1c2b3a4f1c2b3a" +) + +func fakeSpindle(t *testing.T, pipeline map[string]any) *httptest.Server { + upgrader := websocket.Upgrader{} + mux := http.NewServeMux() + reply := func(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(v) + } + mux.HandleFunc("/xrpc/org.tangled.ci.queryPipelines", func(w http.ResponseWriter, r *http.Request) { + reply(w, map[string]any{"pipelines": []any{pipeline}}) + }) + mux.HandleFunc("/xrpc/org.tangled.ci.getPipeline", func(w http.ResponseWriter, r *http.Request) { + reply(w, pipeline) + }) + mux.HandleFunc("/xrpc/org.tangled.ci.subscribePipelineLogs", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("repo") != testRepo { + http.Error(w, `{"error":"InvalidRequest","message":"repo is required"}`, http.StatusBadRequest) + return + } + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + status := func(s string) *string { return &s } + now := time.Now().Format(time.RFC3339) + for _, ev := range []org_tangled.CiSubscribePipelineLogs_Event{ + {Control: &org_tangled.CiSubscribePipelineLogs_Control{Time: now, Workflow: "build", Step: 0, Content: "compile", Status: status("start")}}, + {Data: &org_tangled.CiSubscribePipelineLogs_Data{Time: now, Workflow: "build", Step: 0, Content: "hello from the build\n", Stream: "stdout"}}, + } { + var buf bytes.Buffer + if err := ev.Serialize(&buf); err != nil { + t.Error(err) + return + } + conn.WriteMessage(websocket.BinaryMessage, buf.Bytes()) + } + <-r.Context().Done() + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +func fakeKnot(t *testing.T, spindleDid string) *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/xrpc/com.atproto.repo.getRecord" || r.URL.Query().Get("collection") != org_tangled.RepoManifestNSID { + http.Error(w, `{"error":"RecordNotFound"}`, http.StatusNotFound) + return + } + json.NewEncoder(w).Encode(map[string]any{"value": map[string]any{ + "$type": org_tangled.RepoManifestNSID, + "declaration": map[string]any{"owner": "did:plc:owner", "name": "proj"}, + "spindle": spindleDid, + "createdAt": "2026-10-01T09:12:44Z", + }}) + })) + t.Cleanup(srv.Close) + return srv +} + +func directoryFor(knot *httptest.Server) identity.Directory { + dir := identity.NewMockDirectory() + dir.Insert(identity.Identity{DID: testRepo, Services: map[string]identity.ServiceEndpoint{ + repoident.KnotServiceID: {Type: repoident.KnotServiceType, URL: knot.URL}, + }}) + return dir +} + +func freeAddr(t *testing.T) string { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + return ln.Addr().String() +} + +func dialSSH(t *testing.T, addr string) *gossh.Client { + deadline := time.Now().Add(5 * time.Second) + for { + c, err := gossh.Dial("tcp", addr, &gossh.ClientConfig{User: "anyone", HostKeyCallback: gossh.InsecureIgnoreHostKey()}) + if err == nil { + return c + } + if time.Now().After(deadline) { + t.Fatal(err) + } + time.Sleep(50 * time.Millisecond) + } +} + +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func TestFollowsAWorkflowsLogsFromTheSpindleInTheRepoManifest(t *testing.T) { + name := "build" + pipeline := map[string]any{ + "id": syntax.NewTIDNow(0).String(), + "repo": testRepo, + "commit": testSha, + "trigger": map[string]any{ + "$type": "org.tangled.event.push", "ref": "refs/heads/main", "newSha": testSha, + "oldSha": strings.Repeat("0", 40), "repo": testRepo, + }, + "workflows": []any{map[string]any{"id": name, "name": name, "status": "running", "startedAt": time.Now().Format(time.RFC3339)}}, + } + spindle := fakeSpindle(t, pipeline) + spindleDid := "did:web:localhost%3A" + spindle.URL[strings.LastIndex(spindle.URL, ":")+1:] + knot := fakeKnot(t, spindleDid) + + dir := directoryFor(knot) + + addr := freeAddr(t) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + srv := New(Config{ListenAddr: addr, HostKeyPath: t.TempDir() + "/host_key"}, NewSpindles(dir, true), slog.New(slog.NewTextHandler(io.Discard, nil))) + go srv.ListenAndServe(ctx) + + client := dialSSH(t, addr) + defer client.Close() + sess, err := client.NewSession() + if err != nil { + t.Fatal(err) + } + defer sess.Close() + if err := sess.RequestPty("xterm-256color", 40, 160, gossh.TerminalModes{}); err != nil { + t.Fatal(err) + } + var out syncBuffer + sess.Stdout = &out + sess.Stderr = &out + stdin, err := sess.StdinPipe() + if err != nil { + t.Fatal(err) + } + if err := sess.Start(testRepo + " " + testSha); err != nil { + t.Fatal(err) + } + // wish asks the terminal for its background colour before drawing anything + stdin.Write([]byte("\x1b]11;rgb:0000/0000/0000\x07\x1b[?62c")) + + deadline := time.Now().Add(10 * time.Second) + for !strings.Contains(out.String(), "hello from the build") { + if strings.Contains(out.String(), "error:") || time.Now().After(deadline) { + t.Fatalf("never saw the log line, terminal got:\n%s", out.String()) + } + time.Sleep(50 * time.Millisecond) + } + if !strings.Contains(out.String(), "refs/heads/main") { + t.Errorf("the top bar doesn't name the pushed ref:\n%s", out.String()) + } + stdin.Write([]byte("q")) +} + +func TestSpindleURL(t *testing.T) { + for did, want := range map[string]string{ + "did:web:spindle.example.com": "https://spindle.example.com", + "did:web:localhost%3A6555": "http://localhost:6555", + "did:web:host:with:a:path": "", + "did:plc:abc": "", + } { + got, err := spindleURL(did) + if (err != nil) != (want == "") || got != want { + t.Errorf("spindleURL(%q) = %q, %v, want %q", did, got, err, want) + } + } +} + +func TestNoSpindleInTheManifestIsReportedAsSuch(t *testing.T) { + _, err := NewSpindles(directoryFor(fakeKnot(t, "")), true).Of(context.Background(), testRepo) + if !errors.Is(err, ErrNoSpindle) { + t.Errorf("err = %v, want ErrNoSpindle", err) + } +} + +func TestRefusesLoopbackSpindlesOutsideDev(t *testing.T) { + srv := httptest.NewServer(http.NotFoundHandler()) + defer srv.Close() + addr := strings.TrimPrefix(srv.URL, "http://") + for dev, wantErr := range map[bool]bool{false: true, true: false} { + conn, err := NewSpindles(identity.NewMockDirectory(), dev).dialer.DialContext(context.Background(), "tcp", addr) + if conn != nil { + conn.Close() + } + if (err != nil) != wantErr { + t.Errorf("dev=%v: dial err = %v, want error: %v", dev, err, wantErr) + } + } +} diff --git a/cilogs/error.go b/cilogs/error.go new file mode 100644 index 0000000000000000000000000000000000000000..69ef2f0c93bb0fad65d9122ff225a2fa0f2d338a --- /dev/null +++ b/cilogs/error.go @@ -0,0 +1,31 @@ +package cilogs + +import ( + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" +) + +var ( + colorBrightRed lipgloss.ANSIColor = 9 +) + +type errorModel struct { + renderer *lipgloss.Renderer + message string +} + +func newErrorModel(renderer *lipgloss.Renderer, message string) *errorModel { + return &errorModel{renderer: renderer, message: message} +} + +func (m *errorModel) Init() tea.Cmd { + return tea.Quit +} + +func (m *errorModel) Update(_ tea.Msg) (tea.Model, tea.Cmd) { + return m, tea.Quit +} + +func (m *errorModel) View() string { + return m.renderer.NewStyle().Foreground(colorBrightRed).Render("error: "+m.message) + "\n" +} diff --git a/cilogs/logstream.go b/cilogs/logstream.go new file mode 100644 index 0000000000000000000000000000000000000000..a8d84db0482af34d155c8ebcf44b8cbeffbf676a --- /dev/null +++ b/cilogs/logstream.go @@ -0,0 +1,47 @@ +package cilogs + +import ( + "context" + "time" + + "tangled.org/core/api/org_tangled" +) + +type step struct { + id int64 + name string + command string + lines []string + startTime time.Time + endTime time.Time + finished bool +} + +type logDoneMsg struct { + err error +} + +type logEventMsg struct { + ev *org_tangled.CiSubscribePipelineLogs_Event + events chan *org_tangled.CiSubscribePipelineLogs_Event + done chan error +} + +type eventScheduler struct { + ch chan *org_tangled.CiSubscribePipelineLogs_Event +} + +func newEventScheduler() *eventScheduler { + return &eventScheduler{ch: make(chan *org_tangled.CiSubscribePipelineLogs_Event, 1024)} +} + +func (s *eventScheduler) AddWork(ctx context.Context, _ string, v *org_tangled.CiSubscribePipelineLogs_Event) error { + select { + case s.ch <- v: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *eventScheduler) Shutdown() { close(s.ch) } diff --git a/cilogs/server.go b/cilogs/server.go new file mode 100644 index 0000000000000000000000000000000000000000..5ce69728765348be7c0f326b25a7df84688847f3 --- /dev/null +++ b/cilogs/server.go @@ -0,0 +1,71 @@ +package cilogs + +import ( + "context" + "fmt" + "log/slog" + + "github.com/charmbracelet/ssh" + "github.com/charmbracelet/wish" + tea "github.com/charmbracelet/wish/bubbletea" +) + +type Config struct { + ListenAddr string + HostKeyPath string +} + +type Server struct { + config Config + spindles *Spindles + logger *slog.Logger +} + +func New(cfg Config, spindles *Spindles, logger *slog.Logger) *Server { + return &Server{config: cfg, spindles: spindles, logger: logger} +} + +func (s *Server) ListenAndServe(ctx context.Context) error { + opts := []ssh.Option{ + wish.WithAddress(s.config.ListenAddr), + wish.WithMiddleware( + tea.Middleware(s.teaHandler), + requirePty, + ), + } + + if s.config.HostKeyPath != "" { + opts = append(opts, wish.WithHostKeyPath(s.config.HostKeyPath)) + } + + srv, err := wish.NewServer(opts...) + if err != nil { + return err + } + + go func() { + <-ctx.Done() + s.logger.Info("shutting down SSH log server") + srv.Close() + }() + + s.logger.Info("SSH log server listening", "address", s.config.ListenAddr) + if err := srv.ListenAndServe(); err != ssh.ErrServerClosed { + return err + } + s.logger.Info("SSH log server stopped") + return nil +} + +// requirePty is a middleware that rejects connections without a PTY and tells the user to pass -t. +func requirePty(next ssh.Handler) ssh.Handler { + return func(sess ssh.Session) { + _, _, ok := sess.Pty() + if !ok { + fmt.Fprintf(sess.Stderr(), "error: no terminal allocated\nhint: use `ssh -t` to force PTY allocation\n") + sess.Exit(1) + return + } + next(sess) + } +} diff --git a/cilogs/session.go b/cilogs/session.go new file mode 100644 index 0000000000000000000000000000000000000000..2f9284b0f539768e6433413b26be6dedf4b93529 --- /dev/null +++ b/cilogs/session.go @@ -0,0 +1,64 @@ +package cilogs + +import ( + "errors" + "fmt" + + "github.com/bluesky-social/indigo/atproto/syntax" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/ssh" + wishtea "github.com/charmbracelet/wish/bubbletea" + "tangled.org/core/api/org_tangled" +) + +func (s *Server) teaHandler(sess ssh.Session) (tea.Model, []tea.ProgramOption) { + remote := sess.RemoteAddr().String() + args := sess.Command() + l := s.logger.With("remote", remote) + l.Info("SSH connection", "args", args) + defer l.Info("SSH connection closed", "remote", remote) + + renderer := wishtea.MakeRenderer(sess) + + if len(args) != 2 { + l.Warn("bad invocation", "args", args) + return newErrorModel(renderer, "usage: ssh -t -p "), wishtea.MakeOptions(sess) + } + + repoDID := args[0] + sha := args[1] + + l = l.With("repoDID", repoDID, "sha", sha) + + repo, err := syntax.ParseDID(repoDID) + if err != nil { + l.Warn("bad repo did", "err", err) + return newErrorModel(renderer, fmt.Sprintf("%q is not a repo did", repoDID)), wishtea.MakeOptions(sess) + } + xrpcc, err := s.spindles.Of(sess.Context(), repo) + if err != nil { + l.Warn("no spindle for repo", "err", err) + if errors.Is(err, ErrNoSpindle) { + return newErrorModel(renderer, fmt.Sprintf("no spindle configured for repo %s", repoDID)), wishtea.MakeOptions(sess) + } + return newErrorModel(renderer, fmt.Sprintf("could not look up the spindle for repo %s", repoDID)), wishtea.MakeOptions(sess) + } + l = l.With("spindle", xrpcc.Host) + + out, err := org_tangled.CiQueryPipelines(sess.Context(), xrpcc, []string{sha}, "", 1, repoDID, nil) + if err != nil || len(out.Pipelines) == 0 { + l.Warn("pipeline not found", "err", err) + return newErrorModel(renderer, fmt.Sprintf("pipeline not found for repo %s @ %s", repoDID, sha)), wishtea.MakeOptions(sess) + } + + pipeline := out.Pipelines[0] + if _, err := syntax.ParseRecordKey(pipeline.Id); err != nil { + l.Warn("invalid pipeline id", "id", pipeline.Id, "err", err) + return newErrorModel(renderer, fmt.Sprintf("invalid pipeline id %q", pipeline.Id)), wishtea.MakeOptions(sess) + } + + l.Info("serving pipeline", "pipeline", pipeline.Id, "workflows", len(pipeline.Workflows)) + pty, _, _ := sess.Pty() + opts := append(wishtea.MakeOptions(sess), tea.WithAltScreen()) + return newPipelineModel(renderer, xrpcc, pipeline, pty.Window.Width, pty.Window.Height), opts +} diff --git a/cilogs/spindles.go b/cilogs/spindles.go new file mode 100644 index 0000000000000000000000000000000000000000..96dc67b9a825ec86bdcdc498caa31ec3ab220298 --- /dev/null +++ b/cilogs/spindles.go @@ -0,0 +1,69 @@ +package cilogs + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "strings" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "github.com/gorilla/websocket" + "tangled.org/core/api/org_tangled" + "tangled.org/core/hostutil" + extlexutil "tangled.org/core/lexutil" + "tangled.org/core/spindle/knotclient" +) + +var ErrNoSpindle = errors.New("no spindle configured for this repo") + +type Spindles struct { + knots *knotclient.Client + http *http.Client + dialer *net.Dialer +} + +// the manifest can name any host, so outside dev the calls refuse special addresses +func NewSpindles(dir identity.Directory, dev bool) *Spindles { + httpc := knotclient.HTTPClient(dev) + return &Spindles{knots: knotclient.New(dir, httpc, dev), http: httpc, dialer: knotclient.Dialer(dev)} +} + +func (s *Spindles) Of(ctx context.Context, repo syntax.DID) (*extlexutil.Client, error) { + knot, err := s.knots.KnotOf(ctx, repo) + if err != nil { + return nil, err + } + var manifest org_tangled.RepoManifest + if err := s.knots.GetRecord(ctx, knot, repo, org_tangled.RepoManifestNSID, knotclient.ManifestRkey, &manifest); err != nil { + return nil, err + } + if manifest.Spindle == nil || *manifest.Spindle == "" { + return nil, ErrNoSpindle + } + base, err := spindleURL(*manifest.Spindle) + if err != nil { + return nil, err + } + return &extlexutil.Client{ + Client: indigoxrpc.Client{Host: base, Client: s.http}, + Dialer: websocket.Dialer{NetDialContext: s.dialer.DialContext}, + }, nil +} + +// the spindle serves xrpc on the did:web host, like hostForKnot in web +func spindleURL(did string) (string, error) { + host, ok := strings.CutPrefix(did, "did:web:") + if !ok || strings.Contains(host, ":") { + return "", fmt.Errorf("spindle %q is not a did:web host", did) + } + host, err := url.PathUnescape(host) + if err != nil { + return "", fmt.Errorf("spindle %q: %w", did, err) + } + return hostutil.EnsureHttpScheme(host) +} diff --git a/cilogs/tui.go b/cilogs/tui.go new file mode 100644 index 0000000000000000000000000000000000000000..dca5bdd20fcc3d490f80f8c57e53db6d265d29d6 --- /dev/null +++ b/cilogs/tui.go @@ -0,0 +1,500 @@ +package cilogs + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/charmbracelet/bubbles/spinner" + "github.com/charmbracelet/bubbles/viewport" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/gorilla/websocket" + "tangled.org/core/api/org_tangled" + extlexutil "tangled.org/core/lexutil" +) + +var ( + colorFg lipgloss.NoColor = lipgloss.NoColor{} + colorBlue lipgloss.ANSIColor = 4 + colorBrightBlack lipgloss.ANSIColor = 8 +) + +type tickMsg time.Time + +type statusUpdateMsg struct { + pipeline *org_tangled.CiDefs_Pipeline +} + +type statusUpdateErrMsg struct{ err error } + +type pipelineModel struct { + renderer *lipgloss.Renderer + xrpcc *extlexutil.Client + pipeline *org_tangled.CiDefs_Pipeline + selected int + logs map[string]*workflowLogs + + // pipeline log stream: cancel tears down the consumer goroutine on quit. + // the event/done channels are threaded through log messages, not stored here. + cancel context.CancelFunc + streamDone bool + streamErr error + + spinner spinner.Model + width int + height int +} + +type workflowLogs struct { + steps []step + stepIndex map[int64]int // stepId -> index map + vp viewport.Model + ready bool +} + +func newPipelineModel(renderer *lipgloss.Renderer, xrpcc *extlexutil.Client, pipeline *org_tangled.CiDefs_Pipeline, width, height int) *pipelineModel { + logs := make(map[string]*workflowLogs, len(pipeline.Workflows)) + for _, wf := range pipeline.Workflows { + logs[workflowName(wf)] = &workflowLogs{stepIndex: make(map[int64]int)} + } + sp := spinner.New(spinner.WithSpinner(spinner.Line)) + return &pipelineModel{ + renderer: renderer, + xrpcc: xrpcc, + pipeline: pipeline, + logs: logs, + spinner: sp, + width: width, + height: height, + } +} + +func (m *pipelineModel) Init() tea.Cmd { + return tea.Batch(tick(), m.spinner.Tick, m.subscribeCmd()) +} + +func tick() tea.Cmd { + return tea.Tick(time.Second, func(t time.Time) tea.Msg { return tickMsg(t) }) +} + +// subscribeCmd opens the ci.pipeline.subscribeLogs stream for current pipeline. +// A consumer goroutine pushes decoded events onto the scheduler channel; the +// returned command yields the first event into the bubbletea loop. +func (m *pipelineModel) subscribeCmd() tea.Cmd { + // cancel existing subscriptions just in case + if m.cancel != nil { + m.cancel() + } + sched := newEventScheduler() + done := make(chan error, 1) + ctx, cancel := context.WithCancel(context.Background()) + m.cancel = cancel + + repo, pipelineId := m.pipeline.Repo, m.pipeline.Id + go func() { + err := org_tangled.CiSubscribePipelineLogs(ctx, m.xrpcc, repo, pipelineId, nil, sched) + done <- err + }() + + return readEventCmd(sched.ch, done) +} + +func readEventCmd(events chan *org_tangled.CiSubscribePipelineLogs_Event, done chan error) tea.Cmd { + return func() tea.Msg { + ev, ok := <-events + if !ok { + return logDoneMsg{err: <-done} + } + return logEventMsg{ev: ev, events: events, done: done} + } +} + +// fetchStatusCmd re-fetches the pipeline +func (m *pipelineModel) fetchStatusCmd() tea.Cmd { + repo, pipelineId := m.pipeline.Repo, m.pipeline.Id + return func() tea.Msg { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + out, err := org_tangled.CiGetPipeline(ctx, m.xrpcc, pipelineId, repo) + if err != nil { + return statusUpdateErrMsg{err: fmt.Errorf("refreshing pipeline: %w", err)} + } + return statusUpdateMsg{pipeline: out} + } +} + +func (m *pipelineModel) vpHeight() int { + return max(m.height-2, 1) // topbar + empty line take 2 lines +} + +// resizeViewports updates all viewport dimensions and re-renders their content after a terminal resize. +// +// TODO: can be tedious if we have logs of logs +func (m *pipelineModel) resizeViewports() { + for _, wl := range m.logs { + if !wl.ready { + continue + } + atBottom := wl.vp.AtBottom() + wl.vp.Width = m.width + wl.vp.Height = m.vpHeight() + wl.vp.SetContent(renderLogs(m.renderer, wl, m.width)) + if atBottom { + wl.vp.GotoBottom() + } + } +} + +func (m *pipelineModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width, m.height = msg.Width, msg.Height + m.resizeViewports() + + case tickMsg: + // re-render running workflows so elapsed times advance + m.refreshRunning() + return m, tick() + + case spinner.TickMsg: + var cmd tea.Cmd + m.spinner, cmd = m.spinner.Update(msg) + return m, cmd + + case tea.KeyMsg: + switch msg.String() { + case "q", "ctrl+c": + if m.cancel != nil { + m.cancel() + } + return m, tea.Quit + case "tab", "right", "l": + m.selected = (m.selected + 1) % len(m.pipeline.Workflows) + return m, nil + case "shift+tab", "left", "h": + m.selected = (m.selected - 1 + len(m.pipeline.Workflows)) % len(m.pipeline.Workflows) + return m, nil + } + if wl := m.selectedLogs(); wl != nil && wl.ready { + switch msg.String() { + case "g": + wl.vp.GotoTop() + return m, nil + case "G": + wl.vp.GotoBottom() + return m, nil + case "ctrl+e": + wl.vp.ScrollDown(1) + return m, nil + case "ctrl+y": + wl.vp.ScrollUp(1) + return m, nil + } + var cmd tea.Cmd + wl.vp, cmd = wl.vp.Update(msg) + return m, cmd + } + + case logEventMsg: + m.applyEvent(msg.ev) + return m, readEventCmd(msg.events, msg.done) + + case logDoneMsg: + m.streamDone = true + if !isExpectedClose(msg.err) { + m.streamErr = msg.err + } + m.refreshAll() + // resolve final workflow statuses once now that the stream has ended + return m, m.fetchStatusCmd() + + case statusUpdateMsg: + m.pipeline = msg.pipeline + known := make(map[string]bool, len(m.pipeline.Workflows)) + for _, wf := range m.pipeline.Workflows { + known[workflowName(wf)] = true + } + for name := range m.logs { + if !known[name] { + delete(m.logs, name) + } + } + if m.selected >= len(m.pipeline.Workflows) { + m.selected = max(len(m.pipeline.Workflows)-1, 0) + } + m.refreshAll() + + case statusUpdateErrMsg: + // best-effort final status refresh; ignore failures + } + + return m, nil +} + +func (m *pipelineModel) selectedLogs() *workflowLogs { + if len(m.pipeline.Workflows) < 1+m.selected { + return nil + } + return m.logs[workflowName(m.pipeline.Workflows[m.selected])] +} + +func (m *pipelineModel) initViewport(wl *workflowLogs) { + if wl.ready { + return + } + wl.vp = viewport.New(m.width, m.vpHeight()) + wl.ready = true +} + +// ensureWorkflow returns the log state for a workflow, lazily creating its routing entry. +func (m *pipelineModel) ensureWorkflow(name string) *workflowLogs { + wl, ok := m.logs[name] + if !ok { + wl = &workflowLogs{stepIndex: make(map[int64]int)} + m.logs[name] = wl + } + return wl +} + +// renderWorkflow re-renders a workflow's viewport, preserving bottom-stickiness. +func (m *pipelineModel) renderWorkflow(wl *workflowLogs) { + m.initViewport(wl) + atBottom := wl.vp.AtBottom() + wl.vp.SetContent(renderLogs(m.renderer, wl, m.width)) + if atBottom { + wl.vp.GotoBottom() + } +} + +// refreshAll re-renders every initialized viewport. +func (m *pipelineModel) refreshAll() { + for _, wl := range m.logs { + m.renderWorkflow(wl) + } +} + +// refreshRunning re-renders workflows with unfinished steps so elapsed times advance. +func (m *pipelineModel) refreshRunning() { + if m.streamDone { + return + } + for _, wl := range m.logs { + if !wl.ready { + continue + } + for i := range wl.steps { + if !wl.steps[i].finished { + m.renderWorkflow(wl) + break + } + } + } +} + +// applyEvent routes a decoded subscribeLogs event into the matching workflow. +func (m *pipelineModel) applyEvent(ev *org_tangled.CiSubscribePipelineLogs_Event) { + switch { + case ev.Error != nil: + if ev.Error.Message != "" { + m.streamErr = fmt.Errorf("%s: %s", ev.Error.Error, ev.Error.Message) + } else { + m.streamErr = fmt.Errorf("%s", ev.Error.Error) + } + + case ev.Control != nil: + c := ev.Control + wl := m.ensureWorkflow(c.Workflow) + switch derefStr(c.Status) { + case "start": + wl.stepIndex[c.Step] = len(wl.steps) + wl.steps = append(wl.steps, step{ + id: c.Step, name: c.Content, command: derefStr(c.Command), startTime: parseRFC3339(c.Time), + }) + case "end": + if idx, ok := wl.stepIndex[c.Step]; ok { + wl.steps[idx].endTime, wl.steps[idx].finished = parseRFC3339(c.Time), true + } + } + m.renderWorkflow(wl) + + case ev.Data != nil: + d := ev.Data + wl := m.ensureWorkflow(d.Workflow) + if idx, ok := wl.stepIndex[d.Step]; ok { + wl.steps[idx].lines = append(wl.steps[idx].lines, d.Content) + } + m.renderWorkflow(wl) + } +} + +// isExpectedClose reports whether err is a clean websocket close (or nil). +func isExpectedClose(err error) bool { + if err == nil { + return true + } + var ce *websocket.CloseError + if errors.As(err, &ce) { + switch ce.Code { + case websocket.CloseNormalClosure, websocket.CloseGoingAway, websocket.CloseAbnormalClosure: + return true + } + } + return false +} + +// renderLogs builds the full log content string for a workflow, used as viewport content. +func renderLogs(r *lipgloss.Renderer, wl *workflowLogs, width int) string { + headerStyle := r.NewStyle().Foreground(colorFg).Bold(true) + cmdStyle := r.NewStyle().Foreground(colorBlue).Width(width) + dimStyle := r.NewStyle().Faint(true) + var sb strings.Builder + for i := range wl.steps { + st := &wl.steps[i] + dur := "" + if st.finished { + dur = st.endTime.Sub(st.startTime).Round(time.Millisecond).String() + } else if !st.startTime.IsZero() { + dur = time.Since(st.startTime).Round(time.Second).String() + } + // build overlay: "── name ──...── dur ──" + nameStr := headerStyle.Render(st.name + " ") + durStr := headerStyle.Render(" " + dur + " ") + nameW := lipgloss.Width(nameStr) + durW := lipgloss.Width(durStr) + fillW := max(width-nameW-durW, 0) + fill := dimStyle.Render(strings.Repeat("─", fillW)) + header := nameStr + fill + durStr + sb.WriteString(header + "\n") + if st.command != "" { + sb.WriteString(cmdStyle.Render(st.command) + "\n") + } + for _, l := range st.lines { + sb.WriteString(l + "\n") + } + sb.WriteString("\n") + } + return sb.String() +} + +func (m *pipelineModel) View() string { + body := "" + if wl := m.selectedLogs(); wl != nil && wl.ready { + body = wl.vp.View() + } + if m.streamErr != nil { + body = lipgloss.JoinVertical(lipgloss.Left, body, m.renderer.NewStyle().Foreground(colorBlue).Render("stream error: "+m.streamErr.Error())) + } + return lipgloss.JoinVertical(lipgloss.Left, m.topbarView(), "", body) +} + +// topbarView renders the single-line tab bar with workflow tabs left and trigger info + help right. +func (m *pipelineModel) topbarView() string { + r := m.renderer + activeStyle := r.NewStyle().Background(colorBlue).Foreground(colorFg).Bold(true) + + now := time.Now() + + var tabs strings.Builder + for i, wf := range m.pipeline.Workflows { + status := wf.Status + elapsed := workflowElapsed(wf, now).Round(time.Second).String() + base := " " + statusIcon(status, m.spinner.View()) + " " + workflowName(wf) + if i == m.selected { + tab := base + if elapsed != "" { + tab += " " + elapsed + } + tab += " " + tabs.WriteString(activeStyle.Render(tab)) + } else { + tabs.WriteString(base) + if elapsed != "" { + dim := r.NewStyle().Faint(true) + tabs.WriteString(" " + dim.Render(elapsed)) + } + tabs.WriteString(" ") + } + } + + tabsStr := tabs.String() + infoStr := triggerLine(r, m.pipeline.Trigger, m.pipeline.Commit) + " · " + helpText(r) + + gap := max(m.width-lipgloss.Width(tabsStr)-lipgloss.Width(infoStr), 1) + + return tabsStr + strings.Repeat(" ", gap) + infoStr +} + +func helpText(r *lipgloss.Renderer) string { + key := r.NewStyle().Foreground(colorFg) + action := r.NewStyle().Faint(true) + sep := action.Render(" · ") + + items := []string{ + key.Render("←/→") + " " + action.Render("switch"), + key.Render("↑/↓") + " " + action.Render("scroll"), + key.Render("q") + " " + action.Render("quit"), + } + return strings.Join(items, sep) +} + +func shortSha(sha string) string { + if len(sha) >= 8 { + return sha[:8] + } + return sha +} + +func triggerLine(r *lipgloss.Renderer, t *org_tangled.CiDefs_Pipeline_Trigger, sha string) string { + hash := shortSha(sha) + dim := r.NewStyle().Faint(true) + if t == nil { + return dim.Render(hash) + } + switch { + case t.EventPush != nil: + return t.EventPush.Ref + dim.Render("@"+hash) + dim.Render(" (push)") + case t.EventPullRequest != nil: + pr := t.EventPullRequest + return derefStr(pr.TargetBranch) + dim.Render(" <- "+derefStr(pr.SourceBranch)+"@"+hash) + dim.Render(" (pull-request)") + case t.CiTriggerSchedule != nil: + return t.CiTriggerSchedule.Ref + dim.Render("@"+hash) + dim.Render(" (schedule)") + } + return dim.Render(hash) +} + +func statusIcon(status string, spinnerFrame string) string { + switch status { + case "success": + return "✓" + case "failed": + return "×" + case "running": + return spinnerFrame + case "pending": + return "·" + case "timeout": + return "⌀" + case "cancelled": + return "-" + default: + return "?" + } +} + +func derefStr(s *string) string { + if s == nil { + return "" + } + return *s +} + +func parseRFC3339(s string) time.Time { + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return time.Time{} + } + return t +} diff --git a/cmd/blog/main.go b/cmd/blog/main.go new file mode 100644 index 0000000000000000000000000000000000000000..5e5163cc5621e38f1fd4a76e9ff46ce351405070 --- /dev/null +++ b/cmd/blog/main.go @@ -0,0 +1,50 @@ +// Command blog publishes the blog's posts as standard.site records on the +// Tangled PDS. The site itself is built by Astro (nix/pkgs/blog.nix). +package main + +import ( + "context" + "fmt" + "os" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "tangled.org/core/blog" + tlog "tangled.org/core/log" +) + +const postsDir = "blog/posts" + +func main() { + if len(os.Args) < 2 || os.Args[1] != "publish" { + fmt.Fprintln(os.Stderr, "usage: blog publish") + os.Exit(1) + } + + logger := tlog.New("blog") + if err := runPublish(context.Background()); err != nil { + logger.Error("publish failed", "err", err) + os.Exit(1) + } +} + +func runPublish(ctx context.Context) error { + password := os.Getenv("BLOG_APP_PASSWORD") + if password == "" { + return fmt.Errorf("BLOG_APP_PASSWORD is not set") + } + + posts, err := blog.Posts(postsDir) + if err != nil { + return fmt.Errorf("parsing posts: %w", err) + } + + client, err := atclient.LoginWithPassword(ctx, identity.DefaultDirectory(), syntax.AtIdentifier(blog.RepoDID), password, "", nil) + if err != nil { + return fmt.Errorf("logging in: %w", err) + } + + return blog.Publish(ctx, client, posts) +} diff --git a/cmd/cborgen/cborgen.go b/cmd/cborgen/cborgen.go new file mode 100644 index 0000000000000000000000000000000000000000..2f5ae8bd7f768e93141234c654aee17002d6fe5b --- /dev/null +++ b/cmd/cborgen/cborgen.go @@ -0,0 +1,65 @@ +package main + +import ( + cbg "github.com/whyrusleeping/cbor-gen" + "tangled.org/core/api/org_tangled" +) + +func main() { + genCfg := cbg.Gen{ + MaxStringLength: 1_000_000, + } + + if err := genCfg.WriteMapEncodersToFile( + "api/org_tangled/cbor_gen.go", + "org_tangled", + org_tangled.ActorProfile{}, + org_tangled.CiSubscribePipelineLogs_Control{}, + org_tangled.CiSubscribePipelineLogs_Data{}, + org_tangled.CiTriggerManual{}, + org_tangled.CiTriggerManual_Pair{}, + org_tangled.CiTriggerSchedule{}, + org_tangled.EmbedCommit{}, + org_tangled.EventPullRequest{}, + org_tangled.EventPush{}, + org_tangled.FeedComment{}, + org_tangled.FeedReaction{}, + org_tangled.FeedStar{}, + org_tangled.FeedStar_Record{}, + org_tangled.FeedStar_Repo{}, + org_tangled.FeedSubscription{}, + org_tangled.FeedSubscription_Record{}, + org_tangled.FeedSubscription_Repo{}, + org_tangled.GitOid{}, + org_tangled.GitRefs{}, + org_tangled.GraphAttestation{}, + org_tangled.GraphCollaborator{}, + org_tangled.GraphFollow{}, + org_tangled.GraphVouch{}, + org_tangled.KeySshKey{}, + org_tangled.KeyUnspeccedGpgKey{}, + org_tangled.KnotBlock{}, + org_tangled.KnotMember{}, + org_tangled.LabelDefinition{}, + org_tangled.LabelDefs_LabelToken{}, + org_tangled.MarkupEmbedImage{}, + org_tangled.MarkupEmbedVideo{}, + org_tangled.MarkupMarkdown{}, + org_tangled.MarkupMarkdown_Inline{}, + org_tangled.RepoArtifact{}, + org_tangled.RepoDeclaration{}, + org_tangled.RepoDefs_SourceExternal{}, + org_tangled.RepoDefs_SourceTangled{}, + org_tangled.RepoManifest{}, + org_tangled.RepoManifest_Declaration{}, + org_tangled.RepoReviewUnspeccedRound{}, + org_tangled.ReviewPatch{}, + org_tangled.ReviewPatch_Source{}, + org_tangled.SnippetString{}, + org_tangled.SnippetString_TextFile{}, + org_tangled.TrackTicket{}, + org_tangled.TrackUnspeccedMilestone{}, + ); err != nil { + panic(err) + } +} diff --git a/cmd/cilogs/main.go b/cmd/cilogs/main.go new file mode 100644 index 0000000000000000000000000000000000000000..84da01af8d3114c9ec12a58774d29ac541100403 --- /dev/null +++ b/cmd/cilogs/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + "tangled.org/core/cilogs" + "tangled.org/core/idresolver" + "tangled.org/core/log" +) + +func envOr(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + logger := log.New("cilogs") + cfg := cilogs.Config{ + ListenAddr: envOr("CILOGS_LISTEN_ADDR", "0.0.0.0:3333"), + HostKeyPath: os.Getenv("CILOGS_HOST_KEY_PATH"), + } + dev := os.Getenv("CILOGS_DEV") == "true" + dir := idresolver.DefaultResolver(envOr("CILOGS_PLC_URL", "https://plc.directory")).Directory() + + if err := cilogs.New(cfg, cilogs.NewSpindles(dir, dev), logger).ListenAndServe(ctx); err != nil { + fmt.Fprintf(os.Stderr, "cilogs failed: %v\n", err) + os.Exit(1) + } +} diff --git a/cmd/claimer/emit/emit.go b/cmd/claimer/emit/emit.go new file mode 100644 index 0000000000000000000000000000000000000000..55b8ada38d2b6f4431ccefbb6cf505cd9aab591b --- /dev/null +++ b/cmd/claimer/emit/emit.go @@ -0,0 +1,38 @@ +package emit + +import ( + "strconv" + "strings" +) + +func Esc(v string) string { + return "'" + strings.ReplaceAll(v, "'", "''") + "'" +} + +type Cell struct { + V string + Nil bool + Num bool + Int int64 +} + +func Insert(b *strings.Builder, head string, rows [][]Cell) { + for _, row := range rows { + b.WriteString(head) + b.WriteString("(") + for j, cell := range row { + if j > 0 { + b.WriteString(", ") + } + if cell.Nil { + b.WriteString("NULL") + } else if cell.Num { + b.WriteString(strconv.FormatInt(cell.Int, 10)) + } else { + b.WriteString(Esc(cell.V)) + } + } + b.WriteString(");\n") + } + b.WriteString("\n") +} diff --git a/cmd/claimer/main.go b/cmd/claimer/main.go new file mode 100644 index 0000000000000000000000000000000000000000..77a2690266014bcb820b4b9027c8a74224eae7cb --- /dev/null +++ b/cmd/claimer/main.go @@ -0,0 +1,364 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log/slog" + "net/http" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + + "tangled.org/core/cmd/claimer/emit" + tlog "tangled.org/core/log" +) + +var ( + outFile = flag.String("out", "/tmp/claimer-claims.sql", "where to write the emitted claim SQL") + pdsHost = flag.String("pds", "https://tngl.sh", "PDS host URL") + dryRun = flag.Bool("dry-run", false, "print what would be done without emitting SQL") + verbose = flag.Bool("v", false, "log pagination progress") + apply = flag.Bool("apply", false, "execute the emitted SQL against D1 via wrangler") + yes = flag.Bool("yes", false, "required together with -apply as a destructive-action gate") + local = flag.Bool("local", false, "apply against the local D1 instead of remote") + d1Name = flag.String("d1", "tangled-sites", "D1 database name for wrangler d1 execute") +) + +type repo struct { + DID string `json:"did"` + Head string `json:"head"` +} + +type listReposResponse struct { + Cursor string `json:"cursor"` + Repos []repo `json:"repos"` +} + +type describeRepoResponse struct { + Handle string `json:"handle"` + DID string `json:"did"` +} + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: claimer [flags]\n\n") + fmt.Fprintf(os.Stderr, "Backfills domain_claims for all existing users on the tngl.sh PDS.\n") + fmt.Fprintf(os.Stderr, "DIDs are fetched via com.atproto.sync.listRepos, handles are resolved\n") + fmt.Fprintf(os.Stderr, "via com.atproto.repo.describeRepo (no DNS verification required).\n") + fmt.Fprintf(os.Stderr, "Claims are emitted as literal INSERT OR IGNORE SQL for D1.\n\n") + fmt.Fprintf(os.Stderr, "Flags:\n") + flag.PrintDefaults() + } + flag.Parse() + + ctx := context.Background() + logger := tlog.New("claimer") + ctx = tlog.IntoContext(ctx, logger) + + pdsDomain, err := domainFromURL(*pdsHost) + if err != nil { + logger.Error("invalid pds host", "host", *pdsHost, "error", err) + os.Exit(1) + } + + logger.Info("starting claimer", + "pds", *pdsHost, + "pds_domain", pdsDomain, + "out", *outFile, + "dry_run", *dryRun, + ) + + dids, err := fetchAllDIDs(ctx, *pdsHost, logger) + if err != nil { + logger.Error("failed to fetch repos from PDS", "error", err) + os.Exit(1) + } + + logger.Info("fetched repos", "count", len(dids)) + + suffix := "." + pdsDomain + + var ( + rows [][]emit.Cell + skipped int + errCount int + ) + var b strings.Builder + + for _, did := range dids { + handle, err := describeRepo(ctx, *pdsHost, did) + if err != nil { + logger.Error("failed to describe repo", "did", did, "error", err) + errCount++ + continue + } + + if !strings.HasSuffix(handle, suffix) { + logger.Info("skipping account: handle does not match pds domain", + "did", did, "handle", handle, "suffix", suffix) + skipped++ + continue + } + if !claimableHandle(handle, pdsDomain) { + logger.Warn("skipping account: handle fails the sites-domain label rules", + "did", did, "handle", handle) + skipped++ + continue + } + if !strings.HasSuffix(handle, ".tngl.sh") { + logger.Warn("skipping account: only tngl.sh handle domains are claimed by the backfill", + "did", did, "handle", handle) + skipped++ + continue + } + + if *dryRun { + logger.Info("[dry-run] would claim domain", "did", did, "domain", handle) + continue + } + + if !handleIsPermanent(handle) { + logger.Error("internal invariant broken: non-permanent handle reached emission", + "did", did, "handle", handle) + continue + } + writeClaimSQL(&b, did, handle) + rows = append(rows, []emit.Cell{{V: did}, {V: handle}, {Nil: true}}) + logger.Info("would claim domain", "did", did, "domain", handle) + } + + if *dryRun { + logger.Info("dry run finished", "dids", len(dids), "skipped", skipped, "errors", errCount) + if errCount > 0 { + os.Exit(1) + } + return + } + + emit.Insert(&b, "INSERT OR IGNORE INTO domain_claims (did, domain, deleted) VALUES ", rows) + absOut, err := filepath.Abs(*outFile) + if err != nil { + logger.Error("resolving output path", "path", *outFile, "error", err) + os.Exit(1) + } + if err := os.WriteFile(absOut, []byte(b.String()), 0o644); err != nil { + logger.Error("writing output", "path", absOut, "error", err) + os.Exit(1) + } + + if *apply && !*yes { + logger.Error("-apply is destructive for the target D1 tables; pass -yes to confirm") + os.Exit(1) + } + + if *apply && len(rows) > 0 { + args := []string{"wrangler", "d1", "execute", *d1Name, "--file", absOut} + if !*local { + args = append(args, "--remote") + } + cmd := exec.CommandContext(ctx, "npx", args...) + cmd.Dir = sitesDir() + out, err := cmd.CombinedOutput() + if err != nil { + logger.Error("applying claims to D1 failed", + "error", err, + "output", strings.TrimSpace(string(out)), + ) + os.Exit(1) + } + logger.Info("applied claims to D1", "out", absOut, "d1", *d1Name, "local", *local) + } + + logger.Info("claimer finished", + slog.Group("results", + "claims", len(rows), + "skipped", skipped, + "errors", errCount, + ), + "out", *outFile, + ) + + if errCount > 0 { + os.Exit(1) + } +} + +func writeClaimSQL(b *strings.Builder, did, handle string) { + cutoff := "strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-30 days')" + b.WriteString("UPDATE domain_claims SET deleted = NULL WHERE domain = ") + b.WriteString(emit.Esc(handle)) + b.WriteString(" AND did = ") + b.WriteString(emit.Esc(did)) + b.WriteString(" AND deleted IS NOT NULL;\n") + + b.WriteString("DELETE FROM domain_claims WHERE domain = ") + b.WriteString(emit.Esc(handle)) + b.WriteString(" AND did != ") + b.WriteString(emit.Esc(did)) + b.WriteString(" AND deleted IS NOT NULL AND deleted < ") + b.WriteString(cutoff) + b.WriteString(";\n") + + b.WriteString("UPDATE domain_claims SET domain = ") + b.WriteString(emit.Esc(handle)) + b.WriteString(", deleted = NULL WHERE did = ") + b.WriteString(emit.Esc(did)) + b.WriteString(" AND domain != ") + b.WriteString(emit.Esc(handle)) + b.WriteString(" AND deleted IS NOT NULL AND deleted < ") + b.WriteString(cutoff) + b.WriteString(";\n") +} + +func sitesDir() string { + wd, err := os.Getwd() + if err != nil { + return "sites" + } + for dir := wd; ; dir = path.Dir(dir) { + if _, err := os.Stat(path.Join(dir, "sites", "wrangler.toml")); err == nil { + return path.Join(dir, "sites") + } + if dir == path.Dir(dir) || len(dir) < len(wd)-len("core")-2 { + break + } + } + return "sites" +} + +func handleIsPermanent(handle string) bool { + return strings.HasSuffix(handle, ".tngl.sh") +} + +func claimableHandle(handle, pdsDomain string) bool { + label := strings.TrimSuffix(handle, "."+pdsDomain) + if label == "" || len(label) > 63 || strings.Contains(label, ".") { + return false + } + bytes := []byte(label) + if bytes[0] == '-' || bytes[len(bytes)-1] == '-' { + return false + } + for _, c := range bytes { + if !((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-') { + return false + } + } + return true +} + +func fetchAllDIDs(ctx context.Context, pdsHost string, logger *slog.Logger) ([]string, error) { + var all []string + cursor := "" + + for { + batch, nextCursor, err := listRepos(ctx, pdsHost, cursor) + if err != nil { + return nil, err + } + + for _, r := range batch { + all = append(all, r.DID) + } + + if *verbose { + logger.Info("fetched page", "count", len(batch), "total", len(all), "next_cursor", nextCursor) + } + + if nextCursor == "" || nextCursor == cursor { + break + } + cursor = nextCursor + } + + return all, nil +} + +func listRepos(ctx context.Context, pdsHost, cursor string) ([]repo, string, error) { + params := url.Values{} + params.Set("limit", "100") + if cursor != "" { + params.Set("cursor", cursor) + } + + endpoint := pdsHost + "/xrpc/com.atproto.sync.listRepos?" + params.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, "", fmt.Errorf("building listRepos request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, "", fmt.Errorf("executing listRepos request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + var errBody struct { + Error string `json:"error"` + Message string `json:"message"` + } + json.NewDecoder(resp.Body).Decode(&errBody) + return nil, "", fmt.Errorf("PDS returned %d: %s — %s", resp.StatusCode, errBody.Error, errBody.Message) + } + + var result listReposResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, "", fmt.Errorf("decoding listRepos response: %w", err) + } + + return result.Repos, result.Cursor, nil +} + +func describeRepo(ctx context.Context, pdsHost, did string) (string, error) { + params := url.Values{} + params.Set("repo", did) + + endpoint := pdsHost + "/xrpc/com.atproto.repo.describeRepo?" + params.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return "", fmt.Errorf("building describeRepo request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("executing describeRepo request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + var errBody struct { + Error string `json:"error"` + Message string `json:"message"` + } + json.NewDecoder(resp.Body).Decode(&errBody) + return "", fmt.Errorf("PDS returned %d: %s — %s", resp.StatusCode, errBody.Error, errBody.Message) + } + + var result describeRepoResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", fmt.Errorf("decoding describeRepo response: %w", err) + } + + return result.Handle, nil +} + +func domainFromURL(rawURL string) (string, error) { + if !strings.Contains(rawURL, "://") { + return rawURL, nil + } + u, err := url.Parse(rawURL) + if err != nil { + return "", err + } + host := u.Hostname() + if host == "" { + return "", fmt.Errorf("no hostname in URL %q", rawURL) + } + return host, nil +} diff --git a/cmd/claimer/main_test.go b/cmd/claimer/main_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d5610037c558ff98d548b98279f0a3bceb3c1893 --- /dev/null +++ b/cmd/claimer/main_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "database/sql" + "strings" + "testing" + + _ "github.com/mattn/go-sqlite3" +) + +func TestClaimSQLFreesExpiredDomainBeforeRename(t *testing.T) { + var b strings.Builder + writeClaimSQL(&b, "did:plc:y", "bob.tngl.sh") + statements := strings.Split(strings.TrimSuffix(b.String(), ";\n"), ";\n") + if len(statements) != 3 { + t.Fatalf("claim operations = %d, want undelete, delete, rename", len(statements)) + } + setup := func() *sql.DB { + t.Helper() + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + _, err = db.Exec(`CREATE TABLE domain_claims ( + did TEXT NOT NULL UNIQUE, + domain TEXT NOT NULL UNIQUE, + deleted TEXT + ); + INSERT INTO domain_claims (did, domain, deleted) VALUES + ('did:plc:z', 'bob.tngl.sh', strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-40 days')), + ('did:plc:y', 'bob.tngl.io', strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-40 days'));`) + if err != nil { + t.Fatal(err) + } + return db + } + oldOrder := strings.Join([]string{statements[0], statements[2], statements[1]}, ";\n") + ";" + if _, err := setup().Exec(oldOrder); err == nil || !strings.Contains(err.Error(), "UNIQUE constraint failed") { + t.Fatalf("old rename-before-delete error = %v, want UNIQUE violation", err) + } + db := setup() + if _, err := db.Exec(b.String()); err != nil { + t.Fatalf("claim SQL after freeing the domain: %v", err) + } + var owner string + if err := db.QueryRow("SELECT did FROM domain_claims WHERE domain = 'bob.tngl.sh' AND deleted IS NULL").Scan(&owner); err != nil { + t.Fatal(err) + } + if owner != "did:plc:y" { + t.Fatalf("domain owner = %q, want did:plc:y", owner) + } +} diff --git a/cmd/deliberi/main.go b/cmd/deliberi/main.go new file mode 100644 index 0000000000000000000000000000000000000000..64f74b79990f71e5949aa4b8992560acc485aa87 --- /dev/null +++ b/cmd/deliberi/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/carlmjohnson/versioninfo" + "github.com/urfave/cli/v3" + "tangled.org/core/deliberi" + "tangled.org/core/deliberi/config" + "tangled.org/core/log" +) + +func main() { + if err := run(os.Args); err != nil { + slog.Error("error running deliberi", "err", err) + os.Exit(-1) + } +} + +func run(args []string) error { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + logger := log.New("deliberi") + slog.SetDefault(logger) + ctx = log.IntoContext(ctx, logger) + + app := cli.Command{ + Name: "deliberi", + Usage: "tngl.sh notification + email service", + Version: versioninfo.Short(), + } + app.Commands = []*cli.Command{ + { + Name: "serve", + Usage: "run the deliberi daemon", + Action: runDeliberi, + }, + } + return app.Run(ctx, args) +} + +func runDeliberi(ctx context.Context, cmd *cli.Command) error { + cfg, err := config.Load(ctx) + if err != nil { + return err + } + return deliberi.Run(ctx, cfg) +} diff --git a/cmd/delicurl.sh b/cmd/delicurl.sh new file mode 100755 index 0000000000000000000000000000000000000000..e492d3a52ac69a136c1269e1f76b78cab4184bd4 --- /dev/null +++ b/cmd/delicurl.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# delicurl — curl through service auth for deliberi endpoints. +# +# Authenticates against a PDS with an app password, obtains a service +# auth JWT, and proxies the request to the target URL. +# +# Usage: +# export DELICURL_USER=alice.tngl.sh +# export DELICURL_PASSWORD="xxxx-xxxx-xxxx-xxxx" +# +# # GET (no --data, no -X) +# ./cmd/delicurl.sh \ +# http://localhost:6565/xrpc/org.tangled.temp.notification.getUnreadCount +# +# # POST with body +# ./cmd/delicurl.sh --data '{"read":true}' \ +# http://localhost:6565/xrpc/org.tangled.temp.notification.updateSeen +# +# # POST without body (e.g. markAllRead) +# ./cmd/delicurl.sh -X POST \ +# http://localhost:6565/xrpc/org.tangled.temp.notification.markAllRead + +set -euo pipefail + +PDS="${DELICURL_PDS:-https://pds.tngl.boltless.dev}" +USER="${DELICURL_USER:-}" +PASSWORD="${DELICURL_PASSWORD:-}" +AUD="${DELICURL_AUD:-did:web:deliberi.tngl.boltless.dev}" +LXM="" +DATA="" +HTTP_METHOD="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --method|-m) LXM="$2"; shift 2 ;; + --data|-d) DATA="$2"; shift 2 ;; + -X) HTTP_METHOD="$2"; shift 2 ;; + --pds) PDS="$2"; shift 2 ;; + --aud) AUD="$2"; shift 2 ;; + *) break ;; + esac +done + +URL="${1:-}" +if [[ -z "$URL" ]]; then + echo "Usage: $0 [-X METHOD] [--method lxm] [--data '{}'] [--pds url] [--aud did:web:...] " + echo "" + echo "Environment: DELICURL_USER, DELICURL_PASSWORD (required)" + exit 1 +fi + +# Auto-derive lexicon method from URL when not explicitly set. +# Strip query/fragment, then take the last path segment. +URL_CLEAN="${URL%%\?*}" +URL_CLEAN="${URL_CLEAN%%\#*}" +URL_CLEAN="${URL_CLEAN%%/}" +if [[ -z "$LXM" ]]; then + LXM="${URL_CLEAN##*/}" +fi + +# Derive HTTP method if not explicitly set. +if [[ -z "$HTTP_METHOD" ]]; then + if [[ -n "$DATA" ]]; then + HTTP_METHOD="POST" + else + HTTP_METHOD="GET" + fi +fi + +if [[ -z "$USER" || -z "$PASSWORD" ]]; then + echo "error: DELICURL_USER and DELICURL_PASSWORD must be set" >&2 + exit 1 +fi + +# Step 1: create session → access JWT (POST, JSON body) +SESSION=$(curl -sS -X POST "$PDS/xrpc/com.atproto.server.createSession" \ + -H "Content-Type: application/json" \ + -d '{"identifier":"'"$USER"'","password":"'"$PASSWORD"'"}') +ACCESS_JWT=$(echo "$SESSION" | python3 -c "import sys,json; print(json.load(sys.stdin)['accessJwt'])") + +# Step 2: get service auth token (GET query — aud, exp, lxm are URL params) +EXP=$(( $(date +%s) + 300 )) # 5 min +SA_TOKEN=$(curl -sS -G "$PDS/xrpc/com.atproto.server.getServiceAuth" \ + -H "Authorization: Bearer $ACCESS_JWT" \ + --data-urlencode "aud=$AUD" \ + --data-urlencode "exp=$EXP" \ + --data-urlencode "lxm=$LXM" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['token'])") + +# Step 3: make the request +CURL_ARGS=(-sS -w "\n--- status: %{http_code} ---" -X "$HTTP_METHOD" -H "Authorization: Bearer $SA_TOKEN") +if [[ -n "$DATA" ]]; then + CURL_ARGS+=(-H "Content-Type: application/json" -d "$DATA") +fi +curl "${CURL_ARGS[@]}" "$URL" diff --git a/cmd/dolly/main.go b/cmd/dolly/main.go new file mode 100644 index 0000000000000000000000000000000000000000..aa3b76a78f456da3f6babf0c5bd99118a9ccb633 --- /dev/null +++ b/cmd/dolly/main.go @@ -0,0 +1,259 @@ +package main + +import ( + "bytes" + _ "embed" + "flag" + "fmt" + "image" + "image/color" + "image/png" + "math" + "os" + "path/filepath" + "strconv" + "strings" + "text/template" + + "github.com/srwiley/oksvg" + "github.com/srwiley/rasterx" + "golang.org/x/image/draw" + "tangled.org/core/ico" +) + +func main() { + var ( + size string + fillColor string + output string + templatePath string + kind string + favicon bool + ) + + flag.StringVar(&templatePath, "template", "", "Path to a dolly go-html template file, or a directory of templates") + flag.StringVar(&size, "size", "512", "Output size as WIDTH (height derived from aspect ratio, e.g., 512) or WIDTHxHEIGHT (e.g., 512x512)") + flag.StringVar(&fillColor, "color", "#000000", "Fill color in hex format (e.g., #FF5733)") + flag.StringVar(&output, "output", "dolly.svg", "Output file path (format detected from extension: .svg, .png, or .ico)") + flag.StringVar(&kind, "kind", "logo", "Asset to generate: logo (dolly only) or logotype (dolly + wordmark)") + flag.BoolVar(&favicon, "favicon", false, "Embed a prefers-color-scheme style block so the SVG reacts to dark mode (SVG output only)") + flag.Parse() + + if templatePath == "" { + fmt.Fprintf(os.Stderr, "Empty template path") + os.Exit(1) + } + + if kind != "logo" && kind != "logotype" { + fmt.Fprintf(os.Stderr, "Invalid kind: %s. Must be logo or logotype\n", kind) + os.Exit(1) + } + + width, height, err := parseSize(size) + if err != nil { + fmt.Fprintf(os.Stderr, "Error parsing size: %v\n", err) + os.Exit(1) + } + + // Detect format from file extension + ext := strings.ToLower(filepath.Ext(output)) + format := strings.TrimPrefix(ext, ".") + + if format != "svg" && format != "png" && format != "ico" { + fmt.Fprintf(os.Stderr, "Invalid file extension: %s. Must be .svg, .png, or .ico\n", ext) + os.Exit(1) + } + + if fillColor != "currentColor" && !isValidHexColor(fillColor) { + fmt.Fprintf(os.Stderr, "Invalid color format: %s. Use hex format like #FF5733\n", fillColor) + os.Exit(1) + } + + tpl, err := loadTemplates(templatePath) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to load templates from path %s: %v\n", templatePath, err) + os.Exit(1) + } + + if favicon && format != "svg" { + fmt.Fprintf(os.Stderr, "-favicon is only supported for .svg output\n") + os.Exit(1) + } + + svgData, err := dolly(tpl, "fragments/dolly/"+kind, fillColor, favicon) + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating SVG: %v\n", err) + os.Exit(1) + } + + // Derive height from the SVG's aspect ratio when only a width was given + if height == 0 && format != "svg" { + height, err = deriveHeight(svgData, width) + if err != nil { + fmt.Fprintf(os.Stderr, "Error deriving height: %v\n", err) + os.Exit(1) + } + } + + // Create output directory if it doesn't exist + dir := filepath.Dir(output) + if dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0755); err != nil { + fmt.Fprintf(os.Stderr, "Error creating output directory: %v\n", err) + os.Exit(1) + } + } + + switch format { + case "svg": + err = saveSVG(svgData, output, width, height) + case "png": + err = savePNG(svgData, output, width, height) + case "ico": + err = saveICO(svgData, output, width, height) + } + + if err != nil { + fmt.Fprintf(os.Stderr, "Error saving file: %v\n", err) + os.Exit(1) + } + + if format == "svg" { + // size is irrelevant for svg output; it scales to its viewBox + fmt.Printf("Successfully generated %s\n", output) + } else { + fmt.Printf("Successfully generated %s (%dx%d)\n", output, width, height) + } +} + +func loadTemplates(path string) (*template.Template, error) { + info, err := os.Stat(path) + if err != nil { + return nil, err + } + + if info.IsDir() { + return template.ParseGlob(filepath.Join(path, "*.html")) + } + + return template.ParseFiles(path) +} + +func dolly(tpl *template.Template, name, hexColor string, favicon bool) ([]byte, error) { + var svgData bytes.Buffer + if err := tpl.ExecuteTemplate(&svgData, name, map[string]any{ + "FillColor": hexColor, + "Classes": "", + "Favicon": favicon, + }); err != nil { + return nil, err + } + + return svgData.Bytes(), nil +} + +func svgToImage(svgData []byte, w, h int) (image.Image, error) { + icon, err := oksvg.ReadIconStream(bytes.NewReader(svgData)) + if err != nil { + return nil, fmt.Errorf("error parsing SVG: %v", err) + } + + icon.SetTarget(0, 0, float64(w), float64(h)) + rgba := image.NewRGBA(image.Rect(0, 0, w, h)) + draw.Draw(rgba, rgba.Bounds(), &image.Uniform{color.Transparent}, image.Point{}, draw.Src) + scanner := rasterx.NewScannerGV(w, h, rgba, rgba.Bounds()) + raster := rasterx.NewDasher(w, h, scanner) + icon.Draw(raster, 1.0) + + return rgba, nil +} + +// parseSize parses WIDTH or WIDTHxHEIGHT. A height of 0 means "derive +// from the SVG's aspect ratio". +func parseSize(size string) (int, int, error) { + if !strings.Contains(size, "x") { + width, err := strconv.Atoi(size) + if err != nil { + return 0, 0, fmt.Errorf("invalid width: %v", err) + } + if width <= 0 { + return 0, 0, fmt.Errorf("width must be positive") + } + return width, 0, nil + } + + parts := strings.Split(size, "x") + if len(parts) != 2 { + return 0, 0, fmt.Errorf("invalid size format, use WIDTH or WIDTHxHEIGHT") + } + + width, err := strconv.Atoi(parts[0]) + if err != nil { + return 0, 0, fmt.Errorf("invalid width: %v", err) + } + + height, err := strconv.Atoi(parts[1]) + if err != nil { + return 0, 0, fmt.Errorf("invalid height: %v", err) + } + + if width <= 0 || height <= 0 { + return 0, 0, fmt.Errorf("width and height must be positive") + } + + return width, height, nil +} + +func deriveHeight(svgData []byte, width int) (int, error) { + icon, err := oksvg.ReadIconStream(bytes.NewReader(svgData)) + if err != nil { + return 0, fmt.Errorf("error parsing SVG: %v", err) + } + + if icon.ViewBox.W <= 0 || icon.ViewBox.H <= 0 { + return 0, fmt.Errorf("SVG has an invalid viewBox (%gx%g)", icon.ViewBox.W, icon.ViewBox.H) + } + + return int(math.Round(float64(width) * icon.ViewBox.H / icon.ViewBox.W)), nil +} + +func isValidHexColor(hex string) bool { + if len(hex) != 7 || hex[0] != '#' { + return false + } + _, err := strconv.ParseUint(hex[1:], 16, 32) + return err == nil +} + +func saveSVG(svgData []byte, filepath string, _, _ int) error { + return os.WriteFile(filepath, svgData, 0644) +} + +func savePNG(svgData []byte, filepath string, width, height int) error { + img, err := svgToImage(svgData, width, height) + if err != nil { + return err + } + + f, err := os.Create(filepath) + if err != nil { + return err + } + defer f.Close() + + return png.Encode(f, img) +} + +func saveICO(svgData []byte, filepath string, width, height int) error { + img, err := svgToImage(svgData, width, height) + if err != nil { + return err + } + + icoData, err := ico.ImageToIco(img) + if err != nil { + return err + } + + return os.WriteFile(filepath, icoData, 0644) +} diff --git a/cmd/dolly/templates/logo.html b/cmd/dolly/templates/logo.html new file mode 100644 index 0000000000000000000000000000000000000000..1683e312d881718a5623ce04b0d6dbf1343150e3 --- /dev/null +++ b/cmd/dolly/templates/logo.html @@ -0,0 +1,22 @@ +{{ define "fragments/dolly/logo" }} + + {{ if .Favicon }} + + {{ end }} + + +{{ end }} diff --git a/cmd/dolly/templates/logotype.html b/cmd/dolly/templates/logotype.html new file mode 100644 index 0000000000000000000000000000000000000000..7e924b169ff072e67dd400a7d4f0e4bf6ca56e44 --- /dev/null +++ b/cmd/dolly/templates/logotype.html @@ -0,0 +1,25 @@ +{{ define "fragments/dolly/logotype" }} + + {{ if .Favicon }} + + {{ end }} + + + + + + + + + +{{ end }} diff --git a/cmd/lexpostgen/main.go b/cmd/lexpostgen/main.go new file mode 100644 index 0000000000000000000000000000000000000000..6ce64ad01eff16555e6c1ecba01426082a99fefe --- /dev/null +++ b/cmd/lexpostgen/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "bytes" + "fmt" + "go/format" + "os" +) + +const pipelinePath = "api/org_tangled/cidefs.go" + +func replaceOnce(source, old, new []byte) []byte { + if bytes.Count(source, old) != 1 { + panic(fmt.Sprintf("expected one generated match, got %d", bytes.Count(source, old))) + } + return bytes.Replace(source, old, new, 1) +} + +func main() { + source, err := os.ReadFile(pipelinePath) + if err != nil { + panic(err) + } + start := []byte("type CiDefs_Pipeline_Trigger struct {\n") + if bytes.Count(source, start) != 1 { + panic("expected one CiDefs_Pipeline_Trigger struct") + } + // patch only the trigger union and its methods; other unions in the file stay as generated + at := bytes.Index(source, start) + stop := len(source) + if next := bytes.Index(source[at+len(start):], []byte("\ntype ")); next >= 0 { + stop = at + len(start) + next + } + head, tail := source[:at], source[stop:] + source = source[at:stop:stop] + end := bytes.Index(source, []byte("\n}")) + 1 + source = append(source[:end:end], append([]byte("\tUnknown json.RawMessage\n"), source[end:]...)...) + source = replaceOnce(source, + []byte("\treturn nil, fmt.Errorf(\"cannot marshal empty enum\")"), + []byte("\tif t.Unknown != nil {\n\t\treturn t.Unknown, nil\n\t}\n\treturn nil, fmt.Errorf(\"cannot marshal empty enum\")"), + ) + source = replaceOnce(source, + []byte("\ttyp, err := util.TypeExtract(b)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tswitch typ {"), + []byte("\ttyp, err := util.TypeExtract(b)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t*t = CiDefs_Pipeline_Trigger{}\n\tswitch typ {"), + ) + source = replaceOnce(source, + []byte("\tdefault:\n\t\treturn nil\n\t}\n}"), + []byte("\tdefault:\n\t\tt.Unknown = append(t.Unknown[:0], b...)\n\t\treturn nil\n\t}\n}"), + ) + source = append(append(append([]byte{}, head...), source...), tail...) + if source, err = format.Source(source); err != nil { + panic(err) + } + if err := os.WriteFile(pipelinePath, source, 0o644); err != nil { + panic(err) + } +} diff --git a/cmd/migrator/main.go b/cmd/migrator/main.go new file mode 100644 index 0000000000000000000000000000000000000000..1baa72fdd0d5d7ce72ae4035db4dc0ad610a4dcf --- /dev/null +++ b/cmd/migrator/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/carlmjohnson/versioninfo" + "github.com/urfave/cli/v3" + "tangled.org/core/log" + "tangled.org/core/migrator" + "tangled.org/core/migrator/config" + "tangled.org/core/migrator/git" +) + +func main() { + if err := run(os.Args); err != nil { + slog.Error("error running migrator", "err", err) + os.Exit(-1) + } +} + +func run(args []string) error { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + logger := log.New("migrator") + slog.SetDefault(logger) + ctx = log.IntoContext(ctx, logger) + + app := cli.Command{ + Name: "migrator", + Usage: "tangled centralized durable GitHub repository importer", + Version: versioninfo.Short(), + } + app.Commands = []*cli.Command{ + { + Name: "serve", + Usage: "run the migrator daemon", + Action: runMigrator, + }, + { + Name: "askpass", + Usage: "internal git askpass helper", + Action: runAskpass, + }, + } + return app.Run(ctx, args) +} + +func runMigrator(ctx context.Context, cmd *cli.Command) error { + cfg, err := config.Load(ctx) + if err != nil { + return err + } + return migrator.Run(ctx, cfg) +} + +func runAskpass(ctx context.Context, cmd *cli.Command) error { + args := cmd.Args().Slice() + if len(args) == 0 { + return fmt.Errorf("no prompt provided to askpass") + } + prompt := args[0] + + token := os.Getenv("MIGRATOR_ASKPASS_TOKEN") + expectedHost := os.Getenv("MIGRATOR_ASKPASS_HOST") + + resp, err := git.HandleAskpass(prompt, expectedHost, token) + if err != nil { + return err + } + + fmt.Println(resp) + return nil +} diff --git a/cmd/prefill-zoekt/main.go b/cmd/prefill-zoekt/main.go new file mode 100644 index 0000000000000000000000000000000000000000..ae8e3304d9f35137caec716a8f2157b8386b5e0c --- /dev/null +++ b/cmd/prefill-zoekt/main.go @@ -0,0 +1,174 @@ +// prefill-zoekt bulk-enqueues zoekt index tasks +// +// It reads a REPOS file with one repo DID per line: +// +// did:plc:repository +// +// For each DID it resolves the knot from the DID document, then resolves the +// remote HEAD branch + commit via `git ls-remote --symref`, and POSTs an +// enqueue request to the indexserver's /admin/enqueueIndex endpoint. +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "log" + "net/http" + "net/url" + "os" + "os/exec" + "strings" + "sync" + "sync/atomic" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/samber/lo" + "github.com/sourcegraph/zoekt" + "tangled.org/core/gitutil" + "tangled.org/core/repoident" +) + +func main() { + reposPath := flag.String("repos", "REPOS", "path to repos list file (one DID per line)") + serverUrl := flag.String("server", "http://localhost:6060", "indexserver base url") + plc := flag.String("plc", "https://plc.directory", "atproto PLC directory url") + concurrency := flag.Int("concurrency", 5, "number of repos to process in parallel") + allowHttp := flag.Bool("allow-http", false, "accept repo DIDs whose knot service endpoint is plaintext http, and skip TLS verification when reading HEAD") + flag.Parse() + + server, err := url.Parse(*serverUrl) + if err != nil { + log.Fatalf("parsing -server %q: %v", *serverUrl, err) + } + if (server.Scheme != "http" && server.Scheme != "https") || server.Host == "" { + log.Fatalf("-server %q must be an http or https URL with a host", *serverUrl) + } + + data, err := os.ReadFile(*reposPath) + if err != nil { + log.Fatalf("reading %s: %v", *reposPath, err) + } + + ctx := context.Background() + dir := identity.BaseDirectory{PLCURL: *plc} + + var ok, fail atomic.Int64 + var wg sync.WaitGroup + sem := make(chan struct{}, *concurrency) + + lo.ForEach(strings.Split(string(data), "\n"), func(line string, i int) { + raw := strings.TrimSpace(line) + if raw == "" { + return + } + + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + + head, knot, err := prefillRepo(ctx, &dir, server, raw, *allowHttp) + if err != nil { + log.Printf("line %d: %s: %v", i+1, raw, err) + fail.Add(1) + return + } + log.Printf("line %d: %s: enqueued %s@%s (knot=%s)", i+1, raw, head.Name, head.Version, knot) + ok.Add(1) + }() + }) + + wg.Wait() + fmt.Printf("done: %d enqueued, %d failed\n", ok.Load(), fail.Load()) +} + +func prefillRepo(ctx context.Context, dir identity.Directory, server *url.URL, raw string, allowHTTP bool) (zoekt.RepositoryBranch, repoident.KnotURL, error) { + var knot repoident.KnotURL + + repoDid, err := repoident.NewRepoDid(raw) + if err != nil { + return zoekt.RepositoryBranch{}, knot, err + } + + ident, err := dir.LookupDID(ctx, syntax.DID(repoDid)) + if err != nil { + return zoekt.RepositoryBranch{}, knot, fmt.Errorf("resolving repo DID: %w", err) + } + + knot, err = repoident.KnotURLFromIdentity(ident, repoident.SchemeFor(allowHTTP)) + if err != nil { + return zoekt.RepositoryBranch{}, knot, fmt.Errorf("resolving knot: %w", err) + } + + head, err := resolveHead(knot, repoDid, allowHTTP) + if err != nil { + return head, knot, fmt.Errorf("resolving HEAD: %w", err) + } + + if err := enqueue(server, repoDid, head); err != nil { + return head, knot, fmt.Errorf("enqueue: %w", err) + } + return head, knot, nil +} + +func resolveHead(knot repoident.KnotURL, repoDid repoident.RepoDid, allowHTTP bool) (zoekt.RepositoryBranch, error) { + remote := knot.JoinPath(repoDid.String()) + args := append( + lo.Ternary(allowHTTP, []string{"-c", "http.sslVerify=false"}, nil), + "ls-remote", "--symref", remote, "HEAD", + ) + out, err := gitutil.Contain(exec.Command("git", args...)).Output() + if err != nil { + return zoekt.RepositoryBranch{}, fmt.Errorf("git ls-remote --symref %s HEAD: %w", remote, err) + } + head := lo.Reduce( + strings.Split(string(out), "\n"), + func(head zoekt.RepositoryBranch, line string, _ int) zoekt.RepositoryBranch { + fields := strings.Fields(line) + if len(fields) < 2 { + return head + } + switch { + case fields[0] == "ref:": + head.Name = strings.TrimPrefix(fields[1], "refs/heads/") + case fields[1] == "HEAD": + head.Version = fields[0] + } + return head + }, + zoekt.RepositoryBranch{}, + ) + if head.Name == "" || head.Version == "" { + return zoekt.RepositoryBranch{}, fmt.Errorf("couldn't resolve HEAD (branch=%q sha=%q)", head.Name, head.Version) + } + return head, nil +} + +func enqueue(server *url.URL, repoDid repoident.RepoDid, head zoekt.RepositoryBranch) error { + body, err := json.Marshal(map[string]any{ + "repo": repoDid.String(), + "branches": []zoekt.RepositoryBranch{head}, + }) + if err != nil { + return err + } + + resp, err := http.Post(server.JoinPath("admin", "enqueueIndex").String(), + "application/json", bytes.NewReader(body)) + if err != nil { + return err + } + defer resp.Body.Close() + + log.Println("status", resp.StatusCode) + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("status %d", resp.StatusCode) + } + return nil +} diff --git a/cmd/sitesd/main.go b/cmd/sitesd/main.go new file mode 100644 index 0000000000000000000000000000000000000000..d6f0bc1d234d8a710af61bb786e4e40ab08f34d0 --- /dev/null +++ b/cmd/sitesd/main.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + + "tangled.org/core/log" + "tangled.org/core/sitesd" +) + +func envOr(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func envBool(key string) bool { + v := os.Getenv(key) + return v == "1" || v == "true" +} + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + logger := log.New("sitesd") + ctx = log.IntoContext(ctx, logger) + + opts := sitesd.Options{ + Listen: envOr("SITESD_LISTEN", ":8080"), + InternalListen: envOr("SITESD_INTERNAL_ADDR", ""), + WorkerURL: envOr("SITESD_WORKER_URL", "https://sites.tangled.org"), + Audience: envOr("SITESD_AUDIENCE", "did:web:sites.tangled.org"), + Issuer: envOr("SITESD_ISSUER", "did:web:sitesd.tangled.network"), + KeyHex: envOr("SITESD_PRIVATE_KEY", ""), + Knots: os.Getenv("SITESD_KNOTS"), + Jetstream: os.Getenv("SITESD_JETSTREAM_ENDPOINT"), + RedisAddr: os.Getenv("SITESD_REDIS"), + Dev: envBool("SITESD_DEV"), + } + + if err := sitesd.Serve(ctx, logger, opts); err != nil { + slog.Error("sitesd failed", "err", err) + fmt.Fprintf(os.Stderr, "sitesd failed: %v\n", err) + os.Exit(1) + } +} diff --git a/cmd/spindle-microvm-run/main.go b/cmd/spindle-microvm-run/main.go new file mode 100644 index 0000000000000000000000000000000000000000..5ab4181b61da34ac20d092c4cf8fcf50668250ec --- /dev/null +++ b/cmd/spindle-microvm-run/main.go @@ -0,0 +1,26 @@ +//go:build linux + +package main + +import ( + "context" + "log/slog" + "os" + + tlog "tangled.org/core/log" +) + +func main() { + cmd := SpindleMicroVMRunCommand() + + logger := tlog.New("spindle-microvm-run") + slog.SetDefault(logger) + + ctx := context.Background() + ctx = tlog.IntoContext(ctx, logger) + + if err := cmd.Run(ctx, os.Args); err != nil { + logger.Error(err.Error()) + os.Exit(-1) + } +} diff --git a/cmd/spindle-microvm-run/main_linux.go b/cmd/spindle-microvm-run/main_linux.go new file mode 100644 index 0000000000000000000000000000000000000000..abbb2a01a8d217f020d57cc54e2682c49782328b --- /dev/null +++ b/cmd/spindle-microvm-run/main_linux.go @@ -0,0 +1,381 @@ +//go:build linux + +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "net" + "os" + "path/filepath" + "time" + + "github.com/mdlayher/vsock" + "github.com/urfave/cli/v3" + agentv1 "tangled.org/core/spindle/agentproto/gen" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engines/microvm" +) + +func SpindleMicroVMRunCommand() *cli.Command { + return &cli.Command{ + Name: "spindle-microvm-run", + Usage: "launch the Spindle base microVM and run one command over vsock", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "image-spec", + Sources: cli.EnvVars("SPINDLE_MICROVM_IMAGE_SPEC"), + Usage: "path to microVM image spec JSON", + }, + &cli.StringFlag{ + Name: "mkfs-ext4", + Usage: "override mkfs.ext4 binary", + }, + &cli.StringFlag{ + Name: "work-dir", + Usage: "directory for per-run disks and sockets", + }, + &cli.UintFlag{ + Name: "cid", + Usage: "guest vsock CID; defaults to a random high CID", + }, + &cli.UintFlag{ + Name: "port", + Value: 10240, + Usage: "host vsock port to listen on", + }, + &cli.UintFlag{ + Name: "memory-mib", + Usage: "override the guest memory size in MiB (defaults to the image spec)", + }, + &cli.BoolFlag{ + Name: "disable-kvm", + Usage: "run without -enable-kvm even if /dev/kvm is available", + }, + &cli.BoolFlag{ + Name: "dev", + Usage: "enable dev mode (allows host network access, disables SSL verification)", + }, + &cli.DurationFlag{ + Name: "qmp-timeout", + Value: 10 * time.Second, + Usage: "how long to wait for qmp to become ready", + }, + &cli.DurationFlag{ + Name: "accept-timeout", + Value: 15 * time.Second, + Usage: "how long to wait for the guest agent after qemu starts", + }, + &cli.DurationFlag{ + Name: "exec-timeout", + Value: 30 * time.Second, + Usage: "timeout for the guest command", + }, + &cli.DurationFlag{ + Name: "cache-upload-wait-timeout", + Aliases: []string{"cache-drain-timeout"}, + Value: 5 * time.Minute, + Usage: "how long to wait for guest cache uploads to finish after the command exits", + }, + &cli.DurationFlag{ + Name: "shutdown-timeout", + Value: 10 * time.Second, + Usage: "how long to wait for qemu to exit after guest powerdown", + }, + &cli.StringFlag{ + Name: "cwd", + Usage: "guest working directory", + }, + &cli.StringSliceFlag{ + Name: "cache-read-url", + Sources: cli.EnvVars("SPINDLE_NIX_CACHE_READ_URLS"), + Usage: "Nix binary cache URL to pass to the guest; repeatable", + }, + &cli.StringSliceFlag{ + Name: "cache-trusted-public-key", + Sources: cli.EnvVars("SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS"), + Usage: "Nix binary cache public key to trust in the guest; repeatable", + }, + &cli.StringFlag{ + Name: "cache-upload-url", + Sources: cli.EnvVars("SPINDLE_NIX_CACHE_UPLOAD_URL"), + Usage: "optional cache upload URL for guest-built store paths", + }, + &cli.BoolFlag{ + Name: "cache-require-signed-uploads", + Sources: cli.EnvVars("SPINDLE_NIX_CACHE_REQUIRE_SIGNED_UPLOADS"), + Usage: "require guest cache uploads to carry a trusted narinfo signature", + }, + &cli.StringFlag{ + Name: "activate-config", + Usage: "JSON user config to activate before exec (e.g. '{\"services\":{\"openssh\":{\"enable\":true}}}')", + }, + &cli.StringFlag{ + Name: "db", + Usage: "path to sqlite database for config cache", + }, + }, + Action: runMicroVMRunDev, + } +} + +func runMicroVMRunDev(ctx context.Context, cmd *cli.Command) error { + imageSpecPath := cmd.String("image-spec") + if imageSpecPath == "" { + return fmt.Errorf("--image-spec or SPINDLE_MICROVM_IMAGE_SPEC is required") + } + + imageSpec, err := microvm.LoadImageSpec(imageSpecPath) + if err != nil { + return err + } + + port := uint32(cmd.Uint("port")) + // tell the guest which host vsock port to dial back on. shuttle reads the + // cmdline params this is so we can run multiple of this process + // concurrently, because otherwise it listens on a specific vsock port, and + // we cant bind to the same port twice... + imageSpec.BootArgs = fmt.Sprintf("%s shuttle.vsock_port=%d", imageSpec.BootArgs, port) + if mib := cmd.Uint("memory-mib"); mib > 0 { + imageSpec.MemoryMiB = int(mib) + } + ln, err := vsock.ListenContextID(vsock.Host, port, nil) + if err != nil { + return fmt.Errorf("listen on vsock port %d: %w", port, err) + } + defer ln.Close() + + vm, err := microvm.StartVM(ctx, microvm.VMConfig{ + Image: imageSpec, + BootTimeout: cmd.Duration("qmp-timeout"), + CID: uint32(cmd.Uint("cid")), + EnableKVM: !cmd.Bool("disable-kvm"), + MkfsExt4: cmd.String("mkfs-ext4"), + WorkDir: cmd.String("work-dir"), + Dev: cmd.Bool("dev"), + }, slog.Default()) + if err != nil { + return err + } + defer vm.Close() + + logs := vm.Logs() + fmt.Fprintf(os.Stderr, "microvm started: cid=%d work-dir=%s serial-log=%s qemu-log=%s\n", + vm.CID(), + vm.WorkDir(), + logs.Serial, + logs.Extra["qemu"], + ) + + logger := slog.Default() + + if cmd.Duration("accept-timeout") > 0 { + if err := ln.SetDeadline(time.Now().Add(cmd.Duration("accept-timeout"))); err != nil { + return fmt.Errorf("set accept deadline: %w", err) + } + } + + argv := cmd.Args().Slice() + if len(argv) == 0 { + argv = []string{"/run/current-system/sw/bin/echo", "hello-from-spindle"} + } + jobID := "spindle-microvm-run" + execID := "dev-1" + var pendingConfigKey string + var pendingConfigToplevel string + var configCacheDB *db.DB + + fmt.Fprintf(os.Stderr, "listening for agent on %s\n", ln.Addr()) + conn, err := acceptExpectedVsockConn(ln, vm.CID(), logger) + if err != nil { + return fmt.Errorf("accept agent connection: %w", err) + } + defer conn.Close() + + upstreams, err := microvm.BuildSubstituterUpstreams(cmd.StringSlice("cache-read-url"), nil) + if err != nil { + return fmt.Errorf("build cache upstreams: %w", err) + } + + var readCache *microvm.SubstituterProxy + if len(cmd.StringSlice("cache-read-url")) > 0 { + var err error + readCache, err = microvm.StartSubstituterProxy(ctx, vm.CID(), upstreams, logger) + if err != nil { + return fmt.Errorf("start read cache proxy: %w", err) + } + defer readCache.Close() + } + + var uploadCache *microvm.SubstituterUploadProxy + if cmd.String("cache-upload-url") != "" { + var err error + uploadCache, err = microvm.StartSubstituterUploadProxy(ctx, vm.CID(), cmd.String("cache-upload-url"), upstreams, filepath.Join(vm.WorkDir(), "upload-cache"), logger, nil, "", "", microvm.UploadCacheOptions{ + TrustedPublicKeys: cmd.StringSlice("cache-trusted-public-key"), + RequireSignedUploads: cmd.Bool("cache-require-signed-uploads"), + }) + if err != nil { + return fmt.Errorf("start upload cache proxy: %w", err) + } + defer uploadCache.Close() + } + dnsProxy, err := microvm.StartDNSProxy(ctx, vm.CID(), logger) + if err != nil { + return fmt.Errorf("start dns proxy: %w", err) + } + defer dnsProxy.Close() + + session := microvm.NewAgentSession(conn, vm.CID(), logger) + + initCtx, cancelInit := context.WithTimeout(ctx, 30*time.Second) + defer cancelInit() + if err := session.Init(initCtx, &agentv1.Init{ + JobId: jobID, + CacheTrustedPublicKeys: cmd.StringSlice("cache-trusted-public-key"), + CacheReadProxyPort: readCache.Port(), + CacheUploadProxyPort: uploadCache.Port(), + DnsProxyPort: dnsProxy.Port(), + }); err != nil { + return fmt.Errorf("init agent: %w", err) + } + + execCtx := ctx + if cmd.Duration("exec-timeout") > 0 { + var cancel context.CancelFunc + execCtx, cancel = context.WithTimeout(ctx, cmd.Duration("exec-timeout")) + defer cancel() + } + + if cmd.String("activate-config") != "" { + actCtx := execCtx + baseHash, err := microvm.BaseConfigHash(imageSpec) + if err != nil { + return fmt.Errorf("calculate base config hash: %w", err) + } + + var configKey string + var cachedToplevel string + if cmd.String("db") != "" { + configCacheDB, err = db.Make(ctx, cmd.String("db")) + if err != nil { + return fmt.Errorf("failed to open database: %w", err) + } + defer configCacheDB.Close() + + configKey, err = microvm.BuildConfigKey(imageSpec, cmd.String("activate-config"), "local") + if err != nil { + return fmt.Errorf("calculate config key: %w", err) + } + + record, err := configCacheDB.GetNixOSToplevelCacheRecord(configKey) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("lookup config cache: %w", err) + } + } else { + cachedToplevel = record.Toplevel + fmt.Printf("realizing cached NixOS config %s\n", cachedToplevel) + } + } + + result, err := session.ActivateConfig(actCtx, "dev-activate", &agentv1.ActivateConfig{ + ConfigKey: configKey, + BaseConfigHash: baseHash, + UserConfig: cmd.String("activate-config"), + Toplevel: cachedToplevel, + }, os.Stderr) + if err != nil { + return fmt.Errorf("activate config: %w", err) + } + fmt.Fprintf(os.Stderr, "activated config toplevel: %s\n", result.Toplevel) + + if configCacheDB != nil && cachedToplevel == "" && result.Toplevel != "" && configKey != "" { + if uploadCache == nil { + fmt.Fprintln(os.Stderr, "skipping config cache metadata commit: no cache upload url configured") + } else { + pendingConfigKey = configKey + pendingConfigToplevel = result.Toplevel + } + } + } + + exitCode, err := session.Exec(execCtx, microvm.AgentExec{ + ID: execID, + ExecStart: &agentv1.ExecStart{ + Argv: argv, + Cwd: cmd.String("cwd"), + }, + Stdout: os.Stdout, + Stderr: os.Stderr, + }) + if err != nil { + if detail := microvm.VMCrashLog(vm); detail != "" { + if parsedErr, ok := microvm.ParseCrashLog(detail); ok { + return fmt.Errorf("%w: %w", parsedErr, err) + } + } + return err + } + + if uploadCache != nil { + uploadWaitCtx := ctx + if cmd.Duration("cache-upload-wait-timeout") > 0 { + var cancel context.CancelFunc + uploadWaitCtx, cancel = context.WithTimeout(ctx, cmd.Duration("cache-upload-wait-timeout")) + defer cancel() + } + uploaded, err := session.Drain(uploadWaitCtx) + if err != nil { + return err + } + fmt.Printf("cache uploaded: %d\n", uploaded) + if configCacheDB != nil && pendingConfigKey != "" && pendingConfigToplevel != "" { + if err := configCacheDB.SaveNixOSToplevelCacheRecord(pendingConfigKey, pendingConfigToplevel); err != nil { + return fmt.Errorf("save config cache: %w", err) + } + } + } + + // mirror the engine shutdown order: ask the agent to power off first, + // then fall back to qemu powerdown / kill + shutdownCtx, cancel := context.WithTimeout(context.Background(), cmd.Duration("shutdown-timeout")) + defer cancel() + poweredOff := false + if err := session.Poweroff(shutdownCtx); err != nil { + fmt.Fprintf(os.Stderr, "agent poweroff: %s\n", err) + } else if err := vm.WaitContext(shutdownCtx); err == nil { + poweredOff = true + } + if !poweredOff { + if err := vm.Shutdown(shutdownCtx); err != nil { + fmt.Fprintf(os.Stderr, "microvm shutdown fallback: %s\n", err) + } + } + + if exitCode != 0 { + return fmt.Errorf("guest command exited with code %d", exitCode) + } + return nil +} + +func acceptExpectedVsockConn(ln *vsock.Listener, allowedCID uint32, logger *slog.Logger) (net.Conn, error) { + for { + conn, err := ln.Accept() + if err != nil { + return nil, err + } + if allowedCID == 0 { + return conn, nil + } + addr, ok := conn.RemoteAddr().(*vsock.Addr) + if ok && addr.ContextID == allowedCID { + return conn, nil + } + remote := conn.RemoteAddr() + _ = conn.Close() + logger.Warn("dropped agent connection from unexpected cid", "remote", remote, "expected", allowedCID) + } +} diff --git a/cmd/spindle-microvm-run/unsupported.go b/cmd/spindle-microvm-run/unsupported.go new file mode 100644 index 0000000000000000000000000000000000000000..cca69a7d16baa1fb9ee39ae6107f8efdcf6f5a93 --- /dev/null +++ b/cmd/spindle-microvm-run/unsupported.go @@ -0,0 +1,13 @@ +//go:build !linux + +package main + +import ( + "fmt" + "os" +) + +func main() { + fmt.Fprintf(os.Stderr, "spindle-microvm-run is only supported on Linux\n") + os.Exit(-1) +} diff --git a/cmd/spindle/ban_cli_test.go b/cmd/spindle/ban_cli_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6b174eb4bd9499eec0ffb86646077441a7d74273 --- /dev/null +++ b/cmd/spindle/ban_cli_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" + + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" +) + +// xrpc refuses a banned owner, which leaves the cli as the only way back in +func TestCLIUnbansABannedOwnerStraightOnTheDB(t *testing.T) { + owner := "did:plc:theowner" + t.Setenv("SPINDLE_SERVER_OWNER", owner) + dbPath := filepath.Join(t.TempDir(), "spindle.db") + + if out, code := runCLI(t, dbPath, "ban", owner); code != 0 { + t.Fatalf("ban = %d: %s", code, out) + } + if out, code := runCLI(t, dbPath, "bans"); code != 0 || !strings.Contains(out, owner) { + t.Fatalf("bans = %d: %q, want the owner listed", code, out) + } + + d, err := db.Make(t.Context(), dbPath) + if err != nil { + t.Fatal(err) + } + srv := &config.Server{Owner: owner} + enrolled, err := db.Enrolls(d, srv, "did:plc:theowner") + d.Close() + if err != nil || enrolled { + t.Fatalf("banned owner enrolls = %v, %v, want a ban to beat being the owner", enrolled, err) + } + + if out, code := runCLI(t, dbPath, "unban", owner); code != 0 { + t.Fatalf("unban = %d: %s", code, out) + } + if out, code := runCLI(t, dbPath, "bans"); code != 0 || strings.Contains(out, owner) { + t.Fatalf("bans after unban = %d: %q, want the owner gone", code, out) + } + d, err = db.Make(t.Context(), dbPath) + if err != nil { + t.Fatal(err) + } + defer d.Close() + enrolled, err = db.Enrolls(d, srv, "did:plc:theowner") + if err != nil || !enrolled { + t.Fatalf("unbanned owner enrolls = %v, %v, want the owner served again", enrolled, err) + } + + if out, code := runCLI(t, dbPath, "unban", owner); code == 0 || !strings.Contains(out, "is not banned") { + t.Fatalf("unbanning someone not banned = %d: %s, want an error", code, out) + } +} diff --git a/cmd/spindle/main.go b/cmd/spindle/main.go new file mode 100644 index 0000000000000000000000000000000000000000..34836c4686e0479091bea1fb3870d7e73a94d0b9 --- /dev/null +++ b/cmd/spindle/main.go @@ -0,0 +1,736 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "log/slog" + "math" + "os" + "os/signal" + "strings" + "syscall" + "text/tabwriter" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/urfave/cli/v3" + + "tangled.org/core/idresolver" + tlog "tangled.org/core/log" + "tangled.org/core/spindle" + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/mill" + "tangled.org/core/spindle/quota" +) + +func main() { + cmd := &cli.Command{ + Name: "spindle", + Usage: "spindle continuous integration runner", + Commands: []*cli.Command{ + Command(), + millCommand(), + executorCommand(), + quotaCommand(), + banCommand(), + unbanCommand(), + bansCommand(), + allowListCommand(), + inviteCommand(), + }, + DefaultCommand: "run", + } + + format := os.Getenv("SPINDLE_LOG_FORMAT") + logger, err := tlog.NewWithFormat("spindle", format) + if err != nil { + fmt.Fprintf(os.Stderr, "invalid log format: %v\n", err) + os.Exit(1) + } + slog.SetDefault(logger) + + ctx := tlog.IntoContext(context.Background(), logger) + ctx, stopSignals := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + defer stopSignals() + + if err := cmd.Run(ctx, os.Args); err != nil { + logger.Error(err.Error()) + os.Exit(-1) + } +} + +func Command() *cli.Command { + return &cli.Command{ + Name: "run", + Usage: "run the spindle server", + Action: func(ctx context.Context, cmd *cli.Command) error { + return spindle.Run(ctx) + }, + } +} + +var dbFlag = &cli.StringFlag{ + Name: "db", + Usage: "path to the spindle sqlite db", + Value: "spindle.db", + Sources: cli.EnvVars("SPINDLE_SERVER_DB_PATH"), +} + +func openDB(ctx context.Context, cmd *cli.Command) (*db.DB, error) { + return db.Make(ctx, cmd.String("db")) +} + +// lets executor operators reset stream state +func executorCommand() *cli.Command { + return &cli.Command{ + Name: "executor", + Usage: "executor host administration", + Commands: []*cli.Command{ + { + Name: "reset-stream", + Usage: "wipe the outbox and start a fresh stream epoch, for when the mill has lost this executor's acked position; restart the executor service afterwards", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + return err + } + epoch := hex.EncodeToString(b[:]) + if err := d.SetOutboxEpoch(epoch); err != nil { + return fmt.Errorf("resetting outbox: %w", err) + } + // clear artifacts waiting on leases from the old + // stream, replaying them under a new epoch would + // trip the mill's lease-epoch check + if err := d.ClearPendingArtifacts(); err != nil { + return fmt.Errorf("clearing pending artifacts: %w", err) + } + fmt.Printf("outbox wiped; new stream epoch %s. restart the executor to reconnect cleanly\n", epoch) + return nil + }, + }, + }, + } +} + +// lets mill operators manage executors +func millCommand() *cli.Command { + return &cli.Command{ + Name: "mill", + Usage: "mill host administration", + Commands: []*cli.Command{ + { + Name: "executor", + Usage: "manage executors allowed to join this mill", + Commands: []*cli.Command{ + { + Name: "token", + Usage: "manage executor credentials", + Commands: []*cli.Command{ + { + Name: "generate", + Usage: "generate and store an executor token", + Flags: []cli.Flag{ + dbFlag, + &cli.DurationFlag{ + Name: "ttl", + Usage: "token lifetime (e.g. 720h); omit for no expiry", + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + token, err := mill.GenerateToken() + if err != nil { + return err + } + var expiresAt *time.Time + if ttl := cmd.Duration("ttl"); ttl > 0 { + exp := time.Now().UTC().Add(ttl) + expiresAt = &exp + } + if err := d.CreateExecutorToken(mill.HashToken(token), expiresAt); err != nil { + return fmt.Errorf("storing executor token: %w", err) + } + fmt.Println(token) + return nil + }, + }, + }, + }, + { + Name: "add", + Usage: "register an executor with an existing token", + ArgsUsage: "", + Flags: []cli.Flag{ + dbFlag, + &cli.StringFlag{ + Name: "token-file", + Usage: "file containing a token created by executor token generate", + }, + &cli.StringSliceFlag{ + Name: "label", + Usage: "authorized labels for this executor", + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + name := cmd.Args().First() + if name == "" { + return fmt.Errorf("usage: spindle mill executor add --token-file ") + } + tokenFile := cmd.String("token-file") + if tokenFile == "" { + return fmt.Errorf("--token-file is required") + } + tokenBytes, err := os.ReadFile(tokenFile) + if err != nil { + return fmt.Errorf("reading token file: %w", err) + } + token := strings.TrimSpace(string(tokenBytes)) + if token == "" { + return fmt.Errorf("token file is empty") + } + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + labels := cmd.StringSlice("label") + if err := d.RegisterExecutor(name, mill.HashToken(token), labels); err != nil { + return fmt.Errorf("registering executor %q: %w", name, err) + } + return nil + }, + }, + { + Name: "list", + Usage: "list registered executors", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + registrations, err := d.ListExecutorRegistrations() + if err != nil { + return err + } + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NAME\tCREATED\tEXPIRES\tLABELS") + for _, registration := range registrations { + expires := "never" + if registration.ExpiresAt != nil { + expires = registration.ExpiresAt.Format(time.RFC3339) + if time.Now().After(*registration.ExpiresAt) { + expires += " (expired)" + } + } + labels := strings.Join(registration.Labels, ",") + if labels == "" { + labels = "-" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", registration.Name, registration.CreatedAt, expires, labels) + } + return w.Flush() + }, + }, + { + Name: "reset-cursor", + Usage: "reset the mill's acked stream position for an executor after mill state loss; takes effect on the executor's next reconnect", + ArgsUsage: "", + Flags: []cli.Flag{ + dbFlag, + &cli.UintFlag{ + Name: "to", + Usage: "skip forward to this acked seqno instead of forgetting everything; " + + "events at or below it are treated as applied (accepts their loss)", + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + name := cmd.Args().First() + if name == "" { + return fmt.Errorf("usage: spindle mill executor reset-cursor [--to ]") + } + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + if cmd.IsSet("to") { + n, err := d.SetExecutorCursors(name, uint64(cmd.Uint("to"))) + if err != nil { + return err + } + if n == 0 { + return fmt.Errorf("no cursor rows for executor %q", name) + } + fmt.Printf("skipped %s forward to seqno %d (%d epoch rows); earlier events are lost\n", name, cmd.Uint("to"), n) + return nil + } + n, err := d.DeleteExecutorCursors(name) + if err != nil { + return err + } + fmt.Printf("forgot %d cursor row(s) for %s; the mill now expects its stream from seqno 1\n", n, name) + return nil + }, + }, + { + Name: "revoke", + Usage: "revoke an executor's token", + ArgsUsage: "", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + name := cmd.Args().First() + if name == "" { + return fmt.Errorf("usage: spindle mill executor revoke ") + } + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + ok, err := d.RevokeExecutorToken(name) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("no such executor identity %q", name) + } + return nil + }, + }, + }, + }, + }, + } +} + +func openQuotaStore(ctx context.Context, cmd *cli.Command) (*db.QuotaStore, error) { + d, err := openDB(ctx, cmd) + if err != nil { + return nil, err + } + return db.NewQuotaStore(d, quota.Defaults{}), nil +} + +type quotaKey struct { + did string + resource string +} + +func quotaCommand() *cli.Command { + return &cli.Command{ + Name: "quota", + Usage: "manage resource quotas for repos and users", + Commands: []*cli.Command{ + { + Name: "usage", + Usage: "show usage against limits for every repo and user", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Len() != 0 { + return fmt.Errorf("unexpected arguments") + } + qs, err := openQuotaStore(ctx, cmd) + if err != nil { + return err + } + defer qs.Close() + + defaults, err := config.LoadQuotaDefaults(ctx) + if err != nil { + return fmt.Errorf("invalid quota configuration: %w", err) + } + + limits, err := qs.ListLimits(ctx) + if err != nil { + return fmt.Errorf("listing limits: %w", err) + } + overrides := make(map[quotaKey]int64, len(limits)) + for _, l := range limits { + overrides[quotaKey{did: l.DID, resource: l.Resource}] = l.Limit + } + + usages, err := qs.ListUsage(ctx) + if err != nil { + return fmt.Errorf("listing usage: %w", err) + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "SCOPE\tDID\tRESOURCE\tUSED\tLIMIT") + + for _, u := range usages { + var limitVal int64 = -1 + if ovLimit, ok := overrides[quotaKey{did: u.DID, resource: u.Resource}]; ok { + limitVal = ovLimit + } else if scopeDefs, ok := defaults[u.Scope]; ok { + if defLimit, ok := scopeDefs[u.Resource]; ok { + limitVal = normalizeDefaultLimit(defLimit) + } + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", string(u.Scope), u.DID, string(u.Resource), formatLimit(u.Resource, u.Used), formatLimit(u.Resource, limitVal)) + } + return w.Flush() + }, + }, + { + Name: "list", + Usage: "list all custom limits", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Len() != 0 { + return fmt.Errorf("unexpected arguments") + } + qs, err := openQuotaStore(ctx, cmd) + if err != nil { + return err + } + defer qs.Close() + limits, err := qs.ListLimits(ctx) + if err != nil { + return fmt.Errorf("listing limits: %w", err) + } + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "DID\tRESOURCE\tLIMIT") + for _, l := range limits { + fmt.Fprintf(w, "%s\t%s\t%s\n", l.DID, string(l.Resource), formatLimit(l.Resource, l.Limit)) + } + return w.Flush() + }, + }, + { + Name: "set", + ArgsUsage: "", + Usage: "give a repo or user a custom limit", + Flags: []cli.Flag{ + dbFlag, + &cli.StringFlag{ + Name: "resource", + Usage: "resource to limit (e.g. workflows, cache_storage_bytes)", + Required: true, + }, + &cli.IntFlag{ + Name: "limit", + Usage: "new limit (MiB for cache_storage_bytes and every *_mib resource)", + }, + &cli.BoolFlag{ + Name: "unlimited", + Usage: "remove the limit entirely", + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + did, err := subjectArg(ctx, cmd) + if err != nil { + return err + } + + resource := cmd.String("resource") + + if err := quota.ValidateOverride(did.String(), resource); err != nil { + return err + } + + hasLimit := cmd.IsSet("limit") + hasUnlimited := cmd.Bool("unlimited") + + if !hasLimit && !hasUnlimited { + return fmt.Errorf("must specify either --limit or --unlimited") + } + if hasLimit && hasUnlimited { + return fmt.Errorf("cannot specify both --limit and --unlimited") + } + + var limitVal int64 = -1 + if hasLimit { + limitInt := cmd.Int("limit") + if limitInt < 0 { + return fmt.Errorf("limit cannot be negative: %d", limitInt) + } + if limitInt == 0 { + return fmt.Errorf("--limit must be at least 1. use --unlimited to remove a limit, or spindle ban to block a repo or user") + } + + if resource == quota.ResourceCacheStorageBytes { + if limitInt > math.MaxInt64/1048576 { + return fmt.Errorf("limit %d MiB would overflow bytes", limitInt) + } + limitVal = int64(limitInt) * 1048576 + } else { + limitVal = int64(limitInt) + } + } + + qs, err := openQuotaStore(ctx, cmd) + if err != nil { + return err + } + defer qs.Close() + if err := qs.SetLimit(ctx, did.String(), resource, limitVal); err != nil { + return fmt.Errorf("setting limit: %w", err) + } + fmt.Printf("quota override set for %s %s\n", did.String(), resource) + return nil + }, + }, + { + Name: "unset", + ArgsUsage: "", + Usage: "remove a custom limit and go back to the defaults", + Flags: []cli.Flag{ + dbFlag, + &cli.StringFlag{ + Name: "resource", + Usage: "resource to limit (e.g. workflows, cache_storage_bytes)", + Required: true, + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + did, err := subjectArg(ctx, cmd) + if err != nil { + return err + } + + resource := cmd.String("resource") + + if err := quota.ValidateOverride(did.String(), resource); err != nil { + return err + } + + qs, err := openQuotaStore(ctx, cmd) + if err != nil { + return err + } + defer qs.Close() + if err := qs.UnsetLimit(ctx, did.String(), resource); err != nil { + return fmt.Errorf("unsetting limit: %w", err) + } + fmt.Printf("quota override removed for %s %s\n", did.String(), resource) + return nil + }, + }, + }, + } +} + +func normalizeDefaultLimit(limit int64) int64 { + if limit == 0 { + return -1 + } + return limit +} + +func formatLimit(resource string, limit int64) string { + if limit < 0 { + return "unlimited" + } + switch resource { + case quota.ResourceCacheStorageBytes: + if limit%(1024*1024) == 0 { + return fmt.Sprintf("%d MiB", limit/(1024*1024)) + } + return fmt.Sprintf("%d B", limit) + case quota.ResourceMemoryMiB, quota.ResourceWorkflowMemoryMiB, quota.ResourceDiskMiB, quota.ResourceWorkflowDiskMiB: + return fmt.Sprintf("%d MiB", limit) + case quota.ResourceWorkflows: + return fmt.Sprintf("%d workflows", limit) + case quota.ResourceVCPUs, quota.ResourceWorkflowVCPUs: + return fmt.Sprintf("%d vCPUs", limit) + case quota.ResourceWebhooks: + return fmt.Sprintf("%d webhooks", limit) + case quota.ResourceCIMinutesMonthly: + return fmt.Sprintf("%d minutes/month", limit) + default: + return fmt.Sprintf("%d", limit) + } +} + +func banCommand() *cli.Command { + return &cli.Command{ + Name: "ban", + ArgsUsage: "", + Usage: "ban a repo or user and wipe their existing data on next startup", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + did, err := subjectArg(ctx, cmd) + if err != nil { + return err + } + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + defer d.Close() + return d.PutBan(db.BanEntry{SubjectDid: did}) + }, + } +} + +func unbanCommand() *cli.Command { + return &cli.Command{ + Name: "unban", + ArgsUsage: "", + Usage: "unban a repo or user", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + did, err := subjectArg(ctx, cmd) + if err != nil { + return err + } + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + defer d.Close() + removed, err := d.DeleteBan(did) + if err != nil { + return err + } + if !removed { + return fmt.Errorf("%s is not banned", did) + } + return nil + }, + } +} + +func bansCommand() *cli.Command { + return &cli.Command{ + Name: "bans", + Usage: "list all bans", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + defer d.Close() + bans, err := d.BanList() + if err != nil { + return err + } + w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(w, "DID\tSINCE") + for _, b := range bans { + fmt.Fprintf(w, "%s\t%s\n", b.SubjectDid, b.CreatedAt) + } + return w.Flush() + }, + } +} + +// writes the db directly, so a running spindle admits the change at once but +// only backfills the person's existing repos on its next start +func allowListCommand() *cli.Command { + return &cli.Command{ + Name: "allowlist", + Usage: "manage who this spindle runs ci for while admission is closed", + Commands: []*cli.Command{ + { + Name: "add", + ArgsUsage: "", + Usage: "let a user run ci on this spindle", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + return withDid(ctx, cmd, func(d *db.DB, did syntax.DID) error { + return db.AddToAllowlist(d, did) + }) + }, + }, + { + Name: "remove", + ArgsUsage: "", + Usage: "take a user off the allow-list", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + return withDid(ctx, cmd, func(d *db.DB, did syntax.DID) error { + return db.RemoveFromAllowlist(d, did) + }) + }, + }, + { + Name: "list", + Usage: "list the allow-list", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + defer d.Close() + dids, err := db.ListAllowlist(d) + if err != nil { + return err + } + for _, did := range dids { + fmt.Println(did) + } + return nil + }, + }, + }, + } +} + +func inviteCommand() *cli.Command { + return &cli.Command{ + Name: "invite", + Usage: "mint a single use invite code that puts whoever redeems it on the allow-list", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + defer d.Close() + code, expires, err := d.MintInvite("cli", time.Now()) + if err != nil { + return err + } + fmt.Printf("%s (expires %s)\n", code, expires.UTC().Format(time.RFC3339)) + return nil + }, + } +} + +func withDid(ctx context.Context, cmd *cli.Command, fn func(d *db.DB, did syntax.DID) error) error { + did, err := subjectArg(ctx, cmd) + if err != nil { + return err + } + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + defer d.Close() + return fn(d, did) +} + +func subjectArg(ctx context.Context, cmd *cli.Command) (syntax.DID, error) { + if cmd.Args().Len() != 1 { + return "", fmt.Errorf("expected ") + } + return resolveSubject(ctx, cmd.Args().Get(0)) +} + +// a did is taken as is, so banning a repo did never needs the network +func resolveSubject(ctx context.Context, arg string) (syntax.DID, error) { + if did, err := syntax.ParseDID(arg); err == nil { + return did, nil + } + plc := os.Getenv("SPINDLE_SERVER_PLC_URL") + if plc == "" { + plc = "https://plc.directory" + } + ident, err := idresolver.DefaultResolver(plc).ResolveAtIdentifier(ctx, arg) + if err != nil { + return "", err + } + return ident.DID, nil +} diff --git a/cmd/spindle/main_test.go b/cmd/spindle/main_test.go new file mode 100644 index 0000000000000000000000000000000000000000..bb5eab95249ba6e81ed30582e107c502cba58550 --- /dev/null +++ b/cmd/spindle/main_test.go @@ -0,0 +1,310 @@ +package main + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/quota" +) + +func TestCLIInvalidLogFormat(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=TestHelperProcess") + cmd.Env = append(os.Environ(), "SPINDLE_LOG_FORMAT=invalid", "WANT_HELPER_PROCESS=1") + var stderr bytes.Buffer + cmd.Stderr = &stderr + err := cmd.Run() + + if err == nil { + t.Fatalf("expected command to fail, but it succeeded") + } + + exitErr, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("expected exit error, got %v", err) + } + if code := exitErr.ExitCode(); code != 1 { + t.Errorf("expected exit code 1, got %d", code) + } + + output := stderr.String() + if !strings.Contains(output, "invalid log format") { + t.Errorf("expected error message to contain 'invalid log format', got: %q", output) + } +} + +func TestHelperProcess(t *testing.T) { + if os.Getenv("WANT_HELPER_PROCESS") != "1" { + return + } + for i, arg := range os.Args { + if arg == "--" { + os.Args = append([]string{os.Args[0]}, os.Args[i+1:]...) + main() + return + } + } + main() +} + +func TestCLIQuotaCommands(t *testing.T) { + tmpDb := filepath.Join(t.TempDir(), "spindle.db") + + tests := []struct { + name string + args []string + wantExitCode int + wantError string + wantOutput string + }{ + { + name: "invalid DID", + args: []string{"quota", "set", "alice", "--resource", "cache_storage_bytes", "--limit", "10"}, + wantExitCode: 255, + wantError: "not a did or handle", + }, + { + name: "invalid resource", + args: []string{"quota", "set", "did:web:alice", "--resource", strings.Repeat("a", 65), "--limit", "10"}, + wantExitCode: 255, + wantError: "invalid resource", + }, + { + name: "conflicting flags limit and unlimited", + args: []string{"quota", "set", "did:web:alice", "--resource", "cache_storage_bytes", "--limit", "10", "--unlimited"}, + wantExitCode: 255, + wantError: "cannot specify both --limit and --unlimited", + }, + { + name: "missing flags limit and unlimited", + args: []string{"quota", "set", "did:web:alice", "--resource", "cache_storage_bytes"}, + wantExitCode: 255, + wantError: "must specify either --limit or --unlimited", + }, + { + name: "zero limit", + args: []string{"quota", "set", "did:web:alice", "--resource", "workflows", "--limit", "0"}, + wantExitCode: 255, + wantError: "--unlimited", + }, + { + name: "cache storage limit overflow", + args: []string{"quota", "set", "did:web:alice", "--resource", "cache_storage_bytes", "--limit", "9223372036854775807"}, + wantExitCode: 255, + wantError: "would overflow bytes", + }, + { + name: "set user cache_storage_bytes limit", + args: []string{"quota", "set", "did:web:alice", "--resource", "cache_storage_bytes", "--limit", "10"}, + wantExitCode: 0, + wantOutput: "quota override set", + }, + { + name: "set user workflows limit", + args: []string{"quota", "set", "did:web:alice", "--resource", "workflows", "--limit", "5"}, + wantExitCode: 0, + wantOutput: "quota override set", + }, + { + name: "set user vcpus limit", + args: []string{"quota", "set", "did:web:alice", "--resource", "vcpus", "--limit", "4"}, + wantExitCode: 0, + wantOutput: "quota override set", + }, + { + name: "set user memory_mib limit", + args: []string{"quota", "set", "did:web:alice", "--resource", "memory_mib", "--limit", "2048"}, + wantExitCode: 0, + wantOutput: "quota override set", + }, + { + name: "set user disk_mib limit", + args: []string{"quota", "set", "did:web:alice", "--resource", "disk_mib", "--limit", "4096"}, + wantExitCode: 0, + wantOutput: "quota override set", + }, + { + name: "set user cache_storage_bytes unlimited", + args: []string{"quota", "set", "did:web:alice", "--resource", "cache_storage_bytes", "--unlimited"}, + wantExitCode: 0, + wantOutput: "quota override set", + }, + { + name: "list quota limits", + args: []string{"quota", "list"}, + wantExitCode: 0, + wantOutput: "LIMIT", + }, + { + name: "usage list", + args: []string{"quota", "usage"}, + wantExitCode: 0, + wantOutput: "LIMIT", + }, + { + name: "unset user cache_storage_bytes limit", + args: []string{"quota", "unset", "did:web:alice", "--resource", "cache_storage_bytes"}, + wantExitCode: 0, + wantOutput: "quota override removed", + }, + { + name: "unset user workflows limit", + args: []string{"quota", "unset", "did:web:alice", "--resource", "workflows"}, + wantExitCode: 0, + wantOutput: "quota override removed", + }, + { + name: "set user generic resource limit", + args: []string{"quota", "set", "did:web:alice", "--resource", "custom_gpu", "--limit", "8"}, + wantExitCode: 0, + wantOutput: "quota override set", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fullArgs := append([]string{"-test.run=TestHelperProcess", "--"}, tt.args...) + fullArgs = append(fullArgs, "--db", tmpDb) + cmd := exec.Command(os.Args[0], fullArgs...) + cmd.Env = append(os.Environ(), + "WANT_HELPER_PROCESS=1", + "SPINDLE_SERVER_HOSTNAME=spindle.example.com", + "SPINDLE_SERVER_OWNER=did:web:spindle.example.com", + ) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + exitCode := 0 + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + exitCode = exitErr.ExitCode() + } else { + t.Fatalf("unexpected run error: %v", err) + } + } + + expectedCode := tt.wantExitCode + if expectedCode == -1 { + expectedCode = 255 + } + if exitCode != expectedCode { + t.Errorf("exitCode = %d, want %d. Stderr: %q, Stdout: %q", exitCode, expectedCode, stderr.String(), stdout.String()) + } + + combined := stdout.String() + "\n" + stderr.String() + if tt.wantError != "" && !strings.Contains(combined, tt.wantError) { + t.Errorf("expected error/output to contain %q, got combined: %q", tt.wantError, combined) + } + if tt.wantOutput != "" && !strings.Contains(combined, tt.wantOutput) { + t.Errorf("expected output to contain %q, got combined: %q", tt.wantOutput, combined) + } + }) + } +} + +func runCLI(t *testing.T, dbPath string, args ...string) (string, int) { + t.Helper() + cmd := exec.Command(os.Args[0], append(append([]string{"-test.run=TestHelperProcess", "--"}, args...), "--db", dbPath)...) + cmd.Env = append(os.Environ(), "WANT_HELPER_PROCESS=1") + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + err := cmd.Run() + if exitErr, ok := err.(*exec.ExitError); ok { + return out.String(), exitErr.ExitCode() + } else if err != nil { + t.Fatalf("running %v: %v", args, err) + } + return out.String(), 0 +} + +func TestCLIAllowListCommands(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "spindle.db") + listed := func() string { + out, code := runCLI(t, dbPath, "allowlist", "list") + if code != 0 { + t.Fatalf("allowlist list = %d: %s", code, out) + } + return out + } + + if out, code := runCLI(t, dbPath, "allowlist", "add", "alice"); code == 0 || !strings.Contains(out, "not a did or handle") { + t.Fatalf("adding something that's neither a did nor a handle = %d: %s", code, out) + } + if out, code := runCLI(t, dbPath, "allowlist", "add", "did:plc:alice"); code != 0 { + t.Fatalf("allowlist add = %d: %s", code, out) + } + if out := listed(); !strings.Contains(out, "did:plc:alice") { + t.Fatalf("alice isn't listed after add: %q", out) + } + if out, code := runCLI(t, dbPath, "allowlist", "remove", "did:plc:alice"); code != 0 { + t.Fatalf("allowlist remove = %d: %s", code, out) + } + if out := listed(); strings.Contains(out, "did:plc:alice") { + t.Fatalf("alice is still listed after remove: %q", out) + } +} + +func TestCLIInviteMintsARedeemableCode(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "spindle.db") + out, code := runCLI(t, dbPath, "invite") + if code != 0 { + t.Fatalf("invite = %d: %s", code, out) + } + // migrations log to the same output, so pick the code out + invite := regexp.MustCompile(`\b[0-9a-f]{32}\b`).FindString(out) + + d, err := db.Make(t.Context(), dbPath) + if err != nil { + t.Fatal(err) + } + defer d.Close() + if err := d.RedeemInvite(invite, "did:plc:bob", time.Now()); err != nil { + t.Fatalf("redeeming the cli's code: %v", err) + } + listed, err := db.AllowlistContains(d, "did:plc:bob") + if err != nil || !listed { + t.Fatalf("bob listed = %v, %v after redeeming", listed, err) + } +} + +func TestCLIListsPerWorkflowLimits(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "spindle.db") + for _, set := range [][]string{ + {"--resource", quota.ResourceWorkflowVCPUs, "--limit", "6"}, + {"--resource", quota.ResourceWorkflowMemoryMiB, "--limit", "8192"}, + {"--resource", quota.ResourceWorkflowDiskMiB, "--limit", "20480"}, + } { + if out, code := runCLI(t, dbPath, append([]string{"quota", "set", "did:web:alice"}, set...)...); code != 0 { + t.Fatalf("quota set %v exited %d: %s", set, code, out) + } + } + out, code := runCLI(t, dbPath, "quota", "list") + if code != 0 { + t.Fatalf("quota list exited %d: %s", code, out) + } + for _, want := range []string{"6 vCPUs", "8192 MiB", "20480 MiB"} { + if !strings.Contains(out, want) { + t.Errorf("quota list = %q, want %q", out, want) + } + } +} + +func TestNormalizeDefaultLimit(t *testing.T) { + if got := normalizeDefaultLimit(0); got != -1 { + t.Fatalf("zero configured limit = %d, want -1", got) + } + if got := normalizeDefaultLimit(2); got != 2 { + t.Fatalf("finite configured limit = %d, want 2", got) + } + if got := formatLimit(quota.ResourceWorkflows, 0); got != "0 workflows" { + t.Fatalf("zero override = %q, want 0 workflows", got) + } +} diff --git a/cmd/verifysig/main.go b/cmd/verifysig/main.go new file mode 100644 index 0000000000000000000000000000000000000000..b0e26b042ec467434060b048aa65ba50b15d28cc --- /dev/null +++ b/cmd/verifysig/main.go @@ -0,0 +1,103 @@ +package main + +import ( + "flag" + "fmt" + "io" + "os" + "strings" + + "tangled.org/core/crypto" +) + +func parseCommitObject(commitData string) (string, string, error) { + lines := strings.Split(commitData, "\n") + + var payloadLines []string + var signatureLines []string + var inSignature bool + var foundSignature bool + + for _, line := range lines { + if strings.HasPrefix(line, "gpgsig ") { + foundSignature = true + inSignature = true + // remove 'gpgsig' prefix + sigLine := strings.TrimPrefix(line, "gpgsig ") + signatureLines = append(signatureLines, sigLine) + continue + } + + if inSignature { + // check if this line is part of the signature (starts with space) + if strings.HasPrefix(line, " ") { + // remove the leading space and add to signature + sigLine := strings.TrimPrefix(line, " ") + signatureLines = append(signatureLines, sigLine) + continue + } else { + // end of signature block + inSignature = false + // this line is part of payload, so add it + payloadLines = append(payloadLines, line) + } + } else { + // regular payload line + payloadLines = append(payloadLines, line) + } + } + + if !foundSignature { + return "", commitData, nil // no signature found, return empty signature and full data as payload + } + + signature := strings.Join(signatureLines, "\n") + payload := strings.Join(payloadLines, "\n") + + return signature, payload, nil +} + +func main() { + var pubkeyPath string + flag.StringVar(&pubkeyPath, "pubkey", "", "Path to the public key file") + flag.Parse() + + var pubKey []byte + var err error + if pubkeyPath != "" { + pubKey, err = os.ReadFile(pubkeyPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error reading public key file: %v\n", err) + os.Exit(1) + } + } + + input, err := io.ReadAll(os.Stdin) + if err != nil { + fmt.Fprintf(os.Stderr, "error reading from stdin: %v\n", err) + os.Exit(1) + } + + commitData := string(input) + + signature, payload, err := parseCommitObject(commitData) + if err != nil { + fmt.Fprintf(os.Stderr, "error parsing commit: %v\n", err) + os.Exit(1) + } + + fmt.Println("signature") + fmt.Println(signature) + fmt.Println() + fmt.Println("payload:") + fmt.Println(payload) + + err, ok := crypto.VerifySignature(pubKey, []byte(signature), []byte(payload)) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + if ok { + fmt.Println("ok") + } +} diff --git a/cmd/zoekt-tngl-indexserver/index.go b/cmd/zoekt-tngl-indexserver/index.go new file mode 100644 index 0000000000000000000000000000000000000000..6ccffd4b93967258db1d1b936fa6f2cec3286e59 --- /dev/null +++ b/cmd/zoekt-tngl-indexserver/index.go @@ -0,0 +1,170 @@ +package main + +import ( + "context" + "crypto/sha1" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/util/ssrf" + "github.com/samber/lo" + "tangled.org/core/api/org_tangled" + "tangled.org/core/repoident" +) + +// 1 MB; match https://sourcegraph.sourcegraph.com/r/github.com/sourcegraph/sourcegraph/-/blob/cmd/searcher/internal/search/store.go?L32 +const MaxFileSize = 1 << 20 + +func gitIndex(ctx context.Context, cfg *Config, dir identity.Directory, req indexRequest) error { + ctx, cancel := context.WithTimeout(ctx, cfg.IndexTimeout) + defer cancel() + + repo, err := loadRepo(ctx, cfg, dir, req.Repo) + if err != nil { + return fmt.Errorf("loading repo %s: %w", req.Repo, err) + } + repo.Branches = req.Branches + + gitDir, err := tmpGitDir(repo.Did.String()) + if err != nil { + return err + } + defer os.RemoveAll(gitDir) // best-effort cleanup + + if err := fetchRepo(ctx, gitDir, repo.CloneURL(), req.Branches); err != nil { + return err + } + + if err := indexRepo(ctx, cfg, gitDir, *repo); err != nil { + return err + } + + return nil +} + +func loadRepo(ctx context.Context, cfg *Config, dir identity.Directory, repoDID repoident.RepoDid) (*Repo, error) { + ident, err := dir.LookupDID(ctx, syntax.DID(repoDID)) + if err != nil { + return nil, err + } + + knot, err := repoident.KnotURLFromIdentity(ident, cfg.KnotScheme) + if err != nil { + return nil, fmt.Errorf("repoDid %s: %w", repoDID, err) + } + + pds := ident.PDSEndpoint() + if pds == "" { + return nil, fmt.Errorf("repoDid %s has no atproto_pds service", repoDID) + } + out, err := atproto.RepoGetRecord(ctx, &atclient.APIClient{Host: pds, Client: &http.Client{Transport: ssrf.PublicOnlyTransport()}}, "", org_tangled.RepoManifestNSID, repoDID.String(), "self") + if err != nil { + return nil, fmt.Errorf("manifest of %s on %s: %w", repoDID, pds, err) + } + manifest, ok := out.Value.Val.(*org_tangled.RepoManifest) + if !ok || manifest.Declaration == nil { + return nil, fmt.Errorf("pds %s returned an invalid manifest for %s", pds, repoDID) + } + owner, err := repoident.NewOwnerDid(manifest.Declaration.Owner) + if err != nil { + return nil, fmt.Errorf("manifest of %s: %w", repoDID, err) + } + slug, err := syntax.ParseRecordKey(strings.ToLower(manifest.Declaration.Name)) + if err != nil { + return nil, fmt.Errorf("manifest of %s: %w", repoDID, err) + } + + return &Repo{ + Did: repoDID, + Owner: owner, + Slug: slug, + Knot: knot, + }, nil +} + +func fetchRepo(ctx context.Context, gitDir, cloneUrl string, branches []indexBranch) error { + // Create a repo to fetch into + if err := executeCmd(ctx, + "git", + // use a random default branch. This is so that HEAD isn't a symref to a + // branch that is indexed. For example if you are indexing + // HEAD,master. Then HEAD would be pointing to master by default. + "-c", "init.defaultBranch=nonExistentBranchBB0FOFCH32", + "init", + // we don't need a working copy + "--bare", + gitDir, + ); err != nil { + return err + } + + fetchArgs := append([]string{ + "-C", gitDir, + "-c", "protocol.version=2", + "fetch", "--depth=1", "--no-tags", + // Git's blob:limit filter excludes blobs whose size is >= the given limit, + // while zoekt indexes files up to and including FileLimit bytes. + fmt.Sprintf("--filter=blob:limit=%d", int64(MaxFileSize)+1), + cloneUrl, + }, lo.Map(branches, func(b indexBranch, _ int) string { return string(b.Version) })...) + + if err := executeCmd(ctx, "git", fetchArgs...); err != nil { + return err + } + + return errors.Join(lo.FilterMap(branches, func(b indexBranch, _ int) (error, bool) { + err := executeCmd(ctx, "git", "-C", gitDir, "update-ref", b.Name.Ref(), string(b.Version)) + if err == nil { + return nil, false + } + return fmt.Errorf("failed update-ref %s to %s: %w", b.Name.Ref(), b.Version, err), true + })...) +} + +func indexRepo(ctx context.Context, cfg *Config, gitDir string, repo Repo) error { + executablePath, err := os.Executable() + if err != nil { + return err + } + + repoJson, err := json.Marshal(repo) + if err != nil { + return err + } + + args := []string{"index"} + args = append(args, "-index-dir", cfg.IndexDir) + args = append(args, "-appview-url", cfg.AppviewUrl) + args = append(args, gitDir, string(repoJson)) + if err := executeCmd(ctx, executablePath, args...); err != nil { + return err + } + return nil +} + +func tmpGitDir(name string) (string, error) { + abs := url.QueryEscape(name) + if len(abs) > 200 { + h := sha1.New() + _, _ = io.WriteString(h, abs) + abs = abs[:200] + fmt.Sprintf("%x", h.Sum(nil))[:8] + } + dir := filepath.Join(os.TempDir(), abs+".git") + if _, err := os.Stat(dir); err == nil { + if err := os.RemoveAll(dir); err != nil { + return "", err + } + } + return dir, nil +} diff --git a/cmd/zoekt-tngl-indexserver/main.go b/cmd/zoekt-tngl-indexserver/main.go new file mode 100644 index 0000000000000000000000000000000000000000..4a197d0512bef30c86eae827f3649886fcddcccb --- /dev/null +++ b/cmd/zoekt-tngl-indexserver/main.go @@ -0,0 +1,315 @@ +// similar to zoekt-dynamic-indexserver, but targetting Tangled repos +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net" + "net/http" + "os" + "os/exec" + "os/signal" + "strconv" + "syscall" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/carlmjohnson/versioninfo" + "github.com/samber/lo" + "github.com/sourcegraph/zoekt/gitindex" + "github.com/sourcegraph/zoekt/index" + "github.com/urfave/cli/v3" + "tangled.org/core/repoident" +) + +func loggedRun(cmd *exec.Cmd) error { + outBuf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + cmd.Stdout = outBuf + cmd.Stderr = errBuf + + log.Printf("run %v", cmd.Args) + if err := cmd.Run(); err != nil { + log.Printf("command %s failed: %v\nOUT: %s\nERR: %s", + cmd.Args, err, outBuf.String(), errBuf.String()) + return fmt.Errorf("command %s failed: %v", cmd.Args, err) + } + + return nil +} + +// This function is declared as var so that we can stub it in test +var executeCmd = func(ctx context.Context, name string, arg ...string) error { + cmd := exec.CommandContext(ctx, name, arg...) + cmd.Stdin = &bytes.Buffer{} + err := loggedRun(cmd) + + return err +} + +func main() { + if err := run(os.Args); err != nil { + log.Fatal(err) + } +} + +func run(args []string) error { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + app := cli.Command{ + Name: "zoekt-tngl-indexserver", + Usage: "tangled zoekt index server", + Version: versioninfo.Short(), + } + app.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "index-dir", + Usage: "directory holding index shards.", + Required: true, + Sources: cli.EnvVars("TANGLED_ZOEKT_INDEX_DIR"), + }, + &cli.StringFlag{ + Name: "appview-url", + Usage: "appview url. used when debugging", + Value: "https://tangled.org", + Sources: cli.EnvVars("TANGLED_ZOEKT_APPVIEW_URL"), + }, + } + app.Commands = []*cli.Command{ + { + Name: "serve", + Usage: "run the index server daemon", + Action: runIndexServer, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "plc-url", + Usage: "atproto PLC directory.", + Value: "https://plc.directory", + Sources: cli.EnvVars("TANGLED_ZOEKT_PLC_URL", "TANGLED_PLC_URL", "ATP_PLC_HOST"), + }, + &cli.DurationFlag{ + Name: "index-timeout", + Usage: "kill index job after this much time.", + Value: time.Hour, + Sources: cli.EnvVars("TANGLED_ZOEKT_INDEX_TIMEOUT"), + }, + &cli.IntFlag{ + Name: "index-concurrency", + Usage: "number of repositories to index concurrently.", + Value: 4, + Sources: cli.EnvVars("TANGLED_ZOEKT_INDEX_CONCURRENCY"), + }, + &cli.IntFlag{ + Name: "index-queue-size", + Usage: "max index queue size", + Value: 100, + Sources: cli.EnvVars("TANGLED_ZOEKT_INDEX_QUEUE_SIZE"), + }, + &cli.StringFlag{ + Name: "listen", + Usage: "listen on this address", + Value: ":6060", + Sources: cli.EnvVars("TANGLED_ZOEKT_SERVER_LISTEN"), + }, + &cli.BoolFlag{ + Name: "allow-http", + Usage: "accept repo DIDs whose knot service endpoint is plaintext http.", + Sources: cli.EnvVars("TANGLED_ZOEKT_ALLOW_HTTP"), + }, + }, + }, + { + Name: "index", + Usage: "manually index a git repository", + Action: runIndex, + Arguments: []cli.Argument{ + &cli.StringArg{ + Name: "git-dir", + UsageText: "path to fetched git repository.", + }, + &cli.StringArg{ + Name: "repo", + UsageText: "json-encoded repository information", + }, + }, + }, + } + + return app.Run(ctx, args) +} + +type Config struct { + // IndexDir is the index directory to use. + IndexDir string + + IndexTimeout time.Duration + + // IndexConcurrency is the number of repositories we index at once. + IndexConcurrency int + IndexQueueSize int + + PlcUrl string + AppviewUrl string + Listen string + + KnotScheme repoident.SchemePolicy +} + +func createMissingDirectories(cfg *Config) { + for _, s := range []string{cfg.IndexDir} { + if err := os.MkdirAll(s, 0o755); err != nil { + log.Fatalf("MkdirAll %s: %v", s, err) + } + } +} + +type Repo struct { + Did repoident.RepoDid + Owner repoident.OwnerDid + Slug syntax.RecordKey + Knot repoident.KnotURL + Branches []indexBranch +} + +func (r *Repo) CloneURL() string { + return r.Knot.JoinPath(r.Did.String()) +} + +func runIndexServer(ctx context.Context, cmd *cli.Command) error { + cfg := &Config{ + IndexDir: cmd.String("index-dir"), + IndexTimeout: cmd.Duration("index-timeout"), + IndexConcurrency: cmd.Int("index-concurrency"), + IndexQueueSize: cmd.Int("index-queue-size"), + PlcUrl: cmd.String("plc-url"), + AppviewUrl: cmd.String("appview-url"), + Listen: cmd.String("listen"), + KnotScheme: repoident.SchemeFor(cmd.Bool("allow-http")), + } + createMissingDirectories(cfg) + + server := NewIndexServer(cfg) + go server.Run(ctx) + + <-ctx.Done() + return ctx.Err() +} + +// sub-process to index a fetched repository +func runIndex(ctx context.Context, cmd *cli.Command) error { + var ( + indexDir = cmd.String("index-dir") + appviewUrl = cmd.String("appview-url") + ) + + gitDir := cmd.StringArg("git-dir") + if gitDir == "" { + return errors.New("git-dir is required.") + } + + repoRaw := cmd.StringArg("repo") + if repoRaw == "" { + return errors.New("repo is required.") + } + + var repo Repo + if err := json.Unmarshal([]byte(repoRaw), &repo); err != nil { + return fmt.Errorf("invalid repo: %w", err) + } + if repo.Did == "" || repo.Owner == "" || repo.Knot.IsZero() { + return fmt.Errorf("repo is missing did, owner, or knot: %q", repoRaw) + } + + branches := lo.Map(repo.Branches, func(b indexBranch, _ int) string { return string(b.Name) }) + + buildOpts := index.Options{} + buildOpts.SetDefaults() + + buildOpts.IndexDir = indexDir + + buildOpts.ShardPrefixOverride = repo.Did.String() + + // Tangled templates + webUrl := fmt.Sprintf("%s/%s", appviewUrl, repo.Did) + buildOpts.RepositoryDescription.CommitURLTemplate = fmt.Sprintf("%s/commit/{{.Version}}", webUrl) + buildOpts.RepositoryDescription.FileURLTemplate = fmt.Sprintf("%s/blob/{{.Version}}/{{.Path}}", webUrl) + buildOpts.RepositoryDescription.LineFragmentTemplate = "#L{{.LineNumber}}" + + buildOpts.RepositoryDescription.Name = repo.Slug.String() + buildOpts.RepositoryDescription.URL = webUrl + buildOpts.RepositoryDescription.Metadata = map[string]string{ + "foo": "bar", // for testing + "did": repo.Did.String(), + "owner": repo.Owner.String(), + "knot": repo.Knot.String(), + } + // buildOpts.RepositoryDescription.Source = gitDir // configured later in IndexGitRepo + buildOpts.RepositoryDescription.Branches = nil + buildOpts.RepositoryDescription.SubRepoMap = nil + buildOpts.RepositoryDescription.RawConfig = map[string]string{ + "priority": strconv.FormatFloat(0.0, 'g', -1, 64), + "public": marshalBool(true), + "fork": marshalBool(false), + // Calculate repo rank based on the latest commit date. + "latestCommitDate": marshalBool(true), + } + buildOpts.RepositoryDescription.Rank = 0 + // buildOpts.RepositoryDescription.IndexOptions = "" // configured later in IndexGitRepo + // buildOpts.RepositoryDescription.LatestCommitDate = _ // configured later in IndexGitRepo + buildOpts.RepositoryDescription.FileTombstones = nil + + gitOpts := gitindex.Options{ + RepoDir: gitDir, + Submodules: false, + Incremental: true, + AllowMissingBranch: false, + RepoCacheDir: "", + BuildOptions: buildOpts, + BranchPrefix: "refs/heads/", + Branches: branches, + DeltaShardNumberFallbackThreshold: 0, + } + if _, err := gitindex.IndexGitRepo(gitOpts); err != nil { + return err + } + return nil +} + +func marshalBool(b bool) string { + if b { + return "1" + } + return "0" +} + +func baseDir(plc string) identity.Directory { + base := identity.BaseDirectory{ + PLCURL: plc, + HTTPClient: http.Client{ + Timeout: time.Second * 10, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + // would want this around 100ms for services doing lots of handle resolution. Impacts PLC connections as well, but not too bad. + IdleConnTimeout: time.Millisecond * 1000, + MaxIdleConns: 100, + }, + }, + Resolver: net.Resolver{ + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{Timeout: time.Second * 3} + return d.DialContext(ctx, network, address) + }, + }, + TryAuthoritativeDNS: true, + // primary Bluesky PDS instance only supports HTTP resolution method + SkipDNSDomainSuffixes: []string{".bsky.social"}, + UserAgent: "indigo-identity/" + versioninfo.Short(), + } + return identity.NewCacheDirectory(&base, 250_000, time.Hour*24, time.Minute*2, time.Minute*5) +} diff --git a/cmd/zoekt-tngl-indexserver/metrics.go b/cmd/zoekt-tngl-indexserver/metrics.go new file mode 100644 index 0000000000000000000000000000000000000000..fcb3962b3156d81c53aaff3ec287ea981ce69193 --- /dev/null +++ b/cmd/zoekt-tngl-indexserver/metrics.go @@ -0,0 +1,17 @@ +package main + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// HTTP metrics +var ( + requestsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "zoekt_dynamic_tangled_requests_total", + Help: "Total number of HTTP requests by status code, method, and route.", + }, + []string{"method", "route", "code"}, + ) +) diff --git a/cmd/zoekt-tngl-indexserver/queue.go b/cmd/zoekt-tngl-indexserver/queue.go new file mode 100644 index 0000000000000000000000000000000000000000..1ab3ea33aa415181ca63994619c597178180514f --- /dev/null +++ b/cmd/zoekt-tngl-indexserver/queue.go @@ -0,0 +1,72 @@ +package main + +import ( + "sync" + + "tangled.org/core/repoident" +) + +// deduplicating index work queue +type Queue struct { + mu sync.Mutex + order []repoident.RepoDid + pending map[repoident.RepoDid]indexRequest + size int +} + +func NewQueue(size int) *Queue { + return &Queue{ + pending: make(map[repoident.RepoDid]indexRequest), + size: size, + } +} + +func (q *Queue) Enqueue(req indexRequest) bool { + q.mu.Lock() + defer q.mu.Unlock() + + if _, exists := q.pending[req.Repo]; exists { + q.pending[req.Repo] = req // replace payload, keep position + return true + } + + if len(q.order) >= q.size { + return false // queue full + } + + q.order = append(q.order, req.Repo) + q.pending[req.Repo] = req + return true +} + +func (q *Queue) Pop() (indexRequest, bool) { + q.mu.Lock() + defer q.mu.Unlock() + + if len(q.order) == 0 { + return indexRequest{}, false + } + + did := q.order[0] + q.order = q.order[1:] + if len(q.order) == 0 { + q.order = nil // release the backing array + } + + req, ok := q.pending[did] + delete(q.pending, did) + return req, ok +} + +func (q *Queue) Snapshot() []indexRequest { + q.mu.Lock() + defer q.mu.Unlock() + + out := make([]indexRequest, 0, len(q.order)) + for _, did := range q.order { + if req, ok := q.pending[did]; ok { + out = append(out, req) + } + } + return out +} diff --git a/cmd/zoekt-tngl-indexserver/server.go b/cmd/zoekt-tngl-indexserver/server.go new file mode 100644 index 0000000000000000000000000000000000000000..61ae6cf6ee2dfa4bcd2c24230062173083265c0a --- /dev/null +++ b/cmd/zoekt-tngl-indexserver/server.go @@ -0,0 +1,212 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/go-git/go-git/v5/plumbing" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/samber/lo" + "tangled.org/core/repoident" +) + +type IndexServer struct { + cfg *Config + dir identity.Directory + queue *Queue +} + +func NewIndexServer(cfg *Config) *IndexServer { + return &IndexServer{ + cfg: cfg, + dir: baseDir(cfg.PlcUrl), + queue: NewQueue(cfg.IndexQueueSize), + } +} + +func (s *IndexServer) Run(ctx context.Context) { + // // Start a goroutine which updates the queue with commits to index. + // go func() { + // }() + + for range s.cfg.IndexConcurrency { + go s.processQueue(ctx) + } + + mux := http.NewServeMux() + mux.HandleFunc("/", s.handleHealth) + mux.HandleFunc("/debug/metrics", s.handleMetrics) + mux.HandleFunc("/debug/queue", s.handleDebugQueue) + mux.HandleFunc("/admin/forceIndex", s.handleForceIndex) + mux.HandleFunc("/admin/enqueueIndex", s.handleEnqueueIndex) + if err := http.ListenAndServe(s.cfg.Listen, mux); err != nil { + log.Fatal(err) + } +} + +func (s *IndexServer) handleHealth(w http.ResponseWriter, r *http.Request) { + // Nothing to do. Just return 200 +} + +func (s *IndexServer) handleMetrics(w http.ResponseWriter, r *http.Request) { + promhttp.Handler().ServeHTTP(w, r) +} + +type branchName string + +func (b branchName) Ref() string { + return lo.Ternary(b == "HEAD", "HEAD", "refs/heads/"+string(b)) +} + +func (b *branchName) UnmarshalText(text []byte) error { + name := branchName(text) + if strings.HasPrefix(string(name), "refs/") { + return fmt.Errorf("branch %q must be a short name, without the refs/ prefix", name) + } + if err := plumbing.ReferenceName(name.Ref()).Validate(); err != nil { + return fmt.Errorf("branch %q isn't a valid ref: %w", name, err) + } + *b = name + return nil +} + +type objectID string + +var objectIDPattern = regexp.MustCompile(`^([0-9a-fA-F]{40}|[0-9a-fA-F]{64})$`) + +func (o *objectID) UnmarshalText(text []byte) error { + if !objectIDPattern.Match(text) { + return fmt.Errorf("%q isn't a sha1 or sha256 object id", text) + } + *o = objectID(text) + return nil +} + +type indexBranch struct { + Name branchName `json:"name"` + Version objectID `json:"version"` +} + +type indexRequest struct { + Repo repoident.RepoDid `json:"repo"` + Branches []indexBranch `json:"branches"` +} + +func decodeIndexRequest(r *http.Request) (indexRequest, error) { + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + var req indexRequest + if err := dec.Decode(&req); err != nil { + return indexRequest{}, err + } + if req.Repo == "" { + return indexRequest{}, errors.New("index request has no repo did") + } + if len(req.Branches) == 0 { + return indexRequest{}, fmt.Errorf("index request for %s has no branches", req.Repo) + } + return req, nil +} + +func (s *IndexServer) handleDebugQueue(w http.ResponseWriter, r *http.Request) { + for _, req := range s.queue.Snapshot() { + fmt.Fprintln(w, req.Repo) + for _, b := range req.Branches { + fmt.Fprintf(w, "\t%s:\t%s\n", b.Name, b.Version) + } + } +} + +func (s *IndexServer) handleEnqueueIndex(w http.ResponseWriter, r *http.Request) { + route := "enqueueIndex" + req, err := decodeIndexRequest(r) + if err != nil { + log.Printf("Error decoding index request: %v", err) + http.Error(w, "JSON parser error", http.StatusBadRequest) + s.incrementRequestsTotal(r.Method, route, http.StatusBadRequest) + return + } + + if !s.queue.Enqueue(req) { + // queue full: reject so the producer retries later + http.Error(w, "index queue full", http.StatusServiceUnavailable) + s.incrementRequestsTotal(r.Method, route, http.StatusServiceUnavailable) + return + } + + w.WriteHeader(http.StatusAccepted) + s.incrementRequestsTotal(r.Method, route, http.StatusAccepted) +} + +func (s *IndexServer) handleForceIndex(w http.ResponseWriter, r *http.Request) { + route := "index" + req, err := decodeIndexRequest(r) + if err != nil { + log.Printf("Error decoding index request: %v", err) + http.Error(w, "JSON parser error", http.StatusBadRequest) + s.incrementRequestsTotal(r.Method, route, http.StatusBadRequest) + return + } + + if err := gitIndex(r.Context(), s.cfg, s.dir, req); err != nil { + s.respondWithError(w, r.Method, route, err) + return + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "success": true, + }) + + s.incrementRequestsTotal(r.Method, route, http.StatusOK) +} + +func (s *IndexServer) respondWithError(w http.ResponseWriter, method, route string, err error) { + responseCode := http.StatusInternalServerError + + log.Print(err) + s.incrementRequestsTotal(method, route, responseCode) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(responseCode) + response := map[string]any{ + "Success": false, + "Error": err.Error(), + } + + _ = json.NewEncoder(w).Encode(response) +} + +func (s *IndexServer) incrementRequestsTotal(method, route string, responseCode int) { + requestsTotal.With(prometheus.Labels{"code": strconv.Itoa(responseCode), "method": method, "route": route}).Inc() +} + +func (s *IndexServer) processQueue(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + default: + } + + req, ok := s.queue.Pop() + if !ok { + time.Sleep(time.Second) + continue + } + + if err := gitIndex(ctx, s.cfg, s.dir, req); err != nil { + log.Printf("indexing repo %s failed: %v", req.Repo, err) + } + } +} diff --git a/cmd/zoekt-tngl-indexserver/server_test.go b/cmd/zoekt-tngl-indexserver/server_test.go new file mode 100644 index 0000000000000000000000000000000000000000..dc3770155bec408bc12b2947c2c796c83c2e339f --- /dev/null +++ b/cmd/zoekt-tngl-indexserver/server_test.go @@ -0,0 +1,84 @@ +package main + +import ( + "fmt" + "net/http/httptest" + "strings" + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/repoident" +) + +var ( + sha1Oid = strings.Repeat("a", 40) + sha256Oid = strings.Repeat("b", 64) +) + +func decodeBody(t *testing.T, body string) (indexRequest, error) { + t.Helper() + return decodeIndexRequest(httptest.NewRequest("POST", "/admin/enqueueIndex", strings.NewReader(body))) +} + +func TestDecodeIndexRequest_Accepts(t *testing.T) { + cases := map[string]struct { + name, oid, wantRef string + }{ + "a branch and a sha1": {name: "main", oid: sha1Oid, wantRef: "refs/heads/main"}, + "HEAD and a sha256": {name: "HEAD", oid: sha256Oid, wantRef: "HEAD"}, + } + for label, tc := range cases { + t.Run(label, func(t *testing.T) { + req, err := decodeBody(t, fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":%q,"Version":%q}]}`, tc.name, tc.oid)) + if err != nil { + t.Fatalf("decodeIndexRequest: %v", err) + } + if req.Repo.String() != "did:plc:limpet" { + t.Errorf("Repo = %q, want did:plc:limpet", req.Repo) + } + want := indexBranch{Name: branchName(tc.name), Version: objectID(tc.oid)} + if len(req.Branches) != 1 || req.Branches[0] != want { + t.Errorf("Branches = %v, want %v", req.Branches, want) + } + if got := req.Branches[0].Name.Ref(); got != tc.wantRef { + t.Errorf("Ref = %q, want %q", got, tc.wantRef) + } + }) + } +} + +func TestDecodeIndexRequest_RejectsBadRequests(t *testing.T) { + cases := map[string]string{ + "branch name is a git option": fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":"-d","Version":%q}]}`, sha1Oid), + "version is a git option": `{"repo":"did:plc:limpet","branches":[{"Name":"main","Version":"--upload-pack=touch /tmp/pwned"}]}`, + "version is a ref": `{"repo":"did:plc:limpet","branches":[{"Name":"main","Version":"refs/heads/main"}]}`, + "version is short hex": `{"repo":"did:plc:limpet","branches":[{"Name":"main","Version":"deadbeef"}]}`, + "branch name walks up": fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":"../../objects","Version":%q}]}`, sha1Oid), + "branch name is a full ref": fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":"refs/heads/main","Version":%q}]}`, sha1Oid), + "branch name is empty": fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":"","Version":%q}]}`, sha1Oid), + "repo isn't a did": fmt.Sprintf(`{"repo":"limpet","branches":[{"Name":"main","Version":%q}]}`, sha1Oid), + "repo is absent": fmt.Sprintf(`{"branches":[{"Name":"main","Version":%q}]}`, sha1Oid), + "branches are absent": `{"repo":"did:plc:limpet"}`, + "branches are empty": `{"repo":"did:plc:limpet","branches":[]}`, + "unknown field": fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":"main","Version":%q}],"shards":3}`, sha1Oid), + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + if _, err := decodeBody(t, body); err == nil { + t.Errorf("decodeIndexRequest accepted %s", body) + } + }) + } +} + +func TestRepoCloneURL(t *testing.T) { + knot, err := repoident.ParseKnotURL("https://knot.oyster.cafe", repoident.RequireHTTPS) + if err != nil { + t.Fatalf("ParseKnotURL: %v", err) + } + repo := Repo{Did: "did:plc:limpet", Owner: "did:plc:akshay", Slug: syntax.RecordKey("3kkkkkkkkkkkk"), Knot: knot} + const want = "https://knot.oyster.cafe/did:plc:limpet" + if got := repo.CloneURL(); got != want { + t.Errorf("CloneURL = %q, want %q", got, want) + } +} diff --git a/crates/gix-chunk/Cargo.toml b/crates/gix-chunk/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..f11761fce97a881bab325454748004fb25c3053b --- /dev/null +++ b/crates/gix-chunk/Cargo.toml @@ -0,0 +1,121 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.85" +name = "gix-chunk" +version = "0.7.3" +authors = ["Sebastian Thiel "] +build = false +include = [ + "/src/**/*", + "/LICENSE-*", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Interact with the git chunk file format used in multi-pack index and commit-graph files" +documentation = "https://github.com/git/git/blob/seen/Documentation/technical/chunk-format.txt" +readme = false +license = "MIT OR Apache-2.0" +repository = "https://github.com/GitoxideLabs/gitoxide" +resolver = "2" + +[lib] +name = "gix_chunk" +path = "src/lib.rs" +test = false +doctest = true + +[dependencies.gix-error] +version = "^0.2.5" + +[lints.clippy] +bool_to_int_with_if = "allow" +borrow_as_ptr = "allow" +cast_lossless = "allow" +cast_possible_truncation = "allow" +cast_possible_wrap = "allow" +cast_precision_loss = "allow" +cast_sign_loss = "allow" +checked_conversions = "allow" +copy_iterator = "allow" +default_trait_access = "allow" +doc_markdown = "allow" +empty_docs = "allow" +enum_glob_use = "allow" +explicit_deref_methods = "allow" +explicit_into_iter_loop = "allow" +explicit_iter_loop = "allow" +filter_map_next = "allow" +fn_params_excessive_bools = "allow" +from_iter_instead_of_collect = "allow" +if_not_else = "allow" +ignored_unit_patterns = "allow" +implicit_clone = "allow" +inconsistent_struct_constructor = "allow" +inefficient_to_string = "allow" +inline_always = "allow" +items_after_statements = "allow" +iter_not_returning_iterator = "allow" +iter_without_into_iter = "allow" +large_enum_variant = "allow" +large_stack_arrays = "allow" +manual_assert = "allow" +manual_is_variant_and = "allow" +manual_let_else = "allow" +manual_string_new = "allow" +many_single_char_names = "allow" +match_bool = "allow" +match_same_arms = "allow" +match_wild_err_arm = "allow" +match_wildcard_for_single_variants = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +module_name_repetitions = "allow" +must_use_candidate = "allow" +mut_mut = "allow" +naive_bytecount = "allow" +needless_continue = "allow" +needless_for_each = "allow" +needless_pass_by_value = "allow" +needless_raw_string_hashes = "allow" +no_effect_underscore_binding = "allow" +option_option = "allow" +range_plus_one = "allow" +redundant_else = "allow" +result_large_err = "allow" +return_self_not_must_use = "allow" +should_panic_without_expect = "allow" +similar_names = "allow" +single_match_else = "allow" +stable_sort_primitive = "allow" +struct_excessive_bools = "allow" +struct_field_names = "allow" +too_long_first_doc_paragraph = "allow" +too_many_lines = "allow" +transmute_ptr_to_ptr = "allow" +trivially_copy_pass_by_ref = "allow" +unnecessary_join = "allow" +unnecessary_wraps = "allow" +unreadable_literal = "allow" +unused_self = "allow" +used_underscore_binding = "allow" +wildcard_imports = "allow" + +[lints.clippy.pedantic] +level = "warn" +priority = -1 + +[lints.rust] diff --git a/crates/gix-chunk/LICENSE-APACHE b/crates/gix-chunk/LICENSE-APACHE new file mode 100644 index 0000000000000000000000000000000000000000..f433b1a53f5b830a205fd2df78e2b34974656c7b --- /dev/null +++ b/crates/gix-chunk/LICENSE-APACHE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/crates/gix-chunk/LICENSE-MIT b/crates/gix-chunk/LICENSE-MIT new file mode 100644 index 0000000000000000000000000000000000000000..969d061e8ba2e38d69391910b1ef0d4869ff18d1 --- /dev/null +++ b/crates/gix-chunk/LICENSE-MIT @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/gix-chunk/src/file/decode.rs b/crates/gix-chunk/src/file/decode.rs new file mode 100644 index 0000000000000000000000000000000000000000..1ad3f952bbc248db86ef50851b39a8fd6806a087 --- /dev/null +++ b/crates/gix-chunk/src/file/decode.rs @@ -0,0 +1,93 @@ +use gix_error::ValidationError; +use gix_error::bstr::ByteSlice; +use std::ops::Range; + +use crate::{file, file::index}; + +impl file::Index { + /// Provided a mapped file at the beginning via `data`, starting at `toc_offset` decode all chunk information to return + /// an index with `num_chunks` chunks. + pub fn from_bytes( + data: &[u8], + toc_offset: usize, + num_chunks: u32, + ) -> Result { + if num_chunks == 0 { + return Err(ValidationError::new( + "Empty chunk indices are not allowed as the point of chunked files is to have chunks.", + )); + } + + let data_len: u64 = data.len() as u64; + let mut chunks = Vec::with_capacity(num_chunks as usize); + let mut toc_entry = &data[toc_offset..]; + let expected_min_size = (num_chunks as usize + 1) * file::Index::ENTRY_SIZE; + if toc_entry.len() < expected_min_size { + Err(format!( + "The table of contents would be {expected_min_size} bytes, but got only {toc_entry_len}", + toc_entry_len = toc_entry.len() + ))?; + } + + for chunk_idx in 0..num_chunks { + let (kind, offset) = toc_entry.split_at(4); + let kind = to_kind(kind); + if kind == crate::SENTINEL { + Err(format!( + "Sentinel value encountered while processing chunks {chunk_idx} of {num_chunks}" + ))?; + } + if chunks.iter().any(|c: &index::Entry| c.kind == kind) { + Err(format!( + "The chunk of kind '{}' was encountered more than once", + kind.as_bstr() + ))?; + } + + let offset = be_u64(offset); + if offset > data_len { + Err(format!( + "The chunk offset {offset} went past the file of length {data_len} - was it truncated?", + ))?; + } + toc_entry = &toc_entry[file::Index::ENTRY_SIZE..]; + let next_offset = be_u64(&toc_entry[4..]); + if next_offset > data_len { + Err(format!( + "The chunk offset {next_offset} went past the file of length {data_len} - was it truncated?" + ))?; + } + if next_offset < offset { + Err("Chunk offsets must not decrease.")?; + } + chunks.push(index::Entry { + kind, + offset: Range { + start: offset, + end: next_offset, + }, + }); + } + + let sentinel = to_kind(&toc_entry[..4]); + if sentinel != crate::SENTINEL { + Err(format!( + "Sentinel value wasn't found, saw '{}'", + sentinel.as_bstr() + ))?; + } + + Ok(file::Index { + chunks, + will_write: false, + }) + } +} + +fn to_kind(data: &[u8]) -> crate::Id { + data[..4].try_into().unwrap() +} + +fn be_u64(data: &[u8]) -> u64 { + u64::from_be_bytes(data[..8].try_into().unwrap()) +} diff --git a/crates/gix-chunk/src/file/index.rs b/crates/gix-chunk/src/file/index.rs new file mode 100644 index 0000000000000000000000000000000000000000..1f017fb951d4be55e6bc4e1d9372d23b2274b1ec --- /dev/null +++ b/crates/gix-chunk/src/file/index.rs @@ -0,0 +1,88 @@ +use std::ops::Range; + +use crate::Id; +use crate::file::Index; +use gix_error::bstr::ByteSlice; +use gix_error::{Message, message}; + +/// An entry of a chunk file index +pub struct Entry { + /// The kind of the chunk file + pub kind: Id, + /// The offset, relative to the beginning of the file, at which to find the chunk and its end. + pub offset: Range, +} + +impl Index { + /// The size of a single index entry in bytes + pub const ENTRY_SIZE: usize = std::mem::size_of::() + std::mem::size_of::(); + /// The smallest possible size of an index, consisting only of the sentinel value pointing past itself. + pub const EMPTY_SIZE: usize = Index::ENTRY_SIZE; + + /// Returns the size in bytes an index with `num_entries` would take. + pub const fn size_for_entries(num_entries: usize) -> usize { + Self::ENTRY_SIZE * (num_entries + 1/*sentinel*/) + } + + /// Find a chunk of `kind` and return its offset into the data if found + pub fn offset_by_id(&self, kind: Id) -> Result, Message> { + self.chunks + .iter() + .find_map(|c| (c.kind == kind).then(|| c.offset.clone())) + .ok_or_else(make_message(kind)) + } + + /// Find a chunk of `kind` and return its offset as usize range into the data if found. + /// + /// + /// # Panics + /// + /// - if the usize conversion fails, which isn't expected as memory maps can't be created if files are too large + /// to require such offsets. + pub fn usize_offset_by_id(&self, kind: Id) -> Result, Message> { + self.chunks + .iter() + .find_map(|c| { + (c.kind == kind).then(|| crate::range::into_usize_or_panic(c.offset.clone())) + }) + .ok_or_else(make_message(kind)) + } + + /// Like [`Index::usize_offset_by_id()`] but with support for validation and transformation using a function. + pub fn validated_usize_offset_by_id( + &self, + kind: Id, + validate: impl FnOnce(Range) -> T, + ) -> Result { + self.chunks + .iter() + .find_map(|c| { + (c.kind == kind).then(|| crate::range::into_usize_or_panic(c.offset.clone())) + }) + .map(validate) + .ok_or_else(make_message(kind)) + } + + /// Find a chunk of `kind` and return its data slice based on its offset. + pub fn data_by_id<'a>(&self, data: &'a [u8], kind: Id) -> Result<&'a [u8], Message> { + let offset = self.offset_by_id(kind)?; + Ok(&data[crate::range::into_usize(offset).ok_or_else(|| { + message("The offsets into the file couldn't be represented by usize") + })?]) + } + + /// Return the end offset of the last chunk, which is the highest offset as well. + /// It's definitely available as we have one or more chunks. + pub fn highest_offset(&self) -> crate::file::Offset { + self.chunks.last().expect("at least one chunk").offset.end + } +} + +fn make_message(kind: Id) -> impl FnOnce() -> Message { + move || { + message!( + "Chunk named '{}' was not found in chunk file index", + kind.as_bstr() + ) + } +} diff --git a/crates/gix-chunk/src/file/mod.rs b/crates/gix-chunk/src/file/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..4ddd94999ab4f5faed7e244ff7342f4e1f7e1764 --- /dev/null +++ b/crates/gix-chunk/src/file/mod.rs @@ -0,0 +1,20 @@ +/// +pub mod decode; +/// +pub mod index; + +/// +pub mod write; + +/// The offset to a chunk as seen relative to the beginning of the file containing it. +pub type Offset = u64; + +/// A chunk file providing a table into the parent data. +pub struct Index { + /// If true, we use `chunks` in a way that facilitates writing them. + will_write: bool, + /// Validated chunks as defined by their index entries. + /// + /// Note that this list cannot be empty. + chunks: Vec, +} diff --git a/crates/gix-chunk/src/file/write.rs b/crates/gix-chunk/src/file/write.rs new file mode 100644 index 0000000000000000000000000000000000000000..ccbbf5621a371b41b700cbc412f74b2dcd3040bb --- /dev/null +++ b/crates/gix-chunk/src/file/write.rs @@ -0,0 +1,139 @@ +use crate::file::{Index, index::Entry}; + +mod write_chunk { + use std::collections::VecDeque; + + use crate::file::index; + + /// A [`Write`][std::io::Write] implementation that validates chunk sizes while allowing the user to know + /// which chunk is to be written next. + pub struct Chunk { + chunks_to_write: VecDeque, + inner: W, + next_chunk: Option, + written_bytes: usize, + } + + impl Chunk + where + W: std::io::Write, + { + pub(crate) fn new(out: W, chunks: VecDeque) -> Chunk + where + W: std::io::Write, + { + Chunk { + chunks_to_write: chunks, + inner: out, + next_chunk: None, + written_bytes: 0, + } + } + } + + impl std::io::Write for Chunk + where + W: std::io::Write, + { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.inner.write(buf)?; + self.written_bytes += written; + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } + } + + impl Chunk { + /// Return the inner writer - should only be called once there is no more chunk to write. + pub fn into_inner(self) -> W { + self.inner + } + /// Return the next chunk-id to write, if there is one. + pub fn next_chunk(&mut self) -> Option { + if let Some(entry) = self.next_chunk.take() { + assert_eq!( + entry.offset.end, self.written_bytes as u64, + "BUG: expected to write {} bytes, but only wrote {} for chunk {:?}", + entry.offset.end, self.written_bytes, entry.kind + ); + } + self.written_bytes = 0; + self.next_chunk = self.chunks_to_write.pop_front(); + self.next_chunk.as_ref().map(|e| e.kind) + } + } +} +pub use write_chunk::Chunk; + +/// Writing +impl Index { + /// Create a new index whose sole purpose is to be receiving chunks using [`plan_chunk()`][Index::plan_chunk()] and to be written to + /// an output using [`into_write()`][Index::into_write()] + pub fn for_writing() -> Self { + Index { + will_write: true, + chunks: Vec::new(), + } + } + /// Plan to write a new chunk as part of the index when [`into_write()`][Index::into_write()] is called. + pub fn plan_chunk(&mut self, chunk: crate::Id, exact_size_on_disk: u64) { + assert!( + self.will_write, + "BUG: create the index with `for_writing()`" + ); + assert!( + !self.chunks.iter().any(|e| e.kind == chunk), + "BUG: must not add chunk of same kind twice: {chunk:?}" + ); + self.chunks.push(Entry { + kind: chunk, + offset: 0..exact_size_on_disk, + }); + } + + /// Return the total size of all planned chunks thus far. + pub fn planned_storage_size(&self) -> u64 { + assert!( + self.will_write, + "BUG: create the index with `for_writing()`" + ); + self.chunks.iter().map(|e| e.offset.end).sum() + } + + /// Return the amount of chunks we currently know. + pub fn num_chunks(&self) -> usize { + self.chunks.len() + } + + /// After [planning all chunks][Index::plan_chunk()] call this method with the destination to write the chunks to. + /// Use the [Chunk] writer to write each chunk in order. + /// `current_offset` is the byte position at which `out` will continue writing. + pub fn into_write(self, mut out: W, current_offset: usize) -> std::io::Result> + where + W: std::io::Write, + { + assert!( + self.will_write, + "BUG: create the index with `for_writing()`, cannot write decoded indices" + ); + // First chunk starts past the table of contents + let mut current_offset = + (current_offset + Self::size_for_entries(self.num_chunks())) as u64; + + for entry in &self.chunks { + out.write_all(&entry.kind)?; + out.write_all(¤t_offset.to_be_bytes())?; + + current_offset += entry.offset.end; + } + + // sentinel to mark end of chunks + out.write_all(&0u32.to_be_bytes())?; + out.write_all(¤t_offset.to_be_bytes())?; + + Ok(Chunk::new(out, self.chunks.into())) + } +} diff --git a/crates/gix-chunk/src/lib.rs b/crates/gix-chunk/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..9cc8d27754c61c8fc95aa5ec4c372e423f76f31f --- /dev/null +++ b/crates/gix-chunk/src/lib.rs @@ -0,0 +1,62 @@ +//! Low-level access to reading and writing chunk file based formats. +//! +//! See the [git documentation](https://github.com/git/git/blob/seen/Documentation/technical/chunk-format.txt) for details. +//! +//! ## Examples +//! +//! ``` +//! # fn main() -> Result<(), Box> { +//! use std::io::Write; +//! +//! let mut index = gix_chunk::file::Index::for_writing(); +//! index.plan_chunk(*b"OIDF", 4); +//! index.plan_chunk(*b"DATA", 3); +//! +//! let mut out = index.into_write(Vec::new(), 0)?; +//! while let Some(kind) = out.next_chunk() { +//! match kind { +//! [b'O', b'I', b'D', b'F'] => out.write_all(b"abcd")?, +//! [b'D', b'A', b'T', b'A'] => out.write_all(b"xyz")?, +//! _ => unreachable!("planned chunks are known"), +//! } +//! } +//! +//! let data = out.into_inner(); +//! let decoded = gix_chunk::file::Index::from_bytes(&data, 0, 2)?; +//! assert_eq!(decoded.data_by_id(&data, *b"OIDF")?, b"abcd"); +//! assert_eq!(decoded.data_by_id(&data, *b"DATA")?, b"xyz"); +//! # Ok(()) } +//! ``` +#![deny(missing_docs, unsafe_code)] + +/// An identifier to describe the kind of chunk, unique within a chunk file, typically in ASCII +pub type Id = [u8; 4]; + +/// A special value denoting the end of the chunk file table of contents. +pub const SENTINEL: Id = [0u8; 4]; + +/// +pub mod range { + use std::ops::Range; + + use crate::file; + + /// Turn a u64 Range into a usize range safely, to make chunk ranges useful in memory mapped files. + pub fn into_usize(Range { start, end }: Range) -> Option> { + let start = start.try_into().ok()?; + let end = end.try_into().ok()?; + Some(Range { start, end }) + } + + /// Similar to [`into_usize()`], but panics assuming that the memory map couldn't be created if offsets + /// stored are too high. + /// + /// This is only true for correctly formed files, as it's entirely possible to provide out of bounds offsets + /// which are checked for separately - we wouldn't be here if that was the case. + pub fn into_usize_or_panic(range: Range) -> Range { + into_usize(range).expect("memory maps can't be created if files are too large") + } +} + +/// +pub mod file; diff --git a/crates/gix-commitgraph/Cargo.toml b/crates/gix-commitgraph/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..8b2e53d6e25d750c0503275878f547e8fd39b1c2 --- /dev/null +++ b/crates/gix-commitgraph/Cargo.toml @@ -0,0 +1,194 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.85" +name = "gix-commitgraph" +version = "0.37.1" +authors = [ + "Conor Davis ", + "Sebastian Thiel ", +] +build = false +include = [ + "/src/**/*", + "/LICENSE-*", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Read-only access to the git commitgraph file format" +documentation = "https://git-scm.com/docs/commit-graph" +readme = false +license = "MIT OR Apache-2.0" +repository = "https://github.com/GitoxideLabs/gitoxide" +resolver = "2" + +[package.metadata.docs.rs] +all-features = true +features = [ + "sha1", + "document-features", +] + +[package.metadata.cargo-machete] +ignored = ["bstr"] + +[features] +parallel = ["gix-features/parallel"] +serde = [ + "dep:serde", + "gix-hash/serde", + "bstr/serde", +] +sha1 = [ + "gix-hash/sha1", + "gix-object/sha1", +] +sha256 = [ + "gix-hash/sha256", + "gix-object/sha256", +] + +[lib] +name = "gix_commitgraph" +path = "src/lib.rs" +doctest = false + +[dependencies.bstr] +version = "1.12.0" +features = ["std"] +default-features = false + +[dependencies.document-features] +version = "0.2.0" +optional = true + +[dependencies.gix-chunk] +version = "^0.7.2" + +[dependencies.gix-error] +version = "^0.2.4" + +[dependencies.gix-features] +version = "^0.48.1" + +[dependencies.gix-hash] +version = "^0.25.1" + +[dependencies.gix-hashtable] +version = "^0.15.1" + +[dependencies.gix-lock] +version = "^23.0.1" + +[dependencies.gix-object] +version = "^0.61.0" + +[dependencies.gix-tempfile] +version = "^23.0.1" +default-features = false + +[dependencies.memmap2] +version = "0.9.10" + +[dependencies.nonempty] +version = "0.12.0" + +[dependencies.serde] +version = "1.0.114" +features = ["derive"] +optional = true +default-features = false + +[dev-dependencies] + +[lints.clippy] +bool_to_int_with_if = "allow" +borrow_as_ptr = "allow" +cast_lossless = "allow" +cast_possible_truncation = "allow" +cast_possible_wrap = "allow" +cast_precision_loss = "allow" +cast_sign_loss = "allow" +checked_conversions = "allow" +copy_iterator = "allow" +default_trait_access = "allow" +doc_markdown = "allow" +empty_docs = "allow" +enum_glob_use = "allow" +explicit_deref_methods = "allow" +explicit_into_iter_loop = "allow" +explicit_iter_loop = "allow" +filter_map_next = "allow" +fn_params_excessive_bools = "allow" +from_iter_instead_of_collect = "allow" +if_not_else = "allow" +ignored_unit_patterns = "allow" +implicit_clone = "allow" +inconsistent_struct_constructor = "allow" +inefficient_to_string = "allow" +inline_always = "allow" +items_after_statements = "allow" +iter_not_returning_iterator = "allow" +iter_without_into_iter = "allow" +large_enum_variant = "allow" +large_stack_arrays = "allow" +manual_assert = "allow" +manual_is_variant_and = "allow" +manual_let_else = "allow" +manual_string_new = "allow" +many_single_char_names = "allow" +match_bool = "allow" +match_same_arms = "allow" +match_wild_err_arm = "allow" +match_wildcard_for_single_variants = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +module_name_repetitions = "allow" +must_use_candidate = "allow" +mut_mut = "allow" +naive_bytecount = "allow" +needless_continue = "allow" +needless_for_each = "allow" +needless_pass_by_value = "allow" +needless_raw_string_hashes = "allow" +no_effect_underscore_binding = "allow" +option_option = "allow" +range_plus_one = "allow" +redundant_else = "allow" +result_large_err = "allow" +return_self_not_must_use = "allow" +should_panic_without_expect = "allow" +similar_names = "allow" +single_match_else = "allow" +stable_sort_primitive = "allow" +struct_excessive_bools = "allow" +struct_field_names = "allow" +too_long_first_doc_paragraph = "allow" +too_many_lines = "allow" +transmute_ptr_to_ptr = "allow" +trivially_copy_pass_by_ref = "allow" +unnecessary_join = "allow" +unnecessary_wraps = "allow" +unreadable_literal = "allow" +unused_self = "allow" +used_underscore_binding = "allow" +wildcard_imports = "allow" + +[lints.clippy.pedantic] +level = "warn" +priority = -1 + +[lints.rust] diff --git a/crates/gix-commitgraph/LICENSE-APACHE b/crates/gix-commitgraph/LICENSE-APACHE new file mode 100644 index 0000000000000000000000000000000000000000..f433b1a53f5b830a205fd2df78e2b34974656c7b --- /dev/null +++ b/crates/gix-commitgraph/LICENSE-APACHE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/crates/gix-commitgraph/LICENSE-MIT b/crates/gix-commitgraph/LICENSE-MIT new file mode 100644 index 0000000000000000000000000000000000000000..969d061e8ba2e38d69391910b1ef0d4869ff18d1 --- /dev/null +++ b/crates/gix-commitgraph/LICENSE-MIT @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/gix-commitgraph/src/access.rs b/crates/gix-commitgraph/src/access.rs new file mode 100644 index 0000000000000000000000000000000000000000..5f4bdd3fd9a5a7fb23f2683c4e4b434f3f1a2e4d --- /dev/null +++ b/crates/gix-commitgraph/src/access.rs @@ -0,0 +1,112 @@ +use crate::{BloomFilterSettings, File, Graph, Position, file, file::Commit}; + +/// Access +impl Graph { + /// Returns the commit at the given position `pos`. + /// + /// # Panics + /// If `pos` is greater or equal to [`num_commits()`][Graph::num_commits()]. + pub fn commit_at(&self, pos: Position) -> Commit<'_> { + let r = self.lookup_by_pos(pos); + r.file.commit_at(r.pos) + } + + /// The kind of hash used in this `Graph`. + /// + /// Note that it is always conforming to the hash used in the owning repository. + pub fn object_hash(&self) -> gix_hash::Kind { + self.files.first().object_hash() + } + + /// Returns the commit matching the given `id`. + pub fn commit_by_id(&self, id: impl AsRef) -> Option> { + let r = self.lookup_by_id(id.as_ref())?; + Some(r.file.commit_at(r.file_pos)) + } + + /// Returns the `hash` at the given position `pos`. + /// + /// # Panics + /// If `pos` is greater or equal to [`num_commits()`][Graph::num_commits()]. + pub fn id_at(&self, pos: Position) -> &gix_hash::oid { + let r = self.lookup_by_pos(pos); + r.file.id_at(r.pos) + } + + /// Iterate over commits in unsorted order. + pub fn iter_commits(&self) -> impl Iterator> { + self.files.iter().flat_map(File::iter_commits) + } + + /// Iterate over commit IDs in unsorted order. + pub fn iter_ids(&self) -> impl Iterator { + self.files.iter().flat_map(File::iter_ids) + } + + /// Translate the given `id` to its position in the file. + pub fn lookup(&self, id: impl AsRef) -> Option { + Some(self.lookup_by_id(id.as_ref())?.graph_pos) + } + + /// Returns the number of commits stored in this file. + pub fn num_commits(&self) -> u32 { + self.files.iter().map(File::num_commits).sum() + } + + /// Return changed-path Bloom filter settings used by the top-most compatible graph layer, if available. + pub fn bloom_filter_settings(&self) -> Option { + self.files + .iter() + .rev() + .find_map(File::bloom_filter_settings) + } +} + +/// Access fundamentals +impl Graph { + fn lookup_by_id(&self, id: &gix_hash::oid) -> Option> { + let mut current_file_start = 0; + for file in &self.files { + if let Some(lex_pos) = file.lookup(id) { + return Some(LookupByIdResult { + file, + file_pos: lex_pos, + graph_pos: Position(current_file_start + lex_pos.0), + }); + } + current_file_start += file.num_commits(); + } + None + } + + fn lookup_by_pos(&self, pos: Position) -> LookupByPositionResult<'_> { + let mut remaining = pos.0; + for (file_index, file) in self.files.iter().enumerate() { + match remaining.checked_sub(file.num_commits()) { + Some(v) => remaining = v, + None => { + return LookupByPositionResult { + file, + _file_index: file_index, + pos: file::Position(remaining), + }; + } + } + } + panic!("graph position too large: {}", pos.0); + } +} + +#[derive(Clone)] +struct LookupByIdResult<'a> { + pub file: &'a File, + pub graph_pos: Position, + pub file_pos: file::Position, +} + +#[derive(Clone)] +struct LookupByPositionResult<'a> { + pub file: &'a File, + pub _file_index: usize, + pub pos: file::Position, +} diff --git a/crates/gix-commitgraph/src/bloom.rs b/crates/gix-commitgraph/src/bloom.rs new file mode 100644 index 0000000000000000000000000000000000000000..f14e7e82c5da4d3e5cb36f78ac7566a7bceb0218 --- /dev/null +++ b/crates/gix-commitgraph/src/bloom.rs @@ -0,0 +1,374 @@ +//! Query support for changed-path Bloom filters stored in commit-graph files. + +use bstr::BStr; + +use crate::{BloomFilterSettings, File, Graph, Position, file, from_be_u32}; + +const SEED0: u32 = 0x293a_e76f; +const SEED1: u32 = 0x7e64_6e2c; +const BITS_PER_WORD: u64 = 8; + +/// Precomputed hash positions for a path using Bloom filter settings. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BloomKey { + h0: u32, + h1: u32, + num_hashes: u32, +} + +impl BloomKey { + /// Build a key for `path`. + /// + /// `path` must use `/` as separator. + pub fn from_path(path: &BStr, settings: BloomFilterSettings) -> Self { + Self::from_bytes(path.as_ref(), settings) + } + + /// Build keys for `path` and each directory prefix. + /// + /// For `a/b/c`, this yields keys for `a/b/c`, `a/b`, and `a`. + /// `path` must use `/` as separator. + pub fn from_path_with_prefixes(path: &BStr, settings: BloomFilterSettings) -> Vec { + let bytes = path.as_ref(); + let mut out = vec![Self::from_bytes(bytes, settings)]; + + let mut idx = bytes.len(); + while idx > 0 { + idx -= 1; + if bytes[idx] == b'/' { + out.push(Self::from_bytes(&bytes[..idx], settings)); + } + } + out + } + + pub(crate) fn from_bytes(path: &[u8], settings: BloomFilterSettings) -> Self { + let (h0, h1) = match settings.hash_version { + 1 => (murmur3_v1(SEED0, path), murmur3_v1(SEED1, path)), + 2 => (murmur3_v2(SEED0, path), murmur3_v2(SEED1, path)), + version => panic!( + "BUG: unsupported Bloom hash version {version} should have been filtered earlier" + ), + }; + Self { + h0, + h1, + num_hashes: settings.num_hashes, + } + } + + /// Set the bits of this key in `filter_data`. + pub(crate) fn add_to(&self, filter_data: &mut [u8]) { + let modulo = (filter_data.len() as u64) * BITS_PER_WORD; + for i in 0..self.num_hashes { + let hash = self.h0.wrapping_add(i.wrapping_mul(self.h1)); + let bit_pos = u64::from(hash) % modulo; + filter_data[(bit_pos / BITS_PER_WORD) as usize] |= 1u8 << (bit_pos % BITS_PER_WORD); + } + } + + /// Query whether this key may be contained in `filter_data`. + /// + /// Returns `None` if the filter is unusable (empty data), `Some(false)` on a definite miss, + /// and `Some(true)` on a possible hit. + pub fn contains(&self, filter_data: &[u8]) -> Option { + let modulo = (filter_data.len() as u64) * BITS_PER_WORD; + if modulo == 0 { + return None; + } + + for i in 0..self.num_hashes { + let hash = self.h0.wrapping_add(i.wrapping_mul(self.h1)); + let bit_pos = u64::from(hash) % modulo; + let byte_pos = (bit_pos / BITS_PER_WORD) as usize; + let mask = 1u8 << (bit_pos % BITS_PER_WORD); + if filter_data[byte_pos] & mask == 0 { + return Some(false); + } + } + Some(true) + } +} + +impl File { + /// Query if `path` may be present in the changed-path Bloom filter for commit `pos`. + /// + /// Checks the full path and each of its leading directories, which reduces false positives. + pub fn maybe_contains_path(&self, pos: file::Position, path: &BStr) -> Option { + let settings = self.bloom_filter_settings?; + let data = self.bloom_filter_at(pos)?; + let keys = BloomKey::from_path_with_prefixes(path, settings); + for key in &keys { + match key.contains(data) { + Some(false) => return Some(false), + None => return None, + Some(true) => {} + } + } + Some(true) + } + + /// Query if all `keys` may be present in the changed-path Bloom filter for commit `pos`. + pub fn maybe_contains_all_keys(&self, pos: file::Position, keys: &[BloomKey]) -> Option { + let data = self.bloom_filter_at(pos)?; + if data.is_empty() { + return None; + } + Some(keys.iter().all(|key| key.contains(data) == Some(true))) + } + + /// Return the raw changed-path Bloom filter bytes stored for commit `pos`, as written in the `BDAT` chunk. + /// + /// Returns `None` if this file has no usable Bloom filters or `pos` is out of bounds. + /// An empty slice means the filter wasn't computed, which is distinct from a filter for no changed paths. + /// Interpret the bytes with [`File::bloom_filter_settings()`]. + pub fn bloom_filter_at(&self, pos: file::Position) -> Option<&[u8]> { + self.bloom_filter_settings?; + let index_offset = self.bloom_filter_index_offset?; + let data_offset = self.bloom_filter_data_offset?; + if pos.0 >= self.num_commits() { + return None; + } + + let lex = pos.0 as usize; + let end = from_be_u32(&self.data[index_offset + lex * 4..][..4]); + let start = if lex == 0 { + 0 + } else { + from_be_u32(&self.data[index_offset + (lex - 1) * 4..][..4]) + }; + let start = start as usize; + let end = end as usize; + if start > end || end > self.bloom_filter_data_len { + return None; + } + let start = data_offset.checked_add(start)?; + let end = data_offset.checked_add(end)?; + self.data.get(start..end) + } +} + +impl Graph { + /// Query by commit id if `path` may be present in changed-path Bloom filters. + pub fn maybe_contains_path_by_id( + &self, + id: impl AsRef, + path: &BStr, + ) -> Option { + let pos = self.lookup(id)?; + self.maybe_contains_path(pos, path) + } + + /// Query by graph position if `path` may be present in changed-path Bloom filters. + pub fn maybe_contains_path(&self, pos: Position, path: &BStr) -> Option { + self.commit_at(pos).maybe_contains_path(path) + } +} + +pub(crate) fn murmur3_v1(seed: u32, data: &[u8]) -> u32 { + murmur3(seed, data, true) +} + +pub(crate) fn murmur3_v2(seed: u32, data: &[u8]) -> u32 { + murmur3(seed, data, false) +} + +fn murmur3(seed: u32, data: &[u8], sign_extend: bool) -> u32 { + const C1: u32 = 0xcc9e_2d51; + const C2: u32 = 0x1b87_3593; + const R1: u32 = 15; + const R2: u32 = 13; + const M: u32 = 5; + const N: u32 = 0xe654_6b64; + + let byte_to_u32 = |byte: u8| { + if sign_extend { + u32::from_ne_bytes(i32::from(i8::from_ne_bytes([byte])).to_ne_bytes()) + } else { + u32::from(byte) + } + }; + + let mut seed = seed; + let chunks = data.chunks_exact(4); + let tail = chunks.remainder(); + for chunk in chunks { + let byte1 = byte_to_u32(chunk[0]); + let byte2 = byte_to_u32(chunk[1]) << 8; + let byte3 = byte_to_u32(chunk[2]) << 16; + let byte4 = byte_to_u32(chunk[3]) << 24; + let mut k = byte1 | byte2 | byte3 | byte4; + k = k.wrapping_mul(C1); + k = k.rotate_left(R1); + k = k.wrapping_mul(C2); + + seed ^= k; + seed = seed.rotate_left(R2).wrapping_mul(M).wrapping_add(N); + } + + let mut k1 = 0u32; + match tail.len() { + 3 => { + k1 ^= byte_to_u32(tail[2]) << 16; + k1 ^= byte_to_u32(tail[1]) << 8; + k1 ^= byte_to_u32(tail[0]); + } + 2 => { + k1 ^= byte_to_u32(tail[1]) << 8; + k1 ^= byte_to_u32(tail[0]); + } + 1 => { + k1 ^= byte_to_u32(tail[0]); + } + 0 => {} + _ => unreachable!("remainder is shorter than 4 bytes"), + } + if !tail.is_empty() { + k1 = k1.wrapping_mul(C1); + k1 = k1.rotate_left(R1); + k1 = k1.wrapping_mul(C2); + seed ^= k1; + } + + seed ^= data.len() as u32; + seed ^= seed >> 16; + seed = seed.wrapping_mul(0x85eb_ca6b); + seed ^= seed >> 13; + seed = seed.wrapping_mul(0xc2b2_ae35); + seed ^= seed >> 16; + seed +} + +#[cfg(test)] +mod tests { + use super::{BITS_PER_WORD, BloomKey, murmur3_v2}; + use crate::BloomFilterSettings; + use bstr::BStr; + + fn filter_bytes_for_path(path: &[u8], settings: BloomFilterSettings, len: usize) -> Vec { + let key = BloomKey::from_path(BStr::new(path), settings); + let mut out = vec![0u8; len]; + let modulo = (len as u64) * BITS_PER_WORD; + for i in 0..key.num_hashes { + let hash = key.h0.wrapping_add(i.wrapping_mul(key.h1)); + let bit_pos = u64::from(hash) % modulo; + let byte_pos = (bit_pos / BITS_PER_WORD) as usize; + out[byte_pos] |= 1u8 << (bit_pos % BITS_PER_WORD); + } + out + } + + #[test] + fn murmur3_known_vectors_match_git_and_reference_values() { + assert_eq!(murmur3_v2(0, b""), 0x0000_0000); + assert_eq!(murmur3_v2(0, b"Hello world!"), 0x627b_0c2c); + assert_eq!( + murmur3_v2(0, b"The quick brown fox jumps over the lazy dog"), + 0x2e4f_f723 + ); + } + + #[test] + fn murmur3_v2_matches_git_high_bit_vector() { + assert_eq!(murmur3_v2(0, b"\x99\xaa\xbb\xcc\xdd\xee\xff"), 0xa183_ccfd); + } + + #[test] + fn bloom_key_for_empty_path_matches_git_v1_vector() { + let settings = BloomFilterSettings { + hash_version: 1, + num_hashes: 7, + bits_per_entry: 10, + }; + let key = BloomKey::from_path(BStr::new(b""), settings); + assert_eq!( + (0..key.num_hashes) + .map(|i| key.h0.wrapping_add(i.wrapping_mul(key.h1))) + .collect::>(), + &[ + 0x5615_800c, + 0x5b96_6560, + 0x6117_4ab4, + 0x6698_3008, + 0x6c19_155c, + 0x7199_fab0, + 0x771a_e004 + ] + ); + } + + #[test] + fn bloom_key_for_empty_path_matches_git_v2_vector() { + let settings = BloomFilterSettings { + hash_version: 2, + num_hashes: 7, + bits_per_entry: 10, + }; + let key = BloomKey::from_path(BStr::new(b""), settings); + assert_eq!( + (0..key.num_hashes) + .map(|i| key.h0.wrapping_add(i.wrapping_mul(key.h1))) + .collect::>(), + &[ + 0x5615_800c, + 0x5b96_6560, + 0x6117_4ab4, + 0x6698_3008, + 0x6c19_155c, + 0x7199_fab0, + 0x771a_e004 + ] + ); + } + + #[test] + fn bloom_key_for_high_bit_path_differs_between_versions() { + let path = BStr::new(b"\xc2\xa2"); + let v1 = BloomKey::from_path( + path, + BloomFilterSettings { + hash_version: 1, + num_hashes: 7, + bits_per_entry: 10, + }, + ); + let v2 = BloomKey::from_path( + path, + BloomFilterSettings { + hash_version: 2, + num_hashes: 7, + bits_per_entry: 10, + }, + ); + assert_ne!(v1, v2); + } + + #[test] + fn bloom_filter_for_high_bit_path_matches_git_v1_and_v2_vectors() { + let path = b"\xc2\xa2"; + assert_eq!( + filter_bytes_for_path( + path, + BloomFilterSettings { + hash_version: 1, + num_hashes: 7, + bits_per_entry: 10, + }, + 2, + ), + vec![0x52, 0xa9] + ); + assert_eq!( + filter_bytes_for_path( + path, + BloomFilterSettings { + hash_version: 2, + num_hashes: 7, + bits_per_entry: 10, + }, + 2, + ), + vec![0xc0, 0x1f] + ); + } +} diff --git a/crates/gix-commitgraph/src/file/access.rs b/crates/gix-commitgraph/src/file/access.rs new file mode 100644 index 0000000000000000000000000000000000000000..94fbd2b6ff2a7557fb77f4f447a82fc6b3821550 --- /dev/null +++ b/crates/gix-commitgraph/src/file/access.rs @@ -0,0 +1,165 @@ +use std::{ + fmt::{Debug, Formatter}, + path::Path, +}; + +use crate::{ + BloomFilterSettings, File, + file::{self, COMMIT_DATA_ENTRY_SIZE_SANS_HASH, commit::Commit}, +}; + +/// Access +impl File { + /// The number of base graphs that this file depends on. + pub fn base_graph_count(&self) -> u8 { + self.base_graph_count + } + + /// Returns the commit data for the commit located at the given lexicographical position. + /// + /// `pos` must range from 0 to `self.num_commits()`. + /// + /// # Panics + /// + /// Panics if `pos` is out of bounds. + pub fn commit_at(&self, pos: file::Position) -> Commit<'_> { + Commit::new(self, pos) + } + + /// The kind of hash used in this File. + /// + /// Note that it is always conforming to the hash used in the owning repository. + pub fn object_hash(&self) -> gix_hash::Kind { + self.object_hash + } + + /// Returns an object id at the given index in our list of (sorted) hashes. + /// The position ranges from 0 to `self.num_commits()` + // copied from gix-odb/src/pack/index/ext + pub fn id_at(&self, pos: file::Position) -> &gix_hash::oid { + assert!( + pos.0 < self.num_commits(), + "expected lexicographical position less than {}, got {}", + self.num_commits(), + pos.0 + ); + let pos: usize = pos + .0 + .try_into() + .expect("an architecture able to hold 32 bits of integer"); + let start = self.oid_lookup_offset + (pos * self.hash_len); + gix_hash::oid::from_bytes_unchecked(&self.data[start..][..self.hash_len]) + } + + /// Return an iterator over all object hashes stored in the base graph. + pub fn iter_base_graph_ids(&self) -> impl Iterator { + let start = self.base_graphs_list_offset.unwrap_or(0); + let base_graphs_list = + &self.data[start..][..self.hash_len * usize::from(self.base_graph_count)]; + base_graphs_list + .chunks_exact(self.hash_len) + .map(gix_hash::oid::from_bytes_unchecked) + } + + /// return an iterator over all commits in this file. + pub fn iter_commits(&self) -> impl Iterator> { + (0..self.num_commits()).map(move |i| self.commit_at(file::Position(i))) + } + + /// Return an iterator over all object hashes stored in this file. + pub fn iter_ids(&self) -> impl Iterator { + (0..self.num_commits()).map(move |i| self.id_at(file::Position(i))) + } + + /// Translate the given object hash to its position within this file, if present. + // copied from gix-odb/src/pack/index/ext + pub fn lookup(&self, id: impl AsRef) -> Option { + self.lookup_inner(id.as_ref()) + } + + fn lookup_inner(&self, id: &gix_hash::oid) -> Option { + let first_byte = usize::from(id.first_byte()); + let mut upper_bound = self.fan[first_byte]; + let mut lower_bound = if first_byte != 0 { + self.fan[first_byte - 1] + } else { + 0 + }; + + while lower_bound < upper_bound { + let mid = u32::midpoint(lower_bound, upper_bound); + let mid_sha = self.id_at(file::Position(mid)); + + use std::cmp::Ordering::*; + match id.cmp(mid_sha) { + Less => upper_bound = mid, + Equal => return Some(file::Position(mid)), + Greater => lower_bound = mid + 1, + } + } + None + } + + /// Returns the number of commits in this graph file. + /// + /// The maximum valid `file::Position` that can be used with this file is one less than + /// `num_commits()`. + pub fn num_commits(&self) -> u32 { + self.fan[255] + } + + /// Returns the path to this file. + pub fn path(&self) -> &Path { + &self.path + } + + /// Return changed-path Bloom filter settings if this file has a usable Bloom index and data pair. + pub fn bloom_filter_settings(&self) -> Option { + self.bloom_filter_settings + } +} + +impl File { + /// Returns the byte slice for the given commit in this file's Commit Data (CDAT) chunk. + pub(crate) fn commit_data_bytes(&self, pos: file::Position) -> &[u8] { + assert!( + pos.0 < self.num_commits(), + "expected lexicographical position less than {}, got {}", + self.num_commits(), + pos.0 + ); + let pos: usize = pos + .0 + .try_into() + .expect("an architecture able to hold 32 bits of integer"); + let entry_size = self.hash_len + COMMIT_DATA_ENTRY_SIZE_SANS_HASH; + let start = self.commit_data_offset + (pos * entry_size); + &self.data[start..][..entry_size] + } + + /// Returns the byte slice for this file's entire Extra Edge List (EDGE) chunk. + pub(crate) fn extra_edges_data(&self) -> Option<&[u8]> { + Some(&self.data[self.extra_edges_list_range.clone()?]) + } + + pub(crate) fn has_generation_data(&self) -> bool { + self.generation_data_offset.is_some() + } + + pub(crate) fn disable_generation_data(&mut self) { + self.read_generation_data = false; + } + + pub(crate) fn clear_bloom_filters(&mut self) { + self.bloom_filter_data_len = 0; + self.bloom_filter_data_offset = None; + self.bloom_filter_index_offset = None; + self.bloom_filter_settings = None; + } +} + +impl Debug for File { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, r#"File("{:?}")"#, self.path.display()) + } +} diff --git a/crates/gix-commitgraph/src/file/commit.rs b/crates/gix-commitgraph/src/file/commit.rs new file mode 100644 index 0000000000000000000000000000000000000000..e3db867b00dc01aaf25c8ddb70969773ab6d5ecb --- /dev/null +++ b/crates/gix-commitgraph/src/file/commit.rs @@ -0,0 +1,329 @@ +//! Low-level operations on individual commits. +use crate::{ + File, Position, + bloom::BloomKey, + file::{ + self, CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW, EXTENDED_EDGES_MASK, LAST_EXTENDED_EDGE_MASK, + NO_PARENT, + }, + from_be_u32, +}; +use bstr::BStr; +use gix_error::{Message, message}; +use std::{ + fmt::{Debug, Formatter}, + slice::Chunks, +}; + +/// A commit as stored in a [`File`]. +#[derive(Copy, Clone)] +pub struct Commit<'a> { + pub(crate) file: &'a File, + pos: file::Position, + // We can parse the below fields lazily if needed. + commit_timestamp: u64, + generation: u32, + parent1: ParentEdge, + parent2: ParentEdge, + root_tree_id: &'a gix_hash::oid, +} + +impl<'a> Commit<'a> { + pub(crate) fn new(file: &'a File, pos: file::Position) -> Self { + let bytes = file.commit_data_bytes(pos); + Commit { + file, + pos, + root_tree_id: gix_hash::oid::from_bytes_unchecked(&bytes[..file.hash_len]), + parent1: ParentEdge::from_raw(from_be_u32(&bytes[file.hash_len..][..4])), + parent2: ParentEdge::from_raw(from_be_u32(&bytes[file.hash_len + 4..][..4])), + generation: from_be_u32(&bytes[file.hash_len + 8..][..4]) >> 2, + commit_timestamp: u64::from_be_bytes( + bytes[file.hash_len + 8..][..8].try_into().unwrap(), + ) & 0x0003_ffff_ffff, + } + } + + /// Returns the committer timestamp of this commit. + /// + /// The value is the number of seconds since 1970-01-01 00:00:00 UTC. + pub fn committer_timestamp(&self) -> u64 { + self.commit_timestamp + } + + /// Returns the generation number of this commit. + /// + /// Commits without parents have generation number 1. Commits with parents have a generation + /// number that is the max of their parents' generation numbers + 1. + pub fn generation(&self) -> u32 { + self.generation + } + + /// Returns the corrected commit date of this commit (generation number v2), or `None` if the + /// owning graph has no generation data. + /// + /// It is the larger of this commit's committer timestamp and its parents' corrected commit dates plus one. + pub fn corrected_commit_date(&self) -> Result, Message> { + let file = self.file; + let Some(start) = file + .generation_data_offset + .filter(|_| file.read_generation_data) + else { + return Ok(None); + }; + let offset = from_be_u32(&file.data[start + self.pos.0 as usize * 4..][..4]); + let offset = if offset & CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW == 0 { + u64::from(offset) + } else { + let range = file.generation_data_overflow_range.clone().ok_or_else(|| { + message!( + "Commit {} requires overflow generation data, but there is none", + self.id() + ) + })?; + let index = (offset ^ CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW) as usize; + let bytes = file.data[range] + .get(index * 8..) + .and_then(|bytes| bytes.get(..8)) + .ok_or_else(|| { + message!( + "Commit {} has overflow generation data index {index} out of range", + self.id() + ) + })?; + u64::from_be_bytes(bytes.try_into().expect("8 bytes")) + }; + self.commit_timestamp + .checked_add(offset) + .map(Some) + .ok_or_else(|| { + message!( + "Commit {} has a corrected commit date that overflows", + self.id() + ) + }) + } + + /// Returns an iterator over the parent positions for lookup in the owning [Graph][crate::Graph]. + pub fn iter_parents(self) -> Parents<'a> { + // I didn't find a combinator approach that a) was as strict as ParentIterator, b) supported + // fuse-after-first-error behavior, and b) was significantly shorter or more understandable + // than ParentIterator. So here we are. + Parents { + commit_data: self, + state: ParentIteratorState::First, + } + } + + /// Returns the hash of this commit. + pub fn id(&self) -> &'a gix_hash::oid { + self.file.id_at(self.pos) + } + + /// Returns the first parent of this commit. + pub fn parent1(&self) -> Result, Message> { + self.iter_parents().next().transpose() + } + + /// Returns the position at which this commit is stored in the parent [File]. + pub fn position(&self) -> file::Position { + self.pos + } + + /// Return the hash of the tree this commit points to. + pub fn root_tree_id(&self) -> &gix_hash::oid { + self.root_tree_id + } + + /// Return the raw bytes of this commit's changed-path Bloom filter, see [`File::bloom_filter_at()`]. + pub fn bloom_filter(&self) -> Option<&'a [u8]> { + self.file.bloom_filter_at(self.pos) + } + + /// Query if `path` may be present in this commit's changed-path Bloom filter. + pub fn maybe_contains_path(&self, path: &BStr) -> Option { + self.file.maybe_contains_path(self.pos, path) + } + + /// Query if all `keys` may be present in this commit's changed-path Bloom filter. + pub fn maybe_contains_all_keys(&self, keys: &[BloomKey]) -> Option { + self.file.maybe_contains_all_keys(self.pos, keys) + } +} + +impl Debug for Commit<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Commit {{ id: {}, lex_pos: {}, generation: {}, root_tree_id: {}, parent1: {:?}, parent2: {:?} }}", + self.id(), + self.pos, + self.generation(), + self.root_tree_id(), + self.parent1, + self.parent2, + ) + } +} + +impl Eq for Commit<'_> {} + +impl PartialEq for Commit<'_> { + fn eq(&self, other: &Self) -> bool { + std::ptr::eq(self.file, other.file) && self.pos == other.pos + } +} + +/// An iterator over parents of a [`Commit`]. +pub struct Parents<'a> { + commit_data: Commit<'a>, + state: ParentIteratorState<'a>, +} + +impl Iterator for Parents<'_> { + type Item = Result; + + fn next(&mut self) -> Option { + let state = std::mem::replace(&mut self.state, ParentIteratorState::Exhausted); + match state { + ParentIteratorState::First => match self.commit_data.parent1 { + ParentEdge::None => match self.commit_data.parent2 { + ParentEdge::None => None, + _ => Some(Err(message!( + "commit {} has a second parent but not a first parent", + self.commit_data.id() + ))), + }, + ParentEdge::GraphPosition(pos) => { + self.state = ParentIteratorState::Second; + Some(Ok(pos)) + } + ParentEdge::ExtraEdgeIndex(_) => Some(Err(message!( + "commit {}'s first parent is an extra edge index, which is invalid", + self.commit_data.id(), + ))), + }, + ParentIteratorState::Second => match self.commit_data.parent2 { + ParentEdge::None => None, + ParentEdge::GraphPosition(pos) => Some(Ok(pos)), + ParentEdge::ExtraEdgeIndex(extra_edge_index) => { + if let Some(extra_edges_list) = self.commit_data.file.extra_edges_data() { + let start_offset: usize = extra_edge_index + .try_into() + .expect("an architecture able to hold 32 bits of integer"); + let start_offset = start_offset + .checked_mul(4) + .expect("an extended edge index small enough to fit in usize"); + if let Some(tail) = extra_edges_list.get(start_offset..) { + self.state = ParentIteratorState::Extra(tail.chunks(4)); + // This recursive call is what blocks me from replacing ParentIterator + // with a std::iter::from_fn closure. + self.next() + } else { + Some(Err(message!( + "commit {}'s extra edges overflows the commit-graph file's extra edges list", + self.commit_data.id() + ))) + } + } else { + Some(Err(message!( + "commit {} has extra edges, but commit-graph file has no extra edges list", + self.commit_data.id() + ))) + } + } + }, + ParentIteratorState::Extra(mut chunks) => { + if let Some(chunk) = chunks.next() { + let extra_edge = from_be_u32(chunk); + match ExtraEdge::from_raw(extra_edge) { + ExtraEdge::Internal(pos) => { + self.state = ParentIteratorState::Extra(chunks); + Some(Ok(pos)) + } + ExtraEdge::Last(pos) => Some(Ok(pos)), + } + } else { + Some(Err(message!( + "commit {}'s extra edges overflows the commit-graph file's extra edges list", + self.commit_data.id() + ))) + } + } + ParentIteratorState::Exhausted => None, + } + } + + fn size_hint(&self) -> (usize, Option) { + match ( + &self.state, + self.commit_data.parent1, + self.commit_data.parent2, + ) { + (ParentIteratorState::First, ParentEdge::None, ParentEdge::None) => (0, Some(0)), + (ParentIteratorState::First, ParentEdge::None, _) => (1, Some(1)), + (ParentIteratorState::First, ParentEdge::GraphPosition(_), ParentEdge::None) => { + (1, Some(1)) + } + ( + ParentIteratorState::First, + ParentEdge::GraphPosition(_), + ParentEdge::GraphPosition(_), + ) => (2, Some(2)), + ( + ParentIteratorState::First, + ParentEdge::GraphPosition(_), + ParentEdge::ExtraEdgeIndex(_), + ) => (3, None), + (ParentIteratorState::First, ParentEdge::ExtraEdgeIndex(_), _) => (1, Some(1)), + (ParentIteratorState::Second, _, ParentEdge::None) => (0, Some(0)), + (ParentIteratorState::Second, _, ParentEdge::GraphPosition(_)) => (1, Some(1)), + (ParentIteratorState::Second, _, ParentEdge::ExtraEdgeIndex(_)) => (2, None), + (ParentIteratorState::Extra(_), _, _) => (1, None), + (ParentIteratorState::Exhausted, _, _) => (0, Some(0)), + } + } +} + +#[derive(Debug)] +enum ParentIteratorState<'a> { + First, + Second, + Extra(Chunks<'a, u8>), + Exhausted, +} + +#[derive(Clone, Copy, Debug)] +enum ParentEdge { + None, + GraphPosition(Position), + ExtraEdgeIndex(u32), +} + +impl ParentEdge { + pub fn from_raw(raw: u32) -> ParentEdge { + if raw == NO_PARENT { + return ParentEdge::None; + } + if raw & EXTENDED_EDGES_MASK != 0 { + ParentEdge::ExtraEdgeIndex(raw & !EXTENDED_EDGES_MASK) + } else { + ParentEdge::GraphPosition(Position(raw)) + } + } +} + +enum ExtraEdge { + Internal(Position), + Last(Position), +} + +impl ExtraEdge { + pub fn from_raw(raw: u32) -> Self { + if raw & LAST_EXTENDED_EDGE_MASK != 0 { + Self::Last(Position(raw & !LAST_EXTENDED_EDGE_MASK)) + } else { + Self::Internal(Position(raw)) + } + } +} diff --git a/crates/gix-commitgraph/src/file/init.rs b/crates/gix-commitgraph/src/file/init.rs new file mode 100644 index 0000000000000000000000000000000000000000..3e743f1f3a1ed1323183aa9c3d5fb2af07787511 --- /dev/null +++ b/crates/gix-commitgraph/src/file/init.rs @@ -0,0 +1,268 @@ +use std::path::{Path, PathBuf}; + +use gix_error::{ErrorExt, Exn, Message, ResultExt, message}; + +use crate::{ + BloomFilterSettings, File, + file::{ + BASE_GRAPHS_LIST_CHUNK_ID, BLOOM_FILTER_DATA_CHUNK_ID, BLOOM_FILTER_HEADER_SIZE, + BLOOM_FILTER_INDEX_CHUNK_ID, COMMIT_DATA_CHUNK_ID, COMMIT_DATA_ENTRY_SIZE_SANS_HASH, + EXTENDED_EDGES_LIST_CHUNK_ID, FAN_LEN, GENERATION_DATA_CHUNK_ID, + GENERATION_DATA_OVERFLOW_CHUNK_ID, HEADER_LEN, OID_FAN_CHUNK_ID, OID_LOOKUP_CHUNK_ID, + SIGNATURE, + }, + from_be_u32, +}; + +const MIN_FILE_SIZE: usize = HEADER_LEN + + gix_chunk::file::Index::size_for_entries(3 /*OIDF, OIDL, CDAT*/) + + FAN_LEN * 4 /* FANOUT TABLE CHUNK OIDF */ + + gix_hash::Kind::shortest().len_in_bytes(); + +impl File { + /// Try to parse the commit graph file at `path`. + pub fn at(path: impl AsRef) -> Result> { + Self::try_from(path.as_ref()) + } + + /// A lower-level constructor which constructs a new instance directly from the mapping in `data`, + /// assuming that it originated from `path`. + /// + /// Note that `path` is only used for verification of the hash its basename contains, but otherwise + /// is not of importance. + pub fn new(data: memmap2::Mmap, path: PathBuf) -> Result> { + let data_size = data.len(); + if data_size < MIN_FILE_SIZE { + return Err(message("Commit-graph file too small even for an empty graph").raise()); + } + + let mut ofs = 0; + if &data[ofs..ofs + SIGNATURE.len()] != SIGNATURE { + return Err( + message("Commit-graph file does not start with expected signature").raise(), + ); + } + ofs += SIGNATURE.len(); + + match data[ofs] { + 1 => (), + x => { + return Err(message!("Unsupported commit-graph file version: {x}").raise()); + } + } + ofs += 1; + + let object_hash = gix_hash::Kind::try_from(data[ofs]).map_err(|v| { + message!("Commit-graph file uses unsupported hash version: {v}").raise() + })?; + ofs += 1; + + let chunk_count = data[ofs]; + // Can assert chunk_count >= MIN_CHUNKS here, but later OIDF+OIDL+CDAT presence checks make + // it redundant. + ofs += 1; + + let base_graph_count = data[ofs]; + ofs += 1; + + let chunks = gix_chunk::file::Index::from_bytes(&data, ofs, u32::from(chunk_count)) + .or_raise(|| { + message!( + "Couldn't read commit-graph file with {chunk_count} chunks at offset {ofs}" + ) + })?; + + let base_graphs_list_offset = chunks + .validated_usize_offset_by_id(BASE_GRAPHS_LIST_CHUNK_ID, |chunk_range| { + let chunk_size = chunk_range.len(); + if chunk_size % object_hash.len_in_bytes() != 0 { + return Err(message!("Commit-graph chunk {BASE_GRAPHS_LIST_CHUNK_ID:?} has invalid size: {msg}", + msg = format!( + "chunk size {} is not a multiple of {}", + chunk_size, + object_hash.len_in_bytes() + ), + ).raise()); + } + let chunk_base_graph_count: u32 = (chunk_size / object_hash.len_in_bytes()) + .try_into() + .expect("base graph count to fit in 32-bits"); + if chunk_base_graph_count != u32::from(base_graph_count) { + return Err(message!("Commit-graph {BASE_GRAPHS_LIST_CHUNK_ID:?} chunk contains {chunk_base_graph_count} base graphs, but commit-graph file header claims {base_graph_count} base graphs").raise()) + } + Ok(chunk_range.start) + }) + .ok() + .transpose()?; + + let (commit_data_offset, commit_data_count): (_, u32) = chunks + .validated_usize_offset_by_id(COMMIT_DATA_CHUNK_ID, |chunk_range| { + let chunk_size = chunk_range.len(); + + let entry_size = object_hash.len_in_bytes() + COMMIT_DATA_ENTRY_SIZE_SANS_HASH; + if chunk_size % entry_size != 0 { + return Err(message!("Commit-graph chunk {COMMIT_DATA_CHUNK_ID:?} has invalid size: chunk size {chunk_size} is not a multiple of {entry_size}").raise()) + } + Ok(( + chunk_range.start, + (chunk_size / entry_size) + .try_into() + .expect("number of commits in CDAT chunk to fit in 32 bits"), + )) + })??; + + let fan_offset = chunks + .validated_usize_offset_by_id(OID_FAN_CHUNK_ID, |chunk_range| { + let chunk_size = chunk_range.len(); + + let expected_size = 4 * FAN_LEN; + if chunk_size != expected_size { + return Err(message!("Commit-graph chunk {OID_FAN_CHUNK_ID:?} has invalid size: expected chunk length {expected_size}, got {chunk_size}").raise()) + } + Ok(chunk_range.start) + })? + .or_raise(|| message("Error getting offset for OID fan chunk"))?; + + let (oid_lookup_offset, oid_lookup_count): (_, u32) = chunks + .validated_usize_offset_by_id(OID_LOOKUP_CHUNK_ID, |chunk_range| { + let chunk_size = chunk_range.len(); + + if chunk_size % object_hash.len_in_bytes() != 0 { + return Err(message!("Commit-graph chunk {OID_LOOKUP_CHUNK_ID:?} has invalid size: chunk size {chunk_size} is not a multiple of {hash_len}", hash_len = object_hash.len_in_bytes()).raise()) + } + Ok(( + chunk_range.start, + (chunk_size / object_hash.len_in_bytes()) + .try_into() + .expect("number of commits in OIDL chunk to fit in 32 bits"), + )) + })? + .or_raise(|| message("Error getting offset for OID lookup chunk"))?; + + let extra_edges_list_range = chunks.usize_offset_by_id(EXTENDED_EDGES_LIST_CHUNK_ID).ok(); + // A wrongly sized generation data chunk is ignored instead of rejecting the file. + let generation_data_offset = chunks + .usize_offset_by_id(GENERATION_DATA_CHUNK_ID) + .ok() + .filter(|range| range.len() / 4 == commit_data_count as usize) + .map(|range| range.start); + // The overflow chunk is only checked when reading from it. + let generation_data_overflow_range = chunks + .usize_offset_by_id(GENERATION_DATA_OVERFLOW_CHUNK_ID) + .ok(); + let bloom_filter_index_range = chunks.usize_offset_by_id(BLOOM_FILTER_INDEX_CHUNK_ID).ok(); + let bloom_filter_data_range = chunks.usize_offset_by_id(BLOOM_FILTER_DATA_CHUNK_ID).ok(); + + let mut bloom_filter_data_len = 0usize; + let mut bloom_filter_data_offset = None; + let mut bloom_filter_index_offset = None; + let mut bloom_filter_settings = None; + if let (Some(index_range), Some(data_range)) = ( + bloom_filter_index_range.as_ref(), + bloom_filter_data_range.as_ref(), + ) { + let expected_index_chunk_size = commit_data_count as usize * std::mem::size_of::(); + if index_range.len() == expected_index_chunk_size + && data_range.len() >= BLOOM_FILTER_HEADER_SIZE + { + let settings = BloomFilterSettings { + hash_version: from_be_u32(&data[data_range.start..][..4]), + num_hashes: from_be_u32(&data[data_range.start + 4..][..4]), + bits_per_entry: from_be_u32(&data[data_range.start + 8..][..4]), + }; + let bloom_data_payload_len = data_range.len() - BLOOM_FILTER_HEADER_SIZE; + if settings.is_supported() { + bloom_filter_settings = Some(settings); + bloom_filter_data_len = bloom_data_payload_len; + bloom_filter_data_offset = Some(data_range.start + BLOOM_FILTER_HEADER_SIZE); + bloom_filter_index_offset = Some(index_range.start); + } + } + } + + let trailer = &data[chunks.highest_offset() as usize..]; + if trailer.len() != object_hash.len_in_bytes() { + return Err(message!( + "Expected commit-graph trailer to contain {} bytes, got {}", + object_hash.len_in_bytes(), + trailer.len() + ) + .raise()); + } + + if base_graph_count > 0 && base_graphs_list_offset.is_none() { + return Err(message!( + "Chunk named {BASE_GRAPHS_LIST_CHUNK_ID:?} was not found in chunk file index" + ) + .into()); + } + + let (fan, _) = read_fan(&data[fan_offset..]); + if oid_lookup_count != fan[255] { + return Err(message!("Commit-graph {OID_FAN_CHUNK_ID:?} chunk contains {chunk1_commits} commits, but {OID_LOOKUP_CHUNK_ID:?} chunk contains {chunk2_commits} commits", + chunk1_commits = fan[255], + chunk2_commits = oid_lookup_count, + ).raise()); + } + if commit_data_count != fan[255] { + return Err( + message!("Commit-graph {OID_FAN_CHUNK_ID:?} chunk contains {chunk1_commits} commits, but {COMMIT_DATA_CHUNK_ID:?} chunk contains {chunk2_commits} commits", + chunk1_commits = fan[255], + chunk2_commits = commit_data_count, + ).raise(), + ); + } + Ok(File { + base_graph_count, + base_graphs_list_offset, + bloom_filter_data_len, + bloom_filter_data_offset, + bloom_filter_index_offset, + bloom_filter_settings, + commit_data_offset, + data, + extra_edges_list_range, + fan, + generation_data_offset, + generation_data_overflow_range, + read_generation_data: generation_data_offset.is_some(), + oid_lookup_offset, + path, + hash_len: object_hash.len_in_bytes(), + object_hash, + }) + } +} + +impl TryFrom<&Path> for File { + type Error = Exn; + + fn try_from(path: &Path) -> Result { + let data = std::fs::File::open(path) + .and_then(|file| { + // SAFETY: we have to take the risk of somebody changing the file underneath. Git never writes into the same file. + #[allow(unsafe_code)] + unsafe { + memmap2::MmapOptions::new().map_copy_read_only(&file) + } + }) + .or_raise(|| { + message!( + "Could not open commit-graph file at '{path}'", + path = path.display() + ) + })?; + Self::new(data, path.to_owned()) + } +} + +// Copied from gix-odb/pack/index/init.rs +fn read_fan(d: &[u8]) -> ([u32; FAN_LEN], usize) { + assert!(d.len() >= FAN_LEN * 4); + + let mut fan = [0; FAN_LEN]; + for (c, f) in d.chunks_exact(4).zip(fan.iter_mut()) { + *f = u32::from_be_bytes(c.try_into().unwrap()); + } + (fan, FAN_LEN * 4) +} diff --git a/crates/gix-commitgraph/src/file/mod.rs b/crates/gix-commitgraph/src/file/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..9c05a60d0b6986b5fd3916572af5b67486fb040e --- /dev/null +++ b/crates/gix-commitgraph/src/file/mod.rs @@ -0,0 +1,52 @@ +//! Operations on a single commit-graph file. + +use std::fmt::{Display, Formatter}; + +pub use self::commit::Commit; + +mod access; +pub mod commit; +mod init; +pub mod verify; + +const COMMIT_DATA_ENTRY_SIZE_SANS_HASH: usize = 16; +pub(crate) const FAN_LEN: usize = 256; +const HEADER_LEN: usize = 8; + +pub(crate) const SIGNATURE: &[u8] = b"CGPH"; + +pub(crate) type ChunkId = gix_chunk::Id; +pub(crate) const BASE_GRAPHS_LIST_CHUNK_ID: ChunkId = *b"BASE"; +pub(crate) const BLOOM_FILTER_DATA_CHUNK_ID: ChunkId = *b"BDAT"; +pub(crate) const BLOOM_FILTER_INDEX_CHUNK_ID: ChunkId = *b"BIDX"; +const BLOOM_FILTER_HEADER_SIZE: usize = 3 * std::mem::size_of::(); +pub(crate) const COMMIT_DATA_CHUNK_ID: ChunkId = *b"CDAT"; +pub(crate) const EXTENDED_EDGES_LIST_CHUNK_ID: ChunkId = *b"EDGE"; +pub(crate) const GENERATION_DATA_CHUNK_ID: ChunkId = *b"GDA2"; +pub(crate) const GENERATION_DATA_OVERFLOW_CHUNK_ID: ChunkId = *b"GDO2"; +pub(crate) const OID_FAN_CHUNK_ID: ChunkId = *b"OIDF"; +pub(crate) const OID_LOOKUP_CHUNK_ID: ChunkId = *b"OIDL"; + +// Note that git's commit-graph-format.txt as of v2.28.0 gives an incorrect value 0x0700_0000 for +// NO_PARENT. Fixed in https://github.com/git/git/commit/4d515253afcef985e94400adbfed7044959f9121 . +pub(crate) const NO_PARENT: u32 = 0x7000_0000; +pub(crate) const EXTENDED_EDGES_MASK: u32 = 0x8000_0000; +pub(crate) const LAST_EXTENDED_EDGE_MASK: u32 = 0x8000_0000; +pub(crate) const CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW: u32 = 0x8000_0000; + +/// The position of a given commit within a graph file, starting at 0. +/// +/// Commits within a graph file are sorted in lexicographical order by OID; a commit's lexicographical position +/// is its position in this ordering. If a commit graph spans multiple files, each file's commits +/// start at lexicographical position 0, so it is unique across a single file but is not unique across +/// the whole commit graph. Each commit also has a graph position ([`Position`][crate::Position]), +/// which is unique across the whole commit graph. +/// In order to avoid accidentally mixing lexicographical positions with graph positions, distinct types are used for each. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct Position(pub u32); + +impl Display for Position { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} diff --git a/crates/gix-commitgraph/src/file/verify.rs b/crates/gix-commitgraph/src/file/verify.rs new file mode 100644 index 0000000000000000000000000000000000000000..b1c5c6a953f68a540f77a4625b35976d9bc01647 --- /dev/null +++ b/crates/gix-commitgraph/src/file/verify.rs @@ -0,0 +1,155 @@ +//! Auxiliary types used in commit graph file verification methods. +use std::{ + cmp::{max, min}, + collections::HashMap, + path::Path, +}; + +use gix_error::{ErrorExt, Exn, Message, ResultExt, message}; + +use crate::{File, GENERATION_NUMBER_INFINITY, GENERATION_NUMBER_MAX, file}; + +/// The positive result of [`File::traverse()`] providing some statistical information. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct Outcome { + /// The largest encountered [`file::Commit`] generation number. + pub max_generation: u32, + /// The smallest encountered [`file::Commit`] generation number. + pub min_generation: u32, + /// The largest number of parents in a single [`file::Commit`]. + pub max_parents: u32, + /// The total number of [`commits`][file::Commit]s seen in the iteration. + pub num_commits: u32, + /// A mapping of `N -> number of commits with N parents`. + pub parent_counts: HashMap, +} + +/// Verification +impl File { + /// Returns the trailing checksum over the entire content of this file. + pub fn checksum(&self) -> &gix_hash::oid { + gix_hash::oid::from_bytes_unchecked(&self.data[self.data.len() - self.hash_len..]) + } + + /// Traverse all [commits][file::Commit] stored in this file and call `processor(commit) -> Result<(), Error>` on it. + /// + /// If the `processor` fails, the iteration will be stopped and the entire call results in the respective error. + pub fn traverse<'a, Processor>( + &'a self, + mut processor: Processor, + ) -> Result> + where + Processor: FnMut(&file::Commit<'a>) -> Result<(), Exn>, + { + self.verify_checksum()?; + verify_split_chain_filename_hash(&self.path, self.checksum())?; + + let null_id = self.object_hash().null_ref(); + + let mut stats = Outcome { + max_generation: 0, + max_parents: 0, + min_generation: GENERATION_NUMBER_INFINITY, + num_commits: self.num_commits(), + parent_counts: HashMap::new(), + }; + + // TODO: Verify self.fan values as we go. + let mut prev_id: &gix_hash::oid = null_id; + for commit in self.iter_commits() { + if commit.id() <= prev_id { + if commit.id() == null_id { + return Err(message!( + "commit at file position {} has invalid ID {}", + commit.position(), + commit.id() + ) + .raise()); + } + return Err(message!( + "commit at file position {} with ID {} is out of order relative to its predecessor with ID {prev_id}", + commit.position(), + commit.id() + ) + .raise()); + } + if commit.root_tree_id() == null_id { + return Err(message!( + "commit {} has invalid root tree ID {}", + commit.id(), + commit.root_tree_id() + ) + .raise()); + } + if commit.generation() > GENERATION_NUMBER_MAX { + return Err(message!( + "commit {} has invalid generation {}", + commit.id(), + commit.generation() + ) + .raise()); + } + + processor(&commit) + .or_raise(|| message!("processor failed on commit {}", commit.id()))?; + + stats.max_generation = max(stats.max_generation, commit.generation()); + stats.min_generation = min(stats.min_generation, commit.generation()); + let parent_count = commit + .iter_parents() + .try_fold(0u32, |acc, pos| pos.map(|_| acc + 1))?; + *stats.parent_counts.entry(parent_count).or_insert(0) += 1; + prev_id = commit.id(); + } + + if stats.min_generation == GENERATION_NUMBER_INFINITY { + stats.min_generation = 0; + } + + Ok(stats) + } + + /// Assure the [`checksum`][File::checksum()] matches the actual checksum over all content of this file, excluding the trailing + /// checksum itself. + /// + /// Return the actual checksum on success or [`Exn`] if there is a mismatch. + pub fn verify_checksum(&self) -> Result> { + // Even though we could use gix_hash::bytes_of_file(…), this would require extending our + // Error type to support io::Error. As we only gain progress, there probably isn't much value + // as these files are usually small enough to process them in less than a second, even for the large ones. + // But it's possible, once a progress instance is passed. + let data_len_without_trailer = self.data.len() - self.hash_len; + let mut hasher = gix_hash::hasher(self.object_hash()); + hasher.update(&self.data[..data_len_without_trailer]); + let actual = hasher + .try_finalize() + .map_err(|e| message!("failed to hash commit graph file: {e}").raise())?; + actual + .verify(self.checksum()) + .map_err(|e| message!("{e}").raise())?; + Ok(actual) + } +} + +/// If the given path's filename matches "graph-{hash}.graph", check that `hash` matches the +/// expected hash. +fn verify_split_chain_filename_hash( + path: &Path, + expected: &gix_hash::oid, +) -> Result<(), Exn> { + path.file_name() + .and_then(std::ffi::OsStr::to_str) + .and_then(|filename| filename.strip_suffix(".graph")) + .and_then(|stem| stem.strip_prefix("graph-")) + .map_or(Ok(()), |hex| { + match gix_hash::ObjectId::from_hex(hex.as_bytes()) { + Ok(actual) if actual == expected => Ok(()), + _ => Err(message!( + "commit-graph filename should be graph-{}.graph", + expected.to_hex() + ) + .raise()), + } + }) +} diff --git a/crates/gix-commitgraph/src/init.rs b/crates/gix-commitgraph/src/init.rs new file mode 100644 index 0000000000000000000000000000000000000000..74f62fc1dd12feb4beb58b446dd66102d1b223f6 --- /dev/null +++ b/crates/gix-commitgraph/src/init.rs @@ -0,0 +1,147 @@ +use crate::{BloomFilterSettings, File, Graph, MAX_COMMITS}; +use gix_error::{ErrorExt, Exn, Message, ResultExt, message}; +use std::{ + io::{BufRead, BufReader}, + path::Path, +}; + +/// Instantiate a `Graph` from various sources. +impl Graph { + /// Instantiate a commit graph from `path` which may be a directory containing graph files or the graph file itself. + pub fn at(path: &Path) -> Result> { + Self::try_from(path) + } + + /// Instantiate a commit graph from the directory containing all of its files. + /// + /// Loading stops at the first layer that can't be opened or parsed, whose checksum doesn't match its + /// name in the chain, or whose `BASE` chunk doesn't list the layers below it in chain order, keeping only the + /// layers loaded so far. It's an error if no layer could be loaded. + pub fn from_commit_graphs_dir(path: &Path) -> Result> { + let commit_graphs_dir = path; + let chain_file_path = commit_graphs_dir.join("commit-graph-chain"); + let chain_file = std::fs::File::open(&chain_file_path).or_raise(|| { + message!( + "Could not open commit-graph chain file at '{}'", + chain_file_path.display() + ) + })?; + let mut files = Vec::new(); + for line in BufReader::new(chain_file).lines() { + let hash = line.or_raise(|| { + message!( + "Could not read from commit-graph file at '{}'", + chain_file_path.display() + ) + })?; + let Ok(id) = gix_hash::ObjectId::from_hex(hash.as_bytes()) else { + break; + }; + let Ok(file) = File::at(commit_graphs_dir.join(format!("graph-{hash}.graph"))) else { + break; + }; + if file.checksum() != id || !base_graphs_match(&file, &files) { + break; + } + + files.push(file); + } + Ok(Self::new(files)?) + } + + /// Instantiate a commit graph from a `.git/objects/info/commit-graph` or + /// `.git/objects/info/commit-graphs/graph-*.graph` file. + pub fn from_file(path: &Path) -> Result> { + let file = File::at(path) + .or_raise(|| message!("Could not open commit-graph file at '{}'", path.display()))?; + Ok(Self::new(vec![file])?) + } + + /// Instantiate a commit graph from an `.git/objects/info` directory. + pub fn from_info_dir(info_dir: &Path) -> Result> { + Self::from_file(&info_dir.join("commit-graph")) + .or_else(|_| Self::from_commit_graphs_dir(&info_dir.join("commit-graphs"))) + } + + /// Create a new commit graph from a list of `files`. + pub fn new(files: Vec) -> Result { + let mut files = nonempty::NonEmpty::from_vec(files) + .ok_or_else(|| message!("Commit-graph must contain at least one file"))?; + let num_commits: u64 = files.iter().map(|f| u64::from(f.num_commits())).sum(); + if num_commits > u64::from(MAX_COMMITS) { + return Err(message!( + "Commit-graph files contain {num_commits} commits altogether, but only {MAX_COMMITS} commits are allowed" + )); + } + + let mut f1 = files.first(); + for f2 in files.tail() { + if f1.object_hash() != f2.object_hash() { + return Err(message!( + "Commit-graph files mismatch: '{path1}' uses hash {hash1:?}, but '{path2}' uses hash {hash2:?}", + path1 = f1.path().display(), + hash1 = f1.object_hash(), + path2 = f2.path().display(), + hash2 = f2.object_hash(), + )); + } + f1 = f2; + } + + disable_incompatible_bloom_layers(&mut files); + if !files.iter().all(File::has_generation_data) { + files.iter_mut().for_each(File::disable_generation_data); + } + + Ok(Self { files }) + } +} + +/// Return `true` if `file`'s `BASE` chunk lists the layers `below` it in chain order. +fn base_graphs_match(file: &File, below: &[File]) -> bool { + usize::from(file.base_graph_count()) >= below.len() + && file + .iter_base_graph_ids() + .zip(below) + .all(|(base, below)| base == below.checksum()) +} + +fn disable_incompatible_bloom_layers(files: &mut nonempty::NonEmpty) { + let mut preferred: Option = None; + for file in files.iter_mut().rev() { + let Some(settings) = file.bloom_filter_settings else { + continue; + }; + if preferred.is_none() { + preferred = Some(settings); + continue; + } + if preferred != Some(settings) { + file.clear_bloom_filters(); + } + } +} + +impl TryFrom<&Path> for Graph { + type Error = Exn; + + fn try_from(path: &Path) -> Result { + if path.is_file() { + // Assume we are looking at `.git/objects/info/commit-graph` or + // `.git/objects/info/commit-graphs/graph-*.graph`. + Self::from_file(path) + } else if path.is_dir() { + if path.join("commit-graph-chain").is_file() { + Self::from_commit_graphs_dir(path) + } else { + Self::from_info_dir(path) + } + } else { + Err(message!( + "Did not find any files that look like commit graphs at '{}'", + path.display() + ) + .raise()) + } + } +} diff --git a/crates/gix-commitgraph/src/lib.rs b/crates/gix-commitgraph/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..7a9c9d58a2611e2a1b52210c2774895bea00ad50 --- /dev/null +++ b/crates/gix-commitgraph/src/lib.rs @@ -0,0 +1,114 @@ +//! Read, verify, and traverse git commit graphs. +//! +//! A [commit graph][Graph] is an index of commits in the git commit history. +//! The [Graph] stores commit data in a way that accelerates lookups considerably compared to +//! traversing the git history by usual means. +//! +//! As generating the full commit graph from scratch can take some time, git may write new commits +//! to separate [files][File] instead of overwriting the original file. +//! Eventually, git will merge these files together as the number of files grows. +//! ## Feature Flags +#![cfg_attr( + all(doc, feature = "document-features"), + doc = ::document_features::document_features!() +)] +#![cfg_attr(all(doc, feature = "document-features"), feature(doc_cfg))] +#![deny(missing_docs, unsafe_code)] + +use gix_error::{Exn, Message}; +use std::path::Path; + +/// A single commit-graph file. +/// +/// All operations on a `File` are local to that graph file. Since a commit graph can span multiple +/// files, all interesting graph operations belong on [`Graph`]. +pub struct File { + base_graph_count: u8, + base_graphs_list_offset: Option, + bloom_filter_data_len: usize, + bloom_filter_data_offset: Option, + bloom_filter_index_offset: Option, + bloom_filter_settings: Option, + commit_data_offset: usize, + data: memmap2::Mmap, + extra_edges_list_range: Option>, + fan: [u32; file::FAN_LEN], + generation_data_offset: Option, + generation_data_overflow_range: Option>, + /// `false` if generation data is present but unusable because another file in the graph lacks it. + read_generation_data: bool, + oid_lookup_offset: usize, + path: std::path::PathBuf, + hash_len: usize, + object_hash: gix_hash::Kind, +} + +/// Bloom filter parameters as stored in a commit-graph `BDAT` chunk header. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct BloomFilterSettings { + /// Version of the hash algorithm used by changed-path Bloom filters. + pub hash_version: u32, + /// Number of hashes computed for each key. + pub num_hashes: u32, + /// Number of bits per Bloom filter entry. + pub bits_per_entry: u32, +} + +impl BloomFilterSettings { + pub(crate) fn is_supported(&self) -> bool { + match self.hash_version { + // v1 is deprecated, but existing graphs still use it. + 1 | 2 => true, + _ => false, + } + } +} + +/// A complete commit graph. +/// +/// The data in the commit graph may come from a monolithic `objects/info/commit-graph` file, or it +/// may come from one or more `objects/info/commit-graphs/graph-*.graph` files. These files are +/// generated via `git commit-graph write ...` commands. +pub struct Graph { + files: nonempty::NonEmpty, +} + +/// Instantiate a commit graph from an `.git/objects/info` directory, or one of the various commit-graph files. +pub fn at(path: impl AsRef) -> Result> { + Graph::at(path.as_ref()) +} + +mod access; +pub mod bloom; +pub mod file; +/// +pub mod init; +pub mod verify; +pub mod write; + +/// The number of generations that are considered 'infinite' commit history. +pub const GENERATION_NUMBER_INFINITY: u32 = 0xffff_ffff; +/// The largest valid generation number. +/// +/// If a commit's real generation number is larger than this, the commit graph will cap the value to +/// this number. +/// The largest distinct generation number is `GENERATION_NUMBER_MAX - 1`. +pub const GENERATION_NUMBER_MAX: u32 = 0x3fff_ffff; + +/// The maximum number of commits that can be stored in a commit graph. +pub const MAX_COMMITS: u32 = (1 << 30) + (1 << 29) + (1 << 28) - 1; + +#[inline] +pub(crate) fn from_be_u32(b: &[u8]) -> u32 { + u32::from_be_bytes(b.try_into().unwrap()) +} + +/// A generalized position for use in [`Graph`]. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Hash)] +pub struct Position(pub u32); + +impl std::fmt::Display for Position { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} diff --git a/crates/gix-commitgraph/src/verify.rs b/crates/gix-commitgraph/src/verify.rs new file mode 100644 index 0000000000000000000000000000000000000000..7bf77b6e474d1f07eb94c510c736753cbc0cccdc --- /dev/null +++ b/crates/gix-commitgraph/src/verify.rs @@ -0,0 +1,157 @@ +//! Auxiliary types used by graph verification methods. +use std::{ + cmp::{max, min}, + collections::BTreeMap, +}; + +use gix_error::{ErrorExt, Exn, Message, ResultExt, message}; + +use crate::{ + GENERATION_NUMBER_MAX, Graph, Position, + file::{self}, +}; + +/// Statistics gathered while verifying the integrity of the graph as returned by [`Graph::verify_integrity()`]. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct Outcome { + /// The length of the longest path between any two commits in this graph. + /// + /// For example, this will be `Some(9)` for a commit graph containing 10 linear commits. + /// This will be `Some(0)` for a commit graph containing 0 or 1 commits. + /// If the longest path length is too large to fit in a [u32], then this will be [None]. + pub longest_path_length: Option, + /// The total number of commits traversed. + pub num_commits: u32, + /// A mapping of `N -> number of commits with N parents`. + pub parent_counts: BTreeMap, +} + +impl Graph { + /// Traverse all commits in the graph and call `processor(&commit) -> Result<(), E>` on it while verifying checksums. + /// + /// When `processor` returns an error, the entire verification is stopped and the error returned. + pub fn verify_integrity( + &self, + mut processor: impl FnMut(&file::Commit<'_>) -> Result<(), E>, + ) -> Result> + where + E: std::error::Error + Send + Sync + 'static, + { + if self.files.len() > 256 { + // A file in a split chain can only have up to 255 base files. + return Err(message!( + "Commit-graph should be composed of at most 256 files but actually contains {} files", + self.files.len() + ) + .raise()); + } + + let mut stats = Outcome { + longest_path_length: None, + num_commits: 0, + parent_counts: BTreeMap::new(), + }; + let mut max_generation = 0u32; + + // TODO: Detect duplicate commit IDs across different files. Not sure how to do this without + // a separate loop, e.g. self.iter_sorted_ids(). + + let mut file_start_pos = Position(0); + for (file_index, file) in self.files.iter().enumerate() { + if usize::from(file.base_graph_count()) != file_index { + return Err(message!( + "'{}' should have {} base graphs, but claims {} base graphs", + file.path().display(), + file_index, + file.base_graph_count() + ) + .raise()); + } + + for (base_graph_index, (expected, actual)) in self + .files + .iter() + .take(file_index) + .map(crate::File::checksum) + .zip(file.iter_base_graph_ids()) + .enumerate() + { + if actual != expected { + return Err(message!( + "'{}' base graph at index {} should have ID {} but is {}", + file.path().display(), + base_graph_index, + expected, + actual + ) + .raise()); + } + } + + let next_file_start_pos = Position(file_start_pos.0 + file.num_commits()); + let file_stats = file.traverse(|commit| { + let mut max_parent_generation = 0u32; + let mut max_parent_corrected_date = None; + for parent_pos in commit.iter_parents() { + let parent_pos = parent_pos.map_err(|err| err.raise_erased())?; + if parent_pos >= next_file_start_pos { + return Err(message!( + "Commit {} has parent position {parent_pos} that is out of range (should be in range 0-{})", + commit.id(), + Position(next_file_start_pos.0 - 1) + ) + .raise_erased()); + } + let parent = self.commit_at(parent_pos); + max_parent_generation = max(max_parent_generation, parent.generation()); + let parent_corrected_date = parent.corrected_commit_date().map_err(|err| err.raise_erased())?; + max_parent_corrected_date = max(max_parent_corrected_date, parent_corrected_date); + } + + // If the max parent generation is GENERATION_NUMBER_MAX, then this commit's + // generation should be GENERATION_NUMBER_MAX too. + let expected_generation = min(max_parent_generation + 1, GENERATION_NUMBER_MAX); + if commit.generation() != expected_generation { + return Err(message!( + "Commit {}'s generation should be {expected_generation} but is {}", + commit.id(), + commit.generation() + ) + .raise_erased()); + } + + // Only check the lower bound, as dates beyond 34 bits are truncated in the graph. + let corrected_date = commit.corrected_commit_date().map_err(|err| err.raise_erased())?; + if let (Some(corrected_date), Some(max_parent_corrected_date)) = (corrected_date, max_parent_corrected_date) + { + if corrected_date <= max_parent_corrected_date { + return Err(message!( + "Commit {}'s corrected commit date should be greater than {max_parent_corrected_date} but is {corrected_date}", + commit.id(), + ) + .raise_erased()); + } + } + + processor(commit).or_raise_erased(|| message!("processor failed on commit {id}", id = commit.id()))?; + + Ok(()) + })?; + + max_generation = max(max_generation, file_stats.max_generation); + stats.num_commits += file_stats.num_commits; + for (key, value) in file_stats.parent_counts.into_iter() { + *stats.parent_counts.entry(key).or_insert(0) += value; + } + file_start_pos = next_file_start_pos; + } + + stats.longest_path_length = if max_generation < GENERATION_NUMBER_MAX { + Some(max_generation.saturating_sub(1)) + } else { + None + }; + Ok(stats) + } +} diff --git a/crates/gix-commitgraph/src/write/info_dir.rs b/crates/gix-commitgraph/src/write/info_dir.rs new file mode 100644 index 0000000000000000000000000000000000000000..bccd58f70af86945e225d5bb3b5eb6a4b61659a1 --- /dev/null +++ b/crates/gix-commitgraph/src/write/info_dir.rs @@ -0,0 +1,183 @@ +use std::{ + io::Write, + path::{Path, PathBuf}, + time::SystemTime, +}; + +use gix_error::{Exn, Message, ResultExt, message}; +use gix_hash::ObjectId; +use gix_object::Find; + +use super::{Options, Outcome, Split, write}; +use crate::Graph; + +/// Write the commit-graph of the objects whose `info` directory is `info_dir`. +/// +/// `graph` must be the graph in `info_dir`, if any. It's closed before files are replaced. +/// See [`write()`] for `objects`, `tips` and `options`. +/// +/// * With [`Split::No`], `info_dir/commit-graph` is written, and the chain in `info_dir/commit-graphs` removed. +/// * Otherwise, a layer is written to `info_dir/commit-graphs` and `commit-graph-chain` is updated to end with it. +/// If `info_dir/commit-graph` stays below it, it becomes the bottom layer of the chain, otherwise it's removed. +/// +/// Afterwards, `.graph` files in `info_dir/commit-graphs` that aren't in the chain and weren't modified after +/// `expire_time` are deleted. Merged layers count as modified now and `expire_time` defaults to now, +/// so by default they are deleted right away. +/// +/// Returns `None` and changes nothing if there was nothing to write. +pub fn write_to_info_dir( + objects: &(impl Find + Clone + Send), + info_dir: &Path, + graph: Option, + tips: impl IntoIterator, + options: Options, + expire_time: Option, +) -> Result, Exn> { + let graphs_dir = info_dir.join("commit-graphs"); + let layers: Vec<(PathBuf, ObjectId)> = graph + .iter() + .flat_map(|graph| graph.files.iter()) + .map(|file| (file.path().to_owned(), file.checksum().to_owned())) + .collect(); + let lock = |path: &Path| { + gix_lock::File::acquire_to_update_resource(path, gix_lock::acquire::Fail::Immediately, None) + .or_raise(|| message!("Could not lock '{}'", path.display())) + }; + + let (outcome, chain) = if options.split == Split::No { + let path = info_dir.join("commit-graph"); + let mut out = std::io::BufWriter::new(lock(&path)?); + let Some(outcome) = write(objects, graph.as_ref(), tips, options, &mut out)? else { + return Ok(None); + }; + drop(graph); + commit(into_inner(out)?)?; + make_read_only(&path); + remove(&graphs_dir.join("commit-graph-chain"))?; + (outcome, Vec::new()) + } else { + std::fs::create_dir_all(&graphs_dir) + .or_raise(|| message!("Could not create directory '{}'", graphs_dir.display()))?; + // Hold the chain's lock while writing the layer. + let chain_path = graphs_dir.join("commit-graph-chain"); + let mut chain_lock = lock(&chain_path)?; + let layer = gix_tempfile::new( + &graphs_dir, + gix_tempfile::ContainingDirectory::Exists, + gix_tempfile::AutoRemove::Tempfile, + ) + .or_raise(|| { + message!( + "Could not create a temporary file in '{}'", + graphs_dir.display() + ) + })?; + let mut out = std::io::BufWriter::new(layer); + let Some(outcome) = write(objects, graph.as_ref(), tips, options, &mut out)? else { + return Ok(None); + }; + drop(graph); + let layer = into_inner(out)?; + + let layer_path = |checksum: &ObjectId| graphs_dir.join(format!("graph-{checksum}.graph")); + let base_layers = &layers[..outcome.num_base_layers]; + match base_layers.last() { + // A monolithic `commit-graph` kept below the new layer moves into the chain. + Some((path, checksum)) if *path != layer_path(checksum) => { + std::fs::rename(path, layer_path(checksum)).or_raise(|| { + message!("Could not rename '{}' into the chain", path.display()) + })?; + } + Some(_) => {} + None => remove(&info_dir.join("commit-graph"))?, + } + let path = layer_path(&outcome.checksum); + layer + .persist(&path) + .map_err(|err| err.error) + .or_raise(|| message!("Could not write commit-graph layer '{}'", path.display()))?; + make_read_only(&path); + + let chain: Vec = base_layers + .iter() + .map(|(_, checksum)| *checksum) + .chain(Some(outcome.checksum)) + .collect(); + for checksum in &chain { + writeln!(chain_lock, "{checksum}") + .or_raise(|| message!("Could not write commit-graph chain"))?; + } + commit(chain_lock)?; + make_read_only(&chain_path); + + let now = SystemTime::now(); + for (path, _) in &layers[outcome.num_base_layers..] { + if let Ok(file) = std::fs::File::open(path) { + file.set_modified(now).ok(); + } + } + (outcome, chain.iter().map(layer_path).collect()) + }; + expire( + &graphs_dir, + &chain, + expire_time.unwrap_or_else(SystemTime::now), + )?; + Ok(Some(outcome)) +} + +fn into_inner(out: std::io::BufWriter) -> Result> { + out.into_inner() + .map_err(std::io::IntoInnerError::into_error) + .or_raise(|| message!("Could not write commit-graph")) +} + +fn commit(lock: gix_lock::File) -> Result<(), Exn> { + let path = lock.resource_path(); + lock.commit() + .map_err(|err| err.error) + .or_raise(|| message!("Could not write '{}'", path.display()))?; + Ok(()) +} + +/// Delete the `.graph` files in `graphs_dir` that aren't in `chain` and weren't modified after `expire_time`. +fn expire( + graphs_dir: &Path, + chain: &[PathBuf], + expire_time: SystemTime, +) -> Result<(), Exn> { + let Ok(entries) = std::fs::read_dir(graphs_dir) else { + return Ok(()); + }; + for entry in entries.flatten() { + let path = entry.path(); + let expired = entry + .metadata() + .and_then(|meta| meta.modified()) + .is_ok_and(|modified| modified <= expire_time); + if expired && path.extension().is_some_and(|ext| ext == "graph") && !chain.contains(&path) { + std::fs::remove_file(&path).ok(); + } + } + Ok(()) +} + +fn remove(path: &Path) -> Result<(), Exn> { + match std::fs::remove_file(path) { + Err(err) if err.kind() != std::io::ErrorKind::NotFound => { + Err(err).or_raise(|| message!("Could not remove '{}'", path.display())) + } + _ => Ok(()), + } +} + +fn make_read_only(path: &Path) { + #[cfg(unix)] + if let Ok(meta) = std::fs::metadata(path) { + let mut permissions = meta.permissions(); + permissions.set_readonly(true); + std::fs::set_permissions(path, permissions).ok(); + } + #[cfg(not(unix))] + let _ = path; +} diff --git a/crates/gix-commitgraph/src/write/mod.rs b/crates/gix-commitgraph/src/write/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..a601cf40d87f10f9fbe8b30bdd2f01c119a20e40 --- /dev/null +++ b/crates/gix-commitgraph/src/write/mod.rs @@ -0,0 +1,1142 @@ +//! Write commit-graph files. +use std::{cmp::Ordering, collections::HashSet, io::Write}; + +use gix_error::{ErrorExt, Exn, Message, ResultExt, message}; +use gix_hash::ObjectId; +use gix_object::{CommitRefIter, Find, Kind, TagRefIter, TreeRefIter, tree::EntryRef}; + +mod info_dir; +pub use info_dir::write_to_info_dir; + +use crate::{ + BloomFilterSettings, File, Graph, Position, + bloom::BloomKey, + file::{ + BASE_GRAPHS_LIST_CHUNK_ID, BLOOM_FILTER_DATA_CHUNK_ID, BLOOM_FILTER_INDEX_CHUNK_ID, + COMMIT_DATA_CHUNK_ID, CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW, EXTENDED_EDGES_LIST_CHUNK_ID, + EXTENDED_EDGES_MASK, GENERATION_DATA_CHUNK_ID, GENERATION_DATA_OVERFLOW_CHUNK_ID, + LAST_EXTENDED_EDGE_MASK, NO_PARENT, OID_FAN_CHUNK_ID, OID_LOOKUP_CHUNK_ID, SIGNATURE, + }, +}; + +const GENERATION_NUMBER_MAX: u32 = 0x3FFF_FFFF; +const CORRECTED_COMMIT_DATE_OFFSET_MAX: u64 = (1 << 31) - 1; +const COMMIT_DATE_MASK: u64 = 0x3_FFFF_FFFF; +const BLOOM_MAX_CHANGED_PATHS: usize = 512; + +/// Choose the changed-path Bloom filter settings for a write, or `None` to write no filters. +/// +/// * `existing` is the graph being replaced. Only its top layer's settings count, so once a layer is written +/// without filters, later writes have none either. `None` drops filters, like `--no-changed-paths`. +/// * `changed_paths` requests filters even if the existing graph has none, like `--changed-paths`. +/// * `hash_version` (`commitGraph.changedPathsVersion`) overrides the existing graph's version. +pub fn bloom_filter_settings( + existing: Option<&Graph>, + changed_paths: bool, + hash_version: Option, +) -> Option { + let existing = existing.and_then(|graph| graph.files.last().bloom_filter_settings()); + let mut settings = existing.or(changed_paths.then_some(BloomFilterSettings { + hash_version: 1, + num_hashes: 7, + bits_per_entry: 10, + }))?; + settings.hash_version = match hash_version.unwrap_or(settings.hash_version) { + 2 => 2, + _ => 1, + }; + Some(settings) +} + +/// Options for [`write()`]. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct Options { + /// The kind of hash the objects use. + pub object_hash: gix_hash::Kind, + /// Changed-path Bloom filter settings, or `None` to write no filters. See [`bloom_filter_settings()`]. + pub bloom_settings: Option, + /// Compute at most this many filters, lowest generation first, like `--max-new-filters`. + /// The other commits get no filter, which a later write can fill in. `None` means no limit. + pub max_new_filters: Option, + /// Compute filters on at most this many threads, or on all cores if `None`. + pub thread_limit: Option, + /// Write a single graph, or a layer on top of the existing graph's layers. + pub split: Split, +} + +/// How to lay out the graph, like `git commit-graph write --split`. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum Split { + /// Write one graph with all commits, like writing without `--split`. + #[default] + No, + /// Write a layer with the commits that aren't in the existing graph yet, like `--split`. + /// + /// The new layer absorbs the layers below it while the layer below has at most `size_multiple` times as + /// many commits as the new layer (git's `--size-multiple`, 2 by default), or while the new layer has more + /// than `max_commits` (git's `--max-commits`). + Merge { + /// How large the layer below may be relative to the new one to be merged into it. 0 means 2. + size_multiple: u32, + /// Keep merging layers while the new layer has more commits than this. + max_commits: Option, + }, + /// Write a layer with the commits that aren't in the existing graph yet, and never merge, like `--split=no-merge`. + NoMerge, + /// Write a single layer with all commits, like `--split=replace`. + Replace, +} + +/// The outcome of [`write()`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Outcome { + /// The number of commits written to the graph. + pub num_commits: u32, + /// The number of changed-path Bloom filters that were computed. + pub num_filters_computed: usize, + /// The number of changed-path Bloom filters reused from the existing graph despite a different hash version, + /// possible when no path in the commit's or its first parent's tree has a byte with the high bit set. + pub num_filters_upgraded: usize, + /// The checksum at the end of the file. + pub checksum: ObjectId, + /// The number of layers of the existing graph that the written layer is based on, from the bottom. + /// The other layers were merged into it. Always 0 unless writing a layer with [`Split::Merge`] or [`Split::NoMerge`]. + pub num_base_layers: usize, +} + +/// The commits to write, sorted by id. +struct Commits { + ids: Vec, + commits: Vec, + /// The positions of all parents in the graph including its base layers, see [`Commit::parents`]. + parents: Vec, + /// The number of commits in the base layers, which come before the ones written. + num_commits_in_base: u32, +} + +#[derive(Clone, Copy)] +struct Commit { + tree: ObjectId, + /// The range of the commit's parents in [`Commits::parents`]. + parents: (u32, u32), + seconds: i64, + /// The generation and corrected commit date, if taken from the existing graph. + known: Option<(u32, u64)>, +} + +impl Commits { + fn parents(&self, pos: usize) -> &[u32] { + let (start, end) = self.commits[pos].parents; + &self.parents[start as usize..end as usize] + } + + /// Return the commit at `pos` in the graph including its base layers, if it's one of ours. + fn at(&self, pos: u32) -> Option<&Commit> { + pos.checked_sub(self.num_commits_in_base) + .map(|pos| &self.commits[pos as usize]) + } +} + +/// Write a commit-graph for all commits reachable from `tips` into `out`. +/// +/// `graph` is the existing commit-graph, if any. Commits and filters found in it are reused. +/// Add its commits to `tips` to keep them all, like `--append`. +/// +/// With [`Split::Merge`] or [`Split::NoMerge`], only the reachable commits that aren't in `graph` are written, +/// along with the commits of the layers merged into the new one, which are kept if they still exist in `objects`. +/// [`Outcome::num_base_layers`] tells which layers the new one is based on. Nothing is written if there are no new commits. +/// +/// Tags in `tips` are peeled; tips that aren't commits are ignored. +/// It's an error if a tip or a reachable commit or tree is missing. +/// Returns `None` and writes nothing if there are no commits. +/// +/// Filters are computed in parallel, each thread with its own clone of `objects`. +pub fn write( + objects: &(impl Find + Clone + Send), + graph: Option<&Graph>, + tips: impl IntoIterator, + Options { + object_hash, + bloom_settings, + max_new_filters, + thread_limit, + split, + }: Options, + out: &mut impl std::io::Write, +) -> Result, Exn> { + if let Some(graph) = graph.filter(|graph| graph.object_hash() != object_hash) { + return Err(message!( + "The commit-graph uses {}, but the objects use {object_hash}", + graph.object_hash() + ) + .raise()); + } + let Some((commits, base_layers)) = collect(objects, graph, tips, split, object_hash)? else { + return Ok(None); + }; + // Readers ignore generation data unless every layer has it. + let write_generation_data = base_layers + .last() + .is_none_or(|file| file.has_generation_data()); + let generation_data = generation_data(&commits, graph); + let filters = match bloom_settings { + Some(settings) => { + // The budget goes to lower generations first. Without generation data, corrected dates + // aren't computed, so only commit dates count. + let mut order: Vec = (0..commits.ids.len()).collect(); + order.sort_by_key(|&pos| { + let corrected = if write_generation_data { + generation_data[pos].1 + } else { + 0 + }; + (corrected, commits.commits[pos].seconds, pos) + }); + Some(changed_path_filters( + objects, + graph, + &commits, + &order, + settings, + max_new_filters.unwrap_or(usize::MAX), + thread_limit, + object_hash, + )?) + } + None => None, + }; + let checksum = serialize( + &commits, + &generation_data, + write_generation_data, + bloom_settings.zip(filters.as_ref()), + &base_layers, + object_hash, + out, + ) + .or_raise(|| message!("Could not write commit-graph"))?; + Ok(Some(Outcome { + num_commits: commits.ids.len() as u32, + num_filters_computed: filters.as_ref().map_or(0, |filters| filters.ends.len()), + num_filters_upgraded: filters.as_ref().map_or(0, |filters| filters.upgraded), + checksum, + num_base_layers: base_layers.len(), + })) +} + +/// Return the commits to write and the layers of `graph` they are based on, or `None` if there is nothing to write. +fn collect<'a>( + objects: &impl Find, + graph: Option<&'a Graph>, + tips: impl IntoIterator, + split: Split, + object_hash: gix_hash::Kind, +) -> Result)>, Exn> { + let mut seeds = Vec::new(); + for tip in tips { + seeds.extend(peel_to_commit(objects, graph, tip, object_hash)?); + } + let mut walk = Walk::default(); + let layers = graph.filter(|_| matches!(split, Split::Merge { .. } | Split::NoMerge)); + let Some(graph) = layers else { + walk.run(objects, graph, seeds, 0, &HashSet::new(), object_hash)?; + if walk.ids.is_empty() && split != Split::Replace { + return Ok(None); + } + return Ok(Some((walk.into_commits(graph, 0)?, Vec::new()))); + }; + + // The number of new commits decides how many layers are merged, then the merged layers' commits are added. + walk.run( + objects, + Some(graph), + seeds, + graph.num_commits(), + &HashSet::new(), + object_hash, + )?; + if walk.ids.is_empty() { + return Ok(None); + } + let files: Vec<&File> = graph.files.iter().collect(); + let num_base_layers = num_base_layers(&files, walk.ids.len(), split); + let base_layers = files[..num_base_layers].to_vec(); + let num_commits_in_base: u32 = base_layers.iter().map(|file| file.num_commits()).sum(); + let (mut merged, mut missing) = (Vec::new(), HashSet::new()); + let mut buf = Vec::new(); + for pos in num_commits_in_base..graph.num_commits() { + let id = graph.id_at(Position(pos)).to_owned(); + // Drop commits that don't exist anymore. + let exists = objects + .try_find(&id, &mut buf) + .map_err(|err| message!("Could not read object {id}: {err}"))? + .is_some(); + if exists { + merged.push(id); + } else { + missing.insert(id); + } + } + walk.run( + objects, + Some(graph), + merged, + num_commits_in_base, + &missing, + object_hash, + )?; + Ok(Some(( + walk.into_commits(Some(graph), num_commits_in_base)?, + base_layers, + ))) +} + +/// Return how many layers of `files` stay below a new layer with `num_new_commits`, see [`Split::Merge`]. +fn num_base_layers(files: &[&File], num_new_commits: usize, split: Split) -> usize { + let mut num_base_layers = files.len(); + if let Split::Merge { + size_multiple, + max_commits, + } = split + { + let size_multiple = u64::from(if size_multiple == 0 { 2 } else { size_multiple }); + let mut num_commits = num_new_commits as u64; + while let Some(below) = num_base_layers + .checked_sub(1) + .map(|index| u64::from(files[index].num_commits())) + { + let too_many = max_commits.is_some_and(|max| num_commits > u64::from(max)); + if below > size_multiple * num_commits && !too_many { + break; + } + num_commits += below; + num_base_layers -= 1; + } + } + num_base_layers +} + +/// The commits found so far, in the order they were found. +#[derive(Default)] +struct Walk { + index: gix_hashtable::HashMap, + ids: Vec, + commits: Vec, + /// The parents of all commits, see [`Commit::parents`]. + parent_ids: Vec, +} + +impl Walk { + /// Add `seeds` and the commits reachable from them, except the ones at positions before `stop_before` in `graph`. + /// It's an error to reach `missing` commits. + fn run( + &mut self, + objects: &impl Find, + graph: Option<&Graph>, + mut stack: Vec, + stop_before: u32, + missing: &HashSet, + object_hash: gix_hash::Kind, + ) -> Result<(), Exn> { + let mut buf = Vec::new(); + while let Some(id) = stack.pop() { + if self.index.contains_key(&id) + || (stop_before > 0 + && graph + .and_then(|graph| graph.lookup(id)) + .is_some_and(|pos| pos.0 < stop_before)) + { + continue; + } + if missing.contains(&id) { + return Err(message!( + "Commit {id} is in the commit-graph, but it could not be found" + ) + .raise()); + } + let start = self.parent_ids.len(); + let mut commit = + match graph.and_then(|graph| from_graph(graph, &id, &mut self.parent_ids)) { + Some(commit) => commit, + None => { + self.parent_ids.truncate(start); + decode_commit(objects, &id, &mut buf, &mut self.parent_ids, object_hash)? + } + }; + commit.parents = (start as u32, self.parent_ids.len() as u32); + stack.extend_from_slice(&self.parent_ids[start..]); + self.index.insert(id, self.ids.len() as u32); + self.ids.push(id); + self.commits.push(commit); + } + Ok(()) + } + + /// Sort the commits by id, placing them after the `num_commits_in_base` commits of `graph`'s base layers. + fn into_commits( + self, + graph: Option<&Graph>, + num_commits_in_base: u32, + ) -> Result> { + let mut order: Vec = (0..self.ids.len() as u32).collect(); + order.sort_unstable_by_key(|&index| self.ids[index as usize]); + let mut position = vec![0u32; self.ids.len()]; + for (pos, &index) in order.iter().enumerate() { + position[index as usize] = num_commits_in_base + pos as u32; + } + let mut sorted = Commits { + ids: Vec::with_capacity(self.ids.len()), + commits: Vec::with_capacity(self.ids.len()), + parents: Vec::with_capacity(self.parent_ids.len()), + num_commits_in_base, + }; + for index in order { + let (id, mut commit) = (self.ids[index as usize], self.commits[index as usize]); + let (start, end) = commit.parents; + let new_start = sorted.parents.len() as u32; + for parent in &self.parent_ids[start as usize..end as usize] { + let pos = match self.index.get(parent) { + Some(&index) => position[index as usize], + None => graph + .and_then(|graph| graph.lookup(parent)) + .map(|pos| pos.0) + .filter(|&pos| pos < num_commits_in_base) + .ok_or_else(|| message!("Parent {parent} of commit {id} is missing"))?, + }; + sorted.parents.push(pos); + } + commit.parents = (new_start, sorted.parents.len() as u32); + sorted.ids.push(id); + sorted.commits.push(commit); + } + Ok(sorted) + } +} + +/// Read `id` from `graph` and put its parents into `parents`, or return `None` if it's not there or lacks generation data. +/// The returned commit's parents aren't set. +/// +/// Without generation data, the corrected date needs the full commit date, which the graph stores in 34 bits only. +fn from_graph(graph: &Graph, id: &ObjectId, parents: &mut Vec) -> Option { + let commit = graph.commit_by_id(id)?; + let corrected = commit.corrected_commit_date().ok()??; + for parent in commit.iter_parents() { + let pos = parent.ok().filter(|pos| pos.0 < graph.num_commits())?; + parents.push(graph.id_at(pos).to_owned()); + } + Some(Commit { + tree: commit.root_tree_id().to_owned(), + parents: (0, 0), + seconds: commit.committer_timestamp() as i64, + known: Some((commit.generation(), corrected)), + }) +} + +/// Decode commit `id` from `objects` and put its parents into `parents`. The returned commit's parents aren't set. +fn decode_commit( + objects: &impl Find, + id: &ObjectId, + buf: &mut Vec, + parents: &mut Vec, + object_hash: gix_hash::Kind, +) -> Result> { + use gix_object::commit::ref_iter::Token; + let data = find(objects, id, buf)?; + if data.kind != Kind::Commit { + return Err(message!("Object {id} is a {}, expected a commit", data.kind).raise()); + } + let (mut tree, mut seconds) = (None, 0); + for token in CommitRefIter::from_bytes(data.data, object_hash) { + match token.or_raise(|| message!("Could not decode commit {id}"))? { + Token::Tree { id } => tree = Some(id), + Token::Parent { id } => parents.push(id), + Token::Committer { signature } => { + seconds = signature.seconds(); + break; + } + _ => {} + } + } + Ok(Commit { + tree: tree.ok_or_else(|| message!("Commit {id} has no tree"))?, + parents: (0, 0), + seconds, + known: None, + }) +} + +fn find<'a>( + objects: &impl Find, + id: &ObjectId, + buf: &'a mut Vec, +) -> Result, Exn> { + objects + .try_find(id, buf) + .map_err(|err| message!("Could not read object {id}: {err}"))? + .ok_or_else(|| message!("Object {id} could not be found").into()) +} + +/// Peel `id` to a commit, or return `None` if it points to another kind of object. +/// Commits in `graph` aren't read. +fn peel_to_commit( + objects: &impl Find, + graph: Option<&Graph>, + mut id: ObjectId, + object_hash: gix_hash::Kind, +) -> Result, Exn> { + let mut buf = Vec::new(); + loop { + if graph.is_some_and(|graph| graph.lookup(id).is_some()) { + return Ok(Some(id)); + } + let data = find(objects, &id, &mut buf)?; + match data.kind { + Kind::Commit => return Ok(Some(id)), + Kind::Tag => { + id = TagRefIter::from_bytes(data.data, object_hash) + .target_id() + .or_raise(|| message!("Could not decode tag {id}"))?; + } + _ => return Ok(None), + } + } +} + +/// Return the generation and corrected commit date of each commit. +/// Commits taken from the existing graph keep theirs, and so do parents in its base layers. +fn generation_data(commits: &Commits, graph: Option<&Graph>) -> Vec<(u32, u64)> { + let mut out: Vec> = + commits.commits.iter().map(|commit| commit.known).collect(); + let num_commits_in_base = commits.num_commits_in_base; + let value = + |out: &[Option<(u32, u64)>], parent: u32| match parent.checked_sub(num_commits_in_base) { + Some(pos) => out[pos as usize], + None => Some(base_generation_data( + graph.expect("parents in base layers are in the graph"), + parent, + )), + }; + let mut stack = Vec::new(); + for root in 0..out.len() { + stack.push(root); + while let Some(&top) = stack.last() { + if out[top].is_some() { + stack.pop(); + continue; + } + let parents = commits.parents(top); + let before = stack.len(); + stack.extend( + parents + .iter() + .filter(|&&parent| value(&out, parent).is_none()) + .map(|&parent| (parent - num_commits_in_base) as usize), + ); + if stack.len() == before { + let (generation, date) = + parents.iter().fold((0, 0), |(generation, date), &parent| { + let (parent_generation, parent_date) = + value(&out, parent).expect("parents are done"); + (generation.max(parent_generation), date.max(parent_date)) + }); + let seconds = commits.commits[top].seconds.max(0) as u64; + out[top] = Some(( + generation.saturating_add(1).min(GENERATION_NUMBER_MAX), + if seconds > date { seconds } else { date + 1 }, + )); + stack.pop(); + } + } + } + out.into_iter() + .map(|data| data.expect("every commit was visited")) + .collect() +} + +/// Return the generation and corrected commit date of the commit at `pos` in `graph`. +/// If the graph's generation data can't be used, the generation is taken as corrected date. +fn base_generation_data(graph: &Graph, pos: u32) -> (u32, u64) { + let commit = graph.commit_at(Position(pos)); + let generation = commit.generation(); + let corrected = commit.corrected_commit_date().ok().flatten(); + (generation, corrected.unwrap_or(u64::from(generation))) +} + +/// Where a commit's filter comes from. +#[derive(Clone, Copy)] +enum Filter<'a> { + /// Not computed, written with zero length. + Missing, + /// Reused from the existing graph. + Stored(&'a [u8]), + /// The filter computed at this index. + Computed(usize), +} + +/// The changed-path Bloom filter of each commit. +struct Filters<'a> { + by_position: Vec>, + /// The computed filters, one after another. + data: Vec, + /// The end of each computed filter in `data`. + ends: Vec, + /// How many stored filters were upgraded to another hash version. + upgraded: usize, +} + +impl Filters<'_> { + fn get(&self, pos: usize) -> &[u8] { + match self.by_position[pos] { + Filter::Missing => &[], + Filter::Stored(filter) => filter, + Filter::Computed(index) => { + let start = index + .checked_sub(1) + .map_or(0, |previous| self.ends[previous]); + &self.data[start..self.ends[index]] + } + } + } +} + +/// Return the filters of the commits in `order`. +/// Commits beyond the `max_new_filters` budget get no filter, unless one with the same hash version can be reused. +#[allow(clippy::too_many_arguments)] +fn changed_path_filters<'a>( + objects: &(impl Find + Clone + Send), + graph: Option<&'a Graph>, + commits: &Commits, + order: &[usize], + settings: BloomFilterSettings, + max_new_filters: usize, + thread_limit: Option, + object_hash: gix_hash::Kind, +) -> Result, Exn> { + let mut by_position = vec![Filter::Missing; commits.ids.len()]; + let mut upgraded = 0; + let mut clean_trees = HashSet::new(); + // The first parent's tree and the commit's tree of each filter to compute. + let mut jobs: Vec<(Option, ObjectId)> = Vec::new(); + for &pos in order { + let tree = commits.commits[pos].tree; + let parent_tree = commits + .parents(pos) + .first() + .map(|&parent| match commits.at(parent) { + Some(commit) => commit.tree, + None => graph + .expect("parents in base layers are in the graph") + .commit_at(Position(parent)) + .root_tree_id() + .to_owned(), + }); + let may_compute = jobs.len() < max_new_filters; + if let Some((stored, version)) = + graph.and_then(|graph| stored_filter(graph, &commits.ids[pos], settings)) + { + let reuse = version == settings.hash_version + || (may_compute + && !has_high_bit_paths(objects, Some(tree), &mut clean_trees, object_hash)? + && !has_high_bit_paths(objects, parent_tree, &mut clean_trees, object_hash)?); + if reuse { + upgraded += usize::from(version != settings.hash_version); + by_position[pos] = Filter::Stored(stored); + continue; + } + } + if may_compute { + by_position[pos] = Filter::Computed(jobs.len()); + jobs.push((parent_tree, tree)); + } + } + drop(clean_trees); + // Newest first: packs store newer objects as bases of older ones, and only the objects + // read are cached, so this way the trees just read are the bases of the next ones. + jobs.reverse(); + for filter in &mut by_position { + if let Filter::Computed(index) = filter { + *index = jobs.len() - 1 - *index; + } + } + + // Large chunks of consecutive commits keep each thread's trees close together, which matters more than + // balancing the load: with 10000 the linux kernel's filters are computed 15% faster than with 1000. + let chunk_size = + (jobs.len() / (gix_features::parallel::num_threads(thread_limit) * 8)).clamp(1, 10_000); + let chunks = jobs.chunks(chunk_size); + let num_chunks = chunks.len(); + let computed = gix_features::parallel::in_parallel( + chunks.enumerate(), + thread_limit, + { + let objects = objects.clone(); + move |_| (objects, TreeDiff::default()) + }, + |(index, chunk), (objects, diff)| -> Result<_, Exn> { + let mut data = Vec::new(); + let mut ends = Vec::with_capacity(chunk.len()); + for &(parent_tree, tree) in chunk { + diff.changed_paths(objects, parent_tree, tree, object_hash)?; + diff.changes.add_filter_to(&mut data, settings); + ends.push(data.len()); + } + Ok((index, data, ends)) + }, + InOrder(vec![None; num_chunks]), + )?; + + let mut filters = Filters { + by_position, + data: Vec::with_capacity(computed.iter().map(|(data, _)| data.len()).sum()), + ends: Vec::with_capacity(jobs.len()), + upgraded, + }; + for (data, ends) in computed { + let offset = filters.data.len(); + filters.data.extend_from_slice(&data); + filters + .ends + .extend(ends.into_iter().map(|end| offset + end)); + } + Ok(filters) +} + +/// Collect the computed filters of each chunk in chunk order. +struct InOrder(Vec, Vec)>>); + +impl gix_features::parallel::Reduce for InOrder { + type Input = Result<(usize, Vec, Vec), Exn>; + type FeedProduce = (); + type Output = Vec<(Vec, Vec)>; + type Error = Exn; + + fn feed(&mut self, item: Self::Input) -> Result<(), Self::Error> { + let (index, data, ends) = item?; + self.0[index] = Some((data, ends)); + Ok(()) + } + + fn finalize(self) -> Result { + Ok(self + .0 + .into_iter() + .map(|chunk| chunk.expect("every chunk was computed")) + .collect()) + } +} + +/// Return the filter `graph` has for `id` and its hash version, unless it wasn't computed or used other parameters. +fn stored_filter<'a>( + graph: &'a Graph, + id: &ObjectId, + settings: BloomFilterSettings, +) -> Option<(&'a [u8], u32)> { + let commit = graph.commit_by_id(id)?; + let stored = commit.file.bloom_filter_settings()?; + let filter = commit.bloom_filter().filter(|filter| !filter.is_empty())?; + (stored.num_hashes == settings.num_hashes && stored.bits_per_entry == settings.bits_per_entry) + .then_some((filter, stored.hash_version)) +} + +/// Return `true` if a path in `tree` has a byte with the high bit set. +/// Trees without such paths are added to `clean`. +fn has_high_bit_paths( + objects: &impl Find, + tree: Option, + clean: &mut HashSet, + object_hash: gix_hash::Kind, +) -> Result> { + let mut visited = HashSet::new(); + let mut stack: Vec = tree.into_iter().collect(); + let mut buf = Vec::new(); + while let Some(id) = stack.pop() { + if id == ObjectId::empty_tree(object_hash) || clean.contains(&id) || !visited.insert(id) { + continue; + } + let data = find(objects, &id, &mut buf)?; + for entry in TreeRefIter::from_bytes(data.data, object_hash) { + let entry = entry.or_raise(|| message!("Could not decode tree {id}"))?; + if entry.filename.iter().any(|byte| byte & 0x80 != 0) { + return Ok(true); + } + if entry.mode.is_tree() { + stack.push(entry.oid.to_owned()); + } + } + } + clean.extend(visited); + Ok(false) +} + +/// A recursive tree diff without rename detection that stops after [`BLOOM_MAX_CHANGED_PATHS`] changes. +/// Its buffers are reused across diffs. +#[derive(Default)] +struct TreeDiff { + parent: Vec, + commit: Vec, + changes: Changes, +} + +#[derive(Default)] +struct Changes { + /// Pairs of subtrees still to compare, with their path and a trailing slash. + pending: Vec<(Option, Option, Vec)>, + /// The changed paths one after another, each ending at an offset in `ends`. + paths: Vec, + ends: Vec, + /// If there were more than [`BLOOM_MAX_CHANGED_PATHS`] changes. + too_many: bool, +} + +impl TreeDiff { + /// Diff `parent` against `commit`, remembering the paths of all non-tree entries that differ. + fn changed_paths( + &mut self, + objects: &impl Find, + parent: Option, + commit: ObjectId, + object_hash: gix_hash::Kind, + ) -> Result<(), Exn> { + let changes = &mut self.changes; + changes.paths.clear(); + changes.ends.clear(); + changes.too_many = false; + changes.pending.clear(); + changes.pending.push((parent, Some(commit), Vec::new())); + while let Some((parent, commit, prefix)) = changes.pending.pop() { + let mut lhs = tree_iter(objects, parent, &mut self.parent, object_hash)?; + let mut rhs = tree_iter(objects, commit, &mut self.commit, object_hash)?; + let (mut left, mut right) = + (next_entry(&mut lhs, parent)?, next_entry(&mut rhs, commit)?); + loop { + let order = match (&left, &right) { + (None, None) => break, + (Some(_), None) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + (Some(left), Some(right)) => left.cmp(right), + }; + match order { + Ordering::Less => { + let entry = left.take().expect("present"); + changes.change(&prefix, entry, true); + left = next_entry(&mut lhs, parent)?; + } + Ordering::Greater => { + let entry = right.take().expect("present"); + changes.change(&prefix, entry, false); + right = next_entry(&mut rhs, commit)?; + } + Ordering::Equal => { + let (old, new) = ( + left.take().expect("present"), + right.take().expect("present"), + ); + if old.oid != new.oid || old.mode.kind() != new.mode.kind() { + if new.mode.is_tree() { + changes.pending.push(( + Some(old.oid.to_owned()), + Some(new.oid.to_owned()), + subtree_prefix(&prefix, new.filename), + )); + } else { + changes.push_path(&prefix, new.filename); + } + } + left = next_entry(&mut lhs, parent)?; + right = next_entry(&mut rhs, commit)?; + } + } + if changes.too_many { + changes.pending.clear(); + return Ok(()); + } + } + } + Ok(()) + } +} + +impl Changes { + /// Record `entry` as only being in the parent tree (`removed`) or only in the commit's tree. + fn change(&mut self, prefix: &[u8], entry: EntryRef<'_>, removed: bool) { + if entry.mode.is_tree() { + let oid = Some(entry.oid.to_owned()); + let prefix = subtree_prefix(prefix, entry.filename); + self.pending.push(if removed { + (oid, None, prefix) + } else { + (None, oid, prefix) + }); + } else { + self.push_path(prefix, entry.filename); + } + } + + fn push_path(&mut self, prefix: &[u8], name: &[u8]) { + if self.ends.len() == BLOOM_MAX_CHANGED_PATHS { + self.too_many = true; + return; + } + self.paths.extend_from_slice(prefix); + self.paths.extend_from_slice(name); + self.ends.push(self.paths.len()); + } + + /// Append the filter of the last diff to `out`. + fn add_filter_to(&self, out: &mut Vec, settings: BloomFilterSettings) { + const TOO_LARGE: u8 = 0xFF; + if self.too_many { + out.push(TOO_LARGE); + return; + } + let paths: HashSet<&[u8]> = self + .ends + .iter() + .scan(0, |start, &end| { + let path = &self.paths[*start..end]; + *start = end; + Some(path) + }) + .flat_map(prefixes) + .collect(); + if paths.len() > BLOOM_MAX_CHANGED_PATHS { + out.push(TOO_LARGE); + return; + } + let bits = paths.len() * settings.bits_per_entry as usize; + let start = out.len(); + out.resize(start + bits.div_ceil(8).max(1), 0); + for path in paths { + BloomKey::from_bytes(path, settings).add_to(&mut out[start..]); + } + } +} + +fn subtree_prefix(prefix: &[u8], name: &[u8]) -> Vec { + [prefix, name, b"/"].concat() +} + +/// Read `tree` into `buf`, returning no entries if it's `None` or not a tree. +fn tree_iter<'a>( + objects: &impl Find, + tree: Option, + buf: &'a mut Vec, + object_hash: gix_hash::Kind, +) -> Result, Exn> { + let Some(id) = tree.filter(|id| *id != ObjectId::empty_tree(object_hash)) else { + return Ok(TreeRefIter::from_bytes(&[], object_hash)); + }; + let data = find(objects, &id, buf)?; + Ok(TreeRefIter::from_bytes( + if data.kind == Kind::Tree { + data.data + } else { + &[] + }, + object_hash, + )) +} + +fn next_entry<'a>( + iter: &mut TreeRefIter<'a>, + tree: Option, +) -> Result>, Exn> { + iter.next().transpose().or_raise(|| { + message!( + "Could not decode tree {}", + tree.map_or_else(String::new, |tree| tree.to_string()) + ) + }) +} + +/// Return `path` and each of its leading directories. +fn prefixes(path: &[u8]) -> impl Iterator { + std::iter::once(path).chain( + path.iter() + .enumerate() + .filter(|(_, byte)| **byte == b'/') + .map(|(index, _)| &path[..index]), + ) +} + +/// Write the graph into `out` and return its checksum. +/// `base_layers` are the layers the graph is based on, from the bottom. +fn serialize( + commits: &Commits, + generation_data: &[(u32, u64)], + write_generation_data: bool, + filters: Option<(BloomFilterSettings, &Filters<'_>)>, + base_layers: &[&File], + object_hash: gix_hash::Kind, + out: &mut impl std::io::Write, +) -> std::io::Result { + let num_commits = commits.ids.len() as u64; + let hash_len = object_hash.len_in_bytes() as u64; + // The offset is relative to the 34-bit commit date stored in `CDAT`. + let offsets = || { + commits + .commits + .iter() + .zip(generation_data) + .map(|(commit, (_, corrected))| { + corrected.saturating_sub(commit.seconds.max(0) as u64 & COMMIT_DATE_MASK) + }) + }; + let num_overflows = if write_generation_data { + offsets() + .filter(|&offset| offset > CORRECTED_COMMIT_DATE_OFFSET_MAX) + .count() as u64 + } else { + 0 + }; + let num_edges: u64 = (0..commits.ids.len()) + .map(|pos| commits.parents(pos).len()) + .filter(|&len| len > 2) + .map(|len| len as u64 - 1) + .sum(); + + let mut index = gix_chunk::file::Index::for_writing(); + index.plan_chunk(OID_FAN_CHUNK_ID, 256 * 4); + index.plan_chunk(OID_LOOKUP_CHUNK_ID, num_commits * hash_len); + index.plan_chunk(COMMIT_DATA_CHUNK_ID, num_commits * (hash_len + 16)); + if write_generation_data { + index.plan_chunk(GENERATION_DATA_CHUNK_ID, num_commits * 4); + } + if num_overflows > 0 { + index.plan_chunk(GENERATION_DATA_OVERFLOW_CHUNK_ID, num_overflows * 8); + } + if num_edges > 0 { + index.plan_chunk(EXTENDED_EDGES_LIST_CHUNK_ID, num_edges * 4); + } + if let Some((_, filters)) = filters { + let filter_bytes: u64 = (0..commits.ids.len()) + .map(|pos| filters.get(pos).len() as u64) + .sum(); + index.plan_chunk(BLOOM_FILTER_INDEX_CHUNK_ID, num_commits * 4); + index.plan_chunk(BLOOM_FILTER_DATA_CHUNK_ID, 12 + filter_bytes); + } + if !base_layers.is_empty() { + index.plan_chunk( + BASE_GRAPHS_LIST_CHUNK_ID, + base_layers.len() as u64 * hash_len, + ); + } + + let mut out = std::io::BufWriter::new(gix_hash::io::Write::new(out, object_hash)); + out.write_all(SIGNATURE)?; + let hash_version = match object_hash { + gix_hash::Kind::Sha1 => 1u8, + _ => 2u8, + }; + let num_base_layers = u8::try_from(base_layers.len()) + .map_err(|_| std::io::Error::other("Too many base layers"))?; + out.write_all(&[1, hash_version, index.num_chunks() as u8, num_base_layers])?; + let mut out = index.into_write(out, SIGNATURE.len() + 4)?; + while let Some(chunk) = out.next_chunk() { + match chunk { + OID_FAN_CHUNK_ID => { + let mut buckets = [0u32; 256]; + for id in &commits.ids { + buckets[id.as_slice()[0] as usize] += 1; + } + let mut total = 0; + for count in buckets { + total += count; + out.write_all(&total.to_be_bytes())?; + } + } + OID_LOOKUP_CHUNK_ID => { + for id in &commits.ids { + out.write_all(id.as_slice())?; + } + } + COMMIT_DATA_CHUNK_ID => { + let mut num_edges = 0; + for (pos, (commit, (generation, _))) in + commits.commits.iter().zip(generation_data).enumerate() + { + out.write_all(commit.tree.as_slice())?; + let fields = match commits.parents(pos) { + [] => [NO_PARENT, NO_PARENT], + [first] => [*first, NO_PARENT], + [first, second] => [*first, *second], + [first, rest @ ..] => { + let fields = [*first, EXTENDED_EDGES_MASK | num_edges]; + num_edges += rest.len() as u32; + fields + } + }; + out.write_all(&fields[0].to_be_bytes())?; + out.write_all(&fields[1].to_be_bytes())?; + let date = (commit.seconds.max(0) as u64) & COMMIT_DATE_MASK; + out.write_all(&((u64::from(*generation) << 34) | date).to_be_bytes())?; + } + } + GENERATION_DATA_CHUNK_ID => { + let mut num_overflows = 0; + for offset in offsets() { + let packed = if offset > CORRECTED_COMMIT_DATE_OFFSET_MAX { + num_overflows += 1; + CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW | (num_overflows - 1) + } else { + offset as u32 + }; + out.write_all(&packed.to_be_bytes())?; + } + } + GENERATION_DATA_OVERFLOW_CHUNK_ID => { + for offset in offsets().filter(|&offset| offset > CORRECTED_COMMIT_DATE_OFFSET_MAX) + { + out.write_all(&offset.to_be_bytes())?; + } + } + EXTENDED_EDGES_LIST_CHUNK_ID => { + for pos in 0..commits.ids.len() { + if let [_, rest @ ..] = commits.parents(pos) + && rest.len() > 1 + { + let (last, init) = rest.split_last().expect("more than one"); + for parent in init { + out.write_all(&parent.to_be_bytes())?; + } + out.write_all(&(last | LAST_EXTENDED_EDGE_MASK).to_be_bytes())?; + } + } + } + BLOOM_FILTER_INDEX_CHUNK_ID => { + let (_, filters) = filters.expect("planned with filters"); + let mut end = 0u32; + for pos in 0..commits.ids.len() { + end = end.saturating_add(filters.get(pos).len() as u32); + out.write_all(&end.to_be_bytes())?; + } + } + BLOOM_FILTER_DATA_CHUNK_ID => { + let (settings, filters) = filters.expect("planned with filters"); + for value in [ + settings.hash_version, + settings.num_hashes, + settings.bits_per_entry, + ] { + out.write_all(&value.to_be_bytes())?; + } + for pos in 0..commits.ids.len() { + out.write_all(filters.get(pos))?; + } + } + BASE_GRAPHS_LIST_CHUNK_ID => { + for file in base_layers { + out.write_all(file.checksum().as_bytes())?; + } + } + unknown => unreachable!("BUG: forgot to write chunk {unknown:?}"), + } + } + let out = out + .into_inner() + .into_inner() + .map_err(std::io::IntoInnerError::into_error)?; + let checksum = out.hash.try_finalize().map_err(std::io::Error::other)?; + out.inner.write_all(checksum.as_slice())?; + Ok(checksum) +} diff --git a/crates/gix-rebase/Cargo.toml b/crates/gix-rebase/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..31dd6be32e2c93e07a59f35db3cb9e50ad708e80 --- /dev/null +++ b/crates/gix-rebase/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "gix-rebase" +description = "A crate of the gitoxide project implementing rebase, i.e. replaying commits onto a new base" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[lib] +doctest = false + +[features] +## Enable support for the SHA-1 hash by enabling the respective feature in the `gix-hash` crate. +sha1 = ["gix-hash/sha1"] +## Enable support for the SHA-256 hash by enabling the respective feature in the `gix-hash` crate. +sha256 = ["gix-hash/sha256"] + +[dependencies] +gix-actor = { workspace = true } +gix-diff = { workspace = true } +gix-hash = { workspace = true } +gix-merge = { workspace = true } +gix-object = { workspace = true } + +thiserror = { workspace = true } + +[dev-dependencies] +gix = { workspace = true } +tempfile = { workspace = true } + +[package.metadata.docs.rs] +all-features = true diff --git a/crates/gix-rebase/README.md b/crates/gix-rebase/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cd2e3c1c33383d1b6cb5297e8f17107e57572300 --- /dev/null +++ b/crates/gix-rebase/README.md @@ -0,0 +1,4 @@ +# gix-rebase + +NOTE: The behavior is not byte-identical to git or jj! +It does not support patch-id/change-id based deduplication. diff --git a/crates/gix-rebase/src/lib.rs b/crates/gix-rebase/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..73ae74469489baa2ff67d38cde2929941e375016 --- /dev/null +++ b/crates/gix-rebase/src/lib.rs @@ -0,0 +1,6 @@ +pub mod replay; +pub use replay::function::replay; + +/// The extra commit headers that are dropped from a replayed commit because rewriting it +/// invalidates them. +pub const STALE_HEADERS: &[&str] = &["gpgsig", "gpgsig-sha256", "mergetag"]; diff --git a/crates/gix-rebase/src/replay/function.rs b/crates/gix-rebase/src/replay/function.rs new file mode 100644 index 0000000000000000000000000000000000000000..47919da3cffaaf54143702becfa44f7d85bf62b8 --- /dev/null +++ b/crates/gix-rebase/src/replay/function.rs @@ -0,0 +1,126 @@ +use gix_hash::ObjectId; +use gix_object::FindExt; + +use super::{Complete, Conflict, EmptyBehavior, Error, Options, Outcome, Pick}; + +/// Replay `picks` onto `onto`, one cherry-pick at a time, and return the new tip. +/// +/// `picks` are the commits to replay, **oldest first**, each with at most one parent. +/// +/// `labels` name the sides for conflict markers, and are only observable when `options` are relaxed +/// enough to let a marked-up blob through. +#[allow(clippy::too_many_arguments)] +pub fn replay( + onto: &gix_hash::oid, + picks: &[ObjectId], + committer: &gix_actor::Signature, + labels: gix_merge::blob::builtin_driver::text::Labels<'_>, + objects: &(impl gix_object::FindObjectOrHeader + gix_object::Write), + diff_state: &mut gix_diff::tree::State, + diff_resource_cache: &mut gix_diff::blob::Platform, + blob_merge: &mut gix_merge::blob::Platform, + options: Options, +) -> Result { + let Options { + mut tree_merge, + treat_as_unresolved, + empty, + } = options; + tree_merge.fail_on_conflict = Some(treat_as_unresolved); + + let empty_tree = ObjectId::empty_tree(onto.kind()); + let mut new_parent = onto.to_owned(); + let mut new_parent_tree = commit_tree(objects, &new_parent)?; + let mut out = Vec::with_capacity(picks.len()); + let mut buf = Vec::new(); + + for pick in picks { + let old_commit = objects.find_commit(pick, &mut buf)?.to_owned()?; + let old_parent_tree = match old_commit.parents.as_slice() { + [] => empty_tree, + [one] => commit_tree(objects, one)?, + many => { + return Err(Error::MergeCommit { + id: *pick, + parents: many.len(), + }); + } + }; + + let was_empty = old_parent_tree == old_commit.tree; + + // When the new parent's tree is already the commit's own base, the merge would have *ours* + // equal to the base and could only ever come out as *theirs*, so take that answer directly. + let new_tree = if new_parent_tree == old_parent_tree { + old_commit.tree + } else { + let merge = gix_merge::tree( + &old_parent_tree, + &new_parent_tree, + &old_commit.tree, + labels, + objects, + |buf| objects.write_buf(gix_object::Kind::Blob, buf), + diff_state, + diff_resource_cache, + blob_merge, + tree_merge.clone(), + ) + .map_err(|err| Error::TreeMerge(Box::new(err)))?; + if merge + .conflicts + .iter() + .any(|conflict| conflict.is_unresolved(treat_as_unresolved)) + { + return Ok(Outcome::Conflict(Conflict { + at: *pick, + onto: new_parent, + picks: out, + conflicts: merge.conflicts, + })); + } + let mut editor = merge.tree; + editor + .write(|tree| objects.write(tree)) + .map_err(Error::WriteTree)? + }; + + // TODO: gate for only one parent when we support `--rebase-merges` (follow jj behavior) + let is_empty = new_parent_tree == new_tree; + let should_abandon = match empty { + EmptyBehavior::Keep => false, + EmptyBehavior::AbandonWasEmpty => is_empty && was_empty, + EmptyBehavior::AbandonNewlyEmpty => is_empty && !was_empty, + EmptyBehavior::AbandonAllEmpty => is_empty, + }; + if should_abandon { + out.push(Pick::Dropped); + continue; + } + + let mut rewritten = old_commit; + rewritten.tree = new_tree; + rewritten.parents = std::iter::once(new_parent).collect(); + rewritten.committer = committer.clone(); + rewritten + .extra_headers + .retain(|(name, _)| !crate::STALE_HEADERS.iter().any(|stale| name == stale)); + + new_parent = objects.write(&rewritten).map_err(Error::WriteCommit)?; + new_parent_tree = new_tree; + out.push(Pick::Rewritten(new_parent)); + } + + Ok(Outcome::Complete(Complete { + tip: new_parent, + picks: out, + })) +} + +fn commit_tree( + objects: &impl gix_object::FindObjectOrHeader, + id: &gix_hash::oid, +) -> Result { + let mut buf = Vec::new(); + Ok(objects.find_commit(id, &mut buf)?.tree()) +} diff --git a/crates/gix-rebase/src/replay/mod.rs b/crates/gix-rebase/src/replay/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..409b9cbaaae26e33ddd6e4f5da97817077d6567a --- /dev/null +++ b/crates/gix-rebase/src/replay/mod.rs @@ -0,0 +1,150 @@ +use gix_hash::ObjectId; + +/// What to do with a commit that changes nothing once it has been replayed. +/// [conflict](Outcome::Conflict). +#[derive(Default, Debug, Copy, Clone, Eq, PartialEq)] +pub enum EmptyBehavior { + /// Keep every commit, even one that changes nothing once replayed. + /// + /// `git rebase --keep-empty --empty=keep`. + Keep, + /// Drop commits that were already empty relative to their own parent, but keep ones that only + /// became empty by being replayed. + /// + /// `git rebase --no-keep-empty --empty=keep`. This is the one case jj cannot express. + AbandonWasEmpty, + /// Drop commits that had changes of their own but lost them by being replayed, and keep ones that + /// were empty to begin with — making one of those took `git commit --allow-empty`, so it was + /// deliberate. + /// + /// `git rebase --keep-empty --empty=drop`, which is `git rebase`'s default, and ours. + #[default] + AbandonNewlyEmpty, + /// Drop every commit that changes nothing once replayed, however it came to be that way. + /// + /// `git rebase --no-keep-empty --empty=drop`. + AbandonAllEmpty, +} + +/// Options for use in [`replay()`](crate::replay()). +#[derive(Debug, Clone)] +pub struct Options { + /// The options to use for each individual tree-merge. + /// + /// Note that [`fail_on_conflict`](gix_merge::tree::Options::fail_on_conflict) is always + /// overridden with [`Self::treat_as_unresolved`], as there is no point in merging the rest of a + /// tree that is about to be thrown away. + /// + /// Keep [`rewrites`](gix_merge::tree::Options::rewrites) enabled to match `git rebase`, whose + /// merge backend performs rename and directory-rename detection. + pub tree_merge: gix_merge::tree::Options, + /// Which conflicts are bad enough to stop the replay. + /// + /// [`TreatAsUnresolved::git()`](gix_merge::tree::TreatAsUnresolved::git()) matches what `git` + /// itself would refuse to commit. + pub treat_as_unresolved: gix_merge::tree::TreatAsUnresolved, + /// What to do with a commit whose replay changes nothing. + pub empty: EmptyBehavior, +} + +impl Default for Options { + fn default() -> Self { + Options { + tree_merge: gix_merge::tree::Options::default(), + treat_as_unresolved: gix_merge::tree::TreatAsUnresolved::git(), + empty: EmptyBehavior::default(), + } + } +} + +/// What happened to a single commit that was handed to [`replay()`](crate::replay()). +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum Pick { + /// The commit was replayed as a new commit with the given id. + Rewritten(ObjectId), + /// The commit was dropped, as replaying it wouldn't have changed anything. + Dropped, +} + +/// The result of replaying every commit without hitting a conflict. +#[derive(Debug, Clone)] +pub struct Complete { + /// The id of the last commit that was written, or the `onto` commit if every commit was dropped. + pub tip: ObjectId, + /// One entry per input commit, in input order. + pub picks: Vec, +} + +impl Complete { + /// Return the ids of all commits that were actually written, oldest first. + pub fn rewritten(&self) -> impl Iterator + '_ { + self.picks.iter().filter_map(|pick| match pick { + Pick::Rewritten(id) => Some(*id), + Pick::Dropped => None, + }) + } + + /// Return the amount of commits that were dropped. + pub fn dropped(&self) -> usize { + self.picks + .iter() + .filter(|pick| **pick == Pick::Dropped) + .count() + } +} + +/// The conflict that made a replay stop. +#[derive(Debug, Clone)] +pub struct Conflict { + /// The commit that couldn't be replayed. + pub at: ObjectId, + /// The tip it was being replayed onto. + pub onto: ObjectId, + /// The commits that were written before the conflict was hit, oldest first. + pub picks: Vec, + /// The conflicts that were encountered, including those that could be resolved automatically. + pub conflicts: Vec, +} + +impl Conflict { + /// Return the paths of all conflicts that `how` considers unresolved, in the order they were + /// encountered. + pub fn unresolved_paths( + &self, + how: gix_merge::tree::TreatAsUnresolved, + ) -> impl Iterator + '_ { + self.conflicts + .iter() + .filter(move |conflict| conflict.is_unresolved(how)) + .map(|conflict| conflict.changes_in_resolution().1.location()) + } +} + +/// The outcome of [`replay()`](crate::replay()). +#[derive(Debug, Clone)] +pub enum Outcome { + Complete(Complete), + Conflict(Conflict), +} + +/// The error returned by [`replay()`](crate::replay()). +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error("Could not find a commit to replay, the commit it is based on, or the tip")] + FindCommit(#[from] gix_object::find::existing_object::Error), + #[error("Could not decode a commit that is to be replayed")] + DecodeCommit(#[from] gix_object::decode::Error), + #[error( + "Commit {id} has {parents} parents, but only commits with a single parent can be replayed" + )] + MergeCommit { id: ObjectId, parents: usize }, + #[error(transparent)] + TreeMerge(#[from] Box), + #[error("Could not write the merged tree of a replayed commit into the object database: {0}")] + WriteTree(gix_object::write::Error), + #[error("Could not write a replayed commit into the object database: {0}")] + WriteCommit(gix_object::write::Error), +} + +pub(super) mod function; diff --git a/crates/gix-rebase/tests/fixtures/rebase-baseline.sh b/crates/gix-rebase/tests/fixtures/rebase-baseline.sh new file mode 100755 index 0000000000000000000000000000000000000000..6b5ce3e1d2de2f2af5560dd3b0c430541f3802f6 --- /dev/null +++ b/crates/gix-rebase/tests/fixtures/rebase-baseline.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# Build one repository per rebase case and record what `git rebase` makes of it, so the Rust side can +# be held to the real thing. +# +# Each case ends up as a directory containing a repository with these refs: +# +# onto the commit to replay onto +# source the tip of the commits to replay +# expected what `git rebase onto source` produced, absent if git stopped on a conflict +# +# A line per case is appended to `baseline.cases`: +set -eu -o pipefail + +export GIT_CONFIG_GLOBAL=/dev/null +export GIT_CONFIG_SYSTEM=/dev/null +export GIT_AUTHOR_NAME=author +export GIT_AUTHOR_EMAIL=author@example.com +export GIT_COMMITTER_NAME=committer +export GIT_COMMITTER_EMAIL=committer@example.com +export GIT_TERMINAL_PROMPT=0 + +# Keep this in sync with `REBASE_COMMITTER_*` in tests/rebase/main.rs: the replayed commits must be +# byte-identical to git's, which means the committer has to match down to the second. +readonly REBASE_DATE="1700009999 +0000" + +tick_time=1700000000 +function tick() { + tick_time=$((tick_time + 60)) + export GIT_AUTHOR_DATE="$tick_time +0000" + export GIT_COMMITTER_DATE="$tick_time +0000" +} + +function new_case() { + local name=${1:?case name} + mkdir -p "$name" + cd "$name" + tick_time=1700000000 + tick + git init -q -b main . + git config core.autocrlf false +} + +# Run the reference rebase and record the result. Must be called from inside a case directory, with +# `onto` and `source` already pointing somewhere. +# +# The rebase runs on a *detached* HEAD on purpose: rebasing the `source` branch would move it to the +# result, and then `onto..source` would be git's output rather than the input we want to replay. +function record() { + local name=${1:?case name} + shift + git symbolic-ref HEAD refs/heads/source + git reset -q --hard source + git clean -qfdx + git checkout -q --detach source + local conflicted=0 + if GIT_COMMITTER_DATE="$REBASE_DATE" git rebase "$@" onto >/dev/null 2>&1; then + git branch -f expected HEAD + else + conflicted=1 + git rebase --abort >/dev/null 2>&1 || true + fi + echo "$name $conflicted" >> ../baseline.cases + cd .. +} + +function commit() { + tick + local message=${1:?message} + git add -A + git commit -q --allow-empty -m "$message" +} + +# A commit carrying arbitrary extra headers, written by hand because `git commit` cannot make one. +# Usage: commit_with_headers ... +function commit_with_headers() { + tick + local message=${1:?message} + shift + git add -A + local tree parent + tree="$(git write-tree)" + parent="$(git rev-parse HEAD)" + { + echo "tree $tree" + echo "parent $parent" + echo "author $GIT_AUTHOR_NAME <$GIT_AUTHOR_EMAIL> $GIT_AUTHOR_DATE" + echo "committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE" + local header + for header in "$@"; do echo "$header"; done + echo + echo "$message" + } > .commit-raw + local id + id="$(git hash-object -t commit -w --stdin < .commit-raw)" + rm -f .commit-raw + git update-ref HEAD "$id" +} + +# --- linear: three commits replayed onto an advanced base ------------------------------------- +new_case linear + echo base > base.txt && commit base + git branch fork + echo one > one.txt && commit "onto one" + echo two > two.txt && commit "onto two" + git branch onto + git symbolic-ref HEAD refs/heads/fork && git reset -q --hard fork && git clean -qfdx + echo a > a.txt && commit "pick a" + echo b > b.txt && commit "pick b" + echo c > c.txt && commit "pick c" + git branch source +record linear + +# --- extra headers: git drops them, we must not ----------------------------------------------- +new_case extra-headers + echo base > base.txt && commit base + git branch fork + echo one > one.txt && commit "onto one" + git branch onto + git symbolic-ref HEAD refs/heads/fork && git reset -q --hard fork && git clean -qfdx + echo a > a.txt && commit_with_headers "pick with change-id" \ + "change-id zzzzkzkzkzkzkzkzkzkzkzkzkzkz" "x-custom just because" + echo b > b.txt && commit "plain follow-up" + git branch source +record extra-headers + +# --- a stale signature must be stripped ------------------------------------------------------- +new_case stale-signature + echo base > base.txt && commit base + git branch fork + echo one > one.txt && commit "onto one" + git branch onto + git symbolic-ref HEAD refs/heads/fork && git reset -q --hard fork && git clean -qfdx + echo a > a.txt && commit_with_headers "signed pick" \ + "gpgsig -----BEGIN PGP SIGNATURE-----" \ + "mergetag not-really-a-tag" + git branch source +record stale-signature + +# --- content conflict on the same line -------------------------------------------------------- +new_case conflict + printf 'one\ntwo\nthree\n' > file.txt && commit base + git branch fork + printf 'one\nONTO\nthree\n' > file.txt && commit "onto changes the middle" + git branch onto + git symbolic-ref HEAD refs/heads/fork && git reset -q --hard fork && git clean -qfdx + printf 'one\nSOURCE\nthree\n' > file.txt && commit "source changes the middle" + git branch source +record conflict + +# --- already applied upstream: becomes empty, so git drops it --------------------------------- +new_case becomes-empty + echo base > base.txt && commit base + git branch fork + echo shared > shared.txt && commit "onto adds shared" + git branch onto + git symbolic-ref HEAD refs/heads/fork && git reset -q --hard fork && git clean -qfdx + echo shared > shared.txt && commit "source adds the very same shared" + echo after > after.txt && commit "and something else" + git branch source +record becomes-empty + +# --- started empty: git keeps it -------------------------------------------------------------- +new_case starts-empty + echo base > base.txt && commit base + git branch fork + echo one > one.txt && commit "onto one" + git branch onto + git symbolic-ref HEAD refs/heads/fork && git reset -q --hard fork && git clean -qfdx + commit "deliberately empty" + echo a > a.txt && commit "pick a" + git branch source +record starts-empty + +# --- rename on one side, edit on the other: rename detection has to kick in -------------------- +new_case rename-and-edit + printf 'alpha\nbeta\ngamma\ndelta\nepsilon\nzeta\neta\ntheta\n' > old-name.txt && commit base + git branch fork + git mv old-name.txt new-name.txt && commit "onto renames the file" + git branch onto + git symbolic-ref HEAD refs/heads/fork && git reset -q --hard fork && git clean -qfdx + printf 'alpha\nbeta\ngamma\ndelta\nepsilon\nzeta\neta\nTHETA\n' > old-name.txt && commit "source edits the file" + git branch source +record rename-and-edit + +# --- the source is already based on the tip: pure reparenting ---------------------------------- +new_case already-based + echo base > base.txt && commit base + git branch onto + echo a > a.txt && commit "pick a" + echo b > b.txt && commit "pick b" + git branch source +record already-based --no-ff diff --git a/crates/gix-rebase/tests/rebase/baseline.rs b/crates/gix-rebase/tests/rebase/baseline.rs new file mode 100644 index 0000000000000000000000000000000000000000..aa675e39f888b454c0b1ac29a751c2dfc9782990 --- /dev/null +++ b/crates/gix-rebase/tests/rebase/baseline.rs @@ -0,0 +1,351 @@ +use gix_rebase::replay::{self, EmptyBehavior, Pick}; + +use crate::{Case, complete, conflict, fixture}; + +/// `git rebase` throws every extra commit header away, which is the one thing this crate exists not +/// to do. Those cases get their own assertions below instead of an oid comparison. +const DIVERGES_FROM_GIT: &[&str] = &["extra-headers"]; + +#[test] +fn every_case_matches_git() -> crate::Result { + let cases = std::fs::read_to_string(fixture().join("baseline.cases"))?; + let mut seen = 0; + for line in cases.lines().filter(|line| !line.trim().is_empty()) { + let (name, conflicted) = line + .split_once(' ') + .expect("` ` per line"); + let conflicted = conflicted.trim() == "1"; + seen += 1; + + let case = Case::open(name); + let outcome = case.replay(case.git_default_options())?; + + if conflicted { + let stopped = conflict(outcome); + assert!( + case.expected().is_empty(), + "{name}: git conflicted, so it cannot have recorded a result" + ); + assert!( + stopped + .unresolved_paths(replay::Options::default().treat_as_unresolved) + .next() + .is_some(), + "{name}: a conflict must name at least one path" + ); + continue; + } + + let ours = complete(outcome); + let theirs = case.expected(); + + if DIVERGES_FROM_GIT.contains(&name) { + assert_eq!( + ours.rewritten().count(), + theirs.len(), + "{name}: we must replay the same commits as git, even where the objects differ" + ); + continue; + } + + assert_eq!( + ours.rewritten().collect::>(), + theirs, + "{name}: the replayed commits must be byte-identical to git's" + ); + assert_eq!( + ours.tip, + theirs.last().copied().unwrap_or(case.onto), + "{name}: the tip must be git's tip" + ); + } + assert_eq!(seen, 8, "all fixture cases ran; adjust when adding one"); + Ok(()) +} + +#[test] +fn extra_headers_survive_even_though_git_drops_them() -> crate::Result { + let case = Case::open("extra-headers"); + let ours = complete(case.replay(case.git_default_options())?); + let theirs = case.expected(); + let picks = case.picks(); + + let (ours_first, theirs_first) = (ours.rewritten().next().expect("a commit"), theirs[0]); + assert_ne!( + ours_first, theirs_first, + "keeping the headers necessarily changes the object id" + ); + + assert_eq!( + case.header(ours_first, "change-id").as_deref(), + Some("zzzzkzkzkzkzkzkzkzkzkzkzkzkz"), + "the `change-id` is carried over verbatim" + ); + assert_eq!( + case.header(ours_first, "x-custom").as_deref(), + Some("just because"), + "headers we know nothing about are carried over too" + ); + assert_eq!( + case.header(theirs_first, "change-id"), + None, + "git, for comparison, drops them" + ); + + // Everything else has to be exactly what git produced. + let (mine, gits, original) = ( + case.commit(ours_first), + case.commit(theirs_first), + case.commit(picks[0]), + ); + assert_eq!(mine.tree, gits.tree, "same merge result"); + assert_eq!(mine.parents, gits.parents, "same new parent"); + assert_eq!(mine.author, original.author, "the author is preserved"); + assert_eq!(mine.author, gits.author, "git preserves it the same way"); + assert_eq!(mine.committer, gits.committer, "same fresh committer"); + assert_eq!(mine.message, gits.message, "same message"); + + // The follow-up commit carries no headers of its own, so everything about it matches git's -- + // except its parent, and therefore its id, because our first commit is a different object. + let (plain, gits_plain) = ( + case.commit(ours.rewritten().nth(1).expect("a second commit")), + case.commit(theirs[1]), + ); + assert_eq!(plain.tree, gits_plain.tree); + assert_eq!(plain.author, gits_plain.author); + assert_eq!(plain.committer, gits_plain.committer); + assert_eq!(plain.message, gits_plain.message); + assert!(plain.extra_headers.is_empty()); + assert_eq!( + plain.parents.as_slice(), + [ours_first], + "it follows our commit, which is why its own id differs from git's" + ); + Ok(()) +} + +#[test] +fn a_stale_signature_is_stripped() -> crate::Result { + let case = Case::open("stale-signature"); + let ours = complete(case.replay(case.git_default_options())?); + let rewritten = ours.rewritten().next().expect("a commit"); + + assert_eq!( + case.commit(rewritten).extra_headers, + Vec::new(), + "a signature and a mergetag cannot survive a rewrite, so they go" + ); + assert_eq!( + vec![rewritten], + case.expected(), + "which makes the commit byte-identical to git's" + ); + Ok(()) +} + +#[test] +fn a_commit_that_becomes_empty_is_dropped_but_can_be_kept() -> crate::Result { + let case = Case::open("becomes-empty"); + let picks = case.picks(); + assert_eq!(picks.len(), 2, "the fixture replays two commits"); + + let dropped = complete(case.replay(case.git_default_options())?); + assert_eq!( + dropped.picks[0], + Pick::Dropped, + "the commit whose changes are already upstream goes, as `--empty=drop` is git's default" + ); + assert_eq!(dropped.dropped(), 1); + assert_eq!( + dropped.rewritten().collect::>(), + case.expected(), + "leaving exactly what git left" + ); + + let kept = complete(case.replay(replay::Options { + empty: EmptyBehavior::Keep, + ..case.git_default_options() + })?); + assert_eq!(kept.dropped(), 0, "`--empty=keep` keeps it"); + assert_eq!(kept.rewritten().count(), 2); + Ok(()) +} + +#[test] +fn a_commit_that_started_empty_is_kept_but_can_be_dropped() -> crate::Result { + let case = Case::open("starts-empty"); + let kept = complete(case.replay(case.git_default_options())?); + assert_eq!( + kept.rewritten().collect::>(), + case.expected(), + "an intentionally empty commit is kept, byte-identical to git's" + ); + assert_eq!(kept.dropped(), 0); + + let dropped = complete(case.replay(replay::Options { + empty: EmptyBehavior::AbandonWasEmpty, + ..case.git_default_options() + })?); + assert_eq!(dropped.dropped(), 1, "`--no-keep-empty` drops it"); + Ok(()) +} + +/// The four variants are the four combinations of git's `--empty` and `--keep-empty`, so pin every +/// one of them against both shapes of empty commit. `becomes-empty` replays a commit whose changes +/// are already upstream alongside one that isn't; `starts-empty` replays a deliberately empty commit +/// alongside one that isn't. Each fixture therefore has exactly one droppable commit, and which +/// variants drop it is the whole behaviour. +#[test] +fn each_empty_behaviour_is_one_pair_of_git_flags() -> crate::Result { + let grid = [ + // (behaviour, dropped in `becomes-empty`, dropped in `starts-empty`) + (EmptyBehavior::Keep, 0, 0), + (EmptyBehavior::AbandonWasEmpty, 0, 1), + (EmptyBehavior::AbandonNewlyEmpty, 1, 0), + (EmptyBehavior::AbandonAllEmpty, 1, 1), + ]; + for (empty, becomes_empty, starts_empty) in grid { + for (name, want) in [ + ("becomes-empty", becomes_empty), + ("starts-empty", starts_empty), + ] { + let case = Case::open(name); + let ours = complete(case.replay(replay::Options { + empty, + ..case.git_default_options() + })?); + assert_eq!(ours.dropped(), want, "{name} under {empty:?}"); + assert_eq!( + ours.rewritten().count() + ours.dropped(), + case.picks().len(), + "{name} under {empty:?}: every pick has to be accounted for" + ); + } + } + + assert_eq!( + EmptyBehavior::default(), + EmptyBehavior::AbandonNewlyEmpty, + "our default is git's: keep a deliberately empty commit, drop a redundant one" + ); + Ok(()) +} + +#[test] +fn renames_are_detected_like_git_does() -> crate::Result { + let case = Case::open("rename-and-edit"); + let ours = complete(case.replay(case.git_default_options())?); + assert_eq!( + ours.rewritten().collect::>(), + case.expected(), + "the edit has to follow the rename, or the trees won't match git's" + ); + + // Guard the guard: with rename tracking off this is a modify/delete conflict, so a regression in + // how the options are wired up cannot pass silently. + let without_rewrites = conflict(case.replay(replay::Options { + tree_merge: gix_merge::tree::Options { + rewrites: None, + ..case.git_default_options().tree_merge + }, + ..case.git_default_options() + })?); + assert_eq!( + without_rewrites + .unresolved_paths(replay::Options::default().treat_as_unresolved) + .map(ToString::to_string) + .collect::>(), + vec!["old-name.txt".to_string()], + "without rename tracking the edit has nowhere to go" + ); + Ok(()) +} + +#[test] +fn a_source_already_based_on_the_tip_is_only_reparented() -> crate::Result { + let case = Case::open("already-based"); + let ours = complete(case.replay(case.git_default_options())?); + assert_eq!( + ours.rewritten().collect::>(), + case.expected(), + "git rewrites them too, because the committer changes" + ); + + let picks = case.picks(); + for (original, rewritten) in picks.iter().zip(ours.rewritten()) { + let (before, after) = (case.commit(*original), case.commit(rewritten)); + assert_eq!( + after.tree, before.tree, + "no merge happened, so the tree is untouched" + ); + assert_eq!(after.author, before.author); + assert_eq!( + after.committer, + crate::rebase_committer(), + "only the committer moved" + ); + } + Ok(()) +} + +#[test] +fn a_conflict_reports_the_offending_path_and_writes_no_commit() -> crate::Result { + let case = Case::open("conflict"); + let picks = case.picks(); + let stopped = conflict(case.replay(case.git_default_options())?); + + assert_eq!( + stopped.at, picks[0], + "it stops at the commit that conflicted" + ); + assert_eq!(stopped.onto, case.onto, "having written nothing before it"); + assert!(stopped.picks.is_empty()); + assert_eq!( + stopped + .unresolved_paths(replay::Options::default().treat_as_unresolved) + .map(ToString::to_string) + .collect::>(), + vec!["file.txt".to_string()], + ); + Ok(()) +} + +#[test] +fn a_merge_commit_cannot_be_replayed() -> crate::Result { + // Build a merge commit out of the two diverged sides the `conflict` fixture already has. + let case = Case::open("conflict"); + let merge = { + let mut commit = case.commit(case.source); + commit.parents.push(case.onto); + gix_object::Write::write(&case.repo, &commit).expect("the merge commit can be written") + }; + + let err = case + .replay_onto(case.onto, &[merge], case.git_default_options()) + .expect_err("a merge commit is refused"); + assert!( + matches!(err, replay::Error::MergeCommit { parents: 2, .. }), + "got {err:?}" + ); + Ok(()) +} + +#[test] +fn a_root_commit_is_replayed_against_the_empty_tree() -> crate::Result { + // `linear`'s first commit has no parent; replaying it onto `onto` is what `git rebase --root` + // would do, and its changes are already there, so it drops. + let case = Case::open("linear"); + let root = { + let mut ids: Vec<_> = case + .repo + .rev_walk([case.onto]) + .all()? + .map(|info| info.expect("commit is present").id) + .collect(); + ids.pop().expect("a root commit") + }; + let ours = complete(case.replay_onto(case.onto, &[root], case.git_default_options())?); + assert_eq!(ours.picks, vec![Pick::Dropped]); + assert_eq!(ours.tip, case.onto, "nothing was written"); + Ok(()) +} diff --git a/crates/gix-rebase/tests/rebase/main.rs b/crates/gix-rebase/tests/rebase/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..d1b2ef061b8870b028181a63186674ea544b07ed --- /dev/null +++ b/crates/gix-rebase/tests/rebase/main.rs @@ -0,0 +1,193 @@ +//! Everything here is held against the real `git` binary: `tests/fixtures/rebase-baseline.sh` builds +//! a repository per case and records what `git rebase` made of it, and the tests replay the same +//! input through [`gix_rebase::replay`] and compare. +use std::path::{Path, PathBuf}; + +use gix_hash::ObjectId; +use gix_rebase::replay::{self, Outcome}; + +mod baseline; + +/// What every test returns, mirroring how the `gix` crates do it. +pub type Result = std::result::Result<(), Box>; + +/// The committer that `git rebase` was run with in the fixture. It has to match to the second, or +/// the replayed commits cannot be byte-identical to git's. +/// +/// Keep in sync with `REBASE_DATE` in `tests/fixtures/rebase-baseline.sh`. +const REBASE_COMMITTER_SECONDS: i64 = 1_700_009_999; + +pub fn rebase_committer() -> gix_actor::Signature { + gix_actor::Signature { + name: "committer".into(), + email: "committer@example.com".into(), + time: gix::date::Time { + seconds: REBASE_COMMITTER_SECONDS, + offset: 0, + }, + } +} + +/// Run the fixture script once per test binary and hand out the directory it produced. +pub fn fixture() -> &'static Path { + static FIXTURE: std::sync::OnceLock = std::sync::OnceLock::new(); + FIXTURE.get_or_init(|| { + let script = + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/rebase-baseline.sh"); + // Not a tempdir: keeping it around makes a failure inspectable with plain `git`. + let root = Path::new(env!("CARGO_TARGET_TMPDIR")).join("rebase-baseline"); + if root.exists() { + std::fs::remove_dir_all(&root).expect("fixture directory can be removed"); + } + std::fs::create_dir_all(&root).expect("fixture directory can be created"); + let status = std::process::Command::new("bash") + .arg(&script) + .current_dir(&root) + .status() + .expect("`bash` is available to run the fixture"); + assert!(status.success(), "fixture script failed: {script:?}"); + root + }) +} + +pub struct Case { + pub repo: gix::Repository, + pub onto: ObjectId, + pub source: ObjectId, +} + +impl Case { + pub fn open(name: &str) -> Self { + let repo = gix::open(fixture().join(name)).expect("fixture repository opens"); + let onto = single_ref(&repo, "onto"); + let source = single_ref(&repo, "source"); + Case { repo, onto, source } + } + + /// The commits `git rebase` would have put on its todo-list: everything in `onto..source` that + /// isn't a merge, oldest first. + pub fn picks(&self) -> Vec { + let mut picks: Vec<_> = self + .repo + .rev_walk([self.source]) + .with_hidden([self.onto]) + .all() + .expect("revwalk starts") + .map(|info| info.expect("commit is present")) + .filter(|info| info.parent_ids.len() < 2) + .map(|info| info.id) + .collect(); + picks.reverse(); + picks + } + + /// The commits `git rebase` actually produced, oldest first. Empty if the case conflicted. + pub fn expected(&self) -> Vec { + let Ok(expected) = self.repo.rev_parse_single("expected") else { + return Vec::new(); + }; + let mut ids: Vec<_> = self + .repo + .rev_walk([expected.detach()]) + .with_hidden([self.onto]) + .all() + .expect("revwalk starts") + .map(|info| info.expect("commit is present").id) + .collect(); + ids.reverse(); + ids + } + + pub fn replay(&self, options: replay::Options) -> std::result::Result { + self.replay_onto(self.onto, &self.picks(), options) + } + + pub fn replay_onto( + &self, + onto: ObjectId, + picks: &[ObjectId], + options: replay::Options, + ) -> std::result::Result { + let mut diff_state = gix_diff::tree::State::default(); + let mut diff_resource_cache = self + .repo + .diff_resource_cache_for_tree_diff() + .expect("diff cache can be created"); + let mut blob_merge = self + .repo + .merge_resource_cache(Default::default()) + .expect("merge cache can be created"); + gix_rebase::replay( + &onto, + picks, + &rebase_committer(), + Default::default(), + &self.repo, + &mut diff_state, + &mut diff_resource_cache, + &mut blob_merge, + options, + ) + } + + /// The options to use to match `git rebase`'s defaults, taken from the repository's own config. + pub fn git_default_options(&self) -> replay::Options { + replay::Options { + tree_merge: self + .repo + .tree_merge_options() + .expect("merge options can be read from config") + .into(), + ..Default::default() + } + } + + pub fn commit(&self, id: ObjectId) -> gix_object::Commit { + self.repo + .find_object(id) + .expect("object is present") + .into_commit() + .decode() + .expect("commit decodes") + .to_owned() + .expect("commit can be owned") + } + + pub fn header(&self, id: ObjectId, name: &str) -> Option { + self.commit(id) + .extra_headers + .into_iter() + .find(|(field, _)| field == name) + .map(|(_, value)| value.to_string()) + } +} + +fn single_ref(repo: &gix::Repository, name: &str) -> ObjectId { + repo.rev_parse_single(name) + .unwrap_or_else(|err| panic!("fixture has a `{name}` ref: {err}")) + .detach() +} + +pub fn complete(outcome: Outcome) -> replay::Complete { + match outcome { + Outcome::Complete(complete) => complete, + Outcome::Conflict(conflict) => { + panic!( + "expected the replay to complete, but it conflicted at {}", + conflict.at + ) + } + } +} + +pub fn conflict(outcome: Outcome) -> replay::Conflict { + match outcome { + Outcome::Conflict(conflict) => conflict, + Outcome::Complete(complete) => { + panic!( + "expected a conflict, but the replay completed at {}", + complete.tip + ) + } + } +} diff --git a/crates/gix-receive-pack/Cargo.toml b/crates/gix-receive-pack/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..751a0baaf11a84f375bff479a1d2e3bac4544a11 --- /dev/null +++ b/crates/gix-receive-pack/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "gix-receive-pack" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[lib] +doctest = false + +[dependencies] +gix-features = { workspace = true, features = ["progress"] } +gix-hash = { workspace = true } +gix-object = { workspace = true } +gix-packetline = { workspace = true } +gix-ref = { workspace = true } +gix-transport = { workspace = true, features = ["async-client"] } + +futures-lite = { workspace = true } + +thiserror = { workspace = true } + +[dev-dependencies] +gix = { workspace = true } + +[package.metadata.docs.rs] +all-features = true diff --git a/crates/gix-receive-pack/src/advertisement.rs b/crates/gix-receive-pack/src/advertisement.rs new file mode 100644 index 0000000000000000000000000000000000000000..89b9654ee7a7fa40bb49fd897b7c2ffc5fa68b89 --- /dev/null +++ b/crates/gix-receive-pack/src/advertisement.rs @@ -0,0 +1,194 @@ +use std::collections::HashMap; + +use gix_hash::ObjectId; +use gix_object::bstr::{BStr, BString, ByteSlice}; +use gix_ref::FullName; +use gix_transport::client::Capabilities; + +use crate::ReceivePackAdvertisement; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("expected ` ` in the ref advertisement, got {0:?}")] + MalformedRefLine(BString), + #[error("the remote advertised {name:?}, which is not a valid reference name")] + InvalidRefName { + name: BString, + source: gix_ref::name::Error, + }, + #[error("the remote advertised the unknown hash algorithm {0:?}")] + UnsupportedObjectFormat(BString), + #[error(transparent)] + ObjectId(#[from] gix_hash::decode::Error), +} + +impl ReceivePackAdvertisement { + pub fn object_hash(&self) -> Result { + self.capabilities.object_hash() + } +} + +pub trait CapabilitiesExt { + fn object_hash(&self) -> Result; +} + +impl CapabilitiesExt for Capabilities { + fn object_hash(&self) -> Result { + match self.capability("object-format").and_then(|c| c.value()) { + Some(format) => format + .to_str() + .ok() + .and_then(|f| f.parse().ok()) + .ok_or_else(|| Error::UnsupportedObjectFormat(format.into())), + None => Ok(gix_hash::Kind::Sha1), + } + } +} + +pub fn from_v1_lines(lines: &BStr) -> Result<(HashMap, Vec), Error> { + let mut remote_refs = HashMap::new(); + let mut advertised_haves = Vec::new(); + + for line in lines.lines() { + let line = line.trim_end().as_bstr(); + if line.is_empty() { + continue; + } + let Some(space) = line.find_byte(b' ') else { + return Err(Error::MalformedRefLine(line.into())); + }; + let (hex, name) = line.split_at(space); + let name = name[1..].as_bstr(); + if name.is_empty() { + return Err(Error::MalformedRefLine(line.into())); + } + // `shallow ` is not used on `git-receive-pack`. + if hex == b"shallow" { + continue; + } + // happens on two cases we don't care: + // 1. peeled refs: ` refs/tags/v1^{}` + // 2. empty repository: ` capabilities^{}` + if name.ends_with(b"^{}") { + continue; + } + let id = ObjectId::from_hex(hex)?; + if name == ".have" { + advertised_haves.push(id); + continue; + } + let name = FullName::try_from(name).map_err(|source| Error::InvalidRefName { + name: name.into(), + source, + })?; + remote_refs.insert(name, id); + } + + Ok((remote_refs, advertised_haves)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const A: &str = "1111111111111111111111111111111111111111"; + const B: &str = "2222222222222222222222222222222222222222"; + const NULL: &str = "0000000000000000000000000000000000000000"; + + fn parse(lines: &str) -> (HashMap, Vec) { + from_v1_lines(lines.into()).expect("valid advertisement") + } + + #[test] + fn haves_are_split_out_before_refname_validation() { + let (refs, haves) = parse(&format!( + "{A} refs/heads/main\n\ + {B} .have\n\ + {A} .have\n" + )); + assert_eq!(refs.len(), 1, "`.have` is not a ref"); + assert_eq!( + refs[&FullName::try_from("refs/heads/main").unwrap()], + ObjectId::from_hex(A.as_bytes()).unwrap() + ); + assert_eq!( + haves, + [ + ObjectId::from_hex(B.as_bytes()).unwrap(), + ObjectId::from_hex(A.as_bytes()).unwrap() + ], + "duplicates of an advertised ref are kept: they are a set of objects, not a map" + ); + } + + #[test] + fn peeled_tags_and_the_empty_repo_carrier_are_skipped() { + let (refs, haves) = parse(&format!( + "{NULL} capabilities^{{}}\n\ + {A} refs/tags/v1\n\ + {B} refs/tags/v1^{{}}\n" + )); + assert_eq!(refs.len(), 1); + assert_eq!( + refs[&FullName::try_from("refs/tags/v1").unwrap()], + ObjectId::from_hex(A.as_bytes()).unwrap(), + "the unpeeled value is the one `expected_old` must match" + ); + assert!(haves.is_empty()); + } + + #[test] + fn shallow_boundaries_are_tolerated() { + let (refs, _) = parse(&format!("shallow {A}\n{B} refs/heads/main\n")); + assert_eq!(refs.len(), 1); + } + + #[test] + fn empty_advertisement_is_not_an_error() { + let (refs, haves) = parse(""); + assert!(refs.is_empty() && haves.is_empty()); + } + + #[test] + fn a_line_without_a_space_is_rejected() { + assert!(matches!( + from_v1_lines(A.into()), + Err(Error::MalformedRefLine(_)) + )); + } + + #[test] + fn an_invalid_refname_is_rejected_by_name() { + let err = from_v1_lines(format!("{A} refs/heads/..\n").as_str().into()).unwrap_err(); + assert!(matches!(err, Error::InvalidRefName { .. }), "got {err:?}"); + } + + #[test] + fn object_format_defaults_to_sha1() { + let advertised = |caps: &str| { + let (capabilities, _) = + gix_transport::client::Capabilities::from_bytes(format!("\0{caps}").as_bytes()) + .unwrap(); + ReceivePackAdvertisement { + server_protocol_version: gix_transport::Protocol::V1, + capabilities, + remote_refs: HashMap::new(), + advertised_haves: Vec::new(), + } + }; + assert_eq!( + advertised("report-status").object_hash().unwrap(), + gix_hash::Kind::Sha1 + ); + assert_eq!( + advertised("report-status object-format=sha256") + .object_hash() + .unwrap(), + gix_hash::Kind::Sha256 + ); + assert!(matches!( + advertised("report-status object-format=sha3").object_hash(), + Err(Error::UnsupportedObjectFormat(_)) + )); + } +} diff --git a/crates/gix-receive-pack/src/command.rs b/crates/gix-receive-pack/src/command.rs new file mode 100644 index 0000000000000000000000000000000000000000..ea09af5cbd8f733157c297207d31f13c85dd6d24 --- /dev/null +++ b/crates/gix-receive-pack/src/command.rs @@ -0,0 +1,360 @@ +use gix_object::bstr::{BStr, BString, ByteVec}; + +use crate::{Options, Update}; + +/// The command block could not be built. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("there is nothing to push")] + NoUpdates, + #[error("the receiving end does not support `{0}`")] + UnsupportedCapability(&'static str), + #[error(transparent)] + Io(#[from] std::io::Error), +} + +/// Intersect what this push needs with what the remote advertised, producing the capability list +/// that rides the first command. +pub(crate) fn select_capabilities( + caps: &gix_transport::client::Capabilities, + updates: &[Update], + opts: &Options<'_>, + agent: Option<&str>, +) -> Result { + // TODO(boltless): support `report-status-v2` + let mut out = BString::from("report-status"); + let mut want = |name: &'static str| { + if caps.contains(name) { + out.push(b' '); + out.push_str(name); + true + } else { + false + } + }; + + want("ofs-delta"); + want("side-band-64k"); + + if opts.quiet && !want("quiet") { + return Err(Error::UnsupportedCapability("quiet")); + } + if opts.atomic && !want("atomic") { + return Err(Error::UnsupportedCapability("atomic")); + } + if !opts.push_options.is_empty() && !want("push-options") { + return Err(Error::UnsupportedCapability("push-options")); + } + if let Some(format) = caps.capability("object-format").and_then(|c| c.value()) { + out.push_str(" object-format="); + out.push_str(format); + } + if let Some(agent) = agent.filter(|_| caps.contains("agent")) { + out.push_str(" agent="); + out.push_str(agent); + } + + if updates.iter().any(|u| u.new.is_null()) && !caps.contains("delete-refs") { + return Err(Error::UnsupportedCapability("delete-refs")); + } + + Ok(out) +} + +/// Encode the reference update request that precedes the packfile: +/// +/// ```text +/// \0 <- capabilities ride the FIRST command only +/// <- bare thereafter +/// 0000 +/// <- only when there are options... +/// 0000 <- ...and then this second flush is mandatory +/// ``` +pub(crate) fn encode_command_block( + updates: &[Update], + capabilities: &BStr, + push_options: &[BString], +) -> Result, Error> { + use gix_packetline::blocking_io::encode; + use std::io::Write; + + if updates.is_empty() { + return Err(Error::NoUpdates); + } + + let mut out = Vec::new(); + let mut line = Vec::new(); + + for (index, update) in updates.iter().enumerate() { + line.clear(); + write!( + &mut line, + "{} {} {}", + update.expected_old.to_hex(), + update.new.to_hex(), + update.name + )?; + if index == 0 { + line.push(0); + line.extend_from_slice(capabilities); + } + encode::text_to_write(&line, &mut out)?; + } + encode::flush_to_write(&mut out)?; + + if !push_options.is_empty() { + for option in push_options { + encode::text_to_write(option, &mut out)?; + } + encode::flush_to_write(&mut out)?; + } + + Ok(out) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use gix_hash::ObjectId; + use gix_object::bstr::ByteSlice; + + use crate::ReceivePackAdvertisement; + + use super::*; + + const A: &str = "1111111111111111111111111111111111111111"; + const B: &str = "2222222222222222222222222222222222222222"; + const NULL: &str = "0000000000000000000000000000000000000000"; + + fn oid(hex: &str) -> ObjectId { + ObjectId::from_hex(hex.as_bytes()).unwrap() + } + + /// Frame a payload as `text_to_write` would: 4 hex length digits over payload + `\n`. + fn pkt(payload: &str) -> String { + format!("{:04x}{payload}\n", payload.len() + 5) + } + + fn advertised(caps: &str) -> ReceivePackAdvertisement { + let (capabilities, _) = + gix_transport::client::Capabilities::from_bytes(format!("\0{caps}").as_bytes()) + .unwrap(); + ReceivePackAdvertisement { + server_protocol_version: gix_transport::Protocol::V1, + capabilities, + remote_refs: HashMap::new(), + advertised_haves: Vec::new(), + } + } + + fn opts() -> Options<'static> { + Options { + atomic: false, + push_options: &[], + quiet: false, + } + } + + fn update(name: &str, old: &str, new: &str) -> Update { + Update { + name: gix_ref::FullName::try_from(name).unwrap(), + expected_old: oid(old), + new: oid(new), + } + } + + #[test] + fn capabilities_ride_the_first_command_only() { + let block = encode_command_block( + &[ + update("refs/heads/main", A, B), + update("refs/heads/next", A, B), + ], + "report-status side-band-64k".into(), + &[], + ) + .unwrap(); + assert_eq!( + block.to_str().unwrap(), + format!( + "{}{}0000", + pkt(&format!( + "{A} {B} refs/heads/main\0report-status side-band-64k" + )), + pkt(&format!("{A} {B} refs/heads/next")) + ), + "one NUL in the whole block, and exactly one newline per line" + ); + } + + #[test] + fn creates_and_deletes_use_the_null_id() { + let block = encode_command_block( + &[ + update("refs/heads/new", NULL, B), + update("refs/heads/gone", A, NULL), + ], + "report-status".into(), + &[], + ) + .unwrap(); + assert_eq!( + block.to_str().unwrap(), + format!( + "{}{}0000", + pkt(&format!("{NULL} {B} refs/heads/new\0report-status")), + pkt(&format!("{A} {NULL} refs/heads/gone")) + ) + ); + } + + #[test] + fn a_null_id_goes_on_the_wire_at_the_width_the_caller_gave_it() { + let block = encode_command_block( + &[Update { + name: gix_ref::FullName::try_from("refs/heads/gone").unwrap(), + expected_old: ObjectId::from_hex(&[b'1'; 64]).unwrap(), + new: ObjectId::null(gix_hash::Kind::Sha256), + }], + "report-status delete-refs".into(), + &[], + ) + .unwrap(); + assert!( + block.to_str().unwrap().contains(&"0".repeat(64)), + "64 zeroes, not 40: {:?}", + block.to_str().unwrap() + ); + } + + #[test] + fn push_options_add_a_second_flush() { + let one = |options: &[BString]| { + encode_command_block( + &[update("refs/heads/main", A, B)], + "report-status".into(), + options, + ) + .unwrap() + }; + assert!( + one(&[]).ends_with(b"0000"), + "one flush when there are no options" + ); + assert_eq!( + one(&["ci.skip".into()]).to_str().unwrap(), + format!( + "{}0000{}0000", + pkt(&format!("{A} {B} refs/heads/main\0report-status")), + pkt("ci.skip") + ) + ); + } + + #[test] + fn nothing_to_push_is_an_error() { + assert!(matches!( + encode_command_block(&[], "report-status".into(), &[]), + Err(Error::NoUpdates) + )); + } + + #[test] + fn selection_only_asks_for_what_was_advertised() { + let adv = advertised("report-status ofs-delta side-band-64k agent=git/2.0"); + let caps = select_capabilities( + &adv.capabilities, + &[update("refs/heads/main", A, B)], + &opts(), + Some("gix/1.0"), + ) + .unwrap(); + assert_eq!(caps, "report-status ofs-delta side-band-64k agent=gix/1.0"); + + let bare = advertised("report-status"); + let caps = select_capabilities( + &bare.capabilities, + &[update("refs/heads/main", A, B)], + &opts(), + Some("gix/1.0"), + ) + .unwrap(); + assert_eq!( + caps, "report-status", + "no agent, no side-band, no ofs-delta" + ); + } + + #[test] + fn opt_in_capabilities_that_were_not_advertised_are_errors() { + let adv = advertised("report-status"); + let up = [update("refs/heads/main", A, B)]; + + let atomic = Options { + atomic: true, + ..opts() + }; + assert!(matches!( + select_capabilities(&adv.capabilities, &up, &atomic, None), + Err(Error::UnsupportedCapability("atomic")) + )); + + let options = [BString::from("ci.skip")]; + let with_options = Options { + push_options: &options, + ..opts() + }; + assert!(matches!( + select_capabilities(&adv.capabilities, &up, &with_options, None), + Err(Error::UnsupportedCapability("push-options")) + )); + } + + #[test] + fn the_advertised_object_format_is_echoed() { + let up = [update("refs/heads/main", A, B)]; + let caps = select_capabilities( + &advertised("report-status object-format=sha256").capabilities, + &up, + &opts(), + None, + ) + .unwrap(); + assert_eq!(caps, "report-status object-format=sha256"); + + let caps = select_capabilities( + &advertised("report-status").capabilities, + &up, + &opts(), + None, + ) + .unwrap(); + assert_eq!(caps, "report-status", "sha1 remotes never say"); + } + + #[test] + fn deleting_needs_delete_refs_but_never_sends_it() { + let delete = [update("refs/heads/gone", A, NULL)]; + assert!(matches!( + select_capabilities( + &advertised("report-status").capabilities, + &delete, + &opts(), + None + ), + Err(Error::UnsupportedCapability("delete-refs")) + )); + let caps = select_capabilities( + &advertised("report-status delete-refs").capabilities, + &delete, + &opts(), + None, + ) + .unwrap(); + assert_eq!( + caps, "report-status", + "the client does not echo `delete-refs`" + ); + } +} diff --git a/crates/gix-receive-pack/src/error.rs b/crates/gix-receive-pack/src/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..2a3646c89373e96894986b26119043f0179c9467 --- /dev/null +++ b/crates/gix-receive-pack/src/error.rs @@ -0,0 +1,19 @@ +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("the remote does not support `report-status`")] + NoReportStatus, + #[error("the push was interrupted")] + Interrupted, + #[error(transparent)] + Advertisement(#[from] crate::advertisement::Error), + #[error(transparent)] + Command(#[from] crate::command::Error), + #[error(transparent)] + Report(#[from] crate::report::Error), + #[error(transparent)] + Transport(#[from] gix_transport::client::Error), + #[error(transparent)] + Decode(#[from] gix_packetline::decode::Error), + #[error(transparent)] + Io(#[from] std::io::Error), +} diff --git a/crates/gix-receive-pack/src/lib.rs b/crates/gix-receive-pack/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..d0866f80544c71b1473b588027ab11b311f5e8d8 --- /dev/null +++ b/crates/gix-receive-pack/src/lib.rs @@ -0,0 +1,12 @@ +pub mod advertisement; +pub mod command; +pub mod report; + +mod error; +pub use error::Error; + +mod session; +pub use session::Session; + +mod types; +pub use types::*; diff --git a/crates/gix-receive-pack/src/report.rs b/crates/gix-receive-pack/src/report.rs new file mode 100644 index 0000000000000000000000000000000000000000..26a0b137033cfd788dbc1fa168062edca4a91ffb --- /dev/null +++ b/crates/gix-receive-pack/src/report.rs @@ -0,0 +1,185 @@ +use gix_object::bstr::{BStr, BString, ByteSlice, ByteVec}; + +use crate::RefUpdateStatus; + +/// Strip the pkt-line framing the report keeps *inside* the side-band. +/// +/// With `side-band-64k` band 1 does not carry report text, it carries the whole `report-status` +/// pkt-line stream, chunked wherever the remote's band buffer ended (`pkt::frame_report` in knot, +/// `send-pack.c`'s second packetline reader in git). So demultiplexing the band leaves +/// `000eunpack ok\n`, not `unpack ok` — one more layer to peel before [`parse`] sees text. +/// +/// Stops at the first flush, which is the report's own terminator. +pub(crate) fn unframe(mut input: &[u8]) -> Result { + use gix_packetline::decode::{Stream, streaming}; + + let mut out = BString::default(); + while !input.is_empty() { + let (line, consumed) = match streaming(input)? { + Stream::Complete { + line, + bytes_consumed, + } => (line, bytes_consumed), + Stream::Incomplete { .. } => break, + }; + input = &input[consumed..]; + let Some(data) = line.as_slice() else { break }; + out.push_str(data); + if out.last() != Some(&b'\n') { + out.push(b'\n'); + } + } + Ok(out) +} + +/// The push produced no usable per-reference report. +/// +/// [`Error::Unpack`] is a *legible* reply: the remote spoke the protocol and refused the packfile, +/// so nothing was applied. The other two mean the reply itself could not be read. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("the remote could not unpack the packfile: {0}")] + Unpack(BString), + #[error("the remote sent no report at all")] + Empty, + #[error( + "expected `unpack `, `ok ` or `ng ` in the report, got {0:?}" + )] + MalformedLine(BString), +} + +/// Parse the `report-status` reply +pub fn parse_report_v1(lines: &BStr) -> Result, Error> { + let mut lines = lines + .lines() + .map(|line| line.trim_end().as_bstr()) + .filter(|line| !line.is_empty()); + + let first = lines.next().ok_or(Error::Empty)?; + let unpack = first + .strip_prefix(b"unpack ".as_slice()) + .ok_or_else(|| Error::MalformedLine(first.into()))?; + if unpack != b"ok" { + return Err(Error::Unpack(unpack.into())); + } + + let mut out = Vec::new(); + for line in lines { + let status = if let Some(name) = line.strip_prefix(b"ok ".as_slice()) { + RefUpdateStatus { + name: name.into(), + status: Ok(()), + } + } else if let Some(rest) = line.strip_prefix(b"ng ".as_slice()) { + // The grammar makes the reason mandatory, but a bare `ng ` is still a legible + // rejection: report it with an empty reason rather than failing the whole reply. + let (name, reason) = match rest.find_byte(b' ') { + Some(space) => (&rest[..space], &rest[space + 1..]), + None => (rest, &[][..]), + }; + RefUpdateStatus { + name: name.into(), + status: Err(reason.into()), + } + } else { + return Err(Error::MalformedLine(line.into())); + }; + out.push(status); + } + + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ok(name: &str) -> RefUpdateStatus { + RefUpdateStatus { + name: name.into(), + status: Ok(()), + } + } + + fn ng(name: &str, why: &str) -> RefUpdateStatus { + RefUpdateStatus { + name: name.into(), + status: Err(why.into()), + } + } + + #[test] + fn a_mixed_reply_keeps_the_remotes_order() { + let report = parse_report_v1( + "unpack ok\n\ + ok refs/heads/main\n\ + ng refs/heads/next non-fast-forward\n\ + ok refs/heads/gone\n" + .into(), + ) + .unwrap(); + assert_eq!( + report, + [ + ok("refs/heads/main"), + ng("refs/heads/next", "non-fast-forward"), + ok("refs/heads/gone") + ] + ); + } + + #[test] + fn every_ref_rejected_is_still_a_completed_exchange() { + let report = + parse_report_v1("unpack ok\nng refs/heads/main pre-receive hook declined\n".into()) + .expect("the pack unpacked"); + assert_eq!(report, [ng("refs/heads/main", "pre-receive hook declined")]); + } + + #[test] + fn a_failed_unpack_discards_the_per_ref_lines() { + let err = parse_report_v1( + "unpack index-pack abnormal exit\nng refs/heads/main unpacker error\n".into(), + ) + .unwrap_err(); + assert_eq!(err, Error::Unpack("index-pack abnormal exit".into())); + } + + #[test] + fn a_reason_less_rejection_is_kept() { + let report = parse_report_v1("unpack ok\nng refs/heads/main\n".into()).unwrap(); + assert_eq!(report, [ng("refs/heads/main", "")]); + } + + /// What actually arrives on band 1: the report is a pkt-line stream of its own, and the band + /// may cut it anywhere. + #[test] + fn a_side_band_report_is_unframed_before_it_is_parsed() { + let framed = b"000eunpack ok\n0017ok refs/heads/main\n0000"; + let text = unframe(framed).expect("well-formed framing"); + assert_eq!(text, "unpack ok\nok refs/heads/main\n"); + assert_eq!( + parse_report_v1(text.as_ref()).unwrap(), + [ok("refs/heads/main")] + ); + + // Trailing junk after the report's own flush is not part of the report. + assert_eq!(unframe(b"000eunpack ok\n0000ffff").unwrap(), "unpack ok\n"); + // A cut-off length prefix ends the report instead of failing the whole push. + assert_eq!(unframe(b"000eunpack ok\n00").unwrap(), "unpack ok\n"); + } + + #[test] + fn an_empty_reply_is_an_error() { + assert!(matches!(parse_report_v1("".into()), Err(Error::Empty))); + } + + #[test] + fn an_unknown_line_is_an_error() { + // we don't support `report-status-v2` yet. + assert!(matches!( + parse_report_v1("unpack ok\nok refs/heads/main\noption forced-update\n".into()), + Err(Error::MalformedLine(_)) + )); + } +} diff --git a/crates/gix-receive-pack/src/session.rs b/crates/gix-receive-pack/src/session.rs new file mode 100644 index 0000000000000000000000000000000000000000..5aa37e1c0fc400fdf594bcf3ecfa451e9a05380f --- /dev/null +++ b/crates/gix-receive-pack/src/session.rs @@ -0,0 +1,191 @@ +use std::ops::ControlFlow; +use std::sync::atomic::{AtomicBool, Ordering}; + +use futures_lite::{AsyncBufReadExt as _, AsyncWriteExt}; +use gix_features::progress::Progress; +use gix_object::bstr::{BString, ByteSlice, ByteVec}; +use gix_transport::Service; +use gix_transport::client::async_io::{ + HandleProgress, ReadlineBufRead, SetServiceResponse, Transport, +}; +use gix_transport::client::{MessageKind, WriteMode}; + +use crate::command::{encode_command_block, select_capabilities}; +use crate::{ + Error, Options, ReceivePackAdvertisement, RefUpdateStatus, Update, advertisement, report, +}; + +pub struct Session<'a, T> { + transport: &'a mut T, + advertised: ReceivePackAdvertisement, + /// Value for `agent=`, emitted only if the remote advertised `agent`. + agent: Option<&'a str>, +} + +impl<'a, T> Session<'a, T> +where + T: Transport, +{ + /// Perform the `git-receive-pack` handshake over `transport` and capture what the remote said. + pub async fn handshake(transport: &'a mut T, agent: Option<&'a str>) -> Result { + // Scoped because `SetServiceResponse` borrows `transport` mutably: everything it owns has + // to become owned data before `Self` can take the transport back — rust-lang/rust#76149. + let advertised = { + let result = transport.handshake(Service::ReceivePack, &[]).await; + let SetServiceResponse { + actual_protocol, + capabilities, + refs, + } = result?; + + // `refs` is `None` under protocol v2, which has no push command at all; the + // `report-status` check below then rejects the session, as v2 never advertises it. + let mut lines = BString::default(); + if let Some(mut refs) = refs { + while let Some(line) = refs.readline().await { + // Two layers: transport failure, then framing failure. + let Some(line) = line??.as_bstr() else { + // A non-data line: the flush ending the advertisement. + break; + }; + lines.push_str(line); + if lines.last() != Some(&b'\n') { + lines.push(b'\n'); + } + } + } + + // let supported_versions: Vec<_> = transport.supported_protocol_versions().into(); + // if !supported_versions.is_empty() && !supported_versions.contains(&actual_protocol) { + // return Err(Error::TransportProtocolPolicyViolation { + // actual_version: actual_protocol, + // }); + // } + + let (remote_refs, advertised_haves) = advertisement::from_v1_lines(lines.as_bstr())?; + ReceivePackAdvertisement { + server_protocol_version: actual_protocol, + capabilities, + remote_refs, + advertised_haves, + } + }; + + // TODO(boltless): support `report-status-v2` with v2 report parser. + if !advertised.capabilities.contains("report-status") { + return Err(Error::NoReportStatus); + } + + Ok(Session { + transport, + advertised, + agent, + }) + } + + pub fn advertised(&self) -> &ReceivePackAdvertisement { + &self.advertised + } + + /// Perform one push operation. + /// + /// `pack` must be `None` for a deletion-only push, not an empty pack: `git-receive-pack` reads + /// no pack after a delete-only command block, so an empty one is left unread on the wire. + pub async fn push( + &mut self, + trace_packetlines: bool, + updates: &[Update], + pack: Option<&[u8]>, + opts: Options<'_>, + progress: &mut impl Progress, + should_interrupt: &AtomicBool, + ) -> Result, Error> { + let capabilities = + select_capabilities(&self.advertised.capabilities, updates, &opts, self.agent)?; + let commands = encode_command_block(updates, capabilities.as_bstr(), opts.push_options)?; + + // `Binary` maps to `PostBodyDataKind::Unbounded`, which puts an HTTP backend into streaming + // mode instead of buffering the whole packfile. `MessageKind::Flush` is inert: only + // `into_read()` writes it, and we take `into_parts()` — our flush is already in `commands`. + let request = + self.transport + .request(WriteMode::Binary, MessageKind::Flush, trace_packetlines)?; + + // `RequestWriter`'s `AsyncWrite` frames every write as a packet line, even under `Binary`; + // `into_parts()` gives the unwrapped writer. The commands are framed already, and the pack + // must not be framed at all. + let (mut writer, mut reader) = request.into_parts(); + writer.write_all(&commands).await?; + + if let Some(pack) = pack { + // Chunked because cancellation is cooperative: one `write_all` of a large pack would + // never look at the flag. + for chunk in pack.chunks(64 * 1024) { + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + writer.write_all(chunk).await?; + } + } + writer.flush().await?; + + // Before any read: on a streaming POST the two halves are interdependent, and getting this + // wrong deadlocks rather than errors. + drop(writer); + + // Only with a side-band: once a handler is set `poll_fill_buf` runs every line through + // `decode_band()`, so on an unmultiplexed stream the `u` of `unpack ok` reads as band 117. + let sideband = capabilities.split_str(" ").any(|c| c == b"side-band-64k"); + if sideband { + let progress = &*progress; + reader.set_progress_handler(Some(Box::new(move |is_err: bool, data: &[u8]| { + // Bands 2 and 3 verbatim: turning `phase: 42% (7/16)` back into a progress bar is + // `gix_protocol::RemoteProgress`'s job, which this crate does not depend on. + let text = data.as_bstr().trim_end(); + if !text.is_empty() { + let text = text.to_str_lossy().into_owned(); + if is_err { + progress.fail(text); + } else { + progress.info(text); + } + } + if should_interrupt.load(Ordering::Relaxed) { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + }) as HandleProgress<'_>)); + } + + // Not `readline()`: on a `WithSidebands` reader it forwards to `read_data_line()`, which + // skips all sideband handling. The demultiplexer lives in `poll_fill_buf`, so the report + // has to come through the `AsyncBufRead` surface, which ends in an empty slice. + let mut lines = BString::default(); + loop { + let line = reader.fill_buf().await?; + if line.is_empty() { + break; + } + let consumed = line.len(); + lines.push_str(line); + // One read is one report line only on the unmultiplexed stream: a side-band payload is + // a pkt-line stream of its own, cut wherever the remote's band buffer ended. + if !sideband && lines.last() != Some(&b'\n') { + lines.push(b'\n'); + } + reader.consume(consumed); + + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + } + + let lines = if sideband { + report::unframe(&lines)? + } else { + lines + }; + Ok(report::parse_report_v1(lines.as_bstr())?) + } +} diff --git a/crates/gix-receive-pack/src/types.rs b/crates/gix-receive-pack/src/types.rs new file mode 100644 index 0000000000000000000000000000000000000000..db76e6ef2ddf3a6b05a4b3057998b530106bc69d --- /dev/null +++ b/crates/gix-receive-pack/src/types.rs @@ -0,0 +1,43 @@ +//! The vocabulary of a push: what the remote offers, what the caller asks for, what came back. +//! +//! Parsing and encoding live in the phase modules ([`crate::advertisement`], [`crate::command`], +//! [`crate::report`]); this is only the data they move between them. + +use std::collections::HashMap; + +use gix_hash::ObjectId; +use gix_object::bstr::BString; +use gix_ref::FullName; + +/// Everything the remote said in the `git-receive-pack` handshake. +/// +/// Parsed by [`crate::advertisement::from_v1_lines`], reached through [`crate::Session::advertised`]. +pub struct ReceivePackAdvertisement { + pub server_protocol_version: gix_transport::Protocol, + // state + pub capabilities: gix_transport::client::Capabilities, + pub remote_refs: HashMap, + pub advertised_haves: Vec, +} + +pub struct Update { + pub name: FullName, + pub expected_old: ObjectId, + pub new: ObjectId, +} + +/// Opt-in capabilities. Each is refused when the remote did not advertise it, rather than sent and +/// silently ignored. Checked in [`crate::Session::upload_pack`]; a caller that wants to fail +/// *before* spending seconds on a pack can consult [`crate::Session::advertised`] first. +pub struct Options<'a> { + pub atomic: bool, + pub push_options: &'a [BString], + pub quiet: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RefUpdateStatus { + pub name: BString, + /// `Err` is the `ng` reason from remote. + pub status: Result<(), BString>, +} diff --git a/crates/knot-capability/Cargo.toml b/crates/knot-capability/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..9fb78042ebab143e4e2cfc130ae20b7cc4e5c455 --- /dev/null +++ b/crates/knot-capability/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "knot-capability" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true diff --git a/crates/knot-capability/src/lib.rs b/crates/knot-capability/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..9bdb33df48643a2a426d48a7644ec7f07a9d8d13 --- /dev/null +++ b/crates/knot-capability/src/lib.rs @@ -0,0 +1,116 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum Capability { + Tangled1, + KnotAcl, + RepoDidInput, + KeepCommit, + AtprotoFirehose, + RepoTokenMint, + SocialWrites, + Foster, +} + +impl Capability { + const ALL: &'static [Capability] = &[ + Capability::Tangled1, + Capability::KnotAcl, + Capability::RepoDidInput, + Capability::KeepCommit, + Capability::AtprotoFirehose, + Capability::RepoTokenMint, + Capability::SocialWrites, + Capability::Foster, + ]; + + pub const SERVED: &'static [Capability] = &[ + Capability::Tangled1, + Capability::KnotAcl, + Capability::RepoDidInput, + Capability::AtprotoFirehose, + Capability::RepoTokenMint, + Capability::SocialWrites, + ]; + + pub const fn token(self) -> &'static str { + match self { + Capability::Tangled1 => "tangled1.0", + Capability::KnotAcl => "knot-acl", + Capability::RepoDidInput => "repo-did-input", + Capability::KeepCommit => "knot-keepcommit", + Capability::AtprotoFirehose => "atproto-firehose", + Capability::RepoTokenMint => "knot-repo-token-mint", + Capability::SocialWrites => "knot-social-writes", + Capability::Foster => "knot-foster", + } + } + + pub fn parse(token: &str) -> Option { + Self::ALL.iter().copied().find(|cap| cap.token() == token) + } + + pub fn served(fostering: bool) -> impl IntoIterator { + Self::SERVED + .iter() + .copied() + .chain(fostering.then_some(Capability::Foster)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_capability_parses_back_from_its_token() { + Capability::ALL.iter().copied().for_each(|cap| { + assert_eq!(Capability::parse(cap.token()), Some(cap), "{cap:?}"); + }); + } + + #[test] + fn tokens_are_distinct() { + let tokens: Vec<&str> = Capability::ALL.iter().map(|cap| cap.token()).collect(); + let mut unique = tokens.clone(); + unique.sort_unstable(); + unique.dedup(); + assert_eq!(unique.len(), tokens.len(), "two variants share a token"); + } + + #[test] + fn unknown_tokens_are_rejected() { + ["", "knot_acl", "KNOT-ACL", "atproto-firehose "] + .iter() + .for_each(|token| assert_eq!(Capability::parse(token), None, "{token:?}")); + } + + #[test] + fn served_capabilities_are_real_capabilities() { + Capability::SERVED.iter().copied().for_each(|cap| { + assert!(Capability::ALL.contains(&cap), "{cap:?} is not in ALL"); + }); + } + + #[test] + fn served_without_fostering_is_the_static_list() { + assert!( + Capability::served(false) + .into_iter() + .eq(Capability::SERVED.iter().copied()) + ); + } + + #[test] + fn served_with_fostering_appends_exactly_foster() { + let served: Vec<_> = Capability::served(true).into_iter().collect(); + assert_eq!(served.len(), Capability::SERVED.len() + 1); + assert!( + served[..Capability::SERVED.len()] + .iter() + .eq(Capability::SERVED.iter()) + ); + assert_eq!( + *served.last().expect("one more than SERVED"), + Capability::Foster + ); + } +} diff --git a/crates/lexicons/Cargo.toml b/crates/lexicons/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..203cfff6ee016f265f8223e4970fff1c1521e724 --- /dev/null +++ b/crates/lexicons/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "lexicons" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bytes = { workspace = true } +chrono = { workspace = true } +cid = { workspace = true } +jacquard-common = { workspace = true } +jacquard-derive = { workspace = true } +jacquard-lexicon = { workspace = true } +miette = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } + +[build-dependencies] +anyhow = { workspace = true } +jacquard-lexicon = { workspace = true, features = ["codegen"] } +walkdir = { workspace = true } + +[features] +default = ["sh_tangled", "org_tangled"] +com_atproto = [] +com_bad_example = [] +sh_tangled = ["com_atproto"] +# org.tangled.temp.search.searchCode#fileResult.repo refs sh.tangled.repo.defs#repoViewBasic +org_tangled = ["sh_tangled"] +streaming = ["jacquard-common/websocket"] + +[lints.clippy] +unnecessary_lazy_evaluations = "allow" +unit_arg = "allow" diff --git a/crates/lexicons/build.rs b/crates/lexicons/build.rs new file mode 100644 index 0000000000000000000000000000000000000000..34bfc2a21b2886bde59dcb25ff53cebeb6b54b4d --- /dev/null +++ b/crates/lexicons/build.rs @@ -0,0 +1,105 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use jacquard_lexicon::codegen::{CodeGenerator, CodegenMode}; +use jacquard_lexicon::corpus::LexiconCorpus; +use jacquard_lexicon::lexicon::LexUserType; +use walkdir::WalkDir; + +const DEFAULT_LEXICONS_SUBDIR: &str = "lexicons"; +const STAGED_SUBDIR: &str = "lexicons-staged"; +const GENERATED_SUBDIR: &str = "_lex"; +const GENERATED_ROOT_FILE: &str = "_lex_root.rs"; +const RECORD_COLLECTIONS_FILE: &str = "record_collections.rs"; + +fn main() -> Result<()> { + let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR")?); + let workspace_root = manifest_dir + .parent() + .and_then(Path::parent) + .context("resolve workspace root from manifest dir")? + .to_path_buf(); + + let lexicons_dir = std::env::var("TANGLED_LEXICONS_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| workspace_root.join(DEFAULT_LEXICONS_SUBDIR)); + + println!("cargo:rerun-if-env-changed=TANGLED_LEXICONS_DIR"); + println!("cargo:rerun-if-changed={}", lexicons_dir.display()); + println!("cargo:rerun-if-changed=build.rs"); + + let out_dir = PathBuf::from(std::env::var("OUT_DIR")?); + + let staged = out_dir.join(STAGED_SUBDIR); + if staged.exists() { + std::fs::remove_dir_all(&staged).context("clean staged lexicons")?; + } + stage_lexicons(&lexicons_dir, &staged)?; + + let corpus = LexiconCorpus::load_from_dir(&staged) + .map_err(|e| anyhow::anyhow!("load lexicon corpus: {e:?}"))?; + + let generated = out_dir.join(GENERATED_SUBDIR); + if generated.exists() { + std::fs::remove_dir_all(&generated).context("clean generated dir")?; + } + std::fs::create_dir_all(&generated).context("create generated dir")?; + + let codegen = CodeGenerator::with_mode(&corpus, "crate", CodegenMode::Pretty); + codegen + .write_to_disk(&generated) + .map_err(|e| anyhow::anyhow!("write generated code: {e:?}"))?; + write_generated_root(&generated, &out_dir.join(GENERATED_ROOT_FILE))?; + + write_record_collections(&corpus, &out_dir.join(RECORD_COLLECTIONS_FILE))?; + + Ok(()) +} + +fn write_record_collections(corpus: &LexiconCorpus, path: &Path) -> Result<()> { + let mut nsids: Vec<&str> = corpus + .iter() + .filter(|(_, doc)| matches!(doc.defs.get("main"), Some(LexUserType::Record(_)))) + .map(|(nsid, _)| nsid.as_str()) + .collect(); + nsids.sort_unstable(); + let entries: String = nsids + .iter() + .map(|nsid| format!(" {nsid:?},\n")) + .collect(); + let source = format!( + "/// every record collection in the lexicons, sorted so lookups can binary search\n\ + pub const RECORD_COLLECTIONS: &[&str] = &[\n{entries}];\n" + ); + std::fs::write(path, source).context("write record collections") +} + +/// absolute #[path] so the generated `mod x;` files resolve inside OUT_DIR, not next to lib.rs +fn write_generated_root(generated: &Path, path: &Path) -> Result<()> { + let source = format!( + "#[path = {:?}]\nmod generated;\npub use generated::*;\n", + generated.join("lib.rs") + ); + std::fs::write(path, source).context("write generated root") +} + +fn stage_lexicons(src: &Path, dst: &Path) -> Result<()> { + WalkDir::new(src) + .into_iter() + .filter_map(std::result::Result::ok) + .filter(|e| e.file_type().is_file()) + .filter(|e| { + e.path() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("json")) + }) + .try_for_each(|entry| -> Result<()> { + let rel = entry.path().strip_prefix(src).context("strip src prefix")?; + let target = dst.join(rel); + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).context("create staged parent")?; + } + std::fs::copy(entry.path(), &target).context("copy lexicon file")?; + Ok(()) + }) +} diff --git a/crates/lexicons/src/lib.rs b/crates/lexicons/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..1a20aad81c525998b740f4746f42acac8b08efe7 --- /dev/null +++ b/crates/lexicons/src/lib.rs @@ -0,0 +1,22 @@ +extern crate alloc; + +#[allow(non_snake_case, unused_imports)] +#[allow( + clippy::absurd_extreme_comparisons, + clippy::collapsible_if, + clippy::len_zero, + clippy::manual_strip, + clippy::needless_update, + clippy::new_ret_no_self, + clippy::new_without_default, + clippy::should_implement_trait, + clippy::type_complexity +)] +#[rustfmt::skip] +mod _lex { + include!(concat!(env!("OUT_DIR"), "/_lex_root.rs")); +} + +pub use _lex::*; + +include!(concat!(env!("OUT_DIR"), "/record_collections.rs")); diff --git a/crates/tangled-axum/Cargo.toml b/crates/tangled-axum/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..1acd971c56152161af21ea1c9a4273c484178b6f --- /dev/null +++ b/crates/tangled-axum/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "tangled-axum" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +axum = { workspace = true } +bobbin-runtime = { workspace = true } +jacquard-axum = { workspace = true } +jacquard-common = { workspace = true } +jacquard-identity = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/crates/tangled-axum/src/atproto.rs b/crates/tangled-axum/src/atproto.rs new file mode 100644 index 0000000000000000000000000000000000000000..c5d47649b40870f9fb4a880a0cee1b1fe1b9bbfa --- /dev/null +++ b/crates/tangled-axum/src/atproto.rs @@ -0,0 +1,179 @@ +use std::future::Future; + +use axum::extract::FromRequestParts; +use axum::http::header; +use axum::http::request::Parts; +use bobbin_runtime::UnixMicros; +use jacquard_axum::service_auth::{ + ReplayKey, ReplayStore, ServiceAuth, ServiceAuthConfig, ServiceAuthError, +}; +use jacquard_common::IntoStatic as _; +use jacquard_common::SmolStr; +use jacquard_common::service_auth; +use jacquard_common::types::crypto::KeyCodec; +use jacquard_common::types::string::{Did, DidService}; +use jacquard_identity::resolver::IdentityResolver; + +pub trait AtprotoService: Send + Sync { + type Resolver: IdentityResolver + Send + Sync; + + fn resolver(&self) -> &Self::Resolver; + + fn verify_claims( + &self, + claims: &service_auth::ServiceAuthClaims, + ) -> impl Future> + Send; +} + +/// Proof that claims passed a [`ClaimsPolicy`]. +#[derive(Debug)] +pub struct Verified(()); + +pub struct ClaimsPolicy<'a> { + audience: Option>, + allowed_services: &'a [SmolStr], + require_lxm: bool, + replay: Option<&'a dyn ReplayStore>, +} + +impl<'a> ClaimsPolicy<'a> { + pub fn skip_audience(mut self) -> Self { + self.audience = None; + self + } + + pub fn require_lxm(mut self, require: bool) -> Self { + self.require_lxm = require; + self + } + + pub async fn check( + &self, + claims: &service_auth::ServiceAuthClaims, + now: UnixMicros, + ) -> Result { + if let Some(expected) = self.audience.as_ref() { + if claims.aud.audience().as_str() != expected.as_str() { + return Err(service_auth::ServiceAuthError::AudienceMismatch { + expected: expected.clone().into_static(), + actual: DidService::new_owned(claims.aud.as_str()).unwrap(), + } + .into()); + } + + if !self.allowed_services.is_empty() + && let Some(service) = claims.aud.service() + && !self + .allowed_services + .iter() + .any(|allowed| allowed.as_str() == service) + { + return Err(service_auth::ServiceAuthError::ServiceIdMismatch { + allowed: self.allowed_services.to_vec(), + actual: Some(SmolStr::new(service)), + } + .into()); + } + } + + let now = i64::try_from(now.raw() / 1_000_000).unwrap_or(i64::MAX); + if claims.exp <= now { + return Err(service_auth::ServiceAuthError::Expired { + exp: claims.exp, + now, + } + .into()); + } + + if self.require_lxm && claims.lxm.is_none() { + return Err(ServiceAuthError::MethodBindingRequired); + } + + if let Some(store) = self.replay { + let jti = claims.jti.as_ref().ok_or(ServiceAuthError::MissingJti)?; + let key = ReplayKey::new( + claims.iss.clone().into_static(), + claims.aud.clone().into_static(), + jti.clone(), + ); + store.check_and_insert(key, claims.exp).await?; + } + + Ok(Verified(())) + } +} + +impl<'a, R: IdentityResolver> From<&'a ServiceAuthConfig> for ClaimsPolicy<'a> { + fn from(config: &'a ServiceAuthConfig) -> Self { + Self { + audience: Some(ServiceAuth::service_did(config)), + allowed_services: ServiceAuth::allowed_services(config), + require_lxm: ServiceAuth::require_lxm(config), + replay: ServiceAuth::replay_protection_enabled(config) + .then(|| ServiceAuth::replay_store(config)), + } + } +} + +pub async fn resolve_signing_key( + resolver: &R, + iss: &Did, +) -> Result +where + R: IdentityResolver + Sync, +{ + let doc = resolver.resolve_did_doc_owned(iss).await.map_err(|e| { + ServiceAuthError::DidResolutionFailed { + did: iss.clone().into_static(), + source: Box::new(e), + } + })?; + + let signing_key = doc + .atproto_public_key() + .map_err(|e| ServiceAuthError::InvalidKey(e.to_string()))? + .ok_or_else(|| ServiceAuthError::NoSigningKey(iss.clone().into_static())) + .map(|key| match key.codec { + KeyCodec::P256 => Ok(service_auth::PublicKey::from_p256_bytes(&key.bytes)), + KeyCodec::Secp256k1 => Ok(service_auth::PublicKey::from_k256_bytes(&key.bytes)), + codec => Err(ServiceAuthError::InvalidKey(format!( + "{codec:?} does not sign service auth tokens" + ))), + })???; + + Ok(signing_key) +} + +/// [`jacquard_axum::service_auth::ExtractServiceAuth`] with the claims policy delegated +/// to the state, plus the raw token. +pub struct ExtractAnyServiceAuth(pub service_auth::ServiceAuthClaims, pub String); + +impl FromRequestParts for ExtractAnyServiceAuth { + type Rejection = ServiceAuthError; + + async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + let token = bearer_token_from_parts(parts)?.ok_or(ServiceAuthError::MissingAuthHeader)?; + let parsed = service_auth::parse_jwt(token)?; + + let signing_key = resolve_signing_key(state.resolver(), &parsed.claims().iss).await?; + service_auth::verify_signature(&parsed, &signing_key)?; + + let Verified(()) = state.verify_claims(parsed.claims()).await?; + + Ok(Self(parsed.into_claims(), token.to_owned())) + } +} + +fn bearer_token_from_parts(parts: &Parts) -> Result, ServiceAuthError> { + let Some(auth_header) = parts.headers.get(header::AUTHORIZATION) else { + return Ok(None); + }; + + let auth_str = auth_header + .to_str() + .map_err(|_| ServiceAuthError::InvalidAuthHeader)?; + let token = auth_str + .strip_prefix("Bearer ") + .ok_or(ServiceAuthError::InvalidAuthHeader)?; + Ok(Some(token)) +} diff --git a/crates/tangled-axum/src/lib.rs b/crates/tangled-axum/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..224322fa01dd0a3f462193befa59d3fe1411643f --- /dev/null +++ b/crates/tangled-axum/src/lib.rs @@ -0,0 +1 @@ +pub mod atproto; diff --git a/crates/tranquil-gate/Cargo.toml b/crates/tranquil-gate/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..62bea575167905d4651aa2e04ec8d8046101d63b --- /dev/null +++ b/crates/tranquil-gate/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "tranquil-gate" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow.workspace = true +axum.workspace = true +base64.workspace = true +bobbin-runtime.workspace = true +clap.workspace = true +confique.workspace = true +bs58.workspace = true +jacquard-axum.workspace = true +jacquard-common.workspace = true +jacquard-identity.workspace = true +k256.workspace = true +reqwest = { workspace = true, features = ["query"] } +serde.workspace = true +serde_json.workspace = true +tangled-axum.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +[dev-dependencies] +tempfile.workspace = true +wiremock.workspace = true +tower.workspace = true diff --git a/crates/tranquil-gate/config.example.toml b/crates/tranquil-gate/config.example.toml new file mode 100644 index 0000000000000000000000000000000000000000..352b9e07a282dd0a96122d9ffc2c30ce949aebd3 --- /dev/null +++ b/crates/tranquil-gate/config.example.toml @@ -0,0 +1,13 @@ +bind = "0.0.0.0:3100" +did = "did:web:gate.example" +# Supply the base64-encoded secp256k1 private key using GATE_SIGNING_KEY. +tranquil_url = "https://tranquil.example" +tranquil_did = "did:web:tranquil.example" +deliberi_url = "https://deliberi.example" +deliberi_did = "did:web:deliberi.example" +plc_url = "https://plc.directory" +# extra_ca_file = "/etc/tranquil-gate/ca.pem" + +[log] +format = "text" # or "json" +filter = "info,tranquil_gate=debug" diff --git a/crates/tranquil-gate/src/config.rs b/crates/tranquil-gate/src/config.rs new file mode 100644 index 0000000000000000000000000000000000000000..bdcff4243d03a5e5a70a05c509c8e537e4547c20 --- /dev/null +++ b/crates/tranquil-gate/src/config.rs @@ -0,0 +1,111 @@ +use std::{ + net::SocketAddr, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, ensure}; +use base64::{Engine, engine::general_purpose::STANDARD}; +use confique::Config as _; +use jacquard_common::types::string::{Did, Handle}; +use k256::ecdsa::SigningKey; + +#[derive(confique::Config)] +pub struct Config { + #[config(env = "GATE_BIND", default = "0.0.0.0:3100")] + pub bind: SocketAddr, + #[config(env = "GATE_DID")] + pub did: String, + #[config(env = "GATE_SIGNING_KEY")] + pub signing_key: String, + #[config(env = "GATE_TRANQUIL_URL")] + pub tranquil_url: String, + #[config(env = "GATE_TRANQUIL_DID")] + pub tranquil_did: String, + #[config(env = "GATE_HANDLE_DOMAIN")] + pub handle_domain: String, + #[config(env = "GATE_HANDLE_PEERS", parse_env = confique::env::parse::list_by_comma, default = [])] + pub handle_peers: Vec, + #[config(env = "GATE_DELIBERI_URL")] + pub deliberi_url: String, + #[config(env = "GATE_DELIBERI_DID")] + pub deliberi_did: String, + #[config(env = "GATE_PLC_URL")] + pub plc_url: String, + #[config(env = "GATE_EXTRA_CA_FILE")] + pub extra_ca_file: Option, + #[config(env = "GATE_REQUIRE_VERIFIED_EMAIL", default = true)] + pub require_verified_email: bool, + #[config(nested)] + pub log: LogConfig, +} + +#[derive(confique::Config)] +pub struct LogConfig { + #[config(env = "GATE_LOG_FORMAT", default = "text")] + pub format: String, + #[config(env = "RUST_LOG", default = "info")] + pub filter: String, +} + +impl Config { + pub fn load(path: Option<&Path>) -> anyhow::Result { + let mut builder = Self::builder().env(); + if let Some(path) = path { + builder = builder.preloaded(confique::File::new(path)?.required().load()?); + } + let config = builder + .file("/etc/tranquil-gate/config.toml") + .load() + .context("load gate configuration")?; + config.validate()?; + Ok(config) + } + + pub fn validate(&self) -> anyhow::Result<()> { + for (name, value) in [ + ("did", &self.did), + ("tranquil_did", &self.tranquil_did), + ("deliberi_did", &self.deliberi_did), + ] { + Did::new(value.as_str()).with_context(|| format!("invalid {name}"))?; + } + Handle::new(format!("label.{}", self.handle_domain).as_str()) + .context("handle_domain must be a domain that handles can be made under")?; + let peers = self.handle_peers.iter().map(|peer| ("handle_peers", peer)); + for (name, value) in [ + ("tranquil_url", &self.tranquil_url), + ("deliberi_url", &self.deliberi_url), + ("plc_url", &self.plc_url), + ] + .into_iter() + .chain(peers) + { + let url = reqwest::Url::parse(value).with_context(|| format!("invalid {name}"))?; + ensure!( + matches!(url.scheme(), "http" | "https") + && url.host_str().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none(), + "{name} must be an HTTP(S) base URL without credentials, query, or fragment" + ); + } + self.signing_key()?; + ensure!( + matches!(self.log.format.as_str(), "text" | "json"), + "log.format must be text or json" + ); + tracing_subscriber::EnvFilter::try_new(format!("info,{}", self.log.filter)) + .context("invalid log.filter")?; + Ok(()) + } + + pub(crate) fn signing_key(&self) -> anyhow::Result { + let bytes = STANDARD + .decode(&self.signing_key) + .context("signing_key must be base64")?; + SigningKey::from_slice(&bytes) + .context("signing_key must be a 32-byte secp256k1 private key") + } +} diff --git a/crates/tranquil-gate/src/gate.rs b/crates/tranquil-gate/src/gate.rs new file mode 100644 index 0000000000000000000000000000000000000000..6e921184ebdd77e3c59a33b80a4a93478866300c --- /dev/null +++ b/crates/tranquil-gate/src/gate.rs @@ -0,0 +1,144 @@ +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use k256::ecdsa::{Signature, SigningKey, signature::Signer}; +use serde::Deserialize; +use serde_json::json; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::{Error, now, relay}; +use axum::http::StatusCode; + +#[derive(Deserialize)] +struct Committers { + committers: Vec, +} + +/// Registration policy gating delegated account creation on Tranquil. +/// +/// Build with [`Gate::builder`], then call [`Gate::check`] for each request. +pub struct Gate { + http: reqwest::Client, + gate_did: String, + signing_key: SigningKey, + deliberi_url: String, + deliberi_did: String, + require_verified_email: bool, +} + +pub struct GateBuilder { + http: reqwest::Client, + gate_did: String, + signing_key: SigningKey, + deliberi_url: String, + deliberi_did: String, + require_verified_email: bool, +} + +impl Gate { + pub fn builder( + http: reqwest::Client, + gate_did: impl Into, + signing_key: SigningKey, + deliberi_url: impl Into, + deliberi_did: impl Into, + ) -> GateBuilder { + GateBuilder { + http, + gate_did: gate_did.into(), + signing_key, + deliberi_url: deliberi_url.into(), + deliberi_did: deliberi_did.into(), + require_verified_email: true, + } + } + + /// Tranquil enforces the per-controller delegate cap itself, keyed on the + /// authenticated controller, so the gate only checks what Tranquil cannot. + pub async fn check(&self, did: &str) -> Result<(), Error> { + if self.require_verified_email { + self.check_verified_email(did).await?; + } + Ok(()) + } + + async fn check_verified_email(&self, did: &str) -> Result<(), Error> { + let response = self + .http + .post(format!( + "{}/xrpc/sh.tangled.identity.resolveCommitters", + self.deliberi_url + )) + .bearer_auth(self.resolver_token()) + .json(&json!({"actor": did})) + .send() + .await?; + let verified: Committers = relay(response).await?.json().await?; + if !verified.committers.iter().any(|value| is_email(value)) { + return Err(Error::code(StatusCode::FORBIDDEN, "VerifiedEmailRequired")); + } + Ok(()) + } + + fn resolver_token(&self) -> String { + let timestamp = now(); + let claims = json!({"iss": self.gate_did, "aud": self.deliberi_did, "iat": timestamp, + "exp": timestamp + 60, "lxm": "sh.tangled.identity.resolveCommitters", + "jti": format!("{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos())}); + let input = format!( + "{}.{}", + URL_SAFE_NO_PAD.encode(br#"{"alg":"ES256K","typ":"JWT"}"#), + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()) + ); + let signature: Signature = self.signing_key.sign(input.as_bytes()); + format!("{input}.{}", URL_SAFE_NO_PAD.encode(signature.to_bytes())) + } +} + +impl GateBuilder { + pub fn require_verified_email(mut self, require: bool) -> Self { + self.require_verified_email = require; + self + } + + pub fn build(self) -> Gate { + Gate { + http: self.http, + gate_did: self.gate_did, + signing_key: self.signing_key, + deliberi_url: self.deliberi_url, + deliberi_did: self.deliberi_did, + require_verified_email: self.require_verified_email, + } + } +} + +fn is_email(value: &str) -> bool { + let value = value.trim(); + !value.starts_with("did:") + && value.len() <= 320 + && !value.chars().any(char::is_whitespace) + && !value.chars().any(char::is_control) + && value.split_once('@').is_some_and(|(local, domain)| { + !local.is_empty() && !domain.is_empty() && !domain.contains('@') + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn email_filter() { + for value in [ + "did:plc:abcdefghijklmnopqrstuvwx", + "", + "@host", + "user@", + "a@@b", + "a\0@b", + "a b@host", + ] { + assert!(!is_email(value)); + } + assert!(is_email("person@example.com")); + } +} diff --git a/crates/tranquil-gate/src/lib.rs b/crates/tranquil-gate/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..06164996bdb36b4b3c61658c4d1c3eb3d5874032 --- /dev/null +++ b/crates/tranquil-gate/src/lib.rs @@ -0,0 +1,337 @@ +pub mod config; +mod gate; + +use axum::{ + Json, Router, + extract::{DefaultBodyLimit, State}, + http::StatusCode, + response::{IntoResponse, Response}, + routing::{get, post}, +}; +use bobbin_runtime::{ReqwestHttp, UnixMicros}; +use gate::Gate; +use jacquard_axum::service_auth::{ServiceAuthConfig, ServiceAuthError}; +use jacquard_common::{deps::fluent_uri::Uri, service_auth::ServiceAuthClaims, types::string::Did}; +use jacquard_identity::{JacquardResolver, resolver::PlcSource}; +use k256::ecdsa::SigningKey; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::{ + borrow::Cow, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tangled_axum::atproto::{AtprotoService, ClaimsPolicy, ExtractAnyServiceAuth, Verified}; + +const OWNER: &str = "atproto repo:* blob:*/* rpc:* identity:* account:*?action=manage transition:generic transition:chat.bsky transition:email"; +type Directory = JacquardResolver; + +struct RuntimeConfig { + did: String, + tranquil_url: String, + signing_key: SigningKey, + handle_domain: String, + handle_hosts: Vec, + reserved_handles: Vec, +} + +#[derive(Clone)] +pub struct App { + config: Arc, + http: reqwest::Client, + auth: ServiceAuthConfig>, + gate: Arc, +} + +impl App { + /// Validate configuration and prepare shared clients, auth policy, and quota lock. + pub fn new(config: &config::Config) -> anyhow::Result { + config.validate()?; + let mut http = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(30)) + .redirect(reqwest::redirect::Policy::none()); + if let Some(path) = &config.extra_ca_file { + http = + http.add_root_certificate(reqwest::Certificate::from_pem(&std::fs::read(path)?)?); + } + let http = http.build()?; + let plc = format!("{}/", config.plc_url.trim_end_matches('/')); + let directory = Arc::new( + JacquardResolver::new(ReqwestHttp::new(http.clone()), Default::default()) + .with_plc_source(PlcSource::PlcDirectory { + base: Uri::parse(plc.as_str())?.to_owned(), + }), + ); + let signing_key = config.signing_key()?; + let gate = Gate::builder( + http.clone(), + config.did.clone(), + signing_key.clone(), + config.deliberi_url.trim_end_matches('/'), + config.deliberi_did.clone(), + ) + .require_verified_email(config.require_verified_email) + .build(); + Ok(Self { + config: Arc::new(RuntimeConfig { + did: config.did.clone(), + tranquil_url: config.tranquil_url.trim_end_matches('/').into(), + signing_key, + handle_domain: config.handle_domain.to_ascii_lowercase(), + handle_hosts: std::iter::once(&config.tranquil_url) + .chain(&config.handle_peers) + .map(|url| url.trim_end_matches('/').to_owned()) + .collect(), + reserved_handles: [&config.did, &config.tranquil_did] + .into_iter() + .filter_map(|did| did.strip_prefix("did:web:")) + .map(str::to_ascii_lowercase) + .collect(), + }), + http, + auth: ServiceAuthConfig::new(Did::new_owned(config.tranquil_did.clone())?, directory), + gate: Arc::new(gate), + }) + } +} + +impl AtprotoService for App { + type Resolver = Directory; + fn resolver(&self) -> &Directory { + self.auth.resolver() + } + async fn verify_claims( + &self, + claims: &ServiceAuthClaims, + ) -> Result { + if claims.lxm.as_ref().map(|v| v.as_str()) != Some("farm.tranquil.delegation.createAccount") + { + return Err(ServiceAuthError::MethodBindingRequired); + } + ClaimsPolicy::from(&self.auth) + .require_lxm(true) + .check(claims, UnixMicros::new(now() * 1_000_000)) + .await + } +} + +pub(crate) fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock") + .as_secs() +} + +/// An xrpc error payload: `{"error": ..., "message": ...}`. +pub(crate) struct Error { + pub(crate) status: StatusCode, + pub(crate) error: Cow<'static, str>, + pub(crate) message: Option, +} +impl Error { + pub(crate) fn code(status: StatusCode, error: &'static str) -> Self { + Self { + status, + error: Cow::Borrowed(error), + message: None, + } + } +} +impl IntoResponse for Error { + fn into_response(self) -> Response { + let mut body = json!({"error": self.error}); + if let Some(message) = self.message { + body["message"] = Value::String(message); + } + (self.status, Json(body)).into_response() + } +} +pub(crate) fn upstream(e: impl std::fmt::Display) -> Error { + tracing::warn!(error = %e, "upstream request failed"); + Error::code(StatusCode::BAD_GATEWAY, "UpstreamUnavailable") +} + +impl From for Error { + fn from(error: reqwest::Error) -> Self { + upstream(error) + } +} + +#[derive(Default, Deserialize)] +struct UpstreamError { + error: Option, + message: Option, +} + +/// Pass a 2xx through, relay a 4xx with the upstream's own error code so the +/// caller can tell a taken handle from a delegate cap, and fail closed on 5xx. +pub(crate) async fn relay(response: reqwest::Response) -> Result { + let status = response.status(); + if status.is_success() { + return Ok(response); + } + if status.is_client_error() { + let payload: UpstreamError = match response.json().await { + Ok(payload) => payload, + Err(e) => { + tracing::warn!(%status, error = %e, "upstream error body unreadable"); + UpstreamError::default() + } + }; + return Err(Error { + status, + error: payload + .error + .map_or(Cow::Borrowed("UpstreamRejected"), Cow::Owned), + message: payload.message, + }); + } + Err(upstream(format!("upstream responded {status}"))) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct CreateInput { + handle: String, +} +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +struct Account { + did: String, + handle: String, + #[serde(default)] + controller_did: String, +} + +// tranquil gives a delegated account whatever handle it's asked for without +// resolving it, so the gate pins the domain +fn qualify(handle: &str, domain: &str) -> Option { + let handle = handle.to_ascii_lowercase(); + let label = handle.strip_suffix(domain)?.strip_suffix('.')?; + let valid = !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); + valid.then_some(handle) +} + +// worded like tranquil's own refusal, since the web shows it as is +fn handle_not_available() -> Error { + Error { + message: Some("Handle already taken".into()), + ..Error::code(StatusCode::BAD_REQUEST, "HandleNotAvailable") + } +} + +impl App { + // pdses only answer describeRepo for their own accounts, so anything but + // RepoNotFound counts as taken, and an unreachable host fails closed + async fn ensure_available(&self, handle: &str) -> Result<(), Error> { + for host in &self.config.handle_hosts { + let response = self + .http + .get(format!("{host}/xrpc/com.atproto.repo.describeRepo")) + .query(&[("repo", handle)]) + .send() + .await?; + match response.status() { + status if status.is_success() => return Err(handle_not_available()), + StatusCode::BAD_REQUEST => { + let payload: UpstreamError = response.json().await.unwrap_or_default(); + if payload.error.as_deref() != Some("RepoNotFound") { + return Err(handle_not_available()); + } + } + status => return Err(upstream(format!("{host} responded {status}"))), + } + } + Ok(()) + } + + async fn create(&self, actor: &str, token: &str, input: CreateInput) -> Result { + let cfg = &self.config; + + let handle = qualify(&input.handle, &cfg.handle_domain) + .ok_or_else(|| Error::code(StatusCode::BAD_REQUEST, "InvalidHandle"))?; + if cfg.reserved_handles.contains(&handle) { + return Err(handle_not_available()); + } + self.gate.check(actor).await?; + self.ensure_available(&handle).await?; + + let response = self + .http + .post(format!( + "{}/xrpc/_delegation.createDelegatedAccount", + cfg.tranquil_url + )) + .bearer_auth(token) + .json(&json!({"handle": handle, "controllerScopes": OWNER})) + .send() + .await?; + let mut account: Account = relay(response).await?.json().await?; + account.controller_did = actor.to_owned(); + tracing::info!(controller = actor, did = %account.did, "created delegated account"); + Ok(account) + } +} + +async fn create( + State(app): State, + ExtractAnyServiceAuth(claims, token): ExtractAnyServiceAuth, + Json(input): Json, +) -> Result, Error> { + if input.handle.is_empty() || input.handle.len() > 253 || input.handle.trim() != input.handle { + return Err(Error::code(StatusCode::BAD_REQUEST, "InvalidHandle")); + } + tokio::spawn(async move { app.create(claims.iss.as_str(), &token, input).await }) + .await + .map_err(upstream)? + .map(Json) +} + +async fn did_document(State(app): State) -> Json { + let did = &app.config.did; + let mut key = vec![0xe7, 0x01]; // secp256k1-pub multicodec + key.extend_from_slice( + app.config + .signing_key + .verifying_key() + .to_encoded_point(true) + .as_bytes(), + ); + Json(json!({ + "@context": ["https://www.w3.org/ns/did/v1"], + "id": did, + "verificationMethod": [ + { + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": format!("z{}", bs58::encode(key).into_string()) + } + ] + })) +} + +async fn health() -> Json { + Json(json!({"status": "ok"})) +} + +pub fn router(app: App) -> Router { + Router::new() + .route("/xrpc/_health", get(health)) + .route("/.well-known/did.json", get(did_document)) + .route("/xrpc/org.tangled.delegation.createAccount", post(create)) + // old web builds still post here + .route("/xrpc/sh.tangled.delegation.createAccount", post(create)) + .layer(DefaultBodyLimit::max(4096)) + .with_state(app) +} + +#[cfg(test)] +mod tests; diff --git a/crates/tranquil-gate/src/main.rs b/crates/tranquil-gate/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..c9fab79a529ca700c0d76b0ff123f42e2cb01367 --- /dev/null +++ b/crates/tranquil-gate/src/main.rs @@ -0,0 +1,50 @@ +use std::path::PathBuf; + +use clap::Parser; +use tranquil_gate::{App, config::Config, router}; + +mod telemetry; + +#[derive(Parser)] +#[command(about = "Provision Tranquil delegated accounts for verified Tangled users")] +struct Cli { + #[arg(short, long, env = "GATE_CONFIG", value_name = "FILE")] + config: Option, + + #[arg(long)] + check_config: bool, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + let config = Config::load(cli.config.as_deref())?; + telemetry::init(&config.log)?; + let app = App::new(&config)?; + if cli.check_config { + println!("configuration is valid"); + return Ok(()); + } + let listener = tokio::net::TcpListener::bind(config.bind).await?; + tracing::info!(address = %listener.local_addr()?, "tranquil-gate listening"); + axum::serve(listener, router(app)) + .with_graceful_shutdown(shutdown_signal()) + .await?; + Ok(()) +} + +async fn shutdown_signal() { + #[cfg(unix)] + let terminate = async { + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("install SIGTERM handler") + .recv() + .await; + }; + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + tokio::select! { + _ = tokio::signal::ctrl_c() => {}, + _ = terminate => {}, + } +} diff --git a/crates/tranquil-gate/src/telemetry.rs b/crates/tranquil-gate/src/telemetry.rs new file mode 100644 index 0000000000000000000000000000000000000000..f8cf4352ea3207f5fc2f770aba22a3fcacf0b00e --- /dev/null +++ b/crates/tranquil-gate/src/telemetry.rs @@ -0,0 +1,26 @@ +use std::sync::Arc; + +use bobbin_runtime::{Clock, SystemClock}; +use tracing_subscriber::{EnvFilter, filter::LevelFilter}; +use tranquil_gate::config::LogConfig; + +struct ClockTimer(Arc); + +impl tracing_subscriber::fmt::time::FormatTime for ClockTimer { + fn format_time(&self, w: &mut tracing_subscriber::fmt::format::Writer<'_>) -> std::fmt::Result { + write!(w, "{}", self.0.now_unix_micros().raw()) + } +} + +pub fn init(config: &LogConfig) -> anyhow::Result<()> { + let filter = EnvFilter::try_new(format!("{},{}", LevelFilter::INFO, config.filter))?; + let subscriber = tracing_subscriber::fmt() + .with_env_filter(filter) + .with_timer(ClockTimer(Arc::new(SystemClock::new()))); + let result = match config.format.as_str() { + "text" => subscriber.try_init(), + "json" => subscriber.json().try_init(), + _ => anyhow::bail!("log.format must be text or json"), + }; + result.map_err(|error| anyhow::anyhow!("initialize tracing: {error}")) +} diff --git a/crates/tranquil-gate/src/tests.rs b/crates/tranquil-gate/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..c68f3c1b5c531617f54b64452d391fd5b5e489fb --- /dev/null +++ b/crates/tranquil-gate/src/tests.rs @@ -0,0 +1,381 @@ +use super::*; +use axum::{body::Body, http::Request}; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use k256::ecdsa::{Signature, signature::Signer}; +use tower::ServiceExt; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_json, header, method, path, query_param}, +}; + +const ACTOR: &str = "did:plc:abcdefghijklmnopqrstuvwx"; +const GATE: &str = "did:web:gate.example"; +const CREATE: &str = "farm.tranquil.delegation.createAccount"; +const RESOLVE: &str = "sh.tangled.identity.resolveCommitters"; + +async fn setup() -> (App, MockServer) { + setup_with_peers(&[]).await +} + +async fn setup_with_peers(peers: &[&MockServer]) -> (App, MockServer) { + let server = MockServer::start().await; + let http = reqwest::Client::new(); + let directory = Arc::new( + JacquardResolver::new(ReqwestHttp::new(http.clone()), Default::default()).with_plc_source( + PlcSource::PlcDirectory { + base: Uri::parse(format!("{}/", server.uri()).as_str()) + .unwrap() + .to_owned(), + }, + ), + ); + let signing_key = SigningKey::from_slice(&[1; 32]).unwrap(); + let gate = gate::Gate::builder( + http.clone(), + GATE, + signing_key.clone(), + server.uri(), + "did:web:deliberi.example", + ) + .build(); + let app = App { + config: Arc::new(RuntimeConfig { + did: GATE.into(), + tranquil_url: server.uri(), + signing_key, + handle_domain: "example".into(), + handle_hosts: std::iter::once(server.uri()) + .chain(peers.iter().map(|peer| peer.uri())) + .collect(), + reserved_handles: vec!["gate.example".into()], + }), + http, + auth: ServiceAuthConfig::new(Did::new_owned(GATE).unwrap(), directory), + gate: Arc::new(gate), + }; + let mut key = vec![0xe7, 1]; + key.extend_from_slice( + app.config + .signing_key + .verifying_key() + .to_encoded_point(true) + .as_bytes(), + ); + Mock::given(method("GET")).and(path(format!("/{ACTOR}"))).respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": ACTOR, "verificationMethod": [{"id": format!("{ACTOR}#atproto"), "type":"Multikey", "controller": ACTOR, + "publicKeyMultibase": format!("z{}", bs58::encode(key).into_string())}] + }))).mount(&server).await; + (app, server) +} + +fn token(app: &App, aud: &str, lxm: &str, exp: u64, key: Option<&SigningKey>) -> String { + let payload = json!({"iss": ACTOR, "aud": aud, "lxm": lxm, "iat": now(), "exp": exp, "jti": format!("test-{lxm}")}); + let input = format!( + "{}.{}", + URL_SAFE_NO_PAD.encode(br#"{"alg":"ES256K","typ":"JWT"}"#), + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&payload).unwrap()) + ); + let signature: Signature = key + .unwrap_or(&app.config.signing_key) + .sign(input.as_bytes()); + format!("{input}.{}", URL_SAFE_NO_PAD.encode(signature.to_bytes())) +} + +const ROUTE: &str = "/xrpc/org.tangled.delegation.createAccount"; +const LEGACY_ROUTE: &str = "/xrpc/sh.tangled.delegation.createAccount"; + +async fn request(app: App, token: Option<&str>) -> Response { + request_at(ROUTE, app, token).await +} + +async fn request_at(route: &str, app: App, token: Option<&str>) -> Response { + let mut request = Request::post(route).header("content-type", "application/json"); + if let Some(token) = token { + request = request.header("authorization", format!("Bearer {token}")); + } + router(app) + .oneshot( + request + .body(Body::from(json!({"handle":"org.example"}).to_string())) + .unwrap(), + ) + .await + .unwrap() +} + +async fn describe_repo(server: &MockServer, status: u16, body: Value) { + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.describeRepo")) + .and(query_param("repo", "org.example")) + .respond_with(ResponseTemplate::new(status).set_body_json(body)) + .mount(server) + .await; +} + +async fn handle_free(server: &MockServer) { + describe_repo(server, 400, json!({"error": "RepoNotFound"})).await; +} + +async fn email(server: &MockServer, values: Vec<&str>) { + Mock::given(method("POST")) + .and(path(format!("/xrpc/{RESOLVE}"))) + .and(body_json(json!({"actor": ACTOR}))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"committers": values}))) + .mount(server) + .await; +} + +#[tokio::test] +async fn rejects_missing_wrong_audience_expired_wrong_method_and_bad_signature() { + let (app, server) = setup().await; + let bad_key = SigningKey::from_slice(&[2; 32]).unwrap(); + let tokens = [ + None, + Some(token( + &app, + "did:web:other.example", + CREATE, + now() + 60, + None, + )), + Some(token(&app, GATE, CREATE, now() - 1, None)), + Some(token(&app, GATE, RESOLVE, now() + 60, None)), + Some(token(&app, GATE, CREATE, now() + 60, Some(&bad_key))), + ]; + for token in tokens { + assert!( + request(app.clone(), token.as_deref()) + .await + .status() + .is_client_error() + ); + } + assert!( + server + .received_requests() + .await + .unwrap() + .iter() + .all(|r| r.method == "GET") + ); +} + +#[tokio::test] +async fn requires_an_email_not_the_did_fallback_and_rejects_replay() { + let (app, server) = setup().await; + email(&server, vec![ACTOR]).await; + let jwt = token(&app, GATE, CREATE, now() + 60, None); + let response = request(app.clone(), Some(&jwt)).await; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let body = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + assert!( + std::str::from_utf8(&body) + .unwrap() + .contains("VerifiedEmailRequired") + ); + assert!(request(app, Some(&jwt)).await.status().is_client_error()); + assert_eq!( + server + .received_requests() + .await + .unwrap() + .iter() + .filter(|r| r.method == "POST") + .count(), + 1 + ); +} + +#[tokio::test] +async fn upstream_client_errors_pass_through_and_server_errors_fail_closed() { + let (app, server) = setup().await; + let input = || CreateInput { + handle: "org.example".into(), + }; + // No email mock yet, so the resolver 404s and the gate relays that status. + let error = app + .create(ACTOR, "create-token", input()) + .await + .err() + .unwrap(); + assert_eq!(error.status, StatusCode::NOT_FOUND); + assert_eq!(error.error, "UpstreamRejected"); + email(&server, vec!["person@example.com"]).await; + handle_free(&server).await; + // Tranquil rejecting the creation is relayed with Tranquil's own status, + // error code, and message, so a taken handle is distinguishable from the + // per-controller cap. + Mock::given(path("/xrpc/_delegation.createDelegatedAccount")) + .respond_with(ResponseTemplate::new(400).set_body_json( + json!({"error": "InvalidDelegation", "message": "delegate cap reached"}), + )) + .up_to_n_times(1) + .mount(&server) + .await; + let error = app + .create(ACTOR, "create-token", input()) + .await + .err() + .unwrap(); + assert_eq!(error.status, StatusCode::BAD_REQUEST); + assert_eq!(error.error, "InvalidDelegation"); + assert_eq!(error.message.as_deref(), Some("delegate cap reached")); + Mock::given(path("/xrpc/_delegation.createDelegatedAccount")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + let error = app + .create(ACTOR, "create-token", input()) + .await + .err() + .unwrap(); + assert_eq!(error.status, StatusCode::BAD_GATEWAY); + assert_eq!(error.error, "UpstreamUnavailable"); +} + +#[tokio::test] +async fn rejects_missing_nonce() { + let (app, _) = setup().await; + let claims: ServiceAuthClaims = serde_json::from_value(json!({ + "iss": ACTOR, "aud": GATE, "iat": now(), "exp": now() + 60, "lxm": CREATE, "jti": null + })) + .unwrap(); + assert!(app.verify_claims(&claims).await.is_err()); +} + +#[tokio::test] +async fn the_legacy_route_reaches_the_handler() { + let (app, _) = setup().await; + let response = request_at(LEGACY_ROUTE, app, None).await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn forwards_the_create_token_without_controller_overrides() { + let (app, server) = setup().await; + let create = token(&app, GATE, CREATE, now() + 60, None); + email(&server, vec!["person@example.com"]).await; + handle_free(&server).await; + Mock::given(method("POST")) + .and(path("/xrpc/_delegation.createDelegatedAccount")) + .and(header("authorization", format!("Bearer {create}"))) + .and(body_json( + json!({"handle":"org.example", "controllerScopes":OWNER}), + )) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({"did":"did:plc:org", "handle":"org.example"})), + ) + .expect(1) + .mount(&server) + .await; + let response = request(app, Some(&create)).await; + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + let account: Account = serde_json::from_slice(&body).unwrap(); + assert_eq!(account.controller_did, ACTOR); + // The gate never enumerates delegates or opens a session of its own. + assert!(server.received_requests().await.unwrap().iter().all(|r| { + (r.url.query().is_none() || r.url.path().ends_with("describeRepo")) + && !r.url.path().contains("createSession") + && !r.url.path().contains("listControlledAccounts") + })); +} + +#[test] +fn qualifies_only_single_labels_under_the_handle_domain() { + assert_eq!( + qualify("Acme.Example", "example").as_deref(), + Some("acme.example") + ); + assert_eq!( + qualify("a-1.example", "example").as_deref(), + Some("a-1.example") + ); + for handle in [ + "example", + ".example", + "acme.other", + "acme.notexample", + "a.b.example", + "-acme.example", + "acme-.example", + "ac_me.example", + "acme.example.evil", + ] { + assert_eq!(qualify(handle, "example"), None, "{handle}"); + } +} + +#[tokio::test] +async fn refuses_handles_outside_the_domain_or_reserved_before_checking_email() { + let (app, server) = setup().await; + for (handle, code) in [ + ("org.elsewhere", "InvalidHandle"), + ("gate.example", "HandleNotAvailable"), + ] { + let error = app + .create( + ACTOR, + "create-token", + CreateInput { + handle: handle.into(), + }, + ) + .await + .err() + .unwrap(); + assert_eq!(error.error, code, "{handle}"); + } + assert!(server.received_requests().await.unwrap().is_empty()); +} + +#[tokio::test] +async fn refuses_a_handle_any_pds_in_the_domain_already_holds() { + let input = || CreateInput { + handle: "org.example".into(), + }; + for (status, body) in [ + ( + 200, + json!({"did": "did:plc:someone", "handle": "org.example"}), + ), + (400, json!({"error": "RepoDeactivated"})), + ] { + let peer = MockServer::start().await; + describe_repo(&peer, status, body).await; + let (app, server) = setup_with_peers(&[&peer]).await; + email(&server, vec!["person@example.com"]).await; + handle_free(&server).await; + let error = app + .create(ACTOR, "create-token", input()) + .await + .err() + .unwrap(); + assert_eq!(error.error, "HandleNotAvailable"); + assert!( + server + .received_requests() + .await + .unwrap() + .iter() + .all(|r| !r.url.path().contains("createDelegatedAccount")) + ); + } + + let peer = MockServer::start().await; + describe_repo(&peer, 503, json!({})).await; + let (app, server) = setup_with_peers(&[&peer]).await; + email(&server, vec!["person@example.com"]).await; + handle_free(&server).await; + let error = app + .create(ACTOR, "create-token", input()) + .await + .err() + .unwrap(); + assert_eq!(error.error, "UpstreamUnavailable"); +} diff --git a/crates/tranquil-gate/tests/config.rs b/crates/tranquil-gate/tests/config.rs new file mode 100644 index 0000000000000000000000000000000000000000..e7f9aefa8cd275a2e90ea852ffab46e9baf3c5c2 --- /dev/null +++ b/crates/tranquil-gate/tests/config.rs @@ -0,0 +1,136 @@ +use std::process::{Command, Output}; + +const CONFIG: &str = r#" +did = "did:web:gate.example" +signing_key = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=" +tranquil_url = "https://tranquil.example" +tranquil_did = "did:web:tranquil.example" +handle_domain = "orgs.test" +deliberi_url = "https://deliberi.example" +deliberi_did = "did:web:deliberi.example" +plc_url = "https://plc.directory" +[log] +format = "json" +filter = "tranquil_gate=debug" +"#; + +fn check(config: &str, env: &[(&str, &str)]) -> Output { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + std::fs::write(&path, config).unwrap(); + Command::new(env!("CARGO_BIN_EXE_tranquil-gate")) + .env_clear() + .args(["--check-config", "--config"]) + .arg(path) + .envs(env.iter().copied()) + .output() + .unwrap() +} + +#[test] +fn toml_and_environment_overrides() { + let output = check(CONFIG, &[]); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + // An invalid file value must be replaced before validation. + assert!( + check( + &CONFIG.replace("https://tranquil.example", "invalid"), + &[("GATE_TRANQUIL_URL", "http://127.0.0.1:3000")] + ) + .status + .success() + ); + assert!( + check( + &CONFIG.replace("format = \"json\"", "format = \"invalid\""), + &[("GATE_LOG_FORMAT", "text")] + ) + .status + .success() + ); + assert!( + check( + &CONFIG.replace("tranquil_gate=debug", "[invalid"), + &[("RUST_LOG", "info")] + ) + .status + .success() + ); +} + +#[test] +fn invalid_configuration_fails_before_listening() { + for (name, value, message) in [ + ("GATE_BIND", "invalid", "bind"), + ("GATE_DID", "invalid", "did"), + ( + "GATE_TRANQUIL_URL", + "https://example.com?query", + "tranquil_url", + ), + ("GATE_SIGNING_KEY", "AAAA", "signing_key"), + ("GATE_HANDLE_DOMAIN", "not a domain", "handle_domain"), + ( + "GATE_HANDLE_PEERS", + "https://pds.test,https://example.com?query", + "handle_peers", + ), + ("GATE_LOG_FORMAT", "invalid", "log.format"), + ("RUST_LOG", "[invalid", "log.filter"), + ( + "GATE_EXTRA_CA_FILE", + "/nonexistent/gate-ca.pem", + "No such file", + ), + ] { + let output = check(CONFIG, &[(name, value)]); + assert!(!output.status.success(), "{name} accepted"); + let error = String::from_utf8_lossy(&output.stderr); + assert!(error.contains(message), "{name}: {error}"); + } +} + +#[test] +fn explicit_missing_file_is_an_error() { + let output = Command::new(env!("CARGO_BIN_EXE_tranquil-gate")) + .env_clear() + .env("GATE_CONFIG", "/nonexistent/gate-config.toml") + .arg("--check-config") + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("gate-config.toml")); +} + +#[test] +fn existing_environment_only_configuration_works() { + let output = check( + "", + &[ + ("GATE_DID", "did:web:gate.example"), + ( + "GATE_SIGNING_KEY", + "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=", + ), + ("GATE_TRANQUIL_URL", "https://tranquil.example"), + ("GATE_TRANQUIL_DID", "did:web:tranquil.example"), + ("GATE_HANDLE_DOMAIN", "orgs.test"), + ( + "GATE_HANDLE_PEERS", + "https://pds.test,http://127.0.0.1:3001", + ), + ("GATE_DELIBERI_URL", "https://deliberi.example"), + ("GATE_DELIBERI_DID", "did:web:deliberi.example"), + ("GATE_PLC_URL", "https://plc.directory"), + ], + ); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/crates/trusted-proxies/Cargo.toml b/crates/trusted-proxies/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..9a510a2a07e9b2c7a3451c469c99a0a4f3ea7aa5 --- /dev/null +++ b/crates/trusted-proxies/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "trusted-proxies" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +ipnet = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/trusted-proxies/src/lib.rs b/crates/trusted-proxies/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..1b305c4f6a46c491c29a4c00f6988c7cf4b88f9f --- /dev/null +++ b/crates/trusted-proxies/src/lib.rs @@ -0,0 +1,308 @@ +use std::collections::BTreeSet; +use std::convert::Infallible; +use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::str::FromStr; + +use ipnet::IpNet; + +pub const MAX_HOPS: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct ProxyNet(IpNet); + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ProxyNetError { + #[error("`{0}` isn't an IP address or a CIDR block")] + Unreadable(String), + #[error("a blank entry isn't an IP address or a CIDR block, so drop it or fill it in")] + Blank, +} + +impl ProxyNet { + fn contains(self, peer: IpAddr) -> bool { + match peer.to_canonical() { + IpAddr::V4(v4) => { + self.0.contains(&IpAddr::V4(v4)) + || self.0.contains(&IpAddr::V6(v4.to_ipv6_mapped())) + } + v6 => self.0.contains(&v6), + } + } +} + +impl FromStr for ProxyNet { + type Err = ProxyNetError; + + fn from_str(raw: &str) -> Result { + match raw.trim() { + "" => Err(ProxyNetError::Blank), + entry => entry + .parse::() + .ok() + .or_else(|| entry.parse::().ok().map(IpNet::from)) + .map(|net| Self(canonical(net))) + .ok_or_else(|| ProxyNetError::Unreadable(entry.to_owned())), + } + } +} + +fn canonical(net: IpNet) -> IpNet { + match net { + IpNet::V6(v6) => match (v6.network().to_canonical(), v6.prefix_len()) { + (v4 @ IpAddr::V4(_), len @ 96..) => IpNet::new_assert(v4, len - 96), + _ => net, + }, + IpNet::V4(_) => net, + } +} + +fn chain_address(entry: &str) -> Option { + entry + .parse::() + .ok() + .or_else(|| entry.parse::().ok().map(|hop| hop.ip())) + .or_else(|| { + entry + .strip_prefix('[') + .and_then(|rest| rest.strip_suffix(']')) + .and_then(|inner| inner.parse::().ok()) + .map(IpAddr::V6) + }) +} + +pub fn comma_separated(raw: &str) -> Result, Infallible> { + Ok(raw + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .map(str::to_owned) + .collect()) +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TrustedProxies(BTreeSet); + +impl TrustedProxies { + pub fn parse<'a>(entries: impl IntoIterator) -> Result { + entries + .into_iter() + .map(ProxyNet::from_str) + .collect::, _>>() + .map(Self) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn contains(&self, peer: IpAddr) -> bool { + self.0.iter().any(|net| net.contains(peer)) + } + + pub fn rightmost_untrusted<'a, I>(&self, chain: I) -> Option + where + I: IntoIterator, + I::IntoIter: DoubleEndedIterator, + { + chain + .into_iter() + .rev() + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .take(MAX_HOPS) + .find_map(|entry| match chain_address(entry) { + Some(address) if self.contains(address) => None, + parsed => Some(parsed), + }) + .flatten() + .map(|address| address.to_canonical()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ip(value: &str) -> IpAddr { + value.parse().unwrap() + } + + fn trusting<'a>(entries: impl IntoIterator) -> TrustedProxies { + TrustedProxies::parse(entries).unwrap() + } + + fn client(proxies: &TrustedProxies, chain: &str) -> Option { + proxies.rightmost_untrusted(chain.split(',')) + } + + #[test] + fn rightmost_untrusted_takes_the_rightmost_entry_with_an_empty_list() { + let anyone = TrustedProxies::default(); + [ + ("203.0.113.7, 198.51.100.4", Some("198.51.100.4")), + (" 192.0.2.1 ", Some("192.0.2.1")), + ("not-an-ip", None), + ("", None), + ] + .iter() + .for_each(|&(chain, expected)| { + assert_eq!(client(&anyone, chain), expected.map(ip), "{chain:?}"); + }); + } + + #[test] + fn rightmost_untrusted_steps_over_every_listed_hop_and_stops_where_it_cant_parse() { + let proxies = trusting(["198.51.100.4"]); + [ + ( + "203.0.113.7, 198.51.100.4", + Some("203.0.113.7"), + "the hop left of the listed proxy is the client", + ), + ( + "203.0.113.7, 198.51.100.4, 198.51.100.4", + Some("203.0.113.7"), + "two listed hops in a row are both stepped over", + ), + ( + "203.0.113.7, 192.0.2.1, 198.51.100.4", + Some("192.0.2.1"), + "rightmost_untrusted stops at the first unlisted hop, even with a client still further left", + ), + ( + "198.51.100.4", + None, + "rightmost_untrusted won't find a client in a chain of listed hops alone", + ), + ( + "203.0.113.7, not-an-ip, 198.51.100.4", + None, + "rightmost_untrusted stops at an entry it can't parse, because a client can write anything left of the proxy", + ), + ( + "203.0.113.7, 198.51.100.4,", + Some("203.0.113.7"), + "a stray comma mustn't cost the header, since a client can't claim anything through a blank", + ), + ( + ",203.0.113.7,, 198.51.100.4", + Some("203.0.113.7"), + "a blank anywhere else in the chain reads the same way", + ), + ( + "203.0.113.7:4321, 198.51.100.4", + Some("203.0.113.7"), + "some proxies write the hop with the port it connected from", + ), + ( + "[2001:db8::1]:443, 198.51.100.4", + Some("2001:db8::1"), + "and bracket an IPv6 hop when they do", + ), + ( + "[2001:db8::1], 198.51.100.4", + Some("2001:db8::1"), + "brackets turn up without a port too", + ), + ( + "2001:db8::1, 198.51.100.4", + Some("2001:db8::1"), + "a bare IPv6 hop doesn't need unwrapping", + ), + ] + .iter() + .for_each(|&(chain, expected, why)| { + assert_eq!(client(&proxies, chain), expected.map(ip), "{why}: {chain:?}"); + }); + assert_eq!( + client( + &trusting(["198.51.100.0/24"]), + "203.0.113.7, 198.51.100.9, 198.51.100.10" + ), + Some(ip("203.0.113.7")), + "one CIDR entry covers every hop inside the block" + ); + } + + #[test] + fn rightmost_untrusted_reads_only_the_last_max_hops_entries() { + let proxies = trusting(["198.51.100.4"]); + let padded = |hops| { + std::iter::once("203.0.113.7") + .chain(std::iter::repeat_n("198.51.100.4", hops)) + .collect::>() + .join(",") + }; + assert_eq!( + client(&proxies, &padded(MAX_HOPS - 1)), + Some(ip("203.0.113.7")), + "a chain within MAX_HOPS still reaches the client behind every listed hop" + ); + assert_eq!( + client(&proxies, &padded(MAX_HOPS)), + None, + "the peer answers for the address it connected from when the last MAX_HOPS entries are all listed hops" + ); + } + + #[test] + fn an_entry_covers_every_address_it_spans_in_either_spelling() { + [ + (&["198.51.100.0/24"][..], "198.51.100.4", true, "a CIDR entry covers the addresses inside it"), + (&["198.51.100.0/24"], "198.51.100.255", true, "up to the last address in the block"), + (&["198.51.100.0/24"], "198.51.101.1", false, "and stops at the block boundary"), + (&["2001:db8::/32"], "2001:db8::dead:beef", true, "an IPv6 block reads the same way"), + (&["2001:db8::/32"], "2001:db9::1", false, "and stops at its boundary too"), + (&["2001:db8::/32"], "198.51.100.4", false, "an IPv6 block that isn't v4-mapped won't cover an IPv4 address"), + (&["::ffff:198.51.100.0/120"], "198.51.100.4", true, "a v4-mapped block covers the plain v4 addresses inside it"), + (&["::ffff:198.51.100.0/120"], "::ffff:198.51.100.4", true, "in either spelling"), + (&["::ffff:198.51.100.0/120"], "198.51.101.4", false, "and stops at the folded block boundary"), + (&["::ffff:0:0/96"], "203.0.113.7", true, "the whole v4-mapped range folds to every IPv4 address"), + (&["::ffff:0:0/95"], "203.0.113.7", true, "the match has to try the mapped spelling too, because a prefix under 96 keeps the block in IPv6, where the plain v4 spelling of a peer would miss it"), + (&["::/0"], "203.0.113.7", true, "an operator who lists every IPv6 address has listed every v4-mapped address with it"), + (&["127.0.0.1", " ::1 "], "127.0.0.1", true, "a bare address is a single host, and its entry may be padded"), + (&["127.0.0.1", " ::1 "], "::ffff:127.0.0.1", true, "which a dual-stack listener may report mapped"), + (&["127.0.0.1", " ::1 "], "::1", true, "the IPv6 loopback is its own entry"), + (&["127.0.0.1", " ::1 "], "127.0.0.2", false, "and the host next door is outside all of them"), + (&[], "203.0.113.7", false, "a caller that reads an empty list as trusting every peer has to say so itself, since rightmost_untrusted would step over every entry and never find a client"), + ] + .iter() + .for_each(|&(entries, peer, expected, why)| { + assert_eq!( + trusting(entries.iter().copied()).contains(ip(peer)), + expected, + "{why}: {entries:?} against {peer}" + ); + }); + assert!(TrustedProxies::default().is_empty()); + } + + #[test] + fn parse_refuses_an_entry_it_cant_read_and_quotes_it() { + assert_eq!( + TrustedProxies::parse(["127.0.0.1:5555"]) + .unwrap_err() + .to_string(), + "`127.0.0.1:5555` isn't an IP address or a CIDR block", + ); + [vec![""], vec![" "], vec!["127.0.0.1", "\t"]] + .into_iter() + .for_each(|entries| { + assert_eq!( + TrustedProxies::parse(entries.iter().copied()), + Err(ProxyNetError::Blank), + "discarding the blank would leave a list the operator filled in reading as empty. A caller is free to read an empty list as trusting every peer: {entries:?}" + ); + }); + } + + #[test] + fn comma_separated_discards_the_gaps_a_separator_leaves_behind() { + assert_eq!( + comma_separated("127.0.0.1, 173.245.48.0/20,").unwrap(), + vec!["127.0.0.1".to_owned(), "173.245.48.0/20".to_owned()], + "comma_separated mustn't leave a blank for parse to refuse, since a trailing separator belongs to the format" + ); + assert!(comma_separated("").unwrap().is_empty()); + } +} diff --git a/crypto/verify.go b/crypto/verify.go new file mode 100644 index 0000000000000000000000000000000000000000..a5c014c049035eeb507ea43d50a6ce8ce48ee248 --- /dev/null +++ b/crypto/verify.go @@ -0,0 +1,42 @@ +package crypto + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "fmt" + + "github.com/hiddeco/sshsig" + "golang.org/x/crypto/ssh" +) + +func VerifySignature(pubKey, signature, payload []byte) (error, bool) { + pub, _, _, _, err := ssh.ParseAuthorizedKey(pubKey) + if err != nil { + return fmt.Errorf("failed to parse public key: %w", err), false + } + + sig, err := sshsig.Unarmor(signature) + if err != nil { + return fmt.Errorf("failed to parse signature: %w", err), false + } + + buf := bytes.NewBuffer(payload) + // we use sha-512 because ed25519 keys require it internally; rsa keys support + // multiple algorithms but sha-512 is most secure, and git's ssh signing defaults + // to sha-512 for all key types anyway. + err = sshsig.Verify(buf, sig, pub, sshsig.HashSHA512, "git") + + return err, err == nil +} + +// SSHFingerprint computes the fingerprint of the supplied ssh pubkey. +func SSHFingerprint(pubKey string) (string, error) { + pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pubKey)) + if err != nil { + return "", err + } + + hash := sha256.Sum256(pk.Marshal()) + return "SHA256:" + base64.StdEncoding.EncodeToString(hash[:]), nil +} diff --git a/crypto/verify_test.go b/crypto/verify_test.go new file mode 100644 index 0000000000000000000000000000000000000000..873d98af22ea1e74f64752ae92ba5cd480798e0f --- /dev/null +++ b/crypto/verify_test.go @@ -0,0 +1,134 @@ +package crypto + +import ( + "bytes" + "crypto/ed25519" + "crypto/rand" + "strings" + "testing" + + "github.com/hiddeco/sshsig" + "golang.org/x/crypto/ssh" +) + +// testKey generates an ephemeral ed25519 key pair and returns the ssh.Signer +// and the public key in authorized_keys format. +func testKey(t *testing.T) (ssh.Signer, []byte) { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate ed25519 key: %v", err) + } + signer, err := ssh.NewSignerFromKey(priv) + if err != nil { + t.Fatalf("create signer: %v", err) + } + return signer, ssh.MarshalAuthorizedKey(signer.PublicKey()) +} + +// testSign signs payload with signer using the same parameters as VerifySignature +// expects (SHA-512, "git" namespace) and returns the armored signature. +func testSign(t *testing.T, signer ssh.Signer, payload []byte) []byte { + t.Helper() + sig, err := sshsig.Sign(bytes.NewReader(payload), signer, sshsig.HashSHA512, "git") + if err != nil { + t.Fatalf("sign payload: %v", err) + } + return sshsig.Armor(sig) +} + +func TestSSHFingerprint(t *testing.T) { + t.Run("valid key returns SHA256 fingerprint", func(t *testing.T) { + _, pubKeyBytes := testKey(t) + fp, err := SSHFingerprint(string(pubKeyBytes)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.HasPrefix(fp, "SHA256:") { + t.Errorf("fingerprint %q does not start with SHA256:", fp) + } + }) + + t.Run("same key returns identical fingerprint", func(t *testing.T) { + _, pubKeyBytes := testKey(t) + fp1, _ := SSHFingerprint(string(pubKeyBytes)) + fp2, _ := SSHFingerprint(string(pubKeyBytes)) + if fp1 != fp2 { + t.Errorf("fingerprint not deterministic: %q != %q", fp1, fp2) + } + }) + + t.Run("different keys return different fingerprints", func(t *testing.T) { + _, pub1 := testKey(t) + _, pub2 := testKey(t) + fp1, _ := SSHFingerprint(string(pub1)) + fp2, _ := SSHFingerprint(string(pub2)) + if fp1 == fp2 { + t.Error("different keys produced the same fingerprint") + } + }) + + t.Run("malformed key returns error", func(t *testing.T) { + _, err := SSHFingerprint("not a valid ssh public key") + if err == nil { + t.Error("expected error for malformed key") + } + }) +} + +func TestVerifySignature(t *testing.T) { + signer, pubKeyBytes := testKey(t) + payload := []byte("test payload") + armoredSig := testSign(t, signer, payload) + + t.Run("valid signature verifies successfully", func(t *testing.T) { + err, ok := VerifySignature(pubKeyBytes, armoredSig, payload) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !ok { + t.Error("expected ok=true for valid signature") + } + }) + + t.Run("malformed public key returns error", func(t *testing.T) { + err, ok := VerifySignature([]byte("not a valid key"), armoredSig, payload) + if err == nil { + t.Error("expected error for malformed public key") + } + if ok { + t.Error("expected ok=false") + } + }) + + t.Run("malformed signature returns error", func(t *testing.T) { + err, ok := VerifySignature(pubKeyBytes, []byte("not a valid signature"), payload) + if err == nil { + t.Error("expected error for malformed signature") + } + if ok { + t.Error("expected ok=false") + } + }) + + t.Run("tampered payload fails verification", func(t *testing.T) { + err, ok := VerifySignature(pubKeyBytes, armoredSig, []byte("tampered")) + if err == nil { + t.Error("expected error for tampered payload") + } + if ok { + t.Error("expected ok=false for tampered payload") + } + }) + + t.Run("wrong public key fails verification", func(t *testing.T) { + _, otherPubKey := testKey(t) + err, ok := VerifySignature(otherPubKey, armoredSig, payload) + if err == nil { + t.Error("expected error for wrong public key") + } + if ok { + t.Error("expected ok=false for wrong public key") + } + }) +} diff --git a/default.nix b/default.nix new file mode 100644 index 0000000000000000000000000000000000000000..3d72192f4094ec075a9af79f2c28863396d17d71 --- /dev/null +++ b/default.nix @@ -0,0 +1,16 @@ +# Default setup from https://git.lix.systems/lix-project/flake-compat +let + lockFile = builtins.fromJSON (builtins.readFile ./flake.lock); + flake-compat-node = lockFile.nodes.${lockFile.nodes.root.inputs.flake-compat}; + flake-compat = builtins.fetchTarball { + inherit (flake-compat-node.locked) url; + sha256 = flake-compat-node.locked.narHash; + }; + + flake = ( + import flake-compat { + src = ./.; + } + ); +in + flake.defaultNix diff --git a/deliberi/account_ingest_test.go b/deliberi/account_ingest_test.go new file mode 100644 index 0000000000000000000000000000000000000000..03574064c38f2e5ccdee4afc30b56b292214f737 --- /dev/null +++ b/deliberi/account_ingest_test.go @@ -0,0 +1,65 @@ +package deliberi + +import ( + "slices" + "testing" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + deldb "tangled.org/core/deliberi/db" + models "tangled.org/core/deliberi/models" + "tangled.org/core/jetstream" +) + +type kvDeletes struct{ keys []string } + +func (k *kvDeletes) PutEmailDid(string, string) {} +func (k *kvDeletes) SetPrimaryEmail(string, string) {} +func (k *kvDeletes) DeleteEmailKey(email string) { k.keys = append(k.keys, email) } +func (k *kvDeletes) DeletePrimaryEmail(did string) { k.keys = append(k.keys, "__primary:"+did) } + +func accountFrame(did string, active bool, status string) *jetstream.Event { + frameClock++ + acct := &comatproto.SyncSubscribeRepos_Account{Did: did, Active: active} + if status != "" { + acct.Status = &status + } + return &jetstream.Event{Did: did, TimeUS: frameClock, Kind: jetstream.EventKindAccount, Account: acct} +} + +func TestAccountDeletionDropsEmails(t *testing.T) { + const gone, kept = "did:plc:gone", "did:plc:kept" + i := newTestIngester(t, fakeBobbin{}) + rec := &kvDeletes{} + i.emailKV = rec + for _, e := range []models.Email{ + {Did: gone, Address: "gone@example.com", Verified: true, Primary: true}, + {Did: gone, Address: "alt@example.com", Verified: true}, + {Did: kept, Address: "kept@example.com", Verified: true, Primary: true}, + } { + if err := deldb.AddEmail(i.db, e); err != nil { + t.Fatal(err) + } + } + + mustProcess(t, i, + accountFrame(gone, false, "deactivated"), + accountFrame(kept, false, "takendown"), + ) + if len(rec.keys) != 0 { + t.Fatalf("reversible statuses touched KV: %v", rec.keys) + } + + mustProcess(t, i, accountFrame(gone, false, "deleted")) + + if left, _ := deldb.GetAllEmails(i.db, gone); len(left) != 0 { + t.Errorf("deleted account still has emails: %v", left) + } + if left, _ := deldb.GetAllEmails(i.db, kept); len(left) != 1 { + t.Errorf("other account lost emails: %v", left) + } + slices.Sort(rec.keys) + want := []string{"__primary:" + gone, "alt@example.com", "gone@example.com"} + if !slices.Equal(rec.keys, want) { + t.Errorf("KV deletes = %v, want %v", rec.keys, want) + } +} diff --git a/deliberi/analytics/analytics.go b/deliberi/analytics/analytics.go new file mode 100644 index 0000000000000000000000000000000000000000..dcc02a8b7dd03aecb50e647363fd0d6de65c5233 --- /dev/null +++ b/deliberi/analytics/analytics.go @@ -0,0 +1,67 @@ +// Package analytics sends best-effort product events to posthog. A nil *Client is +// valid and drops every event, so callers don't check whether analytics is on. +package analytics + +import ( + "log/slog" + "time" + + "github.com/google/uuid" + "github.com/posthog/posthog-go" + "tangled.org/core/deliberi/config" +) + +// the same namespace for every event, so an event's uuid only depends on its key +var eventNamespace = uuid.MustParse("3f6b2c8e-1d4a-4f7e-9b2a-6c5d8e9f0a1b") + +type Client struct { + ph posthog.Client + env string + logger *slog.Logger +} + +// New returns nil when no api key is configured +func New(cfg config.PosthogConfig, logger *slog.Logger) *Client { + if cfg.ApiKey == "" { + return nil + } + ph, err := posthog.NewWithConfig(cfg.ApiKey, posthog.Config{Endpoint: cfg.Endpoint}) + if err != nil { + logger.Warn("posthog client failed to start, analytics off", "err", err) + return nil + } + logger.Info("posthog analytics enabled", "endpoint", cfg.Endpoint, "env", cfg.Env) + return &Client{ph: ph, env: cfg.Env, logger: logger} +} + +// Capture queues an event. key identifies the occurrence (a record version, an +// account), so a replayed event keeps its uuid and posthog drops the duplicate +func (c *Client) Capture(event, distinctID, key string, properties posthog.Properties) { + if c == nil { + return + } + if properties == nil { + properties = posthog.NewProperties() + } + properties.Set("environment", c.env) + err := c.ph.Enqueue(posthog.Capture{ + Uuid: uuid.NewSHA1(eventNamespace, []byte(event+" "+key)).String(), + DistinctId: distinctID, + Event: event, + Timestamp: time.Now().UTC(), + Properties: properties, + }) + if err != nil { + c.logger.Warn("failed to enqueue posthog event", "event", event, "err", err) + } +} + +// Close flushes queued events +func (c *Client) Close() { + if c == nil { + return + } + if err := c.ph.Close(); err != nil { + c.logger.Warn("closing posthog client", "err", err) + } +} diff --git a/deliberi/bobbin.go b/deliberi/bobbin.go new file mode 100644 index 0000000000000000000000000000000000000000..cdc21b366bab7b9bb1d5093435ea79598e83c0f0 --- /dev/null +++ b/deliberi/bobbin.go @@ -0,0 +1,104 @@ +package deliberi + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "github.com/samber/lo" + "tangled.org/core/api/org_tangled" +) + +type bobbinLookup interface { + RepoOwner(ctx context.Context, repoDid string) (ownerDid, name string, err error) + TicketAuthor(ctx context.Context, ticketUri string) (string, error) +} + +type inviteLister interface { + ListCollaboratorInvitesBy(ctx context.Context, subjectDid string) (InviteAnswer, error) +} + +type InviteOffer struct { + Uri string `json:"uri"` + KnotDid string `json:"knot"` + RepoDid string `json:"repo"` + AddedBy string `json:"addedBy"` + CreatedAt time.Time `json:"createdAt"` +} + +type InviteAnswer struct { + Offers []InviteOffer `json:"items"` + Pending []string `json:"pending"` + Truncated bool `json:"truncated"` +} + +type bobbinClient struct { + xc *indigoxrpc.Client +} + +func newBobbinClient(apiUrl string) *bobbinClient { + return &bobbinClient{ + xc: &indigoxrpc.Client{ + Host: strings.TrimRight(apiUrl, "/"), + Client: &http.Client{Timeout: 10 * time.Second}, + }, + } +} + +// the record can't name the opener once someone else edits it, so ask bobbin +func (c *bobbinClient) TicketAuthor(ctx context.Context, ticketUri string) (string, error) { + out, err := org_tangled.TrackGetTicketView(ctx, c.xc, ticketUri) + if err != nil { + return "", fmt.Errorf("calling %s: %w", org_tangled.TrackGetTicketViewNSID, err) + } + if out.Author == nil { + return "", fmt.Errorf("ticket view for %s has no author", ticketUri) + } + return out.Author.Did, nil +} + +// bobbin serves getRepoByRepoDid from the repo's org.tangled.repo.manifest +type repoByRepoDid struct { + Value repoManifest `json:"value"` +} + +func (c *bobbinClient) RepoOwner(ctx context.Context, repoDid string) (string, string, error) { + var out repoByRepoDid + params := map[string]any{"repoDid": repoDid} + if err := c.xc.Do(ctx, indigoxrpc.Query, "", org_tangled.RepoGetRepoByRepoDidNSID, params, nil, &out); err != nil { + return "", "", fmt.Errorf("calling %s: %w", org_tangled.RepoGetRepoByRepoDidNSID, err) + } + + decl := out.Value.Declaration + if decl == nil || decl.Owner == "" { + return "", "", fmt.Errorf("manifest for %s has no declared owner", repoDid) + } + return decl.Owner, decl.Name, nil +} + +func (c *bobbinClient) ListCollaboratorInvitesBy(ctx context.Context, subjectDid string) (InviteAnswer, error) { + var out InviteAnswer + params := map[string]any{"subject": subjectDid} + if err := c.xc.Do(ctx, indigoxrpc.Query, "", org_tangled.RepoListCollaboratorInvitesByNSID, params, nil, &out); err != nil { + return InviteAnswer{}, fmt.Errorf("calling %s: %w", org_tangled.RepoListCollaboratorInvitesByNSID, err) + } + + if blank, found := lo.Find(out.Offers, func(offer InviteOffer) bool { + return offer.Uri == "" || offer.KnotDid == "" || offer.RepoDid == "" + }); found { + missing := "record" + if blank.Uri != "" { + missing = lo.Ternary(blank.KnotDid == "", "knot", "repo") + } + return InviteAnswer{}, fmt.Errorf( + "%s served offer missing its %s, so we couldn't tell it apart from a withdrawal", + org_tangled.RepoListCollaboratorInvitesByNSID, + missing, + ) + } + + return out, nil +} diff --git a/deliberi/bobbin_test.go b/deliberi/bobbin_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7523df34079573ab64f15fcab1a8494d12976029 --- /dev/null +++ b/deliberi/bobbin_test.go @@ -0,0 +1,132 @@ +package deliberi + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "tangled.org/core/api/org_tangled" +) + +const collaboratorInvitesByBody = `{"items":[{"uri":"at://did:plc:scallop/sh.tangled.repo.collaboratorInvite/did:plc:limpet","knot":"did:web:knot.oyster.cafe","repo":"did:plc:scallop","addedBy":"did:plc:boltless","createdAt":"2026-06-01T00:00:00.000000Z"}],"pending":[],"truncated":false}` + +func serving(t *testing.T, path, body string) *bobbinClient { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != path { + t.Errorf("path = %q, want %q", r.URL.Path, path) + http.NotFound(w, r) + return + } + if r.URL.Query().Get("subject") == "" { + t.Error("subject param missing; bobbin requires it") + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, body) + })) + t.Cleanup(srv.Close) + return newBobbinClient(srv.URL) +} + +func TestListCollaboratorInvitesByDecodesBobbinsBody(t *testing.T) { + c := serving(t, "/xrpc/"+org_tangled.RepoListCollaboratorInvitesByNSID, collaboratorInvitesByBody) + + answer, err := c.ListCollaboratorInvitesBy(context.Background(), "did:plc:limpet") + if err != nil { + t.Fatalf("ListCollaboratorInvitesBy: %v", err) + } + want := InviteOffer{ + Uri: "at://did:plc:scallop/sh.tangled.repo.collaboratorInvite/did:plc:limpet", + KnotDid: "did:web:knot.oyster.cafe", + RepoDid: "did:plc:scallop", + AddedBy: "did:plc:boltless", + CreatedAt: time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC), + } + if len(answer.Offers) != 1 || answer.Offers[0] != want { + t.Fatalf("offers = %+v, want one %+v", answer.Offers, want) + } +} + +func TestPartialAnswerListsWhatBobbinCouldntRead(t *testing.T) { + const body = `{"items":[],"pending":["did:web:cold.knot","did:web:other.knot"],"truncated":true}` + c := serving(t, "/xrpc/"+org_tangled.RepoListCollaboratorInvitesByNSID, body) + + answer, err := c.ListCollaboratorInvitesBy(context.Background(), "did:plc:limpet") + if err != nil { + t.Fatalf("ListCollaboratorInvitesBy: %v", err) + } + if len(answer.Pending) != 2 { + t.Fatalf("pending = %v, want both knots", answer.Pending) + } + if !answer.Truncated { + t.Fatal("truncated dropped in decode, so we'd read cut list as whole offer set") + } +} + +func TestUnreadableCreatedAtFailsListing(t *testing.T) { + const body = `{"items":[{"uri":"at://did:plc:scallop/sh.tangled.repo.collaboratorInvite/did:plc:limpet","knot":"did:web:knot.example","repo":"did:plc:scallop","addedBy":"did:plc:akshay","createdAt":"whenever"}]}` + c := serving(t, "/xrpc/"+org_tangled.RepoListCollaboratorInvitesByNSID, body) + + if _, err := c.ListCollaboratorInvitesBy(context.Background(), "did:plc:limpet"); err == nil { + t.Fatal("we can't stamp rows from timestamps that won't read") + } +} + +func TestOfferMissingItsKnotFailsListing(t *testing.T) { + const body = `{"items":[{"uri":"at://did:plc:scallop/sh.tangled.repo.collaboratorInvite/did:plc:limpet","repo":"did:plc:scallop","addedBy":"did:plc:akshay","createdAt":"2026-06-01T00:00:00.000000Z"}],"pending":[],"truncated":false}` + c := serving(t, "/xrpc/"+org_tangled.RepoListCollaboratorInvitesByNSID, body) + + if _, err := c.ListCollaboratorInvitesBy(context.Background(), "did:plc:limpet"); err == nil { + t.Fatal("offers missing their knot can't be checked against pending, so accepting one would make its absence look like withdrawal") + } +} + +func TestOfferMissingItsRepoFailsListing(t *testing.T) { + const body = `{"items":[{"uri":"at://did:plc:scallop/sh.tangled.repo.collaboratorInvite/did:plc:limpet","knot":"did:web:knot.example","addedBy":"did:plc:akshay","createdAt":"2026-06-01T00:00:00.000000Z"}],"pending":[],"truncated":false}` + c := serving(t, "/xrpc/"+org_tangled.RepoListCollaboratorInvitesByNSID, body) + + if _, err := c.ListCollaboratorInvitesBy(context.Background(), "did:plc:limpet"); err == nil { + t.Fatal("a repo-less offer has nothing to link or accept against, so the row would be unusable") + } +} + +func TestWarming503FailsListing(t *testing.T) { + const warming = `{"error":"Warming","message":"invite index is still warming, so it can't list offers yet"}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + fmt.Fprint(w, warming) + })) + t.Cleanup(srv.Close) + + c := newBobbinClient(srv.URL) + if _, err := c.ListCollaboratorInvitesBy(context.Background(), "did:plc:limpet"); err == nil { + t.Fatal("503 decoded as empty listing, so reconcile would read every offer as withdrawn") + } +} + +func TestRepoOwnerReadsManifestDeclaration(t *testing.T) { + // the manifest lives in the repo's own account, so the uri authority is the + // repo did and must not be mistaken for the owner + const body = `{"uri":"at://did:plc:therepo/org.tangled.repo.manifest/self","cid":"bafyfake","knot":"did:web:knot.example",` + + `"value":{"$type":"org.tangled.repo.manifest","createdAt":"2026-01-01T00:00:00Z","declaration":{"owner":"did:plc:bob","name":"my-repo"}}}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Query().Get("repoDid"); got != "did:plc:therepo" { + t.Errorf("repoDid param = %q", got) + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, body) + })) + t.Cleanup(srv.Close) + + owner, name, err := newBobbinClient(srv.URL).RepoOwner(context.Background(), "did:plc:therepo") + if err != nil { + t.Fatalf("RepoOwner: %v", err) + } + if owner != "did:plc:bob" || name != "my-repo" { + t.Fatalf("owner/name = %q/%q, want did:plc:bob/my-repo", owner, name) + } +} diff --git a/deliberi/config/config.go b/deliberi/config/config.go new file mode 100644 index 0000000000000000000000000000000000000000..2e79396de02dbc8383564a5ccc0f6ac00f25c032 --- /dev/null +++ b/deliberi/config/config.go @@ -0,0 +1,89 @@ +package config + +import ( + "context" + "fmt" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/sethvargo/go-envconfig" +) + +type Config struct { + ListenAddr string `env:"DELIBERI_LISTEN_ADDR, default=0.0.0.0:6565"` + + // public hostname; derives deliberi's did:web, the audience for service auth + Hostname string `env:"DELIBERI_HOSTNAME, required"` + + DbPath string `env:"DELIBERI_DB_PATH, default=deliberi.db"` + + PlcUrl string `env:"DELIBERI_PLC_URL, default=https://plc.directory"` + JetstreamEndpoint string `env:"DELIBERI_JETSTREAM_ENDPOINT, default=wss://index.tangled.network/subscribe"` + + // listRecipients subscriber fan-out; plain internal xrpc, no auth + BobbinApiUrl string `env:"DELIBERI_BOBBIN_API_URL, default=https://api.tangled.org"` + + BaseURL string `env:"DELIBERI_BASE_URL, default=https://tangled.org"` + + Pds PdsConfig `env:",prefix=DELIBERI_PDS_"` + Resend ResendConfig `env:",prefix=DELIBERI_RESEND_"` + Posthog PosthogConfig `env:",prefix=DELIBERI_POSTHOG_"` + + // optional; any unset leaves the KV cache client nil + CloudflareAccountID string `env:"CLOUDFLARE_ACCOUNT_ID"` + KvApiToken string `env:"DELIBERI_KV_API_TOKEN"` + KvNamespaceID string `env:"DELIBERI_KV_NAMESPACE_ID"` + + TurnstileSecretKey string `env:"DELIBERI_TURNSTILE_SECRET_KEY"` + TurnstileVerifyURL string `env:"DELIBERI_TURNSTILE_VERIFY_URL, default=https://challenges.cloudflare.com/turnstile/v0/siteverify"` + TrustedServiceDIDs []string `env:"DELIBERI_TRUSTED_SERVICE_DIDS"` + + Dev bool `env:"DELIBERI_DEV, default=false"` +} + +type PdsConfig struct { + Host string `env:"HOST, default=https://tngl.sh"` + UserDomain string `env:"USER_DOMAIN, default=.tngl.sh"` + AdminSecret string `env:"ADMIN_SECRET"` +} + +type ResendConfig struct { + ApiKey string `env:"API_KEY"` + SentFrom string `env:"SENT_FROM, default=noreply@notifs.tangled.sh"` + AssetsURL string `env:"ASSETS_URL, default=https://assets.tangled.network/email/"` + NewsletterSegmentId string `env:"NEWSLETTER_SEGMENT_ID"` +} + +// an empty api key turns analytics off +type PosthogConfig struct { + ApiKey string `env:"API_KEY"` + Endpoint string `env:"ENDPOINT, default=https://eu.i.posthog.com"` + Env string `env:"ENV, default=dev"` +} + +func (c *Config) SignupEnabled() bool { + return c.Pds.AdminSecret != "" +} + +func (c *Config) TrustsService(did string) bool { + for _, trusted := range c.TrustedServiceDIDs { + if did == trusted { + return true + } + } + return false +} + +func Load(ctx context.Context) (*Config, error) { + var cfg Config + if err := envconfig.Process(ctx, &cfg); err != nil { + return nil, err + } + for i, value := range cfg.TrustedServiceDIDs { + did, err := syntax.ParseDID(value) + if err != nil { + return nil, fmt.Errorf("DELIBERI_TRUSTED_SERVICE_DIDS: %w", err) + } + cfg.TrustedServiceDIDs[i] = did.String() + } + return &cfg, nil +} diff --git a/deliberi/config/config_test.go b/deliberi/config/config_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ef8a80f3c1aa917deac45d7c75517d1387fd92a6 --- /dev/null +++ b/deliberi/config/config_test.go @@ -0,0 +1,44 @@ +package config + +import ( + "context" + "testing" +) + +func TestLoadValidatesTrustedServiceDIDs(t *testing.T) { + for _, test := range []struct { + name string + value string + wantError bool + trusted []string + }{ + {name: "none"}, + {name: "one", value: "did:web:mirror.example", trusted: []string{"did:web:mirror.example"}}, + { + name: "many", + value: "did:web:mirror.example,did:web:indexer.example", + trusted: []string{"did:web:mirror.example", "did:web:indexer.example"}, + }, + {name: "one invalid", value: "did:web:mirror.example,not-a-did", wantError: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Setenv("DELIBERI_HOSTNAME", "deliberi.example") + t.Setenv("DELIBERI_TRUSTED_SERVICE_DIDS", test.value) + cfg, err := Load(context.Background()) + if (err != nil) != test.wantError { + t.Fatalf("Load() error = %v, wantError = %v", err, test.wantError) + } + if err != nil { + return + } + if got, want := len(cfg.TrustedServiceDIDs), len(test.trusted); got != want { + t.Fatalf("trusted service DIDs = %v, want %v", cfg.TrustedServiceDIDs, test.trusted) + } + for _, did := range test.trusted { + if !cfg.TrustsService(did) { + t.Errorf("TrustsService(%q) = false", did) + } + } + }) + } +} diff --git a/deliberi/db/cache.go b/deliberi/db/cache.go new file mode 100644 index 0000000000000000000000000000000000000000..2bc80737f26cfdd9ae91dab36029a57cb1b05002 --- /dev/null +++ b/deliberi/db/cache.go @@ -0,0 +1,64 @@ +package db + +import "tangled.org/core/deliberi/models" + +func PutRepoName(e Execer, repoDid, ownerDid, name string) error { + _, err := e.Exec( + `insert into repo_names (repo_did, name, owner_did) values (?, ?, ?) + on conflict(repo_did) do update set name = excluded.name, owner_did = excluded.owner_did`, + repoDid, name, ownerDid, + ) + return err +} + +func GetRepoName(e Execer, repoDid string) string { + var name string + _ = e.QueryRow(`select name from repo_names where repo_did = ?`, repoDid).Scan(&name) + return name +} + +func GetRepoOwner(e Execer, repoDid string) string { + var owner string + _ = e.QueryRow(`select owner_did from repo_names where repo_did = ?`, repoDid).Scan(&owner) + return owner +} + +type Entity struct { + Title string + RepoDid string + Kind models.EntityKind + State string + // who opened it. any non-empty value overwrites the stored one, so never pass the latest editor + Author string + Cid string +} + +// empty fields keep the stored value, title always overwrites +func PutEntity(e Execer, atUri string, ent Entity) error { + _, err := e.Exec( + `insert into entity_titles (at_uri, title, repo_did, kind, state, author_did, cid) values (?, ?, ?, ?, ?, ?, ?) + on conflict(at_uri) do update set + title = excluded.title, + repo_did = case when excluded.repo_did != '' then excluded.repo_did else entity_titles.repo_did end, + kind = case when excluded.kind != '' then excluded.kind else entity_titles.kind end, + state = case when excluded.state != '' then excluded.state else entity_titles.state end, + author_did = case when excluded.author_did != '' then excluded.author_did else entity_titles.author_did end, + cid = case when excluded.cid != '' then excluded.cid else entity_titles.cid end`, + atUri, ent.Title, ent.RepoDid, string(ent.Kind), ent.State, ent.Author, ent.Cid, + ) + return err +} + +func GetEntity(e Execer, atUri string) Entity { + var ent Entity + var kind string + _ = e.QueryRow(`select title, repo_did, kind, state, author_did, cid from entity_titles where at_uri = ?`, atUri). + Scan(&ent.Title, &ent.RepoDid, &kind, &ent.State, &ent.Author, &ent.Cid) + ent.Kind = models.EntityKind(kind) + return ent +} + +func DeleteEntity(e Execer, atUri string) error { + _, err := e.Exec(`delete from entity_titles where at_uri = ?`, atUri) + return err +} diff --git a/deliberi/db/cursor.go b/deliberi/db/cursor.go new file mode 100644 index 0000000000000000000000000000000000000000..f0da2b92561da3d3371135dd06dc951f638a6c6b --- /dev/null +++ b/deliberi/db/cursor.go @@ -0,0 +1,31 @@ +package db + +import ( + "database/sql" + "errors" +) + +// false on a fresh db, so the client replays from 0 instead of the live tail +func GetCursor(e Execer) (timeUs int64, ok bool, err error) { + err = e.QueryRow(`select last_time_us from jetstream_cursor where id = 0`).Scan(&timeUs) + if errors.Is(err, sql.ErrNoRows) { + return 0, false, nil + } + return timeUs, err == nil, err +} + +// max keeps a rewound replay from dragging the cursor back +func SaveCursor(e Execer, timeUs int64) error { + _, err := e.Exec( + `insert into jetstream_cursor (id, last_time_us) values (0, ?) + on conflict(id) do update set last_time_us = max(last_time_us, excluded.last_time_us)`, + timeUs, + ) + return err +} + +func GetBootstrapUs(e Execer) int64 { + var us int64 + _ = e.QueryRow(`select time_us from ingest_bootstrap where id = 0`).Scan(&us) + return us +} diff --git a/deliberi/db/db.go b/deliberi/db/db.go new file mode 100644 index 0000000000000000000000000000000000000000..e4accc89621892f1d706b85cac3d1ce7b97acdcb --- /dev/null +++ b/deliberi/db/db.go @@ -0,0 +1,335 @@ +package db + +import ( + "context" + "database/sql" + "log/slog" + "strings" + "time" + + "tangled.org/core/log" + "tangled.org/core/orm" + "tangled.org/core/sqlite" +) + +type DB struct { + *sql.DB + logger *slog.Logger +} + +type Execer interface { + Query(query string, args ...any) (*sql.Rows, error) + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + QueryRow(query string, args ...any) *sql.Row + Exec(query string, args ...any) (sql.Result, error) +} + +func Make(ctx context.Context, dbPath string) (*DB, error) { + logger := log.SubLogger(log.FromContext(ctx), "db") + + db, err := sqlite.Open(dbPath) + if err != nil { + return nil, err + } + + conn, err := db.Conn(ctx) + if err != nil { + return nil, err + } + defer conn.Close() + + _, err = conn.ExecContext(ctx, schema) + if err != nil { + return nil, err + } + + if err := runMigrations(conn, logger); err != nil { + return nil, err + } + + return &DB{db, logger}, nil +} + +func (d *DB) Close() error { + return d.DB.Close() +} + +const schema = ` +create table if not exists emails ( + id integer primary key autoincrement, + did text not null, + email text not null, + verified integer not null default 0, + verification_code text not null, + last_sent text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + is_primary integer not null default 0, + created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + unique(did, email) +); + +create table if not exists signups_inflight ( + id integer primary key autoincrement, + email text not null unique, + invite_code text not null, + verification_token text not null default '', + created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + expires_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '+24 hours')), + last_sent text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) +); +-- the verification_token index is built by the signup-token-index migration, +-- never here: a pre-token database must gain the column before the index. + +-- one materialized row per (recipient, source record). deliberi owns these; +-- read/emailed are inline. unique(recipient_did, at_uri) dedupes fan-out. +create table if not exists notifications ( + id integer primary key autoincrement, + recipient_did text not null, + at_uri text not null, + type text not null, + actor_did text not null, + repo_did text not null default '', + knot_did text not null default '', + entity_at text not null default '', + entity_kind text not null default '', + entity_title text not null default '', + read integer not null default 0, + emailed integer not null default 0, + created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + unique(recipient_did, at_uri) +); +create index if not exists idx_deliberi_notifs_recipient on notifications(recipient_did, read, created); +create index if not exists idx_deliberi_notifs_digest on notifications(recipient_did, emailed, read, created); + +-- small denormalization caches the ingester fills from manifest/ticket +-- records so notifications render human names without extra lookups. +create table if not exists repo_names ( + repo_did text primary key, + name text not null, + owner_did text not null default '' +); +-- repo_did and kind let a comment find its parent ticket's repo and kind. +create table if not exists entity_titles ( + at_uri text primary key, + title text not null, + repo_did text not null default '', + kind text not null default '' +); + +create table if not exists jetstream_cursor ( + id integer primary key check (id = 0), + last_time_us integer not null +); + +-- newest commit rev applied per record uri. deletes keep their row as a +-- tombstone so an older create replayed later can't bring the record back. +create table if not exists record_revs ( + uri text primary key, + rev text not null +); + +-- events stamped before this time_us only build state and never notify +create table if not exists ingest_bootstrap ( + id integer primary key check (id = 0), + time_us integer not null +); + +create table if not exists repo_collaborators ( + repo_did text not null, + did text not null, + primary key (repo_did, did) +); + +-- root is the top level comment of the thread, itself for a top level comment +create table if not exists comments ( + uri text primary key, + ticket_uri text not null, + root text not null, + author_did text not null +); +create index if not exists idx_deliberi_comments_ticket on comments(ticket_uri); +create index if not exists idx_deliberi_comments_root on comments(root); + +-- patch record uri -> ticket uri, so comments on a patch find their ticket +create table if not exists patches ( + uri text primary key, + ticket_uri text not null +); + +-- subject is a record at-uri or a repo did. a delete frame only carries the +-- rkey, so rows are kept per (author, rkey) and resolved per subject on read. +create table if not exists subscriptions ( + author_did text not null, + rkey text not null, + subject text not null, + ignore integer not null default 0, + primary key (author_did, rkey) +); +create index if not exists idx_deliberi_subscriptions_subject on subscriptions(subject); + +create table if not exists migrations ( + name text primary key +); + +create table if not exists notification_preferences ( + id integer primary key autoincrement, + user_did text not null unique, + repo_starred integer not null default 1, + issue_created integer not null default 1, + issue_commented integer not null default 1, + pull_created integer not null default 1, + pull_commented integer not null default 1, + followed integer not null default 1, + user_mentioned integer not null default 1, + email_notifications integer not null default 0, + issue_closed integer not null default 1, + pull_merged integer not null default 1 +); + +create table if not exists newsletter_preferences ( + id integer primary key autoincrement, + user_did text not null unique, + status text not null check (status in ('subscribed', 'dismissed')), + email text, + updated_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) +); +` + +func runMigrations(conn *sql.Conn, logger *slog.Logger) error { + if err := orm.RunMigration(conn, logger, "add-owner-did-to-repo-names", func(tx *sql.Tx) error { + _, err := tx.Exec(`alter table repo_names add column owner_did text not null default ''`) + if err != nil && !isColumnExistsErr(err) { + return err + } + return nil + }); err != nil { + return err + } + if err := orm.RunMigration(conn, logger, "email-canonical-unique", func(tx *sql.Tx) error { + // legacy appview data could hold case-variant or repeated addresses; keep + // one row per canonical address: verified wins, else primary, else earliest + if _, err := tx.Exec(` + delete from emails where id not in ( + select id from ( + select id, row_number() over ( + partition by lower(trim(email)) + order by verified desc, is_primary desc, id asc + ) as rn + from emails + ) where rn = 1 + ) + `); err != nil { + return err + } + // survivors keep legacy casing; normalize so handler lookups match + if _, err := tx.Exec(`update emails set email = lower(trim(email))`); err != nil { + return err + } + // legacy AddEmail could auto-promote an unverified first row; the + // invariant is verified-only primary + if _, err := tx.Exec(`update emails set is_primary = false where verified = false`); err != nil { + return err + } + _, err := tx.Exec(`create unique index if not exists emails_email_lower_unique on emails (lower(email))`) + return err + }); err != nil { + return err + } + if err := orm.RunMigration(conn, logger, "signup-expiry", func(tx *sql.Tx) error { + if _, err := tx.Exec(`alter table signups_inflight add column expires_at text not null default ''`); err != nil && !isColumnExistsErr(err) { + return err + } + // rows predating the column get a 24h lifetime from migration time; new + // rows already default to +24h at insert (see the schema) + _, err := tx.Exec(`update signups_inflight set expires_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '+24 hours') where expires_at = ''`) + return err + }); err != nil { + return err + } + if err := orm.RunMigration(conn, logger, "signup-token-index", func(tx *sql.Tx) error { + // '' rows from before the column existed get unique tokens so the index holds + if _, err := tx.Exec(`alter table signups_inflight add column verification_token text not null default ''`); err != nil && !isColumnExistsErr(err) { + return err + } + if _, err := tx.Exec(`update signups_inflight set verification_token = lower(hex(randomblob(32))) where verification_token = ''`); err != nil { + return err + } + _, err := tx.Exec(`create unique index if not exists signups_inflight_verification_token_unique on signups_inflight (verification_token)`) + return err + }); err != nil { + return err + } + if err := orm.RunMigration(conn, logger, "signup-resend-cooldown", func(tx *sql.Tx) error { + // "-1 day": an upgraded row must never block a first send + if _, err := tx.Exec(`alter table signups_inflight add column last_sent text not null default ''`); err != nil && !isColumnExistsErr(err) { + return err + } + _, err := tx.Exec(`update signups_inflight set last_sent = strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-1 day') where last_sent = ''`) + return err + }); err != nil { + return err + } + if err := orm.RunMigration(conn, logger, "notification-knot-did", func(tx *sql.Tx) error { + if _, err := tx.Exec(`alter table notifications add column knot_did text not null default ''`); err != nil && !isColumnExistsErr(err) { + return err + } + return nil + }); err != nil { + return err + } + if err := orm.RunMigration(conn, logger, "drop-knot-invited-notifications", func(tx *sql.Tx) error { + _, err := tx.Exec(`delete from notifications where type = 'knot_invited'`) + return err + }); err != nil { + return err + } + if err := orm.RunMigration(conn, logger, "entity-kind", func(tx *sql.Tx) error { + if _, err := tx.Exec(`alter table notifications add column entity_kind text not null default ''`); err != nil && !isColumnExistsErr(err) { + return err + } + if _, err := tx.Exec(`alter table entity_titles add column kind text not null default ''`); err != nil && !isColumnExistsErr(err) { + return err + } + return nil + }); err != nil { + return err + } + if err := orm.RunMigration(conn, logger, "ticket-state", func(tx *sql.Tx) error { + for _, col := range []string{"state", "author_did", "cid"} { + if _, err := tx.Exec(`alter table entity_titles add column ` + col + ` text not null default ''`); err != nil && !isColumnExistsErr(err) { + return err + } + } + return nil + }); err != nil { + return err + } + if err := orm.RunMigration(conn, logger, "state-notification-prefs", func(tx *sql.Tx) error { + for _, col := range []string{"issue_closed", "pull_merged"} { + if _, err := tx.Exec(`alter table notification_preferences add column ` + col + ` integer not null default 1`); err != nil && !isColumnExistsErr(err) { + return err + } + } + return nil + }); err != nil { + return err + } + return orm.RunMigration(conn, logger, "ingest-bootstrap", func(tx *sql.Tx) error { + // existing db gets 0 so old history still notifies, a fresh one starts at now + _, err := tx.Exec(` + insert or ignore into ingest_bootstrap (id, time_us) + select 0, case when exists (select 1 from jetstream_cursor) then 0 else ? end`, + time.Now().UnixMicro(), + ) + return err + }) +} + +func isColumnExistsErr(err error) bool { + return err != nil && strings.Contains(err.Error(), "duplicate column name") +} + +// reports whether err is a sqlite unique-constraint violation (e.g. the +// lower(email) reservation index) +func IsUniqueConstraintErr(err error) bool { + return err != nil && strings.Contains(err.Error(), "UNIQUE constraint failed") +} diff --git a/deliberi/db/email.go b/deliberi/db/email.go new file mode 100644 index 0000000000000000000000000000000000000000..0715ef8a5bc1283149ab7aa57f2891465c3bf05e --- /dev/null +++ b/deliberi/db/email.go @@ -0,0 +1,351 @@ +package db + +import ( + "strings" + "time" + + "tangled.org/core/deliberi/models" +) + +func GetPrimaryEmail(e Execer, did string) (models.Email, error) { + query := ` + select id, did, email, verified, is_primary, verification_code, last_sent, created + from emails + where did = ? and is_primary = true + ` + var email models.Email + var createdStr string + var lastSent string + err := e.QueryRow(query, did).Scan(&email.ID, &email.Did, &email.Address, &email.Verified, &email.Primary, &email.VerificationCode, &lastSent, &createdStr) + if err != nil { + return models.Email{}, err + } + email.CreatedAt, err = time.Parse(time.RFC3339, createdStr) + if err != nil { + return models.Email{}, err + } + parsedTime, err := time.Parse(time.RFC3339, lastSent) + if err != nil { + return models.Email{}, err + } + email.LastSent = &parsedTime + return email, nil +} + +func GetEmail(e Execer, did string, em string) (models.Email, error) { + query := ` + select id, did, email, verified, is_primary, verification_code, last_sent, created + from emails + where did = ? and email = ? + ` + var email models.Email + var createdStr string + var lastSent string + err := e.QueryRow(query, did, em).Scan(&email.ID, &email.Did, &email.Address, &email.Verified, &email.Primary, &email.VerificationCode, &lastSent, &createdStr) + if err != nil { + return models.Email{}, err + } + email.CreatedAt, err = time.Parse(time.RFC3339, createdStr) + if err != nil { + return models.Email{}, err + } + parsedTime, err := time.Parse(time.RFC3339, lastSent) + if err != nil { + return models.Email{}, err + } + email.LastSent = &parsedTime + return email, nil +} + +func GetDidForEmail(e Execer, em string) (string, error) { + query := ` + select did + from emails + where email = ? + ` + var did string + err := e.QueryRow(query, em).Scan(&did) + if err != nil { + return "", err + } + return did, nil +} + +func GetEmailToDid(e Execer, emails []string, isVerifiedFilter bool) (map[string]string, error) { + if len(emails) == 0 { + return make(map[string]string), nil + } + + verifiedFilter := 0 + if isVerifiedFilter { + verifiedFilter = 1 + } + + assoc := make(map[string]string) + + placeholders := make([]string, 0, len(emails)) + args := make([]any, 1, len(emails)+1) + + args[0] = verifiedFilter + for _, email := range emails { + if strings.HasPrefix(email, "did:") { + assoc[email] = email + continue + } + placeholders = append(placeholders, "?") + args = append(args, email) + } + + if len(placeholders) == 0 { + return assoc, nil + } + + query := ` + select email, did + from emails + where + verified = ? + and email in (` + strings.Join(placeholders, ",") + `) + ` + + rows, err := e.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var email, did string + if err := rows.Scan(&email, &did); err != nil { + return nil, err + } + assoc[email] = did + } + + return assoc, rows.Err() +} + +func GetVerifiedEmailAddresses(e Execer, did string) ([]string, error) { + rows, err := e.Query(` + select lower(email) + from emails + where did = ? and verified = 1 + order by lower(email) + limit 250`, did) + if err != nil { + return nil, err + } + defer rows.Close() + var addresses []string + for rows.Next() { + var address string + if err := rows.Scan(&address); err != nil { + return nil, err + } + addresses = append(addresses, address) + } + return addresses, rows.Err() +} + +func GetVerificationCodeForEmail(e Execer, did string, email string) (string, error) { + query := ` + select verification_code + from emails + where did = ? and email = ? + ` + var code string + err := e.QueryRow(query, did, email).Scan(&code) + if err != nil { + return "", err + } + return code, nil +} + +func CheckEmailExists(e Execer, did string, email string) (bool, error) { + query := ` + select count(*) + from emails + where did = ? and email = ? + ` + var count int + err := e.QueryRow(query, did, email).Scan(&count) + if err != nil { + return false, err + } + return count > 0, nil +} + +func CheckEmailExistsAtAll(e Execer, email string) (bool, error) { + query := ` + select count(*) + from emails + where email = ? + ` + var count int + err := e.QueryRow(query, email).Scan(&count) + if err != nil { + return false, err + } + return count > 0, nil +} + +func GetEmailByToken(e Execer, token string) (models.Email, error) { + query := ` + select did, email, verified, is_primary, verification_code + from emails + where verification_code = ? + limit 1 + ` + var email models.Email + err := e.QueryRow(query, token).Scan(&email.Did, &email.Address, &email.Verified, &email.Primary, &email.VerificationCode) + if err != nil { + return models.Email{}, err + } + return email, nil +} + +func AddEmail(e Execer, email models.Email) error { + countQuery := ` + select count(*) + from emails + where did = ? + ` + var count int + err := e.QueryRow(countQuery, email.Did).Scan(&count) + if err != nil { + return err + } + + if count == 0 { + email.Primary = true + } + + query := ` + insert into emails (did, email, verified, is_primary, verification_code) + values (?, ?, ?, ?, ?) + ` + _, err = e.Exec(query, email.Did, email.Address, email.Verified, email.Primary, email.VerificationCode) + return err +} + +func DeleteEmail(e Execer, did string, email string) error { + query := ` + delete from emails + where did = ? and email = ? + ` + _, err := e.Exec(query, did, email) + return err +} + +func DeleteAccountEmails(e Execer, did string) ([]string, error) { + rows, err := e.Query(`delete from emails where did = ? returning email`, did) + if err != nil { + return nil, err + } + defer rows.Close() + var emails []string + for rows.Next() { + var email string + if err := rows.Scan(&email); err != nil { + return nil, err + } + emails = append(emails, email) + } + return emails, rows.Err() +} + +func MarkEmailVerified(e Execer, did string, email string) error { + query := ` + update emails + set verified = true + where did = ? and email = ? + ` + _, err := e.Exec(query, did, email) + return err +} + +func MakeEmailPrimary(e Execer, did string, email string) error { + query1 := ` + update emails + set is_primary = false + where did = ? + ` + _, err := e.Exec(query1, did) + if err != nil { + return err + } + + query2 := ` + update emails + set is_primary = true + where did = ? and email = ? + ` + _, err = e.Exec(query2, did, email) + return err +} + +func GetAllEmails(e Execer, did string) ([]models.Email, error) { + query := ` + select did, email, verified, is_primary, verification_code, last_sent, created + from emails + where did = ? + ` + rows, err := e.Query(query, did) + if err != nil { + return nil, err + } + defer rows.Close() + + var emails []models.Email + for rows.Next() { + var email models.Email + var createdStr string + var lastSent string + err := rows.Scan(&email.Did, &email.Address, &email.Verified, &email.Primary, &email.VerificationCode, &lastSent, &createdStr) + if err != nil { + return nil, err + } + email.CreatedAt, err = time.Parse(time.RFC3339, createdStr) + if err != nil { + return nil, err + } + parsedTime, err := time.Parse(time.RFC3339, lastSent) + if err != nil { + return nil, err + } + email.LastSent = &parsedTime + emails = append(emails, email) + } + return emails, nil +} + +func InsertUnverifiedEmail(e Execer, did string, email string, code string) error { + query := ` + insert into emails (did, email, verified, is_primary, verification_code) + values (?, ?, 0, 0, ?) + ` + _, err := e.Exec(query, did, email, code) + return err +} + +func CountVerifiedEmails(e Execer, did string) (int, error) { + query := ` + select count(*) + from emails + where did = ? and verified = true + ` + var count int + err := e.QueryRow(query, did).Scan(&count) + return count, err +} + +func UpdateVerificationCode(e Execer, did string, email string, code string) error { + query := ` + update emails + set verification_code = ?, + last_sent = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') + where did = ? and email = ? + ` + _, err := e.Exec(query, code, did, email) + return err +} diff --git a/deliberi/db/ingest.go b/deliberi/db/ingest.go new file mode 100644 index 0000000000000000000000000000000000000000..8d20f8a4b0cb4067ffa2e23f2fad415e0580f591 --- /dev/null +++ b/deliberi/db/ingest.go @@ -0,0 +1,180 @@ +package db + +import ( + "database/sql" + "errors" + "strings" +) + +// tids sort as strings, so > is enough +func RevIsNew(e Execer, uri, rev string) bool { + var stored string + _ = e.QueryRow(`select rev from record_revs where uri = ?`, uri).Scan(&stored) + return rev > stored +} + +func PutRev(e Execer, uri, rev string) error { + _, err := e.Exec( + `insert into record_revs (uri, rev) values (?, ?) + on conflict(uri) do update set rev = excluded.rev`, + uri, rev, + ) + return err +} + +// the rev is written with its notices, so having one means they landed +// sql.ErrNoRows means the record was never ingested +func GetIngested(e Execer, uri string) (rev, cid string, err error) { + if err := e.QueryRow(`select rev from record_revs where uri = ?`, uri).Scan(&rev); err != nil { + return "", "", err + } + // no entity row just means the record has no cid + if err := e.QueryRow(`select cid from entity_titles where at_uri = ?`, uri).Scan(&cid); err != nil && !errors.Is(err, sql.ErrNoRows) { + return "", "", err + } + return rev, cid, nil +} + +func PutCollaborator(e Execer, repoDid, did string) error { + _, err := e.Exec(`insert or ignore into repo_collaborators (repo_did, did) values (?, ?)`, repoDid, did) + return err +} + +func DeleteCollaborator(e Execer, repoDid, did string) error { + _, err := e.Exec(`delete from repo_collaborators where repo_did = ? and did = ?`, repoDid, did) + return err +} + +func ListCollaborators(e Execer, repoDid string) ([]string, error) { + return queryStrings(e, `select did from repo_collaborators where repo_did = ?`, repoDid) +} + +type Comment struct { + Uri string + TicketUri string + ReplyTo string + Author string +} + +func PutComment(e Execer, c Comment) error { + root := c.Uri + if c.ReplyTo != "" { + root = c.ReplyTo + var parentRoot string + if err := e.QueryRow(`select root from comments where uri = ?`, c.ReplyTo).Scan(&parentRoot); err == nil { + root = parentRoot + } + } + _, err := e.Exec( + `insert into comments (uri, ticket_uri, root, author_did) values (?, ?, ?, ?) + on conflict(uri) do nothing`, + c.Uri, c.TicketUri, root, c.Author, + ) + return err +} + +func DeleteComment(e Execer, uri string) error { + _, err := e.Exec(`delete from comments where uri = ?`, uri) + return err +} + +// the parent may be the root itself, which has no comments row +func ThreadParticipants(e Execer, parentUri string) ([]string, error) { + return queryStrings(e, + `select distinct author_did from comments + where root = coalesce((select root from comments where uri = ?), ?)`, + parentUri, parentUri, + ) +} + +func TicketParticipants(e Execer, ticketUri string) ([]string, error) { + return queryStrings(e, + `select distinct author_did from comments + where ticket_uri = ? or ticket_uri in (select uri from patches where ticket_uri = ?) + union select author_did from entity_titles where at_uri = ? and author_did != ''`, + ticketUri, ticketUri, ticketUri, + ) +} + +func PutPatch(e Execer, uri, ticketUri string) error { + _, err := e.Exec( + `insert into patches (uri, ticket_uri) values (?, ?) + on conflict(uri) do update set ticket_uri = excluded.ticket_uri`, + uri, ticketUri, + ) + return err +} + +func DeletePatch(e Execer, uri string) error { + _, err := e.Exec(`delete from patches where uri = ?`, uri) + return err +} + +func GetPatchTicket(e Execer, uri string) string { + var ticketUri string + _ = e.QueryRow(`select ticket_uri from patches where uri = ?`, uri).Scan(&ticketUri) + return ticketUri +} + +func PutSubscription(e Execer, author, rkey, subject string, ignore bool) error { + _, err := e.Exec( + `insert into subscriptions (author_did, rkey, subject, ignore) values (?, ?, ?, ?) + on conflict(author_did, rkey) do update set subject = excluded.subject, ignore = excluded.ignore`, + author, rkey, subject, ignore, + ) + return err +} + +func DeleteSubscription(e Execer, author, rkey string) error { + _, err := e.Exec(`delete from subscriptions where author_did = ? and rkey = ?`, author, rkey) + return err +} + +// max rkey is the newest since tids sort by time +func ListSubscribers(e Execer, subjects ...string) (subscribers, ignorers []string, err error) { + ph := strings.TrimSuffix(strings.Repeat("?, ", len(subjects)), ", ") + args := make([]any, len(subjects)) + for i, s := range subjects { + args[i] = s + } + rows, err := e.Query(` + select author_did, ignore from subscriptions s + where subject in (`+ph+`) + and rkey = (select max(rkey) from subscriptions where author_did = s.author_did and subject = s.subject)`, + args..., + ) + if err != nil { + return nil, nil, err + } + defer rows.Close() + for rows.Next() { + var did string + var ignore bool + if err := rows.Scan(&did, &ignore); err != nil { + return nil, nil, err + } + if ignore { + ignorers = append(ignorers, did) + } else { + subscribers = append(subscribers, did) + } + } + return subscribers, ignorers, rows.Err() +} + +func queryStrings(e Execer, query string, args ...any) ([]string, error) { + rows, err := e.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var s string + if err := rows.Scan(&s); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} diff --git a/deliberi/db/newsletter.go b/deliberi/db/newsletter.go new file mode 100644 index 0000000000000000000000000000000000000000..700cf11927881ce2ab277dc3104cf0245f98a7e6 --- /dev/null +++ b/deliberi/db/newsletter.go @@ -0,0 +1,52 @@ +package db + +import ( + "database/sql" + "errors" + "fmt" +) + +// both hide the signup prompt; kept apart so dismissed-but-unsubscribed users +// can be re-prompted or reconciled with resend later +const ( + NewsletterStatusSubscribed = "subscribed" + NewsletterStatusDismissed = "dismissed" +) + +// an empty email keeps whatever address the row already holds +func UpsertNewsletterPref(e Execer, did, status, email string) error { + if status != NewsletterStatusSubscribed && status != NewsletterStatusDismissed { + return fmt.Errorf("invalid newsletter status %q", status) + } + + var emailArg any + if email != "" { + emailArg = email + } + + _, err := e.Exec( + `insert into newsletter_preferences (user_did, status, email, updated_at) + values (?, ?, ?, strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) + on conflict(user_did) do update set + status = excluded.status, + email = coalesce(excluded.email, newsletter_preferences.email), + updated_at = excluded.updated_at`, + did, + status, + emailArg, + ) + if err != nil { + return fmt.Errorf("upsert newsletter pref: %w", err) + } + return nil +} + +// "" when the account has no row +func GetNewsletterStatus(e Execer, did string) (string, error) { + var status string + err := e.QueryRow(`select status from newsletter_preferences where user_did = ?`, did).Scan(&status) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return status, err +} diff --git a/deliberi/db/notifications.go b/deliberi/db/notifications.go new file mode 100644 index 0000000000000000000000000000000000000000..16f687779f35891947e2fe80de036262940798fa --- /dev/null +++ b/deliberi/db/notifications.go @@ -0,0 +1,260 @@ +package db + +import ( + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/deliberi/models" + "tangled.org/core/orm" +) + +// CreateNotification inserts a row, deduped on (recipient_did, at_uri). +func CreateNotification(e Execer, n *models.Notification) error { + if n.Created.IsZero() { + n.Created = time.Now().UTC() + } + query := ` + insert into notifications + (recipient_did, at_uri, type, actor_did, repo_did, knot_did, entity_at, entity_kind, entity_title, read, created) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + on conflict(recipient_did, at_uri) do nothing + ` + res, err := e.Exec(query, + n.RecipientDid, n.AtUri, string(n.Type), n.ActorDid, + n.RepoDid, n.KnotDid, n.EntityAt, string(n.EntityKind), n.EntityTitle, n.Read, + n.Created.UTC().Format(time.RFC3339), + ) + if err != nil { + return fmt.Errorf("failed to create notification: %w", err) + } + if id, err := res.LastInsertId(); err == nil { + n.ID = id + } + return nil +} + +const notifCols = `id, recipient_did, at_uri, type, actor_did, repo_did, knot_did, entity_at, entity_kind, entity_title, read, emailed, created` + +func scanNotification(rows interface{ Scan(...any) error }) (*models.Notification, error) { + var n models.Notification + var typeStr, kindStr, createdStr string + if err := rows.Scan( + &n.ID, &n.RecipientDid, &n.AtUri, &typeStr, &n.ActorDid, + &n.RepoDid, &n.KnotDid, &n.EntityAt, &kindStr, &n.EntityTitle, &n.Read, &n.Emailed, &createdStr, + ); err != nil { + return nil, err + } + n.Type = models.NotificationType(typeStr) + n.EntityKind = models.EntityKind(kindStr) + n.Created, _ = time.Parse(time.RFC3339, createdStr) + return &n, nil +} + +func GetNotifications(e Execer, recipientDid string, limit int, filters ...orm.Filter) ([]*models.Notification, error) { + conds := []string{"recipient_did = ?"} + args := []any{recipientDid} + for _, f := range filters { + conds = append(conds, f.Condition()) + args = append(args, f.Arg()...) + } + query := fmt.Sprintf("select %s from notifications where %s order by created desc", notifCols, strings.Join(conds, " and ")) + if limit > 0 { + query += fmt.Sprintf(" limit %d", limit) + } + rows, err := e.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []*models.Notification + for rows.Next() { + n, err := scanNotification(rows) + if err != nil { + return nil, err + } + out = append(out, n) + } + return out, rows.Err() +} + +func CountNotifications(e Execer, recipientDid string, filters ...orm.Filter) (int64, error) { + conds := []string{"recipient_did = ?"} + args := []any{recipientDid} + for _, f := range filters { + conds = append(conds, f.Condition()) + args = append(args, f.Arg()...) + } + query := fmt.Sprintf("select count(*) from notifications where %s", strings.Join(conds, " and ")) + var count int64 + if err := e.QueryRow(query, args...).Scan(&count); err != nil { + return 0, err + } + return count, nil +} + +func MarkRead(e Execer, recipientDid, atUri string, read bool) error { + _, err := e.Exec(`update notifications set read = ? where recipient_did = ? and at_uri = ?`, read, recipientDid, atUri) + return err +} + +func DeleteNotification(e Execer, recipientDid, atUri string) error { + _, err := e.Exec(`delete from notifications where recipient_did = ? and at_uri = ?`, recipientDid, atUri) + return err +} + +func MarkAllRead(e Execer, recipientDid string) error { + _, err := e.Exec(`update notifications set read = 1 where recipient_did = ? and read = 0`, recipientDid) + return err +} + +// state changes key on a rev, not the record uri, so match them by entity uri +func DeleteNotificationsAbout(e Execer, entityAt string) error { + _, err := e.Exec(`delete from notifications where entity_at = ?`, entityAt) + return err +} + +func DeleteNotificationsFrom(e Execer, atUri string) error { + _, err := e.Exec(`delete from notifications where at_uri = ?`, atUri) + return err +} + +func MarkEntityRead(e Execer, recipientDid, entityAt string) error { + _, err := e.Exec(`update notifications set read = 1 where recipient_did = ? and entity_at = ? and read = 0`, recipientDid, entityAt) + return err +} + +func MarkEmailed(e Execer, ids []int64) error { + if len(ids) == 0 { + return nil + } + ph := make([]string, len(ids)) + args := make([]any, len(ids)) + for i, id := range ids { + ph[i] = "?" + args[i] = id + } + _, err := e.Exec(fmt.Sprintf(`update notifications set emailed = 1 where id in (%s)`, strings.Join(ph, ", ")), args...) + return err +} + +func GetPendingEmailDigestRecipients(e Execer, olderThan time.Time) ([]string, error) { + ph := make([]string, len(models.EmailNotificationTypes)) + args := []any{olderThan.UTC().Format(time.RFC3339)} + for i, t := range models.EmailNotificationTypes { + ph[i] = "?" + args = append(args, string(t)) + } + query := fmt.Sprintf(` + select distinct n.recipient_did + from notifications n + join notification_preferences np on np.user_did = n.recipient_did + join emails em on em.did = n.recipient_did and em.is_primary = 1 and em.verified = 1 + where n.emailed = 0 and n.read = 0 and n.created < ? + and np.email_notifications = 1 + and n.type in (%s) + `, strings.Join(ph, ", ")) + rows, err := e.Query(query, args...) + if err != nil { + return nil, fmt.Errorf("failed to query digest recipients: %w", err) + } + defer rows.Close() + var dids []string + for rows.Next() { + var did string + if err := rows.Scan(&did); err != nil { + return nil, err + } + dids = append(dids, did) + } + return dids, rows.Err() +} + +func GetPendingNotificationsForEmailDigest(e Execer, recipientDid string, olderThan time.Time) ([]*models.Notification, error) { + ph := make([]string, len(models.EmailNotificationTypes)) + args := []any{recipientDid, olderThan.UTC().Format(time.RFC3339)} + for i, t := range models.EmailNotificationTypes { + ph[i] = "?" + args = append(args, string(t)) + } + query := fmt.Sprintf(` + select %s from notifications + where recipient_did = ? and emailed = 0 and read = 0 and created < ? + and type in (%s) + order by created desc + `, notifCols, strings.Join(ph, ", ")) + rows, err := e.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []*models.Notification + for rows.Next() { + n, err := scanNotification(rows) + if err != nil { + return nil, err + } + // repo names are not stored on the row: resolve them here, where the + // digest needs them, so a rename shows up in the next email. + n.RepoName = GetRepoName(e, n.RepoDid) + n.RepoOwner = GetRepoOwner(e, n.RepoDid) + out = append(out, n) + } + return out, rows.Err() +} + +func GetNotificationPreference(e Execer, userDid string) (*models.NotificationPreferences, error) { + query := ` + select id, user_did, repo_starred, issue_created, issue_commented, pull_created, + pull_commented, followed, user_mentioned, email_notifications, issue_closed, pull_merged + from notification_preferences + where user_did = ? + ` + var p models.NotificationPreferences + var userDidStr string + err := e.QueryRow(query, userDid).Scan( + &p.ID, &userDidStr, &p.RepoStarred, &p.IssueCreated, &p.IssueCommented, &p.PullCreated, + &p.PullCommented, &p.Followed, &p.UserMentioned, &p.EmailNotifications, &p.IssueClosed, &p.PullMerged, + ) + if errors.Is(err, sql.ErrNoRows) { + // no row yet: defaults so reads never fail for an uncustomized user + return models.DefaultNotificationPreferences(syntax.DID(userDid)), nil + } + if err != nil { + // a real failure must not read as "user wants everything": callers + // deliver on these prefs, so opting out has to survive a db error. + return nil, fmt.Errorf("failed to query notification preferences: %w", err) + } + p.UserDid = syntax.DID(userDidStr) + return &p, nil +} + +func UpsertNotificationPreferences(e Execer, prefs *models.NotificationPreferences) error { + query := ` + insert into notification_preferences + (user_did, repo_starred, issue_created, issue_commented, pull_created, + pull_commented, followed, user_mentioned, email_notifications, issue_closed, pull_merged) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + on conflict(user_did) do update set + repo_starred = excluded.repo_starred, + issue_created = excluded.issue_created, + issue_commented = excluded.issue_commented, + pull_created = excluded.pull_created, + pull_commented = excluded.pull_commented, + followed = excluded.followed, + user_mentioned = excluded.user_mentioned, + email_notifications = excluded.email_notifications, + issue_closed = excluded.issue_closed, + pull_merged = excluded.pull_merged + ` + _, err := e.Exec(query, + prefs.UserDid.String(), + prefs.RepoStarred, prefs.IssueCreated, prefs.IssueCommented, prefs.PullCreated, + prefs.PullCommented, prefs.Followed, prefs.UserMentioned, prefs.EmailNotifications, + prefs.IssueClosed, prefs.PullMerged, + ) + return err +} diff --git a/deliberi/db/notifications_test.go b/deliberi/db/notifications_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8976df25f8d26c42907f08c23b5cb2cab209cb45 --- /dev/null +++ b/deliberi/db/notifications_test.go @@ -0,0 +1,187 @@ +package db + +import ( + "context" + "path/filepath" + "testing" + "time" + + "tangled.org/core/deliberi/models" + "tangled.org/core/orm" +) + +func testDB(t *testing.T) *DB { + t.Helper() + d, err := Make(context.Background(), filepath.Join(t.TempDir(), "x.db")) + if err != nil { + t.Fatalf("Make: %v", err) + } + t.Cleanup(func() { d.Close() }) + return d +} + +// one source record fans out to a row per recipient; read state is per row +// (recipient_did, at_uri), so marking one recipient's row read leaves another +// recipient's untouched. +func TestNotificationsIsolatedPerRecipient(t *testing.T) { + d := testDB(t) + const uri = "at://did:plc:author/sh.tangled.feed.comment/abc" + + for _, did := range []string{"did:plc:alice", "did:plc:bob"} { + if err := CreateNotification(d, &models.Notification{ + RecipientDid: did, + AtUri: uri, + Type: models.NotificationTypeIssueCommented, + ActorDid: "did:plc:author", + }); err != nil { + t.Fatalf("CreateNotification %s: %v", did, err) + } + } + + if err := MarkRead(d, "did:plc:alice", uri, true); err != nil { + t.Fatalf("MarkRead: %v", err) + } + + aliceUnread, _ := CountNotifications(d, "did:plc:alice", orm.FilterEq("read", 0)) + bobUnread, _ := CountNotifications(d, "did:plc:bob", orm.FilterEq("read", 0)) + if aliceUnread != 0 { + t.Fatalf("alice unread = %d, want 0", aliceUnread) + } + if bobUnread != 1 { + t.Fatalf("bob unread = %d, want 1 (must not inherit alice's read)", bobUnread) + } +} + +// CreateNotification dedupes on (recipient_did, at_uri): a firehose replay or a +// user both subscribed and mentioned yields a single row. +func TestCreateNotificationDedupe(t *testing.T) { + d := testDB(t) + const uri = "at://did:plc:repo/sh.tangled.repo.issue/abc" + + for range 2 { + if err := CreateNotification(d, &models.Notification{ + RecipientDid: "did:plc:alice", + AtUri: uri, + Type: models.NotificationTypeIssueCreated, + ActorDid: "did:plc:author", + }); err != nil { + t.Fatalf("CreateNotification: %v", err) + } + } + + count, _ := CountNotifications(d, "did:plc:alice") + if count != 1 { + t.Fatalf("row count = %d, want 1 (deduped)", count) + } +} + +const inviteUri = "at://did:plc:scallop/sh.tangled.repo.collaboratorInvite/did:plc:alice" + +func inviteOffer(recipientDid string) *models.Notification { + return &models.Notification{ + RecipientDid: recipientDid, + AtUri: inviteUri, + Type: models.NotificationTypeCollaboratorInvited, + ActorDid: "did:plc:admin", + RepoDid: "did:plc:scallop", + KnotDid: "did:web:knot.example", + } +} + +func TestDeleteNotificationIsolatedPerRecipient(t *testing.T) { + d := testDB(t) + + for _, did := range []string{"did:plc:alice", "did:plc:bob"} { + if err := CreateNotification(d, inviteOffer(did)); err != nil { + t.Fatalf("CreateNotification %s: %v", did, err) + } + } + + if err := DeleteNotification(d, "did:plc:alice", inviteUri); err != nil { + t.Fatalf("DeleteNotification: %v", err) + } + + aliceRows, _ := CountNotifications(d, "did:plc:alice") + bobRows, _ := CountNotifications(d, "did:plc:bob") + if aliceRows != 0 { + t.Fatalf("alice rows = %d, want 0", aliceRows) + } + if bobRows != 1 { + t.Fatalf("bob rows = %d, want 1 (alice's delete mustn't reach bob)", bobRows) + } +} + +func TestCreateAfterDeleteInserts(t *testing.T) { + d := testDB(t) + + if err := CreateNotification(d, inviteOffer("did:plc:alice")); err != nil { + t.Fatalf("first CreateNotification: %v", err) + } + if err := DeleteNotification(d, "did:plc:alice", inviteUri); err != nil { + t.Fatalf("DeleteNotification: %v", err) + } + if err := CreateNotification(d, inviteOffer("did:plc:alice")); err != nil { + t.Fatalf("second CreateNotification: %v", err) + } + + rows, err := GetNotifications(d, "did:plc:alice", 0) + if err != nil { + t.Fatalf("GetNotifications: %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d, want 1 (re-offer must insert)", len(rows)) + } + if rows[0].KnotDid != "did:web:knot.example" { + t.Fatalf("knotDid = %q, want did:web:knot.example", rows[0].KnotDid) + } +} + +func TestCreateNotificationKeepsSuppliedCreated(t *testing.T) { + d := testDB(t) + made := time.Date(2026, 8, 24, 9, 30, 15, 0, time.UTC) + + offer := inviteOffer("did:plc:alice") + offer.Created = made + if err := CreateNotification(d, offer); err != nil { + t.Fatalf("CreateNotification: %v", err) + } + + rows, err := GetNotifications(d, "did:plc:alice", 0) + if err != nil { + t.Fatalf("GetNotifications: %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d, want 1", len(rows)) + } + if !rows[0].Created.Equal(made) { + t.Fatalf("created = %s, want %s (offer's own timestamp)", rows[0].Created, made) + } +} + +func TestDigestNotificationsCarryTheRepoOwner(t *testing.T) { + d := testDB(t) + if err := PutRepoName(d, "did:plc:repo", "did:plc:owner", "core"); err != nil { + t.Fatalf("PutRepoName: %v", err) + } + if err := CreateNotification(d, &models.Notification{ + RecipientDid: "did:plc:alice", + AtUri: "at://did:plc:repo/org.tangled.track.ticket/749", + Type: models.NotificationTypeIssueCreated, + ActorDid: "did:plc:author", + RepoDid: "did:plc:repo", + Created: time.Date(2026, 8, 24, 9, 30, 15, 0, time.UTC), + }); err != nil { + t.Fatalf("CreateNotification: %v", err) + } + + rows, err := GetPendingNotificationsForEmailDigest(d, "did:plc:alice", time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatalf("GetPendingNotificationsForEmailDigest: %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d, want 1", len(rows)) + } + if rows[0].RepoOwner != "did:plc:owner" || rows[0].RepoName != "core" { + t.Fatalf("repo owner %q name %q, want did:plc:owner core", rows[0].RepoOwner, rows[0].RepoName) + } +} diff --git a/deliberi/db/signup.go b/deliberi/db/signup.go new file mode 100644 index 0000000000000000000000000000000000000000..8d0ccf02422d62d7c884572f09354054ce2ed170 --- /dev/null +++ b/deliberi/db/signup.go @@ -0,0 +1,51 @@ +package db + +import ( + "tangled.org/core/deliberi/models" +) + +func AddInflightSignup(e Execer, signup models.InflightSignup) error { + // strftime because migrated DBs keep constant '' column defaults + query := `insert or replace into signups_inflight (email, invite_code, verification_token, expires_at, last_sent) + values (?, ?, ?, strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '+24 hours'), strftime('%Y-%m-%dT%H:%M:%SZ', 'now'))` + _, err := e.Exec(query, signup.Email, signup.InviteCode, signup.VerificationToken) + return err +} + +func GetInflightByToken(e Execer, token string) (models.InflightSignup, error) { + query := `select email, invite_code, verification_token, last_sent + from signups_inflight + where verification_token = ? and expires_at > strftime('%Y-%m-%dT%H:%M:%SZ', 'now')` + var s models.InflightSignup + err := e.QueryRow(query, token).Scan(&s.Email, &s.InviteCode, &s.VerificationToken, &s.LastSent) + return s, err +} + +// GetInflightByEmail returns the live (unexpired) row; sql.ErrNoRows when none. +func GetInflightByEmail(e Execer, email string) (models.InflightSignup, error) { + query := `select email, invite_code, verification_token, last_sent + from signups_inflight + where email = ? and expires_at > strftime('%Y-%m-%dT%H:%M:%SZ', 'now')` + var s models.InflightSignup + err := e.QueryRow(query, email).Scan(&s.Email, &s.InviteCode, &s.VerificationToken, &s.LastSent) + return s, err +} + +func UpdateInflightLastSent(e Execer, email string) error { + _, err := e.Exec(`update signups_inflight set last_sent = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') where email = ?`, email) + return err +} + +// DeleteExpiredInflightSignups frees the PDS invite codes of stale signups; +// handlers call it on a lookup miss. +func DeleteExpiredInflightSignups(e Execer) error { + _, err := e.Exec(`delete from signups_inflight + where expires_at <> '' and expires_at <= strftime('%Y-%m-%dT%H:%M:%SZ', 'now')`) + return err +} + +func DeleteInflightSignup(e Execer, email string) error { + query := `delete from signups_inflight where email = ?` + _, err := e.Exec(query, email) + return err +} diff --git a/deliberi/db/signup_test.go b/deliberi/db/signup_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d203f829bb2899c71ad7d2a91539c4c0d04ef57d --- /dev/null +++ b/deliberi/db/signup_test.go @@ -0,0 +1,111 @@ +package db + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing" + + "tangled.org/core/deliberi/models" +) + +func TestMakeUpgradesLegacySignupsInflight(t *testing.T) { + path := filepath.Join(t.TempDir(), "legacy.db") + + legacy, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + if _, err := legacy.Exec(`create table signups_inflight ( + id integer primary key autoincrement, + email text not null unique, + invite_code text not null, + created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) + )`); err != nil { + t.Fatalf("create legacy table: %v", err) + } + if _, err := legacy.Exec(`insert into signups_inflight (email, invite_code) values + ('a@example.com', 'code-a'), + ('b@example.com', 'code-b')`); err != nil { + t.Fatalf("seed legacy rows: %v", err) + } + if err := legacy.Close(); err != nil { + t.Fatalf("close legacy db: %v", err) + } + + d, err := Make(context.Background(), path) + if err != nil { + t.Fatalf("Make on legacy db: %v", err) + } + defer d.Close() + + rows, err := d.Query(`select email, verification_token, expires_at, last_sent from signups_inflight order by email`) + if err != nil { + t.Fatalf("query upgraded rows: %v", err) + } + defer rows.Close() + + type row struct { + email, token, expires, lastSent string + } + var got []row + for rows.Next() { + var r row + if err := rows.Scan(&r.email, &r.token, &r.expires, &r.lastSent); err != nil { + t.Fatalf("scan: %v", err) + } + got = append(got, r) + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + if len(got) != 2 { + t.Fatalf("row count = %d, want 2", len(got)) + } + tokens := map[string]bool{} + for _, r := range got { + if len(r.token) != 64 { + t.Errorf("token %q for %s not 64 chars", r.token, r.email) + } + if r.expires == "" { + t.Errorf("expires_at empty for %s", r.email) + } + if r.lastSent == "" { + t.Errorf("last_sent empty for %s", r.email) + } + if tokens[r.token] { + t.Errorf("duplicate token %q", r.token) + } + tokens[r.token] = true + } + + var indexName string + err = d.QueryRow(`select name from sqlite_master + where type = 'index' and name = 'signups_inflight_verification_token_unique'`).Scan(&indexName) + if err != nil { + t.Fatalf("unique index missing after upgrade: %v", err) + } + if !strings.Contains(indexName, "verification_token_unique") { + t.Fatalf("unexpected index name %q", indexName) + } +} + +func TestMakeFreshSignupsInflight(t *testing.T) { + d, err := Make(context.Background(), filepath.Join(t.TempDir(), "fresh.db")) + if err != nil { + t.Fatalf("Make: %v", err) + } + defer d.Close() + + if err := AddInflightSignup(d, models.InflightSignup{Email: "c@example.com", InviteCode: "code-c", VerificationToken: strings.Repeat("ab", 32)}); err != nil { + t.Fatalf("AddInflightSignup: %v", err) + } + row, err := GetInflightByEmail(d, "c@example.com") + if err != nil { + t.Fatalf("GetInflightByEmail: %v", err) + } + if row.LastSent == "" { + t.Errorf("last_sent not set on insert") + } +} diff --git a/deliberi/deliberi.go b/deliberi/deliberi.go new file mode 100644 index 0000000000000000000000000000000000000000..5c54b70da5e78b9a9b162587b2d19ecca32663a0 --- /dev/null +++ b/deliberi/deliberi.go @@ -0,0 +1,104 @@ +package deliberi + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/go-chi/chi/v5" + "tangled.org/core/deliberi/analytics" + "tangled.org/core/deliberi/config" + deldb "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/kv" + "tangled.org/core/deliberi/mailer" + delxrpc "tangled.org/core/deliberi/xrpc" + "tangled.org/core/idresolver" + "tangled.org/core/log" + "tangled.org/core/xrpc/serviceauth" +) + +func Run(ctx context.Context, cfg *config.Config) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + logger := log.FromContext(ctx) + + database, err := deldb.Make(ctx, cfg.DbPath) + if err != nil { + return fmt.Errorf("initializing db: %w", err) + } + + resolver := idresolver.DefaultResolver(cfg.PlcUrl) + + events := analytics.New(cfg.Posthog, log.SubLogger(logger, "analytics")) + defer events.Close() + + var emailKV kv.EmailKV + if cfg.KvApiToken != "" && cfg.CloudflareAccountID != "" && cfg.KvNamespaceID != "" { + emailKV = kv.NewClient(cfg.KvApiToken, cfg.CloudflareAccountID, cfg.KvNamespaceID, + log.SubLogger(logger, "kv")) + } + + bobbin := newBobbinClient(cfg.BobbinApiUrl) + ingester := NewIngester(database, bobbin, resolver, cfg.JetstreamEndpoint, log.SubLogger(logger, "ingest")) + ingester.analytics = events + ingester.emailKV = emailKV + ingestDone := make(chan struct{}) + go func() { + defer close(ingestDone) + if err := ingester.Run(ctx); err != nil { + logger.Error("ingester stopped", "err", err) + cancel() + } + }() + + sender := mailer.New(cfg.Resend, log.SubLogger(logger, "email")) + dispatcher := NewDispatcher(database, sender, cfg.Resend, cfg.BaseURL, resolver, log.SubLogger(logger, "digest"), cfg.Dev) + go dispatcher.Start(ctx) + + serviceAuth := serviceauth.NewServiceAuth(logger, resolver.Directory(), serviceauth.DidWeb(cfg.Hostname).String()) + x := &delxrpc.Xrpc{ + DB: database, + Config: cfg, + Logger: log.SubLogger(logger, "xrpc"), + ServiceAuth: serviceAuth, + IdResolver: resolver, + Sender: sender, + Invites: NewInviteSync(database, bobbin, log.SubLogger(logger, "invites")), + Analytics: events, + KV: emailKV, + } + + srv := &http.Server{Addr: cfg.ListenAddr, Handler: chiMount(x)} + go func() { + logger.Info("starting http server", "addr", cfg.ListenAddr) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + logger.Error("http server failed", "err", err) + cancel() + } + }() + + logger.Info("startup complete") + <-ctx.Done() + logger.Info("received shutdown signal", "reason", ctx.Err()) + + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer shutdownCancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + logger.Error("http shutdown", "err", err) + } + // ingest has to finish its transaction before the db closes + <-ingestDone + if err := database.Close(); err != nil { + logger.Error("db close", "err", err) + } + logger.Info("shutdown complete") + return nil +} + +func chiMount(x *delxrpc.Xrpc) http.Handler { + mux := chi.NewRouter() + mux.Mount("/xrpc", x.Router()) + return mux +} diff --git a/deliberi/digest.go b/deliberi/digest.go new file mode 100644 index 0000000000000000000000000000000000000000..2a8fc010c8f1dbafae9ecd5447096528d65568a4 --- /dev/null +++ b/deliberi/digest.go @@ -0,0 +1,330 @@ +package deliberi + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "html/template" + "log/slog" + "slices" + "strings" + gotemplate "text/template" + "time" + + "tangled.org/core/deliberi/config" + deldb "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/emailtemplate" + "tangled.org/core/deliberi/mailer" + "tangled.org/core/deliberi/models" + "tangled.org/core/idresolver" +) + +const threadTextTmpl = `Hi {{.RecipientHandle}}, + +New activity on {{wrap 70 "" .Title}}{{if .Repo}} in {{.Repo}}{{end}}: + +{{range .Events}}- {{wrap 70 " " .Header}} +{{end}} +View it on Tangled: {{.URL}} + +--- +Manage notification settings: {{.SettingsURL}} +` + +const threadBodyHTMLTmpl = `

Hi {{.RecipientHandle}},

+

New activity on {{.Title}}{{if .Repo}} in {{.Repo}}{{end}}:

+ + {{range .Events}} + + + + {{end}} +
+ + + + + +
+ + +

{{.Actor}} {{.Phrase}}

+
+
+

+ View it on Tangled +

+

+ Manage notification settings +

` + +type threadEvent struct { + IconURL string + Actor string + Phrase string + Header string +} + +type threadData struct { + RecipientHandle string + Title string + Repo string + Events []threadEvent + URL string + SettingsURL string +} + +type threadEmail struct { + subject, text, html string + headers map[string]string +} + +func notifPhrase(n *models.Notification) string { + switch n.Type { + case models.NotificationTypeIssueCreated: + return "opened this issue" + case models.NotificationTypePullCreated: + return "opened this pull request" + case models.NotificationTypeIssueCommented, models.NotificationTypePullCommented: + return "commented" + case models.NotificationTypeIssueClosed: + return "closed this issue" + case models.NotificationTypeIssueReopen: + return "reopened this issue" + case models.NotificationTypePullMerged: + return "merged this pull request" + case models.NotificationTypePullClosed: + return "closed this pull request" + case models.NotificationTypePullReopen: + return "reopened this pull request" + case models.NotificationTypeUserMentioned: + return "mentioned you" + default: + return "updated this" + } +} + +func wordwrap(width int, indent, text string) string { + words := strings.Fields(text) + if len(words) == 0 { + return text + } + var b strings.Builder + col := 0 + for i, w := range words { + if i == 0 { + b.WriteString(w) + col = len(w) + continue + } + if col+1+len(w) > width { + b.WriteString("\n" + indent) + b.WriteString(w) + col = len(indent) + len(w) + } else { + b.WriteByte(' ') + b.WriteString(w) + col += 1 + len(w) + } + } + return b.String() +} + +// every email about one issue or pull points at the same root id, which mail +// clients thread on. the root itself is never sent, so no id is ever reused +func threadRoot(entityAt, domain string) string { + sum := sha256.Sum256([]byte(entityAt)) + return "" +} + +type emailSender interface { + Send(mailer.Message) error + Domain() string +} + +type Dispatcher struct { + db *deldb.DB + sender emailSender + baseURL string + assetsURL string + resolver *idresolver.Resolver + logger *slog.Logger + batchWait time.Duration + interval time.Duration + + textTmpl *gotemplate.Template + htmlTmpl *template.Template +} + +func NewDispatcher(database *deldb.DB, sender emailSender, resend config.ResendConfig, baseURL string, resolver *idresolver.Resolver, logger *slog.Logger, dev bool) *Dispatcher { + batchWait := 10 * time.Minute + interval := 5 * time.Minute + if dev { + batchWait = 30 * time.Second + interval = 15 * time.Second + } + return &Dispatcher{ + db: database, + sender: sender, + baseURL: strings.TrimRight(baseURL, "/"), + assetsURL: strings.TrimRight(resend.AssetsURL, "/") + "/", + resolver: resolver, + logger: logger, + batchWait: batchWait, + interval: interval, + textTmpl: gotemplate.Must(gotemplate.New("thread-text").Funcs(gotemplate.FuncMap{"wrap": wordwrap}).Parse(threadTextTmpl)), + htmlTmpl: template.Must(template.New("thread-body").Parse(threadBodyHTMLTmpl)), + } +} + +func (d *Dispatcher) Start(ctx context.Context) { + d.logger.Info("email dispatcher started", "interval", d.interval, "batchWait", d.batchWait) + ticker := time.NewTicker(d.interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + d.dispatch(ctx) + case <-ctx.Done(): + d.logger.Info("email dispatcher stopped") + return + } + } +} + +func (d *Dispatcher) dispatch(ctx context.Context) { + cutoff := time.Now().Add(-d.batchWait) + recipients, err := deldb.GetPendingEmailDigestRecipients(d.db, cutoff) + if err != nil { + d.logger.Error("email dispatcher: failed to get recipients", "err", err) + return + } + for _, did := range recipients { + if err := d.sendPending(ctx, did, cutoff); err != nil { + d.logger.Error("email dispatcher: failed to send", "did", did, "err", err) + } + } +} + +func (d *Dispatcher) sendPending(ctx context.Context, recipientDid string, cutoff time.Time) error { + em, err := deldb.GetPrimaryEmail(d.db, recipientDid) + if err != nil || !em.Verified { + return nil + } + + notifs, err := deldb.GetPendingNotificationsForEmailDigest(d.db, recipientDid, cutoff) + if err != nil { + return fmt.Errorf("get pending notifications: %w", err) + } + if len(notifs) == 0 { + return nil + } + + handle := d.handle(ctx, recipientDid) + + var order []string + byEntity := map[string][]*models.Notification{} + for _, n := range notifs { + if _, ok := byEntity[n.EntityAt]; !ok { + order = append(order, n.EntityAt) + } + byEntity[n.EntityAt] = append(byEntity[n.EntityAt], n) + } + + var failed error + for _, entity := range order { + group := byEntity[entity] + slices.SortStableFunc(group, func(a, b *models.Notification) int { return a.Created.Compare(b.Created) }) + + mail, err := d.renderThread(ctx, handle, group) + if err != nil { + failed = fmt.Errorf("render %s: %w", entity, err) + continue + } + if err := d.sender.Send(mailer.Message{To: em.Address, Subject: mail.subject, Text: mail.text, Html: mail.html, Headers: mail.headers}); err != nil { + failed = fmt.Errorf("send %s: %w", entity, err) + continue + } + d.logger.Info("email dispatcher: sent", "did", recipientDid, "entity", entity, "notifications", len(group)) + + ids := make([]int64, len(group)) + for i, n := range group { + ids[i] = n.ID + } + if err := deldb.MarkEmailed(d.db, ids); err != nil { + d.logger.Error("email dispatcher: failed to mark emailed", "did", recipientDid, "err", err) + } + } + return failed +} + +func (d *Dispatcher) handle(ctx context.Context, did string) string { + if id, err := d.resolver.ResolveIdent(ctx, did); err == nil && !id.Handle.IsInvalidHandle() { + return id.Handle.String() + } + return did +} + +// group is one entity's notifications, oldest first +func (d *Dispatcher) renderThread(ctx context.Context, recipientHandle string, group []*models.Notification) (mail threadEmail, err error) { + latest := group[len(group)-1] + + repo := "" + if latest.RepoDid != "" { + repo = latest.RepoOwnerHandle(d.resolver) + if latest.RepoName != "" { + repo += "/" + latest.RepoName + } + } + title := latest.EntityTitle + if title == "" { + title = "an issue" + if latest.EntityKind == models.EntityKindPull { + title = "a pull request" + } + } + + events := make([]threadEvent, 0, len(group)) + for _, n := range group { + actor := d.handle(ctx, n.ActorDid) + phrase := notifPhrase(n) + events = append(events, threadEvent{ + IconURL: d.assetsURL + n.Icon() + ".png", + Actor: actor, + Phrase: phrase, + Header: actor + " " + phrase, + }) + } + + data := threadData{ + RecipientHandle: recipientHandle, + Title: title, + Repo: repo, + Events: events, + URL: d.baseURL + latest.URL(d.resolver), + SettingsURL: d.baseURL + "/settings/notifications", + } + + // mail clients thread on matching subjects too, so this must not vary between emails + mail.subject = title + if repo != "" { + mail.subject = "[" + repo + "] " + title + } + root := threadRoot(latest.EntityAt, d.sender.Domain()) + mail.headers = map[string]string{"In-Reply-To": root, "References": root} + + var textBuf bytes.Buffer + if err = d.textTmpl.Execute(&textBuf, data); err != nil { + return + } + mail.text = textBuf.String() + + var htmlBuf bytes.Buffer + if err = d.htmlTmpl.Execute(&htmlBuf, data); err != nil { + return + } + mail.html = emailtemplate.Shell(mail.subject, htmlBuf.String(), d.assetsURL) + return +} diff --git a/deliberi/digest_test.go b/deliberi/digest_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2f56ce6eb428db6e835828d0cf61b757ed0c1767 --- /dev/null +++ b/deliberi/digest_test.go @@ -0,0 +1,114 @@ +package deliberi + +import ( + "context" + "io" + "log/slog" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/deliberi/config" + deldb "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/mailer" + "tangled.org/core/deliberi/models" + "tangled.org/core/idresolver" +) + +type sentMail struct{ sent []mailer.Message } + +func (s *sentMail) Send(m mailer.Message) error { s.sent = append(s.sent, m); return nil } +func (s *sentMail) Domain() string { return "notifs.example" } + +func TestEmailsAboutOneTicketShareAThread(t *testing.T) { + const recipient = "did:plc:recipient" + const issue = "at://did:plc:repo/org.tangled.track.ticket/issue1" + const other = "at://did:plc:repo/org.tangled.track.ticket/issue2" + + database, err := deldb.Make(context.Background(), filepath.Join(t.TempDir(), "x.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + if err := deldb.AddEmail(database, models.Email{Did: recipient, Address: "r@example.com", Verified: true, Primary: true}); err != nil { + t.Fatal(err) + } + prefs := models.DefaultNotificationPreferences(syntax.DID(recipient)) + prefs.EmailNotifications = true + if err := deldb.UpsertNotificationPreferences(database, prefs); err != nil { + t.Fatal(err) + } + + sender := &sentMail{} + d := NewDispatcher(database, sender, config.ResendConfig{}, "https://tangled.test", + idresolver.DefaultResolver("http://127.0.0.1:1"), slog.New(slog.NewTextHandler(io.Discard, nil)), true) + + start := time.Now().Add(-time.Hour) + seq := 0 + notify := func(entity, title string, typ models.NotificationType) { + seq++ + n := &models.Notification{ + RecipientDid: recipient, + AtUri: entity + "#" + string(rune('a'+seq)), + Type: typ, + ActorDid: "did:plc:actor", + EntityAt: entity, + EntityKind: models.EntityKindIssue, + EntityTitle: title, + Created: start.Add(time.Duration(seq) * time.Minute), + } + if err := deldb.CreateNotification(database, n); err != nil { + t.Fatal(err) + } + } + + notify(issue, "Crash on start", models.NotificationTypeIssueCommented) + notify(issue, "Crash on start", models.NotificationTypeIssueCommented) + notify(other, "Docs typo", models.NotificationTypeIssueCreated) + d.dispatch(context.Background()) + + notify(issue, "Crash on start", models.NotificationTypeIssueClosed) + notify(issue, "Crash on start", models.NotificationTypeIssueReopen) + d.dispatch(context.Background()) + + if len(sender.sent) != 3 { + t.Fatalf("sent %d emails, want one per ticket per batch (3)", len(sender.sent)) + } + var issueMails []mailer.Message + var otherMail mailer.Message + for _, m := range sender.sent { + if strings.Contains(m.Subject, "Docs typo") { + otherMail = m + } else { + issueMails = append(issueMails, m) + } + } + if len(issueMails) != 2 { + t.Fatalf("issue emails = %d, want 2", len(issueMails)) + } + first, second := issueMails[0], issueMails[1] + if first.Subject != second.Subject { + t.Errorf("subjects differ within a thread: %q vs %q", first.Subject, second.Subject) + } + root := first.Headers["References"] + if root == "" || second.Headers["References"] != root || second.Headers["In-Reply-To"] != root { + t.Errorf("issue emails don't share a thread root: %v / %v", first.Headers, second.Headers) + } + if otherMail.Headers["References"] == root { + t.Errorf("a different ticket joined the same thread") + } + if !strings.Contains(first.Text, "commented") || strings.Contains(first.Text, "closed") { + t.Errorf("first batch should hold only the comments:\n%s", first.Text) + } + closed, reopened := strings.Index(second.Text, "closed this issue"), strings.Index(second.Text, "reopened this issue") + if closed < 0 || reopened < 0 || closed > reopened { + t.Errorf("second batch should list close then reopen:\n%s", second.Text) + } + + d.dispatch(context.Background()) + if len(sender.sent) != 3 { + t.Errorf("a quiet tick resent mail: %d", len(sender.sent)) + } +} diff --git a/deliberi/email/email.go b/deliberi/email/email.go new file mode 100644 index 0000000000000000000000000000000000000000..b6d3568a2cf200db923bc02e61b36eeec24b5e56 --- /dev/null +++ b/deliberi/email/email.go @@ -0,0 +1,102 @@ +package email + +import ( + "fmt" + "net" + "net/mail" + "strings" + + "github.com/resend/resend-go/v3" +) + +type Email struct { + From string + To string + Subject string + Text string + Html string + APIKey string + Headers map[string]string +} + +func SendEmail(email Email) error { + client := resend.NewClient(email.APIKey) + _, err := client.Emails.Send(&resend.SendEmailRequest{ + From: email.From, + To: []string{email.To}, + Subject: email.Subject, + Text: email.Text, + Html: email.Html, + Headers: email.Headers, + }) + if err != nil { + return fmt.Errorf("error sending email: %w", err) + } + return nil +} + +// AddNewsletterContact creates a global contact in Resend and adds them to the newsletter segment. +func AddNewsletterContact(apiKey, segmentID, emailAddr string) error { + client := resend.NewClient(apiKey) + _, err := client.Contacts.Create(&resend.CreateContactRequest{ + Email: emailAddr, + }) + if err != nil { + return fmt.Errorf("error creating contact: %w", err) + } + _, err = client.Contacts.Segments.Add(&resend.AddContactSegmentRequest{ + SegmentId: segmentID, + Email: emailAddr, + }) + if err != nil { + return fmt.Errorf("error adding contact to newsletter segment: %w", err) + } + return nil +} + +func IsValidEmail(email string) bool { + // Reject whitespace (ParseAddress normalizes it away) + if strings.ContainsAny(email, " \t\n\r") { + return false + } + + // Use stdlib RFC 5322 parser + addr, err := mail.ParseAddress(email) + if err != nil { + return false + } + + // Split email into local and domain parts + parts := strings.Split(addr.Address, "@") + domain := parts[1] + + canonical := coalesceToCanonicalName(domain) + mx, err := net.LookupMX(canonical) + + // Don't check err here; mx will only contain valid mx records, and we should + // only fallback to an implicit mx if there are no mx records defined (whether + // they are valid or not). + if len(mx) != 0 { + return true + } + + if err != nil { + // If the domain resolves to an address, assume it's an implicit mx. + address, _ := net.LookupIP(canonical) + if len(address) != 0 { + return true + } + } + + return false +} + +func coalesceToCanonicalName(domain string) string { + canonical, err := net.LookupCNAME(domain) + if err != nil { + // net.LookupCNAME() returns an error if there is no cname record *and* no + // a/aaaa records, but there may still be mx records. + return domain + } + return canonical +} diff --git a/deliberi/email/email_test.go b/deliberi/email/email_test.go new file mode 100644 index 0000000000000000000000000000000000000000..9d7f4edbcab6bf5d1bb7cc10bed149ffe1dad3af --- /dev/null +++ b/deliberi/email/email_test.go @@ -0,0 +1,53 @@ +package email + +import ( + "testing" +) + +func TestIsValidEmail(t *testing.T) { + tests := []struct { + name string + email string + want bool + }{ + // Valid emails using RFC 2606 reserved domains + {"standard email", "user@example.com", true}, + {"single char local", "a@example.com", true}, + {"dot in middle", "first.last@example.com", true}, + {"multiple dots", "a.b.c@example.com", true}, + {"plus tag", "user+tag@example.com", true}, + {"numbers", "user123@example.com", true}, + {"example.org", "user@example.org", true}, + {"example.net", "user@example.net", true}, + + // Invalid format - rejected by mail.ParseAddress + {"empty string", "", false}, + {"no at sign", "userexample.com", false}, + {"no domain", "user@", false}, + {"no local part", "@example.com", false}, + {"double at", "user@@example.com", false}, + {"just at sign", "@", false}, + {"leading dot", ".user@example.com", false}, + {"trailing dot", "user.@example.com", false}, + {"consecutive dots", "user..name@example.com", false}, + + // Whitespace - rejected before parsing + {"space in local", "user @example.com", false}, + {"space in domain", "user@ example.com", false}, + {"tab", "user\t@example.com", false}, + {"newline", "user\n@example.com", false}, + + // MX lookup - using RFC 2606 reserved TLDs (guaranteed no MX) + {"invalid TLD", "user@example.invalid", false}, + {"test TLD", "user@mail.test", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := IsValidEmail(tt.email) + if got != tt.want { + t.Errorf("IsValidEmail(%q) = %v, want %v", tt.email, got, tt.want) + } + }) + } +} diff --git a/deliberi/emailtemplate/template.go b/deliberi/emailtemplate/template.go new file mode 100644 index 0000000000000000000000000000000000000000..329ad3975e09bbf8d8fbae9433c6d494ba56956f --- /dev/null +++ b/deliberi/emailtemplate/template.go @@ -0,0 +1,45 @@ +// Package emailtemplate is the branded HTML shell shared by every deliberi email. +package emailtemplate + +import "strings" + +// Shell wraps title and bodyHTML in the Tangled-branded email document. +func Shell(title, bodyHTML, assetsURL string) string { + assets := strings.TrimSuffix(assetsURL, "/") + return ` + + + + +` + title + ` + + + + + + + + + +
+ + + + + + +
+
+ Tangled +
+` + bodyHTML + ` +
+

Tangled Labs Oy. © 2026 · tangled.org

+
+
+
+ +` +} diff --git a/deliberi/ingest.go b/deliberi/ingest.go new file mode 100644 index 0000000000000000000000000000000000000000..ed6e589687ca94c872f3169e38147631cced13fa --- /dev/null +++ b/deliberi/ingest.go @@ -0,0 +1,660 @@ +package deliberi + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "slices" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/posthog/posthog-go" + "tangled.org/core/deliberi/analytics" + deldb "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/kv" + models "tangled.org/core/deliberi/models" + "tangled.org/core/idresolver" + "tangled.org/core/jetstream" + "tangled.org/core/sets" +) + +const ( + maxMentions = 8 + idleCursorEvery = time.Minute + // rewinding replays events, the rev check in plan drops the repeats + cursorRewind = 5 * time.Minute +) + +type Ingester struct { + db *deldb.DB + bobbin bobbinLookup + idResolver *idresolver.Resolver + endpoint string + logger *slog.Logger + // nil when analytics is off + analytics *analytics.Client + // nil when the KV cache is off + emailKV kv.EmailKV + + idleCursorUs int64 +} + +var ingestCollections = []string{ + repoManifestNSID, + ticketNSID, + commentNSID, + patchNSID, + collaboratorNSID, + subscriptionNSID, + starNSID, + followNSID, +} + +// star and follow stay out, their handlers only notify on live creates +var revGuarded = []string{ + repoManifestNSID, + ticketNSID, + commentNSID, + patchNSID, + collaboratorNSID, + subscriptionNSID, +} + +func NewIngester(database *deldb.DB, bobbin bobbinLookup, idRes *idresolver.Resolver, endpoint string, logger *slog.Logger) *Ingester { + return &Ingester{ + db: database, + bobbin: bobbin, + idResolver: idRes, + endpoint: endpoint, + logger: logger, + } +} + +func (i *Ingester) Run(ctx context.Context) error { + stream, err := jetstream.NewStream(jetstream.StreamConfig{ + Endpoint: i.endpoint, + Collections: ingestCollections, + // hydrant only sends backfilled commits when historical is asked for + EventTypes: []string{"live", "historical"}, + Cursor: func() (int64, bool, error) { return deldb.GetCursor(i.db) }, + Rewind: cursorRewind, + UserAgent: "deliberi", + }, i.logger) + if err != nil { + return err + } + return stream.Run(ctx, i.process) +} + +// runs in the event tx, so network calls belong in plan +type writer func(x deldb.Execer) error + +func (i *Ingester) process(ctx context.Context, e *jetstream.Event) error { + if e.Kind == jetstream.EventKindAccount && e.Account != nil { + return i.processAccount(e) + } + var write writer + if e.Kind == jetstream.EventKindCommit && e.Commit != nil { + write = i.plan(ctx, e) + } + if write == nil { + return i.advance(e.TimeUS) + } + + tx, err := i.db.Begin() + if err != nil { + return fmt.Errorf("starting event tx: %w", err) + } + defer tx.Rollback() + + if err := write(tx); err != nil { + return err + } + if err := deldb.SaveCursor(tx, e.TimeUS); err != nil { + return fmt.Errorf("saving cursor: %w", err) + } + return tx.Commit() +} + +// only "deleted" is final, deactivated and takendown accounts can come back +func (i *Ingester) processAccount(e *jetstream.Event) error { + if e.Account.Active || e.Account.Status == nil || *e.Account.Status != "deleted" { + return i.advance(e.TimeUS) + } + tx, err := i.db.Begin() + if err != nil { + return fmt.Errorf("starting account tx: %w", err) + } + defer tx.Rollback() + emails, err := deldb.DeleteAccountEmails(tx, e.Did) + if err != nil { + return fmt.Errorf("deleting emails for %s: %w", e.Did, err) + } + if err := deldb.SaveCursor(tx, e.TimeUS); err != nil { + return fmt.Errorf("saving cursor: %w", err) + } + if err := tx.Commit(); err != nil { + return err + } + if len(emails) > 0 { + i.logger.Info("removed emails of deleted account", "did", e.Did, "count", len(emails)) + } + if i.emailKV != nil { + for _, email := range emails { + i.emailKV.DeleteEmailKey(email) + } + i.emailKV.DeletePrimaryEmail(e.Did) + } + return nil +} + +func (i *Ingester) advance(timeUs int64) error { + if timeUs-i.idleCursorUs < idleCursorEvery.Microseconds() { + return nil + } + i.idleCursorUs = timeUs + return deldb.SaveCursor(i.db, timeUs) +} + +func (i *Ingester) plan(ctx context.Context, e *jetstream.Event) writer { + c := e.Commit + uri := fmt.Sprintf("at://%s/%s/%s", e.Did, c.Collection, c.RKey) + + guarded := c.Rev != "" && slices.Contains(revGuarded, c.Collection) + if guarded && !deldb.RevIsNew(i.db, uri, c.Rev) { + return nil + } + + notify := i.notifying(e) + var write writer + switch c.Collection { + case repoManifestNSID: + write = i.planManifest(e, uri) + case ticketNSID: + write = i.planTicket(ctx, e, uri, notify) + case commentNSID: + write = i.planComment(ctx, e, uri, notify) + case patchNSID: + write = i.planPatch(e, uri) + case collaboratorNSID: + write = i.planCollaborator(e) + case subscriptionNSID: + write = i.planSubscription(e, uri) + case starNSID: + write = i.planStar(ctx, e, uri, notify) + case followNSID: + write = i.planFollow(e, uri, notify) + } + if write == nil || !guarded { + return write + } + return func(x deldb.Execer) error { + if err := deldb.PutRev(x, uri, c.Rev); err != nil { + return err + } + return write(x) + } +} + +// bootstrap time is the cutoff, anything older only builds state +func (i *Ingester) notifying(e *jetstream.Event) bool { + return e.Commit.IsLive() && e.TimeUS >= deldb.GetBootstrapUs(i.db) +} + +func (i *Ingester) planManifest(e *jetstream.Event, uri string) writer { + if e.Commit.Operation == jetstream.CommitOperationDelete || len(e.Commit.Record) == 0 { + return nil + } + var rec repoManifest + if err := json.Unmarshal(e.Commit.Record, &rec); err != nil { + i.logger.Warn("decoding manifest record", "err", err, "uri", uri) + return nil + } + if rec.Declaration == nil || rec.Declaration.Owner == "" { + return nil + } + return func(x deldb.Execer) error { + return deldb.PutRepoName(x, e.Did, rec.Declaration.Owner, rec.Declaration.Name) + } +} + +func (i *Ingester) planTicket(ctx context.Context, e *jetstream.Event, uri string, notify bool) writer { + c := e.Commit + if c.Operation == jetstream.CommitOperationDelete { + return func(x deldb.Execer) error { + if err := deldb.DeleteEntity(x, uri); err != nil { + return err + } + return deldb.DeleteNotificationsAbout(x, uri) + } + } + if len(c.Record) == 0 { + return nil + } + var rec ticketRecord + if err := json.Unmarshal(c.Record, &rec); err != nil { + i.logger.Warn("decoding ticket record", "err", err, "uri", uri) + return nil + } + + actor := editorOr(rec.Editor, e.Did) + known := deldb.GetEntity(i.db, uri) + next := deldb.Entity{ + Title: rec.title(), + RepoDid: e.Did, + Kind: rec.kind(), + State: rec.state(), + Cid: c.CID, + Author: known.Author, + } + // a backfill sends the latest version as a create, and a maintainer may have + // written that one, so only a live create names who opened the ticket + opened := c.IsLive() && c.Operation == jetstream.CommitOperationCreate && known.Kind == "" + if opened { + next.Author = actor + } + store := func(x deldb.Execer) error { return deldb.PutEntity(x, uri, next) } + if !notify { + return store + } + i.fillOpener(ctx, uri, &next) + + var kind models.NotificationType + switch { + case opened: + kind = models.NotificationTypeIssueCreated + if next.Kind == models.EntityKindPull { + kind = models.NotificationTypePullCreated + } + case known.State != "": + kind = stateNotification(next.Kind, known.State, next.State) + } + // todo: assigned and unassigned, once the knot takes assignees + if kind == "" { + return store + } + + owner := i.repoOwner(ctx, e.Did) + var mentions []string + if opened { + mentions = i.mentionDids(ctx, actor, bodyText(rec.Body)) + } + return func(x deldb.Execer) error { + if err := store(x); err != nil { + return err + } + collaborators, err := deldb.ListCollaborators(x, e.Did) + if err != nil { + return err + } + recipients := append([]string{owner}, collaborators...) + n := models.Notification{ + AtUri: uri, + Type: kind, + ActorDid: actor, + RepoDid: e.Did, + EntityAt: uri, + EntityKind: next.Kind, + EntityTitle: next.Title, + Created: time.UnixMicro(e.TimeUS).UTC(), + } + if !opened { + // a ticket can close, reopen and close again, so each change gets its own uri + n.AtUri = uri + "#" + c.Rev + participants, err := deldb.TicketParticipants(x, uri) + if err != nil { + return err + } + recipients = append(recipients, participants...) + } + if err := i.fanOut(x, n, recipients, mentions); err != nil { + return err + } + if event := stateEvent(kind); event != "" { + // the ticket's author is the subject, as the appview's events had it + i.analytics.Capture(event, editorOr(next.Author, actor), uri+"#"+c.Rev, posthog.NewProperties(). + Set("repo_did", e.Did). + Set("uri", uri). + Set("actor", actor)) + } + return nil + } +} + +// the appview's names, so existing dashboards keep counting +func stateEvent(kind models.NotificationType) string { + switch kind { + case models.NotificationTypeIssueClosed: + return "issue_closed" + case models.NotificationTypeIssueReopen: + return "issue_reopen" + case models.NotificationTypePullClosed: + return "pull_closed" + case models.NotificationTypePullReopen: + return "pull_reopen" + case models.NotificationTypePullMerged: + return "pull_merged" + } + return "" +} + +func stateNotification(kind models.EntityKind, from, to string) models.NotificationType { + if from == to { + return "" + } + pull := kind == models.EntityKindPull + pick := func(issueType, pullType models.NotificationType) models.NotificationType { + if pull { + return pullType + } + return issueType + } + switch to { + case ticketStateClosed: + return pick(models.NotificationTypeIssueClosed, models.NotificationTypePullClosed) + case ticketStateCompleted: + // completed on a pull means it merged + return pick(models.NotificationTypeIssueClosed, models.NotificationTypePullMerged) + case ticketStateOpen: + return pick(models.NotificationTypeIssueReopen, models.NotificationTypePullReopen) + } + return "" +} + +func (i *Ingester) planComment(ctx context.Context, e *jetstream.Event, uri string, notify bool) writer { + c := e.Commit + if c.Operation == jetstream.CommitOperationDelete { + return func(x deldb.Execer) error { + if err := deldb.DeleteComment(x, uri); err != nil { + return err + } + return deldb.DeleteNotificationsFrom(x, uri) + } + } + // an edit can't change the author or reply target, so only creates count + if c.Operation != jetstream.CommitOperationCreate || len(c.Record) == 0 { + return nil + } + var rec commentRecord + if err := json.Unmarshal(c.Record, &rec); err != nil { + i.logger.Warn("decoding comment record", "err", err, "uri", uri) + return nil + } + if rec.Subject == nil { + return nil + } + subject, err := syntax.ParseATURI(rec.Subject.Uri) + if err != nil || subject.Authority().String() != e.Did { + return nil + } + ticketUri := subject.String() + switch subject.Collection().String() { + case ticketNSID: + case patchNSID: + // a backfill sends a repo's comments before its patches, so an unknown patch + // stays the key and TicketParticipants finds it once the patch arrives + if t := deldb.GetPatchTicket(i.db, ticketUri); t != "" { + ticketUri = t + } + default: + return nil + } + + actor := editorOr(rec.Editor, e.Did) + comment := deldb.Comment{Uri: uri, TicketUri: ticketUri, Author: actor} + if rec.ReplyTo != nil { + comment.ReplyTo = rec.ReplyTo.Uri + } + store := func(x deldb.Execer) error { return deldb.PutComment(x, comment) } + if !notify { + return store + } + // no ticket yet, or a deleted one, so there's nothing to name in a notice + ent := deldb.GetEntity(i.db, ticketUri) + if ent.Kind == "" { + return store + } + learned := i.fillOpener(ctx, ticketUri, &ent) + owner := i.repoOwner(ctx, e.Did) + mentions := i.mentionDids(ctx, actor, bodyText(rec.Body)) + + return func(x deldb.Execer) error { + if err := store(x); err != nil { + return err + } + if learned { + if err := deldb.PutEntity(x, ticketUri, ent); err != nil { + return err + } + } + + var participants []string + var err error + switch { + case ent.Kind == models.EntityKindPull: + participants, err = deldb.TicketParticipants(x, ticketUri) + case comment.ReplyTo != "": + participants, err = deldb.ThreadParticipants(x, comment.ReplyTo) + default: + participants = []string{ent.Author} + } + if err != nil { + return err + } + + n := models.Notification{ + AtUri: uri, + Type: models.NotificationTypeIssueCommented, + ActorDid: actor, + RepoDid: e.Did, + EntityAt: ticketUri, + EntityKind: ent.Kind, + EntityTitle: ent.Title, + Created: time.UnixMicro(e.TimeUS).UTC(), + } + if ent.Kind == models.EntityKindPull { + n.Type = models.NotificationTypePullCommented + } + return i.fanOut(x, n, append([]string{owner}, participants...), mentions) + } +} + +func (i *Ingester) planPatch(e *jetstream.Event, uri string) writer { + if e.Commit.Operation == jetstream.CommitOperationDelete { + return func(x deldb.Execer) error { return deldb.DeletePatch(x, uri) } + } + var rec patchRecord + if err := json.Unmarshal(e.Commit.Record, &rec); err != nil || rec.Ticket == "" { + i.logger.Warn("decoding patch record", "err", err, "uri", uri) + return nil + } + ticketUri := fmt.Sprintf("at://%s/%s/%s", e.Did, ticketNSID, rec.Ticket) + return func(x deldb.Execer) error { return deldb.PutPatch(x, uri, ticketUri) } +} + +func (i *Ingester) planCollaborator(e *jetstream.Event) writer { + did, err := syntax.ParseDID(e.Commit.RKey) + if err != nil { + return nil + } + if e.Commit.Operation == jetstream.CommitOperationDelete { + return func(x deldb.Execer) error { return deldb.DeleteCollaborator(x, e.Did, did.String()) } + } + return func(x deldb.Execer) error { return deldb.PutCollaborator(x, e.Did, did.String()) } +} + +func (i *Ingester) planSubscription(e *jetstream.Event, uri string) writer { + c := e.Commit + if c.Operation == jetstream.CommitOperationDelete { + return func(x deldb.Execer) error { return deldb.DeleteSubscription(x, e.Did, c.RKey) } + } + var rec subscriptionRecord + if err := json.Unmarshal(c.Record, &rec); err != nil || rec.subject() == "" { + i.logger.Warn("decoding subscription record", "err", err, "uri", uri) + return nil + } + return func(x deldb.Execer) error { + return deldb.PutSubscription(x, e.Did, c.RKey, rec.subject(), rec.Ignore) + } +} + +// undoing a star or a follow takes its notice back, the same as erasing a comment +func takeBack(uri string) writer { + return func(x deldb.Execer) error { return deldb.DeleteNotificationsFrom(x, uri) } +} + +func (i *Ingester) planStar(ctx context.Context, e *jetstream.Event, uri string, notify bool) writer { + if e.Commit.Operation == jetstream.CommitOperationDelete { + return takeBack(uri) + } + if e.Commit.Operation != jetstream.CommitOperationCreate || !notify { + return nil + } + var rec starRecord + if err := json.Unmarshal(e.Commit.Record, &rec); err != nil { + i.logger.Warn("decoding star record", "err", err, "uri", uri) + return nil + } + if rec.Subject == nil || rec.Subject.Type != starRepoType || rec.Subject.Identity == "" { + return nil + } + repoDid := rec.Subject.Identity + ownerDid := i.repoOwner(ctx, repoDid) + if ownerDid == "" { + i.logger.Warn("star: could not resolve repo owner, skipping notification", "repoDid", repoDid) + return nil + } + return func(x deldb.Execer) error { + return i.deliver(x, ownerDid, models.Notification{ + AtUri: uri, + Type: models.NotificationTypeRepoStarred, + ActorDid: e.Did, + RepoDid: repoDid, + Created: time.UnixMicro(e.TimeUS).UTC(), + }) + } +} + +func (i *Ingester) planFollow(e *jetstream.Event, uri string, notify bool) writer { + if e.Commit.Operation == jetstream.CommitOperationDelete { + return takeBack(uri) + } + if e.Commit.Operation != jetstream.CommitOperationCreate || !notify { + return nil + } + subject, err := syntax.ParseDID(e.Commit.RKey) + if err != nil { + return nil + } + return func(x deldb.Execer) error { + return i.deliver(x, subject.String(), models.Notification{ + AtUri: uri, + Type: models.NotificationTypeFollowed, + ActorDid: e.Did, + Created: time.UnixMicro(e.TimeUS).UTC(), + }) + } +} + +func (i *Ingester) fanOut(x deldb.Execer, n models.Notification, implicit, mentions []string) error { + // over the cap is spam, so drop them all instead of trimming to the cap + if len(mentions) > maxMentions { + mentions = nil + } + + recipients := sets.New[string]() + for _, did := range implicit { + if did != "" { + recipients.Insert(did) + } + } + subscribers, ignorers, err := deldb.ListSubscribers(x, n.EntityAt, n.RepoDid) + if err != nil { + return fmt.Errorf("listing subscribers: %w", err) + } + for _, did := range subscribers { + recipients.Insert(did) + } + for _, did := range mentions { + recipients.Remove(did) + } + // a mute beats everything, the owner too + for _, did := range ignorers { + recipients.Remove(did) + } + + for did := range recipients.All() { + if err := i.deliver(x, did, n); err != nil { + return err + } + } + + // mentions ignore mutes, so a muted thread still pings you by name + mention := n + mention.Type = models.NotificationTypeUserMentioned + for _, did := range mentions { + if err := i.deliver(x, did, mention); err != nil { + return err + } + } + return nil +} + +func (i *Ingester) repoOwner(ctx context.Context, repoDid string) string { + if owner := deldb.GetRepoOwner(i.db, repoDid); owner != "" { + return owner + } + if i.bobbin == nil { + return "" + } + + owner, name, err := i.bobbin.RepoOwner(ctx, repoDid) + if err != nil { + i.logger.Warn("resolving repo owner", "err", err, "repoDid", repoDid) + return "" + } + if owner == "" { + return "" + } + + // a nameless record must not blank out a name we already cached + if name == "" { + name = deldb.GetRepoName(i.db, repoDid) + } + if err := deldb.PutRepoName(i.db, repoDid, owner, name); err != nil { + i.logger.Warn("caching repo owner", "err", err, "repoDid", repoDid) + } + return owner +} + +// a backfilled ticket only names its last editor, so bobbin says who opened it. true when it found out +func (i *Ingester) fillOpener(ctx context.Context, ticketUri string, ent *deldb.Entity) bool { + if ent.Author != "" || i.bobbin == nil { + return false + } + author, err := i.bobbin.TicketAuthor(ctx, ticketUri) + if err != nil { + i.logger.Warn("looking up who opened a ticket", "err", err, "uri", ticketUri) + return false + } + ent.Author = author + return author != "" +} + +func (i *Ingester) deliver(x deldb.Execer, recipientDid string, n models.Notification) error { + if recipientDid == "" || recipientDid == n.ActorDid { + return nil + } + + prefs, err := deldb.GetNotificationPreference(x, recipientDid) + if err != nil { + return fmt.Errorf("loading prefs for %s: %w", recipientDid, err) + } + if !prefs.ShouldNotify(n.Type) { + return nil + } + + n.RecipientDid = recipientDid + return deldb.CreateNotification(x, &n) +} diff --git a/deliberi/ingest_test.go b/deliberi/ingest_test.go new file mode 100644 index 0000000000000000000000000000000000000000..edd7c5266a3e39497b06a317afee44b6dd3ab289 --- /dev/null +++ b/deliberi/ingest_test.go @@ -0,0 +1,1135 @@ +package deliberi + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/gorilla/websocket" + "tangled.org/core/deliberi/analytics" + "tangled.org/core/deliberi/config" + deldb "tangled.org/core/deliberi/db" + models "tangled.org/core/deliberi/models" + "tangled.org/core/jetstream" +) + +type fakeBobbin struct { + owners map[string]string + repoNames map[string]string + ownerErr error + // proves cache hits skip bobbin + ownerCalls *int + + authors map[string]string +} + +func (f fakeBobbin) RepoOwner(ctx context.Context, repoDid string) (string, string, error) { + if f.ownerCalls != nil { + *f.ownerCalls++ + } + if f.ownerErr != nil { + return "", "", f.ownerErr + } + return f.owners[repoDid], f.repoNames[repoDid], nil +} + +func (f fakeBobbin) TicketAuthor(ctx context.Context, ticketUri string) (string, error) { + if author, ok := f.authors[ticketUri]; ok { + return author, nil + } + return "", io.ErrUnexpectedEOF +} + +const ( + ownerDid = "did:plc:bob" + editorDid = "did:plc:alice" + collabDid = "did:plc:carol" + otherDid = "did:plc:dave" + ticketUri = "at://" + repoDid + "/org.tangled.track.ticket/1" +) + +// an hour ahead of the db, so new frames count as live +var frameClock = time.Now().Add(time.Hour).UnixMicro() + +func frame(did, op, collection, rkey, rev, record string) *jetstream.Event { + frameClock++ + live := true + return &jetstream.Event{ + Did: did, + TimeUS: frameClock, + Kind: jetstream.EventKindCommit, + Commit: &jetstream.Commit{ + Rev: rev, + Operation: op, + Collection: collection, + RKey: rkey, + Record: []byte(record), + CID: "cid-" + rev, + Live: &live, + }, + } +} + +func backfill(e *jetstream.Event) *jetstream.Event { + live := false + e.Commit.Live = &live + return e +} + +func newTestIngester(t *testing.T, b bobbinLookup) *Ingester { + t.Helper() + database, err := deldb.Make(context.Background(), filepath.Join(t.TempDir(), "x.db")) + if err != nil { + t.Fatalf("make db: %v", err) + } + t.Cleanup(func() { database.Close() }) + i := NewIngester(database, b, nil, "ws://hydrant/subscribe", slog.New(slog.NewTextHandler(io.Discard, nil))) + if err := deldb.PutRepoName(i.db, repoDid, ownerDid, "my-repo"); err != nil { + t.Fatalf("seed repo: %v", err) + } + return i +} + +func mustProcess(t *testing.T, i *Ingester, events ...*jetstream.Event) { + t.Helper() + for _, e := range events { + if err := i.process(context.Background(), e); err != nil { + t.Fatalf("process: %v", err) + } + } +} + +func countFor(t *testing.T, i *Ingester, did string) int64 { + t.Helper() + n, err := deldb.CountNotifications(i.db, did) + if err != nil { + t.Fatalf("count: %v", err) + } + return n +} + +func notifsFor(t *testing.T, i *Ingester, did string) []*models.Notification { + t.Helper() + ns, err := deldb.GetNotifications(i.db, did, 0) + if err != nil { + t.Fatalf("get notifications: %v", err) + } + return ns +} + +func onlyFor(t *testing.T, i *Ingester, did string) *models.Notification { + t.Helper() + ns := notifsFor(t, i, did) + if len(ns) != 1 { + t.Fatalf("%s has %d notifications, want 1", did, len(ns)) + } + return ns[0] +} + +func wantTypes(t *testing.T, i *Ingester, want map[string]models.NotificationType) { + t.Helper() + for _, did := range []string{ownerDid, editorDid, collabDid, otherDid, "did:plc:sub", "did:plc:ignorer"} { + ns := notifsFor(t, i, did) + wantType, ok := want[did] + switch { + case !ok && len(ns) != 0: + t.Errorf("%s has %d notifications (first %q), want none", did, len(ns), ns[0].Type) + case ok && (len(ns) != 1 || ns[0].Type != wantType): + t.Errorf("%s notifications = %d, want exactly one %q", did, len(ns), wantType) + } + } +} + +// rows can share a created second, so order is no help +func byType(t *testing.T, i *Ingester, did string, want models.NotificationType) *models.Notification { + t.Helper() + for _, n := range notifsFor(t, i, did) { + if n.Type == want { + return n + } + } + t.Fatalf("%s has no %q notification", did, want) + return nil +} + +func ticketJSON(state, editor, extra string) string { + return `{"$type":"org.tangled.track.ticket","createdAt":"2026-01-01T00:00:00Z","state":"` + state + `",` + + `"title":{"$type":"org.tangled.markup.markdown#inline","text":"the ticket"},` + extra + + `"x-tngl-editor":"` + editor + `"}` +} + +const pullExtra = `"targetBranch":"main",` + +func ticketFrame(op, rev, state, editor, extra string) *jetstream.Event { + return frame(repoDid, op, ticketNSID, "1", rev, ticketJSON(state, editor, extra)) +} + +func subscriptionJSON(subject string, ignore bool) string { + return `{"$type":"org.tangled.feed.subscription","createdAt":"2026-01-01T00:00:00Z","ignore":` + strconv.FormatBool(ignore) + `,"subject":` + subject + `}` +} + +func recordSubject(uri string) string { + return `{"$type":"org.tangled.feed.subscription#record","uri":"` + uri + `"}` +} + +func repoSubject(did string) string { + return `{"$type":"org.tangled.feed.subscription#repo","did":"` + did + `"}` +} + +func subscribe(user, rkey, subject string, ignore bool) *jetstream.Event { + return frame(user, jetstream.CommitOperationCreate, subscriptionNSID, rkey, "rev-"+rkey, subscriptionJSON(subject, ignore)) +} + +func collaboratorFrame(op, rev, did string) *jetstream.Event { + return frame(repoDid, op, collaboratorNSID, did, rev, `{"$type":"org.tangled.graph.collaborator","createdAt":"2026-01-01T00:00:00Z"}`) +} + +func commentJSON(subjectUri, replyTo, editor string) string { + reply := "" + if replyTo != "" { + reply = `"replyTo":{"uri":"` + replyTo + `","cid":"bafyfake"},` + } + return `{"$type":"org.tangled.feed.comment","createdAt":"2026-01-01T00:00:00Z",` + + `"body":{"$type":"org.tangled.markup.markdown","text":"looks good"},` + reply + + `"subject":{"uri":"` + subjectUri + `","cid":"bafyfake"},"x-tngl-editor":"` + editor + `"}` +} + +func commentUri(rkey string) string { + return "at://" + repoDid + "/org.tangled.feed.comment/" + rkey +} + +func commentFrame(rkey, rev, subjectUri, replyTo, editor string) *jetstream.Event { + return frame(repoDid, jetstream.CommitOperationCreate, commentNSID, rkey, rev, commentJSON(subjectUri, replyTo, editor)) +} + +func TestLiveCreateNotifiesOwnerCollaboratorsAndSubscribers(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + collaboratorFrame(jetstream.CommitOperationCreate, "r1", collabDid), + subscribe("did:plc:sub", "s1", recordSubject(ticketUri), false), + subscribe(editorDid, "s2", repoSubject(repoDid), false), + subscribe("did:plc:ignorer", "s3", repoSubject(repoDid), true), + subscribe(ownerDid, "s4", recordSubject(ticketUri), true), + ticketFrame(jetstream.CommitOperationCreate, "r2", "open", editorDid, ""), + ) + + wantTypes(t, i, map[string]models.NotificationType{ + collabDid: models.NotificationTypeIssueCreated, + "did:plc:sub": models.NotificationTypeIssueCreated, + }) + + if got := countFor(t, i, ownerDid); got != 0 { + t.Errorf("owner rows = %d, want 0 (ignored the ticket)", got) + } + n := onlyFor(t, i, collabDid) + if n.ActorDid != editorDid || n.EntityAt != ticketUri || n.RepoDid != repoDid || n.EntityTitle != "the ticket" || n.EntityKind != models.EntityKindIssue { + t.Errorf("notification = %+v", n) + } + if n.AtUri != ticketUri { + t.Errorf("key = %q, want the ticket uri", n.AtUri) + } +} + +func TestLiveCreateOfPullNotifiesOwner(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, pullExtra)) + + n := onlyFor(t, i, ownerDid) + if n.Type != models.NotificationTypePullCreated || n.EntityKind != models.EntityKindPull { + t.Errorf("type/kind = %q/%q, want pull_created/pull", n.Type, n.EntityKind) + } +} + +func TestBackfillStoresStateWithoutNotifying(t *testing.T) { + i := newTestIngester(t, fakeBobbin{authors: map[string]string{ticketUri: editorDid}}) + mustProcess(t, i, + backfill(ticketFrame(jetstream.CommitOperationCreate, "r1", "open", otherDid, "")), + backfill(ticketFrame(jetstream.CommitOperationUpdate, "r2", "closed", otherDid, "")), + ) + + if got := countFor(t, i, ownerDid); got != 0 { + t.Fatalf("owner rows = %d, want 0 for a backfill", got) + } + got := deldb.GetEntity(i.db, ticketUri) + if got.State != "closed" || got.Cid != "cid-r2" || got.Title != "the ticket" { + t.Errorf("stored ticket = %+v", got) + } + if got.Author != "" { + t.Errorf("author = %q, want none yet: a snapshot's editor isn't the author and a replay doesn't ask bobbin", got.Author) + } + + // bobbin supplies the opener on a live update, not the frame's editor + mustProcess(t, i, ticketFrame(jetstream.CommitOperationUpdate, "r3", "open", otherDid, "")) + if got := deldb.GetEntity(i.db, ticketUri).Author; got != editorDid { + t.Errorf("author = %q, want bobbin's original author once a live update notifies", got) + } + if n := onlyFor(t, i, editorDid); n.Type != models.NotificationTypeIssueReopen { + t.Errorf("author's notification = %q, want issue_reopen", n.Type) + } +} + +func TestBackfillSetsBaselineForTheNextLiveUpdate(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + backfill(ticketFrame(jetstream.CommitOperationCreate, "r1", "closed", editorDid, "")), + ticketFrame(jetstream.CommitOperationUpdate, "r2", "open", otherDid, ""), + ) + + if n := onlyFor(t, i, ownerDid); n.Type != models.NotificationTypeIssueReopen { + t.Errorf("type = %q, want issue_reopen", n.Type) + } +} + +func TestLiveUpdateWithoutRowOnlyAdoptsTheTicket(t *testing.T) { + i := newTestIngester(t, fakeBobbin{authors: map[string]string{ticketUri: editorDid}}) + mustProcess(t, i, ticketFrame(jetstream.CommitOperationUpdate, "r2", "closed", otherDid, "")) + + if got := countFor(t, i, ownerDid); got != 0 { + t.Fatalf("owner rows = %d, want 0 (no baseline to diff)", got) + } + if got := deldb.GetEntity(i.db, ticketUri); got.Author != editorDid || got.State != "closed" { + t.Errorf("adopted ticket = %+v", got) + } +} + +func TestCloseByMaintainerNotifiesEveryone(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + collaboratorFrame(jetstream.CommitOperationCreate, "r1", collabDid), + subscribe("did:plc:sub", "s1", repoSubject(repoDid), false), + subscribe("did:plc:ignorer", "s2", recordSubject(ticketUri), true), + ticketFrame(jetstream.CommitOperationCreate, "r2", "open", editorDid, ""), + commentFrame("c1", "r3", ticketUri, "", otherDid), + ) + for _, did := range []string{ownerDid, editorDid, collabDid, otherDid, "did:plc:sub"} { + if err := deldb.DeleteNotification(i.db, did, ticketUri); err != nil { + t.Fatal(err) + } + if err := deldb.DeleteNotification(i.db, did, commentUri("c1")); err != nil { + t.Fatal(err) + } + } + + mustProcess(t, i, ticketFrame(jetstream.CommitOperationUpdate, "r4", "closed", collabDid, "")) + + wantTypes(t, i, map[string]models.NotificationType{ + ownerDid: models.NotificationTypeIssueClosed, + editorDid: models.NotificationTypeIssueClosed, + otherDid: models.NotificationTypeIssueClosed, + "did:plc:sub": models.NotificationTypeIssueClosed, + }) + n := onlyFor(t, i, ownerDid) + if n.ActorDid != collabDid || n.AtUri != ticketUri+"#r4" { + t.Errorf("actor/key = %q/%q", n.ActorDid, n.AtUri) + } +} + +func TestIgnoreBeatsTheOwnerOnStateChange(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, ""), + subscribe(ownerDid, "s1", recordSubject(ticketUri), true), + ) + mustProcess(t, i, ticketFrame(jetstream.CommitOperationUpdate, "r2", "closed", editorDid, "")) + + if got := notifsFor(t, i, ownerDid); len(got) != 1 || got[0].Type != models.NotificationTypeIssueCreated { + t.Errorf("owner notifications = %+v, want only the earlier created", got) + } +} + +func TestPullStateChanges(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, pullExtra)) + + for n, step := range []struct { + state string + want models.NotificationType + }{ + {"closed", models.NotificationTypePullClosed}, + {"open", models.NotificationTypePullReopen}, + {"completed", models.NotificationTypePullMerged}, + } { + mustProcess(t, i, ticketFrame(jetstream.CommitOperationUpdate, "r"+strconv.Itoa(n+2), step.state, otherDid, pullExtra)) + n := byType(t, i, ownerDid, step.want) + if n.EntityKind != models.EntityKindPull || n.ActorDid != otherDid { + t.Errorf("after %s: %q is %q by %q", step.state, step.want, n.EntityKind, n.ActorDid) + } + } + if got := countFor(t, i, otherDid); got != 0 { + t.Errorf("actor rows = %d, want 0", got) + } +} + +type capturedEvent struct { + Uuid string `json:"uuid"` + Event string `json:"event"` + DistinctId string `json:"distinct_id"` + Properties map[string]any `json:"properties"` +} + +// a posthog stand-in collecting what the client batches to /batch/ +func fakePosthog(t *testing.T) (*analytics.Client, func() []capturedEvent) { + t.Helper() + var events []capturedEvent + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Batch []capturedEvent `json:"batch"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decoding posthog batch: %v", err) + } + events = append(events, body.Batch...) + })) + t.Cleanup(srv.Close) + client := analytics.New(config.PosthogConfig{ApiKey: "key", Endpoint: srv.URL, Env: "test"}, + slog.New(slog.NewTextHandler(io.Discard, nil))) + return client, func() []capturedEvent { + client.Close() + return events + } +} + +func TestStateChangesReachPosthogUnderTheAppviewNames(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + client, flushed := fakePosthog(t) + i.analytics = client + frames := []*jetstream.Event{ + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, pullExtra), + ticketFrame(jetstream.CommitOperationUpdate, "r2", "closed", otherDid, pullExtra), + ticketFrame(jetstream.CommitOperationUpdate, "r3", "open", otherDid, pullExtra), + ticketFrame(jetstream.CommitOperationUpdate, "r4", "completed", otherDid, pullExtra), + } + mustProcess(t, i, frames...) + // a replay from another hydrant must not count twice + mustProcess(t, i, frames...) + + events := flushed() + want := []string{"pull_closed", "pull_reopen", "pull_merged"} + if len(events) != len(want) { + t.Fatalf("got %d events %v, want %v", len(events), events, want) + } + for n, e := range events { + if e.Event != want[n] { + t.Errorf("event %d = %q, want %q", n, e.Event, want[n]) + } + // the author is the subject, the maintainer who acted is a property + if e.DistinctId != editorDid || e.Properties["actor"] != otherDid { + t.Errorf("%s: distinct id %q, actor %v, want %q and %q", e.Event, e.DistinctId, e.Properties["actor"], editorDid, otherDid) + } + if e.Properties["repo_did"] != repoDid || e.Properties["uri"] != ticketUri || e.Properties["environment"] != "test" { + t.Errorf("%s: properties %v", e.Event, e.Properties) + } + } +} + +func TestIssueReopen(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, ""), + ticketFrame(jetstream.CommitOperationUpdate, "r2", "closed", editorDid, ""), + ticketFrame(jetstream.CommitOperationUpdate, "r3", "open", editorDid, ""), + ) + if got := countFor(t, i, ownerDid); got != 3 { + t.Fatalf("owner rows = %d, want created, closed and reopen", got) + } + byType(t, i, ownerDid, models.NotificationTypeIssueReopen) +} + +func TestEditsThatKeepTheStateDoNotNotify(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, ""), + ticketFrame(jetstream.CommitOperationUpdate, "r2", "open", otherDid, `"tangledLabels":[{"key":"bug"}],`), + ) + if got := countFor(t, i, ownerDid); got != 1 { + t.Fatalf("owner rows = %d, want only the created", got) + } + if got := deldb.GetEntity(i.db, ticketUri); got.Cid != "cid-r2" { + t.Errorf("cid = %q, want the edit stored", got.Cid) + } +} + +func TestReplayDoesNotNotifyTwiceOrReverseTheState(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + frames := []*jetstream.Event{ + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, ""), + ticketFrame(jetstream.CommitOperationUpdate, "r2", "closed", editorDid, ""), + ticketFrame(jetstream.CommitOperationUpdate, "r3", "open", editorDid, ""), + } + mustProcess(t, i, frames...) + mustProcess(t, i, frames...) + // a stale r2 close can show up late from another hydrant + mustProcess(t, i, ticketFrame(jetstream.CommitOperationUpdate, "r2", "closed", editorDid, "")) + + if got := countFor(t, i, ownerDid); got != 3 { + t.Errorf("owner rows = %d, want 3", got) + } + if got := deldb.GetEntity(i.db, ticketUri); got.State != "open" || got.Cid != "cid-r3" { + t.Errorf("stored ticket = %+v, want the newest version untouched", got) + } +} + +func TestDeleteTombstoneBlocksAnOlderCreate(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, ""), + frame(repoDid, jetstream.CommitOperationDelete, ticketNSID, "1", "r2", ""), + ) + if got := deldb.GetEntity(i.db, ticketUri); got.Kind != "" { + t.Fatalf("ticket after delete = %+v, want gone", got) + } + + mustProcess(t, i, ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, "")) + + if got := deldb.GetEntity(i.db, ticketUri); got.Kind != "" { + t.Errorf("an older create revived the ticket: %+v", got) + } + // a replayed create must not bring back a notice the erase removed + if got := countFor(t, i, ownerDid); got != 0 { + t.Errorf("owner rows = %d, want 0", got) + } +} + +func TestCommentRecipientsFollowLegacyRules(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, ""), + commentFrame("top", "r2", ticketUri, "", otherDid), + commentFrame("reply", "r3", ticketUri, commentUri("top"), collabDid), + commentFrame("lonely", "r4", ticketUri, "", "did:plc:sub"), + ) + for _, did := range []string{ownerDid, editorDid, collabDid, otherDid, "did:plc:sub"} { + for _, key := range []string{ticketUri, commentUri("top"), commentUri("reply"), commentUri("lonely")} { + if err := deldb.DeleteNotification(i.db, did, key); err != nil { + t.Fatal(err) + } + } + } + + mustProcess(t, i, commentFrame("c1", "r5", ticketUri, "", "did:plc:ignorer")) + wantTypes(t, i, map[string]models.NotificationType{ + ownerDid: models.NotificationTypeIssueCommented, + editorDid: models.NotificationTypeIssueCommented, + }) + for _, did := range []string{ownerDid, editorDid} { + if err := deldb.DeleteNotification(i.db, did, commentUri("c1")); err != nil { + t.Fatal(err) + } + } + + mustProcess(t, i, commentFrame("c2", "r6", ticketUri, commentUri("top"), editorDid)) + wantTypes(t, i, map[string]models.NotificationType{ + ownerDid: models.NotificationTypeIssueCommented, + collabDid: models.NotificationTypeIssueCommented, + otherDid: models.NotificationTypeIssueCommented, + }) +} + +func TestPullCommentTellsEveryParticipant(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, pullExtra), + commentFrame("c1", "r2", ticketUri, "", otherDid), + commentFrame("c2", "r3", ticketUri, "", "did:plc:sub"), + ) + + got := notifsFor(t, i, otherDid) + if len(got) != 1 || got[0].AtUri != commentUri("c2") || got[0].Type != models.NotificationTypePullCommented || got[0].EntityKind != models.EntityKindPull { + t.Errorf("earlier commenter notifications = %+v, want the later pull comment", got) + } + if got := countFor(t, i, editorDid); got != 2 { + t.Errorf("pull author rows = %d, want both comments", got) + } + if got := countFor(t, i, collabDid); got != 0 { + t.Errorf("stranger rows = %d, want 0", got) + } +} + +func TestCommentOnPatchResolvesToTheTicket(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + patchUri := "at://" + repoDid + "/org.tangled.review.patch/bafypatch" + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, pullExtra), + frame(repoDid, jetstream.CommitOperationCreate, patchNSID, "bafypatch", "r2", `{"ticket":"1","base":"a","head":"b","createdAt":"2026-01-01T00:00:00Z"}`), + commentFrame("c1", "r3", patchUri, "", otherDid), + ) + + n := onlyFor(t, i, editorDid) + if n.Type != models.NotificationTypePullCommented || n.EntityAt != ticketUri { + t.Errorf("notification = %+v, want a pull comment on the ticket", n) + } +} + +// hydrant backfills a repo in key order, so feed.comment, then review.patch, then track.ticket +func TestBackfillKeepsCommentsSentBeforeTheirTicket(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + patchUri := "at://" + repoDid + "/org.tangled.review.patch/bafypatch" + mustProcess(t, i, + backfill(commentFrame("c1", "r1", ticketUri, "", otherDid)), + backfill(commentFrame("c2", "r2", patchUri, "", "did:plc:sub")), + backfill(frame(repoDid, jetstream.CommitOperationCreate, patchNSID, "bafypatch", "r3", `{"ticket":"1","base":"a","head":"b","createdAt":"2026-01-01T00:00:00Z"}`)), + backfill(ticketFrame(jetstream.CommitOperationCreate, "r4", "open", editorDid, pullExtra)), + ticketFrame(jetstream.CommitOperationUpdate, "r5", "closed", ownerDid, pullExtra), + ) + + for _, did := range []string{otherDid, "did:plc:sub"} { + if n := onlyFor(t, i, did); n.Type != models.NotificationTypePullClosed { + t.Errorf("%s got %q, want the close", did, n.Type) + } + } +} + +// like a comment on a deleted ticket. nothing to name in a notice, but the commenter still counts +func TestLiveCommentOnAnUnseenTicketOnlyKeepsTheCommenter(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + commentFrame("c1", "r1", ticketUri, "", otherDid), + // the first version deliberi sees is only a baseline + ticketFrame(jetstream.CommitOperationUpdate, "r2", "closed", ownerDid, ""), + ticketFrame(jetstream.CommitOperationUpdate, "r3", "open", ownerDid, ""), + ) + if n := onlyFor(t, i, otherDid); n.Type != models.NotificationTypeIssueReopen { + t.Errorf("commenter got %q, want only the reopen", n.Type) + } +} + +func TestCommentOnAnotherReposTicketIgnored(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + const foreign = "at://did:plc:elsewhere/org.tangled.track.ticket/1" + if err := deldb.PutEntity(i.db, foreign, deldb.Entity{Title: "t", RepoDid: "did:plc:elsewhere", Kind: models.EntityKindIssue, Author: editorDid}); err != nil { + t.Fatalf("PutEntity: %v", err) + } + + mustProcess(t, i, commentFrame("c1", "r1", foreign, "", otherDid)) + + if got := countFor(t, i, ownerDid); got != 0 { + t.Fatalf("rows = %d, want 0 (a repo can't comment into another repo's ticket)", got) + } +} + +func TestCommentEditAndBackfillDoNotNotify(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + backfill(ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, "")), + backfill(commentFrame("c1", "r2", ticketUri, "", otherDid)), + frame(repoDid, jetstream.CommitOperationUpdate, commentNSID, "c1", "r3", commentJSON(ticketUri, "", otherDid)), + ) + if got := countFor(t, i, ownerDid); got != 0 { + t.Fatalf("rows = %d, want 0", got) + } + // a backfilled comment still makes its author a participant + mustProcess(t, i, ticketFrame(jetstream.CommitOperationUpdate, "r4", "closed", editorDid, "")) + if got := countFor(t, i, otherDid); got != 1 { + t.Errorf("backfilled commenter rows = %d, want the close", got) + } +} + +func TestEditorlessCommentAttributedToRepo(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, ""), + frame(repoDid, jetstream.CommitOperationCreate, commentNSID, "c1", "r2", + `{"$type":"org.tangled.feed.comment","createdAt":"2026-01-01T00:00:00Z",`+ + `"body":{"$type":"org.tangled.markup.markdown","text":"hi"},"subject":{"uri":"`+ticketUri+`","cid":"bafyfake"}}`), + ) + if n := byType(t, i, editorDid, models.NotificationTypeIssueCommented); n.ActorDid != repoDid { + t.Errorf("actor = %q, want the repo did", n.ActorDid) + } +} + +func TestCommentOnUnknownTicketTakesAuthorFromBobbin(t *testing.T) { + i := newTestIngester(t, fakeBobbin{authors: map[string]string{ticketUri: editorDid}}) + if err := deldb.PutEntity(i.db, ticketUri, deldb.Entity{Title: "old", RepoDid: repoDid, Kind: models.EntityKindIssue}); err != nil { + t.Fatal(err) + } + + mustProcess(t, i, commentFrame("c1", "r1", ticketUri, "", otherDid)) + + if n := onlyFor(t, i, editorDid); n.Type != models.NotificationTypeIssueCommented { + t.Errorf("type = %q", n.Type) + } + if got := deldb.GetEntity(i.db, ticketUri).Author; got != editorDid { + t.Errorf("cached author = %q", got) + } +} + +func TestSubscriptionResolution(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + const sub = "did:plc:sub" + resolve := func() (subs, ignored []string) { + subs, ignored, err := deldb.ListSubscribers(i.db, ticketUri, repoDid) + if err != nil { + t.Fatal(err) + } + return subs, ignored + } + + mustProcess(t, i, subscribe(sub, "3a", recordSubject(ticketUri), false)) + if subs, ign := resolve(); !slices.Equal(subs, []string{sub}) || len(ign) != 0 { + t.Fatalf("subscribed: subs=%v ignored=%v", subs, ign) + } + + mustProcess(t, i, subscribe(sub, "3b", recordSubject(ticketUri), true)) + if subs, ign := resolve(); len(subs) != 0 || !slices.Equal(ign, []string{sub}) { + t.Fatalf("ignored: subs=%v ignored=%v", subs, ign) + } + + // 3a2 sorts before 3b, so arriving late it can't undo the ignore + mustProcess(t, i, subscribe(sub, "3a2", recordSubject(ticketUri), false)) + if subs, ign := resolve(); len(subs) != 0 || !slices.Equal(ign, []string{sub}) { + t.Fatalf("stale duplicate: subs=%v ignored=%v", subs, ign) + } + + mustProcess(t, i, frame(sub, jetstream.CommitOperationDelete, subscriptionNSID, "3b", "rev-3b-del", "")) + if subs, ign := resolve(); !slices.Equal(subs, []string{sub}) || len(ign) != 0 { + t.Fatalf("after delete: subs=%v ignored=%v", subs, ign) + } + + mustProcess(t, i, + frame(sub, jetstream.CommitOperationDelete, subscriptionNSID, "3a", "rev-3a-del", ""), + frame(sub, jetstream.CommitOperationDelete, subscriptionNSID, "3a2", "rev-3a2-del", ""), + ) + if subs, ign := resolve(); len(subs) != 0 || len(ign) != 0 { + t.Fatalf("all deleted: subs=%v ignored=%v", subs, ign) + } + + mustProcess(t, i, subscribe(sub, "3c", repoSubject(repoDid), false)) + if subs, _ := resolve(); !slices.Equal(subs, []string{sub}) { + t.Fatalf("repo level: subs=%v", subs) + } +} + +func TestCollaboratorCreateAndDelete(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, collaboratorFrame(jetstream.CommitOperationCreate, "r1", collabDid)) + got, err := deldb.ListCollaborators(i.db, repoDid) + if err != nil || !slices.Equal(got, []string{collabDid}) { + t.Fatalf("collaborators = %v, %v", got, err) + } + + mustProcess(t, i, collaboratorFrame(jetstream.CommitOperationDelete, "r2", collabDid), collaboratorFrame(jetstream.CommitOperationCreate, "r1", collabDid)) + if got, _ := deldb.ListCollaborators(i.db, repoDid); len(got) != 0 { + t.Fatalf("collaborators after revoke = %v, want none even with the old create replayed", got) + } + + mustProcess(t, i, collaboratorFrame(jetstream.CommitOperationCreate, "r3", collabDid)) + if got, _ := deldb.ListCollaborators(i.db, repoDid); !slices.Equal(got, []string{collabDid}) { + t.Fatalf("collaborators after re-add = %v", got) + } +} + +func TestBootstrapHistoryStaysSilent(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + bootstrap := deldb.GetBootstrapUs(i.db) + if bootstrap == 0 { + t.Fatal("a fresh db must record when it was created") + } + + old := ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, "") + old.TimeUS = bootstrap - 1_000_000 + mustProcess(t, i, old) + if got := countFor(t, i, ownerDid); got != 0 { + t.Fatalf("owner rows = %d, want 0 for history from before the db existed", got) + } + + // history built the state silently, so this close still notifies + mustProcess(t, i, ticketFrame(jetstream.CommitOperationUpdate, "r2", "closed", editorDid, "")) + if n := onlyFor(t, i, ownerDid); n.Type != models.NotificationTypeIssueClosed { + t.Errorf("type = %q, want issue_closed", n.Type) + } +} + +func TestUpgradedDatabaseIsNotSilenced(t *testing.T) { + path := filepath.Join(t.TempDir(), "x.db") + d, err := deldb.Make(context.Background(), path) + if err != nil { + t.Fatal(err) + } + for _, q := range []string{ + `delete from ingest_bootstrap`, + `delete from migrations where name = 'ingest-bootstrap'`, + `insert into jetstream_cursor (id, last_time_us) values (0, 5)`, + } { + if _, err := d.Exec(q); err != nil { + t.Fatal(err) + } + } + d.Close() + + d, err = deldb.Make(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer d.Close() + if got := deldb.GetBootstrapUs(d); got != 0 { + t.Errorf("bootstrap = %d, want 0 for a db that already ingested", got) + } +} + +func TestEventAndCursorCommitTogether(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + e := ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, "") + mustProcess(t, i, e) + if us, ok, _ := deldb.GetCursor(i.db); !ok || us != e.TimeUS { + t.Fatalf("cursor = %d/%v, want %d", us, ok, e.TimeUS) + } + + // a failed write must not move the cursor or the rev, or a retry gets skipped + if _, err := i.db.Exec(`drop table repo_collaborators`); err != nil { + t.Fatal(err) + } + failing := collaboratorFrame(jetstream.CommitOperationCreate, "r2", collabDid) + if err := i.process(context.Background(), failing); err == nil { + t.Fatal("process succeeded without its table") + } + if us, _, _ := deldb.GetCursor(i.db); us != e.TimeUS { + t.Errorf("cursor = %d, want it left at %d", us, e.TimeUS) + } + if !deldb.RevIsNew(i.db, "at://"+repoDid+"/"+collaboratorNSID+"/"+collabDid, "r2") { + t.Error("the failed event's rev was kept, a retry would be skipped") + } +} + +func TestCursorNeverMovesBack(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + if _, ok, err := deldb.GetCursor(i.db); err != nil || ok { + t.Fatalf("fresh cursor ok = %v, %v, want none", ok, err) + } + + now := time.Now().UnixMicro() + for _, us := range []int64{now, now - 1_000_000} { + if err := deldb.SaveCursor(i.db, us); err != nil { + t.Fatal(err) + } + } + if got, _, err := deldb.GetCursor(i.db); err != nil || got != now { + t.Fatalf("cursor = %d, %v, want %d", got, err, now) + } +} + +func TestRunIngestsFromTheStreamAndMovesTheCursor(t *testing.T) { + e := ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, "") + frameJSON, err := json.Marshal(e) + if err != nil { + t.Fatal(err) + } + var query url.Values + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query = r.URL.Query() + con, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err != nil { + return + } + defer con.Close() + con.WriteMessage(websocket.TextMessage, frameJSON) + con.ReadMessage() + })) + defer srv.Close() + + i := newTestIngester(t, fakeBobbin{}) + i.endpoint = "ws" + strings.TrimPrefix(srv.URL, "http") + "/subscribe" + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- i.Run(ctx) }() + + deadline := time.Now().Add(5 * time.Second) + for deldb.GetEntity(i.db, ticketUri).Kind == "" { + if time.Now().After(deadline) { + t.Fatal("the streamed ticket never landed") + } + time.Sleep(10 * time.Millisecond) + } + if us, _, _ := deldb.GetCursor(i.db); us != e.TimeUS { + t.Errorf("cursor = %d, want %d", us, e.TimeUS) + } + if query.Get("cursor") != "0" || !slices.Equal(query["wantedEventTypes"], []string{"live", "historical"}) { + t.Errorf("query = %v, want a replay from 0 for live and historical", query) + } + cancel() + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestDisabledPrefsSuppressStateNotifications(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + prefs := models.DefaultNotificationPreferences(syntax.DID(ownerDid)) + prefs.IssueClosed = false + prefs.PullMerged = false + if err := deldb.UpsertNotificationPreferences(i.db, prefs); err != nil { + t.Fatal(err) + } + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, ""), + ticketFrame(jetstream.CommitOperationUpdate, "r2", "closed", editorDid, ""), + frame(repoDid, jetstream.CommitOperationCreate, ticketNSID, "2", "r3", ticketJSON("open", editorDid, pullExtra)), + frame(repoDid, jetstream.CommitOperationUpdate, ticketNSID, "2", "r4", ticketJSON("completed", editorDid, pullExtra)), + ) + for _, n := range notifsFor(t, i, ownerDid) { + if n.Type == models.NotificationTypeIssueClosed || n.Type == models.NotificationTypePullMerged { + t.Errorf("owner got %q with that pref off", n.Type) + } + } + if got := countFor(t, i, ownerDid); got != 2 { + t.Errorf("owner rows = %d, want the two created", got) + } +} + +func TestFanOutMentions(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + n := models.Notification{AtUri: "at://src", Type: models.NotificationTypeIssueCreated, ActorDid: editorDid, RepoDid: repoDid, EntityAt: ticketUri, EntityKind: models.EntityKindIssue} + + if err := i.fanOut(i.db, n, []string{ownerDid, otherDid}, []string{otherDid}); err != nil { + t.Fatal(err) + } + if got := onlyFor(t, i, otherDid); got.Type != models.NotificationTypeUserMentioned || got.EntityKind != models.EntityKindIssue { + t.Errorf("mentioned user got %+v, want the mention in place of the generic one", got) + } + if got := onlyFor(t, i, ownerDid); got.Type != models.NotificationTypeIssueCreated { + t.Errorf("owner got %q", got.Type) + } +} + +func TestMoreThanEightMentionsIsSpam(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + n := models.Notification{AtUri: "at://src", Type: models.NotificationTypeIssueCreated, ActorDid: editorDid, RepoDid: repoDid, EntityAt: ticketUri, EntityKind: models.EntityKindIssue} + var mentions []string + for k := range 9 { + mentions = append(mentions, "did:plc:m"+strconv.Itoa(k)) + } + + if err := i.fanOut(i.db, n, []string{ownerDid, mentions[0]}, mentions); err != nil { + t.Fatal(err) + } + for _, did := range mentions { + if k := countFor(t, i, did); did != mentions[0] && k != 0 { + t.Errorf("%s was told about a spammy mention", did) + } + } + // mentions[0] is a plain recipient too, so it keeps the generic notice + if n := onlyFor(t, i, mentions[0]); n.Type != models.NotificationTypeIssueCreated { + t.Errorf("type = %q, want the generic notification", n.Type) + } +} + +func TestMentionStillReachesAMutedThread(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, subscribe(otherDid, "s1", recordSubject(ticketUri), true)) + n := models.Notification{AtUri: "at://src", Type: models.NotificationTypeIssueCreated, ActorDid: editorDid, RepoDid: repoDid, EntityAt: ticketUri, EntityKind: models.EntityKindIssue} + + if err := i.fanOut(i.db, n, []string{otherDid}, []string{otherDid}); err != nil { + t.Fatal(err) + } + if got := onlyFor(t, i, otherDid); got.Type != models.NotificationTypeUserMentioned { + t.Errorf("type = %q", got.Type) + } +} + +func TestManifestSeedsOwnerAndName(t *testing.T) { + calls := 0 + i := newTestIngester(t, fakeBobbin{ownerCalls: &calls}) + const other = "did:plc:other" + + mustProcess(t, i, + frame(other, jetstream.CommitOperationCreate, repoManifestNSID, "self", "r1", + `{"$type":"org.tangled.repo.manifest","createdAt":"2026-01-01T00:00:00Z","declaration":{"owner":"`+ownerDid+`","name":"my-repo"}}`), + starFrame(editorDid, other, "star1"), + ) + + if got := deldb.GetRepoName(i.db, other); got != "my-repo" { + t.Errorf("cached name = %q, want my-repo", got) + } + if got := countFor(t, i, ownerDid); got != 1 { + t.Fatalf("owner rows = %d, want 1", got) + } + if calls != 0 { + t.Errorf("RepoOwner calls = %d, want 0 (manifest cache hit must not hit bobbin)", calls) + } +} + +func starFrame(actorDid, repoDid, rkey string) *jetstream.Event { + return frame(actorDid, jetstream.CommitOperationCreate, starNSID, rkey, "rev-"+rkey, + `{"$type":"org.tangled.feed.star","createdAt":"2026-01-01T00:00:00Z","subject":{"$type":"org.tangled.feed.star#repo","identity":"`+repoDid+`"}}`) +} + +func TestStarResolvesOwnerFromBobbinOnCacheMiss(t *testing.T) { + const other = "did:plc:other" + i := newTestIngester(t, fakeBobbin{owners: map[string]string{other: ownerDid}, repoNames: map[string]string{other: "my-repo"}}) + + mustProcess(t, i, starFrame(editorDid, other, "star1")) + + n := onlyFor(t, i, ownerDid) + if n.Type != models.NotificationTypeRepoStarred || n.ActorDid != editorDid || n.RepoDid != other { + t.Errorf("notification = %+v", n) + } + if got := deldb.GetRepoOwner(i.db, other); got != ownerDid { + t.Errorf("cached owner = %q, want %q", got, ownerDid) + } +} + +func TestStarKeepsCachedNameWhenNameMissing(t *testing.T) { + const other = "did:plc:other" + i := newTestIngester(t, fakeBobbin{owners: map[string]string{other: ownerDid}}) + if err := deldb.PutRepoName(i.db, other, "", "my-repo"); err != nil { + t.Fatalf("PutRepoName: %v", err) + } + + mustProcess(t, i, starFrame(editorDid, other, "star1")) + + if got := deldb.GetRepoName(i.db, other); got != "my-repo" { + t.Errorf("cached name = %q, want my-repo", got) + } +} + +func TestStarSkipsWhenOwnerUnresolvable(t *testing.T) { + i := newTestIngester(t, fakeBobbin{ownerErr: io.ErrUnexpectedEOF}) + + mustProcess(t, i, starFrame(editorDid, "did:plc:other", "star1")) + + if got := countFor(t, i, ownerDid); got != 0 { + t.Fatalf("owner rows = %d, want 0", got) + } +} + +func TestBackfilledStarAndFollowAreSilent(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + backfill(starFrame(editorDid, repoDid, "star1")), + backfill(frame(editorDid, jetstream.CommitOperationCreate, followNSID, ownerDid, "r1", `{"$type":"org.tangled.graph.follow","createdAt":"2026-01-01T00:00:00Z"}`)), + ) + if got := countFor(t, i, ownerDid); got != 0 { + t.Fatalf("owner rows = %d, want 0", got) + } +} + +func TestFollowNotifiesRkeySubject(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + + mustProcess(t, i, frame(editorDid, jetstream.CommitOperationCreate, followNSID, ownerDid, "r1", `{"$type":"org.tangled.graph.follow","createdAt":"2026-01-01T00:00:00Z"}`)) + + if n := onlyFor(t, i, ownerDid); n.Type != models.NotificationTypeFollowed || n.ActorDid != editorDid { + t.Errorf("notification = %+v", n) + } +} + +func TestUndoingAStarOrFollowTakesTheNoticeBack(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + const follow = `{"$type":"org.tangled.graph.follow","createdAt":"2026-01-01T00:00:00Z"}` + mustProcess(t, i, + starFrame(editorDid, repoDid, "star1"), + frame(editorDid, jetstream.CommitOperationCreate, followNSID, ownerDid, "r1", follow), + ) + if got := countFor(t, i, ownerDid); got != 2 { + t.Fatalf("owner rows = %d, want the star and the follow", got) + } + + mustProcess(t, i, + frame(editorDid, jetstream.CommitOperationDelete, starNSID, "star1", "r2", ""), + frame(editorDid, jetstream.CommitOperationDelete, followNSID, ownerDid, "r3", ""), + ) + if got := countFor(t, i, ownerDid); got != 0 { + t.Errorf("owner rows = %d, want both taken back", got) + } +} + +func TestPutEntityKeepsKnownFields(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + first := deldb.Entity{Title: "t1", RepoDid: repoDid, Kind: models.EntityKindPull, State: "open", Author: editorDid, Cid: "c1"} + if err := deldb.PutEntity(i.db, ticketUri, first); err != nil { + t.Fatalf("first put: %v", err) + } + if err := deldb.PutEntity(i.db, ticketUri, deldb.Entity{Title: "t2"}); err != nil { + t.Fatalf("second put: %v", err) + } + want := first + want.Title = "t2" + if got := deldb.GetEntity(i.db, ticketUri); got != want { + t.Errorf("entity = %+v, want %+v", got, want) + } +} + +func TestCreatedNotificationUsesEventTime(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + e := ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, "") + mustProcess(t, i, e) + if got := onlyFor(t, i, ownerDid).Created; got.UnixMicro()/1_000_000 != e.TimeUS/1_000_000 { + t.Errorf("created = %v, want the event time", got) + } +} + +func TestIngestedRevIsCommittedWithItsNotices(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, ""), + commentFrame("c1", "r2", ticketUri, "", otherDid), + ) + + rev, _, err := deldb.GetIngested(i.db, commentUri("c1")) + if err != nil || rev != "r2" { + t.Fatalf("comment ingested = %q/%v, want rev r2", rev, err) + } + if n := byType(t, i, ownerDid, models.NotificationTypeIssueCommented); n.AtUri != commentUri("c1") { + t.Errorf("notice key = %q, want the comment uri the rev was stored under", n.AtUri) + } + if rev, cid, err := deldb.GetIngested(i.db, ticketUri); err != nil || rev != "r1" || cid != "cid-r1" { + t.Errorf("ticket ingested = %q/%q/%v, want r1 and cid-r1", rev, cid, err) + } + if _, _, err := deldb.GetIngested(i.db, commentUri("never")); !errors.Is(err, sql.ErrNoRows) { + t.Errorf("a record that was never processed answered %v, want sql.ErrNoRows", err) + } +} + +func TestErasedTicketTakesItsNoticesWithIt(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, ""), + commentFrame("c1", "r2", ticketUri, "", otherDid), + ticketFrame(jetstream.CommitOperationUpdate, "r3", "closed", collabDid, ""), + ) + if got := countFor(t, i, ownerDid); got != 3 { + t.Fatalf("owner rows before the erase = %d, want created, commented and closed", got) + } + + mustProcess(t, i, frame(repoDid, jetstream.CommitOperationDelete, ticketNSID, "1", "r4", "")) + + for _, did := range []string{ownerDid, editorDid} { + if got := countFor(t, i, did); got != 0 { + t.Errorf("%s still has %d notices about the erased ticket", did, got) + } + } +} + +func TestErasedCommentTakesOnlyItsOwnNoticesWithIt(t *testing.T) { + i := newTestIngester(t, fakeBobbin{}) + mustProcess(t, i, + ticketFrame(jetstream.CommitOperationCreate, "r1", "open", editorDid, ""), + commentFrame("c1", "r2", ticketUri, "", otherDid), + commentFrame("c2", "r3", ticketUri, "", otherDid), + ) + + mustProcess(t, i, frame(repoDid, jetstream.CommitOperationDelete, commentNSID, "c1", "r4", "")) + + got := map[models.NotificationType]int{} + for _, n := range notifsFor(t, i, ownerDid) { + got[n.Type]++ + if n.AtUri == commentUri("c1") { + t.Errorf("the erased comment's notice is still there") + } + } + if got[models.NotificationTypeIssueCommented] != 1 || got[models.NotificationTypeIssueCreated] != 1 { + t.Errorf("owner notices = %v, want the created one and the other comment's", got) + } +} diff --git a/deliberi/invites.go b/deliberi/invites.go new file mode 100644 index 0000000000000000000000000000000000000000..464427ba311b3327216b8ca3d0874b7bfe844ec8 --- /dev/null +++ b/deliberi/invites.go @@ -0,0 +1,160 @@ +package deliberi + +import ( + "context" + "log/slog" + "maps" + "sync" + "time" + + "github.com/samber/lo" + deldb "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/models" + "tangled.org/core/orm" +) + +const ( + inviteReconcileInterval = 10 * time.Second + inviteFetchTimeout = 2 * time.Second +) + +var inviteNotificationTypes = []models.NotificationType{ + models.NotificationTypeCollaboratorInvited, +} + +type InviteSync struct { + db *deldb.DB + invites inviteLister + logger *slog.Logger + + mu sync.Mutex + lastRun map[string]time.Time + swept time.Time +} + +func NewInviteSync(database *deldb.DB, invites inviteLister, logger *slog.Logger) *InviteSync { + return &InviteSync{ + db: database, + invites: invites, + logger: logger, + lastRun: map[string]time.Time{}, + } +} + +func (s *InviteSync) Reconcile(ctx context.Context, recipientDid string) { + if !s.claim(recipientDid) { + return + } + + answer, err := s.fetch(ctx, recipientDid) + if err != nil { + s.logger.Warn("listing invites, keeping stored rows", "err", err, "recipient", recipientDid) + return + } + + stored, err := deldb.GetNotifications(s.db, recipientDid, 0, orm.FilterIn("type", inviteNotificationTypes)) + if err != nil { + s.logger.Warn("reading stored invite rows", "err", err, "recipient", recipientDid) + return + } + + offered := keySet(answer.Offers, func(o InviteOffer) string { return o.Uri }) + unread := keySet(answer.Pending, func(knot string) string { return knot }) + held := keySet(stored, func(n *models.Notification) string { return n.AtUri }) + + withdrawn := lo.Filter(stored, func(n *models.Notification, _ int) bool { + _, standing := offered[n.AtUri] + _, cold := unread[n.KnotDid] + return !standing && !cold && !answer.Truncated + }) + fresh := lo.Map( + lo.Filter(answer.Offers, func(o InviteOffer, _ int) bool { + _, cached := held[o.Uri] + return !cached && o.AddedBy != recipientDid + }), + func(o InviteOffer, _ int) *models.Notification { + return inviteNotification(recipientDid, o) + }, + ) + + if err := s.settle(recipientDid, withdrawn, fresh); err != nil { + s.logger.Warn("writing reconciled invite rows", "err", err, "recipient", recipientDid) + } +} + +func (s *InviteSync) settle(recipientDid string, withdrawn, fresh []*models.Notification) error { + if len(withdrawn) == 0 && len(fresh) == 0 { + return nil + } + + tx, err := s.db.Begin() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + if err := firstErr(withdrawn, func(n *models.Notification) error { + return deldb.DeleteNotification(tx, recipientDid, n.AtUri) + }); err != nil { + return err + } + if err := firstErr(fresh, func(n *models.Notification) error { + return deldb.CreateNotification(tx, n) + }); err != nil { + return err + } + + return tx.Commit() +} + +func (s *InviteSync) claim(recipientDid string) bool { + now := time.Now() + s.mu.Lock() + defer s.mu.Unlock() + + if now.Sub(s.swept) >= inviteReconcileInterval { + maps.DeleteFunc(s.lastRun, func(_ string, at time.Time) bool { + return now.Sub(at) >= inviteReconcileInterval + }) + s.swept = now + } + if at, seen := s.lastRun[recipientDid]; seen && now.Sub(at) < inviteReconcileInterval { + return false + } + s.lastRun[recipientDid] = now + return true +} + +func (s *InviteSync) fetch(ctx context.Context, recipientDid string) (InviteAnswer, error) { + ctx, cancel := context.WithTimeout(ctx, inviteFetchTimeout) + defer cancel() + + return s.invites.ListCollaboratorInvitesBy(ctx, recipientDid) +} + +func keySet[T any, K comparable](items []T, key func(T) K) map[K]struct{} { + return lo.SliceToMap(items, func(item T) (K, struct{}) { + return key(item), struct{}{} + }) +} + +func firstErr[T any](items []T, act func(T) error) error { + return lo.Reduce(items, func(err error, item T, _ int) error { + if err != nil { + return err + } + return act(item) + }, nil) +} + +func inviteNotification(recipientDid string, o InviteOffer) *models.Notification { + return &models.Notification{ + RecipientDid: recipientDid, + AtUri: o.Uri, + Type: models.NotificationTypeCollaboratorInvited, + ActorDid: o.AddedBy, + KnotDid: o.KnotDid, + RepoDid: o.RepoDid, + Created: o.CreatedAt, + } +} diff --git a/deliberi/invites_test.go b/deliberi/invites_test.go new file mode 100644 index 0000000000000000000000000000000000000000..78fb694918155808fca5f370f881469f8454d9aa --- /dev/null +++ b/deliberi/invites_test.go @@ -0,0 +1,254 @@ +package deliberi + +import ( + "context" + "errors" + "io" + "log/slog" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + deldb "tangled.org/core/deliberi/db" + models "tangled.org/core/deliberi/models" + "tangled.org/core/orm" +) + +const ( + invitee = "did:plc:invitee" + knotDid = "did:web:knot.example" + repoDid = "did:plc:repo" + collabUri = "at://" + repoDid + "/sh.tangled.repo.collaboratorInvite/" + invitee + otherUri = "at://" + repoDid + "/sh.tangled.repo.collaboratorInvite/did:plc:olaren" +) + +var offerMade = time.Date(2026, 8, 30, 12, 0, 0, 0, time.UTC) + +type fakeLister struct { + collabs []InviteOffer + pending []string + truncated bool + err error + calls *atomic.Int64 +} + +func (f *fakeLister) ListCollaboratorInvitesBy(ctx context.Context, subjectDid string) (InviteAnswer, error) { + if f.calls != nil { + f.calls.Add(1) + } + if f.err != nil { + return InviteAnswer{}, f.err + } + return InviteAnswer{Offers: f.collabs, Pending: f.pending, Truncated: f.truncated}, nil +} + +func offerAt(uri, admin string) InviteOffer { + return InviteOffer{ + Uri: uri, + KnotDid: knotDid, + RepoDid: repoDid, + AddedBy: admin, + CreatedAt: offerMade, + } +} + +func newTestInviteSync(t *testing.T, l inviteLister) *InviteSync { + t.Helper() + database, err := deldb.Make(context.Background(), filepath.Join(t.TempDir(), "x.db")) + if err != nil { + t.Fatalf("make db: %v", err) + } + t.Cleanup(func() { database.Close() }) + return NewInviteSync(database, l, slog.New(slog.NewTextHandler(io.Discard, nil))) +} + +func inviteRows(t *testing.T, s *InviteSync, did string) []*models.Notification { + t.Helper() + rows, err := deldb.GetNotifications(s.db, did, 0, orm.FilterIn("type", inviteNotificationTypes)) + if err != nil { + t.Fatalf("read invite rows: %v", err) + } + return rows +} + +func expire(s *InviteSync, did string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.lastRun, did) +} + +func TestReconcileCreatesRowFromOffer(t *testing.T) { + s := newTestInviteSync(t, &fakeLister{ + collabs: []InviteOffer{offerAt(collabUri, "did:plc:boltless")}, + }) + + s.Reconcile(context.Background(), invitee) + + rows := inviteRows(t, s, invitee) + if len(rows) != 1 { + t.Fatalf("row count = %d, want 1 (one per offer)", len(rows)) + } + + row := rows[0] + if row.AtUri != collabUri { + t.Errorf("atUri = %q, want the offer's record %q", row.AtUri, collabUri) + } + if row.Type != models.NotificationTypeCollaboratorInvited { + t.Errorf("type = %q, want %q", row.Type, models.NotificationTypeCollaboratorInvited) + } + if row.ActorDid != "did:plc:boltless" { + t.Errorf("actorDid = %q, want inviting admin", row.ActorDid) + } + if row.RepoDid != repoDid { + t.Errorf("repoDid = %q, want %q", row.RepoDid, repoDid) + } + if row.KnotDid != knotDid { + t.Errorf("knotDid = %q, want knot serving this offer", row.KnotDid) + } + if !row.Created.Equal(offerMade) { + t.Errorf("created = %s, want offer's own %s", row.Created, offerMade) + } +} + +func TestReconcileDeletesOnlyWhatBobbinVouchesFor(t *testing.T) { + cases := []struct { + name string + then func(l *fakeLister) + want int + }{ + {"offer withdrawn", func(l *fakeLister) { l.collabs = nil }, 0}, + { + "offering knot pending", + func(l *fakeLister) { l.collabs, l.pending = nil, []string{knotDid} }, + 1, + }, + { + "answer truncated", + func(l *fakeLister) { + l.collabs, l.truncated = []InviteOffer{offerAt(otherUri, "did:plc:akshay")}, true + }, + 2, + }, + {"bobbin can't answer", func(l *fakeLister) { l.err = errors.New("bobbin unreachable") }, 1}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + lister := &fakeLister{collabs: []InviteOffer{offerAt(collabUri, "did:plc:akshay")}} + s := newTestInviteSync(t, lister) + s.Reconcile(context.Background(), invitee) + if got := len(inviteRows(t, s, invitee)); got != 1 { + t.Fatalf("row count after offer = %d, want 1", got) + } + + tc.then(lister) + expire(s, invitee) + s.Reconcile(context.Background(), invitee) + + if got := len(inviteRows(t, s, invitee)); got != tc.want { + t.Fatalf("row count once %s = %d, want %d", tc.name, got, tc.want) + } + }) + } +} + +func TestReconcileDeletesOncePendingKnotIsRead(t *testing.T) { + lister := &fakeLister{ + collabs: []InviteOffer{offerAt(collabUri, "did:plc:akshay")}, + pending: []string{knotDid}, + } + s := newTestInviteSync(t, lister) + s.Reconcile(context.Background(), invitee) + + lister.collabs, lister.pending = nil, nil + expire(s, invitee) + s.Reconcile(context.Background(), invitee) + + if got := len(inviteRows(t, s, invitee)); got != 0 { + t.Fatalf("row count once bobbin has read knot = %d, want 0", got) + } +} + +func TestReconcileReofferAfterWithdrawal(t *testing.T) { + lister := &fakeLister{collabs: []InviteOffer{offerAt(collabUri, "did:plc:akshay")}} + s := newTestInviteSync(t, lister) + s.Reconcile(context.Background(), invitee) + + lister.collabs = nil + expire(s, invitee) + s.Reconcile(context.Background(), invitee) + + reoffer := offerAt(collabUri, "did:plc:akshay") + reoffer.CreatedAt = time.Date(2026, 8, 31, 9, 15, 0, 0, time.UTC) + lister.collabs = []InviteOffer{reoffer} + expire(s, invitee) + s.Reconcile(context.Background(), invitee) + + rows := inviteRows(t, s, invitee) + if len(rows) != 1 { + t.Fatalf("row count after re-offer = %d, want 1", len(rows)) + } + if !rows[0].Created.Equal(reoffer.CreatedAt) { + t.Fatalf("created = %s, want re-offer's %s", rows[0].Created, reoffer.CreatedAt) + } +} + +func TestReconcileSkipsSelfInvite(t *testing.T) { + s := newTestInviteSync(t, &fakeLister{collabs: []InviteOffer{offerAt(collabUri, invitee)}}) + + s.Reconcile(context.Background(), invitee) + + if got := len(inviteRows(t, s, invitee)); got != 0 { + t.Fatalf("row count for self-invite = %d, want 0", got) + } +} + +func TestReconcileAsksBobbinOncePerWindow(t *testing.T) { + var calls atomic.Int64 + s := newTestInviteSync(t, &fakeLister{calls: &calls}) + + var racers sync.WaitGroup + racers.Add(16) + for range 16 { + go func() { + defer racers.Done() + s.Reconcile(context.Background(), invitee) + }() + } + racers.Wait() + if got := calls.Load(); got != 1 { + t.Fatalf("listings across 16 concurrent bells = %d, want 1 (one pass)", got) + } + + expire(s, invitee) + s.Reconcile(context.Background(), invitee) + if got := calls.Load(); got != 2 { + t.Fatalf("listings once window passes = %d, want 2", got) + } +} + +func TestReconcileForgetsRecipientsPastWindow(t *testing.T) { + s := newTestInviteSync(t, &fakeLister{}) + + s.Reconcile(context.Background(), "did:plc:one") + s.Reconcile(context.Background(), "did:plc:two") + + s.mu.Lock() + stale := time.Now().Add(-2 * inviteReconcileInterval) + s.lastRun["did:plc:one"] = stale + s.swept = stale + s.mu.Unlock() + + s.Reconcile(context.Background(), "did:plc:three") + + s.mu.Lock() + defer s.mu.Unlock() + if _, held := s.lastRun["did:plc:one"]; held { + t.Fatal("recipient who didn't read this window is still held in debounce") + } + if _, held := s.lastRun["did:plc:two"]; !held { + t.Fatal("sweep took recipient still inside window") + } +} diff --git a/deliberi/kv/kv.go b/deliberi/kv/kv.go new file mode 100644 index 0000000000000000000000000000000000000000..1def6c24d97e436d4f199594e5a6c07cbd0f8f2c --- /dev/null +++ b/deliberi/kv/kv.go @@ -0,0 +1,108 @@ +// Package kv keeps the EMAIL_DID namespace in sync with deliberi's email mutations, fire-and-forget. +package kv + +import ( + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "strings" + "time" +) + +type EmailKV interface { + PutEmailDid(email, did string) + SetPrimaryEmail(did, email string) + DeleteEmailKey(email string) + DeletePrimaryEmail(did string) +} + +type Client struct { + apiToken string + baseURL string + client *http.Client + logger *slog.Logger +} + +func NewClient(token, accountID, namespaceID string, logger *slog.Logger) *Client { + if logger == nil { + logger = slog.Default() + } + return &Client{ + apiToken: token, + baseURL: fmt.Sprintf( + "https://api.cloudflare.com/client/v4/accounts/%s/storage/kv/namespaces/%s/", + accountID, namespaceID, + ), + client: &http.Client{Timeout: 10 * time.Second}, + logger: logger, + } +} + +func (c *Client) PutEmailDid(email, did string) { + c.write("put email→did", strings.ToLower(email), did) +} + +func (c *Client) SetPrimaryEmail(did, email string) { + c.write("put primary→email", "__primary:"+did, email) +} + +func (c *Client) DeleteEmailKey(email string) { + c.delete("delete email key", strings.ToLower(email)) +} + +func (c *Client) DeletePrimaryEmail(did string) { + c.delete("delete primary email", "__primary:"+did) +} + +func (c *Client) write(action, key, value string) { + if c == nil { + return + } + target := c.baseURL + "values/" + url.PathEscape(key) + req, err := http.NewRequest(http.MethodPut, target, strings.NewReader(value)) + if err != nil { + c.logger.Warn("kv "+action+" failed to build request", "err", err, "key", key) + return + } + req.Header.Set("Authorization", "Bearer "+c.apiToken) + req.Header.Set("Content-Type", "text/plain") + resp, err := c.client.Do(req) + if err != nil { + c.logger.Warn("kv "+action+" request failed", "err", err, "key", key) + return + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + c.logger.Warn("kv "+action+" rejected", "status", resp.Status, "key", key, "body", string(body)) + return + } + c.logger.Info("kv "+action+" ok", "key", key) +} + +func (c *Client) delete(action, key string) { + if c == nil { + return + } + target := c.baseURL + "values/" + url.PathEscape(key) + req, err := http.NewRequest(http.MethodDelete, target, nil) + if err != nil { + c.logger.Warn("kv "+action+" failed to build request", "err", err, "key", key) + return + } + req.Header.Set("Authorization", "Bearer "+c.apiToken) + resp, err := c.client.Do(req) + if err != nil { + c.logger.Warn("kv "+action+" request failed", "err", err, "key", key) + return + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + c.logger.Warn("kv "+action+" rejected", "status", resp.Status, "key", key, "body", string(body)) + return + } + c.logger.Info("kv "+action+" ok", "key", key) +} diff --git a/deliberi/kv/kv_test.go b/deliberi/kv/kv_test.go new file mode 100644 index 0000000000000000000000000000000000000000..be4831a5cbc21515cf74730a38bea5e6df2f6376 --- /dev/null +++ b/deliberi/kv/kv_test.go @@ -0,0 +1,190 @@ +package kv + +import ( + "bytes" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +type recordedRequest struct { + method string + path string + body string + auth string + ctype string +} + +func newRecordingClient(t *testing.T) (*Client, func() []recordedRequest) { + t.Helper() + + var mu sync.Mutex + var reqs []recordedRequest + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + mu.Lock() + reqs = append(reqs, recordedRequest{ + method: r.Method, + path: r.URL.Path, + body: string(body), + auth: r.Header.Get("Authorization"), + ctype: r.Header.Get("Content-Type"), + }) + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + + c := &Client{ + apiToken: "test-token", + baseURL: srv.URL + "/accounts/acct/storage/kv/namespaces/ns/", + client: srv.Client(), + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + + snapshot := func() []recordedRequest { + mu.Lock() + defer mu.Unlock() + out := make([]recordedRequest, len(reqs)) + copy(out, reqs) + return out + } + return c, snapshot +} + +func TestPutEmailDid(t *testing.T) { + c, snapshot := newRecordingClient(t) + + c.PutEmailDid("Bob@Example.com", "did:plc:bob") + + got := snapshot() + if len(got) != 1 { + t.Fatalf("got %d requests, want 1", len(got)) + } + r := got[0] + if r.method != http.MethodPut { + t.Errorf("method = %s, want PUT", r.method) + } + // url.PathEscape leaves @ unescaped; the worker looks up the literal key + if !strings.HasSuffix(r.path, "/values/bob@example.com") { + t.Errorf("path = %s, want suffix /values/bob@example.com", r.path) + } + if r.body != "did:plc:bob" { + t.Errorf("body = %q, want did:plc:bob", r.body) + } + if r.auth != "Bearer test-token" { + t.Errorf("auth = %q, want Bearer test-token", r.auth) + } + if r.ctype != "text/plain" { + t.Errorf("content-type = %q, want text/plain", r.ctype) + } + if !strings.Contains(r.path, "bob@example.com") { + t.Errorf("key was not lowercased: %s", r.path) + } +} + +func TestSetPrimaryEmail(t *testing.T) { + c, snapshot := newRecordingClient(t) + + c.SetPrimaryEmail("did:plc:bob", "bob@example.com") + + got := snapshot() + if len(got) != 1 { + t.Fatalf("got %d requests, want 1", len(got)) + } + r := got[0] + if r.method != http.MethodPut { + t.Errorf("method = %s, want PUT", r.method) + } + if !strings.HasSuffix(r.path, "/values/__primary:did:plc:bob") { + t.Errorf("path = %s, want suffix /values/__primary:did:plc:bob", r.path) + } + if r.body != "bob@example.com" { + t.Errorf("body = %q, want bob@example.com", r.body) + } +} + +func TestDeleteEmailKey(t *testing.T) { + c, snapshot := newRecordingClient(t) + + c.DeleteEmailKey("Bob@Example.com") + + got := snapshot() + if len(got) != 1 { + t.Fatalf("got %d requests, want 1", len(got)) + } + r := got[0] + if r.method != http.MethodDelete { + t.Errorf("method = %s, want DELETE", r.method) + } + if !strings.HasSuffix(r.path, "/values/bob@example.com") { + t.Errorf("path = %s, want suffix /values/bob@example.com", r.path) + } + if r.auth != "Bearer test-token" { + t.Errorf("auth = %q, want Bearer test-token", r.auth) + } + if r.ctype != "" { + t.Errorf("content-type = %q, want none on a bodyless DELETE", r.ctype) + } +} + +func TestFailureLoggedNotFatal(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(srv.Close) + + c := &Client{ + apiToken: "test-token", + baseURL: srv.URL + "/", + client: srv.Client(), + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + + c.PutEmailDid("bob@example.com", "did:plc:bob") + c.DeleteEmailKey("bob@example.com") +} + +func TestSuccessLogged(t *testing.T) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + + c := &Client{apiToken: "t", baseURL: srv.URL + "/", client: srv.Client(), logger: logger} + c.PutEmailDid("bob@example.com", "did:plc:bob") + c.DeleteEmailKey("bob@example.com") + + out := buf.String() + for _, want := range []string{"put email", "delete email", "bob@example.com"} { + if !strings.Contains(out, want) { + t.Errorf("missing %q in success log:\n%s", want, out) + } + } +} + +func TestNewClientSetsTimeout(t *testing.T) { + c := NewClient("t", "acct", "ns", slog.New(slog.NewTextHandler(io.Discard, nil))) + if c.client.Timeout != 10*time.Second { + t.Errorf("client timeout = %v, want 10s", c.client.Timeout) + } + if c.baseURL == "" || c.apiToken != "t" { + t.Errorf("client url/token not configured: %q %q", c.baseURL, c.apiToken) + } +} + +func TestNilClientSafe(t *testing.T) { + var c *Client + + c.PutEmailDid("bob@example.com", "did:plc:bob") + c.SetPrimaryEmail("did:plc:bob", "bob@example.com") + c.DeleteEmailKey("bob@example.com") +} diff --git a/deliberi/mailer/mailer.go b/deliberi/mailer/mailer.go new file mode 100644 index 0000000000000000000000000000000000000000..bce65100c5892d898db988f0ceb4f30296c901b9 --- /dev/null +++ b/deliberi/mailer/mailer.go @@ -0,0 +1,77 @@ +package mailer + +import ( + "errors" + "fmt" + "log/slog" + "strings" + + "tangled.org/core/deliberi/config" + appviewemail "tangled.org/core/deliberi/email" +) + +type Sender struct { + apiKey string + from string + newsletterSegmentId string + logger *slog.Logger +} + +type Message struct { + To string + Subject string + Text string + Html string + Headers map[string]string +} + +func New(resend config.ResendConfig, logger *slog.Logger) *Sender { + return &Sender{ + apiKey: resend.ApiKey, + from: resend.SentFrom, + newsletterSegmentId: resend.NewsletterSegmentId, + logger: logger, + } +} + +// Domain is what message ids minted for this sender end in +func (s *Sender) Domain() string { + _, domain, _ := strings.Cut(s.from, "@") + return domain +} + +// Send delivers one email. with no Resend key it prints to stdout instead of +// sending, and reports success so callers proceed normally in dev. +func (s *Sender) Send(msg Message) error { + from := "Tangled <" + s.from + ">" + + if s.apiKey == "" { + s.logger.Info("resend api key unset; writing email to stdout", "to", msg.To, "subject", msg.Subject) + var headers strings.Builder + for k, v := range msg.Headers { + fmt.Fprintf(&headers, "%s: %s\n", strings.ToLower(k), v) + } + fmt.Printf("\n=== deliberi email (stdout, no resend key) ===\nfrom: %s\nto: %s\nsubject: %s\n%s\n%s\n===============================================\n\n", from, msg.To, msg.Subject, headers.String(), msg.Text) + return nil + } + + return appviewemail.SendEmail(appviewemail.Email{ + APIKey: s.apiKey, + From: from, + To: msg.To, + Subject: msg.Subject, + Text: msg.Text, + Html: msg.Html, + Headers: msg.Headers, + }) +} + +var ErrNewsletterUnconfigured = errors.New("resend api key or newsletter segment id unset") + +// resend owns the mailing list itself +func (s *Sender) AddNewsletterContact(addr string) error { + if s.apiKey == "" || s.newsletterSegmentId == "" { + return ErrNewsletterUnconfigured + } + return appviewemail.AddNewsletterContact(s.apiKey, s.newsletterSegmentId, addr) +} diff --git a/deliberi/markup/atlink.go b/deliberi/markup/atlink.go new file mode 100644 index 0000000000000000000000000000000000000000..5c4318537bb7d4ed599c9f0a5986fe74ef4dcac5 --- /dev/null +++ b/deliberi/markup/atlink.go @@ -0,0 +1,121 @@ +// heavily inspired by: https://github.com/kaleocheng/goldmark-extensions + +package markup + +import ( + "regexp" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +// An AtNode struct represents an AtNode +type AtNode struct { + Handle string + ast.BaseInline +} + +var _ ast.Node = &AtNode{} + +// Dump implements Node.Dump. +func (n *AtNode) Dump(source []byte, level int) { + ast.DumpHelper(n, source, level, nil, nil) +} + +// KindAt is a NodeKind of the At node. +var KindAt = ast.NewNodeKind("At") + +// Kind implements Node.Kind. +func (n *AtNode) Kind() ast.NodeKind { + return KindAt +} + +var atRegexp = regexp.MustCompile(`(^|\s|\()(@)([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\b)`) +var markdownLinkRegexp = regexp.MustCompile(`(?ms)\[.*\]\(.*\)`) + +type atParser struct{} + +// NewAtParser return a new InlineParser that parses +// at expressions. +func NewAtParser() parser.InlineParser { + return &atParser{} +} + +func (s *atParser) Trigger() []byte { + return []byte{'@'} +} + +func (s *atParser) Parse(parent ast.Node, block text.Reader, pc parser.Context) ast.Node { + line, segment := block.PeekLine() + m := atRegexp.FindSubmatchIndex(line) + if m == nil { + return nil + } + + // Check for all links in the markdown to see if the handle found is inside one + linksIndexes := markdownLinkRegexp.FindAllIndex(block.Source(), -1) + for _, linkMatch := range linksIndexes { + if linkMatch[0] < segment.Start && segment.Start < linkMatch[1] { + return nil + } + } + + atSegment := text.NewSegment(segment.Start, segment.Start+m[1]) + block.Advance(m[1]) + node := &AtNode{} + node.AppendChild(node, ast.NewTextSegment(atSegment)) + node.Handle = string(atSegment.Value(block.Source())[1:]) + return node +} + +// atHtmlRenderer is a renderer.NodeRenderer implementation that +// renders At nodes. +type atHtmlRenderer struct { + html.Config +} + +// NewAtHTMLRenderer returns a new AtHTMLRenderer. +func NewAtHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { + r := &atHtmlRenderer{ + Config: html.NewConfig(), + } + for _, opt := range opts { + opt.SetHTMLOption(&r.Config) + } + return r +} + +// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. +func (r *atHtmlRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(KindAt, r.renderAt) +} + +func (r *atHtmlRenderer) renderAt(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + w.WriteString(``) + } else { + w.WriteString("") + } + return ast.WalkContinue, nil +} + +type atExt struct{} + +// At is an extension that allow you to use at expression like '@user.bsky.social' . +var AtExt = &atExt{} + +func (e *atExt) Extend(m goldmark.Markdown) { + m.Parser().AddOptions(parser.WithInlineParsers( + util.Prioritized(NewAtParser(), 500), + )) + m.Renderer().AddOptions(renderer.WithNodeRenderers( + util.Prioritized(NewAtHTMLRenderer(), 500), + )) +} diff --git a/deliberi/mentions.go b/deliberi/mentions.go new file mode 100644 index 0000000000000000000000000000000000000000..c41c258445ee329dc0b8ea62634c2a7ece183b39 --- /dev/null +++ b/deliberi/mentions.go @@ -0,0 +1,61 @@ +package deliberi + +import ( + "context" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + + markupext "tangled.org/core/deliberi/markup" +) + +var mentionParser = goldmark.New(goldmark.WithExtensions(markupext.AtExt)) + +func extractMentionHandles(body string) []string { + if body == "" { + return nil + } + + doc := mentionParser.Parser().Parse(text.NewReader([]byte(body))) + + var handles []string + seen := make(map[string]struct{}) + _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering || n.Kind() != markupext.KindAt { + return ast.WalkContinue, nil + } + handle := n.(*markupext.AtNode).Handle + if _, ok := seen[handle]; ok { + return ast.WalkContinue, nil + } + seen[handle] = struct{}{} + handles = append(handles, handle) + return ast.WalkContinue, nil + }) + return handles +} + +// mentionDids resolves the @handles in body to DIDs, dropping the author and +// any handle that fails to resolve. +func (i *Ingester) mentionDids(ctx context.Context, authorDid, body string) []string { + handles := extractMentionHandles(body) + if len(handles) == 0 { + return nil + } + + var dids []string + for _, handle := range handles { + id, err := i.idResolver.ResolveIdent(ctx, handle) + if err != nil { + i.logger.Warn("resolving mention", "err", err, "handle", handle) + continue + } + did := id.DID.String() + if did == authorDid { + continue + } + dids = append(dids, did) + } + return dids +} diff --git a/deliberi/mentions_test.go b/deliberi/mentions_test.go new file mode 100644 index 0000000000000000000000000000000000000000..cd98c1a9bc891a3a7b2ef071ff347b847dd1edfd --- /dev/null +++ b/deliberi/mentions_test.go @@ -0,0 +1,34 @@ +package deliberi + +import ( + "reflect" + "testing" +) + +func TestExtractMentionHandles(t *testing.T) { + got := extractMentionHandles("hey @bob.tngl.sh and @carol.tngl.sh, thanks!") + want := []string{"bob.tngl.sh", "carol.tngl.sh"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +} + +func TestExtractMentionHandlesDedupes(t *testing.T) { + got := extractMentionHandles("@bob.tngl.sh ping @bob.tngl.sh again") + want := []string{"bob.tngl.sh"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +} + +func TestExtractMentionHandlesNone(t *testing.T) { + if got := extractMentionHandles("no mentions in this body"); got != nil { + t.Fatalf("got %v, want nil", got) + } +} + +func TestExtractMentionHandlesEmptyBody(t *testing.T) { + if got := extractMentionHandles(""); got != nil { + t.Fatalf("got %v, want nil", got) + } +} diff --git a/deliberi/models/email.go b/deliberi/models/email.go new file mode 100644 index 0000000000000000000000000000000000000000..d3189abab06439c9e713236f6cd3665ce3065cae --- /dev/null +++ b/deliberi/models/email.go @@ -0,0 +1,14 @@ +package models + +import "time" + +type Email struct { + ID int64 + Did string + Address string + Verified bool + Primary bool + VerificationCode string + LastSent *time.Time + CreatedAt time.Time +} diff --git a/deliberi/models/notifications.go b/deliberi/models/notifications.go new file mode 100644 index 0000000000000000000000000000000000000000..1209b15688c94f3fef90a8fda38f31879482c295 --- /dev/null +++ b/deliberi/models/notifications.go @@ -0,0 +1,229 @@ +package models + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/idresolver" +) + +type NotificationType string + +const ( + NotificationTypeRepoStarred NotificationType = "repo_starred" + NotificationTypeIssueCreated NotificationType = "issue_created" + NotificationTypeIssueCommented NotificationType = "issue_commented" + NotificationTypePullCreated NotificationType = "pull_created" + NotificationTypePullCommented NotificationType = "pull_commented" + NotificationTypeFollowed NotificationType = "followed" + NotificationTypeUserMentioned NotificationType = "user_mentioned" + NotificationTypeCollaboratorInvited NotificationType = "collaborator_invited" + NotificationTypeIssueClosed NotificationType = "issue_closed" + NotificationTypeIssueReopen NotificationType = "issue_reopen" + NotificationTypePullMerged NotificationType = "pull_merged" + NotificationTypePullClosed NotificationType = "pull_closed" + NotificationTypePullReopen NotificationType = "pull_reopen" +) + +// issues and pulls share one record type (org.tangled.track.ticket), so the +// kind is persisted rather than read off the entity's collection +type EntityKind string + +const ( + EntityKindIssue EntityKind = "issue" + EntityKindPull EntityKind = "pull" +) + +var SocialNotificationTypes = []NotificationType{ + NotificationTypeRepoStarred, + NotificationTypeFollowed, +} + +var WorkNotificationTypes = []NotificationType{ + NotificationTypeIssueCreated, + NotificationTypeIssueCommented, + NotificationTypePullCreated, + NotificationTypePullCommented, + NotificationTypeUserMentioned, + NotificationTypeCollaboratorInvited, + NotificationTypeIssueClosed, + NotificationTypeIssueReopen, + NotificationTypePullMerged, + NotificationTypePullClosed, + NotificationTypePullReopen, +} + +// email digest types; social types (repo_starred, followed) are excluded. +var EmailNotificationTypes = []NotificationType{ + NotificationTypeIssueCreated, + NotificationTypeIssueCommented, + NotificationTypePullCreated, + NotificationTypePullCommented, + NotificationTypeUserMentioned, + NotificationTypeIssueClosed, + NotificationTypeIssueReopen, + NotificationTypePullMerged, + NotificationTypePullClosed, + NotificationTypePullReopen, +} + +type Notification struct { + ID int64 + RecipientDid string + AtUri string // source record (comment/ticket/star/follow), or ticket#rev for state changes; dedupe key + Type NotificationType + ActorDid string + RepoDid string + KnotDid string + RepoName string // not stored; filled from the repo cache when the digest reads + RepoOwner string // not stored; filled from the repo cache when the digest reads + EntityAt string // related ticket at-uri (for links); source itself for tickets + EntityKind EntityKind + EntityTitle string + Read bool + Emailed bool + Created time.Time +} + +func (n *Notification) Icon() string { + switch n.Type { + case NotificationTypeRepoStarred: + return "star" + case NotificationTypeIssueCreated, NotificationTypeIssueReopen: + return "circle-dot" + case NotificationTypeIssueClosed: + return "ban" + case NotificationTypePullMerged: + return "git-merge" + case NotificationTypePullClosed: + return "git-pull-request-closed" + case NotificationTypeIssueCommented, NotificationTypePullCommented: + return "message-square" + case NotificationTypePullCreated, NotificationTypePullReopen: + return "git-pull-request-create" + case NotificationTypeFollowed: + return "user-plus" + case NotificationTypeUserMentioned: + return "at-sign" + case NotificationTypeCollaboratorInvited: + return "users-round" + default: + return "" + } +} + +func resolveHandle(res *idresolver.Resolver, did string) string { + if id, err := res.ResolveIdent(context.Background(), did); err == nil && !id.Handle.IsInvalidHandle() { + return id.Handle.String() + } + return did +} + +// a repo did has no handle of its own, so a repo is named by its owner +func (n *Notification) RepoOwnerHandle(res *idresolver.Resolver) string { + if n.RepoOwner == "" { + return n.RepoDid + } + return resolveHandle(res, n.RepoOwner) +} + +func (n *Notification) URL(res *idresolver.Resolver) string { + switch n.Type { + case NotificationTypeFollowed: + return "/" + resolveHandle(res, n.ActorDid) + } + if n.RepoDid == "" || n.RepoName == "" { + return "" + } + repoHandle := n.RepoOwnerHandle(res) + slug := strings.ToLower(n.RepoName) + switch { + case n.EntityAt == "": + case n.EntityKind == EntityKindIssue: + return fmt.Sprintf("/%s/%s/issues/%s", repoHandle, slug, entityID(n.EntityAt)) + case n.EntityKind == EntityKindPull: + return fmt.Sprintf("/%s/%s/pulls/%s", repoHandle, slug, entityID(n.EntityAt)) + } + return fmt.Sprintf("/%s/%s", repoHandle, slug) +} + +// the web app routes issues and pulls by their record key, but the entity is +// often stored as the record's full at-uri +func entityID(entity string) string { + if uri, err := syntax.ParseATURI(entity); err == nil && uri.RecordKey() != "" { + return uri.RecordKey().String() + } + return entity +} + +func Category(t NotificationType) string { + for _, st := range SocialNotificationTypes { + if st == t { + return "social" + } + } + return "work" +} + +type NotificationPreferences struct { + ID int64 + UserDid syntax.DID + RepoStarred bool + IssueCreated bool + IssueCommented bool + PullCreated bool + PullCommented bool + Followed bool + UserMentioned bool + EmailNotifications bool + IssueClosed bool + PullMerged bool +} + +func (prefs *NotificationPreferences) ShouldNotify(t NotificationType) bool { + switch t { + case NotificationTypeRepoStarred: + return prefs.RepoStarred + case NotificationTypeIssueCreated: + return prefs.IssueCreated + case NotificationTypeIssueCommented: + return prefs.IssueCommented + case NotificationTypePullCreated: + return prefs.PullCreated + case NotificationTypePullCommented: + return prefs.PullCommented + case NotificationTypeFollowed: + return prefs.Followed + case NotificationTypeUserMentioned: + return prefs.UserMentioned + case NotificationTypeIssueClosed: + return prefs.IssueClosed + case NotificationTypeIssueReopen: + return prefs.IssueCreated + case NotificationTypePullMerged, NotificationTypePullClosed: + return prefs.PullMerged + case NotificationTypePullReopen: + return prefs.PullCreated + default: + return false + } +} + +func DefaultNotificationPreferences(user syntax.DID) *NotificationPreferences { + return &NotificationPreferences{ + UserDid: user, + RepoStarred: true, + IssueCreated: true, + IssueCommented: true, + PullCreated: true, + PullCommented: true, + Followed: true, + UserMentioned: true, + EmailNotifications: false, + IssueClosed: true, + PullMerged: true, + } +} diff --git a/deliberi/models/notifications_test.go b/deliberi/models/notifications_test.go new file mode 100644 index 0000000000000000000000000000000000000000..4ccdb89c11b05b108f98fb78d24702dfa76e8f54 --- /dev/null +++ b/deliberi/models/notifications_test.go @@ -0,0 +1,98 @@ +package models + +import ( + "slices" + "testing" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "tangled.org/core/idresolver" +) + +func TestStateTypesFollowLegacyPreferences(t *testing.T) { + prefs := DefaultNotificationPreferences("did:plc:me") + prefs.IssueClosed = false + prefs.PullMerged = false + + for typ, want := range map[NotificationType]bool{ + NotificationTypeIssueClosed: false, + NotificationTypePullMerged: false, + NotificationTypePullClosed: false, + NotificationTypeIssueReopen: true, + NotificationTypePullReopen: true, + } { + if got := prefs.ShouldNotify(typ); got != want { + t.Errorf("ShouldNotify(%s) = %v, want %v", typ, got, want) + } + } + + prefs.IssueCreated = false + prefs.PullCreated = false + if prefs.ShouldNotify(NotificationTypeIssueReopen) || prefs.ShouldNotify(NotificationTypePullReopen) { + t.Error("reopen types ride on the created prefs") + } +} + +func TestStateTypesAreWorkEmailAndHaveIcons(t *testing.T) { + for _, typ := range []NotificationType{ + NotificationTypeIssueClosed, NotificationTypeIssueReopen, + NotificationTypePullMerged, NotificationTypePullClosed, NotificationTypePullReopen, + } { + if Category(typ) != "work" { + t.Errorf("%s category = %q, want work", typ, Category(typ)) + } + if (&Notification{Type: typ}).Icon() == "" { + t.Errorf("%s has no icon", typ) + } + inWork := slices.Contains(WorkNotificationTypes, typ) + inEmail := slices.Contains(EmailNotificationTypes, typ) + if !inWork || !inEmail { + t.Errorf("%s work=%v email=%v, want both", typ, inWork, inEmail) + } + } +} + +func TestRepoIsNamedByItsOwnersHandle(t *testing.T) { + dir := identity.NewMockDirectory() + dir.Insert(identity.Identity{DID: syntax.DID("did:plc:owner"), Handle: syntax.Handle("tangled.org")}) + res := idresolver.NewMockResolver(dir) + + n := &Notification{ + Type: NotificationTypeIssueCreated, + RepoDid: "did:plc:repo", + RepoOwner: "did:plc:owner", + RepoName: "core", + EntityAt: "749", + EntityKind: EntityKindIssue, + } + if got := n.RepoOwnerHandle(res); got != "tangled.org" { + t.Errorf("RepoOwnerHandle = %q, want tangled.org", got) + } + if got := n.URL(res); got != "/tangled.org/core/issues/749" { + t.Errorf("URL = %q, want /tangled.org/core/issues/749", got) + } +} + +func TestIssueAndPullLinksUseTheRecordKeyOfAnAtURI(t *testing.T) { + dir := identity.NewMockDirectory() + dir.Insert(identity.Identity{DID: syntax.DID("did:plc:owner"), Handle: syntax.Handle("tangled.org")}) + res := idresolver.NewMockResolver(dir) + + for kind, want := range map[EntityKind]string{ + EntityKindIssue: "/tangled.org/core/issues/4604", + EntityKindPull: "/tangled.org/core/pulls/4604", + } { + n := &Notification{ + Type: NotificationTypeIssueCreated, + RepoDid: "did:plc:repo", + RepoOwner: "did:plc:owner", + RepoName: "core", + EntityAt: "at://did:plc:j5hmlfdrwkvtxm7cjmu7j2is/org.tangled.track.ticket/4604", + EntityKind: kind, + } + if got := n.URL(res); got != want { + t.Errorf("URL = %q, want %q", got, want) + } + } +} diff --git a/deliberi/models/signup.go b/deliberi/models/signup.go new file mode 100644 index 0000000000000000000000000000000000000000..8bc038b35ba0856f851cb757aad1faf5d85ab502 --- /dev/null +++ b/deliberi/models/signup.go @@ -0,0 +1,13 @@ +package models + +import "time" + +type InflightSignup struct { + Id int64 + Email string + InviteCode string + VerificationToken string + Created time.Time + // ISO-8601 (UTC) last sent; gates the email throttle. + LastSent string +} diff --git a/deliberi/readme.md b/deliberi/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..311913a0422eb9079b5a6951a0d475dcda864c48 --- /dev/null +++ b/deliberi/readme.md @@ -0,0 +1,2 @@ +deliberi is the off-protocol data server. It handles notifications fan-out and +their preferences, and email delivery. diff --git a/deliberi/records.go b/deliberi/records.go new file mode 100644 index 0000000000000000000000000000000000000000..70e5543d6ae925da0552deff12b5498d0c65e269 --- /dev/null +++ b/deliberi/records.go @@ -0,0 +1,122 @@ +package deliberi + +import "tangled.org/core/deliberi/models" + +// api/org_tangled has no bindings for these records yet, so decode only the +// fields the ingester reads +const ( + repoManifestNSID = "org.tangled.repo.manifest" + ticketNSID = "org.tangled.track.ticket" + commentNSID = "org.tangled.feed.comment" + starNSID = "org.tangled.feed.star" + followNSID = "org.tangled.graph.follow" + collaboratorNSID = "org.tangled.graph.collaborator" + subscriptionNSID = "org.tangled.feed.subscription" + patchNSID = "org.tangled.review.patch" + + starRepoType = "org.tangled.feed.star#repo" + + ticketStateOpen = "open" + ticketStateClosed = "closed" + ticketStateCompleted = "completed" +) + +type markdownText struct { + Text string `json:"text"` +} + +type repoDeclaration struct { + Owner string `json:"owner"` + Name string `json:"name"` +} + +type repoManifest struct { + Declaration *repoDeclaration `json:"declaration"` +} + +// repo-owned records (tickets, comments) live in the repo's account; the knot +// stamps the writing user into x-tngl-editor +type ticketRecord struct { + State string `json:"state"` + Title *markdownText `json:"title"` + Body *markdownText `json:"body"` + TargetBranch *string `json:"targetBranch"` + Editor string `json:"x-tngl-editor"` +} + +func (t *ticketRecord) kind() models.EntityKind { + if t.TargetBranch != nil { + return models.EntityKindPull + } + return models.EntityKindIssue +} + +// the knot leaves state off when a ticket is open +func (t *ticketRecord) state() string { + if t.State == "" { + return ticketStateOpen + } + return t.State +} + +func (t *ticketRecord) title() string { + if t.Title == nil { + return "" + } + return t.Title.Text +} + +type commentRecord struct { + Subject *struct { + Uri string `json:"uri"` + } `json:"subject"` + ReplyTo *struct { + Uri string `json:"uri"` + } `json:"replyTo"` + Body *markdownText `json:"body"` + Editor string `json:"x-tngl-editor"` +} + +type patchRecord struct { + Ticket string `json:"ticket"` +} + +type subscriptionRecord struct { + Subject *struct { + Uri string `json:"uri"` + Did string `json:"did"` + } `json:"subject"` + Ignore bool `json:"ignore"` +} + +func (s *subscriptionRecord) subject() string { + if s.Subject == nil { + return "" + } + if s.Subject.Uri != "" { + return s.Subject.Uri + } + return s.Subject.Did +} + +type starRecord struct { + Subject *struct { + Type string `json:"$type"` + Identity string `json:"identity"` + } `json:"subject"` +} + +func bodyText(m *markdownText) string { + if m == nil { + return "" + } + return m.Text +} + +// matches bobbin: a record without an editor is attributed to the repo +func editorOr(editor, repoDid string) string { + if editor == "" { + return repoDid + } + return editor +} diff --git a/deliberi/xrpc/account.go b/deliberi/xrpc/account.go new file mode 100644 index 0000000000000000000000000000000000000000..1aa770cfa56fca5f4d59a9f0e5d63324cda5bd1b --- /dev/null +++ b/deliberi/xrpc/account.go @@ -0,0 +1,343 @@ +package xrpc + +import ( + "crypto/rand" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "strings" + + tangled "tangled.org/core/api/org_tangled" + db "tangled.org/core/deliberi/db" + appviewemail "tangled.org/core/deliberi/email" + "tangled.org/core/deliberi/emailtemplate" + "tangled.org/core/deliberi/mailer" + xrpcerr "tangled.org/core/xrpc/errors" +) + +func (x *Xrpc) AccountListEmails(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "AccountListEmails") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + emails, err := db.GetAllEmails(x.DB, did) + if err != nil { + l.Error("failed to get emails", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + items := make([]*tangled.TempAccountListEmails_Email, 0, len(emails)) + for _, e := range emails { + items = append(items, &tangled.TempAccountListEmails_Email{ + Address: e.Address, + Verified: e.Verified, + Primary: e.Primary, + CreatedAt: e.CreatedAt.UTC().Format(timeFormat), + }) + } + + x.writeJSON(w, &tangled.TempAccountListEmails_Output{Emails: items}) +} + +func (x *Xrpc) AccountDeleteEmail(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "AccountDeleteEmail") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + var input tangled.TempAccountDeleteEmail_Input + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + addr := canonicalEmail(input.Email) + + existing, err := db.GetEmail(x.DB, did, addr) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + writeError(w, xrpcErrorTag("EmailNotFound", "the email address is not associated with this account"), http.StatusNotFound) + return + } + l.Error("failed to get email", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + if existing.Primary { + writeError(w, xrpcErrorTag("CannotDeletePrimary", "the primary email address cannot be deleted; set another address as primary first"), http.StatusBadRequest) + return + } + + if err := db.DeleteEmail(x.DB, did, addr); err != nil { + l.Error("failed to delete email", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + if x.KV != nil { + x.KV.DeleteEmailKey(addr) + } + + w.WriteHeader(http.StatusOK) +} + +func (x *Xrpc) AccountSetPrimaryEmail(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "AccountSetPrimaryEmail") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + var input tangled.TempAccountSetPrimaryEmail_Input + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + addr := canonicalEmail(input.Email) + + existing, err := db.GetEmail(x.DB, did, addr) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + writeError(w, xrpcErrorTag("EmailNotFound", "the email address is not associated with this account"), http.StatusNotFound) + return + } + l.Error("failed to get email", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + if !existing.Verified { + writeError(w, xrpcErrorTag("EmailNotVerified", "the email address must be verified before it can be made primary"), http.StatusBadRequest) + return + } + + revertPds, err := x.syncPdsEmail(did, addr) + if err != nil { + l.Error("failed to update PDS email", "err", err, "did", did) + writeError(w, errUpstream, http.StatusBadGateway) + return + } + + if err := db.MakeEmailPrimary(x.DB, did, addr); err != nil { + revertPds() + l.Error("failed to set primary email", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + if x.KV != nil { + x.KV.PutEmailDid(addr, did) + x.KV.SetPrimaryEmail(did, addr) + } + + w.WriteHeader(http.StatusOK) +} + +func canonicalEmail(input string) string { + return strings.ToLower(strings.TrimSpace(input)) +} + +func generateVerificationToken() (string, error) { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +func (x *Xrpc) verifyLink(token string) string { + return strings.TrimSuffix(x.Config.BaseURL, "/") + "/verify/email?token=" + token +} + +func (x *Xrpc) AccountAddEmail(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "AccountAddEmail") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + var input tangled.TempAccountAddEmail_Input + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + addr := canonicalEmail(input.Email) + + if !appviewemail.IsValidEmail(addr) { + writeError(w, xrpcErrorTag("InvalidEmail", "invalid email address"), http.StatusBadRequest) + return + } + + token, err := generateVerificationToken() + if err != nil { + l.Error("failed to generate verification token", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + existsAtAll, err := db.CheckEmailExistsAtAll(x.DB, addr) + if err != nil { + l.Error("failed to check email existence", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + if existsAtAll { + existsForDid, err := db.CheckEmailExists(x.DB, did, addr) + if err != nil { + l.Error("failed to check email for did", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + if existsForDid { + existing, err := db.GetEmail(x.DB, did, addr) + if err != nil { + l.Error("failed to get email", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + if existing.Verified { + writeError(w, xrpcErrorTag("EmailAlreadyRegistered", "this email address is already on your account"), http.StatusConflict) + return + } + if err := db.UpdateVerificationCode(x.DB, did, addr, token); err != nil { + l.Error("failed to update verification token", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + } else { + writeError(w, xrpcErrorTag("EmailAlreadyRegistered", "an account already exists for this email"), http.StatusConflict) + return + } + } else { + if err := db.InsertUnverifiedEmail(x.DB, did, addr, token); err != nil { + if db.IsUniqueConstraintErr(err) { + writeError(w, xrpcErrorTag("EmailAlreadyRegistered", "an account already exists for this email"), http.StatusConflict) + return + } + l.Error("failed to insert email", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + } + + link := x.verifyLink(token) + text := "Verify your email address on Tangled: " + link + body := `

Verify your email address on Tangled with the link below.

+

+ Verify email address +

+

Or open ` + link + `

` + html := emailtemplate.Shell("Verify your email on Tangled", body, x.Config.Resend.AssetsURL) + if err := x.Sender.Send(mailer.Message{To: addr, Subject: "Verify your email on Tangled", Text: text, Html: html}); err != nil { + l.Error("failed to send verification email", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) +} + +func (x *Xrpc) AccountVerifyEmail(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "AccountVerifyEmail") + + var input tangled.TempAccountVerifyEmail_Input + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + token := strings.TrimSpace(input.Token) + if token == "" { + writeError(w, xrpcErrorTag("InvalidCode", "invalid or expired verification link"), http.StatusBadRequest) + return + } + + // count + promote plus the token burn, in one transaction; the conditional + // burn is the single-use gate, so exactly one concurrent click wins + tx, err := x.DB.BeginTx(r.Context(), nil) + if err != nil { + l.Error("failed to begin verification transaction", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + defer tx.Rollback() + + row, err := db.GetEmailByToken(tx, token) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + writeError(w, xrpcErrorTag("InvalidCode", "invalid or expired verification link"), http.StatusBadRequest) + return + } + l.Error("failed to look up verification token", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + result, err := tx.Exec(`update emails set verified = true, verification_code = '' + where verification_code = ? and verified = false`, token) + if err != nil { + l.Error("failed to mark email verified", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + redeemed, err := result.RowsAffected() + if err != nil { + l.Error("failed to read redeemed rows", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + if redeemed == 0 { + writeError(w, xrpcErrorTag("InvalidCode", "invalid or expired verification link"), http.StatusBadRequest) + return + } + + verifiedCount, err := db.CountVerifiedEmails(tx, row.Did) + if err != nil { + l.Error("failed to count verified emails", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + becamePrimary := verifiedCount == 1 + revertPds := func() {} + if becamePrimary { + if err := db.MakeEmailPrimary(tx, row.Did, row.Address); err != nil { + l.Error("failed to make email primary", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + revertPds, err = x.syncPdsEmail(row.Did, row.Address) + if err != nil { + l.Error("failed to update PDS email", "err", err, "did", row.Did) + writeError(w, errUpstream, http.StatusBadGateway) + return + } + } + if err := tx.Commit(); err != nil { + revertPds() + l.Error("failed to commit verification", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + l.Info("email verified", "email", row.Address, "primary", becamePrimary) + + if x.KV != nil { + x.KV.PutEmailDid(row.Address, row.Did) + if becamePrimary { + x.KV.SetPrimaryEmail(row.Did, row.Address) + } + } + + x.writeJSON(w, &tangled.TempAccountVerifyEmail_Output{Email: row.Address}) +} diff --git a/deliberi/xrpc/account_test.go b/deliberi/xrpc/account_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f5bf91b263017f92d1871ddce919e394c6a19446 --- /dev/null +++ b/deliberi/xrpc/account_test.go @@ -0,0 +1,529 @@ +package xrpc + +import ( + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "sort" + "strings" + "sync" + "testing" + + tangled "tangled.org/core/api/org_tangled" + config "tangled.org/core/deliberi/config" + db "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/kv" + "tangled.org/core/deliberi/mailer" + "tangled.org/core/deliberi/models" +) + +type kvRecorder struct { + mu sync.Mutex + emailDids []string + primaries []string + deletes []string +} + +func (k *kvRecorder) PutEmailDid(email, did string) { + k.mu.Lock() + defer k.mu.Unlock() + k.emailDids = append(k.emailDids, email+"|"+did) +} + +func (k *kvRecorder) SetPrimaryEmail(did, email string) { + k.mu.Lock() + defer k.mu.Unlock() + k.primaries = append(k.primaries, did+"|"+email) +} + +func (k *kvRecorder) DeleteEmailKey(email string) { + k.mu.Lock() + defer k.mu.Unlock() + k.deletes = append(k.deletes, email) +} + +func (k *kvRecorder) DeletePrimaryEmail(did string) { + k.mu.Lock() + defer k.mu.Unlock() + k.deletes = append(k.deletes, "__primary:"+did) +} + +func (k *kvRecorder) snapshot() (emailDids, primaries, deletes []string) { + k.mu.Lock() + defer k.mu.Unlock() + return append([]string(nil), k.emailDids...), append([]string(nil), k.primaries...), append([]string(nil), k.deletes...) +} + +func (k *kvRecorder) assertIdle(t *testing.T) { + t.Helper() + emailDids, primaries, deletes := k.snapshot() + if len(emailDids) != 0 || len(primaries) != 0 || len(deletes) != 0 { + t.Errorf("unexpected KV writes: puts=%v primaries=%v deletes=%v", emailDids, primaries, deletes) + } +} + +var _ kv.EmailKV = (*kvRecorder)(nil) + +func newEmailXrpc(t *testing.T) (http.Handler, *db.DB, *kvRecorder, func(nsid string) string) { + t.Helper() + rec := &kvRecorder{} + x, router, d, sign := newTestXrpcFull(t, func(x *Xrpc) { + x.Sender = mailer.New(config.ResendConfig{}, slog.New(slog.NewTextHandler(io.Discard, nil))) + x.KV = rec + }) + _ = x + return router, d, rec, sign +} + +func postJSON(t *testing.T, router http.Handler, nsid, body string, sign func(nsid string) string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/"+nsid, strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+sign(nsid)) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + return rec +} + +func postOpen(t *testing.T, router http.Handler, nsid, body string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/"+nsid, strings.NewReader(body)) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + return rec +} + +func emailByAddress(t *testing.T, d *db.DB, did, addr string) models.Email { + t.Helper() + emails, err := db.GetAllEmails(d, did) + if err != nil { + t.Fatalf("GetAllEmails: %v", err) + } + for _, e := range emails { + if e.Address == addr { + return e + } + } + t.Fatalf("no email row for %s on %s", addr, did) + return models.Email{} +} + +func assertVerificationToken(t *testing.T, token string) { + t.Helper() + if len(token) != 64 { + t.Errorf("token %q not 64 hex characters", token) + } + for _, c := range token { + if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') { + t.Errorf("token %q contains non-hex character", token) + } + } +} + +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + old := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + os.Stdout = w + defer func() { os.Stdout = old }() + fn() + w.Close() + data, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read stdout: %v", err) + } + return string(data) +} + +func seedVerifiedSecondary(t *testing.T, d *db.DB, did, addr string) { + t.Helper() + if err := db.InsertUnverifiedEmail(d, did, addr, "000000"); err != nil { + t.Fatalf("InsertUnverifiedEmail: %v", err) + } + if err := db.MarkEmailVerified(d, did, addr); err != nil { + t.Fatalf("MarkEmailVerified: %v", err) + } +} + +func TestAccountAddEmailCanonicalizesAndInsertsUnverified(t *testing.T) { + router, d, kvRec, sign := newEmailXrpc(t) + + emitted := captureStdout(t, func() { + rec := postJSON(t, router, "org.tangled.temp.account.addEmail", `{"email":" Alice@Example.com "}`, sign) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + }) + + row := emailByAddress(t, d, testActor, "alice@example.com") + if row.Verified { + t.Errorf("email verified before link submission") + } + if row.Primary { + t.Errorf("unverified email marked primary") + } + assertVerificationToken(t, row.VerificationCode) + if !strings.Contains(emitted, "alice@example.com") || + !strings.Contains(emitted, "/verify/email?token="+row.VerificationCode) { + t.Errorf("verification email missing canonical address or link; emitted:\n%s", emitted) + } + kvRec.assertIdle(t) +} + +func TestAccountAddEmailRejectsRegisteredElsewhere(t *testing.T) { + router, d, kvRec, sign := newEmailXrpc(t) + + other := "did:plc:someone-else" + if err := db.AddEmail(d, models.Email{Did: other, Address: "taken@example.com", Verified: true, Primary: true}); err != nil { + t.Fatalf("AddEmail: %v", err) + } + + rec := postJSON(t, router, "org.tangled.temp.account.addEmail", `{"email":"taken@example.com"}`, sign) + if rec.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409; body=%s", rec.Code, rec.Body.String()) + } + + exists, err := db.CheckEmailExists(d, testActor, "taken@example.com") + if err != nil { + t.Fatalf("CheckEmailExists: %v", err) + } + if exists { + t.Errorf("caller gained a row for the rejected address") + } + kvRec.assertIdle(t) +} + +func TestAccountAddEmailRejectsVerifiedOwnAddress(t *testing.T) { + router, d, kvRec, sign := newEmailXrpc(t) + + if err := db.AddEmail(d, models.Email{Did: testActor, Address: "mine@example.com", Verified: true, Primary: true}); err != nil { + t.Fatalf("AddEmail: %v", err) + } + + rec := postJSON(t, router, "org.tangled.temp.account.addEmail", `{"email":"mine@example.com"}`, sign) + if rec.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409; body=%s", rec.Code, rec.Body.String()) + } + + code, err := db.GetVerificationCodeForEmail(d, testActor, "mine@example.com") + if err != nil { + t.Fatalf("GetVerificationCodeForEmail: %v", err) + } + if code != "" { + t.Errorf("resend overwrote the state of an already-verified address") + } + kvRec.assertIdle(t) +} + +func TestAccountAddEmailResendsForUnverified(t *testing.T) { + router, d, kvRec, sign := newEmailXrpc(t) + + if err := db.InsertUnverifiedEmail(d, testActor, "retry@example.com", "000000"); err != nil { + t.Fatalf("InsertUnverifiedEmail: %v", err) + } + + emitted := captureStdout(t, func() { + rec := postJSON(t, router, "org.tangled.temp.account.addEmail", `{"email":"retry@example.com"}`, sign) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (resend); body=%s", rec.Code, rec.Body.String()) + } + }) + + token, err := db.GetVerificationCodeForEmail(d, testActor, "retry@example.com") + if err != nil { + t.Fatalf("GetVerificationCodeForEmail: %v", err) + } + if token == "000000" { + t.Errorf("resend kept the old verification token") + } + assertVerificationToken(t, token) + if !strings.Contains(emitted, "retry@example.com") || + !strings.Contains(emitted, "/verify/email?token="+token) { + t.Errorf("resend email missing address or fresh link; emitted:\n%s", emitted) + } + // the superseded token must not verify anything anymore + if rec := postOpen(t, router, "org.tangled.temp.account.verifyEmail", `{"token":"000000"}`); rec.Code != http.StatusBadRequest { + t.Errorf("superseded token status = %d, want 400", rec.Code) + } + row := emailByAddress(t, d, testActor, "retry@example.com") + if row.Verified || row.VerificationCode != token { + t.Errorf("superseded token mutated the row: %+v", row) + } + kvRec.assertIdle(t) +} + +const verifTokenA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +func TestAccountVerifyEmailFirstBecomesPrimaryAndWritesKV(t *testing.T) { + router, d, kvRec, _ := newEmailXrpc(t) + + if err := db.InsertUnverifiedEmail(d, testActor, "bob@example.com", verifTokenA); err != nil { + t.Fatalf("InsertUnverifiedEmail: %v", err) + } + + rec := postOpen(t, router, "org.tangled.temp.account.verifyEmail", `{"token":"`+verifTokenA+`"}`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var out tangled.TempAccountVerifyEmail_Output + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v; body=%s", err, rec.Body.String()) + } + if out.Email != "bob@example.com" { + t.Errorf("output email = %q, want bob@example.com", out.Email) + } + + row := emailByAddress(t, d, testActor, "bob@example.com") + if !row.Verified { + t.Errorf("email not verified after link submission") + } + if !row.Primary { + t.Errorf("first verified email not promoted to primary") + } + if row.VerificationCode != "" { + t.Errorf("verification token not burned: %q", row.VerificationCode) + } + + emailDids, primaries, _ := kvRec.snapshot() + if len(emailDids) != 1 || emailDids[0] != "bob@example.com|"+testActor { + t.Errorf("PutEmailDid calls = %v, want [bob@example.com|%s]", emailDids, testActor) + } + if len(primaries) != 1 || primaries[0] != testActor+"|bob@example.com" { + t.Errorf("SetPrimaryEmail calls = %v, want [%s|bob@example.com]", primaries, testActor) + } +} + +func TestAccountVerifyEmailKeepsExistingPrimary(t *testing.T) { + router, d, kvRec, _ := newEmailXrpc(t) + + if err := db.AddEmail(d, models.Email{Did: testActor, Address: "primary@example.com", Verified: true, Primary: true}); err != nil { + t.Fatalf("AddEmail: %v", err) + } + if err := db.InsertUnverifiedEmail(d, testActor, "second@example.com", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); err != nil { + t.Fatalf("InsertUnverifiedEmail: %v", err) + } + + rec := postOpen(t, router, "org.tangled.temp.account.verifyEmail", `{"token":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + + row := emailByAddress(t, d, testActor, "second@example.com") + if !row.Verified { + t.Errorf("email not verified") + } + if row.Primary { + t.Errorf("verified secondary took over an existing primary") + } + if row.VerificationCode != "" { + t.Errorf("secondary verify did not burn the token: %q", row.VerificationCode) + } + if primary := emailByAddress(t, d, testActor, "primary@example.com"); !primary.Primary { + t.Errorf("existing primary lost its flag") + } + + emailDids, primaries, _ := kvRec.snapshot() + if len(emailDids) != 1 || emailDids[0] != "second@example.com|"+testActor { + t.Errorf("PutEmailDid calls = %v, want [second@example.com|%s]", emailDids, testActor) + } + if len(primaries) != 0 { + t.Errorf("SetPrimaryEmail called for a secondary verify: %v", primaries) + } +} + +func TestAccountVerifyEmailBadToken(t *testing.T) { + router, d, kvRec, _ := newEmailXrpc(t) + + if err := db.InsertUnverifiedEmail(d, testActor, "bob@example.com", verifTokenA); err != nil { + t.Fatalf("InsertUnverifiedEmail: %v", err) + } + + rec := postOpen(t, router, "org.tangled.temp.account.verifyEmail", `{"token":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}`) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String()) + } + + row := emailByAddress(t, d, testActor, "bob@example.com") + if row.Verified { + t.Errorf("email verified despite bad token") + } + if row.VerificationCode != verifTokenA { + t.Errorf("bad token mutated the stored verification token") + } + if row.Primary { + t.Errorf("row promoted despite bad token") + } + kvRec.assertIdle(t) +} + +func TestAccountVerifyEmailTokenSingleUse(t *testing.T) { + router, d, _, _ := newEmailXrpc(t) + + if err := db.InsertUnverifiedEmail(d, testActor, "bob@example.com", verifTokenA); err != nil { + t.Fatalf("InsertUnverifiedEmail: %v", err) + } + + body := `{"token":"` + verifTokenA + `"}` + if rec := postOpen(t, router, "org.tangled.temp.account.verifyEmail", body); rec.Code != http.StatusOK { + t.Fatalf("first use status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if rec := postOpen(t, router, "org.tangled.temp.account.verifyEmail", body); rec.Code != http.StatusBadRequest { + t.Fatalf("second use status = %d, want 400 (burned token)", rec.Code) + } +} + +func TestAccountVerifyEmailTokenConcurrent(t *testing.T) { + router, d, kvRec, _ := newEmailXrpc(t) + + if err := db.InsertUnverifiedEmail(d, testActor, "bob@example.com", verifTokenA); err != nil { + t.Fatalf("InsertUnverifiedEmail: %v", err) + } + + body := `{"token":"` + verifTokenA + `"}` + nsid := "org.tangled.temp.account.verifyEmail" + serve := func() int { + req := httptest.NewRequest(http.MethodPost, "/"+nsid, strings.NewReader(body)) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + return rec.Code + } + codes := make(chan int, 2) + go func() { codes <- serve() }() + go func() { codes <- serve() }() + + got := []int{<-codes, <-codes} + sort.Ints(got) + if got[0] != http.StatusOK || got[1] != http.StatusBadRequest { + t.Fatalf("concurrent statuses = %v, want one 200 and one 400", got) + } + + row := emailByAddress(t, d, testActor, "bob@example.com") + if !row.Verified || !row.Primary { + t.Errorf("winner state: verified=%v primary=%v, want both true", row.Verified, row.Primary) + } + if row.VerificationCode != "" { + t.Errorf("token not burned after concurrent verify: %q", row.VerificationCode) + } + emailDids, primaries, _ := kvRec.snapshot() + if len(emailDids) != 1 || len(primaries) != 1 { + t.Errorf("KV calls = %v / %v, want exactly one of each", emailDids, primaries) + } +} + +func TestAccountSetPrimaryEmailWritesKV(t *testing.T) { + router, d, kvRec, sign := newEmailXrpc(t) + + const oldPrimary = "primary@example.com" + const newPrimary = "secondary@example.com" + if err := db.AddEmail(d, models.Email{Did: testActor, Address: oldPrimary, Verified: true, Primary: true}); err != nil { + t.Fatalf("AddEmail: %v", err) + } + seedVerifiedSecondary(t, d, testActor, newPrimary) + + rec := postJSON(t, router, "org.tangled.temp.account.setPrimaryEmail", `{"email":"secondary@example.com"}`, sign) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + + emailDids, primaries, _ := kvRec.snapshot() + if len(emailDids) != 1 || emailDids[0] != newPrimary+"|"+testActor { + t.Errorf("PutEmailDid calls = %v, want [%s|%s]", emailDids, newPrimary, testActor) + } + if len(primaries) != 1 || primaries[0] != testActor+"|"+newPrimary { + t.Errorf("SetPrimaryEmail calls = %v, want [%s|%s]", primaries, testActor, newPrimary) + } + + got := emailByAddress(t, d, testActor, newPrimary) + if !got.Primary { + t.Errorf("db did not promote %s", newPrimary) + } + if old := emailByAddress(t, d, testActor, oldPrimary); old.Primary { + t.Errorf("db kept %s as primary", oldPrimary) + } +} + +func TestAccountDeleteEmailWritesKV(t *testing.T) { + router, d, kvRec, sign := newEmailXrpc(t) + + const primary = "primary@example.com" + const secondary = "secondary@example.com" + if err := db.AddEmail(d, models.Email{Did: testActor, Address: primary, Verified: true, Primary: true}); err != nil { + t.Fatalf("AddEmail: %v", err) + } + seedVerifiedSecondary(t, d, testActor, secondary) + + rec := postJSON(t, router, "org.tangled.temp.account.deleteEmail", `{"email":"secondary@example.com"}`, sign) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + + _, _, deletes := kvRec.snapshot() + if len(deletes) != 1 || deletes[0] != secondary { + t.Errorf("DeleteEmailKey calls = %v, want [%s]", deletes, secondary) + } + + emails, err := db.GetAllEmails(d, testActor) + if err != nil { + t.Fatalf("GetAllEmails: %v", err) + } + for _, e := range emails { + if e.Address == secondary { + t.Errorf("deleted row %s still present", secondary) + } + if e.Address == primary && !e.Primary { + t.Errorf("primary flag lost on %s", primary) + } + } +} + +func TestAccountDeleteUnverifiedEmail(t *testing.T) { + router, d, kvRec, sign := newEmailXrpc(t) + + if err := db.InsertUnverifiedEmail(d, testActor, "pending@example.com", "111111"); err != nil { + t.Fatalf("InsertUnverifiedEmail: %v", err) + } + + rec := postJSON(t, router, "org.tangled.temp.account.deleteEmail", `{"email":"pending@example.com"}`, sign) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + + emails, err := db.GetAllEmails(d, testActor) + if err != nil { + t.Fatalf("GetAllEmails: %v", err) + } + if len(emails) != 0 { + t.Errorf("pending row not removed: %+v", emails) + } + _, _, deletes := kvRec.snapshot() + if len(deletes) != 1 || deletes[0] != "pending@example.com" { + t.Errorf("DeleteEmailKey calls = %v, want [pending@example.com]", deletes) + } +} + +func TestAccountDeletePrimaryRejected(t *testing.T) { + router, d, kvRec, sign := newEmailXrpc(t) + + if err := db.AddEmail(d, models.Email{Did: testActor, Address: "primary@example.com", Verified: true, Primary: true}); err != nil { + t.Fatalf("AddEmail: %v", err) + } + + rec := postJSON(t, router, "org.tangled.temp.account.deleteEmail", `{"email":"primary@example.com"}`, sign) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String()) + } + + row := emailByAddress(t, d, testActor, "primary@example.com") + if !row.Primary { + t.Errorf("primary flag dropped despite rejection") + } + _, _, deletes := kvRec.snapshot() + if len(deletes) != 0 { + t.Errorf("KV delete called despite rejection: %v", deletes) + } +} diff --git a/deliberi/xrpc/ingested.go b/deliberi/xrpc/ingested.go new file mode 100644 index 0000000000000000000000000000000000000000..e170ef86c3282db11cc6a7b7532a0cf3d1c13ef8 --- /dev/null +++ b/deliberi/xrpc/ingested.go @@ -0,0 +1,30 @@ +package xrpc + +import ( + "database/sql" + "errors" + "net/http" + + db "tangled.org/core/deliberi/db" +) + +type ingestedOutput struct { + Rev string `json:"rev"` + Cid string `json:"cid,omitempty"` +} + +// e2e uses this to prove a notice was never made +// only mounted in dev, it exposes ingest internals +func (x *Xrpc) ingested(w http.ResponseWriter, r *http.Request) { + rev, cid, err := db.GetIngested(x.DB, r.URL.Query().Get("uri")) + if errors.Is(err, sql.ErrNoRows) { + writeError(w, xrpcErrorTag("NotIngested", "no record with that uri has been processed"), http.StatusNotFound) + return + } + if err != nil { + x.Logger.Error("failed to read ingested rev", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + x.writeJSON(w, ingestedOutput{Rev: rev, Cid: cid}) +} diff --git a/deliberi/xrpc/ingested_test.go b/deliberi/xrpc/ingested_test.go new file mode 100644 index 0000000000000000000000000000000000000000..655c036bfe74962f388352d02c0c78dd95f135af --- /dev/null +++ b/deliberi/xrpc/ingested_test.go @@ -0,0 +1,75 @@ +package xrpc + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/models" +) + +const ingestedUri = "at://did:plc:repo/org.tangled.track.ticket/1" + +func ingestedAt(t *testing.T, router http.Handler, uri string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/_ingested?uri="+uri, nil) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + return rec +} + +func TestIngestedAnswersWithTheRevAndCidInDev(t *testing.T) { + _, router, d, _ := newTestXrpcFull(t, func(x *Xrpc) { x.Config.Dev = true }) + if err := db.PutRev(d, ingestedUri, "3rev"); err != nil { + t.Fatalf("PutRev: %v", err) + } + if err := db.PutEntity(d, ingestedUri, db.Entity{Title: "t", Kind: models.EntityKindIssue, Cid: "bafycid"}); err != nil { + t.Fatalf("PutEntity: %v", err) + } + + rec := ingestedAt(t, router, ingestedUri) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", rec.Code, rec.Body) + } + var out ingestedOutput + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v", err) + } + if out.Rev != "3rev" || out.Cid != "bafycid" { + t.Errorf("answer = %+v, want rev 3rev and cid bafycid", out) + } +} + +func TestIngestedIsNotFoundForAnUnknownRecordInDev(t *testing.T) { + _, router, _, _ := newTestXrpcFull(t, func(x *Xrpc) { x.Config.Dev = true }) + rec := ingestedAt(t, router, ingestedUri) + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", rec.Code) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("content type = %q, a json 404 is how the route is told from a missing one", ct) + } +} + +func TestIngestedRouteDoesNotExistOutsideDev(t *testing.T) { + _, router, d, _ := newTestXrpcFull(t, nil) + if err := db.PutRev(d, ingestedUri, "3rev"); err != nil { + t.Fatalf("PutRev: %v", err) + } + rec := ingestedAt(t, router, ingestedUri) + if rec.Code != http.StatusNotFound || rec.Header().Get("Content-Type") == "application/json" { + t.Fatalf("answer = %d %q, want chi's plain 404", rec.Code, rec.Header().Get("Content-Type")) + } +} + +func TestIngestedIsAServerErrorWhenTheDbBreaks(t *testing.T) { + _, router, d, _ := newTestXrpcFull(t, func(x *Xrpc) { x.Config.Dev = true }) + if err := d.Close(); err != nil { + t.Fatalf("close: %v", err) + } + if rec := ingestedAt(t, router, ingestedUri); rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500 so a broken db isn't read as not yet", rec.Code) + } +} diff --git a/deliberi/xrpc/internal_committers.go b/deliberi/xrpc/internal_committers.go new file mode 100644 index 0000000000000000000000000000000000000000..9dd1c5f38b10d81b80f669e5ff2ac1c197f210b5 --- /dev/null +++ b/deliberi/xrpc/internal_committers.go @@ -0,0 +1,44 @@ +package xrpc + +import ( + "encoding/json" + "net/http" + + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/api/tangled" + db "tangled.org/core/deliberi/db" +) + +func (x *Xrpc) ResolveCommitters(w http.ResponseWriter, r *http.Request) { + caller, ok := actorDid(r) + if !ok || x.Config == nil || len(x.Config.TrustedServiceDIDs) == 0 { + http.NotFound(w, r) + return + } + if !x.Config.TrustsService(caller) { + writeError(w, xrpcErrorTag("Forbidden", "caller is not authorized"), http.StatusForbidden) + return + } + + var input tangled.IdentityResolveCommitters_Input + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4*1024)) + if err := decoder.Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + actor, err := syntax.ParseDID(input.Actor) + if err != nil { + writeError(w, badRequestError("actor must be a DID"), http.StatusBadRequest) + return + } + addresses, err := db.GetVerifiedEmailAddresses(x.DB, actor.String()) + if err != nil { + x.Logger.Error("listing verified committer emails failed", "actor", actor, "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + committers := make([]string, 1, len(addresses)+1) + committers[0] = actor.String() + committers = append(committers, addresses...) + x.writeJSON(w, tangled.IdentityResolveCommitters_Output{Committers: committers}) +} diff --git a/deliberi/xrpc/internal_committers_test.go b/deliberi/xrpc/internal_committers_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8f2f990bfff0b1b67b8c3ae3b792c450ef391134 --- /dev/null +++ b/deliberi/xrpc/internal_committers_test.go @@ -0,0 +1,77 @@ +package xrpc + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "tangled.org/core/api/tangled" + db "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/models" +) + +func TestResolveCommitters(t *testing.T) { + _, router, database, sign := newTestXrpcFull(t, func(x *Xrpc) { + x.Config.TrustedServiceDIDs = []string{"did:web:another.example", testActor} + }) + for _, email := range []models.Email{ + {Did: testActor, Address: "Verified@Example.com", Verified: true, VerificationCode: ""}, + {Did: testActor, Address: "pending@example.com", Verified: false, VerificationCode: "pending"}, + } { + if err := db.AddEmail(database, email); err != nil { + t.Fatal(err) + } + } + req := httptest.NewRequest( + http.MethodPost, + "/"+tangled.IdentityResolveCommittersNSID, + strings.NewReader(`{"actor":"`+testActor+`"}`), + ) + req.Header.Set("Authorization", "Bearer "+sign(tangled.IdentityResolveCommittersNSID)) + response := httptest.NewRecorder() + router.ServeHTTP(response, req) + if response.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", response.Code, response.Body.String()) + } + var output tangled.IdentityResolveCommitters_Output + if err := json.Unmarshal(response.Body.Bytes(), &output); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(output.Committers, ","), testActor+",verified@example.com"; got != want { + t.Fatalf("committers = %q, want %q", got, want) + } +} + +func TestResolveCommittersRejectsAnotherService(t *testing.T) { + _, router, _, sign := newTestXrpcFull(t, func(x *Xrpc) { + x.Config.TrustedServiceDIDs = []string{"did:web:mirror.example", "did:web:indexer.example"} + }) + req := httptest.NewRequest( + http.MethodPost, + "/"+tangled.IdentityResolveCommittersNSID, + strings.NewReader(`{"actor":"`+testActor+`"}`), + ) + req.Header.Set("Authorization", "Bearer "+sign(tangled.IdentityResolveCommittersNSID)) + response := httptest.NewRecorder() + router.ServeHTTP(response, req) + if response.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403", response.Code) + } +} + +func TestResolveCommittersIsDisabledWithoutTrustedServices(t *testing.T) { + router, _, sign := newTestXrpc(t) + req := httptest.NewRequest( + http.MethodPost, + "/"+tangled.IdentityResolveCommittersNSID, + strings.NewReader(`{"actor":"`+testActor+`"}`), + ) + req.Header.Set("Authorization", "Bearer "+sign(tangled.IdentityResolveCommittersNSID)) + response := httptest.NewRecorder() + router.ServeHTTP(response, req) + if response.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", response.Code) + } +} diff --git a/deliberi/xrpc/newsletter.go b/deliberi/xrpc/newsletter.go new file mode 100644 index 0000000000000000000000000000000000000000..ca49d4f298542e58b77350e84d719da19a474738 --- /dev/null +++ b/deliberi/xrpc/newsletter.go @@ -0,0 +1,114 @@ +package xrpc + +import ( + "database/sql" + "encoding/json" + "errors" + "net/http" + + tangled "tangled.org/core/api/org_tangled" + db "tangled.org/core/deliberi/db" + appviewemail "tangled.org/core/deliberi/email" + xrpcerr "tangled.org/core/xrpc/errors" +) + +var errNoVerifiedPrimaryEmail = xrpcErrorTag("NoVerifiedPrimaryEmail", "the account has no verified primary email address") + +func (x *Xrpc) AccountSubscribeNewsletter(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "AccountSubscribeNewsletter") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + primary, err := db.GetPrimaryEmail(x.DB, did) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + writeError(w, errNoVerifiedPrimaryEmail, http.StatusBadRequest) + return + } + l.Error("failed to get primary email", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + if !primary.Verified { + writeError(w, errNoVerifiedPrimaryEmail, http.StatusBadRequest) + return + } + + if err := db.UpsertNewsletterPref(x.DB, did, db.NewsletterStatusSubscribed, primary.Address); err != nil { + l.Error("failed to persist newsletter preference", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + go func() { + if err := x.Sender.AddNewsletterContact(primary.Address); err != nil { + l.Error("failed to add newsletter contact", "err", err) + } + }() + + w.WriteHeader(http.StatusOK) +} + +func (x *Xrpc) AccountDismissNewsletter(w http.ResponseWriter, r *http.Request) { + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + if err := db.UpsertNewsletterPref(x.DB, did, db.NewsletterStatusDismissed, ""); err != nil { + x.Logger.Error("failed to persist newsletter dismissal", "handler", "AccountDismissNewsletter", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) +} + +func (x *Xrpc) AccountGetNewsletterPreference(w http.ResponseWriter, r *http.Request) { + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + status, err := db.GetNewsletterStatus(x.DB, did) + if err != nil { + x.Logger.Error("failed to get newsletter preference", "handler", "AccountGetNewsletterPreference", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + out := &tangled.TempAccountGetNewsletterPreference_Output{} + if status != "" { + out.Status = &status + } + x.writeJSON(w, out) +} + +// open to anonymous visitors, like the appview's form; nothing is stored, the +// contact lives only in resend +func (x *Xrpc) NewsletterSignup(w http.ResponseWriter, r *http.Request) { + var input tangled.TempNewsletterSignup_Input + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + addr := canonicalEmail(input.Email) + if !appviewemail.IsValidEmail(addr) { + writeError(w, xrpcErrorTag("InvalidEmail", "invalid email address"), http.StatusBadRequest) + return + } + + go func() { + if err := x.Sender.AddNewsletterContact(addr); err != nil { + x.Logger.Error("failed to add newsletter contact", "handler", "NewsletterSignup", "err", err) + } + }() + + w.WriteHeader(http.StatusOK) +} diff --git a/deliberi/xrpc/newsletter_test.go b/deliberi/xrpc/newsletter_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3aa419fe59437c001a06e0a372a9411ca20f4eea --- /dev/null +++ b/deliberi/xrpc/newsletter_test.go @@ -0,0 +1,186 @@ +package xrpc + +import ( + "database/sql" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + tangled "tangled.org/core/api/org_tangled" + db "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/models" +) + +type newsletterRow struct { + status string + email sql.NullString +} + +func newsletterPref(t *testing.T, d *db.DB, did string) (newsletterRow, bool) { + t.Helper() + var row newsletterRow + err := d.QueryRow(`select status, email from newsletter_preferences where user_did = ?`, did).Scan(&row.status, &row.email) + if errors.Is(err, sql.ErrNoRows) { + return row, false + } + if err != nil { + t.Fatalf("query newsletter_preferences: %v", err) + } + return row, true +} + +func assertErrorTag(t *testing.T, body []byte, want string) { + t.Helper() + var out struct { + Error string `json:"error"` + } + if err := json.Unmarshal(body, &out); err != nil { + t.Fatalf("decode error body: %v; body=%s", err, body) + } + if out.Error != want { + t.Errorf("error = %q, want %q", out.Error, want) + } +} + +func TestSubscribeNewsletterUsesVerifiedPrimary(t *testing.T) { + router, d, _, sign := newEmailXrpc(t) + if err := db.AddEmail(d, models.Email{Did: testActor, Address: "alice@example.com", Verified: true, Primary: true}); err != nil { + t.Fatalf("AddEmail: %v", err) + } + if err := db.AddEmail(d, models.Email{Did: testActor, Address: "alt@example.com", Verified: true}); err != nil { + t.Fatalf("AddEmail: %v", err) + } + + rec := postJSON(t, router, tangled.TempAccountSubscribeNewsletterNSID, "", sign) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + + row, ok := newsletterPref(t, d, testActor) + if !ok { + t.Fatal("no newsletter preference row") + } + if row.status != db.NewsletterStatusSubscribed || row.email.String != "alice@example.com" { + t.Errorf("row = %+v, want subscribed with alice@example.com", row) + } +} + +func TestSubscribeNewsletterRejectsMissingOrUnverifiedPrimary(t *testing.T) { + for name, seed := range map[string]func(*db.DB) error{ + "no email": func(*db.DB) error { return nil }, + "unverified primary": func(d *db.DB) error { + return db.AddEmail(d, models.Email{Did: testActor, Address: "pending@example.com", VerificationCode: "pending"}) + }, + } { + t.Run(name, func(t *testing.T) { + router, d, _, sign := newEmailXrpc(t) + if err := seed(d); err != nil { + t.Fatalf("seed: %v", err) + } + + rec := postJSON(t, router, tangled.TempAccountSubscribeNewsletterNSID, "", sign) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String()) + } + assertErrorTag(t, rec.Body.Bytes(), "NoVerifiedPrimaryEmail") + if row, ok := newsletterPref(t, d, testActor); ok { + t.Errorf("rejected subscribe wrote a row: %+v", row) + } + }) + } +} + +func TestDismissNewsletterKeepsSubscribedAddress(t *testing.T) { + router, d, _, sign := newEmailXrpc(t) + if err := db.AddEmail(d, models.Email{Did: testActor, Address: "alice@example.com", Verified: true, Primary: true}); err != nil { + t.Fatalf("AddEmail: %v", err) + } + + if rec := postJSON(t, router, tangled.TempAccountSubscribeNewsletterNSID, "", sign); rec.Code != http.StatusOK { + t.Fatalf("subscribe status = %d; body=%s", rec.Code, rec.Body.String()) + } + if rec := postJSON(t, router, tangled.TempAccountDismissNewsletterNSID, "", sign); rec.Code != http.StatusOK { + t.Fatalf("dismiss status = %d; body=%s", rec.Code, rec.Body.String()) + } + + row, ok := newsletterPref(t, d, testActor) + if !ok { + t.Fatal("no newsletter preference row") + } + if row.status != db.NewsletterStatusDismissed || row.email.String != "alice@example.com" { + t.Errorf("row = %+v, want dismissed keeping alice@example.com", row) + } +} + +func TestDismissNewsletterWithoutEmail(t *testing.T) { + router, d, _, sign := newEmailXrpc(t) + + if rec := postJSON(t, router, tangled.TempAccountDismissNewsletterNSID, "", sign); rec.Code != http.StatusOK { + t.Fatalf("status = %d; body=%s", rec.Code, rec.Body.String()) + } + + row, ok := newsletterPref(t, d, testActor) + if !ok { + t.Fatal("no newsletter preference row") + } + if row.status != db.NewsletterStatusDismissed || row.email.Valid { + t.Errorf("row = %+v, want dismissed with null email", row) + } +} + +func getNewsletterStatus(t *testing.T, router http.Handler, sign func(string) string) *string { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/"+tangled.TempAccountGetNewsletterPreferenceNSID, nil) + req.Header.Set("Authorization", "Bearer "+sign(tangled.TempAccountGetNewsletterPreferenceNSID)) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d; body=%s", rec.Code, rec.Body.String()) + } + var out tangled.TempAccountGetNewsletterPreference_Output + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v; body=%s", err, rec.Body.String()) + } + return out.Status +} + +func TestGetNewsletterPreferenceTracksChoice(t *testing.T) { + router, _, _, sign := newEmailXrpc(t) + + if got := getNewsletterStatus(t, router, sign); got != nil { + t.Fatalf("status = %q before any choice, want absent", *got) + } + if rec := postJSON(t, router, tangled.TempAccountDismissNewsletterNSID, "", sign); rec.Code != http.StatusOK { + t.Fatalf("dismiss status = %d", rec.Code) + } + if got := getNewsletterStatus(t, router, sign); got == nil || *got != db.NewsletterStatusDismissed { + t.Fatalf("status = %v after dismiss, want dismissed", got) + } +} + +func TestNewsletterSignupIsOpen(t *testing.T) { + router, d, _, _ := newEmailXrpc(t) + + if rec := postOpen(t, router, tangled.TempNewsletterSignupNSID, `{"email":" Visitor@Example.com "}`); rec.Code != http.StatusOK { + t.Fatalf("status = %d without auth; body=%s", rec.Code, rec.Body.String()) + } + var rows int + if err := d.QueryRow(`select count(*) from newsletter_preferences`).Scan(&rows); err != nil { + t.Fatal(err) + } + if rows != 0 { + t.Errorf("anonymous signup stored %d preference rows", rows) + } +} + +func TestNewsletterSignupRejectsInvalidEmail(t *testing.T) { + router, _, _, _ := newEmailXrpc(t) + + rec := postOpen(t, router, tangled.TempNewsletterSignupNSID, `{"email":"not-an-email"}`) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", rec.Code) + } + assertErrorTag(t, rec.Body.Bytes(), "InvalidEmail") +} diff --git a/deliberi/xrpc/notifications.go b/deliberi/xrpc/notifications.go new file mode 100644 index 0000000000000000000000000000000000000000..67a2be8422ebc585c17b78077bf16698e1629492 --- /dev/null +++ b/deliberi/xrpc/notifications.go @@ -0,0 +1,262 @@ +package xrpc + +import ( + "encoding/json" + "net/http" + "strconv" + + "github.com/bluesky-social/indigo/atproto/syntax" + tangled "tangled.org/core/api/org_tangled" + db "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/models" + "tangled.org/core/orm" + xrpcerr "tangled.org/core/xrpc/errors" +) + +func (x *Xrpc) NotificationList(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "NotificationList") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + if x.Invites != nil { + x.Invites.Reconcile(r.Context(), did) + } + + q := r.URL.Query() + filters := []orm.Filter{} + if q.Get("read") == "unread" { + filters = append(filters, orm.FilterEq("read", 0)) + } + switch q.Get("category") { + case "social": + filters = append(filters, orm.FilterIn("type", models.SocialNotificationTypes)) + case "work": + filters = append(filters, orm.FilterIn("type", models.WorkNotificationTypes)) + } + + limit := 50 + if s := q.Get("limit"); s != "" { + if n, err := strconv.Atoi(s); err == nil && n > 0 && n <= 100 { + limit = n + } + } + + notifs, err := db.GetNotifications(x.DB, did, limit, filters...) + if err != nil { + l.Error("failed to list notifications", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + unreadBase := []orm.Filter{orm.FilterEq("read", 0)} + workUnread, _ := db.CountNotifications(x.DB, did, append(unreadBase, orm.FilterIn("type", models.WorkNotificationTypes))...) + socialUnread, _ := db.CountNotifications(x.DB, did, append(unreadBase, orm.FilterIn("type", models.SocialNotificationTypes))...) + + items := make([]*tangled.TempNotificationListNotifications_Notification, 0, len(notifs)) + for _, n := range notifs { + item := &tangled.TempNotificationListNotifications_Notification{ + Uri: n.AtUri, + Type: string(n.Type), + Category: models.Category(n.Type), + ActorDid: n.ActorDid, + Read: n.Read, + CreatedAt: n.Created.Format(timeFormat), + } + if n.RepoDid != "" { + item.RepoDid = &n.RepoDid + } + if n.EntityAt != "" { + switch n.EntityKind { + case models.EntityKindIssue: + item.IssueAt = &n.EntityAt + case models.EntityKindPull: + item.PullAt = &n.EntityAt + } + } + items = append(items, item) + } + + x.writeJSON(w, &tangled.TempNotificationListNotifications_Output{ + Notifications: items, + SocialUnreadCount: socialUnread, + WorkUnreadCount: workUnread, + }) +} + +func (x *Xrpc) NotificationGetUnreadCount(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "NotificationGetUnreadCount") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + if x.Invites != nil { + x.Invites.Reconcile(r.Context(), did) + } + + count, err := db.CountNotifications(x.DB, did, orm.FilterEq("read", 0)) + if err != nil { + l.Error("failed to count unread notifications", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + x.writeJSON(w, &tangled.TempNotificationGetUnreadCount_Output{Count: count}) +} + +func (x *Xrpc) NotificationUpdateSeen(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "NotificationUpdateSeen") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + var input tangled.TempNotificationUpdateSeen_Input + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + if input.Uri == "" { + writeError(w, badRequestError("uri is required"), http.StatusBadRequest) + return + } + + if err := db.MarkRead(x.DB, did, input.Uri, input.Read); err != nil { + l.Error("failed to update notification read state", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) +} + +func (x *Xrpc) NotificationMarkEntityRead(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "NotificationMarkEntityRead") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + var input tangled.TempNotificationMarkEntityRead_Input + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + if input.EntityAt == "" { + writeError(w, badRequestError("entityAt is required"), http.StatusBadRequest) + return + } + + if err := db.MarkEntityRead(x.DB, did, input.EntityAt); err != nil { + l.Error("failed to mark entity notifications read", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) +} + +func (x *Xrpc) NotificationMarkAllRead(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "NotificationMarkAllRead") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + if err := db.MarkAllRead(x.DB, did); err != nil { + l.Error("failed to mark all read", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) +} + +func (x *Xrpc) NotificationGetPreferences(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "NotificationGetPreferences") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + prefs, err := db.GetNotificationPreference(x.DB, did) + if err != nil { + l.Error("failed to get notification preferences", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + x.writeJSON(w, &tangled.TempNotificationGetPreferences_Preferences{ + EmailNotifications: prefs.EmailNotifications, + Followed: prefs.Followed, + IssueClosed: prefs.IssueClosed, + IssueCommented: prefs.IssueCommented, + IssueCreated: prefs.IssueCreated, + PullCommented: prefs.PullCommented, + PullCreated: prefs.PullCreated, + PullMerged: prefs.PullMerged, + RepoStarred: prefs.RepoStarred, + UserMentioned: prefs.UserMentioned, + }) +} + +func (x *Xrpc) NotificationUpdatePreferences(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "NotificationUpdatePreferences") + + did, ok := actorDid(r) + if !ok { + writeError(w, xrpcerr.MissingActorDidError, http.StatusForbidden) + return + } + + var input tangled.TempNotificationUpdatePreferences_Input + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + + existing, err := db.GetNotificationPreference(x.DB, did) + if err != nil { + l.Error("failed to get existing notification preferences", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + prefs := &models.NotificationPreferences{ + UserDid: syntax.DID(did), + RepoStarred: applyBoolPtr(existing.RepoStarred, input.RepoStarred), + IssueCreated: applyBoolPtr(existing.IssueCreated, input.IssueCreated), + IssueCommented: applyBoolPtr(existing.IssueCommented, input.IssueCommented), + PullCreated: applyBoolPtr(existing.PullCreated, input.PullCreated), + PullCommented: applyBoolPtr(existing.PullCommented, input.PullCommented), + Followed: applyBoolPtr(existing.Followed, input.Followed), + UserMentioned: applyBoolPtr(existing.UserMentioned, input.UserMentioned), + EmailNotifications: applyBoolPtr(existing.EmailNotifications, input.EmailNotifications), + IssueClosed: applyBoolPtr(existing.IssueClosed, input.IssueClosed), + PullMerged: applyBoolPtr(existing.PullMerged, input.PullMerged), + } + + if err := db.UpsertNotificationPreferences(x.DB, prefs); err != nil { + l.Error("failed to update notification preferences", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) +} + +func applyBoolPtr(existing bool, update *bool) bool { + if update != nil { + return *update + } + return existing +} diff --git a/deliberi/xrpc/pds_email.go b/deliberi/xrpc/pds_email.go new file mode 100644 index 0000000000000000000000000000000000000000..48d09288bf394793a371472274a71276a7394268 --- /dev/null +++ b/deliberi/xrpc/pds_email.go @@ -0,0 +1,68 @@ +package xrpc + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" +) + +// found is false when the account does not live on the configured pds +func (x *Xrpc) pdsAccountEmail(did string) (email string, found bool, err error) { + req, err := http.NewRequest(http.MethodGet, x.Config.Pds.Host+"/xrpc/com.atproto.admin.getAccountInfo?did="+url.QueryEscape(did), nil) + if err != nil { + return "", false, err + } + req.SetBasicAuth("admin", x.Config.Pds.AdminSecret) + resp, err := pdsClient.Do(req) + if err != nil { + return "", false, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusBadRequest { + return "", false, nil + } + if resp.StatusCode != http.StatusOK { + return "", false, pdsError(resp, "get account info") + } + var info struct { + Email string `json:"email"` + } + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + return "", false, fmt.Errorf("failed to decode account info response: %w", err) + } + return info.Email, true, nil +} + +func (x *Xrpc) pdsUpdateAccountEmail(did, email string) error { + resp, err := x.pdsRequest("com.atproto.admin.updateAccountEmail", map[string]string{"account": did, "email": email}, true) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return pdsError(resp, "update account email") + } + return nil +} + +// the returned func puts back the email the pds had, which may differ from deliberi's old primary +func (x *Xrpc) syncPdsEmail(did, addr string) (revert func(), err error) { + noop := func() {} + if !x.Config.SignupEnabled() { + return noop, nil + } + prev, found, err := x.pdsAccountEmail(did) + if err != nil || !found || strings.EqualFold(prev, addr) { + return noop, err + } + if err := x.pdsUpdateAccountEmail(did, addr); err != nil { + return noop, err + } + return func() { + if err := x.pdsUpdateAccountEmail(did, prev); err != nil { + x.Logger.Error("failed to restore PDS email", "err", err, "did", did) + } + }, nil +} diff --git a/deliberi/xrpc/pds_email_test.go b/deliberi/xrpc/pds_email_test.go new file mode 100644 index 0000000000000000000000000000000000000000..1138a7cb6e41ca146b6b1068d4513912fdf7fe37 --- /dev/null +++ b/deliberi/xrpc/pds_email_test.go @@ -0,0 +1,244 @@ +package xrpc + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + db "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/models" +) + +type pdsEmails struct { + mu sync.Mutex + emails map[string]string + updates []string + failUpdate bool + hang bool +} + +func (p *pdsEmails) email(did string) string { + p.mu.Lock() + defer p.mu.Unlock() + return p.emails[did] +} + +func (p *pdsEmails) updateLog() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.updates...) +} + +func newPdsEmailStub(t *testing.T, p *pdsEmails) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + authed := func(r *http.Request) bool { + user, pass, ok := r.BasicAuth() + return ok && user == "admin" && pass == testAdminSecret + } + mux.HandleFunc("/xrpc/com.atproto.admin.getAccountInfo", func(w http.ResponseWriter, r *http.Request) { + if !authed(r) { + w.WriteHeader(http.StatusUnauthorized) + return + } + if p.hang { + <-r.Context().Done() + return + } + p.mu.Lock() + email, ok := p.emails[r.URL.Query().Get("did")] + p.mu.Unlock() + if !ok { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"error":"InvalidRequest","message":"Account not found"}`)) + return + } + json.NewEncoder(w).Encode(map[string]string{"email": email}) + }) + mux.HandleFunc("/xrpc/com.atproto.admin.updateAccountEmail", func(w http.ResponseWriter, r *http.Request) { + if !authed(r) { + w.WriteHeader(http.StatusUnauthorized) + return + } + var body map[string]string + json.NewDecoder(r.Body).Decode(&body) + p.mu.Lock() + defer p.mu.Unlock() + if p.failUpdate { + w.WriteHeader(http.StatusInternalServerError) + return + } + p.emails[body["account"]] = body["email"] + p.updates = append(p.updates, body["account"]+"|"+body["email"]) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +func newPdsEmailXrpc(t *testing.T, p *pdsEmails) (http.Handler, *db.DB, *kvRecorder, func(nsid string) string) { + t.Helper() + srv := newPdsEmailStub(t, p) + rec := &kvRecorder{} + _, router, d, sign := newTestXrpcFull(t, func(x *Xrpc) { + x.Config.Pds.AdminSecret = testAdminSecret + x.Config.Pds.Host = srv.URL + x.KV = rec + }) + return router, d, rec, sign +} + +func seedPrimaryAndSecondary(t *testing.T, d *db.DB, primary, secondary string) { + t.Helper() + if err := db.AddEmail(d, models.Email{Did: testActor, Address: primary, Verified: true, Primary: true}); err != nil { + t.Fatalf("AddEmail: %v", err) + } + seedVerifiedSecondary(t, d, testActor, secondary) +} + +func assertPrimary(t *testing.T, d *db.DB, want, notWant string) { + t.Helper() + if !emailByAddress(t, d, testActor, want).Primary { + t.Errorf("%s is not primary", want) + } + if emailByAddress(t, d, testActor, notWant).Primary { + t.Errorf("%s is primary", notWant) + } +} + +const setPrimaryNsid = "org.tangled.temp.account.setPrimaryEmail" + +func TestAccountSetPrimaryEmailUpdatesPdsEmail(t *testing.T) { + p := &pdsEmails{emails: map[string]string{testActor: "old@example.com"}} + router, d, _, sign := newPdsEmailXrpc(t, p) + seedPrimaryAndSecondary(t, d, "old@example.com", "new@example.com") + + rec := postJSON(t, router, setPrimaryNsid, `{"email":"new@example.com"}`, sign) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if got := p.email(testActor); got != "new@example.com" { + t.Errorf("pds email = %q, want new@example.com", got) + } + assertPrimary(t, d, "new@example.com", "old@example.com") +} + +func TestAccountSetPrimaryEmailPdsFailureChangesNothing(t *testing.T) { + p := &pdsEmails{emails: map[string]string{testActor: "old@example.com"}, failUpdate: true} + router, d, kvRec, sign := newPdsEmailXrpc(t, p) + seedPrimaryAndSecondary(t, d, "old@example.com", "new@example.com") + + rec := postJSON(t, router, setPrimaryNsid, `{"email":"new@example.com"}`, sign) + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502; body=%s", rec.Code, rec.Body.String()) + } + assertPrimary(t, d, "old@example.com", "new@example.com") + kvRec.assertIdle(t) + if got := p.email(testActor); got != "old@example.com" { + t.Errorf("pds email = %q, want old@example.com", got) + } +} + +func TestAccountSetPrimaryEmailGivesUpOnAHungPds(t *testing.T) { + prev := pdsClient.Timeout + pdsClient.Timeout = 50 * time.Millisecond + t.Cleanup(func() { pdsClient.Timeout = prev }) + p := &pdsEmails{emails: map[string]string{testActor: "old@example.com"}, hang: true} + router, d, kvRec, sign := newPdsEmailXrpc(t, p) + seedPrimaryAndSecondary(t, d, "old@example.com", "new@example.com") + + rec := postJSON(t, router, setPrimaryNsid, `{"email":"new@example.com"}`, sign) + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502; body=%s", rec.Code, rec.Body.String()) + } + assertPrimary(t, d, "old@example.com", "new@example.com") + kvRec.assertIdle(t) +} + +func TestAccountSetPrimaryEmailSkipsAccountsOffThisPds(t *testing.T) { + p := &pdsEmails{emails: map[string]string{"did:plc:someoneelse": "other@example.com"}} + router, d, _, sign := newPdsEmailXrpc(t, p) + seedPrimaryAndSecondary(t, d, "old@example.com", "new@example.com") + + rec := postJSON(t, router, setPrimaryNsid, `{"email":"new@example.com"}`, sign) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if got := p.updateLog(); len(got) != 0 { + t.Errorf("pds updates = %v, want none", got) + } + assertPrimary(t, d, "new@example.com", "old@example.com") +} + +func TestAccountSetPrimaryEmailSkipsUpdateWhenPdsAlreadyMatches(t *testing.T) { + p := &pdsEmails{emails: map[string]string{testActor: "New@Example.com"}} + router, d, _, sign := newPdsEmailXrpc(t, p) + seedPrimaryAndSecondary(t, d, "old@example.com", "new@example.com") + + rec := postJSON(t, router, setPrimaryNsid, `{"email":"new@example.com"}`, sign) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if got := p.updateLog(); len(got) != 0 { + t.Errorf("pds updates = %v, want none", got) + } +} + +func TestAccountSetPrimaryEmailRevertsPdsWhenDbWriteFails(t *testing.T) { + p := &pdsEmails{emails: map[string]string{testActor: "drifted@example.com"}} + router, d, kvRec, sign := newPdsEmailXrpc(t, p) + seedPrimaryAndSecondary(t, d, "old@example.com", "new@example.com") + if _, err := d.Exec(`create trigger refuse_primary before update of is_primary on emails + begin select raise(abort, 'refused'); end`); err != nil { + t.Fatalf("create trigger: %v", err) + } + + rec := postJSON(t, router, setPrimaryNsid, `{"email":"new@example.com"}`, sign) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500; body=%s", rec.Code, rec.Body.String()) + } + if got := p.email(testActor); got != "drifted@example.com" { + t.Errorf("pds email = %q, want the value read before the change", got) + } + if got := p.updateLog(); len(got) != 2 { + t.Errorf("pds updates = %v, want the change and its revert", got) + } + kvRec.assertIdle(t) +} + +func TestAccountVerifyEmailFirstPrimaryUpdatesPdsEmail(t *testing.T) { + p := &pdsEmails{emails: map[string]string{testActor: "old@example.com"}} + router, d, _, _ := newPdsEmailXrpc(t, p) + if err := db.InsertUnverifiedEmail(d, testActor, "bob@example.com", verifTokenA); err != nil { + t.Fatalf("InsertUnverifiedEmail: %v", err) + } + + rec := postOpen(t, router, "org.tangled.temp.account.verifyEmail", `{"token":"`+verifTokenA+`"}`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if got := p.email(testActor); got != "bob@example.com" { + t.Errorf("pds email = %q, want bob@example.com", got) + } +} + +func TestAccountVerifyEmailPdsFailureKeepsTokenUnspent(t *testing.T) { + p := &pdsEmails{emails: map[string]string{testActor: "old@example.com"}, failUpdate: true} + router, d, kvRec, _ := newPdsEmailXrpc(t, p) + if err := db.InsertUnverifiedEmail(d, testActor, "bob@example.com", verifTokenA); err != nil { + t.Fatalf("InsertUnverifiedEmail: %v", err) + } + + rec := postOpen(t, router, "org.tangled.temp.account.verifyEmail", `{"token":"`+verifTokenA+`"}`) + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502; body=%s", rec.Code, rec.Body.String()) + } + row := emailByAddress(t, d, testActor, "bob@example.com") + if row.Verified || row.Primary || row.VerificationCode != verifTokenA { + t.Errorf("failed verify changed the row: %+v", row) + } + kvRec.assertIdle(t) +} diff --git a/deliberi/xrpc/signup.go b/deliberi/xrpc/signup.go new file mode 100644 index 0000000000000000000000000000000000000000..d1e309415495fc0c0009e75aedacd0210aa04668 --- /dev/null +++ b/deliberi/xrpc/signup.go @@ -0,0 +1,555 @@ +package xrpc + +import ( + "bytes" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + tangled "tangled.org/core/api/org_tangled" + db "tangled.org/core/deliberi/db" + appviewemail "tangled.org/core/deliberi/email" + "tangled.org/core/deliberi/emailtemplate" + "tangled.org/core/deliberi/mailer" + "tangled.org/core/deliberi/models" +) + +var subdomainRegex = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{2,61}[a-z0-9])?$`) + +var errEmailAlreadyRegistered = errors.New("email already registered") + +type pdsRejection struct { + tag string + message string +} + +func (e *pdsRejection) Error() string { return e.tag + ": " + e.message } + +var pdsRejections = map[string]pdsRejection{ + "HandleNotAvailable": {"UsernameNotAvailable", "this username is already taken"}, + "InvalidHandle": {"InvalidUsername", "this username is not allowed"}, + "InvalidPassword": {"InvalidPassword", "this password is not allowed"}, +} + +const signupEmailCooldown = time.Minute + +func withinSignupCooldown(lastSent string) bool { + t, err := time.Parse(time.RFC3339, lastSent) + if err != nil { + return false + } + return time.Since(t) < signupEmailCooldown +} + +var ( + errCaptchaRejected = errors.New("captcha verification failed") + errCaptchaNotConfigured = errors.New("turnstile secret key not configured") +) + +var turnstileClient = &http.Client{Timeout: 10 * time.Second} + +var pdsClient = &http.Client{Timeout: 10 * time.Second} + +type turnstileResponse struct { + Success bool `json:"success"` + ErrorCodes []string `json:"error-codes,omitempty"` + ChallengeTs string `json:"challenge_ts,omitempty"` + Hostname string `json:"hostname,omitempty"` +} + +func (x *Xrpc) verifyTurnstile(token string, r *http.Request) error { + if token == "" { + return errCaptchaRejected + } + if x.Config.TurnstileSecretKey == "" { + return errCaptchaNotConfigured + } + + data := url.Values{} + data.Set("secret", x.Config.TurnstileSecretKey) + data.Set("response", token) + + if remoteIP := r.Header.Get("CF-Connecting-IP"); remoteIP != "" { + data.Set("remoteip", remoteIP) + } else if remoteIP := r.Header.Get("X-Forwarded-For"); remoteIP != "" { + if ips := strings.Split(remoteIP, ","); len(ips) > 0 { + data.Set("remoteip", strings.TrimSpace(ips[0])) + } + } else { + data.Set("remoteip", r.RemoteAddr) + } + + req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, x.Config.TurnstileVerifyURL, strings.NewReader(data.Encode())) + if err != nil { + return fmt.Errorf("failed to build turnstile request: %w", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := turnstileClient.Do(req) + if err != nil { + return fmt.Errorf("failed to verify turnstile token: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return fmt.Errorf("turnstile siteverify status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + var ts turnstileResponse + if err := json.NewDecoder(resp.Body).Decode(&ts); err != nil { + return fmt.Errorf("failed to decode turnstile response: %w", err) + } + if !ts.Success { + x.Logger.Warn("turnstile rejected token", "error_codes", ts.ErrorCodes) + return errCaptchaRejected + } + return nil +} + +func isValidSubdomain(name string) bool { + return len(name) >= 4 && len(name) <= 63 && subdomainRegex.MatchString(name) +} + +func (x *Xrpc) AccountBeginSignup(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "AccountBeginSignup") + + if !x.Config.SignupEnabled() { + writeError(w, xrpcErrorTag("SignupDisabled", "signing up is not currently enabled"), http.StatusFailedDependency) + return + } + + var input tangled.TempAccountBeginSignup_Input + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + + if err := x.verifyTurnstile(input.TurnstileToken, r); err != nil { + switch { + case errors.Is(err, errCaptchaRejected): + l.Warn("captcha verification failed", "err", err) + writeError(w, xrpcErrorTag("CaptchaFailed", "captcha verification failed, please try again"), http.StatusBadRequest) + case errors.Is(err, errCaptchaNotConfigured): + l.Error("turnstile secret key not configured; refusing signups") + writeError(w, errInternal, http.StatusInternalServerError) + default: + l.Error("turnstile verification upstream failed", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + } + return + } + + email := canonicalEmail(input.Email) + + if !appviewemail.IsValidEmail(email) { + writeError(w, xrpcErrorTag("InvalidEmail", "invalid email address"), http.StatusBadRequest) + return + } + + exists, err := db.CheckEmailExistsAtAll(x.DB, email) + if err != nil { + l.Error("failed to check email existence", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + if exists { + writeError(w, xrpcErrorTag("EmailAlreadyRegistered", "an account already exists for this email"), http.StatusConflict) + return + } + + if pending, err := db.GetInflightByEmail(x.DB, email); err == nil && withinSignupCooldown(pending.LastSent) { + writeError(w, xrpcErrorTag("TooManyRequests", "please wait a moment before requesting another email"), http.StatusTooManyRequests) + return + } else if err != nil && !errors.Is(err, sql.ErrNoRows) { + l.Error("failed to check pending signup", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + token, err := generateVerificationToken() + if err != nil { + l.Error("failed to generate verification token", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + // the invite code never leaves deliberi: the emailed link only carries the + // verification token, and completeSignup redeems the stored code internally + code, err := x.pdsCreateInviteCode() + if err != nil { + l.Error("failed to create invite code", "err", err) + writeError(w, errUpstream, http.StatusBadGateway) + return + } + + if err := db.AddInflightSignup(x.DB, models.InflightSignup{Email: email, InviteCode: code, VerificationToken: token}); err != nil { + l.Error("failed to add inflight signup", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + link := x.signupLink(token) + text, html := signupLinkEmail(link, x.Config.Resend.AssetsURL) + if err := x.Sender.Send(mailer.Message{To: email, Subject: "Finish signing up for Tangled", Text: text, Html: html}); err != nil { + l.Error("failed to send signup email", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) +} + +// AccountResendSignup re-mails the same live link — same token, no new invite code. +func (x *Xrpc) AccountResendSignup(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "AccountResendSignup") + + if !x.Config.SignupEnabled() { + writeError(w, xrpcErrorTag("SignupDisabled", "signing up is not currently enabled"), http.StatusFailedDependency) + return + } + + var input tangled.TempAccountResendSignup_Input + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + email := canonicalEmail(input.Email) + if !appviewemail.IsValidEmail(email) { + writeError(w, xrpcErrorTag("InvalidEmail", "invalid email address"), http.StatusBadRequest) + return + } + + inflight, err := db.GetInflightByEmail(x.DB, email) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + if serr := db.DeleteExpiredInflightSignups(x.DB); serr != nil { + l.Error("failed to sweep expired signups", "err", serr) + } + writeError(w, xrpcErrorTag("NoPendingSignup", "no link is waiting for this email; sign up again"), http.StatusNotFound) + return + } + l.Error("failed to get inflight signup", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + if withinSignupCooldown(inflight.LastSent) { + writeError(w, xrpcErrorTag("TooManyRequests", "please wait a moment before requesting another email"), http.StatusTooManyRequests) + return + } + + link := x.signupLink(inflight.VerificationToken) + text, html := signupLinkEmail(link, x.Config.Resend.AssetsURL) + if err := x.Sender.Send(mailer.Message{To: email, Subject: "Finish signing up for Tangled", Text: text, Html: html}); err != nil { + l.Error("failed to resend signup email", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + if err := db.UpdateInflightLastSent(x.DB, email); err != nil { + l.Error("failed to record resend", "err", err) + } + + w.WriteHeader(http.StatusOK) +} + +func (x *Xrpc) AccountGetSignup(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "AccountGetSignup") + + if !x.Config.SignupEnabled() { + writeError(w, xrpcErrorTag("SignupDisabled", "signing up is not currently enabled"), http.StatusFailedDependency) + return + } + + token := strings.TrimSpace(r.URL.Query().Get("token")) + if token == "" { + writeError(w, xrpcErrorTag("InvalidCode", "invalid or expired verification link"), http.StatusBadRequest) + return + } + + inflight, err := db.GetInflightByToken(x.DB, token) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + writeError(w, xrpcErrorTag("InvalidCode", "invalid or expired verification link"), http.StatusBadRequest) + return + } + l.Error("failed to get inflight signup for token", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + + x.writeJSON(w, &tangled.TempAccountGetSignup_Output{Email: inflight.Email, Pds: x.Config.Pds.Host}) +} + +func (x *Xrpc) signupLink(token string) string { + return strings.TrimSuffix(x.Config.BaseURL, "/") + "/signup/verify?token=" + token +} + +const signupLinkNotice = "This link expires in 24 hours. If you didn't request it, you can ignore this email." + +func signupLinkEmail(link, assetsURL string) (text, html string) { + text = "Finish signing up for Tangled: " + link + "\n\n" + signupLinkNotice + body := `

Finish signing up for Tangled with the link below.

+

+ Finish signing up +

+

Or open ` + link + `

+

` + signupLinkNotice + `

` + return text, emailtemplate.Shell("Finish signing up for Tangled", body, assetsURL) +} + +func (x *Xrpc) AccountCompleteSignup(w http.ResponseWriter, r *http.Request) { + l := x.Logger.With("handler", "AccountCompleteSignup") + + if !x.Config.SignupEnabled() { + writeError(w, xrpcErrorTag("SignupDisabled", "signing up is not currently enabled"), http.StatusFailedDependency) + return + } + + var input tangled.TempAccountCompleteSignup_Input + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + writeError(w, errBadRequestBody, http.StatusBadRequest) + return + } + + token := strings.TrimSpace(input.Token) + if token == "" { + writeError(w, xrpcErrorTag("InvalidCode", "invalid or expired verification link"), http.StatusBadRequest) + return + } + + if !isValidSubdomain(input.Username) { + writeError(w, xrpcErrorTag("InvalidUsername", "invalid username"), http.StatusBadRequest) + return + } + + inflight, err := db.GetInflightByToken(x.DB, token) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + if serr := db.DeleteExpiredInflightSignups(x.DB); serr != nil { + l.Error("failed to sweep expired signups", "err", serr) + } + writeError(w, xrpcErrorTag("InvalidCode", "invalid or expired verification link"), http.StatusBadRequest) + return + } + l.Error("failed to get inflight signup for token", "err", err) + writeError(w, errInternal, http.StatusInternalServerError) + return + } + emailAddr := canonicalEmail(inflight.Email) + + did, handle, err := x.provisionAccount(input.Username, input.Password, emailAddr, inflight.InviteCode) + if err != nil { + if errors.Is(err, errEmailAlreadyRegistered) { + writeError(w, xrpcErrorTag("EmailAlreadyRegistered", "an account already exists for this email"), http.StatusConflict) + return + } + var rejected *pdsRejection + if errors.As(err, &rejected) { + writeError(w, xrpcErrorTag(rejected.tag, rejected.message), http.StatusBadRequest) + return + } + l.Error("failed to provision account", "err", err) + writeError(w, errUpstream, http.StatusBadGateway) + return + } + + // the account is the occurrence: one signup per did + x.Analytics.Capture("signup", did, did, nil) + + go func() { + if err := db.DeleteInflightSignup(x.DB, emailAddr); err != nil { + l.Error("failed to delete inflight signup", "err", err) + } + }() + + out := &tangled.TempAccountCompleteSignup_Output{Did: did, Handle: handle} + if input.RequestUri != nil && *input.RequestUri != "" { + // a stock pds has no such route, and the signup still succeeds + authorization, err := x.pdsPreauthorize(did, *input.RequestUri) + if err != nil { + l.Warn("pds did not authorize the new account", "err", err, "did", did) + } else { + out.Authorization = authorization + } + } + x.writeJSON(w, out) +} + +func (x *Xrpc) provisionAccount(username, password, emailAddr, code string) (did, handle string, err error) { + success := false + emailAdded := false + defer func() { + if success { + return + } + x.Logger.Info("rolling back signup", "username", username, "did", did) + if did != "" { + if derr := x.pdsDeleteAccount(did); derr != nil { + x.Logger.Error("failed to roll back PDS account", "err", derr, "did", did) + } + } + if emailAdded { + if derr := db.DeleteEmail(x.DB, did, emailAddr); derr != nil { + x.Logger.Error("failed to roll back email row", "err", derr, "email", emailAddr) + } + } + }() + + did, handle, err = x.pdsCreateAccount(username, password, emailAddr, code) + if err != nil { + return did, handle, err + } + + if err = db.AddEmail(x.DB, models.Email{Did: did, Address: emailAddr, Verified: true, Primary: true}); err != nil { + if db.IsUniqueConstraintErr(err) { + // keep did/handle so the defer rolls the PDS account back + return did, handle, errEmailAlreadyRegistered + } + return did, handle, err + } + emailAdded = true + + if x.KV != nil { + x.KV.PutEmailDid(emailAddr, did) + x.KV.SetPrimaryEmail(did, emailAddr) + } + + success = true + return did, handle, nil +} + +func (x *Xrpc) pdsRequest(endpoint string, body any, useAuth bool) (*http.Response, error) { + return x.pdsPost("/xrpc/"+endpoint, body, useAuth) +} + +func (x *Xrpc) pdsPost(path string, body any, useAuth bool) (*http.Response, error) { + jsonData, err := json.Marshal(body) + if err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodPost, x.Config.Pds.Host+path, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + if useAuth { + req.SetBasicAuth("admin", x.Config.Pds.AdminSecret) + } + return pdsClient.Do(req) +} + +func pdsError(resp *http.Response, action string) error { + var e struct { + Error string `json:"error"` + Message string `json:"message"` + } + b, _ := io.ReadAll(resp.Body) + if err := json.Unmarshal(b, &e); err == nil && e.Message != "" { + return fmt.Errorf("failed to %s: %s - %s", action, e.Error, e.Message) + } + return fmt.Errorf("failed to %s, status %d", action, resp.StatusCode) +} + +func createAccountError(resp *http.Response) error { + var e struct { + Error string `json:"error"` + Message string `json:"message"` + } + b, _ := io.ReadAll(resp.Body) + if err := json.Unmarshal(b, &e); err != nil { + return fmt.Errorf("failed to create account, status %d", resp.StatusCode) + } + if rejection, ok := pdsRejections[e.Error]; ok { + return &rejection + } + return fmt.Errorf("failed to create account: %s - %s", e.Error, e.Message) +} + +func (x *Xrpc) pdsCreateInviteCode() (string, error) { + resp, err := x.pdsRequest("com.atproto.server.createInviteCode", map[string]any{"useCount": 1}, true) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", pdsError(resp, "create invite code") + } + var result map[string]string + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", fmt.Errorf("failed to decode invite code response: %w", err) + } + return result["code"], nil +} + +func (x *Xrpc) pdsCreateAccount(username, password, emailAddr, code string) (did, handle string, err error) { + parsed, err := url.Parse(x.Config.Pds.Host) + if err != nil { + return "", "", fmt.Errorf("invalid PDS host URL: %w", err) + } + handle = fmt.Sprintf("%s.%s", username, parsed.Hostname()) + + body := map[string]string{ + "email": emailAddr, + "handle": handle, + "password": password, + "inviteCode": code, + } + resp, err := x.pdsRequest("com.atproto.server.createAccount", body, false) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", "", createAccountError(resp) + } + + var result struct { + DID string `json:"did"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", "", fmt.Errorf("failed to decode create account response: %w", err) + } + return result.DID, handle, nil +} + +// only tangled's pds build has this route (pds/preauthorize.mjs) +const pdsPreauthorizePath = "/_preauthorize" + +func (x *Xrpc) pdsPreauthorize(did, requestUri string) (*tangled.TempAccountCompleteSignup_Authorization, error) { + resp, err := x.pdsPost(pdsPreauthorizePath, map[string]string{"did": did, "requestUri": requestUri}, true) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, pdsError(resp, "preauthorize") + } + var out tangled.TempAccountCompleteSignup_Authorization + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, fmt.Errorf("failed to decode preauthorize response: %w", err) + } + if out.Code == "" || out.State == "" || out.Iss == "" { + return nil, errors.New("preauthorize response is missing code, state or iss") + } + return &out, nil +} + +func (x *Xrpc) pdsDeleteAccount(did string) error { + resp, err := x.pdsRequest("com.atproto.admin.deleteAccount", map[string]string{"did": did}, true) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return pdsError(resp, "delete account") + } + return nil +} diff --git a/deliberi/xrpc/signup_test.go b/deliberi/xrpc/signup_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d663aa0cce7e6b9748d3843cbf20bb46ae16da54 --- /dev/null +++ b/deliberi/xrpc/signup_test.go @@ -0,0 +1,879 @@ +package xrpc + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + config "tangled.org/core/deliberi/config" + db "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/mailer" + "tangled.org/core/deliberi/models" +) + +const ( + testInviteCode = "invite-code-abc123" + testNewDid = "did:plc:newuser" +) + +const testAdminSecret = "test-admin-secret" + +// pdsRecorder stubs the PDS endpoints the signup handlers call and records them. +type pdsRecorder struct { + mu sync.Mutex + inviteMints int + createAccount []map[string]any + adminDeletes []string + rejectCreate string + preauthorize []map[string]string + noPreauthorize bool +} + +func (p *pdsRecorder) Preauthorizations() []map[string]string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]map[string]string(nil), p.preauthorize...) +} + +func (p *pdsRecorder) InviteMints() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.inviteMints +} + +func (p *pdsRecorder) CreateAccounts() []map[string]any { + p.mu.Lock() + defer p.mu.Unlock() + return append([]map[string]any(nil), p.createAccount...) +} + +func (p *pdsRecorder) AdminDeletes() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.adminDeletes...) +} + +func newPdsStub(t *testing.T, rec *pdsRecorder) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/com.atproto.server.createInviteCode", func(w http.ResponseWriter, r *http.Request) { + user, pass, ok := r.BasicAuth() + if !ok || user != "admin" || pass != testAdminSecret { + w.WriteHeader(http.StatusUnauthorized) + return + } + rec.mu.Lock() + rec.inviteMints++ + rec.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"code":"`+testInviteCode+`"}`) + }) + mux.HandleFunc("/xrpc/com.atproto.server.createAccount", func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + json.NewDecoder(r.Body).Decode(&body) + rec.mu.Lock() + rec.createAccount = append(rec.createAccount, body) + reject := rec.rejectCreate + rec.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if reject != "" { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprint(w, `{"error":"`+reject+`","message":"rejected by pds"}`) + return + } + fmt.Fprint(w, `{"did":"`+testNewDid+`"}`) + }) + mux.HandleFunc("/xrpc/com.atproto.admin.deleteAccount", func(w http.ResponseWriter, r *http.Request) { + user, pass, ok := r.BasicAuth() + if !ok || user != "admin" || pass != testAdminSecret { + w.WriteHeader(http.StatusUnauthorized) + return + } + var body map[string]string + json.NewDecoder(r.Body).Decode(&body) + rec.mu.Lock() + rec.adminDeletes = append(rec.adminDeletes, body["did"]) + rec.mu.Unlock() + w.WriteHeader(http.StatusOK) + }) + mux.HandleFunc(pdsPreauthorizePath, func(w http.ResponseWriter, r *http.Request) { + rec.mu.Lock() + missing := rec.noPreauthorize + rec.mu.Unlock() + if missing { + http.NotFound(w, r) + return + } + user, pass, ok := r.BasicAuth() + if !ok || user != "admin" || pass != testAdminSecret { + w.WriteHeader(http.StatusUnauthorized) + return + } + var body map[string]string + json.NewDecoder(r.Body).Decode(&body) + rec.mu.Lock() + rec.preauthorize = append(rec.preauthorize, body) + rec.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"code":"cod-123","state":"state-456","iss":"https://pds.example"}`) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +func newTurnstileStub(t *testing.T, success bool) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if success { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"success":true}`) + return + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"success":false,"error-codes":["invalid-input-response"]}`) + })) + t.Cleanup(srv.Close) + return srv +} + +func newTurnstileStatusStub(t *testing.T, status int) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + fmt.Fprint(w, `{"success":false,"error-codes":["internal-error"]}`) + })) + t.Cleanup(srv.Close) + return srv +} + +func newTurnstileSlowStub(t *testing.T) (*httptest.Server, func()) { + t.Helper() + release := make(chan struct{}) + var once sync.Once + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-time.After(2 * time.Second): + } + })) + t.Cleanup(srv.Close) + return srv, func() { once.Do(func() { close(release) }) } +} + +func newSignupXrpc(t *testing.T, configure ...func(*Xrpc)) (http.Handler, *db.DB, *kvRecorder, *pdsRecorder) { + t.Helper() + rec := &kvRecorder{} + pds := &pdsRecorder{} + srv := newPdsStub(t, pds) + turnstile := newTurnstileStub(t, true) + _, router, d, _ := newTestXrpcFull(t, func(x *Xrpc) { + x.Config.Pds.AdminSecret = testAdminSecret + x.Config.Pds.Host = srv.URL + x.Config.BaseURL = "https://tangled.example" + x.Config.TurnstileSecretKey = "test-turnstile-secret" + x.Config.TurnstileVerifyURL = turnstile.URL + x.Sender = mailer.New(config.ResendConfig{}, slog.New(slog.NewTextHandler(io.Discard, nil))) + x.KV = rec + for _, c := range configure { + c(x) + } + }) + return router, d, rec, pds +} + +func withTurnstile(srv *httptest.Server) func(*Xrpc) { + return func(x *Xrpc) { x.Config.TurnstileVerifyURL = srv.URL } +} + +func TestAccountBeginSignupStoresTokenAndSendsLink(t *testing.T) { + router, d, kvRec, _ := newSignupXrpc(t) + + emitted := captureStdout(t, func() { + rec := postOpen(t, router, "org.tangled.temp.account.beginSignup", `{"email":" Alice@Example.com ","turnstileToken":"test"}`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + }) + + if !strings.Contains(emitted, "to: alice@example.com") { + t.Fatalf("email not addressed to canonical alice@example.com in:\n%s", emitted) + } + + token := "" + invite := "" + rows, err := d.Query(`select verification_token, invite_code from signups_inflight where email = 'alice@example.com'`) + if err != nil { + t.Fatalf("query inflight: %v", err) + } + defer rows.Close() + if rows.Next() { + rows.Scan(&token, &invite) + } + if token == "" { + t.Fatal("inflight row missing verification_token") + } + assertVerificationToken(t, token) + if invite == "" { + t.Fatal("inflight row missing invite_code") + } + + if !strings.Contains(emitted, "/signup/verify?token="+token) { + t.Fatalf("email link missing verification token; got:\n%s", emitted) + } + if strings.Contains(emitted, invite) { + t.Fatalf("email leaked the invite code %q:\n%s", invite, emitted) + } + + kvRec.assertIdle(t) +} + +func TestAccountBeginSignupRejectsRegisteredEmail(t *testing.T) { + router, d, kvRec, _ := newSignupXrpc(t) + + if err := db.AddEmail(d, models.Email{Did: testNewDid, Address: "alice@example.com", Verified: true, Primary: true}); err != nil { + t.Fatalf("AddEmail: %v", err) + } + + rec := postOpen(t, router, "org.tangled.temp.account.beginSignup", `{"email":"alice@example.com","turnstileToken":"test"}`) + if rec.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409", rec.Code) + } + if !strings.Contains(rec.Body.String(), "EmailAlreadyRegistered") { + t.Fatalf("error body = %s, want EmailAlreadyRegistered", rec.Body.String()) + } + kvRec.assertIdle(t) +} + +func TestAccountBeginSignupRejectsMissingTurnstile(t *testing.T) { + router, d, kvRec, pds := newSignupXrpc(t) + + rec := postOpen(t, router, "org.tangled.temp.account.beginSignup", `{"email":"kate@example.com"}`) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "CaptchaFailed") { + t.Fatalf("error body = %s, want CaptchaFailed", rec.Body.String()) + } + if mints := pds.InviteMints(); mints != 0 { + t.Fatalf("invite minted without a captcha (mints=%d)", mints) + } + if n := inflightCount(t, d, "kate@example.com"); n != 0 { + t.Fatalf("inflight row created without a captcha (count=%d)", n) + } + kvRec.assertIdle(t) +} + +func TestAccountBeginSignupRejectsInvalidTurnstile(t *testing.T) { + router, d, kvRec, pds := newSignupXrpc(t, withTurnstile(newTurnstileStub(t, false))) + + rec := postOpen(t, router, "org.tangled.temp.account.beginSignup", + `{"email":"kate@example.com","turnstileToken":"bogus"}`) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "CaptchaFailed") { + t.Fatalf("error body = %s, want CaptchaFailed", rec.Body.String()) + } + if mints := pds.InviteMints(); mints != 0 { + t.Fatalf("invite minted for a rejected captcha (mints=%d)", mints) + } + if n := inflightCount(t, d, "kate@example.com"); n != 0 { + t.Fatalf("inflight row created for a rejected captcha (count=%d)", n) + } + kvRec.assertIdle(t) +} + +func TestAccountBeginSignupFailsClosedWithoutSecret(t *testing.T) { + _, router, _, _ := newTestXrpcFull(t, func(x *Xrpc) { + x.Config.Pds.AdminSecret = testAdminSecret + x.Config.BaseURL = "https://tangled.example" + x.Config.TurnstileVerifyURL = newTurnstileStub(t, true).URL + }) + + rec := postOpen(t, router, "org.tangled.temp.account.beginSignup", + `{"email":"kate@example.com","turnstileToken":"test"}`) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500; body=%s", rec.Code, rec.Body.String()) + } +} + +func TestAccountBeginSignupUpstreamErrorIsInternal(t *testing.T) { + router, d, kvRec, pds := newSignupXrpc(t, withTurnstile(newTurnstileStatusStub(t, http.StatusInternalServerError))) + + rec := postOpen(t, router, "org.tangled.temp.account.beginSignup", + `{"email":"kate@example.com","turnstileToken":"test"}`) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500; body=%s", rec.Code, rec.Body.String()) + } + if mints := pds.InviteMints(); mints != 0 { + t.Fatalf("invite minted despite upstream failure (mints=%d)", mints) + } + if n := inflightCount(t, d, "kate@example.com"); n != 0 { + t.Fatalf("inflight row created despite upstream failure (count=%d)", n) + } + kvRec.assertIdle(t) +} + +func TestAccountBeginSignupStalledSiteverifyDoesNotHang(t *testing.T) { + slow, release := newTurnstileSlowStub(t) + router, _, kvRec, _ := newSignupXrpc(t, withTurnstile(slow)) + defer release() + + old := turnstileClient + turnstileClient = &http.Client{Timeout: 50 * time.Millisecond} + t.Cleanup(func() { turnstileClient = old }) + + start := time.Now() + rec := postOpen(t, router, "org.tangled.temp.account.beginSignup", + `{"email":"kate@example.com","turnstileToken":"test"}`) + elapsed := time.Since(start) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500; body=%s", rec.Code, rec.Body.String()) + } + if elapsed > 2*time.Second { + t.Fatalf("signup hung on a stalled siteverify: took %s", elapsed) + } + kvRec.assertIdle(t) +} + +func TestAccountCompleteSignupWithToken(t *testing.T) { + router, d, kvRec, pds := newSignupXrpc(t) + + token, err := generateVerificationToken() + if err != nil { + t.Fatalf("generate token: %v", err) + } + if err := db.AddInflightSignup(d, models.InflightSignup{Email: "bob@example.com", InviteCode: testInviteCode, VerificationToken: token}); err != nil { + t.Fatalf("seed inflight: %v", err) + } + + rec := postOpen(t, router, "org.tangled.temp.account.completeSignup", + `{"token":"`+token+`","username":"bobby","password":"password1234"}`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var out tangledTempAccountCompleteSignupOutput + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v; body=%s", err, rec.Body.String()) + } + if out.Did != testNewDid { + t.Fatalf("did = %q, want %q", out.Did, testNewDid) + } + if out.Handle == "" || !strings.HasPrefix(out.Handle, "bobby.") { + t.Fatalf("handle = %q, want bobby.", out.Handle) + } + + accounts := pds.CreateAccounts() + if len(accounts) != 1 { + t.Fatalf("createAccount calls = %d, want 1", len(accounts)) + } + body := accounts[0] + if body["inviteCode"] != testInviteCode { + t.Fatalf("createAccount inviteCode = %v, want %q", body["inviteCode"], testInviteCode) + } + if body["email"] != "bob@example.com" { + t.Fatalf("createAccount email = %v, want bob@example.com", body["email"]) + } + if body["handle"] != out.Handle { + t.Fatalf("createAccount handle = %v, want %q", body["handle"], out.Handle) + } + + row := emailByAddress(t, d, testNewDid, "bob@example.com") + if !row.Verified || !row.Primary { + t.Fatalf("email row = verified:%v primary:%v, want verified+primary", row.Verified, row.Primary) + } + + emailDids, primaries, deletes := kvRec.snapshot() + if len(emailDids) != 1 || emailDids[0] != "bob@example.com|"+testNewDid { + t.Errorf("PutEmailDid calls = %v, want [bob@example.com|%s]", emailDids, testNewDid) + } + if len(primaries) != 1 || primaries[0] != testNewDid+"|bob@example.com" { + t.Errorf("SetPrimaryEmail calls = %v, want [%s|bob@example.com]", primaries, testNewDid) + } + if len(deletes) != 0 { + t.Errorf("unexpected KV deletes: %v", deletes) + } + + // deletion runs in a goroutine; poll briefly + deadline := time.Now().Add(2 * time.Second) + for { + _, err := db.GetInflightByToken(d, token) + if errors.Is(err, sql.ErrNoRows) { + break + } else if err == nil && time.Now().After(deadline) { + t.Fatal("inflight row still present after signup") + } else if err != nil { + t.Fatalf("GetInflightByToken: %v", err) + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestAccountCompleteSignupBadToken(t *testing.T) { + router, d, kvRec, _ := newSignupXrpc(t) + + token, _ := generateVerificationToken() + if err := db.AddInflightSignup(d, models.InflightSignup{Email: "bob@example.com", InviteCode: testInviteCode, VerificationToken: token}); err != nil { + t.Fatalf("seed inflight: %v", err) + } + + rec := postOpen(t, router, "org.tangled.temp.account.completeSignup", + `{"token":"deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef","username":"bobby","password":"password1234"}`) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "InvalidCode") { + t.Fatalf("error body = %s, want InvalidCode", rec.Body.String()) + } + + if _, err := db.GetInflightByToken(d, token); err != nil { + t.Fatalf("inflight row lost on bad token: %v", err) + } + kvRec.assertIdle(t) +} + +func TestAccountCompleteSignupInvalidUsername(t *testing.T) { + router, _, kvRec, _ := newSignupXrpc(t) + + rec := postOpen(t, router, "org.tangled.temp.account.completeSignup", + `{"token":"`+strings.Repeat("ab", 32)+`","username":"UPPER","password":"password1234"}`) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "InvalidUsername") { + t.Fatalf("error body = %s, want InvalidUsername", rec.Body.String()) + } + kvRec.assertIdle(t) +} + +func TestAccountCompleteSignupPdsRejection(t *testing.T) { + for pdsTag, want := range map[string]string{ + "HandleNotAvailable": "UsernameNotAvailable", + "InvalidHandle": "InvalidUsername", + "InvalidPassword": "InvalidPassword", + } { + t.Run(pdsTag, func(t *testing.T) { + router, d, kvRec, pds := newSignupXrpc(t) + pds.rejectCreate = pdsTag + + token, _ := generateVerificationToken() + if err := db.AddInflightSignup(d, models.InflightSignup{Email: "bob@example.com", InviteCode: testInviteCode, VerificationToken: token}); err != nil { + t.Fatalf("seed inflight: %v", err) + } + + rec := postOpen(t, router, "org.tangled.temp.account.completeSignup", + `{"token":"`+token+`","username":"bobby","password":"password1234"}`) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), `"`+want+`"`) { + t.Fatalf("error body = %s, want %s", rec.Body.String(), want) + } + if _, err := db.GetInflightByToken(d, token); err != nil { + t.Fatalf("inflight row lost on a rejected username: %v", err) + } + if deletes := pds.AdminDeletes(); len(deletes) != 0 { + t.Fatalf("admin deletes = %v, want none", deletes) + } + kvRec.assertIdle(t) + }) + } +} + +func TestAccountCompleteSignupMissingToken(t *testing.T) { + router, _, kvRec, _ := newSignupXrpc(t) + + rec := postOpen(t, router, "org.tangled.temp.account.completeSignup", + `{"token":" ","username":"bobby","password":"password1234"}`) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "InvalidCode") { + t.Fatalf("error body = %s, want InvalidCode", rec.Body.String()) + } + kvRec.assertIdle(t) +} + +func inflightByEmail(t *testing.T, d *db.DB, email string) models.InflightSignup { + t.Helper() + row, err := db.GetInflightByEmail(d, email) + if err != nil { + t.Fatalf("GetInflightByEmail(%s): %v", email, err) + } + return row +} + +func expireInflight(t *testing.T, d *db.DB, email string) { + t.Helper() + if _, err := d.Exec(`update signups_inflight set expires_at = '2000-01-01T00:00:00Z' where email = ?`, email); err != nil { + t.Fatalf("expire inflight: %v", err) + } +} + +func backdateInflightSent(t *testing.T, d *db.DB, email string) { + t.Helper() + if _, err := d.Exec(`update signups_inflight set last_sent = strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-2 minutes') where email = ?`, email); err != nil { + t.Fatalf("backdate last_sent: %v", err) + } +} + +func inflightCount(t *testing.T, d *db.DB, email string) int { + t.Helper() + var n int + if err := d.QueryRow(`select count(*) from signups_inflight where email = ?`, email).Scan(&n); err != nil { + t.Fatalf("count inflight: %v", err) + } + return n +} + +func TestAccountResendSignupResendsSameLink(t *testing.T) { + router, d, kvRec, pds := newSignupXrpc(t) + + rec := postOpen(t, router, "org.tangled.temp.account.beginSignup", `{"email":"Erin@Example.com","turnstileToken":"test"}`) + if rec.Code != http.StatusOK { + t.Fatalf("begin status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + if mints := pds.InviteMints(); mints != 1 { + t.Fatalf("invite mints after begin = %d, want 1", mints) + } + row := inflightByEmail(t, d, "erin@example.com") + backdateInflightSent(t, d, "erin@example.com") + + emitted := captureStdout(t, func() { + res := postOpen(t, router, "org.tangled.temp.account.resendSignup", `{"email":"erin@example.com"}`) + if res.Code != http.StatusOK { + t.Fatalf("resend status = %d, want 200; body=%s", res.Code, res.Body.String()) + } + }) + + if !strings.Contains(emitted, "to: erin@example.com") { + t.Fatalf("resend not addressed to erin@example.com in:\n%s", emitted) + } + if !strings.Contains(emitted, "/signup/verify?token="+row.VerificationToken) { + t.Fatalf("resend link differs from the stored token in:\n%s", emitted) + } + if mints := pds.InviteMints(); mints != 1 { + t.Fatalf("invite mints after resend = %d, want still 1 (no new code)", mints) + } + if after := inflightByEmail(t, d, "erin@example.com"); after.VerificationToken != row.VerificationToken { + t.Fatalf("resend rotated the token: %q -> %q", row.VerificationToken, after.VerificationToken) + } + kvRec.assertIdle(t) +} + +func TestAccountResendSignupNoPending(t *testing.T) { + router, _, kvRec, _ := newSignupXrpc(t) + + rec := postOpen(t, router, "org.tangled.temp.account.resendSignup", `{"email":"nobody@example.com"}`) + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "NoPendingSignup") { + t.Fatalf("error body = %s, want NoPendingSignup", rec.Body.String()) + } + kvRec.assertIdle(t) +} + +func TestAccountResendSignupExpiredSweeps(t *testing.T) { + router, d, kvRec, _ := newSignupXrpc(t) + + token, _ := generateVerificationToken() + if err := db.AddInflightSignup(d, models.InflightSignup{Email: "erin@example.com", InviteCode: testInviteCode, VerificationToken: token}); err != nil { + t.Fatalf("seed inflight: %v", err) + } + expireInflight(t, d, "erin@example.com") + + rec := postOpen(t, router, "org.tangled.temp.account.resendSignup", `{"email":"erin@example.com"}`) + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "NoPendingSignup") { + t.Fatalf("error body = %s, want NoPendingSignup", rec.Body.String()) + } + if n := inflightCount(t, d, "erin@example.com"); n != 0 { + t.Fatalf("expired row still present after resend miss (count=%d)", n) + } + kvRec.assertIdle(t) +} + +func TestAccountCompleteSignupExpiredToken(t *testing.T) { + router, d, kvRec, pds := newSignupXrpc(t) + + token, _ := generateVerificationToken() + if err := db.AddInflightSignup(d, models.InflightSignup{Email: "erin@example.com", InviteCode: testInviteCode, VerificationToken: token}); err != nil { + t.Fatalf("seed inflight: %v", err) + } + expireInflight(t, d, "erin@example.com") + + rec := postOpen(t, router, "org.tangled.temp.account.completeSignup", + `{"token":"`+token+`","username":"erin","password":"password1234"}`) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "InvalidCode") { + t.Fatalf("error body = %s, want InvalidCode", rec.Body.String()) + } + if n := inflightCount(t, d, "erin@example.com"); n != 0 { + t.Fatalf("expired row still present after completeSignup miss (count=%d)", n) + } + if accounts := pds.CreateAccounts(); len(accounts) != 0 { + t.Fatalf("createAccount called with an expired token: %v", accounts) + } + kvRec.assertIdle(t) +} + +func TestAccountBeginSignupNormalizesTrailingSlashBaseURL(t *testing.T) { + router, _, kvRec, _ := newSignupXrpc(t, func(x *Xrpc) { x.Config.BaseURL = "https://tangled.example/" }) + + emitted := captureStdout(t, func() { + rec := postOpen(t, router, "org.tangled.temp.account.beginSignup", `{"email":"gina-signup@example.com","turnstileToken":"test"}`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + }) + + if strings.Contains(emitted, "//signup/verify") { + t.Fatalf("emailed link contains a double slash:\n%s", emitted) + } + if !strings.Contains(emitted, "https://tangled.example/signup/verify?token=") { + t.Fatalf("emailed link missing normalized base in:\n%s", emitted) + } + kvRec.assertIdle(t) +} + +func TestAccountBeginSignupCooldown(t *testing.T) { + router, _, kvRec, _ := newSignupXrpc(t) + + first := postOpen(t, router, "org.tangled.temp.account.beginSignup", `{"email":"henry-signup@example.com","turnstileToken":"test"}`) + if first.Code != http.StatusOK { + t.Fatalf("first begin status = %d, want 200; body=%s", first.Code, first.Body.String()) + } + + second := postOpen(t, router, "org.tangled.temp.account.beginSignup", `{"email":"henry-signup@example.com","turnstileToken":"test"}`) + if second.Code != http.StatusTooManyRequests { + t.Fatalf("immediate re-begin status = %d, want 429; body=%s", second.Code, second.Body.String()) + } + if !strings.Contains(second.Body.String(), "TooManyRequests") { + t.Fatalf("error body = %s, want TooManyRequests", second.Body.String()) + } + kvRec.assertIdle(t) +} + +func TestAccountResendSignupCooldown(t *testing.T) { + router, d, kvRec, _ := newSignupXrpc(t) + + token, _ := generateVerificationToken() + if err := db.AddInflightSignup(d, models.InflightSignup{Email: "ivan-signup@example.com", InviteCode: testInviteCode, VerificationToken: token}); err != nil { + t.Fatalf("seed inflight: %v", err) + } + backdateInflightSent(t, d, "ivan-signup@example.com") + + first := postOpen(t, router, "org.tangled.temp.account.resendSignup", `{"email":"ivan-signup@example.com"}`) + if first.Code != http.StatusOK { + t.Fatalf("first resend status = %d, want 200; body=%s", first.Code, first.Body.String()) + } + + second := postOpen(t, router, "org.tangled.temp.account.resendSignup", `{"email":"ivan-signup@example.com"}`) + if second.Code != http.StatusTooManyRequests { + t.Fatalf("immediate re-resend status = %d, want 429; body=%s", second.Code, second.Body.String()) + } + kvRec.assertIdle(t) +} + +func TestAccountCompleteSignupRollsBackOnAddEmailConflict(t *testing.T) { + router, d, kvRec, pds := newSignupXrpc(t) + + if err := db.AddEmail(d, models.Email{Did: "did:plc:other", Address: "judy@example.com", Verified: true, Primary: true}); err != nil { + t.Fatalf("seed competing email: %v", err) + } + token, _ := generateVerificationToken() + if err := db.AddInflightSignup(d, models.InflightSignup{Email: "judy@example.com", InviteCode: testInviteCode, VerificationToken: token}); err != nil { + t.Fatalf("seed inflight: %v", err) + } + + rec := postOpen(t, router, "org.tangled.temp.account.completeSignup", + `{"token":"`+token+`","username":"judy","password":"password1234"}`) + if rec.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "EmailAlreadyRegistered") { + t.Fatalf("error body = %s, want EmailAlreadyRegistered", rec.Body.String()) + } + + deletes := pds.AdminDeletes() + if len(deletes) != 1 || deletes[0] != testNewDid { + t.Fatalf("admin.deleteAccount calls = %v, want [%s]", deletes, testNewDid) + } + emails, err := db.GetAllEmails(d, testNewDid) + if err != nil { + t.Fatalf("GetAllEmails: %v", err) + } + if len(emails) != 0 { + t.Fatalf("email row leaked for rolled-back account: %v", emails) + } + if _, err := db.GetInflightByToken(d, token); err != nil { + t.Fatalf("inflight row lost on conflict: %v", err) + } + kvRec.assertIdle(t) +} + +// typed mirror of the generated output so the assertions don't depend on codegen +type tangledTempAccountCompleteSignupOutput struct { + Did string `json:"did"` + Handle string `json:"handle"` + Authorization *struct { + Code string `json:"code"` + State string `json:"state"` + Iss string `json:"iss"` + } `json:"authorization"` +} + +func completeSignupWithRequest(t *testing.T, requestUri string, configure func(*pdsRecorder)) (tangledTempAccountCompleteSignupOutput, *pdsRecorder) { + t.Helper() + router, d, _, pds := newSignupXrpc(t) + if configure != nil { + configure(pds) + } + token, _ := generateVerificationToken() + if err := db.AddInflightSignup(d, models.InflightSignup{Email: "lena@example.com", InviteCode: testInviteCode, VerificationToken: token}); err != nil { + t.Fatalf("seed inflight: %v", err) + } + body := map[string]string{"token": token, "username": "lenaa", "password": "password1234"} + if requestUri != "" { + body["requestUri"] = requestUri + } + raw, _ := json.Marshal(body) + rec := postOpen(t, router, "org.tangled.temp.account.completeSignup", string(raw)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var out tangledTempAccountCompleteSignupOutput + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v; body=%s", err, rec.Body.String()) + } + if out.Did != testNewDid { + t.Fatalf("did = %q, want %q", out.Did, testNewDid) + } + if deletes := pds.AdminDeletes(); len(deletes) != 0 { + t.Fatalf("admin deletes = %v, want the account kept", deletes) + } + return out, pds +} + +func TestAccountCompleteSignupAuthorizesTheBrowsersRequest(t *testing.T) { + const requestUri = "urn:ietf:params:oauth:request_uri:req-abc" + out, pds := completeSignupWithRequest(t, requestUri, nil) + + calls := pds.Preauthorizations() + if len(calls) != 1 || calls[0]["did"] != testNewDid || calls[0]["requestUri"] != requestUri { + t.Fatalf("preauthorize calls = %v, want one for %s with %s", calls, testNewDid, requestUri) + } + if out.Authorization == nil { + t.Fatal("authorization missing from the output") + } + if out.Authorization.Code != "cod-123" || out.Authorization.State != "state-456" || out.Authorization.Iss != "https://pds.example" { + t.Fatalf("authorization = %+v, want the pds's code, state and iss", *out.Authorization) + } +} + +func TestAccountCompleteSignupWithoutRequestSkipsThePds(t *testing.T) { + out, pds := completeSignupWithRequest(t, "", nil) + if calls := pds.Preauthorizations(); len(calls) != 0 { + t.Fatalf("preauthorize calls = %v, want none", calls) + } + if out.Authorization != nil { + t.Fatalf("authorization = %+v, want none", *out.Authorization) + } +} + +func TestAccountCompleteSignupOnAStockPdsStillCreatesTheAccount(t *testing.T) { + out, _ := completeSignupWithRequest(t, "urn:ietf:params:oauth:request_uri:req-abc", func(p *pdsRecorder) { + p.noPreauthorize = true + }) + if out.Authorization != nil { + t.Fatalf("authorization = %+v, want none from a pds without the route", *out.Authorization) + } +} + +func getSignup(t *testing.T, router http.Handler, token string) *httptest.ResponseRecorder { + t.Helper() + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/org.tangled.temp.account.getSignup?token="+token, nil)) + return rec +} + +func TestAccountGetSignupReturnsEmail(t *testing.T) { + var pdsHost string + router, d, kvRec, _ := newSignupXrpc(t, func(x *Xrpc) { pdsHost = x.Config.Pds.Host }) + + token, _ := generateVerificationToken() + if err := db.AddInflightSignup(d, models.InflightSignup{Email: "fay@example.com", InviteCode: testInviteCode, VerificationToken: token}); err != nil { + t.Fatalf("seed inflight: %v", err) + } + + rec := getSignup(t, router, token) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var out struct { + Email string `json:"email"` + Pds string `json:"pds"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v; body=%s", err, rec.Body.String()) + } + if out.Email != "fay@example.com" { + t.Fatalf("email = %q, want fay@example.com", out.Email) + } + if out.Pds != pdsHost { + t.Fatalf("pds = %q, want %q", out.Pds, pdsHost) + } + if _, err := db.GetInflightByToken(d, token); err != nil { + t.Fatalf("lookup consumed the inflight row: %v", err) + } + kvRec.assertIdle(t) +} + +func TestAccountGetSignupDeadLink(t *testing.T) { + router, d, _, _ := newSignupXrpc(t) + + expired, _ := generateVerificationToken() + if err := db.AddInflightSignup(d, models.InflightSignup{Email: "gus@example.com", InviteCode: testInviteCode, VerificationToken: expired}); err != nil { + t.Fatalf("seed inflight: %v", err) + } + expireInflight(t, d, "gus@example.com") + + for name, token := range map[string]string{"unknown": strings.Repeat("ab", 32), "expired": expired, "empty": ""} { + t.Run(name, func(t *testing.T) { + rec := getSignup(t, router, token) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "InvalidCode") { + t.Fatalf("error body = %s, want InvalidCode", rec.Body.String()) + } + }) + } +} + +func TestSignupLinkEmailStatesExpiry(t *testing.T) { + text, html := signupLinkEmail("https://tangled.example/signup/verify?token=abc", "https://assets.example") + for name, body := range map[string]string{"text": text, "html": html} { + if !strings.Contains(body, "expires in 24 hours") { + t.Errorf("%s body does not state the link expiry: %s", name, body) + } + if !strings.Contains(body, "https://tangled.example/signup/verify?token=abc") { + t.Errorf("%s body lost the link: %s", name, body) + } + } +} diff --git a/deliberi/xrpc/xrpc.go b/deliberi/xrpc/xrpc.go new file mode 100644 index 0000000000000000000000000000000000000000..3653cc53e346e62f51a80c40c234a1c2355b0dc2 --- /dev/null +++ b/deliberi/xrpc/xrpc.go @@ -0,0 +1,148 @@ +package xrpc + +import ( + "context" + "encoding/json" + "log/slog" + "net/http" + "runtime/debug" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-chi/chi/v5" + tangled "tangled.org/core/api/org_tangled" + shTangled "tangled.org/core/api/tangled" + "tangled.org/core/deliberi/analytics" + config "tangled.org/core/deliberi/config" + db "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/kv" + "tangled.org/core/deliberi/mailer" + "tangled.org/core/idresolver" + xrpcerr "tangled.org/core/xrpc/errors" + "tangled.org/core/xrpc/serviceauth" +) + +const ActorDid = serviceauth.ActorDid + +type Xrpc struct { + DB *db.DB + Config *config.Config + Logger *slog.Logger + ServiceAuth *serviceauth.ServiceAuth + IdResolver *idresolver.Resolver + Sender *mailer.Sender + KV kv.EmailKV + Invites InviteReconciler + // nil when analytics is off + Analytics *analytics.Client +} + +type InviteReconciler interface { + Reconcile(ctx context.Context, recipientDid string) +} + +func (x *Xrpc) Router() http.Handler { + r := chi.NewRouter() + r.Use(x.cors) + + r.Get("/_health", x.health) + if x.Config.Dev { + r.Get("/_ingested", x.ingested) + } + r.Post("/"+tangled.TempAccountBeginSignupNSID, x.AccountBeginSignup) + r.Post("/"+tangled.TempAccountResendSignupNSID, x.AccountResendSignup) + r.Get("/"+tangled.TempAccountGetSignupNSID, x.AccountGetSignup) + r.Post("/"+tangled.TempAccountCompleteSignupNSID, x.AccountCompleteSignup) + r.Post("/"+tangled.TempAccountVerifyEmailNSID, x.AccountVerifyEmail) + r.Post("/"+tangled.TempNewsletterSignupNSID, x.NewsletterSignup) + r.Group(func(r chi.Router) { + r.Use(x.ServiceAuth.VerifyServiceAuth) + + r.Post("/"+shTangled.IdentityResolveCommittersNSID, x.ResolveCommitters) + + r.Get("/"+tangled.TempNotificationListNotificationsNSID, x.NotificationList) + r.Get("/"+tangled.TempNotificationGetUnreadCountNSID, x.NotificationGetUnreadCount) + r.Post("/"+tangled.TempNotificationUpdateSeenNSID, x.NotificationUpdateSeen) + r.Post("/"+tangled.TempNotificationMarkAllReadNSID, x.NotificationMarkAllRead) + r.Post("/"+tangled.TempNotificationMarkEntityReadNSID, x.NotificationMarkEntityRead) + r.Get("/"+tangled.TempNotificationGetPreferencesNSID, x.NotificationGetPreferences) + r.Post("/"+tangled.TempNotificationUpdatePreferencesNSID, x.NotificationUpdatePreferences) + + r.Get("/"+tangled.TempAccountListEmailsNSID, x.AccountListEmails) + r.Post("/"+tangled.TempAccountDeleteEmailNSID, x.AccountDeleteEmail) + r.Post("/"+tangled.TempAccountSetPrimaryEmailNSID, x.AccountSetPrimaryEmail) + r.Post("/"+tangled.TempAccountAddEmailNSID, x.AccountAddEmail) + r.Post("/"+tangled.TempAccountSubscribeNewsletterNSID, x.AccountSubscribeNewsletter) + r.Post("/"+tangled.TempAccountDismissNewsletterNSID, x.AccountDismissNewsletter) + r.Get("/"+tangled.TempAccountGetNewsletterPreferenceNSID, x.AccountGetNewsletterPreference) + }) + + return r +} + +const timeFormat = "2006-01-02T15:04:05.000Z" + +func (x *Xrpc) health(w http.ResponseWriter, r *http.Request) { + x.writeJSON(w, map[string]string{"version": serviceVersion()}) +} + +func serviceVersion() string { + info, ok := debug.ReadBuildInfo() + if !ok { + return "dev" + } + for _, s := range info.Settings { + if s.Key == "vcs.revision" && s.Value != "" { + return s.Value + } + } + return "dev" +} + +// cors opens the xrpc endpoints to browser origins; bearer auth, not cookies, +// makes a wildcard origin safe +func (x *Xrpc) cors(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization") + w.Header().Set("Access-Control-Max-Age", "86400") + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusNoContent) + return + } + next.ServeHTTP(w, r) + }) +} + +func writeError(w http.ResponseWriter, e xrpcerr.XrpcError, status int) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(e) +} + +func (x *Xrpc) writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(v) +} + +func actorDid(r *http.Request) (string, bool) { + did, ok := r.Context().Value(ActorDid).(syntax.DID) + if !ok { + return "", false + } + return did.String(), true +} + +var ( + errInternal = xrpcErrorTag("InternalError", "internal server error") + errBadRequestBody = xrpcErrorTag("InvalidRequest", "invalid request body") + errUpstream = xrpcErrorTag("UpstreamError", "an upstream service failed") +) + +func xrpcErrorTag(tag, message string) xrpcerr.XrpcError { + return xrpcerr.NewXrpcError(xrpcerr.WithTag(tag), xrpcerr.WithMessage(message)) +} + +func badRequestError(message string) xrpcerr.XrpcError { + return xrpcErrorTag("InvalidRequest", message) +} diff --git a/deliberi/xrpc/xrpc_test.go b/deliberi/xrpc/xrpc_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2b21cf55cf1c6c23ece2a480785ae26b71a152e5 --- /dev/null +++ b/deliberi/xrpc/xrpc_test.go @@ -0,0 +1,398 @@ +package xrpc + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/auth" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + tangled "tangled.org/core/api/org_tangled" + config "tangled.org/core/deliberi/config" + db "tangled.org/core/deliberi/db" + "tangled.org/core/deliberi/models" + "tangled.org/core/orm" + "tangled.org/core/xrpc/serviceauth" +) + +const ( + testActor = "did:plc:tester" + testAudience = "did:web:test.example" +) + +func newTestXrpc(t *testing.T) (http.Handler, *db.DB, func(nsid string) string) { + x, router, d, sign := newTestXrpcFull(t, nil) + _ = x + return router, d, sign +} + +func newTestXrpcFull(t *testing.T, configure func(*Xrpc)) (*Xrpc, http.Handler, *db.DB, func(nsid string) string) { + t.Helper() + + d, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("db.Make: %v", err) + } + t.Cleanup(func() { d.Close() }) + + priv, err := atcrypto.GeneratePrivateKeyP256() + if err != nil { + t.Fatalf("generate key: %v", err) + } + pub, err := priv.PublicKey() + if err != nil { + t.Fatalf("derive pubkey: %v", err) + } + + dir := identity.NewMockDirectory() + dir.Insert(identity.Identity{ + DID: syntax.DID(testActor), + Keys: map[string]identity.VerificationMethod{ + "atproto": {Type: "Multikey", PublicKeyMultibase: pub.Multibase()}, + }, + }) + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + x := &Xrpc{ + DB: d, + Config: &config.Config{}, + Logger: logger, + ServiceAuth: serviceauth.NewServiceAuth(logger, dir, testAudience), + } + if configure != nil { + configure(x) + } + + sign := func(nsid string) string { + lxm := syntax.NSID(nsid) + token, err := auth.SignServiceAuth(syntax.DID(testActor), testAudience, time.Minute, &lxm, priv) + if err != nil { + t.Fatalf("sign service auth: %v", err) + } + return token + } + + return x, x.Router(), d, sign +} + +func TestHealth(t *testing.T) { + router, _, _ := newTestXrpc(t) + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/_health", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + var body map[string]string + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v; body=%s", err, rec.Body.String()) + } + if body["version"] == "" { + t.Fatalf("missing version in %s", rec.Body.String()) + } +} + +func TestServiceAuthRequired(t *testing.T) { + router, _, _ := newTestXrpc(t) + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/org.tangled.temp.notification.getUnreadCount", nil)) + + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403 without a service-auth token", rec.Code) + } +} + +func TestWrongLexiconTokenRejected(t *testing.T) { + router, _, sign := newTestXrpc(t) + + req := httptest.NewRequest(http.MethodGet, "/org.tangled.temp.notification.getUnreadCount", nil) + req.Header.Set("Authorization", "Bearer "+sign("org.tangled.temp.notification.listNotifications")) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403 for a token bound to a different method", rec.Code) + } +} + +func TestNotificationGetUnreadCount(t *testing.T) { + router, d, sign := newTestXrpc(t) + + nsid := "org.tangled.temp.notification.getUnreadCount" + call := func() int { + req := httptest.NewRequest(http.MethodGet, "/"+nsid, nil) + req.Header.Set("Authorization", "Bearer "+sign(nsid)) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var out struct { + Count int `json:"count"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v; body=%s", err, rec.Body.String()) + } + return out.Count + } + + if got := call(); got != 0 { + t.Fatalf("empty db count = %d, want 0", got) + } + + uri := "at://did:plc:repo/org.tangled.track.ticket/abc" + if err := db.CreateNotification(d, &models.Notification{ + RecipientDid: testActor, + AtUri: uri, + Type: models.NotificationTypeIssueCreated, + ActorDid: "did:plc:someone", + }); err != nil { + t.Fatalf("CreateNotification: %v", err) + } + + if got := call(); got != 1 { + t.Fatalf("count after one unread = %d, want 1", got) + } + + if err := db.MarkRead(d, testActor, uri, true); err != nil { + t.Fatalf("MarkRead: %v", err) + } + if got := call(); got != 0 { + t.Fatalf("count after marking read = %d, want 0", got) + } +} + +func TestNotificationList(t *testing.T) { + router, d, sign := newTestXrpc(t) + + uri := "at://did:plc:repo/org.tangled.track.ticket/abc" + pullUri := "at://did:plc:repo/org.tangled.track.ticket/def" + for _, n := range []*models.Notification{ + { + RecipientDid: testActor, + AtUri: uri, + Type: models.NotificationTypeIssueCreated, + ActorDid: "did:plc:someone", + RepoDid: "did:plc:repo", + EntityAt: uri, + EntityKind: models.EntityKindIssue, + EntityTitle: "a bug report", + Created: time.Now().Add(-time.Minute), + }, + { + RecipientDid: testActor, + AtUri: "at://did:plc:repo/org.tangled.feed.comment/c1", + Type: models.NotificationTypeUserMentioned, + ActorDid: "did:plc:someone", + RepoDid: "did:plc:repo", + EntityAt: pullUri, + EntityKind: models.EntityKindPull, + }, + } { + if err := db.CreateNotification(d, n); err != nil { + t.Fatalf("CreateNotification: %v", err) + } + } + + list := func(query string) tangled.TempNotificationListNotifications_Output { + nsid := "org.tangled.temp.notification.listNotifications" + req := httptest.NewRequest(http.MethodGet, "/"+nsid+query, nil) + req.Header.Set("Authorization", "Bearer "+sign(nsid)) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var out tangled.TempNotificationListNotifications_Output + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v; body=%s", err, rec.Body.String()) + } + return out + } + + out := list("") + if len(out.Notifications) != 2 { + t.Fatalf("got %d notifications, want 2", len(out.Notifications)) + } + // same collection for both: only the persisted kind tells pull from issue + mention := out.Notifications[0] + if mention.PullAt == nil || *mention.PullAt != pullUri || mention.IssueAt != nil { + t.Fatalf("mention pullAt/issueAt = %v/%v, want %q/nil", mention.PullAt, mention.IssueAt, pullUri) + } + n := out.Notifications[1] + if n.Uri != uri { + t.Fatalf("uri = %q, want %q", n.Uri, uri) + } + if n.Type != string(models.NotificationTypeIssueCreated) { + t.Fatalf("type = %q, want issue_created", n.Type) + } + if n.Category != "work" { + t.Fatalf("category = %q, want work", n.Category) + } + if n.RepoDid == nil || *n.RepoDid != "did:plc:repo" { + t.Fatalf("repoDid = %v, want did:plc:repo", n.RepoDid) + } + if n.IssueAt == nil || *n.IssueAt != uri || n.PullAt != nil { + t.Fatalf("issueAt/pullAt = %v/%v, want %q/nil", n.IssueAt, n.PullAt, uri) + } + if out.WorkUnreadCount != 2 { + t.Fatalf("workUnreadCount = %d, want 2", out.WorkUnreadCount) + } + + if err := db.MarkRead(d, testActor, uri, true); err != nil { + t.Fatalf("MarkRead: %v", err) + } + if out := list("?read=unread"); len(out.Notifications) != 1 { + t.Fatalf("unread list after read = %d, want 1", len(out.Notifications)) + } +} + +func TestUpdateSeenPersists(t *testing.T) { + router, d, sign := newTestXrpc(t) + + uri := "at://did:plc:repo/org.tangled.track.ticket/xyz" + if err := db.CreateNotification(d, &models.Notification{ + RecipientDid: testActor, + AtUri: uri, + Type: models.NotificationTypeIssueCreated, + ActorDid: "did:plc:someone", + }); err != nil { + t.Fatalf("CreateNotification: %v", err) + } + + nsid := "org.tangled.temp.notification.updateSeen" + req := httptest.NewRequest(http.MethodPost, "/"+nsid, strings.NewReader(`{"uri":"`+uri+`","read":true}`)) + req.Header.Set("Authorization", "Bearer "+sign(nsid)) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + + unread, err := db.CountNotifications(d, testActor, orm.FilterEq("read", 0)) + if err != nil { + t.Fatalf("CountNotifications: %v", err) + } + if unread != 0 { + t.Fatalf("unread after updateSeen = %d, want 0", unread) + } +} + +func TestStatePreferencesRoundTrip(t *testing.T) { + router, d, sign := newTestXrpc(t) + + call := func(method, nsid, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, "/"+nsid, strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+sign(nsid)) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s status = %d; body=%s", nsid, rec.Code, rec.Body.String()) + } + return rec + } + + call(http.MethodPost, "org.tangled.temp.notification.updatePreferences", `{"issueClosed":false,"pullMerged":false}`) + + var got tangled.TempNotificationGetPreferences_Preferences + rec := call(http.MethodGet, "org.tangled.temp.notification.getPreferences", "") + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.IssueClosed || got.PullMerged || !got.IssueCreated { + t.Errorf("preferences = %+v, want issueClosed and pullMerged off and the rest untouched", got) + } + prefs, err := db.GetNotificationPreference(d, testActor) + if err != nil || prefs.IssueClosed || prefs.PullMerged { + t.Errorf("stored prefs = %+v, %v", prefs, err) + } +} + +func TestMarkEntityReadPersists(t *testing.T) { + router, d, sign := newTestXrpc(t) + + const entityAt = "at://did:plc:repo/org.tangled.track.ticket/xyz" + + for _, n := range []*models.Notification{ + {RecipientDid: testActor, AtUri: entityAt, Type: models.NotificationTypeIssueCreated, ActorDid: "did:plc:someone", EntityAt: entityAt}, + {RecipientDid: testActor, AtUri: "at://did:plc:repo/org.tangled.feed.comment/abc", Type: models.NotificationTypeIssueCommented, ActorDid: "did:plc:commenter", EntityAt: entityAt}, + {RecipientDid: testActor, AtUri: "at://did:plc:repo/org.tangled.track.ticket/other", Type: models.NotificationTypeIssueCreated, ActorDid: "did:plc:someone", EntityAt: "at://did:plc:repo/org.tangled.track.ticket/other"}, + } { + if err := db.CreateNotification(d, n); err != nil { + t.Fatalf("CreateNotification: %v", err) + } + } + + nsid := "org.tangled.temp.notification.markEntityRead" + req := httptest.NewRequest(http.MethodPost, "/"+nsid, strings.NewReader(`{"entityAt":"`+entityAt+`"}`)) + req.Header.Set("Authorization", "Bearer "+sign(nsid)) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + + unread, err := db.CountNotifications(d, testActor, orm.FilterEq("read", 0)) + if err != nil { + t.Fatalf("CountNotifications: %v", err) + } + if unread != 1 { + t.Fatalf("unread after markEntityRead = %d, want 1 (only the unrelated one)", unread) + } +} + +func onlyNotification( + t *testing.T, + router http.Handler, + sign func(string) string, +) (*tangled.TempNotificationListNotifications_Notification, tangled.TempNotificationListNotifications_Output) { + t.Helper() + const nsid = "org.tangled.temp.notification.listNotifications" + req := httptest.NewRequest(http.MethodGet, "/"+nsid, nil) + req.Header.Set("Authorization", "Bearer "+sign(nsid)) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + + var out tangled.TempNotificationListNotifications_Output + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v; body=%s", err, rec.Body.String()) + } + if len(out.Notifications) != 1 { + t.Fatalf("got %d notifications, want 1", len(out.Notifications)) + } + return out.Notifications[0], out +} + +func TestNotificationListHidesKnotOnCollaboratorOffer(t *testing.T) { + router, d, sign := newTestXrpc(t) + + if err := db.CreateNotification(d, &models.Notification{ + RecipientDid: testActor, + AtUri: "at://did:plc:scallop/org.tangled.graph.collaborator/" + testActor, + Type: models.NotificationTypeCollaboratorInvited, + ActorDid: "did:plc:boltless", + RepoDid: "did:plc:scallop", + KnotDid: "did:web:knot.example", + }); err != nil { + t.Fatalf("CreateNotification: %v", err) + } + + n, _ := onlyNotification(t, router, sign) + if n.RepoDid == nil || *n.RepoDid != "did:plc:scallop" { + t.Fatalf("repoDid = %v, want did:plc:scallop", n.RepoDid) + } +} diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..8c4985a4fe04043a16026050e56e71f827cbff31 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +dist/ +public/fonts/ diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs new file mode 100644 index 0000000000000000000000000000000000000000..3caf6acfab8ae4376336d3be934a12d7e2c7e409 --- /dev/null +++ b/docs/astro.config.mjs @@ -0,0 +1,229 @@ +// @ts-check +import { fileURLToPath } from "node:url"; +import { defineConfig } from "astro/config"; +import starlight from "@astrojs/starlight"; +import starlightLlmActions from "starlight-llm-actions"; +import starlightLinksValidator from "starlight-links-validator"; +import sitemap from "@astrojs/sitemap"; +import svelte from "@astrojs/svelte"; +import tailwindcss from "@tailwindcss/vite"; +import Icons from "unplugin-icons/vite"; +import { fontsBase } from "../web/src/fontsBase.mjs"; + +// the ui components are shared with the web app, imported straight from its +// source rather than published as a package +const webLib = fileURLToPath(new URL("../web/src/lib", import.meta.url)); +const base = "/docs"; + +const svelteStyleFallback = { + name: "svelte-style-fallback", + enforce: "pre", + transform(code, id) { + if (!id.includes("svelte&type=style&lang.css")) return; + if (!code.includes("]*>([\s\S]*?)<\/style>/i); + return { code: match ? match[1] : "", map: null }; + }, +}; + +export default defineConfig({ + site: "https://tangled.org", + // served at tangled.org/docs by the docs worker. the build lands in dist/docs + // because static assets on workers are matched against the full request path + base, + outDir: "./dist/docs", + integrations: [ + starlight({ + title: "Tangled Docs", + titleDelimiter: "·", + disable404Route: true, + routeMiddleware: ["./src/starlightRouteData.ts"], + tableOfContents: { minHeadingLevel: 2, maxHeadingLevel: 2 }, + plugins: [ + starlightLlmActions({ + renderMarkdown: "simple", + linkAlternate: true, + llmsTxt: { exclude: ["index"] }, + }), + starlightLinksValidator(), + ], + description: + "Documentation for Tangled — decentralized code collaboration built on the AT Protocol.", + social: [ + { icon: "x.com", label: "X", href: "https://x.com/tangled_org" }, + { + icon: "blueSky", + label: "Bluesky", + href: "https://bsky.app/profile/tangled.org", + }, + ], + favicon: "/favicon.svg", + head: [ + { + tag: "link", + attrs: { + rel: "preload", + href: `${base}/fonts/TimelessSansVF.woff2`, + as: "font", + type: "font/woff2", + crossorigin: "anonymous", + }, + }, + { + tag: "link", + // head entries are emitted verbatim; starlight only rebases `favicon` + attrs: { rel: "apple-touch-icon", href: "/docs/apple-touch-icon.png" }, + }, + ], + editLink: { + baseUrl: "https://tangled.org/@tangled.org/core/blob/master/docs/", + }, + customCss: [ + "./src/styles/global.css", + "./src/styles/tangled.css", + "./src/styles/custom.css", + "./src/styles/diagrams.css", + ], + components: { + Header: "./src/components/Header.astro", + Hero: "./src/components/Hero.astro", + MarkdownContent: "./src/components/MarkdownContent.astro", + MobileMenuFooter: "./src/components/MobileMenuFooter.astro", + PageTitle: "./src/components/PageTitle.astro", + Sidebar: "./src/components/Sidebar.astro", + SiteTitle: "./src/components/SiteTitle.astro", + ThemeProvider: "./src/components/ThemeProvider.astro", + }, + expressiveCode: { + // same palettes as the web app's syntax tokens + themes: ["catppuccin-macchiato", "catppuccin-latte"], + // a plain bordered block like tangled.org's, no editor or terminal chrome + defaultProps: { frame: "none" }, + styleOverrides: { + // expressive-code adds borderWidth to this for the outer + // radius, so subtract it to land on --radius-md exactly + borderRadius: "calc(var(--radius-md) - 1px)", + borderColor: "var(--color-border-default)", + borderWidth: "1px", + codeBackground: "var(--color-background-canvas)", + codeFontFamily: "var(--font-mono)", + codeFontSize: "0.8125rem", + codeLineHeight: "1.5385", + codePaddingBlock: "0.75rem", + codePaddingInline: "0.75rem", + frames: { + shadowColor: "transparent", + frameBoxShadowCssValue: "none", + }, + }, + }, + sidebar: [ + { + label: "Getting started", + items: [ + "introduction", + "quick-start", + "comparison-with-github", + "migrating-from-github", + "why-atproto", + ], + }, + { + label: "Using Tangled", + items: [ + "guides/profile", + "guides/repositories", + "guides/pull-requests", + "guides/access", + "guides/vouching", + "guides/notifications", + "guides/organizations", + "guides/signing-commits", + "guides/importing", + "sites", + "troubleshooting", + ], + }, + { + label: "Repositories", + items: ["knots/repositories", "knots/self-hosting"], + }, + { + label: "Running CI", + items: [ + "spindles", + "spindles/writing-workflows", + "spindles/workflows", + "spindles/secrets", + "spindles/webhooks", + "spindles/self-hosting", + ], + }, + { + label: "Self-hosting", + items: [ + "self-hosting", + "self-hosting/knot", + "self-hosting/spindle", + "self-hosting/frontend", + "self-hosting/indexer", + ], + }, + { + label: "Integrations", + items: ["integrations/discord", "integrations/contributing"], + }, + { + label: "Upgrading", + items: ["upgrading/knots-and-spindles", "upgrading/knot-2"], + }, + { + label: "API", + items: ["api/knot", "api/spindle", "api/bobbin"], + }, + { + label: "Reference", + items: [ + "reference/records", + "reference/identifiers", + "reference/knot-config", + "reference/spindle-config", + ], + }, + { + label: "Contributing", + items: ["contributing/hacking", "contributing/guide"], + }, + ], + }), + sitemap(), + svelte(), + ], + vite: { + plugins: [svelteStyleFallback, tailwindcss(), Icons({ compiler: "svelte" }), fontsBase(base)], + resolve: { + // same aliases the web app's components import through + alias: { + $lib: webLib, + // the web app's own imports (sveltekit 3 subpath imports) + "#lib": webLib, + "$icon/": "~icons/lucide/", + // routing and environment are sveltekit-only; components that build hrefs or + // guard browser apis import them + "$app/paths": fileURLToPath(new URL("./src/stubs/app-paths.ts", import.meta.url)), + "$app/env": fileURLToPath(new URL("./src/stubs/app-env.ts", import.meta.url)), + }, + // web/ has no node_modules in the nix build: resolve the components' + // runtime deps (and svelte itself, which must be a single copy) from here + dedupe: ["svelte", "tailwind-variants", "tailwind-merge"], + }, + server: { + fs: { + allow: [ + fileURLToPath(new URL(".", import.meta.url)), + fileURLToPath(new URL("../web/src", import.meta.url)), + ], + }, + }, + }, +}); diff --git a/docs/package-lock.json b/docs/package-lock.json new file mode 100644 index 0000000000000000000000000000000000000000..8da0e7ead26f82621be3b3948073204afcf1585f --- /dev/null +++ b/docs/package-lock.json @@ -0,0 +1,8435 @@ +{ + "name": "tangled-docs", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tangled-docs", + "version": "0.0.1", + "dependencies": { + "@astrojs/mdx": "^4.3.14", + "@astrojs/sitemap": "^3.7.4", + "@astrojs/starlight": "^0.36.0", + "@astrojs/starlight-tailwind": "^4.0.2", + "@astrojs/svelte": "^7.2.5", + "@tailwindcss/vite": "^4.3.3", + "astro": "^5.14.0", + "hast-util-select": "^6.0.4", + "rehype-parse": "^9.0.1", + "rehype-remark": "^10.0.1", + "remark-gfm": "^4.0.1", + "remark-stringify": "^11.0.0", + "sharp": "^0.34.0", + "starlight-links-validator": "^0.19.2", + "starlight-llm-actions": "0.13.0", + "svelte": "^5.57.2", + "tailwind-merge": "^3.7.0", + "tailwind-variants": "^3.3.1", + "tailwindcss": "^4.3.3", + "unified": "^11.0.5", + "unist-util-remove": "^4.0.0" + }, + "devDependencies": { + "@iconify-json/lucide": "^1.2.140", + "typescript": "^5.9.3", + "unplugin-icons": "^24.0.0" + } + }, + "node_modules/@antfu/install-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-1.1.0.tgz", + "integrity": "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "package-manager-detector": "^1.3.0", + "tinyexec": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@astrojs/compiler": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/@astrojs/compiler/-/compiler-2.13.1.tgz", + "integrity": "sha512-f3FN83d2G/v32ipNClRKgYv30onQlMZX1vCeZMjPsMMPl1mDpmbl0+N5BYo4S/ofzqJyS5hvwacEo0CCVDn/Qg==", + "license": "MIT" + }, + "node_modules/@astrojs/internal-helpers": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/@astrojs/internal-helpers/-/internal-helpers-0.7.6.tgz", + "integrity": "sha512-GOle7smBWKfMSP8osUIGOlB5kaHdQLV3foCsf+5Q9Wsuu+C6Fs3Ez/ttXmhjZ1HkSgsogcM1RXSjjOVieHq16Q==", + "license": "MIT" + }, + "node_modules/@astrojs/markdown-remark": { + "version": "6.3.11", + "resolved": "https://registry.npmjs.org/@astrojs/markdown-remark/-/markdown-remark-6.3.11.tgz", + "integrity": "sha512-hcaxX/5aC6lQgHeGh1i+aauvSwIT6cfyFjKWvExYSxUhZZBBdvCliOtu06gbQyhbe0pGJNoNmqNlQZ5zYUuIyQ==", + "license": "MIT", + "dependencies": { + "@astrojs/internal-helpers": "0.7.6", + "@astrojs/prism": "3.3.0", + "github-slugger": "^2.0.0", + "hast-util-from-html": "^2.0.3", + "hast-util-to-text": "^4.0.2", + "import-meta-resolve": "^4.2.0", + "js-yaml": "^4.1.1", + "mdast-util-definitions": "^6.0.0", + "rehype-raw": "^7.0.0", + "rehype-stringify": "^10.0.1", + "remark-gfm": "^4.0.1", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.1.2", + "remark-smartypants": "^3.0.2", + "shiki": "^3.21.0", + "smol-toml": "^1.6.0", + "unified": "^11.0.5", + "unist-util-remove-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "unist-util-visit-parents": "^6.0.2", + "vfile": "^6.0.3" + } + }, + "node_modules/@astrojs/mdx": { + "version": "4.3.14", + "resolved": "https://registry.npmjs.org/@astrojs/mdx/-/mdx-4.3.14.tgz", + "integrity": "sha512-FBrqJQORVm+rkRa2TS5CjU9PBA6hkhrwLVBSS9A77gN2+iehvjq1w6yya/d0YKC7osiVorKkr3Qd9wNbl0ZkGA==", + "license": "MIT", + "dependencies": { + "@astrojs/markdown-remark": "6.3.11", + "@mdx-js/mdx": "^3.1.1", + "acorn": "^8.15.0", + "es-module-lexer": "^1.7.0", + "estree-util-visit": "^2.0.0", + "hast-util-to-html": "^9.0.5", + "piccolore": "^0.1.3", + "rehype-raw": "^7.0.0", + "remark-gfm": "^4.0.1", + "remark-smartypants": "^3.0.2", + "source-map": "^0.7.6", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.3" + }, + "engines": { + "node": "18.20.8 || ^20.3.0 || >=22.0.0" + }, + "peerDependencies": { + "astro": "^5.0.0" + } + }, + "node_modules/@astrojs/prism": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@astrojs/prism/-/prism-3.3.0.tgz", + "integrity": "sha512-q8VwfU/fDZNoDOf+r7jUnMC2//H2l0TuQ6FkGJL8vD8nw/q5KiL3DS1KKBI3QhI9UQhpJ5dc7AtqfbXWuOgLCQ==", + "license": "MIT", + "dependencies": { + "prismjs": "^1.30.0" + }, + "engines": { + "node": "18.20.8 || ^20.3.0 || >=22.0.0" + } + }, + "node_modules/@astrojs/sitemap": { + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/@astrojs/sitemap/-/sitemap-3.7.4.tgz", + "integrity": "sha512-LbKNC24bdUWcQf/pThB6qLlSqHojxGjZDURIzFocY8rlWnAn2t74nnhnK6S5x0NHriHoAduLEpVjRykmeGiVvA==", + "license": "MIT", + "dependencies": { + "sitemap": "^9.0.0", + "zod": "^4.3.6" + } + }, + "node_modules/@astrojs/starlight": { + "version": "0.36.3", + "resolved": "https://registry.npmjs.org/@astrojs/starlight/-/starlight-0.36.3.tgz", + "integrity": "sha512-5cm4QVQHUP6ZE52O43TtUpsTvLKdZa9XEs4l3suzuY7Ymsbz4ojtoL9NhistbMqM+/qk6fm6SmxbOL6hQ/LfNA==", + "license": "MIT", + "dependencies": { + "@astrojs/markdown-remark": "^6.3.1", + "@astrojs/mdx": "^4.2.3", + "@astrojs/sitemap": "^3.3.0", + "@pagefind/default-ui": "^1.3.0", + "@types/hast": "^3.0.4", + "@types/js-yaml": "^4.0.9", + "@types/mdast": "^4.0.4", + "astro-expressive-code": "^0.41.1", + "bcp-47": "^2.1.0", + "hast-util-from-html": "^2.0.1", + "hast-util-select": "^6.0.2", + "hast-util-to-string": "^3.0.0", + "hastscript": "^9.0.0", + "i18next": "^23.11.5", + "js-yaml": "^4.1.0", + "klona": "^2.0.6", + "mdast-util-directive": "^3.0.0", + "mdast-util-to-markdown": "^2.1.0", + "mdast-util-to-string": "^4.0.0", + "pagefind": "^1.3.0", + "rehype": "^13.0.1", + "rehype-format": "^5.0.0", + "remark-directive": "^3.0.0", + "ultrahtml": "^1.6.0", + "unified": "^11.0.5", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.2" + }, + "peerDependencies": { + "astro": "^5.5.0" + } + }, + "node_modules/@astrojs/starlight-tailwind": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@astrojs/starlight-tailwind/-/starlight-tailwind-4.0.2.tgz", + "integrity": "sha512-SYN/6zq6hJO5tWqbQ2tWT9/jd8ubUkzkBCcF94vByC/ZJ20Mi5GPjFvAh89Yky/aIM+jXxT6W5q4p6l58GKHiQ==", + "license": "MIT", + "peerDependencies": { + "@astrojs/starlight": ">=0.34.0", + "tailwindcss": "^4.0.0" + } + }, + "node_modules/@astrojs/svelte": { + "version": "7.2.5", + "resolved": "https://registry.npmjs.org/@astrojs/svelte/-/svelte-7.2.5.tgz", + "integrity": "sha512-Tl5aF/dYbzzd7sLpxMBX6pRz3yJ1B4pilt9G3GJbj0I0/doJHIEmerNQsnlxX0/InNKUhMXXN8wyyet9VhA+Zw==", + "license": "MIT", + "dependencies": { + "@sveltejs/vite-plugin-svelte": "^5.1.1", + "svelte2tsx": "^0.7.46", + "vite": "^6.4.1" + }, + "engines": { + "node": "18.20.8 || ^20.3.0 || >=22.0.0" + }, + "peerDependencies": { + "astro": "^5.0.0", + "svelte": "^5.1.16", + "typescript": "^5.3.3" + } + }, + "node_modules/@astrojs/telemetry": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@astrojs/telemetry/-/telemetry-3.3.0.tgz", + "integrity": "sha512-UFBgfeldP06qu6khs/yY+q1cDAaArM2/7AEIqQ9Cuvf7B1hNLq0xDrZkct+QoIGyjq56y8IaE2I3CTvG99mlhQ==", + "license": "MIT", + "dependencies": { + "ci-info": "^4.2.0", + "debug": "^4.4.0", + "dlv": "^1.1.3", + "dset": "^3.1.4", + "is-docker": "^3.0.0", + "is-wsl": "^3.1.0", + "which-pm-runs": "^1.1.0" + }, + "engines": { + "node": "18.20.8 || ^20.3.0 || >=22.0.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.7" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@capsizecss/unpack": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@capsizecss/unpack/-/unpack-4.0.1.tgz", + "integrity": "sha512-CuNiSqg7+e1cO/GjffyMOm5Tt2jUF9CWHHnvQ/UkqvtkGfHdgwEC0wpmq7fkN3gxwpRnrAN0WzO3vREKmNolMQ==", + "license": "MIT", + "dependencies": { + "fontkitten": "^1.0.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ctrl/tinycolor": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@ctrl/tinycolor/-/tinycolor-4.2.0.tgz", + "integrity": "sha512-kzyuwOAQnXJNLS9PSyrk0CWk35nWJW/zl/6KvnTBMFK65gm7U1/Z5BqjxeapjZCIhQcM/DsrEmcbRwDyXyXK4A==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@expressive-code/core": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/@expressive-code/core/-/core-0.41.7.tgz", + "integrity": "sha512-ck92uZYZ9Wba2zxkiZLsZGi9N54pMSAVdrI9uW3Oo9AtLglD5RmrdTwbYPCT2S/jC36JGB2i+pnQtBm/Ib2+dg==", + "license": "MIT", + "dependencies": { + "@ctrl/tinycolor": "^4.0.4", + "hast-util-select": "^6.0.2", + "hast-util-to-html": "^9.0.1", + "hast-util-to-text": "^4.0.1", + "hastscript": "^9.0.0", + "postcss": "^8.4.38", + "postcss-nested": "^6.0.1", + "unist-util-visit": "^5.0.0", + "unist-util-visit-parents": "^6.0.1" + } + }, + "node_modules/@expressive-code/plugin-frames": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/@expressive-code/plugin-frames/-/plugin-frames-0.41.7.tgz", + "integrity": "sha512-diKtxjQw/979cTglRFaMCY/sR6hWF0kSMg8jsKLXaZBSfGS0I/Hoe7Qds3vVEgeoW+GHHQzMcwvgx/MOIXhrTA==", + "license": "MIT", + "dependencies": { + "@expressive-code/core": "^0.41.7" + } + }, + "node_modules/@expressive-code/plugin-shiki": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/@expressive-code/plugin-shiki/-/plugin-shiki-0.41.7.tgz", + "integrity": "sha512-DL605bLrUOgqTdZ0Ot5MlTaWzppRkzzqzeGEu7ODnHF39IkEBbFdsC7pbl3LbUQ1DFtnfx6rD54k/cdofbW6KQ==", + "license": "MIT", + "dependencies": { + "@expressive-code/core": "^0.41.7", + "shiki": "^3.2.2" + } + }, + "node_modules/@expressive-code/plugin-text-markers": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/@expressive-code/plugin-text-markers/-/plugin-text-markers-0.41.7.tgz", + "integrity": "sha512-Ewpwuc5t6eFdZmWlFyeuy3e1PTQC0jFvw2Q+2bpcWXbOZhPLsT7+h8lsSIJxb5mS7wZko7cKyQ2RLYDyK6Fpmw==", + "license": "MIT", + "dependencies": { + "@expressive-code/core": "^0.41.7" + } + }, + "node_modules/@iconify-json/lucide": { + "version": "1.2.140", + "resolved": "https://registry.npmjs.org/@iconify-json/lucide/-/lucide-1.2.140.tgz", + "integrity": "sha512-/br6j8K8IFcZscs7gJ20PWaaonQeA4jcGK0Bdt+4EDra2Zbtpb36RGaGOJr5opz6zJtXaZdMCYCXa0dN/y4c0g==", + "dev": true, + "license": "ISC", + "dependencies": { + "@iconify/types": "*" + } + }, + "node_modules/@iconify/types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@iconify/types/-/types-2.0.0.tgz", + "integrity": "sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@iconify/utils": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/@iconify/utils/-/utils-3.1.7.tgz", + "integrity": "sha512-JZHlwdID+dy+lTgbYC8NEC4zeugqeYsc6jewvzb4c58kHauJn+X7rNwQjxz5p2qSjqaEeQoLkCIQ9v/H4PK0/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@antfu/install-pkg": "^2.0.1", + "@iconify/types": "^2.0.0", + "import-meta-resolve": "^4.2.0" + } + }, + "node_modules/@iconify/utils/node_modules/@antfu/install-pkg": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-2.1.0.tgz", + "integrity": "sha512-sdg9NxU3zR4Mnawfbc/x6GB5Wf17WYud5qOuEuxXjaKpYpMkISSJEjItGebXJ2bQ4DIcly4NYH23mtkGJjvKUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "package-manager-detector": "^1.8.0", + "tinyexec": "^1.3.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@mdx-js/mdx": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-3.1.1.tgz", + "integrity": "sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdx": "^2.0.0", + "acorn": "^8.0.0", + "collapse-white-space": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "estree-util-scope": "^1.0.0", + "estree-walker": "^3.0.0", + "hast-util-to-jsx-runtime": "^2.0.0", + "markdown-extensions": "^2.0.0", + "recma-build-jsx": "^1.0.0", + "recma-jsx": "^1.0.0", + "recma-stringify": "^1.0.0", + "rehype-recma": "^1.0.0", + "remark-mdx": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.0.0", + "source-map": "^0.7.0", + "unified": "^11.0.0", + "unist-util-position-from-estree": "^2.0.0", + "unist-util-stringify-position": "^4.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/@oslojs/encoding": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@oslojs/encoding/-/encoding-1.1.0.tgz", + "integrity": "sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==", + "license": "MIT" + }, + "node_modules/@pagefind/darwin-arm64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/darwin-arm64/-/darwin-arm64-1.5.2.tgz", + "integrity": "sha512-MXpI+7HsAdPkvJ0gk9xj9g541BCqBZOBbdwj9g6lB5LCj6kSV6nqDSjzcAJwvOsfu0fjwvC8hQU+ecfhp+MpiQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@pagefind/darwin-x64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/darwin-x64/-/darwin-x64-1.5.2.tgz", + "integrity": "sha512-IojxFWMEJe0RQ7PQ3KXQsPIImNsbpPYpoZ+QUDrL8fAl/O27IX+LVLs74/UzEZy5uA2LD8Nz1AiwKr72vrkZQw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@pagefind/default-ui": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/default-ui/-/default-ui-1.5.2.tgz", + "integrity": "sha512-pm1LMnQg8N2B3n2TnjKlhaFihpz6zTiA4HiGQ6/slKO/+8K9CAU5kcjdSSPgpuk1PMuuN4hxLipUIifnrkl3Sg==", + "license": "MIT" + }, + "node_modules/@pagefind/freebsd-x64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/freebsd-x64/-/freebsd-x64-1.5.2.tgz", + "integrity": "sha512-7EVzo9+0w+2cbe671BtMj10UlNo83I+HrLVLfRxO731svHRJKUfJ/mo05gU14pe9PCfpKNQT8FS3Xc/oDN6pOA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@pagefind/linux-arm64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/linux-arm64/-/linux-arm64-1.5.2.tgz", + "integrity": "sha512-Ovt9+K35sqzn8H3ZMXGwls4TD/wMJuvRtShHIsmUQREmaxjrDEX7gHckRCrwYJ4XE1H1p6HkLz3wukrAnsfXQw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@pagefind/linux-x64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/linux-x64/-/linux-x64-1.5.2.tgz", + "integrity": "sha512-V+tFqHKXhQKq/WqPBD67AFy7scn1/aZID00ws4fSDd+1daSi5UHR9VVlRrOUYKxn3VuFQYRD7lYXdZK1WED1YA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@pagefind/windows-arm64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/windows-arm64/-/windows-arm64-1.5.2.tgz", + "integrity": "sha512-hN9Nh90fNW61nNRCW9ZyQrAj/mD0eRvmJ8NlTUzkbuW8kIzGJUi3cxjFkEcMZ5h/8FsKWD/VcouZl4yo1F7B6g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@pagefind/windows-x64": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@pagefind/windows-x64/-/windows-x64-1.5.2.tgz", + "integrity": "sha512-Fa2Iyw7kaDRzGMfNYNUXNW2zbL5FQVDgSOcbDHdzBrDEdpqOqg8TcZ68F22ol6NJ9IGzvUdmeyZypLW5dyhqsg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/pluginutils": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.4.0.tgz", + "integrity": "sha512-MfPp06CjRLfXQ3wY0R8vJDYBy/MvVcc9OulEfR0B8Iv9ko+GCNaRZ+EpJYFl27LhKsZK0o420sYCRHCjfCgeUg==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-walker": "^2.0.2", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/pluginutils/node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.2.tgz", + "integrity": "sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.62.2.tgz", + "integrity": "sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.62.2.tgz", + "integrity": "sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.62.2.tgz", + "integrity": "sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.62.2.tgz", + "integrity": "sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.62.2.tgz", + "integrity": "sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.62.2.tgz", + "integrity": "sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.62.2.tgz", + "integrity": "sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==", + "cpu": [ + "arm" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.62.2.tgz", + "integrity": "sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.62.2.tgz", + "integrity": "sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.62.2.tgz", + "integrity": "sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==", + "cpu": [ + "loong64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.62.2.tgz", + "integrity": "sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==", + "cpu": [ + "loong64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.62.2.tgz", + "integrity": "sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.62.2.tgz", + "integrity": "sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==", + "cpu": [ + "ppc64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.62.2.tgz", + "integrity": "sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.62.2.tgz", + "integrity": "sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==", + "cpu": [ + "riscv64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.62.2.tgz", + "integrity": "sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.62.2.tgz", + "integrity": "sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.62.2.tgz", + "integrity": "sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.62.2.tgz", + "integrity": "sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.62.2.tgz", + "integrity": "sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.62.2.tgz", + "integrity": "sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.62.2.tgz", + "integrity": "sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.62.2.tgz", + "integrity": "sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.62.2.tgz", + "integrity": "sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@shikijs/core": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-3.23.0.tgz", + "integrity": "sha512-NSWQz0riNb67xthdm5br6lAkvpDJRTgB36fxlo37ZzM2yq0PQFFzbd8psqC2XMPgCzo1fW6cVi18+ArJ44wqgA==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.23.0", + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4", + "hast-util-to-html": "^9.0.5" + } + }, + "node_modules/@shikijs/engine-javascript": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-3.23.0.tgz", + "integrity": "sha512-aHt9eiGFobmWR5uqJUViySI1bHMqrAgamWE1TYSUoftkAeCCAiGawPMwM+VCadylQtF4V3VNOZ5LmfItH5f3yA==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.23.0", + "@shikijs/vscode-textmate": "^10.0.2", + "oniguruma-to-es": "^4.3.4" + } + }, + "node_modules/@shikijs/engine-oniguruma": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-3.23.0.tgz", + "integrity": "sha512-1nWINwKXxKKLqPibT5f4pAFLej9oZzQTsby8942OTlsJzOBZ0MWKiwzMsd+jhzu8YPCHAswGnnN1YtQfirL35g==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.23.0", + "@shikijs/vscode-textmate": "^10.0.2" + } + }, + "node_modules/@shikijs/langs": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-3.23.0.tgz", + "integrity": "sha512-2Ep4W3Re5aB1/62RSYQInK9mM3HsLeB91cHqznAJMuylqjzNVAVCMnNWRHFtcNHXsoNRayP9z1qj4Sq3nMqYXg==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.23.0" + } + }, + "node_modules/@shikijs/themes": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/themes/-/themes-3.23.0.tgz", + "integrity": "sha512-5qySYa1ZgAT18HR/ypENL9cUSGOeI2x+4IvYJu4JgVJdizn6kG4ia5Q1jDEOi7gTbN4RbuYtmHh0W3eccOrjMA==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.23.0" + } + }, + "node_modules/@shikijs/types": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-3.23.0.tgz", + "integrity": "sha512-3JZ5HXOZfYjsYSk0yPwBrkupyYSLpAE26Qc0HLghhZNGTZg/SKxXIIgoxOpmmeQP0RRSDJTk1/vPfw9tbw+jSQ==", + "license": "MIT", + "dependencies": { + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4" + } + }, + "node_modules/@shikijs/vscode-textmate": { + "version": "10.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-10.0.2.tgz", + "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", + "license": "MIT" + }, + "node_modules/@sveltejs/acorn-typescript": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.13.tgz", + "integrity": "sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^8.9.0" + } + }, + "node_modules/@sveltejs/vite-plugin-svelte": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-5.1.1.tgz", + "integrity": "sha512-Y1Cs7hhTc+a5E9Va/xwKlAJoariQyHY+5zBgCZg4PFWNYQ1nMN9sjK1zhw1gK69DuqVP++sht/1GZg1aRwmAXQ==", + "license": "MIT", + "dependencies": { + "@sveltejs/vite-plugin-svelte-inspector": "^4.0.1", + "debug": "^4.4.1", + "deepmerge": "^4.3.1", + "kleur": "^4.1.5", + "magic-string": "^0.30.17", + "vitefu": "^1.0.6" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22" + }, + "peerDependencies": { + "svelte": "^5.0.0", + "vite": "^6.0.0" + } + }, + "node_modules/@sveltejs/vite-plugin-svelte-inspector": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-4.0.1.tgz", + "integrity": "sha512-J/Nmb2Q2y7mck2hyCX4ckVHcR5tu2J+MtBEQqpDrrgELZ2uvraQcK/ioCV61AqkdXFgriksOKIceDcQmqnGhVw==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.7" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22" + }, + "peerDependencies": { + "@sveltejs/vite-plugin-svelte": "^5.0.0", + "svelte": "^5.0.0", + "vite": "^6.0.0" + } + }, + "node_modules/@sveltejs/vite-plugin-svelte/node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/@tailwindcss/node": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.3.tgz", + "integrity": "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==", + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "enhanced-resolve": "^5.24.1", + "jiti": "^2.7.0", + "lightningcss": "1.32.0", + "magic-string": "^0.30.21", + "source-map-js": "^1.2.1", + "tailwindcss": "4.3.3" + } + }, + "node_modules/@tailwindcss/oxide": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.3.3.tgz", + "integrity": "sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==", + "license": "MIT", + "engines": { + "node": ">= 20" + }, + "optionalDependencies": { + "@tailwindcss/oxide-android-arm64": "4.3.3", + "@tailwindcss/oxide-darwin-arm64": "4.3.3", + "@tailwindcss/oxide-darwin-x64": "4.3.3", + "@tailwindcss/oxide-freebsd-x64": "4.3.3", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.3.3", + "@tailwindcss/oxide-linux-arm64-gnu": "4.3.3", + "@tailwindcss/oxide-linux-arm64-musl": "4.3.3", + "@tailwindcss/oxide-linux-x64-gnu": "4.3.3", + "@tailwindcss/oxide-linux-x64-musl": "4.3.3", + "@tailwindcss/oxide-wasm32-wasi": "4.3.3", + "@tailwindcss/oxide-win32-arm64-msvc": "4.3.3", + "@tailwindcss/oxide-win32-x64-msvc": "4.3.3" + } + }, + "node_modules/@tailwindcss/oxide-android-arm64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.3.3.tgz", + "integrity": "sha512-Y85A2gmPSkl5Ve5qR86GL4HT509cFqQh1aes9p3sSkyTPwt0Pppf3GkwGe4JPACcRYjgJIEhQgM6dBClnr0NYw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-arm64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.3.3.tgz", + "integrity": "sha512-BiaWatpBcERQFDlOjRDpIVXuFK5PJez5SA4JMg6VYZdBYU+qKfV/vqjcIs+IYmtitf1xYQZTwXvU/8y4lfZUGw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-x64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.3.3.tgz", + "integrity": "sha512-fAeUqfV5ndhxRwai8cXGzdLvul9utWOmeTkv69unv4ZXixjn61Z+p9lCWdwOwA3TYboG3BwdVuN/RDjhBRl0mw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-freebsd-x64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.3.3.tgz", + "integrity": "sha512-iyf5bV6+wnAlflVeEy7R25dupxTNECZN5QMI0qNT6eT+EgaGdZcKhGkr5SdoaWiLJ3spLqIY9VCeSGrwmtg4kw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.3.3.tgz", + "integrity": "sha512-aAYUprJAJQWWbRrPvtjdroZ56Md+JM8pMiopS6xGEwDfLhqj+2ver2p4nU4Mb3CRqcMmNBjo8KkUgcxhkzVQGQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.3.3.tgz", + "integrity": "sha512-nDxldcEENOxZRzC2uu9jrutZdAAQtb+8WWDCSnWL1zvBk1+FN+x6MtDViPB5AJMfttVCUhehGWus3XBPgatM/w==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-musl": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.3.3.tgz", + "integrity": "sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-gnu": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.3.3.tgz", + "integrity": "sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-musl": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.3.3.tgz", + "integrity": "sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.3.3.tgz", + "integrity": "sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==", + "bundleDependencies": [ + "@napi-rs/wasm-runtime", + "@emnapi/core", + "@emnapi/runtime", + "@tybys/wasm-util", + "@emnapi/wasi-threads", + "tslib" + ], + "cpu": [ + "wasm32" + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.11.1", + "@emnapi/runtime": "^1.11.1", + "@emnapi/wasi-threads": "^1.2.2", + "@napi-rs/wasm-runtime": "^1.1.4", + "@tybys/wasm-util": "^0.10.2", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.11.1", + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.11.1", + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.4", + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": { + "version": "0.10.2", + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": { + "version": "2.8.1", + "inBundle": true, + "license": "0BSD", + "optional": true + }, + "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.3.tgz", + "integrity": "sha512-3rc292Ca2ceK6Ulcc/bAVnTs/3nDtoPhyEKlgPv+yQJQi/JS/AMJlqzxvlDacL1nekbrcf6bTqp/jV4qgnPxNQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-win32-x64-msvc": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.3.3.tgz", + "integrity": "sha512-yJ0pwIVc/nYeGoV02WtsN8KYyLQv7kyI2wDnkezyJlGGjkd4QLwDGAwl47YpPJeuI0M0ObaXGSPjvWDPeTPggw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/vite": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.3.3.tgz", + "integrity": "sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==", + "license": "MIT", + "dependencies": { + "@tailwindcss/node": "4.3.3", + "@tailwindcss/oxide": "4.3.3", + "tailwindcss": "4.3.3" + }, + "peerDependencies": { + "vite": "^5.2.0 || ^6 || ^7 || ^8" + } + }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "license": "MIT" + }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/@types/hast": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.5.tgz", + "integrity": "sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "license": "MIT" + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdx": { + "version": "2.0.14", + "resolved": "https://registry.npmjs.org/@types/mdx/-/mdx-2.0.14.tgz", + "integrity": "sha512-T48PeuJtvLosNTPVhfnIp3i/n3a4g4Bad7YCq5k64D4u7NwDrAotikQ+5+sjtUvBmxCMlbo3dVL+C2dP0rWHzg==", + "license": "MIT" + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/nlcst": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/nlcst/-/nlcst-2.0.3.tgz", + "integrity": "sha512-vSYNSDe6Ix3q+6Z7ri9lyWqgGhJTmzRjZRqyq15N0Z/1/UnVsno9G/N40NBijoYx2seFDIl0+B2mgAb9mezUCA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/node": { + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/picomatch": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/picomatch/-/picomatch-3.0.2.tgz", + "integrity": "sha512-n0i8TD3UDB7paoMMxA3Y65vUncFJXjcUf7lQY7YyKGl6031FNjfsLs6pdLFCy2GNFxItPJG8GvvpbZc2skH7WA==", + "license": "MIT" + }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.3.tgz", + "integrity": "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg==", + "license": "ISC" + }, + "node_modules/acorn": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", + "integrity": "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ansi-align": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", + "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", + "license": "ISC", + "dependencies": { + "string-width": "^4.1.0" + } + }, + "node_modules/ansi-align/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-align/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/ansi-align/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-align/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-escapes": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", + "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", + "license": "MIT", + "dependencies": { + "environment": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/anymatch/node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "license": "MIT" + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/aria-query": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.2.tgz", + "integrity": "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==", + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/array-iterate": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/array-iterate/-/array-iterate-2.0.1.tgz", + "integrity": "sha512-I1jXZMjAgCMmxT4qxXfPXa6SthSoE8h6gkSI9BGGNv8mP8G/v0blc+qFnZu6K42vTOiuME596QaLO0TP3Lk0xg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/astring": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/astring/-/astring-1.9.0.tgz", + "integrity": "sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==", + "license": "MIT", + "bin": { + "astring": "bin/astring" + } + }, + "node_modules/astro": { + "version": "5.18.2", + "resolved": "https://registry.npmjs.org/astro/-/astro-5.18.2.tgz", + "integrity": "sha512-TnFwLnAXty5MXKPDGuKXqK4AMBXG+FH6RUdK7Oyc3gyfNoFIthT+4eRbzOK43bdRlLaZuxgciDSjgtggZ3OtGQ==", + "license": "MIT", + "dependencies": { + "@astrojs/compiler": "^2.13.0", + "@astrojs/internal-helpers": "0.7.6", + "@astrojs/markdown-remark": "6.3.11", + "@astrojs/telemetry": "3.3.0", + "@capsizecss/unpack": "^4.0.0", + "@oslojs/encoding": "^1.1.0", + "@rollup/pluginutils": "^5.3.0", + "acorn": "^8.15.0", + "aria-query": "^5.3.2", + "axobject-query": "^4.1.0", + "boxen": "8.0.1", + "ci-info": "^4.3.1", + "clsx": "^2.1.1", + "common-ancestor-path": "^1.0.1", + "cookie": "^1.1.1", + "cssesc": "^3.0.0", + "debug": "^4.4.3", + "deterministic-object-hash": "^2.0.2", + "devalue": "^5.6.2", + "diff": "^8.0.3", + "dlv": "^1.1.3", + "dset": "^3.1.4", + "es-module-lexer": "^1.7.0", + "esbuild": "^0.27.3", + "estree-walker": "^3.0.3", + "flattie": "^1.1.1", + "fontace": "~0.4.0", + "github-slugger": "^2.0.0", + "html-escaper": "3.0.3", + "http-cache-semantics": "^4.2.0", + "import-meta-resolve": "^4.2.0", + "js-yaml": "^4.1.1", + "magic-string": "^0.30.21", + "magicast": "^0.5.1", + "mrmime": "^2.0.1", + "neotraverse": "^0.6.18", + "p-limit": "^6.2.0", + "p-queue": "^8.1.1", + "package-manager-detector": "^1.6.0", + "piccolore": "^0.1.3", + "picomatch": "^4.0.3", + "prompts": "^2.4.2", + "rehype": "^13.0.2", + "semver": "^7.7.3", + "shiki": "^3.21.0", + "smol-toml": "^1.6.0", + "svgo": "^4.0.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tsconfck": "^3.1.6", + "ultrahtml": "^1.6.0", + "unifont": "~0.7.3", + "unist-util-visit": "^5.0.0", + "unstorage": "^1.17.4", + "vfile": "^6.0.3", + "vite": "^6.4.1", + "vitefu": "^1.1.1", + "xxhash-wasm": "^1.1.0", + "yargs-parser": "^21.1.1", + "yocto-spinner": "^0.2.3", + "zod": "^3.25.76", + "zod-to-json-schema": "^3.25.1", + "zod-to-ts": "^1.2.0" + }, + "bin": { + "astro": "astro.js" + }, + "engines": { + "node": "18.20.8 || ^20.3.0 || >=22.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/astrodotbuild" + }, + "optionalDependencies": { + "sharp": "^0.34.0" + } + }, + "node_modules/astro-expressive-code": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/astro-expressive-code/-/astro-expressive-code-0.41.7.tgz", + "integrity": "sha512-hUpogGc6DdAd+I7pPXsctyYPRBJDK7Q7d06s4cyP0Vz3OcbziP3FNzN0jZci1BpCvLn9675DvS7B9ctKKX64JQ==", + "license": "MIT", + "dependencies": { + "rehype-expressive-code": "^0.41.7" + }, + "peerDependencies": { + "astro": "^4.0.0-beta || ^5.0.0-beta || ^3.3.0 || ^6.0.0-beta" + } + }, + "node_modules/astro/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/astro/node_modules/zod-to-ts": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/zod-to-ts/-/zod-to-ts-1.2.0.tgz", + "integrity": "sha512-x30XE43V+InwGpvTySRNz9kB7qFU8DlyEy7BsSTCHPH1R0QasMmHWZDCzYm6bVXtj/9NNJAZF3jW8rzFvH5OFA==", + "peerDependencies": { + "typescript": "^4.9.4 || ^5.0.2", + "zod": "^3" + } + }, + "node_modules/axobject-query": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", + "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/base-64": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/base-64/-/base-64-1.0.0.tgz", + "integrity": "sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==", + "license": "MIT" + }, + "node_modules/bcp-47": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/bcp-47/-/bcp-47-2.1.1.tgz", + "integrity": "sha512-KLw+H/gd2p4zly1X7Yh/qziuyae5/w/QFnvTng9eZL5fvszL7Whl3MBoWF8yxL7ksUjBfOD+OxkytiqbBpG+Fw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/bcp-47-match": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/bcp-47-match/-/bcp-47-match-2.0.3.tgz", + "integrity": "sha512-JtTezzbAibu8G0R9op9zb3vcWZd9JF6M0xOYGPn0fNCd7wOpRB1mU2mH9T8gaBGbAAyIIVgB2G7xG0GP98zMAQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, + "node_modules/boxen": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-8.0.1.tgz", + "integrity": "sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw==", + "license": "MIT", + "dependencies": { + "ansi-align": "^3.0.1", + "camelcase": "^8.0.0", + "chalk": "^5.3.0", + "cli-boxes": "^3.0.0", + "string-width": "^7.2.0", + "type-fest": "^4.21.0", + "widest-line": "^5.0.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/camelcase": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-8.0.0.tgz", + "integrity": "sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/chokidar": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", + "license": "MIT", + "dependencies": { + "readdirp": "^5.0.0" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/ci-info": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.4.0.tgz", + "integrity": "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-boxes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-3.0.0.tgz", + "integrity": "sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/collapse-white-space": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-2.1.0.tgz", + "integrity": "sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/commander": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", + "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", + "license": "MIT", + "engines": { + "node": ">=16" + } + }, + "node_modules/common-ancestor-path": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/common-ancestor-path/-/common-ancestor-path-1.0.1.tgz", + "integrity": "sha512-L3sHRo1pXXEqX8VU28kfgUY+YGsk09hPqZiZmLacNib6XNTCM8ubYeT7ryXQw8asB1sKgcU5lkB7ONug08aB8w==", + "license": "ISC" + }, + "node_modules/confbox": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.3.1.tgz", + "integrity": "sha512-cKUSoKa8YxFZZSmraVi7onONx3amu77ngK3kGpsYHDH7drPwCRkQE1RYMPlLRrMtnciRj274XNRxcHxnKmDSnA==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cookie-es": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/cookie-es/-/cookie-es-1.2.3.tgz", + "integrity": "sha512-lXVyvUvrNXblMqzIRrxHb57UUVmqsSWlxqt3XIjCkUP0wDAf6uicO6KMbEgYrMNtEvWgWHwe42CKxPu9MYAnWw==", + "license": "MIT" + }, + "node_modules/crossws": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/crossws/-/crossws-0.3.5.tgz", + "integrity": "sha512-ojKiDvcmByhwa8YYqbQI/hg7MEU0NC03+pSdEq4ZUnZR9xXpwk7E43SMNGkn+JxJGPFtNvQ48+vV2p+P1ml5PA==", + "license": "MIT", + "dependencies": { + "uncrypto": "^0.1.3" + } + }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-selector-parser": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/css-selector-parser/-/css-selector-parser-3.3.0.tgz", + "integrity": "sha512-Y2asgMGFqJKF4fq4xHDSlFYIkeVfRsm69lQC1q9kbEsH5XtnINTMrweLkjYMeaUgiXBy/uvKeO/a1JHTNnmB2g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, + "node_modules/css-tree": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", + "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", + "license": "MIT", + "dependencies": { + "mdn-data": "2.27.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "license": "MIT", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/csso": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/csso/-/csso-5.0.5.tgz", + "integrity": "sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==", + "license": "MIT", + "dependencies": { + "css-tree": "~2.2.0" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/csso/node_modules/css-tree": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.2.1.tgz", + "integrity": "sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==", + "license": "MIT", + "dependencies": { + "mdn-data": "2.0.28", + "source-map-js": "^1.0.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/csso/node_modules/mdn-data": { + "version": "2.0.28", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.28.tgz", + "integrity": "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==", + "license": "CC0-1.0" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dedent-js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dedent-js/-/dedent-js-1.0.1.tgz", + "integrity": "sha512-OUepMozQULMLUmhxS95Vudo0jb0UchLimi3+pQ2plj61Fcy8axbP9hbiD4Sz6DPqn6XG3kfmziVfQ1rSys5AJQ==", + "license": "MIT" + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/defu": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.7.tgz", + "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==", + "license": "MIT" + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/destr": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/destr/-/destr-2.0.5.tgz", + "integrity": "sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==", + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/deterministic-object-hash": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/deterministic-object-hash/-/deterministic-object-hash-2.0.2.tgz", + "integrity": "sha512-KxektNH63SrbfUyDiwXqRb1rLwKt33AmMv+5Nhsw1kqZ13SJBRTgZHtGbE+hH3a1mVW1cz+4pqSWVPAtLVXTzQ==", + "license": "MIT", + "dependencies": { + "base-64": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/devalue": { + "version": "5.9.4", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.4.tgz", + "integrity": "sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==", + "license": "MIT" + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/diff": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", + "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/direction": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/direction/-/direction-2.0.1.tgz", + "integrity": "sha512-9S6m9Sukh1cZNknO1CWAr2QAWsbKLafQiyM5gZ7VgXHeuaoUwffKN4q6NC4A/Mf9iiPlOXQEKW/Mv/mh9/3YFA==", + "license": "MIT", + "bin": { + "direction": "cli.js" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dlv": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", + "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", + "license": "MIT" + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/dom-serializer/node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dset": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz", + "integrity": "sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/enhanced-resolve": { + "version": "5.26.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.26.0.tgz", + "integrity": "sha512-9vhedylFonb2YGogzUKX6+Ja72gOJbN1QHAqdrvqLwhdl/QWbKopzoUC9EbQNVsAns/bx/4uyqalrQAoy1IByw==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.3.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/environment": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", + "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "license": "MIT" + }, + "node_modules/esast-util-from-estree": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/esast-util-from-estree/-/esast-util-from-estree-2.0.0.tgz", + "integrity": "sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-visit": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/esast-util-from-js": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/esast-util-from-js/-/esast-util-from-js-2.0.1.tgz", + "integrity": "sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "acorn": "^8.0.0", + "esast-util-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, + "node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/esm-env": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.2.tgz", + "integrity": "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==", + "license": "MIT" + }, + "node_modules/esrap": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.4.0.tgz", + "integrity": "sha512-CQPcneEHw/czj32uwQC52nYnax8H4SXt25FCM+LIwGZ49cMIxpOj3bP8wmzcI+Z0HIy5KkClaPhRLfCySzwHPA==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + }, + "peerDependencies": { + "@typescript-eslint/types": "^8.2.0" + }, + "peerDependenciesMeta": { + "@typescript-eslint/types": { + "optional": true + } + } + }, + "node_modules/estree-util-attach-comments": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-attach-comments/-/estree-util-attach-comments-3.0.0.tgz", + "integrity": "sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-build-jsx": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/estree-util-build-jsx/-/estree-util-build-jsx-3.0.1.tgz", + "integrity": "sha512-8U5eiL6BTrPxp/CHbs2yMgP8ftMhR5ww1eIKoWRMlqvltHF8fZn5LRDvTKuxD3DUn+shRbLGqXemcP51oFCsGQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "estree-walker": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-is-identifier-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", + "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-scope": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/estree-util-scope/-/estree-util-scope-1.0.0.tgz", + "integrity": "sha512-2CAASclonf+JFWBNJPndcOpA8EMJwa0Q8LUFJEKqXLW6+qBvbFZuF5gItbQOs/umBUkjviCSDCbBwU2cXbmrhQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-to-js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/estree-util-to-js/-/estree-util-to-js-2.0.0.tgz", + "integrity": "sha512-WDF+xj5rRWmD5tj6bIqRi6CkLIXbbNQUcxQHzGysQzvHmdYG2G7p/Tf0J0gpxGgkeMZNTIjT/AoSvC9Xehcgdg==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "astring": "^1.8.0", + "source-map": "^0.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-visit": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-2.0.0.tgz", + "integrity": "sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/expressive-code": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/expressive-code/-/expressive-code-0.41.7.tgz", + "integrity": "sha512-2wZjC8OQ3TaVEMcBtYY4Va3lo6J+Ai9jf3d4dbhURMJcU4Pbqe6EcHe424MIZI0VHUA1bR6xdpoHYi3yxokWqA==", + "license": "MIT", + "dependencies": { + "@expressive-code/core": "^0.41.7", + "@expressive-code/plugin-frames": "^0.41.7", + "@expressive-code/plugin-shiki": "^0.41.7", + "@expressive-code/plugin-text-markers": "^0.41.7" + } + }, + "node_modules/exsolve": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/exsolve/-/exsolve-1.1.3.tgz", + "integrity": "sha512-kuzL9zherLQoqDeQaS55AiiIg1kb4KYXaSusTtgZTPlvKP2gGCI82LNmBNh5RjDVUXIcmgx+/Mx7vPwRzBbhJA==", + "dev": true, + "license": "MIT" + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/flattie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/flattie/-/flattie-1.1.1.tgz", + "integrity": "sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/fontace": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/fontace/-/fontace-0.4.1.tgz", + "integrity": "sha512-lDMvbAzSnHmbYMTEld5qdtvNH2/pWpICOqpean9IgC7vUbUJc3k+k5Dokp85CegamqQpFbXf0rAVkbzpyTA8aw==", + "license": "MIT", + "dependencies": { + "fontkitten": "^1.0.2" + } + }, + "node_modules/fontkitten": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/fontkitten/-/fontkitten-1.0.3.tgz", + "integrity": "sha512-Wp1zXWPVUPBmfoa3Cqc9ctaKuzKAV6uLstRqlR56kSjplf5uAce+qeyYym7F+PHbGTk+tCEdkCW6RD7DX/gBZw==", + "license": "MIT", + "dependencies": { + "tiny-inflate": "^1.0.3" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/github-slugger": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-2.0.0.tgz", + "integrity": "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==", + "license": "ISC" + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/h3": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/h3/-/h3-1.15.11.tgz", + "integrity": "sha512-L3THSe2MPeBwgIZVSH5zLdBBU90TOxarvhK9d04IDY2AmVS8j2Jz2LIWtwsGOU3lu2I5jCN7FNvVfY2+XyF+mg==", + "license": "MIT", + "dependencies": { + "cookie-es": "^1.2.3", + "crossws": "^0.3.5", + "defu": "^6.1.6", + "destr": "^2.0.5", + "iron-webcrypto": "^1.2.1", + "node-mock-http": "^1.0.4", + "radix3": "^1.1.2", + "ufo": "^1.6.3", + "uncrypto": "^0.1.3" + } + }, + "node_modules/has-flag": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-5.0.1.tgz", + "integrity": "sha512-CsNUt5x9LUdx6hnk/E2SZLsDyvfqANZSUq4+D3D8RzDJ2M+HDTIkF60ibS1vHaK55vzgiZw1bEPFG9yH7l33wA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/hast-util-embedded": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-embedded/-/hast-util-embedded-3.0.0.tgz", + "integrity": "sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-is-element": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-format": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/hast-util-format/-/hast-util-format-1.1.0.tgz", + "integrity": "sha512-yY1UDz6bC9rDvCWHpx12aIBGRG7krurX0p0Fm6pT547LwDIZZiNr8a+IHDogorAdreULSEzP82Nlv5SZkHZcjA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-embedded": "^3.0.0", + "hast-util-minify-whitespace": "^1.0.0", + "hast-util-phrasing": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "html-whitespace-sensitive-tag-names": "^3.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-from-html": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hast-util-from-html/-/hast-util-from-html-2.0.3.tgz", + "integrity": "sha512-CUSRHXyKjzHov8yKsQjGOElXy/3EKpyX56ELnkHH34vDVw1N1XSQ1ZcAvTyAPtGqLTuKP/uxM+aLkSPqF/EtMw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "devlop": "^1.1.0", + "hast-util-from-parse5": "^8.0.0", + "parse5": "^7.0.0", + "vfile": "^6.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-from-parse5": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.3.tgz", + "integrity": "sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "hastscript": "^9.0.0", + "property-information": "^7.0.0", + "vfile": "^6.0.0", + "vfile-location": "^5.0.0", + "web-namespaces": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-has-property": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-has-property/-/hast-util-has-property-3.0.0.tgz", + "integrity": "sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-is-body-ok-link": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hast-util-is-body-ok-link/-/hast-util-is-body-ok-link-3.0.1.tgz", + "integrity": "sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-is-element": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-3.0.0.tgz", + "integrity": "sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-minify-whitespace": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hast-util-minify-whitespace/-/hast-util-minify-whitespace-1.0.1.tgz", + "integrity": "sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-embedded": "^3.0.0", + "hast-util-is-element": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-parse-selector": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz", + "integrity": "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-phrasing": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hast-util-phrasing/-/hast-util-phrasing-3.0.1.tgz", + "integrity": "sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-embedded": "^3.0.0", + "hast-util-has-property": "^3.0.0", + "hast-util-is-body-ok-link": "^3.0.0", + "hast-util-is-element": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-raw": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-9.1.0.tgz", + "integrity": "sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "@ungap/structured-clone": "^1.0.0", + "hast-util-from-parse5": "^8.0.0", + "hast-util-to-parse5": "^8.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "parse5": "^7.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-select": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/hast-util-select/-/hast-util-select-6.0.4.tgz", + "integrity": "sha512-RqGS1ZgI0MwxLaKLDxjprynNzINEkRHY2i8ln4DDjgv9ZhcYVIHN9rlpiYsqtFwrgpYU361SyWDQcGNIBVu3lw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "bcp-47-match": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "css-selector-parser": "^3.0.0", + "devlop": "^1.0.0", + "direction": "^2.0.0", + "hast-util-has-property": "^3.0.0", + "hast-util-to-string": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "nth-check": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-estree": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/hast-util-to-estree/-/hast-util-to-estree-3.1.3.tgz", + "integrity": "sha512-48+B/rJWAp0jamNbAAf9M7Uf//UVqAoMmgXhBdxTDJLGKY+LRnZ99qcG+Qjl5HfMpYNzS5v4EAwVEF34LeAj7w==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-attach-comments": "^3.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-js": "^1.0.0", + "unist-util-position": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-html": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.5.tgz", + "integrity": "sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^3.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "stringify-entities": "^4.0.0", + "zwitch": "^2.0.4" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-jsx-runtime": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz", + "integrity": "sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-js": "^1.0.0", + "unist-util-position": "^5.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-mdast": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/hast-util-to-mdast/-/hast-util-to-mdast-10.1.2.tgz", + "integrity": "sha512-FiCRI7NmOvM4y+f5w32jPRzcxDIz+PUqDwEqn1A+1q2cdp3B8Gx7aVrXORdOKjMNDQsD1ogOr896+0jJHW1EFQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "hast-util-phrasing": "^3.0.0", + "hast-util-to-html": "^9.0.0", + "hast-util-to-text": "^4.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-hast": "^13.0.0", + "mdast-util-to-string": "^4.0.0", + "rehype-minify-whitespace": "^6.0.0", + "trim-trailing-lines": "^2.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-parse5": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-8.0.1.tgz", + "integrity": "sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-string": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-3.0.1.tgz", + "integrity": "sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-text": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-4.0.2.tgz", + "integrity": "sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "hast-util-is-element": "^3.0.0", + "unist-util-find-after": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", + "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hastscript": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-9.0.1.tgz", + "integrity": "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-parse-selector": "^4.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/html-escaper": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", + "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==", + "license": "MIT" + }, + "node_modules/html-void-elements": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz", + "integrity": "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/html-whitespace-sensitive-tag-names": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/html-whitespace-sensitive-tag-names/-/html-whitespace-sensitive-tag-names-3.0.1.tgz", + "integrity": "sha512-q+310vW8zmymYHALr1da4HyXUQ0zgiIwIicEfotYPWGN0OJVEN/58IJ3A4GBYcEq3LGAZqKb+ugvP0GNB9CEAA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "license": "BSD-2-Clause" + }, + "node_modules/i18next": { + "version": "23.16.8", + "resolved": "https://registry.npmjs.org/i18next/-/i18next-23.16.8.tgz", + "integrity": "sha512-06r/TitrM88Mg5FdUXAKL96dJMzgqLE5dv3ryBAra4KCwD9mJ4ndOTS95ZuymIGoE+2hzfdaMak2X11/es7ZWg==", + "funding": [ + { + "type": "individual", + "url": "https://locize.com" + }, + { + "type": "individual", + "url": "https://locize.com/i18next.html" + }, + { + "type": "individual", + "url": "https://www.i18next.com/how-to/faq#i18next-is-awesome.-how-can-i-support-the-project" + } + ], + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.23.2" + } + }, + "node_modules/import-meta-resolve": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz", + "integrity": "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/inline-style-parser": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.7.tgz", + "integrity": "sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==", + "license": "MIT" + }, + "node_modules/iron-webcrypto": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/iron-webcrypto/-/iron-webcrypto-1.2.1.tgz", + "integrity": "sha512-feOM6FaSr6rEABp/eDfVseKyTMDt+KGpeB35SkVn9Tyn0CqvVsY3EwI0v5i8nMHyJnzCIQf7nsy3p41TPkJZhg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/brc-dd" + } + }, + "node_modules/is-absolute-url": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-absolute-url/-/is-absolute-url-4.0.1.tgz", + "integrity": "sha512-/51/TKE88Lmm7Gc4/8btclNXWS+g50wXhYJq8HWIBAGUBnoAdRu1aXeh364t/O7wXDAcTJDP8PNuNKWUDWie+A==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-docker": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", + "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-inside-container": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", + "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", + "license": "MIT", + "dependencies": { + "is-docker": "^3.0.0" + }, + "bin": { + "is-inside-container": "cli.js" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-reference": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz", + "integrity": "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.6" + } + }, + "node_modules/is-wsl": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", + "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", + "license": "MIT", + "dependencies": { + "is-inside-container": "^1.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/js-yaml": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", + "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/klona": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/klona/-/klona-2.0.6.tgz", + "integrity": "sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/local-pkg": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-1.2.1.tgz", + "integrity": "sha512-++gUqRDEvcnN6Zhqrr+y/CkVEHhlrR96vZn3nZZPYzMcBUyBtTKzB9NadClFIsIVSsu+3i9tfk/erqy9kAmt7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "mlly": "^1.7.4", + "pkg-types": "^2.3.0", + "quansync": "^0.2.11" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/locate-character": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz", + "integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==", + "license": "MIT" + }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/lru-cache": { + "version": "11.5.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", + "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/magicast": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.3.tgz", + "integrity": "sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.3", + "@babel/types": "^7.29.0", + "source-map-js": "^1.2.1" + } + }, + "node_modules/markdown-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/markdown-extensions/-/markdown-extensions-2.0.0.tgz", + "integrity": "sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/markdown-table": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", + "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/mdast-util-definitions": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-definitions/-/mdast-util-definitions-6.0.0.tgz", + "integrity": "sha512-scTllyX6pnYNZH/AIp/0ePz6s4cZtARxImwoPJ7kS42n+MnVsI4XbnG6d4ibehRIldYMWM2LD7ImQblVhUejVQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-directive": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-directive/-/mdast-util-directive-3.1.0.tgz", + "integrity": "sha512-I3fNFt+DHmpWCYAT7quoM6lHf9wuqtI+oCOfvILnoicNIqjh5E3dEJWiXuYME2gNe8vl1iMQwyUHa7bgFmak6Q==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-find-and-replace": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.2.tgz", + "integrity": "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", + "integrity": "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", + "integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-gfm-autolink-literal": "^2.0.0", + "mdast-util-gfm-footnote": "^2.0.0", + "mdast-util-gfm-strikethrough": "^2.0.0", + "mdast-util-gfm-table": "^2.0.0", + "mdast-util-gfm-task-list-item": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", + "integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-find-and-replace": "^3.0.0", + "micromark-util-character": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz", + "integrity": "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", + "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", + "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx/-/mdast-util-mdx-3.0.0.tgz", + "integrity": "sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-expression": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", + "integrity": "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-jsx": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.2.0.tgz", + "integrity": "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdxjs-esm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", + "integrity": "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-hast": { + "version": "13.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz", + "integrity": "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "devlop": "^1.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "trim-lines": "^3.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz", + "integrity": "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdn-data": { + "version": "2.27.1", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", + "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", + "license": "CC0-1.0" + }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-directive": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/micromark-extension-directive/-/micromark-extension-directive-3.0.2.tgz", + "integrity": "sha512-wjcXHgk+PPdmvR58Le9d7zQYWy+vKEU9Se44p2CrCDPiLr2FMyiT4Fyb5UFKFC66wGB3kPlgD7q3TnoqPS7SZA==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "parse-entities": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "license": "MIT", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.1.tgz", + "integrity": "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdx-expression": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-3.0.1.tgz", + "integrity": "sha512-dD/ADLJ1AeMvSAKBwO22zG22N4ybhe7kFIZ3LsDI0GlsNr2A3KYxb0LdC1u5rj4Nw+CHKY0RVdnHX8vj8ejm4Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-mdx-expression": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-mdx-jsx": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-3.0.2.tgz", + "integrity": "sha512-e5+q1DjMh62LZAJOnDraSSbDMvGJ8x3cbjygy2qFEi7HCeUT4BDKCvMozPozcD6WmOt6sVvYDNBKhFSz3kjOVQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "micromark-factory-mdx-expression": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdx-md": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-md/-/micromark-extension-mdx-md-2.0.0.tgz", + "integrity": "sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs/-/micromark-extension-mdxjs-3.0.0.tgz", + "integrity": "sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ==", + "license": "MIT", + "dependencies": { + "acorn": "^8.0.0", + "acorn-jsx": "^5.0.0", + "micromark-extension-mdx-expression": "^3.0.0", + "micromark-extension-mdx-jsx": "^3.0.0", + "micromark-extension-mdx-md": "^2.0.0", + "micromark-extension-mdxjs-esm": "^3.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs-esm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs-esm/-/micromark-extension-mdxjs-esm-3.0.0.tgz", + "integrity": "sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-mdx-expression": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-factory-mdx-expression/-/micromark-factory-mdx-expression-2.0.3.tgz", + "integrity": "sha512-kQnEtA3vzucU2BkrIa8/VaSAsP+EJ3CKOvhMuJgOEGg9KDC6OAY6nSnNDVRiVNRqj7Y4SlSzcStaH/5jge8JdQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-events-to-acorn": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-util-events-to-acorn/-/micromark-util-events-to-acorn-2.0.3.tgz", + "integrity": "sha512-jmsiEIiZ1n7X1Rr5k8wVExBQCg5jy4UXVADItHmNk1zkwEVhBuIUKRu3fqv+hs4nxLISi2DQGlqIOGiFxgbfHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "estree-util-visit": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/mlly": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.2.tgz", + "integrity": "sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.16.0", + "pathe": "^2.0.3", + "pkg-types": "^1.3.1", + "ufo": "^1.6.3" + } + }, + "node_modules/mlly/node_modules/confbox": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz", + "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/mlly/node_modules/pkg-types": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz", + "integrity": "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "confbox": "^0.1.8", + "mlly": "^1.7.4", + "pathe": "^2.0.1" + } + }, + "node_modules/mrmime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", + "integrity": "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/neotraverse": { + "version": "0.6.18", + "resolved": "https://registry.npmjs.org/neotraverse/-/neotraverse-0.6.18.tgz", + "integrity": "sha512-Z4SmBUweYa09+o6pG+eASabEpP6QkQ70yHj351pQoEXIs8uHbaU2DWVmzBANKgflPa47A50PtB2+NgRpQvr7vA==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/nlcst-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/nlcst-to-string/-/nlcst-to-string-4.0.0.tgz", + "integrity": "sha512-YKLBCcUYKAg0FNlOBT6aI91qFmSiFKiluk655WzPF+DDMA02qIyy8uiRqI8QXtcFpEvll12LpL5MXqEmAZ+dcA==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/node-fetch-native": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.7.tgz", + "integrity": "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==", + "license": "MIT" + }, + "node_modules/node-mock-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/node-mock-http/-/node-mock-http-1.0.4.tgz", + "integrity": "sha512-8DY+kFsDkNXy1sJglUfuODx1/opAGJGyrTuFqEoN90oRc2Vk0ZbD4K2qmKXBBEhZQzdKHIVfEJpDU8Ak2NJEvQ==", + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/obug": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/ofetch": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/ofetch/-/ofetch-1.5.1.tgz", + "integrity": "sha512-2W4oUZlVaqAPAil6FUg/difl6YhqhUR7x2eZY4bQCko22UXg3hptq9KLQdqFClV+Wu85UX7hNtdGTngi/1BxcA==", + "license": "MIT", + "dependencies": { + "destr": "^2.0.5", + "node-fetch-native": "^1.6.7", + "ufo": "^1.6.1" + } + }, + "node_modules/ohash": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/ohash/-/ohash-2.0.11.tgz", + "integrity": "sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==", + "license": "MIT" + }, + "node_modules/oniguruma-parser": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/oniguruma-parser/-/oniguruma-parser-0.12.2.tgz", + "integrity": "sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==", + "license": "MIT" + }, + "node_modules/oniguruma-to-es": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/oniguruma-to-es/-/oniguruma-to-es-4.3.6.tgz", + "integrity": "sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==", + "license": "MIT", + "dependencies": { + "oniguruma-parser": "^0.12.2", + "regex": "^6.1.0", + "regex-recursion": "^6.0.2" + } + }, + "node_modules/p-limit": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.2.0.tgz", + "integrity": "sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^1.1.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-queue": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.1.1.tgz", + "integrity": "sha512-aNZ+VfjobsWryoiPnEApGGmf5WmNsCo9xu8dfaYamG5qaLP7ClhLN6NgsFe6SwJ2UbLEBK5dv9x8Mn5+RVhMWQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.1", + "p-timeout": "^6.1.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz", + "integrity": "sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-manager-detector": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.9.0.tgz", + "integrity": "sha512-zAMfbta9nPA8AKkA4WdmNAT8R8s6Dk0/xwPsqNnRidMFcLo1YkLuPDTgs2f64XdzS8vbLNnoJ+uUIkgbz7oRHA==", + "license": "MIT" + }, + "node_modules/pagefind": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/pagefind/-/pagefind-1.5.2.tgz", + "integrity": "sha512-XTUaK0hXMCu2jszWE584JGQT7y284TmMV9l/HX3rnG5uo3rHI/uHU56XTyyyPFjeWEBxECbAi0CaFDJOONtG0Q==", + "license": "MIT", + "bin": { + "pagefind": "lib/runner/bin.cjs" + }, + "optionalDependencies": { + "@pagefind/darwin-arm64": "1.5.2", + "@pagefind/darwin-x64": "1.5.2", + "@pagefind/freebsd-x64": "1.5.2", + "@pagefind/linux-arm64": "1.5.2", + "@pagefind/linux-x64": "1.5.2", + "@pagefind/windows-arm64": "1.5.2", + "@pagefind/windows-x64": "1.5.2" + } + }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/parse-latin": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse-latin/-/parse-latin-7.0.0.tgz", + "integrity": "sha512-mhHgobPPua5kZ98EF4HWiH167JWBfl4pvAIXXdbaVohtK7a6YBOy56kvhCqduqyo/f3yrHFWmqmiMg/BkBkYYQ==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0", + "@types/unist": "^3.0.0", + "nlcst-to-string": "^4.0.0", + "unist-util-modify-children": "^4.0.0", + "unist-util-visit-children": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/piccolore": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/piccolore/-/piccolore-0.1.3.tgz", + "integrity": "sha512-o8bTeDWjE086iwKrROaDf31K0qC/BENdm15/uH9usSC/uZjJOKb2YGiVHfLY4GhwsERiPI1jmwI2XrA7ACOxVw==", + "license": "ISC" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pkg-types": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-2.3.3.tgz", + "integrity": "sha512-j/lCFdcppV0JxWpCEITdbDltBxPP6cHT+yNJ6Go2OgoSA9518X847X9z0p6LtA4Nc16+eQzCZjRrWanTGvHJ5w==", + "dev": true, + "license": "MIT", + "dependencies": { + "confbox": "^0.3.1", + "exsolve": "^1.1.1", + "pathe": "^2.0.3" + } + }, + "node_modules/postcss": { + "version": "8.5.19", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.19.tgz", + "integrity": "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-nested": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.2.0.tgz", + "integrity": "sha512-HQbt28KulC5AJzG+cZtj9kvKB93CFCdLvog1WFLf1D+xmMvPGlBstkpTEZfK5+AN9hfJocyBFCNiqyS48bpgzQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "postcss-selector-parser": "^6.1.1" + }, + "engines": { + "node": ">=12.0" + }, + "peerDependencies": { + "postcss": "^8.2.14" + } + }, + "node_modules/postcss-selector-parser": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/prismjs": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.30.0.tgz", + "integrity": "sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/property-information": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.2.0.tgz", + "integrity": "sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/quansync": { + "version": "0.2.11", + "resolved": "https://registry.npmjs.org/quansync/-/quansync-0.2.11.tgz", + "integrity": "sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT" + }, + "node_modules/radix3": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/radix3/-/radix3-1.1.2.tgz", + "integrity": "sha512-b484I/7b8rDEdSDKckSSBA8knMpcdsXudlE/LNL639wFoHKwLbEkQFZHWEYwDC0wa0FKUcCY+GAF73Z7wxNVFA==", + "license": "MIT" + }, + "node_modules/readdirp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", + "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/recma-build-jsx": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-build-jsx/-/recma-build-jsx-1.0.0.tgz", + "integrity": "sha512-8GtdyqaBcDfva+GUKDr3nev3VpKAhup1+RvkMvUxURHpW7QyIvk9F5wz7Vzo06CEMSilw6uArgRqhpiUcWp8ew==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-util-build-jsx": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/recma-jsx": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/recma-jsx/-/recma-jsx-1.0.1.tgz", + "integrity": "sha512-huSIy7VU2Z5OLv6oFLosQGGDqPqdO1iq6bWNAdhzMxSJP7RAso4fCZ1cKu8j9YHCZf3TPrq4dw3okhrylgcd7w==", + "license": "MIT", + "dependencies": { + "acorn-jsx": "^5.0.0", + "estree-util-to-js": "^2.0.0", + "recma-parse": "^1.0.0", + "recma-stringify": "^1.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/recma-parse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-parse/-/recma-parse-1.0.0.tgz", + "integrity": "sha512-OYLsIGBB5Y5wjnSnQW6t3Xg7q3fQ7FWbw/vcXtORTnyaSFscOtABg+7Pnz6YZ6c27fG1/aN8CjfwoUEUIdwqWQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "esast-util-from-js": "^2.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/recma-stringify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-stringify/-/recma-stringify-1.0.0.tgz", + "integrity": "sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-util-to-js": "^2.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/regex/-/regex-6.1.0.tgz", + "integrity": "sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==", + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-recursion": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/regex-recursion/-/regex-recursion-6.0.2.tgz", + "integrity": "sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==", + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-utilities": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/regex-utilities/-/regex-utilities-2.3.0.tgz", + "integrity": "sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==", + "license": "MIT" + }, + "node_modules/rehype": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/rehype/-/rehype-13.0.2.tgz", + "integrity": "sha512-j31mdaRFrwFRUIlxGeuPXXKWQxet52RBQRvCmzl5eCefn/KGbomK5GMHNMsOJf55fgo3qw5tST5neDuarDYR2A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "rehype-parse": "^9.0.0", + "rehype-stringify": "^10.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-expressive-code": { + "version": "0.41.7", + "resolved": "https://registry.npmjs.org/rehype-expressive-code/-/rehype-expressive-code-0.41.7.tgz", + "integrity": "sha512-25f8ZMSF1d9CMscX7Cft0TSQIqdwjce2gDOvQ+d/w0FovsMwrSt3ODP4P3Z7wO1jsIJ4eYyaDRnIR/27bd/EMQ==", + "license": "MIT", + "dependencies": { + "expressive-code": "^0.41.7" + } + }, + "node_modules/rehype-format": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/rehype-format/-/rehype-format-5.0.1.tgz", + "integrity": "sha512-zvmVru9uB0josBVpr946OR8ui7nJEdzZobwLOOqHb/OOD88W0Vk2SqLwoVOj0fM6IPCCO6TaV9CvQvJMWwukFQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-format": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-minify-whitespace": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/rehype-minify-whitespace/-/rehype-minify-whitespace-6.0.2.tgz", + "integrity": "sha512-Zk0pyQ06A3Lyxhe9vGtOtzz3Z0+qZ5+7icZ/PL/2x1SHPbKao5oB/g/rlc6BCTajqBb33JcOe71Ye1oFsuYbnw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-minify-whitespace": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-parse": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/rehype-parse/-/rehype-parse-9.0.1.tgz", + "integrity": "sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-from-html": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-raw": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz", + "integrity": "sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-raw": "^9.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-recma": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/rehype-recma/-/rehype-recma-1.0.0.tgz", + "integrity": "sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "hast-util-to-estree": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-remark": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/rehype-remark/-/rehype-remark-10.0.1.tgz", + "integrity": "sha512-EmDndlb5NVwXGfUa4c9GPK+lXeItTilLhE6ADSaQuHr4JUlKw9MidzGzx4HpqZrNCt6vnHmEifXQiiA+CEnjYQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "hast-util-to-mdast": "^10.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-stringify": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/rehype-stringify/-/rehype-stringify-10.0.1.tgz", + "integrity": "sha512-k9ecfXHmIPuFVI61B9DeLPN0qFHfawM6RsuX48hoqlaKSF61RskNjSm1lI8PhBEM0MRdLxVVm4WmTqJQccH9mA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-to-html": "^9.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-directive": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/remark-directive/-/remark-directive-3.0.1.tgz", + "integrity": "sha512-gwglrEQEZcZYgVyG1tQuA+h58EZfq5CSULw7J90AFuCTyib1thgHPoqQ+h9iFvU6R+vnZ5oNFQR5QKgGpk741A==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-directive": "^3.0.0", + "micromark-extension-directive": "^3.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-gfm": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz", + "integrity": "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-gfm": "^3.0.0", + "micromark-extension-gfm": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-mdx": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/remark-mdx/-/remark-mdx-3.1.1.tgz", + "integrity": "sha512-Pjj2IYlUY3+D8x00UJsIOg5BEvfMyeI+2uLPn9VO9Wg4MEtN/VTIq2NEJQfde9PnX15KgtHyl9S0BcTnWrIuWg==", + "license": "MIT", + "dependencies": { + "mdast-util-mdx": "^3.0.0", + "micromark-extension-mdxjs": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-rehype": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/remark-rehype/-/remark-rehype-11.1.2.tgz", + "integrity": "sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "mdast-util-to-hast": "^13.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-smartypants": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/remark-smartypants/-/remark-smartypants-3.0.2.tgz", + "integrity": "sha512-ILTWeOriIluwEvPjv67v7Blgrcx+LZOkAUVtKI3putuhlZm84FnqDORNXPPm+HY3NdZOMhyDwZ1E+eZB/Df5dA==", + "license": "MIT", + "dependencies": { + "retext": "^9.0.0", + "retext-smartypants": "^6.0.0", + "unified": "^11.0.4", + "unist-util-visit": "^5.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/remark-stringify": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", + "integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-to-markdown": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/retext": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/retext/-/retext-9.0.0.tgz", + "integrity": "sha512-sbMDcpHCNjvlheSgMfEcVrZko3cDzdbe1x/e7G66dFp0Ff7Mldvi2uv6JkJQzdRcvLYE8CA8Oe8siQx8ZOgTcA==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0", + "retext-latin": "^4.0.0", + "retext-stringify": "^4.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/retext-latin": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/retext-latin/-/retext-latin-4.0.0.tgz", + "integrity": "sha512-hv9woG7Fy0M9IlRQloq/N6atV82NxLGveq+3H2WOi79dtIYWN8OaxogDm77f8YnVXJL2VD3bbqowu5E3EMhBYA==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0", + "parse-latin": "^7.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/retext-smartypants": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/retext-smartypants/-/retext-smartypants-6.2.0.tgz", + "integrity": "sha512-kk0jOU7+zGv//kfjXEBjdIryL1Acl4i9XNkHxtM7Tm5lFiCog576fjNC9hjoR7LTKQ0DsPWy09JummSsH1uqfQ==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0", + "nlcst-to-string": "^4.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/retext-stringify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/retext-stringify/-/retext-stringify-4.0.0.tgz", + "integrity": "sha512-rtfN/0o8kL1e+78+uxPTqu1Klt0yPzKuQ2BfWwwfgIUSayyzxpM1PJzkKt4V8803uB9qSy32MvI7Xep9khTpiA==", + "license": "MIT", + "dependencies": { + "@types/nlcst": "^2.0.0", + "nlcst-to-string": "^4.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rollup": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.2.tgz", + "integrity": "sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==", + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.62.2", + "@rollup/rollup-android-arm64": "4.62.2", + "@rollup/rollup-darwin-arm64": "4.62.2", + "@rollup/rollup-darwin-x64": "4.62.2", + "@rollup/rollup-freebsd-arm64": "4.62.2", + "@rollup/rollup-freebsd-x64": "4.62.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.62.2", + "@rollup/rollup-linux-arm-musleabihf": "4.62.2", + "@rollup/rollup-linux-arm64-gnu": "4.62.2", + "@rollup/rollup-linux-arm64-musl": "4.62.2", + "@rollup/rollup-linux-loong64-gnu": "4.62.2", + "@rollup/rollup-linux-loong64-musl": "4.62.2", + "@rollup/rollup-linux-ppc64-gnu": "4.62.2", + "@rollup/rollup-linux-ppc64-musl": "4.62.2", + "@rollup/rollup-linux-riscv64-gnu": "4.62.2", + "@rollup/rollup-linux-riscv64-musl": "4.62.2", + "@rollup/rollup-linux-s390x-gnu": "4.62.2", + "@rollup/rollup-linux-x64-gnu": "4.62.2", + "@rollup/rollup-linux-x64-musl": "4.62.2", + "@rollup/rollup-openbsd-x64": "4.62.2", + "@rollup/rollup-openharmony-arm64": "4.62.2", + "@rollup/rollup-win32-arm64-msvc": "4.62.2", + "@rollup/rollup-win32-ia32-msvc": "4.62.2", + "@rollup/rollup-win32-x64-gnu": "4.62.2", + "@rollup/rollup-win32-x64-msvc": "4.62.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/sax": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", + "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=11.0.0" + } + }, + "node_modules/scule": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/scule/-/scule-1.3.0.tgz", + "integrity": "sha512-6FtHJEvt+pVMIB9IBY+IcCJ6Z5f1iQnytgyfKMhDKgmzYG+TeH/wx1y3l27rshSbLiSanrR9ffZDrEsmjlQF2g==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/shiki": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-3.23.0.tgz", + "integrity": "sha512-55Dj73uq9ZXL5zyeRPzHQsK7Nbyt6Y10k5s7OjuFZGMhpp4r/rsLBH0o/0fstIzX1Lep9VxefWljK/SKCzygIA==", + "license": "MIT", + "dependencies": { + "@shikijs/core": "3.23.0", + "@shikijs/engine-javascript": "3.23.0", + "@shikijs/engine-oniguruma": "3.23.0", + "@shikijs/langs": "3.23.0", + "@shikijs/themes": "3.23.0", + "@shikijs/types": "3.23.0", + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4" + } + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "license": "MIT" + }, + "node_modules/sitemap": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-9.0.1.tgz", + "integrity": "sha512-S6hzjGJSG3d6if0YoF5kTyeRJvia6FSTBroE5fQ0bu1QNxyJqhhinfUsXi9fH3MgtXODWvwo2BDyQSnhPQ88uQ==", + "license": "MIT", + "dependencies": { + "@types/node": "^24.9.2", + "@types/sax": "^1.2.1", + "arg": "^5.0.0", + "sax": "^1.4.1" + }, + "bin": { + "sitemap": "dist/esm/cli.js" + }, + "engines": { + "node": ">=20.19.5", + "npm": ">=10.8.2" + } + }, + "node_modules/smol-toml": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.0.tgz", + "integrity": "sha512-aqVvWoyO21L23mb+drl4RmMXbf6N7FdHjAhTRA9ZBL7apWBgfWC16KjrASI+1p9GAroljyMHj6fK67i0UiTNvQ==", + "license": "BSD-3-Clause", + "engines": { + "node": ">= 18" + }, + "funding": { + "url": "https://github.com/sponsors/cyyynthia" + } + }, + "node_modules/source-map": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", + "integrity": "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==", + "license": "BSD-3-Clause", + "engines": { + "node": ">= 12" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/starlight-links-validator": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/starlight-links-validator/-/starlight-links-validator-0.19.2.tgz", + "integrity": "sha512-IHeK3R78fsmv53VfRkGbXkwK1CQEUBHM9QPzBEyoAxjZ/ssi5gjV+F4oNNUppTR48iPp+lEY0MTAmvkX7yNnkw==", + "license": "MIT", + "dependencies": { + "@types/picomatch": "^3.0.1", + "github-slugger": "^2.0.0", + "hast-util-from-html": "^2.0.3", + "hast-util-has-property": "^3.0.0", + "is-absolute-url": "^4.0.1", + "kleur": "^4.1.5", + "mdast-util-mdx-jsx": "^3.1.3", + "mdast-util-to-string": "^4.0.0", + "picomatch": "^4.0.2", + "terminal-link": "^5.0.0", + "unist-util-visit": "^5.0.0" + }, + "engines": { + "node": ">=18.17.1" + }, + "peerDependencies": { + "@astrojs/starlight": ">=0.32.0", + "astro": ">=5.1.5" + } + }, + "node_modules/starlight-links-validator/node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/starlight-llm-actions": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/starlight-llm-actions/-/starlight-llm-actions-0.13.0.tgz", + "integrity": "sha512-U2Aw6iegQr9i6uQ4bteMcwcUtZ0qKgZOUA0vm3td1OL/Es0StC+ixB2yNkFswl9cXNDX1CElgx+YsivENePzGw==", + "license": "MIT", + "dependencies": { + "picomatch": "^4", + "zod": "^4.4.3" + }, + "engines": { + "node": "^22.0.0 || >=24.0.0" + }, + "peerDependencies": { + "@astrojs/mdx": ">=4.0.0", + "@astrojs/starlight": ">=0.36.0", + "astro": ">=5.6.0", + "hast-util-select": ">=6.0.0", + "rehype-parse": ">=9.0.0", + "rehype-remark": ">=10.0.0", + "remark-gfm": ">=4.0.0", + "remark-stringify": ">=11.0.0", + "unified": ">=11.0.0", + "unist-util-remove": ">=4.0.0", + "vite": ">=5.0.0" + }, + "peerDependenciesMeta": { + "@astrojs/mdx": { + "optional": true + }, + "hast-util-select": { + "optional": true + }, + "rehype-parse": { + "optional": true + }, + "rehype-remark": { + "optional": true + }, + "remark-gfm": { + "optional": true + }, + "remark-stringify": { + "optional": true + }, + "unified": { + "optional": true + }, + "unist-util-remove": { + "optional": true + } + } + }, + "node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "license": "MIT", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/style-to-js": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/style-to-js/-/style-to-js-1.1.21.tgz", + "integrity": "sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ==", + "license": "MIT", + "dependencies": { + "style-to-object": "1.0.14" + } + }, + "node_modules/style-to-object": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.14.tgz", + "integrity": "sha512-LIN7rULI0jBscWQYaSswptyderlarFkjQ+t79nzty8tcIAceVomEVlLzH5VP4Cmsv6MtKhs7qaAiwlcp+Mgaxw==", + "license": "MIT", + "dependencies": { + "inline-style-parser": "0.2.7" + } + }, + "node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/supports-hyperlinks": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-4.6.0.tgz", + "integrity": "sha512-FL/q/YePS4tFFTNiBOR+zPk2ZrKheqNtmTJ/9OUZ4e1wKHJofDbGQSmSsxS0XNnabSkqexryCaXGvEKz1vf9UQ==", + "license": "MIT", + "dependencies": { + "has-flag": "^5.0.1", + "supports-color": "^10.2.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/chalk/supports-hyperlinks?sponsor=1" + } + }, + "node_modules/svelte": { + "version": "5.57.2", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.57.2.tgz", + "integrity": "sha512-1YiPCAXT0Ct6g1vOMmSLa0Nsc3QRMvtDF32ihgglGVrugIuIK38RyDiFNXVDhtQ81TJjSO57NdMw6DyYLqSGDQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "@jridgewell/sourcemap-codec": "^1.6.0", + "@sveltejs/acorn-typescript": "^1.0.13", + "@types/estree": "^1.0.9", + "acorn": "^8.18.0", + "aria-query": "5.3.1", + "axobject-query": "^4.1.0", + "clsx": "^2.1.1", + "devalue": "^5.9.2", + "esm-env": "^1.2.1", + "esrap": "^2.3.6", + "is-reference": "^3.0.3", + "locate-character": "^3.0.0", + "magic-string": "^0.30.11", + "zimmerframe": "^1.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/svelte/node_modules/aria-query": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.1.tgz", + "integrity": "sha512-Z/ZeOgVl7bcSYZ/u/rh0fOpvEpq//LZmdbkXyc7syVzjPAhfOa9ebsdTSjEBDU4vs5nC98Kfduj1uFo0qyET3g==", + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/svelte2tsx": { + "version": "0.7.61", + "resolved": "https://registry.npmjs.org/svelte2tsx/-/svelte2tsx-0.7.61.tgz", + "integrity": "sha512-EpQ/+UHITBULeUojx/LLD3uTYasZXcX1BrqlrzfLUnT9vdUhaOWK59a3ryWcFU8L0sY1SP+gRhJ2Beiis98+qw==", + "license": "MIT", + "dependencies": { + "dedent-js": "^1.0.1", + "scule": "^1.3.0" + }, + "peerDependencies": { + "svelte": "^3.55 || ^4.0.0-next.0 || ^4.0 || ^5.0.0-next.0", + "typescript": "^4.9.4 || ^5.0.0 || ^6.0.0" + } + }, + "node_modules/svgo": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-4.0.2.tgz", + "integrity": "sha512-ekx94z1rRc5LDi6oSUaeRnYhd0UOJxdtQCL2rF8xpWxD3TPAsISWOrxezqGovqS38GRZOdpDfvQe3ts6F7nsng==", + "license": "MIT", + "dependencies": { + "commander": "^11.1.0", + "css-select": "^5.1.0", + "css-tree": "^3.0.1", + "css-what": "^6.1.0", + "csso": "^5.0.5", + "picocolors": "^1.1.1", + "sax": "^1.5.0" + }, + "bin": { + "svgo": "bin/svgo.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/svgo" + } + }, + "node_modules/tailwind-merge": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.7.0.tgz", + "integrity": "sha512-XPPUyAc+cvspz3lHTcR/QgPfW2A0lv/xQNIjX3HGhLR+Nq2lHaLq5MtTesHn8GUr3W3DguT2KT5x3NVgRtYwmA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/dcastil" + } + }, + "node_modules/tailwind-variants": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/tailwind-variants/-/tailwind-variants-3.3.1.tgz", + "integrity": "sha512-4pAvwUtM4HKBiRZftncAbpn6V9Hhwoa5Fl7O2u5zbp7Z5Cvu+/o/6+176WY3WCEES209543quG8zFIcXCsc5Jw==", + "license": "MIT", + "engines": { + "node": ">=16.9.x", + "pnpm": ">=7.x" + }, + "peerDependencies": { + "tailwind-merge": ">=3.0.0", + "tailwindcss": "*" + }, + "peerDependenciesMeta": { + "tailwind-merge": { + "optional": true + }, + "tailwindcss": { + "optional": true + } + } + }, + "node_modules/tailwindcss": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.3.tgz", + "integrity": "sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==", + "license": "MIT" + }, + "node_modules/tapable": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/terminal-link": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-5.0.0.tgz", + "integrity": "sha512-qFAy10MTMwjzjU8U16YS4YoZD+NQLHzLssFMNqgravjbvIPNiqkGFR4yjhJfmY9R5OFU7+yHxc6y+uGHkKwLRA==", + "license": "MIT", + "dependencies": { + "ansi-escapes": "^7.0.0", + "supports-hyperlinks": "^4.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tiny-inflate": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", + "integrity": "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==", + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.1.tgz", + "integrity": "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trim-trailing-lines": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/trim-trailing-lines/-/trim-trailing-lines-2.1.0.tgz", + "integrity": "sha512-5UR5Biq4VlVOtzqkm2AZlgvSlDJtME46uV0br0gENbwN4l5+mMKT4b9gJKqWtuL2zAIqajGJGuvbCbcAJUZqBg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/tsconfck": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/tsconfck/-/tsconfck-3.1.6.tgz", + "integrity": "sha512-ks6Vjr/jEw0P1gmOVwutM3B7fWxoWBL2KRDb1JfqGVawBmO5UsvmWOQFGHBPl5yxYz4eERr19E6L7NMv+Fej4w==", + "deprecated": "unmaintained", + "license": "MIT", + "bin": { + "tsconfck": "bin/tsconfck.js" + }, + "engines": { + "node": "^18 || >=20" + }, + "peerDependencies": { + "typescript": "^5.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, + "node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/ufo": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.4.tgz", + "integrity": "sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==", + "license": "MIT" + }, + "node_modules/ultrahtml": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/ultrahtml/-/ultrahtml-1.7.0.tgz", + "integrity": "sha512-2xRd0VHoAQE4M+vF/DvFFB7pUV0ZxTW1TLi7lHQWnF/Sb5TPeEUV/l+hxcNnGO00ZXGnR0voCMmYRKQf+rvJ2g==", + "license": "MIT" + }, + "node_modules/uncrypto": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/uncrypto/-/uncrypto-0.1.3.tgz", + "integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==", + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "license": "MIT" + }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unifont": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/unifont/-/unifont-0.7.4.tgz", + "integrity": "sha512-oHeis4/xl42HUIeHuNZRGEvxj5AaIKR+bHPNegRq5LV1gdc3jundpONbjglKpihmJf+dswygdMJn3eftGIMemg==", + "license": "MIT", + "dependencies": { + "css-tree": "^3.1.0", + "ofetch": "^1.5.1", + "ohash": "^2.0.11" + } + }, + "node_modules/unist-util-find-after": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-5.0.0.tgz", + "integrity": "sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", + "integrity": "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-modify-children": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-modify-children/-/unist-util-modify-children-4.0.0.tgz", + "integrity": "sha512-+tdN5fGNddvsQdIzUF3Xx82CU9sMM+fA0dLgR9vOmT0oPT2jH+P1nd5lSqfCfXAw+93NhcXNY2qqvTUtE4cQkw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "array-iterate": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position-from-estree": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position-from-estree/-/unist-util-position-from-estree-2.0.0.tgz", + "integrity": "sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-remove": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-remove/-/unist-util-remove-4.0.0.tgz", + "integrity": "sha512-b4gokeGId57UVRX/eVKej5gXqGlc9+trkORhFJpu9raqZkZhU0zm8Doi05+HaiBsMEIJowL+2WtQ5ItjsngPXg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-remove-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-5.0.0.tgz", + "integrity": "sha512-Hp5Kh3wLxv0PHj9m2yZhhLt58KzPtEYKQQ4yxfYFEO7EvHwzyDYnduhHnY1mDxoqr7VUwVuHXk9RXKIiYS1N8Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", + "integrity": "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-children": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unist-util-visit-children/-/unist-util-visit-children-3.0.0.tgz", + "integrity": "sha512-RgmdTfSBOg04sdPcpTSD1jzoNBjt9a80/ZCzp5cI9n1qPzLZWF9YdvWGN2zmTumP1HWhXKdUWexjy/Wy/lJ7tA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.2.tgz", + "integrity": "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unplugin": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/unplugin/-/unplugin-3.4.0.tgz", + "integrity": "sha512-9skdIFlCsPdFV7wUfZxNsFInlW+7nJmGu2gkTu0OUhF56aXGsHab9x52/QhdJ4lC7ZDPWTxbiC4ANqVlsuaW3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "picomatch": "^4.0.7", + "webpack-virtual-modules": "^0.6.2" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "@farmfe/core": "*", + "@rsbuild/core": "*", + "@rspack/core": "*", + "bun-types-no-globals": "*", + "esbuild": "*", + "rolldown": "*", + "rollup": "*", + "unloader": "*", + "vite": "*", + "webpack": "*" + }, + "peerDependenciesMeta": { + "@farmfe/core": { + "optional": true + }, + "@rsbuild/core": { + "optional": true + }, + "@rspack/core": { + "optional": true + }, + "bun-types-no-globals": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "rolldown": { + "optional": true + }, + "rollup": { + "optional": true + }, + "unloader": { + "optional": true + }, + "vite": { + "optional": true + }, + "webpack": { + "optional": true + } + } + }, + "node_modules/unplugin-icons": { + "version": "24.0.0", + "resolved": "https://registry.npmjs.org/unplugin-icons/-/unplugin-icons-24.0.0.tgz", + "integrity": "sha512-wV6c7I0PtiIP8KIU7Z1Ukyc2uzgY9ind3RXNu7j63L9qTWDn7CzLnW9FD57DKbfGpWKQv3qyV2PCtBG8Y5Llcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@antfu/install-pkg": "^1.1.0", + "@iconify/utils": "^3.1.7", + "local-pkg": "^1.2.1", + "obug": "^2.2.1", + "unplugin": "^3.3.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@svgr/core": ">=7.0.0", + "@svgx/core": "^1.0.1", + "@vue/compiler-sfc": "^3.0.2", + "@vue/compiler-vapor": "^3.6.0-rc.5", + "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "@svgr/core": { + "optional": true + }, + "@svgx/core": { + "optional": true + }, + "@vue/compiler-sfc": { + "optional": true + }, + "@vue/compiler-vapor": { + "optional": true + }, + "svelte": { + "optional": true + } + } + }, + "node_modules/unstorage": { + "version": "1.17.5", + "resolved": "https://registry.npmjs.org/unstorage/-/unstorage-1.17.5.tgz", + "integrity": "sha512-0i3iqvRfx29hkNntHyQvJTpf5W9dQ9ZadSoRU8+xVlhVtT7jAX57fazYO9EHvcRCfBCyi5YRya7XCDOsbTgkPg==", + "license": "MIT", + "dependencies": { + "anymatch": "^3.1.3", + "chokidar": "^5.0.0", + "destr": "^2.0.5", + "h3": "^1.15.10", + "lru-cache": "^11.2.7", + "node-fetch-native": "^1.6.7", + "ofetch": "^1.5.1", + "ufo": "^1.6.3" + }, + "peerDependencies": { + "@azure/app-configuration": "^1.8.0", + "@azure/cosmos": "^4.2.0", + "@azure/data-tables": "^13.3.0", + "@azure/identity": "^4.6.0", + "@azure/keyvault-secrets": "^4.9.0", + "@azure/storage-blob": "^12.26.0", + "@capacitor/preferences": "^6 || ^7 || ^8", + "@deno/kv": ">=0.9.0", + "@netlify/blobs": "^6.5.0 || ^7.0.0 || ^8.1.0 || ^9.0.0 || ^10.0.0", + "@planetscale/database": "^1.19.0", + "@upstash/redis": "^1.34.3", + "@vercel/blob": ">=0.27.1", + "@vercel/functions": "^2.2.12 || ^3.0.0", + "@vercel/kv": "^1 || ^2 || ^3", + "aws4fetch": "^1.0.20", + "db0": ">=0.2.1", + "idb-keyval": "^6.2.1", + "ioredis": "^5.4.2", + "uploadthing": "^7.4.4" + }, + "peerDependenciesMeta": { + "@azure/app-configuration": { + "optional": true + }, + "@azure/cosmos": { + "optional": true + }, + "@azure/data-tables": { + "optional": true + }, + "@azure/identity": { + "optional": true + }, + "@azure/keyvault-secrets": { + "optional": true + }, + "@azure/storage-blob": { + "optional": true + }, + "@capacitor/preferences": { + "optional": true + }, + "@deno/kv": { + "optional": true + }, + "@netlify/blobs": { + "optional": true + }, + "@planetscale/database": { + "optional": true + }, + "@upstash/redis": { + "optional": true + }, + "@vercel/blob": { + "optional": true + }, + "@vercel/functions": { + "optional": true + }, + "@vercel/kv": { + "optional": true + }, + "aws4fetch": { + "optional": true + }, + "db0": { + "optional": true + }, + "idb-keyval": { + "optional": true + }, + "ioredis": { + "optional": true + }, + "uploadthing": { + "optional": true + } + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-location": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-5.0.3.tgz", + "integrity": "sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", + "integrity": "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vite": { + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz", + "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/vitefu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.1.3.tgz", + "integrity": "sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg==", + "license": "MIT", + "workspaces": [ + "tests/deps/*", + "tests/projects/*", + "tests/projects/workspace/packages/*" + ], + "peerDependencies": { + "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "vite": { + "optional": true + } + } + }, + "node_modules/web-namespaces": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz", + "integrity": "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/webpack-virtual-modules": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/webpack-virtual-modules/-/webpack-virtual-modules-0.6.2.tgz", + "integrity": "sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/which-pm-runs": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.1.0.tgz", + "integrity": "sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/widest-line": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-5.0.0.tgz", + "integrity": "sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA==", + "license": "MIT", + "dependencies": { + "string-width": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/xxhash-wasm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.1.0.tgz", + "integrity": "sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==", + "license": "MIT" + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", + "license": "MIT", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/yocto-spinner": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/yocto-spinner/-/yocto-spinner-0.2.3.tgz", + "integrity": "sha512-sqBChb33loEnkoXte1bLg45bEBsOP9N1kzQh5JZNKj/0rik4zAPTNSAVPj3uQAdc6slYJ0Ksc403G2XgxsJQFQ==", + "license": "MIT", + "dependencies": { + "yoctocolors": "^2.1.1" + }, + "engines": { + "node": ">=18.19" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/yoctocolors": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yoctocolors/-/yoctocolors-2.1.2.tgz", + "integrity": "sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zimmerframe": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/zimmerframe/-/zimmerframe-1.1.5.tgz", + "integrity": "sha512-msJxIvYDYcoNL+PJsu+7qmpDWsYmAxTY+2TNYXXF0hzBzBk0BMecOqDOG/EckUoKCuKwObfbugIl8QpqHDXeFA==", + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + } + } +} diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 0000000000000000000000000000000000000000..831a121b7f402397b72c0b5c5bbe84aed1e58d14 --- /dev/null +++ b/docs/package.json @@ -0,0 +1,39 @@ +{ + "name": "tangled-docs", + "type": "module", + "version": "0.0.1", + "private": true, + "scripts": { + "dev": "astro dev", + "build": "astro build", + "preview": "astro preview" + }, + "dependencies": { + "@astrojs/mdx": "^4.3.14", + "@astrojs/sitemap": "^3.7.4", + "@astrojs/starlight": "^0.36.0", + "@astrojs/starlight-tailwind": "^4.0.2", + "@astrojs/svelte": "^7.2.5", + "@tailwindcss/vite": "^4.3.3", + "astro": "^5.14.0", + "hast-util-select": "^6.0.4", + "rehype-parse": "^9.0.1", + "rehype-remark": "^10.0.1", + "remark-gfm": "^4.0.1", + "remark-stringify": "^11.0.0", + "sharp": "^0.34.0", + "starlight-links-validator": "^0.19.2", + "starlight-llm-actions": "0.13.0", + "svelte": "^5.57.2", + "tailwind-merge": "^3.7.0", + "tailwind-variants": "^3.3.1", + "tailwindcss": "^4.3.3", + "unified": "^11.0.5", + "unist-util-remove": "^4.0.0" + }, + "devDependencies": { + "@iconify-json/lucide": "^1.2.140", + "typescript": "^5.9.3", + "unplugin-icons": "^24.0.0" + } +} diff --git a/docs/pnpm-lock.yaml b/docs/pnpm-lock.yaml new file mode 100644 index 0000000000000000000000000000000000000000..111283b0a35dc1ba2a8deb706195290582d558e2 --- /dev/null +++ b/docs/pnpm-lock.yaml @@ -0,0 +1,5454 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@astrojs/mdx': + specifier: ^4.3.14 + version: 4.3.14(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)) + '@astrojs/sitemap': + specifier: ^3.7.4 + version: 3.7.4 + '@astrojs/starlight': + specifier: ^0.36.0 + version: 0.36.3(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)) + '@astrojs/starlight-tailwind': + specifier: ^4.0.2 + version: 4.0.2(@astrojs/starlight@0.36.3(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)))(tailwindcss@4.3.3) + '@astrojs/svelte': + specifier: ^7.2.5 + version: 7.2.5(@types/node@24.13.3)(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1))(jiti@2.7.0)(lightningcss@1.32.0)(svelte@5.57.2)(typescript@5.9.3)(yaml@2.9.1) + '@tailwindcss/vite': + specifier: ^4.3.3 + version: 4.3.3(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)) + astro: + specifier: ^5.14.0 + version: 5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1) + hast-util-select: + specifier: ^6.0.4 + version: 6.0.4 + rehype-parse: + specifier: ^9.0.1 + version: 9.0.1 + rehype-remark: + specifier: ^10.0.1 + version: 10.0.1 + remark-gfm: + specifier: ^4.0.1 + version: 4.0.1 + remark-stringify: + specifier: ^11.0.0 + version: 11.0.0 + sharp: + specifier: ^0.34.0 + version: 0.34.5 + starlight-links-validator: + specifier: ^0.19.2 + version: 0.19.2(@astrojs/starlight@0.36.3(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)))(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)) + starlight-llm-actions: + specifier: 0.13.0 + version: 0.13.0(@astrojs/mdx@4.3.14(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)))(@astrojs/starlight@0.36.3(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)))(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1))(hast-util-select@6.0.4)(rehype-parse@9.0.1)(rehype-remark@10.0.1)(remark-gfm@4.0.1)(remark-stringify@11.0.0)(unified@11.0.5)(unist-util-remove@4.0.0)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)) + svelte: + specifier: ^5.57.2 + version: 5.57.2 + tailwind-merge: + specifier: ^3.7.0 + version: 3.7.0 + tailwind-variants: + specifier: ^3.3.1 + version: 3.3.1(tailwind-merge@3.7.0)(tailwindcss@4.3.3) + tailwindcss: + specifier: ^4.3.3 + version: 4.3.3 + unified: + specifier: ^11.0.5 + version: 11.0.5 + unist-util-remove: + specifier: ^4.0.0 + version: 4.0.0 + devDependencies: + '@iconify-json/lucide': + specifier: ^1.2.140 + version: 1.2.142 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + unplugin-icons: + specifier: ^24.0.0 + version: 24.0.0(esbuild@0.27.7)(rollup@4.62.4)(svelte@5.57.2)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)) + +packages: + + '@antfu/install-pkg@1.1.0': + resolution: {integrity: sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==} + + '@antfu/install-pkg@2.1.0': + resolution: {integrity: sha512-sdg9NxU3zR4Mnawfbc/x6GB5Wf17WYud5qOuEuxXjaKpYpMkISSJEjItGebXJ2bQ4DIcly4NYH23mtkGJjvKUw==} + + '@astrojs/compiler@2.13.1': + resolution: {integrity: sha512-f3FN83d2G/v32ipNClRKgYv30onQlMZX1vCeZMjPsMMPl1mDpmbl0+N5BYo4S/ofzqJyS5hvwacEo0CCVDn/Qg==} + + '@astrojs/internal-helpers@0.7.6': + resolution: {integrity: sha512-GOle7smBWKfMSP8osUIGOlB5kaHdQLV3foCsf+5Q9Wsuu+C6Fs3Ez/ttXmhjZ1HkSgsogcM1RXSjjOVieHq16Q==} + + '@astrojs/markdown-remark@6.3.11': + resolution: {integrity: sha512-hcaxX/5aC6lQgHeGh1i+aauvSwIT6cfyFjKWvExYSxUhZZBBdvCliOtu06gbQyhbe0pGJNoNmqNlQZ5zYUuIyQ==} + + '@astrojs/mdx@4.3.14': + resolution: {integrity: sha512-FBrqJQORVm+rkRa2TS5CjU9PBA6hkhrwLVBSS9A77gN2+iehvjq1w6yya/d0YKC7osiVorKkr3Qd9wNbl0ZkGA==} + engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0} + peerDependencies: + astro: ^5.0.0 + + '@astrojs/prism@3.3.0': + resolution: {integrity: sha512-q8VwfU/fDZNoDOf+r7jUnMC2//H2l0TuQ6FkGJL8vD8nw/q5KiL3DS1KKBI3QhI9UQhpJ5dc7AtqfbXWuOgLCQ==} + engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0} + + '@astrojs/sitemap@3.7.3': + resolution: {integrity: sha512-f8euLVsyeAmAkSm/1M2Kb8sL8byQmfgbvBNaHFItCheTj/IpiJYSEWVcqDHZ/yEHxiS7+w87mQkzwZaPHmk5GA==} + + '@astrojs/sitemap@3.7.4': + resolution: {integrity: sha512-LbKNC24bdUWcQf/pThB6qLlSqHojxGjZDURIzFocY8rlWnAn2t74nnhnK6S5x0NHriHoAduLEpVjRykmeGiVvA==} + + '@astrojs/starlight-tailwind@4.0.2': + resolution: {integrity: sha512-SYN/6zq6hJO5tWqbQ2tWT9/jd8ubUkzkBCcF94vByC/ZJ20Mi5GPjFvAh89Yky/aIM+jXxT6W5q4p6l58GKHiQ==} + peerDependencies: + '@astrojs/starlight': '>=0.34.0' + tailwindcss: ^4.0.0 + + '@astrojs/starlight@0.36.3': + resolution: {integrity: sha512-5cm4QVQHUP6ZE52O43TtUpsTvLKdZa9XEs4l3suzuY7Ymsbz4ojtoL9NhistbMqM+/qk6fm6SmxbOL6hQ/LfNA==} + peerDependencies: + astro: ^5.5.0 + + '@astrojs/svelte@7.2.5': + resolution: {integrity: sha512-Tl5aF/dYbzzd7sLpxMBX6pRz3yJ1B4pilt9G3GJbj0I0/doJHIEmerNQsnlxX0/InNKUhMXXN8wyyet9VhA+Zw==} + engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0} + peerDependencies: + astro: ^5.0.0 + svelte: ^5.1.16 + typescript: ^5.3.3 + + '@astrojs/telemetry@3.3.0': + resolution: {integrity: sha512-UFBgfeldP06qu6khs/yY+q1cDAaArM2/7AEIqQ9Cuvf7B1hNLq0xDrZkct+QoIGyjq56y8IaE2I3CTvG99mlhQ==} + engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0} + + '@babel/helper-string-parser@7.29.7': + resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.29.8': + resolution: {integrity: sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==} + engines: {node: '>=6.0.0'} + hasBin: true + + '@babel/runtime@7.29.7': + resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} + engines: {node: '>=6.9.0'} + + '@babel/types@7.29.8': + resolution: {integrity: sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==} + engines: {node: '>=6.9.0'} + + '@capsizecss/unpack@4.0.1': + resolution: {integrity: sha512-CuNiSqg7+e1cO/GjffyMOm5Tt2jUF9CWHHnvQ/UkqvtkGfHdgwEC0wpmq7fkN3gxwpRnrAN0WzO3vREKmNolMQ==} + engines: {node: '>=18'} + + '@ctrl/tinycolor@4.2.0': + resolution: {integrity: sha512-kzyuwOAQnXJNLS9PSyrk0CWk35nWJW/zl/6KvnTBMFK65gm7U1/Z5BqjxeapjZCIhQcM/DsrEmcbRwDyXyXK4A==} + engines: {node: '>=14'} + + '@emnapi/runtime@1.11.3': + resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} + + '@esbuild/aix-ppc64@0.25.12': + resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/aix-ppc64@0.27.7': + resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.25.12': + resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm64@0.27.7': + resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.25.12': + resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-arm@0.27.7': + resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.25.12': + resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/android-x64@0.27.7': + resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.25.12': + resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-arm64@0.27.7': + resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.25.12': + resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.7': + resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.25.12': + resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-arm64@0.27.7': + resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.25.12': + resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.7': + resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.25.12': + resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm64@0.27.7': + resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.25.12': + resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-arm@0.27.7': + resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.25.12': + resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-ia32@0.27.7': + resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.25.12': + resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-loong64@0.27.7': + resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.25.12': + resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-mips64el@0.27.7': + resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.25.12': + resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-ppc64@0.27.7': + resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.25.12': + resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.7': + resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.25.12': + resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-s390x@0.27.7': + resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.25.12': + resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/linux-x64@0.27.7': + resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.12': + resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-arm64@0.27.7': + resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.25.12': + resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.7': + resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.12': + resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-arm64@0.27.7': + resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.25.12': + resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.7': + resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.25.12': + resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/openharmony-arm64@0.27.7': + resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.25.12': + resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/sunos-x64@0.27.7': + resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.25.12': + resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.27.7': + resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.25.12': + resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-ia32@0.27.7': + resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.25.12': + resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@esbuild/win32-x64@0.27.7': + resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@expressive-code/core@0.41.7': + resolution: {integrity: sha512-ck92uZYZ9Wba2zxkiZLsZGi9N54pMSAVdrI9uW3Oo9AtLglD5RmrdTwbYPCT2S/jC36JGB2i+pnQtBm/Ib2+dg==} + + '@expressive-code/plugin-frames@0.41.7': + resolution: {integrity: sha512-diKtxjQw/979cTglRFaMCY/sR6hWF0kSMg8jsKLXaZBSfGS0I/Hoe7Qds3vVEgeoW+GHHQzMcwvgx/MOIXhrTA==} + + '@expressive-code/plugin-shiki@0.41.7': + resolution: {integrity: sha512-DL605bLrUOgqTdZ0Ot5MlTaWzppRkzzqzeGEu7ODnHF39IkEBbFdsC7pbl3LbUQ1DFtnfx6rD54k/cdofbW6KQ==} + + '@expressive-code/plugin-text-markers@0.41.7': + resolution: {integrity: sha512-Ewpwuc5t6eFdZmWlFyeuy3e1PTQC0jFvw2Q+2bpcWXbOZhPLsT7+h8lsSIJxb5mS7wZko7cKyQ2RLYDyK6Fpmw==} + + '@iconify-json/lucide@1.2.142': + resolution: {integrity: sha512-vEGX5X52/l77jRMcmxXk3PiEebldDUs8WFdM8X3Evq1S9p5F+o0VZhx8sqaQsmFEB9Ukp5ND5Q/0g2kQ/LF+rQ==} + + '@iconify/types@2.0.0': + resolution: {integrity: sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==} + + '@iconify/utils@3.1.7': + resolution: {integrity: sha512-JZHlwdID+dy+lTgbYC8NEC4zeugqeYsc6jewvzb4c58kHauJn+X7rNwQjxz5p2qSjqaEeQoLkCIQ9v/H4PK0/w==} + + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.34.5': + resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.34.5': + resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.2.4': + resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.2.4': + resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.2.4': + resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-arm@1.2.4': + resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-ppc64@1.2.4': + resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-riscv64@1.2.4': + resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-s390x@1.2.4': + resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-x64@1.2.4': + resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-linux-arm64@0.34.5': + resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-arm@0.34.5': + resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-ppc64@0.34.5': + resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-riscv64@0.34.5': + resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-s390x@0.34.5': + resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-x64@0.34.5': + resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-linuxmusl-arm64@0.34.5': + resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-linuxmusl-x64@0.34.5': + resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-wasm32@0.34.5': + resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.34.5': + resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.34.5': + resolution: {integrity: sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.34.5': + resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/remapping@2.3.5': + resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@mdx-js/mdx@3.1.1': + resolution: {integrity: sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ==} + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + resolution: {integrity: sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@oslojs/encoding@1.1.0': + resolution: {integrity: sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==} + + '@pagefind/darwin-arm64@1.5.2': + resolution: {integrity: sha512-MXpI+7HsAdPkvJ0gk9xj9g541BCqBZOBbdwj9g6lB5LCj6kSV6nqDSjzcAJwvOsfu0fjwvC8hQU+ecfhp+MpiQ==} + cpu: [arm64] + os: [darwin] + + '@pagefind/darwin-x64@1.5.2': + resolution: {integrity: sha512-IojxFWMEJe0RQ7PQ3KXQsPIImNsbpPYpoZ+QUDrL8fAl/O27IX+LVLs74/UzEZy5uA2LD8Nz1AiwKr72vrkZQw==} + cpu: [x64] + os: [darwin] + + '@pagefind/default-ui@1.5.2': + resolution: {integrity: sha512-pm1LMnQg8N2B3n2TnjKlhaFihpz6zTiA4HiGQ6/slKO/+8K9CAU5kcjdSSPgpuk1PMuuN4hxLipUIifnrkl3Sg==} + + '@pagefind/freebsd-x64@1.5.2': + resolution: {integrity: sha512-7EVzo9+0w+2cbe671BtMj10UlNo83I+HrLVLfRxO731svHRJKUfJ/mo05gU14pe9PCfpKNQT8FS3Xc/oDN6pOA==} + cpu: [x64] + os: [freebsd] + + '@pagefind/linux-arm64@1.5.2': + resolution: {integrity: sha512-Ovt9+K35sqzn8H3ZMXGwls4TD/wMJuvRtShHIsmUQREmaxjrDEX7gHckRCrwYJ4XE1H1p6HkLz3wukrAnsfXQw==} + cpu: [arm64] + os: [linux] + + '@pagefind/linux-x64@1.5.2': + resolution: {integrity: sha512-V+tFqHKXhQKq/WqPBD67AFy7scn1/aZID00ws4fSDd+1daSi5UHR9VVlRrOUYKxn3VuFQYRD7lYXdZK1WED1YA==} + cpu: [x64] + os: [linux] + + '@pagefind/windows-arm64@1.5.2': + resolution: {integrity: sha512-hN9Nh90fNW61nNRCW9ZyQrAj/mD0eRvmJ8NlTUzkbuW8kIzGJUi3cxjFkEcMZ5h/8FsKWD/VcouZl4yo1F7B6g==} + cpu: [arm64] + os: [win32] + + '@pagefind/windows-x64@1.5.2': + resolution: {integrity: sha512-Fa2Iyw7kaDRzGMfNYNUXNW2zbL5FQVDgSOcbDHdzBrDEdpqOqg8TcZ68F22ol6NJ9IGzvUdmeyZypLW5dyhqsg==} + cpu: [x64] + os: [win32] + + '@rollup/pluginutils@5.4.0': + resolution: {integrity: sha512-MfPp06CjRLfXQ3wY0R8vJDYBy/MvVcc9OulEfR0B8Iv9ko+GCNaRZ+EpJYFl27LhKsZK0o420sYCRHCjfCgeUg==} + engines: {node: '>=14.0.0'} + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + + '@rollup/rollup-android-arm-eabi@4.62.4': + resolution: {integrity: sha512-RrPokAb7dmbxFoeO3TloqHyOjgye8RkBhSqmp4aJMIex4c9r46ZstPnleDQOq1t46VOVjwIuwNogIqbodV1Vvg==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.62.4': + resolution: {integrity: sha512-JKuJc+pnpks2pjy7L/N3v/cAkZxYlnmuZoD840ldbMI5KDbC4iO9NKwPKYdjYFCMAIIlBzYSFHxIJVYzRo2/8A==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.62.4': + resolution: {integrity: sha512-krw5uS2STmvJ02x0uTXHbqQNuz+9eZ1iw+qXk9dmW2gvV4jV7O2hEoOnuhFrpOPiel1mBFtqbxYZZtC46hXLOw==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.62.4': + resolution: {integrity: sha512-wsTxtgApb4PrOsNJIm0FZ1h3WvCC+k9uxLJ4ad75hgoS4NiRes2SoJFlDAyMwiUY8IssDqGcHbXuN0sx1tfF1A==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.62.4': + resolution: {integrity: sha512-GUOnQlyZe3yAXhWOtOMsn5Qkrv5E5mZXa0thbARWi5Ei2szlVXJFQhddZ4HbAzh8q92w5twp+CQvs/eFanz9YQ==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.62.4': + resolution: {integrity: sha512-/Y7f3QuxjzPKsjA/rfEDa3+0vXqyjmJ50Ln8dPpCmWkKTrUoWHG1cWhTqaAMLob2m2nESWuC7yGrREz019Ztqg==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.62.4': + resolution: {integrity: sha512-81wiiX3v7aqy+T+bT61TJ78yJjRquqFFTTbAPt08imfQQzkPIW8t6aJbkTagtCCrXMNc9D66+geqlK7ydLPNqA==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm-musleabihf@4.62.4': + resolution: {integrity: sha512-9kmDIvNZqdoHOBZgNtpTBeLWYO/LVipM3H/j62P8848/l/VPEQL6N3uxU9pvP1oZAsXyC2MEnFP3ovRjo7WYNQ==} + cpu: [arm] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-arm64-gnu@4.62.4': + resolution: {integrity: sha512-CcnXHWnXg69g+DX5VWL3FHts3qMRN2uVEHX+BZvGLdd07/gXkn3ePjYtO1LDJvxkGKVHMclKBRa1QUTH+6toYQ==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm64-musl@4.62.4': + resolution: {integrity: sha512-iFOibiHnTRuhrWLlRsOQFdZJJIa7S8OwkneJr4ocALP16u5yk6lWLINFwhHaEqBFMsKDUZofLkGos7+CPzGB3g==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-loong64-gnu@4.62.4': + resolution: {integrity: sha512-XnWYMI7euHlb5a871xPja+Gm7DRCFU+FGRrtS2sMq9N8FvqtpagUy6gD4YOemC5MRk9xbh8+jYMEJbigFQwsgA==} + cpu: [loong64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-loong64-musl@4.62.4': + resolution: {integrity: sha512-qGDAlO0U8xedCcsdRm9oaoQY8DAx/QT7uIxJWhCdx0ceIWX783UC9QSYkdpzAe29wNiVfp24+bZdQmn49o45SQ==} + cpu: [loong64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-ppc64-gnu@4.62.4': + resolution: {integrity: sha512-ru4H6ezD7ysA5EiEK6qkkaEb4modH8CTej6kUy/gQi20u3kB3G7Zn8snXXkeJSCOFKG/rbPPtM/+9Wgas1961w==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-ppc64-musl@4.62.4': + resolution: {integrity: sha512-2W4MO5WQVJnbJaZdvDb9rhBDuFU1nKIepPFpJUBsTh2k1YY2g+ODViaWuyOAjQ5cOP7NvrvLzt3wvHOoiAvc7w==} + cpu: [ppc64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-riscv64-gnu@4.62.4': + resolution: {integrity: sha512-+fxjfuoAmVMCYV5QyjoIpu0cp5DOiOTeqYFk1AVaxGr+/ravWLX89XfQmptsoWcaVy/TGf2hexzbUOrCQIL1CQ==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-riscv64-musl@4.62.4': + resolution: {integrity: sha512-jTn8JfHGL4djjFxPuM06LmNUJDsst2jeVlsd9OmIH6zc5sC9K6rIuO4YajXatLUpBmBKl6b35ro1QZocLi+tcA==} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-s390x-gnu@4.62.4': + resolution: {integrity: sha512-oCJCJL4pXsoDcP2QZ+JVlPTIRc6266zsIaeJJsWImmF7HO0W8nb6HuSgZlMWxJwaPf8ehbSw8yo0EUw925hKsA==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-gnu@4.62.4': + resolution: {integrity: sha512-W69hukhZ3KKNRCaMIEzKvcFye42hh0FE1+YoYaf5+Ikacuftoco6yO/xouz0hc5d5W/s3yBro5jRiuEE/Q5vUw==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-musl@4.62.4': + resolution: {integrity: sha512-qiXbGG2jkjXhzXpsFZSR2Xpb8DN/UaxYsbb/STbuR/6fpaDgRmmaq1B/LmtF2wQFOFOSsK2jdE0RZ3a0zHn4QA==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rollup/rollup-openbsd-x64@4.62.4': + resolution: {integrity: sha512-nWeM//hxv8mIo6jD7Hu4o48DVmV9pbV6gsKaWU+4NFyqHoPKwrkRiZGLKUhOBk8qNmDmpwFtPKg80Bo/Tn4xiQ==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.62.4': + resolution: {integrity: sha512-s62SQ/vgsRSvMwDkOEfTqfgASF0f26ZNaQuTA6Aok5lrikf89yI2W0gFHvZb2Jpgc6N8JnOKZgCK2iciO3CsxQ==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.62.4': + resolution: {integrity: sha512-J6wGf8TVGbXJq+HH+ttTvrcfNKPbuZecV6KT1B8I18BC5IURUh5kl4Yl5OEP5eFIUoI5BWxCsyYMhFsDx8kekw==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.62.4': + resolution: {integrity: sha512-zmfrQd/0wu6oJs8Vq8KwY/YtsKSsLtKe/HwAP4Wqy8LhWjeT55fHRAkOhYQ12wI3ayS4Tt12d5CDRD7N96SAYQ==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.62.4': + resolution: {integrity: sha512-qPzHqdj9rfUD+w79dtE07zi/kFwKyCJqplp5K5ygeLTp7jLpAoc16OAH39HSmRC9UpozaecsleI8uAdEj6v2yw==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.62.4': + resolution: {integrity: sha512-zD6NdeWEByGE9QF9vCrlJ5YQB4oq9q91kPZS37Jwj5hOkvR1lTBSpsKhKDw4IJtbQ35LsTS1HD9DZYGKIshU1Q==} + cpu: [x64] + os: [win32] + + '@shikijs/core@3.23.0': + resolution: {integrity: sha512-NSWQz0riNb67xthdm5br6lAkvpDJRTgB36fxlo37ZzM2yq0PQFFzbd8psqC2XMPgCzo1fW6cVi18+ArJ44wqgA==} + + '@shikijs/engine-javascript@3.23.0': + resolution: {integrity: sha512-aHt9eiGFobmWR5uqJUViySI1bHMqrAgamWE1TYSUoftkAeCCAiGawPMwM+VCadylQtF4V3VNOZ5LmfItH5f3yA==} + + '@shikijs/engine-oniguruma@3.23.0': + resolution: {integrity: sha512-1nWINwKXxKKLqPibT5f4pAFLej9oZzQTsby8942OTlsJzOBZ0MWKiwzMsd+jhzu8YPCHAswGnnN1YtQfirL35g==} + + '@shikijs/langs@3.23.0': + resolution: {integrity: sha512-2Ep4W3Re5aB1/62RSYQInK9mM3HsLeB91cHqznAJMuylqjzNVAVCMnNWRHFtcNHXsoNRayP9z1qj4Sq3nMqYXg==} + + '@shikijs/themes@3.23.0': + resolution: {integrity: sha512-5qySYa1ZgAT18HR/ypENL9cUSGOeI2x+4IvYJu4JgVJdizn6kG4ia5Q1jDEOi7gTbN4RbuYtmHh0W3eccOrjMA==} + + '@shikijs/types@3.23.0': + resolution: {integrity: sha512-3JZ5HXOZfYjsYSk0yPwBrkupyYSLpAE26Qc0HLghhZNGTZg/SKxXIIgoxOpmmeQP0RRSDJTk1/vPfw9tbw+jSQ==} + + '@shikijs/vscode-textmate@10.0.2': + resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==} + + '@sveltejs/acorn-typescript@1.0.13': + resolution: {integrity: sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ==} + peerDependencies: + acorn: ^8.9.0 + + '@sveltejs/vite-plugin-svelte-inspector@4.0.1': + resolution: {integrity: sha512-J/Nmb2Q2y7mck2hyCX4ckVHcR5tu2J+MtBEQqpDrrgELZ2uvraQcK/ioCV61AqkdXFgriksOKIceDcQmqnGhVw==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22} + peerDependencies: + '@sveltejs/vite-plugin-svelte': ^5.0.0 + svelte: ^5.0.0 + vite: ^6.0.0 + + '@sveltejs/vite-plugin-svelte@5.1.1': + resolution: {integrity: sha512-Y1Cs7hhTc+a5E9Va/xwKlAJoariQyHY+5zBgCZg4PFWNYQ1nMN9sjK1zhw1gK69DuqVP++sht/1GZg1aRwmAXQ==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22} + peerDependencies: + svelte: ^5.0.0 + vite: ^6.0.0 + + '@tailwindcss/node@4.3.3': + resolution: {integrity: sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==} + + '@tailwindcss/oxide-android-arm64@4.3.3': + resolution: {integrity: sha512-Y85A2gmPSkl5Ve5qR86GL4HT509cFqQh1aes9p3sSkyTPwt0Pppf3GkwGe4JPACcRYjgJIEhQgM6dBClnr0NYw==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [android] + + '@tailwindcss/oxide-darwin-arm64@4.3.3': + resolution: {integrity: sha512-BiaWatpBcERQFDlOjRDpIVXuFK5PJez5SA4JMg6VYZdBYU+qKfV/vqjcIs+IYmtitf1xYQZTwXvU/8y4lfZUGw==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [darwin] + + '@tailwindcss/oxide-darwin-x64@4.3.3': + resolution: {integrity: sha512-fAeUqfV5ndhxRwai8cXGzdLvul9utWOmeTkv69unv4ZXixjn61Z+p9lCWdwOwA3TYboG3BwdVuN/RDjhBRl0mw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [darwin] + + '@tailwindcss/oxide-freebsd-x64@4.3.3': + resolution: {integrity: sha512-iyf5bV6+wnAlflVeEy7R25dupxTNECZN5QMI0qNT6eT+EgaGdZcKhGkr5SdoaWiLJ3spLqIY9VCeSGrwmtg4kw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [freebsd] + + '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.3': + resolution: {integrity: sha512-aAYUprJAJQWWbRrPvtjdroZ56Md+JM8pMiopS6xGEwDfLhqj+2ver2p4nU4Mb3CRqcMmNBjo8KkUgcxhkzVQGQ==} + engines: {node: '>= 20'} + cpu: [arm] + os: [linux] + + '@tailwindcss/oxide-linux-arm64-gnu@4.3.3': + resolution: {integrity: sha512-nDxldcEENOxZRzC2uu9jrutZdAAQtb+8WWDCSnWL1zvBk1+FN+x6MtDViPB5AJMfttVCUhehGWus3XBPgatM/w==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@tailwindcss/oxide-linux-arm64-musl@4.3.3': + resolution: {integrity: sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@tailwindcss/oxide-linux-x64-gnu@4.3.3': + resolution: {integrity: sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==} + engines: {node: '>= 20'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@tailwindcss/oxide-linux-x64-musl@4.3.3': + resolution: {integrity: sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==} + engines: {node: '>= 20'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@tailwindcss/oxide-wasm32-wasi@4.3.3': + resolution: {integrity: sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + bundledDependencies: + - '@napi-rs/wasm-runtime' + - '@emnapi/core' + - '@emnapi/runtime' + - '@tybys/wasm-util' + - '@emnapi/wasi-threads' + - tslib + + '@tailwindcss/oxide-win32-arm64-msvc@4.3.3': + resolution: {integrity: sha512-3rc292Ca2ceK6Ulcc/bAVnTs/3nDtoPhyEKlgPv+yQJQi/JS/AMJlqzxvlDacL1nekbrcf6bTqp/jV4qgnPxNQ==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [win32] + + '@tailwindcss/oxide-win32-x64-msvc@4.3.3': + resolution: {integrity: sha512-yJ0pwIVc/nYeGoV02WtsN8KYyLQv7kyI2wDnkezyJlGGjkd4QLwDGAwl47YpPJeuI0M0ObaXGSPjvWDPeTPggw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [win32] + + '@tailwindcss/oxide@4.3.3': + resolution: {integrity: sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==} + engines: {node: '>= 20'} + + '@tailwindcss/vite@4.3.3': + resolution: {integrity: sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==} + peerDependencies: + vite: ^5.2.0 || ^6 || ^7 || ^8 + + '@types/debug@4.1.13': + resolution: {integrity: sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==} + + '@types/estree-jsx@1.0.5': + resolution: {integrity: sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/hast@3.0.5': + resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} + + '@types/js-yaml@4.0.9': + resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==} + + '@types/mdast@4.0.4': + resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} + + '@types/mdx@2.0.14': + resolution: {integrity: sha512-T48PeuJtvLosNTPVhfnIp3i/n3a4g4Bad7YCq5k64D4u7NwDrAotikQ+5+sjtUvBmxCMlbo3dVL+C2dP0rWHzg==} + + '@types/ms@2.1.0': + resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} + + '@types/nlcst@2.0.3': + resolution: {integrity: sha512-vSYNSDe6Ix3q+6Z7ri9lyWqgGhJTmzRjZRqyq15N0Z/1/UnVsno9G/N40NBijoYx2seFDIl0+B2mgAb9mezUCA==} + + '@types/node@24.13.3': + resolution: {integrity: sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==} + + '@types/picomatch@3.0.2': + resolution: {integrity: sha512-n0i8TD3UDB7paoMMxA3Y65vUncFJXjcUf7lQY7YyKGl6031FNjfsLs6pdLFCy2GNFxItPJG8GvvpbZc2skH7WA==} + + '@types/sax@1.2.7': + resolution: {integrity: sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==} + + '@types/unist@2.0.11': + resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} + + '@types/unist@3.0.3': + resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} + + '@ungap/structured-clone@1.3.3': + resolution: {integrity: sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg==} + + acorn-jsx@5.3.2: + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + + acorn@8.18.0: + resolution: {integrity: sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==} + engines: {node: '>=0.4.0'} + hasBin: true + + ansi-align@3.0.1: + resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} + + ansi-escapes@7.3.0: + resolution: {integrity: sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==} + engines: {node: '>=18'} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-regex@6.2.2: + resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} + engines: {node: '>=12'} + + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + + anymatch@3.1.3: + resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} + engines: {node: '>= 8'} + + arg@5.0.2: + resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} + + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + + aria-query@5.3.1: + resolution: {integrity: sha512-Z/ZeOgVl7bcSYZ/u/rh0fOpvEpq//LZmdbkXyc7syVzjPAhfOa9ebsdTSjEBDU4vs5nC98Kfduj1uFo0qyET3g==} + engines: {node: '>= 0.4'} + + aria-query@5.3.2: + resolution: {integrity: sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==} + engines: {node: '>= 0.4'} + + array-iterate@2.0.1: + resolution: {integrity: sha512-I1jXZMjAgCMmxT4qxXfPXa6SthSoE8h6gkSI9BGGNv8mP8G/v0blc+qFnZu6K42vTOiuME596QaLO0TP3Lk0xg==} + + astring@1.9.0: + resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} + hasBin: true + + astro-expressive-code@0.41.7: + resolution: {integrity: sha512-hUpogGc6DdAd+I7pPXsctyYPRBJDK7Q7d06s4cyP0Vz3OcbziP3FNzN0jZci1BpCvLn9675DvS7B9ctKKX64JQ==} + peerDependencies: + astro: ^4.0.0-beta || ^5.0.0-beta || ^3.3.0 || ^6.0.0-beta + + astro@5.18.2: + resolution: {integrity: sha512-TnFwLnAXty5MXKPDGuKXqK4AMBXG+FH6RUdK7Oyc3gyfNoFIthT+4eRbzOK43bdRlLaZuxgciDSjgtggZ3OtGQ==} + engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0'} + hasBin: true + + axobject-query@4.1.0: + resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} + engines: {node: '>= 0.4'} + + bail@2.0.2: + resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} + + base-64@1.0.0: + resolution: {integrity: sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==} + + bcp-47-match@2.0.3: + resolution: {integrity: sha512-JtTezzbAibu8G0R9op9zb3vcWZd9JF6M0xOYGPn0fNCd7wOpRB1mU2mH9T8gaBGbAAyIIVgB2G7xG0GP98zMAQ==} + + bcp-47@2.1.1: + resolution: {integrity: sha512-KLw+H/gd2p4zly1X7Yh/qziuyae5/w/QFnvTng9eZL5fvszL7Whl3MBoWF8yxL7ksUjBfOD+OxkytiqbBpG+Fw==} + + boolbase@1.0.0: + resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + + boxen@8.0.1: + resolution: {integrity: sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw==} + engines: {node: '>=18'} + + camelcase@8.0.0: + resolution: {integrity: sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==} + engines: {node: '>=16'} + + ccount@2.0.1: + resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + + chalk@5.6.2: + resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + + character-entities-html4@2.1.0: + resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} + + character-entities-legacy@3.0.0: + resolution: {integrity: sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==} + + character-entities@2.0.2: + resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} + + character-reference-invalid@2.0.1: + resolution: {integrity: sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==} + + chokidar@5.0.0: + resolution: {integrity: sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==} + engines: {node: '>= 20.19.0'} + + ci-info@4.4.0: + resolution: {integrity: sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==} + engines: {node: '>=8'} + + cli-boxes@3.0.0: + resolution: {integrity: sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==} + engines: {node: '>=10'} + + clsx@2.1.1: + resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} + engines: {node: '>=6'} + + collapse-white-space@2.1.0: + resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==} + + comma-separated-tokens@2.0.3: + resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + + commander@11.1.0: + resolution: {integrity: sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==} + engines: {node: '>=16'} + + common-ancestor-path@1.0.1: + resolution: {integrity: sha512-L3sHRo1pXXEqX8VU28kfgUY+YGsk09hPqZiZmLacNib6XNTCM8ubYeT7ryXQw8asB1sKgcU5lkB7ONug08aB8w==} + + confbox@0.1.8: + resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} + + confbox@0.3.1: + resolution: {integrity: sha512-cKUSoKa8YxFZZSmraVi7onONx3amu77ngK3kGpsYHDH7drPwCRkQE1RYMPlLRrMtnciRj274XNRxcHxnKmDSnA==} + + cookie-es@1.2.3: + resolution: {integrity: sha512-lXVyvUvrNXblMqzIRrxHb57UUVmqsSWlxqt3XIjCkUP0wDAf6uicO6KMbEgYrMNtEvWgWHwe42CKxPu9MYAnWw==} + + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + + crossws@0.3.5: + resolution: {integrity: sha512-ojKiDvcmByhwa8YYqbQI/hg7MEU0NC03+pSdEq4ZUnZR9xXpwk7E43SMNGkn+JxJGPFtNvQ48+vV2p+P1ml5PA==} + + css-select@5.2.2: + resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==} + + css-selector-parser@3.3.0: + resolution: {integrity: sha512-Y2asgMGFqJKF4fq4xHDSlFYIkeVfRsm69lQC1q9kbEsH5XtnINTMrweLkjYMeaUgiXBy/uvKeO/a1JHTNnmB2g==} + + css-tree@2.2.1: + resolution: {integrity: sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} + + css-tree@3.2.1: + resolution: {integrity: sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0} + + css-what@6.2.2: + resolution: {integrity: sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==} + engines: {node: '>= 6'} + + cssesc@3.0.0: + resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} + engines: {node: '>=4'} + hasBin: true + + csso@5.0.5: + resolution: {integrity: sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + decode-named-character-reference@1.3.0: + resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} + + dedent-js@1.0.1: + resolution: {integrity: sha512-OUepMozQULMLUmhxS95Vudo0jb0UchLimi3+pQ2plj61Fcy8axbP9hbiD4Sz6DPqn6XG3kfmziVfQ1rSys5AJQ==} + + deepmerge@4.3.1: + resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} + engines: {node: '>=0.10.0'} + + defu@6.1.7: + resolution: {integrity: sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==} + + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + + destr@2.0.5: + resolution: {integrity: sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + deterministic-object-hash@2.0.2: + resolution: {integrity: sha512-KxektNH63SrbfUyDiwXqRb1rLwKt33AmMv+5Nhsw1kqZ13SJBRTgZHtGbE+hH3a1mVW1cz+4pqSWVPAtLVXTzQ==} + engines: {node: '>=18'} + + devalue@5.9.0: + resolution: {integrity: sha512-RWrqdArjvPbsATEhOPUo6Wndc/iWnkWKlhIrdlF3zMMYo/c3CVtoaVAyLtWxz5h8nSlkHzxnzV2uLydPXmtF+A==} + + devalue@5.9.4: + resolution: {integrity: sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==} + + devlop@1.1.0: + resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} + + diff@8.0.4: + resolution: {integrity: sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==} + engines: {node: '>=0.3.1'} + + direction@2.0.1: + resolution: {integrity: sha512-9S6m9Sukh1cZNknO1CWAr2QAWsbKLafQiyM5gZ7VgXHeuaoUwffKN4q6NC4A/Mf9iiPlOXQEKW/Mv/mh9/3YFA==} + hasBin: true + + dlv@1.1.3: + resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==} + + dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + + domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + + domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + + domutils@3.2.2: + resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==} + + dset@3.1.4: + resolution: {integrity: sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==} + engines: {node: '>=4'} + + emoji-regex@10.6.0: + resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + enhanced-resolve@5.26.0: + resolution: {integrity: sha512-9vhedylFonb2YGogzUKX6+Ja72gOJbN1QHAqdrvqLwhdl/QWbKopzoUC9EbQNVsAns/bx/4uyqalrQAoy1IByw==} + engines: {node: '>=10.13.0'} + + entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + + entities@6.0.1: + resolution: {integrity: sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==} + engines: {node: '>=0.12'} + + environment@1.1.0: + resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==} + engines: {node: '>=18'} + + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + + esast-util-from-estree@2.0.0: + resolution: {integrity: sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==} + + esast-util-from-js@2.0.1: + resolution: {integrity: sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw==} + + esbuild@0.25.12: + resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} + engines: {node: '>=18'} + hasBin: true + + esbuild@0.27.7: + resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==} + engines: {node: '>=18'} + hasBin: true + + escape-string-regexp@5.0.0: + resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} + engines: {node: '>=12'} + + esm-env@1.2.2: + resolution: {integrity: sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==} + + esrap@2.4.0: + resolution: {integrity: sha512-CQPcneEHw/czj32uwQC52nYnax8H4SXt25FCM+LIwGZ49cMIxpOj3bP8wmzcI+Z0HIy5KkClaPhRLfCySzwHPA==} + peerDependencies: + '@typescript-eslint/types': ^8.2.0 + peerDependenciesMeta: + '@typescript-eslint/types': + optional: true + + estree-util-attach-comments@3.0.0: + resolution: {integrity: sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw==} + + estree-util-build-jsx@3.0.1: + resolution: {integrity: sha512-8U5eiL6BTrPxp/CHbs2yMgP8ftMhR5ww1eIKoWRMlqvltHF8fZn5LRDvTKuxD3DUn+shRbLGqXemcP51oFCsGQ==} + + estree-util-is-identifier-name@3.0.0: + resolution: {integrity: sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==} + + estree-util-scope@1.0.0: + resolution: {integrity: sha512-2CAASclonf+JFWBNJPndcOpA8EMJwa0Q8LUFJEKqXLW6+qBvbFZuF5gItbQOs/umBUkjviCSDCbBwU2cXbmrhQ==} + + estree-util-to-js@2.0.0: + resolution: {integrity: sha512-WDF+xj5rRWmD5tj6bIqRi6CkLIXbbNQUcxQHzGysQzvHmdYG2G7p/Tf0J0gpxGgkeMZNTIjT/AoSvC9Xehcgdg==} + + estree-util-visit@2.0.0: + resolution: {integrity: sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==} + + estree-walker@2.0.2: + resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==} + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + eventemitter3@5.0.4: + resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + + expressive-code@0.41.7: + resolution: {integrity: sha512-2wZjC8OQ3TaVEMcBtYY4Va3lo6J+Ai9jf3d4dbhURMJcU4Pbqe6EcHe424MIZI0VHUA1bR6xdpoHYi3yxokWqA==} + + exsolve@1.1.3: + resolution: {integrity: sha512-kuzL9zherLQoqDeQaS55AiiIg1kb4KYXaSusTtgZTPlvKP2gGCI82LNmBNh5RjDVUXIcmgx+/Mx7vPwRzBbhJA==} + + extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + flattie@1.1.1: + resolution: {integrity: sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ==} + engines: {node: '>=8'} + + fontace@0.4.1: + resolution: {integrity: sha512-lDMvbAzSnHmbYMTEld5qdtvNH2/pWpICOqpean9IgC7vUbUJc3k+k5Dokp85CegamqQpFbXf0rAVkbzpyTA8aw==} + + fontkitten@1.0.3: + resolution: {integrity: sha512-Wp1zXWPVUPBmfoa3Cqc9ctaKuzKAV6uLstRqlR56kSjplf5uAce+qeyYym7F+PHbGTk+tCEdkCW6RD7DX/gBZw==} + engines: {node: '>=20'} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + get-east-asian-width@1.6.0: + resolution: {integrity: sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==} + engines: {node: '>=18'} + + github-slugger@2.0.0: + resolution: {integrity: sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==} + + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + + h3@1.15.11: + resolution: {integrity: sha512-L3THSe2MPeBwgIZVSH5zLdBBU90TOxarvhK9d04IDY2AmVS8j2Jz2LIWtwsGOU3lu2I5jCN7FNvVfY2+XyF+mg==} + + has-flag@5.0.1: + resolution: {integrity: sha512-CsNUt5x9LUdx6hnk/E2SZLsDyvfqANZSUq4+D3D8RzDJ2M+HDTIkF60ibS1vHaK55vzgiZw1bEPFG9yH7l33wA==} + engines: {node: '>=12'} + + hast-util-embedded@3.0.0: + resolution: {integrity: sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==} + + hast-util-format@1.1.0: + resolution: {integrity: sha512-yY1UDz6bC9rDvCWHpx12aIBGRG7krurX0p0Fm6pT547LwDIZZiNr8a+IHDogorAdreULSEzP82Nlv5SZkHZcjA==} + + hast-util-from-html@2.0.3: + resolution: {integrity: sha512-CUSRHXyKjzHov8yKsQjGOElXy/3EKpyX56ELnkHH34vDVw1N1XSQ1ZcAvTyAPtGqLTuKP/uxM+aLkSPqF/EtMw==} + + hast-util-from-parse5@8.0.3: + resolution: {integrity: sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==} + + hast-util-has-property@3.0.0: + resolution: {integrity: sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==} + + hast-util-is-body-ok-link@3.0.1: + resolution: {integrity: sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==} + + hast-util-is-element@3.0.0: + resolution: {integrity: sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==} + + hast-util-minify-whitespace@1.0.1: + resolution: {integrity: sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==} + + hast-util-parse-selector@4.0.0: + resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} + + hast-util-phrasing@3.0.1: + resolution: {integrity: sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==} + + hast-util-raw@9.1.0: + resolution: {integrity: sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==} + + hast-util-select@6.0.4: + resolution: {integrity: sha512-RqGS1ZgI0MwxLaKLDxjprynNzINEkRHY2i8ln4DDjgv9ZhcYVIHN9rlpiYsqtFwrgpYU361SyWDQcGNIBVu3lw==} + + hast-util-to-estree@3.1.3: + resolution: {integrity: sha512-48+B/rJWAp0jamNbAAf9M7Uf//UVqAoMmgXhBdxTDJLGKY+LRnZ99qcG+Qjl5HfMpYNzS5v4EAwVEF34LeAj7w==} + + hast-util-to-html@9.0.5: + resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==} + + hast-util-to-jsx-runtime@2.3.6: + resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} + + hast-util-to-mdast@10.1.2: + resolution: {integrity: sha512-FiCRI7NmOvM4y+f5w32jPRzcxDIz+PUqDwEqn1A+1q2cdp3B8Gx7aVrXORdOKjMNDQsD1ogOr896+0jJHW1EFQ==} + + hast-util-to-parse5@8.0.1: + resolution: {integrity: sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==} + + hast-util-to-string@3.0.1: + resolution: {integrity: sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A==} + + hast-util-to-text@4.0.2: + resolution: {integrity: sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==} + + hast-util-whitespace@3.0.0: + resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==} + + hastscript@9.0.1: + resolution: {integrity: sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==} + + html-escaper@3.0.3: + resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==} + + html-void-elements@3.0.0: + resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + + html-whitespace-sensitive-tag-names@3.0.1: + resolution: {integrity: sha512-q+310vW8zmymYHALr1da4HyXUQ0zgiIwIicEfotYPWGN0OJVEN/58IJ3A4GBYcEq3LGAZqKb+ugvP0GNB9CEAA==} + + http-cache-semantics@4.2.0: + resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} + + i18next@23.16.8: + resolution: {integrity: sha512-06r/TitrM88Mg5FdUXAKL96dJMzgqLE5dv3ryBAra4KCwD9mJ4ndOTS95ZuymIGoE+2hzfdaMak2X11/es7ZWg==} + + import-meta-resolve@4.2.0: + resolution: {integrity: sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==} + + inline-style-parser@0.2.7: + resolution: {integrity: sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==} + + iron-webcrypto@1.2.1: + resolution: {integrity: sha512-feOM6FaSr6rEABp/eDfVseKyTMDt+KGpeB35SkVn9Tyn0CqvVsY3EwI0v5i8nMHyJnzCIQf7nsy3p41TPkJZhg==} + + is-absolute-url@4.0.1: + resolution: {integrity: sha512-/51/TKE88Lmm7Gc4/8btclNXWS+g50wXhYJq8HWIBAGUBnoAdRu1aXeh364t/O7wXDAcTJDP8PNuNKWUDWie+A==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + is-alphabetical@2.0.1: + resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} + + is-alphanumerical@2.0.1: + resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==} + + is-decimal@2.0.1: + resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} + + is-docker@3.0.0: + resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + hasBin: true + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + is-hexadecimal@2.0.1: + resolution: {integrity: sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==} + + is-inside-container@1.0.0: + resolution: {integrity: sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==} + engines: {node: '>=14.16'} + hasBin: true + + is-plain-obj@4.1.0: + resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} + engines: {node: '>=12'} + + is-reference@3.0.3: + resolution: {integrity: sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==} + + is-wsl@3.1.1: + resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} + engines: {node: '>=16'} + + jiti@2.7.0: + resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} + hasBin: true + + js-yaml@4.3.1: + resolution: {integrity: sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==} + hasBin: true + + kleur@3.0.3: + resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} + engines: {node: '>=6'} + + kleur@4.1.5: + resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} + engines: {node: '>=6'} + + klona@2.0.6: + resolution: {integrity: sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==} + engines: {node: '>= 8'} + + lightningcss-android-arm64@1.32.0: + resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.32.0: + resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.32.0: + resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.32.0: + resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.32.0: + resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.32.0: + resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + lightningcss-linux-arm64-musl@1.32.0: + resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + lightningcss-linux-x64-gnu@1.32.0: + resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + lightningcss-linux-x64-musl@1.32.0: + resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + lightningcss-win32-arm64-msvc@1.32.0: + resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.32.0: + resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.32.0: + resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} + engines: {node: '>= 12.0.0'} + + local-pkg@1.2.1: + resolution: {integrity: sha512-++gUqRDEvcnN6Zhqrr+y/CkVEHhlrR96vZn3nZZPYzMcBUyBtTKzB9NadClFIsIVSsu+3i9tfk/erqy9kAmt7Q==} + engines: {node: '>=14'} + + locate-character@3.0.0: + resolution: {integrity: sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==} + + longest-streak@3.1.0: + resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} + + lru-cache@11.5.2: + resolution: {integrity: sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==} + engines: {node: 20 || >=22} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + magicast@0.5.4: + resolution: {integrity: sha512-llBEhWm1SacoRwgHUoQJYtwp4PBLF4faQi5TCpIGyGs9n4y5+juI0tDgyKIfpqxckRHaHzouUEph3THklWh03w==} + + markdown-extensions@2.0.0: + resolution: {integrity: sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==} + engines: {node: '>=16'} + + markdown-table@3.0.4: + resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==} + + mdast-util-definitions@6.0.0: + resolution: {integrity: sha512-scTllyX6pnYNZH/AIp/0ePz6s4cZtARxImwoPJ7kS42n+MnVsI4XbnG6d4ibehRIldYMWM2LD7ImQblVhUejVQ==} + + mdast-util-directive@3.1.0: + resolution: {integrity: sha512-I3fNFt+DHmpWCYAT7quoM6lHf9wuqtI+oCOfvILnoicNIqjh5E3dEJWiXuYME2gNe8vl1iMQwyUHa7bgFmak6Q==} + + mdast-util-find-and-replace@3.0.2: + resolution: {integrity: sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==} + + mdast-util-from-markdown@2.0.3: + resolution: {integrity: sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==} + + mdast-util-gfm-autolink-literal@2.0.1: + resolution: {integrity: sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==} + + mdast-util-gfm-footnote@2.1.0: + resolution: {integrity: sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==} + + mdast-util-gfm-strikethrough@2.0.0: + resolution: {integrity: sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==} + + mdast-util-gfm-table@2.0.0: + resolution: {integrity: sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==} + + mdast-util-gfm-task-list-item@2.0.0: + resolution: {integrity: sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==} + + mdast-util-gfm@3.1.0: + resolution: {integrity: sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==} + + mdast-util-mdx-expression@2.0.1: + resolution: {integrity: sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==} + + mdast-util-mdx-jsx@3.2.0: + resolution: {integrity: sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q==} + + mdast-util-mdx@3.0.0: + resolution: {integrity: sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w==} + + mdast-util-mdxjs-esm@2.0.1: + resolution: {integrity: sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==} + + mdast-util-phrasing@4.1.0: + resolution: {integrity: sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==} + + mdast-util-to-hast@13.2.1: + resolution: {integrity: sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==} + + mdast-util-to-markdown@2.1.2: + resolution: {integrity: sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==} + + mdast-util-to-string@4.0.0: + resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==} + + mdn-data@2.0.28: + resolution: {integrity: sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==} + + mdn-data@2.27.1: + resolution: {integrity: sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==} + + micromark-core-commonmark@2.0.3: + resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==} + + micromark-extension-directive@3.0.2: + resolution: {integrity: sha512-wjcXHgk+PPdmvR58Le9d7zQYWy+vKEU9Se44p2CrCDPiLr2FMyiT4Fyb5UFKFC66wGB3kPlgD7q3TnoqPS7SZA==} + + micromark-extension-gfm-autolink-literal@2.1.0: + resolution: {integrity: sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==} + + micromark-extension-gfm-footnote@2.1.0: + resolution: {integrity: sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==} + + micromark-extension-gfm-strikethrough@2.1.0: + resolution: {integrity: sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==} + + micromark-extension-gfm-table@2.1.1: + resolution: {integrity: sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==} + + micromark-extension-gfm-tagfilter@2.0.0: + resolution: {integrity: sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==} + + micromark-extension-gfm-task-list-item@2.1.0: + resolution: {integrity: sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==} + + micromark-extension-gfm@3.0.0: + resolution: {integrity: sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==} + + micromark-extension-mdx-expression@3.0.1: + resolution: {integrity: sha512-dD/ADLJ1AeMvSAKBwO22zG22N4ybhe7kFIZ3LsDI0GlsNr2A3KYxb0LdC1u5rj4Nw+CHKY0RVdnHX8vj8ejm4Q==} + + micromark-extension-mdx-jsx@3.0.2: + resolution: {integrity: sha512-e5+q1DjMh62LZAJOnDraSSbDMvGJ8x3cbjygy2qFEi7HCeUT4BDKCvMozPozcD6WmOt6sVvYDNBKhFSz3kjOVQ==} + + micromark-extension-mdx-md@2.0.0: + resolution: {integrity: sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ==} + + micromark-extension-mdxjs-esm@3.0.0: + resolution: {integrity: sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A==} + + micromark-extension-mdxjs@3.0.0: + resolution: {integrity: sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ==} + + micromark-factory-destination@2.0.1: + resolution: {integrity: sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==} + + micromark-factory-label@2.0.1: + resolution: {integrity: sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==} + + micromark-factory-mdx-expression@2.0.3: + resolution: {integrity: sha512-kQnEtA3vzucU2BkrIa8/VaSAsP+EJ3CKOvhMuJgOEGg9KDC6OAY6nSnNDVRiVNRqj7Y4SlSzcStaH/5jge8JdQ==} + + micromark-factory-space@2.0.1: + resolution: {integrity: sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==} + + micromark-factory-title@2.0.1: + resolution: {integrity: sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==} + + micromark-factory-whitespace@2.0.1: + resolution: {integrity: sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==} + + micromark-util-character@2.1.1: + resolution: {integrity: sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==} + + micromark-util-chunked@2.0.1: + resolution: {integrity: sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==} + + micromark-util-classify-character@2.0.1: + resolution: {integrity: sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==} + + micromark-util-combine-extensions@2.0.1: + resolution: {integrity: sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==} + + micromark-util-decode-numeric-character-reference@2.0.2: + resolution: {integrity: sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==} + + micromark-util-decode-string@2.0.1: + resolution: {integrity: sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==} + + micromark-util-encode@2.0.1: + resolution: {integrity: sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==} + + micromark-util-events-to-acorn@2.0.3: + resolution: {integrity: sha512-jmsiEIiZ1n7X1Rr5k8wVExBQCg5jy4UXVADItHmNk1zkwEVhBuIUKRu3fqv+hs4nxLISi2DQGlqIOGiFxgbfHg==} + + micromark-util-html-tag-name@2.0.1: + resolution: {integrity: sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==} + + micromark-util-normalize-identifier@2.0.1: + resolution: {integrity: sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==} + + micromark-util-resolve-all@2.0.1: + resolution: {integrity: sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==} + + micromark-util-sanitize-uri@2.0.1: + resolution: {integrity: sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==} + + micromark-util-subtokenize@2.1.0: + resolution: {integrity: sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==} + + micromark-util-symbol@2.0.1: + resolution: {integrity: sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==} + + micromark-util-types@2.0.2: + resolution: {integrity: sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==} + + micromark@4.0.2: + resolution: {integrity: sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==} + + mlly@1.8.2: + resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + + mrmime@2.0.1: + resolution: {integrity: sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==} + engines: {node: '>=10'} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.17: + resolution: {integrity: sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + neotraverse@0.6.18: + resolution: {integrity: sha512-Z4SmBUweYa09+o6pG+eASabEpP6QkQ70yHj351pQoEXIs8uHbaU2DWVmzBANKgflPa47A50PtB2+NgRpQvr7vA==} + engines: {node: '>= 10'} + + nlcst-to-string@4.0.0: + resolution: {integrity: sha512-YKLBCcUYKAg0FNlOBT6aI91qFmSiFKiluk655WzPF+DDMA02qIyy8uiRqI8QXtcFpEvll12LpL5MXqEmAZ+dcA==} + + node-fetch-native@1.6.7: + resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==} + + node-mock-http@1.0.5: + resolution: {integrity: sha512-KQyt/wLjG3TAc7DOUhpqWzgd4ERxR80JOlTK5VE5R1S12IaPVN5qkj4klBce9HPG1Njuup4Sb5bljaT34lIyjw==} + + normalize-path@3.0.0: + resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} + engines: {node: '>=0.10.0'} + + nth-check@2.1.1: + resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + + obug@2.2.1: + resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} + engines: {node: '>=12.20.0'} + + ofetch@1.5.1: + resolution: {integrity: sha512-2W4oUZlVaqAPAil6FUg/difl6YhqhUR7x2eZY4bQCko22UXg3hptq9KLQdqFClV+Wu85UX7hNtdGTngi/1BxcA==} + + ohash@2.0.11: + resolution: {integrity: sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==} + + oniguruma-parser@0.12.2: + resolution: {integrity: sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==} + + oniguruma-to-es@4.3.6: + resolution: {integrity: sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==} + + p-limit@6.2.0: + resolution: {integrity: sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==} + engines: {node: '>=18'} + + p-queue@8.1.1: + resolution: {integrity: sha512-aNZ+VfjobsWryoiPnEApGGmf5WmNsCo9xu8dfaYamG5qaLP7ClhLN6NgsFe6SwJ2UbLEBK5dv9x8Mn5+RVhMWQ==} + engines: {node: '>=18'} + + p-timeout@6.1.4: + resolution: {integrity: sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg==} + engines: {node: '>=14.16'} + + package-manager-detector@1.8.0: + resolution: {integrity: sha512-yQA4H19AmPEoMUeavPMDIe1higySl/gH/yaQrkT/s07Qp+7pp2hYz30N3z2l5BkjVkF9Ow6o0wjJamm2y7Sn0A==} + + pagefind@1.5.2: + resolution: {integrity: sha512-XTUaK0hXMCu2jszWE584JGQT7y284TmMV9l/HX3rnG5uo3rHI/uHU56XTyyyPFjeWEBxECbAi0CaFDJOONtG0Q==} + hasBin: true + + parse-entities@4.0.2: + resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} + + parse-latin@7.0.0: + resolution: {integrity: sha512-mhHgobPPua5kZ98EF4HWiH167JWBfl4pvAIXXdbaVohtK7a6YBOy56kvhCqduqyo/f3yrHFWmqmiMg/BkBkYYQ==} + + parse5@7.3.0: + resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + piccolore@0.1.3: + resolution: {integrity: sha512-o8bTeDWjE086iwKrROaDf31K0qC/BENdm15/uH9usSC/uZjJOKb2YGiVHfLY4GhwsERiPI1jmwI2XrA7ACOxVw==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@2.3.2: + resolution: {integrity: sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==} + engines: {node: '>=8.6'} + + picomatch@4.0.5: + resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} + engines: {node: '>=12'} + + picomatch@4.0.7: + resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} + engines: {node: '>=12'} + + pkg-types@1.3.1: + resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + + pkg-types@2.3.3: + resolution: {integrity: sha512-j/lCFdcppV0JxWpCEITdbDltBxPP6cHT+yNJ6Go2OgoSA9518X847X9z0p6LtA4Nc16+eQzCZjRrWanTGvHJ5w==} + + postcss-nested@6.2.0: + resolution: {integrity: sha512-HQbt28KulC5AJzG+cZtj9kvKB93CFCdLvog1WFLf1D+xmMvPGlBstkpTEZfK5+AN9hfJocyBFCNiqyS48bpgzQ==} + engines: {node: '>=12.0'} + peerDependencies: + postcss: ^8.2.14 + + postcss-selector-parser@6.1.4: + resolution: {integrity: sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==} + engines: {node: '>=4'} + + postcss@8.5.25: + resolution: {integrity: sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==} + engines: {node: ^10 || ^12 || >=14} + + prismjs@1.30.0: + resolution: {integrity: sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==} + engines: {node: '>=6'} + + prompts@2.4.2: + resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==} + engines: {node: '>= 6'} + + property-information@7.2.0: + resolution: {integrity: sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==} + + quansync@0.2.11: + resolution: {integrity: sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==} + + radix3@1.1.2: + resolution: {integrity: sha512-b484I/7b8rDEdSDKckSSBA8knMpcdsXudlE/LNL639wFoHKwLbEkQFZHWEYwDC0wa0FKUcCY+GAF73Z7wxNVFA==} + + readdirp@5.0.0: + resolution: {integrity: sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==} + engines: {node: '>= 20.19.0'} + + recma-build-jsx@1.0.0: + resolution: {integrity: sha512-8GtdyqaBcDfva+GUKDr3nev3VpKAhup1+RvkMvUxURHpW7QyIvk9F5wz7Vzo06CEMSilw6uArgRqhpiUcWp8ew==} + + recma-jsx@1.0.1: + resolution: {integrity: sha512-huSIy7VU2Z5OLv6oFLosQGGDqPqdO1iq6bWNAdhzMxSJP7RAso4fCZ1cKu8j9YHCZf3TPrq4dw3okhrylgcd7w==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + + recma-parse@1.0.0: + resolution: {integrity: sha512-OYLsIGBB5Y5wjnSnQW6t3Xg7q3fQ7FWbw/vcXtORTnyaSFscOtABg+7Pnz6YZ6c27fG1/aN8CjfwoUEUIdwqWQ==} + + recma-stringify@1.0.0: + resolution: {integrity: sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g==} + + regex-recursion@6.0.2: + resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==} + + regex-utilities@2.3.0: + resolution: {integrity: sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==} + + regex@6.1.0: + resolution: {integrity: sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==} + + rehype-expressive-code@0.41.7: + resolution: {integrity: sha512-25f8ZMSF1d9CMscX7Cft0TSQIqdwjce2gDOvQ+d/w0FovsMwrSt3ODP4P3Z7wO1jsIJ4eYyaDRnIR/27bd/EMQ==} + + rehype-format@5.0.1: + resolution: {integrity: sha512-zvmVru9uB0josBVpr946OR8ui7nJEdzZobwLOOqHb/OOD88W0Vk2SqLwoVOj0fM6IPCCO6TaV9CvQvJMWwukFQ==} + + rehype-minify-whitespace@6.0.2: + resolution: {integrity: sha512-Zk0pyQ06A3Lyxhe9vGtOtzz3Z0+qZ5+7icZ/PL/2x1SHPbKao5oB/g/rlc6BCTajqBb33JcOe71Ye1oFsuYbnw==} + + rehype-parse@9.0.1: + resolution: {integrity: sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==} + + rehype-raw@7.0.0: + resolution: {integrity: sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==} + + rehype-recma@1.0.0: + resolution: {integrity: sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw==} + + rehype-remark@10.0.1: + resolution: {integrity: sha512-EmDndlb5NVwXGfUa4c9GPK+lXeItTilLhE6ADSaQuHr4JUlKw9MidzGzx4HpqZrNCt6vnHmEifXQiiA+CEnjYQ==} + + rehype-stringify@10.0.1: + resolution: {integrity: sha512-k9ecfXHmIPuFVI61B9DeLPN0qFHfawM6RsuX48hoqlaKSF61RskNjSm1lI8PhBEM0MRdLxVVm4WmTqJQccH9mA==} + + rehype@13.0.2: + resolution: {integrity: sha512-j31mdaRFrwFRUIlxGeuPXXKWQxet52RBQRvCmzl5eCefn/KGbomK5GMHNMsOJf55fgo3qw5tST5neDuarDYR2A==} + + remark-directive@3.0.1: + resolution: {integrity: sha512-gwglrEQEZcZYgVyG1tQuA+h58EZfq5CSULw7J90AFuCTyib1thgHPoqQ+h9iFvU6R+vnZ5oNFQR5QKgGpk741A==} + + remark-gfm@4.0.1: + resolution: {integrity: sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==} + + remark-mdx@3.1.1: + resolution: {integrity: sha512-Pjj2IYlUY3+D8x00UJsIOg5BEvfMyeI+2uLPn9VO9Wg4MEtN/VTIq2NEJQfde9PnX15KgtHyl9S0BcTnWrIuWg==} + + remark-parse@11.0.0: + resolution: {integrity: sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==} + + remark-rehype@11.1.2: + resolution: {integrity: sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw==} + + remark-smartypants@3.0.3: + resolution: {integrity: sha512-gCaK+ndZ0hYezlqFegHFCVh2CQemsi0Npdh1qVM9bxlUFknjkbP6VmojWhddOCrbK0PbbacmYLWfTULRiT1eWA==} + engines: {node: '>=16.0.0'} + + remark-stringify@11.0.0: + resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==} + + retext-latin@4.0.0: + resolution: {integrity: sha512-hv9woG7Fy0M9IlRQloq/N6atV82NxLGveq+3H2WOi79dtIYWN8OaxogDm77f8YnVXJL2VD3bbqowu5E3EMhBYA==} + + retext-smartypants@6.2.0: + resolution: {integrity: sha512-kk0jOU7+zGv//kfjXEBjdIryL1Acl4i9XNkHxtM7Tm5lFiCog576fjNC9hjoR7LTKQ0DsPWy09JummSsH1uqfQ==} + + retext-stringify@4.0.0: + resolution: {integrity: sha512-rtfN/0o8kL1e+78+uxPTqu1Klt0yPzKuQ2BfWwwfgIUSayyzxpM1PJzkKt4V8803uB9qSy32MvI7Xep9khTpiA==} + + retext@9.0.0: + resolution: {integrity: sha512-sbMDcpHCNjvlheSgMfEcVrZko3cDzdbe1x/e7G66dFp0Ff7Mldvi2uv6JkJQzdRcvLYE8CA8Oe8siQx8ZOgTcA==} + + rollup@4.62.4: + resolution: {integrity: sha512-RXOqwaPsBGjMNMa4sQjDjHieHEZDFoj/Rdr46l2MU5DfEs16wHJPC2RPTPHWhNl+M3aI472LLqFkFKut4SblOg==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + sax@1.6.1: + resolution: {integrity: sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q==} + engines: {node: '>=11.0.0'} + + scule@1.3.0: + resolution: {integrity: sha512-6FtHJEvt+pVMIB9IBY+IcCJ6Z5f1iQnytgyfKMhDKgmzYG+TeH/wx1y3l27rshSbLiSanrR9ffZDrEsmjlQF2g==} + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + sharp@0.34.5: + resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + + shiki@3.23.0: + resolution: {integrity: sha512-55Dj73uq9ZXL5zyeRPzHQsK7Nbyt6Y10k5s7OjuFZGMhpp4r/rsLBH0o/0fstIzX1Lep9VxefWljK/SKCzygIA==} + + sisteransi@1.0.5: + resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} + + sitemap@9.0.1: + resolution: {integrity: sha512-S6hzjGJSG3d6if0YoF5kTyeRJvia6FSTBroE5fQ0bu1QNxyJqhhinfUsXi9fH3MgtXODWvwo2BDyQSnhPQ88uQ==} + engines: {node: '>=20.19.5', npm: '>=10.8.2'} + hasBin: true + + smol-toml@1.7.1: + resolution: {integrity: sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ==} + engines: {node: '>= 18'} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + source-map@0.7.6: + resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} + engines: {node: '>= 12'} + + space-separated-tokens@2.0.2: + resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + + starlight-links-validator@0.19.2: + resolution: {integrity: sha512-IHeK3R78fsmv53VfRkGbXkwK1CQEUBHM9QPzBEyoAxjZ/ssi5gjV+F4oNNUppTR48iPp+lEY0MTAmvkX7yNnkw==} + engines: {node: '>=18.17.1'} + peerDependencies: + '@astrojs/starlight': '>=0.32.0' + astro: '>=5.1.5' + + starlight-llm-actions@0.13.0: + resolution: {integrity: sha512-U2Aw6iegQr9i6uQ4bteMcwcUtZ0qKgZOUA0vm3td1OL/Es0StC+ixB2yNkFswl9cXNDX1CElgx+YsivENePzGw==} + engines: {node: ^22.0.0 || >=24.0.0} + peerDependencies: + '@astrojs/mdx': '>=4.0.0' + '@astrojs/starlight': '>=0.36.0' + astro: '>=5.6.0' + hast-util-select: '>=6.0.0' + rehype-parse: '>=9.0.0' + rehype-remark: '>=10.0.0' + remark-gfm: '>=4.0.0' + remark-stringify: '>=11.0.0' + unified: '>=11.0.0' + unist-util-remove: '>=4.0.0' + vite: '>=5.0.0' + peerDependenciesMeta: + '@astrojs/mdx': + optional: true + hast-util-select: + optional: true + rehype-parse: + optional: true + rehype-remark: + optional: true + remark-gfm: + optional: true + remark-stringify: + optional: true + unified: + optional: true + unist-util-remove: + optional: true + + stream-replace-string@2.0.0: + resolution: {integrity: sha512-TlnjJ1C0QrmxRNrON00JvaFFlNh5TTG00APw23j74ET7gkQpTASi6/L2fuiav8pzK715HXtUeClpBTw2NPSn6w==} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} + + stringify-entities@4.0.4: + resolution: {integrity: sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-ansi@7.2.0: + resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} + engines: {node: '>=12'} + + style-to-js@1.1.21: + resolution: {integrity: sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ==} + + style-to-object@1.0.14: + resolution: {integrity: sha512-LIN7rULI0jBscWQYaSswptyderlarFkjQ+t79nzty8tcIAceVomEVlLzH5VP4Cmsv6MtKhs7qaAiwlcp+Mgaxw==} + + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + + supports-hyperlinks@4.6.0: + resolution: {integrity: sha512-FL/q/YePS4tFFTNiBOR+zPk2ZrKheqNtmTJ/9OUZ4e1wKHJofDbGQSmSsxS0XNnabSkqexryCaXGvEKz1vf9UQ==} + engines: {node: '>=20'} + + svelte2tsx@0.7.61: + resolution: {integrity: sha512-EpQ/+UHITBULeUojx/LLD3uTYasZXcX1BrqlrzfLUnT9vdUhaOWK59a3ryWcFU8L0sY1SP+gRhJ2Beiis98+qw==} + peerDependencies: + svelte: ^3.55 || ^4.0.0-next.0 || ^4.0 || ^5.0.0-next.0 + typescript: ^4.9.4 || ^5.0.0 || ^6.0.0 + + svelte@5.57.2: + resolution: {integrity: sha512-1YiPCAXT0Ct6g1vOMmSLa0Nsc3QRMvtDF32ihgglGVrugIuIK38RyDiFNXVDhtQ81TJjSO57NdMw6DyYLqSGDQ==} + engines: {node: '>=18'} + + svgo@4.0.2: + resolution: {integrity: sha512-ekx94z1rRc5LDi6oSUaeRnYhd0UOJxdtQCL2rF8xpWxD3TPAsISWOrxezqGovqS38GRZOdpDfvQe3ts6F7nsng==} + engines: {node: '>=16'} + hasBin: true + + tailwind-merge@3.7.0: + resolution: {integrity: sha512-XPPUyAc+cvspz3lHTcR/QgPfW2A0lv/xQNIjX3HGhLR+Nq2lHaLq5MtTesHn8GUr3W3DguT2KT5x3NVgRtYwmA==} + + tailwind-variants@3.3.1: + resolution: {integrity: sha512-4pAvwUtM4HKBiRZftncAbpn6V9Hhwoa5Fl7O2u5zbp7Z5Cvu+/o/6+176WY3WCEES209543quG8zFIcXCsc5Jw==} + engines: {node: '>=16.9.x', pnpm: '>=7.x'} + peerDependencies: + tailwind-merge: '>=3.0.0' + tailwindcss: '*' + peerDependenciesMeta: + tailwind-merge: + optional: true + tailwindcss: + optional: true + + tailwindcss@4.3.3: + resolution: {integrity: sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==} + + tapable@2.3.3: + resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} + engines: {node: '>=6'} + + terminal-link@5.0.0: + resolution: {integrity: sha512-qFAy10MTMwjzjU8U16YS4YoZD+NQLHzLssFMNqgravjbvIPNiqkGFR4yjhJfmY9R5OFU7+yHxc6y+uGHkKwLRA==} + engines: {node: '>=20'} + + tiny-inflate@1.0.3: + resolution: {integrity: sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==} + + tinyexec@1.3.0: + resolution: {integrity: sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==} + engines: {node: '>=18'} + + tinyexec@1.3.1: + resolution: {integrity: sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==} + engines: {node: '>=18'} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + trim-lines@3.0.1: + resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} + + trim-trailing-lines@2.1.0: + resolution: {integrity: sha512-5UR5Biq4VlVOtzqkm2AZlgvSlDJtME46uV0br0gENbwN4l5+mMKT4b9gJKqWtuL2zAIqajGJGuvbCbcAJUZqBg==} + + trough@2.2.0: + resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} + + tsconfck@3.1.6: + resolution: {integrity: sha512-ks6Vjr/jEw0P1gmOVwutM3B7fWxoWBL2KRDb1JfqGVawBmO5UsvmWOQFGHBPl5yxYz4eERr19E6L7NMv+Fej4w==} + engines: {node: ^18 || >=20} + deprecated: unmaintained + hasBin: true + peerDependencies: + typescript: ^5.0.0 + peerDependenciesMeta: + typescript: + optional: true + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + type-fest@4.41.0: + resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} + engines: {node: '>=16'} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + ufo@1.6.4: + resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} + + ultrahtml@1.7.0: + resolution: {integrity: sha512-2xRd0VHoAQE4M+vF/DvFFB7pUV0ZxTW1TLi7lHQWnF/Sb5TPeEUV/l+hxcNnGO00ZXGnR0voCMmYRKQf+rvJ2g==} + + uncrypto@0.1.3: + resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==} + + undici-types@7.18.2: + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + + unified@11.0.5: + resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==} + + unifont@0.7.4: + resolution: {integrity: sha512-oHeis4/xl42HUIeHuNZRGEvxj5AaIKR+bHPNegRq5LV1gdc3jundpONbjglKpihmJf+dswygdMJn3eftGIMemg==} + + unist-util-find-after@5.0.0: + resolution: {integrity: sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==} + + unist-util-is@6.0.1: + resolution: {integrity: sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==} + + unist-util-modify-children@4.0.0: + resolution: {integrity: sha512-+tdN5fGNddvsQdIzUF3Xx82CU9sMM+fA0dLgR9vOmT0oPT2jH+P1nd5lSqfCfXAw+93NhcXNY2qqvTUtE4cQkw==} + + unist-util-position-from-estree@2.0.0: + resolution: {integrity: sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ==} + + unist-util-position@5.0.0: + resolution: {integrity: sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==} + + unist-util-remove-position@5.0.0: + resolution: {integrity: sha512-Hp5Kh3wLxv0PHj9m2yZhhLt58KzPtEYKQQ4yxfYFEO7EvHwzyDYnduhHnY1mDxoqr7VUwVuHXk9RXKIiYS1N8Q==} + + unist-util-remove@4.0.0: + resolution: {integrity: sha512-b4gokeGId57UVRX/eVKej5gXqGlc9+trkORhFJpu9raqZkZhU0zm8Doi05+HaiBsMEIJowL+2WtQ5ItjsngPXg==} + + unist-util-stringify-position@4.0.0: + resolution: {integrity: sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==} + + unist-util-visit-children@3.0.0: + resolution: {integrity: sha512-RgmdTfSBOg04sdPcpTSD1jzoNBjt9a80/ZCzp5cI9n1qPzLZWF9YdvWGN2zmTumP1HWhXKdUWexjy/Wy/lJ7tA==} + + unist-util-visit-parents@6.0.2: + resolution: {integrity: sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==} + + unist-util-visit@5.1.0: + resolution: {integrity: sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==} + + unplugin-icons@24.0.0: + resolution: {integrity: sha512-wV6c7I0PtiIP8KIU7Z1Ukyc2uzgY9ind3RXNu7j63L9qTWDn7CzLnW9FD57DKbfGpWKQv3qyV2PCtBG8Y5Llcg==} + peerDependencies: + '@svgr/core': '>=7.0.0' + '@svgx/core': ^1.0.1 + '@vue/compiler-sfc': ^3.0.2 + '@vue/compiler-vapor': ^3.6.0-rc.5 + svelte: ^3.0.0 || ^4.0.0 || ^5.0.0 + peerDependenciesMeta: + '@svgr/core': + optional: true + '@svgx/core': + optional: true + '@vue/compiler-sfc': + optional: true + '@vue/compiler-vapor': + optional: true + svelte: + optional: true + + unplugin@3.4.0: + resolution: {integrity: sha512-9skdIFlCsPdFV7wUfZxNsFInlW+7nJmGu2gkTu0OUhF56aXGsHab9x52/QhdJ4lC7ZDPWTxbiC4ANqVlsuaW3w==} + engines: {node: ^20.19.0 || >=22.12.0} + peerDependencies: + '@farmfe/core': '*' + '@rsbuild/core': '*' + '@rspack/core': '*' + bun-types-no-globals: '*' + esbuild: '*' + rolldown: '*' + rollup: '*' + unloader: '*' + vite: '*' + webpack: '*' + peerDependenciesMeta: + '@farmfe/core': + optional: true + '@rsbuild/core': + optional: true + '@rspack/core': + optional: true + bun-types-no-globals: + optional: true + esbuild: + optional: true + rolldown: + optional: true + rollup: + optional: true + unloader: + optional: true + vite: + optional: true + webpack: + optional: true + + unstorage@1.17.5: + resolution: {integrity: sha512-0i3iqvRfx29hkNntHyQvJTpf5W9dQ9ZadSoRU8+xVlhVtT7jAX57fazYO9EHvcRCfBCyi5YRya7XCDOsbTgkPg==} + peerDependencies: + '@azure/app-configuration': ^1.8.0 + '@azure/cosmos': ^4.2.0 + '@azure/data-tables': ^13.3.0 + '@azure/identity': ^4.6.0 + '@azure/keyvault-secrets': ^4.9.0 + '@azure/storage-blob': ^12.26.0 + '@capacitor/preferences': ^6 || ^7 || ^8 + '@deno/kv': '>=0.9.0' + '@netlify/blobs': ^6.5.0 || ^7.0.0 || ^8.1.0 || ^9.0.0 || ^10.0.0 + '@planetscale/database': ^1.19.0 + '@upstash/redis': ^1.34.3 + '@vercel/blob': '>=0.27.1' + '@vercel/functions': ^2.2.12 || ^3.0.0 + '@vercel/kv': ^1 || ^2 || ^3 + aws4fetch: ^1.0.20 + db0: '>=0.2.1' + idb-keyval: ^6.2.1 + ioredis: ^5.4.2 + uploadthing: ^7.4.4 + peerDependenciesMeta: + '@azure/app-configuration': + optional: true + '@azure/cosmos': + optional: true + '@azure/data-tables': + optional: true + '@azure/identity': + optional: true + '@azure/keyvault-secrets': + optional: true + '@azure/storage-blob': + optional: true + '@capacitor/preferences': + optional: true + '@deno/kv': + optional: true + '@netlify/blobs': + optional: true + '@planetscale/database': + optional: true + '@upstash/redis': + optional: true + '@vercel/blob': + optional: true + '@vercel/functions': + optional: true + '@vercel/kv': + optional: true + aws4fetch: + optional: true + db0: + optional: true + idb-keyval: + optional: true + ioredis: + optional: true + uploadthing: + optional: true + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + vfile-location@5.0.3: + resolution: {integrity: sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==} + + vfile-message@4.0.3: + resolution: {integrity: sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==} + + vfile@6.0.3: + resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} + + vite@6.4.3: + resolution: {integrity: sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + peerDependencies: + '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + jiti: '>=1.21.0' + less: '*' + lightningcss: ^1.21.0 + sass: '*' + sass-embedded: '*' + stylus: '*' + sugarss: '*' + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + jiti: + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitefu@1.1.3: + resolution: {integrity: sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg==} + peerDependencies: + vite: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + vite: + optional: true + + web-namespaces@2.0.1: + resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + + webpack-virtual-modules@0.6.2: + resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==} + + which-pm-runs@1.1.0: + resolution: {integrity: sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==} + engines: {node: '>=4'} + + widest-line@5.0.0: + resolution: {integrity: sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA==} + engines: {node: '>=18'} + + wrap-ansi@9.0.2: + resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} + engines: {node: '>=18'} + + xxhash-wasm@1.1.0: + resolution: {integrity: sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==} + + yaml@2.9.1: + resolution: {integrity: sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==} + engines: {node: '>= 14.6'} + hasBin: true + + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yocto-queue@1.2.2: + resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} + engines: {node: '>=12.20'} + + yocto-spinner@0.2.3: + resolution: {integrity: sha512-sqBChb33loEnkoXte1bLg45bEBsOP9N1kzQh5JZNKj/0rik4zAPTNSAVPj3uQAdc6slYJ0Ksc403G2XgxsJQFQ==} + engines: {node: '>=18.19'} + + yoctocolors@2.2.0: + resolution: {integrity: sha512-xYqdZFUK/VYazNl/oCDYN+3WloWQwMfZxBoiNt6qNyk+xfOdi598muWE42rNZFp1kNOiqW936q5RhUdnpqElSg==} + engines: {node: '>=18'} + + zimmerframe@1.1.5: + resolution: {integrity: sha512-msJxIvYDYcoNL+PJsu+7qmpDWsYmAxTY+2TNYXXF0hzBzBk0BMecOqDOG/EckUoKCuKwObfbugIl8QpqHDXeFA==} + + zod-to-json-schema@3.25.2: + resolution: {integrity: sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==} + peerDependencies: + zod: ^3.25.28 || ^4 + + zod-to-ts@1.2.0: + resolution: {integrity: sha512-x30XE43V+InwGpvTySRNz9kB7qFU8DlyEy7BsSTCHPH1R0QasMmHWZDCzYm6bVXtj/9NNJAZF3jW8rzFvH5OFA==} + peerDependencies: + typescript: ^4.9.4 || ^5.0.2 + zod: ^3 + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + + zod@4.4.3: + resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} + + zwitch@2.0.4: + resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} + +snapshots: + + '@antfu/install-pkg@1.1.0': + dependencies: + package-manager-detector: 1.8.0 + tinyexec: 1.3.0 + + '@antfu/install-pkg@2.1.0': + dependencies: + package-manager-detector: 1.8.0 + tinyexec: 1.3.1 + + '@astrojs/compiler@2.13.1': {} + + '@astrojs/internal-helpers@0.7.6': {} + + '@astrojs/markdown-remark@6.3.11': + dependencies: + '@astrojs/internal-helpers': 0.7.6 + '@astrojs/prism': 3.3.0 + github-slugger: 2.0.0 + hast-util-from-html: 2.0.3 + hast-util-to-text: 4.0.2 + import-meta-resolve: 4.2.0 + js-yaml: 4.3.1 + mdast-util-definitions: 6.0.0 + rehype-raw: 7.0.0 + rehype-stringify: 10.0.1 + remark-gfm: 4.0.1 + remark-parse: 11.0.0 + remark-rehype: 11.1.2 + remark-smartypants: 3.0.3 + shiki: 3.23.0 + smol-toml: 1.7.1 + unified: 11.0.5 + unist-util-remove-position: 5.0.0 + unist-util-visit: 5.1.0 + unist-util-visit-parents: 6.0.2 + vfile: 6.0.3 + transitivePeerDependencies: + - supports-color + + '@astrojs/mdx@4.3.14(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1))': + dependencies: + '@astrojs/markdown-remark': 6.3.11 + '@mdx-js/mdx': 3.1.1 + acorn: 8.18.0 + astro: 5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1) + es-module-lexer: 1.7.0 + estree-util-visit: 2.0.0 + hast-util-to-html: 9.0.5 + piccolore: 0.1.3 + rehype-raw: 7.0.0 + remark-gfm: 4.0.1 + remark-smartypants: 3.0.3 + source-map: 0.7.6 + unist-util-visit: 5.1.0 + vfile: 6.0.3 + transitivePeerDependencies: + - supports-color + + '@astrojs/prism@3.3.0': + dependencies: + prismjs: 1.30.0 + + '@astrojs/sitemap@3.7.3': + dependencies: + sitemap: 9.0.1 + stream-replace-string: 2.0.0 + zod: 4.4.3 + + '@astrojs/sitemap@3.7.4': + dependencies: + sitemap: 9.0.1 + zod: 4.4.3 + + '@astrojs/starlight-tailwind@4.0.2(@astrojs/starlight@0.36.3(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)))(tailwindcss@4.3.3)': + dependencies: + '@astrojs/starlight': 0.36.3(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)) + tailwindcss: 4.3.3 + + '@astrojs/starlight@0.36.3(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1))': + dependencies: + '@astrojs/markdown-remark': 6.3.11 + '@astrojs/mdx': 4.3.14(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)) + '@astrojs/sitemap': 3.7.3 + '@pagefind/default-ui': 1.5.2 + '@types/hast': 3.0.5 + '@types/js-yaml': 4.0.9 + '@types/mdast': 4.0.4 + astro: 5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1) + astro-expressive-code: 0.41.7(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)) + bcp-47: 2.1.1 + hast-util-from-html: 2.0.3 + hast-util-select: 6.0.4 + hast-util-to-string: 3.0.1 + hastscript: 9.0.1 + i18next: 23.16.8 + js-yaml: 4.3.1 + klona: 2.0.6 + mdast-util-directive: 3.1.0 + mdast-util-to-markdown: 2.1.2 + mdast-util-to-string: 4.0.0 + pagefind: 1.5.2 + rehype: 13.0.2 + rehype-format: 5.0.1 + remark-directive: 3.0.1 + ultrahtml: 1.7.0 + unified: 11.0.5 + unist-util-visit: 5.1.0 + vfile: 6.0.3 + transitivePeerDependencies: + - supports-color + + '@astrojs/svelte@7.2.5(@types/node@24.13.3)(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1))(jiti@2.7.0)(lightningcss@1.32.0)(svelte@5.57.2)(typescript@5.9.3)(yaml@2.9.1)': + dependencies: + '@sveltejs/vite-plugin-svelte': 5.1.1(svelte@5.57.2)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)) + astro: 5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1) + svelte: 5.57.2 + svelte2tsx: 0.7.61(svelte@5.57.2)(typescript@5.9.3) + typescript: 5.9.3 + vite: 6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1) + transitivePeerDependencies: + - '@types/node' + - jiti + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + + '@astrojs/telemetry@3.3.0': + dependencies: + ci-info: 4.4.0 + debug: 4.4.3 + dlv: 1.1.3 + dset: 3.1.4 + is-docker: 3.0.0 + is-wsl: 3.1.1 + which-pm-runs: 1.1.0 + transitivePeerDependencies: + - supports-color + + '@babel/helper-string-parser@7.29.7': {} + + '@babel/helper-validator-identifier@7.29.7': {} + + '@babel/parser@7.29.8': + dependencies: + '@babel/types': 7.29.8 + + '@babel/runtime@7.29.7': {} + + '@babel/types@7.29.8': + dependencies: + '@babel/helper-string-parser': 7.29.7 + '@babel/helper-validator-identifier': 7.29.7 + + '@capsizecss/unpack@4.0.1': + dependencies: + fontkitten: 1.0.3 + + '@ctrl/tinycolor@4.2.0': {} + + '@emnapi/runtime@1.11.3': + dependencies: + tslib: 2.8.1 + optional: true + + '@esbuild/aix-ppc64@0.25.12': + optional: true + + '@esbuild/aix-ppc64@0.27.7': + optional: true + + '@esbuild/android-arm64@0.25.12': + optional: true + + '@esbuild/android-arm64@0.27.7': + optional: true + + '@esbuild/android-arm@0.25.12': + optional: true + + '@esbuild/android-arm@0.27.7': + optional: true + + '@esbuild/android-x64@0.25.12': + optional: true + + '@esbuild/android-x64@0.27.7': + optional: true + + '@esbuild/darwin-arm64@0.25.12': + optional: true + + '@esbuild/darwin-arm64@0.27.7': + optional: true + + '@esbuild/darwin-x64@0.25.12': + optional: true + + '@esbuild/darwin-x64@0.27.7': + optional: true + + '@esbuild/freebsd-arm64@0.25.12': + optional: true + + '@esbuild/freebsd-arm64@0.27.7': + optional: true + + '@esbuild/freebsd-x64@0.25.12': + optional: true + + '@esbuild/freebsd-x64@0.27.7': + optional: true + + '@esbuild/linux-arm64@0.25.12': + optional: true + + '@esbuild/linux-arm64@0.27.7': + optional: true + + '@esbuild/linux-arm@0.25.12': + optional: true + + '@esbuild/linux-arm@0.27.7': + optional: true + + '@esbuild/linux-ia32@0.25.12': + optional: true + + '@esbuild/linux-ia32@0.27.7': + optional: true + + '@esbuild/linux-loong64@0.25.12': + optional: true + + '@esbuild/linux-loong64@0.27.7': + optional: true + + '@esbuild/linux-mips64el@0.25.12': + optional: true + + '@esbuild/linux-mips64el@0.27.7': + optional: true + + '@esbuild/linux-ppc64@0.25.12': + optional: true + + '@esbuild/linux-ppc64@0.27.7': + optional: true + + '@esbuild/linux-riscv64@0.25.12': + optional: true + + '@esbuild/linux-riscv64@0.27.7': + optional: true + + '@esbuild/linux-s390x@0.25.12': + optional: true + + '@esbuild/linux-s390x@0.27.7': + optional: true + + '@esbuild/linux-x64@0.25.12': + optional: true + + '@esbuild/linux-x64@0.27.7': + optional: true + + '@esbuild/netbsd-arm64@0.25.12': + optional: true + + '@esbuild/netbsd-arm64@0.27.7': + optional: true + + '@esbuild/netbsd-x64@0.25.12': + optional: true + + '@esbuild/netbsd-x64@0.27.7': + optional: true + + '@esbuild/openbsd-arm64@0.25.12': + optional: true + + '@esbuild/openbsd-arm64@0.27.7': + optional: true + + '@esbuild/openbsd-x64@0.25.12': + optional: true + + '@esbuild/openbsd-x64@0.27.7': + optional: true + + '@esbuild/openharmony-arm64@0.25.12': + optional: true + + '@esbuild/openharmony-arm64@0.27.7': + optional: true + + '@esbuild/sunos-x64@0.25.12': + optional: true + + '@esbuild/sunos-x64@0.27.7': + optional: true + + '@esbuild/win32-arm64@0.25.12': + optional: true + + '@esbuild/win32-arm64@0.27.7': + optional: true + + '@esbuild/win32-ia32@0.25.12': + optional: true + + '@esbuild/win32-ia32@0.27.7': + optional: true + + '@esbuild/win32-x64@0.25.12': + optional: true + + '@esbuild/win32-x64@0.27.7': + optional: true + + '@expressive-code/core@0.41.7': + dependencies: + '@ctrl/tinycolor': 4.2.0 + hast-util-select: 6.0.4 + hast-util-to-html: 9.0.5 + hast-util-to-text: 4.0.2 + hastscript: 9.0.1 + postcss: 8.5.25 + postcss-nested: 6.2.0(postcss@8.5.25) + unist-util-visit: 5.1.0 + unist-util-visit-parents: 6.0.2 + + '@expressive-code/plugin-frames@0.41.7': + dependencies: + '@expressive-code/core': 0.41.7 + + '@expressive-code/plugin-shiki@0.41.7': + dependencies: + '@expressive-code/core': 0.41.7 + shiki: 3.23.0 + + '@expressive-code/plugin-text-markers@0.41.7': + dependencies: + '@expressive-code/core': 0.41.7 + + '@iconify-json/lucide@1.2.142': + dependencies: + '@iconify/types': 2.0.0 + + '@iconify/types@2.0.0': {} + + '@iconify/utils@3.1.7': + dependencies: + '@antfu/install-pkg': 2.1.0 + '@iconify/types': 2.0.0 + import-meta-resolve: 4.2.0 + + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.2.4 + optional: true + + '@img/sharp-darwin-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.2.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.2.4': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-s390x@1.2.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + optional: true + + '@img/sharp-linux-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.2.4 + optional: true + + '@img/sharp-linux-arm@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.2.4 + optional: true + + '@img/sharp-linux-ppc64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.2.4 + optional: true + + '@img/sharp-linux-riscv64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.2.4 + optional: true + + '@img/sharp-linux-s390x@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.2.4 + optional: true + + '@img/sharp-linux-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + optional: true + + '@img/sharp-wasm32@0.34.5': + dependencies: + '@emnapi/runtime': 1.11.3 + optional: true + + '@img/sharp-win32-arm64@0.34.5': + optional: true + + '@img/sharp-win32-ia32@0.34.5': + optional: true + + '@img/sharp-win32-x64@0.34.5': + optional: true + + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/remapping@2.3.5': + dependencies: + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@jridgewell/sourcemap-codec@1.6.0': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@mdx-js/mdx@3.1.1': + dependencies: + '@types/estree': 1.0.9 + '@types/estree-jsx': 1.0.5 + '@types/hast': 3.0.5 + '@types/mdx': 2.0.14 + acorn: 8.18.0 + collapse-white-space: 2.1.0 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + estree-util-scope: 1.0.0 + estree-walker: 3.0.3 + hast-util-to-jsx-runtime: 2.3.6 + markdown-extensions: 2.0.0 + recma-build-jsx: 1.0.0 + recma-jsx: 1.0.1(acorn@8.18.0) + recma-stringify: 1.0.0 + rehype-recma: 1.0.0 + remark-mdx: 3.1.1 + remark-parse: 11.0.0 + remark-rehype: 11.1.2 + source-map: 0.7.6 + unified: 11.0.5 + unist-util-position-from-estree: 2.0.0 + unist-util-stringify-position: 4.0.0 + unist-util-visit: 5.1.0 + vfile: 6.0.3 + transitivePeerDependencies: + - supports-color + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + optional: true + + '@oslojs/encoding@1.1.0': {} + + '@pagefind/darwin-arm64@1.5.2': + optional: true + + '@pagefind/darwin-x64@1.5.2': + optional: true + + '@pagefind/default-ui@1.5.2': {} + + '@pagefind/freebsd-x64@1.5.2': + optional: true + + '@pagefind/linux-arm64@1.5.2': + optional: true + + '@pagefind/linux-x64@1.5.2': + optional: true + + '@pagefind/windows-arm64@1.5.2': + optional: true + + '@pagefind/windows-x64@1.5.2': + optional: true + + '@rollup/pluginutils@5.4.0(rollup@4.62.4)': + dependencies: + '@types/estree': 1.0.9 + estree-walker: 2.0.2 + picomatch: 4.0.7 + optionalDependencies: + rollup: 4.62.4 + + '@rollup/rollup-android-arm-eabi@4.62.4': + optional: true + + '@rollup/rollup-android-arm64@4.62.4': + optional: true + + '@rollup/rollup-darwin-arm64@4.62.4': + optional: true + + '@rollup/rollup-darwin-x64@4.62.4': + optional: true + + '@rollup/rollup-freebsd-arm64@4.62.4': + optional: true + + '@rollup/rollup-freebsd-x64@4.62.4': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.62.4': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.62.4': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.62.4': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.62.4': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.62.4': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.62.4': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.62.4': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.62.4': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.62.4': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.62.4': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.62.4': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.62.4': + optional: true + + '@rollup/rollup-linux-x64-musl@4.62.4': + optional: true + + '@rollup/rollup-openbsd-x64@4.62.4': + optional: true + + '@rollup/rollup-openharmony-arm64@4.62.4': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.62.4': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.62.4': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.62.4': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.62.4': + optional: true + + '@shikijs/core@3.23.0': + dependencies: + '@shikijs/types': 3.23.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + hast-util-to-html: 9.0.5 + + '@shikijs/engine-javascript@3.23.0': + dependencies: + '@shikijs/types': 3.23.0 + '@shikijs/vscode-textmate': 10.0.2 + oniguruma-to-es: 4.3.6 + + '@shikijs/engine-oniguruma@3.23.0': + dependencies: + '@shikijs/types': 3.23.0 + '@shikijs/vscode-textmate': 10.0.2 + + '@shikijs/langs@3.23.0': + dependencies: + '@shikijs/types': 3.23.0 + + '@shikijs/themes@3.23.0': + dependencies: + '@shikijs/types': 3.23.0 + + '@shikijs/types@3.23.0': + dependencies: + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + + '@shikijs/vscode-textmate@10.0.2': {} + + '@sveltejs/acorn-typescript@1.0.13(acorn@8.18.0)': + dependencies: + acorn: 8.18.0 + + '@sveltejs/vite-plugin-svelte-inspector@4.0.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.57.2)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)))(svelte@5.57.2)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1))': + dependencies: + '@sveltejs/vite-plugin-svelte': 5.1.1(svelte@5.57.2)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)) + debug: 4.4.3 + svelte: 5.57.2 + vite: 6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1) + transitivePeerDependencies: + - supports-color + + '@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.57.2)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1))': + dependencies: + '@sveltejs/vite-plugin-svelte-inspector': 4.0.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.57.2)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)))(svelte@5.57.2)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)) + debug: 4.4.3 + deepmerge: 4.3.1 + kleur: 4.1.5 + magic-string: 0.30.21 + svelte: 5.57.2 + vite: 6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1) + vitefu: 1.1.3(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)) + transitivePeerDependencies: + - supports-color + + '@tailwindcss/node@4.3.3': + dependencies: + '@jridgewell/remapping': 2.3.5 + enhanced-resolve: 5.26.0 + jiti: 2.7.0 + lightningcss: 1.32.0 + magic-string: 0.30.21 + source-map-js: 1.2.1 + tailwindcss: 4.3.3 + + '@tailwindcss/oxide-android-arm64@4.3.3': + optional: true + + '@tailwindcss/oxide-darwin-arm64@4.3.3': + optional: true + + '@tailwindcss/oxide-darwin-x64@4.3.3': + optional: true + + '@tailwindcss/oxide-freebsd-x64@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-arm64-gnu@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-arm64-musl@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-x64-gnu@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-x64-musl@4.3.3': + optional: true + + '@tailwindcss/oxide-wasm32-wasi@4.3.3': + optional: true + + '@tailwindcss/oxide-win32-arm64-msvc@4.3.3': + optional: true + + '@tailwindcss/oxide-win32-x64-msvc@4.3.3': + optional: true + + '@tailwindcss/oxide@4.3.3': + optionalDependencies: + '@tailwindcss/oxide-android-arm64': 4.3.3 + '@tailwindcss/oxide-darwin-arm64': 4.3.3 + '@tailwindcss/oxide-darwin-x64': 4.3.3 + '@tailwindcss/oxide-freebsd-x64': 4.3.3 + '@tailwindcss/oxide-linux-arm-gnueabihf': 4.3.3 + '@tailwindcss/oxide-linux-arm64-gnu': 4.3.3 + '@tailwindcss/oxide-linux-arm64-musl': 4.3.3 + '@tailwindcss/oxide-linux-x64-gnu': 4.3.3 + '@tailwindcss/oxide-linux-x64-musl': 4.3.3 + '@tailwindcss/oxide-wasm32-wasi': 4.3.3 + '@tailwindcss/oxide-win32-arm64-msvc': 4.3.3 + '@tailwindcss/oxide-win32-x64-msvc': 4.3.3 + + '@tailwindcss/vite@4.3.3(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1))': + dependencies: + '@tailwindcss/node': 4.3.3 + '@tailwindcss/oxide': 4.3.3 + tailwindcss: 4.3.3 + vite: 6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1) + + '@types/debug@4.1.13': + dependencies: + '@types/ms': 2.1.0 + + '@types/estree-jsx@1.0.5': + dependencies: + '@types/estree': 1.0.9 + + '@types/estree@1.0.9': {} + + '@types/hast@3.0.5': + dependencies: + '@types/unist': 3.0.3 + + '@types/js-yaml@4.0.9': {} + + '@types/mdast@4.0.4': + dependencies: + '@types/unist': 3.0.3 + + '@types/mdx@2.0.14': {} + + '@types/ms@2.1.0': {} + + '@types/nlcst@2.0.3': + dependencies: + '@types/unist': 3.0.3 + + '@types/node@24.13.3': + dependencies: + undici-types: 7.18.2 + + '@types/picomatch@3.0.2': {} + + '@types/sax@1.2.7': + dependencies: + '@types/node': 24.13.3 + + '@types/unist@2.0.11': {} + + '@types/unist@3.0.3': {} + + '@ungap/structured-clone@1.3.3': {} + + acorn-jsx@5.3.2(acorn@8.18.0): + dependencies: + acorn: 8.18.0 + + acorn@8.18.0: {} + + ansi-align@3.0.1: + dependencies: + string-width: 4.2.3 + + ansi-escapes@7.3.0: + dependencies: + environment: 1.1.0 + + ansi-regex@5.0.1: {} + + ansi-regex@6.2.2: {} + + ansi-styles@6.2.3: {} + + anymatch@3.1.3: + dependencies: + normalize-path: 3.0.0 + picomatch: 2.3.2 + + arg@5.0.2: {} + + argparse@2.0.1: {} + + aria-query@5.3.1: {} + + aria-query@5.3.2: {} + + array-iterate@2.0.1: {} + + astring@1.9.0: {} + + astro-expressive-code@0.41.7(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)): + dependencies: + astro: 5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1) + rehype-expressive-code: 0.41.7 + + astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1): + dependencies: + '@astrojs/compiler': 2.13.1 + '@astrojs/internal-helpers': 0.7.6 + '@astrojs/markdown-remark': 6.3.11 + '@astrojs/telemetry': 3.3.0 + '@capsizecss/unpack': 4.0.1 + '@oslojs/encoding': 1.1.0 + '@rollup/pluginutils': 5.4.0(rollup@4.62.4) + acorn: 8.18.0 + aria-query: 5.3.2 + axobject-query: 4.1.0 + boxen: 8.0.1 + ci-info: 4.4.0 + clsx: 2.1.1 + common-ancestor-path: 1.0.1 + cookie: 1.1.1 + cssesc: 3.0.0 + debug: 4.4.3 + deterministic-object-hash: 2.0.2 + devalue: 5.9.0 + diff: 8.0.4 + dlv: 1.1.3 + dset: 3.1.4 + es-module-lexer: 1.7.0 + esbuild: 0.27.7 + estree-walker: 3.0.3 + flattie: 1.1.1 + fontace: 0.4.1 + github-slugger: 2.0.0 + html-escaper: 3.0.3 + http-cache-semantics: 4.2.0 + import-meta-resolve: 4.2.0 + js-yaml: 4.3.1 + magic-string: 0.30.21 + magicast: 0.5.4 + mrmime: 2.0.1 + neotraverse: 0.6.18 + p-limit: 6.2.0 + p-queue: 8.1.1 + package-manager-detector: 1.8.0 + piccolore: 0.1.3 + picomatch: 4.0.5 + prompts: 2.4.2 + rehype: 13.0.2 + semver: 7.8.5 + shiki: 3.23.0 + smol-toml: 1.7.1 + svgo: 4.0.2 + tinyexec: 1.3.0 + tinyglobby: 0.2.17 + tsconfck: 3.1.6(typescript@5.9.3) + ultrahtml: 1.7.0 + unifont: 0.7.4 + unist-util-visit: 5.1.0 + unstorage: 1.17.5 + vfile: 6.0.3 + vite: 6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1) + vitefu: 1.1.3(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)) + xxhash-wasm: 1.1.0 + yargs-parser: 21.1.1 + yocto-spinner: 0.2.3 + zod: 3.25.76 + zod-to-json-schema: 3.25.2(zod@3.25.76) + zod-to-ts: 1.2.0(typescript@5.9.3)(zod@3.25.76) + optionalDependencies: + sharp: 0.34.5 + transitivePeerDependencies: + - '@azure/app-configuration' + - '@azure/cosmos' + - '@azure/data-tables' + - '@azure/identity' + - '@azure/keyvault-secrets' + - '@azure/storage-blob' + - '@capacitor/preferences' + - '@deno/kv' + - '@netlify/blobs' + - '@planetscale/database' + - '@types/node' + - '@upstash/redis' + - '@vercel/blob' + - '@vercel/functions' + - '@vercel/kv' + - aws4fetch + - db0 + - idb-keyval + - ioredis + - jiti + - less + - lightningcss + - rollup + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - typescript + - uploadthing + - yaml + + axobject-query@4.1.0: {} + + bail@2.0.2: {} + + base-64@1.0.0: {} + + bcp-47-match@2.0.3: {} + + bcp-47@2.1.1: + dependencies: + is-alphabetical: 2.0.1 + is-alphanumerical: 2.0.1 + is-decimal: 2.0.1 + + boolbase@1.0.0: {} + + boxen@8.0.1: + dependencies: + ansi-align: 3.0.1 + camelcase: 8.0.0 + chalk: 5.6.2 + cli-boxes: 3.0.0 + string-width: 7.2.0 + type-fest: 4.41.0 + widest-line: 5.0.0 + wrap-ansi: 9.0.2 + + camelcase@8.0.0: {} + + ccount@2.0.1: {} + + chalk@5.6.2: {} + + character-entities-html4@2.1.0: {} + + character-entities-legacy@3.0.0: {} + + character-entities@2.0.2: {} + + character-reference-invalid@2.0.1: {} + + chokidar@5.0.0: + dependencies: + readdirp: 5.0.0 + + ci-info@4.4.0: {} + + cli-boxes@3.0.0: {} + + clsx@2.1.1: {} + + collapse-white-space@2.1.0: {} + + comma-separated-tokens@2.0.3: {} + + commander@11.1.0: {} + + common-ancestor-path@1.0.1: {} + + confbox@0.1.8: {} + + confbox@0.3.1: {} + + cookie-es@1.2.3: {} + + cookie@1.1.1: {} + + crossws@0.3.5: + dependencies: + uncrypto: 0.1.3 + + css-select@5.2.2: + dependencies: + boolbase: 1.0.0 + css-what: 6.2.2 + domhandler: 5.0.3 + domutils: 3.2.2 + nth-check: 2.1.1 + + css-selector-parser@3.3.0: {} + + css-tree@2.2.1: + dependencies: + mdn-data: 2.0.28 + source-map-js: 1.2.1 + + css-tree@3.2.1: + dependencies: + mdn-data: 2.27.1 + source-map-js: 1.2.1 + + css-what@6.2.2: {} + + cssesc@3.0.0: {} + + csso@5.0.5: + dependencies: + css-tree: 2.2.1 + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + decode-named-character-reference@1.3.0: + dependencies: + character-entities: 2.0.2 + + dedent-js@1.0.1: {} + + deepmerge@4.3.1: {} + + defu@6.1.7: {} + + dequal@2.0.3: {} + + destr@2.0.5: {} + + detect-libc@2.1.2: {} + + deterministic-object-hash@2.0.2: + dependencies: + base-64: 1.0.0 + + devalue@5.9.0: {} + + devalue@5.9.4: {} + + devlop@1.1.0: + dependencies: + dequal: 2.0.3 + + diff@8.0.4: {} + + direction@2.0.1: {} + + dlv@1.1.3: {} + + dom-serializer@2.0.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + + domelementtype@2.3.0: {} + + domhandler@5.0.3: + dependencies: + domelementtype: 2.3.0 + + domutils@3.2.2: + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + + dset@3.1.4: {} + + emoji-regex@10.6.0: {} + + emoji-regex@8.0.0: {} + + enhanced-resolve@5.26.0: + dependencies: + graceful-fs: 4.2.11 + tapable: 2.3.3 + + entities@4.5.0: {} + + entities@6.0.1: {} + + environment@1.1.0: {} + + es-module-lexer@1.7.0: {} + + esast-util-from-estree@2.0.0: + dependencies: + '@types/estree-jsx': 1.0.5 + devlop: 1.1.0 + estree-util-visit: 2.0.0 + unist-util-position-from-estree: 2.0.0 + + esast-util-from-js@2.0.1: + dependencies: + '@types/estree-jsx': 1.0.5 + acorn: 8.18.0 + esast-util-from-estree: 2.0.0 + vfile-message: 4.0.3 + + esbuild@0.25.12: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.12 + '@esbuild/android-arm': 0.25.12 + '@esbuild/android-arm64': 0.25.12 + '@esbuild/android-x64': 0.25.12 + '@esbuild/darwin-arm64': 0.25.12 + '@esbuild/darwin-x64': 0.25.12 + '@esbuild/freebsd-arm64': 0.25.12 + '@esbuild/freebsd-x64': 0.25.12 + '@esbuild/linux-arm': 0.25.12 + '@esbuild/linux-arm64': 0.25.12 + '@esbuild/linux-ia32': 0.25.12 + '@esbuild/linux-loong64': 0.25.12 + '@esbuild/linux-mips64el': 0.25.12 + '@esbuild/linux-ppc64': 0.25.12 + '@esbuild/linux-riscv64': 0.25.12 + '@esbuild/linux-s390x': 0.25.12 + '@esbuild/linux-x64': 0.25.12 + '@esbuild/netbsd-arm64': 0.25.12 + '@esbuild/netbsd-x64': 0.25.12 + '@esbuild/openbsd-arm64': 0.25.12 + '@esbuild/openbsd-x64': 0.25.12 + '@esbuild/openharmony-arm64': 0.25.12 + '@esbuild/sunos-x64': 0.25.12 + '@esbuild/win32-arm64': 0.25.12 + '@esbuild/win32-ia32': 0.25.12 + '@esbuild/win32-x64': 0.25.12 + + esbuild@0.27.7: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.7 + '@esbuild/android-arm': 0.27.7 + '@esbuild/android-arm64': 0.27.7 + '@esbuild/android-x64': 0.27.7 + '@esbuild/darwin-arm64': 0.27.7 + '@esbuild/darwin-x64': 0.27.7 + '@esbuild/freebsd-arm64': 0.27.7 + '@esbuild/freebsd-x64': 0.27.7 + '@esbuild/linux-arm': 0.27.7 + '@esbuild/linux-arm64': 0.27.7 + '@esbuild/linux-ia32': 0.27.7 + '@esbuild/linux-loong64': 0.27.7 + '@esbuild/linux-mips64el': 0.27.7 + '@esbuild/linux-ppc64': 0.27.7 + '@esbuild/linux-riscv64': 0.27.7 + '@esbuild/linux-s390x': 0.27.7 + '@esbuild/linux-x64': 0.27.7 + '@esbuild/netbsd-arm64': 0.27.7 + '@esbuild/netbsd-x64': 0.27.7 + '@esbuild/openbsd-arm64': 0.27.7 + '@esbuild/openbsd-x64': 0.27.7 + '@esbuild/openharmony-arm64': 0.27.7 + '@esbuild/sunos-x64': 0.27.7 + '@esbuild/win32-arm64': 0.27.7 + '@esbuild/win32-ia32': 0.27.7 + '@esbuild/win32-x64': 0.27.7 + + escape-string-regexp@5.0.0: {} + + esm-env@1.2.2: {} + + esrap@2.4.0: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + estree-util-attach-comments@3.0.0: + dependencies: + '@types/estree': 1.0.9 + + estree-util-build-jsx@3.0.1: + dependencies: + '@types/estree-jsx': 1.0.5 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + estree-walker: 3.0.3 + + estree-util-is-identifier-name@3.0.0: {} + + estree-util-scope@1.0.0: + dependencies: + '@types/estree': 1.0.9 + devlop: 1.1.0 + + estree-util-to-js@2.0.0: + dependencies: + '@types/estree-jsx': 1.0.5 + astring: 1.9.0 + source-map: 0.7.6 + + estree-util-visit@2.0.0: + dependencies: + '@types/estree-jsx': 1.0.5 + '@types/unist': 3.0.3 + + estree-walker@2.0.2: {} + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + eventemitter3@5.0.4: {} + + expressive-code@0.41.7: + dependencies: + '@expressive-code/core': 0.41.7 + '@expressive-code/plugin-frames': 0.41.7 + '@expressive-code/plugin-shiki': 0.41.7 + '@expressive-code/plugin-text-markers': 0.41.7 + + exsolve@1.1.3: {} + + extend@3.0.2: {} + + fdir@6.5.0(picomatch@4.0.7): + optionalDependencies: + picomatch: 4.0.7 + + flattie@1.1.1: {} + + fontace@0.4.1: + dependencies: + fontkitten: 1.0.3 + + fontkitten@1.0.3: + dependencies: + tiny-inflate: 1.0.3 + + fsevents@2.3.3: + optional: true + + get-east-asian-width@1.6.0: {} + + github-slugger@2.0.0: {} + + graceful-fs@4.2.11: {} + + h3@1.15.11: + dependencies: + cookie-es: 1.2.3 + crossws: 0.3.5 + defu: 6.1.7 + destr: 2.0.5 + iron-webcrypto: 1.2.1 + node-mock-http: 1.0.5 + radix3: 1.1.2 + ufo: 1.6.4 + uncrypto: 0.1.3 + + has-flag@5.0.1: {} + + hast-util-embedded@3.0.0: + dependencies: + '@types/hast': 3.0.5 + hast-util-is-element: 3.0.0 + + hast-util-format@1.1.0: + dependencies: + '@types/hast': 3.0.5 + hast-util-embedded: 3.0.0 + hast-util-minify-whitespace: 1.0.1 + hast-util-phrasing: 3.0.1 + hast-util-whitespace: 3.0.0 + html-whitespace-sensitive-tag-names: 3.0.1 + unist-util-visit-parents: 6.0.2 + + hast-util-from-html@2.0.3: + dependencies: + '@types/hast': 3.0.5 + devlop: 1.1.0 + hast-util-from-parse5: 8.0.3 + parse5: 7.3.0 + vfile: 6.0.3 + vfile-message: 4.0.3 + + hast-util-from-parse5@8.0.3: + dependencies: + '@types/hast': 3.0.5 + '@types/unist': 3.0.3 + devlop: 1.1.0 + hastscript: 9.0.1 + property-information: 7.2.0 + vfile: 6.0.3 + vfile-location: 5.0.3 + web-namespaces: 2.0.1 + + hast-util-has-property@3.0.0: + dependencies: + '@types/hast': 3.0.5 + + hast-util-is-body-ok-link@3.0.1: + dependencies: + '@types/hast': 3.0.5 + + hast-util-is-element@3.0.0: + dependencies: + '@types/hast': 3.0.5 + + hast-util-minify-whitespace@1.0.1: + dependencies: + '@types/hast': 3.0.5 + hast-util-embedded: 3.0.0 + hast-util-is-element: 3.0.0 + hast-util-whitespace: 3.0.0 + unist-util-is: 6.0.1 + + hast-util-parse-selector@4.0.0: + dependencies: + '@types/hast': 3.0.5 + + hast-util-phrasing@3.0.1: + dependencies: + '@types/hast': 3.0.5 + hast-util-embedded: 3.0.0 + hast-util-has-property: 3.0.0 + hast-util-is-body-ok-link: 3.0.1 + hast-util-is-element: 3.0.0 + + hast-util-raw@9.1.0: + dependencies: + '@types/hast': 3.0.5 + '@types/unist': 3.0.3 + '@ungap/structured-clone': 1.3.3 + hast-util-from-parse5: 8.0.3 + hast-util-to-parse5: 8.0.1 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.2.1 + parse5: 7.3.0 + unist-util-position: 5.0.0 + unist-util-visit: 5.1.0 + vfile: 6.0.3 + web-namespaces: 2.0.1 + zwitch: 2.0.4 + + hast-util-select@6.0.4: + dependencies: + '@types/hast': 3.0.5 + '@types/unist': 3.0.3 + bcp-47-match: 2.0.3 + comma-separated-tokens: 2.0.3 + css-selector-parser: 3.3.0 + devlop: 1.1.0 + direction: 2.0.1 + hast-util-has-property: 3.0.0 + hast-util-to-string: 3.0.1 + hast-util-whitespace: 3.0.0 + nth-check: 2.1.1 + property-information: 7.2.0 + space-separated-tokens: 2.0.2 + unist-util-visit: 5.1.0 + zwitch: 2.0.4 + + hast-util-to-estree@3.1.3: + dependencies: + '@types/estree': 1.0.9 + '@types/estree-jsx': 1.0.5 + '@types/hast': 3.0.5 + comma-separated-tokens: 2.0.3 + devlop: 1.1.0 + estree-util-attach-comments: 3.0.0 + estree-util-is-identifier-name: 3.0.0 + hast-util-whitespace: 3.0.0 + mdast-util-mdx-expression: 2.0.1 + mdast-util-mdx-jsx: 3.2.0 + mdast-util-mdxjs-esm: 2.0.1 + property-information: 7.2.0 + space-separated-tokens: 2.0.2 + style-to-js: 1.1.21 + unist-util-position: 5.0.0 + zwitch: 2.0.4 + transitivePeerDependencies: + - supports-color + + hast-util-to-html@9.0.5: + dependencies: + '@types/hast': 3.0.5 + '@types/unist': 3.0.3 + ccount: 2.0.1 + comma-separated-tokens: 2.0.3 + hast-util-whitespace: 3.0.0 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.2.1 + property-information: 7.2.0 + space-separated-tokens: 2.0.2 + stringify-entities: 4.0.4 + zwitch: 2.0.4 + + hast-util-to-jsx-runtime@2.3.6: + dependencies: + '@types/estree': 1.0.9 + '@types/hast': 3.0.5 + '@types/unist': 3.0.3 + comma-separated-tokens: 2.0.3 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + hast-util-whitespace: 3.0.0 + mdast-util-mdx-expression: 2.0.1 + mdast-util-mdx-jsx: 3.2.0 + mdast-util-mdxjs-esm: 2.0.1 + property-information: 7.2.0 + space-separated-tokens: 2.0.2 + style-to-js: 1.1.21 + unist-util-position: 5.0.0 + vfile-message: 4.0.3 + transitivePeerDependencies: + - supports-color + + hast-util-to-mdast@10.1.2: + dependencies: + '@types/hast': 3.0.5 + '@types/mdast': 4.0.4 + '@ungap/structured-clone': 1.3.3 + hast-util-phrasing: 3.0.1 + hast-util-to-html: 9.0.5 + hast-util-to-text: 4.0.2 + hast-util-whitespace: 3.0.0 + mdast-util-phrasing: 4.1.0 + mdast-util-to-hast: 13.2.1 + mdast-util-to-string: 4.0.0 + rehype-minify-whitespace: 6.0.2 + trim-trailing-lines: 2.1.0 + unist-util-position: 5.0.0 + unist-util-visit: 5.1.0 + + hast-util-to-parse5@8.0.1: + dependencies: + '@types/hast': 3.0.5 + comma-separated-tokens: 2.0.3 + devlop: 1.1.0 + property-information: 7.2.0 + space-separated-tokens: 2.0.2 + web-namespaces: 2.0.1 + zwitch: 2.0.4 + + hast-util-to-string@3.0.1: + dependencies: + '@types/hast': 3.0.5 + + hast-util-to-text@4.0.2: + dependencies: + '@types/hast': 3.0.5 + '@types/unist': 3.0.3 + hast-util-is-element: 3.0.0 + unist-util-find-after: 5.0.0 + + hast-util-whitespace@3.0.0: + dependencies: + '@types/hast': 3.0.5 + + hastscript@9.0.1: + dependencies: + '@types/hast': 3.0.5 + comma-separated-tokens: 2.0.3 + hast-util-parse-selector: 4.0.0 + property-information: 7.2.0 + space-separated-tokens: 2.0.2 + + html-escaper@3.0.3: {} + + html-void-elements@3.0.0: {} + + html-whitespace-sensitive-tag-names@3.0.1: {} + + http-cache-semantics@4.2.0: {} + + i18next@23.16.8: + dependencies: + '@babel/runtime': 7.29.7 + + import-meta-resolve@4.2.0: {} + + inline-style-parser@0.2.7: {} + + iron-webcrypto@1.2.1: {} + + is-absolute-url@4.0.1: {} + + is-alphabetical@2.0.1: {} + + is-alphanumerical@2.0.1: + dependencies: + is-alphabetical: 2.0.1 + is-decimal: 2.0.1 + + is-decimal@2.0.1: {} + + is-docker@3.0.0: {} + + is-fullwidth-code-point@3.0.0: {} + + is-hexadecimal@2.0.1: {} + + is-inside-container@1.0.0: + dependencies: + is-docker: 3.0.0 + + is-plain-obj@4.1.0: {} + + is-reference@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + is-wsl@3.1.1: + dependencies: + is-inside-container: 1.0.0 + + jiti@2.7.0: {} + + js-yaml@4.3.1: + dependencies: + argparse: 2.0.1 + + kleur@3.0.3: {} + + kleur@4.1.5: {} + + klona@2.0.6: {} + + lightningcss-android-arm64@1.32.0: + optional: true + + lightningcss-darwin-arm64@1.32.0: + optional: true + + lightningcss-darwin-x64@1.32.0: + optional: true + + lightningcss-freebsd-x64@1.32.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.32.0: + optional: true + + lightningcss-linux-arm64-gnu@1.32.0: + optional: true + + lightningcss-linux-arm64-musl@1.32.0: + optional: true + + lightningcss-linux-x64-gnu@1.32.0: + optional: true + + lightningcss-linux-x64-musl@1.32.0: + optional: true + + lightningcss-win32-arm64-msvc@1.32.0: + optional: true + + lightningcss-win32-x64-msvc@1.32.0: + optional: true + + lightningcss@1.32.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.32.0 + lightningcss-darwin-arm64: 1.32.0 + lightningcss-darwin-x64: 1.32.0 + lightningcss-freebsd-x64: 1.32.0 + lightningcss-linux-arm-gnueabihf: 1.32.0 + lightningcss-linux-arm64-gnu: 1.32.0 + lightningcss-linux-arm64-musl: 1.32.0 + lightningcss-linux-x64-gnu: 1.32.0 + lightningcss-linux-x64-musl: 1.32.0 + lightningcss-win32-arm64-msvc: 1.32.0 + lightningcss-win32-x64-msvc: 1.32.0 + + local-pkg@1.2.1: + dependencies: + mlly: 1.8.2 + pkg-types: 2.3.3 + quansync: 0.2.11 + + locate-character@3.0.0: {} + + longest-streak@3.1.0: {} + + lru-cache@11.5.2: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + magicast@0.5.4: + dependencies: + '@babel/parser': 7.29.8 + '@babel/types': 7.29.8 + source-map-js: 1.2.1 + + markdown-extensions@2.0.0: {} + + markdown-table@3.0.4: {} + + mdast-util-definitions@6.0.0: + dependencies: + '@types/mdast': 4.0.4 + '@types/unist': 3.0.3 + unist-util-visit: 5.1.0 + + mdast-util-directive@3.1.0: + dependencies: + '@types/mdast': 4.0.4 + '@types/unist': 3.0.3 + ccount: 2.0.1 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + parse-entities: 4.0.2 + stringify-entities: 4.0.4 + unist-util-visit-parents: 6.0.2 + transitivePeerDependencies: + - supports-color + + mdast-util-find-and-replace@3.0.2: + dependencies: + '@types/mdast': 4.0.4 + escape-string-regexp: 5.0.0 + unist-util-is: 6.0.1 + unist-util-visit-parents: 6.0.2 + + mdast-util-from-markdown@2.0.3: + dependencies: + '@types/mdast': 4.0.4 + '@types/unist': 3.0.3 + decode-named-character-reference: 1.3.0 + devlop: 1.1.0 + mdast-util-to-string: 4.0.0 + micromark: 4.0.2 + micromark-util-decode-numeric-character-reference: 2.0.2 + micromark-util-decode-string: 2.0.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + unist-util-stringify-position: 4.0.0 + transitivePeerDependencies: + - supports-color + + mdast-util-gfm-autolink-literal@2.0.1: + dependencies: + '@types/mdast': 4.0.4 + ccount: 2.0.1 + devlop: 1.1.0 + mdast-util-find-and-replace: 3.0.2 + micromark-util-character: 2.1.1 + + mdast-util-gfm-footnote@2.1.0: + dependencies: + '@types/mdast': 4.0.4 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + micromark-util-normalize-identifier: 2.0.1 + transitivePeerDependencies: + - supports-color + + mdast-util-gfm-strikethrough@2.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + + mdast-util-gfm-table@2.0.0: + dependencies: + '@types/mdast': 4.0.4 + devlop: 1.1.0 + markdown-table: 3.0.4 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + + mdast-util-gfm-task-list-item@2.0.0: + dependencies: + '@types/mdast': 4.0.4 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + + mdast-util-gfm@3.1.0: + dependencies: + mdast-util-from-markdown: 2.0.3 + mdast-util-gfm-autolink-literal: 2.0.1 + mdast-util-gfm-footnote: 2.1.0 + mdast-util-gfm-strikethrough: 2.0.0 + mdast-util-gfm-table: 2.0.0 + mdast-util-gfm-task-list-item: 2.0.0 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + + mdast-util-mdx-expression@2.0.1: + dependencies: + '@types/estree-jsx': 1.0.5 + '@types/hast': 3.0.5 + '@types/mdast': 4.0.4 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + + mdast-util-mdx-jsx@3.2.0: + dependencies: + '@types/estree-jsx': 1.0.5 + '@types/hast': 3.0.5 + '@types/mdast': 4.0.4 + '@types/unist': 3.0.3 + ccount: 2.0.1 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + parse-entities: 4.0.2 + stringify-entities: 4.0.4 + unist-util-stringify-position: 4.0.0 + vfile-message: 4.0.3 + transitivePeerDependencies: + - supports-color + + mdast-util-mdx@3.0.0: + dependencies: + mdast-util-from-markdown: 2.0.3 + mdast-util-mdx-expression: 2.0.1 + mdast-util-mdx-jsx: 3.2.0 + mdast-util-mdxjs-esm: 2.0.1 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + + mdast-util-mdxjs-esm@2.0.1: + dependencies: + '@types/estree-jsx': 1.0.5 + '@types/hast': 3.0.5 + '@types/mdast': 4.0.4 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + + mdast-util-phrasing@4.1.0: + dependencies: + '@types/mdast': 4.0.4 + unist-util-is: 6.0.1 + + mdast-util-to-hast@13.2.1: + dependencies: + '@types/hast': 3.0.5 + '@types/mdast': 4.0.4 + '@ungap/structured-clone': 1.3.3 + devlop: 1.1.0 + micromark-util-sanitize-uri: 2.0.1 + trim-lines: 3.0.1 + unist-util-position: 5.0.0 + unist-util-visit: 5.1.0 + vfile: 6.0.3 + + mdast-util-to-markdown@2.1.2: + dependencies: + '@types/mdast': 4.0.4 + '@types/unist': 3.0.3 + longest-streak: 3.1.0 + mdast-util-phrasing: 4.1.0 + mdast-util-to-string: 4.0.0 + micromark-util-classify-character: 2.0.1 + micromark-util-decode-string: 2.0.1 + unist-util-visit: 5.1.0 + zwitch: 2.0.4 + + mdast-util-to-string@4.0.0: + dependencies: + '@types/mdast': 4.0.4 + + mdn-data@2.0.28: {} + + mdn-data@2.27.1: {} + + micromark-core-commonmark@2.0.3: + dependencies: + decode-named-character-reference: 1.3.0 + devlop: 1.1.0 + micromark-factory-destination: 2.0.1 + micromark-factory-label: 2.0.1 + micromark-factory-space: 2.0.1 + micromark-factory-title: 2.0.1 + micromark-factory-whitespace: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-chunked: 2.0.1 + micromark-util-classify-character: 2.0.1 + micromark-util-html-tag-name: 2.0.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-resolve-all: 2.0.1 + micromark-util-subtokenize: 2.1.0 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-directive@3.0.2: + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.1 + micromark-factory-whitespace: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + parse-entities: 4.0.2 + + micromark-extension-gfm-autolink-literal@2.1.0: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-sanitize-uri: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-gfm-footnote@2.1.0: + dependencies: + devlop: 1.1.0 + micromark-core-commonmark: 2.0.3 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-sanitize-uri: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-gfm-strikethrough@2.1.0: + dependencies: + devlop: 1.1.0 + micromark-util-chunked: 2.0.1 + micromark-util-classify-character: 2.0.1 + micromark-util-resolve-all: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-gfm-table@2.1.1: + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-gfm-tagfilter@2.0.0: + dependencies: + micromark-util-types: 2.0.2 + + micromark-extension-gfm-task-list-item@2.1.0: + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-gfm@3.0.0: + dependencies: + micromark-extension-gfm-autolink-literal: 2.1.0 + micromark-extension-gfm-footnote: 2.1.0 + micromark-extension-gfm-strikethrough: 2.1.0 + micromark-extension-gfm-table: 2.1.1 + micromark-extension-gfm-tagfilter: 2.0.0 + micromark-extension-gfm-task-list-item: 2.1.0 + micromark-util-combine-extensions: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-mdx-expression@3.0.1: + dependencies: + '@types/estree': 1.0.9 + devlop: 1.1.0 + micromark-factory-mdx-expression: 2.0.3 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-events-to-acorn: 2.0.3 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-mdx-jsx@3.0.2: + dependencies: + '@types/estree': 1.0.9 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + micromark-factory-mdx-expression: 2.0.3 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-events-to-acorn: 2.0.3 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + vfile-message: 4.0.3 + + micromark-extension-mdx-md@2.0.0: + dependencies: + micromark-util-types: 2.0.2 + + micromark-extension-mdxjs-esm@3.0.0: + dependencies: + '@types/estree': 1.0.9 + devlop: 1.1.0 + micromark-core-commonmark: 2.0.3 + micromark-util-character: 2.1.1 + micromark-util-events-to-acorn: 2.0.3 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + unist-util-position-from-estree: 2.0.0 + vfile-message: 4.0.3 + + micromark-extension-mdxjs@3.0.0: + dependencies: + acorn: 8.18.0 + acorn-jsx: 5.3.2(acorn@8.18.0) + micromark-extension-mdx-expression: 3.0.1 + micromark-extension-mdx-jsx: 3.0.2 + micromark-extension-mdx-md: 2.0.0 + micromark-extension-mdxjs-esm: 3.0.0 + micromark-util-combine-extensions: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-factory-destination@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-factory-label@2.0.1: + dependencies: + devlop: 1.1.0 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-factory-mdx-expression@2.0.3: + dependencies: + '@types/estree': 1.0.9 + devlop: 1.1.0 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-events-to-acorn: 2.0.3 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + unist-util-position-from-estree: 2.0.0 + vfile-message: 4.0.3 + + micromark-factory-space@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-types: 2.0.2 + + micromark-factory-title@2.0.1: + dependencies: + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-factory-whitespace@2.0.1: + dependencies: + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-util-character@2.1.1: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-util-chunked@2.0.1: + dependencies: + micromark-util-symbol: 2.0.1 + + micromark-util-classify-character@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-util-combine-extensions@2.0.1: + dependencies: + micromark-util-chunked: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-util-decode-numeric-character-reference@2.0.2: + dependencies: + micromark-util-symbol: 2.0.1 + + micromark-util-decode-string@2.0.1: + dependencies: + decode-named-character-reference: 1.3.0 + micromark-util-character: 2.1.1 + micromark-util-decode-numeric-character-reference: 2.0.2 + micromark-util-symbol: 2.0.1 + + micromark-util-encode@2.0.1: {} + + micromark-util-events-to-acorn@2.0.3: + dependencies: + '@types/estree': 1.0.9 + '@types/unist': 3.0.3 + devlop: 1.1.0 + estree-util-visit: 2.0.0 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + vfile-message: 4.0.3 + + micromark-util-html-tag-name@2.0.1: {} + + micromark-util-normalize-identifier@2.0.1: + dependencies: + micromark-util-symbol: 2.0.1 + + micromark-util-resolve-all@2.0.1: + dependencies: + micromark-util-types: 2.0.2 + + micromark-util-sanitize-uri@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-encode: 2.0.1 + micromark-util-symbol: 2.0.1 + + micromark-util-subtokenize@2.1.0: + dependencies: + devlop: 1.1.0 + micromark-util-chunked: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-util-symbol@2.0.1: {} + + micromark-util-types@2.0.2: {} + + micromark@4.0.2: + dependencies: + '@types/debug': 4.1.13 + debug: 4.4.3 + decode-named-character-reference: 1.3.0 + devlop: 1.1.0 + micromark-core-commonmark: 2.0.3 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-chunked: 2.0.1 + micromark-util-combine-extensions: 2.0.1 + micromark-util-decode-numeric-character-reference: 2.0.2 + micromark-util-encode: 2.0.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-resolve-all: 2.0.1 + micromark-util-sanitize-uri: 2.0.1 + micromark-util-subtokenize: 2.1.0 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + transitivePeerDependencies: + - supports-color + + mlly@1.8.2: + dependencies: + acorn: 8.18.0 + pathe: 2.0.3 + pkg-types: 1.3.1 + ufo: 1.6.4 + + mrmime@2.0.1: {} + + ms@2.1.3: {} + + nanoid@3.3.17: {} + + neotraverse@0.6.18: {} + + nlcst-to-string@4.0.0: + dependencies: + '@types/nlcst': 2.0.3 + + node-fetch-native@1.6.7: {} + + node-mock-http@1.0.5: {} + + normalize-path@3.0.0: {} + + nth-check@2.1.1: + dependencies: + boolbase: 1.0.0 + + obug@2.2.1: {} + + ofetch@1.5.1: + dependencies: + destr: 2.0.5 + node-fetch-native: 1.6.7 + ufo: 1.6.4 + + ohash@2.0.11: {} + + oniguruma-parser@0.12.2: {} + + oniguruma-to-es@4.3.6: + dependencies: + oniguruma-parser: 0.12.2 + regex: 6.1.0 + regex-recursion: 6.0.2 + + p-limit@6.2.0: + dependencies: + yocto-queue: 1.2.2 + + p-queue@8.1.1: + dependencies: + eventemitter3: 5.0.4 + p-timeout: 6.1.4 + + p-timeout@6.1.4: {} + + package-manager-detector@1.8.0: {} + + pagefind@1.5.2: + optionalDependencies: + '@pagefind/darwin-arm64': 1.5.2 + '@pagefind/darwin-x64': 1.5.2 + '@pagefind/freebsd-x64': 1.5.2 + '@pagefind/linux-arm64': 1.5.2 + '@pagefind/linux-x64': 1.5.2 + '@pagefind/windows-arm64': 1.5.2 + '@pagefind/windows-x64': 1.5.2 + + parse-entities@4.0.2: + dependencies: + '@types/unist': 2.0.11 + character-entities-legacy: 3.0.0 + character-reference-invalid: 2.0.1 + decode-named-character-reference: 1.3.0 + is-alphanumerical: 2.0.1 + is-decimal: 2.0.1 + is-hexadecimal: 2.0.1 + + parse-latin@7.0.0: + dependencies: + '@types/nlcst': 2.0.3 + '@types/unist': 3.0.3 + nlcst-to-string: 4.0.0 + unist-util-modify-children: 4.0.0 + unist-util-visit-children: 3.0.0 + vfile: 6.0.3 + + parse5@7.3.0: + dependencies: + entities: 6.0.1 + + pathe@2.0.3: {} + + piccolore@0.1.3: {} + + picocolors@1.1.1: {} + + picomatch@2.3.2: {} + + picomatch@4.0.5: {} + + picomatch@4.0.7: {} + + pkg-types@1.3.1: + dependencies: + confbox: 0.1.8 + mlly: 1.8.2 + pathe: 2.0.3 + + pkg-types@2.3.3: + dependencies: + confbox: 0.3.1 + exsolve: 1.1.3 + pathe: 2.0.3 + + postcss-nested@6.2.0(postcss@8.5.25): + dependencies: + postcss: 8.5.25 + postcss-selector-parser: 6.1.4 + + postcss-selector-parser@6.1.4: + dependencies: + cssesc: 3.0.0 + util-deprecate: 1.0.2 + + postcss@8.5.25: + dependencies: + nanoid: 3.3.17 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + prismjs@1.30.0: {} + + prompts@2.4.2: + dependencies: + kleur: 3.0.3 + sisteransi: 1.0.5 + + property-information@7.2.0: {} + + quansync@0.2.11: {} + + radix3@1.1.2: {} + + readdirp@5.0.0: {} + + recma-build-jsx@1.0.0: + dependencies: + '@types/estree': 1.0.9 + estree-util-build-jsx: 3.0.1 + vfile: 6.0.3 + + recma-jsx@1.0.1(acorn@8.18.0): + dependencies: + acorn: 8.18.0 + acorn-jsx: 5.3.2(acorn@8.18.0) + estree-util-to-js: 2.0.0 + recma-parse: 1.0.0 + recma-stringify: 1.0.0 + unified: 11.0.5 + + recma-parse@1.0.0: + dependencies: + '@types/estree': 1.0.9 + esast-util-from-js: 2.0.1 + unified: 11.0.5 + vfile: 6.0.3 + + recma-stringify@1.0.0: + dependencies: + '@types/estree': 1.0.9 + estree-util-to-js: 2.0.0 + unified: 11.0.5 + vfile: 6.0.3 + + regex-recursion@6.0.2: + dependencies: + regex-utilities: 2.3.0 + + regex-utilities@2.3.0: {} + + regex@6.1.0: + dependencies: + regex-utilities: 2.3.0 + + rehype-expressive-code@0.41.7: + dependencies: + expressive-code: 0.41.7 + + rehype-format@5.0.1: + dependencies: + '@types/hast': 3.0.5 + hast-util-format: 1.1.0 + + rehype-minify-whitespace@6.0.2: + dependencies: + '@types/hast': 3.0.5 + hast-util-minify-whitespace: 1.0.1 + + rehype-parse@9.0.1: + dependencies: + '@types/hast': 3.0.5 + hast-util-from-html: 2.0.3 + unified: 11.0.5 + + rehype-raw@7.0.0: + dependencies: + '@types/hast': 3.0.5 + hast-util-raw: 9.1.0 + vfile: 6.0.3 + + rehype-recma@1.0.0: + dependencies: + '@types/estree': 1.0.9 + '@types/hast': 3.0.5 + hast-util-to-estree: 3.1.3 + transitivePeerDependencies: + - supports-color + + rehype-remark@10.0.1: + dependencies: + '@types/hast': 3.0.5 + '@types/mdast': 4.0.4 + hast-util-to-mdast: 10.1.2 + unified: 11.0.5 + vfile: 6.0.3 + + rehype-stringify@10.0.1: + dependencies: + '@types/hast': 3.0.5 + hast-util-to-html: 9.0.5 + unified: 11.0.5 + + rehype@13.0.2: + dependencies: + '@types/hast': 3.0.5 + rehype-parse: 9.0.1 + rehype-stringify: 10.0.1 + unified: 11.0.5 + + remark-directive@3.0.1: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-directive: 3.1.0 + micromark-extension-directive: 3.0.2 + unified: 11.0.5 + transitivePeerDependencies: + - supports-color + + remark-gfm@4.0.1: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-gfm: 3.1.0 + micromark-extension-gfm: 3.0.0 + remark-parse: 11.0.0 + remark-stringify: 11.0.0 + unified: 11.0.5 + transitivePeerDependencies: + - supports-color + + remark-mdx@3.1.1: + dependencies: + mdast-util-mdx: 3.0.0 + micromark-extension-mdxjs: 3.0.0 + transitivePeerDependencies: + - supports-color + + remark-parse@11.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-from-markdown: 2.0.3 + micromark-util-types: 2.0.2 + unified: 11.0.5 + transitivePeerDependencies: + - supports-color + + remark-rehype@11.1.2: + dependencies: + '@types/hast': 3.0.5 + '@types/mdast': 4.0.4 + mdast-util-to-hast: 13.2.1 + unified: 11.0.5 + vfile: 6.0.3 + + remark-smartypants@3.0.3: + dependencies: + retext: 9.0.0 + retext-smartypants: 6.2.0 + unified: 11.0.5 + unist-util-visit: 5.1.0 + + remark-stringify@11.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-to-markdown: 2.1.2 + unified: 11.0.5 + + retext-latin@4.0.0: + dependencies: + '@types/nlcst': 2.0.3 + parse-latin: 7.0.0 + unified: 11.0.5 + + retext-smartypants@6.2.0: + dependencies: + '@types/nlcst': 2.0.3 + nlcst-to-string: 4.0.0 + unist-util-visit: 5.1.0 + + retext-stringify@4.0.0: + dependencies: + '@types/nlcst': 2.0.3 + nlcst-to-string: 4.0.0 + unified: 11.0.5 + + retext@9.0.0: + dependencies: + '@types/nlcst': 2.0.3 + retext-latin: 4.0.0 + retext-stringify: 4.0.0 + unified: 11.0.5 + + rollup@4.62.4: + dependencies: + '@types/estree': 1.0.9 + optionalDependencies: + '@napi-rs/lzma-linux-x64-gnu': 1.5.1 + '@rollup/rollup-android-arm-eabi': 4.62.4 + '@rollup/rollup-android-arm64': 4.62.4 + '@rollup/rollup-darwin-arm64': 4.62.4 + '@rollup/rollup-darwin-x64': 4.62.4 + '@rollup/rollup-freebsd-arm64': 4.62.4 + '@rollup/rollup-freebsd-x64': 4.62.4 + '@rollup/rollup-linux-arm-gnueabihf': 4.62.4 + '@rollup/rollup-linux-arm-musleabihf': 4.62.4 + '@rollup/rollup-linux-arm64-gnu': 4.62.4 + '@rollup/rollup-linux-arm64-musl': 4.62.4 + '@rollup/rollup-linux-loong64-gnu': 4.62.4 + '@rollup/rollup-linux-loong64-musl': 4.62.4 + '@rollup/rollup-linux-ppc64-gnu': 4.62.4 + '@rollup/rollup-linux-ppc64-musl': 4.62.4 + '@rollup/rollup-linux-riscv64-gnu': 4.62.4 + '@rollup/rollup-linux-riscv64-musl': 4.62.4 + '@rollup/rollup-linux-s390x-gnu': 4.62.4 + '@rollup/rollup-linux-x64-gnu': 4.62.4 + '@rollup/rollup-linux-x64-musl': 4.62.4 + '@rollup/rollup-openbsd-x64': 4.62.4 + '@rollup/rollup-openharmony-arm64': 4.62.4 + '@rollup/rollup-win32-arm64-msvc': 4.62.4 + '@rollup/rollup-win32-ia32-msvc': 4.62.4 + '@rollup/rollup-win32-x64-gnu': 4.62.4 + '@rollup/rollup-win32-x64-msvc': 4.62.4 + fsevents: 2.3.3 + + sax@1.6.1: {} + + scule@1.3.0: {} + + semver@7.8.5: {} + + sharp@0.34.5: + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.8.5 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.34.5 + '@img/sharp-darwin-x64': 0.34.5 + '@img/sharp-libvips-darwin-arm64': 1.2.4 + '@img/sharp-libvips-darwin-x64': 1.2.4 + '@img/sharp-libvips-linux-arm': 1.2.4 + '@img/sharp-libvips-linux-arm64': 1.2.4 + '@img/sharp-libvips-linux-ppc64': 1.2.4 + '@img/sharp-libvips-linux-riscv64': 1.2.4 + '@img/sharp-libvips-linux-s390x': 1.2.4 + '@img/sharp-libvips-linux-x64': 1.2.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + '@img/sharp-linux-arm': 0.34.5 + '@img/sharp-linux-arm64': 0.34.5 + '@img/sharp-linux-ppc64': 0.34.5 + '@img/sharp-linux-riscv64': 0.34.5 + '@img/sharp-linux-s390x': 0.34.5 + '@img/sharp-linux-x64': 0.34.5 + '@img/sharp-linuxmusl-arm64': 0.34.5 + '@img/sharp-linuxmusl-x64': 0.34.5 + '@img/sharp-wasm32': 0.34.5 + '@img/sharp-win32-arm64': 0.34.5 + '@img/sharp-win32-ia32': 0.34.5 + '@img/sharp-win32-x64': 0.34.5 + + shiki@3.23.0: + dependencies: + '@shikijs/core': 3.23.0 + '@shikijs/engine-javascript': 3.23.0 + '@shikijs/engine-oniguruma': 3.23.0 + '@shikijs/langs': 3.23.0 + '@shikijs/themes': 3.23.0 + '@shikijs/types': 3.23.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + + sisteransi@1.0.5: {} + + sitemap@9.0.1: + dependencies: + '@types/node': 24.13.3 + '@types/sax': 1.2.7 + arg: 5.0.2 + sax: 1.6.1 + + smol-toml@1.7.1: {} + + source-map-js@1.2.1: {} + + source-map@0.7.6: {} + + space-separated-tokens@2.0.2: {} + + starlight-links-validator@0.19.2(@astrojs/starlight@0.36.3(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)))(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)): + dependencies: + '@astrojs/starlight': 0.36.3(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)) + '@types/picomatch': 3.0.2 + astro: 5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1) + github-slugger: 2.0.0 + hast-util-from-html: 2.0.3 + hast-util-has-property: 3.0.0 + is-absolute-url: 4.0.1 + kleur: 4.1.5 + mdast-util-mdx-jsx: 3.2.0 + mdast-util-to-string: 4.0.0 + picomatch: 4.0.7 + terminal-link: 5.0.0 + unist-util-visit: 5.1.0 + transitivePeerDependencies: + - supports-color + + starlight-llm-actions@0.13.0(@astrojs/mdx@4.3.14(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)))(@astrojs/starlight@0.36.3(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)))(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1))(hast-util-select@6.0.4)(rehype-parse@9.0.1)(rehype-remark@10.0.1)(remark-gfm@4.0.1)(remark-stringify@11.0.0)(unified@11.0.5)(unist-util-remove@4.0.0)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)): + dependencies: + '@astrojs/starlight': 0.36.3(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)) + astro: 5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1) + picomatch: 4.0.7 + vite: 6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1) + zod: 4.4.3 + optionalDependencies: + '@astrojs/mdx': 4.3.14(astro@5.18.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.4)(typescript@5.9.3)(yaml@2.9.1)) + hast-util-select: 6.0.4 + rehype-parse: 9.0.1 + rehype-remark: 10.0.1 + remark-gfm: 4.0.1 + remark-stringify: 11.0.0 + unified: 11.0.5 + unist-util-remove: 4.0.0 + + stream-replace-string@2.0.0: {} + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + string-width@7.2.0: + dependencies: + emoji-regex: 10.6.0 + get-east-asian-width: 1.6.0 + strip-ansi: 7.2.0 + + stringify-entities@4.0.4: + dependencies: + character-entities-html4: 2.1.0 + character-entities-legacy: 3.0.0 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + strip-ansi@7.2.0: + dependencies: + ansi-regex: 6.2.2 + + style-to-js@1.1.21: + dependencies: + style-to-object: 1.0.14 + + style-to-object@1.0.14: + dependencies: + inline-style-parser: 0.2.7 + + supports-color@10.2.2: {} + + supports-hyperlinks@4.6.0: + dependencies: + has-flag: 5.0.1 + supports-color: 10.2.2 + + svelte2tsx@0.7.61(svelte@5.57.2)(typescript@5.9.3): + dependencies: + dedent-js: 1.0.1 + scule: 1.3.0 + svelte: 5.57.2 + typescript: 5.9.3 + + svelte@5.57.2: + dependencies: + '@jridgewell/remapping': 2.3.5 + '@jridgewell/sourcemap-codec': 1.6.0 + '@sveltejs/acorn-typescript': 1.0.13(acorn@8.18.0) + '@types/estree': 1.0.9 + acorn: 8.18.0 + aria-query: 5.3.1 + axobject-query: 4.1.0 + clsx: 2.1.1 + devalue: 5.9.4 + esm-env: 1.2.2 + esrap: 2.4.0 + is-reference: 3.0.3 + locate-character: 3.0.0 + magic-string: 0.30.21 + zimmerframe: 1.1.5 + transitivePeerDependencies: + - '@typescript-eslint/types' + + svgo@4.0.2: + dependencies: + commander: 11.1.0 + css-select: 5.2.2 + css-tree: 3.2.1 + css-what: 6.2.2 + csso: 5.0.5 + picocolors: 1.1.1 + sax: 1.6.1 + + tailwind-merge@3.7.0: {} + + tailwind-variants@3.3.1(tailwind-merge@3.7.0)(tailwindcss@4.3.3): + optionalDependencies: + tailwind-merge: 3.7.0 + tailwindcss: 4.3.3 + + tailwindcss@4.3.3: {} + + tapable@2.3.3: {} + + terminal-link@5.0.0: + dependencies: + ansi-escapes: 7.3.0 + supports-hyperlinks: 4.6.0 + + tiny-inflate@1.0.3: {} + + tinyexec@1.3.0: {} + + tinyexec@1.3.1: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + + trim-lines@3.0.1: {} + + trim-trailing-lines@2.1.0: {} + + trough@2.2.0: {} + + tsconfck@3.1.6(typescript@5.9.3): + optionalDependencies: + typescript: 5.9.3 + + tslib@2.8.1: + optional: true + + type-fest@4.41.0: {} + + typescript@5.9.3: {} + + ufo@1.6.4: {} + + ultrahtml@1.7.0: {} + + uncrypto@0.1.3: {} + + undici-types@7.18.2: {} + + unified@11.0.5: + dependencies: + '@types/unist': 3.0.3 + bail: 2.0.2 + devlop: 1.1.0 + extend: 3.0.2 + is-plain-obj: 4.1.0 + trough: 2.2.0 + vfile: 6.0.3 + + unifont@0.7.4: + dependencies: + css-tree: 3.2.1 + ofetch: 1.5.1 + ohash: 2.0.11 + + unist-util-find-after@5.0.0: + dependencies: + '@types/unist': 3.0.3 + unist-util-is: 6.0.1 + + unist-util-is@6.0.1: + dependencies: + '@types/unist': 3.0.3 + + unist-util-modify-children@4.0.0: + dependencies: + '@types/unist': 3.0.3 + array-iterate: 2.0.1 + + unist-util-position-from-estree@2.0.0: + dependencies: + '@types/unist': 3.0.3 + + unist-util-position@5.0.0: + dependencies: + '@types/unist': 3.0.3 + + unist-util-remove-position@5.0.0: + dependencies: + '@types/unist': 3.0.3 + unist-util-visit: 5.1.0 + + unist-util-remove@4.0.0: + dependencies: + '@types/unist': 3.0.3 + unist-util-is: 6.0.1 + unist-util-visit-parents: 6.0.2 + + unist-util-stringify-position@4.0.0: + dependencies: + '@types/unist': 3.0.3 + + unist-util-visit-children@3.0.0: + dependencies: + '@types/unist': 3.0.3 + + unist-util-visit-parents@6.0.2: + dependencies: + '@types/unist': 3.0.3 + unist-util-is: 6.0.1 + + unist-util-visit@5.1.0: + dependencies: + '@types/unist': 3.0.3 + unist-util-is: 6.0.1 + unist-util-visit-parents: 6.0.2 + + unplugin-icons@24.0.0(esbuild@0.27.7)(rollup@4.62.4)(svelte@5.57.2)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)): + dependencies: + '@antfu/install-pkg': 1.1.0 + '@iconify/utils': 3.1.7 + local-pkg: 1.2.1 + obug: 2.2.1 + unplugin: 3.4.0(esbuild@0.27.7)(rollup@4.62.4)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)) + optionalDependencies: + svelte: 5.57.2 + transitivePeerDependencies: + - '@farmfe/core' + - '@rsbuild/core' + - '@rspack/core' + - bun-types-no-globals + - esbuild + - rolldown + - rollup + - unloader + - vite + - webpack + + unplugin@3.4.0(esbuild@0.27.7)(rollup@4.62.4)(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)): + dependencies: + '@jridgewell/remapping': 2.3.5 + picomatch: 4.0.7 + webpack-virtual-modules: 0.6.2 + optionalDependencies: + esbuild: 0.27.7 + rollup: 4.62.4 + vite: 6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1) + + unstorage@1.17.5: + dependencies: + anymatch: 3.1.3 + chokidar: 5.0.0 + destr: 2.0.5 + h3: 1.15.11 + lru-cache: 11.5.2 + node-fetch-native: 1.6.7 + ofetch: 1.5.1 + ufo: 1.6.4 + + util-deprecate@1.0.2: {} + + vfile-location@5.0.3: + dependencies: + '@types/unist': 3.0.3 + vfile: 6.0.3 + + vfile-message@4.0.3: + dependencies: + '@types/unist': 3.0.3 + unist-util-stringify-position: 4.0.0 + + vfile@6.0.3: + dependencies: + '@types/unist': 3.0.3 + vfile-message: 4.0.3 + + vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1): + dependencies: + esbuild: 0.25.12 + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + postcss: 8.5.25 + rollup: 4.62.4 + tinyglobby: 0.2.17 + optionalDependencies: + '@types/node': 24.13.3 + fsevents: 2.3.3 + jiti: 2.7.0 + lightningcss: 1.32.0 + yaml: 2.9.1 + + vitefu@1.1.3(vite@6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1)): + optionalDependencies: + vite: 6.4.3(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.1) + + web-namespaces@2.0.1: {} + + webpack-virtual-modules@0.6.2: {} + + which-pm-runs@1.1.0: {} + + widest-line@5.0.0: + dependencies: + string-width: 7.2.0 + + wrap-ansi@9.0.2: + dependencies: + ansi-styles: 6.2.3 + string-width: 7.2.0 + strip-ansi: 7.2.0 + + xxhash-wasm@1.1.0: {} + + yaml@2.9.1: + optional: true + + yargs-parser@21.1.1: {} + + yocto-queue@1.2.2: {} + + yocto-spinner@0.2.3: + dependencies: + yoctocolors: 2.2.0 + + yoctocolors@2.2.0: {} + + zimmerframe@1.1.5: {} + + zod-to-json-schema@3.25.2(zod@3.25.76): + dependencies: + zod: 3.25.76 + + zod-to-ts@1.2.0(typescript@5.9.3)(zod@3.25.76): + dependencies: + typescript: 5.9.3 + zod: 3.25.76 + + zod@3.25.76: {} + + zod@4.4.3: {} + + zwitch@2.0.4: {} diff --git a/docs/pnpm-workspace.yaml b/docs/pnpm-workspace.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbb26c82767012ed9c65d677831a4dc7dd4782d7 --- /dev/null +++ b/docs/pnpm-workspace.yaml @@ -0,0 +1,3 @@ +allowBuilds: + esbuild: true + sharp: true diff --git a/docs/public/apple-touch-icon.png b/docs/public/apple-touch-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..2fad9b55a19590c6e1e97f66aa1e47e050d03782 GIT binary patch literal 4666 zcmV-A62yqd~vVrCZ;OoG{z*;fHva%Ss1vncRt6j-N{rLcJtr(sQ1MVaP!2Q5y8b+OLAx;$)n#+wTwsx+AvepLz&C*Hv>PW;Elvb}AAte4lQ-nYmNpF% z9oYq#5rM&)%I|>vv>VNaVcH6KCad4>z>h^MNO1 z%WJ@aE~f%hbKI=PC*#>ICF2GMB=+JiZ zI&fj+TXHVzx|g;=RF!NYc7bgCYj>emNTM*=0{(g6F>OU+AWbvsA&WE}uwogZzbp<#iR!bZcqgh)KeYJk zw3|^oSq^NkVNBX1bhaw~nYdUc>O;1;U#&4pYdZM%0S^HODe92$$LkV1WL3zQ+B&Pd zv51=gF88 z!gvY8_M8OjiJolg@&$29(RPeU61U8d!0yqqtDFdFH&l{p$#nUON5vX%n~_W%$-wrGU$=OqrjeSbxR})u|P)Z#ho7Nv?#nJ z8FL~cvW4Q!c^et+Hn9%^{wsOQB_d-tiL&)Ty9ZhsZ1(SF0tfolEs-e1I+;(;Z1o>o z4d16EV@^a&A8@@K^ld`(lw&wBts&oDA}l`?C-?~k{&qQVcbBYt`hk_euks%JOa%Lm zvCHjp>%w)p*ED=FPX}H${JU&UIaP8vA}+n%aM2>Gvg`0;ZABvD3E5?-D@Py(y^FRc9ym{8ha}!6`;kARt%OI$ z0ONaP+p11{6Zn?45|Oe{TLI5Z?9obL>xnyB98o9UCiEV7X!h!nQ?oiSR~CVT+KNO> zUHZBmyZ4BbP1RWe94{_~iT2zgbLqOWAtz!Td}lj34<-JT&qmN+zGM#K`LC@=8JVhW zAr<7Z2>Q*3yg=u@ZRN2M!$jxc({&Ovhd*mFvs{9@By0#J(WKgDQZ)q#ynBJaH z89qI5JMdF&B`VDCfq&6f!ZFz>&C?NlDr{x71RY6~W2D7p+$zXRJyppY*TfWwbyih+ zB&+Trz)p&KtOPDqe7|c}WdHMn2!2^M@NHz1nEVzJ08zVNUC1x9 z3Vk&KMQkM0(e_Dl6XF@*en~7kvSgXpbg0*$;zfkIY~yXXMtE`rikQj^fnrtvN1ON0 zl@z~4*w3JTt2$lF_sH(@gVOHbvT_A-c~V;uQ#mz2=bo3N!-9f75#Kee^ z=Ne9FPO>cb;}%X}uUVGuid!T`NtJng=pRmx;MdhjW(Nqq-YdprqR(o}vahqK>#?5d zXTm)Z^mS2rmWEIG^rqk|u`!fa+$RTHTy}WXvh2exo_*4?Y?n+|e0N#0ALB-YeyvlO ze`V$dI^woT@@UiX{T5M+{x~$!C^19v-DTMjY^HIMb-kW09r;#((HRK^X39@Il{+&~ zUEA@Lgzl{xD~EoJjRrkcaY53Kr2%hmo9-{Kk+i+0>}B!nLl)2a;vQ`U3UK?7lU<`h zXA7OH^>k{<4+6S4jx&7jX*mEf#U@Ci%@4&F|EsM)0lwevKWr=WIPDedkQ@-OQC1u3 zWE-XRPQ!8sfp>VSn++2p=y%)Qjg13y^FhTDiE9x9^MP4`-Zi>iR;3oaVp;aChR;1K zgNZit!~v`-T-kU&(dy*wa`BPXCS)u&o7g0_MrZ0CS|k>=57q}9NXqh>B#7NeD}#Ph z{28>_B<8P2aIHrphbF~#P;Q3aY7@eGb&MI&s=88Fi?>a8Cc-C~TjRX)hWrfAFzL}@3fC1;X z+HLA&(3vrxrW9IhPT=?1tQl+~O>JZGtZz?38(!2_AcyJVOtlGljhM}2zv4%yj?54I zKAVN!R-Saw`Wm((rs+>JZ^-PSB_^Z7#R?g+LWg~|S?Jk}x?5O08&fdym6nG`Avvql z)~Sg>ua~Ju$P*-+Ijsw9P9tq1HQ78HTd3O@4f^W5&~K*mwpEfbb*3A=a`El^+{|i= z2vRmT&TWFp$L78pn?4Upj>(SLEJT(`Ug{9rX{LS^E?I2&+}0LlRymk_pSA)G%$IP< zxr2`F%6h5fkM`|1d$5Nx56GF*ng*`1xgcv}{4oV;9~M#E@K4vQ_jZ@Rz`~ z1^(8Qqa8MApPVSo%-I(Je`%y!^p0;@Z4Hoq^bz%y^!4Y`TJ*SnQ{2)(hcbAmRa63_sM;psLro;2uXZB$O zy=*RNxU-S1EN?cO%6$bp`ZU>!YcB7!oWI*M=jQiR%N>}% zF|23klhHhU^rflX0*umbfFX{QSa2>Yc(;Ye+IGNA9m=}Mpp4ytUo`z)_O+N?U|zYs zY?Xp7+z zhM}EK|5T#v1Eq}R5)9G>-w0^PDe6kHVpA~*WR2mI|75vlT)-C$pKRh>Y2^ezm#J*`vX5x>W6cDG>l6(%|j!J{FBFJhWpX6 z@;=}<7SFGjZL#0UO_W|4y$=S)d(<|jY8_Ctx`ALq-3B_UB z3KU>Ug%ggp<8aGM5V^QA=#M%oClqB*=t6S>aH6&n9_Rx;p{+zSJ4W#7vX$5asWj3r zC!E@C=1XpN*;p<{=#Vj-UGYwb&OE-k&hqwn)i~PabuSkNy|;rhG&n(90Rzqgj?-4c zGuz7Ex)WR&w936N2b`>}Ko0vz!lZ8CBd!W^VbH2~ze@0~h!?Y?CDT@jMEx=FVQmE( zc_e~`MNzq5yYX6(4KjB`&^94uMI>PpHteAQTxYz4seCC4w~3x-1SObHvOk4w&4X4)L&cfRc!x1 zAun`5mE_96p?rHAa+!C#M3ZB@E+h1i+!`pO>sT3;wCfv-3`ocab)Ji$F5QzlxwWyvv@NEG4ME(&q&`+I;~noT5ZSS-OPqgfOV zF-vYNClWTyaAC}Na+;Ye&N_*NDb5B#AF&W#6|XZVw|En^=S|>DSMC$O+!`lK68J<|if6!3o$=oo{X8Q9)}@F`$* z1dZ!TRso;Rsq-On8m6tQrrZ|DzEn6&@@{K4a#?xVBI3{a5$LNW3nB_E?~SpN=R~`) zi^|W$?a_uGMWAj?C0?F(XEe(!;7Et{90xobLA`uR(XUwT=#`z27*5$9 zk;tPyQdR5>JQG13T}w92_*)O{RSP~0+$jDap*m!ZoM!oq;y*;cToBbZLRYdJI3_^5 zRi05YYEScM+>goD+L*v~+Z^~_1jbla<^y{LZo67AQbvJq1CL8QkRlzBjl>T~TD2IG z@SiRtoVH;Wk(&c0Tk1&26lOjd4NL^yBa!oWl~HzU8U5DFR@-8Ux<6OK06*v=;-zE8 z%Po$hv=uQVYtz3MgH9yca$MvK#sTs7-cGxr64hb2Oo#p$fdMzl3}A2V#z|C{?d7`C z3S~pSC(zDMqD$BoxF~S4)8~Q9tFn7hmqlT>k@ef5l8x&qar|*{KXgz|FK5ew`Ced- zgK~qB47whSk{zBsfE|Ht@uIu|&^*NnAWvDb^#ANHmf%oF-~T wGU!CDNCus#70I9zwIUgGqE`HW00030|5f^4hi=v$n*aa+07*qoM6N<$f{VoCD*ylh literal 0 HcmV?d00001 diff --git a/docs/public/favicon.svg b/docs/public/favicon.svg new file mode 100644 index 0000000000000000000000000000000000000000..d5013ddcfdcf3195e814021e31b7e785cea40d87 --- /dev/null +++ b/docs/public/favicon.svg @@ -0,0 +1,21 @@ + + + + + + + diff --git a/docs/scripts/xrpc-api.mjs b/docs/scripts/xrpc-api.mjs new file mode 100644 index 0000000000000000000000000000000000000000..3a6914331ead8cf0f021bdafe4d211b6cd79a5c7 --- /dev/null +++ b/docs/scripts/xrpc-api.mjs @@ -0,0 +1,220 @@ +// generates docs/src/content/docs/api/*.md from the service routers and the lexicons +// run from anywhere: node docs/scripts/xrpc-api.mjs + +import { readFileSync, readdirSync, statSync, writeFileSync, mkdirSync } from "node:fs"; +import { join, dirname, relative } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), "../.."); +const out = join(root, "docs/src/content/docs/api"); + +const services = [ + { + slug: "knot", + title: "Knot", + description: "XRPC methods served by a knot.", + intro: "Served by `knot-server` at `https:///xrpc/`.", + rust: ["knot2/crates/knot-xrpc/src", "knot2/crates/knot-server/src"], + }, + { + slug: "spindle", + title: "Spindle", + description: "XRPC methods served by a spindle.", + intro: "Served by `spindle` at `https:///xrpc/`.", + go: ["spindle/xrpc/xrpc.go"], + }, + { + slug: "bobbin", + title: "Bobbin", + description: "XRPC methods served by bobbin, the Tangled API.", + intro: "Served by `bobbin` at `https:///xrpc/`. Ours is `https://api.tangled.org`.", + rust: ["bobbin/crates/xrpc/src", "bobbin/crates/bobbin/src"], + }, +]; + +function walk(dir, ext) { + const files = []; + for (const name of readdirSync(dir)) { + const path = join(dir, name); + if (statSync(path).isDirectory()) { + if (name !== "tests") files.push(...walk(path, ext)); + } else if (name.endsWith(ext) && name !== "tests.rs" && !name.endsWith("_test.go")) { + files.push(path); + } + } + return files; +} + +// strip test modules so test-only routes are not picked up +function nonTest(src) { + let out = ""; + let at = 0; + for (let i = src.indexOf("#[cfg(test)]"); i !== -1; i = src.indexOf("#[cfg(test)]", at)) { + out += src.slice(at, i); + const semi = src.indexOf(";", i); + let j = src.indexOf("{", i); + if (j === -1 || (semi !== -1 && semi < j)) { + at = semi + 1; + continue; + } + for (let depth = 0; j < src.length; j++) { + if (src[j] === "{") depth++; + else if (src[j] === "}" && --depth === 0) break; + } + at = j + 1; + } + return out + src.slice(at); +} + +function rustMethods(dirs) { + const files = dirs.flatMap((d) => walk(join(root, d), ".rs")).map((f) => nonTest(readFileSync(f, "utf8"))); + const consts = new Map(); + for (const src of files) { + for (const m of src.matchAll(/const\s+(\w+)\s*:\s*&(?:'static\s+)?str\s*=\s*"([^"]+)"/g)) { + consts.set(m[1], m[2]); + } + } + const methods = new Set(); + for (const src of files) { + for (const m of src.matchAll(/\.route\(\s*(?:"([^"]+)"|([\w:]+))\s*,/g)) { + const path = m[1] ?? consts.get(m[2].split("::").pop()); + if (path?.startsWith("/xrpc/") && path.includes(".")) methods.add(path.slice("/xrpc/".length)); + } + for (const m of src.matchAll(/const\s+\w+_NSIDS\s*:\s*&\[&str\]\s*=\s*&\[([^\]]*)\]/g)) { + for (const n of m[1].matchAll(/"([^"]+)"/g)) methods.add(n[1]); + } + } + return methods; +} + +function goMethods(files) { + const consts = new Map(); + for (const f of walk(join(root, "api"), ".go")) { + for (const m of readFileSync(f, "utf8").matchAll(/(\w+NSID)\s*=\s*"([^"]+)"/g)) consts.set(m[1], m[2]); + } + const methods = new Set(); + for (const f of files) { + const src = readFileSync(join(root, f), "utf8"); + for (const m of src.matchAll(/r\.(?:Get|Post)\("\/"\s*\+\s*\w+\.(\w+NSID)/g)) { + if (consts.has(m[1])) methods.add(consts.get(m[1])); + } + } + return methods; +} + +const lexicons = new Map(); +for (const f of walk(join(root, "lexicons"), ".json")) { + const doc = JSON.parse(readFileSync(f, "utf8")); + if (doc.id) lexicons.set(doc.id, doc); +} + +function resolve(ref, base) { + const [id, name] = ref.startsWith("#") ? [base, ref.slice(1)] : ref.split("#"); + const doc = lexicons.get(id); + return doc && { def: doc.defs[name ?? "main"], id }; +} + +function refName(ref, base) { + if (ref.startsWith("#")) return ref.slice(1); + const [id, name] = ref.split("#"); + return id === base ? name : name ? `${id}#${name}` : id; +} + +function typeOf(field, base) { + switch (field.type) { + case "ref": + return `\`${refName(field.ref, base)}\``; + case "union": + return field.refs.map((r) => `\`${refName(r, base)}\``).join(" or "); + case "array": + return `array of ${typeOf(field.items, base)}`; + case "string": + return field.format ? `string, ${field.format}` : "string"; + default: + return field.type; + } +} + +function fieldLine(name, field, required, base) { + const tags = [typeOf(field, base)]; + if (required) tags.push("required"); + const values = field.enum ?? field.knownValues; + if (values) tags.push(`${field.enum ? "one of" : "known values"} ${values.map((v) => `\`${v}\``).join(", ")}`); + if (field.default !== undefined) tags.push(`default \`${JSON.stringify(field.default)}\``); + if (field.minimum !== undefined && field.maximum !== undefined) tags.push(`${field.minimum} to ${field.maximum}`); + const description = field.description ? `: ${field.description.trim()}` : ""; + return `- \`${name}\` (${tags.join(", ")})${description}`; +} + +function objectLines(schema, base) { + if (schema?.type === "ref") { + const target = resolve(schema.ref, base); + if (!target) return [`- \`${refName(schema.ref, base)}\``]; + return objectLines(target.def, target.id); + } + if (schema?.type === "union") return [`- ${typeOf(schema, base)}`]; + const props = Object.entries(schema?.properties ?? {}); + if (props.length === 0) return ["- None."]; + const required = new Set(schema.required ?? []); + return props.map(([name, field]) => fieldLine(name, field, required.has(name), base)); +} + +function bodyLines(label, body, base) { + if (!body) return []; + const lines = [`${label}:`, ""]; + if (body.schema) { + if (body.encoding && body.encoding !== "application/json") lines.push(`- Encoding: \`${body.encoding}\``); + lines.push(...objectLines(body.schema, base)); + } else { + lines.push(`- Encoding: \`${body.encoding}\``); + } + return [...lines, ""]; +} + +const verbs = { query: "`GET`", procedure: "`POST`", subscription: "WebSocket" }; + +function methodSection(nsid) { + const lines = [`## ${nsid}`, ""]; + const main = lexicons.get(nsid)?.defs?.main; + if (!main) { + const note = nsid.startsWith("com.atproto.") + ? "Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto)." + : "No lexicon in this repository."; + return [...lines, note, ""]; + } + lines.push(main.description ? `${verbs[main.type]}. ${main.description.trim()}` : `${verbs[main.type]}.`, ""); + if (main.parameters && Object.keys(main.parameters.properties ?? {}).length) { + lines.push("Params:", "", ...objectLines(main.parameters, nsid), ""); + } + lines.push(...bodyLines("Input", main.input, nsid)); + lines.push(...bodyLines("Output", main.output, nsid)); + if (main.type === "subscription" && main.message) lines.push("Messages:", "", ...objectLines(main.message.schema, nsid), ""); + if (main.errors?.length) { + lines.push("Errors:", "", ...main.errors.map((e) => `- \`${e.name}\`${e.description ? `: ${e.description.trim()}` : ""}`), ""); + } + return lines; +} + +mkdirSync(out, { recursive: true }); +for (const service of services) { + const methods = [ + ...(service.rust ? rustMethods(service.rust) : []), + ...(service.go ? goMethods(service.go) : []), + ] + .filter((nsid) => !nsid.startsWith("sh.tangled.") && !nsid.startsWith("org.tangled.temp.") && !nsid.startsWith("blue.microcosm.")) + .sort(); + const lines = [ + "---", + `title: ${service.title}`, + `description: ${service.description}`, + "---", + "", + ``, + "", + service.intro, + "", + ...methods.flatMap(methodSection), + ]; + writeFileSync(join(out, `${service.slug}.md`), lines.join("\n").trimEnd() + "\n"); + console.log(`${service.slug}: ${methods.length} methods`); +} diff --git a/docs/src/assets/diagrams/bobbin.svg b/docs/src/assets/diagrams/bobbin.svg new file mode 100644 index 0000000000000000000000000000000000000000..b9c448a30ae3ed721bc23c5215e4b8e025de143b --- /dev/null +++ b/docs/src/assets/diagrams/bobbin.svg @@ -0,0 +1,70 @@ + + Hydrant collects org.tangled records; bobbin replays them through a four-stage pipeline into in-memory indexes and serves XRPC + + + + + + + + + bobbin (memory only) + + + + + firehoses + + + /stream + + + + + + + + + XRPC + + + + PDSes + users, orgs + + + Knots + repo accounts + + + Hydrant + org.tangled.* only + + + Prep + decode frames + + + Claim + serial, cursor order + + + Resolve + parallel, identities + + + Commit + serial + + + Indexes + edges, claims, tickets, search, actors + + + Web + /xrpc proxy + + + Deliberi + listRecipients + diff --git a/docs/src/assets/diagrams/spindle.svg b/docs/src/assets/diagrams/spindle.svg new file mode 100644 index 0000000000000000000000000000000000000000..b2b1d3eb2dfffb14bb810d6332c45297c41094dd --- /dev/null +++ b/docs/src/assets/diagrams/spindle.svg @@ -0,0 +1,73 @@ + + A spindle mill ingests events and places workflows on executors, which run each workflow in a microVM + + + + + + + + + executor host A + + executor host B + + + + repos, members + + + pushes + + + trigger, logs + + + dials /mill + + + + + + + + + logs, artifacts + + + + Hydrant + /subscribe + + + Knots + subscribeRepos + + + Web + XRPC, websocket + + + Spindle mill + ingest, place, serve + + + Artifact store + disk or S3 + + + Executor + seats, labels + + + microVM + shuttle agent over vsock + + + Executor + seats, labels + + + microVM + shuttle agent over vsock + diff --git a/docs/src/components/Diagram.astro b/docs/src/components/Diagram.astro new file mode 100644 index 0000000000000000000000000000000000000000..556dfb0e41d62a96538cf46179cdcf1f83817a3c --- /dev/null +++ b/docs/src/components/Diagram.astro @@ -0,0 +1,15 @@ +--- +// inlines a diagram svg so it can use the theme's css variables (see the +// .diagram rules in styles/diagrams.css). import the file with ?raw +interface Props { + svg: string; + caption?: string; +} + +const { svg, caption } = Astro.props; +--- + +
+ + {caption &&
{caption}
} +
diff --git a/docs/src/components/Header.astro b/docs/src/components/Header.astro new file mode 100644 index 0000000000000000000000000000000000000000..a01ef257044deb966990e4824ff8253f7d699a15 --- /dev/null +++ b/docs/src/components/Header.astro @@ -0,0 +1,16 @@ +--- +// only shown on narrow screens and on pages without a sidebar (the landing page). +// pages with a sidebar carry search in the sidebar, so it renders here only when +// there is none: one search per page +import Search from "./Search.astro"; +import SiteTitle from "./SiteTitle.astro"; + +const { hasSidebar } = Astro.locals.starlightRoute; +--- + +
+ +
+ {!hasSidebar && } +
+
diff --git a/docs/src/components/Hero.astro b/docs/src/components/Hero.astro new file mode 100644 index 0000000000000000000000000000000000000000..8fbcfd554c422f96ed05c9aa75b449e62a189d67 --- /dev/null +++ b/docs/src/components/Hero.astro @@ -0,0 +1,158 @@ +--- +// starlight's Hero (0.36), with the actions rendered as the web app's +// buttons. markup and styles otherwise match upstream +import { Image } from "astro:assets"; +import Button from "$lib/components/ui/Button.svelte"; +import ArrowRight from "~icons/lucide/arrow-right"; + +const { data } = Astro.locals.starlightRoute.entry; +const { title = data.title, tagline, image, actions = [] } = data.hero || {}; + +const imageAttrs = { + loading: "eager" as const, + decoding: "async" as const, + width: 400, + height: 400, + alt: image?.alt || "", +}; + +let darkImage: ImageMetadata | undefined; +let lightImage: ImageMetadata | undefined; +let rawHtml: string | undefined; +if (image) { + if ("file" in image) { + darkImage = image.file; + } else if ("dark" in image) { + darkImage = image.dark; + lightImage = image.light; + } else { + rawHtml = image.html; + } +} + +// the primary action is the web app's call to action, arrow on the right. +// the others need a text color: the web app sets it on every , starlight doesn't +const variants = { primary: "primary", secondary: "default", minimal: "ghost" } as const; +--- + +
+ { + darkImage && ( + + ) + } + {lightImage && } + {rawHtml &&
} +
+
+

+ {tagline &&
} +
+ { + actions.length > 0 && ( +
+ {actions.map(({ link: href, text, variant = "primary" }) => ( + + ))} +
+ ) + } +

+
+ + diff --git a/docs/src/components/MarkdownContent.astro b/docs/src/components/MarkdownContent.astro new file mode 100644 index 0000000000000000000000000000000000000000..cc3ab85c0be3c749564feb175315e6ede1008046 --- /dev/null +++ b/docs/src/components/MarkdownContent.astro @@ -0,0 +1,7 @@ +--- +// starlight's content wrapper, plus the web app's `markup` class, so pages render +// like a markdown file does on tangled.org (web/src/markup.css) +import Default from "@astrojs/starlight/components/MarkdownContent.astro"; +--- + +
 diff --git a/docs/src/components/MobileMenuFooter.astro b/docs/src/components/MobileMenuFooter.astro new file mode 100644 index 0000000000000000000000000000000000000000..0bf4c5a2d16d2569a181a33910175c72c249a33f --- /dev/null +++ b/docs/src/components/MobileMenuFooter.astro @@ -0,0 +1,3 @@ +--- +// there are no social links or languages, so the mobile menu needs no footer +--- diff --git a/docs/src/components/PageTitle.astro b/docs/src/components/PageTitle.astro new file mode 100644 index 0000000000000000000000000000000000000000..25d9347efa5e146e6ca71e014dc0a3838572fc2b --- /dev/null +++ b/docs/src/components/PageTitle.astro @@ -0,0 +1,5 @@ +--- +import Default from "@astrojs/starlight/components/PageTitle.astro"; +--- + + diff --git a/docs/src/components/PullSelectors.svelte b/docs/src/components/PullSelectors.svelte new file mode 100644 index 0000000000000000000000000000000000000000..e2dc7eab61bd2976f235d69501d482912983d754 --- /dev/null +++ b/docs/src/components/PullSelectors.svelte @@ -0,0 +1,142 @@ + + + + (versionRange = next)} /> + + {#snippet commitRow(_index: number, item: Item, actions: RowActions)} + {#if item === "base"} +
+ + + Base + +
+ {:else} +
+ +
+ + {item.subject} +
+ +
+ {/if} + {/snippet} + + + diff --git a/docs/src/components/RepoCreationName.svelte b/docs/src/components/RepoCreationName.svelte new file mode 100644 index 0000000000000000000000000000000000000000..0534392cb4cd2d98d2130836664184f70d78b869 --- /dev/null +++ b/docs/src/components/RepoCreationName.svelte @@ -0,0 +1,92 @@ + + + + + {#snippet children({ id })} +
+ + {#snippet trigger()} + + + {selected.handle} + + + +
+ {/snippet} + + + + diff --git a/docs/src/components/Search.astro b/docs/src/components/Search.astro new file mode 100644 index 0000000000000000000000000000000000000000..dca5e4d615b6cbc16dab43206aa04520cc3d9d6a --- /dev/null +++ b/docs/src/components/Search.astro @@ -0,0 +1,142 @@ +--- +// starlight's search, with each result's excerpt cut down to a little context +// before its first match, so the match survives the one-line clamp in tangled.css. +// excerpts that start or stop mid-sentence are marked so the stylesheet can add +// an ellipsis at that end. results show a page of cards at a time, the last +// one faded, and scrolling it into view shows the next page, asking pagefind +// for more once the shown cards run low. the arrow keys move through the results +// while focus stays in the input, and enter opens the active one. each card's +// title is prefixed with its page's title, unless the card is the page itself +import Default from "@astrojs/starlight/components/Search.astro"; +--- + + + + diff --git a/docs/src/components/Sidebar.astro b/docs/src/components/Sidebar.astro new file mode 100644 index 0000000000000000000000000000000000000000..e671d602935028ed6413be3b150a19812b23dcc8 --- /dev/null +++ b/docs/src/components/Sidebar.astro @@ -0,0 +1,79 @@ +--- +// the top of the page lives here instead of a top bar: logo, +// then search, then the nav. the nav is starlight's sidebar data rendered flat: +// group headings and links, no collapsing, no indentation (styles in tangled.css) +import type { StarlightRouteData } from "@astrojs/starlight/route-data"; +import Search from "./Search.astro"; +import SidebarPersister from "@astrojs/starlight/components/SidebarPersister.astro"; +import SiteTitle from "./SiteTitle.astro"; + +type Entry = StarlightRouteData["sidebar"][number]; + +const { sidebar } = Astro.locals.starlightRoute; +const links = (entries: Entry[]) => entries.filter((e) => e.type === "link"); +--- + +
+ +
+
+ + + + + + + diff --git a/docs/src/components/SiteTitle.astro b/docs/src/components/SiteTitle.astro new file mode 100644 index 0000000000000000000000000000000000000000..9d5a63557c51b769cd1269327f5dce4f7ede25b6 --- /dev/null +++ b/docs/src/components/SiteTitle.astro @@ -0,0 +1,10 @@ +--- +// the shared web logo, so the wordmark has one source +import Logo from "$lib/components/ui/Logo.svelte"; + +const { siteTitle, siteTitleHref } = Astro.locals.starlightRoute; +--- + + + + diff --git a/docs/src/components/ThemeProvider.astro b/docs/src/components/ThemeProvider.astro new file mode 100644 index 0000000000000000000000000000000000000000..cab622dc96835f7c6d63e43b7c3e823f41a11289 --- /dev/null +++ b/docs/src/components/ThemeProvider.astro @@ -0,0 +1,10 @@ + diff --git a/docs/src/components/VouchCardDemo.svelte b/docs/src/components/VouchCardDemo.svelte new file mode 100644 index 0000000000000000000000000000000000000000..19c8449766614e243044f26c9e9d2381fe56a1a5 --- /dev/null +++ b/docs/src/components/VouchCardDemo.svelte @@ -0,0 +1,38 @@ + + +
+
+ +
+
+ + diff --git a/docs/src/content.config.ts b/docs/src/content.config.ts new file mode 100644 index 0000000000000000000000000000000000000000..7fbcf2c33262d746e2fdfb8970347b6a8226a7ed --- /dev/null +++ b/docs/src/content.config.ts @@ -0,0 +1,7 @@ +import { defineCollection } from "astro:content"; +import { docsLoader } from "@astrojs/starlight/loaders"; +import { docsSchema } from "@astrojs/starlight/schema"; + +export const collections = { + docs: defineCollection({ loader: docsLoader(), schema: docsSchema() }), +}; diff --git a/docs/src/content/docs/api/bobbin.md b/docs/src/content/docs/api/bobbin.md new file mode 100644 index 0000000000000000000000000000000000000000..abd77fa36a969b01e0a7767ded0730ed95b831af --- /dev/null +++ b/docs/src/content/docs/api/bobbin.md @@ -0,0 +1,946 @@ +--- +title: Bobbin +description: XRPC methods served by bobbin, the Tangled API. +--- + + + +Served by `bobbin` at `https:///xrpc/`. Ours is `https://api.tangled.org`. + +## org.tangled.actor.getProfile + +`GET`. + +Params: + +- `actor` (string, at-uri, required): AT-URI of the sh.tangled.actor.profile record to fetch. + +Output: + +- `uri` (string, at-uri, required) +- `cid` (string, cid) +- `value` (unknown, required): Embedded sh.tangled.actor.profile record. + +## org.tangled.actor.getProfileActivity + +`GET`. Get recent Tangled activity for an actor, grouped by calendar month. + +Params: + +- `actor` (string, did, required): Actor whose activity to return. +- `months` (integer, default `7`, 1 to 12): Number of UTC calendar months to include, including the current month. + +Output: + +- `months` (array of `activityMonth`, required) +- `truncated` (boolean, required): True when the result reached the server activity-item safety limit. + +## org.tangled.actor.getProfiles + +`GET`. + +Params: + +- `actors` (array of string, at-uri, required): AT-URIs of the sh.tangled.actor.profile records to fetch. At most 50 per request. + +Output: + +- `items` (array of `recordView`, required) + +## org.tangled.actor.getTrending + +`GET`. Get trending people and repositories. + +Output: + +- `people` (array of `org.tangled.actor.defs#profileViewDetailed`, required) +- `repos` (array of `org.tangled.repo.defs#repoViewBasic`, required) + +## org.tangled.actor.searchActorsTypeahead + +`GET`. Find actors by their verified handle. + +Params: + +- `q` (string, required): The beginning of a handle. A leading @ is optional. +- `limit` (integer, default `10`, 1 to 100) + +Output: + +- `actors` (array of `actor`, required) + +## org.tangled.bobbin.awaitRecord + +`GET`. Wait for a record written to a PDS to reach this bobbin instance, then report what happened to it. + +Params: + +- `uri` (string, at-uri, required): AT-URI of the record that was written. +- `cid` (string, cid): CID of the revision that was written, as returned by createRecord, putRecord or applyWrites. Omit it only when awaiting a delete, which has no CID. any other responds with whatever revision of the record settles first. + +Output: + +- `status` (string, required, known values `indexed`, `deleted`, `rejected`): What bobbin did with the record. +- `cid` (string, cid): CID of the revision this result is about. +- `reason` (string, known values `unknownCollection`, `undecodable`, `invalid`, `missingBody`, `notIndexed`): Why a rejected record will not appear. `invalid` means the record decoded but breaks a lexicon constraint, such as a comment body past its length limit. +- `detail` (string): Opaque error message which can be more detailed than the reason. + +Errors: + +- `NotSettled`: The record had not reached the appview before the wait ran out +- `Overloaded`: Too many callers are already waiting, retry later or poll the record instead + +## org.tangled.bobbin.getCoverage + +No lexicon in this repository. + +## org.tangled.feed.countComments + +`GET`. + +Params: + +- `subject` (string, at-uri, required): Record AT-URI the comments attach to. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.feed.countCommentsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose comment authorings to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.feed.countReactions + +`GET`. + +Params: + +- `subject` (string, at-uri, required): Record AT-URI the reactions target. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.feed.countReactionsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose reaction authorings to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.feed.countStars + +`GET`. + +Params: + +- `subject` (string, required): Repo DID to list star edges for. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.feed.countStarsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose star authorings to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.feed.getStar + +`GET`. + +Params: + +- `actor` (string, did, required): DID of the potential stargazer. +- `subject` (string, required): Repo DID to check. + +Output: + +- `uri` (string, at-uri, required): Uri of actor's star record for subject. + +## org.tangled.feed.getTimeline + +`GET`. + +Params: + +- `viewer` (string, did): TODO: deprecate this and use auth instead. +- `followingOnly` (boolean) +- `limit` (integer, default `50`, 1 to 100) +- `cursor` (string) + +Output: + +- `feed` (array of `timelineItem`, required) +- `cursor` (string) + +## org.tangled.feed.listComments + +`GET`. + +Params: + +- `subject` (string, at-uri, required): Record AT-URI the comments attach to. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `org.tangled.feed.defs#commentView`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.feed.listCommentsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose comment authorings to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `org.tangled.feed.listComments#listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.feed.listReactions + +`GET`. + +Params: + +- `subject` (string, at-uri, required): Record AT-URI the reactions target. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.feed.listReactionsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose reaction authorings to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `org.tangled.feed.listReactions#listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.feed.listStars + +`GET`. + +Params: + +- `subject` (string, required): Repo DID to list star edges for. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.feed.listStarsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose star authorings to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `org.tangled.feed.listStars#listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.feed.subscription.getForActor + +No lexicon in this repository. + +## org.tangled.graph.countFollows + +`GET`. + +Params: + +- `subject` (string, did, required): Followee DID whose inbound follows to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.graph.countFollowsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose follow authorings to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.graph.getFollow + +`GET`. + +Params: + +- `actor` (string, did, required): DID of the potential follower. +- `subject` (string, did, required): Followee DID to check. + +Output: + +- `uri` (string, at-uri, required): Uri of actor's follow record for subject. + +## org.tangled.graph.listFollows + +`GET`. + +Params: + +- `subject` (string, did, required): Followee DID whose inbound follows to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.graph.listFollowsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose follow authorings to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `org.tangled.graph.listFollows#listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.graph.listNetworkVouches + +`GET`. List vouches involving one or more actors as seen through a viewer's vouch network: vouches the viewer's network placed on an actor, and vouches an actor placed on the viewer's network. Requires authentication. + +Params: + +- `actors` (array of string, did, required): DIDs of the profiles whose network vouches to list. At most 50 per request. +- `cursor` (string): Pagination cursor. Only accepted when a single actor is requested. +- `limit` (integer, default `50`, 1 to 100): Maximum number of vouches per actor. + +Output: + +- `items` (array of `listItem`, required) +- `cursor` (string) + +## org.tangled.label.countDefinitions + +`GET`. + +Params: + +- `subject` (string, required): DID of the repo whose label definitions to count. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.label.listDefinitions + +`GET`. + +Params: + +- `subject` (string, required): DID of the repo whose label definitions to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.publicKey.countKeys + +`GET`. + +Params: + +- `subject` (string, did, required): Owner DID whose public-key records to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.publicKey.getPublicKey + +`GET`. + +Params: + +- `publicKey` (string, at-uri, required): AT-URI of the sh.tangled.publicKey record to fetch. + +Output: + +- `uri` (string, at-uri, required) +- `cid` (string, cid) +- `value` (unknown, required): Embedded sh.tangled.publicKey record. + +## org.tangled.publicKey.listKeys + +`GET`. + +Params: + +- `subject` (string, did, required): Owner DID whose public-key records to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.query.enrichResponse + +`POST`. Run an XRPC query and hydrate its response with a data sidecar: typed payloads for every did / at-uri / strong-ref found in the response. + +Input: + +- `xrpc` (string, nsid, required): NSID of the inner query to run, e.g. org.tangled.repo.listRepos. +- `params` (unknown): Parameters for the inner query, exactly as it declares them. +- `enrich` (array of `linkDescriptor`, required): Payloads to compute for each found reference. +- `viewer` (string, did): DID whose own relation to each reference is looked up for viewer payloads and views. Required when any descriptor requests a viewer payload. + +Output: + +- `output` (unknown, required): The inner query's response, unmodified. +- `data` (unknown, required): Reference (did or at-uri) -> link source -> payload type nsid -> payload. References no descriptor applies to are absent. + +Errors: + +- `UnknownQuery`: The xrpc parameter does not name a query this server can execute +- `InvalidTargetPath`: A targets entry is not a valid record path +- `InvalidLinkDescriptor`: An enrich entry names an unknown payload type, collection, or path + +## org.tangled.repo.countArtifacts + +`GET`. + +Params: + +- `subject` (string, required): Repo or release identifier whose artifacts to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.repo.countArtifactsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose artifact authorings to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.repo.countCollaborators + +`GET`. + +Params: + +- `subject` (string, did, required): Repo DID whose collaborator records to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.repo.countCollaboratorsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose collaborator authorings to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.repo.countForks + +`GET`. + +Params: + +- `subject` (string, did, required): Repo DID to count forks of. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.repo.countIssues + +`GET`. + +Params: + +- `subject` (string, did, required): Repo DID to list issues for +- `author` (string, did): Restrict to issues authored by this user DID. +- `state` (string, known values `open`, `closed`): Restrict to issues whose latest derived state matches. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.repo.countPulls + +`GET`. + +Params: + +- `subject` (string, did, required): Repo DID to list pulls for +- `author` (string, did): Restrict to pulls authored by this user DID. +- `status` (string, known values `open`, `closed`, `merged`): Restrict to pulls whose latest derived status matches. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.repo.countRepos + +`GET`. + +Params: + +- `subject` (string, did, required): Owner DID whose repo records to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.repo.getRepo + +`GET`. + +Params: + +- `repo` (string, at-uri, required): AT-URI of the sh.tangled.repo record to fetch. + +Output: + +- `uri` (string, at-uri, required) +- `cid` (string, cid) +- `value` (unknown, required): Embedded sh.tangled.repo record. +- `knot` (string): Host of the knot named by the #tangled_knot service in the repo's DID document. Absent when it names none. + +## org.tangled.repo.getRepoByName + +`GET`. + +Params: + +- `owner` (string, did, required): DID of the account that owns the repo. +- `name` (string, required): Name of the repo as it appears in its url. + +Output: + +- `uri` (string, at-uri, required) +- `cid` (string, cid) +- `value` (unknown, required): Embedded sh.tangled.repo record. +- `knot` (string): Host of the knot named by the #tangled_knot service in the repo's DID document. Absent when it names none. + +## org.tangled.repo.getRepoByRepoDid + +`GET`. + +Params: + +- `repoDid` (string, did, required): Repo DID whose sh.tangled.repo record to fetch. + +Output: + +- `uri` (string, at-uri, required) +- `cid` (string, cid) +- `value` (unknown, required): Embedded sh.tangled.repo record. +- `knot` (string): Host of the knot named by the #tangled_knot service in the repo's DID document. Absent when it names none. + +## org.tangled.repo.getRepos + +`GET`. + +Params: + +- `repos` (array of string, at-uri, required): AT-URIs of the sh.tangled.repo records to fetch. At most 50 per request. + +Output: + +- `items` (array of `recordView`, required) + +## org.tangled.repo.getReposByRepoDids + +`GET`. + +Params: + +- `dids` (array of string, did, required): Repo DIDs to fetch. DIDs that resolve to none are omitted from the response. + +Output: + +- `items` (array of `listItem`, required) + +## org.tangled.repo.listArtifacts + +`GET`. + +Params: + +- `subject` (string, required): Repo or release identifier whose artifacts to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.repo.listArtifactsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose artifact authorings to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `org.tangled.repo.listArtifacts#listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.repo.listCollaboratorInvitesBy + +`GET`. Collaboration offers an actor DID has waiting for them. The inverse of sh.tangled.repo.listCollaboratorInvites, which knots serve only for their own repo. + +Params: + +- `subject` (string, did, required): Actor DID offered collaboration. + +Output: + +- `items` (array of `inviteItem`, required) +- `pending` (array of string, did, required): Knots that this index hasn't read yet +- `truncated` (boolean, required): Per-subject dropped offers from items. If truncated is true, missing offers won't say if they were withdrawn, same with pending knots. + +## org.tangled.repo.listCollaborators + +`GET`. + +Params: + +- `subject` (string, did, required): Repo DID whose collaborator records to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by effectiveSince. + +Output: + +- `items` (array of `listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.repo.listCollaboratorsBy + +`GET`. + +Params: + +- `subject` (string, did, required): Actor DID whose collaborator authorings to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `org.tangled.repo.listCollaborators#listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.repo.listForks + +`GET`. + +Params: + +- `subject` (string, did, required): Repo DID to list forks of. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.repo.listIssues + +`GET`. + +Params: + +- `subject` (string, did, required): Repo DID to list issues for +- `author` (string, did): Restrict to issues authored by this user DID. +- `state` (string, known values `open`, `closed`): Restrict to issues whose latest derived state matches. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `issueListItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.repo.listPulls + +`GET`. + +Params: + +- `subject` (string, did, required): Repo DID to list pulls for +- `author` (string, did): Restrict to pulls authored by this user DID. +- `status` (string, known values `open`, `closed`, `merged`): Restrict to pulls whose latest derived status matches. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `pullListItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.repo.listRepos + +`GET`. + +Params: + +- `subject` (string, did, required): Owner DID whose repo records to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.search.query + +`GET`. Full-text and faceted search over indexed records. + +Params: + +- `q` (string): Search query: free text plus key:value qualifiers. See the docs for more information. +- `nsid` (string, nsid): Restrict to records of this collection. +- `author` (string, did): Restrict to records authored by this DID. +- `repo` (string, did): Restrict to records under this repo DID. +- `since` (string, datetime): Restrict to records created at or after this time. +- `until` (string, datetime): Restrict to records created at or before this time. +- `kind` (string, known values `issue`, `pull`): Restrict tickets to issues or pull requests. +- `label` (array of string): Restrict tickets to those with all of these label names. Requires repo. +- `sort` (string, known values `relevance`, `created-desc`, `created-asc`): Result order. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for numbered pages. Mutually exclusive with cursor. +- `limit` (integer, default `50`, 1 to 1000) + +Output: + +- `hits` (array of `hit`, required) +- `cursor` (string) +- `total` (integer): Number of matches across all pages. +- `stateCounts` (`stateCounts`): Matches per ticket state, without the state filters. + +Errors: + +- `InvalidQuery`: The query failed to parse. + +## org.tangled.string.countStrings + +`GET`. + +Params: + +- `subject` (string, required): Scope identifier whose string records to list. + +Output: + +- `count` (integer, required): Total number of matching records. +- `distinctAuthors` (integer, required): Number of distinct authors among the matching records. + +## org.tangled.string.getString + +`GET`. + +Params: + +- `uri` (string, at-uri, required): AT-URI of the sh.tangled.string record to fetch. + +Output: + +- `uri` (string, at-uri, required) +- `cid` (string, cid) +- `value` (unknown, required): Embedded sh.tangled.string record. + +## org.tangled.string.listStrings + +`GET`. + +Params: + +- `subject` (string, required): Scope identifier whose string records to list. +- `cursor` (string): Pagination cursor +- `offset` (integer): Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor. +- `limit` (integer, default `50`, 1 to 1000) +- `order` (string, known values `asc`, `desc`, default `"desc"`): Sort direction by createdAt. + +Output: + +- `items` (array of `listItem`, required) +- `cursor` (string) +- `total` (integer): Total items in the full list; omitted for filtered or merged views + +## org.tangled.track.getTicket + +No lexicon in this repository. + +## org.tangled.track.getTicketView + +`GET`. + +Params: + +- `uri` (string, at-uri, required) + +Output: + +- `uri` (string, at-uri, required) +- `cid` (string, cid, required) +- `repo` (`org.tangled.repo.defs#repoViewBasic`, required) +- `author` (`org.tangled.actor.defs#profileViewBasic`, required): The account that opened the ticket, the editor of its first version. Not the account that last edited or closed it. +- `title` (`org.tangled.markup.markdown#inline`, required) +- `body` (`org.tangled.markup.markdown`) +- `state` (`ticketState`, required) +- `tangledLabels` (array of `org.tangled.label.defs#labelToken`) +- `targetBranch` (string): Present when the ticket is a pull request. +- `createdAt` (string, datetime, required) + +## org.tangled.track.getTickets + +No lexicon in this repository. diff --git a/docs/src/content/docs/api/knot.md b/docs/src/content/docs/api/knot.md new file mode 100644 index 0000000000000000000000000000000000000000..b21bb79715647f23fd0bb8020216977410099617 --- /dev/null +++ b/docs/src/content/docs/api/knot.md @@ -0,0 +1,510 @@ +--- +title: Knot +description: XRPC methods served by a knot. +--- + + + +Served by `knot-server` at `https:///xrpc/`. + +## com.atproto.identity.resolveHandle + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.repo.createRecord + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.repo.deleteRecord + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.repo.describeRepo + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.repo.getRecord + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.repo.listRecords + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.repo.putRecord + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.server.describeServer + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.sync.getBlob + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.sync.getBlocks + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.sync.getLatestCommit + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.sync.getRecord + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.sync.getRepo + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.sync.getRepoStatus + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.sync.listBlobs + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.sync.listRepos + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + +## com.atproto.sync.subscribeRepos + +Standard atproto method, see the [atproto lexicons](https://github.com/bluesky-social/atproto/tree/main/lexicons/com/atproto). + + + +## org.tangled.git.receivePack + +`POST`. No-op xrpc endpoint for `git-push` permission scope. + +## org.tangled.git.setDefaultBranch + +`POST`. Set the default branch of this repository + +Input: + +- `repo` (string, did, required): DID of the repository +- `branch` (string, required): Branch name without the refs/heads/ prefix + +Output: + +- `uri` (string, at-uri, required): The repository's org.tangled.git.refs record +- `cid` (string, cid, required): Version of the refs record after the move + +## org.tangled.knot.describeKnot + +`GET`. + +Output: + +- `did` (string, did, required) +- `version` (string, required) +- `capabilities` (array of string): Protocol capability tokens this knot implements. Knots that omit this field are treated as legacy. +- `links` (`links`): URLs of service policy documents. +- `contact` (`contact`): Contact information + +## org.tangled.repo.createRepo + +`POST`. Create a new repository + +Input: + +- `name` (string, record-key, required): Name of the repository. +- `did` (string, did): Optional user-provided DID to use as the repo identity instead of minting one. +- `defaultBranch` (string): Default branch of the repository. +- `spindle` (string, did) +- `description` (string) +- `website` (string, uri): Any URI related to the repo +- `topics` (array of string): Topics related to the repo +- `source` (`org.tangled.repo.defs#sourceTangled` or `org.tangled.repo.defs#sourceExternal`): A source to clone from, populate this when forking or importing a repository. +- `createdAt` (string, datetime) + +Output: + +- `did` (string, did) +- `key` (string): Multibase-encoded public signing key the knot holds for this repository + +## org.tangled.repo.deleteRepo + +`POST`. Delete a repository + +Input: + +- `repo` (string, did, required) +- `force` (boolean): Skip the check for a declaration record on the owner's PDS. Knot admins only. + +## org.tangled.repo.getImportStatus + +`GET`. How far the knot got importing a repository from its source. Only the repository owner and knot admins see the real reason; everyone else gets a generic one. + +Params: + +- `repo` (string, did, required): DID of the repository + +Output: + +- `status` (string, known values `fetching`, `present`, `partial`, `failed`): Where the import from the source is. Omitted when the repository wasn't created from a source. +- `reason` (string): Why a partial or failed import ended that way + +Errors: + +- `RepoNotFound`: Repo is not registered on this knot +- `InvalidRequest`: Invalid request parameters + +## org.tangled.repo.listRecordVersions + +`GET`. Every version of a record this knot hosts, oldest first unless order says otherwise. A version's bytes are fetched by content id through com.atproto.sync.getBlocks. + +Params: + +- `uri` (string, at-uri, required): Record in a repo this knot hosts +- `limit` (integer, default `100`, 1 to 500) +- `cursor` (string): Pagination cursor +- `order` (string, known values `asc`, `desc`, default `"asc"`): asc lists the oldest version first, desc the newest. + +Output: + +- `uri` (string, at-uri, required) +- `versions` (array of `version`, required) +- `cursor` (string): Cursor for the next page, left out when the page already reaches the newest version + +Errors: + +- `InvalidRequest`: The uri isn't an at-uri, or the collection isn't versioned +- `NotFound`: No such record on this knot +- `ProjectionWarming`: The record's change history is still being folded; retry shortly + +## org.tangled.repo.mintToken + +`POST`. Mint a service-auth JWT whose issuer is this repo's DID, signed by the knot that owns the repo's signing key. When calling this, you'll need the same perms as git push, ie. being repo owner/collaborator/knot-admin. + +Input: + +- `repo` (string, did, required): DID of the repo that becomes the token's issuer. Must be the audience of a given calling session. +- `aud` (string, did, required): Audience of the minted token, a DID service ref for the service, that the token is addressed to. +- `lxm` (string, nsid, required): Lexicon method to which minted token binds. +- `exp` (integer, required): Unix seconds. Must be in the future, and at most 60s after mint time. + +Output: + +- `token` (string, required): The signed service-auth JWT. + +## org.tangled.repo.reserveKey + +`POST`. Reserve a repo identity under a DID the caller brings. The knot seals and holds a signing key for the repo DID; the caller publishes the matching public key in that DID's document before calling org.tangled.repo.createRepo with the same DID. + +Input: + +- `repo` (string, did, required): The caller-minted DID to reserve + +Output: + +- `repo` (string, did, required) +- `key` (string, required): Multibase-encoded public signing key the knot holds for this repo identity + +## org.tangled.review.submitPatch + +`POST`. Create a patch record, fetching its head from the source repository first. The only way to write org.tangled.review.patch records. + +Input: + +- `repo` (string, did, required): DID of the repository +- `patch` (`patch`, required) + +Output: + +- `uri` (string, at-uri, required): AT-URI of created org.tangled.review.patch record. +- `cid` (string, cid, required): CID of created org.tangled.review.patch record. diff --git a/docs/src/content/docs/api/spindle.md b/docs/src/content/docs/api/spindle.md new file mode 100644 index 0000000000000000000000000000000000000000..33a59fe6e5ddffa12a776c716d3e9dc9f9fde4e5 --- /dev/null +++ b/docs/src/content/docs/api/spindle.md @@ -0,0 +1,236 @@ +--- +title: Spindle +description: XRPC methods served by a spindle. +--- + + + +Served by `spindle` at `https:///xrpc/`. + +## org.tangled.ci.approvePipeline + +`POST`. Approve blocked pipeline. + +Input: + +- `repo` (string, did, required): git repository DID +- `pipeline` (string, record-key, required): pipeline id + +Errors: + +- `PipelineNotFound`: Pipeline not found +- `PipelineNotBlocked`: Pipeline has no workflow waiting for approval + +## org.tangled.ci.getWorkflowDefinition + +`GET`. + +Params: + +- `repo` (string, did, required) +- `workflow` (string, required) + +Output: + +- `workflow` (`org.tangled.ci.workflow`, required) + +Errors: + +- `InvalidRequest`: Invalid request parameters +- `RepoNotFound`: Repository not found +- `WorkflowNotFound`: Workflow not found + +## org.tangled.ci.listWorkflowDefinitions + +`GET`. + +Params: + +- `repo` (string, did, required) +- `ref` (string, default `"HEAD"`) + +Output: + +- `workflows` (array of `org.tangled.ci.workflow`, required) + +Errors: + +- `InvalidRequest`: Invalid request parameters +- `RepoNotFound`: Repository not found +- `RefNotFound`: No branch or tag of the repository has this name + +## org.tangled.ci.retryPipeline + +`POST`. Duplicate existing pipeline for rerun. + +Input: + +- `repo` (string, did, required): git repository DID +- `pipeline` (string, record-key, required): pipeline id +- `workflows` (array of string): Workflow names to filter. When not provided, entire pipeline will be duplicated. + +Output: + +- `pipeline` (string, record-key, required): Scheduled pipeline id + +Errors: + +- `PipelineNotFound`: Pipeline not found +- `PipelineNotRetryable`: Pipeline is still blocked or has no stored spec to rerun + +## org.tangled.event.notifyEvent + +`POST`. notify an event from external source + +Input: + +- `repo` (string, did, required): Repository DID +- `event` (`org.tangled.ci.trigger.manual` or `org.tangled.event.push`, required): Tangled event object + +Output: + +- `pipeline` (string): Spindle-local id of the scheduled pipeline +- `message` (string): Text returned by the server for this push, like how to follow a started CI. + +Errors: + +- `InvalidRequest`: Invalid request parameters + +## org.tangled.secret.addSecret + +`POST`. Add a CI secret + +Input: + +- `repo` (string, did, required) +- `key` (string, required) +- `value` (string, required) + +Errors: + +- `InvalidRequest`: Invalid request parameters +- `RepoNotFound`: Repository not found +- `SecretAlreadyExists`: A secret with this key already exists. + +## org.tangled.secret.listSecrets + +`GET`. + +Params: + +- `repo` (string, did, required) + +Output: + +- `secrets` (array of `secret`, required) + +Errors: + +- `InvalidRequest`: Invalid request parameters +- `RepoNotFound`: Repository not found + +## org.tangled.secret.removeSecret + +`POST`. Remove a CI secret + +Input: + +- `repo` (string, did, required) +- `key` (string, required) + +Errors: + +- `InvalidRequest`: Invalid request parameters +- `RepoNotFound`: Repository not found +- `SecretNotFound`: The requested secret does not exist. + +## org.tangled.webhook.createWebhook + +`POST`. Create a new webhook for a repository. + +Input: + +- `repo` (string, did, required): DID of the repository as minted by the knot. +- `url` (string, uri, required): Endpoint URL that will receive webhook payloads. +- `active` (boolean, default `true`): Whether the webhook should be active immediately. +- `events` (array of `org.tangled.webhook.defs#eventType`, required): Event types to subscribe to. +- `secret` (string): Optional HMAC secret used to sign payloads. If omitted, payloads are not signed. + +Output: + +- `id` (string, required): ID of the newly created webhook. + +Errors: + +- `NoEventsSelected`: At least one event type must be specified. + +## org.tangled.webhook.deleteWebhook + +`POST`. Delete a webhook from a repository. + +Input: + +- `repo` (string, did, required): DID of the repository as minted by the knot. +- `webhook` (string, required): Webhook ID to delete. + +Errors: + +- `WebhookNotFound` + +## org.tangled.webhook.listWebhookDeliveries + +`GET`. List recent delivery attempts for a webhook. + +Params: + +- `repo` (string, did, required): DID of the repository as minted by the knot. +- `webhooks` (array of string, required) +- `limit` (integer, default `50`, 1 to 100) + +Output: + +- `deliveries` (array of `org.tangled.webhook.defs#delivery`, required) + +## org.tangled.webhook.listWebhooks + +`GET`. List webhooks configured for a repository. + +Params: + +- `repo` (string, did, required): DID of the repository as minted by the knot. + +Output: + +- `webhooks` (array of `org.tangled.webhook.defs#webhook`, required) + +## org.tangled.webhook.retryWebhookDelivery + +`POST`. Re-send a specific webhook delivery. + +Input: + +- `repo` (string, did, required): DID of the repository as minted by the knot. +- `webhook` (string, required): Webhook ID that owns the delivery. +- `delivery` (string, required): Delivery ID to retry. + +Errors: + +- `WebhookNotFound` +- `DeliveryNotFound` + +## org.tangled.webhook.updateWebhook + +`POST`. Update an existing webhook. Only supplied fields are changed. + +Input: + +- `repo` (string, did, required): DID of the repository as minted by the knot. +- `webhook` (string, required): Webhook ID to update. +- `url` (string, uri) +- `secret` (string) +- `active` (boolean) +- `events` (array of `org.tangled.webhook.defs#eventType`) + +Errors: + +- `WebhookNotFound` diff --git a/docs/src/content/docs/comparison-with-github.md b/docs/src/content/docs/comparison-with-github.md new file mode 100644 index 0000000000000000000000000000000000000000..b1b539674faf8cc48c217b1e8a3880a8f09c9e82 --- /dev/null +++ b/docs/src/content/docs/comparison-with-github.md @@ -0,0 +1,57 @@ +--- +title: Comparison with GitHub +description: + How GitHub concepts map onto Tangled, and where the models differ. +--- + +Most of what you do on GitHub has a direct counterpart on Tangled. Git +itself is unchanged: commits, branches, tags and remotes work the same. +What differs is who stores what: your account, your git data, and your +CI can each live on a different server. Tangled is +federated: those servers can be run by +different people, including you, and still work together. + +## Accounts + +GitHub hosts every account itself. On Tangled, your account lives with +an account provider (a PDS), and Tangled is one app that uses it. If you +sign up with Tangled you get a handle like `alice.tngl.sh`. If you +already have an atproto account, sign in with that handle instead; there +is nothing to link. + +## Repositories and hosting + +On GitHub, the website and the git server are one service. On Tangled, +the git server is a **knot**, and anyone can run one. Host repositories +with Tangled on the knots we run, or on your own knot; the web app shows +both the same way. + +Issues and pull requests are stored on the same knot as the code, not in +your account. That keeps a project's discussion with the project, and +the knot enforces who may write it. + +## Pull requests + +A GitHub pull request tracks a moving branch: force-push and the old +commits disappear from the review. In contrast, pull request submissions +in Tangled are immutable. + +When you create a pull request, it creates an immutable "round". To +update your pull request, you may push to the same branch, and hit +"resubmit" on the interface. This creates a new round. Each round +records a fixed base and head commit, reviewers can read earlier rounds +and observe a pull request as it evolves. + +Tangled also has support for the Jujutsu version control system: it can +automatically detect Jujutsu change-ids and can calculate interdiffs +across rounds. [Pull requests](/docs/guides/pull-requests/) covers the +workflow and its current limits. + +## CI + +Workflows live in `.tangled/workflows/*.yml` and run on a **spindle**: +run CI with Tangled, or run your own spindle. [Writing +workflows](/docs/spindles/writing-workflows/) has working examples. + +To move an existing repository over, see +[Migrating from GitHub](/docs/migrating-from-github/). diff --git a/docs/src/content/docs/contributing/guide.md b/docs/src/content/docs/contributing/guide.md new file mode 100644 index 0000000000000000000000000000000000000000..5efa716eb9e4bcb967c0c6f122db7d2b9795e9d5 --- /dev/null +++ b/docs/src/content/docs/contributing/guide.md @@ -0,0 +1,137 @@ +--- +title: Contribution guide +description: Commit guidelines, code formatting, proposals, and DCO. +--- + +## Commit guidelines + +We follow a commit style similar to the Go project. Please keep commits: + +- **atomic**: each commit should represent one logical change +- **descriptive**: the commit message should clearly describe what the + change does and why it's needed + +### Message format + +``` +/: + +Optional longer description can go here, if necessary. Explain what the +change does and why, especially if not obvious. Reference relevant +issues or PRs when applicable. These can be links for now since we don't +auto-link issues/PRs yet. +``` + +Here are some examples: + +``` +bobbin/xrpc: fix cursor overflow in listIssues + +The cursor was parsed as u32, so offsets past 4 billion wrapped +around and returned the first page again. +``` + +``` +knot2/knot-ssh: improve error checking in upload-pack +``` + +### General notes + +- PRs get merged "as-is" (fast-forward)—like applying a patch-series + using `git am`. At present, there is no squashing—so please author + your commits as they would appear on `master`, following the above + guidelines. +- If there is a lot of nesting, for example "web: + src/lib/components/repo: ...", it can be truncated to "web: repo: + ...". If the change affects many subdirectories, you may abbreviate to + just the top-level name, e.g. "spindle: ..." or "knot2: ...". Use + "all:" for tree-wide changes. +- The prefix must name a real top-level path. Conventional-commit + prefixes such as `fix:` or `feat:` are rejected by the commit-lint CI + check (`.tangled/hooks/commit-msg-lint.sh`). +- Keep commits lowercased with no trailing period. +- Use the imperative mood in the summary line (e.g., "fix bug" not + "fixed bug" or "fixes bug"). +- Keep the summary line under 76 characters. +- Follow the same formatting for PR titles if filled manually. +- Don't include unrelated changes in the same commit. +- Avoid noisy commit messages like "wip" or "final fix"—rewrite history + before submitting if necessary. + +## Code formatting + +Go, Rust and Nix are formatted with [`treefmt`](https://treefmt.com): +run `nix fmt` (or `treefmt` in the devshell). The web app formats with +its own pinned Prettier: run `pnpm format` in `web/`. + +## Proposals for bigger changes + +Small fixes like typos, minor bugs, or trivial refactors can be +submitted directly as PRs. + +For larger changes—especially those introducing new features, +significant refactoring, or altering system behavior—please open a +proposal first. This helps us evaluate the scope, design, and potential +impact before implementation. + +Create a new issue titled: + +``` +proposal: : +``` + +In the description, explain: + +- What the change is +- Why it's needed +- How you plan to implement it (roughly) +- Any open questions or tradeoffs + +We'll use the issue thread to discuss and refine the idea before moving +forward. + +## Developer Certificate of Origin (DCO) + +We require all contributors to certify that they have the right to +submit the code they're contributing. To do this, we follow the +[Developer Certificate of Origin (DCO)](https://developercertificate.org/). + +By signing your commits, you're stating that the contribution is your +own work, or that you have the right to submit it under the project's +license. This helps us keep things clean and legally sound. + +To sign your commit, just add the `-s` flag when committing: + +```sh +git commit -s -m "your commit message" +``` + +This appends a line like: + +``` +Signed-off-by: Your Name +``` + +We won't merge commits if they aren't signed off. If you forget, you can +amend the last commit like this: + +```sh +git commit --amend -s +``` + +If you're submitting a PR with multiple commits, make sure each one is +signed. + +For [jj](https://jj-vcs.github.io/jj/latest/) users, you can run the +following command to make it sign off commits in the tangled repo: + +```shell +# Safety check, should say "No matching config key..." +jj config list templates.commit_trailers +# The command below may need to be adjusted if the command above returned something. +jj config set --repo templates.commit_trailers "format_signed_off_by_trailer(self)" +``` + +Refer to the +[jujutsu documentation](https://jj-vcs.github.io/jj/latest/config/#commit-trailers) +for more information. diff --git a/docs/src/content/docs/contributing/hacking.md b/docs/src/content/docs/contributing/hacking.md new file mode 100644 index 0000000000000000000000000000000000000000..492e4d8cc1138cf478e768cce6e9e8e941a43b64 --- /dev/null +++ b/docs/src/content/docs/contributing/hacking.md @@ -0,0 +1,15 @@ +--- +title: Local development +description: Run the whole Tangled stack locally with localinfra/pc. +--- + +`localinfra/pc` runs a local copy of the whole network as plain +processes under [process-compose](https://github.com/F1bonacc1/process-compose): +a PLC directory, two PDSes, a knot, hydrant, bobbin, knotmirror, +deliberi, spindle and the web app, all behind Caddy on fake public +hostnames under `*.tngl.boltless.dev`. Start it with +`nix run .#localinfra`, and stop it with `bin/pc down`. Services run +what is already built, so build your change and restart that service, +for example `bin/pc process restart knot2`. Optional services such as +the real spindle engine and observability run under profiles. +`localinfra/pc/README.md` has the full reference. diff --git a/docs/src/content/docs/guides/access.md b/docs/src/content/docs/guides/access.md new file mode 100644 index 0000000000000000000000000000000000000000..e90fb360e34edececb2392051b2bfd6752ead5e2 --- /dev/null +++ b/docs/src/content/docs/guides/access.md @@ -0,0 +1,102 @@ +--- +title: Access and collaborators +description: + Who can push, merge, moderate and administer a repository, and how + collaborators and contribution policies work. +--- + +Access to a repository is decided by its knot. The knot knows four kinds +of people: the repository **owner**, the repository's **collaborators**, +the knot's **members**, and the knot's **admins**. Everyone else is a +stranger. The web app only hides controls you can't use; the knot +enforces the rules. + +## Roles + +| Role | Who | Where it is recorded | +| ------------ | ------------------------------------------------- | ------------------------------ | +| Owner | The account that created the repository | The knot's repository registry | +| Collaborator | Accounts the owner added | The repository, on the knot | +| Knot member | Accounts admitted to the knot, usually by invite | The knot's member list | +| Knot admin | DIDs listed in the knot's `server.admins` setting | The knot's config file | + +A knot also keeps a blocklist. A blocked account can't push, write +issues or comments, or create repositories on that knot, whatever its +other roles. Admins can't be blocked. + +## What each role can do + +| Action | Owner | Collaborator | Knot admin | Anyone else | +| ---------------------------------------------------------------- | ----- | ------------ | ---------- | ----------- | +| Push, delete branches | Yes | Yes | No | No | +| Merge pull requests | Yes | Yes | No | No | +| Rename on the knot, set default branch | Yes | Yes | No | No | +| Close, reopen, edit or delete others' issues, pulls and comments | Yes | Yes | Yes | No | +| Apply labels (web app) | Yes | Yes | No | No | +| Define labels | Yes | No | No | No | +| Add or remove collaborators | Yes | No | No | No | +| Set the contribution policy | Yes | No | Yes | No | +| Delete the repository | Yes | No | Yes | No | +| Open issues and pull requests, comment, react | Yes | Yes | By policy | By policy | +| Manage pipeline secrets (on the spindle) | Yes | Yes | No | No | + +Knot admins moderate every repository on their knot, but they are not +maintainers of other people's repositories: they can't push, merge or +define labels there. Knot members get no rights over other people's +repositories; membership only matters for creating repositories and for +the `members` contribution policy. + +## Collaborators + +The owner adds collaborators under **Settings → Access → Add +collaborator** by searching for a handle. The web app sends the change +to the knot, and from then on the collaborator can push and merge. The +collaborator sees the repository's **Settings** tab with the default +branch and pipeline secrets, but not the owner-only sections. Removing a +collaborator on the same page takes effect on the knot immediately. + +## Contribution policy + +Each repository has a contribution policy that decides who may open +issues and pull requests, comment and react. It doesn't affect pushing, +which always needs the owner or a collaborator. + +| Policy | Who may contribute | +| --------------- | ------------------------------------------------- | +| `anyone` | Any account the knot hasn't blocked | +| `members` | The owner, collaborators, and members of the knot | +| `collaborators` | The owner and collaborators only | + +A new repository takes the knot's default, set by +`repo.contribution_policy` in the knot config, which is +`anyone` unless the operator changes it. When the policy refuses +you, the knot answers: + +```text +Please ask the repo owner to add you as a collaborator, or to open the contribution policy to members or anyone, thanks. +``` + +## Knot membership + +Members can create repositories on a knot whose `acl.admission` is +`closed`, the default. With `open` admission, anyone not blocked can +create repositories. Admins create invite codes with +`knot-server admin create-invite-code`; you redeem one through an invite +link (`/invite?knot=&code=`), which also saves the knot in +**Settings → Knots**. Invite codes expire after two days and don't +survive a knot restart. + +## When access fails + +Check where the action happens: + +- **Push and clone**: the knot. It matches your SSH key against the keys + in your PDS, caching them for about an hour, so a key you just added + or removed can take a moment to take effect. +- **Issues, pulls, comments, merges**: the knot, using the roles and + policy above. +- **Pipeline secrets and runs**: the spindle, which keeps its own view + of owners and collaborators. +- **Profile, stars, follows**: your PDS. + +[Troubleshooting](/docs/troubleshooting/) covers common SSH failures. diff --git a/docs/src/content/docs/guides/importing.md b/docs/src/content/docs/guides/importing.md new file mode 100644 index 0000000000000000000000000000000000000000..621a65875ef97f0c2663342f4ec48511e281b8b8 --- /dev/null +++ b/docs/src/content/docs/guides/importing.md @@ -0,0 +1,37 @@ +--- +title: Importing repositories +description: + Import a repository from a Git URL or from GitHub (including private + repositories), follow the import, and keep pushing to two hosts. +--- + +The [importer](https://tangled.org/repo/migrate) copies a repository's +Git history, branches, and tags from another host into a new repository, +hosted with Tangled or on your own knot. Issues, pull requests, and +other forge data are **not** imported presently. + +You don't need the importer if you have the repository locally: create +an empty repository and push to it, as shown in +[Quick start](/docs/quick-start/). + +## Importing from a URL + +1. Open the importer and choose the URL option. +2. Paste a public Git URL, such as `https://example.com/repository.git`. +3. Set the name, an optional description, and the knot to import into + (your saved knots). +4. Start the import. + +Tangled checks that the source is reachable, then your knot fetches it +directly. + +## Importing from GitHub + +The GitHub importer handles public and private repositories: + +1. Head to the [importer](https://tangled.org/repo/migrate). +2. Authorize the importer to grant read access to your GitHub account. +3. Configure repositories and migrate them! + +Read the [Migrating from GitHub](/docs/migrating-from-github/) guide for +more. diff --git a/docs/src/content/docs/guides/notifications.md b/docs/src/content/docs/guides/notifications.md new file mode 100644 index 0000000000000000000000000000000000000000..f874c130b01cb0cf931208a16e95aa213adab640 --- /dev/null +++ b/docs/src/content/docs/guides/notifications.md @@ -0,0 +1,42 @@ +--- +title: Notifications and emails +description: + What triggers a notification, how subscriptions decide who receives + it, and how to set preferences and notification emails. +--- + +Open your [inbox](https://tangled.org/notifications) to read your +notifications; the bell in the header shows the unread count. + +There are two categories to notifications, **work** and **social**. +Things that might need your attention typically go into the **work** +category, a new issue notification for example. Events such as stars and +follows are categorized as **social**. + +## Subscriptions + +In addition to default notifications, you can subscribe to repositories, +issues and pull requests to be notified on status changes: + +- **Watch** on a repository page subscribes you to the whole repository: + new issues, new pull requests, and every comment. +- **Subscribe** on an issue or pull request subscribes you to that + thread. +- Opening an issue or pull request, or commenting on one, subscribes you + to that thread automatically. + +## Notification emails + +With **Email notifications** on, Tangled emails you about unread work +notifications (new issues and pull requests, comments, state changes, +mentions). Each issue or pull request gets its own email thread: a +comment, two more comments, then a close and a reopen all land in the +same conversation in your mail client. Tangled waits about ten minutes +after an event before sending, so a burst of activity on one issue +arrives as one message in that thread. Stars, follows, and invites stay +in the web inbox. + +Emails go only to your **verified primary** email. Add and verify one +under **Settings → Emails** first: Tangled sends a verification link, +and the address shows as verified once you open it. Without a verified +primary address, no email is sent. diff --git a/docs/src/content/docs/guides/organizations.mdx b/docs/src/content/docs/guides/organizations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..d6520035b570103f2855d02b4a93c218c98dbd1e --- /dev/null +++ b/docs/src/content/docs/guides/organizations.mdx @@ -0,0 +1,50 @@ +--- +title: Organizations +description: + Create an organization account, invite members, act as the + organization, and leave it. +--- + +import RepoCreationName from "../../../components/RepoCreationName.svelte"; + +An organization on Tangled is a real atproto account with its own DID +and handle. Members are accounts that have been granted control of it. + +## Creating an organization + +1. Verify an email address under **Settings → Emails**. Creation is + refused without one. +2. Open the [**New + Organization**](https://tangled.org/organizations/new) and choose a + name +3. Tangled creates the account `.tngl.sh` with you as its + controller and marks it as an organization. + +You may be asked to sign in again so Tangled can request permission to +create the account. Each user can create a upto two organizations. Just +like regular user accounts, you can change the handle of your +organization from `.tngl.sh` to be a domain that you own. + +## Members and roles + +The creator is the **owner** and can manage the account. Invited people +are **members**: open the organization's profile, go to **Settings → +Members**, and add accounts. Members can act as the organization but +can't manage the account. Owners can remove members from the same tab. + +## Acting as the organization + +Organizations are passwordless, delegate accounts. Owners and members +can act as the organization in certain parts of the interface, such as +when creating a new repository or when forking a repository. The account +picker in the new-repository form lets you choose which account you +create the repository under: + + + +## Leaving + +**Settings → Organizations** lists the organizations you control. Select +**Leave** to give up your access. You lose access to the organization's +repositories and settings immediately, and an owner has to invite you +back. diff --git a/docs/src/content/docs/guides/profile.md b/docs/src/content/docs/guides/profile.md new file mode 100644 index 0000000000000000000000000000000000000000..f33ccfbe736e50d814bb4dac6e97fb1beb34aa7d --- /dev/null +++ b/docs/src/content/docs/guides/profile.md @@ -0,0 +1,55 @@ +--- +title: Your profile +description: + What your profile shows, how to edit it, and the DID behind your + handle. +--- + +Your profile lives at `tangled.org/`. It shows who you are, +what you work on, and how you're connected to other people on Tangled. + +Your profile is a public record in your own account, so it goes wherever +your account goes. Anyone can read it, independent of any web app. + +## Edit your profile + +Select **Edit** on your profile to change your bio, pronouns, location +and up to five links. Change your avatar under **Settings → Profile**. +Without one, Tangled uses your Bluesky avatar. + +## Your DID + +Your handle, like `alice.tngl.sh`, is a name. Behind it is a DID, a +permanent identifier like `did:plc:7iza6de2dwap2sbkpav7c6c6`. Your +repositories, stars, follows and comments all point at the DID, not the +handle. + +That's what makes a handle easy to change: you change it with your +account provider, and everything you've done on Tangled follows. Links +to your old handle stop working, but a link with your DID in place of +the handle always reaches you. + +Your DID is shown under **Settings → Profile**. + +## What your profile shows + +The top of your profile shows your avatar, handle, pronouns, bio, +location and links, and how many people follow you and you follow. +Select either number to see the list. + +Below that are tabs: + +- **Overview**: your pinned repositories and an activity chart of the + last six months of commits, pull requests, issues and new + repositories. +- **Repositories**: every repository you own. +- **Starred**: repositories you've starred. +- **Strings**: code snippets you've shared. +- **Vouches**: accounts you've vouched for or denounced. See + [Vouch network](/docs/guides/vouching/). + +On someone else's profile, **Follow** and **Vouch** take the place of +**Edit**. + +An [organization](/docs/guides/organizations/) has its own profile layout, +with Overview and Repositories tabs. diff --git a/docs/src/content/docs/guides/pull-requests.mdx b/docs/src/content/docs/guides/pull-requests.mdx new file mode 100644 index 0000000000000000000000000000000000000000..eb346054aeb5db0dcb159c1134f79a92aa551704 --- /dev/null +++ b/docs/src/content/docs/guides/pull-requests.mdx @@ -0,0 +1,105 @@ +--- +title: Pull requests +description: Propose changes on a branch, review them round by round, and merge + them. +--- + +import PullSelectors from "../../../components/PullSelectors.svelte"; + +A pull request is a proposal to merge one branch into another branch. It +tells other people about the changes that you pushed to a branch. Other +people can discuss the changes and review them. You can add more commits +before you merge the changes into the target branch. + +On Tangled, a pull request is a series of immutable **rounds**. When you +create a pull request, Tangled makes round 0. To update the pull +request, push to the same branch and select **Resubmit**. This makes a +new round. Each round records a fixed base commit and a fixed head +commit. Reviewers can read earlier rounds. Thus they can see how the +pull request changes. A round cannot be modified once created, you can +only submit a new round. + +## Opening a pull request + +The **New pull** page (`/pulls/new`) asks for a source branch, a target +branch, a title, and a body in markdown. The target branch is the +default branch, unless you select a different branch. You cannot use the +default branch as the source. + +The page shows the commits that the pull request includes, and the diff +against the target branch. + +When you push a branch that is not the default branch, the push output +includes a link. The link opens the form with the two branches already +correct: + +```sh +git push origin fix-parser +``` + +The link has this form: + +`https://tangled.org/alice.tngl.sh/my-project/pulls/new?source=branch&sourceBranch=fix-parser&targetBranch=main` + +When you submit the form, Tangled creates the pull request and its first +round, round 0. Round 0 records the merge base of the two branches and +the tip of the source branch. + +## Rounds + +Each round holds a base commit and a head commit. You cannot edit a +round after Tangled creates it. + +This separates work on a branch from a request for review. You can push +fixups, squash, and rebase as often as you want. These actions do not +change what the reviewers read. When the branch is ready, submit a new +round. Earlier rounds stay available, together with the discussion that +occurred while they were current. + +## Reviewing + +Every round of a pull request has a URL. Thus you can send a reviewer a +link to the exact view that you want. Since rounds are immutable, +reviewers can always drop comments without them being outdated by an +update to the PR. + +## Interdiffs and JJ VCS + +As your PR evolves, reviewers might want to only review the part of the +PR that has changed across versions. To enable this form of review, +Tangled has native support for interdiff-style reviews: + +- First, you must use the Jujustu version control system to manage your + repository +- All commits in your PR must have [Jujutsu + change-ids](https://docs.jj-vcs.dev/latest/glossary/#change-id) + associated with them + +When you submit a second version to your PR, you will see all submitted +versions in a version selector dropdown, here, you can use use +Shift + Left Click to compare against a previous +version, and Tangled will calculate an interdiff to show you exactly +what changed between the two rounds. + +Tangled also shows Jujutsu change-ids across the UI if commits have an +associated change-id header. + + + +## Merging + +The merge box checks the newest round against the current tip of the +target branch. It reports if the round applies cleanly. The repository +owner and the collaborators can then select **Merge**. Several merge +strategies are also available to choose from. + +## URLs + +| Path | Shows | +| ------------------------------------------------------- | --------------------------------------------- | +| `///pulls/` | A pull request, at its newest round | +| `///pulls//latest` | The newest round | +| `///pulls//` | One round | +| `///pulls///..` | A range of commits in a round | +| `///pulls//..` | The interdiff between two rounds | +| `///pulls//../` | That interdiff narrowed to one Jujutsu change | diff --git a/docs/src/content/docs/guides/repositories.md b/docs/src/content/docs/guides/repositories.md new file mode 100644 index 0000000000000000000000000000000000000000..f00691e5b6b6e6298882372a3440086c59016e3f --- /dev/null +++ b/docs/src/content/docs/guides/repositories.md @@ -0,0 +1,102 @@ +--- +title: Repositories +description: + Create, fork, clone, browse, configure, rename and delete repositories + on Tangled. +--- + +A Tangled repository is just a git repository on a knot. The knot stores +the git data and the repository's issues and pull requests and your +account links you to it as the owner. + +## Creating a repository + +Head to the [new repository](https://tangled.org/repo/new) page to +create a repository. If you control an organization account, the owner +selector next to the name lets you create the repository under the +organization instead. The name becomes the URL: +`https://tangled.org//`. + +## Where the repository lives + +The knot holds the git data, the issues and pull requests. When creating +a repository, you can choose a knot to host the repository on: + +- **Host with Tangled** on `knot1.tangled.sh`, the default. +- **Self-host your own knot** to keep the data on a server you control. + See [Run a knot](/docs/self-hosting/knot/). + +Presently, repositories cannot migrate from one knot to another, but +this is in the works. + +## Forking + +**Fork** on a repository opens a similar form with the name and +description filled in. Just like the repository creation flow, you can +choose a destination knot for the forked repository. + +## Importing + +**Migrate** (`/repo/migrate`) imports a repository from GitHub or from +any git URL. [Importing repositories](/docs/guides/importing/) guide contains +more about the importer tool. + +## Clone URLs + +The **Clone** menu shows two forms of each URL: a human-readable +version, typically `/`, and a permalink +version, that uses the repository DID. The permalink version continues +to work across handle changes and repository renames (although Tangled +will do its best to maintain redirects in such cases): + +| Transport | Named | Permalink | +| --------- | ---------------------------------------------- | ------------------------------------ | +| HTTPS | `https://tangled.org/alice.tngl.sh/my-project` | `https://tangled.org/did:plc:abc123` | +| SSH | `git@tangled.org:alice.tngl.sh/my-project` | `git@tangled.org:did:plc:abc123` | + +The SSH host is the knot's hostname. For self-hosted knots, the SSH host +and port depend on your SSH setup. A trailing `.git` is accepted on the +named form but not on the DID form. + +Pushing needs an SSH key in your account and push access to the +repository. Some refs under `refs/` are reserved by the knot for its own +bookkeeping, and pushes to them are refused. + +The Clone menu also offers `.tar.gz` and `.zip` archives of the current +ref. + +## Renaming + +**Settings → General → Rename** changes the display name of a +repository. Old names will continue to work on the web as well as for +clones, they will be simply be redirected. Git remotes will also +continue to work across renames. + +## Deleting + +**Settings → General → Delete** asks you to type the repository name, +then asks the knot to delete the repository. This deletes the git data, +issues and pull requests on the knot and can't be undone. It can take a +few seconds for the repository to entirely disappear from Tangled. + +## URLs + +All of these work with any branch, tag or commit as ``: + +| Path | Shows | +| -------------------------------------- | ----------------------------------------| +| `//` | Repository index page | +| `///tree//` | A directory | +| `///blob//` | A file | +| `///raw//` | Raw `text/plain` file contents | +| `///history//` | Commits that touched a file | +| `///commits/` | Commit log | +| `///commit/` | One commit and its diff | +| `///commit/.patch` | The commit as `git format-patch` output | +| `///commit/.diff` | The commit's diff only | +| `///branches`, `/tags` | Branches and tags | + +`.patch` and `.diff` need a full or abbreviated commit hash, not a +branch name. They don't work for a root commit yet, which has no parent +to diff against. + diff --git a/docs/src/content/docs/guides/signing-commits.md b/docs/src/content/docs/guides/signing-commits.md new file mode 100644 index 0000000000000000000000000000000000000000..1348b75e3ae55f524a92d970fca08da2e3af8b4e --- /dev/null +++ b/docs/src/content/docs/guides/signing-commits.md @@ -0,0 +1,48 @@ +--- +title: Signing commits +description: + Sign commits with SSH or GPG, and how Tangled ties commit emails to + your account. +--- + +Tangled supports SSH commit signatures. For verified +commits, Tangled displays a green badge and the fingerprint +of the SSH key used to sign that commit. + + +## Signing with SSH + +To sign commits using your SSH key in git: + + +```sh +git config --global gpg.format ssh +git config --global user.signingkey ~/.ssh/id_ed25519.pub +git config --global commit.gpgsign true +``` + +The key you sign with can be any key that has been added to your Tangled +account in **Settings → SSH Keys**. + +If you are using Jujutsu, add this to your config: + +```toml +[signing] +behavior = "own" +backend = "ssh" +key = "~/.ssh/id_ed25519.pub" +``` + +## Linking commits to your account + +Tangled maps a commit to your account when its author email is either: + +- one of your **verified** emails (add them under **Settings → Emails** + and open the link sent to each), or +- your DID itself, for example `git config user.email did:plc:abcd1234`. + +Authors of linked commits are listed correctly in the repository page, +in blame views, and in `Signed-off-by:` trailers. Only verified emails +count, so nobody can claim your commits by adding your address to their +account. Commits with an unverified or unknown email still exist and +display normally; they just aren't linked to your Tangled account. diff --git a/docs/src/content/docs/guides/vouching.mdx b/docs/src/content/docs/guides/vouching.mdx new file mode 100644 index 0000000000000000000000000000000000000000..3289d94a9cf8a371079b013e313899fea7920c52 --- /dev/null +++ b/docs/src/content/docs/guides/vouching.mdx @@ -0,0 +1,61 @@ +--- +title: Vouch network +description: + Vouch for or denounce accounts, and see the trust signals from your + network. +--- + +import VouchCardDemo from "../../../components/VouchCardDemo.svelte"; + +Vouching serves as a signal of trust to your circle. You can vouch or +denounce users that you interact with. It is a signal of trust to your +circle. Your circle can view vouch descisions made by you. Vouching on +Tangled is inspired by the [vouch](https://github.com/mitchellh/vouch) +project by Mitchell Hashimoto. Read the [blog +post](https://blog.tangled.org/vouching/) to know more. + + + +## Vouch or denounce + +Open a profile and select **Vouch**. Then select **Vouch** or +**Denounce**. You can add an optional reason, up to 256 characters. When +vouching for a user from the a PR submission, Tangled automatically adds +the PR record to the vouch record's evidences list. + +## The network + +Tangled collects vouch records from every account. It shows a "hat" on a +profile at points of interaction: + +- issues and issue comments, and +- pull requests and pull request comments. + +A green hat is a vouch. A red hat is a denouncement. + +A hat appears only in two cases: + +- you vouched for or denounced the account, or +- an account you vouched for made a decision about it. + +Thus you see the decisions of your circle. Select the hat to see who +made the decision. Only a vouch extends your circle: a denouncement does +not add the account to your network. Two people can see different hats +for the same account. + +## The effects of vouching + +A vouch or a denouncement does not gate any permissions at the moment. +It does not hide anyone's issues or pull requests. It shows a hat only. +Use the hat to inform your decision. A denouncement has no consequence +for the other account at this time. Vouching is simply used to build +your trust network and inform your circle, actions are left up to the +end user. + +:::caution + +Reasons and evidence are public data in your account. Anyone can read +them, independent of any web app. Do not put private reports or personal +information in them. + +::: diff --git a/docs/src/content/docs/index.mdx b/docs/src/content/docs/index.mdx new file mode 100644 index 0000000000000000000000000000000000000000..884b84b691322d2777b18ff4bd60c5791bef307a --- /dev/null +++ b/docs/src/content/docs/index.mdx @@ -0,0 +1,112 @@ +--- +title: Tangled docs +description: + Guides and reference for Tangled, your code forge for the next decade. +head: + - tag: title + content: Tangled Docs +tableOfContents: false +hero: + title: The most hackable forge + tagline: + Tangled is a social code-collaboration platform built on an open + protocol. It is designed to be self-hostable, hackable and + extensible. + actions: + - text: Quick start + link: /docs/quick-start/ + variant: primary + - text: What is Tangled? + link: /docs/introduction/ + variant: minimal +--- + +import ButtonCard from "$lib/components/ui/ButtonCard.svelte"; +import Rocket from "~icons/lucide/rocket"; +import GitCompareArrows from "~icons/lucide/git-compare-arrows"; +import Github from "~icons/lucide/github"; +import HardDrive from "~icons/lucide/hard-drive"; +import Braces from "~icons/lucide/braces"; +import FileJson from "~icons/lucide/file-json"; +import MapIcon from "~icons/lucide/map"; +import Plug from "~icons/lucide/plug"; +import Hammer from "~icons/lucide/hammer"; +import BookA from "~icons/lucide/book-a"; + +
+ + + + +
+ +## Hacking + +
+ + + + + + +
diff --git a/docs/src/content/docs/integrations/contributing.md b/docs/src/content/docs/integrations/contributing.md new file mode 100644 index 0000000000000000000000000000000000000000..dcfc80daf2418811a84fa81f569b4290041b45b8 --- /dev/null +++ b/docs/src/content/docs/integrations/contributing.md @@ -0,0 +1,43 @@ +--- +title: Contributing an integration +description: + Build an integration that connects Tangled to another service, and + add its documentation here. +--- + +You can build an integration that connects Tangled to a different +service using the public [API](/docs/api/bobbin) or by simply listening +for events on a firehose. If you write an integration, get in touch, and +we could get it documented here! + +## Listening for events + +Each repository has its own DID. The records of a repository, for +example issues, pull requests and artifacts, can be filtered and +streamed using [Jetstream](https://github.com/bluesky-social/jetstream). + +Some records are owned by users, and not repositories. For example, a +star is in the repository of the user that gives the star. + +Refer to [Records](/docs/reference/records/) for the record types. + +## Get more data + +Use the [Bobbin API](/docs/api/bobbin/) at `https://api.tangled.org` +to get data that is not in the records. For example: + +- `org.tangled.repo.getRepoByName` gives the DID of a repository. +- `org.tangled.temp.git.listTags` gives the tags and their messages. +- `org.tangled.feed.countStars` gives the star count. + +## Add documentation + +When your integration is ready, add a page for it here: + +1. Add a Markdown file in `docs/src/content/docs/integrations/`. +2. Add the page to the **Integrations** section of the sidebar in + `docs/astro.config.mjs`. +3. Tell users how to install the integration and how to use it. Keep + the page short. +4. Send a pull request. Refer to the + [Contribution guide](/docs/contributing/guide/). diff --git a/docs/src/content/docs/integrations/discord.md b/docs/src/content/docs/integrations/discord.md new file mode 100644 index 0000000000000000000000000000000000000000..29b045a8fafd3fa83fd3037fd8befb5ee268716d --- /dev/null +++ b/docs/src/content/docs/integrations/discord.md @@ -0,0 +1,70 @@ +--- +title: Discord +description: + Add the Tangled bot to a Discord server and post repository events + to a channel. +--- + +The Tangled bot posts repository events to a Discord channel. It can +watch any public repository on Tangled. + +## Add the bot to a server + +1. Open the [install link](https://discord-bot.tangled.org/invite). +2. Select your server and select **Authorize**. + +You must have the **Manage Server** permission to add the bot. + +## Watch a repository + +In a channel, type: + +``` +/tangled watch repo:tangled.org/core +``` + +:::note +If the `/tangled` commands do not show, disable the legacy chat input. +In Discord, go to **User Settings → Accessibility → Audio & Screen +Reader** and turn off **Use the legacy chat input**. +::: + +The bot posts these events in that channel: + +| Event | What the bot posts | +| ------------------ | ----------------------------------------------- | +| **Issues** | Each new issue, with its title and body. | +| **Pull requests** | Each new pull request, with its diff stats. | +| **Tags** | Each new tag, with its message. | +| **Artifacts** | Each new artifact, with a link to its tag. | +| **Star milestones**| The star count when it reaches 1, 2, 4, 8, ... | + +To post in a different channel, set the `channel` option. To turn off +an event, set its option to `False`. For example, this command posts +only issues and pull requests: + +``` +/tangled watch repo:tangled.org/core tags:False artifacts:False stars:False +``` + +To change the events later, run `/tangled watch` again for the same +repository. + +## Watch a collection + +The bot can also post each new record of a collection in a +repository, for example `org.tangled.feed.comment`: + +``` +/tangled collection add repo:tangled.org/core nsid:org.tangled.feed.comment +``` + +To stop, use `/tangled collection remove`. + +## Other commands + +- `/tangled list` shows all repositories that the server watches. +- `/tangled unwatch` stops all events for a repository. + +Only members with the **Manage Server** permission can use these +commands. A server can watch a maximum of 50 repositories. diff --git a/docs/src/content/docs/introduction.md b/docs/src/content/docs/introduction.md new file mode 100644 index 0000000000000000000000000000000000000000..857132ea99b5902bedc80d54553240747208499f --- /dev/null +++ b/docs/src/content/docs/introduction.md @@ -0,0 +1,62 @@ +--- +title: What is Tangled +description: + Tangled is a git forge built on the AT Protocol that you can use + hosted, or self-host in part or in full. +--- + +Tangled is a social code forge: you can host git repositories, +collaborate on issues and pull requests, follow other devs and star +their repos. However, Tangled is different from the rest: + +- It is federated: you can self-host a server and still interact with + the broader network +- It is open source +- It is entirely self-hostable: you can host a single piece, like the + repository storage component, or the CI component, or all of it +- It is hackable + +The individual servers talk to each other through public interfaces, so +you can replace any of them with your own: people have written knots +that store repositories on code.storage, or Cloudflare Artifacts, and +spindles that runs CI on Buildkite or Tekton! + +That being said, you don't need to run anything to use Tangled. Sign in, +create a repository, and push. [Quick start](/docs/quick-start/) walks +through it. + +## How it is split + +Your account lives with an **account provider** (a PDS, or Personal Data +Server), the same kind of account you might already use for Bluesky. It +holds your profile, follows, stars, SSH public keys, and a small entry +for each repository you own. + +Git data lives on a **knot**, a git server. Each repository has its own +identity (a DID), and its issues, pull requests, comments, reactions and +labels are stored next to the code on the knot. + +**Spindles** run CI: they execute the workflows when you push. + +The web app saves your changes to your account or to the knot, and shows +everything gathered from across the network. No web app holds a +privileged copy of your data: it all comes from accounts and knots. + +## Hosted and self-hosted + +You can host with Tangled, or self-host. Tangled is federated: knots, +spindles, accounts and web apps run by different people work together, +so you can run all of it, some of it, or none of it. + +| Component | What we run | You can run your own | +| ----------------- | ----------------------------------------------- | -----------------------------------------------------------------------------| +| Account (PDS) | Signup gives you a `.tngl.sh` account | Any atproto PDS works | +| Knot | Shared knots, the default is `knot1.tangled.sh` | See [Run a knot](/docs/self-hosting/knot/) | +| Spindle | `spindle.tangled.sh` | See [Run a spindle](/docs/self-hosting/spindle/) | +| Web app and index | The website at `tangled.org` | See [Run an index](/docs/self-hosting/indexer/) and [Run a frontend](/docs/self-hosting/frontend/) | + +Typically, users self-host a knot: your code then lives on a machine you +control, while the web app still shows it and accepts issues and pull +requests against it. A self-hosted spindle lets you run CI on your own +hardware. Run every piece yourself and you have a fully independent +Tangled. diff --git a/docs/src/content/docs/knots/index.md b/docs/src/content/docs/knots/index.md new file mode 100644 index 0000000000000000000000000000000000000000..24c19c255d0f94983298918a5274b1611fe7052d --- /dev/null +++ b/docs/src/content/docs/knots/index.md @@ -0,0 +1,198 @@ +--- +title: Knot +description: + What a knot does, what you can do with it, and how it talks to the + rest of Tangled. +--- + +A knot is the server that hosts your git repositories. It is a single +Rust binary, `knot-server`, that serves git over SSH and HTTPS and also +acts as the atproto PDS for every repository it hosts. You don't need to +run a knot to use Tangled: host with Tangled on the knots we run, or +self-host your own. + +```sh +git clone knot.example.com:alice.bsky.social/proj # SSH, by owner and name +git clone https://knot.example.com/did:plc:abc123 # HTTPS, by repo DID +``` + +## What a knot does + +- **Git hosting.** Clone, fetch and push over SSH (`git-upload-pack`, + `git-receive-pack`, `git-upload-archive`), smart HTTP, and Git LFS. + The SSH server is built in: there is no `sshd`, + `AuthorizedKeysCommand` or `git` unix user to configure. +- **A PDS for repo accounts.** Every repository has its own `did:plc` + repo DID. The knot holds that DID's + rotation and signing key, and its DID document points `#atproto_pds` + and `#tangled_knot` at the knot. Issues, pulls, comments, reactions, + label definitions and patches are records in the repo DID's account on + the knot, not in the author's PDS. +- **Access control.** Admins, members, repo owners, collaborators and a + blocklist decide who can create repos, push and write records. +- **Events.** A `com.atproto.sync.subscribeRepos` firehose that the + indexer, mirrors and spindles follow. + +## Git as the database + +The knot has no SQL database. All state lives under `repo.scan_path`: + +- Each repo is a bare git repository, sharded by DID + (`///`). +- ACLs and records are signed **collaborative objects** (COBs): change + graphs stored under `refs/cobs//`, with snapshots under + `refs/cob-checkpoints/*`. The repo DID's atproto commit chain is + derived from them and stored at `refs/atproto/commit`. +- A **meta-repo**, sharded under the knot's own `did:web`, holds the + member list, blocklist, repo registry (owner and name to repo DID, + policies), the queue of pending PLC operations and the firehose + sequence number. At boot the knot rebuilds its in-memory index from + the meta-repo and answers "warming" until it is ready. + +The only other state is a sealed key store, unlocked by the key in the +env var named by `secrets.master_key_env` (usually `KNOT_MASTER_KEY`). + +:::caution + +The knot's key is the sole rotation key for every repo DID it hosts. If +you lose the master key or the sealed store, git data stays readable but +you lose control of every repo DID. Keep the master key off the server. + +::: + +## Roles and access + +| Role | Where it's stored | What it allows | +| ------------ | ------------------------------------------------------------- | ------------------------------------------------------------------- | +| Admin | `server.admins` in config (the first entry is the knot owner) | Manage members, ban accounts, moderate any repo's text | +| Member | Members COB in the meta-repo | Create repos when admission is closed | +| Repo owner | Repo registry | Push, manage collaborators, set contribution policy, rename, delete | +| Collaborator | Collaborators COB in the repo | Push, merge pulls, moderate the repo | +| Blocked | Blocklist in the meta-repo | Nothing; overrides every other role | + +**Admission** (`[acl] admission`) is `closed` by default: only members +can create repos. Set it to `open` to let any account create repos. + +**Contribution policy** decides who may write records (issues, comments, +pulls) into a repo, separately from who may push. Values are `anyone`, +`members` and `collaborators`; the default comes from +`[repo] contribution_policy` (`anyone`). Owners change it per +repo through the manifest's `contributionPolicy`, with +`com.atproto.repo.putRecord` on `org.tangled.repo.manifest`. Pushing +always requires being the owner or a collaborator, and not being +blocked. + +Membership and collaborators are records the knot keeps itself, written +with `com.atproto.repo.createRecord` and removed with `deleteRecord`. +Admins add members as `org.tangled.knot.member` in the knot's own +account (`knot-server admin member add` does this), and owners add +collaborators as `org.tangled.graph.collaborator` in the repo's +account; the record key is the subject's DID. The web UI manages +collaborators. Members can also join with an **invite code**: codes are single +use, expire after two days, and live in memory, so a restart discards +them. The landing URL is `https:///invite/`. + +## Clone URLs + +A git repo can be pushed/pulled by its owner + name, or by its own repo-DID. The owner can be a DID or an atproto handle. + +Over SSH: + +```sh +git clone knot.oyster.cafe:nel.pet/squid +git clone knot.oyster.cafe:did:plc:nel/squid +git clone knot.oyster.cafe:did:plc:barnacle +``` + +Over HTTPS, the same: + +```sh +git clone https://knot.oyster.cafe/nel.pet/squid +git clone https://knot.oyster.cafe/did:plc:nel/squid +git clone https://knot.oyster.cafe/did:plc:barnacle +``` + +Push works over both, of course. For HTTP pushing, it is up to the user to find a good Tangled-CLI or something that can put the right things in the git credential helper such that a service auth token is minted and used on push. + +Appending `.git` onto the end of the repo name is optional, so `did:plc:nel/squid.git` goes to the same repo as `did:plc:nel/squid`. Note: that only applies to the repo name variant though - `did:plc:barnacle.git` is interpreted as a DID that literally ends with `.git`. This would be made better from better DID parsing, since a `did:plc` can't have dots, only a `did:web` can. Lewis will get around to that eventually. + +## Pushing + +SSH push identifies you by matching the offered key against the +`org.tangled.key.sshKey` records in your PDS whose `usages` include +`auth` (or omit `usages`). The knot caches keys for an +hour (`[keyfill] ttl_secs`). If your agent offers many keys, add +`IdentitiesOnly yes` for the knot host in `~/.ssh/config`. + +HTTPS push takes an atproto service-auth token bound to +`org.tangled.git.receivePack`, as a Bearer token or as the Basic-auth password, +usually from a git credential helper. + +After a push, git prints **push messages**: an acknowledgement, an "Open +pull request" link for new branches, workflow compile diagnostics, and a +whatever the repository's spindle answered about the pipeline it scheduled, +such as how to follow its logs over SSH. The text comes from +`[messages.*]` templates. Pass `-o skip-ci` to skip CI for a push. + +A Knot won't allow any pushes to the following reserved refs: `refs/cobs/*`, +`refs/cob-checkpoints/*`, `refs/hidden/*` and `refs/atproto/*`. On the flip-side, `refs/cobs/*` will be fetchable read-only when using an explicit ref-prefix. There +are no server-side hooks. + +## How it talks to the rest + +| Peer | Direction | What | +| -------------------------------------- | --------- | -------------------------------------------------------------------------------------------------- | +| Web | in | `org.tangled.repo.createRepo`, ACL procedures, `com.atproto.repo.*` record writes (aud = repo DID) | +| Hydrant | out | Follows `subscribeRepos` for `#commit`, `#identity` and `#account` frames | +| Spindle | out | Follows `subscribeRepos` for `org.tangled.git.refs` record ops; clones over HTTPS | +| Knotmirror | in/out | Fetches packs; pushes merges and branch deletes with your forwarded token | +| PLC directory | out | Submits genesis and update ops for repo DIDs, with retries | +| User PDSes | out | Reads identities and SSH keys | + +After every push, the knot rewrites the single `org.tangled.git.refs` +record (rkey `self`) with a snapshot of every ref and the default +branch, then emits a knot-signed `org.tangled.git.pushEvent` frame +(repo, pusher, ref edits, push options) on the firehose. + +`org.tangled.knot.describeKnot` returns the knot's DID, version and +capabilities: `tangled1.0`, `knot-acl`, `repo-did-input`, +`atproto-firehose` and `knot-social-writes`. + +## Operations + +**Maintenance** runs in the background: commit-graph, multi-pack index, +bitmaps and a geometric repack every six hours, with prune and reflog +grace periods (`[maintenance]`). + +**Health:** `curl https:///xrpc/_health` returns the version, or +503 if the LFS store is unwritable. `/xrpc/org.tangled.knot.describeKnot` +describes the knot, and `/.well-known/did.json` the knot's DID document. + +**Admin CLI.** `knot-server admin` posts to the knot's own +`https://` with Basic auth. It only works when +`[acl] legacy_admin_secret_env` names an env var holding a shared +secret. + +```sh +knot-server admin create-invite-code +knot-server admin member add did:plc:alice +knot-server admin member remove did:plc:alice +knot-server admin user ban did:plc:spammer +knot-server admin user unban did:plc:spammer +``` + +`knot-server config-template` prints a full config, and +`knot-server validate` checks one. + +:::note + +New knots default to `[git] object_format = "sha256"`. SHA-256 repos +can't push to or fetch from SHA-1 repos, and the setting only affects +newly created repos. Knots migrated with `knot-migrate` keep `sha1`. + +::: + +[Run a knot](/docs/self-hosting/knot/) covers installation, +[Knot configuration](/docs/reference/knot-config/) lists every setting, and +[Golang to Rust knot migration guide](/docs/upgrading/knot-2/) covers moving from +the Go knot. diff --git a/docs/src/content/docs/knots/repositories.md b/docs/src/content/docs/knots/repositories.md new file mode 100644 index 0000000000000000000000000000000000000000..62a6ed7d52bfbe8bdd4a0dafa88f8e590c3c7bd7 --- /dev/null +++ b/docs/src/content/docs/knots/repositories.md @@ -0,0 +1,58 @@ +--- +title: Your first repository +description: Add an SSH key, create a repository, and push to it. +--- + +Every repository lives on a knot, the server that stores its git data, +issues and pull requests. By default, Tangled hosts your repositories on +`knot1.tangled.sh`, so you don't need to run anything. To keep them on a +server you control, see [Self-hosting](/docs/knots/self-hosting/). + +## Add an SSH key + +Pushing needs a public SSH key in your account. Make one if you don't +have one: + +```sh +ssh-keygen -t ed25519 -C "alice@example.com" +cat ~/.ssh/id_ed25519.pub +``` + +Paste the output into **Settings → Keys → Add SSH key**. + +If your SSH agent holds many keys, pin the right one in +`~/.ssh/config`: + +``` +Host tangled.org + User git + IdentityFile ~/.ssh/id_ed25519 + IdentitiesOnly yes +``` + +## Create a repository + +Open [New Repository](https://tangled.org/repo/new), give it a name, and +keep **Host with Tangled** selected. + +## Push + +Push an existing project to it: + +```sh +cd my-project +git remote add origin git@tangled.org:alice.tngl.sh/my-project +git push -u origin main +``` + +To move a repository you already push elsewhere, change `origin` and +push every branch and tag: + +```sh +git remote set-url origin git@tangled.org:alice.tngl.sh/my-project +git push origin --all +git push origin --tags +``` + +[Repositories](/docs/guides/repositories/) covers cloning, forking and +access. diff --git a/docs/src/content/docs/knots/self-hosting.md b/docs/src/content/docs/knots/self-hosting.md new file mode 100644 index 0000000000000000000000000000000000000000..4868cb52d66ca9df35acf433fe87d8f0dffec54f --- /dev/null +++ b/docs/src/content/docs/knots/self-hosting.md @@ -0,0 +1,229 @@ +--- +title: Self-hosting +description: + Run your own knot, the server that stores git repositories and their + repo accounts. +--- + +Run your own knot to keep your repositories on a server you control. +The web app still shows them, and people can still open issues and pull +requests against them. The knot is a single Rust binary, +`knot-server`, with no database: git is the only store, plus a sealed +key file. [Run a knot](/docs/self-hosting/knot/) is the short version of +this page. + +If you already run the Go knot, follow +[Golang to Rust knot migration guide](/docs/upgrading/knot-2/) instead. + +## Requirements + +- A Linux machine that stays online. +- A DNS name. It becomes the knot's identity, `did:web:`, so + choose one you'll keep. +- Your DID, which becomes the first admin. +- TLS, from a reverse proxy or from the knot itself. + +## Master key and sealed keys + +The knot keeps its signing keys in a sealed file, encrypted with a +master key from an environment variable. Make the key once: + +```sh +printf 'KNOT_MASTER_KEY=%s\n' "$(openssl rand -base64 32)" > knot.env +``` + +:::caution + +Back up the master key away from the knot. Without it, the knot loses +control of every repository's identity. + +::: + +## Ports, Taylor's version (ha ha, 22 joke) + +A knot has two listeners where HTTP defaults to `[::]:5555`, SSH defaults to `[::]:2222`. + +Thus far in Tangled in general, SSH is heavily used because the old Tangled knot implementation didn't do HTTP pushing. Tangled generally auto-generates a URL for SSH git'ing that looks like this: + +``` +git clone git@knot.oyster.cafe:did:plc:barnacle +``` + +However notice that from Tangled's client, it builds a clone URL with no port number in, because it always assumes 22. If the knot's SSH listener is listening on another port, all of its users have gotta either rewrite the URL as `ssh://git@knot.oyster.cafe:2222/did:plc:barnacle`, or add a block to their `~/.ssh/config`. + +So **give the knot port 22 if one can**! It doesn't need unix `git` user / shell account, so the only thing in the way is the remote computer's own sshd, which is usually already taking port 22. + +If the reader is nodding along instead of saying "no Lewis I won't change the remote computer's sshd because I like not bricking it" then: + +Moving sshd is something that can lock one out of one's own server, so do it in this order: + +1. Open a second SSH session to the server and keep it open for this whole procedure. If step 5 goes wrong, having this session open might be the saving grace. +2. Find out which thing owns the port with `systemctl is-enabled ssh.socket`, since each answer sends you to a different file in step 3: + - `enabled`: edit the socket unit. A `Port` line in `sshd_config` does nothing at all here, and editing it is the usual way to lose an afternoon. Ask me how I know. + - `disabled`, or no systemd: edit `Port` in `sshd_config`; sshd holds the port itself. +3. Edit `/etc/ssh/sshd_config` and set `Port 2200` or whatever free port one likes. Leave `Port 22` in place as well for now, so sshd listens on both. Under the socket unit, `systemctl edit ssh.socket` takes a `[Socket]` section with `ListenStream=2200` instead, such that the new port joins whatever's already listening. +4. Open the new port in the firewall if there is one. On ufw that's `ufw allow 2200/tcp` (I think!! Untested). On a cloud provider one will probably also have to deal with it / open it in their proprietary config. +5. Restart sshd, or `systemctl restart ssh.socket` for the socket unit, and **from the local computer, in a third terminal**, confirm `ssh -p 2200 root@the.server` works before continuing. +6. Once confirmed, only now remove `Port 22` from `sshd_config`, restart sshd once more, & give the port to the knot. Under the socket unit that's an empty `ListenStream=` above the `ListenStream=2200`, since a drop-in only adds to the port it inherits. When running the binary directly, that means `ssh_listen_addr = "[::]:22"`. Comparatively, in a container it entails publishing the container's 2222 as the host's 22, which is what the compose file in [Run a knot](/docs/self-hosting/knot/#run-with-docker) does. + +If one would rather not move sshd at all, another cool option for having knot2 on port 22 is a second IP address on the remote computer. Bind sshd to one with `ListenAddress`, bind the knot to the other with `ssh_listen_addr = ":22"`, and just put the knot's DNS record on that second address. Leaving the knot on `[::]:22` would wildcard-bind every address on the box and would collide with sshd no matter which single IP that sshd listens on. + +HTTP can stay on 5555 behind a reverse proxy, or move to 443 if one wants the knot to terminate TLS itself. + +## Configuration + +```sh +knot-server config-template > config.toml +knot-server --config config.toml validate +``` + +The required keys are `server.hostname`, `server.admins`, +`server.ssh_host_key_file`, `repo.scan_path`, `secrets.sealed_key_file`, +`secrets.master_key_env` and `atproto.plc_directory`. Any key can also +come from a `KNOT_*` environment variable. + +:::note + +New repositories use SHA-256. If your users mirror from SHA-1 hosts +such as GitHub, set `git.object_format = "sha1"` before anyone creates a +repository. + +:::: + +## Install on NixOS + +```nix +{ + imports = [ tangled.nixosModules.knot-rs ]; + + services.tangled.knot-rs = { + enable = true; + environmentFile = "/etc/knot/knot.env"; + settings = { + server = { + hostname = "knot.example.com"; + admins = [ "did:plc:yourdid" ]; + }; + atproto.plc_directory = "https://plc.directory"; + lfs.store_path = "/var/lib/knot/lfs"; + contact.email = "your@contact.email"; + }; + }; +} +``` + +The knot listens on `127.0.0.1:5555` and keeps its state in +`/var/lib/knot`. + +## Install with a container + +Use `atcr.io/tangled.org/knot:2`. It runs as uid 1000, so `chown` your +data directories to `1000:1000`, and set `KNOT_SCAN_PATH`, +`KNOT_SEALED_KEY_FILE` and `KNOT_SSH_HOST_KEY_FILE` to your mounted +paths. [Run a knot](/docs/self-hosting/knot/#run-with-docker) has a full +compose file. + +## Running under systemd + +We show an example [`systemd/knot.service`](https://tangled.org/did:plc:j5hmlfdrwkvtxm7cjmu7j2is/blob/master/systemd/knot.service) installable to `/etc/systemd/system/` for running the binary straight on the machine, then `systemctl enable --now knot`. + +It runs the knot as `git` (well remember this is just for fun, not because we *have* to have a git user mind you) from `/usr/local/bin/knot-server`, so edit `User=`, `Group=` and `ExecStart=` if one's setup differs. `ProtectSystem=strict` and `ReadWritePaths=/var/lib/knot` such that anything the knot writes outside the tree gets its own `ReadWritePaths=` entry, an LFS store or an ACME cache on another disk being the usual suspects. `ProtectHome=true` has to go if any of the paths is under `/home`. The two `CAP_NET_BIND_SERVICE` lines let it bind a port below 1024, and both can go when SSH stays on 2222 and HTTP on 5555. + +## TLS + +**Behind a reverse proxy.** Proxy to port 5555, and don't add CORS headers at the proxy. Set `xrpc.trusted_proxy_header = "x-forwarded-for"` when doing this, otherwise every client looks like it comes from the proxy and the ratelimiter will treat them as one very busy mister. Only set it behind a proxy the operator controls, since a direct client can like, invent that header. + +Add `xrpc.trusted_proxies = ["fd00:1::4", "10.89.0.4"]` for example, one entry per address that the proxy connects from, so the knot honors that header from the proxy alone & ratelimits anyone else by the address they connected from. A CIDR block will work too, `["173.245.48.0/20"]` covers a whole provider's edge. If several proxies you control are in the path, list them all. Mister knot will read the chain right -> left, iterate over every entry the list covers, and take the first entry it doesn't. It will read 32 entries at most, and the knot will ratelimit by the address the request connected from when the list covers all 32. + +The knot can also terminate TLS itself (and that's the only way to get its HTTP3 support) because a plain TCP frontend can't proxy QUIC. Using a certificate the operator already manages: + +```toml +[server] +listen_addr = "[::]:443" + +[tls] +cert_path = "/data/tls/fullchain.pem" +key_path = "/data/tls/privkey.pem" +``` + +Or let it fetch its own via ACME: + +```toml +[server] +listen_addr = "[::]:443" + +[tls] +acme_enabled = true +acme_cache_dir = "/data/acme" +acme_contact = "nel@oyster.cafe" +``` + +ACME here uses the TLS-ALPN-01 challenge, so the knot has to be the receiver (in the phone sense) ponging on 443 for the configured hostname. Set `acme_staging = true` while testing such that a typo doesn't nuke the Let's Encrypt ratelimit. Leave `trusted_proxy_header` unset in this mode, and open 443/udp in the firewall if one wants HTTP3 to be reachable. + +## First run + +```sh +curl https://knot.example.com/xrpc/_health +``` + +Then add the knot under Settings → Knots in the web app. + +## Members and invites + +By default only admins can create repositories. Set +`acl.admission = "open"` to let anyone in, or add members with the +admin commands. These need `acl.legacy_admin_secret_env` set: + +```sh +knot-server --config config.toml admin member add did:plc:friend +knot-server --config config.toml admin create-invite-code +``` + +## Git LFS + +We use the now-required `lfs.store_path` to have the knot save +LFS objects, atproto record attachments (from issues and pulls, say) +and soon LOPs objects too. Suggestion: choose a directory that's +alongside the repos folder, like `/var/lib/knot/lfs` beside +`/var/lib/knot/repos` for example. If you don't plan to have actual +git repos using this for massive files, then it'll be fine on the +same disk as the git repos; if you *do* plan to have massive files, +consider putting the LFS folder on a different disk entirely like an +HDD. Wherever you put the folder, keep it out of the `repo.scan_path`. + +## Backups + +Back these things up! It's of the utmost importance. + +1. The master key, wherever one keeps it. +2. `secrets/sealed.bin`, the sealed key store. It's useless without the master key & the master key is useless without it, so treat them as a pair. +3. `repos/`, which is every repo plus the knot's own ACL data. Git is the database after all, so that one directory is the knot's entire state. +4. `lfs/`, the LFS/LOPs/ atproto record attachments. +5. I guess the SSH host key, though much less dire if it is lost and has to be changed. + +## Updates + +Stop the knot, replace the binary or image, and start it again. Read +[Upgrading knots and spindles](/docs/upgrading/knots-and-spindles/) +first. + +Pre-built images are at `atcr.io/tangled.org/knot:latest`, +with `knot-server` and `knot-migrate` in them. +The image comes from [@tangled.org/knot-docker](https://tangled.org/did:plc:f5s5la5wlofsxidb3zemdune) +instead of the `knot2/Containerfile` in this repo. + +It runs as its own `knot` user at uid 1000, +so chown the data directories to `1000:1000` before the first start. + +The knot drains at `SIGTERM` time, so in-flight clones/pushes get up to 40s to finish before they're cut off. + +## Troubleshooting + +- The knot won't start: run `knot-server validate` with the same config + and environment. +- A push is rejected for an SSH key: publish the key on the user's PDS, + then wait up to an hour for the knot's cache. +- Every user is rate-limited together: set `xrpc.trusted_proxy_header` + and `xrpc.trusted_proxies`. + +[Knot Configuration](/docs/reference/knot-config/) lists every setting. diff --git a/docs/src/content/docs/migrating-from-github.md b/docs/src/content/docs/migrating-from-github.md new file mode 100644 index 0000000000000000000000000000000000000000..0bbb242e00bdef4fbf1a8f495e9876d1bd9d8b4d --- /dev/null +++ b/docs/src/content/docs/migrating-from-github.md @@ -0,0 +1,107 @@ +--- +title: Migrating from GitHub +description: + Copy a repository from GitHub with the importer, then configure the Git + remotes to push to Tangled. +--- + +You can migrate your repositories, associated emails, and SSH keys from +GitHub to Tangled using the [importer +tool](https://tangled.org/repo/migrate). You can read more about the +importer tool [here](/docs/guides/importing/). You may also migrate +repositories without the importer. + +## Using the importer tool + +This is a quick and easy way to migrate most of your data from GitHub to +Tangled: + +1. Head to the [importer](https://tangled.org/repo/migrate). +2. Authorize the importer to grant read access to your GitHub account. +3. Configure repositories and migrate them! + +The importer requries read-only access to your GitHub account, and you +can revoke access once repositories are imported. If you wish to avoid +the tool, you can also migrate without it, as done below. + +## Without the importer tool + +You can move a repository without the importer. Use a local clone. + +1. Create an empty repository on Tangled. [Quick start](/docs/quick-start/) + shows the procedure. +2. Open the repository on your computer. + + ```sh + cd /path/to/your/repository + ``` + +3. Show the current remote. + + ```sh + git remote -v + ``` + + The output might look like: + + ```text + origin git@github.com:alice/my-project.git (fetch) + origin git@github.com:alice/my-project.git (push) + ``` + +4. Set the remote URL to your Tangled repository. + + ```sh + git remote set-url origin git@tangled.org:alice.tngl.sh/my-project + ``` + +5. Push all branches and tags. + + ```sh + git push origin --all + git push origin --tags + ``` + +All commits, branches and tags are now on Tangled. + +## Pushing to GitHub and Tangled + +To keep GitHub and Tangled in step, use one remote with two push URLs. +Start with a repository whose remote points to GitHub. Git fetches from +the first URL. Git pushes to both URLs. + +1. Show the current remote. + + ```sh + git remote -v + ``` + + ```text + origin git@github.com:alice/my-project.git (fetch) + origin git@github.com:alice/my-project.git (push) + ``` + +2. Add the GitHub URL as the first push URL. Git then uses it to fetch + only. + + ```sh + git remote set-url --add --push origin git@github.com:alice/my-project.git + ``` + +3. Add the Tangled URL as the second push URL. + + ```sh + git remote set-url --add --push origin git@tangled.org:alice.tngl.sh/my-project + ``` + +4. Show the result. + + ```sh + git remote -v + ``` + + ```text + origin git@github.com:alice/my-project.git (fetch) + origin git@github.com:alice/my-project.git (push) + origin git@tangled.org:alice.tngl.sh/my-project (push) + ``` diff --git a/docs/src/content/docs/quick-start.md b/docs/src/content/docs/quick-start.md new file mode 100644 index 0000000000000000000000000000000000000000..d6db3e421ab616d9e91a3502dc25054415f38e5f --- /dev/null +++ b/docs/src/content/docs/quick-start.md @@ -0,0 +1,112 @@ +--- +title: Quick start +description: + Sign in, add an SSH key, create a repository on a knot, and push your + first commit. +--- + +This page takes you from no account to a pushed repository on +[tangled.org](https://tangled.org). It takes a few minutes and needs +only `git` and `ssh`. + +```sh +git remote add origin git@tangled.org:alice.tngl.sh/my-project +git push -u origin main +``` + +## Sign in or sign up + +Tangled accounts are AT Protocol accounts. If you already have one (a +Bluesky account, for example), sign in at +[tangled.org/login](https://tangled.org/login) with your handle, such as +`alice.bsky.social`. Your browser is sent to your account provider to +approve access; Tangled never sees your password. + +If you don't have an account, [sign up](https://tangled.org/signup). You +enter an email address, choose a username and password. The result is an +account with a handle like `alice.tngl.sh`, on the PDS we run. It works +in any other atproto application too. + +You can sign in to several accounts and switch between them from the +account menu. + +## Onboarding + +The first sign-in opens a short, optional setup. If you connect +GitHub, Tangled can import your avatar, SSH keys and email addresses +from it. + +Add at least one verified email if you want email notifications or want +your commits linked to your profile. + +## Add an SSH key + +Pushing over SSH needs a public key published in your account. Generate +one if you don't have one: + +```sh +ssh-keygen -t ed25519 -C "alice@example.com" +cat ~/.ssh/id_ed25519.pub +``` + +Paste the output into **Settings → Keys → Add SSH key** +(`/settings/keys/new`) with a title such as "Work laptop". The key is +saved in your account. + +If your SSH agent holds many keys, the knot may give up before it sees +the right one. Pin the key in `~/.ssh/config`: + +```sh +Host tangled.org + User git + IdentityFile ~/.ssh/id_ed25519 + IdentitiesOnly yes +``` + +Use your knot's hostname instead of `tangled.org` for repositories on a +self-hosted knot. + +## Create a repository + +Head to the [New Repository](https://tangled.org/repo/new) page and fill in: + +1. **Repository name**: letters, digits, `.`, `_` and `-`, up to 100 + characters. A trailing `.git` is dropped, and `self` is reserved. +2. **Description**: optional, up to 140 characters. +3. **Default branch**: `main` unless you change it. +4. **Hosting**: by default the repository is hosted with Tangled. If you + self-host a knot, you can host it there instead; see [Running a + knot](/docs/self-hosting/knot/). +5. **CI**: optional, under **Advanced**. Run it with Tangled, or on a + spindle you self-host. + +## Push + +An empty repository page shows its remote. Push an existing project to +it: + +```sh +cd my-project +git remote add origin git@tangled.org:alice.tngl.sh/my-project +git push -u origin main +``` + +For repositories hosted by Tangled, the SSH host is `tangled.org`. On a +self-hosted knot it is the knot's own hostname, for example +`git@knot.example.com:alice.tngl.sh/my-project`. Note that this depends +on your self-hosted knot's SSH setup. + +## Moving an existing repository + +To move a repository you already push elsewhere, point `origin` at +Tangled and push every branch and tag: + +```sh +git remote set-url origin git@tangled.org:alice.tngl.sh/my-project +git push origin --all +git push origin --tags +``` + +If you are coming from GitHub, check out the [Migration +guide](/docs/migrating-from-github/), that walks you through a mass +repository migration tool. diff --git a/docs/src/content/docs/reference/identifiers.md b/docs/src/content/docs/reference/identifiers.md new file mode 100644 index 0000000000000000000000000000000000000000..bc0ff2575bec805e8fbcbf14d3e801913b39e56e --- /dev/null +++ b/docs/src/content/docs/reference/identifiers.md @@ -0,0 +1,170 @@ +--- +title: Identifiers and Git URLs +description: + DIDs, handles, AT URIs, record keys, repository addressing, clone + URLs, reserved refs and push options. +--- + +Tangled names accounts, repositories and records with AT Protocol +identifiers, and names commits and branches with ordinary Git +identifiers. This page lists each form, what it points at, and when it +changes. + +```sh +git clone git@knot.example.com:alice.example.com/proj +git clone https://knot.example.com/did:plc:4kx2vqmn7d3rjzy5lbk6hq3c +``` + +| Identifier | Example | Names | Changes when | +| -------------- | --------------------------------------------- | -------------------------------- | -------------------------------------- | +| Handle | `alice.example.com` | An account | The owner changes it | +| Account DID | `did:plc:…` | A user or organization | Never | +| Repository DID | `did:plc:…` | One repository | Never, including renames and transfers | +| Service DID | `did:web:knot.example.com` | A knot, spindle or other service | Its hostname changes | +| AT URI | `at://did:plc:…/org.tangled.track.ticket/3l…` | One record | The record is deleted | +| CID | `bafyrei…` | One version of a record | The record changes | +| NSID | `org.tangled.track.ticket` | A record type or XRPC method | Never | +| Git object ID | `a1b2…` (40 or 64 hex) | One commit, tree or blob | Its content changes | +| Git ref | `refs/heads/main` | A branch or tag | It moves on push | +| Pipeline ID | spindle-local | One CI run on one spindle | Never; meaningless on another spindle | + +## Handles and DIDs + +A handle is the readable name of an account and resolves to a DID. +Records and permissions store the DID, so changing a handle does not +rewrite anything. Web URLs accept either form: +`https://tangled.org/alice.example.com/proj` and +`https://tangled.org/did:plc:…/proj` show the same repository. + +Users and repositories use `did:plc`. Knots and spindles use `did:web` +derived from their hostname: a knot at `knot.example.com` is +`did:web:knot.example.com`, and its DID document is served at +`https://knot.example.com/.well-known/did.json`. + +## Repository DIDs + +Every repository has its own `did:plc`, separate from its owner's. The +knot that hosts it mints it (or adopts one the client prepared with +`org.tangled.repo.reserveKey`) and holds its rotation and signing key. +Its DID document lists the knot as both `#atproto_pds` and +`#tangled_knot`. + +The repository DID is the stable handle for everything about the +repository: its records, CI pipelines, secrets, webhooks and git mirror +all key on it. The human name is claimed by the owner, under a slug that +is the lowercased name: + +- The repository account holds `org.tangled.repo.manifest/self` with + `declaration: {owner, name}`. +- The owner's PDS holds `org.tangled.repo.declaration/` with + `repo: `. + +A repository is only visible under `owner/slug` while both sides agree. +Renaming changes the name, the slug and the declaration; the repository +DID stays the same. + +## AT URIs and record keys + +```text +at://// +``` + +The first segment is the account whose repository holds the record, +which is not always the author. Issues, pull requests, comments, +reactions, labels and patches live in the **repository DID's** account +on the knot, so their URIs start with the repository DID: + +```text +at://did:plc:repoxyz/org.tangled.track.ticket/3lzq4w5qk2c2a +``` + +The person who wrote such a record is in its `x-tngl-editor` field, +stamped by the knot. Stars, follows, vouches, profiles and strings live +in the author's own PDS and start with the author's DID. + +Record key (rkey) rules depend on the collection: + +| Rule | Collections | +| ----------------------- | ------------------------------------------------------------------------ | +| `self` | `actor.profile`, `repo.manifest`, `git.refs` | +| Repository slug | `repo.declaration` | +| Subject DID | `graph.follow`, `graph.vouch`, `graph.collaborator` | +| The record's own CID | `review.patch` | +| TID, minted by the knot | `track.ticket`, `feed.comment`, `feed.reaction` written through the knot | +| TID | everything else | + +Web URLs for issues and pull requests use the ticket's rkey: +`/alice.example.com/proj/issues/3lzq4w5qk2c2a`. There are no sequential +numbers. + +[Records](/docs/reference/records/) lists every collection. + +## Git object format + +New repositories on a knot use SHA-256 by default, so object IDs are 64 +hex characters. Repositories migrated from the Go knot keep SHA-1 (40 +characters). A SHA-256 repository can't push to or fetch from a SHA-1 +one. Don't assume a fixed hash length in tools. + +## Clone URLs + +A knot serves each repository under two addresses: owner plus name, or +the repository DID. The owner can be a handle or a DID. + +Over SSH: + +```sh +git clone git@knot.example.com:alice.example.com/proj +git clone git@knot.example.com:did:plc:alice/proj +git clone git@knot.example.com:did:plc:repoxyz +``` + +Over HTTPS: + +```sh +git clone https://knot.example.com/alice.example.com/proj +git clone https://knot.example.com/did:plc:alice/proj +git clone https://knot.example.com/did:plc:repoxyz +``` + +- A trailing `.git` is accepted on the name form (`proj.git`) but + **not** on the DID form: `did:plc:repoxyz.git` won't resolve. +- The knot's SSH listener defaults to port 2222. If the knot doesn't + listen on 22, use `ssh://git@knot.example.com:2222/did:plc:repoxyz` or + a `Host` block in `~/.ssh/config`. +- The DID form keeps working after a rename; the name form follows the + current slug. + +SSH push authenticates with a key published in your PDS. If your agent +offers several keys and the knot rejects them, add `IdentitiesOnly yes` +for that host. HTTPS push takes a service-auth token bound to +`org.tangled.git.receivePack` as the Bearer token or Basic password, supplied +by a credential helper. + +## Reserved refs + +Knots don't allow pushes to these namespaces. All of these are also not under ref advertisement, other than `refs/cobs/*` which can be +fetched with an explicit `refs/cobs/` ref-prefix, read only. + +| Ref prefix | Holds | +| ------------------------ | ------------------------------------------------------------------------------------------------------- | +| `refs/cobs/*` | Collaborative objects: signed change graphs for collaborators, tickets, comments and more. Append-only. | +| `refs/cob-checkpoints/*` | Knot-written COB snapshots. | +| `refs/hidden/*` | Nothing. Legacy fork staging refs are deleted on boot; the prefix stays reserved. | +| `refs/atproto/*` | The repository DID's atproto commit chain, PLC queue and tombstones. | + +## Push options + +Push options pass through to the knot's `org.tangled.git.pushEvent` (at +most 10) and to spindle. + +```sh +git push -o skip-ci origin main +``` + +| Option | Effect | +| ---------------------- | ------------------------------------------------------------------------- | +| `skip-ci` or `ci-skip` | Spindle does not start pipelines for this push. Push webhooks still fire. | + +[Workflow Reference](/docs/spindles/workflows/) describes how pushes trigger +pipelines. diff --git a/docs/src/content/docs/reference/knot-config.md b/docs/src/content/docs/reference/knot-config.md new file mode 100644 index 0000000000000000000000000000000000000000..f7a6641e0b83d6ec0577450f63eef1ac7e0a49e2 --- /dev/null +++ b/docs/src/content/docs/reference/knot-config.md @@ -0,0 +1,352 @@ +--- +title: Knot configuration +description: knot-server configuration keys, environment variables and defaults. +--- + +`knot-server` reads TOML from the file given with `--config` (or `KNOT_CONFIG`), then from `/etc/knot/config.toml`. Each key also has a `KNOT_*` environment variable, and environment variables take precedence over both files. Sizes are in bytes; durations are in the unit named by the key suffix (`_ms`, `_secs`). + +## `[server]` + +| Key | Env var | Default | Description | +| --------------------------------- | -------------------------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `hostname` | `KNOT_HOSTNAME` | required | Public hostname. The knot's identity is `did:web:`. | +| `admins` | `KNOT_ADMINS` | required | Admin DIDs. The first entry is the knot's service owner. Comma-separated as an env var. | +| `listen_addr` | `KNOT_LISTEN_ADDR` | `[::]:5555` | HTTP listen address for XRPC, smart HTTP git, LFS and the homepage. | +| `listen_header_timeout_ms` | `KNOT_LISTEN_HEADER_TIMEOUT_MS` | `10000` | Time allowed to receive request headers. | +| `listen_idle_timeout_ms` | `KNOT_LISTEN_IDLE_TIMEOUT_MS` | `60000` | Idle connection timeout. Must be at least `listen_header_timeout_ms`. | +| `listen_max_connections` | `KNOT_LISTEN_MAX_CONNECTIONS` | `1024` | Maximum concurrent HTTP connections. | +| `listen_rate_limit_per_second` | `KNOT_LISTEN_RATE_LIMIT_PER_SECOND` | `50` | HTTP request rate limit per client IP. | +| `listen_rate_limit_burst` | `KNOT_LISTEN_RATE_LIMIT_BURST` | `200` | Burst size for the per-client-IP request rate limit. | +| `listen_max_inflight_requests` | `KNOT_LISTEN_MAX_INFLIGHT_REQUESTS` | `1024` | Maximum HTTP requests in flight. Excess requests get `503`. | +| `listen_request_timeout_ms` | `KNOT_LISTEN_REQUEST_TIMEOUT_MS` | `60000` | Timeout for an HTTP request, except streaming writes. | +| `listen_body_timeout_ms` | `KNOT_LISTEN_BODY_TIMEOUT_MS` | `30000` | Inactivity timeout while reading a request body. | +| `listen_write_request_timeout_ms` | `KNOT_LISTEN_WRITE_REQUEST_TIMEOUT_MS` | `1800000` | Timeout for streaming writes: `git-receive-pack` POSTs and LFS object PUTs. | +| `internal_listen_addr` | `KNOT_INTERNAL_LISTEN_ADDR` | `[::1]:5444` | Listen address of the mTLS admin listener. Used only when `tls.mtls_enabled` is set. | +| `ssh_listen_addr` | `KNOT_SSH_LISTEN_ADDR` | `[::]:2222` | Listen address of the built-in SSH server for git and LFS. | +| `ssh_host_key_file` | `KNOT_SSH_HOST_KEY_FILE` | required | SSH host key path. Absolute. Generated on start if absent. | +| `ssh_max_pack_bytes` | `KNOT_SSH_MAX_PACK_BYTES` | `8589934592` | Largest pack accepted in one SSH push. | +| `ssh_max_inflight_per_peer` | `KNOT_SSH_MAX_INFLIGHT_PER_PEER` | `4` | Maximum number of SSH git operations that one client address can run at the same time. | +| `ssh_proxy_protocol_from` | `KNOT_SSH_PROXY_PROTOCOL_FROM` | `[]` | IPs or CIDR blocks allowed to send a PROXY protocol v1 header on SSH connections. SSH rate limits then key on the header's source address. Comma-separated as an env var. | +| `appview_endpoint` | `KNOT_APPVIEW_ENDPOINT` | `https://tangled.org` | Base URL of the appview, used to build links in push messages. | + +## `[tls]` + +| Key | Env var | Default | Description | +| --------------------- | ------------------------------ | ------- | ----------------------------------------------------------------------------------------------- | +| `cert_path` | `KNOT_TLS_CERT_PATH` | unset | PEM certificate chain for TLS on the knot. Absolute. Set together with `key_path`. | +| `key_path` | `KNOT_TLS_KEY_PATH` | unset | PEM private key for `cert_path`. Absolute. | +| `http3` | `KNOT_TLS_HTTP3` | `true` | Serve HTTP/3 over QUIC when the knot has a certificate. | +| `acme_enabled` | `KNOT_TLS_ACME_ENABLED` | `false` | Obtain the certificate over ACME (TLS-ALPN-01). Can't combine with `cert_path`/`key_path`. | +| `acme_cache_dir` | `KNOT_TLS_ACME_CACHE_DIR` | unset | ACME account and certificate cache directory. Absolute. Required with `acme_enabled`. | +| `acme_contact` | `KNOT_TLS_ACME_CONTACT` | unset | Contact email for the ACME account. Required with `acme_enabled`. | +| `acme_staging` | `KNOT_TLS_ACME_STAGING` | `false` | Use the Let's Encrypt staging directory. | +| `mtls_enabled` | `KNOT_TLS_MTLS_ENABLED` | `false` | Serve the mTLS admin listener on `server.internal_listen_addr`. Requires a server certificate. | +| `mtls_client_ca_path` | `KNOT_TLS_MTLS_CLIENT_CA_PATH` | unset | CA bundle that admin client certificates must chain to. Absolute. Required with `mtls_enabled`. | +| `mtls_admin_spki_pin` | `KNOT_TLS_MTLS_ADMIN_SPKI_PIN` | unset | Base64 SHA-256 SPKI pin of the admin client certificate. Required with `mtls_enabled`. | + +## `[acl]` + +| Key | Env var | Default | Description | +| ------------------------- | ------------------------------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `admission` | `KNOT_ADMISSION` | `closed` | Repo creation policy. `closed`: only knot members. `open`: any account that isn't blocked. | +| `legacy_admin_secret_env` | `KNOT_LEGACY_ADMIN_SECRET_ENV` | unset | Name of an env var holding a shared admin secret. When set and non-empty, the knot accepts Basic Auth as the first DID in `server.admins` on `createRecord`/`deleteRecord` for `org.tangled.knot.member` and `org.tangled.knot.block`, and serves the invite code admin method. Used by `knot-server admin`. | + +## `[repo]` + +| Key | Env var | Default | Description | +| --------------------- | --------------------------- | --------------- | -------------------------------------------------------------------------------------------------- | +| `scan_path` | `KNOT_SCAN_PATH` | required | Root directory of all repositories and the knot's meta-repo. Absolute. Must exist and be writable. | +| `default_branch` | `KNOT_DEFAULT_BRANCH` | `main` | Default branch of new repositories. | +| `contribution_policy` | `KNOT_CONTRIBUTION_POLICY` | `anyone` | Contribution policy: `anyone`, `members` or `collaborators`. | +| `write_burst` | `KNOT_REPO_WRITE_BURST` | `30` | Per-account burst for authenticated writes. | +| `write_refill_ms` | `KNOT_REPO_WRITE_REFILL_MS` | `500` | Refill interval of the per-account write bucket. 1 to 60000. | + +## `[git]` + +| Key | Env var | Default | Description | +| --------------- | ------------------------ | -------- | ------------------------------------------------------ | +| `object_format` | `KNOT_GIT_OBJECT_FORMAT` | `sha256` | Object format of new repositories: `sha256` or `sha1`. | + +## `[secrets]` + +| Key | Env var | Default | Description | +| ----------------- | ---------------------- | -------- | ----------------------------------------------------------------------------------------------------- | +| `sealed_key_file` | `KNOT_SEALED_KEY_FILE` | required | Sealed key store holding the knot and per-repo signing keys. Absolute. Created if absent. | +| `master_key_env` | `KNOT_MASTER_KEY_ENV` | required | Name of the env var holding the master key that unseals `sealed_key_file`: base64, at least 32 bytes. | + +## `[http]` + +| Key | Env var | Default | Description | +| ------------------------- | ----------------------------------- | ---------- | ----------------------------------------------------------------------------- | +| `connect_timeout_ms` | `KNOT_HTTP_CONNECT_TIMEOUT_MS` | `5000` | Connect timeout for outbound HTTP requests. | +| `read_timeout_ms` | `KNOT_HTTP_READ_TIMEOUT_MS` | `30000` | Read timeout for outbound HTTP requests. | +| `request_timeout_ms` | `KNOT_HTTP_REQUEST_TIMEOUT_MS` | `60000` | Overall timeout for outbound HTTP requests. | +| `max_response_bytes` | `KNOT_HTTP_MAX_RESPONSE_BYTES` | `16777216` | Largest outbound HTTP response body read. | +| `block_private_addresses` | `KNOT_HTTP_BLOCK_PRIVATE_ADDRESSES` | `true` | Refuse outbound requests to loopback, private and other non-public addresses. | + +## `[atproto]` + +| Key | Env var | Default | Description | +| ------------------- | ------------------------ | -------- | ------------------------------------------------------------------------------------------------------------ | +| `plc_directory` | `KNOT_PLC_DIRECTORY` | required | PLC directory URL. Must be `https`. | +| `plc_sweep_width` | `KNOT_PLC_SWEEP_WIDTH` | `1` | Num of concurrent PLC reconciliations the directory sweep runs, must be between 1 and 64. | +| `repo_signing_keys` | `KNOT_REPO_SIGNING_KEYS` | unset | Path to a per-repo signing key archive from `knot-migrate`. Its keys are sealed into the key store on start. | + +## `[xrpc]` + +| Key | Env var | Default | Description | +| ------------------------------ | ---------------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `max_body_bytes` | `KNOT_XRPC_MAX_BODY_BYTES` | `65536` | XRPC request body limit. | +| `max_response_bytes` | `KNOT_XRPC_MAX_RESPONSE_BYTES` | `5242880` | XRPC response size limit. | +| `max_archive_bytes` | `KNOT_XRPC_MAX_ARCHIVE_BYTES` | `1073741824` | Upper bound on one archive across `org.tangled.git.getArchive`, `git archive --remote` over SSH and the smart HTTP archive route. | +| `tree_last_commit_budget_ms` | `KNOT_XRPC_TREE_LAST_COMMIT_BUDGET_MS` | `300` | Time budget for last-commit lookup in tree listings. | +| `blob_last_commit_budget_ms` | `KNOT_XRPC_BLOB_LAST_COMMIT_BUDGET_MS` | `2000` | Time budget for last-commit lookup for a blob. | +| `languages_budget_ms` | `KNOT_XRPC_LANGUAGES_BUDGET_MS` | `1000` | Time budget for language statistics. | +| `max_patch_bytes` | `KNOT_XRPC_MAX_PATCH_BYTES` | `16777216` | Body limit for the `seedRepo` procedure. | +| `max_patch_decompressed_bytes` | `KNOT_XRPC_MAX_PATCH_DECOMPRESSED_BYTES` | `134217728` | Limit on the total decompressed size of a patch the merge procedures parse. | +| `preauth_burst` | `KNOT_XRPC_PREAUTH_BURST` | `20` | Per-peer burst for unauthenticated requests. | +| `preauth_refill_ms` | `KNOT_XRPC_PREAUTH_REFILL_MS` | `100` | Refill interval of the per-peer pre-auth bucket. | +| `per_peer_inflight` | `KNOT_XRPC_PER_PEER_INFLIGHT` | `8` | Concurrent pre-auth requests per peer. | +| `global_inflight` | `KNOT_XRPC_GLOBAL_INFLIGHT` | `64` | Concurrent pre-auth requests across all peers. Must be at least `per_peer_inflight`. | +| `max_pending_reservations` | `KNOT_XRPC_MAX_PENDING_RESERVATIONS` | `256` | Knot-wide limit on reserved repository keys awaiting creation. | +| `per_actor_reservations` | `KNOT_XRPC_PER_ACTOR_RESERVATIONS` | `32` | Per-account limit on reserved repository keys awaiting creation. | +| `reservation_ttl_secs` | `KNOT_XRPC_RESERVATION_TTL_SECS` | `3600` | Time a reserved repository key is held before it lapses and its sealed key is reclaimed. | +| `fork_max_pack_bytes` | `KNOT_XRPC_FORK_MAX_PACK_BYTES` | `1073741824` | Largest pack fetched when creating a fork. | +| `import_max_pack_bytes` | `KNOT_XRPC_IMPORT_MAX_PACK_BYTES` | `268435456` | Largest pack fetched when importing a repository. | +| `fork_fetch_timeout_ms` | `KNOT_XRPC_FORK_FETCH_TIMEOUT_MS` | `600000` | Request timeout for the fork fetch. | +| `trusted_proxy_header` | `KNOT_XRPC_TRUSTED_PROXY_HEADER` | unset | Header a reverse proxy appends the client address to, such as `x-forwarded-for`. Read right to left; the first entry not covered by `trusted_proxies` is the client. Unset uses the socket peer address. | +| `trusted_proxies` | `KNOT_XRPC_TRUSTED_PROXIES` | `[]` | IPs or CIDR blocks whose `trusted_proxy_header` is honored. At most 32 hops are read. Empty honors the header from every peer and takes its rightmost entry. Requires `trusted_proxy_header`. Comma-separated as an env var. | +| `events_replay_buffer` | `KNOT_XRPC_EVENTS_REPLAY_BUFFER` | `4096` | Events kept for `subscribeRepos` cursor replay. | +| `events_replay_bytes` | `KNOT_XRPC_EVENTS_REPLAY_BYTES` | `67108864` | Byte cap on the replay buffer. | +| `events_max_subscribers` | `KNOT_XRPC_EVENTS_MAX_SUBSCRIBERS` | `256` | Maximum concurrent `subscribeRepos` subscribers. | +| `events_max_per_peer` | `KNOT_XRPC_EVENTS_MAX_PER_PEER` | `8` | Maximum `subscribeRepos` subscriptions per peer. | + +## `[maintenance]` + +| Key | Env var | Default | Description | +| --------------------------- | -------------------------------------------- | ---------- | -------------------------------------------------------------------------------- | +| `enabled` | `KNOT_MAINTENANCE_ENABLED` | `true` | Run background git maintenance. | +| `commit_graph` | `KNOT_MAINTENANCE_COMMIT_GRAPH` | `true` | Write commit-graph files. | +| `multi_pack_index` | `KNOT_MAINTENANCE_MULTI_PACK_INDEX` | `true` | Write multi-pack indexes. | +| `bitmap` | `KNOT_MAINTENANCE_BITMAP` | `true` | Write reachability bitmaps. | +| `interval_secs` | `KNOT_MAINTENANCE_INTERVAL_SECS` | `21600` | Interval between maintenance passes. | +| `repack_max_objects` | `KNOT_MAINTENANCE_REPACK_MAX_OBJECTS` | `16000000` | A full repack is skipped for a repository with more reachable objects than this. | +| `repack_geometric_factor` | `KNOT_MAINTENANCE_REPACK_GEOMETRIC_FACTOR` | `2` | Geometric factor for pack roll-up. At least 2. | +| `repack_loose_threshold` | `KNOT_MAINTENANCE_REPACK_LOOSE_THRESHOLD` | `6700` | A repack is due when a repository has more loose objects than this. | +| `prune_grace_secs` | `KNOT_MAINTENANCE_PRUNE_GRACE_SECS` | `1209600` | Unreachable objects younger than this are kept. | +| `full_repack_interval_secs` | `KNOT_MAINTENANCE_FULL_REPACK_INTERVAL_SECS` | `432000` | Minimum interval between full repacks and reflog expiry of a repository. | +| `reflog_expire_secs` | `KNOT_MAINTENANCE_REFLOG_EXPIRE_SECS` | `7776000` | Reflog retention. | +| `large_push_bytes` | `KNOT_MAINTENANCE_LARGE_PUSH_BYTES` | `52428800` | A push pack of at least this size triggers maintenance of that repository. | + +## `[pack_cache]` + +| Key | Env var | Default | Description | +| ----------------- | --------------------------------- | ------------ | --------------------------------------------------------------- | +| `enabled` | `KNOT_PACK_CACHE_ENABLED` | `true` | Cache generated packs for repeated identical fetches. | +| `ttl_secs` | `KNOT_PACK_CACHE_TTL_SECS` | `60` | Lifetime of a cached pack. | +| `max_entry_bytes` | `KNOT_PACK_CACHE_MAX_ENTRY_BYTES` | `67108864` | Largest single cached pack. | +| `max_total_bytes` | `KNOT_PACK_CACHE_MAX_TOTAL_BYTES` | `2147483648` | Total pack cache size, clamped to a share of the memory budget. | + +## `[pack]` + +| Key | Env var | Default | Description | +| ---------------------------- | -------------------------------------- | ------------- | --------------------------------------------------- | +| `max_objects` | `KNOT_PACK_MAX_OBJECTS` | `16000000` | Most objects in one generated pack. | +| `max_total_bytes` | `KNOT_PACK_MAX_TOTAL_BYTES` | `68719476736` | Largest generated pack. | +| `selection_max_objects` | `KNOT_PACK_SELECTION_MAX_OBJECTS` | `16000000` | Most objects walked when selecting objects to send. | +| `selection_time_budget_secs` | `KNOT_PACK_SELECTION_TIME_BUDGET_SECS` | `600` | Time budget for object selection. | + +## `[lfs]` + +| Key | Env var | Default | Description | +| ------------------------ | --------------------------------- | ------------ | ------------------------------------------------------------------------------------------- | +| `store_path` | `KNOT_LFS_STORE_PATH` | required | LFS objects and atproto record attachments; must be absolute path, must exist + be writable, must be outside the `repo.scan_path`. | +| `max_object_bytes` | `KNOT_LFS_MAX_OBJECT_BYTES` | `5368709120` | Largest LFS object. | +| `free_space_floor_bytes` | `KNOT_LFS_FREE_SPACE_FLOOR_BYTES` | `1073741824` | Uploads are refused when free disk space is below this. | +| `gc_grace_secs` | `KNOT_LFS_GC_GRACE_SECS` | `1209600` | Unreferenced LFS objects younger than this are kept. | +| `gc_interval_secs` | `KNOT_LFS_GC_INTERVAL_SECS` | `21600` | Interval between LFS garbage collection passes. | +| `max_ssh_transfers` | `KNOT_LFS_MAX_SSH_TRANSFERS` | `16` | Concurrent LFS transfers over SSH. | +| `max_http_downloads` | `KNOT_LFS_MAX_HTTP_DOWNLOADS` | `64` | Concurrent LFS downloads over HTTP. | + +## `[social]` + +| Key | Env var | Default | Description | +| ------------------- | ------------------------------- | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `repo_budget_bytes` | `KNOT_SOCIAL_REPO_BUDGET_BYTES` | `1073741824` | Byte budget over the live record versions of one repository, across all collections. A write past the budget gets `429`. | +| `foster_url` | `KNOT_SOCIAL_FOSTER_URL` | `https://knot2.mock.tangled.org` | Foster knot the knot pulls shadowed social records from at boot, for the one-time 2026-10-28 upgrade. Empty disables the pull. | +| `foster_boot_width` | `KNOT_SOCIAL_FOSTER_BOOT_WIDTH` | `4` | Num of concurrent handovers the boot-time foster pull will do, must be between 1 and 256. | + +## `[keyfill]` + +| Key | Env var | Default | Description | +| ---------------------- | ----------------------------------- | ------- | ----------------------------------------------------------------------------------------- | +| `key_budget_mib` | `KNOT_KEYFILL_KEY_BUDGET_MIB` | `64` | Memory budget for cached SSH keys, in MiB. 1 to 1048576. | +| `ttl_secs` | `KNOT_KEYFILL_TTL_SECS` | `3600` | Time a fetched SSH key set stays valid. | +| `reprieve_retry_secs` | `KNOT_KEYFILL_REPRIEVE_RETRY_SECS` | `300` | Retry interval while a key refresh fails. | +| `reprieve_budget_secs` | `KNOT_KEYFILL_REPRIEVE_BUDGET_SECS` | `21600` | Time cached keys stay usable while refresh fails. Must be at least `reprieve_retry_secs`. | + +## `[resources]` + +| Key | Env var | Default | Description | +| ------------------ | ----------------------- | ------- | ------------------------------------------------------------------------ | +| `max_threads` | `KNOT_MAX_THREADS` | `0` | Worker thread cap. `0` uses the available parallelism. | +| `max_memory_bytes` | `KNOT_MAX_MEMORY_BYTES` | `0` | Memory budget. `0` derives it from the cgroup limit or available memory. | + +## `[homepage]` + +| Key | Env var | Default | Description | +| --------- | ----------------------- | ------- | --------------------------------------------------------------------------- | +| `enabled` | `KNOT_HOMEPAGE_ENABLED` | `true` | Serve an HTML homepage at `/`. | +| `path` | `KNOT_HOMEPAGE_PATH` | unset | HTML file served as the homepage. Absolute. Unset serves the built-in page. | + +## `[links]` + +| Key | Env var | Default | Description | +| ------------------ | ----------------------------- | ------- | --------------------------------------------------------------- | +| `privacy_policy` | `KNOT_LINKS_PRIVACY_POLICY` | unset | Privacy policy URL, served by `org.tangled.knot.describeKnot`. | +| `terms_of_service` | `KNOT_LINKS_TERMS_OF_SERVICE` | unset | Terms of service URL, served by `org.tangled.knot.describeKnot`. | + +## `[contact]` + +| Key | Env var | Default | Description | +| ------- | -------------------- | -------- | --------------------------------------------------------------- | +| `email` | `KNOT_CONTACT_EMAIL` | required | Operator contact email, served by `org.tangled.knot.describeKnot`. | + +## `[messages.*]` + +Text the knot sends to git clients. Placeholders in braces are filled in at runtime. Multi-line keys take a list of lines or one newline-separated string. + +### `[messages.push]` + +| Key | Env var | Default | Description | +| ---------------- | ----------------------------------- | ------------------------------------------------------------- | --------------------------------------------- | +| `ack` | `KNOT_MESSAGES_PUSH_ACK` | `["{knot} received {refs}."]` | Multi-line. Placeholders: `{knot}`, `{refs}`. | +| `pull_request` | `KNOT_MESSAGES_PUSH_PULL_REQUEST` | `["", "-> Open pull request for this branch:", " {url}", ""]` | Multi-line. Placeholders: `{url}`. | +| `pipeline_clean` | `KNOT_MESSAGES_PUSH_PIPELINE_CLEAN` | `["pipeline compiled with no diagnostics"]` | Multi-line. Placeholders: none. | +| `pipeline_none` | `KNOT_MESSAGES_PUSH_PIPELINE_NONE` | `["no pipelines to compile"]` | Multi-line. Placeholders: none. | + +### `[messages.fetch]` + +| Key | Env var | Default | Description | +| ------------- | --------------------------------- | ----------------------------------------- | ------------------------------------ | +| `motd` | `KNOT_MESSAGES_FETCH_MOTD` | `["Thanks for using {knot}!"]` | Multi-line. Placeholders: `{knot}`. | +| `enumerating` | `KNOT_MESSAGES_FETCH_ENUMERATING` | `["Enumerating objects: {count}, done."]` | Multi-line. Placeholders: `{count}`. | +| `total` | `KNOT_MESSAGES_FETCH_TOTAL` | `["Total {count}, done."]` | Multi-line. Placeholders: `{count}`. | +| `fatal` | `KNOT_MESSAGES_FETCH_FATAL` | `knot: {error}` | Placeholders: `{error}`. | + +### `[messages.reject]` + +| Key | Env var | Default | Description | +| --------------------------- | ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `reserved_refs` | `KNOT_MESSAGES_REJECT_RESERVED_REFS` | `refs/cobs/*, refs/cob-checkpoints/*, refs/hidden/*, refs/atproto/* and refs/tngl/* are reserved and can't be pushed` | Placeholders: none. | +| `cob_create_only` | `KNOT_MESSAGES_REJECT_COB_CREATE_ONLY` | `existing refs/cobs/* object cannot be modified or deleted over the wire` | Placeholders: none. | +| `cob_delete` | `KNOT_MESSAGES_REJECT_COB_DELETE` | `refs/cobs/* stores append-only collaborative objects and cannot be deleted` | Placeholders: none. | +| `hidden_reserved` | `KNOT_MESSAGES_REJECT_HIDDEN_RESERVED` | `refs/hidden/* is reserved and can't be pushed` | Placeholders: none. | +| `atproto_reserved` | `KNOT_MESSAGES_REJECT_ATPROTO_RESERVED` | `refs/atproto/* is reserved for the knot's atproto commit chain and can't be pushed` | Placeholders: none. | +| `tngl_reserved` | `KNOT_MESSAGES_REJECT_TNGL_RESERVED` | `refs/tngl/* is reserved for knot-written keep refs and can't be pushed` | Placeholders: none. | +| `checkpoint_reserved` | `KNOT_MESSAGES_REJECT_CHECKPOINT_RESERVED` | `refs/cob-checkpoints/* is reserved for knot-written collaborative-object snapshots and can't be pushed` | Placeholders: none. | +| `cob_verification` | `KNOT_MESSAGES_REJECT_COB_VERIFICATION` | `collaborative-object verification failed: {error}` | Placeholders: `{error}`. | +| `ref_exists` | `KNOT_MESSAGES_REJECT_REF_EXISTS` | `reference already exists` | Placeholders: none. | +| `stale_old_value` | `KNOT_MESSAGES_REJECT_STALE_OLD_VALUE` | `stale info: old value doesn't match` | Placeholders: none. | +| `missing_objects` | `KNOT_MESSAGES_REJECT_MISSING_OBJECTS` | `missing necessary objects` | Placeholders: none. | +| `missing_objects_for` | `KNOT_MESSAGES_REJECT_MISSING_OBJECTS_FOR` | `missing necessary objects for {ref}` | Placeholders: `{ref}`. | +| `atomic_failed` | `KNOT_MESSAGES_REJECT_ATOMIC_FAILED` | `atomic transaction failed` | Placeholders: none. | +| `atomic_aborted` | `KNOT_MESSAGES_REJECT_ATOMIC_ABORTED` | `atomic push aborted` | Placeholders: none. | +| `authorization_unavailable` | `KNOT_MESSAGES_REJECT_AUTHORIZATION_UNAVAILABLE` | `authorization unavailable` | Placeholders: none. | +| `unpacker_error` | `KNOT_MESSAGES_REJECT_UNPACKER_ERROR` | `unpacker error` | Placeholders: none. | +| `ref_snapshot_unavailable` | `KNOT_MESSAGES_REJECT_REF_SNAPSHOT_UNAVAILABLE` | `ref snapshot unavailable` | Placeholders: none. | +| `object_migration_failed` | `KNOT_MESSAGES_REJECT_OBJECT_MIGRATION_FAILED` | `object migration failed` | Placeholders: none. | + +### `[messages.ssh]` + +| Key | Env var | Default | Description | +| ---------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------- | +| `greeting` | `KNOT_MESSAGES_SSH_GREETING` | `["Hi {user}! You're authenticated to {knot} knot.", "This knot serves git over ssh, so there's no shell here. :P", "Clone repo with: git clone {knot}:"]` | Multi-line. Placeholders: `{user}`, `{knot}`. | +| `greeting_unknown` | `KNOT_MESSAGES_SSH_GREETING_UNKNOWN` | `["Hi there! This is the {knot} knot.", "This knot serves git over ssh, so there's no shell here. :P", "Clone repo with: git clone {knot}:", "Publish your ssh key to your atproto account so this knot can identify your pushes.", "Put your handle in the url, as in yourhandle@{knot}:, so your ssh client can find your registered key on its own."]` | Multi-line. Placeholders: `{knot}`. | +| `unsupported_command` | `KNOT_MESSAGES_SSH_UNSUPPORTED_COMMAND` | `knot: unsupported command` | Placeholders: none. | +| `too_many_operations` | `KNOT_MESSAGES_SSH_TOO_MANY_OPERATIONS` | `knot: too many concurrent operations from your address, try again shortly` | Placeholders: none. | +| `repo_not_found` | `KNOT_MESSAGES_SSH_REPO_NOT_FOUND` | `knot: repository not found` | Placeholders: none. | +| `index_warming` | `KNOT_MESSAGES_SSH_INDEX_WARMING` | `knot: repository index is warming, retry shortly` | Placeholders: none. | +| `key_not_registered` | `KNOT_MESSAGES_SSH_KEY_NOT_REGISTERED` | `knot: this ssh key doesn't match any key published by the accounts that may push here. Authorized: {authorized}. If your agent offers several keys, add -o IdentitiesOnly=yes so it offers your registered key.` | Placeholders: `{authorized}`. | +| `identity_unavailable` | `KNOT_MESSAGES_SSH_IDENTITY_UNAVAILABLE` | `knot: couldn't read the account records needed to check your ssh key, retry shortly` | Placeholders: none. | +| `push_denied` | `KNOT_MESSAGES_SSH_PUSH_DENIED` | `knot: you aren't authorized to push to this repository.` | Placeholders: none. | +| `shutting_down` | `KNOT_MESSAGES_SSH_SHUTTING_DOWN` | `knot: server is shutting down` | Placeholders: none. | +| `archive_malformed` | `KNOT_MESSAGES_SSH_ARCHIVE_MALFORMED` | `knot: malformed upload-archive request` | Placeholders: none. | +| `archive_timeout` | `KNOT_MESSAGES_SSH_ARCHIVE_TIMEOUT` | `knot: upload-archive request timed out` | Placeholders: none. | +| `archive_failed` | `KNOT_MESSAGES_SSH_ARCHIVE_FAILED` | `knot: upload-archive failed` | Placeholders: none. | +| `advertise_failed` | `KNOT_MESSAGES_SSH_ADVERTISE_FAILED` | `knot: cannot advertise refs` | Placeholders: none. | +| `push_too_large` | `KNOT_MESSAGES_SSH_PUSH_TOO_LARGE` | `knot: push exceeds configured size limit` | Placeholders: none. | +| `receive_deadline` | `KNOT_MESSAGES_SSH_RECEIVE_DEADLINE` | `knot: receive exceeded its time budget` | Placeholders: none. | +| `malformed_pack` | `KNOT_MESSAGES_SSH_MALFORMED_PACK` | `knot: malformed pack stream` | Placeholders: none. | +| `receive_read_error` | `KNOT_MESSAGES_SSH_RECEIVE_READ_ERROR` | `knot: receive read error` | Placeholders: none. | +| `receive_ended_early` | `KNOT_MESSAGES_SSH_RECEIVE_ENDED_EARLY` | `knot: receive stream ended early` | Placeholders: none. | +| `receive_failed` | `KNOT_MESSAGES_SSH_RECEIVE_FAILED` | `knot: receive-pack failed` | Placeholders: none. | + +### `[messages.http]` + +| Key | Env var | Default | Description | +| --------------------- | ---------------------------------------- | -------------------------------------------------- | ------------------------ | +| `push_denied` | `KNOT_MESSAGES_HTTP_PUSH_DENIED` | `you aren't authorized to push to this repository` | Placeholders: none. | +| `repo_not_found` | `KNOT_MESSAGES_HTTP_REPO_NOT_FOUND` | `repository not found` | Placeholders: none. | +| `push_too_large` | `KNOT_MESSAGES_HTTP_PUSH_TOO_LARGE` | `push exceeds the configured size limit` | Placeholders: none. | +| `malformed_pack` | `KNOT_MESSAGES_HTTP_MALFORMED_PACK` | `malformed pack stream: {error}` | Placeholders: `{error}`. | +| `receive_ended_early` | `KNOT_MESSAGES_HTTP_RECEIVE_ENDED_EARLY` | `receive stream ended early` | Placeholders: none. | + +### `[messages.lfs]` + +| Key | Env var | Default | Description | +| --------------------- | --------------------------------------- | --------------------------------------------------------- | ----------------------------------------- | +| `invalid_oid` | `KNOT_MESSAGES_LFS_INVALID_OID` | `invalid LFS oid {value}` | Placeholders: `{value}`. | +| `hash_mismatch` | `KNOT_MESSAGES_LFS_HASH_MISMATCH` | `oid mismatch, declared {declared}, computed {computed}` | Placeholders: `{declared}`, `{computed}`. | +| `size_mismatch` | `KNOT_MESSAGES_LFS_SIZE_MISMATCH` | `size mismatch, declared {declared}, received {received}` | Placeholders: `{declared}`, `{received}`. | +| `size_limit_exceeded` | `KNOT_MESSAGES_LFS_SIZE_LIMIT_EXCEEDED` | `object size {declared} exceeds limit {limit}` | Placeholders: `{declared}`, `{limit}`. | +| `free_space_denied` | `KNOT_MESSAGES_LFS_FREE_SPACE_DENIED` | `free space {free} below floor {floor}` | Placeholders: `{free}`, `{floor}`. | +| `not_found` | `KNOT_MESSAGES_LFS_NOT_FOUND` | `object {oid} not found` | Placeholders: `{oid}`. | +| `framing` | `KNOT_MESSAGES_LFS_FRAMING` | `protocol framing fault: {detail}` | Placeholders: `{detail}`. | +| `too_many` | `KNOT_MESSAGES_LFS_TOO_MANY` | `too many {what} in one message, limit {limit}` | Placeholders: `{what}`, `{limit}`. | +| `unknown_command` | `KNOT_MESSAGES_LFS_UNKNOWN_COMMAND` | `unknown command {command}` | Placeholders: `{command}`. | +| `unsupported_version` | `KNOT_MESSAGES_LFS_UNSUPPORTED_VERSION` | `unsupported version {version}` | Placeholders: `{version}`. | +| `unsupported_hash` | `KNOT_MESSAGES_LFS_UNSUPPORTED_HASH` | `unsupported hash algorithm {algorithm}` | Placeholders: `{algorithm}`. | +| `put_on_download` | `KNOT_MESSAGES_LFS_PUT_ON_DOWNLOAD` | `put-object isn't allowed on a download channel` | Placeholders: none. | +| `verify_on_download` | `KNOT_MESSAGES_LFS_VERIFY_ON_DOWNLOAD` | `verify-object isn't allowed on a download channel` | Placeholders: none. | +| `get_on_upload` | `KNOT_MESSAGES_LFS_GET_ON_UPLOAD` | `get-object isn't allowed on an upload channel` | Placeholders: none. | +| `put_no_body` | `KNOT_MESSAGES_LFS_PUT_NO_BODY` | `put-object is missing its object body` | Placeholders: none. | + +## NixOS module + +`services.tangled.knot-rs` renders `settings` to `/etc/knot/config.toml`, checks it with `knot-server validate --config-only` at build time when the build platform can run the binary, and runs `knot-server --config /etc/knot/config.toml`. Keys under `settings` not listed below pass through unchanged. + +| Option | Default | Description | +| -------------------------------------- | ---------------------------- | --------------------------------------------------------------------------------------------------------- | +| `enable` | `false` | Enable the knot service. | +| `package` | none | Package providing `knot-server`. | +| `user` | `"knot"` | User the knot runs as and owner of the repositories. | +| `group` | value of `user` | Group the knot runs as. | +| `stateDir` | `"/var/lib/knot"` | State directory; base for the path defaults below. | +| `openFirewall` | `true` | Open the TCP port of each non-loopback listen address, and the UDP port for HTTP/3 when TLS is on. | +| `environmentFile` | `null` | systemd `EnvironmentFile`. Must be set; holds the master key. Its `KNOT_*` variables override `settings`. | +| `settings.server.hostname` | none | `server.hostname`. | +| `settings.server.admins` | none | `server.admins`. Non-empty list. | +| `settings.server.listen_addr` | `"127.0.0.1:5555"` | `server.listen_addr`. | +| `settings.server.internal_listen_addr` | `"[::1]:5444"` | `server.internal_listen_addr`. | +| `settings.server.ssh_listen_addr` | `"[::]:2222"` | `server.ssh_listen_addr`. Must not be a port `services.openssh` listens on. | +| `settings.server.ssh_host_key_file` | `"${stateDir}/ssh_host_key"` | `server.ssh_host_key_file`. | +| `settings.acl.admission` | `"closed"` | `acl.admission`: `closed` or `open`. | +| `settings.repo.scan_path` | `"${stateDir}/repos"` | `repo.scan_path`. | +| `settings.git.object_format` | `"sha256"` | `git.object_format`: `sha1` or `sha256`. | +| `settings.secrets.sealed_key_file` | `"${stateDir}/sealed-keys"` | `secrets.sealed_key_file`. | +| `settings.secrets.master_key_env` | `"KNOT_MASTER_KEY"` | `secrets.master_key_env`. | +| `settings.atproto.plc_directory` | none | `atproto.plc_directory`. | +| `settings.contact.email` | none | `contact.email`. | +| `settings.xrpc.trusted_proxy_header` | `null` | `xrpc.trusted_proxy_header`. | +| `settings.lfs.store_path` | none | `lfs.store_path`. | +| `settings.tls.cert_path` | `null` | `tls.cert_path`. | +| `settings.tls.key_path` | `null` | `tls.key_path`. | +| `settings.tls.http3` | `true` | `tls.http3`. | +| `settings.tls.acme_enabled` | `false` | `tls.acme_enabled`. Requires a non-loopback `listen_addr`. | +| `settings.tls.acme_cache_dir` | `"${stateDir}/acme"` | `tls.acme_cache_dir`. | +| `settings.tls.acme_contact` | `null` | `tls.acme_contact`. | +| `settings.tls.acme_staging` | `false` | `tls.acme_staging`. | +| `settings.tls.mtls_enabled` | `false` | `tls.mtls_enabled`. | +| `settings.tls.mtls_client_ca_path` | `null` | `tls.mtls_client_ca_path`. | +| `settings.tls.mtls_admin_spki_pin` | `null` | `tls.mtls_admin_spki_pin`. | diff --git a/docs/src/content/docs/reference/records.md b/docs/src/content/docs/reference/records.md new file mode 100644 index 0000000000000000000000000000000000000000..b2fc0d59659a47ed1cebea07ddcb7d004f76573e --- /dev/null +++ b/docs/src/content/docs/reference/records.md @@ -0,0 +1,99 @@ +--- +title: Records +description: + Every org.tangled.* record type, where it is stored, and its record + key rule, plus the legacy sh.tangled.* records still in use. +--- + +Tangled data lives in AT Protocol records. Each record is stored either +in a **user's PDS** (things a person owns: profile, stars, follows) or +in the **repository DID's account on its knot** (things about a +repository: issues, pull requests, comments). Schemas are in +[`lexicons/org/tangled/`](https://tangled.org/tangled.org/core/tree/master/lexicons/org/tangled). + +```sh +$ curl "https://knot.example.com/xrpc/com.atproto.repo.listRecords?repo=did:plc:repoxyz&collection=org.tangled.track.ticket&limit=1" +``` + +Records in a repository account are written through the knot +(`com.atproto.repo.createRecord` and friends, or by the knot itself). +The knot checks the caller's permission and stamps the record with +`x-tngl-editor`, the DID of the account that made the change. + +Location key: + +- **User PDS**: the author's own account. +- **Repo account**: the repository DID's account, hosted by its knot. + +## Repositories + +| NSID | Stored in | Record key | Purpose | +| -------------------------------- | ---------------- | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `org.tangled.repo.manifest` | Repo account | `self` | Repository metadata (description, website, topics, spindle DID, fork or import source) and the `declaration {owner, name}` half of the ownership claim. Written by the knot. | +| `org.tangled.repo.declaration` | User PDS (owner) | Repository slug | The owner's half of the claim: `{repo: repoDid}`. A repository is listed under `owner/slug` only while manifest and declaration agree. | +| `org.tangled.graph.collaborator` | Repo account | Invitee DID | Collaboration offer. Indexed by bobbin. The repo owner will create it by sending a request to `com.atproto.repo.createRecord` with this NSID, conversely removing it with `deleteRecord`; the knot grants collaborator access from the record's presence. | +| `org.tangled.label.definition` | Repo account | any | A label in the repository's catalogue: `{name, color}`. | + +## Issues and pull requests + +| NSID | Stored in | Record key | Purpose | +| -------------------------- | ------------ | ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `org.tangled.track.ticket` | Repo account | TID (minted by the knot) | An issue, or a pull request when `targetBranch` is set. Holds title, body, state (`open`, `closed`, `completed`) and labels. Only the author or a maintainer can change state. | +| `org.tangled.review.patch` | Repo account | The record's own CID | One immutable revision of a pull request: `{ticket, base, head}` commit IDs, plus an optional `source` naming the repo DID and branch it came from. Each resubmission adds a patch. | + +## Discussion and reactions + +| NSID | Stored in | Record key | Purpose | +| ------------------------------- | ------------ | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `org.tangled.feed.comment` | Repo account | TID (minted by the knot) | A comment on a ticket, patch or string, with optional `replyTo`. Images and video attach through the markdown `blobs`. | +| `org.tangled.feed.reaction` | Repo account | TID (minted by the knot) | An emoji reaction on a record. Create and delete only. | +| `org.tangled.graph.attestation` | User PDS | TID | The author's claim that a knot-written record is really theirs: `{subject, change}` where `change` is the COB change ID, from the write's `x-tngl-change`. | + +## Social graph + +| NSID | Stored in | Record key | Purpose | +| ------------------------------- | --------- | ----------- | --------------------------------------------------------------------------------------- | +| `org.tangled.actor.profile` | User PDS | `self` | Profile: avatar, bio, links, location, pronouns, pinned repositories, `isOrganization`. | +| `org.tangled.feed.star` | User PDS | TID | Star on a repository DID or a record. | +| `org.tangled.feed.subscription` | User PDS | TID | Subscription to a repository or a thread; `ignore` mutes it. Drives notifications. | +| `org.tangled.graph.follow` | User PDS | Subject DID | Follow another account. | +| `org.tangled.graph.vouch` | User PDS | Subject DID | `vouch` or `denounce` another account, with a reason and up to 10 evidence links. | + +## Keys and strings + +| NSID | Stored in | Record key | Purpose | +| ---------------------------- | --------- | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `org.tangled.key.sshKey` | User PDS | TID | SSH public key with `usages` `auth` and/or `sign`. Written by web key settings and GitHub import. The knot authenticates pushes with keys whose `usages` include `auth` (or omit `usages`); bobbin indexes it. | +| `org.tangled.snippet.string` | User PDS | any (TID preferred) | A string (gist): description plus one text file blob up to 1 MB. | + +## Git + +| NSID | Stored in | Record key | Purpose | +| ---------------------- | ------------ | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `org.tangled.git.refs` | Repo account | `self` | Snapshot of every ref and the default branch, rewritten by the knot after each push. Knotmirror syncs from it. Spindle reads the default branch from it. | + + + +## Shared objects + +Embedded in other records, never stored alone: `org.tangled.git.oid`, +`org.tangled.embed.commit`, +`org.tangled.markup.markdown` (text up to 256 KiB plus blobs), +`org.tangled.markup.embed.image`, `org.tangled.markup.embed.video`, +`org.tangled.label.defs`, `org.tangled.repo.defs`. + +The [API](/docs/api/bobbin/) pages list the methods that read and +write these records. diff --git a/docs/src/content/docs/reference/spindle-config.md b/docs/src/content/docs/reference/spindle-config.md new file mode 100644 index 0000000000000000000000000000000000000000..4c7b7e5b07e111156c02af15c6173847b9c7bcb6 --- /dev/null +++ b/docs/src/content/docs/reference/spindle-config.md @@ -0,0 +1,226 @@ +--- +title: Spindle configuration +description: Every environment variable read by spindle and cilogs, with its default. +--- + +Spindle is configured by environment variables only. Durations use Go +syntax (`5m`, `720h`) and lists are comma-separated. + +## Role + +| Variable | Default | Description | +| -------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------- | +| `SPINDLE_ROLE` | `standalone` | `standalone` ingests events and runs jobs, `mill` ingests events and places jobs on executors, `executor` dials a mill and runs jobs. | +| `SPINDLE_LOG_FORMAT` | `text` | Format of logs on stderr: `text` or `json`. | + +## Server + +| Variable | Default | Description | +| ------------------------------------------ | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `SPINDLE_SERVER_HOSTNAME` | required | Public hostname. The spindle DID is `did:web:`. | +| `SPINDLE_SERVER_OWNER` | required in standalone and mill | DID of the spindle owner, always an admin. | +| `SPINDLE_SERVER_ADMINS` | unset | DIDs of additional admins. Each entry must be a valid DID. | +| `SPINDLE_SERVER_ADMISSION` | `closed` | `closed` serves admins and the allow-list, `open` serves anyone who is not banned. | +| `SPINDLE_SERVER_KEY_PATH` | required in standalone and mill | PEM file with the EC P-256 private key behind `/.well-known/did.json`. | +| `SPINDLE_SERVER_LISTEN_ADDR` | `0.0.0.0:6555` | HTTP listen address. | +| `SPINDLE_SERVER_METRICS_LISTEN_ADDR` | `127.0.0.1:9091` | Prometheus metrics listen address. | +| `SPINDLE_SERVER_DB_PATH` | `spindle.db` | SQLite database path. Also the default `--db` of the admin subcommands. | +| `SPINDLE_SERVER_REPO_DIR` | `repos` | Directory of per-repo checkouts. | +| `SPINDLE_SERVER_JETSTREAM_ENDPOINT` | `wss://jetstream1.us-west.bsky.network/subscribe` | Jetstream endpoint for live events. | +| `SPINDLE_SERVER_PLC_URL` | `https://plc.directory` | PLC directory for DID resolution. | +| `SPINDLE_SERVER_DEV` | `false` | Development mode: allows plain HTTP and private-network addresses for knots, webhooks, clones and backfill. | +| `SPINDLE_SERVER_DEV_EXTRA_HOSTS` | unset | Hostnames mapped to `host-gateway` in nixery workflow containers. | +| `SPINDLE_SERVER_LOG_DIR` | `/var/log/spindle` | Directory of workflow log files. In standalone, the disk artifact store when `SPINDLE_ARTIFACT_STORES_DISK_DIR` is unset. | +| `SPINDLE_SERVER_LOG_MAX_BYTES` | `33554432` | Maximum size of one workflow's stored log, in bytes. Output past it is dropped. `0` means no cap. | +| `SPINDLE_SERVER_MAX_JOB_COUNT` | `2` | Pipelines that run at once. | +| `SPINDLE_SERVER_REPO_JOB_WORKERS` | `256` | Repos refreshed at once, shared between knots. Must be greater than zero. | +| `SPINDLE_SERVER_NOTIFY_SLOTS` | `64` | Knot notify events processed at once, shared between knots. Must be greater than zero. | +| `SPINDLE_SERVER_PRIORITY_KNOTS` | unset | Knot hosts or `did:web` DIDs that get reserved repo job workers and notify slots. | +| `SPINDLE_SERVER_PRIORITY_REPO_JOB_WORKERS` | `0` | Repo job workers reserved for priority knots. `0` means a quarter of `SPINDLE_SERVER_REPO_JOB_WORKERS`. | +| `SPINDLE_SERVER_PRIORITY_NOTIFY_SLOTS` | `0` | Notify slots reserved for priority knots. `0` means a quarter of `SPINDLE_SERVER_NOTIFY_SLOTS`. | +| `SPINDLE_SERVER_DOCKER_SOCKET` | unset | Docker socket mounted into nixery workflow containers and set as their `DOCKER_HOST`. | +| `SPINDLE_SERVER_CORS_ORIGINS` | `*` | Origins allowed to call XRPC from a browser. Set it empty to disable CORS. | +| `SPINDLE_SERVER_LOGS_ADDR` | unset | `host:port` of a cilogs SSH server, returned to a knot whose push scheduled a pipeline. | + +## Contact and links + +Served by `org.tangled.spindle.describeSpindle`. + +| Variable | Default | Description | +| -------------------------------- | ------------------------------- | --------------------------------- | +| `SPINDLE_CONTACT_EMAIL` | required in standalone and mill | Operator contact email. | +| `SPINDLE_LINKS_PRIVACY_POLICY` | unset | Privacy policy URL. Absolute. | +| `SPINDLE_LINKS_TERMS_OF_SERVICE` | unset | Terms of service URL. Absolute. | + +## Backfill + +| Variable | Default | Description | +| ---------------------------- | --------- | ---------------------------------------------------------------------------------- | +| `SPINDLE_SERVER_HYDRANT_DIR` | `hydrant` | Directory of the embedded hydrant database, its API socket and the spindle cursor. | + +## Secrets + +| Variable | Default | Description | +| ------------------------------------------- | ----------------------- | --------------------------------------------------------------------------------- | +| `SPINDLE_SERVER_SECRETS_PROVIDER` | `sqlite` | `sqlite` stores secrets in the spindle database, `openbao` uses an OpenBao Proxy. | +| `SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR` | `http://127.0.0.1:8200` | OpenBao Proxy address. | +| `SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT` | `spindle` | KV v2 mount. | + +## Nixery engine + +| Variable | Default | Description | +| --------------------------------------------------- | ------------------- | -------------------------------------------- | +| `SPINDLE_NIXERY_PIPELINES_NIXERY` | `nixery.tangled.sh` | Nixery host that builds workflow images. | +| `SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT` | `5m` | Default workflow timeout. | +| `SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB` | `6144` | Memory limit per workflow container, in MiB. | +| `SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS` | `8` | Workflow containers running at once. | + +## MicroVM engine + +| Variable | Default | Description | +| --------------------------------------------------- | --------------- | ----------------------------------------------------------------------------- | +| `SPINDLE_MICROVM_PIPELINES_IMAGE_DIR` | unset | Directory of microVM images. Required by microVM workflows. | +| `SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR` | system temp dir | Directory of per-workflow temporary disks. | +| `SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE` | `nixos-x86_64` | Image used when a workflow sets no `image`. | +| `SPINDLE_MICROVM_PIPELINES_AGENT_PORT` | `10240` | vsock port of the guest agent. | +| `SPINDLE_MICROVM_PIPELINES_ENABLE_KVM` | `true` | Use KVM acceleration. | +| `SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT` | `5m` | Default workflow timeout. | +| `SPINDLE_MICROVM_PIPELINES_AGING_THRESHOLD` | `30s` | Queue time after which smaller requests can no longer go ahead of a workflow. | +| `SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB` | `0` | Memory across all running microVMs, in MiB. `0` means unlimited. | +| `SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS` | `0` | vCPUs across all running microVMs. `0` means unlimited. | +| `SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB` | `0` | Disk across all running microVMs, in MiB. `0` means unlimited. | +| `SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB` | `0` | Memory one workflow may request, in MiB. `0` means unlimited. | +| `SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS` | `0` | vCPUs one workflow may request. `0` means unlimited. | +| `SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB` | `0` | Disk one workflow may request, in MiB. `0` means unlimited. | + +### Cgroups + +| Variable | Default | Description | +| ------------------------------------------------------------ | ------- | ------------------------------------------------------------------------------------------------------- | +| `SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS` | `false` | Run each workflow's processes in a per-workflow cgroup. | +| `SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT` | `self` | Parent cgroup. `self` is the spindle process's own cgroup. | +| `SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX` | `4096` | Process limit per workflow cgroup. | +| `SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB` | unset | Swap limit per workflow cgroup, in MiB. | +| `SPINDLE_MICROVM_PIPELINES_CGROUP_CPU_MAX_PERCENT` | `0` | CPU quota as a percentage of one core. `0` caps each VM at its vCPU count, negative disables the limit. | +| `SPINDLE_MICROVM_PIPELINES_CGROUP_IO_WEIGHT` | `0` | IO weight, 1 to 10000. `0` leaves IO unlimited. | +| `SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB` | `512` | `memory.min` of the spindle supervisor cgroup, in MiB. | + +### Debug SSH + +| Variable | Default | Description | +| --------------------------------------------------- | -------------- | ----------------------------------------------------------- | +| `SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_ENABLED` | `false` | Allow SSH into the VM of a failed workflow. | +| `SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_LISTEN_ADDR` | `0.0.0.0:2222` | Debug SSH listen address. | +| `SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST` | unset | Host in the printed SSH command. | +| `SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_JUMP_HOST` | unset | Jump host in the printed SSH command. | +| `SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST_KEY_PATH` | unset | Host private key. Unset generates one next to the database. | +| `SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_GRACE_PERIOD` | `5m` | Time a failed workflow's VM stays up after failure. | + +## Nix cache + +| Variable | Default | Description | +| ------------------------------------------ | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `SPINDLE_NIX_CACHE_READ_URLS` | unset | Binary caches microVM guests read from. | +| `SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS` | unset | Trusted public keys for those caches. | +| `SPINDLE_NIX_CACHE_UPLOAD_URL` | unset | Shared binary cache that guest builds are uploaded to. Safe only with trusted owners; see [Self-hosting](/docs/spindles/self-hosting/#caches). | +| `SPINDLE_NIX_CACHE_REQUIRE_SIGNED_UPLOADS` | `false` | Require uploaded narinfos to be signed by a trusted key. The key must reach only repositories you trust. | +| `SPINDLE_NIX_CACHE_ATTIC_URL` | unset | atticd URL. Every repository gets its own cache there. Cannot be set with the upload URL. Executors set it too. Reach it over loopback or TLS. | +| `SPINDLE_NIX_CACHE_ATTIC_KEY_PATH` | unset | PEM file with the RSA private key that signs attic tokens. Set with the attic URL on a standalone spindle or mill; executors ignore it. | +| `SPINDLE_NIX_CACHE_RUN_UPLOAD_MIB` | `32768` | Raw NAR MiB one run may upload to a repository cache without a limit; with one, twice the limit. Zero is no cap. | +| `SPINDLE_NIX_CACHE_MAX_MIB` | `0` | MiB all repository caches in attic may hold together, by uncompressed size. Past it, least recently used paths are evicted. Zero is no cap. | +| `SPINDLE_NIX_CACHE_RETENTION` | `336h` | Evict repository cache paths unused for this long. Zero turns it off. | +| `SPINDLE_NIX_CACHE_MAX_STAGED_MIB` | `5120` | Upload staging limit, in MiB. Must be greater than zero. | +| `SPINDLE_NIX_CACHE_MAX_STAGED_OBJECTS` | `1024` | Upload staging object limit. Must be greater than zero. | + +## CI cache + +| Variable | Default | Description | +| ------------------------------------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SPINDLE_CACHE_BACKEND` | unset | `disk` or `s3`. Unset disables the cache. | +| `SPINDLE_CACHE_DISK_DIR` | `cache` next to the database | Directory of the `disk` backend. | +| `SPINDLE_CACHE_S3_BUCKET` | unset | Unversioned bucket of the `s3` backend. | +| `SPINDLE_CACHE_S3_PREFIX` | `spindle/cache` | Key prefix in the bucket. | +| `SPINDLE_CACHE_STORE_ID` | unset | Identity of a shared cache store. A mill places cached workflows only on executors with the same value; unset on a mill disables the cache. | +| `SPINDLE_CACHE_RETENTION` | `720h` | Time since last use after which an entry is deleted. `0` keeps entries forever. | +| `SPINDLE_CACHE_PRUNE_INTERVAL` | `1h` | Interval between prunes of expired entries, of entries past `SPINDLE_CACHE_MAX_BYTES`, of repository nix caches, and of users and repos over their cache storage quota. | +| `SPINDLE_CACHE_MAX_ENTRIES_PER_OWNER` | `1000` | Entries across all repos of one owner. `0` means unlimited. | +| `SPINDLE_CACHE_MAX_BYTES` | `0` | Bytes stored across the whole cache. Past it, the least recently used entries not being restored are evicted. `0` means unlimited. | + +## Artifact stores + +| Variable | Default | Description | +| ----------------------------------- | ----------- | -------------------------------------------------------- | +| `SPINDLE_ARTIFACT_STORES_DISK_DIR` | unset | Directory of the disk store. | +| `SPINDLE_ARTIFACT_STORES_S3_BUCKET` | unset | Bucket of the S3 store. | +| `SPINDLE_ARTIFACT_STORES_S3_REGION` | `us-east-1` | Region of the S3 store. | +| `SPINDLE_S3_LOG_BUCKET` | unset | Deprecated alias of `SPINDLE_ARTIFACT_STORES_S3_BUCKET`. | + +## Quotas + +| Variable | Default | Description | +| ----------------------------------------------- | ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SPINDLE_QUOTA_{USER,REPO}_WORKFLOWS` | `0` | Concurrent workflows. `0` means unlimited, negative is rejected. | +| `SPINDLE_QUOTA_{USER,REPO}_VCPUS` | `0` | Concurrent vCPUs. `0` means unlimited, negative is rejected. | +| `SPINDLE_QUOTA_{USER,REPO}_MEMORY_MIB` | `0` | Concurrent memory, in MiB. A microVM workflow counts its image's memory. `0` means unlimited, negative is rejected. | +| `SPINDLE_QUOTA_{USER,REPO}_DISK_MIB` | `0` | Concurrent disk, in MiB. `0` means unlimited, negative is rejected. | +| `SPINDLE_QUOTA_{USER,REPO}_CACHE_STORAGE_MIB` | `5120` for users, `0` for repos | Cache storage, in MiB, for repository nix cache paths and `cache:` entries together. Up to twice the limit is accepted, and the prune evicts the least recently used of either back under it. `0` means unlimited, negative is rejected. | +| `SPINDLE_QUOTA_{USER,REPO}_WEBHOOKS` | `0` | Webhooks. `0` means unlimited, negative is rejected. | +| `SPINDLE_QUOTA_{USER,REPO}_CI_MINUTES_MONTHLY` | `0` | CI minutes per calendar month. `0` means unlimited, negative is rejected. | +| `SPINDLE_QUOTA_{USER,REPO}_WORKFLOW_VCPUS` | `0` | vCPUs a single workflow may request. A bigger one is refused rather than queued. `0` means unlimited, negative is rejected. | +| `SPINDLE_QUOTA_{USER,REPO}_WORKFLOW_MEMORY_MIB` | `0` | Memory a single workflow may request, in MiB. A bigger one is refused rather than queued. `0` means unlimited, negative is rejected. | +| `SPINDLE_QUOTA_{USER,REPO}_WORKFLOW_DISK_MIB` | `0` | Disk a single workflow may request, in MiB. A bigger one is refused rather than queued. `0` means unlimited, negative is rejected. | + +## OIDC + +| Variable | Default | Description | +| ----------------------- | -------------- | ------------------------------------------------------------------------------ | +| `SPINDLE_OIDC_ISSUER` | unset | `iss` claim of workflow ID tokens. Must be set with `SPINDLE_OIDC_KEY_PATH`. | +| `SPINDLE_OIDC_KEY_PATH` | unset | PEM file with an EC P-256 private key. Must be set with `SPINDLE_OIDC_ISSUER`. | +| `SPINDLE_OIDC_KEY_ID` | key thumbprint | `kid` header. Defaults to the RFC 7638 thumbprint of the key. | + +## Mill and executors + +| Variable | Default | Role | Description | +| -------------------------------------- | ------- | -------------- | --------------------------------------------------------------------------------------------------------------- | +| `SPINDLE_MILL_URL` | unset | executor | Mill websocket endpoint. Required in executor, rejected in other roles. | +| `SPINDLE_MILL_SHARED_SECRET` | unset | executor | Executor token for the mill. Required in executor. | +| `SPINDLE_MILL_SEATS` | `4` | executor | Seats advertised to the mill. | +| `SPINDLE_MILL_LABELS` | unset | executor | Capability labels advertised to the mill. | +| `SPINDLE_MILL_ARTIFACT_STORE` | unset | mill, executor | Name of the artifact store shared by the mill and its executors. Optional when exactly one store is configured. | +| `SPINDLE_MILL_MAX_PENDING` | `100` | mill | Pending job queue limit. | +| `SPINDLE_MILL_RECONNECT_GRACE` | `45s` | mill | Reconnect window of an executor before its leases fail. | +| `SPINDLE_MILL_CANCEL_ACK_TIMEOUT` | `10s` | mill | Wait for an executor to acknowledge a cancel. | +| `SPINDLE_MILL_CANCEL_TEARDOWN_TIMEOUT` | `10m` | mill | Wait for a cancelled job to tear down. | +| `SPINDLE_MILL_DRAIN_TIMEOUT` | `20m` | all | Drain time on shutdown. Must be greater than zero. | +| `SPINDLE_MILL_JUMP_LISTEN_ADDR` | unset | mill | SSH jump listen address for debug sessions on executors. Requires `SPINDLE_MILL_JUMP_HOST_KEY_PATH`. | +| `SPINDLE_MILL_JUMP_HOST_KEY_PATH` | unset | mill | Host key of the jump listener. | +| `SPINDLE_MILL_DEBUG_EXECUTOR_PORT` | `2223` | mill | Executor debug SSH port that the jump host forwards to. | +| `SPINDLE_MILL_MAX_JUMP_CONNECTIONS` | `128` | mill | Concurrent jump connections. Must be greater than zero when the jump listener is set. | + +## Scheduled workflows + +| Variable | Default | Description | +| ----------------------------------- | ------- | ----------------------------------------------------------------------- | +| `SPINDLE_SCHEDULE_MINIMUM_INTERVAL` | `5m` | Shortest schedule interval. Must be a positive whole number of minutes. | +| `SPINDLE_SCHEDULE_CONCURRENCY` | `4` | Scheduled runs started at once. Must be greater than zero. | + +## cilogs + +| Variable | Default | Description | +| ---------------------- | ----------------------- | --------------------------------------------------------------- | +| `CILOGS_LISTEN_ADDR` | `0.0.0.0:3333` | SSH listen address. | +| `CILOGS_HOST_KEY_PATH` | unset | Host key path, created when missing. Unset uses `./id_ed25519`. | +| `CILOGS_PLC_URL` | `https://plc.directory` | PLC directory for DID resolution. | +| `CILOGS_DEV` | unset | `true` allows plain HTTP and private-network addresses. | + +## Telemetry + +| Variable | Default | Description | +| ------------------------------ | --------- | -------------------------------------------------------- | +| `SPINDLE_TRACING_ENDPOINT` | unset | OTLP/HTTP endpoint for traces. Unset disables tracing. | +| `SPINDLE_TRACING_SERVICE_NAME` | `spindle` | Service name on traces. | +| `SPINDLE_TRACING_SAMPLE_RATIO` | `1` | Sampling ratio, 0 to 1. | +| `SPINDLE_TRACING_INSECURE` | `false` | Use plain HTTP to the collector. | +| `SPINDLE_LOGGING_ENDPOINT` | unset | OTLP/HTTP endpoint for logs. Unset keeps logs on stderr. | +| `SPINDLE_LOGGING_SERVICE_NAME` | `spindle` | Service name on logs. | +| `SPINDLE_LOGGING_INSECURE` | `false` | Use plain HTTP to the collector. | diff --git a/docs/src/content/docs/self-hosting/frontend.md b/docs/src/content/docs/self-hosting/frontend.md new file mode 100644 index 0000000000000000000000000000000000000000..292a00abc62d293ec5f5241903e04eb0441be8ed --- /dev/null +++ b/docs/src/content/docs/self-hosting/frontend.md @@ -0,0 +1,6 @@ +--- +title: Run a frontend +description: Run the Tangled web app on your own domain. +--- + +Coming soon. diff --git a/docs/src/content/docs/self-hosting/index.mdx b/docs/src/content/docs/self-hosting/index.mdx new file mode 100644 index 0000000000000000000000000000000000000000..b4575079043a9203f86c7e8f89138b6b2270296e --- /dev/null +++ b/docs/src/content/docs/self-hosting/index.mdx @@ -0,0 +1,12 @@ +--- +title: Choosing what to run +description: The Tangled services that you can self-host. +--- + +You can self-host some or all of these services: + +- **[Knot](/docs/self-hosting/knot/)**: Stores your git repositories. This is typically what you want. +- **[Spindle](/docs/self-hosting/spindle/)**: Runs CI pipelines for your repositories. +- **[Client](/docs/self-hosting/frontend/)**: Shows repositories, issues, and pull requests in a web browser. +- **[Index](/docs/self-hosting/indexer/)**: Collects records from PDSes and knots and serves them to the client. +- **PDS**: Keeps your account, you identity and your social data. diff --git a/docs/src/content/docs/self-hosting/indexer.md b/docs/src/content/docs/self-hosting/indexer.md new file mode 100644 index 0000000000000000000000000000000000000000..88ba66354417cb321b21bc2b49ecee9f65267bd6 --- /dev/null +++ b/docs/src/content/docs/self-hosting/indexer.md @@ -0,0 +1,38 @@ +--- +title: Run an index +description: Run your own index of the Tangled network. +--- + +The index is `bobbin`, the service behind the [Tangled API](/docs/api/bobbin/). +It keeps everything in memory and rebuilds it from a +[hydrant](https://tangled.org/ptr.pet/hydrant) stream on every start, so +it needs no database. It also needs a slingshot for records and identities. + +Build it: + +```sh +nix build .#bobbin +``` + +Configure it with environment variables: + +| Variable | Value | +| --- | --- | +| `BOBBIN_SERVICE_DID` | Required. The index's own DID, for example `did:web:index.example.com`. | +| `BOBBIN_HYDRANT_URL` | Your hydrant, for example `wss://hydrant.example.com`. | +| `BOBBIN_SLINGSHOT_URL` | Your slingshot. Hydrant serves this too, so it is often the same host. | +| `BOBBIN_MIRROR_URL` | Your knotmirror. Without it, repository pages have no code. | +| `BOBBIN_BIND` | Address to listen on. Defaults to `127.0.0.1:8090`. | +| `BOBBIN_CORS_ORIGINS` | Origins of the web apps that call the index, for example `https://tangled.example.com`. | +| `BOBBIN_TRUSTED_PROXIES` | IPs or CIDRs of your reverse proxy, so knots rate-limit per visitor. | + +Every other setting is optional. `bobbin config-template` prints all of them +with their defaults, and you can put them in a TOML file passed with +`--config` instead. Check the configuration, then start it: + +```sh +./result/bin/bobbin validate +./result/bin/bobbin +``` + +Put a reverse proxy in front of it for TLS. diff --git a/docs/src/content/docs/self-hosting/knot.md b/docs/src/content/docs/self-hosting/knot.md new file mode 100644 index 0000000000000000000000000000000000000000..96868c114cfd52682c38ff2e8f0264e37daefd3a --- /dev/null +++ b/docs/src/content/docs/self-hosting/knot.md @@ -0,0 +1,150 @@ +--- +title: Run a knot +description: Run a knot, the server that stores your git repositories. +--- + +A knot stores your git repositories. You can run it on NixOS or in a +container. + +If you already run the Go knot, follow +[Golang to Rust knot migration guide](/docs/upgrading/knot-2/) instead. + +## Before you start + +You need: + +- A Linux machine that stays online. +- A DNS name for the knot, for example `knot.example.com`. Choose a name + that you will keep, because the knot uses it as its identity. +- Your DID. If you do not know it, look up your handle at + [pds.ls](https://pds.ls). +- A reverse proxy, for example Caddy, to give the knot TLS. + +Make a master key and keep it in an env file: + +```sh +printf 'KNOT_MASTER_KEY=%s\n' "$(openssl rand -base64 32)" > knot.env +chmod 600 knot.env +``` + +:::caution + +Keep a backup copy of the master key on a different machine. If you +lose it, the knot cannot control your repositories. + +::: + +## Run on NixOS + +Put `knot.env` at `/etc/knot/knot.env`. Then add the Tangled flake and +enable the `knot-rs` module: + +```nix +{ + inputs.tangled.url = "git+https://tangled.org/did:plc:j5hmlfdrwkvtxm7cjmu7j2is"; + outputs = { nixpkgs, tangled, ... }: { + nixosConfigurations.knot = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ tangled.nixosModules.knot-rs ./knot.nix ]; + }; + }; +} +``` + +```nix +# knot.nix +{ + services.tangled.knot-rs = { + enable = true; + environmentFile = "/etc/knot/knot.env"; + settings = { + server = { + hostname = "knot.example.com"; + admins = [ "did:plc:yourdid" ]; + }; + atproto.plc_directory = "https://plc.directory"; + lfs.store_path = "/var/lib/knot/lfs"; + contact.email = "your@contact.email"; + }; + }; +} +``` + +The knot listens on `127.0.0.1:5555`. Point your reverse proxy at it. + +## Run with Docker + +Make a `config.toml`: + +```toml +[server] +hostname = "knot.example.com" +admins = ["did:plc:yourdid"] +listen_addr = "[::]:5555" +ssh_host_key_file = "/data/ssh/host_key" + +[repo] +scan_path = "/data/repos" + +[secrets] +sealed_key_file = "/data/secrets/sealed-keys" +master_key_env = "KNOT_MASTER_KEY" + +[atproto] +plc_directory = "https://plc.directory" + +[lfs] +store_path = "/data/lfs" + +[contact] +email = "you@example.com" +``` + +Make a `compose.yaml`: + +```yaml +services: + knot: + image: atcr.io/tangled.org/knot:2 + restart: unless-stopped + env_file: ./knot.env + command: ["--config", "/etc/knot/config.toml"] + environment: + KNOT_SCAN_PATH: /data/repos + KNOT_SEALED_KEY_FILE: /data/secrets/sealed-keys + KNOT_SSH_HOST_KEY_FILE: /data/ssh/host_key + KNOT_MASTER_KEY_ENV: KNOT_MASTER_KEY + ports: + - "2222:2222" + - "127.0.0.1:5555:5555" + volumes: + - ./config.toml:/etc/knot/config.toml:ro + - ./repos:/data/repos + - ./secrets:/data/secrets + - ./ssh:/data/ssh + - ./lfs:/data/lfs +``` + +Then start it: + +```sh +mkdir -p repos secrets ssh lfs +sudo chown -R 1000:1000 repos secrets ssh lfs +docker compose up -d +``` + +Point your reverse proxy at `127.0.0.1:5555`. + +## Check that it works + +```sh +curl https://knot.example.com/xrpc/_health +``` + +Then, in the Tangled web app, go to Settings → Knots and add your knot. +You can now make a repository on it and push to it. + +Users clone over SSH on port 2222. To use port 22, see +["Ports, Taylor's version"](/docs/knots/self-hosting/#ports-taylors-version-ha-ha-22-joke). +[Self-hosting](/docs/knots/self-hosting/) explains a knot in detail. [Knot Configuration](/docs/reference/knot-config/) lists +all settings. diff --git a/docs/src/content/docs/self-hosting/spindle.md b/docs/src/content/docs/self-hosting/spindle.md new file mode 100644 index 0000000000000000000000000000000000000000..dd4031513320b16751b0a3442ec2ab27bb3dc566 --- /dev/null +++ b/docs/src/content/docs/self-hosting/spindle.md @@ -0,0 +1,86 @@ +--- +title: Run a spindle +description: Run a spindle, the server that runs CI for your repositories. +--- + +A spindle runs CI pipelines for your repositories. You do not need your +own spindle to use CI. Run one if you want CI on your own machines, or +secrets that stay on them. + +## Before you start + +You need: + +- A Linux machine with KVM that stays online. +- A DNS name for the spindle, for example `spindle.example.com`. Choose + a name that you will keep, because the spindle uses it as its + identity. +- Your DID. If you do not know it, look up your handle at + [pds.ls](https://pds.ls). +- A reverse proxy, for example Caddy, to give the spindle TLS. + +Make a signing key: + +```sh +openssl ecparam -name prime256v1 -genkey -noout -out /var/lib/secrets/spindle-key.pem +chmod 600 /var/lib/secrets/spindle-key.pem +``` + +:::caution + +Keep a backup copy of the signing key. If you lose it, other services +stop trusting the spindle for some time. + +::: + +## Run on NixOS + +Add the Tangled flake and enable the `spindle` module: + +```nix +{ + imports = [ tangled.nixosModules.spindle ]; + + services.tangled.spindle = { + enable = true; + server = { + hostname = "spindle.example.com"; + owner = "did:plc:yourdid"; + }; + keyPath = "/var/lib/secrets/spindle-key.pem"; + contact.email = "admin@spindle.example.com"; + artifactStores.s3.bucket = ""; + }; + + systemd.tmpfiles.rules = [ + "L+ /var/lib/spindle/images/nixos - - - - ${tangled.packages.x86_64-linux.spindle-nixos-image}" + ]; +} +``` + +The spindle listens on port 6555. Point your reverse proxy at it. The +proxy must also pass websocket upgrades. + +## Let people use it + +At first, the spindle runs CI only for you. To let another person use +it, run this on the spindle machine: + +```sh +spindle allowlist add friend.example.com +``` + +## Check that it works + +```sh +curl https://spindle.example.com/_health +``` + +Then, in the Tangled web app, open a repository's **Settings → +Pipelines** and select your spindle. Push a +[workflow](/docs/spindles/writing-workflows/) to start a run. + +[Self-hosting](/docs/spindles/self-hosting/) explains how a spindle runs +and how to configure it. +[Spindle Configuration](/docs/reference/spindle-config/) lists all +settings. diff --git a/docs/src/content/docs/sites.md b/docs/src/content/docs/sites.md new file mode 100644 index 0000000000000000000000000000000000000000..a7273a78a98fd41703859b70483a561d89262e1e --- /dev/null +++ b/docs/src/content/docs/sites.md @@ -0,0 +1,47 @@ +--- +title: Sites +description: + Serve a static website from a branch and directory of a Tangled + repository. +--- + +Sites serve static files from a repository at a Tangled provided domain. +All users can claim a domain from +[**Settings**](https://tangled.org/settings). + +## Domains + +If your account is on the PDS we run (`*.tngl.sh`), your sites domain is +`.tngl.sh`. Otherwise, claim a `.tngl.io` domain +under **Settings → Sites**; `sites` is reserved. You can hold one domain at a time. Releasing +it disables every site on it, deletes their files, and puts the name in +a 30-day cooldown before anyone else can claim it. + +Custom domains are not supported at the moment. + +## Configuring a site + +In the repository, open **Settings → Sites** and set: + +- Branch: the branch to publish from. +- Deploy directory: the directory served as the site root, e.g. `/`, `/docs`, `/public`. It must contain `index.html`. +- Site type: **Index** (served at the domain root) or **Sub-path** (served at `//`). + +A domain has at most one root site. If another repository +already holds it, only the sub-path option is available. + +A site is redeployed on every push to the configured branch and whenever +its settings change. + +## Serving rules + +- A request for a directory serves its `index.html`, so `/about` serves + `/about/index.html`. +- Directory URLs without a trailing slash are redirected to add one. +- Repositories other than the root site are served under `//`. + +## Disabling a site + +Select **Disable** in the repository's **Settings → Sites**. +The configuration is removed, the site stops being served, +and its files are deleted. diff --git a/docs/src/content/docs/spindles/architecture.mdx b/docs/src/content/docs/spindles/architecture.mdx new file mode 100644 index 0000000000000000000000000000000000000000..f4d425a25f03e7e3e89902fbba6e1901ed8c31e6 --- /dev/null +++ b/docs/src/content/docs/spindles/architecture.mdx @@ -0,0 +1,178 @@ +--- +title: Spindle +description: + Spindle roles, engines, how it learns about repos and pushes, and + where it keeps state. +--- + +import Diagram from "../../../components/Diagram.astro"; +import spindle from "../../../assets/diagrams/spindle.svg?raw"; + +A spindle is Tangled's CI runner: a Go binary, `spindle`, that watches +repos, compiles `.tangled/workflows/*.yml` when something happens, and +runs each workflow in a microVM or container. This page covers how it +works inside. To write workflows, read +[Writing workflows](/docs/spindles/writing-workflows/); to run one, read +[Run a spindle](/docs/self-hosting/spindle/). + + + +## Roles + +The same binary runs in one of three roles, set with `SPINDLE_ROLE`: + +| Role | Ingests events | Runs workflows | Notes | +| ---------------------- | -------------- | -------------- | --------------------------------------------------------------- | +| `standalone` (default) | yes | yes | One host does everything | +| `mill` | yes | no | Places jobs on executors; serves XRPC, logs and `/mill` | +| `executor` | no | yes | Dials the mill at `ws(s):///mill` with a registered token | + +In a mill fleet, the mill runs stand-in engines that mirror the real +ones, so a remote run looks the same as a local one. Placement matches a +workflow's `runs_on` labels against executor `SPINDLE_MILL_LABELS`, then +takes bids for free seats (`SPINDLE_MILL_SEATS`). The winning executor +validates the image and architecture. Secrets cross the wire once, only +to the winner. The mill and its executors must share one artifact store +(`SPINDLE_MILL_ARTIFACT_STORE`) and, if the CI cache is on, one cache +store with a matching `SPINDLE_CACHE_STORE_ID`. + +Executor tokens are managed on the mill: + +```sh +spindle mill executor token generate --ttl 24h +spindle mill executor add --token-file ./runner-1.token --label linux --label x86_64 runner-1 +spindle mill executor list +spindle mill executor revoke runner-1 +``` + +## Engines + +| Engine | Unit of isolation | Notes | +| --------- | ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| `microvm` | One QEMU VM per workflow | Images: `nixos`, `alpine`, `almalinux10`, or your own. NixOS images also accept `services`, `virtualisation`, `substituters` and flake `dependencies` | +| `nixery` | One Docker container per step | Image built from Nix packages by [Nixery](https://nixery.dev); state shared through the workspace | +| `dummy` | none | For tests | + +Each microVM guest runs **shuttle**, a Rust agent that talks to the host +over vsock (port `SPINDLE_MICROVM_PIPELINES_AGENT_PORT`, default 10240). +Shuttle executes steps and PTYs, activates the NixOS configuration, and +proxies DNS and Nix cache traffic. The VM runs in its own namespaces, +with outbound-only networking through slirp4netns, private address +ranges blackholed and a filtering DNS proxy. Guests never hold cache +credentials. + +The microVM scheduler keeps a budget of memory, vCPUs and disk +(`SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_*` and `MAX_WORKFLOW_*`) with +aging and per-user fairness. Without `ENABLE_CGROUPS`, these limits are +bookkeeping only; with it, cgroup v2 enforces them. The +[engine README](https://tangled.org/tangled.org/core/blob/master/spindle/engines/microvm/README.md) +goes deeper. + +## How a spindle learns about repos + +- **Records** arrive on a Jetstream-compatible stream + (`SPINDLE_SERVER_JETSTREAM_ENDPOINT`; on a Tangled network, hydrant's + `/subscribe`). Spindle reads `org.tangled.repo.declaration`, + `org.tangled.repo.manifest`, `org.tangled.graph.collaborator`, + `org.tangled.track.ticket` and `org.tangled.review.patch`. +- **Backfill** comes from an embedded hydrant (`SPINDLE_SERVER_HYDRANT_DIR`). + It only runs while a round is going: spindle starts it for new owners + and for owners whose last round failed, and it fetches their + `org.tangled.repo.declaration`, `repo.manifest` and + `graph.collaborator` records. +- **Pushes** come from the knot. After a push to a repo whose manifest + names this spindle, the knot calls the spindle's + `org.tangled.event.notifyEvent` with its own service auth, and retries + from an outbox if the spindle is busy or down. + +A repo is accepted when its `org.tangled.repo.manifest` record's +`spindle` field equals this spindle's `did:web` (derived from +`SPINDLE_SERVER_HOSTNAME`), its owner's `org.tangled.repo.declaration` +names it back, its owner is admitted (an admin, on the allow-list, or +anyone when `SPINDLE_SERVER_ADMISSION=open`), and neither it nor its +owner is banned. Spindle then +sparse-clones `.tangled/workflows` from the knot over HTTPS. Changing +the field to another spindle, or the manifest going missing, stops +serving the repo but keeps its secrets and history. Only a repo its +knot reports as deleted is wiped, secrets included. + +Triggers are push, pull requests, manual runs +(`org.tangled.ci.triggerPipeline`) and cron schedules. A pull request +whose author has no push access waits for a collaborator's approval. + +## Storage + +| What | Where | +| ---------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | +| Repos, members, collaborators, pipelines, jobs, schedules, feed cursors, quotas, bans, webhooks, executor tokens | SQLite at `SPINDLE_SERVER_DB_PATH` | +| Workflow sparse clones | `SPINDLE_SERVER_REPO_DIR` | +| Logs and artifacts | Disk (`SPINDLE_ARTIFACT_STORES_DISK_DIR`) or S3 (`SPINDLE_ARTIFACT_STORES_S3_*`) | +| CI cache archives | Disk or S3 (`SPINDLE_CACHE_*`); off when unset | + +Pipelines are not PDS records. Spindle serves them as +`org.tangled.ci.defs#pipeline` objects over `org.tangled.ci.queryPipelines`, +`getPipeline` and the `subscribePipelineLogs` websocket. Pipeline IDs +are local to a spindle. + +## Secrets + +Repo secrets are added through `org.tangled.secret.addSecret` and stored +by a provider chosen with `SPINDLE_SERVER_SECRETS_PROVIDER`: `sqlite` +(default, inside the spindle DB) or `openbao` (KV v2 through an OpenBao +Proxy). Secrets are injected as env vars and masked in logs. + +**Trust** gates secrets, cache and OIDC. A pipeline is trusted only when +its checkout comes from the target repo and, for a pull request, from a +branch of it that needed no approval. Manual runs of a fork, fork and +patch pull requests, and approved runs get none of them. + +## Caches and the Nix substituter proxy + +There are two separate caches: + +- **CI cache** (`cache:` in a workflow): archives of paths keyed by + lockfile hashes, stored in disk or S3, with a per-owner entry limit, a + total limit and a pruner (`SPINDLE_CACHE_RETENTION`, default 720h). +- **Nix cache:** a read proxy fans out to `SPINDLE_NIX_CACHE_READ_URLS` + plus the workflow's `substituters`, and built paths are uploaded to + `SPINDLE_NIX_CACHE_UPLOAD_URL`. Guests reach both through host proxies + over vsock. Untrusted runs get no upload proxy. The upload cache is + shared by every repository on the spindle, so it is only safe when + every admitted owner is trusted. `SPINDLE_NIX_CACHE_REQUIRE_SIGNED_UPLOADS` + accepts only uploads signed by `SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS`, + which helps only if the signing key reaches just the repositories you + trust, for example as a repository secret. + +## Quotas, bans, webhooks and OIDC + +- **Quotas** per user and per repo (`SPINDLE_QUOTA_{USER,REPO}_*`: + workflows, vCPUs, memory, disk, cache storage, webhooks, monthly CI minutes, + and the vCPUs, memory and disk of a single workflow; 0 = unlimited). Override at runtime with `spindle quota`. + Cache storage covers a repo's nix cache paths and its `cache:` entries + together, and defaults to 5 GiB per user. Past it, the pruner evicts the + least recently used of either. +- **Bans** with `spindle ban`, `unban` and `bans`. A ban wipes the + repo's state on the spindle. +- **Allow-list and invites** with `spindle allowlist add|remove|list` and + `spindle invite`. +- **Webhooks** for push, `pull_request:*` and `repository:renamed` are + stored and delivered by the spindle (`org.tangled.webhook.*`), + with an SSRF-safe client. See [Webhooks](/docs/spindles/webhooks/). +- **OIDC:** with `SPINDLE_OIDC_ISSUER`, `KEY_PATH` and `KEY_ID` set, a + workflow that declares `audience` gets an ES256 ID token (subject + `repo:ref`) for cloud federation. Spindle serves discovery and JWKS. + +## Endpoints + +Spindle's identity is `did:web:`, the audience +for service-auth calls. It listens on `SPINDLE_SERVER_LISTEN_ADDR` +(default `0.0.0.0:6555`) for `/`, `/_health`, `/.well-known/did.json`, +`/xrpc/*`, `/mill` (mill role) and OIDC discovery, and serves Prometheus metrics on +`127.0.0.1:9091`. + +[Spindle configuration](/docs/reference/spindle-config/) lists every +variable, and [Workflow reference](/docs/spindles/workflows/) documents the +workflow format. diff --git a/docs/src/content/docs/spindles/index.md b/docs/src/content/docs/spindles/index.md new file mode 100644 index 0000000000000000000000000000000000000000..f3a3e6c9e3ce7e44fca3c7f379f746e32e9f582b --- /dev/null +++ b/docs/src/content/docs/spindles/index.md @@ -0,0 +1,94 @@ +--- +title: How CI works +description: + Spindles are Tangled's CI runners. They watch your knot for pushes and + run the workflows in .tangled/workflows/. +sidebar: + order: 0 +--- + +Spindle is a small CI runner service. It watches the knot that hosts +your repository. When you push, it runs the workflow files in +`.tangled/workflows/` on that commit. You can use Tangled's hosted +spindle or [run your own](/docs/self-hosting/spindle/). + +```yaml +# .tangled/workflows/test.yml +when: + - event: push + branch: main + +engine: microvm +image: nixos + +dependencies: + - go + +steps: + - name: test + command: go test ./... +``` + +[Writing workflows](/docs/spindles/writing-workflows/) explains this file +one part at a time. + +## The CI model + +``` +push ──▶ pipeline ──┬──▶ workflow (test.yml) ──▶ step, step, step + └──▶ workflow (lint.yml) ──▶ step, step +``` + +- One trigger event makes one **pipeline**. +- Each workflow file that matches the event becomes a **workflow** in + that pipeline. Workflows run in parallel, each in its own container or + microVM. +- A workflow is a list of **steps** that run in order. The first step + that exits non-zero fails the workflow. The steps after it do not run. + +## From push to run + +``` +you ──git push──▶ knot ──notifyEvent──▶ spindle ──▶ pipeline +``` + +1. You push to your knot. +2. The knot tells the repository's spindle about the push. The spindle + ignores refs that are not branches or tags, and pushes made with the + `skip-ci` push option. +3. The spindle reads the workflow files at the pushed commit and keeps + those whose `when` block matches the event, branch or tag, and + changed paths. If none match, nothing runs. +4. Each matching workflow becomes a run in a pipeline. The repository's + **Pipelines** tab and the commit show its status and logs. + +Behind this: + +- The repository's `org.tangled.repo.manifest` record names its + spindle. When you pick a spindle in **Settings → Pipelines**, the + manifest changes, and that spindle starts to serve the repository. +- The spindle watches the Jetstream for changes to repositories, + collaborators, pull requests and patches. It reads the records + themselves from the knot. +- Pushes and manual runs reach the spindle as an + `org.tangled.event.notifyEvent` call. Only the repository's own knot + can send a push. Pull request runs start from the pull request and + patch records. +- The spindle writes no records. Pipelines, statuses, logs, secrets and + webhooks live in the spindle's own database. The web app reads them + with `org.tangled.ci.*` queries, and logs stream over a WebSocket. + +## The engines + +Spindle has two execution backends, picked per-workflow with the +`engine` field: + +| Engine | Runs on | Use it for | +| --------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | +| `nixery` | A fresh Docker or Podman container for each step, with the image built from your Nix packages by Nixery | Short, stateless builds and tests. | +| `microvm` | The whole workflow inside its own microVM, with a choice of images | Stronger isolation, NixOS services such as PostgreSQL, Docker inside CI. | + +With nixery, state persists across steps only in the +`/tangled/workspace` directory. With microvm, files, packages and +services persist for the whole workflow. Which engines and images exist +depends on the spindle operator. diff --git a/docs/src/content/docs/spindles/secrets.md b/docs/src/content/docs/spindles/secrets.md new file mode 100644 index 0000000000000000000000000000000000000000..7522db2431d75674c89dbcff6d58278f4c9c57cd --- /dev/null +++ b/docs/src/content/docs/spindles/secrets.md @@ -0,0 +1,82 @@ +--- +title: Secrets +description: + Store tokens and passwords for a repository's workflows on its + spindle, and how they reach your workflows. +--- + +A repository secret is a named value, such as an API token, that your +workflows can read but that never appears in the repository. Secrets are +stored on the repository's spindle. + +## Adding a secret + +1. Open the repository's **Settings → Pipelines**. A spindle must be + selected. +2. Under **Secrets**, select **Add secret**. +3. Enter a **Name** (for example `CLOUDFLARE_API_TOKEN`) and a + **Value**, then select **Add**. + +The list shows each secret's name, when it was added, and by whom. +Values are never shown again; to change one, delete it and add it again. +In fact, there is no API to read the value of a secret, these are only +exposed to each workflow at execution time. Delete removes it from the +spindle immediately. + +Names can be 1 to 50 characters. Values can be up to 4096 characters. +Use names that are valid shell variable names, since that's how steps +read them. + +## Using a secret in a step + +Every secret is automatically set as an environment variable in every +step of every workflow in the repository. So adding a secret with the +name `CLOUDFLARE_API_TOKEN` will make that environment variable +available to all steps in the workflow: + +```yaml +steps: + - name: Deploy + command: pnpm exec wrangler deploy # reads $CLOUDFLARE_API_TOKEN +``` + +Any workflow that runs for the repository gets all of the configured +secrets. + +## Masking + +The spindle replaces secret values with `***` in stored and streamed +logs. It also masks the base64 encoding of each value and each line of a +multi-line value. Masking is a safety net, not a guarantee and anyone +who can run a workflow can read the secrets. + +## Visibility of secrets + +- **Managing** secrets needs settings access to the repository on the + spindle: the repository owner and collaborators. +- **Using** secrets only needs a run. Anyone who can push a branch or + tag that matches a workflow can make a step print or send them + somewhere. Treat push access as secret access. +- **Untrusted runs get no secrets.** The spindle withholds secrets, + caches and OIDC tokens from every run whose code isn't a branch of + this repository: manual runs of a fork, pull requests from forks or + patches, and any pull request run a collaborator had to approve, + retries included. Approving a run lets it run untrusted, nothing + more. +- **The spindle operator** can read secrets. They're stored in the + spindle's database, or in OpenBao if the operator configured it. + +:::note + +Secrets live on one spindle and are non-transferrable. If you switch +from one spindle to another, you will need to add secrets again there. + +::: + +## Avoiding long-lived secrets + +For cloud providers that accept OpenID Connect, set +[`audience`](/docs/spindles/workflows/#audience) in the workflow instead of +storing a key. The spindle mints a short-lived token per run as +`TANGLED_ID_TOKEN`, masked like a secret. This needs the spindle +operator to have enabled OIDC. diff --git a/docs/src/content/docs/spindles/self-hosting.md b/docs/src/content/docs/spindles/self-hosting.md new file mode 100644 index 0000000000000000000000000000000000000000..8ba3edb728b517d05070f3ef16da9b7a433fa9aa --- /dev/null +++ b/docs/src/content/docs/spindles/self-hosting.md @@ -0,0 +1,196 @@ +--- +title: Self-hosting +description: + How a spindle runs and how to configure one, from a single NixOS host + to a mill with remote executors. +--- + +A spindle is Tangled's CI runner. It watches the knots of the +repositories it serves, runs their `.tangled/workflows/*.yml` in +microVMs or containers, and serves status and logs over XRPC. [Run a +spindle](/docs/self-hosting/spindle/) is the short version of this page. + +## Signing key + +The spindle signs with an EC P-256 key that it publishes in its DID +document. It never generates one, so make it once and point +`SPINDLE_SERVER_KEY_PATH` (NixOS: `keyPath`) at it: + +```sh +openssl ecparam -name prime256v1 -genkey -noout -out /var/lib/secrets/spindle-key.pem +``` + +## Requirements + +- Linux with KVM for the `microvm` engine, or Docker for the `nixery` + engine. +- A recent `git`. +- A public hostname with TLS. + +## Install on NixOS + +Import `nixosModules.spindle` from the Tangled flake: + +```nix +{ + imports = [ tangled.nixosModules.spindle ]; + + services.tangled.spindle = { + enable = true; + server = { + hostname = "spindle.example.com"; + owner = "did:plc:yourdid"; + }; + keyPath = "/var/lib/secrets/spindle-key.pem"; + contact.email = "admin@spindle.example.com"; + artifactStores.s3.bucket = ""; + }; + + systemd.tmpfiles.rules = [ + "L+ /var/lib/spindle/images/nixos - - - - ${tangled.packages.x86_64-linux.spindle-nixos-image}" + ]; +} +``` + +The module enables the `vhost`, `vsock` and `vhost_vsock` kernel +modules and puts `qemu`, `e2fsprogs`, `slirp4netns`, `iproute2` and +`util-linux` on the service's `PATH`. + +Put it behind an HTTPS reverse proxy to port 6555 that passes websocket +upgrades. State lives in `/var/lib/spindle`, and +secrets such as AWS credentials go in `environmentFile`. +[Run a spindle](/docs/self-hosting/spindle/) walks through a first +install. + +**Elsewhere.** Build with `nix build .#spindle`, set the environment, +and run `spindle`. There is no container image. + +## MicroVM images + +Workflows pick an image by name, for example `image: nixos`. Spindle +looks for it in `SPINDLE_MICROVM_PIPELINES_IMAGE_DIR`. The flake builds +`spindle-nixos-image`, `spindle-alpine-image` and +`spindle-almalinux10-image`. + +## Admins, admission and repositories + +Admins are the owner plus `SPINDLE_SERVER_ADMINS`. With +`SPINDLE_SERVER_ADMISSION=closed` (the default) the spindle serves only +admins and its allow-list; with `open` it serves anyone who isn't +banned. + +```sh +spindle allowlist add friend.example.com +spindle invite +``` + +A repository is served when its manifest names this spindle and its +owner is admitted. + +## Secrets + +Repository secrets live in `spindle.db` (the default) or in OpenBao +(`SPINDLE_SERVER_SECRETS_PROVIDER=openbao`). They are masked in logs and +withheld from fork runs. + +## Logs and artifacts + +Logs are written to `SPINDLE_SERVER_LOG_DIR`, then archived to a disk +directory or an S3 bucket. The NixOS module defaults to the +`tangled-logs` bucket; set `artifactStores.s3.bucket = ""` to keep logs +on disk. + +## CI logs over SSH + +Run a `cilogs` server (`nixosModules.cilogs`) and set +`SPINDLE_SERVER_LOGS_ADDR` to its `host:port`. Pushes then print an +`ssh` command that follows the run. + +## Caches + +- **Nix binary cache:** microVM guests read from + `SPINDLE_NIX_CACHE_READ_URLS` and upload to + `SPINDLE_NIX_CACHE_UPLOAD_URL`. An upload URL (`http(s)://`, `ssh://`, + `ssh-ng://`, `local` or `daemon`) is one cache shared by every + repository, and a workflow can upload any content under any store path + it names. Every later workflow that reads that cache trusts it, and a + `local`, `daemon`, `ssh` or `ssh-ng` upload lands in that host's Nix + store. Only set an upload URL when everyone whose workflows run on the + spindle is trusted: `SPINDLE_SERVER_ADMISSION=closed` with an + allow-list of owners you trust. Untrusted runs (forks, patch pulls and + approved runs) never upload. The spindle logs a warning at startup when + admission is open and an upload URL is set. + `SPINDLE_NIX_CACHE_ATTIC_URL` points at an [attic](https://github.com/zhaofengli/attic) + instead, and gives every repository its own cache there. That is the + safe choice with open admission. The spindle signs short-lived tokens + for the repository's cache with the RSA key in + `SPINDLE_NIX_CACHE_ATTIC_KEY_PATH`, which atticd needs only the public + half of. Only runs of the repository's own code may push to it. Set it + or an upload URL, not both. +- **CI cache:** backs the workflow [`cache`](/docs/spindles/workflows/#cache) + field. Set `SPINDLE_CACHE_BACKEND` to `disk` or `s3`. + `SPINDLE_CACHE_MAX_BYTES` caps the whole cache: a save that would go + past it evicts the least recently used entries, and the pruner trims + back under it after the cap is lowered. Saves still being written can + go over it for a while. The cap is kept by the spindle holding the + cache index, the mill in a mill fleet, so set it there. On NixOS, the + `cache` options set these for the spindle and its local executors. + +## Quotas + +`SPINDLE_QUOTA_USER_*` and `SPINDLE_QUOTA_REPO_*` cap workflows, CPU, +memory, disk, cache and CI minutes. Override them per user: + +```sh +spindle quota set friend.example.com --resource workflows --limit 4 +spindle quota usage +spindle ban did:plc:spammer +``` + +A microVM workflow counts its image's memory. The runner's extra memory +only counts against `SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB`. + +The `WORKFLOW_VCPUS`, `WORKFLOW_MEMORY_MIB` and `WORKFLOW_DISK_MIB` limits +cap each workflow on its own and don't change how many run at once. A +workflow has to fit both its repo's and its owner's limits, or it fails +right away. To keep everyone on the 2 vCPU, 4 GiB images and let one repo +use a 6 vCPU, 8 GiB image: + +```sh +SPINDLE_QUOTA_REPO_WORKFLOW_VCPUS=2 +SPINDLE_QUOTA_REPO_WORKFLOW_MEMORY_MIB=4096 +spindle quota set did:plc:bigrepo --resource workflow_vcpus --limit 6 +spindle quota set did:plc:bigrepo --resource workflow_memory_mib --limit 8192 +``` + +Set the default on one scope and override that same scope. + +## OIDC tokens for workflows + +Set `SPINDLE_OIDC_ISSUER` and `SPINDLE_OIDC_KEY_PATH` (an EC P-256 key) +to give trusted workflows an OIDC token for their `audience:`. + +## Mill and executors + +A spindle can split into a **mill**, which places jobs, and +**executors**, which run them. Set `SPINDLE_ROLE` to `mill` or +`executor`, and register each executor on the mill: + +```sh +spindle mill executor token generate --db /var/lib/spindle/spindle.db > exec-a.token +spindle mill executor add --db /var/lib/spindle/spindle.db --token-file exec-a.token --label kvm exec-a +``` + +The mill and executors must share one artifact store. + +## Troubleshooting + +- Pushes never trigger anything: check that the owner is admitted and + not banned. +- A workflow fails with a missing tool or device: install what the error + names, see Requirements. +- Runs are slow to start: check `SPINDLE_SERVER_MAX_JOB_COUNT` and + `spindle quota usage`. + +[Spindle Configuration](/docs/reference/spindle-config/) lists every +setting. diff --git a/docs/src/content/docs/spindles/webhooks.md b/docs/src/content/docs/spindles/webhooks.md new file mode 100644 index 0000000000000000000000000000000000000000..f0dd886a68f37fcaaee4ed838f4fcbd00e99bbae --- /dev/null +++ b/docs/src/content/docs/spindles/webhooks.md @@ -0,0 +1,291 @@ +--- +title: Webhooks +description: + Send a repository's push, rename and pull request events to your own + HTTP endpoint, with the headers, payloads and signatures it receives. +--- + +A webhook sends an HTTP `POST` with a JSON body to a URL you choose +whenever something happens in a repository. Webhooks are stored and +delivered by the repository's **spindle**, so a repository needs a +spindle selected before it can have any. + +## Adding a webhook + +1. Open the repository's **Settings → Hooks** and select **New + webhook**. +2. **Payload URL**: the endpoint that receives the requests, for example + `https://example.com/webhook`. +3. **Secret** (optional): if set, every request is signed with + HMAC-SHA256 so your endpoint can verify it. Leave it blank to send + unsigned requests. +4. **Events**: pick at least one (see below). +5. **Active**: leave on to start delivering right away. +6. Select **Add**. + +Repository owners and collaborators can manage webhooks. If **Settings → +Hooks** asks you to **Authorize webhooks**, sign in again: webhook +access was added to the app's permissions after your session started. + +To change a webhook, select **Edit**. Leave **New secret** blank to keep +the existing secret, or tick **Remove the existing secret** to stop +signing. The toggle next to each webhook pauses and resumes it without +deleting it. + +## Events + +| Option | Event name | Sent when | +| ------------------------ | -------------------------- | ----------------------------------------- | +| Push events | `push` | A branch or tag is updated on the knot. | +| Repository renamed | `repository:renamed` | The repository's name changes. | +| Pull request opened | `pull_request:created` | A pull request is opened. | +| Pull request resubmitted | `pull_request:resubmitted` | A new round is pushed to a pull request. | +| Pull request merged | `pull_request:merged` | A pull request is merged. | +| Pull request closed | `pull_request:closed` | A pull request is closed without merging. | +| Pull request reopened | `pull_request:reopened` | A closed pull request is reopened. | + +The event name arrives in the `X-Tangled-Event` header. Issue events +aren't available yet. + +## Receiving a delivery + +Your endpoint should: + +1. Read the raw request body before parsing it. The signature covers + those exact bytes. +2. If you set a secret, check `X-Tangled-Signature-256` and reject the + request if it's missing or wrong. +3. Route on `X-Tangled-Event`. +4. Respond with a `2xx` status within 30 seconds. Queue slow work and + respond first. + +## Headers + +```http +POST /webhook HTTP/1.1 +Content-Type: application/json +User-Agent: Tangled-Hook/7b320e5 +X-Tangled-Event: push +X-Tangled-Hook-ID: 12 +X-Tangled-Delivery: 5f0c1c9e-3c1d-4f43-9a55-0c2b8f1e7d21 +X-Tangled-Repo: did:plc:hwevmowznbiukdf6uk5dwrrq/3lzsomerkey +X-Tangled-Signature-256: sha256=9f2c… +``` + +| Header | Value | +| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Content-Type` | `application/json` | +| `User-Agent` | `Tangled-Hook/` for pushes, `Tangled-Hook/rename`, `Tangled-Hook/pull_request`, or `Tangled-Hook/retry` for a manual retry. | +| `X-Tangled-Event` | The event name, e.g. `push` or `pull_request:merged`. | +| `X-Tangled-Hook-ID` | The webhook's numeric id on the spindle. | +| `X-Tangled-Delivery` | A UUID, new for every delivery including retries. | +| `X-Tangled-Repo` | The repository's `full_name` (see below). | +| `X-Tangled-Signature-256` | `sha256=`, only when the webhook has a secret. | + +## Payloads + +### The repository object + +Every payload has a `repository` object: + +```json +{ + "name": "some-repo", + "full_name": "did:plc:hwevmowznbiukdf6uk5dwrrq/3lzsomerkey", + "description": "", + "fork": false, + "html_url": "https://knot.example.com/did:plc:hwevmowznbiukdf6uk5dwrrq/3lzsomerkey", + "clone_url": "https://knot.example.com/did:plc:hwevmowznbiukdf6uk5dwrrq/3lzsomerkey", + "ssh_url": "ssh://git@knot.example.com/did:plc:hwevmowznbiukdf6uk5dwrrq/3lzsomerkey", + "created_at": "2025-09-15T08:57:23Z", + "updated_at": "2025-09-15T08:57:23Z", + "owner": { "did": "did:plc:hwevmowznbiukdf6uk5dwrrq" } +} +``` + +| Field | Meaning | +| --------------------------------------------- | ----------------------------------------------------------------------------------- | +| `full_name` | `/` of the repository record. It doesn't change on rename. | +| `description` | Always empty in the current spindle. | +| `fork` | Always `false` in the current spindle. | +| `html_url`, `clone_url`, `ssh_url` | Built from the repository's **knot** hostname, owner DID and record key. | +| `created_at`, `updated_at` | When the spindle learned about the repository. Both carry the same value. | +| `owner.did` | The repository owner's DID. | +| `website`, `stars_count`, `open_issues_count` | Defined but never sent by the current spindle. | + +### Push + +```json +{ + "ref": "refs/heads/main", + "before": "c04ddf64eddc90e4e2a9846ba3b43e67a0e2865e", + "after": "7b320e5cbee2734071e4310c1d9ae401d8f6cab5", + "pusher": { "did": "did:plc:hwevmowznbiukdf6uk5dwrrq" }, + "repository": { "...": "see above" } +} +``` + +| Field | Meaning | +| ------------ | ------------------------------------------------------------------------------------------------- | +| `ref` | The full ref that changed, e.g. `refs/heads/main` or `refs/tags/v1.0.0`. | +| `before` | The ref's previous commit. | +| `after` | The ref's new commit. | +| `pusher.did` | The account the knot recorded as making the update, or the repository owner if none was recorded. | + +One event is sent per updated ref, so a push of three branches produces +three deliveries. + +### Repository renamed + +```json +{ + "old_name": "some-repo", + "new_name": "better-name", + "repository": { "...": "see above, with the new name" }, + "sender": { "did": "did:plc:hwevmowznbiukdf6uk5dwrrq" } +} +``` + +### Pull request + +All `pull_request:*` events share one shape: + +```json +{ + "action": "created", + "pull_request": { + "number": 0, + "title": "add dark mode", + "body": "implements dark mode as discussed in #2", + "state": "open", + "target_branch": "main", + "source": { "branch": "dark-mode" }, + "round_number": 1, + "owner": { "did": "did:plc:hwevmowznbiukdf6uk5dwrrq" }, + "created_at": "2025-09-15T08:57:23Z" + }, + "repository": { "...": "see above" }, + "sender": { "did": "did:plc:hwevmowznbiukdf6uk5dwrrq" } +} +``` + +| Field | Meaning | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `action` | The event name's suffix: `created`, `resubmitted`, `merged`, `closed` or `reopened`. | +| `pull_request.number` | Always `0` in the current spindle. Don't use it to identify the pull request. | +| `pull_request.title`, `body` | From the pull request. `body` is empty when it has none. | +| `pull_request.state` | `merged` for `merged`, `closed` for `closed`, otherwise `open`. | +| `pull_request.target_branch` | The branch the pull request targets. | +| `pull_request.source` | Present for branch-based and fork-based pull requests, absent for patch-based ones. `branch` is the source branch; `repo` is the source repository, only for forks. | +| `pull_request.round_number` | How many rounds the pull request has. | +| `pull_request.owner.did` | The pull request's author. | +| `pull_request.created_at` | When the pull request was opened. | +| `sender.did` | Who performed the action: the author for `created` and `resubmitted`, the account that changed the state otherwise. | + +The patch itself isn't included. `html_url`, `patch_url` and +`source.sha` are defined in the payload type but not filled in by the +current spindle. + +## Verifying signatures + +When the webhook has a secret, `X-Tangled-Signature-256` is `sha256=` +followed by the hex HMAC-SHA256 of the raw request body, keyed with the +secret. Compute the same HMAC over the bytes you received and compare in +constant time. Reject the request when the header is missing. + +```go +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "strings" +) + +func verifySignature(payload []byte, header, secret string) bool { + if !strings.HasPrefix(header, "sha256=") { + return false + } + signature := strings.TrimPrefix(header, "sha256=") + + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write(payload) + expected := hex.EncodeToString(mac.Sum(nil)) + + return hmac.Equal([]byte(signature), []byte(expected)) +} + +func webhookHandler(w http.ResponseWriter, r *http.Request) { + payload, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + if !verifySignature(payload, r.Header.Get("X-Tangled-Signature-256"), yourSecret) { + http.Error(w, "invalid signature", http.StatusUnauthorized) + return + } + // respond first, process asynchronously + go processWebhook(r.Header.Get("X-Tangled-Event"), payload) + w.WriteHeader(http.StatusOK) +} +``` + +A retried delivery from the **Deliveries** page is signed with the +webhook's **current** secret, so rotating the secret and retrying works. + +## Deliveries and retries + +Each webhook's **Deliveries** page lists recent attempts with the +payload URL, the request body, the response status and body, and whether +it succeeded. The spindle keeps the last 50 deliveries per webhook, with up to 10 KiB +of each response body. +Select **Retry** to send a delivery again; the retry is signed with the +webhook's current secret and gets a new delivery id. + +The spindle tries each delivery up to 3 times, backing off from 1 second +up to 10 seconds, when the connection fails or your endpoint returns a +`5xx` status. A `4xx` response is recorded as a failure and not retried, +so only return `4xx` when the same request can never succeed. Deliveries +are not retried later beyond those 3 attempts. + +Requests are sent from the spindle's network, not from the web app. The +spindle refuses to deliver to private and internal addresses and doesn't +follow redirects, so the payload URL must be a public address that +answers directly. If many events arrive at once and the spindle's +delivery queue is full, the event is recorded as a failed delivery +instead of being sent; retry it from the **Deliveries** page. + +## Example: Discord notification + +```javascript +app.post( + "/webhook", + express.raw({ type: "application/json" }), + async (req, res) => { + // verify req.body (a Buffer) against X-Tangled-Signature-256 first + const payload = JSON.parse(req.body); + res.status(200).send("OK"); + + if (req.get("X-Tangled-Event") !== "push") return; + await fetch("https://discord.com/api/webhooks/...", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + content: `${payload.pusher.did} pushed to ${payload.ref} in ${payload.repository.name}`, + }), + }); + }, +); +``` + +:::caution + +Webhooks live on the spindle. Switching the repository to another +spindle makes the old spindle delete the repository's webhooks and +delivery history; recreate them on the new spindle. + +::: diff --git a/docs/src/content/docs/spindles/workflows.md b/docs/src/content/docs/spindles/workflows.md new file mode 100644 index 0000000000000000000000000000000000000000..3c1a81c252fcde08fed560f4edfbc5168224d35d --- /dev/null +++ b/docs/src/content/docs/spindles/workflows.md @@ -0,0 +1,710 @@ +--- +title: Workflow reference +description: + Every field a spindle workflow file accepts, per engine, with defaults + and validation rules. +--- + +A workflow is a YAML file in `.tangled/workflows/` at the root of your +repository. Your repository's spindle reads every file in that directory +at the triggering commit and runs the ones whose trigger matches. +[Writing Workflows](/docs/spindles/writing-workflows/) walks through a first +workflow; this page lists every field. + +## All fields + +Every field a workflow file accepts. Each one links to its section +below. + +```yaml +# common fields, every engine +when: # list of conditions, any one matches + - event: [push, pull_request, manual, schedule] + branch: ["main", "release-*"] # push and pull_request + tag: ["v*"] # push only + types: [opened, reopened, synchronize, closed, merged] # pull_request only + paths: ["src/**"] # changed files + schedule: # schedule only + - cron: "H 2 * * *" + timezone: Europe/London # default UTC +engine: microvm # microvm, nixery or dummy (required) +clone: + skip: false + depth: 1 + submodules: true + tags: true +environment: + KEY: value +cache: + - key: go-mod # required + hash: [go.sum] + fallback: false + paths: [.gocache] # required + compression-level: 5 # 1 to 19 + when: on-success # or always +steps: + - name: test + command: go test ./... + environment: + KEY: value +runs_on: [arm64] # executor labels +audience: sts.amazonaws.com # oidc token audience + +# microvm only +image: nixos +dependencies: [go, "unstable#zig", "github:nixos/nixpkgs#hello"] +registry: + nixpkgs: github:nixos/nixpkgs/nixos-unstable +substituters: + https://nix-community.cachix.org: "nix-community.cachix.org-1:..." +services: + postgresql: true +virtualisation: + docker: true + +# nixery only +dependencies: + nixpkgs: [go] +``` + +## Pipelines + +One trigger event produces one **pipeline**. Each workflow file that +matches the event becomes a **workflow** in that pipeline and runs in +its own container or microVM, in parallel with the others. A workflow is +a list of **steps** run in order; the first step that exits non-zero +fails the workflow, and later steps don't run. + +A workflow has a set of common fields that apply no matter which engine +you pick: + +| Field | Required | Purpose | +| ----------------------------- | -------- | ---------------------------------------------------------------------- | +| [`when`](#trigger) | no | When the workflow runs. Without it, the workflow runs for every event. | +| [`engine`](#engine) | **yes** | Which engine runs the workflow. | +| [`clone`](#clone-options) | no | How the repository is fetched. | +| [`environment`](#environment) | no | Environment variables for every step. | +| [`cache`](#cache) | no | Directories kept between runs. | +| [`steps`](#steps) | no | The commands to run. | +| [`runs_on`](#runs_on) | no | Executor labels, on spindles that use a mill. | +| [`audience`](#audience) | no | Request an OIDC ID token for this audience. | + +On top of these, each engine has its own options for dependencies and +images. See [Engines](#engines). + +The spindle rejects unknown keys: a typo such as `registre:` fails the +workflow with an error that points at the offending line, instead of +being ignored. + +### Trigger + +The trigger defines when a workflow runs. It's the `when` field, a list +of conditions. The workflow runs if **any** condition matches. Each +condition has these fields: + +- `event`: **required**. A string or a list of one or more of: + - `push`: a branch or tag is pushed to the repository. + - `pull_request`: a pull request targeting this repository changes. + See `types`. + - `manual`: marks the workflow as one you start by hand. The spindle + doesn't evaluate `when` for manual runs at all: it runs the + workflows the caller names, or every workflow. + - `schedule`: the workflow runs according to its `cron` expressions + using the default-branch name and commit saved when spindle last + refreshed the repository's schedules. +- `branch`: Defines which branches the workflow should run for. With + `push`, commits to the branch(es) listed here trigger the workflow. + With `pull_request`, updates to pull requests targeting the branch(es) + listed here trigger the workflow. Supports glob patterns using `*` and + `**` (e.g., `main`, `develop`, `release-*`). Either `branch` or `tag` + (or both) must be specified for `push` events. +- `tag`: Defines which tags the workflow should run for. Only used with + `push`: when tags matching the pattern(s) listed here are pushed, the + workflow triggers. Supports glob patterns using `*` and `**` (e.g., + `v*`, `v1.*`, `release-**`). +- `types`: Only used with `pull_request`. Which pull request actions + trigger the workflow: `opened`, `reopened`, `synchronize` (a new round + was pushed), `closed`, `merged`. Defaults to `opened`, `reopened` and + `synchronize`. +- `paths`: Glob patterns over changed file paths. When set, the + condition only matches if at least one changed file matches a pattern. + For pushes, if the spindle can't compute the list of changed files, + the condition is not skipped on that account. +- `schedule`: A list of scheduled trigger entries. This field is + required for `schedule` events and invalid for other events. Each + entry requires a `cron` field containing a five-field expression + (`minute hour day-of-month month day-of-week`) and accepts an optional + IANA `timezone`, which defaults to `UTC`. Hashed `H` fields are + supported; seconds, nicknames such as `@daily`, and timezone prefixes + are not. A condition containing `schedule` cannot also specify + `branch`, `tag`, `types`, or `paths`. + +`*` matches any run of characters except `/`; `**` also matches `/`. Any +of the list fields accepts a single string instead of a list. + +For example, a workflow that runs when commits are pushed to `main` or +`develop`, when pull requests that target `main` are updated, or by +hand: + +```yaml +when: + - event: ["push", "manual"] + branch: ["main", "develop"] + - event: ["pull_request"] + branch: ["main"] +``` + +To run a deployment workflow when tags matching `v*` are pushed: + +```yaml +when: + - event: ["push"] + tag: ["v*"] +``` + +You can combine branch and tag patterns in a single condition. The +workflow triggers if either matches: + +```yaml +when: + - event: ["push"] + branch: ["main", "release-*"] + tag: ["v*", "stable"] +``` + +Only run when documentation changes, and also when a pull request is +merged: + +```yaml +when: + - event: push + branch: main + paths: ["docs/**", "mkdocs.yml"] + - event: pull_request + branch: main + types: [merged] +``` + +#### Pull request triggers + +A `pull_request` run starts on its own when the pull request's author +has push access to the repository. Any other pull request's run waits, +blocked, until a collaborator approves it. The checkout is the pull +request's source commit; `branch` matches the **target** branch. + +Only a run of a branch of this repository that needed no approval gets +the repository's secrets, caches and ID token. Runs of fork and +patch-based pull requests, and every approved run and its retries, get +none of them: approving lets the code run, untrusted. + +#### Schedule triggers + +Scheduled workflows use the saved default-branch revision, including +after a spindle restart. A condition can contain multiple schedule +entries, each with its own timezone. For example, this runs at a stable +minute during the 02:00 hour in New York and at 09:00 on weekdays in +London: + +```yaml +when: + - event: schedule + schedule: + - cron: "H 2 * * *" + timezone: America/New_York + - cron: "0 9 * * 1-5" + timezone: Europe/London +``` + +`H` spreads scheduled work using a deterministic hash of the repository +DID and workflow filename. The minute stays stable across restarts; +changing either identity changes the hash. Ranges and steps are +supported, for example `H(0-29)/10` or `H/15`. Unbounded day-of-month +hashes use days 1–28. Exact expressions remain exact; spindle logs a +warning recommending `H` rather than silently changing their timing. + +Spindle checks schedules once per minute. The minimum interval defaults +to five minutes and is configured with +`SPINDLE_SCHEDULE_MINIMUM_INTERVAL` (for example `10m`). It must be a +positive whole number of minutes. Spindle rejects workflows whose +combined schedule entries have distinct occurrences closer than this +interval. Validation is best-effort: it examines up to 400 days or 4096 +merged occurrences, whichever comes first. Simultaneous entries count as +one run. Runtime claims also enforce the minimum interval. + +`SPINDLE_SCHEDULE_CONCURRENCY` controls parallel repository dispatches +and startup schedule refreshes (default `4`). Dispatch finishes each +minute's batch before starting the next. Occurrences while spindle is +offline are not replayed. + +#### Skipping CI + +To skip CI for a push, pass a Git push option: + +```sh +git push -o skip-ci +``` + +`ci-skip` is also accepted. The push is recorded as usual; no pipeline +starts for it. + +### Engine + +The **required** `engine` field picks the engine the workflow runs on. A +workflow without it fails with `missing engine`. The engines are: + +- `nixery`: This uses an instance of [Nixery](https://nixery.dev) to run + steps, which allows you to add [dependencies](#dependencies) from + Nixpkgs (https://github.com/NixOS/nixpkgs). You can search for + packages on https://search.nixos.org. Each step runs in its own + container. See [Nixery Engine](#nixery-engine). +- `microvm`: Runs the whole workflow inside its own microVM. Has + configuration features for NixOS images that let you enable services, + run Docker inside the VM, and so on. See + [microVM Engine](#microvm-engine). + +Which engines are available depends on the spindle. A spindle operator +can run either or both. + +```yaml +engine: "nixery" +``` + +### Clone options + +When a workflow starts, its first step fetches the triggering commit +into the workspace. The **optional** `clone` field changes this: + +- `skip`: `true` skips the fetch. Useful when the workflow needs nothing + from the repository. Default `false`. Setting `depth`, `submodules` or + `tags` together with `skip` produces a warning. +- `depth`: How many commits to fetch, counting back from the triggering + commit. `0` or unset fetches 1, only the commit that triggered the + workflow. +- `submodules`: Recursively fetch Git submodules + (https://git-scm.com/book/en/v2/Git-Tools-Submodules). Default `true`. +- `tags`: Fetch tags as well. Default `true`. + +The default settings are: + +```yaml +clone: + skip: false + depth: 1 + submodules: true + tags: true +``` + +The fetch uses the repository's HTTPS clone URL on its knot. The +checkout is detached at the triggering commit. + +### Environment + +The `environment` field defines environment variables available +throughout the entire workflow. **Do not put secrets here: these values +are visible to anyone who can read the repository.** Add secrets in the +repository's settings instead; see [Secrets](/docs/spindles/secrets/). + +```yaml +environment: + GOOS: "linux" + GOARCH: "arm64" + NODE_ENV: "production" + MY_ENV_VAR: "MY_ENV_VALUE" +``` + +By default, the following environment variables are set: + +| Variable | Value | +| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| `CI` | Always `true`. | +| `TANGLED_PIPELINE_ID` | The id of the current pipeline. | +| `TANGLED_PIPELINE_KIND` | One of `push`, `pull_request`, `manual` or `schedule`. | +| `TANGLED_PIPELINE_SOURCE` | DID of the repository the code was checked out from. Differs from the repository's own DID only for manual runs of a fork. | +| `TANGLED_REPO_KNOT` | The repository's knot hostname. | +| `TANGLED_REPO_DID` | The DID of the repository owner. | +| `TANGLED_REPO_REPO_DID` | The repository's own DID. | +| `TANGLED_REPO_NAME` | The name of the repository. | +| `TANGLED_REPO_DEFAULT_BRANCH` | The default branch of the repository. | +| `TANGLED_REPO_URL` | The full clone URL of the repository. | + +These variables identify the selected git revision. SHA variables are +available for every trigger kind; ref variables are available for +pushes, pull requests, schedules, and manual runs that specify a ref: + +| Variable | Value | +| -------------------- | ------------------------------------------------------------------------------------------------------------- | +| `TANGLED_REF` | The full git reference (e.g., `refs/heads/main` or `refs/tags/v1.0.0`). For pull requests, the source branch. | +| `TANGLED_REF_NAME` | The short name of the reference (e.g., `main` or `v1.0.0`). | +| `TANGLED_REF_TYPE` | `branch` or `tag`. | +| `TANGLED_SHA` | The commit SHA that triggered the pipeline. | +| `TANGLED_COMMIT_SHA` | Alias for `TANGLED_SHA`. | + +Schedule triggers additionally provide: + +| Variable | Value | +| ---------------------- | ----------------------------------------------- | +| `TANGLED_SCHEDULED_AT` | The UTC minute selected by the cron expression. | + +These variables are only available when the pipeline is triggered by a +pull request: + +| Variable | Value | +| -------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `TANGLED_PR_SOURCE_BRANCH` | The source branch of the pull request. | +| `TANGLED_PR_TARGET_BRANCH` | The target branch of the pull request. | +| `TANGLED_PR_SOURCE_SHA` | The commit SHA of the source branch. | +| `TANGLED_PR_AT_URI` | The [AT URI](https://atproto.com/specs/at-uri-scheme) of the PR itself, will be of the form `at:///org.tangled.track.ticket/` | +| `TANGLED_PIPELINE_SOURCE_BRANCH` | Same as `TANGLED_PR_SOURCE_BRANCH`. | +| `TANGLED_PIPELINE_TARGET_BRANCH` | Same as `TANGLED_PR_TARGET_BRANCH`. | + +Manual runs expose each input passed with the trigger as +`TANGLED_INPUT_`, with the key upper-cased. + +Repository secrets are added to the environment of every step, and +`TANGLED_ID_TOKEN` when [`audience`](#audience) is set. Step-level +`environment` overrides workflow-level `environment`. + +### Cache + +The `cache` field lets a workflow persist directories across pipeline +runs. Before the first step, the engine looks up each entry and extracts +the matching archive into the workspace; after the steps finish, the +paths are archived again and stored. + +- `key`: **required**. Name the cache is saved under: letters, digits, + `.`, `_` and `-`, starting with a letter or digit, at most 128 + characters. Keys must be unique within a workflow. Entries are scoped + to the repository, engine, workflow file and ref, and carry over from + one commit to the next. A branch or pull with no entry of its own + restores the default branch's, but saves only under its own ref. +- `hash`: **optional** list of repo files (lockfiles, manifests) whose + content is folded into the entry. The entry is stored as + `-`, so editing `go.sum` invalidates the cache + without bumping the key by hand. Paths are relative to the repository + root, are read from git at the commit being built, and can't contain + globs. +- `fallback`: **optional**, default `false`. With `hash` set, when no + entry matches the current hash, restore the newest entry for the same + workflow and ref whose hash differs, then the default branch's. This is how you start from + yesterday's dependencies after a lockfile change. +- `paths`: **required**, at least one. Paths to archive. Relative paths + are anchored at the repository checkout, the directory steps start in + (`/workspace/repo` on microvm, `/tangled/workspace` on nixery). + Absolute paths work too, for caching directories outside the checkout, + but they name engine-specific locations. All paths must be writable by + the CI user and contain no spaces, quotes, shell metacharacters or + `..`. +- `compression-level`: **optional** zstd level, `1` (fastest) to `19` + (smallest). Defaults to `5`. Saving needs `zstd` in the image; without + it the cache isn't saved. +- `when`: **optional** save policy: `on-success` (the default) or + `always`, which also saves after a failed or timed-out run. + +```yaml +cache: + - key: go-mod + hash: + - go.sum + - go.mod + fallback: true + paths: + - .gocache +``` + +Caches are only restored and saved for trusted pipelines (pushes and +same-repository pull requests). Pipelines building untrusted code, like +manual runs of a fork, skip the cache entirely. Saving follows each +entry's `when` policy, and a timed-out run counts as failed. A cache +miss or failure never fails the workflow, but an invalid `cache` entry +does. + +The spindle operator chooses the storage backend and size limits. Past +them, the least recently used caches are evicted first. +See [Run a spindle](/docs/self-hosting/spindle/). If no backend is +configured, `cache` entries are ignored. + +### Steps + +The `steps` field is a list of step objects, each with: + +- `name`: Shown in the run's logs and used to describe what the step + does. +- `command`: The command to run. The step runs in a Bash shell, and its + output is visible on the repository's pipelines page. Any dependencies + you added in your engine's section (see [Engines](#engines)) are + available here. A multi-line `|` block runs as one script. +- `environment`: Like the workflow-level [environment](#environment), + this **optional** key-value map sets environment variables for this + step only. **Do not put secrets here.** + +```yaml +steps: + - name: "Build backend" + command: "go build" + environment: + GOOS: "darwin" + GOARCH: "arm64" + - name: "Build frontend" + command: "npm run build" + environment: + NODE_ENV: "production" +``` + +The whole workflow, including time spent waiting for a free slot, must +finish within the spindle's workflow timeout. The default is 5 minutes; +operators can change it. A workflow that runs out of time ends with +status `timeout`. + +### runs_on + +On a spindle split into a mill and executors, `runs_on` restricts the +workflow to executors carrying all of the listed labels. Labels are +chosen by the operator. If no executor registered with the spindle +carries all of them, the workflow fails right away. On a standalone +spindle it has no effect. + +```yaml +runs_on: [arm64, large] +``` + +### audience + +`audience` asks the spindle to mint an OpenID Connect ID token for this +run and expose it to every step as `TANGLED_ID_TOKEN`. Use it to +authenticate to a cloud provider or other service that trusts the +spindle's issuer, without storing a long-lived credential. + +```yaml +audience: sts.amazonaws.com +``` + +- The value must be printable ASCII, at most 150 characters. It becomes + the token's `aud` claim. +- The token is an ES256 JWT with claims `iss`, `aud`, `sub` + (`:`), `iat`, `nbf`, `exp`, `jti`, `repository`, + `ref`, `sha` and `actor`. `repository` is the repository DID. +- It lives for the workflow's time budget, at least 15 minutes. +- It's masked in logs like a secret. +- It's only minted for trusted pipelines. Manual runs of a fork never + get one. `ref` is empty for pull request runs, and for manual runs + it's only set when the spindle verified that the ref contains the + commit. +- The spindle publishes its keys at `/.well-known/openid-configuration` + and `/jwks`. If the operator hasn't configured OIDC, no token is + issued. + +## Engines + +The common fields above apply to every workflow. Each engine then adds +its own fields on top. Pick an engine with the [`engine`](#engine) field +and use the matching section below. + +### Nixery engine + +Each step runs in a fresh container built by Nixery from your +dependencies. The workspace is shared between steps; nothing else is. + +#### Dependencies + +The `dependencies` field lets you define which dependencies to get, and +from where. It's a key-value map, with the key being the registry to +fetch dependencies from, and the value being the list of dependencies to +fetch. + +The registry URL syntax can be found +[in the Nix manual](https://nix.dev/manual/nix/2.18/command-ref/new-cli/nix3-registry-add). + +Say you want Node.js and Go from `nixpkgs`, and a package called +`my_pkg` you've made in your own repository at +`https://tangled.org/@example.com/my_pkg`: + +```yaml +dependencies: + # nixpkgs + nixpkgs: + - nodejs + - go + # unstable + nixpkgs/nixpkgs-unstable: + - bun + # custom registry + git+https://tangled.org/@example.com/my_pkg: + - my_pkg +``` + +These dependencies are now available to every step. + +#### Complete nixery workflow + +```yaml +# .tangled/workflows/build.yml + +when: + - event: ["push", "manual"] + branch: ["main", "develop"] + - event: ["pull_request"] + branch: ["main"] + +engine: "nixery" + +# using the default values +clone: + skip: false + depth: 1 + submodules: true + +dependencies: + # nixpkgs + nixpkgs: + - nodejs + - go + # custom registry + git+https://tangled.org/@example.com/my_pkg: + - my_pkg + +environment: + GOOS: "linux" + GOARCH: "arm64" + NODE_ENV: "production" + MY_ENV_VAR: "MY_ENV_VALUE" + +steps: + - name: "Build backend" + command: "go build" + environment: + GOOS: "darwin" + GOARCH: "arm64" + - name: "Build frontend" + command: "npm run build" + environment: + NODE_ENV: "production" +``` + +For another example, see the workflow +[Tangled uses to build the project](https://tangled.org/@tangled.org/core/blob/master/.tangled/workflows/build.yml). + +### microVM engine + +The whole workflow runs in one QEMU microVM, so files, installed +packages and running services carry over from one step to the next. +Steps run as the unprivileged `spindle-workflow` user in +`/workspace/repo`. The VM has outbound network access only, with private +address ranges blocked. + +#### Image + +A workflow picks the image to boot with the top-level `image` field: + +```yaml +engine: microvm +image: nixos +``` + +There are two flavours of images: + +- **NixOS images** (e.g. `nixos`): the whole guest is built with Nix, so + you can configure it from the workflow file itself. The + `dependencies`, `services`, `virtualisation`, `registry` and + `substituters` fields below are all understood here, and the guest + builds and activates that configuration before any of your steps run. +- **Non-NixOS images** (e.g. `alpine`, `almalinux10`): there's no NixOS + to configure, so the workflow-level config fields above have no + effect. You still get a full machine to run steps in. + +The available image names depend on what the spindle operator has +installed. `nixos`, `alpine` and `almalinux10` are the images built from +the Tangled repository. If `image` is omitted, the spindle's configured +default image is used (`nixos-x86_64` unless the operator changed it). + +In every image, `nixpkgs` (as in `nix shell nixpkgs#hello` or +``) is the nixpkgs revision the image was built from, and +`nix flake metadata nixpkgs` prints it. `unstable` is the latest +nixpkgs-unstable, as in `nix shell unstable#hello`. + +#### Dependencies + +On the microVM engine, `dependencies` is a flat list of packages that +are made available to every step. This field only applies to **NixOS +images**; for other images, use the image's package manager in a step. + +The guest builds a +[`nix develop`](https://nix.dev/manual/nix/2.18/command-ref/new-cli/nix3-develop)-style +devshell from your dependencies and uses it for each step. For example, +adding `pkg-config` and `openssl` is enough for the `openssl-sys` crate +to build in a Rust project. + +A bare name like `go` is looked up in the image's nixpkgs, so it is the +same `go` as `nixpkgs#go` in a step. You can also point at any flake with +the `flakeref#attr` syntax, so `unstable#go` takes `go` from +nixpkgs-unstable and `github:nixos/nixpkgs#hello` pulls `hello` straight +out of that flake. + +```yaml +dependencies: + - go + - unstable#zig + - github:nixos/nixpkgs#hello +``` + +#### Registry + +The `registry` field remaps flake references, the same way +`nix registry` does. This lets you pin or alias the flakes used by +`dependencies`. + +For example, pin `nixpkgs` to `nixos-unstable` so that the bare `go` +above resolves from unstable, and alias your own flake so you can use +`myflake#tool` in `dependencies`: + +```yaml +registry: + nixpkgs: github:nixos/nixpkgs/nixos-unstable + myflake: github:me/x +``` + +#### Substituters + +The `substituters` field is a map of Nix binary cache URL to its trusted +public key. These are fed into the spindle's read proxy, so the guest +can substitute prebuilt paths from them instead of building everything +from scratch. + +```yaml +substituters: + https://nix-community.cachix.org: "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" +``` + +#### Services and virtualisation + +The `services` and `virtualisation` fields are passed straight through +to NixOS. Anything you could write under `services.*` or +`virtualisation.*` in a NixOS configuration, you can write here, and +it's brought up before any of your steps run. + +As a convenience, `true` works as shorthand for `.enable = true` +anywhere an `enable` option exists (e.g. `virtualisation.docker: true`). + +```yaml +services: + postgresql: + enable: true + ensureDatabases: ["spindle-workflow"] + ensureUsers: + - name: spindle-workflow + ensureDBOwnership: true + +virtualisation: + docker: true +``` + +#### Debugging a failed run + +If the operator enabled debug SSH, a microVM whose step fails is kept +alive for a grace period (5 minutes by default). The step's log ends +with a **Debug shell** entry containing the exact `ssh` command. You +need an SSH key with push access to the repository. + +### Dummy engine + +`engine: dummy` accepts `steps` and `environment` and doesn't execute +anything. It exists for testing spindle itself. diff --git a/docs/src/content/docs/spindles/writing-workflows.md b/docs/src/content/docs/spindles/writing-workflows.md new file mode 100644 index 0000000000000000000000000000000000000000..7acc7ed03f302796dd7f6e9c41acf2f56dbbfd9e --- /dev/null +++ b/docs/src/content/docs/spindles/writing-workflows.md @@ -0,0 +1,535 @@ +--- +title: Writing workflows +description: + A walkthrough of the workflow syntax, one field at a time, with small + examples and ready-made recipes. +--- + +Spindle workflows allow you to write CI/CD pipelines in a simple format. +They're located in the `.tangled/workflows` directory at the root of +your repository, and are defined using YAML. This page shows each part +with a small example. The [workflow reference](/docs/spindles/workflows/) +lists every field. + +Before a workflow can run, the repository needs a spindle. Open the +repository's **Settings → Pipelines** and, under **Spindle**, run CI +with Tangled or select your own spindle. + +## Your first workflow + +Create `.tangled/workflows/check.yml`: + +```yaml +when: + - event: push + branch: main + +engine: microvm +image: nixos + +dependencies: + - go + +steps: + - name: Test + command: go test ./... +``` + +Commit and push it: + +```sh +git add .tangled/workflows/check.yml +git commit -m "Add CI workflow" +git push +``` + +The push starts the first run. Open the repository's **Pipelines** tab +to watch it. + +If a workflow file has a problem, `git push` prints it. Add +`-o verbose-ci` to see the report when there is no problem. + +A workflow has four main parts: + +| Part | Says | +| -------------- | ------------------------ | +| `when` | When the workflow runs. | +| `engine` | Where the workflow runs. | +| `dependencies` | What tools it needs. | +| `steps` | What commands to run. | + +## When it runs + +The `when` field is a list of conditions. The workflow runs if any +condition matches. + +Run on every push to `main`: + +```yaml +when: + - event: push + branch: main +``` + +Run on pull requests that target `main`: + +```yaml +when: + - event: pull_request + branch: main +``` + +Run when tags matching `v*` are pushed: + +```yaml +when: + - event: push + tag: "v*" +``` + +Run only when files in `docs/` change: + +```yaml +when: + - event: push + branch: main + paths: ["docs/**"] +``` + +Run every night, at a stable minute in the 02:00 hour: + +```yaml +when: + - event: schedule + schedule: + - cron: "H 2 * * *" +``` + +Let people start it by hand: + +```yaml +when: + - event: manual +``` + +Combine them. This runs on pushes to `main` or `develop`, on pull +requests that target `main`, and by hand: + +```yaml +when: + - event: ["push", "manual"] + branch: ["main", "develop"] + - event: ["pull_request"] + branch: ["main"] +``` + +To skip CI for a push, pass a Git push option: + +```sh +git push -o skip-ci +``` + +## Where it runs + +The `engine` field picks the engine: + +```yaml +engine: microvm +image: nixos +``` + +```yaml +engine: nixery +``` + +- `microvm` runs the whole workflow inside its own microVM. Files, + packages and services stay from one step to the next. +- `nixery` runs each step in a fresh container. Only the workspace + stays between steps. + +Use `microvm` if you are not sure. With `microvm`, `image` picks the +image to boot: `nixos`, or a non-NixOS image such as `alpine`. + +## Tools + +On `microvm` with a NixOS image, `dependencies` is a list of nixpkgs +packages. You can search for packages on https://search.nixos.org. + +```yaml +dependencies: + - nodejs + - pnpm +``` + +Take a package from any flake with `flakeref#attr`: + +```yaml +dependencies: + - github:nixos/nixpkgs#hello +``` + +On `nixery`, `dependencies` is a map from registry to packages: + +```yaml +dependencies: + nixpkgs: + - nodejs + - go +``` + +On a non-NixOS image, install tools in a step: + +```yaml +steps: + - name: "Install go" + command: apk add go +``` + +## Steps + +Each step has a `name` and a `command`. The step runs in a Bash shell. +Its logs show on the pipelines page. + +```yaml +steps: + - name: "Build" + command: go build ./... + - name: "Test" + command: go test ./... +``` + +Use `|` for a command with more than one line: + +```yaml +steps: + - name: "Lint and test" + command: | + pnpm run lint + pnpm test +``` + +## Environment variables + +Set variables for every step with `environment`. Set variables for one +step with that step's `environment`: + +```yaml +environment: + NODE_ENV: "production" + +steps: + - name: "Build backend" + command: "go build" + environment: + GOOS: "darwin" + GOARCH: "arm64" +``` + +Do not put secrets here, these environment variables are visible to +anyone viewing the repository. Add secrets in the repository's +**Settings → Pipelines → Secrets**. They reach every step as environment +variables. See [Secrets](/docs/spindles/secrets/). + +The spindle also sets `CI=true` and `TANGLED_*` variables that describe +the run: + +```sh +echo "$TANGLED_REF_NAME" # main +echo "$TANGLED_SHA" # the commit that triggered the pipeline +``` + +## Caching + +Keep directories between runs with `cache`: + +```yaml +cache: + - key: go-mod + hash: [go.sum] + paths: [.gocache] +``` + +`hash` folds the content of `go.sum` into the key, so editing `go.sum` +invalidates the cache. Caches are only used for trusted runs, and only when +the spindle operator configured cache storage. + +## Cloning + +By default the spindle fetches only the commit that triggered the +workflow. Fetch more history when a tool needs it, such as +`git describe`: + +```yaml +clone: + depth: 50 +``` + +Skip the clone when the workflow needs nothing from the repository: + +```yaml +clone: + skip: true +``` + +## Recipes + +### Lint, test and build a Node project + +```yaml +when: + - event: ["push", "pull_request"] + branch: ["main"] + +engine: microvm +image: nixos + +dependencies: + - pnpm + +steps: + - name: "Install dependencies" + command: pnpm install --frozen-lockfile + - name: "Lint and test" + command: | + pnpm run lint + pnpm test + - name: "Build" + command: pnpm run build +``` + +### Check formatting + +```yaml +when: + - event: ["push", "pull_request"] + branch: ["main"] + +engine: microvm +image: alpine # slimmer image for checking the formatting + +steps: + - name: "Install go" + command: apk add go + - name: "Check formatting" + command: test -z $(gofmt -l .) +``` + +### Build a Rust project that links OpenSSL + +```yaml +when: + - event: ["push", "pull_request"] + branch: ["main"] + +engine: microvm +image: nixos + +dependencies: + - gcc + - cargo + - rustc + - clippy + - rustfmt + - pkg-config # exports PKG_CONFIG_PATH for the libraries below + - openssl # the C library + headers openssl-sys links against + +steps: + - name: "Check formatting" + command: cargo fmt --check + - name: "Clippy" + command: cargo clippy --all-targets -- -D warnings + - name: "Test" + command: cargo test --all + - name: "Release build" + command: cargo build --release +``` + +### Run migrations and integration tests against PostgreSQL + +```yaml +when: + - event: ["push", "pull_request"] + branch: ["main"] + +engine: microvm +image: nixos + +environment: + DATABASE_URL: "postgresql:///spindle-workflow?host=/run/postgresql" + +dependencies: + - gcc + - cargo + - rustc + - pkg-config + - openssl + - sqlx-cli + +services: + postgresql: + enable: true + # has to be same name as the user for peer auth to work automatically + ensureDatabases: ["spindle-workflow"] + ensureUsers: + - name: spindle-workflow + ensureDBOwnership: true + +steps: + - name: "Run migrations" + command: sqlx migrate run + - name: "Integration tests" + command: cargo test --all +``` + +### Build and push a Docker image on tag + +`REGISTRY_USER` and `REGISTRY_TOKEN` are repository secrets. + +```yaml +when: + - event: ["push"] + tag: ["v*"] + +engine: microvm +image: nixos + +virtualisation: + docker: true + +steps: + - name: "Build and push to ghcr.io" + command: | + set -euo pipefail + + echo "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin + image="ghcr.io/$REGISTRY_USER/myapp:$TANGLED_REF_NAME" + + docker build -t "$image" -t "ghcr.io/$REGISTRY_USER/myapp:latest" . + docker push "$image" + docker push "ghcr.io/$REGISTRY_USER/myapp:latest" +``` + +### Deploy to Cloudflare Workers on tag + +```yaml +# .tangled/workflows/deploy.yml +when: + - event: ["push"] + tag: ["v*"] + +engine: microvm +image: nixos + +dependencies: + - pnpm + +steps: + - name: "Install dependencies" + command: pnpm install --frozen-lockfile + - name: "Deploy worker" + # `wrangler` picks up `CLOUDFLARE_API_TOKEN` from the env. + # set it under **Settings → Pipelines → Secrets**. + command: pnpm exec wrangler deploy +``` + +### Authenticate to a cloud provider without a stored key + +If the spindle operator enabled OIDC, set `audience` and exchange +`TANGLED_ID_TOKEN` for short-lived cloud credentials. Configure the +provider to trust the spindle's issuer and to check the token's +`repository` and `ref` claims. + +```yaml +when: + - event: push + branch: main + +engine: microvm +image: nixos +audience: sts.amazonaws.com + +dependencies: + - awscli2 + +steps: + - name: "Deploy" + command: | + echo "$TANGLED_ID_TOKEN" > /tmp/token + export AWS_ROLE_ARN=arn:aws:iam::123456789012:role/deploy + export AWS_WEB_IDENTITY_TOKEN_FILE=/tmp/token + aws s3 sync dist/ s3://my-bucket/ +``` + +### Attach a release artifact on tag + +This workflow runs when you push a tag. It builds a file and attaches +it to the tag. The file shows with the tag on the repo page. + +The workflow uses [goat](https://github.com/bluesky-social/goat) to get +service tokens from your PDS. Add two secrets under **Settings → +Pipelines → Secrets**: `RELEASE_HANDLE`, the handle of a repo owner or +collaborator, and `RELEASE_APP_PASSWORD`, an app password for that +account. + +```yaml +# .tangled/workflows/release.yml +when: + - event: ["push"] + tag: ["v*"] + +engine: microvm +image: nixos + +dependencies: + - git + - curl + - jq + - unstable#atproto-goat + +steps: + - name: "Build the artifact" + command: | + set -euo pipefail + # Replace this with your build. The limit is 50 MiB. + mkdir -p dist + git archive --format=tar.gz -o "dist/$TANGLED_REPO_NAME-$TANGLED_REF_NAME.tar.gz" \ + --prefix="$TANGLED_REPO_NAME-$TANGLED_REF_NAME/" HEAD + + - name: "Upload the artifact" + command: | + set -euo pipefail + goat account login -u "$RELEASE_HANDLE" -p "$RELEASE_APP_PASSWORD" + file="dist/$TANGLED_REPO_NAME-$TANGLED_REF_NAME.tar.gz" + xrpc="https://$TANGLED_REPO_KNOT/xrpc" + + # Each knot write needs a new token. + token() { + goat account service-auth --aud "$TANGLED_REPO_REPO_DID" --lxm "$1" + } + + # Send the file to the knot first. + blob=$(curl -fsS "$xrpc/com.atproto.repo.uploadBlob" \ + -H "Authorization: Bearer $(token com.atproto.repo.uploadBlob)" \ + -H "Content-Type: application/gzip" \ + --data-binary "@$file" | jq -c .blob) + + # Then make the record using createRecord + # The subject is the commit of the tag (which is HEAD). + jq -n \ + --arg repo "$TANGLED_REPO_REPO_DID" \ + --arg oid "$(git rev-parse HEAD)" \ + --arg name "$(basename "$file")" \ + --arg now "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --argjson blob "$blob" \ + '{repo: $repo, collection: "org.tangled.repo.artifact", record: { + "$type": "org.tangled.repo.artifact", + subject: {"$type": "org.tangled.git.oid", oid: $oid}, + name: $name, + createdAt: $now, + artifact: $blob + }}' | + curl -fsS "$xrpc/com.atproto.repo.createRecord" \ + -H "Authorization: Bearer $(token com.atproto.repo.createRecord)" \ + -H "Content-Type: application/json" \ + --data-binary @- +``` diff --git a/docs/src/content/docs/troubleshooting.md b/docs/src/content/docs/troubleshooting.md new file mode 100644 index 0000000000000000000000000000000000000000..df417d8fab94baa109dfc9aaf38bb44222ff2262 --- /dev/null +++ b/docs/src/content/docs/troubleshooting.md @@ -0,0 +1,100 @@ +--- +title: Troubleshooting +description: + Fixes for sign-in, changes that are slow to appear, rejected pushes, + unreachable knots, empty activity, and commits not linked to your + account. +--- + +Most problems on Tangled come from one of three places: your account +provider (sign-in), the delay before a change shows up in the web app, +or the knot that hosts the repository (pushes). Each section below names +the message you see and what to do. + +## Sign-in fails + +Tangled signs you in with atproto OAuth through your account provider. +If you run your own PDS and sign-in fails or times out: + +- Make sure the PDS clock is synced with NTP (`ntpd -qg`, then keep + `ntpd` running). OAuth tokens are rejected when clocks drift. +- Raise the Node network timeout on the PDS: + `NODE_OPTIONS="--network-family-autoselection-attempt-timeout=500"`. + +## Asked to sign in again + +Some actions need OAuth permissions that older sessions don't have, for +example webhooks, knot invites, creating an organization, reporting, or +checking a repository before import. Tangled detects the missing scope +and shows a **Sign in again** prompt. Signing in again grants the new +scope; nothing is lost. + +## "Saved, but the appview has not caught up yet" + +Your change was saved. It can take a few seconds for it to appear in the +web app; if it takes longer than Tangled is willing to wait, it shows +this message. Reload after a few seconds and the change appears. The +same applies to "Deleted, but the appview has not caught up yet." + +If changes never appear, something is wrong on Tangled's side; report it. + +## Push rejected + +The knot identifies you by matching the SSH key your client offers +against the SSH keys published in the account of everyone allowed to +push. + +```sh +$ git push +knot: this ssh key doesn't match any key published by the accounts that may push here. Authorized: … If your agent offers several keys, add -o IdentitiesOnly=yes so it offers your registered key. +``` + +- Add the key under **Settings → SSH Keys**. +- If your SSH agent holds several keys, it may offer the wrong one + first. Pin the right one: + + ```text + Host knot.example.com + IdentityFile ~/.ssh/id_ed25519 + IdentitiesOnly yes + ``` + +Other messages: + +| Message | Cause | +| ------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `knot: you aren't authorized to push to this repository.` | Your key matched, but you aren't the owner or a collaborator. See [Access and collaborators](/docs/guides/access/). | +| `knot: couldn't read the account records needed to check your ssh key, retry shortly` | The knot couldn't reach an account provider. Retry. | +| `knot: repository index is warming, retry shortly` | The knot just started. Retry. | +| `knot: push exceeds configured size limit` | The pack is larger than the knot allows. Split the push or ask the operator. | + +## Knot unreachable + +Browsing code mostly keeps working when a knot is down, because the web +app keeps its own copy for reading. Anything that has to reach the knot +fails: pushing and cloning from it, creating repositories, opening +issues, pull requests or comments (stored on the knot), merging, and +changing collaborators. Recent pushes may not show up until the knot is +back. Wait for the knot to return, or ask its operator; +[Run a knot](/docs/self-hosting/knot/) has operator-side checks. + +## Empty activity or commit heatmap + +Your profile only counts commits whose author email is one of your +**verified** emails or your DID. + +1. Check `git log --format='%ae' | sort -u` in a repository you pushed + to. +2. Add and verify that address under **Settings → Emails**, or set + `git config user.email did:plc:yourdid` for future commits. + +Counts refresh on their own once the email is verified; past commits +don't need rewriting. + +## Commits not linked to your account + +Same cause as above: the author email isn't verified on your account. +Verify it, or use your DID as the email. + +If you expected a "verified signature" badge: the current frontend +doesn't show one yet, see [Signing commits](/docs/guides/signing-commits/). diff --git a/docs/src/content/docs/upgrading/knot-2.md b/docs/src/content/docs/upgrading/knot-2.md new file mode 100644 index 0000000000000000000000000000000000000000..b1a3f3c7a3fa006bf1656785a57608ac15042b95 --- /dev/null +++ b/docs/src/content/docs/upgrading/knot-2.md @@ -0,0 +1,384 @@ +--- +title: Golang to Rust knot migration guide +description: + Move from the Golang knot to the Rust based knot with the offline knot-migrate + tool. +--- + +The Rust based knot is a second implementation of the knot concept. +It serves the same repositories under the same hostname and the same `did:web`, +and every clone URL and every `at://` reference will keep on working. +You'll move to it by running one offline tool while the knot is stopped. +Upgrading the old knot won't magically move you from the Golang knot to the Rust based knot. +This is intentional; we don't want to give anyone a nasty surprise +if we'd switch out implementations under you or accidentally +brick someone's janky-but-working setup by attempting to shift +a jenga block underneath them. + +Here's what a switchover involves: + +- **Rehearsals:** try as many as you like, with the old knot still serving. +- **Downtime:** from the `systemctl disable` in step 1 to `listening on` in step 5, + mostly the copying, depending on how large your repositories are and how fast the disk is. +- **Parity:** the same knot on the same hostname, + with the same owners, members, & collaborators. +- **Untouched stuff for reversion:** the old database, + and under the default-copy-mode, the old repositories too. + +What do you get for upgrading? + +- The Rust based knot doesn't have a SQLite database. Git is the only database, + so your members, collaborators, & repository owners will be stored in git itself. +- The Rust based knot will serve SSH in-process, so you no longer need `sshd`, + the `AuthorizedKeysCommand`, or the unix `git` account. +- The Rust based knot will read SSH keys live from each user's PDS. +- Repositories created from now on will stay SHA-1, matching what you already have. + Pass `--object-format sha256` to the migration if you'd rather modernize. + +The repo directories will come across whole, hooks and commit-graphs included. +The Rust based knot doesn't do repo hooks, +and it will rewrite commit-graphs on its own maintenance schedule. + +## Before we begin + +Get tools handy! Install both binaries: + +``` +git clone https://tangled.org/did:plc:j5hmlfdrwkvtxm7cjmu7j2is +cd core +nix build .#knot-rs && sudo install -m755 result/bin/knot-server /usr/local/bin/ +nix build .#knot-migrate && sudo install -m755 result/bin/knot-migrate /usr/local/bin/ +``` + +On NixOS, the `knot-rs` module will write the config file, the service unit, +& the state directory for you, +so read [Upgrading on NixOS](#upgrading-on-nixos) alongside this guide. +Run `./result/bin/knot-migrate` straight out of `nix build` instead of installing it. +The knot never runs the migrator, so the module leaves it off the system entirely. + +Make the dir that the Rust based knot will use, owned by the same user that the old knot runs as, +then write a master key for unsealing the per-repository signing keys: + +``` +sudo install -d -o git -g git /var/lib/knot +sudo install -d -m 700 /etc/knot +printf 'KNOT_MASTER_KEY=%s\n' "$(openssl rand -base64 32)" | sudo tee /etc/knot/knot.env > /dev/null +sudo chmod 600 /etc/knot/knot.env +``` + +That writes `/etc/knot/knot.env`, readable only by root: + +``` +KNOT_MASTER_KEY=Xm9x2t2m2WNSVJ+9v0Cq0ftFhwEuLW/8x0aeCcaHiWM= +``` + +**Back it up somewhere off the server.** +Seriously. +Seriously seriously. +The sealed key store is useless without it, +and the keys that it seals will sign every record that the migration writes. + +## Dress rehearsal, with old knot still serving requests + +`knot-migrate` will open its source database read-only, +so you can rehearse as often as you like without stopping anything. +Run it as root, since it'll read your `sshd` host key and the master key alongside the database: + +``` +sudo sh -c 'set -a; . /etc/knot/knot.env; set +a; exec knot-migrate \ + --source-db /home/git/knotserver.db \ + --env-file /home/git/.knot.env \ + --host-key /etc/ssh/ssh_host_ed25519_key \ + --plc-url https://plc.directory \ + --contact-email ops@oyster.cafe \ + --target /var/lib/knot \ + --dry-run' +``` + +The `--env-file` gives your hostname and repository scan path, +so you don't have to repeat them. +The contact email lands in the rendered config's `[contact]` section, +and comes out of `KNOT_CONTACT_EMAIL` in the env file when the flag stays off. +`--dry-run` won't touch the target, +though SQLite will still add `knotserver.db-shm` and `knotserver.db-wal` +beside a write-ahead-log database even to read it. +Rehearse until the exit code is 0 and the last block of the report reads +`scan path: writable`, `host key algorithm: ssh-ed25519`, +and `master key: KNOT_MASTER_KEY decodes to a usable key`. +Any line in the block with an error in it will stop the switchover. + +Mount each host directory at the same location inside the container if you'd rather run it that way, +such that every path in the command is still a host path. +Mount the target too, even though the rehearsal won't write there, +since it will measure the free space and the write permission on the target. +The `atcr.io` image runs as its own unprivileged `knot` user, +so pass `--user 0:0` to read the host key. +An image that you built yourself from `knot2/Containerfile` is distroless, +with its binaries in `/usr/local/bin` instead: + +``` +sudo sh -c 'set -a; . /etc/knot/knot.env; set +a; exec docker run --rm \ + --user 0:0 -v /home/git:/home/git -v /etc/ssh:/etc/ssh:ro \ + -v /var/lib/knot:/var/lib/knot -e KNOT_MASTER_KEY \ + --entrypoint /usr/bin/knot-migrate atcr.io/tangled.org/knot:2 \ + ' +``` + +The `:ro` on `/home/git` will work only while the old knot is still running. +Mount it writably from step 1 onward, rehearsals included, +even though the migration only ever reads your database. +SQLite will create an index file beside a write-ahead-log database to read it at all. +Under a `:ro` mount, with the knot stopped and its log checkpointed, it can't, +and `knot-migrate` errors out with `unable to open database file`. +`--consume-source` will move the repositories out of it, so mount it writably for that too. + +## Reading the report + +Then read what the report says! + +``` +knot owner: did:plc:akshay +members to grant: 3 +repos to adopt: 5 +collaborator grants: 4 + +casbin cross-check drift: +acl-only collaborator grants unioned in: 1 +did:plc:squid <- did:plc:boltless +table-only collaborator grants missing from acl: 1 +did:plc:octopus <- did:plc:dawn +repos with no acl owner marker where the owner regains push: 1 +did:plc:scallop + +skipped repos: 2 +did:plc:conch unrepresentable name "Test knot" +drops collaborator grant for did:plc:boltless +did:plc:tuna no git repository at the source path + +transfer mode: copy +the filesystem checks used /var/lib/knot, since the scan path doesn't exist yet +scan path: writable +room to copy: 89.1GiB free is enough for the 580.0KiB that adoption will copy +host key algorithm: ssh-ed25519 +master key: KNOT_MASTER_KEY decodes to a usable key + +we left the target alone. Re-run without --dry-run to migrate. +``` + +At the top is what's coming across, +`repos to adopt`, `members to grant`, and `collaborator grants`, +and they should match what you think you have. +Under them is every repository that `knot-migrate` will skip, each with its reason. +The last block is the preflight on the target, +where `transfer mode: copy` and `room to copy: ... is enough` +join the three lines you rehearsed against. + +**Skips.** If `repos to adopt` is lower than you expect, the skip list is where the rest went: +a name with a slash, a backslash, whitespace, a control character, `..`, +a bare `.`, or more than 100 bytes, and a record key with anything outside letters, +digits and `.-_:~`. Rename those on tangled.org and rehearse again, or accept losing them. + +**Owner drift.** `knot-migrate` will refuse the whole switchover +when the `acl` table and `repo_keys` are recorded with different owners for a repository. +The drift section prints both DIDs, one repository per line, as ` acl , repo_keys `. +Settle each repository in the old knot's database, +either by deleting the `acl` rows for the wrong owner or by correcting `repo_keys.owner_did`, +then rehearse again. +An extra `acl` owner marker beside the owner in `repo_keys` won't stop anything, +since `knot-migrate` reports it and discards it. + +**Room on disk.** Every disk measurement happens at `/repos`, +where your repositories end up. +Point `--target` at a directory whose parent already exists, +and mount or create the disk you mean to fill before you rehearse. +A rehearsal against your root filesystem measures your root filesystem. +Under `copy`, the default, your existing repositories stay where they are, +and `room to copy` counts a second copy of every repository it adopts. +`--consume-source` will move them instead, in seconds, with no extra room on disk, +and only where source and target are on one filesystem +and this process can write the old scan path. +Once they've moved, only a filesystem snapshot taken beforehand will bring them back, +even from a switchover that stopped halfway. +`knot-migrate` will refuse before it writes anything, +so your repos never end up split across two trees. + +> 🦪 **Note:** if you ran Secure Mode, each repo tree has a per-owner uid, +> so any one owner can read only their own. Root can read all of them. +> `knot-migrate` will refuse outright when it can't read a source path, +> and every repository is in the skip list with the error behind it. +> It will rehearse everything else first either way, +> and report that you need root alongside whatever else is outstanding. +> It skips a path that isn't there at all, as usual. +> `--skip-unreadable` will migrate the rest and leave those repositories on the old knot, +> which is what you want when root can't read them either, +> as with a dead disk or a stale network mount. +> `--consume-source` will empty the old scan path of every repository that it could read, +> so running it alongside `--skip-unreadable` leaves only the unreadable repositories on the old knot. + +## Doing the thing + +**Step 1.** Stop the knot and disable it, +so SQLite checkpoints its WAL and no restart of the machine can take port 5555 back from the Rust based knot. +Leave it installed though, for "Going back": + +``` +sudo systemctl disable --now knotserver +``` + +**Step 2.** Copy database through a read-only connection, so the log stays untouched, +and keep the copy until the switchover is settled. +`knot-migrate` will never write to the source database. +The copy is there for everything else that can go wrong on the day, +and "Going back" has a restore-command: + +``` +sudo sqlite3 "file:/home/git/knotserver.db?mode=ro" \ + ".backup /var/tmp/knotserver-cutover.db" +``` + +**Step 3.** Run the same command that you rehearsed, without `--dry-run`, +then give the tree to the user that the service runs as, since root wrote all of it: + +``` +sudo chown -R git:git /var/lib/knot +``` + +`knot-migrate` will write your repositories, a `config.toml`, the imported host key, +the sealed key store, and `repo-signing-keys.json`. +That last file is every repository's private key in the clear, +the same keys that your old database stored, +so move it somewhere safe and off the server once the migration is done. +Towards the end of the copy, install +[`knot.service`](https://tangled.org/did:plc:j5hmlfdrwkvtxm7cjmu7j2is/blob/master/systemd/knot.service) +to `/etc/systemd/system/`, so step 5 is only a start. + +**Step 4.** Give port 22 to the knot, +if you want SSH clone URLs to keep working without a port number in them. +The order to do that in is under "Ports, Taylor's version" in [Self-hosting a knot](/docs/knots/self-hosting/#ports-taylors-version-ha-ha-22-joke), +since getting it wrong will lock you out of your own server. +Two moves there are migration-only: +move `/etc/ssh/sshd_config.d/authorized_keys_command.conf` outside `sshd_config.d`, +keeping it for going back, +and uncomment `ssh_listen_addr` in `/var/lib/knot/config.toml` to set it to `"[::]:22"`. + +**Step 5.** Start it and follow the log, watching the index rebuild from the meta-repo, +until `index ready` and then `listening on`: + +``` +sudo systemctl enable --now knot +sudo journalctl -fu knot +``` + +The knot rebuilds the index before it takes a connection. +The [systemd section](/docs/knots/self-hosting/#running-under-systemd) covers editing the unit for a different user, +a different binary path, or an LFS store outside `/var/lib/knot`. + +Both knots serve HTTP on port 5555 by default, +so you don't have to touch a reverse proxy already aimed at the old knot. +Configure the proxy in the knot anyway. +The knot will rate-limit by the address that a request arrives from, +and every one of your users arrives from the proxy until you do. +In `/var/lib/knot/config.toml`, +set `xrpc.trusted_proxy_header` to the header that your proxy appends, +and `xrpc.trusted_proxies` to the address that it connects from. +The [TLS section](/docs/knots/self-hosting/#tls) spells both out at length. +An empty `trusted_proxies` leaves the knot honoring the header from every address, +so fill it in whenever anything but your proxy can connect to port 5555, +and keep 5555 off the public internet either way. + +## Upgrading on NixOS + +> 🦪 +> +> NixOS is not my forte. Please read this and absolutely second-guess me. +> I have of course tested this, but still, a boy gets paranoid. + +Steps 1 and 2 read the same, and the `knot-rs` module replaces the rest, +so configure it as the [knot self-hosting docs](/docs/knots/self-hosting/#install-on-nixos) describe. +Set `settings.git.object_format` to whatever you passed `--object-format`, +since the module defaults to `sha256` where the migration defaults to `sha1`. + +Run the migration before your first rebuild. +A knot that starts on an empty state directory won't serve any repositories, +and it will generate a host key of its own, +which `knot-migrate` will then refuse to overwrite. +A rebuild starts the knot the moment the module is on, +so if you want the module in place before you migrate, add this line first: + +``` +systemd.services.knot-rs.wantedBy = lib.mkForce []; +``` + +The knot then stays stopped until you start it by hand. +Migrate, `chown`, run `systemctl start knot-rs`, and delete the line at your next rebuild. + +If the knot already started once, `--force-host-key` writes the imported key over the one it generated. +The fingerprint everybody cached under the old knot is the one that comes back, +so it only surprises whoever connected while the Rust based knot was serving a key of its own. +The `config.toml` that `knot-migrate` writes is only a record of what it chose, +because the file the knot reads is the module's. + +Then run the migration and hand the tree to the module's user, +`knot`, where the Go module used `git`, and rebuild. +The `--source-repos` and `--hostname` flags replace the `--env-file` above, +since the Go module configures the old knot through systemd and will never write an env file. +Append `--dry-run` to rehearse it first, as often as you like: + +``` +nix build .#knot-migrate +sudo sh -c 'set -a; . /etc/knot/knot.env; set +a; exec ./result/bin/knot-migrate \ + --source-db /home/git/knotserver.db \ + --source-repos /home/git \ + --hostname knot.example.com \ + --host-key /etc/ssh/ssh_host_ed25519_key \ + --plc-url https://plc.directory \ + --target /var/lib/knot' +sudo chown -R knot:knot /var/lib/knot +sudo nixos-rebuild switch +``` + +The source paths above are the Go module's defaults, +where `stateDir` is `/home/git` and `repo.scanPath` is the state directory itself, +so read your own out of `services.tangled.knot` if you moved either. +"Check it worked" and everything after it apply unchanged, +with `systemctl status knot-rs` in place of `knot`. + +## Checking that it worked + +``` +curl -s https://knot.example.com/xrpc/_health +curl -s https://knot.example.com/xrpc/org.tangled.knot.describeKnot +ssh-keyscan -t ed25519 your.knot.com +``` + +The health endpoint should answer `{"version":"knot 2.0.0"}`, +`describeKnot` should answer with your knot's `did:web`, +and the keyscan should match the fingerprint that your users already have. +Then clone a repository anonymously, and push to a repository that you own. + +> 🦪 +> +> The Rust based knot will take SSH keys from each user's PDS on an hour's lease, +> where the old knot kept a copy that could outlive the record. +> So when someone's push stops working: is their key record still in their PDS? +> Have they re-added the key in their Tangled settings, which writes it back? +> Have they waited out the lease since? +> Lower `keyfill.ttl_secs` if an hour is too long for you. + +## Going back in case of emergency + +If you ran the migration in its default copy mode, +your old data is exactly where it was. +Put `sshd` back on 22, move `authorized_keys_command.conf` back into `sshd_config.d`, +and start the old knot: + +``` +sudo systemctl disable --now knot +sudo systemctl enable --now knotserver +``` + +Restore `/var/tmp/knotserver-cutover.db` over `/home/git/knotserver.db` only if something wrote to the original, +which the migration itself never will. +If you used `--consume-source`, your repositories moved instead of being copied, +and only a filesystem snapshot taken before the migration +or your own little dir-reverse script would bring repos back. diff --git a/docs/src/content/docs/upgrading/knots-and-spindles.md b/docs/src/content/docs/upgrading/knots-and-spindles.md new file mode 100644 index 0000000000000000000000000000000000000000..972eb7ccdde8b035b453fc521c1ca975f2bc4fd4 --- /dev/null +++ b/docs/src/content/docs/upgrading/knots-and-spindles.md @@ -0,0 +1,188 @@ +--- +title: Upgrading knots and spindles +description: + Steps for upgrading a self-hosted knot or spindle across releases with + breaking changes. +--- + +Sometimes, non-backwards compatible changes are made to the +knot/spindle XRPC APIs. If you host a knot or a spindle, you +will need to follow this guide to upgrade. Typically, this +only requires you to deploy the newest version. + +This document is laid out in reverse-chronological order. +Newer migration guides are listed first, and older guides +are further down the page. + +## Upgrading to v1.20.0 + +This section assumes your knot already runs the Rust based +knot. If you're still on the Go knot, follow the [Golang to +Rust knot migration guide](/docs/upgrading/knot-2/) instead. + +This release moves repository data into the knot. Issues, pull requests, +comments, labels and collaborator grants are now stored on the knot, +rather than in each author's PDS. This release also makes use of the +newly stabilized `org.tangled.*` lexicons. + +To upgrade: + +- Set an operator contact email, this is now mandatory and is served by + `org.tangled.knot.describeKnot`: + * Set `contact.email` in your config + * or `KNOT_CONTACT_EMAIL` in the environment + * or `services.tangled.knot-rs.settings.contact.email` in the nix module +- The LFS store (path) is now mandatory, since we now store atproto + record attachments on the knot itself. Please choose/`mkdir` a good path + on disk for LFS/blobs/LOPs and set `lfs.store_path`/`KNOT_LFS_STORE_PATH` + in the environment you have. We recommend putting it alongside the git + repos path, like say `/var/lib/knot/lfs` beside `/var/lib/knot/repos`. + If you had already set up LFS, nothing needed from you. +- Update the event stream proxy: + * The event stream has moved from `/events` to `/xrpc/com.atproto.sync.subscribeRepos` + * Modify your reverse proxy setup to behave accordingly + + ```nginx + location = /xrpc/com.atproto.sync.subscribeRepos { + proxy_pass http://127.0.0.1:5555; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 1h; + proxy_buffering off; + } + ``` + +- You may also give the indexer a kick to get things + moving: + + ``` + curl -X POST https://index.tangled.network/xrpc/com.atproto.sync.requestCrawl \ + -H 'Content-Type: application/json' \ + -d '{"hostname":"knot.example.com"}' + ``` + +## Upgrading to v1.16.0-alpha + +Starting with v1.16.0-alpha, spindles own CI pipeline data +directly. The appview no longer stores pipeline runs or follows +spindle event streams for pipeline history. Instead, it asks the +configured spindle for pipeline lists, single pipeline details, +workflow logs, retries, and cancellations over XRPC. + +This means that existing pipeline logs / runs won't appear after +you upgrade. Existing pipeline history from the appview cannot be +automatically migrated. If you want to migrate your data, you can +reach out to us and we will send you an SQL file that'll add the data +into your spindle. + +- Upgrade to the latest tag (v1.16.0 or above) +- Head to the [spindle + dashboard](https://tangled.org/settings/spindles) and hit the + "retry" button to verify your spindle + +## Upgrading to v1.15.0-alpha + +With v1.15.0-alpha, a knot itself owns its members and +per-repo collaborators directly. Previously this data was sourced from +PDS records (`sh.tangled.knot.member` and `sh.tangled.repo.collaborator`) +that the appview and the knot both read off the firehose. +The knot is now the source of truth and serves them over XRPC instead: + +- `sh.tangled.knot.addMember`, `sh.tangled.knot.removeMember`, `sh.tangled.knot.listMembers` +- `sh.tangled.repo.addCollaborator`, `sh.tangled.repo.removeCollaborator`, `sh.tangled.repo.listCollaborators` + +Until your knot is upgraded, the appview keeps reading its +members and collaborators from the old firehose-sourced records. +Upgrade to move your knot onto knot-owned access control. + +- Upgrade to the latest tag (v1.15.0 or above) +- Head to the [knot dashboard](https://tangled.org/settings/knots) and + hit the "retry" button to verify your knot + +## Upgrading to v1.14.0-alpha + +Starting with v1.14.0-alpha, the fully knot uses the repoDID as its +canonical handle for repositories. This unlocks repository +renames from the appview UI and changes the wire format for +the following lexicons (`sh.tangled.repo.pull`, `sh.tangled.repo.collaborator`, +`sh.tangled.repo.issue`, `sh.tangled.git.refUpdate`). + +Knots that have not been upgraded may silently drop new push +events, pull requests, issues, and collaborator invites for +repositories they host until upgraded. So upgrade please!!! + +- Upgrade to the latest tag (v1.14.0 or above) +- Head to the [knot dashboard](https://tangled.org/settings/knots) and + hit the "retry" button to verify your knot + +## Upgrading to v1.13.0-alpha + +Starting with v1.13.0-alpha, every repository on a knot is +assigned a DID. This makes repositories stable across +renames and transfers. + +When you upgrade your knot to this version, the server will +automatically mint DIDs for all existing repositories on +startup. This is a one-time process and you may see +additional log output during the first boot as DIDs are +assigned. + +- Upgrade to the latest tag (v1.13.0 or above) +- Head to the [knot dashboard](https://tangled.org/settings/knots) and + hit the "retry" button to verify your knot + +## Upgrading from v1.8.x + +After v1.8.2, the HTTP API for knots and spindles has been +deprecated and replaced with XRPC. Repositories on outdated +knots will not be viewable from the appview. Upgrading is +straightforward however. + +For knots: + +- Upgrade to the latest tag (v1.9.0 or above) +- Head to the [knot dashboard](https://tangled.org/settings/knots) and + hit the "retry" button to verify your knot + +For spindles: + +- Upgrade to the latest tag (v1.9.0 or above) +- Head to the [spindle + dashboard](https://tangled.org/settings/spindles) and hit the + "retry" button to verify your spindle + +## Upgrading from v1.7.x + +After v1.7.0, knot secrets have been deprecated. You no +longer need a secret from the appview to run a knot. All +authorized commands to knots are managed via [Inter-Service +Authentication](https://atproto.com/specs/xrpc#inter-service-authentication-jwt). +Knots will be read-only until upgraded. + +Upgrading is quite easy, in essence: + +- `KNOT_SERVER_SECRET` is no more, you can remove this + environment variable entirely +- `KNOT_SERVER_OWNER` is now required on boot, set this to + your DID. You can find your DID in the + [settings](https://tangled.org/settings) page. +- Restart your knot once you have replaced the environment + variable +- Head to the [knot dashboard](https://tangled.org/settings/knots) and + hit the "retry" button to verify your knot. This simply + writes a `sh.tangled.knot` record to your PDS. + +If you use the nix module, simply bump the flake to the +latest revision, and change your config block like so: + +```diff + services.tangled.knot = { + enable = true; + server = { +- secretFile = /path/to/secret; ++ owner = "did:plc:foo"; + }; + }; +``` diff --git a/docs/src/content/docs/why-atproto.md b/docs/src/content/docs/why-atproto.md new file mode 100644 index 0000000000000000000000000000000000000000..02a272108ec9bdfbbd52464d430c254c29967506 --- /dev/null +++ b/docs/src/content/docs/why-atproto.md @@ -0,0 +1,30 @@ +--- +title: Why AT Protocol +description: + Why Tangled is built on the AT Protocol, and what that gives you over + centralized forges. +--- + +Git is decentralized, but the rest of code collaboration is not. Almost +all open source code lives on one platform, GitHub. When one company +holds all the code, it can fail or change the rules for everyone. Email, +git and IRC have lasted for many years. Central services do not. + +Code collaboration has always made use of two protocols, one for code +transfer and one for communication: + +- **Email flow:** git moves the code, email carries communication. +- **GitHub:** git moves the code, the GitHub website carries + communication. +- **Tangled:** git moves the code, the AT Protocol carries + communication. + +Tangled shares events between git servers, which we call "knots". You +can work on repositories on any server. You can fork across servers. You +can push to a repository on your own server and open a pull request on a +repository on a different server. + +AT Protocol carries the events around the code, like issues and pull +requests. It also carries the social parts: a timeline of events, +follows, and stars. It shares collaborator invites and SSH public keys. +The rest is plain git. diff --git a/docs/src/pages/404.astro b/docs/src/pages/404.astro new file mode 100644 index 0000000000000000000000000000000000000000..35020d87be5a1c65cc860d56c98fcba6be41fffd --- /dev/null +++ b/docs/src/pages/404.astro @@ -0,0 +1,17 @@ +--- +import StarlightPage from "@astrojs/starlight/components/StarlightPage.astro"; +import ButtonCard from "$lib/components/ui/ButtonCard.svelte"; +import Rocket from "~icons/lucide/rocket"; +import BookMarked from "~icons/lucide/book-marked"; +import GitPullRequest from "~icons/lucide/git-pull-request"; +--- + + +

+ The page you are looking for does not exist, or it may have moved. Try + one of these instead. +

+ +
+
+ diff --git a/docs/src/starlightRouteData.ts b/docs/src/starlightRouteData.ts new file mode 100644 index 0000000000000000000000000000000000000000..4277e4eade227a0b026776ecadc070ab2ab84d7d --- /dev/null +++ b/docs/src/starlightRouteData.ts @@ -0,0 +1,22 @@ +import { defineRouteMiddleware } from "@astrojs/starlight/route-data"; +import type { StarlightRouteData } from "@astrojs/starlight/route-data"; + +// starlight always seeds the table of contents with a link back to the page +// title, so `toc` is truthy even on pages with no headings and the "on this +// page" overview renders empty. drop it in that case. +const PAGE_TITLE_ID = "_top"; + +type TocItem = NonNullable["items"][number]; + +function hasSections(items: TocItem[]): boolean { + return items.some( + (item) => item.slug !== PAGE_TITLE_ID || hasSections(item.children), + ); +} + +export const onRequest = defineRouteMiddleware((context) => { + const route = context.locals.starlightRoute; + if (route.toc && !hasSections(route.toc.items)) { + route.toc = undefined; + } +}); diff --git a/docs/src/stubs/app-env.ts b/docs/src/stubs/app-env.ts new file mode 100644 index 0000000000000000000000000000000000000000..589e4ea71c6da9efcd86e5f38cf5209a74be86e3 --- /dev/null +++ b/docs/src/stubs/app-env.ts @@ -0,0 +1,6 @@ +// sveltekit's environment module, stubbed like app-paths.ts for the web +// components the docs render. only `browser` is read +export const browser = typeof window !== "undefined"; +export const dev = import.meta.env.DEV; +export const building = false; +export const version = ""; diff --git a/docs/src/stubs/app-paths.ts b/docs/src/stubs/app-paths.ts new file mode 100644 index 0000000000000000000000000000000000000000..a0a17fa8f9719e72c2bc2b67e1b211dd9256f32c --- /dev/null +++ b/docs/src/stubs/app-paths.ts @@ -0,0 +1,7 @@ +// the docs render the web app's components outside sveltekit, so its routing +// virtual module is stubbed here (see the alias in astro.config.mjs). only +// `resolve` is used, and only for hrefs, so returning the path is enough. +export const base = ""; +export const assets = ""; +export const resolve = (path: string): string => path; +export const asset = (path: string): string => path; diff --git a/docs/src/styles/custom.css b/docs/src/styles/custom.css new file mode 100644 index 0000000000000000000000000000000000000000..2c84c344e3bea1f968bed108426bea4cc1d5a4d5 --- /dev/null +++ b/docs/src/styles/custom.css @@ -0,0 +1,117 @@ +/* + * starlight theme mapped onto the web app's tokens, which switch with the + * theme themselves, so one mapping serves light and dark. the light selector + * matches starlight-tailwind's, which would otherwise win on specificity + */ +:root, +:root[data-theme="light"] { + --sl-font: var(--font-sans); + --sl-font-mono: var(--font-mono); + + /* the top bar and sidebar sit directly on the page background */ + --sl-color-bg-nav: var(--sl-color-black); + --sl-color-bg-sidebar: var(--sl-color-black); + + --sl-color-white: var(--color-foreground-default); + --sl-color-gray-1: var(--color-foreground-default); + --sl-color-gray-2: var(--color-foreground-muted); + --sl-color-gray-3: var(--color-foreground-subtle); + --sl-color-gray-4: var(--color-border-strong); + --sl-color-gray-5: var(--color-border-default); + --sl-color-gray-6: var(--color-background-inset); + --sl-color-gray-7: var(--color-background-subtle); + /* the page background, like the app shell: white in light mode */ + --sl-color-black: var(--color-background-default); + + /* monochrome accents, like the web app's links */ + --sl-color-accent-low: var(--color-background-inset); + --sl-color-accent: var(--color-foreground-muted); + --sl-color-accent-high: var(--color-foreground-default); + + /* the web app's type scale for what markup.css doesn't cover: the page + title, table of contents, and other starlight furniture */ + --sl-text-body: 0.9375rem; /* paragraph-body */ + --sl-line-height: 1.6; /* 24px on the 15px body */ + --sl-text-body-sm: 0.8125rem; /* paragraph-regular */ + --sl-text-sm: 0.8125rem; + --sl-text-h1: 1.875rem; /* heading-2, the app's page titles */ +} + +/* in dark mode the page falls back to the canvas shade, like the app shell */ +:root[data-theme="dark"] { + --sl-color-black: var(--color-background-canvas); +} + +/* landing page */ +.feature-grid { + display: grid; + grid-template-columns: 1fr; + gap: 1rem; + margin-top: 2rem; +} + +@media (min-width: 48rem) { + .feature-grid { + grid-template-columns: repeat(2, 1fr); + } +} + +/* the cards are links, but shouldn't take the prose link's indigo hover color + (the icon inherits currentColor from the anchor) */ +.feature-grid > a, +.feature-grid > a:hover { + color: var(--color-foreground-default); +} + +/* ButtonCard is sized for app chrome (12px). on the landing page the cards + are content, so step them up like the welcome tiles in web: title one + step above the description, the description one step below the body */ +.feature-grid > a { + padding: var(--spacing-md); +} + +.feature-grid > a > div { + gap: 0.25rem; +} + +.feature-grid > a .font-medium { + font-size: 0.9375rem; /* typography-paragraph-body */ + line-height: 1.5rem; +} + +.feature-grid > a span:not(.font-medium) { + font-size: 0.8125rem; /* typography-paragraph-regular */ + line-height: 1.25rem; +} + +.feature-grid > a svg { + width: 1rem; + height: 1rem; +} + +/* + * pages without a table of contents drop `data-has-toc`, and starlight then + * lets the main pane fill the full width and centers its content, so prose + * jumps sideways between pages. size and align it as if the right column were + * still there, leaving the space reserved, so nothing shifts + */ +@media (min-width: 72rem) { + [data-has-sidebar]:not([data-has-toc]) .main-pane { + --sl-content-margin-inline: auto 0; + order: 1; + width: calc( + var(--sl-content-width) + + (100% - var(--sl-content-width) - var(--sl-sidebar-width)) / 2 + ); + } +} + +/* hero tweaks for the landing page */ +[data-has-hero] .hero { + padding-block: 0 3rem; +} + +[data-has-hero] .hero h2 { + font-size: clamp(1.75rem, 4.5vw, 2.5rem); + letter-spacing: -0.02em; +} diff --git a/docs/src/styles/diagrams.css b/docs/src/styles/diagrams.css new file mode 100644 index 0000000000000000000000000000000000000000..b3cc3681509ae9075e29de37e838196c1ac24230 --- /dev/null +++ b/docs/src/styles/diagrams.css @@ -0,0 +1,132 @@ +/* diagram svgs (src/assets/diagrams) carry only shapes and class names; + colors and type come from here so they follow the theme */ +.diagram { + margin-block: 1.5rem; + /* one hue per role, as in the Git Merge 2026 slides */ + --d-pds: #0ea5e9; + --d-relay: #eab308; + --d-index: #22c55e; + --d-client: #a855f7; + --d-knot: #f59e0b; + --d-spindle: #ec4899; +} + +.diagram svg { + display: block; + width: 100%; + height: auto; + max-width: 48rem; + margin-inline: auto; + font-family: var(--font-sans); +} + +.diagram figcaption { + margin-top: 0.5rem; + text-align: center; + font-size: var(--sl-text-body-sm); + color: var(--color-foreground-muted); +} + +/* a service or component */ +.diagram .box { + fill: var(--color-background-default); + stroke: var(--color-border-strong); + stroke-width: 1; +} + +/* the thing the diagram is about */ +.diagram .box-accent { + fill: var(--color-background-info-subtle); + stroke: var(--color-foreground-info); + stroke-width: 1; +} + +/* something outside tangled: a pds, plc, the user's machine */ +.diagram .box-external { + fill: var(--color-background-inset); + stroke: var(--color-border-default); + stroke-width: 1; + stroke-dasharray: 4 3; +} + +/* a dashed region grouping boxes, e.g. one operator's machines */ +.diagram .group { + fill: none; + stroke: var(--color-border-default); + stroke-width: 1; + stroke-dasharray: 6 4; +} + +/* role boxes: tinted fill, full-strength stroke */ +.diagram :is(.box-pds, .box-relay, .box-index, .box-client, .box-knot, .box-spindle) { + stroke-width: 1.5; + fill: color-mix(in oklab, var(--d-role) 16%, var(--color-background-default)); + stroke: var(--d-role); +} + +.diagram .box-pds { --d-role: var(--d-pds); } +.diagram .box-relay { --d-role: var(--d-relay); } +.diagram .box-index { --d-role: var(--d-index); } +.diagram .box-client { --d-role: var(--d-client); } +.diagram .box-knot { --d-role: var(--d-knot); } +.diagram .box-spindle { --d-role: var(--d-spindle); } + +/* something you could run instead: dashed, untinted */ +.diagram .yours { + fill: var(--color-background-default); + stroke-dasharray: 5 4; +} + +.diagram .box.yours { + stroke: var(--color-border-strong); +} + +/* a small tag naming a supporting service */ +.diagram .chip { + fill: var(--color-background-muted); + stroke: none; +} + +.diagram .chip-label { + fill: var(--color-foreground-muted); + font-size: 10.5px; +} + +.diagram .divider { + stroke: var(--color-border-default); + stroke-width: 1; +} + +.diagram .label { + fill: var(--color-foreground-default); + font-size: 13px; + font-weight: 590; +} + +.diagram .sublabel { + fill: var(--color-foreground-muted); + font-size: 11px; +} + +.diagram .group-label, +.diagram .edge-label { + fill: var(--color-foreground-subtle); + font-size: 11px; +} + +.diagram .edge { + fill: none; + stroke: var(--color-foreground-subtle); + stroke-width: 1.25; +} + +.diagram .edge-dashed { + fill: none; + stroke: var(--color-foreground-subtle); + stroke-width: 1.25; + stroke-dasharray: 4 3; +} + +.diagram .arrowhead { + fill: var(--color-foreground-subtle); +} diff --git a/docs/src/styles/global.css b/docs/src/styles/global.css new file mode 100644 index 0000000000000000000000000000000000000000..3f4f67ac4ce94dce4458c24a05154f25944a1322 --- /dev/null +++ b/docs/src/styles/global.css @@ -0,0 +1,57 @@ +/* starlight's recipe orders base before theme, but the web tokens put their + dark overrides in base: theme (variables only) goes first so they win */ +@layer theme, base, starlight, components, utilities; + +@import "@astrojs/starlight-tailwind"; +@import "tailwindcss/theme.css" layer(theme); +@import "tailwindcss/utilities.css" layer(utilities); + +/* the web app's tokens, dark variant and utilities. static, since the other + stylesheets here map starlight onto these variables directly */ +@import "../../../web/src/theme.css" theme(static); + +/* the web app's self-hosted fonts; the files land in public/fonts from + nix/pkgs/web-static-files.nix (docs.nix preBuild, or the dev shell) */ +@import "../../../web/src/fonts.css"; + +/* how tangled.org renders markdown. its rules sit in the components layer, + so they win over starlight's own content styles */ +@import "../../../web/src/markup.css"; + +@source "../../../web/src/lib/components/ui"; +@source "../../../web/src/lib/components/repo"; + +/* preflight resets the components assume. starlight-tailwind restores only the + border reset and the fonts, so the rest is put back here: links outside + content take their parent's color, and native controls -- buttons above all + -- otherwise paint from the UA, which shows up as grey chrome on components + the app styles from scratch */ +@layer base { + a { + color: inherit; + } + + button, + input, + select, + optgroup, + textarea, + ::file-selector-button { + box-sizing: border-box; + margin: 0; + padding: 0; + font: inherit; + font-feature-settings: inherit; + font-variation-settings: inherit; + letter-spacing: inherit; + color: inherit; + border-radius: 0; + background-color: transparent; + opacity: 1; + } + + button, + input:where([type="button"], [type="reset"], [type="submit"]) { + appearance: button; + } +} diff --git a/docs/src/styles/tangled.css b/docs/src/styles/tangled.css new file mode 100644 index 0000000000000000000000000000000000000000..bd18bbd51c3a9b9ed3556fbd73f54422efafc22a --- /dev/null +++ b/docs/src/styles/tangled.css @@ -0,0 +1,649 @@ +/* layout: no top bar on desktop, the sidebar carries logo and search, no + dividers anywhere, a narrow reading column */ +:root, +:root[data-theme="light"] { + --sl-content-width: 36rem; + --sl-sidebar-width: 18.5rem; + --sl-sidebar-pad-x: 3rem; + /* one top inset shared by sidebar, content and table of contents */ + --docs-top: 5rem; + --sl-color-hairline: transparent; + --sl-color-hairline-light: transparent; + --sl-color-hairline-shade: transparent; +} + +@media (min-width: 50rem) { + :root[data-has-sidebar] { + --sl-nav-height: 0rem; + } + + :root[data-has-sidebar] .header { + display: none; + } +} + +/* the landing page keeps a header, but it scrolls away with the page */ +:root:not([data-has-sidebar]) .header { + position: absolute; + background: transparent; +} + +.docs-header, +.docs-header-end { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.docs-header { + justify-content: space-between; + height: 100%; +} + +::selection { + background-color: var(--color-background-info-subtle-hover); + color: var(--color-foreground-default); +} + +/* sidebar */ +@media (min-width: 50rem) { + .sidebar-content { + padding-top: var(--docs-top); + /* more gutter on the left, with the freed space taken from the invisible + right side so the nav keeps its width and long labels don't wrap */ + padding-inline: 5rem 1rem; + gap: 1.75rem; + } + + :root[data-has-sidebar] main > .content-panel:first-of-type { + padding-top: var(--docs-top); + /* the title panel and the body panel each add 1.5rem of padding, which + leaves a 3rem gap under the title; trim it to the reference's 1rem */ + padding-bottom: 0.5rem; + } + + :root[data-has-sidebar] main > .content-panel:first-of-type + .content-panel { + padding-top: 0.5rem; + } + + .right-sidebar-panel { + padding-top: var(--docs-top); + } + + :root[data-has-sidebar] main > .content-panel:first-of-type h1 { + margin-top: 0; + } +} + +/* the "On this page" label is just a bold entry, the same size as the links */ +.right-sidebar-panel h2 { + font-size: var(--sl-text-xs); + font-weight: var(--font-weight-semibold); +} + +/* every ToC entry aligns with the label: no indent for nested headings */ +.right-sidebar-panel nav a { + padding-inline-start: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.sidebar-top { + align-items: center; + justify-content: space-between; +} + +.site-title { + color: var(--sl-color-white); +} + +/* search: a pill at the top of the sidebar that opens the search dialog */ +.sidebar-search button[data-open-modal] { + display: flex; + width: 100%; + height: 2rem; + max-width: none; + gap: 0.5rem; + padding-inline: 0.75rem; + border: 1px solid var(--color-border-default); + border-radius: 9999px; + background-color: var(--color-background-default); + color: var(--color-foreground-subtle); + font-size: 0.875rem; +} + +.sidebar-search button[data-open-modal]:hover { + border-color: var(--color-border-strong); + color: var(--color-foreground-muted); +} + +.sidebar-search button[data-open-modal] > span { + display: block; +} + +.sidebar-search button[data-open-modal] kbd { + margin-inline-start: auto; + background: transparent; + color: var(--color-foreground-subtle); +} + +/* the dialog takes the reading column's width, and its corners follow the + input's radius out by the frame's padding, so the two curves are concentric */ +site-search dialog { + margin-top: 12vh; + border: 1px solid var(--color-border-default); + border-radius: calc(var(--radius-md) + var(--spacing-xs)); + background-color: var(--color-background-default); + box-shadow: 0 24px 48px -12px var(--color-shadow-ambient-strong); +} + +@media (min-width: 50rem) { + site-search dialog { + max-width: var(--sl-content-width); + min-height: 0; + } + + site-search .dialog-frame { + padding: var(--spacing-xs); + } +} + +site-search .dialog-frame { + scrollbar-width: none; +} + +site-search .dialog-frame::-webkit-scrollbar { + display: none; +} + +site-search dialog::backdrop { + /* the web app's modal scrim: black in both themes, so dark mode dims rather + than lightens the page behind the dialog */ + background-color: rgb(0 0 0 / 0.4); + -webkit-backdrop-filter: none; + backdrop-filter: none; +} + +/* the search dialog's input and results are set at the prose size. pagefind + groups results by page; the page titles are hidden so the matches read as + one flat list of cards, which only show a background on hover. the input follows the app's list search + (Input with a search icon and a ghost clear button) */ +#starlight__search { + --pagefind-ui-border-radius: var(--radius-md); + --search-match: #fef08a; +} + +:root[data-theme="dark"] #starlight__search { + --search-match: rgb(250 204 21 / 0.3); +} + +#starlight__search .pagefind-ui__search-input { + height: 2rem; + padding-inline: 1.875rem 2rem; + border: 0; + background-color: var(--color-background-default); + box-shadow: inset 0 0 0 1px var(--color-shadow-ring); + font-size: var(--sl-text-body); + transition: box-shadow 150ms; +} + +#starlight__search input:focus { + outline: none; + box-shadow: + inset 0 0 0 1px var(--color-border-strong), + 0 0 0 1px var(--color-border-strong); +} + +#starlight__search .pagefind-ui__search-input::placeholder { + color: var(--color-foreground-placeholder); + opacity: 1; +} + +#starlight__search .pagefind-ui__form::before { + top: 0.5625rem; + left: 0.5rem; + width: 0.875rem; + height: 0.875rem; + background-color: var(--color-foreground-subtle); +} + +#starlight__search .pagefind-ui__search-clear { + top: 0.25rem; + right: 0.25rem; + width: 1.5rem; + height: 1.5rem; + border-radius: var(--radius-sm); +} + +#starlight__search .pagefind-ui__search-clear:hover { + background-color: var(--color-background-inset); +} + +#starlight__search .pagefind-ui__search-clear::before { + background-color: var(--color-foreground-muted); + mask-size: 0.875rem; + -webkit-mask-size: 0.875rem; +} + +#starlight__search .pagefind-ui__results-area { + margin-top: var(--spacing-xs); +} + +#starlight__search .pagefind-ui__message { + height: auto; + padding: 0 var(--spacing-sm) var(--spacing-xs); + color: var(--color-foreground-subtle); + font-size: var(--sl-text-sm); + font-weight: 400; +} + +#starlight__search :is(.pagefind-ui__results, .pagefind-ui__result-inner) { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 0.125rem; +} + +#starlight__search .pagefind-ui__results > * + * { + margin-top: 0; +} + +#starlight__search .pagefind-ui__result-inner > .pagefind-ui__result-title { + display: none; +} + +#starlight__search .pagefind-ui__result-nested::before { + content: none; +} + +#starlight__search .pagefind-ui__result-nested { + padding: 0.375rem var(--spacing-sm); + border-radius: var(--radius-md); + background-color: transparent; +} + +#starlight__search .pagefind-ui__result-nested .pagefind-ui__result-title { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: var(--sl-text-body); + line-height: var(--sl-line-height); +} + +#starlight__search .pagefind-ui__result-link[data-page]::before { + position: static; + content: attr(data-page) " / "; + color: var(--color-foreground-muted); +} + +#starlight__search .pagefind-ui__result-nested .pagefind-ui__result-link { + color: var(--color-foreground-default); + font-size: inherit; + font-weight: var(--font-weight-medium); +} + +#starlight__search .pagefind-ui__result-excerpt { + min-width: 0; + margin: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--color-foreground-muted); + font-size: var(--sl-text-body); + line-height: var(--sl-line-height); +} + +#starlight__search .pagefind-ui__result-excerpt[data-clipped]::before { + content: "… "; +} + +#starlight__search .pagefind-ui__result-excerpt[data-trails]::after { + content: " …"; +} + +#starlight__search mark { + border-radius: 0.125rem; + background-color: var(--search-match); + color: inherit; + font-weight: inherit; +} + +#starlight__search .pagefind-ui__result-nested:is(:hover, [data-active]) { + outline: none; + background-color: var(--color-background-inset); +} + +#starlight__search .pagefind-ui__result-title:not(:where(.pagefind-ui__result-nested *)):focus-within, +#starlight__search .pagefind-ui__result-nested:focus-within { + outline: none; +} + +#starlight__search .pagefind-ui__result-nested a:focus-visible { + outline: none; +} + +#starlight__search .pagefind-ui__result-nested:has(a:focus-visible) { + outline: 2px solid var(--color-border-focus); + outline-offset: 2px; +} + +/* results show a page of cards at a time (see Search.astro). while more can + load, the last shown card fades out, and scrolling it into view loads more + in place of pagefind's button */ +#starlight__search .pagefind-ui__result-nested[data-overflow], +#starlight__search .pagefind-ui__button { + display: none; +} + +#starlight__search .pagefind-ui__result-nested[data-faded] { + mask-image: linear-gradient(to bottom, rgb(0 0 0 / 0.5), transparent); + pointer-events: none; +} + +/* the sidebar scrolls, but without a visible scrollbar */ +.sidebar-pane { + scrollbar-width: none; +} + +.sidebar-pane::-webkit-scrollbar { + display: none; +} + +/* the mobile menu toggle is a ghost button */ +starlight-menu-button button { + background-color: transparent; + box-shadow: none; + border-radius: var(--radius-md); + color: var(--color-foreground-muted); +} + +starlight-menu-button button:hover, +starlight-menu-button[aria-expanded="true"] button { + background-color: var(--color-background-inset); +} + +starlight-menu-button button:focus-visible { + outline: 2px solid var(--color-border-focus); + outline-offset: 2px; +} + +/* on mobile the header reserves room for the fixed menu button, with a 0.5rem gap */ +@media (max-width: 49.999rem) { + :root[data-has-sidebar] .header { + padding-inline-end: calc(0.5rem + var(--sl-nav-pad-x) + var(--sl-menu-button-size)); + } +} + +/* the nav: one size throughout, bold group headings, whitespace between groups */ +.docs-nav { + font-size: 0.875rem; + line-height: 1.2; +} + +.docs-nav ul { + list-style: none; + margin: 0; + padding: 0; +} + +.docs-nav > * + * { + margin-top: 1.75rem; +} + +.docs-nav h2, +.docs-nav h3 { + margin: 0 0 0.25rem; + font-size: inherit; + line-height: inherit; + color: var(--color-foreground-default); +} + +.docs-nav h2 { + font-weight: var(--font-weight-semibold); +} + +.docs-nav h3 { + font-weight: var(--font-weight-medium); +} + +.docs-nav .subgroup { + margin-block: 0.75rem; +} + +.docs-nav a { + display: block; + padding-block: 0.25rem; + color: var(--color-foreground-muted); + text-decoration: none; +} + +.docs-nav a:hover { + color: var(--color-foreground-default); +} + +.docs-nav a[aria-current="page"] { + color: var(--color-foreground-default); + font-weight: var(--font-weight-medium); +} + +/* content: markup.css does the typography. what's left is fitting it around + the elements starlight adds */ + +/* body text sits a step below the brightest foreground; headings and links + stay bright. markup.css sets the container colour and each heading's, so a + rule on the container re-tints the prose without dimming the headings */ +.markup { + color: var(--color-foreground-muted); +} + +/* tables take the web app's card/panel radius (--radius-md). the outer border + sits on the table (a block scroll container), so its corners follow the + radius cleanly: the cells keep only their inner rules and no border is + clipped at the corners */ +.markup table { + border: 1px solid var(--color-border-default); + border-radius: var(--radius-md); + border-collapse: separate; + border-spacing: 0; +} + +/* code blocks are the same kind of panel, so match the table radius. + markup.css gives every `pre` the smaller inline radius, which also wins over + expressive-code's own border radius, so override it here */ +.markup pre { + border-radius: var(--radius-md); +} + +/* keyboard keys follow kumo-svelte's prose `kbd`: a flat keycap -- a hairline + border on the base surface, no inset shadow -- set in mono at 0.875em */ +.markup kbd { + padding: 0.125rem 0.375rem; + border: 1px solid var(--color-border-default); + border-radius: 0.25rem; + background-color: var(--color-background-default); + color: var(--color-foreground-default); + font-family: var(--font-mono); + font-size: 0.875em; + font-weight: 400; + line-height: inherit; + box-shadow: none; +} + +.markup :is(th, td) { + border: 0; + border-inline-end: 1px solid var(--color-border-default); + border-block-end: 1px solid var(--color-border-default); +} + +/* the last column and row drop their inner rules, the table border closes them */ +.markup tr > :last-child { + border-inline-end: 0; +} + +.markup tbody tr:last-child > * { + border-block-end: 0; +} + +.markup :is(p, li, dd, blockquote) { + overflow-wrap: break-word; +} + +.markup :is(h1, h2, h3, h4, h5, h6) { + text-wrap: balance; +} + +/* starlight wraps each heading with its anchor link. the rule under a + readme's h1 and h2 goes on the wrapper, so it spans the column */ +.markup .sl-heading-wrapper { + margin-top: var(--spacing-xl); + margin-bottom: var(--spacing-sm); +} + +.markup .sl-heading-wrapper > :is(h1, h2, h3, h4, h5, h6) { + margin: 0; + border: 0; + padding: 0; +} + +.markup .sl-heading-wrapper:is(.level-h1, .level-h2) { + border-bottom: 1px solid var(--color-border-default); + padding-bottom: var(--spacing-xs); +} + +.markup > :first-child, +.markup > .sl-heading-wrapper:first-child { + margin-top: 0; +} + +/* keep the shared markup styles' always-on underline: the text stays the + brightest foreground, the underline is muted until hover, then brightens. + this drops the indigo link-hover colour (web/src/markup.css). the heading + anchor keeps its `#` */ +.markup a:not(.sl-anchor-link) { + color: var(--color-foreground-default); + text-decoration-color: var(--color-foreground-muted); +} + +.markup a:not(.sl-anchor-link):hover { + color: var(--color-foreground-default); + text-decoration-color: var(--color-foreground-default); +} + +/* the anchor link, shown as tangled.org's `#` on hover */ +.markup .sl-anchor-link { + margin-inline-start: var(--spacing-xs); + color: var(--color-foreground-placeholder); + text-decoration: none; + opacity: 0; + transition: opacity 150ms; +} + +.markup .sl-heading-wrapper:hover .sl-anchor-link, +.markup .sl-anchor-link:focus-visible { + opacity: 1; +} + +.markup .sl-anchor-link svg { + display: none; +} + +.markup .sl-anchor-link::before { + content: "#"; +} + +/* asides, styled like tangled.org's github alerts */ +.markup .starlight-aside { + margin-block: var(--spacing-sm); + padding: 0 0 0 var(--spacing-md); + border: 0; + border-inline-start: 2px solid var(--color-border-default); + background: transparent; +} + +.markup .starlight-aside__title { + gap: 0; + font-size: inherit; + font-weight: var(--font-weight-semibold); +} + +.markup .starlight-aside__icon { + display: none; +} + +.markup .starlight-aside--note, +.markup .starlight-aside--tip { + border-color: var(--color-background-info-emphasis); +} + +.markup .starlight-aside--note .starlight-aside__title { + color: var(--color-foreground-info); +} + +.markup .starlight-aside--tip { + border-color: var(--color-background-success-emphasis); +} + +.markup .starlight-aside--tip .starlight-aside__title { + color: var(--color-foreground-success); +} + +.markup .starlight-aside--caution { + border-color: var(--color-background-warning-emphasis); +} + +.markup .starlight-aside--caution .starlight-aside__title { + color: var(--color-foreground-warning); +} + +.markup .starlight-aside--danger { + border-color: var(--color-background-danger-emphasis); +} + +.markup .starlight-aside--danger .starlight-aside__title { + color: var(--color-foreground-danger); +} + +/* no next/previous links at the foot of the page */ +.pagination-links { + display: none; +} + +/* expressive code sets its own size on pre; markup.css sizes code at 0.875em of + pre, so pre takes the body size and code lands on tangled.org's 13px */ +.markup .expressive-code pre { + font-size: 0.9375rem; +} + +/* no copy button on code blocks */ +.expressive-code .copy { + display: none; +} + +/* starlight sizes the heading wrapper with its own scale, which the anchor link + inherits; match each level to the heading markup.css gives it */ +.markup .sl-heading-wrapper.level-h1 { + font-size: 1.5rem; +} + +.markup .sl-heading-wrapper.level-h2 { + font-size: 1.25rem; +} + +.markup .sl-heading-wrapper.level-h3 { + font-size: 1rem; +} + +.markup .sl-heading-wrapper:is(.level-h4, .level-h5, .level-h6) { + font-size: 0.8125rem; +} + +/* starlight lets inline code break anywhere; in a table that splits short + identifiers mid-word, so keep them whole and let the table scroll instead */ +.markup :is(td, th) code { + overflow-wrap: normal; + white-space: nowrap; +} + +body { + text-wrap-style: pretty; +} diff --git a/docs/tsconfig.json b/docs/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..8bf91d3bb997dfb0ea4a592b849545656f0fcebe --- /dev/null +++ b/docs/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "astro/tsconfigs/strict", + "include": [".astro/types.d.ts", "**/*"], + "exclude": ["dist"] +} diff --git a/docs/wrangler.jsonc b/docs/wrangler.jsonc new file mode 100644 index 0000000000000000000000000000000000000000..f248d324d810892be6b291dcd5ec44a890eef567 --- /dev/null +++ b/docs/wrangler.jsonc @@ -0,0 +1,21 @@ +{ + "name": "tangled-docs", + "compatibility_date": "2026-10-27", + "workers_dev": true, + "preview_urls": false, + "observability": { + "enabled": true + }, + // assets only, no worker script. astro builds into dist/docs (base = /docs), + // so asset paths line up with tangled.org/docs/* request paths + "assets": { + "directory": "./dist", + "not_found_handling": "404-page" + }, + "routes": [ + { + "pattern": "tangled.org/docs*", + "zone_name": "tangled.org" + } + ] +} diff --git a/e2e/.gitignore b/e2e/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..4f70632a8c64230e639408fba63d14e18b1f72c4 --- /dev/null +++ b/e2e/.gitignore @@ -0,0 +1,4 @@ +node_modules +test-results +playwright-report +.auth diff --git a/e2e/.prettierrc.json b/e2e/.prettierrc.json new file mode 100644 index 0000000000000000000000000000000000000000..56c921750773baa4b7372afb459613126b23d0fb --- /dev/null +++ b/e2e/.prettierrc.json @@ -0,0 +1,6 @@ +{ + "useTabs": true, + "singleQuote": false, + "trailingComma": "none", + "printWidth": 100 +} diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1942a694f4f9134bab5c04c00a3391aa7fdb32f3 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,83 @@ +# e2e + +browser tests that follow a write through every service. a flow drives the web app in a +real browser against the localinfra stack, then checks that every service the write passed +through (pds, knot, hydrant, bobbin, spindle) agrees with what the page showed. + +```sh +pnpm e2e # the whole suite, bringing the stack up if it isn't +pnpm e2e specs/social.spec.ts # one spec +pnpm e2e --report=report.html # also write an html report +pnpm e2e --har=run.har # also write the requests as a har +pnpm e2e --login=ui # every new account logs in through the login page +pnpm e2e:up # start the stack without running anything +pnpm e2e:sweep # cleans up after a crashed run +``` + +with 12 workers its about 4-5 minutes for the whole suite on an m5 macbook. `--workers=6` +or lower might be useful for less RAM / CPU (it uses a ton of cpu in a vm like colima anyway). + +the html report is drawn with the web's own components, so it needs `pnpm install` in `web/` +(or `--web-modules=`). + +the web runs as a prod build (`localinfra/docker-compose.web-build.yml` for compose, the web +service's `setup` for pc), made when the web starts. `pnpm e2e` and `pnpm e2e:up` rebuild it by +themselves when anything under `web/` changed. + +you need the localinfra stack, its dev ca in `localinfra/certs`, and `pnpm install` here. +the stack is either compose or the container-free one in `localinfra/pc`. `E2E_STACK=compose` or +`E2E_STACK=pc` picks, otherwise it is whichever is running, otherwise compose when docker or podman +is installed. for pc run `pnpm e2e:up` (the nix app brings go and cargo), it +starts the stack in the background, and `localinfra/pc/bin/pc down`, from the repo root, stops it. +service urls come from `E2E_*_URL` (see `harness/config.ts`) and default to the localinfra +hostnames. the `deliberi` hop asks deliberi's dev only `/xrpc/_ingested` route, so deliberi has to run with +`DELIBERI_DEV=true` (both localinfra stacks set it). making accounts needs the pds admin password, read from `localinfra/pds.env` or +`E2E_PDS_ADMIN_PASSWORD`. + +## outages + +`outages/` holds flows that stop and start stack processes, so they only exist on the pc stack +when asked for, because they would break flows running beside them: +`E2E_OUTAGES=1 pnpm exec playwright test --project=outages`. they stop spindle through process-compose's socket and +rewind its saved jetstream cursor (`harness/processes.ts`, `E2E_PC_STATE` is the stack's state +dir, `run` sets it). a failed flow starts spindle again in its afterEach. + +## cleanup + +every flow removes its own accounts and repos. `pnpm e2e:sweep` cleans up what the tests +add in case a crash or something occurs. + +## layout + +``` +run checks the stack, rebuilds the web if it changed, starts playwright +up brings the stack up, compose or pc +harness/ + flow.ts what a spec is written with, flow("...").open(...).click(...) + vocabulary.ts the things a spec points at, like buttons, pages and records + shared.ts steps and checks that more than one spec uses + run.ts who acts, which repo, what each step does + services.ts asking pds, knot, etc. what they have after a write + attestations.ts claiming the attestation the page writes to the pds after a knot write + cobgit.ts looking inside the knot's git for what a write left there + versions.ts every version the knot has of a record + stream.ts watching hydrant's event stream for a write to show up + firehose.ts watching the knot and pds for the action and cid of a write + frames.ts bounded frame history and waking checks when a frame arrives + api.ts writes from node, accounts, repos, tickets, pulls + git.ts pushing and cloning over ssh and https like a real git client + net.ts tracing every http request node makes + tabs.ts one browser window per account, waiting for a page to go quiet + signin.ts signs in, through the login page or with a session made in node + auth.setup.ts logs the four seeded accounts in once so flows start signed in + config.ts the service urls and the seeded accounts + global.ts cleans up what a killed run left, starts the shared browser + chores.ts a list of what a run has to cleanup + record.ts remembers what each step did, for the report + lexicons.ts record types for checking what setup wrote + reporter.ts the terminal output + report/ the html report +specs/ one file per product area +tools/sweep.spec.ts deletes leaked accounts, only runs with `pnpm e2e:sweep` +tools/tidy.spec.ts a stopped worker starts this to finish its cleanup +``` diff --git a/e2e/harness/api.ts b/e2e/harness/api.ts new file mode 100644 index 0000000000000000000000000000000000000000..37772f523439b29ad6bb0d97a6d9e77c4e2121a0 --- /dev/null +++ b/e2e/harness/api.ts @@ -0,0 +1,491 @@ +import type { Did, ResourceUri } from "@atcute/lexicons/syntax"; + +import { config, handleOf, seeded, type Account } from "./config"; +import type { Comment, Declaration, LabelDefinition, Profile, Ticket } from "./lexicons"; +import { + get, + ok, + partsOf, + recordUrl, + request, + retry429, + selfUri, + stamp, + xrpc, + type Written +} from "./net"; + +export interface RepoOptions { + spindle?: string; + forkOf?: string; + importFrom?: string; +} + +const sessions = new Map>(); + +export const sessionOf = (who: Account) => { + if (!sessions.has(who.did)) { + const login = request(`${config.pdsUrl}/xrpc/com.atproto.server.createSession`, { + method: "POST", + json: { identifier: who.handle, password: config.password } + }).then(({ status, body }) => { + if (status !== 200) + throw new Error( + `pds refused ${who.name}'s login: ${status} ${JSON.stringify(body)}` + ); + return body.accessJwt as string; + }); + // otherwise a refused login stays cached and every call after it fails + login.catch(() => sessions.delete(who.did)); + sessions.set(who.did, login); + } + return sessions.get(who.did)!; +}; + +// 401 on login means the account is gone, the other texts are not-found answers +const GONE = /not ?found|does not exist|no .* to erase|pds refused .*login: 401/i; +export const alreadyGone = (cause: unknown) => GONE.test(String(cause)); + +export const getOk = async (url: string) => ok(url, await get(url)); + +export const tokenFor = async (who: Account, aud: string, lxm: string): Promise => { + const auth = await request( + xrpc(config.pdsUrl, "com.atproto.server.getServiceAuth", { aud, lxm }), + { token: await sessionOf(who) } + ); + return ok("pds getServiceAuth", auth).token; +}; + +// calls an xrpc on a service using service auth +export const serviceDo = async ( + who: Account, + base: string, + aud: string, + nsid: string, + { json, params }: { json?: object; params?: Record } = {} +) => + request(xrpc(base, nsid, params), { + method: json === undefined ? "GET" : "POST", + token: await tokenFor(who, aud, nsid), + json + }); + +export const spindleGet = (who: Account, nsid: string, params: Record) => + serviceDo(who, config.spindleUrl, config.spindleDid, nsid, { params }); +export const spindlePost = (who: Account, nsid: string, json: object) => + serviceDo(who, config.spindleUrl, config.spindleDid, nsid, { json }); + +const QUOTA = "org.tangled.temp.spindle.quota"; +const spindleAdmin = seeded.alice; +export interface QuotaUsage { + scope: "user" | "repo"; + did: string; + resource: string; + used: number; +} +export const quotas = { + set: async (did: string, resource: string, limit: number) => + ok( + `spindle ${QUOTA}.set`, + await spindlePost(spindleAdmin, `${QUOTA}.set`, { did, resource, limit }) + ), + unset: async (did: string, resource: string) => + ok( + `spindle ${QUOTA}.unset`, + await spindlePost(spindleAdmin, `${QUOTA}.unset`, { did, resource }) + ), + usage: async (did: string) => + (ok(`spindle ${QUOTA}.usage`, await spindleGet(spindleAdmin, `${QUOTA}.usage`, { did })) + .usages as QuotaUsage[]) ?? [] +}; + +// the knot takes a service token once, so every retry mints a new one +const serviceReply = (who: Account, aud: string, nsid: string, input: object) => + retry429(() => serviceDo(who, config.knotUrl, aud, nsid, { json: input })); + +export const asKnot = (who: Account, nsid: string, input: object) => + serviceReply(who, config.knotDid, nsid, input); + +const serviceCall = async (who: Account, aud: string, nsid: string, input: object) => + ok(`knot ${nsid}`, await serviceReply(who, aud, nsid, input)); + +const knotCall = (who: Account, repo: string, nsid: string, input: object) => + serviceCall(who, repo, nsid, { repo, ...input }); + +// unlike knotCall this doesn't throw on a refusal, the caller checks the status +export const knotReply = (who: Account, repo: string, nsid: string, input: object) => + serviceReply(who, repo, nsid, { repo, ...input }); + +const IMPORT_STATUS = "org.tangled.repo.getImportStatus"; +// the token is addressed to the repo, like web's, not to the knot +export const importStatus = async (repo: string, who?: Account) => + request( + xrpc(config.knotUrl, IMPORT_STATUS, { repo }), + who ? { token: await tokenFor(who, repo, IMPORT_STATUS) } : {} + ); + +// knotmirror forwards the token to the knot's receive-pack, so it's minted for that +export const mirrorMerge = async ( + who: Account, + repo: string, + branch: string, + target: string, + source: string +) => + request(xrpc(config.bobbinUrl, "sh.tangled.git.mergeCommit"), { + method: "POST", + token: await tokenFor(who, config.knotDid, "org.tangled.git.receivePack"), + json: { + target: { $type: "sh.tangled.git.mergeCommit#target", repo, branch, commit: target }, + source: { $type: "sh.tangled.git.mergeCommit#source", repo, commit: source }, + style: "rebase" + } + }); + +export type Policy = "anyone" | "collaborators" | "members"; +const MANIFEST = "org.tangled.repo.manifest"; +// the policy has no endpoint of its own because it lives on the manifest +export const setPolicy = async (who: Account, repo: string, policy: Policy) => { + const manifest = await knotRecord(selfUri(repo, MANIFEST)); + if (!manifest) throw new Error(`the knot has no manifest for ${repo}`); + await knotCall(who, repo, "com.atproto.repo.putRecord", { + collection: MANIFEST, + rkey: "self", + record: { ...manifest.value, contributionPolicy: policy }, + swapRecord: manifest.cid + }); +}; + +export const pdsCall = async (who: Account, nsid: string, input: object) => + ok( + `pds ${nsid}`, + await retry429(async () => + request(`${config.pdsUrl}/xrpc/${nsid}`, { + method: "POST", + token: await sessionOf(who), + json: { repo: who.did, ...input } + }) + ) + ); + +const admin = async (nsid: string, json: object) => { + if (!config.pdsAdminPassword) + throw new Error("set E2E_PDS_ADMIN_PASSWORD, localinfra/pds.env has none to read"); + return ok( + `pds ${nsid}`, + await request(`${config.pdsUrl}/xrpc/${nsid}`, { + method: "POST", + basic: `admin:${config.pdsAdminPassword}`, + json + }) + ); +}; + +export const preauthorize = (did: string, requestUri: string) => + request(`${config.pdsUrl}/_preauthorize`, { + method: "POST", + basic: `admin:${config.pdsAdminPassword}`, + json: { did, requestUri } + }); + +const slug = (text: string) => text.toLowerCase().replace(/[^a-z0-9]+/g, "-"); + +// no other test shares an account's stars, tickets or rate limits +// membership is a record in the knot's own repo, keyed by the member's did +const KNOT_MEMBER = "org.tangled.knot.member"; + +export const accounts = { + create: async (label: string): Promise => { + const name = `${slug(label)}-${stamp()}`; + const handle = handleOf(name); + const { code } = await admin("com.atproto.server.createInviteCode", { useCount: 1 }); + const made = ok( + "pds createAccount", + await request(`${config.pdsUrl}/xrpc/com.atproto.server.createAccount`, { + method: "POST", + json: { + email: `${name}@pds.tngl.boltless.dev`, + handle, + password: config.password, + inviteCode: code + } + }) + ); + const account = { name, handle, did: made.did as string }; + // hydrant only indexes an account once it writes a record in a signal collection + try { + await pdsCall(account, "com.atproto.repo.putRecord", { + collection: "org.tangled.actor.profile", + rkey: "self", + record: { $type: "org.tangled.actor.profile" } satisfies Profile + }); + } catch (cause) { + // the caller never gets the account back, so nothing else would clean it up + await admin("com.atproto.admin.deleteAccount", { did: account.did }).catch(() => {}); + throw cause; + } + return account; + }, + // the knot is closed to anyone it hasn't been told about + makeMember: (who: Account) => + serviceCall(seeded.alice, config.knotDid, "com.atproto.repo.createRecord", { + repo: config.knotDid, + collection: KNOT_MEMBER, + rkey: who.did, + record: { $type: KNOT_MEMBER, createdAt: new Date().toISOString() } + }), + removeMember: (who: Account) => + serviceCall(seeded.alice, config.knotDid, "com.atproto.repo.deleteRecord", { + repo: config.knotDid, + collection: KNOT_MEMBER, + rkey: who.did + }), + remove: async (who: Account) => { + await admin("com.atproto.admin.deleteAccount", { did: who.did }); + sessions.delete(who.did); + } +}; + +// the knot answers ok for a repo it doesn't know, so a half-made one is safe to delete +const deleteKnotRepo = (who: Account, repo: { did: string }) => + serviceCall(who, config.knotDid, "org.tangled.repo.deleteRepo", { repo: repo.did }); + +// the knot refuses to delete a repo while its declaration is still on the owner's pds, so that goes first +const removeRepo = async (who: Account, repo: { did: string; name: string }) => { + await pdsCall(who, "com.atproto.repo.deleteRecord", { + collection: "org.tangled.repo.declaration", + rkey: repo.name + }); + await deleteKnotRepo(who, repo); +}; + +export const repos = { + create: async (who: Account, repoName: string, options: RepoOptions = {}) => { + const createdAt = new Date().toISOString(); + const made = await serviceCall(who, config.knotDid, "org.tangled.repo.createRepo", { + name: repoName, + createdAt, + ...(options.spindle ? { spindle: options.spindle } : {}), + ...(options.forkOf + ? { source: { $type: "org.tangled.repo.defs#sourceTangled", did: options.forkOf } } + : {}), + ...(options.importFrom + ? { + source: { + $type: "org.tangled.repo.defs#sourceExternal", + uri: options.importFrom + } + } + : {}) + }); + const declaration = await pdsCall(who, "com.atproto.repo.createRecord", { + collection: "org.tangled.repo.declaration", + rkey: repoName, + record: { + $type: "org.tangled.repo.declaration", + repo: made.did, + createdAt + } satisfies Declaration + }).catch(async (cause) => { + // the caller never gets this repo, so nothing else would remove the knot's half + await deleteKnotRepo(who, { did: made.did }).catch(() => {}); + throw cause; + }); + return { + did: made.did as Did, + name: repoName, + declaration: { uri: declaration.uri as string, cid: declaration.cid as string } + }; + }, + remove: removeRepo +}; + +const TICKET = "org.tangled.track.ticket"; + +// the sites worker keeps claims and site configs itself, behind service auth +const sitesDo = async ( + who: Account, + nsid: string, + input: { json?: object; params?: Record } +) => ok(`sites ${nsid}`, await serviceDo(who, config.sitesUrl, config.sitesDid, nsid, input)); + +export const sites = { + claim: (who: Account, domain: string) => + sitesDo(who, "org.tangled.temp.site.claimDomain", { json: { domain } }), + config: async (who: Account, repoDid: string) => + (await sitesDo(who, "org.tangled.temp.repo.getSiteConfig", { params: { repoDid } })).config, + history: async (who: Account, repoDid: string) => + (await sitesDo(who, "org.tangled.temp.repo.getDeployHistory", { params: { repoDid } })) + .deploys as { status: string; error: string }[], + // returns the reply instead of throwing on a refusal + saveReply: (who: Account, input: object) => + serviceDo(who, config.sitesUrl, config.sitesDid, "org.tangled.temp.repo.updateSiteConfig", { + json: input + }) +}; + +export const decodeRefs = (value: { refs: { $bytes: string } }) => + Buffer.from(value.refs.$bytes, "base64") + .toString() + .split("\n") + .filter(Boolean) + .map((line) => line.split(" ") as [sha: string, ref: string]); + +// a missing record or repo is a 404 or a RecordNotFound/RepoNotFound body +export const knotRecord = async (uri: string) => { + const { status, body } = await get(recordUrl(config.knotUrl, uri)); + if (status === 404 || ["RecordNotFound", "RepoNotFound"].includes(body?.error)) + return undefined; + return ok("knot getRecord", { status, body }); +}; + +export const knotRecords = async (repoDid: string, collection: string) => + ( + await getOk( + xrpc(config.knotUrl, "com.atproto.repo.listRecords", { + repo: repoDid, + collection, + limit: "100" + }) + ) + ).records as { uri: string; cid: string; value: any }[]; + +export const refsOf = async ( + uri: string +): Promise<{ cid: string; named: ReturnType } | undefined> => { + const record = await knotRecord(uri); + return record && { cid: record.cid, named: decodeRefs(record.value) }; +}; + +export const knotRecordFor = async (uri: string, doing: string) => { + const found = await knotRecord(uri); + if (!found) throw new Error(`the knot has no ${uri} to ${doing}`); + return found; +}; + +// the knot only takes an edit that names the version it was read at +export const editCob = async ( + who: Account, + uri: string, + doing: string, + change: (stored: Record) => Record +) => { + const { repo, collection, rkey } = partsOf(uri); + const current = await knotRecordFor(uri, doing); + // the knot sets x-tngl-editor itself, so it can't be sent back + const { "x-tngl-editor": _editor, ...stored } = current.value; + return knotReply(who, repo, "com.atproto.repo.putRecord", { + collection, + rkey, + swapRecord: current.cid, + record: change(stored) + }); +}; + +export const ticketRecord = (title: string, more: Partial = {}) => + ({ + $type: TICKET, + title: { $type: "org.tangled.markup.markdown#inline", text: title }, + state: "open", + createdAt: new Date().toISOString(), + ...more + }) satisfies Ticket; + +export const tickets = { + open: async (who: Account, repo: string, title: string): Promise => { + const made = await knotCall(who, repo, "com.atproto.repo.createRecord", { + collection: TICKET, + record: ticketRecord(title) + }); + return { uri: made.uri, cid: made.cid }; + }, + setState: async (who: Account, ticket: Written, state: "open" | "closed"): Promise => { + const made = ok( + "knot com.atproto.repo.putRecord", + await editCob(who, ticket.uri, "change", (stored) => ({ ...stored, state })) + ); + return { uri: made.uri, cid: made.cid }; + } +}; + +export const pulls = { + open: async ( + who: Account, + repo: Did, + title: string, + targetBranch: string, + range: { base: string; head: string }, + sourceBranch: string + ): Promise<{ ticket: Written; patch: Written }> => { + const ticket = await knotCall(who, repo, "com.atproto.repo.createRecord", { + collection: TICKET, + record: ticketRecord(title, { targetBranch }) + }); + // the knot only takes patches through submitPatch, not createRecord + const patch = await knotCall(who, repo, "org.tangled.review.submitPatch", { + patch: { + ticket: partsOf(ticket.uri).rkey, + base: { oid: range.base }, + head: { oid: range.head }, + sourceRepo: repo, + // web resubmits from the source branch, a patch without one has no resubmit button + sourceBranch + } + }); + return { + ticket: { uri: ticket.uri, cid: ticket.cid }, + patch: { uri: patch.uri, cid: patch.cid } + }; + } +}; + +const COMMENT = "org.tangled.feed.comment"; + +// the subject's cid must be its current version or the knot refuses it +export const comments = { + add: (who: Account, repo: string, subject: Written, text: string) => + knotReply(who, repo, "com.atproto.repo.createRecord", { + collection: COMMENT, + record: { + $type: COMMENT, + subject: { + $type: "com.atproto.repo.strongRef", + uri: subject.uri as ResourceUri, + cid: subject.cid + }, + body: { $type: "org.tangled.markup.markdown", text }, + createdAt: new Date().toISOString() + } satisfies Comment + }) +}; + +const LABEL = "org.tangled.label.definition"; + +// the knot only takes a client rkey on label definitions, and this one is the name +export const labels = { + define: (who: Account, repo: string, name: string) => + knotReply(who, repo, "com.atproto.repo.createRecord", { + collection: LABEL, + rkey: name, + record: { + $type: LABEL, + name, + color: "#a855f7", + createdAt: new Date().toISOString() + } satisfies LabelDefinition + }) +}; + +export const erase = async (who: Account, record: Pick) => { + const { repo, collection, rkey } = partsOf(record.uri); + if (repo === who.did) { + await pdsCall(who, "com.atproto.repo.deleteRecord", { collection, rkey }); + return; + } + const current = await knotRecordFor(record.uri, "erase"); + await knotCall(who, repo, "com.atproto.repo.deleteRecord", { + collection, + rkey, + swapRecord: current.cid + }); +}; diff --git a/e2e/harness/attestations.ts b/e2e/harness/attestations.ts new file mode 100644 index 0000000000000000000000000000000000000000..8f3b81b0447c86eaae9c016da54c9f6c6940789c --- /dev/null +++ b/e2e/harness/attestations.ts @@ -0,0 +1,50 @@ +import { expect } from "@playwright/test"; + +import type { Account } from "./config"; +import type { Written } from "./net"; +import type { Run } from "./run"; +import { knot, lookBefore, pds, settle } from "./services"; +import type { Held } from "./tabs"; +import { versionsOf } from "./versions"; +import { attestationId, record, type Slot } from "./vocabulary"; + +// an attestation is a record in its author's own repo, and the run erases a slot's records as +// the first account to write it, so each account gets a slot of its own +const slots = new Map(); +const attestationBy = ({ name }: Account) => { + const slot = slots.get(name) ?? record(attestationId.nsid); + slots.set(name, slot); + return slot; +}; + +export const isAttested = (held: Held) => held.host === knot.host && held.method !== "delete"; + +// the page attests every knot write by itself, so tracing one has to claim that put too +export const claimAttestation = async (run: Run, written: Written) => { + const slot = attestationBy(run.actor); + // two knot writes in a row put two attestations in the queue, so take the one naming this write + const held = await run.hold( + slot, + (body) => + body.record?.subject?.uri === written.uri && body.record?.subject?.cid === written.cid + ); + expect(held.host, "the attestation should go to the pds").toBe(pds.host); + expect(held.method, "the attestation should be a put").toBe("put"); + + // the same version can come back under a newer change, so the newest match is the one just made + const { versions } = await versionsOf(written.uri); + const change = versions.filter((version) => version.cid === written.cid).at(-1)?.changeId; + expect( + change, + `the knot should list ${written.cid} among the versions of ${written.uri}` + ).toBeDefined(); + expect(held.body.record, "the attestation should name the knot's change").toMatchObject({ + $type: attestationId.nsid, + subject: { uri: written.uri, cid: written.cid }, + change + }); + + const before = { ...(await lookBefore([pds], held)), released: Date.now() }; + const attestation = await run.release(slot, held); + await settle(attestation, [pds], run.actor, attestationId.nsid, held.method, before); +}; diff --git a/e2e/harness/auth.setup.ts b/e2e/harness/auth.setup.ts new file mode 100644 index 0000000000000000000000000000000000000000..404015a95c71e85cd54cc8bea86d06f7e4c50075 --- /dev/null +++ b/e2e/harness/auth.setup.ts @@ -0,0 +1,19 @@ +import { test as setup } from "@playwright/test"; + +import { authFile, seeded } from "./config"; +import { recorded, watchBrowser } from "./record"; +import { logIn } from "./signin"; + +for (const account of Object.values(seeded)) { + setup(`log in ${account.name}`, async ({ page }) => { + watchBrowser(page.context()); + await setup.step("log in", (info) => + recorded( + info, + () => logIn(page, account), + () => page.url() + ) + ); + await page.context().storageState({ path: authFile(account) }); + }); +} diff --git a/e2e/harness/chores.ts b/e2e/harness/chores.ts new file mode 100644 index 0000000000000000000000000000000000000000..a0b692b6f38c1f0cf581896d53b92a20c1247b28 --- /dev/null +++ b/e2e/harness/chores.ts @@ -0,0 +1,229 @@ +import { spawn } from "node:child_process"; +import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { accounts, alreadyGone, erase, quotas, repos } from "./api"; +import type { Account } from "./config"; +import { forgetKey } from "./git"; +import type { Written } from "./net"; +import type { Slot } from "./vocabulary"; + +export type Chore = + | { kind: "record"; by: Account; uri: string } + | { kind: "repo"; owner: Account; made: { did: string; name: string } } + | { kind: "limit"; by: Account; did: string; resource: string } + | { kind: "member"; account: Account } + | { kind: "account"; account: Account }; + +interface Entry { + pid: number; + label: string; + chore: Chore; +} + +interface Undo { + label: string; + id: string; + chore: Chore; +} + +export const LEDGER = resolve(".auth/chores"); + +// flows take minutes, so an old chore with a live pid means that pid got reused +const STALE_MS = 60 * 60_000; + +const alive = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return (cause as NodeJS.ErrnoException).code === "EPERM"; + } +}; + +const stillOwes = (pid: number, id: string) => + alive(pid) && Date.now() - Number(id.slice(0, 13)) < STALE_MS; + +let counter = 0; + +// on disk before this returns, so a kill right after can't lose it +export const owe = (label: string, chore: Chore) => { + mkdirSync(LEDGER, { recursive: true }); + // the counter keeps same-millisecond chores in order, newest first on replay + const stamp = Date.now().toString().padStart(13, "0"); + const id = `${stamp}-${process.pid}-${String(counter++).padStart(6, "0")}`; + const entry: Entry = { pid: process.pid, label, chore }; + writeFileSync(resolve(LEDGER, `${id}.tmp`), JSON.stringify(entry)); + renameSync(resolve(LEDGER, `${id}.tmp`), resolve(LEDGER, `${id}.json`)); + return id; +}; + +export const settled = (id: string) => rmSync(resolve(LEDGER, `${id}.json`), { force: true }); + +// someone else may have finished it already, so gone counts as done +const unlessGone = async (what: () => Promise) => { + try { + await what(); + } catch (cause) { + if (!alreadyGone(cause)) throw cause; + } +}; + +const perform = async (chore: Chore) => { + switch (chore.kind) { + case "record": + return unlessGone(() => erase(chore.by, chore)); + case "repo": + return unlessGone(() => repos.remove(chore.owner, chore.made)); + case "limit": + return quotas.unset(chore.did, chore.resource); + case "member": + return unlessGone(() => accounts.removeMember(chore.account)); + case "account": + try { + await unlessGone(() => accounts.remove(chore.account)); + } finally { + await forgetKey(chore.account); + } + } +}; + +const ownerOf = (chore: Chore) => + "by" in chore ? chore.by : "owner" in chore ? chore.owner : chore.account; + +// an account goes last, and once it's gone nothing it owns can be reached, so it waits +export const holdsUp = (chore: Chore, failed: Set) => + (chore.kind === "member" || chore.kind === "account") && failed.has(chore.account.did); + +export const replay = async (whose: "mine" | "dead") => { + const failed: string[] = []; + const failedFor = new Set(); + mkdirSync(LEDGER, { recursive: true }); + const names = readdirSync(LEDGER).sort().reverse(); + for (const name of names) { + const path = resolve(LEDGER, name); + const pid = Number(name.split("-")[1]); + if (name.endsWith(".tmp")) { + // a tmp file means the process died before the rename, so it never reached the ledger + if (whose === "dead" && !alive(pid)) rmSync(path, { force: true }); + continue; + } + let entry: Entry; + try { + entry = JSON.parse(readFileSync(path, "utf8")); + } catch (cause) { + // its owner can settle it between the listing and the read + if ((cause as NodeJS.ErrnoException).code === "ENOENT") continue; + failed.push(`${name}: the ledger entry can't be read, ${String(cause).slice(0, 100)}`); + continue; + } + if (whose === "mine" ? entry.pid !== process.pid : stillOwes(entry.pid, name)) continue; + if (holdsUp(entry.chore, failedFor)) continue; + try { + await perform(entry.chore); + rmSync(path, { force: true }); + } catch (cause) { + failedFor.add(ownerOf(entry.chore).did); + failed.push(`${entry.label}: ${String(cause).slice(0, 200)}`); + } + } + return failed; +}; + +// stopped from outside, a worker is killed a moment later, long before its tidying is done. so a +// process in its own session waits for it to go and replays what it still owed +let hooked = false; +let helped = false; +const tidyFromOutside = () => { + if (helped) return; + helped = true; + spawn("pnpm", ["exec", "playwright", "test", "--project=sweep", "tidy.spec.ts"], { + detached: true, + stdio: "ignore", + env: { + ...process.env, + E2E_SWEEP: "1", + E2E_SHARED_BROWSER: "0", + E2E_TIDY_FOR: `${process.pid}` + } + }).unref(); +}; + +const tidyWhenStopped = () => { + if (hooked) return; + hooked = true; + for (const signal of ["SIGTERM", "SIGHUP"] as const) { + process.once(signal, () => { + tidyFromOutside(); + void replay("mine").finally(() => process.exit(1)); + }); + } + // a runner killed by the second ctrl+c leaves its workers to disconnect, with no signal at all + process.once("disconnect", () => { + const owed = readdirSync(LEDGER).some((name) => name.split("-")[1] === `${process.pid}`); + if (owed) tidyFromOutside(); + }); +}; + +export class Tidy { + private undo: Undo[] = []; + private closed = false; + private erasers = new Map(); + private owedUris = new Set(); + + constructor() { + tidyWhenStopped(); + } + + // a step still running at timeout lands after close, so it is tried right now + add(label: string, chore: Chore) { + const entry = { label, id: owe(label, chore), chore }; + if (!this.closed) { + this.undo.unshift(entry); + return; + } + void this.attempt(entry).then((failure) => { + if (failure) console.warn("could not tidy up late:", failure); + }); + } + + private async attempt({ label, id, chore }: Undo) { + try { + await perform(chore); + settled(id); + } catch (cause) { + return `${label}: ${String(cause).slice(0, 300)}`; + } + } + + // the first account to write to a slot erases every record version of it + record(slot: Slot, by: Account, written: Written) { + if (this.owedUris.has(written.uri)) return; + this.owedUris.add(written.uri); + if (!this.erasers.has(slot)) this.erasers.set(slot, by); + this.add(`${slot.nsid} record`, { + kind: "record", + by: this.erasers.get(slot)!, + uri: written.uri + }); + } + + slotHasEraser(slot: Slot) { + return this.erasers.has(slot); + } + + // a leftover throws, so a leak fails the test instead of only logging + async close() { + this.closed = true; + const failedFor = new Set(); + const leftovers: string[] = []; + for (const entry of this.undo) { + if (holdsUp(entry.chore, failedFor)) continue; + const failure = await this.attempt(entry); + if (!failure) continue; + failedFor.add(ownerOf(entry.chore).did); + leftovers.push(failure); + } + if (leftovers.length) throw new Error(`could not tidy up:\n ${leftovers.join("\n ")}`); + } +} diff --git a/e2e/harness/cobgit.ts b/e2e/harness/cobgit.ts new file mode 100644 index 0000000000000000000000000000000000000000..f82ae4ddbc132783ebd2e38ecb2aa2771d515207 --- /dev/null +++ b/e2e/harness/cobgit.ts @@ -0,0 +1,273 @@ +import { encode } from "@atcute/cbor"; +import { CODEC_DCBOR, createSync, toString as cidString } from "@atcute/cid"; +import { execFile, spawn } from "node:child_process"; +import { realpath } from "node:fs/promises"; +import type { Socket } from "node:net"; +import { createInterface } from "node:readline"; +import { promisify } from "node:util"; + +import { config } from "./config"; +import { get, partsOf, until } from "./net"; +import { versionsOf, type Version } from "./versions"; + +const exec = promisify(execFile); + +const GIT_IMAGE = process.env.E2E_GIT_IMAGE ?? "docker.io/alpine/git"; + +const installed = (program: string) => + exec(program, ["--version"]).then( + () => true, + () => false + ); + +type Knot = { scan: string } | { engine: string; container: string; scan: string }; + +const findKnot = async (): Promise => { + // outside a container the knot's repos sit on this disk, so E2E_KNOT_SCAN points straight at them + const own = process.env.E2E_KNOT_SCAN; + if (own) return { scan: own }; + const engine = process.env.E2E_ENGINE ?? ((await installed("podman")) ? "podman" : "docker"); + const root = await realpath(".."); + const listed = await exec(engine, [ + "ps", + "-q", + "--filter", + "label=com.docker.compose.service=knot2", + "--filter", + `label=com.docker.compose.project.working_dir=${root}` + ]); + const container = process.env.E2E_KNOT_CONTAINER ?? listed.stdout.trim().split("\n")[0]; + if (!container) + throw new Error( + `${engine} should be running a knot2 container from the compose project in ${root} but lists none, set E2E_KNOT_CONTAINER for a knot started some other way` + ); + const { stdout } = await exec(engine, [ + "inspect", + "--format", + "{{range .Config.Env}}{{println .}}{{end}}", + container + ]); + const scan = /^KNOT_SCAN_PATH=(.+)$/m.exec(stdout)?.[1]; + if (!scan) + throw new Error(`the knot container ${container} should set KNOT_SCAN_PATH but doesn't`); + return { engine, container, scan }; +}; + +const shardOf = (did: string) => { + const [, method, ...rest] = did.split(":"); + const id = rest.join(":"); + return `${method}/${id.slice(0, 2)}/${id.slice(2)}`; +}; + +// one line in, "end" out, so one container serves every check +const LOOP = `check() { + set -eu + cd "$1" + tip=$(git rev-parse -q --verify "refs/cobs/$2/$3^{commit}") || { echo missing; return 0; } + echo "tip $tip" + for change in $(git rev-list --reverse --topo-order "$tip"); do + head=$(git cat-file commit "$change" | sed '/^$/q') + type=$(echo "$head" | sed -n 's/^cob-type //p') + author=$(echo "$head" | sed -n 's/^cob-author //p') + echo "change $change $type $author $(git ls-tree --name-only "$change" | tr '\\n' ,)" + done + if git cat-file -e "$tip:record" 2>/dev/null; then + echo "record $(git cat-file blob "$tip:record" | base64 | tr -d '\\n')" + fi + echo "payload $(git cat-file blob "$3:payload" | base64 | tr -d '\\n')" +} +while read -r repo nsid root; do + (check "$repo" "$nsid" "$root") 2>&1 || echo "failed $?" + echo end +done +`; + +interface Asked { + lines: string[]; + done: (lines: string[]) => void; + fail: (cause: Error) => void; +} + +// the knot's repos are owned by another user, so git refuses them without safe.directory +const SAFE_DIRECTORY = { + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "safe.directory", + GIT_CONFIG_VALUE_0: "*" +}; + +const spawnGit = (box: Knot) => + "engine" in box + ? spawn( + box.engine, + [ + "run", + "-i", + "--rm", + "--network=none", + `--volumes-from=${box.container}:ro`, + ...Object.entries(SAFE_DIRECTORY).map( + ([key, value]) => `--env=${key}=${value}` + ), + "--entrypoint=sh", + GIT_IMAGE, + "-c", + LOOP + ], + { stdio: ["pipe", "pipe", "pipe"] } + ) + : spawn("sh", ["-c", LOOP], { + env: { ...process.env, ...SAFE_DIRECTORY }, + stdio: ["pipe", "pipe", "pipe"] + }); + +const startHelper = async () => { + const box = await findKnot(); + const child = spawnGit(box); + // the loop only ends when stdin closes, and --rm cleans the container up then + const answers = child.stdout as unknown as Socket; + child.unref(); + for (const pipe of [child.stdin, child.stdout, child.stderr]) + (pipe as unknown as Socket).unref(); + const waiting: Asked[] = []; + let said = ""; + child.stderr.on("data", (chunk) => (said = (said + chunk).slice(-2000))); + createInterface({ input: child.stdout }).on("line", (line) => { + const asked = waiting[0]; + if (!asked) return; + if (line !== "end") return void asked.lines.push(line); + waiting.shift(); + if (waiting.length === 0) answers.unref(); + asked.done(asked.lines); + }); + const ended = new Promise((resolve) => { + const gone = (why: string) => resolve(new Error(`the git helper ${why}: ${said.trim()}`)); + child.on("error", (cause) => gone(cause.message)); + child.on("exit", (code) => gone(`exited with ${code}`)); + }); + void ended.then((cause) => { + helper = undefined; + for (const asked of waiting.splice(0)) asked.fail(cause); + }); + return { + scan: box.scan, + ask: (line: string) => + new Promise((done, fail) => { + answers.ref(); + waiting.push({ lines: [], done, fail }); + child.stdin.write(`${line}\n`); + }) + }; +}; + +let helper: ReturnType | undefined; + +interface Change { + id: string; + type: string; + author: string; + files: string[]; +} + +interface CobInGit { + tip: string; + changes: Change[]; + record?: Buffer; + payload: Buffer; +} + +// ls-remote can't see refs/cobs and the knot hides them from reads, so read the disk +const cobInGit = async ( + repoDid: string, + nsid: string, + root: string +): Promise => { + const git = await (helper ??= startHelper()); + const lines = await git.ask(`${git.scan}/${shardOf(repoDid)} ${nsid} ${root}`); + if (lines[0] === "missing") return undefined; + const field = (name: string) => + lines + .filter((line) => line.startsWith(`${name} `)) + .map((line) => line.slice(name.length + 1)); + if (field("payload").length === 0) + throw new Error(`git couldn't read refs/cobs/${nsid}/${root}: ${lines.join(" ")}`); + const record = field("record")[0]; + return { + tip: field("tip")[0], + changes: field("change").map((line) => { + const [id, type, author, files] = line.split(" "); + return { id, type, author, files: files.split(",").filter(Boolean) }; + }), + record: record === undefined ? undefined : Buffer.from(record, "base64"), + payload: Buffer.from(field("payload")[0], "base64") + }; +}; + +const cidOf = (bytes: Buffer) => cidString(createSync(CODEC_DCBOR, bytes)); + +// the root payload holds address {rkey, collection} as dag-cbor, so match those bytes +const namesRecord = (payload: Buffer, collection: string, rkey: string) => + payload.includes(Buffer.concat([encode("address"), encode({ rkey, collection })])); + +const signingKey = async () => { + const { body } = await get(`${config.knotUrl}/.well-known/did.json`); + const methods: { id: string; publicKeyMultibase: string }[] = body?.verificationMethod ?? []; + return methods.find((method) => method.id.endsWith("#atproto"))?.publicKeyMultibase; +}; + +let knotKey: Promise | undefined; + +// a cid of undefined means the record was deleted, so the chain ends in a tombstone +const gitDisagrees = async ( + { uri, cid }: { uri: string; cid?: string }, + versions: Version[] +): Promise => { + const { repo, collection, rkey } = partsOf(uri); + if (versions.length === 0) return "listCobVersions lists no versions to look up in git"; + const root = versions[0].changeId; + const ref = `refs/cobs/${collection}/${root}`; + const [cob, key] = await Promise.all([ + cobInGit(repo, collection, root), + (knotKey ??= signingKey()) + ]); + if (!cob) return `there is no ${ref}`; + const last = versions.at(-1)!; + if (cob.tip !== last.changeId) + return `${ref} points at ${cob.tip} but listCobVersions ends at ${last.changeId}`; + const walked = cob.changes.map((change) => change.id); + const listed = versions.map((version) => version.changeId); + if (JSON.stringify([...walked].sort()) !== JSON.stringify([...listed].sort())) + return `git walks ${walked.join(" ")} from ${ref} but listCobVersions lists ${listed.join(" ")}`; + const stranger = cob.changes.find( + (change) => change.type !== collection || change.author !== key + ); + if (stranger) + return `change ${stranger.id} says cob-type ${stranger.type} by ${stranger.author}, it should be ${collection} by the knot's key ${key}`; + if (!namesRecord(cob.payload, collection, rkey)) + return `the root change of ${ref} doesn't name rkey ${rkey} in ${collection}`; + const holds = cob.changes.at(-1)!.files; + if (cid === undefined) + return holds.includes("tombstone") && !cob.record + ? undefined + : `the tip of ${ref} should hold a tombstone but holds ${holds.join(", ")}`; + if (!cob.record) return `the tip of ${ref} should hold a record but holds ${holds.join(", ")}`; + const hashed = cidOf(cob.record); + return hashed === cid + ? undefined + : `the record blob at the tip of ${ref} hashes to ${hashed} but the write answered ${cid}`; +}; + +// versions are read again only when git disagrees, because a write can land between reads +export const checkGit = (written: { uri: string; cid?: string }, read: Version[]) => { + const { collection } = partsOf(written.uri); + let versions: Version[] | undefined = read; + return until( + `refs/cobs/${collection}/${read[0]?.changeId} should end at ${written.cid ?? "a tombstone"}`, + 10_000, + async () => { + versions ??= (await versionsOf(written.uri)).versions; + const problem = await gitDisagrees(written, versions); + versions = undefined; + return [!problem, problem ?? ""]; + } + ); +}; diff --git a/e2e/harness/config.ts b/e2e/harness/config.ts new file mode 100644 index 0000000000000000000000000000000000000000..80fa65d19cc677bedc6c17e6b705f7c175dead53 --- /dev/null +++ b/e2e/harness/config.ts @@ -0,0 +1,76 @@ +import { readFileSync } from "node:fs"; + +const host = (name: string) => `https://${name}.tngl.boltless.dev`; + +// localinfra's env file has the dev pds admin password, so a local run needs no setup +const devPdsAdmin = () => { + try { + const env = readFileSync("../localinfra/pds.env", "utf8"); + return env.match(/^PDS_ADMIN_PASSWORD=(.+)$/m)?.[1]; + } catch { + return undefined; + } +}; + +const loginThrough = (value = "minted") => { + if (value !== "minted" && value !== "ui") + throw new Error(`--login is ui or minted, not ${value}`); + return value; +}; + +// every stack name resolves to the host's caddy in the browser +const browserArgs = ["--host-resolver-rules=MAP *.tngl.boltless.dev 127.0.0.1"]; + +// E2E_CHROMIUM swaps in a browser where playwright's download can't run +export const browserLaunch = { + args: browserArgs, + executablePath: process.env.E2E_CHROMIUM || undefined +}; + +export const config = { + webUrl: process.env.E2E_WEB_URL ?? host("web"), + pdsUrl: process.env.E2E_PDS_URL ?? host("pds"), + knotUrl: process.env.E2E_KNOT_URL ?? host("knot2"), + hydrantUrl: process.env.E2E_HYDRANT_URL ?? host("hydrant"), + bobbinUrl: process.env.E2E_BOBBIN_URL ?? host("bobbin"), + spindleUrl: process.env.E2E_SPINDLE_URL ?? host("spindle"), + // the service is named mirror here, not knotmirror + knotmirrorUrl: process.env.E2E_KNOTMIRROR_URL ?? host("mirror"), + sitesUrl: process.env.E2E_SITES_URL ?? host("sites"), + ogreUrl: process.env.E2E_OGRE_URL ?? host("ogre"), + // deliberi follows the bobbin url, so a run pointed at another bobbin gets its own deliberi + deliberiUrl: + process.env.E2E_DELIBERI_URL ?? + (process.env.E2E_BOBBIN_URL ?? host("bobbin")).replace("bobbin", "deliberi"), + knotDid: "did:web:knot2.tngl.boltless.dev", + spindleDid: "did:web:spindle.tngl.boltless.dev", + sitesDid: "did:web:sites.tngl.boltless.dev", + sitesDomain: process.env.E2E_SITES_DOMAIN ?? "sites.tngl.boltless.dev", + // ssh isn't behind caddy, it's the port the compose file publishes + knotSsh: process.env.E2E_KNOT_SSH ?? "127.0.0.1:2223", + password: process.env.E2E_PASSWORD ?? "password", + login: loginThrough(process.env.E2E_LOGIN), + pdsAdminPassword: process.env.E2E_PDS_ADMIN_PASSWORD ?? devPdsAdmin() +}; + +export interface Account { + name: string; + handle: string; + did: string; +} + +export const handleOf = (name: string) => `${name}.pds.tngl.boltless.dev`; + +const seed = (name: string, did: string): Account => ({ name, handle: handleOf(name), did }); + +// seeded by localinfra's init-data, so they exist on every stack +export const seeded = { + alice: seed("alice", "did:plc:igzuueb63dytyy3tpaweofuu"), + bob: seed("bob", "did:plc:xbserj4jfuhzkbl3mszkvvrb"), + charlie: seed("charlie", "did:plc:pgkgoz6rpvmgynswviohltpx"), + david: seed("david", "did:plc:ryxq2u2b6sadhhjflpwiefg4") +}; + +export const isSeeded = (account: Account) => + Object.values(seeded).some((known) => known.did === account.did); +export const authFile = (account: Account) => `.auth/${account.name}.json`; diff --git a/e2e/harness/firehose.ts b/e2e/harness/firehose.ts new file mode 100644 index 0000000000000000000000000000000000000000..8e57ae97c882b195db273e5c018d243a1ef08616 --- /dev/null +++ b/e2e/harness/firehose.ts @@ -0,0 +1,208 @@ +import { ComAtprotoSyncSubscribeRepos } from "@atcute/atproto"; +import { FirehoseError, FirehoseSubscription, type MessageOf } from "@atcute/firehose"; +import { WebSocket } from "undici"; + +import { config } from "./config"; +import { Frames } from "./frames"; +import { dispatcher } from "./net"; + +type Action = "create" | "update" | "delete"; +interface Seen { + id: number; + action: Action; + cid?: string; + // a tid, so plain string compare orders it by time + rev: string; + at: number; +} +type Message = MessageOf; + +const BUDGET = 10_000; +const isAction = (action: string): action is Action => + action === "create" || action === "update" || action === "delete"; +const told = (seen: Seen) => `${seen.action}${seen.cid ? ` ${seen.cid}` : ""} #${seen.id}`; + +// the subscription makes its own sockets, so the stack's dns and dev ca ride in on a subclass +class Dispatched extends WebSocket { + constructor(url: string | URL, protocols?: string | string[]) { + super(url, { protocols, dispatcher }); + } +} + +export class Firehose extends Frames { + private stopper?: AbortController; + private last = -1; + private stopped = true; + private downSince?: number; + private lastWords = ""; + private lost = ""; + + constructor( + private name: string, + private base: string + ) { + super(); + } + + async start() { + if (!this.stopped) return; + this.stopped = false; + const stopper = (this.stopper = new AbortController()); + const service = this.base.replace(/^http/, "ws"); + const url = `${service}/xrpc/${ComAtprotoSyncSubscribeRepos.mainSchema.nsid}`; + await new Promise((open, fail) => { + let up = false; + const timer = setTimeout( + () => fail(new Error(`${url} would not open within ${BUDGET / 1000}s`)), + BUDGET + ); + const subscription = new FirehoseSubscription({ + service, + nsid: ComAtprotoSyncSubscribeRepos.mainSchema, + // both servers replay strictly after the cursor, unlike hydrant's inclusive id + params: () => (this.last < 0 ? {} : { cursor: this.last }), + signal: stopper.signal, + ws: { + WebSocket: Dispatched, + minReconnectionDelay: 250, + maxReconnectionDelay: 2_000 + }, + onConnectionOpen: () => { + up = true; + clearTimeout(timer); + this.downSince = undefined; + open(); + }, + onConnectionError: () => { + this.downSince ??= Date.now(); + this.lastWords = `${url} would not open`; + if (up) return; + clearTimeout(timer); + fail(new Error(this.lastWords)); + }, + onConnectionClose: (event) => { + if (this.stopped) return; + this.downSince ??= Date.now(); + this.lastWords ||= `close ${event.code} ${event.reason}`.trim(); + // no first seq means there is no safe cursor to recover the gap from + if (this.last < 0) + this.lost = "disconnected before the first cursor, frames may be missing"; + this.wake(); + }, + onError: (cause) => this.unreadable(cause) + }); + void this.read(subscription); + }).catch((cause) => { + this.stop(); + throw cause; + }); + } + + stop() { + this.stopped = true; + this.stopper?.abort(); + this.wake(); + } + + mark() { + return this.last; + } + + private describe(since: number) { + const parts = [`the watcher is at #${this.last}`]; + if (since < this.forgotUpTo) parts.push(`it forgot frames up to #${this.forgotUpTo}`); + if (this.downSince !== undefined) { + const down = ((Date.now() - this.downSince) / 1000).toFixed(1); + parts.push( + `it has been disconnected for ${down}s (${this.lastWords || "no reason given"})` + ); + } else if (this.lastWords) parts.push(this.lastWords); + if (this.lost) parts.push(this.lost); + return parts.join(", "); + } + + private unreadable(cause: unknown) { + this.lost = + cause instanceof FirehoseError + ? `${this.name} said ${cause.error}: ${cause.message}` + : `could not read a frame: ${(cause as Error).message}`; + this.wake(); + } + + private async read(messages: AsyncIterable) { + try { + for await (const message of messages) { + try { + this.take(message); + } catch (cause) { + this.unreadable(cause); + } + } + } catch (cause) { + if (!this.stopped) this.unreadable(cause); + } + } + + private take(message: Message) { + if (message.$type === "com.atproto.sync.subscribeRepos#info") { + this.lastWords = `${this.name} said ${message.name}: ${message.message}`; + if (message.name === "OutdatedCursor") this.lost = this.lastWords; + this.wake(); + return; + } + if (message.seq <= this.last) return; + if (message.$type === "com.atproto.sync.subscribeRepos#commit") { + if (message.tooBig && message.ops.length === 0) + this.lost = `commit #${message.seq} for ${message.repo} is tooBig and has no ops`; + for (const op of message.ops) { + if (!isAction(op.action)) + throw new Error(`commit #${message.seq} has an invalid op for ${op.path}`); + if (op.action !== "delete" && !op.cid) + throw new Error(`commit #${message.seq} has no cid for ${op.path}`); + this.append(`at://${message.repo}/${op.path}`, { + id: message.seq, + action: op.action, + cid: op.cid?.$link, + rev: message.rev, + at: Date.now() + }); + } + } + this.last = message.seq; + this.lastWords = ""; + this.wake(); + } + + async check({ uri, cid }: { uri: string; cid?: string }, action?: Action, since = -1) { + const wanted = cid === undefined ? "delete" : (action ?? "create or update"); + const what = `${this.name}'s firehose should announce ${wanted} of ${uri}${cid ? ` at ${cid}` : ""}`; + const deadline = Date.now() + BUDGET; + for (;;) { + if (this.stopped) + throw new Error(`${what} but this process never opened it or stopped it`); + const frames = this.seen(uri).filter((frame) => frame.id > since); + const newest = frames.at(-1); + if ( + newest && + newest.cid === cid && + (cid === undefined + ? newest.action === "delete" + : newest.action !== "delete" && (!action || newest.action === action)) + ) + return newest; + const after = since < 0 ? "since the watcher opened" : `after #${since}`; + const said = frames.length + ? `${after} it announced ${frames.slice(-5).map(told).join(", ")}` + : `it announced nothing for that uri ${after}`; + const left = deadline - Date.now(); + if (left <= 0) + throw new Error( + `${what}, gave up after ${BUDGET / 1000}s: ${said}, ${this.describe(since)}` + ); + await this.changed(left); + } + } +} + +export const knotFirehose = new Firehose("knot", config.knotUrl); +export const pdsFirehose = new Firehose("pds", config.pdsUrl); diff --git a/e2e/harness/flow.ts b/e2e/harness/flow.ts new file mode 100644 index 0000000000000000000000000000000000000000..059dbb4c7a6d94b19285ebdf473c1cac0890d501 --- /dev/null +++ b/e2e/harness/flow.ts @@ -0,0 +1,388 @@ +import { expect, test, type Browser, type TestStepInfo } from "@playwright/test"; +import { fileURLToPath } from "node:url"; + +import { setPolicy, type Policy, type RepoOptions } from "./api"; +import { claimAttestation, isAttested } from "./attestations"; +import { browserLaunch, config, type Account } from "./config"; +import type { Reply } from "./net"; +import { recorded } from "./record"; +import { Run, openPull, openTicket, pushFiles, traceNewRepo, waitForMirror } from "./run"; +import { bobbin, hydrant, knot, lookBefore, settle, type Hop } from "./services"; +import { watcher } from "./stream"; +import { knotFirehose, pdsFirehose } from "./firehose"; +import type { Held } from "./tabs"; +import { + replacing, + resolve, + through, + type Ctx, + type Lazy, + type Slot, + type Target, + type Where +} from "./vocabulary"; + +export * from "./vocabulary"; +export { bobbin, deliberi, git, hydrant, knot, knotCopy, knotmirror, pds } from "./services"; +export { note } from "./record"; + +interface Place { + file: string; + line: number; + column: number; +} + +interface Step { + name: string; + run: (run: Run) => Promise; + checkpoint?: boolean; + asked: Error; +} + +// reading .stack is slow, so --list and steps that never run skip it +const askedHere = () => { + const limit = Error.stackTraceLimit; + Error.stackTraceLimit = 30; + const asked = new Error(); + Error.stackTraceLimit = limit; + return asked; +}; + +const placeOf = (frame: string): Place | undefined => { + const found = /\(?([^()\s]+):(\d+):(\d+)\)?$/.exec(frame); + if (!found) return undefined; + // specs load as modules, so their frames come as file:// urls + const file = found[1].startsWith("file:") ? fileURLToPath(found[1]) : found[1]; + return { file, line: +found[2], column: +found[3] }; +}; + +const placesOf = (asked: Error) => { + const places = (asked.stack ?? "") + .split("\n") + .slice(1) + .map(placeOf) + .filter((place): place is Place => !!place && !place.file.endsWith("harness/flow.ts")); + const at = places.find((place) => place.file.includes("/specs/")) ?? places[0]; + const via = places[0] === at ? undefined : places[0]; + return { at, via }; +}; + +const onlyPresent = (verb: string, what: Target) => { + if (what.absent) + throw new Error(`can't ${verb} ${what.describe}, a not() target is only for .sees()`); +}; + +const route = (hops: Hop[]) => hops.map((hop) => hop.name).join(", "); + +// never add a then() here, or playwright treats the chain as a promise +class Flow { + constructor(private steps: Step[]) {} + + private add(name: string, run: Step["run"], checkpoint?: boolean) { + this.steps.push({ name, run, checkpoint, asked: askedHere() }); + return this; + } + + actor(who: string | Account, options: { login?: "ui" } = {}) { + if (typeof who === "string") { + const kind = `new account${options.login ? ", login page" : ""}`; + return this.add(`actor ${who} (${kind})`, (run) => run.switchTo(who, options)); + } + if (options.login) + throw new Error( + `${who.name} is seeded, it logged in through the login page before the run` + ); + return this.add(`actor ${who.name} (seeded)`, (run) => run.switchTo(who, options)); + } + + repo(options: Pick = {}) { + return this.add("repo", (run) => run.own(options)); + } + + fork() { + return this.add("fork", async (run) => { + const upstream = run.repo; + await run.own({ forkOf: upstream.did }); + run.upstream = upstream; + }); + } + + push( + files: Record, + options: { branch?: string; base?: string; options?: string[] } = {} + ) { + return this.add("push", (run) => pushFiles(run, files, options)); + } + + // a page that makes a repo can't make its actor a knot member + member() { + return this.add("member", (run) => run.becomeMember()); + } + + joinedKnot() { + return this.add("joined the knot", async (run) => run.oweMembership()); + } + + mirrored() { + return this.add("mirrored", waitForMirror); + } + + pull(pull: Slot, patch: Slot, options: { title?: string; from: string; into: string }) { + return this.add("pull", (run) => openPull(run, pull, patch, options)); + } + + // for a record a service changed on its own, like a merge closing its ticket + learn(slot: Slot) { + return this.add(`learn ${slot.nsid}`, (run) => run.learn(slot, run.get(slot).uri, "learn")); + } + + ticket(slot: Slot, options: { title?: string; state?: "open" | "closed" } = {}) { + return this.add(`ticket ${options.state ?? "open"}`, (run) => + openTicket(run, slot, options) + ); + } + + open(at: Where) { + return this.add(`open ${at.describe}`, async (run) => { + await run.openTab(); + // the page is inert until it hydrates + await run.page.goto(at.path(run)); + await run.settle(); + }); + } + + window(name = "") { + return this.add(name ? `window ${name}` : "first window", (run) => run.openWindow(name)); + } + + device(name: string) { + return this.add(`device ${name}`, (run) => run.openDevice(name)); + } + + at(here: Where, timeout?: number) { + return this.add(`at ${here.describe}`, async (run) => { + await expect(run.page).toHaveURL(`${config.webUrl}${here.path(run)}`, { timeout }); + }); + } + + click(what: Target) { + onlyPresent("click", what); + return this.add(`click ${what.describe}`, (run) => what.locate(run.page, run).click()); + } + + fill(what: Target, value: Lazy) { + onlyPresent("fill", what); + return this.add(`fill ${what.describe}`, (run) => + what.locate(run.page, run).fill(resolve(value, run)) + ); + } + + press(key: string) { + return this.add(`press ${key}`, async (run) => { + await run.page.keyboard.press(key); + await run.settle(); + }); + } + + reload() { + return this.add("reload", async (run) => { + await run.page.reload(); + await run.settle(); + }); + } + + sees(what: Target, words?: string) { + if (what.absent && words !== undefined) { + throw new Error(`${what.describe} can't say "${words}", there is nothing to read`); + } + return this.add(`sees ${what.describe}${words ? ` saying ${words}` : ""}`, async (run) => { + const found = what.locate(run.page, run); + if (what.absent) await expect(found).toHaveCount(0); + else if (words === undefined) await expect(found).toBeVisible(); + else await expect(found).toHaveText(words); + }); + } + + // the write is held here, so the server hasn't seen it yet + sends( + slot: Slot, + to: { name: string; host: string }, + body: Lazy>, + method?: Held["method"] + ) { + return this.add(`sends ${slot.nsid} to ${to.name}`, async (run) => { + const held = await run.hold(slot); + expect(held.host, `${slot.nsid} should be sent to the ${to.name}`).toBe(to.host); + if (method) expect(held.method, `${slot.nsid} should be a ${method}`).toBe(method); + expect(held.body).toMatchObject(resolve(body, run)); + }); + } + + traceNewRepo(name: Lazy, hops: Hop[]) { + return this.add("trace the new repo", (run) => traceNewRepo(run, resolve(name, run), hops)); + } + + sendsNothing(slot: Slot) { + return this.add(`sends no ${slot.nsid}`, (run) => run.expectNoWrite(slot)); + } + + trace(slot: Slot, hops: Hop[]) { + return this.add(`trace ${slot.nsid} through ${route(hops)}`, async (run) => { + const held = await run.hold(slot); + const before = { ...(await lookBefore(hops, held)), released: Date.now() }; + const written = await run.release(slot, held); + await settle(written, hops, run.actor, slot.nsid, held.method, before, run.everyone); + if (isAttested(held)) await claimAttestation(run, written); + }); + } + + follow(slot: Slot, hops: Hop[]) { + return this.add(`follow ${slot.nsid} through ${route(hops)}`, (run) => + settle(run.get(slot), hops, run.actor, slot.nsid, "settled", undefined, run.everyone) + ); + } + + // the knot's own copy of the record says this state, whatever the page shows + knotStored(slot: Slot, state: string) { + return this.add(`the knot stored it as ${state}`, async (run) => { + expect((await knot.has(run.get(slot))).state).toBe(state); + }); + } + + // no page control for this, so it goes through the api + policy(policy: Policy) { + return this.add(`policy ${policy}`, (run) => setPolicy(run.actor, run.repo.did, policy)); + } + + rejects(name: string, attempt: (ctx: Ctx) => Promise, status: number | number[]) { + const allowed = [status].flat(); + return this.add(`rejects ${name}`, async (run) => { + const reply = await attempt(run); + expect( + allowed, + `${name} answered ${reply.status} ${JSON.stringify(reply.body)}` + ).toContain(reply.status); + }); + } + + refused(slot: Slot, status: number | number[]) { + return this.add(`${slot.nsid} is refused with ${status}`, async (run) => { + await run.answer(slot, await run.hold(slot), status); + }); + } + + // a grant write answers with an empty body, so there's no record to trace + accepted(slot: Slot) { + return this.add(`${slot.nsid} is accepted`, async (run) => { + await run.answer(slot, await run.hold(slot), 200); + }); + } + + // the button only flips once bobbin has the new state, so a held write changes nothing + changeState( + slot: Slot, + to: "open" | "closed", + clicking: Target, + then: Target, + hops: Hop[] = through(knot, hydrant, bobbin) + ) { + return this.click(clicking) + .sends(slot, knot, replacing(slot, { state: to }), "put") + .sees(clicking) + .trace(slot, hops) + .knotStored(slot, to) + .sees(then); + } + + checkpoint(name: string) { + return this.add(`checkpoint: ${name}`, async () => {}, true); + } + + step(name: string, run: (ctx: Ctx) => Promise) { + return this.add(name, run); + } + + use(chain: (flow: Flow) => Flow) { + return chain(this); + } +} + +export type { Flow }; + +// teardown still gets its own time after a test times out, which is why the world closes there +const withWorld = test.extend<{ world: Run }, { browser: Browser; stream: void }>({ + browser: [ + async ({ playwright }, use) => { + const shared = process.env.E2E_BROWSER_WS; + const browser = shared + ? await playwright.chromium.connect(shared) + : await playwright.chromium.launch(browserLaunch); + await use(browser); + await browser.close(); + }, + { scope: "worker" } + ], + // open before the worker's first write so every hop has its frames + stream: [ + async ({}, use) => { + try { + await watcher.start(); + await Promise.all([knotFirehose.start(), pdsFirehose.start()]); + await use(); + } finally { + watcher.stop(); + knotFirehose.stop(); + pdsFirehose.stop(); + } + }, + { scope: "worker", auto: true } + ], + world: async ({ browser }, use) => { + const run = new Run(browser); + await use(run); + await run.close(); + } +}); + +export const flow = (name: string) => { + const steps: Step[] = []; + const asked = askedHere(); + withWorld(name, async ({ world }) => { + const recording = (run: Step["run"]) => (info: TestStepInfo) => + recorded( + info, + () => run(world), + () => world.pageUrl + ); + let reached = "no checkpoint"; + try { + for (const step of steps) { + const { at, via } = placesOf(step.asked); + const { actor, repo } = world.known; + await test.step(step.name, recording(step.run), { + location: at, + params: { + ...(via && { via: `${via.file}:${via.line}` }), + ...(actor && { actor }), + ...(repo && { repo }) + }, + subtitle: repo + }); + if (!step.checkpoint) continue; + reached = step.name; + test.info().annotations.push({ type: "checkpoint", description: step.name }); + } + await test.step( + "nothing else was written", + recording((run) => run.expectQuiet()), + { + location: placesOf(asked).at + } + ); + } catch (cause) { + test.info().annotations.push({ type: "stopped after", description: reached }); + throw cause; + } + }); + return new Flow(steps); +}; diff --git a/e2e/harness/frames.ts b/e2e/harness/frames.ts new file mode 100644 index 0000000000000000000000000000000000000000..3cb85c24a676de40621045af23f96e324fc64274 --- /dev/null +++ b/e2e/harness/frames.ts @@ -0,0 +1,43 @@ +// every worker's writes come through here, and a run is only a few thousand frames +const KEPT = 50_000; + +export class Frames { + private records = new Map(); + private kept = 0; + protected forgotUpTo = -1; + private waiting = new Set<() => void>(); + + seen(uri: string) { + return this.records.get(uri) ?? []; + } + + // wake on the next frame so a check passes the moment it lands + changed(ms: number) { + return new Promise((wake) => { + const done = () => { + clearTimeout(timer); + this.waiting.delete(done); + wake(); + }; + const timer = setTimeout(done, ms); + this.waiting.add(done); + }); + } + + protected wake() { + for (const wake of this.waiting) wake(); + } + + protected append(uri: string, frame: T) { + const seen = this.seen(uri); + seen.push(frame); + this.records.set(uri, seen); + this.kept++; + while (this.kept > KEPT) { + const [oldest, dropped] = this.records.entries().next().value!; + this.records.delete(oldest); + this.kept -= dropped.length; + this.forgotUpTo = Math.max(this.forgotUpTo, dropped.at(-1)!.id); + } + } +} diff --git a/e2e/harness/git.ts b/e2e/harness/git.ts new file mode 100644 index 0000000000000000000000000000000000000000..4e80b760723ae78d028ae2e08102e4b09452959a --- /dev/null +++ b/e2e/harness/git.ts @@ -0,0 +1,263 @@ +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { promisify } from "node:util"; + +import { getOk, pdsCall } from "./api"; +import { config, type Account } from "./config"; +import type { PublicKey } from "./lexicons"; +import { sleep, xrpc } from "./net"; +import type { Ctx } from "./vocabulary"; + +const exec = promisify(execFile); +const KEYS = resolve(".auth/keys"); + +const keygen = async (who: Account) => { + const path = join(KEYS, who.name); + await mkdir(KEYS, { recursive: true }); + await exec("ssh-keygen", [ + "-q", + "-t", + "ed25519", + "-N", + "", + "-C", + `e2e-${who.name}`, + "-f", + path + ]); + return path; +}; + +// keyed by the key too, makeKey rewrites the same path with a new one +const registered = new Set(); + +// the knot picks up org.tangled.key.sshKey from the pds when it connects +const keyFileOf = async (who: Account) => { + const path = join(KEYS, who.name); + await readFile(path).catch(() => keygen(who)); + const line = (await readFile(`${path}.pub`, "utf8")).trim(); + if (registered.has(`${who.did} ${line}`)) return path; + const listed = await getOk( + xrpc(config.pdsUrl, "com.atproto.repo.listRecords", { + repo: who.did, + collection: "org.tangled.key.sshKey" + }) + ); + if (!listed.records.some((r: any) => r.value.publicKey === line)) { + await pdsCall(who, "com.atproto.repo.createRecord", { + collection: "org.tangled.key.sshKey", + record: { + $type: "org.tangled.key.sshKey", + publicKey: line, + name: `e2e-${who.name}`, + createdAt: new Date().toISOString() + } satisfies PublicKey + }); + } + registered.add(`${who.did} ${line}`); + return path; +}; + +// a key is only worth keeping while its account exists +export const forgetKey = async (who: Account) => { + const path = join(KEYS, who.name); + await rm(path, { force: true }); + await rm(`${path}.pub`, { force: true }); +}; + +// BatchMode and ConnectTimeout make a dead host fail fast instead of prompting +// -F /dev/null so that a personal ssh config isn't used +const sshFor = (key: string) => + `ssh -F /dev/null -i ${key} -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR`; + +// as the ssh user the knot reads keys on the spot, plain "git" only sees the ones gathered so far +const remoteOf = (who: Account, repoDid: string) => { + const [host, port] = config.knotSsh.split(":"); + return `ssh://${who.handle}@${host}:${port}/${repoDid}`; +}; + +export const sshAs = async (who: Account) => sshFor(await keyFileOf(who)); + +const gitTo = async (who: Account, repoDid: string) => ({ + ssh: await sshAs(who), + remote: remoteOf(who, repoDid) +}); + +// the knot allows four git ops at once per address, and every worker shares one +const retryWhenBusy = async (run: () => Promise): Promise => { + for (let attempt = 1; ; attempt++) { + try { + return await run(); + } catch (cause: any) { + const busy = /too many concurrent operations/.test(String(cause.stderr ?? cause)); + if (!busy || attempt >= 30) throw cause; + await sleep(200 + Math.random() * 400); + } + } +}; + +const GIT_TIMEOUT = 120_000; + +const git = async (cwd: string, ssh: string, ...args: string[]) => + retryWhenBusy(() => + exec("git", ["-c", "user.name=e2e", "-c", "user.email=e2e@example.com", ...args], { + cwd, + env: { ...process.env, GIT_SSH_COMMAND: ssh }, + timeout: GIT_TIMEOUT + }) + ); + +const inScratch = async (run: (dir: string) => Promise) => { + const dir = await mkdtemp(join(tmpdir(), "e2e-git-")); + try { + return await run(dir); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}; + +export const push = async ( + who: Account, + repoDid: string, + files: Record, + { + branch = "main", + base, + options = [] + }: { branch?: string; base?: string; options?: string[] } = {} +) => + inScratch(async (dir) => { + const { ssh, remote } = await gitTo(who, repoDid); + const heads = Object.keys(await lsRemote(who, repoDid, "refs/heads/")); + if (heads.length > 0) { + const known = heads.includes(`refs/heads/${branch}`); + const start = known ? branch : base; + await git( + dir, + ssh, + "clone", + "-q", + "--depth=1", + ...(start ? ["--single-branch", "--branch", start] : []), + remote, + "." + ); + if (!known) await git(dir, ssh, "checkout", "-q", "-b", branch); + } else { + // knot repos are sha256, and git refuses a sha1 history pushed into one + await git(dir, ssh, "init", "-q", "--object-format=sha256", "-b", branch); + } + for (const [file, text] of Object.entries(files)) { + await mkdir(dirname(join(dir, file)), { recursive: true }); + await writeFile(join(dir, file), text); + } + await git(dir, ssh, "add", "-A"); + await git(dir, ssh, "commit", "-q", "-m", "e2e"); + const sha = (await git(dir, ssh, "rev-parse", "HEAD")).stdout.trim(); + const { stderr } = await git( + dir, + ssh, + "push", + ...options.flatMap((option) => ["-o", option]), + remote, + `${branch}:${branch}` + ); + return { sha, output: stderr }; + }); + +export const lsRemote = async (who: Account, repoDid: string, prefix = "") => { + const { ssh, remote } = await gitTo(who, repoDid); + const { stdout } = await git(tmpdir(), ssh, "ls-remote", remote); + return Object.fromEntries( + stdout + .trim() + .split("\n") + .filter(Boolean) + .map((line) => line.split("\t").reverse()) + .filter(([ref]) => ref.startsWith(prefix)) + ) as Record; +}; + +// git can't resolve the stack's names or trust its dev ca either +const overHttps = (cwd: string, ...args: string[]) => { + const resolve = [config.webUrl, config.knotUrl].flatMap((url) => { + const { hostname, port } = new URL(url); + return ["-c", `http.curloptResolve=${hostname}:${port || 443}:127.0.0.1`]; + }); + return retryWhenBusy(() => + exec("git", ["-c", "http.sslVerify=false", ...resolve, ...args], { + cwd, + timeout: GIT_TIMEOUT + }) + ); +}; + +export const lsRemoteOverHttps = async (url: string) => + (await overHttps(tmpdir(), "ls-remote", url)).stdout; + +export const cloneOverHttps = (url: string) => + inScratch(async (dir) => { + await overHttps(dir, "clone", "-q", url, "."); + return (await overHttps(dir, "rev-parse", "HEAD")).stdout.trim(); + }); + +export const headOf = async (who: Account, repoDid: string) => { + const { ssh, remote } = await gitTo(who, repoDid); + const { stdout } = await git(tmpdir(), ssh, "ls-remote", "--symref", remote, "HEAD"); + return /^ref: refs\/heads\/(\S+)\tHEAD/m.exec(stdout)?.[1]; +}; + +export const deleteBranch = async (who: Account, repoDid: string, branch: string) => + inScratch(async (dir) => { + const { ssh, remote } = await gitTo(who, repoDid); + await git(dir, ssh, "init", "-q", "--object-format=sha256"); + await git(dir, ssh, "push", remote, "--delete", branch); + }); + +export const pushAnnotatedTag = async ( + who: Account, + repoDid: string, + tag: string, + message: string +) => + inScratch(async (dir) => { + const { ssh, remote } = await gitTo(who, repoDid); + await git(dir, ssh, "clone", "-q", "--depth=1", remote, "."); + await git(dir, ssh, "tag", "-a", tag, "-m", message); + await git(dir, ssh, "push", "-q", remote, tag); + }); + +interface Key { + path: string; + pub: string; + fingerprint: string; +} + +const made = new WeakMap(); + +// writes to the path pushes read, so a push uses this key +export const makeKey = async (ctx: Ctx) => { + const path = await keygen(ctx.actor); + const { stdout } = await exec("ssh-keygen", ["-lf", `${path}.pub`]); + const pub = (await readFile(`${path}.pub`, "utf8")).trim(); + made.set(ctx, { path, pub, fingerprint: stdout.split(" ")[1] }); +}; + +export const keyOf = (ctx: Ctx) => made.get(ctx)!; + +// retryWhenBusy keeps a busy knot from reading as a bad key +export const lsRemoteWith = async (ctx: Ctx) => { + try { + await retryWhenBusy(() => + exec("git", ["ls-remote", remoteOf(ctx.actor, ctx.repo.did)], { + env: { ...process.env, GIT_SSH_COMMAND: sshFor(keyOf(ctx).path) }, + timeout: 60_000 + }) + ); + return "listed"; + } catch (cause: any) { + return String(cause.stderr ?? cause); + } +}; diff --git a/e2e/harness/global.ts b/e2e/harness/global.ts new file mode 100644 index 0000000000000000000000000000000000000000..e5168b687ff835845c475695302d63b519dc8c67 --- /dev/null +++ b/e2e/harness/global.ts @@ -0,0 +1,19 @@ +import { chromium } from "@playwright/test"; + +import { replay } from "./chores"; +import { browserLaunch } from "./config"; + +export default async () => { + // a killed run leaves leaks, so the next run reports them and the end fails on them + for (const what of await replay("dead")) console.warn("an earlier run's leftover:", what); + const server = + process.env.E2E_SHARED_BROWSER === "0" + ? undefined + : await chromium.launchServer(browserLaunch); + if (server) process.env.E2E_BROWSER_WS = server.wsEndpoint(); + return async () => { + await server?.close(); + const failed = await replay("dead"); + if (failed.length) throw new Error(`could not tidy up:\n ${failed.join("\n ")}`); + }; +}; diff --git a/e2e/harness/lexicons.ts b/e2e/harness/lexicons.ts new file mode 100644 index 0000000000000000000000000000000000000000..a4c3d595e6d486f9cf8e635ce51a2ce9866b171b --- /dev/null +++ b/e2e/harness/lexicons.ts @@ -0,0 +1,17 @@ +// types only, so a run never loads anything from web/ +import type {} from "../../web/src/lib/api/lexicons/index"; +import type { Records } from "@atcute/lexicons/ambient"; + +export type RecordNsid = keyof Records; + +export type { Main as Profile } from "../../web/src/lib/api/lexicons/types/org/tangled/actor/profile"; +export type { Main as Comment } from "../../web/src/lib/api/lexicons/types/org/tangled/feed/comment"; +export type { Main as Follow } from "../../web/src/lib/api/lexicons/types/org/tangled/graph/follow"; +export type { Main as PublicKey } from "../../web/src/lib/api/lexicons/types/org/tangled/key/sshKey"; +export type { Main as LabelDefinition } from "../../web/src/lib/api/lexicons/types/org/tangled/label/definition"; +export type { Main as Declaration } from "../../web/src/lib/api/lexicons/types/org/tangled/repo/declaration"; +export type { Main as Patch } from "../../web/src/lib/api/lexicons/types/org/tangled/review/patch"; +export type { Main as Ticket } from "../../web/src/lib/api/lexicons/types/org/tangled/track/ticket"; +export type { Main as LegacyRepo } from "../../web/src/lib/api/lexicons/types/sh/tangled/repo"; +export type { Main as Artifact } from "../../web/src/lib/api/lexicons/types/sh/tangled/repo/artifact"; +export type { $input as PocketPreferences } from "../../web/src/lib/api/lexicons/types/com/bad-example/pocket/putPreferences"; diff --git a/e2e/harness/net.ts b/e2e/harness/net.ts new file mode 100644 index 0000000000000000000000000000000000000000..768c9346c4e14467d246500dcca44671a35f27e3 --- /dev/null +++ b/e2e/harness/net.ts @@ -0,0 +1,145 @@ +import { parseCanonicalResourceUri as partsOf } from "@atcute/lexicons/syntax"; +import { Agent, fetch } from "undici"; + +import { keep } from "./record"; + +// node can't resolve the stack's names or trust its dev ca +export const dispatcher = new Agent({ + connect: { + rejectUnauthorized: false, + lookup: (_host, options, callback) => { + const address = { address: "127.0.0.1", family: 4 }; + if (options.all) callback(null, [address]); + else callback(null, address.address, address.family); + } + } +}); + +export interface Reply { + status: number; + body: any; +} + +interface Options { + method?: "GET" | "POST"; + token?: string; + json?: unknown; + basic?: string; +} + +export const request = async ( + url: string, + { method = "GET", token, json, basic }: Options = {} +): Promise => { + const headers: Record = { + ...(token ? { authorization: `Bearer ${token}` } : {}), + ...(basic ? { authorization: `Basic ${Buffer.from(basic).toString("base64")}` } : {}), + ...(json === undefined ? {} : { "content-type": "application/json" }) + }; + const body = json === undefined ? undefined : JSON.stringify(json); + const started = Date.now(); + const sent = { by: "node" as const, method, url, request: { headers, body } }; + // without this a hung service holds the hop for undici's default of minutes + const response = await fetch(url, { + dispatcher, + signal: AbortSignal.timeout(10_000), + method, + headers, + body + }).catch((error) => { + const reason = error.cause?.code ?? error.cause ?? error.message; + keep({ ...sent, ms: Date.now() - started, error: String(reason) }); + // undici's "fetch failed" hides the real reason, which is on error.cause + throw new Error(`${method} ${url}: ${reason}`); + }); + const text = await response.text(); + keep({ + ...sent, + status: response.status, + ms: Date.now() - started, + response: { headers: Object.fromEntries(response.headers), body: text } + }); + return { status: response.status, body: parseBody(text) }; +}; + +export const ok = (what: string, reply: Reply, status = 200) => { + if (reply.status !== status) + throw new Error(`${what} answered ${reply.status} ${JSON.stringify(reply.body)}`); + return reply.body; +}; + +// a failing service may answer with plain text, not json +export const parseBody = (text: string): unknown => { + try { + return JSON.parse(text); + } catch { + return text; + } +}; + +// request() drops the headers, so this makes its own fetch +export const contentTypeOf = async (url: string): Promise => { + const response = await fetch(url, { dispatcher, signal: AbortSignal.timeout(10_000) }); + await response.arrayBuffer(); + return response.headers.get("content-type"); +}; + +export const sleep = (ms: number) => new Promise((wake) => setTimeout(wake, ms)); + +// the knot limits per account and the pds on logins, so each try waits longer +export const retry429 = async ( + go: () => Promise, + tries = 6, + step = 300 +) => { + for (let attempt = 1; ; attempt++) { + const reply = await go(); + if (reply.status !== 429 || attempt === tries) return reply; + await sleep(step * attempt); + } +}; + +export const get = (url: string) => request(url); + +export const xrpc = (base: string, nsid: string, params: Record = {}) => + `${base}/xrpc/${nsid}?${new URLSearchParams(params)}`; + +export interface Written { + uri: string; + cid: string; +} + +export { partsOf }; + +export const selfUri = (did: string, nsid: string) => `at://${did}/${nsid}/self`; + +export const recordUrl = (base: string, uri: string) => { + const { repo, collection, rkey } = partsOf(uri); + return xrpc(base, "com.atproto.repo.getRecord", { repo, collection, rkey }); +}; + +// stampedAt reads the time back out, so a sweep can spot leaked names +export const stamp = () => `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 5)}`; +export const stampedAt = (name: string) => { + const time = /-([0-9a-z]{8})[0-9a-z]{3}$/.exec(name)?.[1]; + return time ? parseInt(time, 36) : undefined; +}; + +// most waits end within a few tries, so the pauses start short +// a long poll already waited on the server, so it skips the pause +export const until = async ( + what: string, + budget: number, + check: () => Promise<[done: boolean, saw: string, value?: T]>, + { longPoll = false } = {} +) => { + const deadline = Date.now() + budget; + for (let pause = 25; ; pause = Math.min(pause * 2, 200)) { + const asked = Date.now(); + const [done, saw, value] = await check(); + if (done) return value as T; + if (Date.now() > deadline) + throw new Error(`${what}, gave up after ${budget / 1000}s: ${saw}`); + if (!longPoll || Date.now() - asked < 1_000) await sleep(pause); + } +}; diff --git a/e2e/harness/processes.ts b/e2e/harness/processes.ts new file mode 100644 index 0000000000000000000000000000000000000000..07a6642a832403f13a6edac2c816e0c5554837f3 --- /dev/null +++ b/e2e/harness/processes.ts @@ -0,0 +1,91 @@ +import { request } from "node:http"; +import { DatabaseSync } from "node:sqlite"; + +import { until } from "./net"; + +const onPc = process.env.E2E_STACK === "pc"; + +const stateDir = () => { + const state = process.env.E2E_PC_STATE; + if (!onPc || !state) throw new Error("this needs the pc stack, run it with E2E_STACK=pc"); + return state; +}; + +interface Processor { + status: string; + is_ready: string; +} + +const pc = (method: string, path: string) => + new Promise<{ status: number; body: string }>((done, fail) => { + request({ socketPath: `${stateDir()}/run/pc.sock`, method, path }, (res) => { + let body = ""; + res.on("data", (chunk) => (body += chunk)); + res.on("end", () => done({ status: res.statusCode ?? 0, body })); + }) + .on("error", fail) + .end(); + }); + +const processOf = async (name: string): Promise => { + const { status, body } = await pc("GET", `/process/${name}`); + if (status !== 200) throw new Error(`process-compose answered ${status} for ${name}: ${body}`); + return JSON.parse(body); +}; + +const command = async (method: string, verb: string, name: string) => { + const { status, body } = await pc(method, `/process/${verb}/${name}`); + if (status !== 200) + throw new Error(`process-compose could not ${verb} ${name}: ${status} ${body}`); +}; + +export const stopProcess = async (name: string) => { + await command("PATCH", "stop", name); + await until(`${name} should have stopped`, 30_000, async () => { + const { status } = await processOf(name); + return [ + status !== "Running" && status !== "Launching" && status !== "Terminating", + `it is ${status}` + ]; + }); +}; + +// use started when the next step has to kill it mid-startup +export const startProcess = async (name: string, upTo: "started" | "ready" = "ready") => { + await command("POST", "start", name); + await until(`${name} should be ${upTo}`, 90_000, async () => { + const { status, is_ready } = await processOf(name); + const there = upTo === "ready" ? is_ready === "Ready" : status === "Running"; + return [there, `it is ${status}, ${is_ready}`]; + }); +}; + +export const isReady = async (name: string) => (await processOf(name)).is_ready === "Ready"; + +// deliberi has no resend key here, so the token comes from its db instead of a mail +export const signupToken = (email: string) => { + const db = new DatabaseSync(`${stateDir()}/data/deliberi/deliberi.db`, { readOnly: true }); + try { + const row = db + .prepare("select verification_token as token from signups_inflight where email = ?") + .get(email); + if (typeof row?.token !== "string") throw new Error(`deliberi has no signup for ${email}`); + return row.token; + } finally { + db.close(); + } +}; + +// stop spindle first, it reads this at boot and keeps writing it +export const rewindJetstreamCursor = (microseconds: number) => { + const db = new DatabaseSync(`${stateDir()}/data/spindle/spindle.db`); + try { + db.exec("pragma busy_timeout = 10000"); + db.prepare("update _jetstream set last_time_us = ? where id = 1").run(microseconds); + const moved = db.prepare("select last_time_us as at from _jetstream where id = 1").get(); + if (moved?.at !== microseconds) + throw new Error(`spindle has no saved cursor to rewind: ${JSON.stringify(moved)}`); + } finally { + db.close(); + } +}; diff --git a/e2e/harness/record.ts b/e2e/harness/record.ts new file mode 100644 index 0000000000000000000000000000000000000000..d98b33f4d95fc659efc8e5f5cfb497f9742234fd --- /dev/null +++ b/e2e/harness/record.ts @@ -0,0 +1,163 @@ +import { AsyncLocalStorage } from "node:async_hooks"; + +import type { BrowserContext, Request, TestStepInfo } from "@playwright/test"; + +import { config } from "./config"; + +export interface Exchange { + by: "node" | "browser"; + method: string; + url: string; + status?: number; + ms: number; + request?: { headers: Record; body?: string }; + response?: { headers: Record; body?: string }; + error?: string; + at?: number; +} + +export interface StepRecord { + notes: Record; + exchanges: Exchange[]; + bytes: number; + dropped: number; + pending: Set>; +} + +const BODY_CAP = 64 * 1024; +const STEP_CAP = 512 * 1024; + +const isLocal = (url: string) => { + const host = new URL(url).hostname; + return host === "localhost" || host === "127.0.0.1" || host.endsWith(".tngl.boltless.dev"); +}; + +// any non local config drops bodies and masks secrets in the url +const keepsBodies = Object.values(config) + .filter((value): value is string => typeof value === "string" && /^https?:\/\//.test(value)) + .every(isLocal); + +const serviceHosts = new Set( + [ + config.pdsUrl, + config.knotUrl, + config.hydrantUrl, + config.bobbinUrl, + config.spindleUrl, + config.deliberiUrl + ].map((url) => new URL(url).host) +); + +const masked = (url: string) => + url.replace(/([?&][^=&]*(?:token|password|secret|code)[^=&]*=)[^&]*/gi, "$1…"); + +const clip = (text: string | undefined) => + text === undefined || text.length <= BODY_CAP + ? text + : `${text.slice(0, BODY_CAP)}\n… truncated, ${text.length} chars in all`; + +const clipped = (part: Exchange["request"]) => part && { ...part, body: clip(part.body) }; + +const steps = new AsyncLocalStorage(); +// browser events have no step async context, so they go to the running top step +let topStep: StepRecord | undefined; + +export const fresh = (): StepRecord => ({ + notes: {}, + exchanges: [], + bytes: 0, + dropped: 0, + pending: new Set() +}); + +const current = () => steps.getStore(); + +export const within = (record: StepRecord, run: () => T) => steps.run(record, run); + +// recording must never fail the step it watches +export const keep = (exchange: Exchange, record = current()) => { + try { + if (!record) return; + const sent = { ...exchange, at: Date.now() - exchange.ms }; + const kept: Exchange = keepsBodies + ? { ...sent, request: clipped(exchange.request), response: clipped(exchange.response) } + : { ...sent, url: masked(exchange.url), request: undefined, response: undefined }; + const size = JSON.stringify(kept).length; + if (record.bytes + size > STEP_CAP) { + record.dropped++; + return; + } + record.bytes += size; + record.exchanges.push(kept); + } catch {} +}; + +export const note = (key: string, value: unknown, record = current()) => { + if (record) record.notes[key] = value; +}; + +export const attach = async (info: TestStepInfo, record: StepRecord) => { + try { + await Promise.race([ + Promise.allSettled(record.pending), + new Promise((settled) => setTimeout(settled, 250)) + ]); + if (record.dropped) record.notes["requests not kept"] = record.dropped; + if (!Object.keys(record.notes).length && !record.exchanges.length) return; + const body = JSON.stringify({ notes: record.notes, exchanges: record.exchanges }); + await info.attach("e2e-step", { body, contentType: "application/json" }); + } catch {} +}; + +export const recorded = async ( + info: TestStepInfo, + run: () => Promise, + pageUrl: () => string | undefined +) => { + const record = fresh(); + topStep = record; + try { + return await within(record, run); + } finally { + const url = pageUrl(); + if (url && url !== "about:blank") note("page", url, record); + await attach(info, record); + } +}; + +// asset requests would drown the service calls, so only those are kept +export const watchBrowser = (context: BrowserContext) => { + const finished = (request: Request) => { + const record = topStep; + if (!record || !serviceHosts.has(new URL(request.url()).host)) return; + const ms = request.timing().responseEnd; + const filled = (async () => { + const response = await request.response(); + keep( + { + by: "browser", + method: request.method(), + url: request.url(), + status: response?.status(), + ms: ms >= 0 ? Math.round(ms) : 0, + request: { + headers: await request.allHeaders(), + body: request.postData() ?? undefined + }, + response: response + ? { + headers: await response.allHeaders(), + body: await response.text().catch(() => undefined) + } + : undefined, + error: request.failure()?.errorText + }, + record + ); + })().catch(() => {}); + record.pending.add(filled); + filled.finally(() => record.pending.delete(filled)); + }; + context.on("requestfinished", finished); + context.on("requestfailed", finished); +}; diff --git a/e2e/harness/report/ExchangeRow.svelte b/e2e/harness/report/ExchangeRow.svelte new file mode 100644 index 0000000000000000000000000000000000000000..4d7355353fe681aea073d200b40ca096dbaa91d0 --- /dev/null +++ b/e2e/harness/report/ExchangeRow.svelte @@ -0,0 +1,214 @@ + + + +{#snippet body(text: string)} +
+
+ body · {bytes(text)} + +
+ 
{pretty(text)}
+ + +
+{/snippet} + +{#snippet message(label: string, part: Exchange["request"])} + {@const headers = Object.entries(part?.headers ?? {})} + {#if headers.length || part?.body} +
+ {label} + {#if headers.length} + 
{headers
+						.map(([name, value]) => `${name}: ${value}`)
+						.join("\n")}
+ {/if} + {#if part?.body} + {@render body(part.body)} + {/if} +
+ {/if} +{/snippet} + +
  • +
    + + +
    +
    + {exchange.url} + +
    + {@render message("request", exchange.request)} + {@render message("response", exchange.response)} +
    +
    +
    • + + diff --git a/e2e/harness/report/Report.svelte b/e2e/harness/report/Report.svelte new file mode 100644 index 0000000000000000000000000000000000000000..18370c77544b219d244e26df0fd46e7b8039df30 --- /dev/null +++ b/e2e/harness/report/Report.svelte @@ -0,0 +1,755 @@ + + +{#snippet cause(flow: FlowData)} + {#if where(flow)}{where(flow)}{" · "}{/if}{brief(reason(flow))} +{/snippet} + +{#snippet stepLine(flow: FlowData, step: StepData, depth: number, tag = "div")} + {@const note = rowNote(flow, step)} + + + + + 0 && "typography-paragraph-small", + step.failed && "font-medium text-foreground-danger-strong" + ]} + title={step.title}>{step.title} + {#if note} + + {/if} + {#if tag === "summary"} +     + {/if} + {elapsed(step.duration)} + +{/snippet} + +{#snippet stepRow(flow: FlowData, step: StepData, depth = 0)} + {@const recorded = hasRecord(step)} +
  • + {#if step.children.length || recorded} +
    0)}> + {@render stepLine(flow, step, depth, "summary")} + {#if recorded} + {@render stepRecord(step, depth)} + {/if} + {#if step.children.length} +
      + {#each step.children as child, index (index)} + {@render stepRow(flow, child, depth + 1)} + {/each} +
    + {/if} +
    + {:else} + {@render stepLine(flow, step, depth)} + {/if} +
    • +{/snippet} + +{#snippet stepRecord(step: StepData, depth: number)} + {@const facts = factsOf(step)} +
    + {#if facts.length} +
    + {#each facts as [key, value] (key)} +
    {key}
    +
    + {value} +
    + {/each} +
    + {/if} + {#if step.exchanges?.length} +
    + {step.exchanges.length} {plural(step.exchanges.length, "request", "requests")} + +
    +
      + {#each step.exchanges as exchange, index (index)} + + {/each} +
    + {/if} +
    +{/snippet} + +{#snippet flowCard( + spec: SpecData, + flow: FlowData, + open: boolean, + place: "group" | "alone" | "spec" = "alone" +)} + {@const { hidden, shown } = split(flow)} + {@const failingStep = flow.steps.find((step) => step.failed)} + {@const failing = failingStep?.location} +
    + + + {#if flow.steps.length} +
      + {#if hidden.length} +
    1. +
      + + +
        + {#each hidden as step, index (index)} + {@render stepRow(flow, step)} + {/each} +
      +
      +
      2. + {/if} + {#each shown as step, index (index)} + {@render stepRow(flow, step)} + {/each} +
    + {/if} + {#if broken(flow)} +
    + {#if failing} +
    + failed at + {failing.file}:{failing.line} + + {#if failingStep?.via} + through + {failingStep.via.file}:{failingStep.via.line} + {/if} +
    + {/if} + {#if flow.failedStep.length > 1} + + {/if} + {#if flow.error} + 
    {flow.error}
    + {/if} + {#if flow.trace} + pnpm exec playwright show-trace {flow.trace} + {/if} +
    + {/if} +
    +{/snippet} + +
    +
    +

    e2e run

    +
    + {#if outcomes.length > 1} + + {/if} + {#each outcomes as outcome (outcome.label)} + {#snippet face()} + + {outcome.count} {outcome.label} + {/snippet} + {#if outcome.show} + + {:else} + + {@render face()} + + {/if} + {/each} +
    + + +
    + + + +
    +
    +
    + + + + + + +
    + + {#each data.errors as message, index (index)} + + {/each} + +
    +

    + {failures.length} + {plural(failures.length, "failure", "failures")} + {#if groups.length < failures.length} + in {groups.length} groups + {/if} +

    + {#each shared as group, index (index)} + +
    + {group.length} flows + failed the same way: + {@render cause(group[0].flow)} +
    +
    + {#each group.slice(0, GROUP_SHOWN) as { spec, flow }, position (anchor(spec.name, flow))} + {@render flowCard(spec, flow, position === 0 && failures.length <= OPEN_UP_TO, "group")} + {/each} + {#if group.length > GROUP_SHOWN} + {@const more = group.length - GROUP_SHOWN} +
    + + +
    + {#each group.slice(GROUP_SHOWN) as { spec, flow } (anchor(spec.name, flow))} + {@render flowCard(spec, flow, false, "group")} + {/each} +
    +
    + {/if} +
    +     + {/each} + {#if alone.length} + +
    + {#each alone as { spec, flow } (anchor(spec.name, flow))} + {@render flowCard(spec, flow, failures.length <= OPEN_UP_TO)} + {/each} +
    +     + {/if} + {#if !failures.length} + + {/if} +
    + +
    +

    specs

    + {#each data.specs as spec (spec.name)} + {@const passed = spec.flows.filter((flow) => icons[flow.status] === "success").length} +
    + + + +
    + {#each failedFirst(spec) as flow (flow.title)} + {@render flowCard(spec, flow, false, "spec")} + {/each} +
    +     +
    + {/each} +
    + +
    +

    slowest flows

    + +
    + {#each slowest as { spec, flow } (anchor(spec.name, flow))} + {@const share = flow.duration / Math.max(slowest[0].flow.duration, 1)} +
    + + + + {spec.name} › + {flow.title} + + {elapsed(flow.duration)} +
    + {/each} +
    +     +
    +
    + + diff --git a/e2e/harness/report/data.ts b/e2e/harness/report/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..61a611423cdfb37d9b8d4938ac75cc0196f1db77 --- /dev/null +++ b/e2e/harness/report/data.ts @@ -0,0 +1,50 @@ +import type { Exchange } from "../record"; + +export type { Exchange }; + +export type FlowStatus = + "passed" | "flaky" | "failed" | "timedOut" | "interrupted" | "skipped" | "did not run"; + +export interface CodeLocation { + file: string; + line: number; +} + +export interface StepData { + title: string; + start: number; + duration: number; + failed: boolean; + location?: CodeLocation; + via?: CodeLocation; + subtitle?: string; + subject?: string; + params?: Record; + notes?: Record; + exchanges?: Exchange[]; + children: StepData[]; +} + +export interface FlowData { + title: string; + status: FlowStatus; + duration: number; + attempts: number; + steps: StepData[]; + failedStep: string[]; + error?: string; + trace?: string; +} + +export interface SpecData { + name: string; + flows: FlowData[]; +} + +export interface ReportData { + root: string; + started: number; + duration: number; + specs: SpecData[]; + errors: string[]; +} diff --git a/e2e/harness/report/render.ts b/e2e/harness/report/render.ts new file mode 100644 index 0000000000000000000000000000000000000000..38d862e4ec1f84c2978ec77668dd9df460873ede --- /dev/null +++ b/e2e/harness/report/render.ts @@ -0,0 +1,148 @@ +import { existsSync } from "node:fs"; +import { readFile, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import type { ReportData } from "./data"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const web = path.resolve(here, "../../../web"); +const modules = path.resolve(process.env.E2E_WEB_MODULES ?? path.join(web, "node_modules")); +const fonts = path.join(web, "static"); + +interface Resolver { + resolve: (id: string, importer: string | undefined, options: object) => Promise; +} + +const bare = /^(?![./])(@[\w.-]+\/)?[\w.-]+(\/|$)/; + +const [input, out] = process.argv.slice(2); +if (!existsSync(path.join(modules, "vite"))) { + console.error( + `${modules} has no vite, run pnpm install in web/ or pass --web-modules=` + ); + process.exit(1); +} + +const require = createRequire(path.join(modules, "noop.js")); +const load = (name: string, from = require) => import(pathToFileURL(from.resolve(name)).href); + +// tailwind only searches up from web/src, so it misses packages installed elsewhere +// take the esm entry, cjs gives tailwind a module object instead of the plugin +const entry = (condition: "style" | "import") => async (id: string) => { + if (!bare.test(id)) return undefined; + const manifest = path.join(modules, id, "package.json"); + const pkg = JSON.parse(await readFile(manifest, "utf8")); + const fallback = condition === "style" ? pkg.style : (pkg.module ?? pkg.main); + return path.join(path.dirname(manifest), pkg.exports?.["."]?.[condition] ?? fallback); +}; + +const renderHtml = async (data: ReportData) => { + const { createServer } = await load("vite"); + const { svelte } = await load("@sveltejs/vite-plugin-svelte"); + const { default: icons } = await load("unplugin-icons/vite"); + const server = await createServer({ + configFile: false, + root: web, + logLevel: "error", + // vite's default cache would land in web/node_modules + cacheDir: path.join(tmpdir(), "e2e-report-vite"), + appType: "custom", + server: { middlewareMode: true, hmr: false, ws: false, watch: null }, + optimizeDeps: { noDiscovery: true, include: [] }, + ssr: { noExternal: true }, + // the tsconfig extends .svelte-kit, which only exists after svelte-kit sync + oxc: { tsconfig: false }, + resolve: { + alias: [ + { find: /^\$lib\//, replacement: `${web}/src/lib/` }, + { find: /^\$icon\//, replacement: "~icons/lucide/" } + ] + }, + plugins: [ + // web/ may have no node_modules, so bare imports fall back to ours + { + name: "web-modules", + enforce: "pre", + async resolveId( + this: Resolver, + id: string, + importer: string | undefined, + options: object + ) { + if (!bare.test(id)) return null; + const own = await this.resolve(id, importer, { ...options, skipSelf: true }); + const fallback = path.join(modules, "noop.js"); + return own ?? this.resolve(id, fallback, { ...options, skipSelf: true }); + } + }, + svelte({ + configFile: false, + compilerOptions: { css: "injected", runes: true, experimental: { async: true } } + }), + icons({ compiler: "svelte", collectionsNodeResolvePath: path.dirname(modules) }) + ] + }); + try { + const { render } = await server.ssrLoadModule("svelte/server"); + const { default: Report } = await server.ssrLoadModule(path.join(here, "Report.svelte")); + return await render(Report, { props: { data } }); + } finally { + await server.close(); + } +}; + +const buildCss = async (html: string) => { + const node = await load( + "@tailwindcss/node", + createRequire(require.resolve("@tailwindcss/vite")) + ); + const compiler = await node.compile(`@import "${web}/src/app.css";`, { + base: path.join(web, "src"), + onDependency: () => {}, + customCssResolver: entry("style"), + customJsResolver: entry("import") + }); + const candidates = new Set(); + for (const [, list] of html.matchAll(/class="([^"]*)"/g)) + for (const name of list.split(/\s+/)) if (name) candidates.add(name); + return inlineFonts(compiler.build([...candidates])); +}; + +// the report opens from disk so /fonts urls miss, and italics are cut for size +const inlineFonts = async (css: string) => { + const faces = css.match(/@font-face\s*{[^}]*}/g) ?? []; + for (const face of faces) { + const url = face.match(/url\("?(\/fonts\/[^")]+)"?\)/)?.[1]; + const file = url && path.join(fonts, url); + if (/font-style:\s*italic/.test(face) || !file || !existsSync(file)) { + css = css.replace(face, ""); + continue; + } + const data = `data:font/woff2;base64,${(await readFile(file)).toString("base64")}`; + css = css.replace(face, face.replace(url, data)); + } + return css; +}; + +const data: ReportData = JSON.parse(await readFile(input, "utf8")); +const { head, body } = await renderHtml(data); +const css = await buildCss(body); +const script = await readFile(path.join(here, "search.js"), "utf8"); +await writeFile( + out, + ` + + + + +e2e report + +${head} + +${body} + +` +); diff --git a/e2e/harness/report/search.js b/e2e/harness/report/search.js new file mode 100644 index 0000000000000000000000000000000000000000..c994fea5a5dfcbcbff14a3d38f7892c53142f39c --- /dev/null +++ b/e2e/harness/report/search.js @@ -0,0 +1,366 @@ +// inlined at the end of the report so the elements exist before it looks them up +const input = document.getElementById("search"); +const count = document.getElementById("search-count"); +const show = document.getElementById("show"); +const flows = [...document.querySelectorAll("[data-search]")]; +const folds = [...document.querySelectorAll("[data-fold], [data-search]")]; +const total = new Set(flows.map((flow) => flow.dataset.key)).size; +for (const fold of folds) fold.dataset.drawn = fold.open ? "open" : ""; + +// search ignores the show filter, which comes back once the box is empty +let filtered; +const apply = () => { + const query = input.value.trim().toLowerCase(); + if (query && filtered === undefined) { + filtered = show.value; + show.value = "all"; + } else if (!query && filtered !== undefined) { + show.value = filtered; + filtered = undefined; + } + pressed(); + for (const flow of flows) flow.hidden = query !== "" && !textOf(flow).includes(query); + const found = flows.filter((flow) => !flow.hidden); + // flows repeat once per failure, so count unique keys + const matches = new Set(found.map((flow) => flow.dataset.key)).size; + for (const fold of folds) { + // only open found flows when there are five or fewer, or the page becomes a wall of cards + const wanted = fold.matches("[data-search]") + ? matches <= 5 + : found.some((flow) => fold.contains(flow)); + fold.open = query ? wanted : fold.dataset.drawn === "open"; + } + for (const section of document.querySelectorAll("[data-section]")) + section.hidden = query !== "" && !section.querySelector("[data-search]:not([hidden])"); + for (const group of document.querySelectorAll("[data-group]")) { + const shown = group.querySelectorAll("[data-search]:not([hidden])").length; + group.hidden = query !== "" && shown === 0; + const label = group.querySelector("[data-count]"); + if (label) + label.textContent = query + ? `${shown} of ${label.dataset.count} flows` + : `${label.dataset.count} flows`; + } + const visible = new Set( + found.filter((flow) => flow.checkVisibility()).map((flow) => flow.dataset.key) + ); + count.textContent = query ? `${visible.size} of ${total} match` : ""; + document.getElementById("search-empty").hidden = !query || visible.size > 0; + reveal(query ? found.filter((flow) => flow.open && flow.checkVisibility()) : [], query); +}; + +// raw bodies can be huge, so this only gets built once a search starts +const index = new Map(); +const textOf = (flow) => { + if (!index.has(flow)) { + const raws = [...flow.querySelectorAll("[data-raw]")].map((pre) => JSON.parse(pre.dataset.raw)); + index.set(flow, [flow.dataset.search, flow.textContent, ...raws].join("\n").toLowerCase()); + } + return index.get(flow); +}; + +const places = [ + ["[data-body]", "body"], + ["[data-request] pre", "header"], + ["pre", "error"], + ["[data-request] > summary, [data-request] .break-all", "url"], + ["dl", "step facts"], + ["[data-search] > summary", "flow title"], + ["summary", "step title"] +]; +let opened = []; + +// the walker only sees text that is drawn, so json bodies get grown first +const reveal = (shown, query) => { + for (const fold of opened) fold.open = false; + opened = []; + const ranges = []; + for (const flow of shown) { + const hits = new Set(); + for (const body of flow.querySelectorAll("[data-body]")) { + if (!body.querySelector("[data-pretty]").textContent.toLowerCase().includes(query)) continue; + drawBody(body); + growAll(body.querySelector("[data-tree]")); + } + const walker = document.createTreeWalker(flow, NodeFilter.SHOW_TEXT); + for (let text = walker.nextNode(); text; text = walker.nextNode()) { + const lower = text.data.toLowerCase(); + const parent = text.parentElement; + if (!lower.includes(query) || parent.closest("[data-hit]")) continue; + const tail = parent.closest("[data-tail]"); + if (tail) { + tail.hidden = false; + tail.nextElementSibling?.remove(); + } + if (parent.closest("[hidden]")) continue; + for (let at = lower.indexOf(query); at >= 0; at = lower.indexOf(query, at + query.length)) { + const range = new Range(); + range.setStart(text, at); + range.setEnd(text, at + query.length); + ranges.push(range); + } + for (let fold = parent.closest("details"); fold && fold !== flow;) { + if (!fold.open) { + fold.open = true; + opened.push(fold); + } + fold = fold.parentElement.closest("details"); + } + const place = places.find(([selector]) => parent.closest(selector)); + hits.add(place && flow.contains(parent.closest(place[0])) ? place[1] : "flow"); + } + const hint = flow.querySelector("[data-hit]"); + hint.textContent = hits.size ? `matched in ${[...hits].join(", ")}` : ""; + hint.hidden = !hits.size; + } + for (const hint of document.querySelectorAll("[data-hit]")) + if (!shown.some((flow) => flow.contains(hint))) hint.hidden = true; + if (window.CSS?.highlights) CSS.highlights.set("search", new Highlight(...ranges)); +}; + +// clipboard api is refused on file:// pages, so fall back to a hidden textarea +const copy = async (text) => { + try { + await navigator.clipboard.writeText(text); + return; + } catch { + const area = document.createElement("textarea"); + area.value = text; + area.style.cssText = "position:fixed;opacity:0"; + document.body.append(area); + area.select(); + document.execCommand("copy"); + area.remove(); + } +}; + +const copied = (element) => { + element.dataset.copied = ""; + clearTimeout(element.copyTimer); + element.copyTimer = setTimeout(() => delete element.dataset.copied, 1200); +}; + +document.addEventListener("click", (event) => { + const target = event.target.closest?.("[data-loc], [data-copy], [data-copy-all]"); + if (!target) return; + event.preventDefault(); + const text = target.hasAttribute("data-copy-all") + ? [...target.parentElement.nextElementSibling.querySelectorAll("[data-copy]")] + .map((button) => button.dataset.copy) + .join("\n\n") + : (target.dataset.copy ?? (event.shiftKey ? target.dataset.abs : target.dataset.loc)); + copy(text).then(() => copied(target)); +}); + +const caret = document.getElementById("json-caret").content.firstElementChild; +const tools = document.getElementById("body-tools").content.firstElementChild; +const STRING_CAP = 300; +const FOLD_FROM = 2; + +const element = (tag, className, text) => { + const node = document.createElement(tag); + if (className) node.className = className; + if (text !== undefined) node.textContent = text; + return node; +}; + +const rawOf = (body) => { + const pretty = body.querySelector("[data-pretty]"); + if (pretty.dataset.raw) return JSON.parse(pretty.dataset.raw); + try { + return JSON.stringify(JSON.parse(pretty.textContent)); + } catch { + return pretty.textContent; + } +}; + +const leaf = (value) => { + if (value === null) return element("span", "nul", "null"); + if (typeof value === "number") return element("span", "num", String(value)); + if (typeof value === "boolean") return element("span", "bool", String(value)); + const text = JSON.stringify(value); + const span = element("span", "str", text.slice(0, STRING_CAP)); + if (text.length > STRING_CAP) { + const tail = element("span", "", text.slice(STRING_CAP)); + tail.hidden = true; + tail.dataset.tail = ""; + const more = element("button", "more", `show all, ${text.length - STRING_CAP} more`); + more.type = "button"; + more.dataset.more = ""; + span.append(tail, more); + } + return span; +}; + +const preview = (value) => { + const brief = (item) => + item !== null && typeof item === "object" + ? Array.isArray(item) + ? "[…]" + : "{…}" + : typeof item === "string" + ? JSON.stringify(item.length > 24 ? `${item.slice(0, 24)}…` : item) + : String(item); + const entries = Array.isArray(value) + ? value.slice(0, 4).map(brief) + : Object.entries(value) + .slice(0, 4) + .map(([key, item]) => `${key}: ${brief(item)}`); + const more = Object.keys(value).length > 4 ? ", …" : ""; + return Array.isArray(value) ? `[${entries.join(", ")}${more}]` : `{${entries.join(", ")}${more}}`; +}; + +const row = (key, tag = "div") => { + const line = element(tag, "row"); + if (key !== undefined) line.append(element("span", "key", key)); + return line; +}; + +const node = (key, value, depth) => { + if (value === null || typeof value !== "object") { + const line = row(key); + line.prepend(element("span", "leaf")); + line.append(leaf(value)); + return line; + } + const count = Object.keys(value).length; + const fold = element("details"); + const summary = row(key, "summary"); + summary.prepend(caret.cloneNode(true)); + const peek = element("span", "", preview(value)); + peek.dataset.preview = ""; + summary.append( + element("span", "punct", Array.isArray(value) ? `[ ${count} ]` : `{ ${count} }`), + peek + ); + fold.append(summary); + fold.json = value; + fold.depth = depth; + if (depth < FOLD_FROM) grow(fold); + return fold; +}; + +const grow = (fold) => { + if (fold.grown) return; + fold.grown = true; + fold.open = true; + const kids = element("div", "kids"); + for (const [key, value] of Object.entries(fold.json)) + kids.append(node(key, value, fold.depth + 1)); + fold.append(kids); +}; + +const growAll = (tree) => { + let closed; + while ((closed = [...tree.querySelectorAll("details")].filter((fold) => !fold.grown)).length) + for (const fold of closed) grow(fold); +}; + +const drawBody = (body) => { + if (body.drawn) return; + body.drawn = true; + body.querySelector("[data-body-tools]").replaceWith(tools.cloneNode(true)); + const pretty = body.querySelector("[data-pretty]"); + let value; + try { + value = JSON.parse(pretty.textContent); + } catch { + body.querySelector("[data-json-only]").hidden = true; + return; + } + const tree = body.querySelector("[data-tree]"); + const top = node(undefined, value, 0); + tree.append(top); + body.querySelector("[data-raw-view]").textContent = rawOf(body); + pretty.hidden = true; + tree.hidden = false; +}; + +const drawIn = (scope) => { + for (const body of scope.querySelectorAll("[data-body]")) drawBody(body); +}; + +const view = (body, which) => { + body.querySelector("[data-tree]").hidden = which !== "tree"; + body.querySelector("[data-raw-view]").hidden = which !== "raw"; + for (const button of body.querySelectorAll("[data-view]")) + button.setAttribute("aria-pressed", String(button.dataset.view === which)); +}; + +document.addEventListener( + "toggle", + (event) => { + const target = event.target; + if (!target.open) return; + if (target.matches("[data-request]")) drawIn(target); + else if (target.closest("[data-tree]") && target.json) grow(target); + }, + true +); +for (const request of document.querySelectorAll("[data-request][open]")) drawIn(request); + +document.addEventListener("click", (event) => { + const target = event.target.closest?.( + "[data-view], [data-unfold], [data-fold-all], [data-copy-body], [data-more]" + ); + if (!target) return; + event.preventDefault(); + const body = target.closest("[data-body]"); + if (target.dataset.view) view(body, target.dataset.view); + else if (target.hasAttribute("data-unfold")) { + view(body, "tree"); + growAll(body.querySelector("[data-tree]")); + for (const fold of body.querySelectorAll("[data-tree] details")) fold.open = true; + } else if (target.hasAttribute("data-fold-all")) { + view(body, "tree"); + for (const fold of body.querySelectorAll("[data-tree] details")) fold.open = false; + body.querySelector("[data-tree] details").open = true; + } else if (target.hasAttribute("data-more")) { + target.previousElementSibling.hidden = false; + target.remove(); + } else copy(rawOf(body)).then(() => copied(target)); +}); + +// editors open a dropped file:// uri, text/plain carries the line for the ones that read it +document.addEventListener("dragstart", (event) => { + const target = event.target.closest?.("[data-loc]"); + if (!target) return; + const [file] = target.dataset.abs.split(/:(?=\d+$)/); + event.dataTransfer.setData("text/uri-list", `file://${file}`); + event.dataTransfer.setData("text/plain", target.dataset.abs); +}); + +for (const button of document.querySelectorAll("[data-copy], [data-copy-all]")) + button.hidden = false; + +document.getElementById("search-box").classList.replace("hidden", "flex"); +const pressed = () => { + for (const button of document.querySelectorAll("[data-show]")) + button.setAttribute("aria-pressed", String(button.dataset.show === show.value)); +}; +document.addEventListener("click", (event) => { + const target = event.target.closest?.("[data-show]"); + if (!target) return; + show.value = target.dataset.show; + show.dispatchEvent(new Event("change")); +}); +pressed(); +let typing; +input.addEventListener("input", () => { + clearTimeout(typing); + typing = setTimeout(apply, 120); +}); +show.addEventListener("change", () => { + filtered = undefined; + apply(); +}); +document.addEventListener("keydown", (event) => { + if (event.key === "/" && document.activeElement !== input) { + event.preventDefault(); + input.focus(); + } else if (event.key === "Escape" && document.activeElement === input) { + input.value = ""; + apply(); + input.blur(); + } +}); diff --git a/e2e/harness/report/write.ts b/e2e/harness/report/write.ts new file mode 100644 index 0000000000000000000000000000000000000000..52b2cd2b8b10e514803afeb32a9ece20b7708d2f --- /dev/null +++ b/e2e/harness/report/write.ts @@ -0,0 +1,89 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import type { Exchange, ReportData, StepData } from "./data"; + +// the render loads web's svelte and tailwind, so it runs in a separate node +export const writeHtmlReport = async (out: string, data: ReportData) => { + const dir = await mkdtemp(path.join(tmpdir(), "e2e-report-")); + const input = path.join(dir, "report.json"); + const render = path.join(path.dirname(fileURLToPath(import.meta.url)), "render.ts"); + try { + await writeFile(input, JSON.stringify(data)); + await promisify(execFile)(process.execPath, [render, input, path.resolve(out)], { + timeout: 120_000, + maxBuffer: 16 * 1024 * 1024 + }); + console.log(`html report: ${out}`); + } catch (error) { + const failure = error as { stderr?: string; message: string }; + const why = (failure.stderr || failure.message).trim().split("\n").slice(0, 20).join("\n"); + console.log(`no html report: ${why}`); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}; + +const pairs = (headers: Record = {}) => + Object.entries(headers).map(([name, value]) => ({ name, value })); + +// only what the steps kept is in the har, for devtools or any har viewer +export const writeHar = async (out: string, data: ReportData) => { + const exchanges = (steps: StepData[]): Exchange[] => + steps.flatMap((step) => [...(step.exchanges ?? []), ...exchanges(step.children)]); + const entries = data.specs + .flatMap((spec) => spec.flows.flatMap((flow) => exchanges(flow.steps))) + .sort((a, b) => (a.at ?? 0) - (b.at ?? 0)) + .map((exchange) => { + const url = new URL(exchange.url); + const type = exchange.response?.headers["content-type"] ?? ""; + return { + startedDateTime: new Date(exchange.at ?? 0).toISOString(), + time: exchange.ms, + request: { + method: exchange.method, + url: exchange.url, + httpVersion: "HTTP/1.1", + headers: pairs(exchange.request?.headers), + queryString: [...url.searchParams].map(([name, value]) => ({ name, value })), + cookies: [], + headersSize: -1, + bodySize: exchange.request?.body?.length ?? 0, + ...(exchange.request?.body !== undefined && { + postData: { + mimeType: exchange.request.headers["content-type"] ?? "", + text: exchange.request.body + } + }) + }, + response: { + status: exchange.status ?? 0, + statusText: exchange.error ?? "", + httpVersion: "HTTP/1.1", + headers: pairs(exchange.response?.headers), + cookies: [], + content: { + size: exchange.response?.body?.length ?? 0, + mimeType: type, + text: exchange.response?.body ?? "" + }, + redirectURL: "", + headersSize: -1, + bodySize: -1 + }, + cache: {}, + timings: { send: 0, wait: exchange.ms, receive: 0 } + }; + }); + try { + const log = { version: "1.2", creator: { name: "e2e", version: "1" }, entries }; + await writeFile(out, JSON.stringify({ log })); + console.log(`har: ${out}`); + } catch (error) { + console.log(`no har: ${(error as Error).message}`); + } +}; diff --git a/e2e/harness/reporter.ts b/e2e/harness/reporter.ts new file mode 100644 index 0000000000000000000000000000000000000000..194b7fafda6bca5227d9c0cbe500f9df270c124e --- /dev/null +++ b/e2e/harness/reporter.ts @@ -0,0 +1,271 @@ +import type { + Reporter, + Suite, + TestCase, + TestError, + TestResult, + TestStep +} from "@playwright/test/reporter"; + +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { stripVTControlCharacters, styleText } from "node:util"; + +import type { CodeLocation, FlowData, ReportData, SpecData, StepData } from "./report/data"; +import { writeHar, writeHtmlReport } from "./report/write"; + +// the stock list names the project and harness/flow.ts:272:2 on every line, which is where every flow is registered + +const green = (text: string) => styleText("green", text); +const red = (text: string) => styleText("red", text); +const dim = (text: string) => styleText("dim", text); + +const specOf = (test: TestCase) => { + let suite: Suite | undefined = test.parent; + while (suite && suite.type !== "file") suite = suite.parent; + return (suite?.title ?? "").replace(/^.*\//, "").replace(/(\.spec)?\.ts$/, ""); +}; + +const titleOf = (test: TestCase) => test.titlePath().slice(3).join(" › "); + +const traceOf = (result: TestResult) => + result.attachments.find((attachment) => attachment.name === "trace")?.path; + +const seconds = (ms: number) => + ms < 60_000 ? `${(ms / 1000).toFixed(1)}s` : `${(ms / 60_000).toFixed(1)}m`; + +// the innermost step that threw, which is the one the flow was on +const failedStep = (steps: TestStep[]): string[] => { + for (const step of steps) { + if (step.category !== "test.step" || !step.error) continue; + return [step.title, ...failedStep(step.steps)]; + } + return []; +}; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +const place = (where: string, line: number): CodeLocation | undefined => { + // esm spec frames give file:// urls, not paths + const file = where.startsWith("file:") ? fileURLToPath(where) : where; + const relative = path.relative(root, file); + // a frame in flow.ts is the loop running every step, not the step + if (relative === "harness/flow.ts") return undefined; + return { file: relative.startsWith("..") ? file : relative, line }; +}; + +// flow.ts sets the via param to the helper's file:line +const via = (step: TestStep) => { + const found = /^(.*):(\d+)$/.exec(String(step.params?.via ?? "")); + return found ? place(found[1], Number(found[2])) : undefined; +}; + +// flow.ts and services.ts write this, see record.ts +const recordOf = (step: TestStep): Pick => { + const kept = step.attachments.find((attachment) => attachment.name === "e2e-step")?.body; + if (!kept) return {}; + try { + return JSON.parse(kept.toString()); + } catch { + return {}; + } +}; + +const subjectOf = (notes: Record = {}) => { + const repo = notes.repo as { name?: string } | undefined; + if (repo?.name) return repo.name; + const records = Object.entries(notes).filter( + (entry): entry is [string, { uri: string }] => + typeof (entry[1] as { uri?: unknown })?.uri === "string" + ); + const last = records.at(-1); + return last && `${last[0].split(".").at(-1)} ${last[1].uri.split("/").at(-1)}`; +}; + +// render.ts reads this json, so one chatty flow gets capped at 2mb +const FLOW_BUDGET = 2 * 1024 * 1024; + +// only top level steps get a location, the nested ones are all services.ts +const stepData = ( + steps: TestStep[], + start: number, + budget: { left: number }, + top = false +): StepData[] => + steps + .filter((step) => step.category === "test.step") + .map((step) => { + const { notes, exchanges } = recordOf(step); + const size = JSON.stringify(exchanges ?? []).length; + const fits = size <= budget.left; + if (fits) budget.left -= size; + const params = Object.entries(step.params ?? {}).filter(([key]) => key !== "via"); + return { + title: step.title, + subtitle: step.subtitle, + subject: subjectOf(notes), + start: step.startTime.getTime() - start, + duration: step.duration, + failed: !!step.error, + ...(top && { + location: step.location && place(step.location.file, step.location.line), + via: via(step) + }), + ...(params.length && { + params: Object.fromEntries(params.map(([k, v]) => [k, String(v)])) + }), + ...(notes && { notes }), + ...(exchanges?.length && + (fits + ? { exchanges } + : { + notes: { + ...notes, + requests: `${exchanges.length} left out, the flow kept 2MB` + } + })), + children: stepData(step.steps, start, budget) + }; + }); + +const flowData = (test: TestCase): FlowData => { + const title = titleOf(test); + const attempt = test.results.at(-1); + if (!attempt) + return { + title, + status: "did not run", + duration: 0, + attempts: 0, + steps: [], + failedStep: [] + }; + const outcome = test.outcome(); + const status = + attempt.status === "passed" ? (outcome === "flaky" ? "flaky" : "passed") : attempt.status; + const steps = stepData(attempt.steps, attempt.startTime.getTime(), { left: FLOW_BUDGET }, true); + const error = attempt.error?.message ?? attempt.error?.value; + return { + title, + status, + duration: attempt.duration, + attempts: test.results.length, + steps, + failedStep: failedStep(attempt.steps), + error: error && stripVTControlCharacters(error).split("\n").slice(0, 200).join("\n"), + trace: traceOf(attempt) + }; +}; + +interface Failure { + test: TestCase; + result: TestResult; +} + +export default class FlowReporter implements Reporter { + private html?: string; + private suite?: Suite; + private errors: string[] = []; + private total = 0; + private done = 0; + private passed = 0; + private flaky = 0; + private interrupted = 0; + private failures: Failure[] = []; + private started = Date.now(); + + constructor(options: { html?: string } = {}) { + this.html = options.html; + } + + printsToStdio() { + return true; + } + + onBegin(_config: unknown, suite: Suite) { + const tests = suite.allTests(); + this.suite = suite; + this.total = tests.length; + this.started = Date.now(); + if (!process.argv.includes("--list")) return; + for (const test of tests) console.log(` ${specOf(test)} › ${titleOf(test)}`); + console.log( + `Total: ${tests.length} tests in ${suite.suites.flatMap((project) => project.suites).length} files` + ); + } + + onStdOut(chunk: string | Buffer) { + process.stdout.write(chunk); + } + + onStdErr(chunk: string | Buffer) { + process.stderr.write(chunk); + } + + onTestEnd(test: TestCase, result: TestResult) { + const retrying = + result.status !== "passed" && + result.status !== "skipped" && + result.retry < test.retries; + if (retrying) return; + this.done++; + const ok = result.status === "passed"; + if (result.status === "interrupted") this.interrupted++; + else if (ok && result.retry > 0) this.flaky++; + else if (ok) this.passed++; + else this.failures.push({ test, result }); + const mark = ok ? green("✓") : result.status === "interrupted" ? dim("-") : red("✗"); + const counter = dim(`${this.done}/${this.total}`); + console.log( + ` ${mark} ${counter} ${specOf(test)} › ${titleOf(test)} ${dim(`(${seconds(result.duration)})`)}` + ); + } + + onError(error: TestError) { + const message = error.message ?? error.value ?? "error"; + this.errors.push(stripVTControlCharacters(message)); + console.log(red(message)); + } + + async onEnd() { + if (process.argv.includes("--list")) return; + for (const [index, { test, result: attempt }] of this.failures.entries()) { + const step = failedStep(attempt.steps); + const trace = traceOf(attempt); + const message = (attempt.error?.message ?? attempt.status) + .split("\n") + .slice(0, 25) + .join("\n"); + console.log(`\n${red(`${index + 1}) ${specOf(test)} › ${titleOf(test)}`)}`); + if (step.length) console.log(` at step: ${step.join(" › ")}`); + console.log(message.replace(/^/gm, " ")); + if (trace) console.log(dim(` pnpm exec playwright show-trace ${trace}`)); + } + const parts = [`${this.passed} passed`]; + if (this.flaky) parts.push(`${this.flaky} flaky`); + if (this.failures.length) parts.push(red(`${this.failures.length} failed`)); + if (this.interrupted) parts.push(`${this.interrupted} interrupted`); + if (this.done < this.total) parts.push(`${this.total - this.done} did not run`); + console.log(`\n${parts.join(", ")} ${dim(`(${seconds(Date.now() - this.started)})`)}`); + const report = this.html || process.env.E2E_REPORT_HAR ? this.report() : undefined; + if (report && this.html) await writeHtmlReport(this.html, report); + if (report && process.env.E2E_REPORT_HAR) + await writeHar(process.env.E2E_REPORT_HAR, report); + } + + private report(): ReportData { + const specs = new Map(); + for (const test of this.suite?.allTests() ?? []) { + const name = specOf(test); + if (!specs.has(name)) specs.set(name, { name, flows: [] }); + specs.get(name)!.flows.push(flowData(test)); + } + return { + root, + started: this.started, + duration: Date.now() - this.started, + specs: [...specs.values()], + errors: this.errors + }; + } +} diff --git a/e2e/harness/run.ts b/e2e/harness/run.ts new file mode 100644 index 0000000000000000000000000000000000000000..cfb3005900023d64bd5d022086998fcf4fdd66e9 --- /dev/null +++ b/e2e/harness/run.ts @@ -0,0 +1,349 @@ +import type { Did } from "@atcute/lexicons/syntax"; +import { expect, type Browser } from "@playwright/test"; + +import { accounts, knotRecordFor, pulls, repos, tickets, type RepoOptions } from "./api"; +import { config, isSeeded, type Account } from "./config"; +import { Tidy, type Chore } from "./chores"; +import { push } from "./git"; +import { selfUri, stamp, type Written } from "./net"; +import { note } from "./record"; +import { bobbin, lookBefore, refsNaming, settle, type Hop } from "./services"; +import { Tabs, type Held } from "./tabs"; +import { + declarationId, + manifestId, + refsId, + repoAt, + unique, + type Ctx, + type Repo, + type Slot +} from "./vocabulary"; + +class People { + private labelled = new Map(); + private members = new Set(); + private throughThePage = new Set(); + private met = new Map(); + who?: Account; + + constructor(private tidy: Tidy) {} + + get actor() { + if (!this.who) throw new Error("this step needs an .actor() before it"); + return this.who; + } + + async switchTo(who: string | Account, options: { login?: "ui" } = {}) { + if (typeof who !== "string") { + this.met.set(who.did, who); + this.who = who; + return; + } + let account = this.labelled.get(who); + if (!account) { + const made = await accounts.create(who); + this.labelled.set(who, made); + this.tidy.add(`account ${made.handle}`, { kind: "account", account: made }); + account = made; + } + if (options.login === "ui") this.throughThePage.add(account.name); + this.met.set(account.did, account); + this.who = account; + } + + get everyone() { + return [...this.met.values()]; + } + + signsInThroughThePage(account: Account) { + return config.login === "ui" || this.throughThePage.has(account.name); + } + + // the knot only lets its members make repos + async ensureMember(account: Account) { + if (isSeeded(account) || this.members.has(account.did)) return; + await accounts.makeMember(account); + this.oweMembership(account); + } + + // for a membership made outside ensureMember, like an invite redeemed in the browser + oweMembership(account: Account) { + this.members.add(account.did); + this.tidy.add(`knot membership of ${account.handle}`, { kind: "member", account }); + } +} + +export class Run implements Ctx { + readonly tidy = new Tidy(); + private people = new People(this.tidy); + private tabs: Tabs; + private home?: Repo; + private parent?: Repo; + private last?: Ctx["pushed"]; + private slots = new Map(); + private window = ""; + + constructor(browser: Browser) { + this.tabs = new Tabs(browser); + } + + get actor() { + return this.people.actor; + } + + // read before the test runs, so this getter must not throw + get known() { + return { actor: this.people.who?.name, repo: this.home?.name }; + } + + get pageUrl() { + try { + return this.page.url(); + } catch { + return undefined; + } + } + + get page() { + return this.tabs.page(this.actor, this.window); + } + + async openWindow(window: string) { + await this.tabs.openWindow(this.actor, window); + this.window = window; + } + + async openDevice(window: string) { + await this.tabs.openDevice(this.actor, window); + this.window = window; + } + + get repo() { + if (!this.home) throw new Error("this step needs a .repo() before it"); + return this.home; + } + + set repo(repo: Repo) { + this.home = repo; + note("repo", { name: repo.name, did: repo.did }); + } + + get upstream() { + if (!this.parent) throw new Error("this step needs a .fork() before it"); + return this.parent; + } + + set upstream(repo: Repo) { + this.parent = repo; + } + + get everyone() { + return this.people.everyone; + } + + get pushed() { + if (!this.last) throw new Error("this step needs a .push() before it"); + return this.last; + } + + set pushed(pushed: Ctx["pushed"]) { + this.last = pushed; + } + + switchTo(who: string | Account, options: { login?: "ui" } = {}) { + this.window = ""; + return this.people.switchTo(who, options); + } + + async own(options: RepoOptions = {}) { + const owner = this.actor; + await this.people.ensureMember(owner); + const made = await repos.create(owner, `e2e-${stamp()}`, options); + // registered before the wait, since a stalled hydrant fails the run right there + this.tidy.add(`repo ${made.name}`, { kind: "repo", owner, made }); + // the web only shows a repo after bobbin has indexed it + await bobbin.indexed(made.declaration); + await bobbin.hasRepo(owner.did, made.name); + this.set(declarationId, made.declaration); + await this.learn(manifestId, selfUri(made.did, manifestId.nsid), "read"); + this.repo = repoAt(owner, { did: made.did, name: made.name, defaultBranch: "main" }); + } + + becomeMember() { + return this.people.ensureMember(this.actor); + } + + oweMembership() { + this.people.oweMembership(this.actor); + } + + adoptRepo(made: { did: Did; name: string }) { + const owner = this.actor; + this.tidy.add(`repo ${made.name}`, { kind: "repo", owner, made }); + this.repo = repoAt(owner, { ...made, defaultBranch: "main" }); + } + + owe(label: string, chore: Chore) { + this.tidy.add(label, chore); + } + + expectNoWrite(slot: Slot) { + return this.tabs.expectNoWrite(this.actor, slot); + } + + get = (slot: Slot) => { + const written = this.slots.get(slot); + if (!written) throw new Error(`nothing has filled the ${slot.nsid} slot yet`); + return written; + }; + + set(slot: Slot, written: Written) { + this.slots.set(slot, written); + note(slot.nsid, written); + } + + // nothing else erases this record, so the run has to + adopt(slot: Slot, written: Written) { + this.set(slot, written); + this.tidy.record(slot, this.actor, written); + } + + async learn(slot: Slot, uri: string, doing: string) { + this.set(slot, { uri, cid: (await knotRecordFor(uri, doing)).cid }); + } + + settle() { + return this.tabs.settle(this.actor); + } + + openTab() { + return this.tabs.open(this.actor, this.people.signsInThroughThePage(this.actor)); + } + + hold(slot: Slot, matching?: (body: any) => boolean) { + return this.tabs.hold(this.actor, slot, matching); + } + + // a write let through here is never recorded, so the slot keeps what it had + answer(slot: Slot, held: Held, status: number | number[]) { + return this.tabs.release(this.actor, slot, held, status); + } + + async release(slot: Slot, held: Held) { + const body = await this.tabs.release(this.actor, slot, held); + if (held.method === "delete") return this.get(slot); + const before = this.slots.get(slot); + const written = { uri: body.uri as string, cid: body.cid as string }; + if (held.method === "put" && before?.cid === written.cid) { + throw new Error(`the put of ${slot.nsid} answered with the cid it started from`); + } + this.set(slot, written); + // a put with nothing to replace is a create under a client rkey + const made = held.method === "create" || (held.method === "put" && !before); + if (made || this.tidy.slotHasEraser(slot)) this.tidy.record(slot, this.actor, written); + return written; + } + + expectQuiet() { + return this.tabs.expectQuiet(); + } + + // tabs close first so nothing writes while the world is torn down + async close() { + await this.tabs.close(); + await this.tidy.close(); + } +} + +export const pushFiles = async ( + run: Run, + files: Record, + options: { branch?: string; base?: string; options?: string[] } +) => { + const { actor, repo } = run; + const branch = options.branch ?? repo.defaultBranch; + const { sha, output } = await push(actor, repo.did, files, { + branch, + base: options.base, + options: options.options + }); + run.pushed = { sha, branch, output }; + repo.heads[branch] = sha; + const uri = selfUri(repo.did, refsId.nsid); + const stored = await refsNaming( + uri, + `the knot should list ${sha} on ${branch} in ${repo.did}`, + (named) => named.some(([commit, ref]) => commit === sha && ref === `refs/heads/${branch}`) + ); + run.set(refsId, { uri, cid: stored.cid }); +}; + +// bobbin's pages read git, so wait for every pushed branch to show up there +export const waitForMirror = async (run: Run) => { + await bobbin.knowsRepo(run.repo.did); + await Promise.all( + Object.entries(run.repo.heads).map(([branch, sha]) => + bobbin.hasBranch(run.repo.did, branch, sha) + ) + ); +}; + +export const openPull = async ( + run: Run, + pull: Slot, + patch: Slot, + options: { title?: string; from: string; into: string } +) => { + const { from, into } = options; + const range = { base: run.repo.heads[into], head: run.repo.heads[from] }; + if (!range.base || !range.head) throw new Error("a pull needs a .push() to both branches"); + const opened = await pulls.open( + run.actor, + run.repo.did, + options.title ?? unique("a pull"), + into, + range, + from + ); + // the patch goes first at teardown, since it names a ticket that has to outlive it + run.adopt(pull, opened.ticket); + run.adopt(patch, opened.patch); + await bobbin.indexed(opened.ticket); + await bobbin.indexed(opened.patch); +}; + +// made through the api, so the flow only clicks through what it tests +export const openTicket = async ( + run: Run, + slot: Slot, + options: { title?: string; state?: "open" | "closed" } +) => { + const opened = await tickets.open(run.actor, run.repo.did, options.title ?? unique("a ticket")); + run.adopt(slot, opened); + if (options.state === "closed") { + run.set(slot, await tickets.setState(run.actor, opened, "closed")); + } + await bobbin.indexed(run.get(slot)); +}; + +// the did sits in the held declaration, so register the repo before letting the write through +export const traceNewRepo = async (run: Run, name: string, hops: Hop[]) => { + const owner = run.actor; + const held = await run.hold(declarationId); + expect(held.host, "the declaration should go to the pds").toBe(new URL(config.pdsUrl).host); + expect(held.method, "the declaration should be a create").toBe("create"); + expect(held.body).toMatchObject({ + repo: owner.did, + collection: declarationId.nsid, + rkey: name, + record: { $type: declarationId.nsid } + }); + const did: Did = held.body.record.repo; + expect(did, "the declaration should name the repo the knot minted").toMatch(/^did:/); + run.adoptRepo({ did, name }); + const before = await lookBefore(hops, held); + const written = await run.release(declarationId, held); + await settle(written, hops, owner, declarationId.nsid, "create", before); + await run.learn(manifestId, selfUri(did, manifestId.nsid), "read"); +}; diff --git a/e2e/harness/services.ts b/e2e/harness/services.ts new file mode 100644 index 0000000000000000000000000000000000000000..11f93833ec12006a4d1c9a39d84d3943ad3148f9 --- /dev/null +++ b/e2e/harness/services.ts @@ -0,0 +1,663 @@ +import { test } from "@playwright/test"; + +import { serviceDo, getOk, refsOf } from "./api"; +import { attach, fresh, note, within } from "./record"; +import { checkGit } from "./cobgit"; +import { config, type Account } from "./config"; +import { lsRemote } from "./git"; +import { knotFirehose, pdsFirehose, type Firehose } from "./firehose"; +import type { PocketPreferences } from "./lexicons"; +import { get, partsOf, recordUrl, until, xrpc, type Reply, type Written } from "./net"; +import { checkStream, streamCarries, watcher } from "./stream"; +import type { Held } from "./tabs"; +import { checkVersions, isCob, versionsBefore, type Version } from "./versions"; + +// read while the write is still held or a hop cannot tell what it changed +export interface Before { + versions?: Version[]; + mark?: number; + knotMark?: number; + pdsMark?: number; + pdsExisted?: boolean; + released?: number; +} + +export interface Settling { + how: Held["method"] | "settled"; + before?: Before; + // whose notices get checked. with nobody here an "absent" check can't fail, so it proves nothing + people?: Account[]; +} + +export interface Hop { + name: string; + present: (written: Written, who: Account, settling?: Settling) => Promise; + absent: (written: Pick, who?: Account, settling?: Settling) => Promise; + look?: (uri: string | undefined) => Promise; +} + +// creates have no rkey yet, the knot mints it, so there is no uri to read +export const lookBefore = async (hops: Hop[], { body }: Held): Promise => { + const uri = body.rkey ? `at://${body.repo}/${body.collection}/${body.rkey}` : undefined; + return Object.assign({}, ...(await Promise.all(hops.map((hop) => hop.look?.(uri))))); +}; + +// one step per check so the report shows how long after the write each service had it +// the tidy replay runs outside a test, so test.info throws there +const step = (title: string, check: () => Promise): Promise => { + try { + test.info(); + } catch { + return check(); + } + return test.step(title, async (info) => { + const record = fresh(); + try { + return await within(record, check); + } finally { + await attach(info, record); + } + }); +}; + +// one hop is one row, so a failing sub-check is named on it +// the hop's own check keeps its plain message +const alongside = async ( + own: Promise, + checks: Record | undefined> +) => { + const named = Object.entries(checks).filter(([, check]) => check); + const [first, ...rest] = await Promise.allSettled([own, ...named.map(([, check]) => check)]); + const failed = [first, ...rest].flatMap((check, at) => + check.status === "rejected" + ? [`${at ? `${named[at - 1][0]}: ` : ""}${(check.reason as Error).message}`] + : [] + ); + if (failed.length) throw new Error(failed.join("\n ")); + return (first as PromiseFulfilledResult).value; +}; + +const recordHop = (name: string, base: string) => { + const has = ({ uri, cid }: Written) => + until(`${name} should have ${uri} at ${cid}`, 10_000, async () => { + const { status, body } = await get(recordUrl(base, uri)); + const saw = + status === 200 + ? `it has ${body.cid}` + : `it answered ${status} ${JSON.stringify(body)}`; + return [status === 200 && body.cid === cid, saw, body.value]; + }); + const lacks = ({ uri }: Pick) => + until(`${name} should have dropped ${uri}`, 10_000, async () => { + const { status, body } = await get(recordUrl(base, uri)); + // down or erroring is not the same as gone, only a 404 counts + const gone = status === 404 || body?.error === "RecordNotFound"; + return [ + gone, + status === 200 ? "it still has it" : `it answered ${status} ${JSON.stringify(body)}` + ]; + }); + return { name, host: new URL(base).host, has, lacks, present: has, absent: lacks }; +}; + +const firehosed = ( + watcher: Firehose, + written: { uri: string; cid?: string }, + settling?: Settling, + since?: number, + existed = true +) => + step("firehose", async () => { + const how = settling?.how; + const action = + how === "put" ? (existed ? "update" : "create") : how === "settled" ? undefined : how; + const frame = await watcher.check(written, action, since); + const released = settling?.before?.released; + if (released) note("arrived", `${frame.at - released} ms after release`); + }); + +const pdsRecords = recordHop("pds", config.pdsUrl); +export const pds = { + ...pdsRecords, + present: (written: Written, _who?: Account, settling?: Settling) => + alongside(pdsRecords.has(written), { + firehose: firehosed( + pdsFirehose, + written, + settling, + settling?.before?.pdsMark, + settling?.before?.pdsExisted + ) + }), + absent: ({ uri }: Pick, _who?: Account, settling?: Settling) => + alongside(pdsRecords.lacks({ uri }), { + firehose: firehosed(pdsFirehose, { uri }, settling, settling?.before?.pdsMark) + }), + look: async (uri: string | undefined): Promise => { + const pdsMark = pdsFirehose.mark(); + if (!uri) return { pdsMark }; + // a pds put can create too, so its method alone cannot tell the action + const { status, body } = await get(recordUrl(config.pdsUrl, uri)); + if (status !== 200 && status !== 404 && body?.error !== "RecordNotFound") + throw new Error( + `pds should say whether ${uri} exists before the write but answered ${status} ${JSON.stringify(body)}` + ); + return { pdsMark, pdsExisted: status === 200 }; + } +}; + +const hydrantDb = recordHop("hydrant db", config.hydrantUrl); +const streamed = ({ uri, cid }: { uri: string; cid?: string }, settling?: Settling) => + streamCarries(uri) + ? step("stream", async () => { + const { mark, released } = settling?.before ?? {}; + const frame = await checkStream({ uri, cid }, settling?.how ?? "settled", mark); + // checkStream only notices a frame on its next wake, so the arrival time comes from the watcher + if (released && frame) note("arrived", `${frame.at - released} ms after release`); + }) + : undefined; +export const hydrant = { + ...hydrantDb, + name: "hydrant", + present: (written: Written, _who?: Account, settling?: Settling) => + alongside( + step("db", () => hydrantDb.has(written)), + { stream: streamed(written, settling) } + ), + absent: ({ uri }: Pick, _who?: Account, settling?: Settling) => + alongside( + step("db", () => hydrantDb.lacks({ uri })), + { stream: streamed({ uri }, settling) } + ), + look: async () => ({ mark: watcher.mark() }) +}; + +const knotRecords = recordHop("knot", config.knotUrl); +// a cob's uri authority is the repo, so the author lives in the record body +const editorIs = async (written: Written, did: string) => { + const value = await knotRecords.has(written); + if (value["x-tngl-editor"] !== did) { + throw new Error( + `knot should show ${did} as the editor of ${written.uri} but x-tngl-editor is ${value["x-tngl-editor"]} (use knotCopy for a record the knot wrote itself)` + ); + } +}; + +// one listCobVersions read feeds both checks, no cid means the record was deleted +const cobChecks = ( + written: { uri: string; cid?: string }, + editor: string | undefined, + settling?: Settling +) => { + const firehose = + isCob(written.uri) || partsOf(written.uri).collection === "org.tangled.git.refs" + ? firehosed(knotFirehose, written, settling, settling?.before?.knotMark) + : undefined; + if (!isCob(written.uri)) return { firehose }; + const chain = step("versions", () => + checkVersions(written, { editor, had: settling?.before?.versions }) + ); + return { + firehose, + versions: chain, + // a chain that never settles fails as versions already, git would only repeat it + // git's step starts once the chain is in, so its bar is only git's time + git: chain.then( + (versions) => step("git", () => checkGit(written, versions)), + () => undefined + ) + }; +}; +const versionsLook = async (uri: string | undefined) => ({ + knotMark: knotFirehose.mark(), + versions: await versionsBefore(uri) +}); + +const knotHop = (checksEditor: boolean): Hop => ({ + name: "knot", + present: (written, who, settling) => { + const editor = checksEditor ? who.did : undefined; + return alongside( + step("record", () => (editor ? editorIs(written, editor) : knotRecords.has(written))), + cobChecks(written, editor, settling) + ); + }, + absent: ({ uri }, who, settling) => + alongside( + step("record", () => knotRecords.lacks({ uri })), + cobChecks({ uri }, checksEditor ? who?.did : undefined, settling) + ), + look: versionsLook +}); + +// the knot wrote this one itself, so there is no x-tngl-editor to check +export const knotCopy = knotHop(false); + +export const knot = { ...knotRecords, ...knotHop(true) }; + +// the knot rewrites the refs record a moment after a push, so wait for it +export const refsNaming = ( + uri: string, + what: string, + wanted: (named: [sha: string, ref: string][]) => boolean +) => + until(what, 10_000, async () => { + const now = await refsOf(uri); + if (!now) return [false, "it has no refs record"]; + const said = now.named.map(([sha, ref]) => `${ref} ${sha}`).join(", "); + return [wanted(now.named), `it names ${said}`, now]; + }); + +export const git: Hop = { + name: "git ls-remote", + present: (written, who) => + until(`git should list exactly the refs ${written.uri} names`, 10_000, async () => { + const stored = await refsOf(written.uri); + if (stored?.cid !== written.cid) + return [false, `the knot has ${stored?.cid ?? "no record"}`]; + const named = Object.fromEntries(stored.named.map(([sha, ref]) => [ref, sha])); + if (Object.keys(named).length === 0) return [false, "the record names no refs"]; + // git lists HEAD, the refs record never does + const listed = await lsRemote(who, partsOf(written.uri).repo, "refs/"); + return [ + JSON.stringify(Object.entries(named).sort()) === + JSON.stringify(Object.entries(listed).sort()), + `git lists ${JSON.stringify(listed)}, the record names ${JSON.stringify(named)}` + ]; + }), + absent: async () => { + throw new Error("git has no refs record to lose"); + } +}; + +const awaitRecord = async ({ uri, cid }: { uri: string; cid?: string }) => + ( + await get( + xrpc(config.bobbinUrl, "org.tangled.bobbin.awaitRecord", { + uri, + ...(cid ? { cid } : {}) + }) + ) + ).body; + +const said = (body: { status?: string; reason?: string; detail?: string }) => + body.status + ? `it said ${body.status} ${body.reason ?? ""} ${body.detail ?? ""}` + : JSON.stringify(body); + +// parallel workers can push pds to hydrant to bobbin past 15 s, hence 30 s +const indexed = (written: Written) => + until( + `bobbin should have indexed ${written.uri} at ${written.cid}`, + 30_000, + async () => { + const body = await awaitRecord(written); + // a rejected record never gets indexed, so throw instead of polling + if (body.status === "rejected") + throw new Error(`bobbin should have indexed ${written.uri} but ${said(body)}`); + return [body.status === "indexed" && body.cid === written.cid, said(body)]; + }, + { longPoll: true } + ); + +const deleted = ({ uri }: Pick) => + until( + `bobbin should have deleted ${uri}`, + 15_000, + async () => { + const body = await awaitRecord({ uri }); + return [body.status === "deleted", said(body)]; + }, + { longPoll: true } + ); + +const mirrorList = async (kind: "Branches" | "Tags", repoDid: string) => { + const { status, body } = await get( + xrpc(config.bobbinUrl, `org.tangled.temp.git.list${kind}`, { repo: repoDid, limit: "100" }) + ); + const names: string[] = (body?.[kind.toLowerCase()] ?? []).map((one: any) => one.name); + return { status, names }; +}; + +export const bobbin = { + name: "bobbin", + indexed, + list: mirrorList, + present: indexed, + absent: deleted, + // a stall here means knot2 is rate limiting hydrant's backfill + repoByName: (owner: string, name: string) => + get(xrpc(config.bobbinUrl, "org.tangled.temp.repo.getRepoByName", { owner, name })), + hasRepo: (ownerDid: string, name: string) => + until( + `bobbin should know the repo ${name} (hydrant may be backing off knot2)`, + 30_000, + async () => { + const { status } = await bobbin.repoByName(ownerDid, name); + return [status === 200, `getRepoByName answered ${status}`]; + } + ), + // the mirror 404s an unsynced repo, even an empty one, and the web reads that as a knot needing an upgrade + knowsRepo: (repoDid: string) => + until(`bobbin should know ${repoDid}`, 60_000, async () => { + const { status } = await mirrorList("Branches", repoDid); + return [status === 200, `listBranches answered ${status}`]; + }), + // the mirror picks a push up a moment after the knot, so pages that read git wait for this first + hasBranch: (repoDid: string, branch: string, sha: string) => + until(`bobbin should have ${branch} at ${sha}`, 60_000, async () => { + const { status, body } = await get( + xrpc(config.bobbinUrl, "org.tangled.temp.git.getRef", { + repo: repoDid, + ref: `refs/heads/${branch}` + }) + ); + return [ + status === 200 && body.oid === sha, + `getRef answered ${status} ${JSON.stringify(body)}` + ]; + }), + countStars: async (subject: string) => { + const body = await getOk( + xrpc(config.bobbinUrl, "org.tangled.feed.countStars", { subject }) + ); + return body.count as number; + }, + listForks: (subject: string) => + getOk(xrpc(config.bobbinUrl, "org.tangled.repo.listForks", { subject, limit: "50" })), + countForks: async (subject: string) => { + const body = await getOk( + xrpc(config.bobbinUrl, "org.tangled.repo.countForks", { subject }) + ); + return body.count as number; + } +}; + +interface Workflow { + name?: string; + id: string; + status: string; + error?: string; +} +const FINISHED = ["success", "failed", "cancelled", "timeout"]; + +const finishedRun = (repoDid: string, sha: string) => + until<{ pipeline: string; workflow: Workflow }>( + "spindle should finish the workflow", + 120_000, + async () => { + const { status, body } = await get( + xrpc(config.spindleUrl, "org.tangled.ci.queryPipelines", { + repo: repoDid, + commits: sha + }) + ); + const runs: { pipeline: string; workflow: Workflow }[] = ( + body?.pipelines ?? [] + ).flatMap((pipeline: { id: string; workflows?: Workflow[] }) => + (pipeline.workflows ?? []).map((workflow) => ({ pipeline: pipeline.id, workflow })) + ); + const done = runs.find((one) => FINISHED.includes(one.workflow.status)); + return [ + Boolean(done), + `it answered ${status} with ${JSON.stringify(runs.map((one) => one.workflow))}`, + done + ]; + } + ); + +// triggers look like org.tangled.event.push, so match on the suffix +export const spindle = { + hasPipeline: (repoDid: string, sha: string, event: "push" | "pullRequest", budget = 30_000) => + until( + `spindle should have a ${event} pipeline for ${repoDid} at ${sha}`, + budget, + async () => { + const { status, body } = await get( + xrpc(config.spindleUrl, "org.tangled.ci.queryPipelines", { + repo: repoDid, + commits: sha + }) + ); + const pipelines: { commit: string; trigger?: { $type?: string } }[] = + body.pipelines ?? []; + const found = pipelines.find( + (p) => p.commit === sha && p.trigger?.$type?.endsWith(`.event.${event}`) + ); + const saw = + status === 200 + ? `it listed ${pipelines.length} pipelines for the commit` + : `it answered ${status}`; + return [Boolean(found), saw, found]; + } + ), + finishedRun, + finishedWorkflow: async (repoDid: string, sha: string) => + (await finishedRun(repoDid, sha)).workflow +}; + +export const settle = async ( + written: Written, + hops: Hop[], + who: Account, + nsid: string, + how: Settling["how"], + before?: Before, + people?: Account[] +) => { + const settling = { how, before, people }; + const checks = await Promise.allSettled( + hops.map((hop) => + step(hop.name, () => + how === "delete" + ? hop.absent(written, who, settling) + : hop.present(written, who, settling) + ) + ) + ); + if (checks.every((check) => check.status === "fulfilled")) return; + const table = checks.map((check, at) => + check.status === "fulfilled" + ? ` ✓ ${hops[at].name}` + : ` ✗ ${hops[at].name} ${(check.reason as Error).message}` + ); + throw new Error([`${nsid} ${written.uri} ${written.cid} (${how})`, ...table].join("\n")); +}; + +// pocket keys the saved lists by the token's aud +const POCKET_AUD = "did:web:tangled.org"; +const KNOT_PREF = "org.tangled.temp.actor.defs#savedKnotPref"; +const SPINDLE_PREF = "org.tangled.temp.actor.defs#savedSpindlePref"; +const RECENTS_PREF = "org.tangled.temp.actor.defs#recentsPref"; + +export interface Saved { + knots: string[]; + spindles: string[]; +} + +type Stored = PocketPreferences["preferences"]; + +const storedBy = async (who: Account): Promise => { + const lxm = "com.bad-example.pocket.getPreferences"; + const reply = await serviceDo(who, config.bobbinUrl, POCKET_AUD, lxm); + // a 400 NotFound means the account never saved anything + if (reply.status === 400 && reply.body?.error === "NotFound") return []; + if (reply.status !== 200) throw new Error(`pocket answered ${JSON.stringify(reply.body)}`); + return reply.body.preferences; +}; + +export const savedBy = async (who: Account): Promise => { + const stored = await storedBy(who); + const hosts = (items: { value: string }[]) => items.map((item) => item.value).sort(); + return { + knots: hosts(stored.flatMap((pref) => (pref.$type === KNOT_PREF ? pref.items : []))), + spindles: hosts(stored.flatMap((pref) => (pref.$type === SPINDLE_PREF ? pref.items : []))) + }; +}; + +export const recentsIn = async (who: Account) => + (await storedBy(who)).flatMap((pref) => (pref.$type === RECENTS_PREF ? pref.items : [])); + +// a single try, saveFor is the one that retries +export const putSaved = async (who: Account, saved: Saved): Promise => { + const lxm = "com.bad-example.pocket.putPreferences"; + const items = (hosts: string[]) => hosts.map((host) => ({ id: host, value: host })); + return serviceDo(who, config.bobbinUrl, POCKET_AUD, lxm, { + json: { + preferences: [ + { $type: KNOT_PREF, items: items(saved.knots) }, + { $type: SPINDLE_PREF, items: items(saved.spindles) } + ] + } satisfies PocketPreferences + }); +}; + +// pocket can't resolve a new account's did doc for a moment +export const saveFor = async (who: Account, saved: Saved) => + until(`pocket should take the save for ${who.handle}`, 15_000, async () => { + const reply = await putSaved(who, saved); + return [reply.status === 200, `it answered ${reply.status} ${JSON.stringify(reply.body)}`]; + }); + +// deliberi only answers the recipient, not other accounts +const deliberiGet = async (who: Account, lxm: string, params?: Record) => { + const aud = `did:web:${new URL(config.deliberiUrl).host}`; + const reply = await serviceDo(who, config.deliberiUrl, aud, lxm, { params }); + if (reply.status !== 200) throw new Error(`deliberi answered ${JSON.stringify(reply.body)}`); + return reply.body; +}; + +export type Toggles = Record; + +export const notificationsOf = (who: Account): Promise => + deliberiGet(who, "org.tangled.temp.notification.getPreferences"); + +export interface Notice { + uri: string; + type: string; + read: boolean; + actorDid: string; + repoDid?: string; + issueAt?: string; + pullAt?: string; +} + +export const noticesOf = async (who: Account): Promise => + (await deliberiGet(who, "org.tangled.temp.notification.listNotifications", { limit: "50" })) + .notifications; + +export const aboutOf = (notice: Notice) => notice.issueAt ?? notice.pullAt; + +// a state change notice is the ticket uri with #rev on the end +const madeFrom = (notice: Notice, uri: string) => + notice.uri === uri || notice.uri.startsWith(`${uri}#`); + +// deliberi only shows each person their own notices, so ask as each one +export const noticesMadeFrom = async (uri: string, people: Account[]) => + ( + await Promise.all( + people.map(async (who) => + (await noticesOf(who)) + .filter((n) => madeFrom(n, uri)) + .map((notice) => ({ who, notice })) + ) + ) + ).flat(); + +// deliberi saves the rev with its notices, so once _ingested shows the rev the notices are written +const ingested = async (written: Written) => { + // an edit keeps the uri, so only a rev at least as new as the write proves it landed + const watcher = isCob(written.uri) ? knotFirehose : pdsFirehose; + const { rev } = await watcher.check(written); + return until( + `deliberi should have ingested ${written.uri} at ${written.cid}`, + 20_000, + async () => { + const { status, body } = await get( + `${config.deliberiUrl}/xrpc/_ingested?${new URLSearchParams({ uri: written.uri })}` + ); + // a json 404 is just unseen, a plain text 404 means the route is missing + if (status === 404 && typeof body === "string") + throw new Error( + `deliberi has no /xrpc/_ingested route, run it with DELIBERI_DEV=true, nothing else can say it processed ${written.uri}` + ); + // only tickets carry a cid, so a missing one doesn't count against it + const has = + status === 200 && + body.rev >= rev && + (body.cid === undefined || body.cid === written.cid); + return [ + has, + `/_ingested answered ${status} ${JSON.stringify(body)}, the write is at rev ${rev}` + ]; + } + ); +}; + +const told = (found: { who: Account; notice: Notice }[]) => + found.map(({ who, notice }) => `${notice.type} to ${who.name}`); + +export const deliberi: Hop = { + name: "deliberi", + present: async (written, _who, settling) => { + await step("ingested", () => ingested(written)); + const found = await noticesMadeFrom(written.uri, settling?.people ?? []); + note("deliberi made", told(found).join(", ") || "no notice"); + }, + absent: async ({ uri }, _who, settling) => { + await until(`deliberi should have no notice made from ${uri}`, 15_000, async () => { + const found = await noticesMadeFrom(uri, settling?.people ?? []); + return [found.length === 0, `it still has ${told(found).join(", ")}`]; + }); + } +}; + +// the opener is the editor of the first version, read from the knot's git +export const knotmirror = { + // the mirror lags the knot a moment, so this polls + openerOf: (uri: string) => + until(`knotmirror should list who opened ${uri}`, 30_000, async () => { + const { repo, collection, rkey } = partsOf(uri); + const { status, body } = await get( + xrpc(config.knotmirrorUrl, "org.tangled.temp.repo.listRecordOrigins", { + repo, + collection, + limit: "1000" + }) + ); + const found = + status === 200 + ? body.origins?.find((o: { rkey: string }) => o.rkey === rkey) + : undefined; + return [ + Boolean(found), + `listRecordOrigins answered ${status} ${JSON.stringify(body)}`, + found?.author as string + ]; + }) +}; + +export const openers = { + view: async (uri: string): Promise => + (await getOk(xrpc(config.bobbinUrl, "org.tangled.track.getTicketView", { uri }))).author + .did, + listed: async (repoDid: string, list: "listIssues" | "listPulls", uri: string) => { + const body = await getOk( + xrpc(config.bobbinUrl, `org.tangled.repo.${list}`, { subject: repoDid, limit: "100" }) + ); + return body.items.find((item: { uri: string }) => item.uri === uri)?.author as + string | undefined; + }, + searched: async (repoDid: string, kind: "issue" | "pull", query: string) => { + const body = await getOk( + xrpc(config.bobbinUrl, "org.tangled.search.query", { + repo: repoDid, + nsid: "org.tangled.track.ticket", + kind, + ...(query ? { q: query } : {}), + limit: "100" + }) + ); + return body.hits as { uri: string; author?: string }[]; + } +}; diff --git a/e2e/harness/shared.ts b/e2e/harness/shared.ts new file mode 100644 index 0000000000000000000000000000000000000000..9d26aecc9633fb53c0ee4f8a50db6dee5d4c8611 --- /dev/null +++ b/e2e/harness/shared.ts @@ -0,0 +1,580 @@ +import { expect } from "@playwright/test"; + +import { knotReply, knotRecordFor, labels, ticketRecord, tickets } from "./api"; +import { config, type Account } from "./config"; +import { flow, type Flow } from "./flow"; +import { deleteBranch, lsRemote, pushAnnotatedTag } from "./git"; +import { get, ok, selfUri, until, xrpc, type Reply, type Written } from "./net"; +import { + bobbin, + knotmirror, + hydrant, + knot, + openers, + pds, + refsNaming, + spindle, + type Hop +} from "./services"; +import { + alerts, + commentId, + commentMenu, + declarationId, + confirmDelete, + discussion, + editLabels, + labelOption, + manifestId, + link, + mergeButton, + moreActions, + not, + page, + patchId, + pullId, + record, + refsId, + renameForm, + rkeyOf, + replacing, + resolve, + saveButton, + subscriptionId, + target, + testid, + text, + through, + ticketId, + triage, + triageChip, + triageToken, + where, + within, + type Ctx, + type Lazy, + type Slot, + type Target +} from "./vocabulary"; + +const remembered = new WeakMap>(); +export const remember = (label: string) => async (ctx: Ctx) => { + remembered.set(ctx, { ...remembered.get(ctx), [label]: ctx.actor }); +}; +export const recall = (ctx: Ctx, label: string) => { + const who = remembered.get(ctx)?.[label]; + if (!who) throw new Error(`no ${label} was remembered`); + return who; +}; + +// the login lives in cookies and web storage, so both have to go +export const forgetTheLogin = async (ctx: Ctx) => { + await ctx.page.context().clearCookies(); + await ctx.page.evaluate(() => { + localStorage.clear(); + sessionStorage.clear(); + }); +}; + +export const acceptConfirm = async (ctx: Ctx) => { + ctx.page.once("dialog", (dialog) => void dialog.accept()); +}; + +export const jsonError = (status: number, error: string, message: string) => ({ + status, + contentType: "application/json", + headers: { "access-control-allow-origin": "*" }, + body: JSON.stringify({ error, message }) +}); + +// only this page fails, and page routes run before the held writes, so the knot never sees it +export const answerWith = + (pattern: string | RegExp, status: number, error: string, message: string) => (ctx: Ctx) => + ctx.page.route(pattern, (route) => route.fulfill(jsonError(status, error, message))); + +export const finished = async (ctx: Ctx, what: string, done: Promise) => + done.catch(async (cause) => { + const said = await alerts.locate(ctx.page, ctx).allInnerTexts(); + throw new Error(`${what}, the page said: ${said.join(" | ") || "nothing"}`, { cause }); + }); + +export const saveFinishes = (ctx: Ctx) => + finished( + ctx, + "the save did not finish", + expect(saveButton.locate(ctx.page, ctx)).toBeDisabled() + ); + +export const learnRefs = (ref: string, there: boolean) => async (ctx: Ctx) => { + const uri = selfUri(ctx.repo.did, refsId.nsid); + const stored = await refsNaming( + uri, + `the knot should ${there ? "list" : "drop"} ${ref} in ${uri}`, + (named) => named.some(([, name]) => name === ref) === there + ); + ctx.get(refsId).cid = stored.cid; +}; + +export const deletesBranch = (branch: string) => async (ctx: Ctx) => + deleteBranch(ctx.actor, ctx.repo.did, branch); + +export const pushesTag = (tag: string, message: string) => async (ctx: Ctx) => + pushAnnotatedTag(ctx.actor, ctx.repo.did, tag, message); + +// bobbin lags the knot by a few seconds, so this polls +export const mirrorLists = + (kind: "Branches" | "Tags", name: string, there: boolean) => async (ctx: Ctx) => + until(`bobbin should ${there ? "list" : "drop"} ${name}`, 60_000, async () => { + const { status, names } = await bobbin.list(kind, ctx.repo.did); + return [ + status === 200 && names.includes(name) === there, + `it answered ${status} ${names}` + ]; + }); + +// the sidebar can't make a label on a new repo, so this goes through the api +export const defineLabel = (name: string) => async (ctx: Ctx) => { + const made = await labels.define(ctx.actor, ctx.repo.did, name); + expect(made.status, `creating the label answered ${JSON.stringify(made.body)}`).toBe(200); + await bobbin.indexed({ uri: made.body.uri, cid: made.body.cid }); +}; + +export const labelOn = (slot: Slot, noun: string) => (flow: Flow) => + flow + .sees(not(triageChip)) + .click(editLabels) + .click(labelOption) + .sends(slot, knot, replacing(slot, { tangledLabels: [triageToken] }), "put") + // the chip shows before the knot has the label + .sees(triageChip) + .trace(slot, through(knot, hydrant, bobbin)) + .step(`the knot's ${noun} carries the label`, async (ctx) => + expect(await labelsOnKnot(ctx, slot)).toEqual([triage]) + ) + .reload() + .sees(triageChip); + +export const labelOff = (slot: Slot, noun: string) => (flow: Flow) => + flow + .click(editLabels) + .click(labelOption) + .sends(slot, knot, replacing(slot), "put") + .sees(not(triageChip)) + .trace(slot, through(knot, hydrant, bobbin)) + .step(`the knot's ${noun} has no label left`, async (ctx) => + expect(await labelsOnKnot(ctx, slot)).toEqual([]) + ) + .reload() + .sees(not(triageChip)); + +export const labelsOnKnot = async (ctx: Ctx, on: Slot) => + ((await knot.has(ctx.get(on))).tangledLabels ?? []).map((token: { key: string }) => token.key); + +export const manifestOf = async (ctx: Ctx) => + (await knotRecordFor(selfUri(ctx.repo.did, manifestId.nsid), "read")).value; + +export const topicsOnKnot = async (ctx: Ctx) => (await manifestOf(ctx)).topics ?? []; + +export const picture = { + name: "me.png", + mimeType: "image/png", + buffer: Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", + "base64" + ) +}; + +export const pdsRecordUrl = (repo: string, collection: string, rkey: string) => + xrpc(config.pdsUrl, "com.atproto.repo.getRecord", { repo, collection, rkey }); + +export const made = (what: string, reply: Reply): Written => { + const { uri, cid } = ok(what, reply); + return { uri, cid }; +}; + +interface IssueOptions { + title: string; + body?: Lazy; + state?: "open" | "closed"; + wait?: boolean; +} + +export const issue = (slot: Slot, options: IssueOptions) => async (ctx: Ctx) => { + const body = options.body === undefined ? undefined : resolve(options.body, ctx); + const reply = await knotReply(ctx.actor, ctx.repo.did, "com.atproto.repo.createRecord", { + collection: ticketId.nsid, + record: ticketRecord( + options.title, + body === undefined ? {} : { body: { $type: "org.tangled.markup.markdown", text: body } } + ) + }); + let written = made(`opening ${options.title}`, reply); + // the knot only creates tickets open, so closing takes a second call + if (options.state === "closed") written = await tickets.setState(ctx.actor, written, "closed"); + ctx.adopt(slot, written); + if (options.wait !== false) await bobbin.indexed(written); +}; + +export const stillStored = + (slot: Slot, said: (value: any) => string, words: string) => async (ctx: Ctx) => { + const stored = await knot.has(ctx.get(slot)); + expect(said(stored)).toBe(words); + await hydrant.has(ctx.get(slot)); + }; + +export const workflow = `engine: dummy +when: + - event: ["push", "pull_request"] + branch: ["main"] +steps: + - name: hello + command: echo "build ok" +`; + +export const workflowPath = ".tangled/workflows/ci.yml"; + +interface Pipeline { + id: string; + commit: string; + trigger?: { $type?: string }; + workflows?: { id: string; status: string }[]; +} + +export const pullPipelines = async (repoDid: string, sha: string) => { + const { status, body } = await get( + xrpc(config.spindleUrl, "org.tangled.ci.queryPipelines", { repo: repoDid, commits: sha }) + ); + expect(status, `spindle answered ${JSON.stringify(body)}`).toBe(200); + return ((body.pipelines ?? []) as Pipeline[]).filter( + (p) => p.commit === sha && p.trigger?.$type?.endsWith(".event.pullRequest") + ); +}; + +// a pull from someone who can't push makes pipelines that wait for the collaborator's approval +export const waitsForApproval = + (sha: (ctx: Ctx) => string, repoDid: (ctx: Ctx) => string = (ctx) => ctx.repo.did) => + (ctx: Ctx) => + until( + `spindle should block the pull pipelines at ${sha(ctx)}`, + 30_000, + async () => { + const found = await pullPipelines(repoDid(ctx), sha(ctx)); + const blocked = + found.length > 0 && + found.every((p) => p.workflows?.every((w) => w.status === "blocked")); + return [ + blocked, + `it listed ${JSON.stringify(found.map((p) => p.workflows))}`, + found + ]; + } + ); + +export const pipelineFor = + (event: "push" | "pullRequest", sha: (ctx: Ctx) => string) => (ctx: Ctx) => + spindle.hasPipeline(ctx.repo.did, sha(ctx), event); + +export const base = { "e2e/base.txt": "base\n" }; + +// the knot only takes patches whose commits are already in the repo, so pulls stay inside one repo +export const branches = { from: "feature", into: "main" }; + +// the pull pages read git through the mirror, so the repo has to be mirrored +export const onBranches = (name: string, options: { ci?: boolean } = {}) => + flow(name) + .actor("owner") + .repo(options.ci ? { spindle: config.spindleDid } : {}) + .push(options.ci ? { ...base, [workflowPath]: workflow } : base) + .push({ "e2e/feature.txt": "feature\n" }, { branch: branches.from, base: branches.into }) + .mirrored(); + +export const sendsPatch = + ({ fork = false } = {}) => + (flow: Flow) => + flow + .sends( + patchId, + knot, + (ctx) => ({ + ...(fork ? { repo: ctx.upstream.did } : {}), + patch: { + ticket: rkeyOf(ctx, pullId), + base: { oid: (fork ? ctx.upstream : ctx.repo).heads[branches.into] }, + head: { oid: ctx.repo.heads[branches.from] }, + sourceRepo: ctx.repo.did + } + }), + "create" + ) + .trace(patchId, through(knot, hydrant, bobbin)); + +export const resubmitsForkPatch = (flow: Flow) => + flow + .click(testid("resubmit")) + .sends( + patchId, + knot, + (ctx) => ({ + repo: ctx.upstream.did, + patch: { ticket: rkeyOf(ctx, pullId), sourceRepo: ctx.repo.did } + }), + "create" + ) + .trace(patchId, through(knot, hydrant, bobbin)); + +export const subscribesToPull = (flow: Flow) => + flow + .sends( + subscriptionId, + pds, + (ctx) => ({ record: { subject: { uri: ctx.get(pullId).uri } } }), + "create" + ) + .trace(subscriptionId, through(pds)); + +// the fork is the flow's repo and the upstream the one it asks to pull from +export const forkedAhead = (name: string, options: { ci?: boolean } = {}) => + flow(name) + .actor("owner") + .repo(options.ci ? { spindle: config.spindleDid } : {}) + .push(options.ci ? { ...base, [workflowPath]: workflow } : base) + .actor("forker") + .fork() + .push({ "e2e/fork.txt": "from the fork\n" }, { branch: branches.from, base: branches.into }) + .step("the mirror has the upstream's main", async (ctx) => { + await bobbin.hasBranch( + ctx.upstream.did, + branches.into, + ctx.upstream.heads[branches.into] + ); + }) + .mirrored(); + +// a fork's first diff read fetches its commits, so the page fails until a reload +export const reloadsUntil = (what: Target) => async (ctx: Ctx) => + until(`reloading should show ${what.describe}`, 60_000, async () => { + const shown = (await what.locate(ctx.page, ctx).count()) > 0; + if (!shown) await ctx.page.reload(); + return [shown, "it is still not on the page"]; + }); + +export const composeFromFork = where( + "the new pull form in fork mode", + (ctx) => + `${ctx.upstream.path}/pulls/new?source=fork&fork=${ctx.repo.did}&sourceBranch=${branches.from}&targetBranch=${branches.into}` +); +export const upstreamPulls = where( + "the upstream's pulls list", + (ctx) => `${ctx.upstream.path}/pulls` +); +export const upstreamPull = where( + "the pull on the upstream", + (ctx) => `${ctx.upstream.path}/pulls/${rkeyOf(ctx, pullId)}` +); + +export const opensForkPull = (title: string) => (flow: Flow) => + flow + .open(composeFromFork) + .fill(testid("title"), title) + .click(testid("submit")) + .sends( + pullId, + knot, + (ctx) => ({ + repo: ctx.upstream.did, + record: { state: "open", title: { text: title }, targetBranch: branches.into } + }), + "create" + ) + .trace(pullId, through(knot, hydrant, bobbin)) + .use(sendsPatch({ fork: true })) + .use(subscribesToPull) + .at(upstreamPulls) + .sees(link(title)); + +// the page marks the ticket completed with its own put, after the merge +export const mergesThrough = (hops: Hop[]) => (flow: Flow) => + flow + .click(mergeButton) + .sends(pullId, knot, replacing(pullId, { state: "completed" }), "put") + .trace(pullId, hops) + .sees(testid("ticketStatePill", { state: "completed" })); + +export const merges = mergesThrough(through(knot, hydrant, bobbin)); + +export const targetIsAt = + (branch: string, who = (ctx: Ctx) => ctx.actor) => + async (ctx: Ctx) => { + const heads = await lsRemote(who(ctx), ctx.repo.did); + expect(heads[`refs/heads/${branches.into}`]).toBe(ctx.repo.heads[branch]); + }; + +// a rebase makes a new tip, so match the tree and commit count, not the sha +const tips = new WeakMap(); +export const mergedTip = (ctx: Ctx) => { + const tip = tips.get(ctx); + if (!tip) throw new Error("no merge was checked yet, use targetTookTheFeature first"); + return tip; +}; + +export const targetTookTheFeature = + (who = (ctx: Ctx) => ctx.actor, { fromFork = false } = {}) => + async (ctx: Ctx) => { + const target = fromFork ? ctx.upstream : ctx.repo; + const before = target.heads[branches.into]; + const head = ctx.repo.heads[branches.from]; + const repos = [...new Set([target.did, ctx.repo.did])] + .map((did) => `repos=${did}`) + .join("&"); + const tip = (await lsRemote(who(ctx), target.did))[`refs/heads/${branches.into}`]; + expect(tip, "the target branch didn't move").not.toBe(before); + tips.set(ctx, tip); + const between = async (to: string) => { + const { status, body } = await get( + `${xrpc(config.bobbinUrl, "org.tangled.temp.git.listCommits", { ranges: `${before}..${to}` })}&${repos}` + ); + return { + status, + body, + commits: (body?.commits ?? []) as { oid: string; parents: string[]; tree: string }[] + }; + }; + await until(`the mirror should list the feature's commits on ${tip}`, 30_000, async () => { + const [merged, feature] = [await between(tip), await between(head)]; + const [tipCommit, headCommit] = [ + merged.commits.find((c) => c.oid === tip), + feature.commits.find((c) => c.oid === head) + ]; + const oldest = merged.commits.find((c) => c.parents[0] === before); + const took = + merged.status === 200 && + feature.commits.length > 0 && + merged.commits.length === feature.commits.length && + Boolean(oldest) && + tipCommit?.tree === headCommit?.tree; + return [ + took, + `merged ${JSON.stringify(merged.body)}, feature ${JSON.stringify(feature.body)}` + ]; + }); + }; + +export const commented = "needs a comment"; +export const note = "same here"; + +export const onCommentedIssue = ( + name: string, + { title = commented, words = note, subscription = through(pds) } = {} +) => + flow(name) + .actor("owner") + .repo() + .ticket(ticketId, { title }) + .open(page.issue(ticketId)) + .fill(discussion, words) + .click(testid("submit")) + .trace(commentId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, subscription) + .sees(within(testid("commentBody"), text(words))); + +export const deletesComment = (flow: Flow) => + flow + .click(commentMenu) + .click(testid("handleDelete")) + .click(confirmDelete) + .sends(commentId, knot, replacing(commentId), "delete") + .trace(commentId, through(knot, hydrant, bobbin)); + +export const deletesIssue = (flow: Flow) => + flow + .click(moreActions) + .click(testid("openDeleteConfirm")) + .click(confirmDelete) + .sends(ticketId, knot, replacing(ticketId), "delete") + .trace(ticketId, through(knot, hydrant, bobbin)); + +// a rename writes the new declaration before deleting the old one, so the new one needs its own slot +const movedDeclarationId = record("org.tangled.repo.declaration"); + +export const renamesTo = (name: (ctx: Ctx) => string) => (flow: Flow) => + flow + .open(renameForm) + .fill(testid("name"), name) + .click(testid("rename")) + .sends(movedDeclarationId, pds, (ctx) => ({ rkey: name(ctx) }), "create") + .trace(movedDeclarationId, through(pds, hydrant, bobbin)) + .trace(manifestId, through(knot, hydrant, bobbin)) + .sends(declarationId, pds, (ctx) => ({ rkey: ctx.repo.name }), "delete") + .trace(declarationId, through(pds, hydrant, bobbin)); + +export const rowOf = (slot: Slot) => + target("the ticket's row", (tab, ctx) => + tab.locator(`[data-testid="ticketRow"][data-rkey=${JSON.stringify(rkeyOf(ctx, slot))}]`) + ); + +export const authorLink = (who: (ctx: Ctx) => Account) => + target("the link to the ticket's author", (tab, ctx) => + // the hidden narrow copy comes first, so take the visible one + tab + .locator(`a[href="/${who(ctx).handle}"]`) + .filter({ visible: true }) + .first() + ); + +export const bobbinNamesOpener = + ( + slot: Slot, + kind: "issue" | "pull", + opener: (ctx: Ctx) => Account, + lastWriter: (ctx: Ctx) => Account + ) => + async (ctx: Ctx) => { + const { uri } = ctx.get(slot); + const list = kind === "issue" ? "listIssues" : "listPulls"; + const [opened, wrote] = [opener(ctx).did, lastWriter(ctx).did]; + await until(`bobbin should name ${opened} as the opener of ${uri}`, 20_000, async () => { + const seen = { + view: await openers.view(uri), + listed: await openers.listed(ctx.repo.did, list, uri), + searched: (await openers.searched(ctx.repo.did, kind, "")).find( + (h) => h.uri === uri + )?.author + }; + const byOpener = await openers.searched(ctx.repo.did, kind, `author:${opened}`); + const byWriter = await openers.searched(ctx.repo.did, kind, `author:${wrote}`); + const right = + Object.values(seen).every((author) => author === opened) && + byOpener.some((h) => h.uri === uri) && + !byWriter.some((h) => h.uri === uri); + return [ + right, + `views name ${JSON.stringify(seen)}, author:opener lists ${byOpener.length}, author:last writer lists ${byWriter.length}` + ]; + }); + }; + +export const knotmirrorNamesOpener = + (slot: Slot, opener: (ctx: Ctx) => Account) => async (ctx: Ctx) => + expect(await knotmirror.openerOf(ctx.get(slot).uri)).toBe(opener(ctx).did); + +export const theFan = (ctx: Ctx) => recall(ctx, "fan"); +export const theOwner = (ctx: Ctx) => recall(ctx, "owner"); + +export const fanPulled = (name: string, title = "a pull from the fan") => + onBranches(name) + .step("remember the owner", remember("owner")) + .actor("fan") + .step("remember the fan", remember("fan")) + .pull(pullId, patchId, { ...branches, title }) + .actor("owner"); + +export const fanOpened = (name: string, title = "opened by somebody else") => + flow(name) + .actor("fan") + .step("remember the fan", remember("fan")) + .actor("owner") + .step("remember the owner", remember("owner")) + .repo() + .actor("fan") + .ticket(ticketId, { title }) + .actor("owner"); diff --git a/e2e/harness/signin.ts b/e2e/harness/signin.ts new file mode 100644 index 0000000000000000000000000000000000000000..96047331da9eb24fb5111d194476c2c464a94583 --- /dev/null +++ b/e2e/harness/signin.ts @@ -0,0 +1,371 @@ +import { createDpopFetch, generateDpopKey, generatePkce } from "@atcute/oauth-crypto"; +import type { BrowserContextOptions, Page } from "@playwright/test"; +import { randomBytes } from "node:crypto"; +import { fetch as undiciFetch, parseCookie, request } from "undici"; + +import { config, type Account } from "./config"; +import { dispatcher, ok, parseBody, retry429 } from "./net"; +import { keep } from "./record"; + +type StorageState = Exclude; +type Cookie = StorageState["cookies"][number]; + +interface Reply { + status: number; + headers: Record; + body: any; +} + +const once = (load: () => Promise) => { + let kept: Promise | undefined; + return () => { + kept ??= load().catch((cause) => { + kept = undefined; + throw cause; + }); + return kept; + }; +}; + +const flatHeaders = (headers: Reply["headers"]) => + Object.fromEntries( + Object.entries(headers).map(([name, value]) => [name, [value ?? ""].flat().join(", ")]) + ); + +// undici keeps the set-cookie headers and doesn't follow redirects +const send = ( + url: string, + method: "GET" | "POST", + headers: Record = {}, + json?: unknown +): Promise => + retry429(async () => { + const sentHeaders = + json === undefined ? headers : { ...headers, "content-type": "application/json" }; + const sentBody = json === undefined ? undefined : JSON.stringify(json); + const started = Date.now(); + const response = await request(url, { + dispatcher, + method, + signal: AbortSignal.timeout(10_000), + headers: sentHeaders, + body: sentBody + }).catch((error) => { + throw new Error(`${method} ${url}: ${error.code ?? error.message}`); + }); + const text = await response.body.text(); + keep({ + by: "node", + method, + url, + status: response.statusCode, + ms: Date.now() - started, + request: { headers: sentHeaders, body: sentBody }, + response: { headers: flatHeaders(response.headers), body: text } + }); + return { status: response.statusCode, headers: response.headers, body: parseBody(text) }; + }); + +const clientMetadata = once(async () => + ok( + "the web's client metadata", + await send(`${config.webUrl}/oauth-client-metadata.json`, "GET") + ) +); + +const authServer = once(async () => { + const resource = ok( + "the pds resource metadata", + await send(`${config.pdsUrl}/.well-known/oauth-protected-resource`, "GET") + ); + const issuer = resource.authorization_servers[0]; + return ok( + "the pds authorization server metadata", + await send(`${issuer}/.well-known/oauth-authorization-server`, "GET") + ); +}); + +// the pds wants the newest dpop nonce echoed back, kept per origin +const nonces = new Map(); + +const dpopPost = (dpop: typeof fetch, url: string, json: unknown) => + retry429(async () => { + const response = await dpop(url, { + method: "POST", + signal: AbortSignal.timeout(10_000), + headers: { "content-type": "application/json" }, + body: JSON.stringify(json) + }).catch((error) => { + throw new Error(`POST ${url}: ${error.cause?.code ?? error.message}`); + }); + return { status: response.status, body: parseBody(await response.text()) }; + }); + +class Jar { + private cookies = new Map(); + + constructor(private host: string) {} + + take(reply: Reply) { + for (const line of [reply.headers["set-cookie"] ?? []].flat()) { + const parsed = parseCookie(line); + if (!parsed) continue; + const { name, value, maxAge, expires } = parsed; + const ends = maxAge === undefined ? expires : Date.now() + maxAge * 1000; + this.cookies.set(name, { + name, + value, + domain: this.host, + path: parsed.path ?? "/", + expires: ends === undefined ? -1 : new Date(ends).getTime() / 1000, + httpOnly: Boolean(parsed.httpOnly), + secure: Boolean(parsed.secure), + sameSite: parsed.sameSite ?? "Lax" + }); + } + } + + get header() { + return this.all.map(({ name, value }) => `${name}=${value}`).join("; "); + } + + value(name: string) { + return this.cookies.get(name)?.value ?? ""; + } + + get all() { + return [...this.cookies.values()]; + } +} + +export interface Client { + client_id: string; + redirect_uris: string[]; + scope: string; +} + +const push = async (dpop: typeof fetch, challenge: string, client: Client, loginHint?: string) => { + const server = await authServer(); + const state = randomBytes(18).toString("base64url"); + const par = ok( + "the pds par", + await dpopPost(dpop, server.pushed_authorization_request_endpoint, { + redirect_uri: client.redirect_uris[0], + code_challenge: challenge, + code_challenge_method: "S256", + state, + ...(loginHint ? { login_hint: loginHint } : {}), + response_mode: "fragment", + response_type: "code", + scope: client.scope, + client_id: client.client_id + }), + 201 + ); + return { server, state, requestUri: par.request_uri as string }; +}; + +const authorize = async (account: Account, dpop: typeof fetch, challenge: string) => { + const client = await clientMetadata(); + const { server, state, requestUri } = await push(dpop, challenge, client, account.handle); + const pds = new URL(server.issuer); + + // opening the page is what sets the device and csrf cookies the calls need + const jar = new Jar(pds.host); + const page = `${server.authorization_endpoint}?${new URLSearchParams({ + client_id: client.client_id, + request_uri: requestUri + })}`; + const opened = await send(page, "GET", { + "sec-fetch-site": "same-site", + "sec-fetch-mode": "navigate", + "sec-fetch-dest": "document", + referer: `${config.webUrl}/` + }); + if (opened.status !== 200) throw new Error(`the pds authorize page answered ${opened.status}`); + jar.take(opened); + + // the api only answers requests that look like they came from its own page + const asThePage = (headers: Record = {}) => ({ + "sec-fetch-site": "same-origin", + "sec-fetch-mode": "same-origin", + "sec-fetch-dest": "empty", + origin: pds.origin, + referer: page, + "x-csrf-token": jar.value("csrf-token"), + cookie: jar.header, + ...headers + }); + const api = `${pds.origin}/@atproto/oauth-provider/~api`; + const signingIn = await send(`${api}/sign-in`, "POST", asThePage(), { + username: account.handle, + password: config.password, + remember: false, + locale: "en" + }); + const signedIn = ok("the pds sign in", signingIn); + jar.take(signingIn); + if (signedIn.account.did !== account.did) + throw new Error(`${account.handle} signed in as ${signedIn.account.did}`); + + const consent = await send( + `${api}/consent`, + "POST", + asThePage({ authorization: `Bearer ${signedIn.ephemeralToken}` }), + { did: account.did, scope: client.scope } + ); + jar.take(consent); + // consent returns a url that only bounces the code on to the web's callback + const sent = new URL(ok("the pds consent", consent).url).searchParams; + const code = sent.get("code"); + if (!code || sent.get("state") !== state || sent.get("iss") !== server.issuer) + throw new Error(`the pds did not send this request's code back: ${sent}`); + return { client, server, code, cookies: jar.all }; +}; + +const dpopFetch = (key: Awaited>) => + createDpopFetch({ + key, + nonces: { + get: (origin) => nonces.get(origin), + set: (origin, nonce) => void nonces.set(origin, nonce) + }, + isAuthServer: true, + // undici's fetch can't read the Request atcute builds and node's fetch can't take an undici 8 + // dispatcher, so unpack the request into a plain url and init for undici + fetch: async (input, init) => { + const req = new Request(input, init); + const body = req.body ? await req.arrayBuffer() : undefined; + const response = await undiciFetch(req.url, { + method: req.method, + headers: [...req.headers], + body, + signal: req.signal, + dispatcher + }); + return response as unknown as Response; + } + }); + +export const pushedRequest = async (client?: Client) => { + const { challenge } = await generatePkce(); + const dpop = dpopFetch(await generateDpopKey(["ES256"])); + return (await push(dpop, challenge, client ?? (await clientMetadata()))).requestUri; +}; + +export const mintSession = async (account: Account): Promise => { + const key = await generateDpopKey(["ES256"]); + const dpop = dpopFetch(key); + const { verifier, challenge } = await generatePkce(); + const { client, server, code, cookies } = await authorize(account, dpop, challenge); + const token = ok( + "the pds token", + await dpopPost(dpop, server.token_endpoint, { + grant_type: "authorization_code", + redirect_uri: client.redirect_uris[0], + code, + code_verifier: verifier, + client_id: client.client_id + }) + ); + if (token.sub !== account.did || token.token_type !== "DPoP") + throw new Error(`the pds gave a ${token.token_type} token for ${token.sub}`); + const granted = new Set(token.scope.split(" ")); + const missing = client.scope.split(" ").filter((scope: string) => !granted.has(scope)); + if (missing.length) throw new Error(`the pds did not grant ${missing.join(" ")}`); + + const now = Date.now(); + const pds = new URL(server.issuer).origin; + const web = new URL(config.webUrl); + const session = { + value: { + dpopKey: key, + info: { + sub: account.did, + // the web reads aud from the did document, which names this pds + aud: new URL(config.pdsUrl).href, + server: { + issuer: server.issuer, + authorization_endpoint: server.authorization_endpoint, + pushed_authorization_request_endpoint: + server.pushed_authorization_request_endpoint, + revocation_endpoint: server.revocation_endpoint, + token_endpoint: server.token_endpoint + } + }, + token: { + scope: token.scope, + refresh: token.refresh_token, + access: token.access_token, + type: "DPoP", + expires_at: now + token.expires_in * 1000 + } + }, + expiresAt: null + }; + const nonce = nonces.get(pds); + const known = nonce + ? { [pds]: { value: nonce, expiresAt: now + 86_400_000, updatedAt: now } } + : {}; + const webCookie = (name: string, value: string): Cookie => ({ + name, + value, + domain: web.host, + path: "/", + expires: now / 1000 + 365 * 24 * 60 * 60, + httpOnly: false, + secure: true, + sameSite: "Lax" + }); + const storage = { + "tangled.currentHandle": account.handle, + "atcute-oauth:sessions": JSON.stringify({ [account.did]: session }), + "tangled.accounts": JSON.stringify([ + { did: account.did, handle: account.handle, addedAt: Math.floor(now / 1000) } + ]), + "atcute-oauth:states": "{}", + "tangled.currentDid": account.did, + "atcute-oauth:dpopNonces": JSON.stringify(known) + }; + return { + cookies: [ + ...cookies, + webCookie("tangled.currentDid", encodeURIComponent(account.did)), + webCookie("tangled.currentHandle", account.handle) + ], + origins: [ + { + origin: web.origin, + localStorage: Object.entries(storage).map(([name, value]) => ({ name, value })) + } + ] + }; +}; + +export const logIn = async (page: Page, account: Account) => { + await page.goto("/login", { waitUntil: "networkidle" }); + await signInHere(page, account); +}; + +export const signInHere = async (page: Page, account: Account) => { + // the form ignores clicks until it hydrates, so the page has to settle first + await page.waitForLoadState("networkidle"); + // the typeahead popover covers the button, so enter submits instead + await page.getByLabel("Handle", { exact: true }).fill(account.handle); + await page.getByLabel("Handle", { exact: true }).press("Enter"); + await page.locator("input[type=password]").fill(config.password); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await page.getByRole("button", { name: "Authorize", exact: true }).click(); + // an account with no profile lands on the welcome wizard, so only leaving /oauth is waited for + await page.waitForURL( + (url) => url.origin === config.webUrl && !url.pathname.startsWith("/oauth") + ); + // currentDid appears before the tokens are stored, so wait for both + await page.waitForFunction( + (did) => + localStorage.getItem("tangled.currentDid") === did && + Boolean(localStorage.getItem("atcute-oauth:sessions")?.includes(did)), + account.did + ); + await page.waitForLoadState("networkidle"); +}; diff --git a/e2e/harness/stream.ts b/e2e/harness/stream.ts new file mode 100644 index 0000000000000000000000000000000000000000..ad8d949c0993dbe051ef4ea43877f089e7e16683 --- /dev/null +++ b/e2e/harness/stream.ts @@ -0,0 +1,203 @@ +import { WebSocket } from "undici"; + +import { config } from "./config"; +import { Frames } from "./frames"; +import { dispatcher, get, partsOf } from "./net"; +import type { Held } from "./tabs"; + +type Action = "create" | "update" | "delete"; + +interface Seen { + id: number; + action: Action; + cid?: string; + live: boolean; + at: number; +} + +// /stats lags the newest id, so start the replay a bit before it +const REPLAY = 1_000; +// frames go out when the db write commits, so this only covers the db hop +const BUDGET = 10_000; + +const uriOf = (record: { did: string; collection: string; rkey: string }) => + `at://${record.did}/${record.collection}/${record.rkey}`; + +// opened before the first flow so a check can look back at earlier frames +class Watcher extends Frames { + private socket?: WebSocket; + private floor = -1; + private last = -1; + private stopped = true; + private retries = 0; + private downSince?: number; + private lastWords = ""; + + get running() { + return !this.stopped; + } + + async start() { + if (this.running) return; + const { status, body } = await get(`${config.hydrantUrl}/stats`); + const stored = body?.counts?.events; + if (status !== 200 || typeof stored !== "number") { + throw new Error( + `hydrant /stats should count its events but answered ${status} ${JSON.stringify(body)}` + ); + } + this.floor = Math.max(0, stored - REPLAY); + this.stopped = false; + await this.connect().catch((cause) => { + this.stop(); + throw cause; + }); + } + + stop() { + this.stopped = true; + this.socket?.close(); + } + + // take it before the write goes through, the frames it causes all get higher ids + mark() { + return Math.max(this.last, this.floor - 1); + } + + // a dead socket must never look like hydrant being quiet + describe(since: number) { + const parts = [`the watcher is at #${this.last}`]; + if (since < this.forgotUpTo) parts.push(`it forgot frames up to #${this.forgotUpTo}`); + if (this.downSince !== undefined) { + const down = ((Date.now() - this.downSince) / 1000).toFixed(1); + parts.push( + `it has been disconnected for ${down}s (${this.lastWords || "no reason given"})` + ); + } + return parts.join(", "); + } + + // a reconnect resumes at the next id and hydrant replays the record frames missed while down + private connect() { + const cursor = this.last >= 0 ? this.last + 1 : this.floor; + const url = `${config.hydrantUrl.replace(/^http/, "ws")}/stream?cursor=${cursor}`; + const socket = new WebSocket(url, { dispatcher }); + this.socket = socket; + socket.addEventListener("message", (event) => this.take(JSON.parse(String(event.data)))); + socket.addEventListener("open", () => { + this.downSince = undefined; + this.retries = 0; + }); + socket.addEventListener("close", (event) => { + if (this.stopped) return; + this.downSince ??= Date.now(); + this.lastWords ||= `close ${event.code} ${event.reason}`.trim(); + const wait = Math.min(250 * 2 ** this.retries++, 2_000); + setTimeout(() => { + if (!this.stopped) this.connect().catch(() => {}); + }, wait); + }); + return new Promise((opened, failed) => { + socket.addEventListener("open", () => opened(), { once: true }); + socket.addEventListener("error", () => failed(new Error(`${url} would not open`)), { + once: true + }); + }); + } + + private take(frame: any) { + if (frame.type === "error") { + this.lastWords = `hydrant said ${frame.error}: ${frame.message}`; + return; + } + if (typeof frame.id !== "number" || frame.id <= this.last) return; + this.last = frame.id; + this.lastWords = ""; + if (frame.type === "record") this.keep(frame.id, frame.record); + this.wake(); + } + + private keep(id: number, record: any) { + const uri = uriOf(record); + this.append(uri, { + id, + action: record.action, + cid: record.cid, + live: record.live, + at: Date.now() + }); + } +} + +export const watcher = new Watcher(); + +const waitFor = async (what: string, check: () => [done: boolean, saw: string, newest?: Seen]) => { + const deadline = Date.now() + BUDGET; + for (;;) { + const [done, saw, newest] = check(); + if (done) return newest; + const left = deadline - Date.now(); + if (left <= 0) throw new Error(`${what}, gave up after ${BUDGET / 1000}s: ${saw}`); + await watcher.changed(left); + } +}; + +const told = (seen: Seen) => + `${seen.action}${seen.cid ? ` ${seen.cid}` : ""} #${seen.id}${seen.live ? "" : " (backfill)"}`; + +const framesOf = (uri: string, since: number) => { + if (!watcher.running) { + throw new Error( + `hydrant's stream should be watched for ${uri} but this process never opened it` + ); + } + const frames = watcher.seen(uri).filter((seen) => seen.id > since); + const after = since < 0 ? "since the watcher opened" : `after #${since}`; + const said = frames.length + ? `${after} it announced ${frames.slice(-5).map(told).join(", ")}` + : `it announced nothing for that uri ${after}`; + return { frames, said: `${said}, ${watcher.describe(since)}` }; +}; + +const replacedBy = (uri: string, seen: Seen) => { + const before = watcher + .seen(uri) + .filter((other) => other.id < seen.id && other.cid !== seen.cid) + .at(-1); + return before?.action === "delete" ? undefined : before; +}; + +export const checkStream = ( + { uri, cid }: { uri: string; cid?: string }, + how: Held["method"] | "settled", + since = -1 +) => { + if (cid === undefined) + return waitFor(`hydrant's stream should announce the delete of ${uri}`, () => { + const { frames, said } = framesOf(uri, since); + const newest = frames.at(-1); + return [newest?.action === "delete", said, newest]; + }); + const wanted = how === "create" ? "a create" : "a create or update"; + return waitFor(`hydrant's stream should announce ${wanted} of ${uri} at ${cid}`, () => { + const { frames, said } = framesOf(uri, since); + const newest = frames.at(-1); + // only the newest frame counts, so an older cid coming back after it fails too + if (newest?.cid !== cid || newest.action === "delete") return [false, said]; + const ours = frames.filter((seen) => seen.cid === cid); + if (how === "create") return [ours.some((seen) => seen.action === "create"), said, newest]; + // a backfill says create for old records too, so only a live create counts as new + const stillThere = ours.map((seen) => + seen.live && seen.action === "create" ? replacedBy(uri, seen) : undefined + ); + if (stillThere.every(Boolean)) + return [ + false, + `${said}, but ${told(stillThere[0]!)} was still there so it had to be an update` + ]; + return [true, said, newest]; + }); +}; + +// hydrant only indexes org.tangled.*, so other collections never reach the stream +export const streamCarries = (uri: string) => partsOf(uri).collection.startsWith("org.tangled."); diff --git a/e2e/harness/tabs.ts b/e2e/harness/tabs.ts new file mode 100644 index 0000000000000000000000000000000000000000..64bbf375a180d1bc2ab8ee829875ecf8a8839f4a --- /dev/null +++ b/e2e/harness/tabs.ts @@ -0,0 +1,267 @@ +import { + test, + type Browser, + type BrowserContext, + type Page, + type Request, + type Route +} from "@playwright/test"; + +import { authFile, isSeeded, type Account } from "./config"; +import { parseBody, sleep, type Reply } from "./net"; +import { watchBrowser } from "./record"; +import { logIn, mintSession } from "./signin"; +import type { Slot } from "./vocabulary"; + +interface Write { + method: Held["method"]; + collection: string; + host: string; + body: any; + route: Route; +} + +export interface Held { + method: "create" | "put" | "delete"; + // the knot and the pds take the same xrpc, so the host is what tells them apart + host: string; + body: any; + release: () => Promise; +} + +const SETTLE_MS = 100; + +const PATCH = "org.tangled.review.patch"; +const SUBMIT_PATCH = "org.tangled.review.submitPatch"; + +const holdWrites = async (where: Page | BrowserContext) => { + const parked: Write[] = []; + const waiting: (() => void)[] = []; + // a parked write is the test's to release, so it doesn't count as busy + const inflight = new Map(); + // playwright sometimes never reports a request as finished, so an old one stops counting as busy + const STALE_MS = 10_000; + const busy = () => [...inflight].filter(([, began]) => Date.now() - began < STALE_MS); + // these can't lead to a write, so they don't hold the page busy + const passive = new Set(["image", "font", "media", "stylesheet"]); + // only code under /src writes records, so this traffic can linger + const background = [ + "/_internal/indexnow", + "deliberi", + "searchActorsTypeahead", + "/node_modules/" + ]; + where.on("request", (request) => { + const url = request.url(); + if (passive.has(request.resourceType()) || background.some((part) => url.includes(part))) + return; + inflight.set(request, Date.now()); + }); + where.on("requestfinished", (request) => inflight.delete(request)); + where.on("requestfailed", (request) => inflight.delete(request)); + + const park = (route: Route, method: Held["method"], collection?: string) => { + inflight.delete(route.request()); + const host = new URL(route.request().url()).host; + const body = route.request().postDataJSON(); + parked.push({ method, collection: collection ?? body.collection, host, body, route }); + waiting.splice(0).forEach((wake) => wake()); + }; + for (const method of ["create", "put", "delete"] as const) + await where.route(`**/xrpc/com.atproto.repo.${method}Record`, (route) => + park(route, method) + ); + // patches use their own xrpc, so the createRecord route above never sees them + await where.route(`**/xrpc/${SUBMIT_PATCH}`, (route) => park(route, "create", PATCH)); + + const describe = (w: Write) => `${w.method} ${w.collection} to ${w.host}`; + + const next = async ( + collection: string, + timeout = 10_000, + matching?: (body: any) => boolean + ): Promise => { + const deadline = Date.now() + timeout; + for (;;) { + const at = parked.findIndex( + (w) => w.collection === collection && (!matching || matching(w.body)) + ); + if (at >= 0) { + const { method, host, body, route } = parked.splice(at, 1)[0]; + return { + method, + host, + body, + release: async () => { + const response = await route.fetch(); + const text = await response.text(); + // the page always gets the real answer, failing it is the test's job + await route.fulfill({ response }); + return { status: response.status(), body: parseBody(text) }; + } + }; + } + const left = deadline - Date.now(); + if (left <= 0) { + const held = parked.map(describe).join(", ") || "nothing"; + const which = matching ? " that matched" : ""; + throw new Error( + `no write to ${collection}${which} was sent, the page sent: ${held}` + ); + } + await new Promise((wake) => { + waiting.push(wake); + setTimeout(wake, left); + }); + } + }; + + const expectNoOthers = () => { + if (parked.length) + throw new Error(`the page also sent: ${parked.map(describe).join(", ")}`); + }; + + // the dev server compiles modules as pages ask, so a write can come late under parallel workers + const quiet = async (silence = 500, limit = 20_000) => { + const deadline = Date.now() + limit; + let since = Date.now(); + while (Date.now() < deadline && Date.now() - since < silence) { + await sleep(50); + if (busy().length) since = Date.now(); + } + if (Date.now() - since < silence) { + throw new Error( + `the page still had a request running after ${limit / 1000}s, so nothing can be said about what it sends next: ${busy() + .map(([request]) => request.url()) + .join(", ")}` + ); + } + }; + + const sent = (collection: string) => parked.some((w) => w.collection === collection); + + return { next, expectNoOthers, quiet, sent }; +}; + +interface Tab { + context: BrowserContext; + windows: Map; + writes: Awaited>; + held: Map; + devices: BrowserContext[]; +} + +export class Tabs { + private tabs = new Map(); + + constructor(private browser: Browser) {} + + private of(account: Account) { + const tab = this.tabs.get(account.name); + if (!tab) throw new Error(`${account.name} has no browser yet, start with an .open()`); + return tab; + } + + page(account: Account, window = "") { + const page = this.of(account).windows.get(window); + if (!page) + throw new Error(`${account.name} has no window ${window}, start with a .window()`); + return page; + } + + async openWindow(account: Account, window: string) { + const tab = this.of(account); + if (!tab.windows.has(window)) tab.windows.set(window, await tab.context.newPage()); + } + + // another browser signed in as the same account, sharing no storage with the first + async openDevice(account: Account, window: string) { + const tab = this.of(account); + if (tab.windows.has(window)) return; + const context = await this.browser.newContext({ + storageState: await this.signedIn(account, false) + }); + watchBrowser(context); + tab.devices.push(context); + tab.windows.set(window, await context.newPage()); + } + + private signedIn(account: Account, throughThePage: boolean) { + if (isSeeded(account)) return authFile(account); + if (throughThePage) return undefined; + return test.step("mint a session", () => mintSession(account)); + } + + async open(account: Account, throughThePage: boolean) { + if (this.tabs.has(account.name)) return; + const storageState = await this.signedIn(account, throughThePage); + const context = await this.browser.newContext({ storageState }); + watchBrowser(context); + const page = await context.newPage(); + const writes = await holdWrites(context); + this.tabs.set(account.name, { + context, + windows: new Map([["", page]]), + writes, + held: new Map(), + devices: [] + }); + if (!isSeeded(account) && throughThePage) + await test.step("log in through the login page", () => logIn(page, account)); + } + + // a slot already holding a write answers with that one, whatever it is asked to match + async hold(account: Account, slot: Slot, matching?: (body: any) => boolean) { + const tab = this.of(account); + let held = tab.held.get(slot); + if (!held) { + held = await tab.writes.next(slot.nsid, undefined, matching); + tab.held.set(slot, held); + } + return held; + } + + async release(account: Account, slot: Slot, held: Held, expected: number | number[] = 200) { + const reply = await held.release(); + this.of(account).held.delete(slot); + if (![expected].flat().includes(reply.status)) { + throw new Error( + `${held.method} ${slot.nsid} answered ${reply.status} ${JSON.stringify(reply.body)}, expected ${expected}` + ); + } + return reply.body; + } + + // playwright's own networkidle always waits 500ms, which every open and reload would pay + async settle(account: Account) { + await this.of(account).writes.quiet(SETTLE_MS); + } + + async expectNoWrite(account: Account, slot: Slot) { + const { writes } = this.of(account); + await writes.quiet(); + if (writes.sent(slot.nsid)) throw new Error(`the page sent a write to ${slot.nsid}`); + } + + async expectQuiet() { + const tabs = [...this.tabs.values()]; + for (const tab of tabs) { + const [unreleased] = tab.held.keys(); + if (unreleased) { + throw new Error( + `the page sent a write to ${unreleased.nsid} that no .trace() let through` + ); + } + } + // wait on every tab before judging any, or a late write from one slips past + await Promise.all(tabs.map((tab) => tab.writes.quiet())); + for (const tab of tabs) tab.writes.expectNoOthers(); + } + + async close() { + for (const tab of this.tabs.values()) { + for (const context of [tab.context, ...tab.devices]) + await context.close().catch(() => {}); + } + } +} diff --git a/e2e/harness/versions.ts b/e2e/harness/versions.ts new file mode 100644 index 0000000000000000000000000000000000000000..e3e430e1981d84dd9684f8cb4e3bdd14e4f9f749 --- /dev/null +++ b/e2e/harness/versions.ts @@ -0,0 +1,80 @@ +import { config } from "./config"; +import { get, partsOf, until, xrpc } from "./net"; + +// listRecordVersions 400s on anything outside knot_cobs::social_collections +const COBS = new Set([ + "org.tangled.track.ticket", + "org.tangled.feed.comment", + "org.tangled.feed.reaction", + "org.tangled.label.definition", + "org.tangled.review.patch", + "org.tangled.repo.manifest" +]); + +export const isCob = (uri: string) => COBS.has(partsOf(uri).collection); + +export interface Version { + cid?: string; + changeId: string; + editor: string; +} + +export const versionsOf = async (uri: string) => { + const { status, body } = await get( + xrpc(config.knotUrl, "org.tangled.repo.listRecordVersions", { uri, limit: "500" }) + ); + return { status, body, versions: (status === 200 ? body.versions : []) as Version[] }; +}; + +const listed = (versions: Version[]) => + versions.map((version) => `${version.cid ?? "erased"} by ${version.editor}`).join(", ") || + "no versions"; + +// no uri before a create, and a chosen rkey like a label slug 404s +export const versionsBefore = async (uri?: string): Promise => + uri && isCob(uri) + ? until( + `the knot should list the versions of ${uri} before the write`, + 10_000, + async () => { + const { status, body, versions } = await versionsOf(uri); + return [ + status === 200 || status === 404, + `it answered ${status} ${JSON.stringify(body)}`, + versions + ]; + } + ) + : []; + +interface Expected { + // left out for records the knot wrote itself, so the check skips it + editor?: string; + // without a read from before the write, only the newest version gets checked + had?: Version[]; +} + +// resolves with the versions so the git check reads the same chain +export const checkVersions = ( + { uri, cid }: { uri: string; cid?: string }, + { editor, had }: Expected +) => { + const count = had ? `${had.length + 1} versions` : "versions"; + const by = editor ? ` by ${editor}` : ""; + return until( + `the knot should list ${count} of ${uri}, the newest ${cid ?? "erased"}${by}`, + 10_000, + async () => { + const { status, body, versions } = await versionsOf(uri); + if (status !== 200) return [false, `it answered ${status} ${JSON.stringify(body)}`]; + const last = versions.at(-1); + const grew = + !had || + (versions.length === had.length + 1 && + had.every((version, at) => versions[at].changeId === version.changeId)); + const newest = last?.cid === cid && (!editor || last?.editor === editor); + const before = had?.length ? `, before the write it listed ${listed(had)}` : ""; + return [grew && newest, `it lists ${listed(versions)}${before}`, versions]; + } + ); +}; diff --git a/e2e/harness/vocabulary.ts b/e2e/harness/vocabulary.ts new file mode 100644 index 0000000000000000000000000000000000000000..4a45fb4300800ad36c31d3795b42766e706b150c --- /dev/null +++ b/e2e/harness/vocabulary.ts @@ -0,0 +1,269 @@ +import type { Locator, Page } from "@playwright/test"; + +import type { Chore } from "./chores"; +import type { Account } from "./config"; +import type { Did } from "@atcute/lexicons/syntax"; + +import type { RecordNsid } from "./lexicons"; +import { partsOf, stamp, type Written } from "./net"; +import type { Hop } from "./services"; + +// declared once at module level, each run fills its own copy +export interface Slot { + nsid: N; +} +export const record = (nsid: N): Slot => ({ nsid }); + +export const declarationId = record("org.tangled.repo.declaration"); + +export const manifestId = record("org.tangled.repo.manifest"); + +export const collaboratorId = record("org.tangled.graph.collaborator"); + +// a push changes this, the page never writes it +export const refsId = record("org.tangled.git.refs"); + +// a pull is a ticket too, its own slot so one flow can hold an issue and a pull at once +// the page also fires a subscribe write on its own, the flow has to let it through +export const ticketId = record("org.tangled.track.ticket"); +export const pullId = record("org.tangled.track.ticket"); +export const patchId = record("org.tangled.review.patch"); + +// a refused write never fills its slot, so the refusal gets its own +export const refusedTicketId = record("org.tangled.track.ticket"); + +export const commentId = record("org.tangled.feed.comment"); +export const subscriptionId = record("org.tangled.feed.subscription"); +export const attestationId = record("org.tangled.graph.attestation"); + +// written to the account's own pds, not the knot +export const profileId = record("org.tangled.actor.profile"); +export const starId = record("org.tangled.feed.star"); +export const followId = record("org.tangled.graph.follow"); +export const vouchId = record("org.tangled.graph.vouch"); +export const publicKeyId = record("org.tangled.key.sshKey"); + +export const labelDefinitionId = record("org.tangled.label.definition"); + +export interface Ctx { + actor: Account; + page: Page; + repo: Repo; + upstream: Repo; + // output is git push's stderr, where the knot's remote: lines land + pushed: { sha: string; branch: string; output: string }; + get: (slot: Slot) => Written; + set: (slot: Slot, written: Written) => void; + adopt: (slot: Slot, written: Written) => void; + adoptRepo: (made: { did: Did; name: string }) => void; + owe: (label: string, chore: Chore) => void; +} + +export const rkeyOf = (ctx: Ctx, slot: Slot) => partsOf(ctx.get(slot).uri).rkey; + +export type Lazy = T | ((ctx: Ctx) => T); + +export const resolve = (value: Lazy, ctx: Ctx): T => + typeof value === "function" ? (value as (ctx: Ctx) => T)(ctx) : value; + +export interface Repo { + did: Did; + name: string; + path: string; + defaultBranch: string; + heads: Record; +} + +export const repoAt = ( + owner: Account, + repo: Pick & Partial> +): Repo => ({ heads: {}, ...repo, path: `/${owner.handle}/${repo.name}` }); + +export interface Where { + describe: string; + path: (ctx: Ctx) => string; +} +export interface Target { + describe: string; + locate: (page: Page, ctx: Ctx) => Locator; + absent?: boolean; +} + +export const where = (describe: string, path: Where["path"]): Where => ({ describe, path }); +export const target = (describe: string, locate: Target["locate"]): Target => ({ + describe, + locate +}); +export const button = (name: string) => + target(`button ${name}`, (page) => page.getByRole("button", { name, exact: true })); +export const link = (name: string) => + target(`link ${name}`, (page) => page.getByRole("link", { name })); +export const field = (label: string) => target(`field ${label}`, (page) => page.getByLabel(label)); +export const placeholder = (hint: string) => + target(`field ${hint}`, (page) => page.getByPlaceholder(hint)); +export const text = (words: string | RegExp) => + target(`text ${words}`, (page) => page.getByText(words)); +// a closed dialog and a layout's hidden twin stay in the dom, so only visible ones count +export const testid = (id: string, attributes: Record = {}) => { + for (const value of Object.values(attributes)) + if (/[\u0000-\u001f]/.test(String(value))) + throw new Error( + `a control character in ${JSON.stringify(value)} can't be matched by a selector` + ); + const only = Object.entries(attributes).map( + ([name, value]) => `[data-${name}=${JSON.stringify(String(value))}]` + ); + const where = Object.entries(attributes).map(([name, value]) => `${name} ${value}`); + return target(["testid", id, ...where].join(" "), (page) => + page + .locator(`[data-testid=${JSON.stringify(id)}]${only.join("")}`) + .filter({ visible: true }) + ); +}; +export const component = (name: string) => + target(`component ${name}`, (page) => page.locator(`[data-component=${JSON.stringify(name)}]`)); +export const not = (what: Target): Target => ({ + ...what, + describe: `no ${what.describe}`, + absent: true +}); + +// a locator answers getBy and locator like a page does, so the cast holds +export const within = (scope: Target, inner: Target): Target => + target(`${inner.describe} in ${scope.describe}`, (page, ctx) => + inner.locate(scope.locate(page, ctx) as unknown as Page, ctx) + ); + +export const through = (...hops: Hop[]) => hops; + +export const unique = (what: string) => `${what} ${stamp()}`; + +// the knot only takes a put or a delete that names the version it read +export const replacing = (slot: Slot, record?: Lazy>) => (ctx: Ctx) => ({ + swapRecord: ctx.get(slot).cid, + ...(record ? { record: resolve(record, ctx) } : {}) +}); + +export const mergeButton = within(testid("mergeChoice"), testid("primaryAction")); + +export const moreActionsOn = (_what: "issue" | "pull request") => + within(component("Ticket"), testid("moreMenu")); +export const moreActions = moreActionsOn("issue"); + +export const menuItem = (name: string) => + target(`menu item ${name}`, (tab) => tab.getByRole("menuitem", { name, exact: true })); + +export const confirmDelete = testid("confirm"); + +export const commentMenu = target("the comment's menu", (tab, ctx) => + within(testid("commentCard", { handle: ctx.actor.handle }), testid("moreMenu")).locate(tab, ctx) +); + +// exact, or a substring also finds the commit messages +export const heading = (name: string) => + target(`heading ${name}`, (tab) => tab.getByRole("heading", { name, exact: true })); + +export const exactLink = (name: string | ((ctx: Ctx) => string)) => + target(`the link ${typeof name === "string" ? name : "to the repo"}`, (tab, ctx) => + tab.getByRole("link", { name: typeof name === "string" ? name : name(ctx), exact: true }) + ); + +// one save bar at the top and one at the bottom, so take the first +export const saveButton = target("the Save button", (tab, ctx) => + testid("save").locate(tab, ctx).first() +); + +export const unsaved = target("the unsaved changes note", (tab, ctx) => + testid("unsavedChanges").locate(tab, ctx).first() +); + +// scoped to the modal, the page says Delete repository twice +export const dialogButton = (name: string) => + target(`dialog button ${name}`, (tab) => + tab.getByRole("dialog").getByRole("button", { name, exact: true }) + ); + +export const nameField = target("the repo name field", (tab) => + tab.locator('[data-testid="confirmName"], [data-testid="name"]').filter({ visible: true }) +); + +export const openDelete = testid("setConfirmDelete"); + +// sveltekit's route announcer is an alert too, so leave it out +export const alerts = target("the page's alerts", (tab) => + tab.locator("[role=alert]:not(#svelte-announcer)") +); + +// check what got written and that the alert is up, not what it says +export const refusal = testid("errorAlert"); +export const staleEdit = testid("staleEdit"); + +export const commitRow = (nth: "first" | "last" | number) => + target(`the ${nth} commit row`, (tab, ctx) => { + const rows = testid("commitRow").locate(tab, ctx); + return typeof nth === "number" ? rows.nth(nth - 1) : rows[nth](); + }); + +// issues and pulls say different words in the comment box +export const discussion = testid("body"); +export const pullDiscussion = testid("reviewComment"); + +export const triage = "triage"; +export const triageToken = { $type: "org.tangled.label.defs#labelToken", key: triage }; +export const editLabels = testid("editLabels"); +// the value starts with a nul no selector can name, so match its end +export const createOption = target("the create option", (page) => + page.locator(`[data-testid="option"][data-value$="create:${triage}"]`).filter({ visible: true }) +); +export const labelOption = target("the triage option", (page) => + page.locator('[data-testid="option"]').filter({ visible: true, hasText: triage }) +); +// the open picker lists the same word too, so scope to the sidebar +export const triageChip = within(testid("labelHref"), text(triage)); + +// the count link has no accessible name, only the number +export const starCount = testid("starsHref"); + +export const topic = "e2e-topic"; +export const topicChip = testid("tag", { value: topic }); + +// the page says Vouch twice, so scope to this dialog +export const vouchDialog = testid("vouch"); +export const inVouchDialog = (name: string) => within(vouchDialog, button(name)); + +export const page = { + repo: where("the repo page", (ctx) => ctx.repo.path), + settings: where("the repo settings", (ctx) => `${ctx.repo.path}/settings`), + profile: where("the actor's profile", (ctx) => `/${ctx.actor.handle}`), + issues: where("the issues list", (ctx) => `${ctx.repo.path}/issues`), + newIssue: where("the new issue form", (ctx) => `${ctx.repo.path}/issues/new`), + pulls: where("the pulls list", (ctx) => `${ctx.repo.path}/pulls`), + newRepo: where("the new repo form", () => "/repo/new"), + newPull: (branches: { from: string; into: string }) => + where( + "the new pull form", + (ctx) => + `${ctx.repo.path}/pulls/new?sourceBranch=${branches.from}&targetBranch=${branches.into}` + ), + pull: (ticket: Slot) => + where("the pull page", (ctx) => `${ctx.repo.path}/pulls/${rkeyOf(ctx, ticket)}`), + issue: (ticket: Slot) => + where("the issue page", (ctx) => `${ctx.repo.path}/issues/${rkeyOf(ctx, ticket)}`) +}; + +export const knots = where("the knots settings", () => "/settings/knots"); +export const newKnot = where("the new knot form", () => "/settings/knots/new"); +export const spindles = where("the spindles settings", () => "/settings/spindles"); +export const newSpindle = where("the new spindle form", () => "/settings/spindles/new"); +export const profileSettings = where("the profile settings", () => "/settings"); +export const keys = where("the keys settings", () => "/settings/keys"); +export const keysNew = where("the new key form", () => "/settings/keys/new"); + +export const settingsAt = (sub: string) => + where(`the ${sub} settings`, (ctx) => `${ctx.repo.path}/settings/${sub}`); +export const newLabel = settingsAt("labels/new"); +export const renameForm = settingsAt("rename"); +export const branchesTab = where("the branches tab", (ctx) => `${ctx.repo.path}/branches`); + +export const loginThen = (path: (ctx: Ctx) => string) => + where("the login page", (ctx) => `/login?return_url=${encodeURIComponent(path(ctx))}`); diff --git a/e2e/harness/web-hash.ts b/e2e/harness/web-hash.ts new file mode 100644 index 0000000000000000000000000000000000000000..3eb6a70b427fe79c600da8d04e2ef0a41c68435b --- /dev/null +++ b/e2e/harness/web-hash.ts @@ -0,0 +1,33 @@ +import { createHash } from "node:crypto"; +import { readFileSync, readdirSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +// what the web container builds from: all of web/ except what it or its tools write there +const generated = new Set([ + "node_modules", + ".svelte-kit", + ".wrangler", + ".git", + "build", + "dist", + "coverage", + "storybook-static" +]); + +const web = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../web"); + +const filesUnder = (dir: string): string[] => + readdirSync(dir, { withFileTypes: true }) + .filter((entry) => !generated.has(entry.name)) + .flatMap((entry) => { + const file = path.join(dir, entry.name); + if (entry.isDirectory()) return filesUnder(file); + return entry.isFile() ? [file] : []; + }); + +const hash = createHash("sha1"); +for (const file of filesUnder(web).sort()) { + hash.update(path.relative(web, file)).update("\0").update(readFileSync(file)); +} +console.log(hash.digest("hex").slice(0, 16)); diff --git a/e2e/harness/webhooks.ts b/e2e/harness/webhooks.ts new file mode 100644 index 0000000000000000000000000000000000000000..447343356f5b1b88121837ae5579c76937b6fe17 --- /dev/null +++ b/e2e/harness/webhooks.ts @@ -0,0 +1,261 @@ +import { expect } from "@playwright/test"; +import { spawn, spawnSync } from "node:child_process"; +import { createHmac } from "node:crypto"; +import { existsSync, readFileSync, readlinkSync } from "node:fs"; +import { createServer, type IncomingHttpHeaders } from "node:http"; +import type { AddressInfo } from "node:net"; +import { createInterface } from "node:readline"; + +import { spindleGet, spindlePost } from "./api"; +import type { Account } from "./config"; +import { sleep, stamp, until } from "./net"; +import { resolve, type Ctx, type Lazy } from "./vocabulary"; + +export const WEBHOOKS = "org.tangled.webhook"; + +export interface Delivery { + method?: string; + headers: IncomingHttpHeaders; + raw: string; + body: any; +} + +// on linux the local stack has its own network namespace that can't reach the host's loopback, +// so the receiver runs inside it and reports over stdout. elsewhere it listens here +const inboxes = new Map(); +let serving: Promise | undefined; + +const receiver = ` +const server = require("node:http").createServer((req, res) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString(); + console.log(JSON.stringify({ url: req.url, method: req.method, headers: req.headers, raw })); + res.end("ok"); + }); +}); +server.listen(0, "127.0.0.1", () => console.log(JSON.stringify({ port: server.address().port }))); +process.stdin.on("end", () => process.exit()); +process.stdin.resume(); +`; + +const take = ( + url: string, + method: string | undefined, + headers: IncomingHttpHeaders, + raw: string +) => { + const inbox = inboxes.get(url); + if (!inbox) return false; + let body: unknown; + try { + body = JSON.parse(raw); + } catch {} + inbox.push({ method, headers, raw, body }); + return true; +}; + +// ns.pid goes stale if linux-ns runs again, so look up process-compose by its socket +const namespaceOfTheStack = () => { + const state = process.env.E2E_PC_STATE; + if (!state) return undefined; + let named: string; + try { + named = readFileSync(`${state}/run/ns.pid`, "utf8").trim(); + } catch { + return undefined; + } + if (existsSync(`/proc/${named}/ns/net`)) return named; + // the wrapper matches the same pgrep but is still in the host namespace + const here = readlinkSync("/proc/self/ns/net"); + const found = spawnSync("pgrep", ["-f", `process-compose up .*${state}/run/pc.sock`], { + encoding: "utf8" + }) + .stdout.split("\n") + .find((pid) => pid && readlinkSync(`/proc/${pid}/ns/net`) !== here); + if (!found) + throw new Error( + `${state}/run/ns.pid names ${named}, which is gone, and no process-compose is in the stack's namespace` + ); + return found; +}; + +const serveInsideTheStack = (ns: string) => + new Promise((ready, failed) => { + const child = spawn( + "nsenter", + ["--user", "--preserve-credentials", "--net", "-t", ns, "node", "-e", receiver], + { stdio: ["pipe", "pipe", "inherit"] } + ); + child.on("error", failed); + child.on("exit", (code) => + failed(new Error(`the receiver in the stack's namespace exited with ${code}`)) + ); + createInterface({ input: child.stdout }).on("line", (line) => { + const said = JSON.parse(line); + if (said.port) return ready(said.port); + take(said.url, said.method, said.headers, said.raw); + }); + // unref or the test run waits on the receiver + child.unref(); + (child.stdout as unknown as { unref(): void }).unref(); + (child.stdin as unknown as { unref(): void }).unref(); + }); + +const serveHere = () => + new Promise((ready) => { + const server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + const found = take( + req.url ?? "", + req.method, + req.headers, + Buffer.concat(chunks).toString() + ); + if (!found) return void res.writeHead(404).end(); + res.end("ok"); + }); + }); + server.unref(); + server.listen(0, "127.0.0.1", () => ready((server.address() as AddressInfo).port)); + }); + +const serve = () => { + const ns = namespaceOfTheStack(); + return (serving ??= ns ? serveInsideTheStack(ns) : serveHere()); +}; + +interface Hook { + id: string; + url: string; + owner: Account; + secret?: string; + inbox: Delivery[]; + taken: number; +} + +const hooks = new WeakMap(); +const hookOf = (ctx: Ctx) => { + const hook = hooks.get(ctx); + if (!hook) throw new Error("this step needs a webhook step before it"); + return hook; +}; + +export const hookOn = + (events: string[], options: { secret?: string; active?: boolean } = {}) => + async (ctx: Ctx) => { + const url = `http://127.0.0.1:${await serve()}/${stamp()}`; + const inbox: Delivery[] = []; + inboxes.set(new URL(url).pathname, inbox); + // spindle doesn't know a new repo for a bit, so this retries + const id = await until("spindle should take the webhook", 15_000, async () => { + const made = await spindlePost(ctx.actor, `${WEBHOOKS}.createWebhook`, { + repo: ctx.repo.did, + url, + events, + ...options + }); + return [ + made.status === 200, + `it answered ${made.status} ${JSON.stringify(made.body)}`, + made.body?.id + ]; + }); + hooks.set(ctx, { id, url, owner: ctx.actor, secret: options.secret, inbox, taken: 0 }); + }; + +export const hookUrl = (ctx: Ctx) => hookOf(ctx).url; +export const hookId = (ctx: Ctx) => hookOf(ctx).id; + +export const dropsHook = async (ctx: Ctx) => { + const hook = hookOf(ctx); + await spindlePost(hook.owner, `${WEBHOOKS}.deleteWebhook`, { + repo: ctx.repo.did, + webhook: hook.id + }); +}; + +interface Logged { + id: string; + webhook: string; + event: string; + url: string; + success: boolean; + requestBody?: { $bytes: string }; + responseCode?: number; +} + +const logOf = async (ctx: Ctx, hook: Hook) => { + const reply = await spindleGet(hook.owner, `${WEBHOOKS}.listWebhookDeliveries`, { + repo: ctx.repo.did, + webhooks: hook.id, + limit: "100" + }); + expect(reply.status, `spindle answered ${JSON.stringify(reply.body)}`).toBe(200); + return reply.body.deliveries as Logged[]; +}; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +// deliveries are taken in order, so an extra one fails here as the wrong event +export const delivers = + (event: string, userAgent: Lazy, check: (body: any, ctx: Ctx) => void) => + async (ctx: Ctx) => { + const hook = hookOf(ctx); + await until(`the receiver should get ${event}`, 30_000, async () => [ + hook.inbox.length > hook.taken, + `it has ${hook.inbox.length} deliveries, ${hook.taken} already checked` + ]); + const got = hook.inbox[hook.taken++]; + const { headers, body } = got; + + expect(got.method).toBe("POST"); + expect(headers["x-tangled-event"]).toBe(event); + expect(headers["content-type"]).toBe("application/json"); + expect(headers["user-agent"]).toBe(resolve(userAgent, ctx)); + expect(headers["x-tangled-hook-id"]).toBe(String(hook.id)); + expect(headers["x-tangled-delivery"]).toMatch(UUID); + expect(headers["x-tangled-repo"]).toBe(body.repository.full_name); + expect(headers["x-tangled-signature-256"]).toBe( + hook.secret + ? `sha256=${createHmac("sha256", hook.secret).update(got.raw).digest("hex")}` + : undefined + ); + expect(body.repository.full_name).toMatch(new RegExp(`^${hook.owner.did}/.+`)); + expect(body.repository.owner).toEqual({ did: hook.owner.did }); + check(body, ctx); + + const deliveryId = headers["x-tangled-delivery"]; + const logged = await until( + `spindle's delivery log should list ${deliveryId}`, + 15_000, + async () => { + const entry = (await logOf(ctx, hook)).find((one) => one.id === deliveryId); + return [Boolean(entry), `it has no such delivery`, entry]; + } + ); + expect(logged).toMatchObject({ + webhook: hook.id, + event, + url: hook.url, + success: true, + responseCode: 200 + }); + expect(Buffer.from(logged.requestBody?.$bytes ?? "", "base64").toString()).toBe(got.raw); + }; + +// a stray duplicate lands within a moment of the first, so sleep 3s first +export const nothingMore = async (ctx: Ctx) => { + const hook = hookOf(ctx); + await sleep(3000); + expect( + hook.inbox.map((one) => one.headers["x-tangled-event"]), + "the receiver got more than the flow expected" + ).toHaveLength(hook.taken); + expect(await logOf(ctx, hook), "spindle logged more than the flow expected").toHaveLength( + hook.taken + ); +}; diff --git a/e2e/outages/knot-outage.spec.ts b/e2e/outages/knot-outage.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..d271ddf9bfecdcc14597588bc76a6850786be76b --- /dev/null +++ b/e2e/outages/knot-outage.spec.ts @@ -0,0 +1,69 @@ +import { expect, test } from "@playwright/test"; + +import { knotRecords, tickets } from "../harness/api"; +import { config } from "../harness/config"; +import { + bobbin, + commentId, + discussion, + flow, + hydrant, + knot, + not, + page, + pds, + refusal, + subscriptionId, + testid, + text, + through, + ticketId +} from "../harness/flow"; +import { get, recordUrl, until } from "../harness/net"; +import { isReady, startProcess, stopProcess } from "../harness/processes"; +import { note } from "../harness/shared"; + +const KNOT = "knot2"; + +// a failed flow can leave the knot down, so the next test starts it again +test.afterEach(async () => { + if (!(await isReady(KNOT))) await startProcess(KNOT); +}); + +flow("a comment sent while the knot is down is refused, lands nowhere, and works once it is back") + .actor("owner") + .repo() + .ticket(ticketId, { title: "the knot will go away" }) + .open(page.issue(ticketId)) + + .step("the knot stops", () => stopProcess(KNOT)) + .fill(discussion, note) + .click(testid("submit")) + // caddy answers for the stopped knot, so it's a 502 to 504 and not a dropped connection + .refused(commentId, [502, 503, 504]) + .sees(refusal) + .step("the box still holds the text", async (ctx) => + expect(discussion.locate(ctx.page, ctx)).toHaveValue(note) + ) + + .step("the knot comes back", () => startProcess(KNOT)) + .step("the knot has no comment", async (ctx) => { + const listed = await until("the knot should answer again", 30_000, async () => { + const records = await knotRecords(ctx.repo.did, commentId.nsid).catch(() => undefined); + return [records !== undefined, "it isn't answering yet", records]; + }); + expect(listed).toHaveLength(0); + }) + // hydrant only redials after its backoff, and backfills what it missed once it does + .step("hydrant listens to the knot again", async (ctx) => { + const probe = await tickets.open(ctx.actor, ctx.repo.did, "is hydrant back"); + await until("hydrant should have the probe", 60_000, async () => { + const { status } = await get(recordUrl(config.hydrantUrl, probe.uri)); + return [status === 200, `it answered ${status}`]; + }); + }) + .click(testid("submit")) + .trace(commentId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .sees(text(note)) + .sees(not(refusal)); diff --git a/e2e/outages/spindle-outage.spec.ts b/e2e/outages/spindle-outage.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..cd73545f2af9fcbfb50c590c0440f34b39996882 --- /dev/null +++ b/e2e/outages/spindle-outage.spec.ts @@ -0,0 +1,120 @@ +import { test } from "@playwright/test"; + +import { knotReply, serviceDo } from "../harness/api"; +import { config, type Account } from "../harness/config"; +import { collaboratorId, flow, type Ctx, type Flow } from "../harness/flow"; +import { sleep, until } from "../harness/net"; +import { isReady, rewindJetstreamCursor, startProcess, stopProcess } from "../harness/processes"; +import { spindle } from "../harness/services"; +import { recall, remember, workflow, workflowPath } from "../harness/shared"; + +const SPINDLE = "spindle"; +const ADD = "com.atproto.repo.createRecord"; +const REMOVE = "com.atproto.repo.deleteRecord"; + +// a flow can die with spindle still stopped, and the next one needs it up +test.afterEach(async () => { + if (!(await isReady(SPINDLE))) await startProcess(SPINDLE); +}); + +// spindle checks its own stored collaborators here, not the knot's +const listWebhooks = (who: Account, ctx: Ctx) => + serviceDo(who, config.spindleUrl, config.spindleDid, "org.tangled.webhook.listWebhooks", { + params: { repo: ctx.repo.did } + }); + +const spindleAllows = (label: string, allowed: boolean) => async (ctx: Ctx) => { + const who = recall(ctx, label); + await until( + `spindle should ${allowed ? "allow" : "refuse"} ${label} on ${ctx.repo.did}`, + 60_000, + async () => { + const { status, body } = await listWebhooks(who, ctx); + return [(status === 200) === allowed, `it answered ${status} ${JSON.stringify(body)}`]; + } + ); +}; + +const collaboration = (nsid: string, label: string) => async (ctx: Ctx) => { + const reply = await knotReply(ctx.actor, ctx.repo.did, nsid, { + collection: collaboratorId.nsid, + rkey: recall(ctx, label).did, + ...(nsid === ADD + ? { record: { $type: collaboratorId.nsid, createdAt: new Date().toISOString() } } + : {}) + }); + if (reply.status !== 200) + throw new Error(`${nsid} for ${label} answered ${JSON.stringify(reply.body)}`); +}; + +// cursor 1 is older than anything jetstream keeps +const longOutage = async () => rewindJetstreamCursor(1); + +const servedRepo = (name: string) => + flow(name) + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ [workflowPath]: workflow }) + .step("spindle has a pipeline for the first push", (ctx) => + spindle.hasPipeline(ctx.repo.did, ctx.pushed.sha, "push") + ); + +const withTwoHelpers = (flow: Flow) => + flow + .actor("keeper") + .step("remember the keeper", remember("keeper")) + .actor("leaver") + .step("remember the leaver", remember("leaver")) + .actor("owner") + .step("the owner adds the leaver", collaboration(ADD, "leaver")) + .step("spindle lets the leaver in", spindleAllows("leaver", true)) + .step("spindle keeps the keeper out", spindleAllows("keeper", false)); + +const whileSpindleIsDown = (flow: Flow) => + flow + .step("spindle stops", () => stopProcess(SPINDLE)) + .step("the owner adds the keeper", collaboration(ADD, "keeper")) + .step("the owner removes the leaver", collaboration(REMOVE, "leaver")) + .step("spindle's saved cursor is older than jetstream's history", longOutage); + +const caughtUp = (flow: Flow) => + flow + .step("spindle lets the keeper in", spindleAllows("keeper", true)) + .step("spindle turns the leaver away", spindleAllows("leaver", false)) + .step("spindle still serves the owner", async (ctx) => { + const { status, body } = await listWebhooks(ctx.actor, ctx); + if (status !== 200) + throw new Error(`spindle answered ${status} ${JSON.stringify(body)}`); + }) + .push({ "e2e/after.txt": "after the outage\n" }) + .step("spindle has a pipeline for a push after the outage", (ctx) => + spindle.hasPipeline(ctx.repo.did, ctx.pushed.sha, "push", 60_000) + ); + +servedRepo("a spindle down for longer than jetstream remembers catches up on collaborators") + .use(withTwoHelpers) + .use(whileSpindleIsDown) + .step("spindle starts", () => startProcess(SPINDLE)) + .use(caughtUp); + +// a boot that never finishes catching up still has to leave the gap for the next one +servedRepo("a spindle stopped again while it catches up still catches up on the next start") + .use(withTwoHelpers) + .use(whileSpindleIsDown) + .step("spindle starts", () => startProcess(SPINDLE, "started")) + .step("spindle stops again at once", () => stopProcess(SPINDLE)) + .step("spindle starts and stops once more", async () => { + await startProcess(SPINDLE, "started"); + await sleep(1_000); + await stopProcess(SPINDLE); + }) + .step("spindle starts for good", () => startProcess(SPINDLE)) + .use(caughtUp); + +servedRepo("a push while spindle is down gets its pipeline once spindle is back") + .step("spindle stops", () => stopProcess(SPINDLE)) + .push({ "e2e/while-down.txt": "pushed while spindle was down\n" }) + .step("spindle starts", () => startProcess(SPINDLE)) + .step("spindle has a pipeline for the push it missed", (ctx) => + spindle.hasPipeline(ctx.repo.did, ctx.pushed.sha, "push", 120_000) + ); diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 0000000000000000000000000000000000000000..082432f19f85a827392dfc8c4b36b5d6c4ccc03c --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,26 @@ +{ + "name": "e2e", + "version": "0.0.0", + "private": true, + "type": "module", + "packageManager": "pnpm@11.17.0", + "scripts": { + "e2e": "./run", + "e2e:up": "./up", + "e2e:sweep": "./run --sweep", + "test": "playwright test" + }, + "devDependencies": { + "@atcute/atproto": "^4.0.4", + "@atcute/cbor": "^2.3.8", + "@atcute/cid": "^2.5.0", + "@atcute/firehose": "^1.2.0", + "@atcute/lexicons": "^2.1.1", + "@atcute/oauth-crypto": "^1.0.1", + "@playwright/test": "^1.61.1", + "@types/node": "^24", + "prettier": "^3.9.9", + "typescript": "^5", + "undici": "^8.11.2" + } +} diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts new file mode 100644 index 0000000000000000000000000000000000000000..efcff28a638aaa46a99f9b2051b2f782603fb339 --- /dev/null +++ b/e2e/playwright.config.ts @@ -0,0 +1,36 @@ +import { defineConfig } from "@playwright/test"; + +import { browserLaunch, config } from "./harness/config"; + +export default defineConfig({ + globalSetup: "./harness/global.ts", + fullyParallel: true, + workers: 12, + timeout: 90_000, + expect: { timeout: 2_000 }, + reporter: [["./harness/reporter.ts", { html: process.env.E2E_REPORT }]], + use: { + baseURL: config.webUrl, + ignoreHTTPSErrors: true, + // screenshots are most of a trace's cost, the dom snapshots show each step + trace: { mode: "retain-on-failure", screenshots: false }, + launchOptions: browserLaunch + }, + projects: [ + { name: "setup", testMatch: "harness/auth.setup.ts" }, + { name: "flows", testDir: "specs", dependencies: ["setup"] }, + // stops stack processes, so it only runs on pc with E2E_OUTAGES=1 + ...(process.env.E2E_STACK === "pc" && process.env.E2E_OUTAGES + ? [ + { + name: "outages", + testDir: "outages", + dependencies: ["setup"], + fullyParallel: false, + timeout: 300_000 + } + ] + : []), + ...(process.env.E2E_SWEEP ? [{ name: "sweep", testDir: "tools" }] : []) + ] +}); diff --git a/e2e/pnpm-lock.yaml b/e2e/pnpm-lock.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56f233773b2a5982f0ea306dcab5a39ba952098a --- /dev/null +++ b/e2e/pnpm-lock.yaml @@ -0,0 +1,346 @@ +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + .: + devDependencies: + "@atcute/atproto": + specifier: ^4.0.4 + version: 4.0.4(@atcute/lexicons@2.1.1) + "@atcute/cbor": + specifier: ^2.3.8 + version: 2.3.8(@atcute/cid@2.5.0) + "@atcute/cid": + specifier: ^2.5.0 + version: 2.5.0 + "@atcute/firehose": + specifier: ^1.2.0 + version: 1.2.0(@atcute/cid@2.5.0)(@atcute/lexicons@2.1.1) + "@atcute/lexicons": + specifier: ^2.1.1 + version: 2.1.1 + "@atcute/oauth-crypto": + specifier: ^1.0.1 + version: 1.0.1(typescript@5.9.3) + "@playwright/test": + specifier: ^1.61.1 + version: 1.63.0 + "@types/node": + specifier: ^24 + version: 24.19.1 + prettier: + specifier: ^3.9.9 + version: 3.9.9 + typescript: + specifier: ^5 + version: 5.9.3 + undici: + specifier: ^8.11.2 + version: 8.11.2 + +packages: + "@atcute/atproto@4.0.4": + resolution: + { + integrity: sha512-aEp5wQVPVbZcFJUwoi54tyT+ehRI/xtPe9T0zIvcryHAVXF/zxHtNboM+htbZLOAGsDFsqzgCyxph0Bafw7L8w== + } + peerDependencies: + "@atcute/lexicons": ^2.0.0 + + "@atcute/cbor@2.3.8": + resolution: + { + integrity: sha512-gazXBcNTr2U3cRaCjfRbq2GKHJnWapUUA3mgehL+0tbVXEty2wf/LIwF9z46gjqjX2DYGGwkybF8lTI3dzHaNQ== + } + peerDependencies: + "@atcute/cid": ^2.5.0 + + "@atcute/cid@2.5.0": + resolution: + { + integrity: sha512-CAZyzUdaQhuC+S7tGjoC7vgyDI2X4JU+CPO/7WjEU7HlNsAGXjSkD++5TlfnSWGDqUeqpjMZjUfFBZYHKzPHew== + } + + "@atcute/firehose@1.2.0": + resolution: + { + integrity: sha512-u6Oh137haGOzWNFbJzakUjjDOHvVNORY5r5H4v9OfHqVzFOYwSDptMYsD2TlRf4BbdtmHfjCb7pL1/D0eMfh6Q== + } + peerDependencies: + "@atcute/lexicons": ^2.1.0 + + "@atcute/lexicons@2.1.1": + resolution: + { + integrity: sha512-kHyqUW8g/Fq9LTctGh0TVBx0AmtecV6VSrRprrSbqKqHe9q8C6y+YPL3EMNq2l93XA1iDHkkvYwLdTcKL/lNbA== + } + + "@atcute/multibase@1.2.5": + resolution: + { + integrity: sha512-cReTONgYpQo/VHD3ZmzPNoyBKJgSk1J4h//cvvdVVJBMar+SjlQ/sUXeTjQfuyfmDKv+TLKhmutLcvbMcQ9Rvw== + } + + "@atcute/oauth-crypto@1.0.1": + resolution: + { + integrity: sha512-ghC8ceFx0r6pi5X3dNoET1K6PNg8afeAOZLD4PKLhrpVYdRtRrgY4uqAe3ojtSIjoS6GXmVm063CoJ2xtg/MoQ== + } + + "@atcute/uint8array@1.2.0": + resolution: + { + integrity: sha512-KoBGTbV4lS8zXNu91osM3FecrH3NUnxNPsBAkaNtn6uf/p240245qoLn7keFpRM0npX522X8R+XBBb2rvtE7Rg== + } + + "@atcute/util-text@1.3.4": + resolution: + { + integrity: sha512-u2UAM7iSM09sQaSG9jtxWFSPgB8boVj50/BoyMvYnhVgGBu+nXIuAcdDUQCsZA44YZgTPPhN/b86JX+jH7SPzQ== + } + + "@mary-ext/event-iterator@1.1.0": + resolution: + { + integrity: sha512-uL6XFBsfAq+h6wnmioLTDMKzgbuKKQTo/2k512W1w1M5SgVroS6+D4MOD+j1K0loEYUJD9rIKV0Fo5ETZlgafw== + } + + "@mary-ext/simple-event-emitter@1.0.2": + resolution: + { + integrity: sha512-A644JdOf+KfTGyX/EYf8AI1Y18mM7369KrhtOVkdEwRa2QIC0Botv7XWnK6/XU7HuQfvtFpCufmHJC3hi4YMuw== + } + + "@oomfware/eval@0.1.0": + resolution: + { + integrity: sha512-EIkukTd3zDQEHUjcfFRDq79Jgm4OEyUrEUvp/SwgjfoH2POaE5cvaswIonFwdt8fSzJF5xtAdgcgIoKD8sPpjg== + } + + "@playwright/test@1.63.0": + resolution: + { + integrity: sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ== + } + engines: { node: ">=20" } + hasBin: true + + "@standard-schema/spec@1.1.0": + resolution: + { + integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w== + } + + "@types/node@24.19.1": + resolution: + { + integrity: sha512-aS3/DG0oM05K0RIXXP+hKjinGG5IgSSVGzswZxW3O0sS3pH4/fycXundUC9XsszgKCk4gHXylTEK6hyFxVxnoQ== + } + + esm-env@1.2.2: + resolution: + { + integrity: sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA== + } + + event-target-polyfill@0.0.4: + resolution: + { + integrity: sha512-Gs6RLjzlLRdT8X9ZipJdIZI/Y6/HhRLyq9RdDlCsnpxr/+Nn6bU2EFGuC94GjxqhM+Nmij2Vcq98yoHrU8uNFQ== + } + + nanoid@5.1.16: + resolution: + { + integrity: sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ== + } + engines: { node: ^18 || >=20 } + hasBin: true + + partysocket@1.3.0: + resolution: + { + integrity: sha512-1zToNyolZFK/7nuAw/K2bZrNzFqaZyRoCEkS+9vG6WSC5ikrN6qWRe96q6ImU51uptz2r+dAwSkwhJVdQi4LiA== + } + peerDependencies: + react: ">=17" + peerDependenciesMeta: + react: + optional: true + + playwright-core@1.63.0: + resolution: + { + integrity: sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg== + } + engines: { node: ">=20" } + hasBin: true + + playwright@1.63.0: + resolution: + { + integrity: sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg== + } + engines: { node: ">=20" } + hasBin: true + + prettier@3.9.9: + resolution: + { + integrity: sha512-Z/CJHIkdujO/OtN7nXUii0Rf3VT5SRuhjBA82Xvu2XhBUgX3nhP67T0LHceBdQLex7OOFGTox+Q5Yg8Jk2Qivg== + } + engines: { node: ">=14" } + hasBin: true + + typescript@5.9.3: + resolution: + { + integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw== + } + engines: { node: ">=14.17" } + hasBin: true + + undici-types@7.24.6: + resolution: + { + integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg== + } + + undici@8.11.2: + resolution: + { + integrity: sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ== + } + engines: { node: ">=22.19.0" } + + unicode-segmenter@0.17.3: + resolution: + { + integrity: sha512-hKZwqBjJDmqNrq1+LjDxck1qLzJFcLLJM2Xq92ORRHOuau6GSHeELmn+uyk6zNH1CK9mogrUJGP9WJCJUQXMAw== + } + + valibot@1.5.0: + resolution: + { + integrity: sha512-nil6AkP2TChWL43Z5uJ6GTxX01CUA+g8LWUM+N/rB9NBbkUMaUsi9PUNzlUPgoKASgmx9f7eGOYpJ04/fSa6FQ== + } + peerDependencies: + typescript: ">=5" + peerDependenciesMeta: + typescript: + optional: true + + yocto-queue@1.2.2: + resolution: + { + integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ== + } + engines: { node: ">=12.20" } + +snapshots: + "@atcute/atproto@4.0.4(@atcute/lexicons@2.1.1)": + dependencies: + "@atcute/lexicons": 2.1.1 + + "@atcute/cbor@2.3.8(@atcute/cid@2.5.0)": + dependencies: + "@atcute/cid": 2.5.0 + "@atcute/multibase": 1.2.5 + "@atcute/uint8array": 1.2.0 + + "@atcute/cid@2.5.0": + dependencies: + "@atcute/multibase": 1.2.5 + "@atcute/uint8array": 1.2.0 + + "@atcute/firehose@1.2.0(@atcute/cid@2.5.0)(@atcute/lexicons@2.1.1)": + dependencies: + "@atcute/cbor": 2.3.8(@atcute/cid@2.5.0) + "@atcute/lexicons": 2.1.1 + "@atcute/uint8array": 1.2.0 + "@mary-ext/event-iterator": 1.1.0 + "@mary-ext/simple-event-emitter": 1.0.2 + partysocket: 1.3.0 + transitivePeerDependencies: + - "@atcute/cid" + - react + + "@atcute/lexicons@2.1.1": + dependencies: + "@atcute/uint8array": 1.2.0 + "@atcute/util-text": 1.3.4 + "@oomfware/eval": 0.1.0 + "@standard-schema/spec": 1.1.0 + esm-env: 1.2.2 + + "@atcute/multibase@1.2.5": + dependencies: + "@atcute/uint8array": 1.2.0 + + "@atcute/oauth-crypto@1.0.1(typescript@5.9.3)": + dependencies: + "@atcute/multibase": 1.2.5 + "@atcute/uint8array": 1.2.0 + nanoid: 5.1.16 + valibot: 1.5.0(typescript@5.9.3) + transitivePeerDependencies: + - typescript + + "@atcute/uint8array@1.2.0": {} + + "@atcute/util-text@1.3.4": + dependencies: + unicode-segmenter: 0.17.3 + + "@mary-ext/event-iterator@1.1.0": + dependencies: + yocto-queue: 1.2.2 + + "@mary-ext/simple-event-emitter@1.0.2": {} + + "@oomfware/eval@0.1.0": {} + + "@playwright/test@1.63.0": + dependencies: + playwright: 1.63.0 + + "@standard-schema/spec@1.1.0": {} + + "@types/node@24.19.1": + dependencies: + undici-types: 7.24.6 + + esm-env@1.2.2: {} + + event-target-polyfill@0.0.4: {} + + nanoid@5.1.16: {} + + partysocket@1.3.0: + dependencies: + event-target-polyfill: 0.0.4 + + playwright-core@1.63.0: {} + + playwright@1.63.0: + dependencies: + playwright-core: 1.63.0 + + prettier@3.9.9: {} + + typescript@5.9.3: {} + + undici-types@7.24.6: {} + + undici@8.11.2: {} + + unicode-segmenter@0.17.3: {} + + valibot@1.5.0(typescript@5.9.3): + optionalDependencies: + typescript: 5.9.3 + + yocto-queue@1.2.2: {} diff --git a/e2e/run b/e2e/run new file mode 100755 index 0000000000000000000000000000000000000000..bb4dd1dbf0690070b64458e69973f32b466bfada --- /dev/null +++ b/e2e/run @@ -0,0 +1,77 @@ +#!/bin/sh +# runs the suite. asks to start the stack if it isn't up, and rebuilds the web when web/ changed +# pnpm e2e the whole suite +# pnpm e2e specs/star.spec.ts one spec +# pnpm e2e --report=report.html also write an html report +# pnpm e2e --har=run.har also write the requests as a har +# pnpm e2e --login=ui every new account logs in through the login page +# pnpm e2e --web-modules=DIR the web node_modules the report loads, default web/node_modules +# pnpm e2e --sweep clean up what a crashed run left +# every other argument goes to playwright +set -eu +here="$(dirname "$0")" + +sweep= +n=$# +while [ "$n" -gt 0 ]; do + arg="$1" + shift + n=$((n - 1)) + case $arg in + --report=*) export E2E_REPORT="${arg#*=}" ;; + --har=*) export E2E_REPORT_HAR="${arg#*=}" ;; + --login=*) export E2E_LOGIN="${arg#*=}" ;; + --web-modules=*) export E2E_WEB_MODULES="${arg#*=}" ;; + --sweep) sweep=1 ;; + *) set -- "$@" "$arg" ;; + esac +done + +E2E_STACK="$("$here/up" --stack)" +export E2E_STACK +if [ "$E2E_STACK" = pc ]; then + root="$(cd "$here/.." && pwd)" + E2E_KNOT_SCAN="${E2E_KNOT_SCAN:-$(here="$root/localinfra/pc" && . "$here/bin/env.sh" && echo "$TNGL_STATE/data/knot2/repositories")}" + export E2E_KNOT_SCAN + E2E_PC_STATE="${E2E_PC_STATE:-$(here="$root/localinfra/pc" && . "$here/bin/env.sh" && echo "$TNGL_STATE")}" + export E2E_PC_STATE + # playwright's own chromium download won't run here, so the nix app supplies one + if [ -z "${E2E_CHROMIUM:-}" ] && command -v nix >/dev/null 2>&1; then + E2E_CHROMIUM=$(cd "$root" && nix run .#localinfra -- --exec printenv E2E_CHROMIUM 2>/dev/null) || E2E_CHROMIUM= + export E2E_CHROMIUM + fi +fi + +status=0 +"$here/up" --check || status=$? +case $status in +0) ;; +2) + # the sweep only removes accounts, it doesn't look at the web + if [ -z "$sweep" ]; then + echo "web/ changed since the web was built, rebuilding it, this takes a few minutes" >&2 + "$here/up" + fi + ;; +*) + if [ ! -t 0 ]; then + echo "the stack isn't up, start it with pnpm e2e:up" >&2 + exit 1 + fi + printf "the e2e stack isn't up, start it? the web build takes a few minutes [y/N] " >&2 + read -r answer + case $answer in + [yY]*) "$here/up" ;; + *) exit 1 ;; + esac + ;; +esac + +cd "$here" +[ -d node_modules ] || pnpm install +[ -n "${E2E_CHROMIUM:-}" ] || pnpm exec playwright install chromium +if [ -n "$sweep" ]; then + export E2E_SWEEP=1 + set -- --project=sweep "$@" +fi +exec pnpm exec playwright test "$@" diff --git a/e2e/specs/access.spec.ts b/e2e/specs/access.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..54b82f2e928fecf00331647731d3a14836d54f09 --- /dev/null +++ b/e2e/specs/access.spec.ts @@ -0,0 +1,413 @@ +import { expect } from "@playwright/test"; + +import { + asKnot, + editCob, + getOk, + knotRecord, + knotRecords, + knotReply, + mirrorMerge +} from "../harness/api"; +import { config, seeded } from "../harness/config"; +import { + bobbin, + collaboratorId, + type Ctx, + type Flow, + flow, + git, + hydrant, + knot, + knotCopy, + link, + manifestId, + not, + page, + patchId, + pds, + pullId, + refsId, + refusal, + refusedTicketId, + subscriptionId, + target, + testid, + text, + through, + ticketId, + where, + within +} from "../harness/flow"; +import { lsRemote, push } from "../harness/git"; +import { partsOf, selfUri, until, xrpc } from "../harness/net"; +import { + base, + branches, + merges, + onBranches, + recall, + remember, + targetTookTheFeature +} from "../harness/shared"; + +const COLLABORATOR = collaboratorId.nsid; +const ADD = "com.atproto.repo.createRecord"; +const REMOVE = "com.atproto.repo.deleteRecord"; + +const access = where("the access settings", (ctx) => `${ctx.repo.path}/settings/access`); +const addCollaboratorForm = where( + "the add collaborator form", + (ctx) => `${ctx.repo.path}/settings/access/new` +); + +const collaboration = (nsid: string, subject: (ctx: Ctx) => string) => (ctx: Ctx) => + knotReply(ctx.actor, ctx.repo.did, nsid, { + collection: COLLABORATOR, + rkey: subject(ctx), + ...(nsid === ADD + ? { record: { $type: COLLABORATOR, createdAt: new Date().toISOString() } } + : {}) + }); + +const collaborate = (nsid: string, label: string) => async (ctx: Ctx) => { + const reply = await collaboration(nsid, (c) => recall(c, label).did)(ctx); + expect(reply.status, `${nsid} answered ${JSON.stringify(reply.body)}`).toBe(200); +}; + +const withHelper = (flow: Flow) => + flow + .actor("helper") + .step("remember the helper", remember("helper")) + .actor("owner") + .step("the owner adds the helper", collaborate(ADD, "helper")); + +const bobbinSubjects = async (repoDid: string): Promise => { + const { items } = await getOk( + xrpc(config.bobbinUrl, "org.tangled.repo.listCollaborators", { subject: repoDid }) + ); + return items.map((item: { subject: string }) => item.subject); +}; + +const knotSubjects = async (repoDid: string): Promise => + (await knotRecords(repoDid, COLLABORATOR)).map((record) => partsOf(record.uri).rkey); + +// the pages read bobbin, so a collaborator must show in bobbin's list too +const listing = ( + service: string, + subjectsOf: (repoDid: string) => Promise, + label: string, + wanted: boolean +): [string, (ctx: Ctx) => Promise] => [ + `${service} ${wanted ? "lists" : "no longer lists"} ${label}`, + async (ctx) => { + const who = recall(ctx, label); + await until( + `${service} should ${wanted ? "list" : "drop"} ${label} on ${ctx.repo.did}`, + 20_000, + async () => { + const now = await subjectsOf(ctx.repo.did); + return [now.includes(who.did) === wanted, `it lists ${JSON.stringify(now)}`]; + } + ); + } +]; +const knotListing = (label: string, wanted: boolean) => + listing("the knot", knotSubjects, label, wanted); +const bobbinListing = (label: string, wanted: boolean) => + listing("bobbin", bobbinSubjects, label, wanted); + +const helperHandle = target("the helper's handle", (page, ctx) => + testid("collaboratorHandle", { handle: recall(ctx, "helper").handle }).locate(page, ctx) +); +const suggestion = target("the helper in the suggestions", (page, ctx) => + testid("selectActor", { handle: recall(ctx, "helper").handle }).locate(page, ctx) +); + +// only the knot's own refusals count, and it words them two ways depending on whether it knows whose key it is +const pushRefused = (label: string, branch: string) => async (ctx: Ctx) => { + const who = recall(ctx, label); + await expect( + push(who, ctx.repo.did, { [`e2e/${label}.txt`]: `${label}\n` }, { branch, base: "main" }) + ).rejects.toThrow( + /you aren't authorized to push to this repository|doesn't match any key published by the accounts that may push here/ + ); +}; + +const branchIsAt = + (branch: string, label = "owner") => + async (ctx: Ctx) => { + const heads = await lsRemote(recall(ctx, label), ctx.repo.did); + expect(heads[`refs/heads/${branch}`]).toBe(ctx.repo.heads[branch]); + }; + +flow("the owner adds a collaborator from the access page") + .actor("owner") + .repo() + .actor("helper") + .step("remember the helper", remember("helper")) + .actor("owner") + + .open(access) + .click(testid("addCollaborator")) + .at(addCollaboratorForm) + .fill(testid("collaborator"), (ctx) => recall(ctx, "helper").handle) + .click(suggestion) + .click(testid("invite")) + .sends( + collaboratorId, + knot, + (ctx) => ({ repo: ctx.repo.did, rkey: recall(ctx, "helper").did }), + "create" + ) + .accepted(collaboratorId) + .at(access) + .step(...knotListing("helper", true)) + .step(...bobbinListing("helper", true)) + .reload() + .sees(helperHandle) + .sees(testid("remove", { standing: "collaborator" })); + +flow("the owner removes a collaborator from the access page") + .actor("owner") + .repo() + .use(withHelper) + .step(...knotListing("helper", true)) + + .open(access) + .sees(helperHandle) + .click(testid("remove", { standing: "collaborator" })) + .sends( + collaboratorId, + knot, + (ctx) => ({ repo: ctx.repo.did, rkey: recall(ctx, "helper").did }), + "delete" + ) + .accepted(collaboratorId) + .step(...knotListing("helper", false)) + .step(...bobbinListing("helper", false)) + .reload() + .sees(not(helperHandle)) + .sees(not(testid("remove", { standing: "collaborator" }))); + +flow("a collaborator pushes a branch to the owner's repo") + .actor("owner") + .repo() + .push(base) + .step("remember the owner", remember("owner")) + .use(withHelper) + + .actor("helper") + .push({ "e2e/helper.txt": "from the helper\n" }, { branch: "from-helper", base: "main" }) + .follow(refsId, through(git, knotCopy, hydrant, bobbin)) + .step("the owner sees the helper's branch over git", branchIsAt("from-helper")); + +flow("a stranger's push is refused and the branch stays where it was") + .actor("owner") + .repo() + .push(base) + .step("remember the owner", remember("owner")) + .actor("nobody") + .step("remember the stranger", remember("nobody")) + .step("the stranger's push is refused", pushRefused("nobody", "main")) + .step("the stranger's new branch is refused too", pushRefused("nobody", "mine")) + .step("the owner's branch didn't move", branchIsAt("main")) + .step("and there is no new branch", async (ctx) => { + const heads = await lsRemote(recall(ctx, "owner"), ctx.repo.did, "refs/heads/"); + expect(Object.keys(heads)).toEqual(["refs/heads/main"]); + }) + .follow(refsId, through(git, knotCopy)); + +flow("a collaborator the owner removes can't push any more") + .actor("owner") + .repo() + .push(base) + .step("remember the owner", remember("owner")) + .use(withHelper) + .actor("helper") + .push({ "e2e/first.txt": "while allowed\n" }, { branch: "while-allowed", base: "main" }) + .follow(refsId, through(git, knotCopy, hydrant, bobbin)) + + .actor("owner") + .step("the owner removes the helper", collaborate(REMOVE, "helper")) + .step(...knotListing("helper", false)) + .actor("helper") + .step("the helper's push is refused", pushRefused("helper", "after-removal")) + .step("the owner's repo has no such branch", async (ctx) => { + const heads = await lsRemote(recall(ctx, "owner"), ctx.repo.did, "refs/heads/"); + expect(Object.keys(heads)).not.toContain("refs/heads/after-removal"); + }); + +const issueTitle = "a collaborator's issue"; + +flow("under the collaborators policy a collaborator opens an issue and a stranger is refused") + .actor("owner") + .repo() + .policy("collaborators") + .use(withHelper) + + .actor("helper") + .open(page.newIssue) + .fill(testid("title"), issueTitle) + .click(testid("submit")) + .sends( + ticketId, + knot, + (ctx) => ({ + repo: ctx.repo.did, + record: { state: "open", title: { text: issueTitle } } + }), + "create" + ) + .trace(ticketId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .at(page.issues) + .sees(link(issueTitle)) + + .actor(seeded.charlie) + .open(page.newIssue) + .fill(testid("title"), "let me in") + .click(testid("submit")) + .refused(refusedTicketId, 403) + .sees(refusal) + .open(page.issues) + .sees(link(issueTitle)) + .sees(not(link("let me in"))); + +onBranches("a collaborator closes and reopens the owner's pull request") + .pull(pullId, patchId, branches) + .step("remember the owner", remember("owner")) + .use(withHelper) + .actor("helper") + .open(page.pull(pullId)) + .sees(testid("requestClose")) + .changeState(pullId, "closed", testid("requestClose"), testid("changeState")) + .changeState(pullId, "open", testid("changeState"), testid("requestClose")) + .reload() + .sees(testid("requestClose")); + +onBranches("a collaborator merges the owner's pull request") + .pull(pullId, patchId, branches) + .step("remember the owner", remember("owner")) + .use(withHelper) + .actor("helper") + .open(page.pull(pullId)) + .sees(testid("mergeChoice")) + .use(merges) + .knotStored(pullId, "completed") + .step( + "the target branch took the feature's commit", + targetTookTheFeature((ctx) => recall(ctx, "owner")) + ); + +const strangersPull = "a pull a stranger may only read"; +const closeThroughApi = async (ctx: Ctx) => { + return editCob(ctx.actor, ctx.get(pullId).uri, "close", (stored) => ({ + ...stored, + state: "closed" + })); +}; + +onBranches("a stranger sees no close, merge or label controls on a pull and the knot refuses them") + .pull(pullId, patchId, { ...branches, title: strangersPull }) + .open(page.pull(pullId)) + .sees(testid("requestClose")) + .sees(testid("mergeChoice")) + .sees(testid("editLabels")) + + .actor(seeded.charlie) + .open(page.pull(pullId)) + .sees(within(testid("ticketTitle"), text(strangersPull))) + .sees(not(testid("requestClose"))) + .sees(not(testid("mergeChoice"))) + .sees(not(testid("editLabels"))) + .rejects( + "charlie merging the pull", + (ctx) => + mirrorMerge( + ctx.actor, + ctx.repo.did, + branches.into, + ctx.repo.heads[branches.into], + ctx.repo.heads[branches.from] + ), + [401, 403, 409] + ) + .rejects("charlie closing the pull", closeThroughApi, 403) + .knotStored(pullId, "open"); + +flow("only the owner is offered the delete button in the repo settings") + .actor("owner") + .repo() + .use(withHelper) + .open(page.settings) + .sees(testid("setConfirmDelete")) + + .actor("helper") + .open(page.settings) + .sees(testid("formDefaultBranch")) + .sees(not(testid("setConfirmDelete"))) + + .actor(seeded.alice) + .open(page.settings) + .sees(testid("errorMessage")) + .sees(not(testid("setConfirmDelete"))); + +// the owner's declaration is still up, which only an admin's force gets past +const aliceDeletes = async (ctx: Ctx) => { + const reply = await asKnot(ctx.actor, "org.tangled.repo.deleteRepo", { + repo: ctx.repo.did, + force: true + }); + expect(reply.status, `the delete answered ${JSON.stringify(reply.body)}`).toBe(200); +}; + +flow("a knot admin deletes a repo she does not own through the api") + .actor("owner") + .repo() + .push(base) + .actor(seeded.alice) + .step("alice deletes it through the api", aliceDeletes) + .step("the knot has dropped the repo", async (ctx) => { + await until(`the knot should have dropped ${ctx.repo.did}`, 10_000, async () => [ + (await knotRecord(selfUri(ctx.repo.did, manifestId.nsid))) === undefined, + "it still has the manifest" + ]); + }); + +flow("a repo a knot admin deleted is gone from bobbin too") + .actor("owner") + .repo() + .step("remember the owner", remember("owner")) + .actor(seeded.alice) + .step("alice deletes it through the api", aliceDeletes) + .step("bobbin no longer knows the repo", async (ctx) => { + await until(`bobbin should have dropped ${ctx.repo.name}`, 20_000, async () => { + const { status } = await bobbin.repoByName(recall(ctx, "owner").did, ctx.repo.name); + return [status !== 200, `getRepoByName answered ${status}`]; + }); + }); + +flow("a collaborator can't add or remove collaborators") + .actor("owner") + .repo() + .use(withHelper) + + .actor("helper") + .rejects( + "the helper adding charlie", + collaboration(ADD, () => seeded.charlie.did), + 403 + ) + .rejects( + "the helper removing themself", + collaboration(REMOVE, (ctx) => ctx.actor.did), + 403 + ) + .step("the knot still lists only the helper", async (ctx) => { + expect(await knotSubjects(ctx.repo.did)).toEqual([recall(ctx, "helper").did]); + }) + .open(access) + .sees(testid("errorMessage")) + .open(addCollaboratorForm) + .sees(testid("errorMessage")); diff --git a/e2e/specs/account.spec.ts b/e2e/specs/account.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..09e87dd848e1ccefd21f53c2d6b0ea4de969f59e --- /dev/null +++ b/e2e/specs/account.spec.ts @@ -0,0 +1,307 @@ +import { expect, type Request, test } from "@playwright/test"; + +import { config, handleOf } from "../harness/config"; +import { + bobbin, + type Ctx, + type Flow, + field, + flow, + git, + hydrant, + keys, + keysNew, + knotCopy, + pds, + profileSettings, + publicKeyId, + refsId, + refusal, + rkeyOf, + target, + testid, + text, + through, + where, + within +} from "../harness/flow"; +import { keyOf, lsRemoteWith, makeKey } from "../harness/git"; +import { get, stamp, until, xrpc } from "../harness/net"; +import { notificationsOf, putSaved, type Toggles } from "../harness/services"; + +const files = { "README.md": "pushed with a key from settings\n" }; + +const addsLaptopKey = (flow: Flow) => + flow + .open(keysNew) + .fill(testid("name"), "laptop") + .fill(testid("key"), (ctx) => keyOf(ctx).pub) + .click(testid("submit")) + .sends( + publicKeyId, + pds, + (ctx) => ({ record: { name: "laptop", publicKey: keyOf(ctx).pub } }), + "create" + ) + .trace(publicKeyId, through(pds, hydrant, bobbin)); + +const knotTakesTheKey = (ctx: Ctx) => + until("the knot should take the key", 30_000, async () => { + const saw = await lsRemoteWith(ctx); + return [saw === "listed", saw]; + }); + +flow("a key added in settings is the key the knot takes for a push") + .actor("pusher") + .repo() + .step("the account makes a key the knot has never heard of", makeKey) + .step("the knot refuses that key", async (ctx) => { + expect(await lsRemoteWith(ctx)).not.toBe("listed"); + }) + + .use(addsLaptopKey) + .step("the knot lists the repo for that key", knotTakesTheKey) + .push(files) + .follow(refsId, through(git, knotCopy, hydrant, bobbin)); + +flow("a key added in settings is listed with its fingerprint") + .actor("lister") + .step("the account makes a key", makeKey) + + .use(addsLaptopKey) + .at(keys) + .sees(text("laptop")) + // the page computes the fingerprint, ssh-keygen is the reference + .sees( + target("the key's fingerprint", (tab, ctx) => + within(testid("keyFingerprint"), text(keyOf(ctx).fingerprint)).locate(tab, ctx) + ) + ); + +const notAKey = "hello, this is not a key"; + +flow("a title and some words that are not an ssh key are refused, not saved") + .actor("typist") + .open(keysNew) + .fill(testid("name"), "junk") + .fill(testid("key"), notAKey) + .click(testid("submit")) + .sees(refusal) + .sendsNothing(publicKeyId) + + .open(keys) + .sees(testid("settingsEmpty")); + +test("a visitor opening the verify email page without a token is told so", async ({ browser }) => { + const context = await browser.newContext(); + const tab = await context.newPage(); + await tab.goto("/verify/email", { waitUntil: "networkidle" }); + await expect(testid("verificationFailedHeading").locate(tab, {} as Ctx)).toBeVisible(); + await expect(testid("verificationErrorMessage").locate(tab, {} as Ctx)).toBeVisible(); + await expect(testid("emails").locate(tab, {} as Ctx)).toBeVisible(); + await context.close(); +}); + +test("a visitor opening the verify email page with a token nobody issued is told it is invalid", async ({ + browser +}) => { + const context = await browser.newContext(); + const tab = await context.newPage(); + await tab.goto(`/verify/email?token=not-a-real-token-${stamp()}`, { waitUntil: "networkidle" }); + await expect(testid("verificationFailedHeading").locate(tab, {} as Ctx)).toBeVisible(); + await expect(testid("verificationErrorMessage").locate(tab, {} as Ctx)).toBeVisible(); + await context.close(); +}); + +const notifications = where("the notification settings", () => "/settings/notifications"); + +const notificationToggle = (key: string) => testid("notificationToggle", { key }); + +const startedWith = new WeakMap(); +const noteNotifications = async (ctx: Ctx) => { + startedWith.set(ctx, await notificationsOf(ctx.actor)); +}; + +// deliberi takes the save, not the pds, so the record holder never sees it +const saves = new WeakMap>(); +const watchTheSave = async (ctx: Ctx) => { + const sent = ctx.page.waitForRequest( + (sent) => sent.url().includes("notification.updatePreferences") && sent.method() === "POST" + ); + sent.catch(() => {}); + saves.set(ctx, sent); +}; + +const flipped = { emailNotifications: true, followed: false }; + +flow( + "notification switches saved in settings are what deliberi keeps and the page shows after a reload" +) + .actor("reader") + .step("deliberi's starting preferences", noteNotifications) + + .open(notifications) + .click(notificationToggle("emailNotifications")) + .click(notificationToggle("followed")) + .step("watch for the save", watchTheSave) + .click(testid("save")) + .step("the page sent only the two switches that changed", async (ctx) => { + const sent = await saves.get(ctx)!; + expect(sent.postDataJSON()).toEqual(flipped); + }) + .step("deliberi keeps the two and leaves the rest", async (ctx) => { + await until(`deliberi should keep ${JSON.stringify(flipped)}`, 10_000, async () => { + const has = await notificationsOf(ctx.actor); + const want = { ...startedWith.get(ctx)!, ...flipped }; + return [ + JSON.stringify(has, Object.keys(want).sort()) === + JSON.stringify(want, Object.keys(want).sort()), + `it keeps ${JSON.stringify(has)}` + ]; + }); + }) + .reload() + .step("the page shows the saved switches", async (ctx) => { + const email = notificationToggle("emailNotifications").locate(ctx.page, ctx); + const followed = notificationToggle("followed").locate(ctx.page, ctx); + await expect(email).toBeChecked(); + await expect(followed).not.toBeChecked(); + await expect(testid("save").locate(ctx.page, ctx)).toBeDisabled(); + }); + +const dangers = [ + { + open: testid("password"), + title: "Change password", + submit: testid("sendResetCodeButton"), + path: "password" + }, + { + open: testid("delete"), + title: "Delete account", + submit: testid("sendDeletionCodeButton"), + path: "delete" + } +]; + +const settingsPage = (name: string) => where(`the ${name} form`, () => `/settings/profile/${name}`); + +for (const danger of dangers) { + flow(`the ${danger.path} form stays off until a password is typed and cancel goes back`) + .actor("owner") + + .open(profileSettings) + .click(danger.open) + .at(settingsPage(danger.path)) + .sees(testid("drillDownTitle")) + .step("the button that acts is off", async (ctx) => { + await expect(danger.submit.locate(ctx.page, ctx)).toBeDisabled(); + }) + .fill(testid("password"), "not the password") + .step("the button that acts is on, and is left alone", async (ctx) => { + await expect(danger.submit.locate(ctx.page, ctx)).toBeEnabled(); + }) + .click(testid("settings")) + .at(profileSettings) + .sees(testid("settingsHeaderTitle")); +} + +const knotRefusesTheKey = (ctx: Ctx) => + until("the knot should refuse the removed key", 60_000, async () => { + const saw = await lsRemoteWith(ctx); + return [saw !== "listed", "the knot still lists the repo for it"]; + }); + +flow("removing a key in settings makes the knot refuse a push with it") + .actor("pusher") + .repo() + .step("the account makes a key", makeKey) + .use(addsLaptopKey) + .step("the knot takes the key", knotTakesTheKey) + + .at(keys) + .sees(text("laptop")) + .click(testid("confirmDelete")) + .click(within(testid("deleteConfirm"), testid("confirm"))) + .sends(publicKeyId, pds, (ctx) => ({ rkey: rkeyOf(ctx, publicKeyId) }), "delete") + .trace(publicKeyId, through(pds, hydrant, bobbin)) + .sees(testid("settingsEmpty")) + .reload() + .sees(testid("settingsEmpty")) + .step("the knot refuses the removed key", knotRefusesTheKey); + +// the non-GET requests, which the record holder never sees +const sentByPage = new WeakMap(); +const watchRequests = async (ctx: Ctx) => { + const seen: Request[] = []; + ctx.page.on("request", (sent) => { + if (!["GET", "OPTIONS", "HEAD"].includes(sent.method())) seen.push(sent); + }); + sentByPage.set(ctx, seen); +}; +const pageSends = (what: string, url: RegExp) => (ctx: Ctx) => + until(`the page should send ${what}`, 10_000, async () => { + const seen = sentByPage.get(ctx)!; + return [ + seen.some((sent) => url.test(sent.url())), + `it sent ${seen.map((sent) => `${sent.method()} ${new URL(sent.url()).pathname}`).join(", ") || "nothing"}` + ]; + }); + +flow("changing the password sends a reset code") + .actor("pwuser") + .open(settingsPage("password")) + .step("watch what the page sends", watchRequests) + .fill(testid("password"), config.password) + .click(testid("sendResetCodeButton")) + .step( + "the page asks for a reset code", + pageSends("a password reset request", /requestPasswordReset/) + ); + +flow("deleting an account asks the pds to mail a deletion code and asks for it") + .actor("goner") + .open(settingsPage("delete")) + .step("watch what the page sends", watchRequests) + .fill(testid("password"), config.password) + .click(testid("sendDeletionCodeButton")) + .step( + "the page asks the pds for a deletion code", + pageSends("a deletion code request", /requestAccountDelete/) + ) + .sees(field("Deletion code")); + +const newHandles = new WeakMap(); +const pickHandle = async (ctx: Ctx) => { + newHandles.set(ctx, handleOf(`rn-${stamp()}`)); +}; +const newHandle = (ctx: Ctx) => newHandles.get(ctx)!; + +flow("changing the handle gives the account the new handle") + .actor("renmr") + .step("pick a handle", pickHandle) + .open(settingsPage("handle")) + .click(testid("ownDomainChosen")) + .fill(testid("handle"), newHandle) + .click(testid("submit")) + .step("the pds knows the account by the new handle", (ctx) => + until(`the pds should call ${ctx.actor.did} ${newHandle(ctx)}`, 15_000, async () => { + const { body } = await get( + xrpc(config.pdsUrl, "com.atproto.repo.describeRepo", { repo: ctx.actor.did }) + ); + return [body?.handle === newHandle(ctx), `it says ${body?.handle}`]; + }) + ) + .open(where("the profile at the new handle", (ctx) => `/${newHandle(ctx)}`)) + .sees( + target("the new handle", (tab, ctx) => + tab.getByText(newHandle(ctx), { exact: true }).first() + ) + ); + +flow("a new account's first preferences save is taken without a retry") + .actor("fresh") + .step("saves a knot at once", async (ctx) => { + const reply = await putSaved(ctx.actor, { knots: ["a.knot.test"], spindles: [] }); + expect(reply.status, `pocket answered ${JSON.stringify(reply.body)}`).toBe(200); + }); diff --git a/e2e/specs/chores.spec.ts b/e2e/specs/chores.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..2724c5c44ac98624c404ab9f47cc09dcb0105581 --- /dev/null +++ b/e2e/specs/chores.spec.ts @@ -0,0 +1,79 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { expect, test } from "@playwright/test"; + +import { accounts, alreadyGone, getOk } from "../harness/api"; +import { type Chore, holdsUp, LEDGER, owe, replay, settled } from "../harness/chores"; +import { config } from "../harness/config"; +import { xrpc } from "../harness/net"; + +const exists = async (did: string) => { + try { + await getOk(xrpc(config.pdsUrl, "com.atproto.repo.describeRepo", { repo: did })); + return true; + } catch { + return false; + } +}; + +// above the pid limit, so no live process can own it +const DEAD = 2 ** 22 + 1; +const asDead = (id: string) => { + const path = resolve(LEDGER, `${id}.json`); + const entry = JSON.parse(readFileSync(path, "utf8")); + writeFileSync(path, JSON.stringify({ ...entry, pid: DEAD })); +}; + +test("a dead run's chores are replayed and a live one's are left alone", async () => { + test.setTimeout(60_000); + const left = await accounts.create("dead"); + const live = await accounts.create("live"); + try { + asDead(owe("account of a run that was killed", { kind: "account", account: left })); + owe("account of this run", { kind: "account", account: live }); + + expect(await replay("dead")).toEqual([]); + expect(await exists(left.did)).toBe(false); + expect(await exists(live.did)).toBe(true); + + expect(await replay("mine")).toEqual([]); + expect(await exists(live.did)).toBe(false); + owe("the same account again", { kind: "account", account: live }); + expect(await replay("mine")).toEqual([]); + } finally { + for (const account of [left, live]) await accounts.remove(account).catch(() => {}); + } +}); + +test("the newest chore is done first, even inside one millisecond", () => { + const ids = Array.from({ length: 100 }, (_, i) => + owe(`chore ${i}`, { kind: "member", account: { name: "x", handle: "x", did: `did:${i}` } }) + ); + try { + expect([...ids].sort().reverse()).toEqual([...ids].reverse()); + } finally { + ids.forEach(settled); + } +}); + +test("an account stays while one of its chores has failed", () => { + const alice = { name: "a", handle: "a", did: "did:a" }; + const bob = { name: "b", handle: "b", did: "did:b" }; + const failed = new Set([alice.did]); + const record: Chore = { kind: "record", by: alice, uri: "at://x/y/z" }; + expect(holdsUp({ kind: "account", account: alice }, failed)).toBe(true); + expect(holdsUp({ kind: "member", account: alice }, failed)).toBe(true); + expect(holdsUp({ kind: "account", account: bob }, failed)).toBe(false); + expect(holdsUp(record, failed)).toBe(false); +}); + +test("only a service saying it is gone counts as gone", () => { + expect(alreadyGone(new Error('knot getRecord answered 400 {"error":"RepoNotFound"}'))).toBe( + true + ); + expect(alreadyGone(new Error("the knot has no at://x/y/z to erase"))).toBe(true); + expect(alreadyGone(new Error("pds refused a's login: 401 AuthenticationRequired"))).toBe(true); + expect(alreadyGone(new Error("pds refused a's login: 429 RateLimitExceeded"))).toBe(false); + expect(alreadyGone(new Error("GET https://pds: ECONNREFUSED"))).toBe(false); +}); diff --git a/e2e/specs/ci-logs.spec.ts b/e2e/specs/ci-logs.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..5f6617338012c9b8867d1b4258a6965c355527d1 --- /dev/null +++ b/e2e/specs/ci-logs.spec.ts @@ -0,0 +1,51 @@ +import { expect } from "@playwright/test"; + +import { config } from "../harness/config"; +import { flow } from "../harness/flow"; +import { workflow, workflowPath } from "../harness/shared"; +import type { Ctx } from "../harness/vocabulary"; + +// localinfra/pc sets this as the spindle's logs address +const LOGS = { host: "spindle.tngl.boltless.dev", port: 3333 }; +// knot2's did document has no spindle service. the knot can't notify it +const NOT_A_SPINDLE = "did:web:knot2.tngl.boltless.dev"; + +const command = (ctx: Ctx) => + `remote: ssh -t -p ${LOGS.port} ${LOGS.host} ${ctx.repo.did} ${ctx.pushed.sha}`; + +const noHint = async (ctx: Ctx) => { + expect(ctx.pushed.output).not.toContain("ssh -t"); +}; + +flow("a push that schedules a pipeline prints the spindle's ci logs command") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ [workflowPath]: workflow }) + .step("the push output has the spindle's ssh command", async (ctx) => { + expect(ctx.pushed.output).toContain("remote: -> Browse CI logs in your terminal:"); + expect(ctx.pushed.output).toContain(command(ctx)); + }); + +flow("a repo without a spindle gets no ci logs command") + .actor("owner") + .repo() + .push({ [workflowPath]: workflow }) + .step("the push output has no ssh command", noHint); + +flow("a push that skips ci gets no ci logs command") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ [workflowPath]: workflow }, { options: ["skip-ci"] }) + .step("the push output has no ssh command", noHint); + +flow("a push without workflows gets no ci logs command") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ "e2e/readme.txt": "no ci here\n" }) + .step("the push output has no ssh command", noHint); + +flow("a spindle the knot can't notify gets no ci logs command") + .actor("owner") + .repo({ spindle: NOT_A_SPINDLE }) + .push({ [workflowPath]: workflow }) + .step("the push output has no ssh command", noHint); diff --git a/e2e/specs/comments.spec.ts b/e2e/specs/comments.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..bd18f5df0f83bebeccb5506357108dbdf4c4fd74 --- /dev/null +++ b/e2e/specs/comments.spec.ts @@ -0,0 +1,376 @@ +import { expect } from "@playwright/test"; + +import { comments } from "../harness/api"; +import { + bobbin, + button, + commentId, + component, + flow, + hydrant, + knot, + not, + page, + pds, + record, + replacing, + subscriptionId, + target, + testid, + text, + through, + ticketId, + within +} from "../harness/flow"; +import { retry429 } from "../harness/net"; +import { + commented, + deletesComment, + made, + note, + onCommentedIssue, + stillStored +} from "../harness/shared"; + +const busy = 101; +const saying = (at: number) => `comment ${String(at).padStart(3, "0")}`; +const exactly = (words: string) => within(testid("commentBody"), text(words)); + +flow("more than a hundred comments still show the newest one") + .actor("owner") + .repo() + .ticket(ticketId, { title: "a busy thread" }) + .step(`${busy} comments`, async (ctx) => { + const write = async (at: number) => { + // a burst this big outlasts the usual back off, so it waits longer and tries more often + const reply = await retry429( + () => comments.add(ctx.actor, ctx.repo.did, ctx.get(ticketId), saying(at)), + 10, + 1_000 + ); + const written = made(`comment ${at}`, reply); + ctx.adopt(commentId, written); + return written; + }; + for (let from = 1; from < busy; from += 4) { + const burst = Array.from({ length: Math.min(4, busy - from) }, (_, at) => from + at); + await Promise.all(burst.map(write)); + } + await bobbin.indexed(await write(busy)); + }) + + .open(page.issue(ticketId)) + .sees(exactly(saying(busy))) + .sees(not(exactly(saying(1)))) + .click(button("Load older comments")) + .sees(exactly(saying(1))) + .sees(exactly(saying(busy))); + +const reply = record("org.tangled.feed.comment"); +const reaction = record("org.tangled.feed.reaction"); + +const replyBox = within(testid("commentEditor", { mode: "reply" }), testid("body")); +const replySubmit = within(testid("commentEditor", { mode: "reply" }), testid("submit")); +const editBox = within(testid("commentEditor", { mode: "edit" }), testid("body")); +const editSubmit = within(testid("commentEditor", { mode: "edit" }), testid("submit")); +const discussion = within(testid("commentEditor", { mode: "new" }), testid("body")); +const commentSubmit = within(testid("commentEditor", { mode: "new" }), testid("submit")); + +const threadTitle = "needs a thread"; +const parentText = "the parent says this"; +const replyText = "and the reply says that"; +const editedParent = "the parent, but better"; + +const issuePicker = within(component("Ticket"), testid("react")); +const commentPicker = within(component("CommentCard"), testid("react")); +const thumbs = testid("handlePick", { kind: "👍" }); +const chip = testid("reactionChip", { kind: "👍" }); +const chipSaying = (count: number) => testid("reactionChip", { kind: "👍", count }); + +// replies sit in the ml-2xl column under their parent, top level comments never do +const nested = (words: string) => within(testid("replies"), text(words)); + +const withOneComment = (name: string) => + onCommentedIssue(name, { + title: threadTitle, + words: parentText, + // a reply looks the subscription up in bobbin, and writes a second one if it's missing + subscription: through(pds, hydrant, bobbin) + }); + +withOneComment("replying under a comment nests it and survives a reload") + .sees(not(nested(parentText))) + .fill(replyBox, replyText) + .click(replySubmit) + .sends( + reply, + knot, + (ctx) => ({ + record: { + subject: { uri: ctx.get(ticketId).uri }, + replyTo: { uri: ctx.get(commentId).uri, cid: ctx.get(commentId).cid }, + body: { text: replyText } + } + }), + "create" + ) + .trace(reply, through(knot, hydrant, bobbin)) + .sees(nested(replyText)) + .reload() + .sees(text(parentText)) + .sees(nested(replyText)) + .sees(not(nested(parentText))) + .step("the knot's copy of the reply names the parent", async (ctx) => { + const stored = await knot.has(ctx.get(reply)); + if (stored.replyTo?.uri !== ctx.get(commentId).uri) + throw new Error(`the reply says it answers ${stored.replyTo?.uri}`); + }); + +flow("reacting to an issue and taking the reaction back") + .actor("owner") + .repo() + .ticket(ticketId, { title: "react to me" }) + + .open(page.issue(ticketId)) + .click(issuePicker) + .click(thumbs) + .sends( + reaction, + knot, + (ctx) => ({ + record: { reaction: "👍", subject: { uri: ctx.get(ticketId).uri } } + }), + "create" + ) + .sees(chipSaying(1)) + .trace(reaction, through(knot, hydrant, bobbin)) + .sees(chipSaying(1)) + .reload() + .sees(chipSaying(1)) + + .click(chip) + .sends(reaction, knot, replacing(reaction), "delete") + .sees(not(chip)) + .trace(reaction, through(knot, hydrant, bobbin)) + .sees(not(chip)) + .reload() + .sees(not(chip)); + +flow("a reaction from each of two people adds up and one taking theirs back leaves the other") + .actor("owner") + .repo() + .ticket(ticketId, { title: "react to me twice" }) + + .open(page.issue(ticketId)) + .click(issuePicker) + .click(thumbs) + .trace(reaction, through(knot, hydrant, bobbin)) + .sees(chipSaying(1)) + + .actor("fan") + .open(page.issue(ticketId)) + .sees(chipSaying(1)) + .click(chip) + .sends(reaction, knot, { record: { reaction: "👍" } }, "create") + .trace(reaction, through(knot, hydrant, bobbin)) + .sees(chipSaying(2)) + .reload() + .sees(chipSaying(2)) + .click(chip) + .sends(reaction, knot, replacing(reaction), "delete") + .trace(reaction, through(knot, hydrant, bobbin)) + .sees(chipSaying(1)) + .reload() + .sees(chipSaying(1)) + + .actor("owner") + .open(page.issue(ticketId)) + .sees(chipSaying(1)); + +withOneComment("reacting to a comment and taking the reaction back") + .click(commentPicker) + .click(thumbs) + .sends( + reaction, + knot, + (ctx) => ({ + record: { + reaction: "👍", + subject: { uri: ctx.get(commentId).uri, cid: ctx.get(commentId).cid } + } + }), + "create" + ) + .trace(reaction, through(knot, hydrant, bobbin)) + .sees(chipSaying(1)) + .reload() + .sees(chipSaying(1)) + + .click(chip) + .sends(reaction, knot, replacing(reaction), "delete") + .trace(reaction, through(knot, hydrant, bobbin)) + .sees(not(chip)) + .reload() + .sees(not(chip)) + .sees(text(parentText)); + +withOneComment("deleting a comment that has replies keeps the replies under a tombstone") + .fill(replyBox, replyText) + .click(replySubmit) + .trace(reply, through(knot, hydrant, bobbin)) + .sees(nested(replyText)) + + .use(deletesComment) + .sees(testid("deletedCommentNotice")) + .sees(not(text(parentText))) + .sees(text(replyText)) + .reload() + .sees(testid("deletedCommentNotice")) + .sees(not(text(parentText))) + .sees(nested(replyText)) + .step("the reply is still on the knot", async (ctx) => { + await knot.has(ctx.get(reply)); + }); + +// scoped to the edited card, so a chip elsewhere can't stand in for it +const chipOnEdited = target("the 👍 chip on the edited comment", (page, ctx) => + testid("commentCard") + .locate(page, ctx) + .filter({ hasText: editedParent }) + .locator('[data-testid="reactionChip"][data-kind="👍"][data-count="1"]') +); + +withOneComment("editing a comment keeps its reactions on screen") + .click(commentPicker) + .click(thumbs) + .trace(reaction, through(knot, hydrant, bobbin)) + .sees(chipSaying(1)) + + .click(within(component("CommentCard"), testid("moreMenu"))) + .click(testid("setEditingUri")) + .fill(editBox, editedParent) + .click(editSubmit) + .sends( + commentId, + knot, + replacing(commentId, (ctx) => ({ + subject: { uri: ctx.get(ticketId).uri }, + body: { text: editedParent } + })), + "put" + ) + .trace(commentId, through(knot, hydrant, bobbin)) + .sees(text(editedParent)) + // before the reload, which would hide it + .sees(chipOnEdited) + .reload() + .sees(text(editedParent)) + .sees(chipOnEdited); + +const title = "the build is broken"; + +const edited = "same here, again"; + +flow("commenting on an issue you opened") + .actor("owner") + .repo() + + .open(page.newIssue) + .fill(testid("title"), title) + .click(testid("submit")) + .trace(ticketId, through(knot, hydrant, bobbin)) + // opening subscribed the author, so the comment must not subscribe again + .trace(subscriptionId, through(pds, hydrant, bobbin)) + + .open(page.issue(ticketId)) + .sees(testid("toggleSubscription", { subscribed: "true" })) + .fill(discussion, note) + .click(commentSubmit) + .trace(commentId, through(knot, hydrant, bobbin)) + .sees(text(note)) + .reload() + .sees(text(note)) + .sees(testid("toggleSubscription", { subscribed: "true" })); + +flow("commenting on an issue") + .actor("owner") + .repo() + .ticket(ticketId, { title: commented }) + + .open(page.issue(ticketId)) + .fill(discussion, note) + .click(commentSubmit) + // the knot only takes a comment that names the version of the ticket it saw + .sends( + commentId, + knot, + (ctx) => ({ + record: { subject: { $type: "com.atproto.repo.strongRef", ...ctx.get(ticketId) } } + }), + "create" + ) + .sees(text(commented)) + .sees(not(text(note))) + .trace(commentId, through(knot, hydrant, bobbin)) + .sees(text(note)) + .trace(subscriptionId, through(pds)) + .reload() + .sees(text(note)); + +onCommentedIssue("editing a comment") + .click(within(component("CommentCard"), testid("moreMenu"))) + .click(testid("setEditingUri")) + .fill(editBox, edited) + .click(editSubmit) + .sends( + commentId, + knot, + // the knot wants an edit to resend the subject too + replacing(commentId, (ctx) => ({ + subject: { uri: ctx.get(ticketId).uri }, + body: { text: edited } + })), + "put" + ) + .trace(commentId, through(knot, hydrant, bobbin)) + .sees(text(edited)) + .reload() + .sees(text(edited)); + +onCommentedIssue("deleting a comment") + .use(deletesComment) + .sees(text(commented)) + .sees(not(text(note))) + .reload() + .sees(text(commented)) + .sees(not(text(note))); + +const draft = "typed and never sent"; + +const cancel = testid("cancel"); + +onCommentedIssue("cancelling a comment edit leaves the comment as it was") + .click(within(component("CommentCard"), testid("moreMenu"))) + .click(testid("setEditingUri")) + .fill(editBox, edited) + .click(cancel) + .sendsNothing(commentId) + .sees(text(note)) + .sees(not(text(edited))) + .reload() + .sees(text(note)) + .sees(not(text(edited))) + .step( + "the knot and hydrant still have the first version", + stillStored(commentId, (value) => value.body.text, note) + ); + +onCommentedIssue("a comment typed and never sent is not written") + .fill(discussion, draft) + .open(page.issues) + .sendsNothing(commentId) + .open(page.issue(ticketId)) + .sees(not(text(draft))) + .step("the box starts empty", async (ctx) => { + await expect(discussion.locate(ctx.page, ctx)).toHaveValue(""); + }); diff --git a/e2e/specs/faults.spec.ts b/e2e/specs/faults.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..84ed797d3de9f24aa2b6ae2b56689e522bb399e7 --- /dev/null +++ b/e2e/specs/faults.spec.ts @@ -0,0 +1,347 @@ +import { expect } from "@playwright/test"; + +import { knotRecords } from "../harness/api"; +import { + bobbin, + commentId, + component, + type Ctx, + discussion, + flow, + hydrant, + knot, + link, + not, + page, + pds, + refusedTicketId, + refusal, + replacing, + staleEdit, + starCount, + starId, + subscriptionId, + text, + testid, + through, + ticketId, + within +} from "../harness/flow"; +import { sleep, until, type Written } from "../harness/net"; +import { answerWith, commented, jsonError, note } from "../harness/shared"; + +const stateChangeAlert = testid("stateChangeError"); + +const bobbinWait = "**/xrpc/org.tangled.bobbin.awaitRecord*"; +const bobbinFails = answerWith(bobbinWait, 504, "UpstreamTimeout", "bobbin took too long"); +const bobbinRecovers = (ctx: Ctx) => ctx.page.unroute(bobbinWait); +const probeFails = answerWith( + "**/xrpc/org.tangled.knot.describeKnot*", + 503, + "ServiceUnavailable", + "the e2e flow cut this read" +); + +// a swapCommit the pds no longer has, so it refuses the star write +const staleCommit = "bafyreie5737gdxlw5i64vzichcalba3z2v5n6icifvx5xytvygrxj2r4gy"; +const racedByAnotherWrite = (ctx: Ctx) => + ctx.page.route("**/xrpc/com.atproto.repo.createRecord", async (route) => { + const body = route.request().postDataJSON(); + if (body?.collection !== starId.nsid) return route.fallback(); + await route.fallback({ postData: JSON.stringify({ ...body, swapCommit: staleCommit }) }); + }); + +// the failure clears after 2s, so check it right after the refusal +const failedStar = testid("star", { failed: "true" }); + +flow("a star the pds refuses rolls back and the button says so") + .actor("owner") + .repo() + .actor("fan") + + .open(page.repo) + .step("the pds will refuse the next star", racedByAnotherWrite) + .click(testid("star", { starred: "false" })) + .sends( + starId, + pds, + (ctx) => ({ + record: { subject: { $type: "org.tangled.feed.star#repo", identity: ctx.repo.did } } + }), + "create" + ) + .sees(testid("star", { starred: "true" })) + .sees(starCount, "1") + .refused(starId, 400) + .sees(failedStar) + .sees(testid("star", { starred: "false" })) + .sees(starCount, "0") + .step("bobbin counts no stars", async (ctx) => + expect(await bobbin.countStars(ctx.repo.did)).toBe(0) + ) + .reload() + .sees(testid("star", { starred: "false" })) + .sees(starCount, "0"); + +// a real burst is hard to hit on purpose, so the 429 is faked for one comment +const rateLimited = "too many writes, slow down"; +const limitsTheNextComment = async (ctx: Ctx) => { + let limited = false; + await ctx.page.route("**/xrpc/com.atproto.repo.createRecord", async (route) => { + if (limited || route.request().postDataJSON()?.collection !== commentId.nsid) + return route.fallback(); + limited = true; + await route.fulfill(jsonError(429, "RateLimitExceeded", rateLimited)); + }); +}; + +flow("a comment the knot rate limits keeps its text and can be sent again") + .actor("owner") + .repo() + .ticket(ticketId, { title: commented }) + + .open(page.issue(ticketId)) + .step("the knot will rate limit the next comment", limitsTheNextComment) + .fill(discussion, note) + .click(testid("submit")) + .sees(refusal) + .step("the box still holds the text", async (ctx) => + expect(discussion.locate(ctx.page, ctx)).toHaveValue(note) + ) + .sendsNothing(commentId) + + .click(testid("submit")) + .sends( + commentId, + knot, + (ctx) => ({ + record: { subject: { $type: "com.atproto.repo.strongRef", ...ctx.get(ticketId) } } + }), + "create" + ) + .trace(commentId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .sees(text(note)) + .sees(not(refusal)) + .reload() + .sees(text(note)); + +flow("two submits of the comment form in one task send one write") + .actor("owner") + .repo() + .ticket(ticketId, { title: commented }) + + .open(page.issue(ticketId)) + .fill(discussion, note) + .step("the form is submitted twice before the page can react", (ctx) => + discussion.locate(ctx.page, ctx).evaluate((box) => { + const form = (box as HTMLTextAreaElement).form!; + form.requestSubmit(); + form.requestSubmit(); + }) + ) + .sends(commentId, knot, { record: { body: { text: note } } }, "create") + .trace(commentId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .sendsNothing(commentId) + .sees(text(note)); + +// the knot saves the comment and answers late, so the page reloads mid-send +const slowAnswer = async (ctx: Ctx) => { + await ctx.page.route("**/xrpc/com.atproto.repo.createRecord", async (route) => { + if (route.request().postDataJSON()?.collection !== commentId.nsid) return route.fallback(); + const response = await route.fetch(); + await sleep(4_000); + // the page this answer was for is gone by then + await route.fulfill({ response }).catch(() => undefined); + }); +}; + +const sentComment = new WeakMap(); + +flow("reloading while a comment is still sending leaves it posted once") + .actor("owner") + .repo() + .ticket(ticketId, { title: commented }) + + .open(page.issue(ticketId)) + .step("the knot is slow to answer", slowAnswer) + .fill(discussion, note) + .click(testid("submit")) + .step("the knot has the comment and the page is still waiting", async (ctx) => { + const [stored] = await until("the knot should have the comment", 10_000, async () => { + const records = await knotRecords(ctx.repo.did, commentId.nsid); + return [records.length === 1, `it lists ${records.length}`, records]; + }); + sentComment.set(ctx, stored); + await expect(testid("submit").locate(ctx.page, ctx)).toBeDisabled(); + }) + .step("the page is reloaded", async (ctx) => { + await ctx.page.reload({ waitUntil: "networkidle" }); + }) + .step("every service has the comment", async (ctx) => { + const written = sentComment.get(ctx)!; + await Promise.all([ + knot.present(written, ctx.actor), + hydrant.present(written), + bobbin.indexed(written) + ]); + }) + .reload() + .sees(text(note)) + .step("there is one comment", async (ctx) => { + await expect(ctx.page.getByText(note, { exact: true })).toHaveCount(1); + expect(await knotRecords(ctx.repo.did, commentId.nsid)).toHaveLength(1); + }); + +// 60 family emoji are about 1500 bytes, over the title's 1024 byte max +const familyOf = (graphemes: number) => "👨‍👩‍👧‍👦".repeat(graphemes); +const tooLong = familyOf(60); +const trimmed = familyOf(40); +const titleField = testid("title"); + +flow("a title over the byte limit is refused with a reason and a shorter one saves") + .actor("owner") + .repo() + + .open(page.newIssue) + .fill(titleField, tooLong) + .click(testid("submit")) + .sends(refusedTicketId, knot, { record: { title: { text: tooLong } } }, "create") + .refused(refusedTicketId, 400) + .sees(refusal) + .step("the form still holds the title", async (ctx) => + expect(titleField.locate(ctx.page, ctx)).toHaveValue(tooLong) + ) + + .fill(titleField, trimmed) + .click(testid("submit")) + .sends(ticketId, knot, { record: { title: { text: trimmed } } }, "create") + .trace(ticketId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .at(page.issues) + .sees(link(trimmed)); + +const slowTitle = "saved while bobbin was slow"; + +flow("creating an issue the knot saved while bobbin was slow does not make a second one") + .actor("owner") + .repo() + + .open(page.newIssue) + .step("bobbin fails the page's wait", bobbinFails) + .fill(titleField, slowTitle) + .click(testid("submit")) + .sends(ticketId, knot, { record: { title: { text: slowTitle } } }, "create") + .trace(ticketId, through(knot, hydrant, bobbin)) + .sees(refusal) + .step("bobbin answers the page again", bobbinRecovers) + + .click(testid("submit")) + .sendsNothing(ticketId) + .trace(subscriptionId, through(pds)) + .step("the knot has one issue with that title", async (ctx) => { + const stored = await knotRecords(ctx.repo.did, ticketId.nsid); + expect(stored.filter((record) => record.value.title.text === slowTitle)).toHaveLength(1); + }); + +flow("closing an issue the knot took while bobbin was slow shows it closed") + .actor("owner") + .repo() + .ticket(ticketId, { title: "close this one" }) + + .open(page.issue(ticketId)) + .step("bobbin fails the page's wait", bobbinFails) + .click(testid("requestStateChange", { state: "open" })) + .sends(ticketId, knot, replacing(ticketId, { state: "closed" }), "put") + .trace(ticketId, through(knot, hydrant, bobbin)) + .knotStored(ticketId, "closed") + .sees(stateChangeAlert) + .sees(testid("requestStateChange", { state: "closed" })) + .step("bobbin answers the page again", bobbinRecovers) + .reload() + .sees(testid("requestStateChange", { state: "closed" })); + +// guards openTicket and openComment against probing, since a failed probe falls back to the pds +const probedTitle = "written while the probe fails"; + +flow("an issue opened while the knot's capability probe fails still lands on the knot") + .actor("owner") + .repo() + + .open(page.newIssue) + .step("the knot's capability probe fails", probeFails) + .fill(titleField, probedTitle) + .click(testid("submit")) + .sends( + ticketId, + knot, + (ctx) => ({ + repo: ctx.repo.did, + record: { $type: ticketId.nsid, title: { text: probedTitle } } + }), + "create" + ) + .trace(ticketId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .at(page.issues) + .sees(link(probedTitle)); + +flow("a comment sent while the knot's capability probe fails still lands on the knot") + .actor("owner") + .repo() + .ticket(ticketId, { title: commented }) + + .open(page.issue(ticketId)) + .step("the knot's capability probe fails", probeFails) + .fill(discussion, note) + .click(testid("submit")) + .sends( + commentId, + knot, + (ctx) => ({ + repo: ctx.repo.did, + record: { $type: commentId.nsid, body: { text: note } } + }), + "create" + ) + .trace(commentId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .sees(text(note)); + +const firstTitle = "a title before anyone edits it"; +const secondTitle = "retitled from the second window"; +const staleTitle = "retitled from the stale window"; +const moreActions = testid("moreMenu"); + +flow("an edit from a page that loaded before another edit is refused") + .actor("owner") + .repo() + .ticket(ticketId, { title: firstTitle }) + + .open(page.issue(ticketId)) + .click(moreActions) + .click(testid("setEditing")) + .fill(titleField, staleTitle) + + .window("second") + .open(page.issue(ticketId)) + .click(moreActions) + .click(testid("setEditing")) + .fill(titleField, secondTitle) + .click(within(component("TicketForm"), testid("submit"))) + .sends(ticketId, knot, replacing(ticketId, { title: { text: secondTitle } }), "put") + .trace(ticketId, through(knot, hydrant, bobbin)) + .sees(within(testid("ticketTitle"), text(secondTitle))) + + .window() + .click(within(component("TicketForm"), testid("submit"))) + .sees(staleEdit) + .sendsNothing(refusedTicketId) + .step("the form still holds the stale window's title", async (ctx) => + expect(titleField.locate(ctx.page, ctx)).toHaveValue(staleTitle) + ) + .step("the knot still has the second title", async (ctx) => { + const stored = await knot.has(ctx.get(ticketId)); + expect(stored.title.text).toBe(secondTitle); + }); diff --git a/e2e/specs/fork-pulls.spec.ts b/e2e/specs/fork-pulls.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..7b73cbeb119280dde5c873fa6d6df88e748b51ab --- /dev/null +++ b/e2e/specs/fork-pulls.spec.ts @@ -0,0 +1,197 @@ +import { expect } from "@playwright/test"; + +import { + button, + commitRow, + type Ctx, + link, + mergeButton, + not, + pullId, + target, + testid, + text, + unique, + where +} from "../harness/flow"; +import { spindle } from "../harness/services"; +import { + answerWith, + branches, + forkedAhead, + merges, + opensForkPull, + reloadsUntil, + resubmitsForkPatch, + waitsForApproval, + targetTookTheFeature, + upstreamPull +} from "../harness/shared"; +import { delivers, dropsHook, hookOn, nothingMore } from "../harness/webhooks"; + +const forkTitle = unique("a fix from my fork"); +const forkHead = (ctx: Ctx) => ctx.repo.heads[branches.from]; +const upstreamDid = (ctx: Ctx) => ctx.upstream.did; +const completed = testid("ticketStatePill", { state: "completed" }); + +const upstreamMerged = where( + "the upstream's merged pulls", + (ctx) => `${ctx.upstream.path}/pulls?state=completed` +); + +forkedAhead("merging a fork pull request moves the upstream's target to the fork's commit") + .use(opensForkPull(forkTitle)) + + .actor("owner") + .open(upstreamPull) + .use(merges) + .knotStored(pullId, "completed") + .step( + "the upstream's target branch took the fork's commit", + targetTookTheFeature((ctx) => ctx.actor, { fromFork: true }) + ) + .checkpoint("the merge moved the upstream's target branch") + .reload() + .sees(completed) + .open(upstreamMerged) + .sees(link(forkTitle)); + +// the mirror misses a fork's head for well under a second, too short to hit on purpose +const mergeCheckRoute = "**/xrpc/sh.tangled.git.temp2.mergeCheck*"; +const mergeCommitRoute = "**/xrpc/sh.tangled.git.mergeCommit"; +const lacksTheHead = (route: string) => (ctx: Ctx) => + answerWith(route, 404, "RevisionNotFound", `commit not found: ${forkHead(ctx)}`)(ctx); +const mirrorWait = testid("mirrorWait"); + +forkedAhead("a fork pull request waits to merge until the mirror has the fork's head") + .use(opensForkPull(forkTitle)) + + .actor("owner") + .open(upstreamPull) + .step("the mirror doesn't have the fork's head yet", lacksTheHead(mergeCheckRoute)) + .reload() + .sees(testid("mirrorWait", { state: "waiting" })) + .step("the merge waits for the mirror", (ctx) => + expect(mergeButton.locate(ctx.page, ctx)).toBeDisabled() + ) + .step("the mirror catches up", (ctx) => ctx.page.unroute(mergeCheckRoute)) + .step("the check tries again by itself", (ctx) => + expect(mirrorWait.locate(ctx.page, ctx)).toHaveCount(0, { timeout: 10_000 }) + ) + .step("the merge loses the race once", lacksTheHead(mergeCommitRoute)) + .click(mergeButton) + .sees( + testid("errorAlert"), + "The mirror hasn't caught up with this pull yet, try again in a moment." + ) + .step("the mirror catches up again", (ctx) => ctx.page.unroute(mergeCommitRoute)) + .use(merges) + .knotStored(pullId, "completed") + .step( + "the upstream's target branch took the fork's commit", + targetTookTheFeature((ctx) => ctx.actor, { fromFork: true }) + ); + +const pushedFile = testid("diffFileName", { path: "e2e/more.txt" }); +// no round labels here, so the commit rows tell the rounds apart +forkedAhead("resubmitting a fork pull request after pushing to the fork's branch") + .use(opensForkPull(forkTitle)) + .push({ "e2e/more.txt": "more\n" }, { branch: branches.from }) + .mirrored() + .open(upstreamPull) + .sees(not(pushedFile)) + .sees(testid("resubmit")) + .use(resubmitsForkPatch) + .reload() + .step("the new file shows up", reloadsUntil(pushedFile)) + .sees(pushedFile) + .sees(commitRow(2)) + .sees(not(commitRow(3))); + +const approveCi = button("Approve CI"); + +forkedAhead("a fork pull request from someone who isn't a collaborator waits for approval", { + ci: true +}) + .use(opensForkPull(forkTitle)) + .step("the opened pull's pipeline is blocked", waitsForApproval(forkHead, upstreamDid)) + .push({ "e2e/more.txt": "more\n" }, { branch: branches.from }) + .mirrored() + .open(upstreamPull) + .use(resubmitsForkPatch) + .reload() + .changeState(pullId, "closed", testid("requestClose"), testid("changeState")) + .changeState(pullId, "open", testid("changeState"), testid("requestClose")) + .step( + "still blocked after the resubmit, the close and the reopen", + waitsForApproval(forkHead, upstreamDid) + ) + .sees(not(approveCi)) + + .actor("owner") + .open(upstreamPull) + .click(approveCi) + .click(button("Approve CI run")) + .sees(text("CI approved")) + .step("spindle runs a workflow at the fork's head", (ctx) => + spindle.finishedWorkflow(ctx.upstream.did, forkHead(ctx)) + ); + +// the hook belongs to the upstream, but the flow's repo is the fork +const onUpstream = (step: (ctx: Ctx) => Promise) => async (ctx: Ctx) => { + const fork = ctx.repo; + ctx.repo = ctx.upstream; + try { + await step(ctx); + } finally { + ctx.repo = fork; + } +}; + +const forks = new WeakMap(); +const rememberFork = async (ctx: Ctx) => { + forks.set(ctx, { did: ctx.repo.did, sha: forkHead(ctx), forker: ctx.actor.did }); +}; + +const pullEvents = ["pull_request:created", "pull_request:merged"]; + +const forkPullDelivered = (action: string, state: string) => + onUpstream( + delivers(`pull_request:${action}`, "Tangled-Hook/pull_request", (body, ctx) => { + const fork = forks.get(ctx); + if (!fork) throw new Error("this step needs the fork remembered first"); + expect(body).toMatchObject({ + action, + sender: { did: ctx.actor.did }, + pull_request: { + title: forkTitle, + state, + target_branch: branches.into, + round_number: 1, + owner: { did: fork.forker }, + source: { repo: fork.did, branch: branches.from, sha: fork.sha } + } + }); + }) + ); + +forkedAhead("fork pull request webhooks name the fork as the source", { ci: true }) + .step("remember the fork", rememberFork) + .actor("owner") + .step("spindle has a pipeline for the first push", (ctx) => + spindle.hasPipeline(ctx.upstream.did, ctx.upstream.heads[branches.into], "push") + ) + .step( + "point a webhook at the upstream", + onUpstream(hookOn(pullEvents, { secret: "e2e-webhook-secret" })) + ) + .actor("forker") + .use(opensForkPull(forkTitle)) + .step("the receiver got the new pull", forkPullDelivered("created", "open")) + + .actor("owner") + .open(upstreamPull) + .use(merges) + .step("the receiver got the merge", forkPullDelivered("merged", "merged")) + .step("nothing else arrived", onUpstream(nothingMore)) + .step("take the webhook away", onUpstream(dropsHook)); diff --git a/e2e/specs/forks.spec.ts b/e2e/specs/forks.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..16f005936e629eae5dbb56872e7a737aa2d339b7 --- /dev/null +++ b/e2e/specs/forks.spec.ts @@ -0,0 +1,113 @@ +import { expect } from "@playwright/test"; + +import { + bobbin, + type Ctx, + flow, + hydrant, + knotCopy, + manifestId, + not, + page, + pds, + target, + testid, + through, + where +} from "../harness/flow"; +import { until } from "../harness/net"; + +const files = { "README.md": "hello from e2e\n" }; + +let upstream = { did: "", handle: "", name: "" }; +let fork = { handle: "", name: "" }; +const noteFork = async (ctx: Ctx) => { + fork = { handle: ctx.actor.handle, name: ctx.repo.name }; +}; +const noteUpstream = async (ctx: Ctx) => { + upstream = { did: ctx.repo.did, handle: ctx.actor.handle, name: ctx.repo.name }; +}; + +const upstreamPage = where("the upstream's page", () => `/${upstream.handle}/${upstream.name}`); +const forksPage = where( + "the upstream's forks page", + () => `/${upstream.handle}/${upstream.name}/forks` +); +const forksCount = testid("forks"); +const forkForm = where("the fork form", (ctx) => `${ctx.repo.path}/fork`); + +const forkCard = target("the fork's card", (tab) => + tab.locator(`[data-component="RepoCardContent"] a[href="/${fork.handle}/${fork.name}"]`).first() +); + +const bobbinListsTheFork = async (ctx: Ctx) => { + const forkUri = `at://${ctx.repo.did}/org.tangled.repo.manifest/self`; + const { items } = await until( + `bobbin should list ${forkUri} as a fork of ${upstream.did}`, + 20_000, + async () => { + const listed = await bobbin.listForks(upstream.did); + const counted = await bobbin.countForks(upstream.did); + const listsIt = listed.items?.some((one: { uri: string }) => one.uri === forkUri); + return [ + Boolean(listsIt) && listed.total === counted && counted === listed.items.length, + `listForks said ${JSON.stringify(listed)}, countForks ${counted}`, + listed + ]; + } + ); + expect(items).toHaveLength(1); + expect(items[0].value.source).toMatchObject({ did: upstream.did }); + expect(items[0].value.declaration).toMatchObject({ + owner: ctx.actor.did, + name: ctx.repo.name + }); +}; + +flow("a repo nobody forked has an empty forks page") + .actor("owner") + .repo() + .push(files) + .step("note the upstream", noteUpstream) + .step("bobbin lists no forks and counts none", async () => { + const listed = await bobbin.listForks(upstream.did); + expect(listed.items).toEqual([]); + expect(await bobbin.countForks(upstream.did)).toBe(0); + }) + .open(upstreamPage) + .sees(forksCount, "0") + .click(forksCount) + .at(forksPage) + .sees(testid("emptyState")) + .reload() + .sees(testid("emptyState")); + +flow("a fork shows on the upstream's forks page") + .actor("owner") + .repo() + .push(files) + .step("note the upstream", noteUpstream) + + .actor("forker") + .member() + .open(upstreamPage) + .click(testid("fork")) + .at(forkForm) + .click(testid("submit")) + .traceNewRepo((ctx) => ctx.repo.name, through(pds, hydrant, bobbin)) + .step("note the fork", noteFork) + .follow(manifestId, through(knotCopy, hydrant, bobbin)) + .step("bobbin lists the fork and agrees with its count", bobbinListsTheFork) + + .open(upstreamPage) + .sees(forksCount, "1") + .click(forksCount) + .at(forksPage) + .sees(forkCard) + .sees(not(testid("emptyState"))) + .reload() + .sees(forkCard) + + .actor("owner") + .open(forksPage) + .sees(forkCard); diff --git a/e2e/specs/git.spec.ts b/e2e/specs/git.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..ecd2dd58aff7df50ef262d82d655d4afca1245ef --- /dev/null +++ b/e2e/specs/git.spec.ts @@ -0,0 +1,420 @@ +import { expect } from "@playwright/test"; +import { execFile } from "node:child_process"; +import { readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { gunzipSync } from "node:zlib"; +import { fetch as undiciFetch } from "undici"; + +import { decodeRefs, pdsCall, sessionOf } from "../harness/api"; +import { type Account, config } from "../harness/config"; +import { + bobbin, + branchesTab, + confirmDelete, + type Ctx, + exactLink, + flow, + git, + heading, + hydrant, + knot, + knotCopy, + link, + not, + page, + pds, + record, + refsId, + target, + text, + testid, + through, + where, + within +} from "../harness/flow"; +import { cloneOverHttps, lsRemote, lsRemoteOverHttps, sshAs } from "../harness/git"; +import { dispatcher, until } from "../harness/net"; +import { base, deletesBranch, learnRefs, mirrorLists, pushesTag } from "../harness/shared"; +import { type Artifact } from "../harness/lexicons"; + +const files = { "README.md": "hello from e2e\n", "src/main.txt": "x\n" }; + +flow("pushing a commit to a new repo") + .actor("owner") + .repo() + .push(files) + .follow(refsId, through(git, knotCopy, hydrant, bobbin)) + .step("the record names the pushed commit and the default branch", async (ctx) => { + const value = await knot.has(ctx.get(refsId)); + expect(decodeRefs(value)).toContainEqual([ctx.pushed.sha, "refs/heads/main"]); + expect(value.defaultBranch).toBe("main"); + }); + +flow("a pushed repo shows its files") + .actor("owner") + .repo() + .push(files) + .mirrored() + .open(page.repo) + .sees(link("README.md")) + .sees(link("src")); + +const exec = promisify(execFile); + +// encoded as one segment, so a slash in a branch name survives +const treeAt = (ref: string, path = "") => + where(`the tree of ${ref}${path && ` at ${path}`}`, (ctx) => + [`${ctx.repo.path}/tree/${encodeURIComponent(ref)}`, path].filter(Boolean).join("/") + ); +const blobOf = (ref: string, path: string) => + where( + `the blob ${path} on ${ref}`, + (ctx) => `${ctx.repo.path}/blob/${encodeURIComponent(ref)}/${path}` + ); +const tagsTab = where("the tags tab", (ctx) => `${ctx.repo.path}/tags`); + +flow("browsing a branch with a slash in its name down to one file") + .actor("owner") + .repo() + .push(base) + .push({ "docs/guide/intro.txt": "browse me\n" }, { branch: "feature/browse", base: "main" }) + .mirrored() + + .open(treeAt("feature/browse")) + .sees(exactLink("docs")) + .click(exactLink("docs")) + .at(treeAt("feature/browse", "docs")) + .click(exactLink("guide")) + .at(treeAt("feature/browse", "docs/guide")) + .click(exactLink("intro.txt")) + .at(blobOf("feature/browse", "docs/guide/intro.txt")) + .sees(text("browse me")) + .step("the knot has the branch at the commit the page read", async (ctx) => { + const heads = await lsRemote(ctx.actor, ctx.repo.did, "refs/heads/"); + expect(heads["refs/heads/feature/browse"]).toBe(ctx.repo.heads["feature/browse"]); + }); + +const readme = "# readme from a branch\n\nsee [the code](src/main.txt) for more\n"; +// the heading has its own anchor link in it, so text alone won't match +const renderedReadmeHeading = target("the readme's heading", (tab) => + tab.locator('[data-testid="renderedMarkdownBody"] h1:has(a[href="#readme-from-a-branch"])') +); + +flow("a readme on a branch renders and its relative link goes to the file") + .actor("owner") + .repo() + .push(base) + .push( + { "README.md": readme, "src/main.txt": "linked file\n" }, + { branch: "feature/readme", base: "main" } + ) + .mirrored() + + .open(treeAt("feature/readme")) + .sees(renderedReadmeHeading) + .click(exactLink("the code")) + .sees(text("linked file")); + +const onlyMain = async (ctx: Ctx) => + expect(Object.keys(await lsRemote(ctx.actor, ctx.repo.did, "refs/heads/"))).toEqual([ + "refs/heads/main" + ]); + +flow("deleting a branch over git takes it off the branches tab") + .actor("owner") + .repo() + .push(base) + .push({ "e2e/doomed.txt": "gone soon\n" }, { branch: "doomed", base: "main" }) + .mirrored() + + .open(branchesTab) + .sees(exactLink("doomed")) + .sees(exactLink("main")) + .step("delete the branch over git", deletesBranch("doomed")) + .step("the knot's record drops it", learnRefs("refs/heads/doomed", false)) + .follow(refsId, through(git, knotCopy, hydrant, bobbin)) + .step("git lists no doomed branch", onlyMain) + .step("bobbin has caught up", mirrorLists("Branches", "doomed", false)) + .reload() + .sees(not(exactLink("doomed"))) + .sees(exactLink("main")); + +flow("an annotated tag shows under tags with its message") + .actor("owner") + .repo() + .push(base) + .step("push the tag over git", pushesTag("v1.0", "the first release")) + .step("the knot's record names it", learnRefs("refs/tags/v1.0", true)) + .follow(refsId, through(knotCopy, hydrant, bobbin)) + .step("git lists the tag", async (ctx) => { + const tags = await lsRemote(ctx.actor, ctx.repo.did, "refs/tags/"); + expect(Object.keys(tags)).toContain("refs/tags/v1.0"); + }) + .step("bobbin has caught up", mirrorLists("Tags", "v1.0", true)) + + .open(tagsTab) + .sees(exactLink("v1.0")) + .sees(within(testid("tagMessageTitle"), text("the first release"))) + .click(exactLink("v1.0")) + .at(where("the tag's page", (ctx) => `${ctx.repo.path}/tags/v1.0`)) + .sees(text("the first release")); + +const cloneUrl = (label: "SSH" | "HTTPS") => testid("cloneUrlValue", { label }); +const codeMenu = within(testid("cloneThisRepository"), testid("toggleMenu")); + +flow("the ssh clone url in the code menu is one git accepts") + .actor("owner") + .repo() + .push(base) + .mirrored() + .open(page.repo) + .click(codeMenu) + .sees(cloneUrl("SSH")) + .step("git lists the repo's branches at the url the page shows", async (ctx) => { + const url = await cloneUrl("SSH").locate(ctx.page, ctx).innerText(); + // the page shows the host with no port, so the port is set by hand + const [host, port] = config.knotSsh.split(":"); + const listed = await until(`git should accept ${url}`, 60_000, async () => { + try { + const { stdout } = await exec("git", ["ls-remote", "--heads", url], { + env: { + ...process.env, + GIT_SSH_COMMAND: `${await sshAs(ctx.actor)} -p ${port} -o HostName=${host}` + }, + timeout: 30_000 + }); + return [true, stdout, stdout]; + } catch (cause: any) { + return [false, String(cause.stderr ?? cause).trim()]; + } + }); + expect(listed).toContain(`${ctx.repo.heads["main"]}\trefs/heads/main`); + }); + +const redirectOf = async (url: string) => { + const res = await undiciFetch(url, { dispatcher, redirect: "manual" }); + await res.arrayBuffer(); + return { status: res.status, location: res.headers.get("location") }; +}; + +flow("the https clone url in the code menu clones through the web app") + .actor("owner") + .repo() + .push(base) + .mirrored() + .open(page.repo) + .click(codeMenu) + .sees(cloneUrl("HTTPS")) + .step("info/refs at that url redirects to the repo did on the knot", async (ctx) => { + const url = await cloneUrl("HTTPS").locate(ctx.page, ctx).innerText(); + const service = "info/refs?service=git-upload-pack"; + expect(await redirectOf(`${url}/${service}`)).toEqual({ + status: 307, + location: `${config.knotUrl}/${ctx.repo.did}/${service}` + }); + }) + .step("git lists and clones the repo at that url", async (ctx) => { + const url = await cloneUrl("HTTPS").locate(ctx.page, ctx).innerText(); + expect(await lsRemoteOverHttps(url)).toContain( + `${ctx.repo.heads["main"]}\trefs/heads/main` + ); + expect(await cloneOverHttps(`${url}.git`)).toBe(ctx.repo.heads["main"]); + }) + .step("git lists the repo at its did permalink", async (ctx) => { + const url = `${config.webUrl}/${ctx.repo.did}`; + expect(await lsRemoteOverHttps(url)).toContain( + `${ctx.repo.heads["main"]}\trefs/heads/main` + ); + }); + +const atHandle = where( + "the repo under its @handle", + (ctx) => `/@${ctx.actor.handle}/${ctx.repo.name}` +); + +flow("an old @handle repo link opens the repo and clones over https") + .actor("owner") + .repo() + .push(base) + .mirrored() + .open(atHandle) + .at(page.repo) + .step("git lists the repo at the @handle url", async (ctx) => { + const url = `${config.webUrl}${atHandle.path(ctx)}`; + expect(await lsRemoteOverHttps(url)).toContain( + `${ctx.repo.heads["main"]}\trefs/heads/main` + ); + }); + +const tarFiles = (tgz: Buffer) => { + const tar = gunzipSync(tgz); + const found: Record = {}; + for (let at = 0; at + 512 <= tar.length && tar[at] !== 0;) { + const cstr = (from: number, to: number) => tar.toString("utf8", from, to).split("\0")[0]; + const size = parseInt(cstr(at + 124, at + 136).trim() || "0", 8); + const kind = String.fromCharCode(tar[at + 156]); + const name = [cstr(at + 345, at + 500), cstr(at, at + 100)].filter(Boolean).join("/"); + if (kind === "0" || kind === "\0") + found[name] = tar.toString("utf8", at + 512, at + 512 + size); + at += 512 + Math.ceil(size / 512) * 512; + } + return found; +}; + +const zipNames = (zip: Buffer) => { + const names: string[] = []; + for (let at = zip.indexOf("PK\x03\x04"); at !== -1; at = zip.indexOf("PK\x03\x04", at + 4)) { + const length = zip.readUInt16LE(at + 26); + names.push(zip.toString("utf8", at + 30, at + 30 + length)); + } + return names; +}; + +const downloadOf = async (ctx: Ctx, name: string) => { + const format = name === ".zip" ? "zip" : name; + const [download] = await Promise.all([ + ctx.page.waitForEvent("download"), + testid("downloadArchiveLink", { format }).locate(ctx.page, ctx).click() + ]); + const saved = join(tmpdir(), `e2e-archive-${Date.now()}-${download.suggestedFilename()}`); + await download.saveAs(saved); + try { + return await readFile(saved); + } finally { + await rm(saved, { force: true }); + } +}; + +const archiveFiles = { "e2e/base.txt": "in the archive\n", "docs/readme.txt": "second file\n" }; + +flow("the tar.gz in the code menu downloads the default branch") + .actor("owner") + .repo() + .push(archiveFiles) + .mirrored() + .open(page.repo) + .click(codeMenu) + .step("the download unpacks to the pushed files under the repo's name", async (ctx) => { + const files = tarFiles(await downloadOf(ctx, "tar.gz")); + for (const [path, contents] of Object.entries(archiveFiles)) + expect(files[`${ctx.repo.name}-main/${path}`]).toBe(contents); + }); + +flow("the zip in the code menu downloads the default branch") + .actor("owner") + .repo() + .push(archiveFiles) + .mirrored() + .open(page.repo) + .click(codeMenu) + .step("the download lists the pushed files under the repo's name", async (ctx) => { + const names = zipNames(await downloadOf(ctx, ".zip")); + for (const path of Object.keys(archiveFiles)) + expect(names).toContain(`${ctx.repo.name}-main/${path}`); + }); + +const artifact = record("sh.tangled.repo.artifact"); +const tagPage = where("the tag's page", (ctx) => `${ctx.repo.path}/tags/v1.0`); +const releaseName = "release.txt"; + +const releaseFile = target("the release file input", (tab) => tab.getByLabel("Upload artifact")); +const releaseLink = exactLink(releaseName); + +flow("uploading a release file to a tag lists it on the tag page") + .actor("owner") + .repo() + .push(base) + .step("push an annotated tag", pushesTag("v1.0", "the first release")) + .step("bobbin lists the tag", mirrorLists("Tags", "v1.0", true)) + + .open(tagPage) + .sees(releaseFile) + .step("choose the file", async (ctx) => { + await releaseFile.locate(ctx.page, ctx).setInputFiles({ + name: releaseName, + mimeType: "text/plain", + buffer: Buffer.from("the release\n") + }); + }) + .sends(artifact, pds, () => ({ record: { $type: artifact.nsid, name: releaseName } }), "create") + .trace(artifact, through(pds, hydrant, bobbin)) + .sees(releaseLink) + .reload() + .sees(releaseLink); + +// no helper for a binary upload here, so this dials the pds directly +const uploadBlob = async (who: Account, bytes: Buffer, mimeType: string) => { + const sent = await undiciFetch(`${config.pdsUrl}/xrpc/com.atproto.repo.uploadBlob`, { + dispatcher, + method: "POST", + headers: { authorization: `Bearer ${await sessionOf(who)}`, "content-type": mimeType }, + body: bytes + }); + const body: any = await sent.json(); + expect(sent.status, `the pds refused the blob: ${JSON.stringify(body)}`).toBe(200); + return body.blob; +}; + +// a sha256 repo's tags can't give the 20 byte hash the lexicon wants, so this is made up +const makeRelease = async (ctx: Ctx) => { + const blob = await uploadBlob(ctx.actor, Buffer.from("the release\n"), "text/plain"); + const made = await pdsCall(ctx.actor, "com.atproto.repo.createRecord", { + collection: artifact.nsid, + record: { + $type: artifact.nsid, + name: releaseName, + repoDid: ctx.repo.did, + tag: { $bytes: Buffer.alloc(20, 7).toString("base64") }, + createdAt: new Date().toISOString(), + artifact: blob + } satisfies Artifact + }); + ctx.adopt(artifact, { uri: made.uri, cid: made.cid }); +}; +const deleteRelease = target("the delete release button", (tab) => + tab.getByRole("button", { name: `Delete ${releaseName}`, exact: true }) +); + +flow("deleting a release file takes it off the tag page and off every service") + .actor("owner") + .repo() + .push(base) + .step("push an annotated tag", pushesTag("v1.0", "the first release")) + .step("bobbin lists the tag", mirrorLists("Tags", "v1.0", true)) + .step("upload a release file", makeRelease) + + .open(tagPage) + .sees(releaseLink) + .sees(deleteRelease) + .click(deleteRelease) + .click(confirmDelete) + .sends(artifact, pds, () => ({ collection: artifact.nsid }), "delete") + .trace(artifact, through(pds, hydrant, bobbin)) + .sees(not(releaseLink)) + .reload() + .sees(not(releaseLink)); + +const deleteBranchButton = target("the delete branch button", (tab) => + tab.getByRole("button", { name: "Delete branch doomed", exact: true }) +); + +flow("deleting a branch from the branches tab removes it from git") + .actor("owner") + .repo() + .push(base) + .push({ "e2e/doomed.txt": "gone soon\n" }, { branch: "doomed", base: "main" }) + .mirrored() + + .open(branchesTab) + .sees(exactLink("doomed")) + .sees(deleteBranchButton) + .click(deleteBranchButton) + .click(confirmDelete) + .step("the knot's record drops it", learnRefs("refs/heads/doomed", false)) + .follow(refsId, through(git, knotCopy, hydrant, bobbin)) + .step("git lists only main", onlyMain) + .reload() + .sees(not(exactLink("doomed"))) + .sees(exactLink("main")); diff --git a/e2e/specs/issues.spec.ts b/e2e/specs/issues.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..4ea0b60f49852a9fe9783e6d3c1c869f8f36d77f --- /dev/null +++ b/e2e/specs/issues.spec.ts @@ -0,0 +1,411 @@ +import { expect } from "@playwright/test"; + +import { editCob, tickets } from "../harness/api"; +import { + bobbin, + commentId, + component, + type Ctx, + flow, + hydrant, + knot, + link, + not, + page, + patchId, + pds, + pullId, + record, + replacing, + subscriptionId, + target, + testid, + text, + through, + ticketId, + unique, + where, + within +} from "../harness/flow"; +import { + authorLink, + bobbinNamesOpener, + branches, + fanOpened, + deletesComment, + deletesIssue, + knotmirrorNamesOpener, + issue, + note, + onBranches, + rowOf, + stillStored, + theFan, + theOwner +} from "../harness/shared"; + +const closedTab = testid("segment", { segment: "closed" }); + +const hostile = [ + "before the html", + ' ', + 'click me bold', + "after the html" +].join(" "); + +// scope to the body, the page's data scripts hold the raw text too +const bodyParagraph = (tab: Ctx["page"]) => tab.locator("p", { hasText: "after the html" }); +const unsafeInBody = target("something unsafe in the body", (tab) => + bodyParagraph(tab).locator("script, [onerror], [onmouseover], a[href^='javascript']") +); + +flow("an issue body with raw html renders inert") + .actor("owner") + .repo() + .step( + "an issue whose body is raw html", + issue(ticketId, { title: "html in the body", body: hostile }) + ) + + .open(page.issue(ticketId)) + .sees(target("the body", bodyParagraph)) + .sees(not(unsafeInBody)) + .sees(target("the bold word", (tab) => bodyParagraph(tab).locator("b", { hasText: "bold" }))) + .step("no handler ran", async (ctx) => + expect(await ctx.page.evaluate("window.__e2e")).toBeUndefined() + ); + +const mention = target("the mention link", (tab, ctx) => + tab.locator(`a.mention[href="/${ctx.actor.handle}"]`, { hasText: `@${ctx.actor.handle}` }) +); + +flow("a mention in an issue body links to that person's profile") + .actor("owner") + .repo() + .step( + "an issue that mentions its author", + issue(ticketId, { + title: "mind the mention", + body: (ctx) => `thanks to @${ctx.actor.handle} for this` + }) + ) + + .open(page.issue(ticketId)) + .sees(mention) + .click(mention) + .at(page.profile); + +flow("a closed tab with nothing closed says so") + .actor("owner") + .repo() + .ticket(ticketId, { title: "still open" }) + + .open(page.issues) + .sees(link("still open")) + .click(closedTab) + .sees(not(link("still open"))) + .sees(not(testid("ticketRow"))); + +const pageItemNumbers = async (ctx: Ctx) => { + const names = await ctx.page.getByRole("link", { name: /page item \d+/ }).allInnerTexts(); + return names.map((name) => /page item (\d+)/.exec(name)![1]); +}; +const onFirstPage = new WeakMap(); + +const many = 31; +const pageSize = 30; + +flow("paging past the first thirty issues keeps the closed tab") + .actor("owner") + .repo() + .step(`${many} closed issues`, async (ctx) => { + for (let at = 1; at <= many; at++) { + await issue(ticketId, { + title: `page item ${String(at).padStart(2, "0")}`, + state: "closed", + wait: at === many + })(ctx); + } + }) + + .open(page.issues) + .sees(testid("emptyState")) + .click(closedTab) + .sees(testid("nextPage")) + .step("the first page lists thirty", async (ctx) => { + const first = await pageItemNumbers(ctx); + expect(first).toHaveLength(pageSize); + onFirstPage.set(ctx, first); + }) + + .click(testid("nextPage")) + .step("the second page is still the closed tab", async (ctx) => { + await expect(ctx.page).toHaveURL(/[?&]page=2/); + await expect(closedTab.locate(ctx.page, ctx)).toHaveAttribute("aria-current", "page"); + }) + .step("the second page lists the one that is left", async (ctx) => { + await expect.poll(async () => (await pageItemNumbers(ctx)).length).toBe(many - pageSize); + const everything = [...onFirstPage.get(ctx)!, ...(await pageItemNumbers(ctx))]; + expect(new Set(everything).size, "an issue is on both pages or missing").toBe(many); + }); + +const other = record("org.tangled.track.ticket"); + +const searchIssues = within(component("TicketSearch"), testid("q")); + +const needle = unique("search needle"); +const hay = unique("unrelated"); + +flow("searching the issue list finds the issue you named") + .actor("owner") + .repo() + .ticket(ticketId, { title: needle }) + .ticket(other, { title: hay }) + + .open(page.issues) + .sees(link(needle)) + .sees(link(hay)) + .fill(searchIssues, needle) + .press("Enter") + .sees(link(needle)) + .sees(not(link(hay))); + +const pulled = unique("a pull that is not an issue"); +const issued = unique("an issue that is not a pull"); + +onBranches("the issues list leaves out pull requests") + .pull(pullId, patchId, { ...branches, title: pulled }) + .ticket(ticketId, { title: issued }) + + .open(page.issues) + .sees(link(issued)) + .sees(not(link(pulled))) + .open(page.issue(pullId)) + .sees(testid("errorTitle", { status: "404" })); + +const countMe = unique("count me"); + +flow("the comment count on the issue list follows comments and their erase") + .actor("owner") + .repo() + .ticket(ticketId, { title: countMe }) + + .open(page.issues) + .sees(testid("commentCount", { count: 0 })) + + .open(page.issue(ticketId)) + .fill(within(testid("commentEditor", { mode: "new" }), testid("body")), note) + .click(within(testid("commentEditor", { mode: "new" }), testid("submit"))) + .trace(commentId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .sees(within(testid("commentBody"), text(note))) + .open(page.issues) + .sees(testid("commentCount", { count: 1 })) + + .open(page.issue(ticketId)) + .use(deletesComment) + .sees(not(within(testid("commentBody"), text(note)))) + .open(page.issues) + .sees(testid("commentCount", { count: 0 })); + +const title = "the build is broken"; +const body = "it says no"; + +const fixedTitle = "a typo in the title"; + +flow("opening an issue") + .actor("owner") + .repo() + + .open(page.newIssue) + .fill(within(component("IssueForm"), testid("title")), title) + .fill(within(component("IssueForm"), testid("body")), body) + .click(within(component("IssueForm"), testid("submit"))) + .sends( + ticketId, + knot, + (ctx) => ({ + repo: ctx.repo.did, + record: { state: "open", title: { text: title }, body: { text: body } } + }), + "create" + ) + .at(page.newIssue) + .trace(ticketId, through(knot, hydrant, bobbin)) + .at(page.issues) + .sees(link(title)) + + .sends( + subscriptionId, + pds, + (ctx) => ({ record: { subject: { uri: ctx.get(ticketId).uri } } }), + "create" + ) + .trace(subscriptionId, through(pds)); + +for (const [verb, from, to] of [ + ["closing", "open", "closed"], + ["reopening", "closed", "open"] +] as const) { + flow(`${verb} an issue`) + .actor("owner") + .repo() + .ticket(ticketId, { title: `${verb} this`, state: from }) + + .open(page.issue(ticketId)) + .changeState( + ticketId, + to, + testid("requestStateChange", { state: from }), + testid("requestStateChange", { state: to }) + ) + .reload() + .sees(testid("requestStateChange", { state: to })); +} + +flow("editing an issue") + .actor("owner") + .repo() + .ticket(ticketId, { title: "a typo in the titel" }) + + .open(page.issue(ticketId)) + .click(within(component("Ticket"), testid("moreMenu"))) + .click(testid("setEditing")) + .fill(within(component("TicketForm"), testid("title")), fixedTitle) + .click(within(component("TicketForm"), testid("submit"))) + .sends(ticketId, knot, replacing(ticketId, { title: { text: fixedTitle } }), "put") + .trace(ticketId, through(knot, hydrant, bobbin)) + .sees(within(testid("ticketTitle"), text(fixedTitle))) + .reload() + .sees(within(testid("ticketTitle"), text(fixedTitle))); + +// the kept ticket proves the list rendered, or "gone" would pass on an empty list +const kept = record("org.tangled.track.ticket"); + +flow("deleting an issue") + .actor("owner") + .repo() + .ticket(kept, { title: "keep this one" }) + .ticket(ticketId, { title: "delete this one" }) + + .open(page.issue(ticketId)) + .use(deletesIssue) + .at(page.issues) + .sees(link("keep this one")) + .sees(not(link("delete this one"))); + +const oldTitle = "a typo in the titel"; +const newTitle = "a typo in the title"; + +flow("cancelling an issue edit leaves the issue as it was") + .actor("owner") + .repo() + .ticket(ticketId, { title: oldTitle }) + + .open(page.issue(ticketId)) + .click(within(component("Ticket"), testid("moreMenu"))) + .click(testid("setEditing")) + .fill(within(component("TicketForm"), testid("title")), newTitle) + .click(within(component("TicketForm"), testid("cancel"))) + .sendsNothing(ticketId) + .sees(within(testid("ticketTitle"), text(oldTitle))) + .sees(not(within(testid("ticketTitle"), text(newTitle)))) + .reload() + .sees(within(testid("ticketTitle"), text(oldTitle))) + .step( + "the knot and hydrant still have the first version", + stillStored(ticketId, (value) => value.title.text, oldTitle) + ); + +flow("cancelling a new issue writes nothing") + .actor("owner") + .repo() + + .open(page.newIssue) + .fill(within(component("IssueForm"), testid("title")), newTitle) + .fill(within(component("IssueForm"), testid("body")), "never mind") + .click(within(component("IssueForm"), testid("cancel"))) + .at(page.issues) + .sendsNothing(ticketId) + .sees(not(link(newTitle))) + .reload() + .sees(not(link(newTitle))); + +flow("an issue closed in another window shows closed after a reload") + .actor("owner") + .repo() + .ticket(ticketId, { title: "closed from the other window" }) + + .open(page.issue(ticketId)) + .sees(testid("requestStateChange", { state: "open" })) + + .window("second") + .open(page.issue(ticketId)) + .changeState( + ticketId, + "closed", + testid("requestStateChange", { state: "open" }), + testid("requestStateChange", { state: "closed" }) + ) + + .window() + .sees(testid("requestStateChange", { state: "open" })) + .reload() + .sees(testid("requestStateChange", { state: "closed" })); + +// bobbin's issue list has no author filter, so this goes through search like the page does +const closedBy = (who: (ctx: Ctx) => { handle: string }) => + where("the closed issues one account opened", (ctx) => { + const query = `is:closed author:${who(ctx).handle}`; + return `${ctx.repo.path}/issues?q=${encodeURIComponent(query)}`; + }); + +fanOpened("an issue the owner closed still belongs to whoever opened it") + .open(page.issue(ticketId)) + .changeState( + ticketId, + "closed", + testid("requestStateChange", { state: "open" }), + testid("requestStateChange", { state: "closed" }) + ) + .step( + "bobbin names the fan as the opener everywhere, and the owner as no one's", + bobbinNamesOpener(ticketId, "issue", theFan, theOwner) + ) + .reload() + .sees(within(component("Ticket"), authorLink(theFan))) + + .open(closedBy(theFan)) + .sees(within(rowOf(ticketId), authorLink(theFan))) + // the row showing proves the list loaded, so an empty owner list means something + .open(closedBy(theOwner)) + .sees(testid("emptyState")) + .sees(not(rowOf(ticketId))); + +fanOpened("an issue the owner edited still belongs to whoever opened it") + // the edit button is for the author only, the owner has to use the api + .step("the owner retitles it through the api", async (ctx) => { + const edited = await editCob(ctx.actor, ctx.get(ticketId).uri, "retitle", (stored) => ({ + ...stored, + title: { ...stored.title, text: fixedTitle } + })); + expect(edited.status, JSON.stringify(edited.body)).toBe(200); + }) + .learn(ticketId) + .follow(ticketId, through(knot, hydrant, bobbin)) + .step( + "bobbin names the fan as the opener everywhere, and the owner as no one's", + bobbinNamesOpener(ticketId, "issue", theFan, theOwner) + ) + .open(page.issue(ticketId)) + .sees(within(testid("ticketTitle"), text(fixedTitle))) + .sees(within(component("Ticket"), authorLink(theFan))); + +fanOpened("knotmirror lists the fan as the opener of an issue the owner closed") + .step("the owner closes it through the api", async (ctx) => { + await tickets.setState(ctx.actor, ctx.get(ticketId), "closed"); + }) + .learn(ticketId) + .follow(ticketId, through(knot, hydrant, bobbin)) + .knotStored(ticketId, "closed") + .step("knotmirror names the fan", knotmirrorNamesOpener(ticketId, theFan)); diff --git a/e2e/specs/knots.spec.ts b/e2e/specs/knots.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..d31e677ed02a949d6d7dbe5f291bf03882206715 --- /dev/null +++ b/e2e/specs/knots.spec.ts @@ -0,0 +1,279 @@ +import { expect } from "@playwright/test"; + +import { knotRecord } from "../harness/api"; +import { config } from "../harness/config"; +import { + type Ctx, + flow, + knots, + newKnot, + newSpindle, + not, + page, + refusal, + spindles, + target, + testid, + text, + where +} from "../harness/flow"; +import { request, stamp, until } from "../harness/net"; +import { type Saved, savedBy, saveFor } from "../harness/services"; +import { answerWith } from "../harness/shared"; + +const knotHost = new URL(config.knotUrl).host; +const invitePage = (code: (ctx: Ctx) => string) => + where("the invite page", (ctx) => `/invite?knot=${knotHost}&code=${code(ctx)}`); + +const adminSecret = process.env.E2E_KNOT_ADMIN_SECRET ?? "super-strong-devonly-key"; + +const codes = new WeakMap(); + +const mintInviteCode = async (ctx: Ctx) => { + const reply = await request(`${config.knotUrl}/xrpc/org.tangled.temp.server.createInviteCode`, { + method: "POST", + basic: `admin:${adminSecret}` + }); + expect(reply.status, `the knot minted no invite code: ${JSON.stringify(reply.body)}`).toBe(200); + codes.set(ctx, (reply.body as { code: string }).code); +}; +const codeOf = (ctx: Ctx) => codes.get(ctx)!; + +const isMember = async (did: string) => + (await knotRecord(`at://${config.knotDid}/org.tangled.knot.member/${did}`)) !== undefined; + +// the save is a pocket.putPreferences, not a record, and leaving before it's sent loses the knot +const saves = new WeakMap>(); +const watchTheSave = async (ctx: Ctx) => { + saves.set( + ctx, + ctx.page.waitForResponse((reply) => reply.url().includes("pocket.putPreferences")) + ); +}; +const theSaveIsDone = (ctx: Ctx) => saves.get(ctx)!; + +const becomesMember = (ctx: Ctx) => + until(`the knot should list ${ctx.actor.did} as a member`, 15_000, async () => { + const member = await isMember(ctx.actor.did); + return [member, member ? "it does" : "it does not"]; + }); + +flow("joining a knot from an invite link makes you a member and saves the knot") + .actor("guest") + .step("the knot mints an invite code", mintInviteCode) + + .open(invitePage(codeOf)) + .sees(testid("joinHeading", { knot: knotHost })) + .joinedKnot() + .step("watch for the page's save", watchTheSave) + .click(testid("join")) + .step("the knot lists the guest as a member", becomesMember) + .step("the page has saved the knot", theSaveIsDone) + + .open(knots) + .sees(text(knotHost)); + +flow("a joined knot says you are in") + .actor("guest") + .step("the knot mints an invite code", mintInviteCode) + + .open(invitePage(codeOf)) + .joinedKnot() + .click(testid("join")) + .step("the knot lists the guest as a member", becomesMember) + .sees(testid("youreInHeading")) + .sees(testid("knots")); + +const failSavedList = answerWith( + /com\.bad-example\.pocket\.getPreferences/, + 500, + "InternalServerError", + "the e2e flow cut this read" +); +const restoreSavedList = (ctx: Ctx) => ctx.page.unroute(/com\.bad-example\.pocket\.getPreferences/); + +const savedSpindle = "spindle.saved.example.test"; +const addedKnot = "knot.added.example.test"; + +flow("adding a knot keeps the saved spindles when the saved list cannot be read") + .actor("saver") + .open(newSpindle) + .fill(testid("address"), savedSpindle) + .click(testid("submit")) + .at(spindles) + .sees(text(savedSpindle)) + + // saving without reading would wipe the spindles, so the add is refused until the read works + .open(newKnot) + .step("the saved list can't be read", failSavedList) + .fill(testid("address"), addedKnot) + .click(testid("submit")) + .sees(refusal) + .at(newKnot) + .step("the saved list can be read again", restoreSavedList) + .click(testid("submit")) + .at(knots) + .sees(text(addedKnot)) + + .open(spindles) + .sees(target("the saved spindle", (tab) => tab.getByText(savedSpindle, { exact: true }))); + +const savedBecomes = (saved: (ctx: Ctx) => Saved) => async (ctx: Ctx) => { + const want = saved(ctx); + await until( + `pocket should keep ${JSON.stringify(want)} for ${ctx.actor.handle}`, + 15_000, + async () => { + const has = await savedBy(ctx.actor); + return [ + JSON.stringify(has) === + JSON.stringify({ knots: want.knots.sort(), spindles: want.spindles.sort() }), + `it keeps ${JSON.stringify(has)}` + ]; + } + ); +}; + +const removeFor = (pick: Pick) => + target(`Remove on the row of the ${pick} host`, (tab, ctx) => + testid("remove", { host: named(pick)(ctx) }).locate(tab, ctx) + ); +const savedHost = (pick: Pick) => + target(`the saved ${pick} host`, (tab, ctx) => + tab.getByText(named(pick)(ctx), { exact: true }) + ); + +const hosts = new WeakMap(); +const pickHosts = async (ctx: Ctx) => { + const tag = stamp(); + hosts.set(ctx, { + a: `a-${tag}.knot.e2e.test`, + b: `b-${tag}.knot.e2e.test`, + s: `s-${tag}.spindle.e2e.test` + }); +}; +type Pick = "a" | "b" | "s"; +const named = (pick: Pick) => (ctx: Ctx) => hosts.get(ctx)![pick]; + +flow("a knot saved in settings is offered when creating a repo") + .actor("owner") + .step("pick hosts", pickHosts) + + .open(newKnot) + .fill(testid("address"), named("a")) + .click(testid("submit")) + .at(knots) + .sees(savedHost("a")) + .step( + "pocket keeps the knot", + savedBecomes((ctx) => ({ knots: [named("a")(ctx)], spindles: [] })) + ) + + .open(page.newRepo) + .sees( + target("the saved knot among the choices", (tab, ctx) => + testid("choice", { value: named("a")(ctx) }).locate(tab, ctx) + ) + ); + +flow("removing a saved knot keeps the other knot and the saved spindle") + .actor("owner") + .step("pick hosts", pickHosts) + .step("two knots and a spindle are saved", (ctx) => + saveFor(ctx.actor, { + knots: [named("a")(ctx), named("b")(ctx)], + spindles: [named("s")(ctx)] + }) + ) + + .open(knots) + .click(removeFor("a")) + .sees(not(savedHost("a"))) + .step( + "pocket keeps one knot and the spindle", + savedBecomes((ctx) => ({ knots: [named("b")(ctx)], spindles: [named("s")(ctx)] })) + ) + .reload() + .sees(savedHost("b")) + .sees(not(savedHost("a"))); + +flow("removing a saved spindle keeps the saved knot") + .actor("owner") + .step("pick hosts", pickHosts) + .step("a knot and a spindle are saved", (ctx) => + saveFor(ctx.actor, { knots: [named("a")(ctx)], spindles: [named("s")(ctx)] }) + ) + + .open(spindles) + .click(removeFor("s")) + .sees(testid("settingsEmpty")) + .step( + "pocket keeps the knot and no spindle", + savedBecomes((ctx) => ({ knots: [named("a")(ctx)], spindles: [] })) + ) + .reload() + .sees(testid("settingsEmpty")) + .open(knots) + .sees(savedHost("a")); + +const addKnotButton = testid("submit"); + +flow("the add knot form is off until an address is typed and cancel saves nothing") + .actor("owner") + .step("pick hosts", pickHosts) + + .open(newKnot) + .step("Add is off", async (ctx) => { + await expect(addKnotButton.locate(ctx.page, ctx)).toBeDisabled(); + }) + .fill(testid("address"), named("a")) + .step("Add is on, and is left alone", async (ctx) => { + await expect(addKnotButton.locate(ctx.page, ctx)).toBeEnabled(); + }) + .click(testid("knots")) + .at(knots) + .sees(testid("settingsEmpty")) + .step( + "pocket keeps nothing", + savedBecomes(() => ({ knots: [], spindles: [] })) + ); + +flow("a knot typed with https and a path is saved as just its host") + .actor("owner") + .step("pick hosts", pickHosts) + + .open(newKnot) + .fill(testid("address"), (ctx) => `https://${named("a")(ctx).toUpperCase()}/some/path`) + .click(addKnotButton) + .at(knots) + .sees(savedHost("a")) + .step( + "pocket keeps the bare host", + savedBecomes((ctx) => ({ knots: [named("a")(ctx)], spindles: [] })) + ); + +// another device saves a knot after this page read the list and before its write lands +const anotherDeviceSavesFirst = (ctx: Ctx) => { + let raced = false; + return ctx.page.route(/com\.bad-example\.pocket\.putPreferences/, async (route) => { + if (!raced && route.request().postData()?.includes(named("a")(ctx))) { + raced = true; + await saveFor(ctx.actor, { knots: [named("b")(ctx)], spindles: [] }); + } + await route.continue(); + }); +}; + +flow("a knot saved on another device while this page saves one keeps both") + .actor("owner") + .step("pick hosts", pickHosts) + + .open(newKnot) + .step("another device saves a knot just before this page's write", anotherDeviceSavesFirst) + .fill(testid("address"), named("a")) + .click(addKnotButton) + .at(knots) + .step( + "pocket keeps both knots", + savedBecomes((ctx) => ({ knots: [named("a")(ctx), named("b")(ctx)], spindles: [] })) + ); diff --git a/e2e/specs/labels.spec.ts b/e2e/specs/labels.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..8d47db56e22b4529ec309ca6b5a2e55ccaacbd6d --- /dev/null +++ b/e2e/specs/labels.spec.ts @@ -0,0 +1,208 @@ +import { expect } from "@playwright/test"; + +import { editCob, knotRecordFor } from "../harness/api"; +import { + bobbin, + confirmDelete, + createOption, + type Ctx, + editLabels, + flow, + hydrant, + knot, + labelDefinitionId, + labelOption, + link, + newLabel, + not, + page, + record, + refusal, + testid, + text, + through, + ticketId, + triage, + triageChip, + triageToken, + unique, + where, + within +} from "../harness/flow"; +import { defineLabel, labelOff, labelOn, labelsOnKnot } from "../harness/shared"; + +const other = record("org.tangled.track.ticket"); + +const labelled = unique("labelled one"); +const plain = unique("plain one"); + +// the sidebar can't make a label for a new repo's owner, so write it through the api +const labelTicket = async (ctx: Ctx) => { + await defineLabel(triage)(ctx); + const put = await editCob(ctx.actor, ctx.get(ticketId).uri, "label", (stored) => ({ + ...stored, + tangledLabels: [triageToken] + })); + expect(put.status, `labelling the ticket answered ${JSON.stringify(put.body)}`).toBe(200); + await bobbin.indexed({ uri: put.body.uri, cid: put.body.cid }); +}; + +const labelList = where( + "the issues list for one label", + (ctx) => `${ctx.repo.path}/issues?q=${encodeURIComponent(`is:open label:${triage}`)}` +); + +flow("a label chip opens the issue list filtered to that label") + .actor("owner") + .repo() + .ticket(ticketId, { title: labelled }) + .ticket(other, { title: plain }) + .step("label the first ticket", labelTicket) + + .open(page.issue(ticketId)) + .sees(triageChip) + .click(triageChip) + .at(labelList) + .sees(link(labelled)) + .sees(not(link(plain))); + +flow("the repo owner can make a label from the issue sidebar") + .actor("owner") + .repo() + .ticket(ticketId, { title: labelled }) + + .open(page.issue(ticketId)) + .click(editLabels) + .fill(testid("search"), triage) + .sees(createOption); + +flow("a label made with spaces and a chosen colour is stored under its slug") + .actor("owner") + .repo() + + .open(newLabel) + .fill(testid("name"), "Good First Issue") + .click(testid("setColor", { value: "#3b82f6" })) + .click(testid("create")) + .sends( + labelDefinitionId, + knot, + (ctx) => ({ + repo: ctx.repo.did, + rkey: "good-first-issue", + record: { $type: labelDefinitionId.nsid, name: "Good First Issue", color: "#3b82f6" } + }), + "create" + ) + .trace(labelDefinitionId, through(knot, hydrant, bobbin)) + .at(page.settings) + .sees(testid("confirmLabelDelete", { name: "Good First Issue" })) + .reload() + .sees(testid("confirmLabelDelete", { name: "Good First Issue" })); + +flow("putting a label on an issue and taking it off") + .actor("owner") + .repo() + .ticket(ticketId, { title: "label me" }) + .step("define the label", defineLabel(triage)) + + .open(page.issue(ticketId)) + .use(labelOn(ticketId, "ticket")) + .use(labelOff(ticketId, "ticket")); + +const settings = where("the repo's settings", (ctx) => `${ctx.repo.path}/settings`); +const deleteLabel = testid("confirmLabelDelete", { name: triage }); + +flow("deleting a label definition takes it off the issue") + .actor("owner") + .repo() + .ticket(ticketId, { title: "labelled, then not" }) + + .open(newLabel) + .fill(testid("name"), triage) + .click(testid("create")) + .sends( + labelDefinitionId, + knot, + (ctx) => ({ + repo: ctx.repo.did, + rkey: triage, + record: { $type: labelDefinitionId.nsid, name: triage } + }), + "create" + ) + .trace(labelDefinitionId, through(knot, hydrant, bobbin)) + .at(settings) + .sees(deleteLabel) + + .open(page.issue(ticketId)) + .click(editLabels) + .click(labelOption) + .trace(ticketId, through(knot, hydrant, bobbin)) + .sees(triageChip) + .step("the knot's ticket carries the label", async (ctx) => + expect(await labelsOnKnot(ctx, ticketId)).toEqual([triage]) + ) + + .open(settings) + .click(deleteLabel) + .click(confirmDelete) + .sends( + labelDefinitionId, + knot, + (ctx) => ({ repo: ctx.repo.did, collection: labelDefinitionId.nsid, rkey: triage }), + "delete" + ) + .trace(labelDefinitionId, through(knot, hydrant, bobbin)) + .sees(not(deleteLabel)) + .open(page.issue(ticketId)) + .sees(text("labelled, then not")) + .sees(not(triageChip)) + .step("the knot's ticket is off the label", async (ctx) => + expect(await labelsOnKnot(ctx, ticketId)).toEqual([]) + ); + +flow("a label name the repo already has is refused by the form") + .actor("owner") + .repo() + .step("define the label", defineLabel(triage)) + + .open(newLabel) + .fill(testid("name"), triage) + .click(testid("create")) + .refused(labelDefinitionId, 409) + .sees(refusal) + .at(newLabel); + +flow("cancelling a new label writes nothing") + .actor("owner") + .repo() + + .open(newLabel) + .fill(testid("name"), triage) + .click(testid("back")) + .at(settings) + .sendsNothing(labelDefinitionId) + .sees(testid("settingsEmpty")) + .reload() + .sees(testid("settingsEmpty")); + +flow("cancelling a label delete keeps the label") + .actor("owner") + .repo() + .step("define the label", defineLabel(triage)) + + .open(settings) + .click(deleteLabel) + .click(within(testid("labelDelete"), testid("close"))) + .sendsNothing(labelDefinitionId) + .sees(deleteLabel) + .reload() + .sees(deleteLabel) + .step("the knot still has the label", async (ctx) => { + const stored = await knotRecordFor( + `at://${ctx.repo.did}/${labelDefinitionId.nsid}/${triage}`, + "read" + ); + expect(stored.value.name).toBe(triage); + }); diff --git a/e2e/specs/markup.spec.ts b/e2e/specs/markup.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..732f9aa35fbb8f1648326259801dd1f77cefd472 --- /dev/null +++ b/e2e/specs/markup.spec.ts @@ -0,0 +1,106 @@ +import { config } from "../harness/config"; +import { + bobbin, + commentId, + flow, + hydrant, + knot, + page, + pds, + record, + subscriptionId, + target, + testid, + through, + ticketId, + within +} from "../harness/flow"; +import { get, until } from "../harness/net"; + +const writing = within(testid("commentEditor", { mode: "new" }), testid("body")); +const sending = within(testid("commentEditor", { mode: "new" }), testid("submit")); + +const inComment = (describe: string, css: string) => + target(describe, (tab) => tab.locator(`[data-testid="commentBody"] ${css}`).first()); +const drawn = inComment("the diagram", "[data-mermaid] svg"); +const coloured = inComment("a highlighted token", 'pre span[style*="--diffs"]'); + +const markdown = [ + "a diagram and some code", + "", + "```mermaid", + "flowchart LR", + " A --> B", + "```", + "", + "```ts", + "const answer: number = 42;", + "```" +].join("\n"); + +flow("a comment's code and diagram are drawn by the server and stay after a reload") + .actor("owner") + .repo() + .ticket(ticketId, { title: "show me the markup" }) + .open(page.issue(ticketId)) + .fill(writing, markdown) + .click(sending) + .trace(commentId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .sees(drawn) + .sees(coloured) + .step("the page's own html already holds both, no script needed", async (ctx) => { + const path = page.issue(ticketId).path(ctx); + await until( + `the html of ${path} should hold the diagram and the code`, + 30_000, + async () => { + const { status, body } = await get(`${config.webUrl}${path}`); + const html = String(body); + const diagram = html.includes("data-mermaid") && html.includes(" testid("handlePick", { kind }); +const chip = (kind: string, count: number) => testid("reactionChip", { kind, count }); + +flow("different emojis on one issue are counted apart across two people") + .actor("owner") + .repo() + .ticket(ticketId, { title: "react a lot" }) + .open(page.issue(ticketId)) + .click(target("the issue's picker", (tab) => tab.locator('[data-testid="react"]').first())) + .click(pick("🎉")) + .trace(first, through(knot, hydrant, bobbin)) + .sees(chip("🎉", 1)) + .click(target("the issue's picker", (tab) => tab.locator('[data-testid="react"]').first())) + .click(pick("🚀")) + .trace(second, through(knot, hydrant, bobbin)) + .sees(chip("🎉", 1)) + .sees(chip("🚀", 1)) + + .actor("fan") + .open(page.issue(ticketId)) + .sees(chip("🎉", 1)) + .sees(chip("🚀", 1)) + .click(testid("reactionChip", { kind: "🎉" })) + .trace(third, through(knot, hydrant, bobbin)) + .sees(chip("🎉", 2)) + .sees(chip("🚀", 1)) + .reload() + .sees(chip("🎉", 2)) + .sees(chip("🚀", 1)); diff --git a/e2e/specs/merges.spec.ts b/e2e/specs/merges.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..282e70f62150ac6bf0c24a7fc590067978a3f41d --- /dev/null +++ b/e2e/specs/merges.spec.ts @@ -0,0 +1,338 @@ +import { mirrorMerge, pulls } from "../harness/api"; +import { + bobbin, + button, + commitRow, + type Ctx, + flow, + type Flow, + knot, + link, + mergeButton, + not, + page, + patchId, + pullId, + refusedTicketId, + rkeyOf, + staleEdit, + target, + testid, + text, + through, + unique, + where, + within +} from "../harness/flow"; +import { + authorLink, + base, + bobbinNamesOpener, + branches, + fanPulled, + forkedAhead, + mergedTip, + merges, + onBranches, + opensForkPull, + pipelineFor, + reloadsUntil, + rowOf, + targetIsAt, + targetTookTheFeature, + theFan, + theOwner, + upstreamPull +} from "../harness/shared"; + +const title = unique("a pull to merge"); +onBranches("a merged pull moves from the open list to the merged list") + .pull(pullId, patchId, { ...branches, title }) + .open(page.pull(pullId)) + .use(merges) + .open(page.pulls) + .sees(not(link(title))) + .open(where("the merged pulls", (ctx) => `${ctx.repo.path}/pulls?state=completed`)) + .sees(link(title)); + +const fanTitle = unique("a pull the owner merges"); +fanPulled("a pull the owner merged still belongs to whoever opened it", fanTitle) + .actor("owner") + .open(page.pull(pullId)) + .use(merges) + .step( + "bobbin names the fan as the opener everywhere, and the owner as no one's", + bobbinNamesOpener(pullId, "pull", theFan, theOwner) + ) + .open(where("the merged pulls", (ctx) => `${ctx.repo.path}/pulls?state=completed`)) + .sees(within(rowOf(pullId), authorLink(theFan))); + +onBranches("the merged branch can be deleted after a merge") + .pull(pullId, patchId, branches) + .open(page.pull(pullId)) + .use(merges) + .sees(testid("openDeleteConfirm")); + +const fanPullsFrom = (name: string, source: string, into: string, base: string) => + onBranches(name) + .push({ "e2e/precious.txt": "precious\n" }, { branch: "precious", base: "main" }) + .mirrored() + .actor("fan") + .step(`the fan opens a pull from ${source} into ${into}`, async (ctx) => { + const opened = await pulls.open( + ctx.actor, + ctx.repo.did, + unique(`a pull from ${source}`), + into, + { base: ctx.repo.heads[base], head: ctx.repo.heads[source] }, + source + ); + ctx.adopt(pullId, opened.ticket); + ctx.adopt(patchId, opened.patch); + await bobbin.indexed(opened.ticket); + await bobbin.indexed(opened.patch); + }) + .actor("owner"); + +const offersNoDelete = (flow: Flow) => + flow + .open(page.pull(pullId)) + .changeState(pullId, "closed", testid("requestClose"), testid("changeState")) + .reload() + .sees(not(button("Delete branch"))); + +fanPullsFrom( + "a closed pull doesn't offer to delete a branch that moved past it", + "precious", + "main", + "main" +) + .push({ "e2e/later.txt": "later\n" }, { branch: "precious" }) + .mirrored() + .use(offersNoDelete); + +fanPullsFrom( + "a closed pull doesn't offer to delete the default branch", + "main", + "feature", + "main" +).use(offersNoDelete); + +onBranches("merging a pull request", { ci: true }) + .pull(pullId, patchId, branches) + .open(page.pull(pullId)) + .use(merges) + .knotStored(pullId, "completed") + .step("the target branch took the feature's commit", targetTookTheFeature()) + .checkpoint("the merge moved the target branch") + .step("spindle has a pipeline for the push the merge made", pipelineFor("push", mergedTip)); + +// the patch names the merge base, so if the target moved after the feature was cut, open from the fork point +const marks = new Map(); +const mark = (name: string) => async (ctx: Ctx) => { + marks.set(`${ctx.repo.did} ${name}`, ctx.pushed.sha); +}; +const marked = (ctx: Ctx, name: string) => { + const sha = marks.get(`${ctx.repo.did} ${name}`); + if (!sha) throw new Error(`no commit was marked ${name} for ${ctx.repo.did}`); + return sha; +}; +const markForkPoint = mark("fork point"); +const openFromForkPoint = (title?: string) => async (ctx: Ctx) => { + const opened = await pulls.open( + ctx.actor, + ctx.repo.did, + title ?? unique("a pull"), + branches.into, + { base: marked(ctx, "fork point"), head: ctx.repo.heads[branches.from] }, + branches.from + ); + ctx.adopt(pullId, opened.ticket); + ctx.adopt(patchId, opened.patch); + await bobbin.indexed(opened.ticket); + await bobbin.indexed(opened.patch); +}; + +// the pull page checks the target's tip through bobbin, so the mirror needs it first +const mirroredTip = async (ctx: Ctx) => { + await bobbin.hasBranch(ctx.repo.did, branches.into, ctx.repo.heads[branches.into]); +}; + +const pullAt = (ctx: Ctx, tail: string) => `${ctx.repo.path}/pulls/${rkeyOf(ctx, pullId)}${tail}`; + +const forkTitle = unique("a fix from my fork"); + +forkedAhead("opening a pull request from a fork to the upstream") + .use(opensForkPull(forkTitle)) + + .actor("owner") + .open(upstreamPull) + .sees(text(forkTitle)) + .step("the diff shows up", reloadsUntil(testid("diffFileName", { path: "e2e/fork.txt" }))) + .sees(testid("diffFileName", { path: "e2e/fork.txt" })); + +const mergeDisabled = target("the disabled merge button", (tab) => + tab + .locator('[data-testid="mergeChoice"] [data-testid="primaryAction"]:disabled') + .filter({ visible: true }) +); +const conflictsSummary = testid("conflictsSummary"); +const conflictedFile = testid("conflictedFile", { filename: "e2e/conflict.txt" }); + +const rejectMerge = (ctx: Ctx) => + mirrorMerge( + ctx.actor, + ctx.repo.did, + branches.into, + ctx.repo.heads[branches.into], + ctx.repo.heads[branches.from] + ); + +flow("a pull whose branches conflict can't be merged and the page says which file") + .actor("owner") + .repo() + .push({ "e2e/conflict.txt": "base\n" }) + .step("mark the fork point", markForkPoint) + .push( + { "e2e/conflict.txt": "the feature's line\n" }, + { branch: branches.from, base: branches.into } + ) + .push({ "e2e/conflict.txt": "the target's line\n" }) + .step("the mirror has the target's new tip", mirroredTip) + .mirrored() + .step("open the pull", openFromForkPoint()) + + .open(page.pull(pullId)) + .sees(conflictsSummary) + .sees(mergeDisabled) + .click(conflictsSummary) + .sees(conflictedFile) + .rejects("merging the conflicting pull", rejectMerge, 409) + .knotStored(pullId, "open") + .step("the target branch didn't move", targetIsAt(branches.into)); + +flow("a pull merges when the target gained another file in the same folder") + .actor("owner") + .repo() + .push(base) + .step("mark the fork point", markForkPoint) + .push({ "e2e/feature.txt": "feature\n" }, { branch: branches.from, base: branches.into }) + .push({ "e2e/target.txt": "target\n" }) + .step("the mirror has the target's new tip", mirroredTip) + .mirrored() + .step("open the pull", openFromForkPoint()) + + .open(page.pull(pullId)) + .sees(not(mergeDisabled)) + .use(merges) + .knotStored(pullId, "completed"); + +const short = (sha: string) => sha.slice(0, 8); + +const changedFiles = (count: number) => testid("changedFilesCount", { count }); +const inDiff = (file: string) => testid("diffFileName", { path: `e2e/${file}` }); + +// two buttons match here, the graph dot and the label, so take the first +const showAllChanges = target("the all changes row", (tab, ctx) => + testid("selectAll").locate(tab, ctx).first() +); + +const twoCommits = (name: string) => + flow(name) + .actor("owner") + .repo() + .push(base) + .step("mark the fork point", markForkPoint) + .push({ "e2e/first.txt": "first\n" }, { branch: branches.from, base: branches.into }) + .step("mark the first commit", mark("first")) + .push({ "e2e/second.txt": "second\n" }, { branch: branches.from }) + .mirrored() + .step("open the pull", openFromForkPoint()); + +twoCommits("narrowing a pull's diff to one commit and back to all of them") + .open(page.pull(pullId)) + .sees(changedFiles(2)) + .click(commitRow("first")) + .at( + where("the newest commit's range", (ctx) => + pullAt( + ctx, + `/0/${short(marked(ctx, "first"))}..${short(ctx.repo.heads[branches.from])}` + ) + ) + ) + .sees(changedFiles(1)) + .sees(inDiff("second.txt")) + .sees(not(inDiff("first.txt"))) + .reload() + .sees(changedFiles(1)) + .sees(inDiff("second.txt")) + .sees(not(inDiff("first.txt"))) + .click(showAllChanges) + .at(where("the whole version", (ctx) => pullAt(ctx, "/0"))) + .sees(changedFiles(2)); + +twoCommits("the oldest commit's button narrows the diff to the first file") + .open(page.pull(pullId)) + .click(commitRow("last")) + .sees(changedFiles(1)) + .sees(inDiff("first.txt")) + .sees(not(inDiff("second.txt"))); + +twoCommits("a range that names no commit in the url goes back to the whole version") + .open( + where("a range naming no commit", (ctx) => + pullAt(ctx, `/0/${"0".repeat(8)}..${"1".repeat(8)}`) + ) + ) + .sees(changedFiles(2)) + .at(where("the version without the range", (ctx) => pullAt(ctx, "/0"))); + +const mergedInAnotherWindow = (name: string) => + onBranches(name) + .pull(pullId, patchId, branches) + .open(page.pull(pullId)) + .sees(mergeButton) + .window("second") + .open(page.pull(pullId)) + .use(merges) + .window() + .click(mergeButton) + .sees(staleEdit) + .sendsNothing(refusedTicketId); + +const closedInAnotherWindow = (name: string) => + onBranches(name) + .pull(pullId, patchId, branches) + .open(page.pull(pullId)) + .sees(mergeButton) + .window("second") + .open(page.pull(pullId)) + .changeState(pullId, "closed", testid("requestClose"), testid("changeState")) + .window() + .click(mergeButton) + .sees(staleEdit) + .sendsNothing(refusedTicketId); + +mergedInAnotherWindow("a second merge from a stale window is refused and the page says why") + .knotStored(pullId, "completed") + .step("the target branch moved once", targetTookTheFeature()); + +mergedInAnotherWindow("a stale window stops offering merge once the knot said it was merged") + .sees(not(mergeButton)) + .reload() + .sees(testid("ticketStatePill", { state: "completed" })); + +closedInAnotherWindow( + "merging a pull that was closed in another window is refused and the page says why" +) + .knotStored(pullId, "closed") + .step("the target branch didn't move", targetIsAt(branches.into)) + .reload() + .sees(testid("changeState")) + .sees(not(mergeButton)); + +closedInAnotherWindow("a stale window shows the pull as closed once the knot refused the merge") + .sees(not(mergeButton)) + .reload() + .sees(testid("changeState")); diff --git a/e2e/specs/opengraph.spec.ts b/e2e/specs/opengraph.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..4fc24c113c19d6853456ee2e2ab00b3a57a01c03 --- /dev/null +++ b/e2e/specs/opengraph.spec.ts @@ -0,0 +1,271 @@ +import { expect } from "@playwright/test"; +import { fetch } from "undici"; + +import { config, seeded } from "../harness/config"; +import { + bobbin, + type Ctx, + flow, + hydrant, + page, + patchId, + pds, + pullId, + starId, + testid, + through, + ticketId, + type Where +} from "../harness/flow"; +import { dispatcher, get, stamp, until } from "../harness/net"; +import { branches, onBranches, remember, theOwner } from "../harness/shared"; + +type Card = Record; + +interface Preview { + status: number; + tags: Record; + card?: Card; +} + +const entities: Record = { + "&": "&", + """: '"', + "'": "'", + "<": "<", + ">": ">" +}; +const unescaped = (value: string) => value.replace(/&(amp|quot|#39|lt|gt);/g, (it) => entities[it]); + +// what a crawler reads: the server html, no script runs +const previewOf = async (path: string): Promise => { + const { status, body } = await get(`${config.webUrl}${path}`); + const tags: Record = {}; + for (const [, key, value] of String(body).matchAll( + / { + expect(preview.tags[key], `the page should say ${key} exactly once`).toHaveLength(1); + return preview.tags[key][0]; +}; + +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +// the header is enough to tell a real card from an error page, the payload says what is on it +const rendered = async (preview: Preview) => { + const url = one(preview, "og:image"); + expect(one(preview, "twitter:image")).toBe(url); + expect(one(preview, "twitter:card")).toBe("summary_large_image"); + const response = await fetch(url, { dispatcher, signal: AbortSignal.timeout(30_000) }); + const bytes = Buffer.from(await response.arrayBuffer()); + expect(response.status, `ogre answered ${bytes.toString().slice(0, 300)}`).toBe(200); + expect(response.headers.get("content-type")).toBe("image/png"); + expect(bytes.subarray(0, 8).equals(PNG), "the card should be a png").toBe(true); + const size = [bytes.readUInt32BE(16), bytes.readUInt32BE(20)]; + expect(size).toEqual([1200, 630]); + expect([one(preview, "og:image:width"), one(preview, "og:image:height")]).toEqual( + size.map(String) + ); + return bytes; +}; + +const cards = new WeakMap(); +const renders = (ctx: Ctx) => cards.get(ctx) ?? []; + +// a cold page can miss stats and languages, ssr only waits a moment for them. keep asking until they show up +const previews = + ( + at: Where, + what: string, + card: (ctx: Ctx) => Card, + more?: (preview: Preview, ctx: Ctx) => void + ) => + async (ctx: Ctx) => { + const path = at.path(ctx); + const wanted = card(ctx); + const preview = await until(`the preview of ${path} should ${what}`, 30_000, async () => { + const seen = await previewOf(path); + const matches = seen.status === 200 && !!seen.card && isLike(seen.card, wanted); + return [ + matches, + `it answered ${seen.status} with ${JSON.stringify(seen.card ?? seen.tags["og:image"])}`, + seen + ]; + }); + // the web is plain http behind caddy in this stack, the scheme isn't worth comparing, just the host + expect(new URL(String(preview.card?.avatarUrl)).host).toBe(new URL(config.webUrl).host); + more?.(preview, ctx); + const bytes = await rendered(preview); + expect( + renders(ctx).some((earlier) => earlier.equals(bytes)), + "the card should look different from the one before" + ).toBe(false); + cards.set(ctx, [...renders(ctx), bytes]); + }; + +const isLike = (seen: Card, wanted: Card): boolean => + Object.entries(wanted).every(([key, value]) => + typeof value === "function" + ? value(seen[key]) + : value === undefined + ? seen[key] === undefined + : seen[key] === value + ); + +const repoCard = (more: Card) => (ctx: Ctx) => ({ + type: "repository", + repoName: ctx.repo.name, + ownerHandle: theOwner(ctx).handle, + ...more +}); + +const titled = (title: (ctx: Ctx) => string) => (preview: Preview, ctx: Ctx) => { + expect(one(preview, "og:title")).toBe(title(ctx)); + expect(one(preview, "twitter:title")).toBe(title(ctx)); +}; + +const GO = "#00ADD8"; +const PYTHON = "#3572A5"; +type Language = { color: string; percentage: number }; +const goOverPython = (languages?: Language[]) => { + const share = (color: string) => languages?.find((l) => l.color === color)?.percentage ?? 0; + return languages?.length === 2 && share(GO) > share(PYTHON) && share(PYTHON) > 0; +}; +const code = { + "cmd/main.go": `package main\n\nimport "fmt"\n\nfunc main() {\n${'\tfmt.Println("hello from go")\n'.repeat(30)}}\n`, + "tool.py": 'print("hello from python")\n' +}; + +flow("a repo's link preview shows its languages once code is pushed") + .actor("owner") + .step("remember the owner", remember("owner")) + .repo() + .step( + "the preview has no languages yet", + previews( + page.repo, + "show an empty repo", + repoCard({ stars: 0, issues: 0, pulls: 0, languages: undefined }), + titled((ctx) => `${ctx.repo.name} by ${ctx.actor.handle} · Tangled`) + ) + ) + .push(code) + .mirrored() + .step( + "the preview shows go and python, go the larger", + previews(page.repo, "show go and python", repoCard({ languages: goOverPython })) + ); + +const starsTheRepo = (ctx: Ctx) => ({ + record: { subject: { $type: "org.tangled.feed.star#repo", identity: ctx.repo.did } } +}); + +onBranches("a repo's link preview counts its stars, open issues and open pulls") + .step("remember the owner", remember("owner")) + .step( + "the preview counts nothing", + previews(page.repo, "count nothing", repoCard({ stars: 0, issues: 0, pulls: 0 })) + ) + .ticket(ticketId) + .pull(pullId, patchId, branches) + .step( + "the preview counts the issue and the pull", + previews( + page.repo, + "count one issue and one pull", + repoCard({ stars: 0, issues: 1, pulls: 1 }) + ) + ) + .actor("fan") + .open(page.repo) + .click(testid("star", { starred: "false" })) + .sends(starId, pds, starsTheRepo, "create") + .trace(starId, through(pds, hydrant, bobbin)) + .step( + "the preview counts the star", + previews(page.repo, "count one star", repoCard({ stars: 1, issues: 1, pulls: 1 })) + ); + +const issueTitle = "the card should say this"; +const pullTitle = "and this one too"; + +onBranches("an issue's and a pull's link previews name them") + .ticket(ticketId, { title: issueTitle }) + .pull(pullId, patchId, { ...branches, title: pullTitle }) + .step( + "the issue's preview names it", + previews( + page.issue(ticketId), + "name the issue", + (ctx) => ({ + type: "issue", + title: issueTitle, + status: "open", + authorHandle: ctx.actor.handle, + repoName: ctx.repo.name, + ownerHandle: ctx.actor.handle + }), + (preview, ctx) => { + titled((c) => `${issueTitle} · ${c.repo.name} · Tangled`)(preview, ctx); + expect(one(preview, "og:type")).toBe("article"); + } + ) + ) + .step( + "the pull's preview names it", + previews( + page.pull(pullId), + "name the pull", + (ctx) => ({ + type: "pullRequest", + title: pullTitle, + status: "open", + authorHandle: ctx.actor.handle, + repoName: ctx.repo.name, + ownerHandle: ctx.actor.handle + }), + (preview) => expect(one(preview, "og:type")).toBe("article") + ) + ); + +flow("a profile's link preview counts its repos") + .actor("owner") + .repo() + .step( + "the profile's preview counts the repo", + previews( + page.profile, + "count one repo", + (ctx) => ({ + type: "profile", + handle: ctx.actor.handle, + repos: 1, + stars: 0, + followers: 0 + }), + (preview, ctx) => { + titled((c) => `${c.actor.handle} (@${c.actor.handle}) · Tangled`)(preview, ctx); + expect(one(preview, "og:type")).toBe("profile"); + } + ) + ); + +flow("a missing repo's link preview names nothing") + .actor(seeded.alice) + .step("the preview of a repo that isn't there has no card", async (ctx) => { + const name = `e2e-missing-${stamp()}`; + const preview = await previewOf(`/${ctx.actor.handle}/${name}`); + expect(preview.status).toBe(404); + expect(preview.card).toBeUndefined(); + expect(JSON.stringify(preview.tags)).not.toContain(name); + }); diff --git a/e2e/specs/permissions.spec.ts b/e2e/specs/permissions.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..a3e0a23a11df097b84008261e068069e8bc7f5bc --- /dev/null +++ b/e2e/specs/permissions.spec.ts @@ -0,0 +1,200 @@ +import { knotReply, ticketRecord, tickets } from "../harness/api"; +import { seeded } from "../harness/config"; +import { + bobbin, + component, + type Ctx, + declarationId, + flow, + hydrant, + knot, + link, + loginThen, + moreActions, + not, + page, + pds, + refusal, + refusedTicketId, + subscriptionId, + testid, + text, + through, + ticketId, + within +} from "../harness/flow"; +import { stamp } from "../harness/net"; +import { deletesIssue, forgetTheLogin } from "../harness/shared"; + +const shownByButton = testid("stateChangeError"); + +const title = "the build is broken"; +const fixedTitle = "a typo in the title"; + +flow("an outsider keeps erase but loses edit and close when the policy tightens") + .actor("owner") + .repo() + .actor(seeded.charlie) + .open(page.newIssue) + .fill(testid("title"), title) + .click(testid("submit")) + .trace(ticketId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + + .actor("owner") + .policy("collaborators") + + .actor(seeded.charlie) + .open(page.issue(ticketId)) + .click(testid("requestStateChange")) + .refused(refusedTicketId, 403) + .sees(shownByButton) + .knotStored(ticketId, "open") + .click(moreActions) + .click(testid("setEditing")) + .fill(testid("title"), fixedTitle) + .click(within(component("TicketForm"), testid("submit"))) + .refused(refusedTicketId, 403) + .sees(refusal) + .knotStored(ticketId, "open") + .click(testid("cancel")) + + .use(deletesIssue) + .at(page.issues) + .sees(not(link(title))); + +flow( + "the issue page stops offering close and edit to an author the policy no longer lets change it" +) + .actor("owner") + .repo() + .actor(seeded.charlie) + .ticket(ticketId, { title }) + .actor("owner") + .policy("collaborators") + .actor(seeded.charlie) + .open(page.issue(ticketId)) + .sees(not(testid("requestStateChange"))); + +flow("a members only repo takes a knot member's issue and refuses a stranger's") + .actor("owner") + .repo() + .policy("members") + + .actor(seeded.bob) + .open(page.newIssue) + .fill(testid("title"), title) + .click(testid("submit")) + .trace(ticketId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .at(page.issues) + .sees(link(title)) + + .actor(seeded.charlie) + .open(page.newIssue) + .fill(testid("title"), "let me in") + .click(testid("submit")) + .refused(refusedTicketId, 403) + .sees(refusal) + .at(page.newIssue) + .open(page.issues) + .sees(link(title)) + .sees(not(link("let me in"))); + +const alicesClose = (ctx: Ctx) => tickets.setState(ctx.actor, ctx.get(ticketId), "closed"); + +flow("a knot admin closes a stranger's issue through the api") + .actor("owner") + .repo() + .actor(seeded.charlie) + .ticket(ticketId, { title }) + + .actor(seeded.alice) + .open(page.issue(ticketId)) + .sees(text(title)) + .sees(not(testid("requestStateChange"))) + .step("alice closes it through the api", alicesClose) + .learn(ticketId) + .follow(ticketId, through(knot, hydrant, bobbin)) + .knotStored(ticketId, "closed"); + +flow("a closed issue still has its author after a knot admin closes it") + .actor("owner") + .repo() + .actor(seeded.charlie) + .ticket(ticketId, { title }) + .actor(seeded.alice) + .step("alice closes it through the api", alicesClose) + .learn(ticketId) + .follow(ticketId, through(knot, hydrant, bobbin)) + .actor(seeded.charlie) + .open(page.issue(ticketId)) + .sees(testid("requestStateChange")); + +flow("a knot admin can close an issue on a collaborators only repo but not open one") + .actor("owner") + .repo() + .actor(seeded.charlie) + .ticket(ticketId, { title }) + .actor("owner") + .policy("collaborators") + + .actor(seeded.alice) + .rejects( + "alice opening an issue", + (ctx) => + knotReply(ctx.actor, ctx.repo.did, "com.atproto.repo.createRecord", { + collection: ticketId.nsid, + record: ticketRecord("let the admin in") + }), + 403 + ) + .step("alice closes it through the api", alicesClose) + .learn(ticketId) + .follow(ticketId, through(knot, hydrant, bobbin)) + .knotStored(ticketId, "closed"); + +flow("a signed out visitor is sent to log in by the pages that need an account") + .actor("owner") + .repo() + .actor(seeded.charlie) + .open(page.repo) + .step("signs out", forgetTheLogin) + .open(page.newIssue) + .at(loginThen((ctx) => `${ctx.repo.path}/issues/new`)) + .open(page.settings) + .at(loginThen((ctx) => `${ctx.repo.path}/settings`)) + .open(page.newRepo) + .at(loginThen(() => "/repo/new")); + +flow("a stranger opening a repo's settings is told they need push access") + .actor("owner") + .repo() + .open(page.settings) + .sees(testid("settingsSectionTitle", { name: "danger" })) + .actor(seeded.charlie) + .open(page.settings) + .sees(testid("errorMessage")) + .sees(not(testid("settingsSectionTitle", { name: "danger" }))); + +flow("a stranger can't create a repo on the closed knot and the form says why") + .actor(seeded.charlie) + .open(page.newRepo) + .fill(testid("name"), () => `e2e-denied-${stamp()}`) + .click(testid("submit")) + .sees(refusal) + .sendsNothing(declarationId) + .at(page.newRepo); + +flow("a stranger sees no close, edit or delete on someone else's issue") + .actor("owner") + .repo() + .ticket(ticketId, { title }) + .actor(seeded.charlie) + .open(page.issue(ticketId)) + .sees(text(title)) + .sees(not(testid("requestStateChange"))) + .click(moreActions) + .sees(testid("navigatorClipboardWriteText")) + .sees(not(testid("setEditing"))) + .sees(not(testid("openDeleteConfirm"))); diff --git a/e2e/specs/pipelines.spec.ts b/e2e/specs/pipelines.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..27d91d947e2cc44f848f69d83c7a1f70da6ec9fa --- /dev/null +++ b/e2e/specs/pipelines.spec.ts @@ -0,0 +1,475 @@ +import { expect, test } from "@playwright/test"; + +import { spindleGet, spindlePost } from "../harness/api"; +import { config } from "../harness/config"; +import { + type Ctx, + type Target, + bobbin, + flow, + hydrant, + knot, + manifestId, + not, + saveButton, + settingsAt, + through, + target, + testid, + text, + where, + within +} from "../harness/flow"; +import { get, stamp, until, xrpc } from "../harness/net"; +import { saveFor, spindle } from "../harness/services"; +import { WEBHOOKS } from "../harness/webhooks"; +import { + acceptConfirm, + finished, + manifestOf, + pipelineFor, + saveFinishes, + workflow, + workflowPath +} from "../harness/shared"; + +flow("pushing a workflow makes a pipeline for the push") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ [workflowPath]: workflow }) + .step( + "spindle has a pipeline for the push", + pipelineFor("push", (ctx) => ctx.pushed.sha) + ); + +const runs = new WeakMap>>(); +const succeeds = async (ctx: Ctx) => { + const run = await spindle.finishedRun(ctx.repo.did, ctx.pushed.sha); + runs.set(ctx, run); + expect(run.workflow.status, JSON.stringify(run.workflow)).toBe("success"); +}; +const finishedWorkflow = where("the finished workflow", (ctx) => { + const { pipeline, workflow } = runs.get(ctx)!; + return `${ctx.repo.path}/pipelines/${pipeline}/workflow/${workflow.id}`; +}); +// the dummy engine prints each step as "$ " +const loggedCommand = text('$ echo "build ok"'); + +// without s3 the archive is the log file itself, so removing it loses the log +flow("a finished workflow's log still shows once spindle has archived it") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ [workflowPath]: workflow }) + .step("spindle runs the workflow to success", succeeds) + .open(finishedWorkflow) + .sees(loggedCommand) + .reload() + .sees(loggedCommand); + +const retryButton = testid("retryPipeline"); +// the retry lands on the new run. spindle lists it beside the first for the same commit +const ranAgain = (ctx: Ctx) => + until("spindle should list the retried run", 30_000, async () => { + const { status, body } = await get( + xrpc(config.spindleUrl, "org.tangled.ci.queryPipelines", { + repo: ctx.repo.did, + commits: ctx.pushed.sha + }) + ); + const ids: string[] = (body.pipelines ?? []).map((p: { id: string }) => p.id); + const first = runs.get(ctx)!.pipeline; + const landed = ids.find( + (id) => id !== first && ctx.page.url().includes(`/pipelines/${id}/`) + ); + return [ + ids.length === 2 && Boolean(landed), + `it answered ${status} with ${ids.join(", ")}` + ]; + }); + +flow("retrying a finished pipeline runs it again and opens the new run") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ [workflowPath]: workflow }) + .step("spindle runs the workflow to success", succeeds) + .open(finishedWorkflow) + .click(retryButton) + .step("the page moves to a second run of the commit", ranAgain) + .sees(loggedCommand); + +flow("someone without push access can't retry a pipeline") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ [workflowPath]: workflow }) + .step("spindle runs the workflow to success", succeeds) + .actor("author") + .open(finishedWorkflow) + .sees(loggedCommand) + .sees(not(retryButton)); + +const pipelineOf = (repo: string, pipeline: string) => + get(xrpc(config.spindleUrl, "org.tangled.ci.getPipeline", { repo, pipeline })); +const logsOf = (repo: string, pipeline: string) => + get(xrpc(config.spindleUrl, "org.tangled.ci.subscribePipelineLogs", { repo, pipeline })); +const firstRepo = new WeakMap(); + +flow("a pipeline isn't found under a repo it doesn't belong to") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ [workflowPath]: workflow }) + .step("spindle runs the workflow to success", succeeds) + .step("spindle answers for the pipeline under its own repo", async (ctx) => { + firstRepo.set(ctx, ctx.repo.did); + const reply = await pipelineOf(ctx.repo.did, runs.get(ctx)!.pipeline); + expect(reply.status, JSON.stringify(reply.body)).toBe(200); + }) + .repo({ spindle: config.spindleDid }) + .rejects( + "reading the first repo's pipeline under the second", + (ctx) => pipelineOf(ctx.repo.did, runs.get(ctx)!.pipeline), + 404 + ) + .rejects( + "following the first repo's pipeline logs under the second", + (ctx) => logsOf(ctx.repo.did, runs.get(ctx)!.pipeline), + 404 + ) + .rejects( + "cancelling the first repo's pipeline under the second", + (ctx) => + spindlePost(ctx.actor, "org.tangled.ci.cancelPipeline", { + repo: ctx.repo.did, + pipeline: runs.get(ctx)!.pipeline + }), + 404 + ) + .step("the pipeline is still the first repo's", async (ctx) => { + const reply = await pipelineOf(firstRepo.get(ctx)!, runs.get(ctx)!.pipeline); + expect(reply.status, JSON.stringify(reply.body)).toBe(200); + }); + +const cron = `engine: dummy +when: + - event: schedule + schedule: + - cron: "* * * * *" +steps: + - name: tick + command: echo tick +`; +interface Listed { + total: number; + pipelines: { id: string; commit: string; trigger?: { $type?: string; scheduledAt?: string } }[]; +} +const scheduledRun = new WeakMap(); + +// localinfra sets a 1m minimum interval, so one fires within two minutes +flow("a pipeline a cron schedule started is listed and readable") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ ".tangled/workflows/tick.yml": cron }) + .step("leave room for the schedule to come round", async () => { + test.setTimeout(test.info().timeout + 150_000); + }) + .step("spindle lists a scheduled pipeline for the head", async (ctx) => { + const found = await until("spindle should list a scheduled pipeline", 150_000, async () => { + const { status, body } = await get( + xrpc(config.spindleUrl, "org.tangled.ci.queryPipelines", { + repo: ctx.repo.did, + triggers: "org.tangled.ci.trigger.schedule" + }) + ); + const listed = body as Listed; + const run = (listed.pipelines ?? []).find((p) => p.commit === ctx.pushed.sha); + return [ + Boolean(run), + `it answered ${status}, counted ${listed.total} and listed ${listed.pipelines?.length}`, + run + ]; + }); + expect(found.trigger?.scheduledAt).toMatch(/^\d{4}-\d\d-\d\dT\d\d:\d\d:00Z$/); + scheduledRun.set(ctx, found.id); + }) + .step("spindle reads the scheduled pipeline back", async (ctx) => { + const reply = await get( + xrpc(config.spindleUrl, "org.tangled.ci.getPipeline", { + repo: ctx.repo.did, + pipeline: scheduledRun.get(ctx)! + }) + ); + expect(reply.status, JSON.stringify(reply.body)).toBe(200); + expect(reply.body.trigger.$type).toBe("org.tangled.ci.trigger.schedule"); + }); + +const hooks = settingsAt("hooks"); +const newHook = settingsAt("hooks/new"); +const pipelines = settingsAt("pipelines"); +const newSecret = settingsAt("pipelines/secrets/new"); + +const landsOn = (what: string, at: (ctx: Ctx) => string) => (ctx: Ctx) => + finished( + ctx, + what, + expect(ctx.page).toHaveURL(`${config.webUrl}${at(ctx)}`, { timeout: 8000 }) + ); + +const SECRETS = "org.tangled.secret"; + +let hookUrl = ""; +let hookId = ""; +const pickHook = async () => { + hookUrl = `https://example.com/e2e/${stamp()}`; +}; +const deleteHook = testid("confirmDelete"); +const newHookLink = testid("newWebhook"); +// hookUrl gets picked per run, so the selector can't be built at load time +const lazily = (label: string, make: () => Target): Target => + target(label, (page, ctx) => make().locate(page, ctx)); +const hookRow = lazily("webhook row", () => testid("webhookRow", { url: hookUrl })); +const hookSwitch = lazily("webhook switch", () => testid("webhookSwitch", { url: hookUrl })); + +interface Hook { + id: string; + url: string; + events: string[]; + active: boolean; +} +const listHooks = (ctx: Ctx) => + spindleGet(ctx.actor, `${WEBHOOKS}.listWebhooks`, { repo: ctx.repo.did }); +const hooksOnSpindle = async (ctx: Ctx) => { + const reply = await listHooks(ctx); + expect(reply.status, `spindle answered ${JSON.stringify(reply.body)}`).toBe(200); + return (reply.body.webhooks ?? []) as Hook[]; +}; +// spindle takes a while to learn a new repo, so the list is polled +const spindleHooks = (what: string, ok: (hooks: Hook[]) => boolean) => async (ctx: Ctx) => { + await until(`spindle should list ${what}`, 10_000, async () => { + const reply = await listHooks(ctx); + return [ + reply.status === 200 && ok(reply.body?.webhooks ?? []), + `it answered ${reply.status} ${JSON.stringify(reply.body)}` + ]; + }); +}; +const makeHook = async (ctx: Ctx) => { + const made = await spindlePost(ctx.actor, `${WEBHOOKS}.createWebhook`, { + repo: ctx.repo.did, + url: hookUrl, + events: ["push"], + active: true + }); + expect(made.status, `spindle refused the webhook: ${JSON.stringify(made.body)}`).toBe(200); + hookId = made.body.id; +}; +const dropHook = async (ctx: Ctx) => { + await spindlePost(ctx.actor, `${WEBHOOKS}.deleteWebhook`, { + repo: ctx.repo.did, + webhook: hookId + }); +}; + +flow("adding a webhook lists it under hooks") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .step("pick the payload url", pickHook) + + .open(hooks) + .sees(newHookLink) + .click(newHookLink) + .at(newHook) + .fill(testid("draftUrl"), () => hookUrl) + .click(testid("add")) + .step( + "the form finishes on the hooks list", + landsOn("the webhook was not added", (ctx) => `${ctx.repo.path}/settings/hooks`) + ) + .sees(hookRow) + .step( + "spindle lists it for push events, switched on", + spindleHooks("the new webhook", (listed) => + listed.some((one) => one.url === hookUrl && one.active && one.events.join() === "push") + ) + ) + .reload() + .sees(hookRow) + .step("take the webhook away", async (ctx) => { + hookId = (await hooksOnSpindle(ctx)).find((one) => one.url === hookUrl)!.id; + await dropHook(ctx); + }); + +flow("deleting a webhook takes it off the list and off the spindle") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .step("pick the payload url", pickHook) + .step("make a webhook on the spindle", makeHook) + + .open(hooks) + .sees(hookRow) + .step("accept the confirm", acceptConfirm) + .sees(deleteHook) + .click(deleteHook) + .sees(not(hookRow)) + .step( + "spindle lists no webhook", + spindleHooks("no webhook", (listed) => listed.length === 0) + ) + .reload() + .sees(not(hookRow)) + .sees(testid("settingsEmpty")); + +flow("switching a webhook off keeps it listed and inactive") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .step("pick the payload url", pickHook) + .step("make a webhook on the spindle", makeHook) + + .open(hooks) + .sees(hookRow) + .step("it starts switched on", async (ctx) => { + await expect(hookSwitch.locate(ctx.page, ctx)).toBeChecked(); + }) + .sees(hookSwitch) + .click(hookSwitch) + .step("the switch goes off", async (ctx) => { + await expect(hookSwitch.locate(ctx.page, ctx)).not.toBeChecked(); + }) + .step( + "spindle keeps it, inactive", + spindleHooks("the webhook switched off", (listed) => + listed.some((one) => one.url === hookUrl && !one.active) + ) + ) + .reload() + .sees(hookRow) + .step("it is still off after a reload", async (ctx) => { + await expect(hookSwitch.locate(ctx.page, ctx)).not.toBeChecked(); + }) + .step("take the webhook away", dropHook); + +let secretKey = ""; +const pickSecret = async () => { + secretKey = `E2E_${stamp().toUpperCase()}`; +}; +const deleteSecret = testid("drop"); +const addSecretLink = testid("addSecret"); +const secretRow = lazily("secret row", () => testid("secretName", { key: secretKey })); +const secretsOnSpindle = + (what: string, ok: (keys: string[], who: string[]) => boolean) => async (ctx: Ctx) => { + await until(`spindle should list ${what}`, 10_000, async () => { + const reply = await spindleGet(ctx.actor, `${SECRETS}.listSecrets`, { + repo: ctx.repo.did + }); + const secrets = (reply.body?.secrets ?? []) as { key: string; createdBy: string }[]; + return [ + reply.status === 200 && + ok( + secrets.map((one) => one.key), + secrets.map((one) => one.createdBy) + ), + `it answered ${reply.status} ${JSON.stringify(reply.body)}` + ]; + }); + }; +const makeSecret = async (ctx: Ctx) => { + const made = await spindlePost(ctx.actor, `${SECRETS}.addSecret`, { + repo: ctx.repo.did, + key: secretKey, + value: "hunter2" + }); + expect(made.status, `spindle refused the secret: ${JSON.stringify(made.body)}`).toBe(200); +}; +const dropSecret = async (ctx: Ctx) => { + await spindlePost(ctx.actor, `${SECRETS}.removeSecret`, { repo: ctx.repo.did, key: secretKey }); +}; + +flow("adding a pipeline secret lists it under pipelines") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .step("pick the secret's name", pickSecret) + + .open(pipelines) + .sees(addSecretLink) + .click(addSecretLink) + .at(newSecret) + .fill(testid("name"), () => secretKey) + .fill(testid("secretValue"), "hunter2") + .click(testid("add")) + .step( + "the form finishes on the pipelines page", + landsOn("the secret was not added", (ctx) => `${ctx.repo.path}/settings/pipelines`) + ) + .sees(secretRow) + .step("spindle lists it, made by the owner", async (ctx) => + secretsOnSpindle( + "the new secret", + (keys, who) => keys.includes(secretKey) && who.every((did) => did === ctx.actor.did) + )(ctx) + ) + .reload() + .sees(secretRow) + .step("take the secret away", dropSecret); + +flow("deleting a pipeline secret takes it off the list and off the spindle") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .step("pick the secret's name", pickSecret) + .step("make a secret on the spindle", makeSecret) + + .open(pipelines) + .sees(secretRow) + .sees(deleteSecret) + .click(deleteSecret) + .sees(not(secretRow)) + .step( + "spindle lists no secret", + secretsOnSpindle("no secret", (keys) => keys.length === 0) + ) + .reload() + .sees(not(secretRow)) + .sees(testid("settingsEmpty")); + +const spindleHost = new URL(config.spindleUrl).host; + +const saveSpindle = (ctx: Ctx) => saveFor(ctx.actor, { knots: [], spindles: [spindleHost] }); + +const spindleOption = (name: string) => + target(`the ${name} spindle option`, (tab) => tab.getByRole("radio", { name })); +const picked = (name: string) => async (ctx: Ctx) => { + await expect(spindleOption(name).locate(ctx.page, ctx)).toBeChecked(); +}; + +const manifestSpindle = async (ctx: Ctx) => (await manifestOf(ctx)).spindle as string | undefined; + +flow("choosing a spindle in the pipeline settings keeps it after a reload") + .actor("owner") + .repo() + .step("save a spindle for the account", saveSpindle) + + .open(pipelines) + .step("no spindle is picked", picked("No spindle")) + .click(spindleOption(spindleHost)) + .click(saveButton) + .trace(manifestId, through(knot, hydrant, bobbin)) + .step("the save finishes, which leaves nothing to save", saveFinishes) + .step("the manifest names the spindle", async (ctx) => { + expect(await manifestSpindle(ctx)).toBe(config.spindleDid); + }) + .reload() + .step("the spindle stays picked", picked(spindleHost)); + +flow("removing a repo's spindle in the pipeline settings keeps it removed after a reload") + .actor("owner") + .repo({ spindle: config.spindleDid }) + + .open(pipelines) + .step("the spindle is picked", picked(spindleHost)) + .click(spindleOption("No spindle")) + .click(saveButton) + .click(within(testid("confirming"), testid("confirm"))) + .trace(manifestId, through(knot, hydrant, bobbin)) + .step("the save finishes, which leaves nothing to save", saveFinishes) + .step("the manifest names no spindle", async (ctx) => { + expect(await manifestSpindle(ctx)).toBeUndefined(); + }) + .reload() + .step("no spindle stays picked", picked("No spindle")); diff --git a/e2e/specs/profile-settings.spec.ts b/e2e/specs/profile-settings.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..2e6a061490295ed10966be2f1722d8db52944fd4 --- /dev/null +++ b/e2e/specs/profile-settings.spec.ts @@ -0,0 +1,315 @@ +import { expect } from "@playwright/test"; + +import { getOk, pdsCall } from "../harness/api"; +import { + bobbin, + component, + type Ctx, + type Flow, + flow, + hydrant, + not, + page, + pds, + profileId, + profileSettings, + saveButton, + target, + testid, + through, + within, + type Repo +} from "../harness/flow"; +import { pdsRecordUrl } from "../harness/shared"; +import { type Profile } from "../harness/lexicons"; + +const profileOnPds = async (ctx: Ctx) => + (await getOk(pdsRecordUrl(ctx.actor.did, profileId.nsid, "self"))).value as Profile; + +const writeProfile = (record: Omit) => async (ctx: Ctx) => { + const written = await pdsCall(ctx.actor, "com.atproto.repo.putRecord", { + collection: profileId.nsid, + rkey: "self", + record: { $type: profileId.nsid, ...record } + }); + await bobbin.indexed(written); +}; + +const linkRow = (index: number) => testid("formLink", { index }); + +const valueOf = (what: ReturnType, expected: string) => async (ctx: Ctx) => { + await expect(what.locate(ctx.page, ctx)).toHaveValue(expected); +}; + +// a cleared row only closes on blur +const blur = (flow: Flow) => flow.press("Tab"); + +flow("the social links show the filled ones and one empty row, and a cleared row closes up") + .actor("owner") + .step("a profile with one link", writeProfile({ links: ["https://example.com/one"] })) + + .open(profileSettings) + .sees(linkRow(0)) + .sees(linkRow(1)) + .sees(not(linkRow(2))) + .step("the first row holds the saved link and the next one is empty", async (ctx) => { + await valueOf(linkRow(0), "https://example.com/one")(ctx); + await valueOf(linkRow(1), "")(ctx); + }) + + .fill(linkRow(1), "two.example.org") + .sees(linkRow(2)) + .sees(not(linkRow(3))) + + .fill(linkRow(0), "") + .use(blur) + .step("the second link took the first row's place", valueOf(linkRow(0), "two.example.org")) + .sees(linkRow(1)) + .sees(not(linkRow(2))) + + .fill(linkRow(1), "https://example.net/three") + .sees(linkRow(2)) + .click(saveButton) + .sends( + profileId, + pds, + { record: { links: ["https://two.example.org/", "https://example.net/three"] } }, + "put" + ) + .trace(profileId, through(pds, hydrant, bobbin)) + .step("the pds keeps the links in order and writes no empty one", async (ctx) => { + expect((await profileOnPds(ctx)).links).toEqual([ + "https://two.example.org/", + "https://example.net/three" + ]); + }) + .reload() + .sees(linkRow(2)) + .sees(not(linkRow(3))) + .step("the form shows the two saved links", async (ctx) => { + await valueOf(linkRow(0), "https://two.example.org/")(ctx); + await valueOf(linkRow(1), "https://example.net/three")(ctx); + await valueOf(linkRow(2), "")(ctx); + }); + +const made = new WeakMap(); +const keep = async (ctx: Ctx) => { + made.set(ctx, [...(made.get(ctx) ?? []), ctx.repo]); +}; +const nth = (ctx: Ctx, index: number) => made.get(ctx)![index]; + +const picker = testid("pinPicker"); +const pickerTrigger = within(picker, testid("trigger")); +const card = (index: number) => + target(`the pinned card of repo ${index + 1}`, (tab, ctx) => + testid("pinnedRepo", { key: nth(ctx, index).did }).locate(tab, ctx) + ); +const optionFor = (index: number) => + target(`the picker option for repo ${index + 1}`, (tab, ctx) => + within(picker, testid("option", { value: nth(ctx, index).did })).locate(tab, ctx) + ); +const gripOf = (index: number) => within(card(index), testid("reorder")); + +const pin = (index: number) => (flow: Flow) => + flow.click(pickerTrigger).click(optionFor(index)).sees(card(index)); + +const listed = async (ctx: Ctx) => { + const keys = await ctx.page + .locator('[data-testid="pinnedRepo"]') + .evaluateAll((items) => items.map((item) => item.getAttribute("data-key"))); + return keys.map((key) => made.get(ctx)!.findIndex((repo) => repo.did === key)); +}; + +const listsInOrder = (expected: number[]) => async (ctx: Ctx) => { + await expect.poll(() => listed(ctx)).toEqual(expected); +}; + +const focusGrip = (index: number) => async (ctx: Ctx) => { + await gripOf(index).locate(ctx.page, ctx).focus(); +}; + +// the cards sit in a grid, so the drop goes to the card nearest the pointer +const dragOnto = (index: number, over: number) => async (ctx: Ctx) => { + const grip = await gripOf(index).locate(ctx.page, ctx).boundingBox(); + const onto = await card(over).locate(ctx.page, ctx).boundingBox(); + if (!grip || !onto) throw new Error("the grip or the card to drag onto is not on the page"); + const { mouse } = ctx.page; + await mouse.move(grip.x + grip.width / 2, grip.y + grip.height / 2); + await mouse.down(); + await mouse.move(onto.x + onto.width / 2, onto.y + onto.height / 2, { steps: 12 }); + await mouse.up(); +}; + +const shownOnProfile = (ctx: Ctx) => + component("PinnedRepositories") + .locate(ctx.page, ctx) + .locator("a") + .evaluateAll((links) => links.map((link) => link.getAttribute("href"))) + .then((hrefs) => [ + ...new Set( + hrefs.flatMap((href) => { + const index = made.get(ctx)!.findIndex((repo) => repo.path === href); + return index === -1 ? [] : [index]; + }) + ) + ]); + +flow("repositories are pinned from the picker, reordered with the keyboard and a drag, and saved") + .actor("owner") + .repo() + .step("remember the first repo", keep) + .repo() + .step("remember the second repo", keep) + .repo() + .step("remember the third repo", keep) + + .open(profileSettings) + .use(pin(0)) + .use(pin(1)) + .use(pin(2)) + .step("they are listed in the order they were picked", listsInOrder([0, 1, 2])) + + .step("focus the third repo's grip", focusGrip(2)) + .press("ArrowUp") + .step("the third repo moved up one", listsInOrder([0, 2, 1])) + + .step("drag the second repo's grip onto the first card", dragOnto(1, 0)) + .step("the second repo is first", listsInOrder([1, 0, 2])) + + .click(within(card(0), testid("unpin"))) + .sees(not(card(0))) + .step("the first repo is no longer listed", listsInOrder([1, 2])) + + .click(saveButton) + .sends( + profileId, + pds, + (ctx) => ({ record: { pinnedRepositories: [nth(ctx, 1).did, nth(ctx, 2).did] } }), + "put" + ) + .trace(profileId, through(pds, hydrant, bobbin)) + .step("the pds keeps the repo dids in the saved order", async (ctx) => { + expect((await profileOnPds(ctx)).pinnedRepositories).toEqual([ + nth(ctx, 1).did, + nth(ctx, 2).did + ]); + }) + + .reload() + .step("the editor lists the saved pins in order", listsInOrder([1, 2])) + .open(page.profile) + .sees(component("PinnedRepositories")) + .step("the profile shows the pinned repos in that order", async (ctx) => { + await expect.poll(() => shownOnProfile(ctx)).toEqual([1, 2]); + }); + +const startEditing = within(component("UserProfileChrome"), testid("startEditing")); +const saveProfile = within(component("UserProfileChrome"), testid("submit")); + +flow("repositories are pinned, reordered and unpinned in place on the profile page, and saved") + .actor("owner") + .repo() + .step("remember the first repo", keep) + .repo() + .step("remember the second repo", keep) + .repo() + .step("remember the third repo", keep) + + .open(page.profile) + .sees(component("PinnedRepositories")) + .click(startEditing) + .sees(within(component("PinnedRepositories"), picker)) + .use(pin(0)) + .use(pin(1)) + .use(pin(2)) + .step("they are listed in the order they were picked", listsInOrder([0, 1, 2])) + + .step("focus the third repo's grip", focusGrip(2)) + .press("ArrowLeft") + .step("the third repo moved back one", listsInOrder([0, 2, 1])) + + .step("drag the second repo's grip onto the first card", dragOnto(1, 0)) + .step("the second repo is first", listsInOrder([1, 0, 2])) + + .click(within(card(0), testid("unpin"))) + .sees(not(card(0))) + .step("the first repo is no longer listed", listsInOrder([1, 2])) + + .click(saveProfile) + .sends( + profileId, + pds, + (ctx) => ({ record: { pinnedRepositories: [nth(ctx, 1).did, nth(ctx, 2).did] } }), + "put" + ) + .trace(profileId, through(pds, hydrant, bobbin)) + .step("the pds keeps the repo dids in the saved order", async (ctx) => { + expect((await profileOnPds(ctx)).pinnedRepositories).toEqual([ + nth(ctx, 1).did, + nth(ctx, 2).did + ]); + }) + .sees(not(picker)) + .step("the profile shows the saved pins in order without a reload", async (ctx) => { + await expect.poll(() => shownOnProfile(ctx)).toEqual([1, 2]); + }) + .reload() + .step("and still does after one", async (ctx) => { + await expect.poll(() => shownOnProfile(ctx)).toEqual([1, 2]); + }); + +// valid dids with no repo behind them, so filling the picker needs no repos +const strangers = [..."abcdef"].map((letter) => `did:plc:${letter.repeat(24)}` as const); + +flow("the picker goes away once six repositories are pinned") + .actor("owner") + .step("a profile with six pins", writeProfile({ pinnedRepositories: strangers })) + + .open(profileSettings) + .sees(testid("pinnedRepo", { key: strangers[5] })) + .step("six cards are listed", async (ctx) => { + await expect(testid("pinnedRepo").locate(ctx.page, ctx)).toHaveCount(6); + }) + .sees(not(picker)) + .click(within(testid("pinnedRepo", { key: strangers[0] }), testid("unpin"))) + .sees(picker) + .step("unpinning one brings the picker back", async (ctx) => { + await expect(pickerTrigger.locate(ctx.page, ctx)).toBeEnabled(); + }); + +const sideBySide = async (ctx: Ctx) => { + const bio = await testid("formDescription").locate(ctx.page, ctx).boundingBox(); + const link = await linkRow(0).locate(ctx.page, ctx).boundingBox(); + if (!bio || !link) throw new Error("the bio or the first link is not on the page"); + return { bio, link }; +}; + +const sizeOfWindow = (width: number, height: number) => async (ctx: Ctx) => { + await ctx.page.setViewportSize({ width, height }); +}; + +flow( + "the profile settings put the bio beside the links on a wide window and stack them on a narrow one" +) + .actor("owner") + + .open(profileSettings) + .sees(testid("formDescription")) + .step("a wide window", sizeOfWindow(1200, 900)) + .step("the bio sits left of the links", async (ctx) => { + await expect + .poll(async () => { + const { bio, link } = await sideBySide(ctx); + return bio.x + bio.width <= link.x; + }) + .toBe(true); + }) + .step("a narrow window", sizeOfWindow(390, 900)) + .step("the links are under the bio", async (ctx) => { + await expect + .poll(async () => { + const { bio, link } = await sideBySide(ctx); + return link.y >= bio.y + bio.height; + }) + .toBe(true); + }); diff --git a/e2e/specs/profile.spec.ts b/e2e/specs/profile.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..ee5a6d9b7c3aabfd6e47ebfc383c5d0616da2827 --- /dev/null +++ b/e2e/specs/profile.spec.ts @@ -0,0 +1,420 @@ +import { expect } from "@playwright/test"; + +import { getOk, pdsCall } from "../harness/api"; +import { config } from "../harness/config"; +import { + bobbin, + component, + type Ctx, + flow, + followId, + hydrant, + not, + page, + pds, + profileId, + profileSettings, + publicKeyId, + refusal, + target, + testid, + text, + through, + vouchId, + where, + within +} from "../harness/flow"; +import { keyOf, makeKey } from "../harness/git"; +import { contentTypeOf, get, stamp, until, xrpc } from "../harness/net"; +import { forgetTheLogin, pdsRecordUrl, picture, recall, remember } from "../harness/shared"; +import { type Profile } from "../harness/lexicons"; + +const avatarSettings = where("the avatar settings", () => "/settings/profile/avatar"); + +const richProfile = (ctx: Ctx) => + ({ + $type: "org.tangled.actor.profile", + description: "before", + pronouns: "she/her", + location: "the knot", + links: ["https://example.com/me"], + pinnedRepositories: [ctx.repo.did] + }) satisfies Profile; + +const writeRichProfile = async (ctx: Ctx) => { + const written = await pdsCall(ctx.actor, "com.atproto.repo.putRecord", { + collection: profileId.nsid, + rkey: "self", + record: richProfile(ctx) + }); + await bobbin.indexed(written); +}; + +const pinnedRepo = target("the pinned repo", (tab, ctx) => + tab.getByRole("link", { name: ctx.repo.name, exact: true }).first() +); + +const profileOnPds = async (ctx: Ctx) => + (await getOk(pdsRecordUrl(ctx.actor.did, profileId.nsid, "self"))).value; + +const bioAfter = (ctx: Ctx) => ({ record: { ...richProfile(ctx), description: "after" } }); + +const keepsTheRest = async (ctx: Ctx) => { + const { description, ...rest } = richProfile(ctx); + expect(await profileOnPds(ctx)).toMatchObject({ ...rest, description: "after" }); +}; + +flow("editing the bio on the profile page keeps the pinned repo and the other fields") + .actor("owner") + .repo() + .step("a profile with a pin, pronouns, a location and a link", writeRichProfile) + + .open(page.profile) + .sees(pinnedRepo) + .click(within(component("UserProfileChrome"), testid("startEditing"))) + .fill(within(component("ProfileEditForm"), testid("description")), "after") + .click(within(component("UserProfileChrome"), testid("submit"))) + .sends(profileId, pds, bioAfter, "put") + .trace(profileId, through(pds, hydrant, bobbin)) + .step("the pds keeps the pin, the pronouns, the location and the link", keepsTheRest) + .sees(within(testid("profileBio"), text("after"))) + .reload() + .sees(within(testid("profileBio"), text("after"))) + .sees(testid("profilePronouns", { pronouns: "she/her" })) + .sees(pinnedRepo); + +const firstSave = within(component("SettingsHeader"), testid("save")); + +flow("editing the bio in profile settings keeps the pinned repo and the other fields") + .actor("owner") + .repo() + .step("a profile with a pin, pronouns, a location and a link", writeRichProfile) + + .open(profileSettings) + .fill(within(component("ProfileSettingsForm"), testid("formDescription")), "after") + .click(firstSave) + .sends(profileId, pds, bioAfter, "put") + .trace(profileId, through(pds, hydrant, bobbin)) + .step("the pds keeps the pin, the pronouns, the location and the link", keepsTheRest) + .reload() + .step("the settings form shows the saved bio", async (ctx) => { + await expect( + within(component("ProfileSettingsForm"), testid("formDescription")).locate( + ctx.page, + ctx + ) + ).toHaveValue("after"); + }) + .open(page.profile) + .sees(within(testid("profileBio"), text("after"))) + .sees(pinnedRepo); + +// the pages never ask for size=52&format=png, so no cached placeholder can answer it +const servedAvatar = (ctx: Ctx) => + contentTypeOf( + `${config.webUrl}/avatar/${encodeURIComponent(ctx.actor.did)}?size=52&format=png` + ); + +flow("an avatar uploaded in the settings is the picture the avatar service serves") + .actor("owner") + + .open(avatarSettings) + // the input hides behind a browse button, so a visible filter can't find it + .step("choose the picture", (ctx) => + ctx.page.locator('[data-testid="avatarFileInput"]').setInputFiles(picture) + ) + .click(testid("uploadAvatar")) + .sends(profileId, pds, { record: { avatar: { mimeType: "image/png" } } }, "put") + .trace(profileId, through(pds, hydrant, bobbin)) + .step("the avatar service serves the picture and not a placeholder", async (ctx) => { + expect(await servedAvatar(ctx)).toBe("image/png"); + }); + +const javascriptLink = target("a link that runs javascript", (tab) => + tab.locator('a[href^="javascript:" i]') +); +const scripted = "javascript:alert(1)"; + +flow("a javascript: link in a profile is not shown as a link") + .actor("owner") + + .open(page.profile) + .click(within(component("UserProfileChrome"), testid("startEditing"))) + .fill(testid("socialLink", { index: 0 }), scripted) + .click(within(component("UserProfileChrome"), testid("submit"))) + .sends(profileId, pds, { record: { links: [scripted] } }, "put") + .trace(profileId, through(pds, hydrant, bobbin)) + .sees(not(javascriptLink)) + .reload() + .sees(not(javascriptLink)); + +flow("the profile settings refuse a link that is not http or https and write nothing") + .actor("owner") + + .open(profileSettings) + .fill(testid("formLink", { index: 0 }), scripted) + .click(firstSave) + .sees(refusal) + .sendsNothing(profileId) + .step("the pds profile has no link", async (ctx) => { + expect((await profileOnPds(ctx)).links).toBeUndefined(); + }); + +const welcome = where("the welcome wizard", () => "/welcome"); + +const next = testid("handleNext"); +const unfollowed = target("the Unfollow that replaced it", (tab) => + testid("follow", { following: "true" }) + .locate(tab, null as any) + .first() +); +const firstFollow = target("the first Follow on the discover step", (tab) => + testid("follow", { following: "false" }) + .locate(tab, null as any) + .first() +); + +const throughTheProfileStep = (name: string) => + flow(name) + .actor("newbie", { login: "ui" }) + .open(welcome) + .sees(within(component("ProfileStep"), testid("stepHeaderTitle"))) + .step("picks a picture", (ctx) => + testid("welcomeAvatarInput").locate(ctx.page, ctx).setInputFiles(picture) + ) + .fill(within(component("ProfileStep"), testid("draftDescription")), "i write small tools") + .fill(within(component("ProfileStep"), testid("draftPronouns")), "she/her") + .fill(within(component("ProfileStep"), testid("draftWebsite")), "example.com") + .click(next) + .sends( + profileId, + pds, + { + record: { + description: "i write small tools", + pronouns: "she/her", + links: ["https://example.com/"] + } + }, + "put" + ) + .trace(profileId, through(pds, hydrant, bobbin)) + .sees(within(component("EmailsStep"), testid("stepHeaderTitle"))); + +throughTheProfileStep("the welcome wizard saves a new account's profile and goes on") + .step("the pds keeps the picture, the words and the link", async (ctx) => { + const value = await profileOnPds(ctx); + expect(value).toMatchObject({ + description: "i write small tools", + pronouns: "she/her", + links: ["https://example.com/"] + }); + expect(value.avatar?.ref?.$link, "the profile has no avatar").toBeTruthy(); + }) + .open(page.profile) + .sees(within(testid("profileBio"), text("i write small tools"))) + .sees(testid("profilePronouns", { pronouns: "she/her" })); + +// advance() ignores nextReady, so discover can be skipped on purpose +throughTheProfileStep("the discover step can be skipped without following anyone") + .click(next) + .sees(within(component("SocialStep"), testid("stepHeaderTitle"))) + .click(next) + .sees(within(component("KeysStep"), testid("stepHeaderTitle"))); + +throughTheProfileStep("following someone on the discover step lets the wizard go on") + .click(next) + .sees(within(component("SocialStep"), testid("stepHeaderTitle"))) + .click(firstFollow) + .sends(followId, pds, { collection: followId.nsid }, "put") + .trace(followId, through(pds, hydrant, bobbin)) + .sees(unfollowed) + .click(next) + .sees(within(component("KeysStep"), testid("stepHeaderTitle"))); + +const lastStep = testid("handleNext"); +const wizardKey = { name: "wizard-laptop" }; + +flow("a new account goes through the whole welcome wizard") + .actor("newbie", { login: "ui" }) + .step("the account makes a key to add", makeKey) + .open(welcome) + .sees(within(component("ProfileStep"), testid("stepHeaderTitle"))) + .step("picks a picture", (ctx) => + testid("welcomeAvatarInput").locate(ctx.page, ctx).setInputFiles(picture) + ) + .fill(within(component("ProfileStep"), testid("draftDescription")), "i write small tools") + .fill(within(component("ProfileStep"), testid("draftPronouns")), "she/her") + .fill(within(component("ProfileStep"), testid("draftWebsite")), "example.com") + .click(next) + .trace(profileId, through(pds, hydrant, bobbin)) + .click(next) + .click(firstFollow) + .trace(followId, through(pds, hydrant, bobbin)) + .sees(unfollowed) + .click(next) + .sees(within(component("KeysStep"), testid("stepHeaderTitle"))) + .fill(within(component("KeysStep"), testid("name")), wizardKey.name) + .fill(within(component("KeysStep"), testid("key")), (ctx) => keyOf(ctx).pub) + .click(within(component("KeysStep"), testid("submit"))) + .sends( + publicKeyId, + pds, + (ctx) => ({ record: { name: wizardKey.name, publicKey: keyOf(ctx).pub } }), + "create" + ) + .trace(publicKeyId, through(pds, hydrant, bobbin)) + .click(next) + .sees(within(component("FinishStep"), testid("stepHeaderTitle"))) + .click(lastStep) + .at(where("the home page", () => "/")); + +// accounts.create already wrote an empty profile, so this takes it back out +const withoutProfile = async (ctx: Ctx) => { + await pdsCall(ctx.actor, "com.atproto.repo.deleteRecord", { + collection: profileId.nsid, + rkey: "self" + }); + // the home page asks bobbin, so wait for bobbin to say the profile is gone + await until("bobbin should have no profile", 15_000, async () => { + const { status } = await get( + xrpc(config.bobbinUrl, "org.tangled.actor.getProfile", { + actor: `at://${ctx.actor.did}/${profileId.nsid}/self` + }) + ); + return [status === 404, `getProfile answered ${status}`]; + }); +}; + +flow("an account with no profile record is sent to the welcome wizard from the home page") + .actor("blank", { login: "ui" }) + .step("the profile record is taken away", withoutProfile) + .open(where("the home page", () => "/")) + .at(welcome) + .sees(within(component("ProfileStep"), testid("stepHeaderTitle"))); + +const ownHandle = target("the account's handle", (tab, ctx) => + tab.getByText(ctx.actor.handle, { exact: true }).first() +); + +flow("an account with no profile record can open its own profile page") + .actor("blank", { login: "ui" }) + .step("the profile record is taken away", withoutProfile) + .open(page.profile) + .sees(ownHandle); + +const vouchedBy = (expected: Record | undefined) => async (ctx: Ctx) => { + const { status, body } = await get( + pdsRecordUrl(ctx.actor.did, vouchId.nsid, recall(ctx, "friend").did) + ); + if (expected === undefined) + expect(status, `the pds still has it: ${JSON.stringify(body)}`).toBe(400); + else expect(body.value).toMatchObject(expected); +}; + +const friendsProfile = where("the friend's profile", (ctx) => `/${recall(ctx, "friend").handle}`); +const vouchModal = testid("vouch"); +const vouchTrigger = testid("vouchTrigger"); +const vouchStanding = (standing: "vouch" | "denounce" | "none") => + testid("vouchTrigger", { standing }); +const vouchOption = (choice: string) => within(vouchModal, testid("setChoice", { value: choice })); +const saveVouch = within(vouchModal, testid("save")); +const reasonBox = testid("reason"); + +flow("changing a vouch to a denounce keeps the new reason and the same record") + .actor("friend") + .step("remember the friend", remember("friend")) + .actor("fan") + + .open(friendsProfile) + .click(vouchTrigger) + .click(vouchOption("vouch")) + .fill(reasonBox, "knows their stuff") + .click(saveVouch) + .trace(vouchId, through(pds, hydrant, bobbin)) + .sees(vouchStanding("vouch")) + + .click(vouchTrigger) + .click(vouchOption("denounce")) + .fill(reasonBox, "changed my mind") + .click(saveVouch) + .sends( + vouchId, + pds, + (ctx) => ({ + rkey: recall(ctx, "friend").did, + record: { kind: "denounce", reason: "changed my mind" } + }), + "put" + ) + .trace(vouchId, through(pds, hydrant, bobbin)) + .sees(not(vouchModal)) + .sees(vouchStanding("denounce")) + .sees(not(vouchStanding("vouch"))) + .step( + "the pds has one vouch record and it is a denounce", + vouchedBy({ kind: "denounce", reason: "changed my mind" }) + ) + .reload() + .sees(vouchStanding("denounce")); + +const signup = where("the sign up page", () => "/signup"); +const signupMail = `signup-${stamp()}@e2e.test`; + +flow("signing up with an email says to check the mail") + .actor("guest", { login: "ui" }) + .open(page.profile) + .step("signs out", forgetTheLogin) + .open(signup) + .sees(not(testid("signupUnavailable"))) + .fill(testid("email"), signupMail) + .step("the sign up button is on", async (ctx) => { + await expect(testid("submit").locate(ctx.page, ctx)).toBeEnabled(); + }) + .click(testid("submit")) + .sees(testid("checkEmailTitle")) + .sees(text(signupMail)); + +flow("vouching for someone and withdrawing it") + .actor("friend") + .step("remember who is being vouched for", remember("friend")) + .actor("fan") + + .open(friendsProfile) + .click(vouchTrigger) + .click(vouchOption("vouch")) + .fill(reasonBox, "knows their stuff") + .click(saveVouch) + .sends( + vouchId, + pds, + (ctx) => ({ + rkey: recall(ctx, "friend").did, + record: { kind: "vouch", reason: "knows their stuff" } + }), + "put" + ) + // the dialog waits on bobbin before it closes, so the trigger changes late + .sees(not(vouchStanding("vouch"))) + .trace(vouchId, through(pds, hydrant, bobbin)) + .step( + "the pds has the vouch and its reason", + vouchedBy({ kind: "vouch", reason: "knows their stuff" }) + ) + .sees(vouchStanding("vouch")) + .reload() + .sees(vouchStanding("vouch")) + + .click(vouchTrigger) + .click(vouchOption("none")) + .click(saveVouch) + .sends(vouchId, pds, (ctx) => ({ rkey: recall(ctx, "friend").did }), "delete") + .sees(vouchStanding("vouch")) + .trace(vouchId, through(pds, hydrant, bobbin)) + // the open dialog has a second Vouch button, so wait for it to close first + .sees(not(vouchModal)) + .sees(vouchStanding("none")) + .step("the pds has dropped the vouch", vouchedBy(undefined)) + .reload() + .sees(vouchStanding("none")) + .sees(not(vouchStanding("vouch"))); diff --git a/e2e/specs/pull-pipelines.spec.ts b/e2e/specs/pull-pipelines.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..a97d0d4d8d68ab06421337a30e6a7fe84b8508ef --- /dev/null +++ b/e2e/specs/pull-pipelines.spec.ts @@ -0,0 +1,123 @@ +import { expect } from "@playwright/test"; + +import { config } from "../harness/config"; +import { + type Ctx, + bobbin, + flow, + hydrant, + knot, + link, + page, + patchId, + pullId, + rkeyOf, + testid, + through, + where +} from "../harness/flow"; +import { until } from "../harness/net"; +import { + base, + branches, + deletesBranch, + merges, + onBranches, + pipelineFor, + pullPipelines, + waitsForApproval, + workflow, + workflowPath +} from "../harness/shared"; + +// only a close or a merge runs this one, the default types leave both out +const doneWorkflow = `engine: dummy +when: + - event: pull_request + branch: ["main"] + types: [closed, merged] +steps: + - name: bye + command: echo "done" +`; +const donePath = ".tangled/workflows/done.yml"; + +const head = (ctx: Ctx) => ctx.repo.heads[branches.from]; + +const runs = (id: string, sha: (ctx: Ctx) => string) => (ctx: Ctx) => + until(`spindle should run ${id} for the pull at ${sha(ctx)}`, 30_000, async () => { + const found = await pullPipelines(ctx.repo.did, sha(ctx)); + const ran = found.some((p) => p.workflows?.some((w) => w.id === id)); + return [ran, `it listed ${found.length} pull request pipelines for the head`]; + }); + +const withDone = (name: string) => + flow(name) + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ ...base, [workflowPath]: workflow, [donePath]: doneWorkflow }) + .push({ "e2e/feature.txt": "feature\n" }, { branch: branches.from, base: branches.into }) + .mirrored() + .pull(pullId, patchId, branches) + .step("spindle runs the default workflow for the opened pull", runs("ci.yml", head)); + +onBranches("opening a pull request runs a pull_request pipeline for its head", { ci: true }) + .pull(pullId, patchId, branches) + .step("spindle has a pull request pipeline for the head", pipelineFor("pullRequest", head)); + +const pullRuns = new WeakMap(); +const pullRun = where("the pull request run", (ctx) => { + const run = pullRuns.get(ctx)!; + return `${ctx.repo.path}/pipelines/${run.id}/workflow/${run.workflow}`; +}); + +onBranches("a pull request run links back to its pull", { ci: true }) + .pull(pullId, patchId, branches) + .step("spindle runs the default workflow for the opened pull", runs("ci.yml", head)) + .step("remember the run", async (ctx) => { + const [run] = await pullPipelines(ctx.repo.did, head(ctx)); + pullRuns.set(ctx, { id: run.id, workflow: run.workflows![0].id }); + }) + .open(pullRun) + .sees(link("Open PR")) + .click(link("Open PR")) + .step("the link lands on the pull", (ctx) => + expect + .poll(() => new URL(ctx.page.url()).pathname) + .toMatch(new RegExp(`^${ctx.repo.path}/pulls/${rkeyOf(ctx, pullId)}(/|$)`)) + ); + +withDone("closing a pull request runs its types: [closed] workflow") + .open(page.pull(pullId)) + .changeState(pullId, "closed", testid("requestClose"), testid("changeState")) + .step("spindle runs the close workflow at the head", runs("done.yml", head)); + +withDone("a merged pull request runs its types: [merged] workflow") + .open(page.pull(pullId)) + .use(merges) + .step("spindle runs the merge workflow at the head", runs("done.yml", head)); + +// the head is then only kept by the knot's keep ref +withDone("closing a pull request whose branch is gone still runs at the old head") + .step("delete the source branch", deletesBranch(branches.from)) + .open(page.pull(pullId)) + .changeState(pullId, "closed", testid("requestClose"), testid("changeState")) + .step("spindle runs the close workflow at the old head", runs("done.yml", head)); + +onBranches("a pull from someone who isn't a collaborator waits for approval", { ci: true }) + .actor("author") + .pull(pullId, patchId, branches) + .step("the opened pull's pipeline is blocked", waitsForApproval(head)) + .open(page.pull(pullId)) + .changeState(pullId, "closed", testid("requestClose"), testid("changeState")) + .changeState(pullId, "open", testid("changeState"), testid("requestClose")) + .step("still blocked after the close and the reopen", waitsForApproval(head)) + .actor("owner") + .push({ "e2e/more.txt": "more\n" }, { branch: branches.from }) + .mirrored() + .actor("author") + .open(page.pull(pullId)) + .click(testid("resubmit")) + .sends(patchId, knot, (ctx) => ({ patch: { ticket: rkeyOf(ctx, pullId) } }), "create") + .trace(patchId, through(knot, hydrant, bobbin)) + .step("the resubmitted head is blocked too", waitsForApproval(head)); diff --git a/e2e/specs/pulls.spec.ts b/e2e/specs/pulls.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..941a5c83133ece43a9d1fb0db6524a376495d64e --- /dev/null +++ b/e2e/specs/pulls.spec.ts @@ -0,0 +1,386 @@ +import { expect } from "@playwright/test"; + +import { comments, editCob } from "../harness/api"; +import { seeded } from "../harness/config"; +import { + bobbin, + commentId, + commitRow, + component, + type Ctx, + type Flow, + hydrant, + knot, + link, + not, + page, + patchId, + pds, + pullId, + record, + refusal, + replacing, + rkeyOf, + subscriptionId, + target, + testid, + text, + through, + triage, + unique, + within +} from "../harness/flow"; +import { lsRemote } from "../harness/git"; +import { + branches, + defineLabel, + labelOff, + labelOn, + merges, + onBranches, + pipelineFor, + recall, + remember, + sendsPatch, + stillStored, + subscribesToPull +} from "../harness/shared"; + +const title = unique("add a feature"); + +onBranches("opening a pull request", { ci: true }) + .open(page.newPull(branches)) + .fill(within(component("PullCompose"), testid("title")), title) + .click(within(component("PullCompose"), testid("submit"))) + .sends( + pullId, + knot, + (ctx) => ({ + repo: ctx.repo.did, + record: { + state: "open", + title: { text: title }, + targetBranch: branches.into + } + }), + "create" + ) + .trace(pullId, through(knot, hydrant, bobbin)) + .use(sendsPatch()) + .use(subscribesToPull) + .at(page.pulls) + .sees(link(title)) + .checkpoint("the pull is listed") + .step( + "spindle has a pipeline for the pull", + pipelineFor("pullRequest", (ctx) => ctx.repo.heads[branches.from]) + ) + .open(page.pull(pullId)) + .sees(testid("workflowName", { name: "ci.yml" })); + +onBranches("closing and reopening a pull request") + .pull(pullId, patchId, branches) + .open(page.pull(pullId)) + .changeState(pullId, "closed", testid("requestClose"), testid("changeState")) + .changeState(pullId, "open", testid("changeState"), testid("requestClose")) + .reload() + .sees(testid("requestClose")); + +// the diff lists a file only once the mirror can read the commits between the branches +const diffedFile = testid("diffFileName", { path: "e2e/feature.txt" }); + +onBranches("the new pull form lists the branch's file in its diff") + .open(page.newPull(branches)) + .sees(diffedFile) + .sees(within(component("PullCompose"), testid("submit"))); + +const viewedTitle = unique("a pull to view"); + +onBranches("the pull page shows the pull's title") + .pull(pullId, patchId, { ...branches, title: viewedTitle }) + .open(page.pull(pullId)) + .sees(not(testid("errorTitle"))) + .sees(within(testid("ticketTitle"), text(viewedTitle))); + +const otherPull = record("org.tangled.track.ticket"); +const otherPatch = record("org.tangled.review.patch"); + +const searchPulls = within(component("TicketSearch"), testid("q")); + +const pullNeedle = unique("search needle"); +const pullHay = unique("unrelated"); + +const onTwoPulls = (name: string) => + onBranches(name) + .pull(pullId, patchId, { ...branches, title: pullNeedle }) + .pull(otherPull, otherPatch, { ...branches, title: pullHay }); + +onTwoPulls("searching the pull list finds the pull you named") + .open(page.pulls) + .sees(link(pullNeedle)) + .sees(link(pullHay)) + .fill(searchPulls, pullNeedle) + .press("Enter") + .sees(link(pullNeedle)) + .sees(not(link(pullHay))); + +const note = "first thoughts"; +const secondThoughts = "second thoughts"; +const commentMenu = within(component("PullReviewComment"), testid("moreMenu")); +const editor = within(component("PullReviewCommentEditor"), testid("body")); +const commentField = within(component("PullReviewCommentForm"), testid("reviewComment")); + +const commented = (name: string) => + onBranches(name) + .pull(pullId, patchId, branches) + .open(page.pull(pullId)) + .fill(commentField, note) + .click(within(component("PullReviewCommentForm"), testid("submit"))) + .sends( + commentId, + knot, + (ctx) => ({ record: { subject: { uri: ctx.get(pullId).uri } } }), + "create" + ) + .trace(commentId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .sees(within(testid("reviewCommentBody"), text(note))); + +commented("commenting on a pull request") + .reload() + .sees(within(testid("reviewCommentBody"), text(note))); + +const startEdit = >(flowSoFar: T) => + flowSoFar + .click(commentMenu) + .click(testid("setEditing")) + .fill(editor, secondThoughts) + .click(within(component("PullReviewCommentEditor"), testid("submit"))) + .sends( + commentId, + knot, + replacing(commentId, (ctx) => ({ + subject: { uri: ctx.get(pullId).uri }, + body: { text: secondThoughts } + })), + "put" + ) + .trace(commentId, through(knot, hydrant, bobbin)); + +startEdit(commented("editing a comment on a pull request")) + .reload() + .sees(within(testid("reviewCommentBody"), text(secondThoughts))) + .sees(not(within(testid("reviewCommentBody"), text(note)))); + +startEdit(commented("an edited pull comment shows without a reload")).sees( + within(testid("reviewCommentBody"), text(secondThoughts)) +); + +const bugTitle = unique("a pull to resubmit"); +const pushedFile = testid("diffFileName", { path: "e2e/more.txt" }); +// round labels only head comments, so commit rows are how to tell rounds apart here +const twoCommitRows = (flow: Flow) => flow.sees(commitRow(2)).sees(not(commitRow(3))); + +onBranches("resubmitting a pull request after pushing to its source branch", { ci: true }) + .pull(pullId, patchId, { ...branches, title: bugTitle }) + .push({ "e2e/more.txt": "more\n" }, { branch: branches.from }) + .mirrored() + .open(page.pull(pullId)) + .sees(not(pushedFile)) + .sees(testid("resubmit")) + .click(testid("resubmit")) + .sends(patchId, knot, (ctx) => ({ patch: { ticket: rkeyOf(ctx, pullId) } }), "create") + .trace(patchId, through(knot, hydrant, bobbin)) + .reload() + .sees(pushedFile) + .use(twoCommitRows) + .checkpoint("the resubmitted round shows in the diff") + .step( + "spindle has a pipeline for the new commit", + pipelineFor("pullRequest", (ctx) => ctx.repo.heads[branches.from]) + ); + +const moreActions = within(component("Ticket"), testid("moreMenu")); +const editPull = within(component("Ticket"), testid("setEditing")); + +const strangerTitle = "a pull from someone who can't contribute"; + +onBranches( + "a stranger's pull is refused when only collaborators may contribute and the page says so" +) + .policy("collaborators") + + .actor(seeded.charlie) + .open(page.newPull(branches)) + .fill(within(component("PullCompose"), testid("title")), strangerTitle) + .click(within(component("PullCompose"), testid("submit"))) + .refused(pullId, 403) + .sees(refusal) + .open(page.pulls) + .sees(not(link(strangerTitle))); + +const otherBranch = "other"; + +// keep it in its own folder, a shared file makes the knot refuse the merge +const withOtherBranch = (name: string) => + onBranches(name) + .push({ "other/other.txt": "other\n" }, { branch: otherBranch, base: branches.into }) + .mirrored() + .pull(pullId, patchId, branches); + +const targetBranchPicker = within( + component("TicketForm"), + within(testid("targetBranch"), testid("trigger")) +); +const otherOption = testid("option", { value: otherBranch }); +const targeting = (branch: string) => testid("targetBranchTag", { branch }); + +withOtherBranch("retargeting a pull request to another branch") + .open(page.pull(pullId)) + .sees(targeting(branches.into)) + .click(moreActions) + .click(editPull) + .click(targetBranchPicker) + .click(otherOption) + .click(within(component("TicketForm"), testid("submit"))) + .sends(pullId, knot, replacing(pullId, { targetBranch: otherBranch }), "put") + .trace(pullId, through(knot, hydrant, bobbin)) + .sees(targeting(otherBranch)) + .step("the knot's copy targets the other branch", async (ctx) => + expect((await knot.has(ctx.get(pullId))).targetBranch).toBe(otherBranch) + ) + .reload() + .sees(targeting(otherBranch)) + .sees(not(targeting(branches.into))); + +// the page won't see this, so the knot's copy is edited directly +const retargetOnKnot = async (ctx: Ctx) => { + const reply = await editCob(ctx.actor, ctx.get(pullId).uri, "retarget", (stored) => ({ + ...stored, + targetBranch: otherBranch + })); + expect(reply.status, `the retarget answered ${JSON.stringify(reply.body)}`).toBe(200); + ctx.set(pullId, { uri: reply.body.uri, cid: reply.body.cid }); +}; + +withOtherBranch("a retargeted pull merges into its new branch and leaves the old one alone") + .step("the pull is retargeted", retargetOnKnot) + .open(page.pull(pullId)) + .sees(targeting(otherBranch)) + .use(merges) + .knotStored(pullId, "completed") + .step("only the other branch moved", async (ctx) => { + const heads = await lsRemote(ctx.actor, ctx.repo.did, "refs/heads/"); + expect(heads[`refs/heads/${branches.into}`]).toBe(ctx.repo.heads[branches.into]); + expect(heads[`refs/heads/${otherBranch}`]).not.toBe(ctx.repo.heads[otherBranch]); + }); + +onBranches("putting a label on a pull request and taking it off") + .pull(pullId, patchId, branches) + .step("define the label", defineLabel(triage)) + + .open(page.pull(pullId)) + .use(labelOn(pullId, "pull")) + .sees(targeting(branches.into)) + .use(labelOff(pullId, "pull")); + +const reviewNote = "looks fine to me"; +const peerSays = async (ctx: Ctx) => { + const reply = await comments.add(ctx.actor, ctx.repo.did, ctx.get(pullId), reviewNote); + expect(reply.status, `the review comment answered ${JSON.stringify(reply.body)}`).toBe(200); + await bobbin.indexed({ uri: reply.body.uri, cid: reply.body.cid }); +}; + +// the wrapper has the same testid, so match the button +const assigneePicker = target("the assignee picker", (tab) => + tab + .locator('[data-component="TicketForm"] button[data-testid="assignee"]') + .filter({ visible: true }) +); +const peerOption = target("the peer in the picker", (tab, ctx) => + testid("option", { value: recall(ctx, "peer").did }).locate(tab, ctx) +); +const assignedTo = target("the assignee in the pull's header", (tab, ctx) => + testid("assignee", { handle: recall(ctx, "peer").handle }).locate(tab, ctx) +); + +onBranches("assigning a pull request to a peer keeps the assignee after a reload") + .pull(pullId, patchId, branches) + .actor("peer") + .step("remember the peer", remember("peer")) + .member() + .step("the peer comments on the pull", peerSays) + + .actor("owner") + .open(page.pull(pullId)) + .click(moreActions) + .click(editPull) + .click(assigneePicker) + .click(peerOption) + .click(within(component("TicketForm"), testid("submit"))) + .sends(pullId, knot, replacing(pullId), "put") + .trace(pullId, through(knot, hydrant, bobbin)) + .sees(assignedTo) + .reload() + .sees(assignedTo); + +const afterWords = "a note on the second round"; +const inRound = (round: string, words: string) => + within( + testid("roundGroup", { version: round.replace(/^v/, "") }), + within(testid("reviewCommentBody"), text(words)) + ); + +onBranches("a comment written after a resubmit files under the new round") + .pull(pullId, patchId, branches) + .push({ "e2e/more.txt": "more\n" }, { branch: branches.from }) + .mirrored() + .open(page.pull(pullId)) + .sees(testid("resubmit")) + .click(testid("resubmit")) + .use(sendsPatch()) + .use(twoCommitRows) + + .fill(commentField, afterWords) + .click(within(component("PullReviewCommentForm"), testid("submit"))) + .sends( + commentId, + knot, + (ctx) => ({ record: { subject: { uri: ctx.get(pullId).uri } } }), + "create" + ) + .trace(commentId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .sees(inRound("v1", afterWords)) + .sees(not(inRound("v0", afterWords))) + .reload() + .sees(inRound("v1", afterWords)) + .sees(not(inRound("v0", afterWords))); + +const edited = "same here, again"; + +const cancel = within(component("PullReviewCommentEditor"), testid("cancel")); + +onBranches("cancelling a pull comment edit leaves the comment as it was") + .pull(pullId, patchId, branches) + .open(page.pull(pullId)) + .fill(commentField, note) + .click(within(component("PullReviewCommentForm"), testid("submit"))) + .trace(commentId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .sees(within(testid("reviewCommentBody"), text(note))) + + .click(commentMenu) + .click(testid("setEditing")) + .fill(editor, edited) + .click(cancel) + .sendsNothing(commentId) + .sees(within(testid("reviewCommentBody"), text(note))) + .sees(not(within(testid("reviewCommentBody"), text(edited)))) + .reload() + .sees(within(testid("reviewCommentBody"), text(note))) + .sees(not(within(testid("reviewCommentBody"), text(edited)))) + .step( + "the knot and hydrant still have the first version", + stillStored(commentId, (value) => value.body.text, note) + ); diff --git a/e2e/specs/quota.spec.ts b/e2e/specs/quota.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..90f79f476dd14011fc1e266b5f6c517dc2cfd6c4 --- /dev/null +++ b/e2e/specs/quota.spec.ts @@ -0,0 +1,91 @@ +import { expect } from "@playwright/test"; + +import { quotas } from "../harness/api"; +import { config, seeded } from "../harness/config"; +import { type Ctx, flow, text, where } from "../harness/flow"; +import { until } from "../harness/net"; +import { spindle } from "../harness/services"; +import { workflow, workflowPath } from "../harness/shared"; + +const MINUTES = "ci_minutes_monthly"; + +const resetsAt = () => { + const now = new Date(); + const next = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1)); + return `${next.toISOString().slice(0, 10)} 00:00 UTC`; +}; +const usedUp = (who: "owner" | "repo") => + `monthly CI minutes used up (${who}): 1 of 1, resets ${resetsAt()}`; + +const ownerDid = (ctx: Ctx) => ctx.actor.did; +const repoDid = (ctx: Ctx) => ctx.repo.did; + +// the chore goes in before the set, so a crash after it still gets unset +const limitToOneMinute = (whose: (ctx: Ctx) => string) => async (ctx: Ctx) => { + const did = whose(ctx); + ctx.owe(`monthly ci minutes limit of ${did}`, { + kind: "limit", + by: seeded.alice, + did, + resource: MINUTES + }); + await quotas.set(did, MINUTES, 1); +}; + +const minutesUsed = async (did: string) => + (await quotas.usage(did)).find((row) => row.resource === MINUTES)?.used ?? 0; + +const usedMinutes = + (what: string, whose: (ctx: Ctx) => string, minutes: number) => async (ctx: Ctx) => { + await until(`spindle should bill ${what} ${minutes} minutes`, 10_000, async () => { + const used = await minutesUsed(whose(ctx)); + return [used === minutes, `it billed ${used}`]; + }); + }; + +const runs = new WeakMap>>(); + +const succeeds = async (ctx: Ctx) => { + const run = await spindle.finishedRun(ctx.repo.did, ctx.pushed.sha); + expect(run.workflow.status, JSON.stringify(run.workflow)).toBe("success"); +}; + +const isDenied = (who: "owner" | "repo") => async (ctx: Ctx) => { + const run = await spindle.finishedRun(ctx.repo.did, ctx.pushed.sha); + runs.set(ctx, run); + expect(run.workflow.status, JSON.stringify(run.workflow)).toBe("failed"); + expect(run.workflow.error).toBe(usedUp(who)); +}; + +const deniedWorkflow = where("the denied workflow", (ctx) => { + const { pipeline, workflow } = runs.get(ctx)!; + return `${ctx.repo.path}/pipelines/${pipeline}/workflow/${workflow.id}`; +}); + +const second = { "e2e/second.txt": "second\n" }; + +flow("a workflow is denied once the owner's monthly ci minutes are used up") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .step("limit the owner to one ci minute a month", limitToOneMinute(ownerDid)) + .push({ [workflowPath]: workflow }) + .step("spindle runs the first workflow to success", succeeds) + .step("spindle billed the owner one of one minutes", usedMinutes("the owner", ownerDid, 1)) + .step("spindle billed the repo one minute too", usedMinutes("the repo", repoDid, 1)) + .push(second) + .step("spindle fails the second workflow, saying the minutes are used up", isDenied("owner")) + .open(deniedWorkflow) + .sees(text(usedUp("owner"))); + +flow("a workflow is denied once the repo's monthly ci minutes are used up") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .step("limit the repo to one ci minute a month", limitToOneMinute(repoDid)) + .push({ [workflowPath]: workflow }) + .step("spindle runs the first workflow to success", succeeds) + .step("spindle billed the repo one of one minutes", usedMinutes("the repo", repoDid, 1)) + .step("spindle billed the unlimited owner too", usedMinutes("the owner", ownerDid, 1)) + .push(second) + .step("spindle fails the second workflow, saying the minutes are used up", isDenied("repo")) + .open(deniedWorkflow) + .sees(text(usedUp("repo"))); diff --git a/e2e/specs/recents.spec.ts b/e2e/specs/recents.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..bc0695924d09d41c4c911f618511d9920d11bcce --- /dev/null +++ b/e2e/specs/recents.spec.ts @@ -0,0 +1,78 @@ +import { expect } from "@playwright/test"; + +import { + type Ctx, + type Flow, + flow, + page, + patchId, + pullId, + rkeyOf, + testid, + text, + ticketId, + unique, + where, + within +} from "../harness/flow"; +import { until } from "../harness/net"; +import { recentsIn } from "../harness/services"; +import { branches, onBranches } from "../harness/shared"; + +const home = where("the home page", () => "/"); +const row = (kind: "repo" | "issue" | "pull") => testid("recentItem", { kind }); + +flow("a new account has no recent pages on the home page") + .actor("owner") + .open(home) + .sees(text("No recent pages yet.")); + +const issueTitle = unique("an issue to come back to"); +const pullTitle = unique("a pull to come back to"); + +const rowPaths = (ctx: Ctx) => + testid("recentItem") + .locate(ctx.page, ctx) + .evaluateAll((links) => links.map((link) => link.getAttribute("href"))); + +const visitedPaths = (ctx: Ctx) => [ + `${ctx.repo.path}/pulls/${rkeyOf(ctx, pullId)}`, + `${ctx.repo.path}/issues/${rkeyOf(ctx, ticketId)}`, + ctx.repo.path +]; + +const visitsRepoIssueAndPull = (chain: Flow) => + chain + .pull(pullId, patchId, { ...branches, title: pullTitle }) + .ticket(ticketId, { title: issueTitle }) + .open(page.repo) + .open(page.issue(ticketId)) + .sees(within(testid("ticketTitle"), text(issueTitle))) + .open(page.pull(pullId)) + .sees(within(testid("ticketTitle"), text(pullTitle))); + +const listsThemNewestFirst = (chain: Flow) => + chain + .open(home) + .step("the pull, the issue and the repo are listed in that order", async (ctx) => { + await expect.poll(() => rowPaths(ctx)).toEqual(visitedPaths(ctx)); + }) + .sees(within(row("pull"), text(pullTitle))) + .sees(within(row("issue"), text(issueTitle))); + +onBranches("the home page lists the repo, issue and pull just opened, newest first") + .use(visitsRepoIssueAndPull) + .use(listsThemNewestFirst) + .click(row("issue")) + .at(page.issue(ticketId)); + +onBranches("recent pages follow the account to another browser") + .use(visitsRepoIssueAndPull) + .step("pocket keeps the three pages", (ctx) => + until(`pocket should keep the recents of ${ctx.actor.handle}`, 30_000, async () => { + const items = await recentsIn(ctx.actor); + return [items.length === 3, `it keeps ${JSON.stringify(items)}`]; + }) + ) + .device("laptop") + .use(listsThemNewestFirst); diff --git a/e2e/specs/refusals.spec.ts b/e2e/specs/refusals.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..9298108d15932ca5d08f222b5b03d244f7272103 --- /dev/null +++ b/e2e/specs/refusals.spec.ts @@ -0,0 +1,333 @@ +import { expect } from "@playwright/test"; + +import { + comments, + editCob, + erase, + knotRecord, + knotRecords, + knotReply, + serviceDo, + spindlePost, + tickets +} from "../harness/api"; +import { config, seeded } from "../harness/config"; +import { + type Ctx, + bobbin, + declarationId, + hydrant, + knot, + replacing, + through, + flow, + not, + page, + pullId, + patchId, + refusal, + settingsAt, + type Target, + testid, + ticketId +} from "../harness/flow"; +import { get, partsOf, request, selfUri, stamp, until, xrpc } from "../harness/net"; +import { spindle } from "../harness/services"; +import { forgetTheLogin, manifestOf, workflow, workflowPath } from "../harness/shared"; + +// has to match KNOT_LEGACY_ADMIN_SECRET in svc/knot2 +const mintInvite = async () => { + const reply = await request(xrpc(config.knotUrl, "org.tangled.temp.server.createInviteCode"), { + method: "POST", + basic: "admin:super-strong-devonly-key", + json: { useCount: 1 } + }); + expect(reply.status, `the knot answered ${JSON.stringify(reply.body)}`).toBe(200); + return reply.body.code as string; +}; +const redeem = (who: Ctx["actor"], code: string) => + serviceDo(who, config.knotUrl, config.knotDid, "org.tangled.temp.server.redeemInviteCode", { + json: { code } + }); +const memberRecord = (did: string) => + knotRecord(`at://${config.knotDid}/org.tangled.knot.member/${did}`); +const invites = new WeakMap(); + +flow("a knot invite that was already used can't make a second member") + .actor("first") + .step("the knot mints a one use invite", async (ctx) => { + invites.set(ctx, await mintInvite()); + }) + .step("the first account redeems it", async (ctx) => { + const reply = await redeem(ctx.actor, invites.get(ctx)!); + expect(reply.status, `the knot answered ${JSON.stringify(reply.body)}`).toBe(200); + }) + .actor("second") + .rejects( + "the second account redeeming the same code", + (ctx) => redeem(ctx.actor, invites.get(ctx)!), + 400 + ) + .step("the knot has no member record for the second account", async (ctx) => { + expect(await memberRecord(ctx.actor.did)).toBeUndefined(); + }); + +const CI = "org.tangled.ci"; + +flow( + "asking spindle about a pipeline, branch or workflow the repo doesn't have is the caller's mistake" +) + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ [workflowPath]: workflow }) + .step("spindle has a pipeline for the push", (ctx) => + spindle.hasPipeline(ctx.repo.did, ctx.pushed.sha, "push") + ) + .rejects( + "cancelling a pipeline that doesn't exist", + (ctx) => + spindlePost(ctx.actor, `${CI}.cancelPipeline`, { + repo: ctx.repo.did, + pipeline: "nonexistent" + }), + 404 + ) + .rejects( + "listing the workflows of a branch that doesn't exist", + (ctx) => + get( + xrpc(config.spindleUrl, `${CI}.listWorkflowDefinitions`, { + repo: ctx.repo.did, + ref: "no-such-branch" + }) + ), + 404 + ) + .rejects( + "running a workflow the repo doesn't have", + (ctx) => + spindlePost(ctx.actor, "org.tangled.event.notifyEvent", { + repo: ctx.repo.did, + event: { + $type: `${CI}.trigger.manual`, + repo: ctx.repo.did, + commit: ctx.pushed.sha, + workflows: ["nope.yml"] + } + }), + 400 + ); + +const hooksNew = settingsAt("hooks/new"); + +// a button that starts off proves nothing, so see it turn on first +const isOn = (button: Target) => async (ctx: Ctx) => + expect(button.locate(ctx.page, ctx)).toBeEnabled(); +const isOff = (button: Target) => async (ctx: Ctx) => + expect(button.locate(ctx.page, ctx)).toBeDisabled(); + +flow("a webhook url that isn't http or https is refused and spindle keeps no webhook") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .open(hooksNew) + .fill(testid("draftUrl"), "https://example.com/hook") + .step("the add turns on", isOn(testid("add"))) + .fill(testid("draftUrl"), "javascript:alert(1)") + .step("the add turns off", isOff(testid("add"))) + .step("spindle lists no webhook", async (ctx) => { + const reply = await serviceDo( + ctx.actor, + config.spindleUrl, + config.spindleDid, + "org.tangled.webhook.listWebhooks", + { params: { repo: ctx.repo.did } } + ); + expect(reply.body?.webhooks ?? []).toEqual([]); + }); + +const secretsNew = settingsAt("pipelines/secrets/new"); +const secretsOf = async (ctx: Ctx): Promise => { + const reply = await serviceDo( + ctx.actor, + config.spindleUrl, + config.spindleDid, + "org.tangled.secret.listSecrets", + { params: { repo: ctx.repo.did } } + ); + expect(reply.status, `spindle answered ${JSON.stringify(reply.body)}`).toBe(200); + return (reply.body.secrets ?? []).map((secret: { key: string }) => secret.key); +}; +const secretsBefore = new WeakMap(); + +flow("a secret key with characters a workflow can't name is refused and spindle keeps nothing") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .step("count spindle's secrets", async (ctx) => { + secretsBefore.set(ctx, await secretsOf(ctx)); + }) + .open(secretsNew) + .fill(testid("name"), "not a key!") + .fill(testid("secretValue"), "hush") + .click(testid("add")) + // spindle refusing it is what puts the alert up, so the write is settled by then + .sees(refusal) + .at(secretsNew) + // other flows can share the repo did, so only this key and the count are ours to check + .step("spindle kept no new secret", async (ctx) => { + const keys = await secretsOf(ctx); + expect(keys).not.toContain("not a key!"); + expect(keys).toHaveLength(secretsBefore.get(ctx)!.length); + }); + +flow("a repo name with characters a repo can't have is refused and nothing is declared") + .actor("owner") + .open(page.newRepo) + .fill(testid("name"), "not a/repo name") + .click(testid("submit")) + .sendsNothing(declarationId) + .at(page.newRepo); + +flow("an issue form with an empty title can't be sent") + .actor("owner") + .repo() + .open(page.newIssue) + .fill(testid("title"), "a title") + .step("the submit turns on", isOn(testid("submit"))) + .fill(testid("title"), " ") + .step("the submit turns off", isOff(testid("submit"))) + .sendsNothing(ticketId); + +const written = new WeakMap(); + +flow("a comment on an issue that was erased is refused and the knot keeps no comment") + .actor("owner") + .repo() + .step("open an issue and erase it", async (ctx) => { + const ticket = await tickets.open(ctx.actor, ctx.repo.did, "here and gone"); + written.set(ctx, ticket); + await erase(ctx.actor, ticket); + }) + .rejects( + "commenting on the erased issue", + (ctx) => comments.add(ctx.actor, ctx.repo.did, written.get(ctx)!, "too late"), + [400, 404, 409] + ) + .step("the knot has no comment", async (ctx) => { + expect(await knotRecords(ctx.repo.did, "org.tangled.feed.comment")).toHaveLength(0); + }); + +flow("editing someone else's comment through the api is refused and the comment stays") + .actor("owner") + .repo() + .step("the owner comments on an issue", async (ctx) => { + const ticket = await tickets.open(ctx.actor, ctx.repo.did, "mine to say"); + const reply = await comments.add(ctx.actor, ctx.repo.did, ticket, "my words"); + expect(reply.status).toBe(200); + written.set(ctx, { uri: reply.body.uri, cid: reply.body.cid }); + }) + .actor(seeded.charlie) + .rejects( + "charlie rewriting the owner's comment", + (ctx) => + editCob(ctx.actor, written.get(ctx)!.uri, "edit", (stored) => ({ + ...stored, + body: { ...stored.body, text: "charlie's words" } + })), + 403 + ) + .step("the knot still has the owner's words", async (ctx) => { + const stored = await knotRecord(written.get(ctx)!.uri); + expect(stored?.value.body.text).toBe("my words"); + }); + +flow("deleting someone else's repo through the api is refused and the repo stays") + .actor("owner") + .repo() + .actor(seeded.charlie) + .rejects( + "charlie deleting the owner's repo", + (ctx) => + serviceDo(ctx.actor, config.knotUrl, config.knotDid, "org.tangled.repo.deleteRepo", { + json: { repo: ctx.repo.did } + }), + [401, 403] + ) + .step("the knot still has the repo's manifest", async (ctx) => { + expect(await knotRecord(selfUri(ctx.repo.did, "org.tangled.repo.manifest"))).toBeDefined(); + }); + +// the knot fetches the head before taking a patch, so an all zero oid gets refused +const headNobodyHas = (ctx: Ctx) => + ctx.page.route("**/xrpc/org.tangled.review.submitPatch", async (route) => { + const body = route.request().postDataJSON(); + const oid = body.patch.head.oid as string; + await route.fallback({ + postData: JSON.stringify({ + ...body, + patch: { ...body.patch, head: { oid: "0".repeat(oid.length) } } + }) + }); + }); + +flow("a pull whose patch the knot refuses leaves no pull ticket behind") + .actor("owner") + .repo() + .push({ "e2e/base.txt": "base\n" }) + .push({ "e2e/feature.txt": "feature\n" }, { branch: "feature", base: "main" }) + .mirrored() + .open(page.newPull({ from: "feature", into: "main" })) + .step("the knot will refuse the patch", headNobodyHas) + .fill(testid("title"), "this one won't take") + .click(testid("submit")) + .sends(pullId, knot, { record: { state: "open" } }, "create") + .trace(pullId, through(knot)) + .refused(patchId, [400, 404, 409, 422]) + // no patch means no diff, so the page deletes the pull ticket it just made + .sends(pullId, knot, replacing(pullId), "delete") + .trace(pullId, through(knot, hydrant, bobbin)) + .sees(refusal) + .step("the knot lists no ticket", async (ctx) => { + expect(await knotRecords(ctx.repo.did, "org.tangled.track.ticket")).toHaveLength(0); + }); + +const names = new WeakMap(); + +flow("renaming a repo to a name the owner already has is refused and nothing moves") + .actor("owner") + .repo() + .step("remember the first repo", async (ctx) => { + names.set(ctx, ctx.repo.name); + }) + .repo() + .open(settingsAt("rename")) + .fill(testid("name"), (ctx) => names.get(ctx)!) + .click(testid("rename")) + .sees(refusal) + .sendsNothing(declarationId) + .step("the knot's manifest keeps the old name", async (ctx) => { + expect((await manifestOf(ctx)).declaration).toMatchObject({ name: ctx.repo.name }); + }); + +const fresh = new WeakMap(); + +flow("a signup with a token the server never sent is refused and makes no account") + .actor("owner") + .open(page.profile) + .step("open the signup verify page signed out", async (ctx) => { + fresh.set(ctx, `nobody${stamp()}`.slice(0, 18)); + await forgetTheLogin(ctx); + await ctx.page.goto("/signup/verify"); + }) + .fill(testid("token"), "not-a-real-token") + .fill(testid("signupUsername"), (ctx) => fresh.get(ctx)!) + .fill(testid("password"), config.password) + .click(testid("submit")) + .sees(refusal) + .step("the pds has no account by that handle", async (ctx) => { + const { status } = await get( + xrpc(config.pdsUrl, "com.atproto.identity.resolveHandle", { + handle: `${fresh.get(ctx)}.pds.tngl.boltless.dev` + }) + ); + expect(status).not.toBe(200); + }); diff --git a/e2e/specs/repo-settings.spec.ts b/e2e/specs/repo-settings.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..102242fdd6a5bb4836e8d6aed318537d7461d1b3 --- /dev/null +++ b/e2e/specs/repo-settings.spec.ts @@ -0,0 +1,428 @@ +import { expect } from "@playwright/test"; + +import { editCob, sites as sitesApi } from "../harness/api"; +import { config } from "../harness/config"; +import { + bobbin, + button, + component, + type Ctx, + exactLink, + flow, + hydrant, + knot, + manifestId, + not, + page, + placeholder, + renameForm, + saveButton, + settingsAt, + target, + text, + testid, + through, + topic, + topicChip, + unique, + unsaved, + where, + within +} from "../harness/flow"; +import { headOf } from "../harness/git"; +import { get, selfUri, until, xrpc } from "../harness/net"; +import { + base, + finished, + manifestOf, + renamesTo, + saveFinishes, + topicsOnKnot +} from "../harness/shared"; + +let renamedTo = ""; +const renamedPath = (ctx: Ctx) => `/${ctx.actor.handle}/${renamedTo}`; +const repoName = (name: () => string) => + target("the repo's name in its header", (tab) => + tab.getByRole("link", { name: name(), exact: true }).first() + ); + +flow("renaming a repo made today moves it to the new name") + .actor("owner") + .repo() + .step("pick the new name", async (ctx) => { + renamedTo = `${ctx.repo.name}-renamed`; + }) + .use(renamesTo(() => renamedTo)) + .step("the rename finishes on the new settings page", async (ctx) => + finished( + ctx, + "the rename did not finish", + ctx.page.waitForURL(`${config.webUrl}${renamedPath(ctx)}/settings`, { timeout: 5000 }) + ) + ) + .open(where("the renamed repo", renamedPath)) + .sees(repoName(() => renamedTo)) + .step("the old name is gone", async (ctx) => { + const old = await ctx.page.goto(`${config.webUrl}${ctx.repo.path}`); + expect(old?.status()).toBe(404); + }) + .step("the knot's manifest names the new repo", async (ctx) => { + expect((await manifestOf(ctx)).declaration).toMatchObject({ name: renamedTo }); + }); + +const blurb = unique("does one thing well"); + +flow("saving a description in the repo settings shows it on the repo page") + .actor("owner") + .repo() + + .open(page.settings) + .fill(testid("formDescription"), blurb) + .click(saveButton) + .trace(manifestId, through(knot, hydrant, bobbin)) + .step("the save finishes, which leaves nothing to save", saveFinishes) + .step("the knot's manifest carries it", async (ctx) => { + expect((await manifestOf(ctx)).description).toBe(blurb); + }) + .open(page.repo) + .reload() + .sees(within(testid("repoDescription"), text(blurb))); + +const sites = settingsAt("sites"); + +const branchPicker = within(testid("formDefaultBranch"), testid("trigger")); +const branchOption = (name: string) => testid("option", { value: name }); + +flow("a branch pushed over git is offered in the default branch picker") + .actor("owner") + .repo() + .push(base) + .push({ "dev-only.txt": "only here\n" }, { branch: "feature/picker", base: "main" }) + .mirrored() + + .open(page.settings) + .sees(branchPicker, "main") + .click(branchPicker) + .sees(branchOption("main")) + .sees(branchOption("feature/picker")); + +flow("changing the default branch in the repo settings moves the knot's head") + .actor("owner") + .repo() + .push(base) + .push({ "dev-only.txt": "only here\n" }, { branch: "develop", base: "main" }) + .mirrored() + + .open(page.settings) + .click(branchPicker) + .click(branchOption("develop")) + .click(saveButton) + .sendsNothing(manifestId) + .step("the knot's head is develop", async (ctx) => { + await until(`the knot should point HEAD at develop`, 20_000, async () => { + const head = await headOf(ctx.actor, ctx.repo.did); + return [head === "develop", `HEAD is ${head}`]; + }); + }) + .step("bobbin reports develop as the default", async (ctx) => { + await until( + `bobbin should report develop as the default of ${ctx.repo.did}`, + 60_000, + async () => { + const { status, body } = await get( + xrpc(config.bobbinUrl, "org.tangled.temp.git.listBranches", { + repo: ctx.repo.did, + limit: "1" + }) + ); + return [ + status === 200 && body.defaultBranch === "develop", + `it said ${status} ${body?.defaultBranch}` + ]; + } + ); + }) + .reload() + .sees(branchPicker, "develop") + .open(page.repo) + .sees(exactLink("dev-only.txt")); + +const liveTag = testid("siteLiveTag"); +const siteHeading = testid("settingsHeaderTitle"); +const directory = testid("formDirectory"); +const directoryIs = (want: string) => async (ctx: Ctx) => { + await expect(directory.locate(ctx.page, ctx)).toHaveValue(want); +}; +const disableButton = testid("confirmDisable"); +const disableForGood = testid("disableSite"); + +// a site needs a claimed domain before the panel offers anything to save +const siteDomain = (ctx: Ctx) => `${ctx.actor.name}.${config.sitesDomain}`; +const claimDomain = (ctx: Ctx) => sitesApi.claim(ctx.actor, siteDomain(ctx)); +const indexSite = target("the index site option", (tab) => + tab.getByRole("radio", { name: "Index site" }) +); +const subpathSite = target("the sub-path site option", (tab) => + tab.getByRole("radio", { name: "Sub-path site" }) +); +const servedAt = (at: (ctx: Ctx) => string, want: number, body?: string) => async (ctx: Ctx) => { + const url = `https://${siteDomain(ctx)}/${at(ctx)}`; + await until( + `${url} should answer ${want}${body ? ` with ${JSON.stringify(body)}` : ""}`, + 90_000, + async () => { + const reply = await get(url); + const done = reply.status === want && (body === undefined || reply.body === body); + return [done, `it said ${reply.status} ${JSON.stringify(reply.body).slice(0, 80)}`]; + } + ); +}; +const served = (path: string, want: number, body?: string) => + servedAt((ctx) => `${ctx.repo.name}/${path}`, want, body); +const servedAtRoot = (path: string, want: number, body?: string) => + servedAt(() => path, want, body); + +flow("a repo that never published a site doesn't say it is live") + .actor("owner") + .repo() + .open(sites) + .sees(siteHeading) + .sees(not(liveTag)) + .sees(testid("settingsEmpty")); + +flow("publishing a site from a directory serves that directory and keeps it after a reload") + .actor("owner") + .repo() + .step("claim a sites domain", claimDomain) + .push({ "index.html": "the root\n", "docs/index.html": "the docs\n" }) + .open(sites) + .fill(directory, "/docs") + .click(saveButton) + .step("the save bar has nothing left to save", saveFinishes) + .step("the site serves the docs index.html", served("index.html", 200, "the docs\n")) + .reload() + .step("the directory is still /docs", directoryIs("/docs")) + .sees(liveTag) + .step("the sites worker keeps /docs", async (ctx) => { + expect(await sitesApi.config(ctx.actor, ctx.repo.did)).toMatchObject({ dir: "/docs" }); + }); + +flow("a published site serves what was pushed and follows the next push") + .actor("owner") + .repo() + .step("claim a sites domain", claimDomain) + .push({ "index.html": "first\n" }) + .open(sites) + // an unconfigured repo can save its prefilled branch, which is what publishes it + .click(saveButton) + .step("the save bar has nothing left to save", saveFinishes) + .step("the site serves the first index.html", served("index.html", 200, "first\n")) + .push({ "index.html": "second\n" }) + .step("the push redeploys the site", served("index.html", 200, "second\n")); + +flow("a published site serves files whose names have spaces or non-ascii characters") + .actor("owner") + .repo() + .step("claim a sites domain", claimDomain) + .push({ "index.html": "home\n", "sp ace.txt": "a space\n", "ü.txt": "an umlaut\n" }) + .open(sites) + .click(saveButton) + .step("the save bar has nothing left to save", saveFinishes) + .step("the site serves index.html", served("index.html", 200, "home\n")) + .step("the site serves sp ace.txt", served(encodeURIComponent("sp ace.txt"), 200, "a space\n")) + .step("the site serves ü.txt", served(encodeURIComponent("ü.txt"), 200, "an umlaut\n")); + +flow("a second repo can't take the index site another repo holds") + .actor("owner") + .repo() + .step("claim a sites domain", claimDomain) + .push({ "index.html": "first\n" }) + .open(sites) + .click(indexSite) + .click(saveButton) + .step("the save bar has nothing left to save", saveFinishes) + .step("the first repo is the index site", servedAtRoot("index.html", 200, "first\n")) + .repo() + .push({ "index.html": "second\n" }) + .open(sites) + .click(indexSite) + .click(saveButton) + .sees(text(/already the index site/)) + .step("the first repo is still the index site", servedAtRoot("index.html", 200, "first\n")); + +flow("the index site's repo can move to a sub-path and back") + .actor("owner") + .repo() + .step("claim a sites domain", claimDomain) + .push({ "index.html": "home\n" }) + .open(sites) + .click(indexSite) + .click(saveButton) + .step("the save bar has nothing left to save", saveFinishes) + .step("the domain root serves it", servedAtRoot("index.html", 200, "home\n")) + .click(subpathSite) + .click(saveButton) + .step("the save bar has nothing left to save", saveFinishes) + .step("it moves under its name", served("index.html", 200, "home\n")) + .step("the domain root has no site", servedAtRoot("index.html", 404)) + .click(indexSite) + .click(saveButton) + .step("the save bar has nothing left to save", saveFinishes) + .step("the domain root serves it again", servedAtRoot("index.html", 200, "home\n")); + +flow("a site config that names the repo something else is refused") + .actor("owner") + .repo() + .step("claim a sites domain", claimDomain) + .push({ "index.html": "home\n" }) + .step("saving it under another name is refused", async (ctx) => { + const reply = await sitesApi.saveReply(ctx.actor, { + repoDid: ctx.repo.did, + name: "not-the-name", + rkey: ctx.repo.name, + knot: new URL(config.knotUrl).host, + branch: ctx.repo.defaultBranch, + dir: "/" + }); + expect(reply).toMatchObject({ status: 409, body: { error: "RepoIdentityMismatch" } }); + }); + +flow("a deploy from a missing directory says which directory, not where sitesd unpacked it") + .actor("owner") + .repo() + .step("claim a sites domain", claimDomain) + .push({ "index.html": "home\n" }) + .open(sites) + .fill(directory, "/does/not/exist") + .click(saveButton) + .step("the save bar has nothing left to save", saveFinishes) + .step("the deploy fails", async (ctx) => { + await until("the deploy history should record a failure", 90_000, async () => { + const deploys = await sitesApi.history(ctx.actor, ctx.repo.did); + return [deploys[0]?.status === "failure", `it has ${JSON.stringify(deploys)}`]; + }); + }) + .reload() + .sees(text('deploy directory "/does/not/exist" does not exist')); + +const accountSites = where("the account's sites settings", () => "/settings/sites"); + +flow("the sites service's own label can't be claimed as a domain") + .actor("owner") + .open(accountSites) + .fill(placeholder("floo"), "sites") + .click(button("Claim")) + .sees(text(/reserved subdomain/)) + .sees(testid("settingsEmpty")); + +flow("disabling a site takes it off the air and stays off after a reload") + .actor("owner") + .repo() + .step("claim a sites domain", claimDomain) + .push({ "index.html": "on air\n" }) + .open(sites) + .click(saveButton) + .step("the save bar has nothing left to save", saveFinishes) + .step("the site is up", served("index.html", 200, "on air\n")) + .click(disableButton) + .click(disableForGood) + // the disable only goes out after a token mint, so reloading straight away can cancel it + .sees(not(liveTag)) + .reload() + .sees(not(liveTag)) + .step("the site is gone", served("index.html", 404)); + +let firstName = ""; + +flow("renaming a repo to a name the owner already has is refused") + .actor("owner") + .repo() + .step("remember the first repo", async (ctx) => { + firstName = ctx.repo.name; + }) + .repo() + + .open(renameForm) + .fill(testid("name"), () => firstName) + .click(testid("rename")) + .step("the form says the name is taken", (ctx) => + finished( + ctx, + "the form did not refuse the name", + expect(testid("errorAlert").locate(ctx.page, ctx)).toBeVisible() + ) + ) + .at(renameForm); + +const noTopics = async (ctx: Ctx) => expect(await topicsOnKnot(ctx)).toEqual([]); + +const savedTopic = async (ctx: Ctx) => { + const reply = await editCob( + ctx.actor, + selfUri(ctx.repo.did, manifestId.nsid), + "save a topic on", + (manifest) => ({ ...manifest, topics: [topic] }) + ); + expect(reply.status, `saving the topic answered ${JSON.stringify(reply.body)}`).toBe(200); + ctx.set(manifestId, reply.body); +}; + +flow("removing a topic that was already saved") + .actor("owner") + .repo() + .step("the account has the repo saved with a topic", savedTopic) + + .open(page.settings) + .sees(topicChip) + .click(testid("removeAt")) + .click(saveButton) + .trace(manifestId, through(knot, hydrant, bobbin)) + .sees(not(unsaved)) + .step("the knot's manifest has no topics", noTopics) + .reload() + .sees(not(topicChip)); + +const resetSettings = within(component("SettingsHeader"), testid("reset")); + +flow("adding a topic in the settings and removing it again") + .actor("owner") + .repo() + + .open(page.settings) + .fill(testid("topics"), topic) + .press("Enter") + .sees(topicChip) + .click(saveButton) + .trace(manifestId, through(knot, hydrant, bobbin)) + .sees(not(unsaved)) + .step("the knot's manifest lists the topic", async (ctx) => + expect(await topicsOnKnot(ctx)).toEqual([topic]) + ) + .reload() + .sees(topicChip) + + .click(testid("removeAt")) + .click(saveButton) + .trace(manifestId, through(knot, hydrant, bobbin)) + .sees(not(unsaved)) + .step("the knot's manifest has no topics", noTopics) + .reload() + .sees(not(topicChip)); + +flow("resetting an unsaved topic in the settings writes nothing") + .actor("owner") + .repo() + + .open(page.settings) + .fill(testid("topics"), topic) + .press("Enter") + .sees(topicChip) + .sees(unsaved) + .click(resetSettings) + .sendsNothing(manifestId) + .sees(not(topicChip)) + .sees(not(unsaved)) + .reload() + .sees(not(topicChip)) + .step("the knot's manifest has no topics", noTopics); diff --git a/e2e/specs/repos.spec.ts b/e2e/specs/repos.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..0b17a5906be3e37d313c7d935697f41cda761356 --- /dev/null +++ b/e2e/specs/repos.spec.ts @@ -0,0 +1,490 @@ +import { expect, test } from "@playwright/test"; + +import { importStatus, knotRecord } from "../harness/api"; +import { config, seeded } from "../harness/config"; +import { + bobbin, + button, + component, + type Ctx, + declarationId, + exactLink, + flow, + type Flow, + git, + hydrant, + knot, + knotCopy, + manifestId, + not, + page, + pds, + refsId, + target, + testid, + text, + through, + where, + within +} from "../harness/flow"; +import { lsRemote, push } from "../harness/git"; +import { partsOf, selfUri, stamp, until } from "../harness/net"; +import { base, forgetTheLogin, manifestOf, mirrorLists, recall, remember } from "../harness/shared"; + +const files = { "README.md": "hello from e2e\n" }; + +flow("forking a repo") + .actor("owner") + .repo() + .push(files) + + .actor("forker") + .fork() + .follow(manifestId, through(knotCopy, hydrant, bobbin)) + .step("the manifest names the upstream", async (ctx) => { + const value = await knot.has(ctx.get(manifestId)); + expect(value.source).toMatchObject({ did: ctx.upstream.did }); + }) + .step("the fork starts from the upstream's commit", async (ctx) => { + expect(await lsRemote(ctx.actor, ctx.repo.did)).toMatchObject({ + "refs/heads/main": ctx.pushed.sha + }); + }); + +flow("pushing to a fork") + .actor("owner") + .repo() + .push(files) + + .actor("forker") + .fork() + .push({ "fork.txt": "only in the fork\n" }) + .follow(refsId, through(git, knotCopy, hydrant, bobbin)) + .step("the fork moved and the upstream didn't", async (ctx) => { + expect((await lsRemote(ctx.actor, ctx.repo.did))["refs/heads/main"]).toBe(ctx.pushed.sha); + expect((await lsRemote(ctx.actor, ctx.upstream.did))["refs/heads/main"]).toBe( + ctx.upstream.heads.main + ); + }); + +// one flow at a time per worker, so a module-level let is safe +let made = ""; + +flow("creating a repo from the new repository form") + .actor("owner") + .member() + .step("pick a name", async () => { + made = `e2e-${stamp()}`; + }) + + .open(page.newRepo) + .fill(testid("name"), () => made) + .fill(testid("description"), "made by the form") + .click(testid("submit")) + .traceNewRepo(() => made, through(pds, hydrant, bobbin)) + .follow(manifestId, through(knotCopy, hydrant, bobbin)) + .step("the manifest names the declaration and carries the description", async (ctx) => { + expect(await manifestOf(ctx)).toMatchObject({ + declaration: { owner: ctx.actor.did, name: made }, + description: "made by the form" + }); + }) + .at(page.repo) + .reload() + .sees( + target("the repo's name", (tab, ctx) => + tab.getByRole("link", { name: ctx.repo.name, exact: true }).first() + ) + ); + +// no .mirrored() and no reload, the form itself has to wait for the mirror +flow("creating a repo from the form opens a page that already works") + .actor("owner") + .member() + .step("pick a name", async () => { + made = `e2e-${stamp()}`; + }) + + .open(page.newRepo) + .fill(testid("name"), () => made) + .click(testid("submit")) + .traceNewRepo(() => made, through(pds, hydrant, bobbin)) + .at(page.repo, 45_000) + .sees(testid("emptyRepoNote")); + +flow("a name the owner already has is refused by the form") + .actor("owner") + .repo() + + .open(page.newRepo) + .fill(testid("name"), (ctx) => ctx.repo.name) + .click(testid("submit")) + .sees(testid("errorAlert")) + .at(page.newRepo); + +const typedName = testid("confirmName"); + +flow("the delete dialog forgets a name you typed and cancelled") + .actor("owner") + .repo() + + .open(page.settings) + .click(testid("setConfirmDelete")) + .fill(typedName, (ctx) => ctx.repo.name) + .click(testid("clearConfirmDelete")) + .click(testid("setConfirmDelete")) + .step("the name field is empty again", async (ctx) => { + await expect(typedName.locate(ctx.page, ctx)).toHaveValue(""); + }) + .step("the delete button is disabled again", async (ctx) => { + await expect(testid("destroy").locate(ctx.page, ctx)).toBeDisabled(); + }); + +flow("a repo nobody pushed to says it is empty") + .actor("owner") + .repo() + .mirrored() + .open(page.repo) + .sees(testid("emptyRepoNote")); + +const confirm = testid("destroy"); + +flow("deleting a repo") + .actor("owner") + .repo() + + .open(page.settings) + .click(testid("setConfirmDelete")) + .fill(testid("confirmName"), (ctx) => ctx.repo.name) + .click(confirm) + .sends(declarationId, pds, (ctx) => ({ rkey: ctx.repo.name }), "delete") + .trace(declarationId, through(pds, hydrant, bobbin)) + // the page only asks the knot once the declaration is gone everywhere + .step("the knot has dropped the repo", (ctx) => + until(`the knot should have dropped ${ctx.repo.did}`, 10_000, async () => [ + (await knotRecord(selfUri(ctx.repo.did, manifestId.nsid))) === undefined, + "it still has the manifest" + ]) + ) + .at(page.profile); + +const forkForm = where("the fork form", (ctx) => `${ctx.repo.path}/fork`); +const forkLink = testid("fork"); +let upstreamOfFork = { did: "", handle: "", name: "" }; +const noteUpstream = async (ctx: Ctx) => { + upstreamOfFork = { did: ctx.repo.did, handle: ctx.actor.handle, name: ctx.repo.name }; +}; + +flow("forking a repo from its Fork button") + .actor("owner") + .repo() + .push(base) + .step("note the upstream", noteUpstream) + .step("remember the owner", remember("owner")) + + .actor("forker") + .member() + .open(page.repo) + .click(forkLink) + .at(forkForm) + .click(testid("submit")) + .traceNewRepo((ctx) => ctx.repo.name, through(pds, hydrant, bobbin)) + .follow(manifestId, through(knotCopy, hydrant, bobbin)) + .step("the manifest names the upstream", async (ctx) => { + expect((await manifestOf(ctx)).source).toMatchObject({ did: upstreamOfFork.did }); + }) + .step("the fork starts from the upstream's commit", async (ctx) => { + expect(await lsRemote(ctx.actor, ctx.repo.did)).toMatchObject({ + "refs/heads/main": (await lsRemote(recall(ctx, "owner"), upstreamOfFork.did))[ + "refs/heads/main" + ] + }); + }) + .at(page.repo) + .reload() + .sees(testid("forkedFromPrefix")) + .sees(exactLink(() => `${upstreamOfFork.handle}/${upstreamOfFork.name}`)); + +// no .mirrored() and no reload, the form itself has to wait for the fork's files to be mirrored +flow("forking a repo from its Fork button opens a page that already has the files") + .actor("owner") + .repo() + .push(base) + + .actor("forker") + .member() + .open(page.repo) + .click(forkLink) + .at(forkForm) + .click(testid("submit")) + .traceNewRepo((ctx) => ctx.repo.name, through(pds, hydrant, bobbin)) + .at(page.repo, 45_000) + // the commit message is also called e2e, so a link by name finds two + .sees(target("the e2e directory", (tab) => tab.locator('a[href$="/tree/main/e2e"]'))); + +const syncFork = button("Sync fork"); +let upstreamHead = ""; + +flow("syncing a fork brings in what the upstream pushed since") + .actor("owner") + .repo() + .push(base) + .step("note the upstream", noteUpstream) + .step("remember the owner", remember("owner")) + + .actor("forker") + .fork() + .step("the upstream moves on", async (ctx) => { + ({ sha: upstreamHead } = await push( + recall(ctx, "owner"), + upstreamOfFork.did, + { "later.txt": "pushed after the fork\n" }, + { branch: "main" } + )); + }) + .open(page.repo) + .sees(syncFork) + .click(syncFork) + .step("the fork's main is the upstream's new commit", async (ctx) => { + await until(`the fork should be at ${upstreamHead}`, 20_000, async () => { + const heads = await lsRemote(ctx.actor, ctx.repo.did, "refs/heads/"); + return [ + heads["refs/heads/main"] === upstreamHead, + `it is at ${heads["refs/heads/main"]}` + ]; + }); + }) + .reload() + .sees(not(syncFork)); + +const migrate = where("the migrate page", () => "/repo/migrate"); +const stubRepo = "https://github.tngl.boltless.dev/octostub/fresh.git"; +const gitUrlMethod = within(component("MethodPicker"), testid("select", { value: "url" })); +const gitUrlField = testid("url"); +const importName = testid("repoConfigName"); +const fetchDetails = testid("fetchDetails"); +const startMigration = testid("importUrl"); + +// this page polls forever, so .open() would never go idle +const goToMigrate = async (ctx: Ctx) => { + await ctx.page.goto(migrate.path(ctx), { waitUntil: "load" }); +}; + +let imported = ""; +const pickImportName = async () => { + imported = `e2e-${stamp()}`; +}; + +for (const [what, url] of [ + ["a plain http git url", "http://example.com/some/repository.git"], + ["a local address", "https://127.0.0.1/some/repository.git"] +]) { + flow(`the import form refuses ${what}`) + .actor("owner") + .member() + .open(page.profile) + .step("open the migrate page", goToMigrate) + .click(gitUrlMethod) + .fill(gitUrlField, url) + .click(fetchDetails) + .sees(testid("errorAlert")); +} + +// the first migration's grant goes through the pds login page +const grantTheMigrator = async (ctx: Ctx) => { + await ctx.page.waitForURL((url) => url.host.startsWith("pds.")); + await ctx.page.locator("input[type=password]").fill(config.password); + await ctx.page.getByRole("button", { name: "Sign in", exact: true }).click(); + await ctx.page.getByRole("button", { name: "Authorize", exact: true }).click(); + await ctx.page.waitForURL((url) => url.origin === config.webUrl); +}; + +const fetched = testid("remoteBranchNotice"); +// without a grant the git url form asks for one with this button instead of redirecting +const authorizeInline = testid("authorizeMigrator"); + +flow("coming back from the migrator grant shows the git url form again") + .actor("owner") + .member() + .open(page.profile) + .step("open the migrate page", goToMigrate) + .click(gitUrlMethod) + .fill(gitUrlField, stubRepo) + .click(fetchDetails) + .sees(fetched) + .click(startMigration) + .click(authorizeInline) + .step("grant the migrator access", grantTheMigrator) + .click(gitUrlMethod) + .sees(gitUrlField); + +const mirrorHas = (branch: string) => mirrorLists("Branches", branch, true); + +const repoOnBobbin = (name: string) => async (ctx: Ctx) => { + const found = await until(`bobbin should know the imported repo ${name}`, 60_000, async () => { + const { status, body } = await bobbin.repoByName(ctx.actor.did, name); + return [status === 200, `getRepoByName answered ${status}`, body]; + }); + // the uri authority of a manifest is the repo did + const did = partsOf(found.uri).repo; + ctx.adoptRepo({ did, name }); +}; + +const stubBranchOnKnot = async (ctx: Ctx) => + until(`the knot should list main on ${ctx.repo.did}`, 60_000, async () => { + const heads = await lsRemote(ctx.actor, ctx.repo.did, "refs/heads/").catch(() => ({})); + return [Object.keys(heads).includes("refs/heads/main"), `it lists ${Object.keys(heads)}`]; + }); + +// the page is the only place a failed migration says why +const migrationFinishes = async (ctx: Ctx) => { + test.setTimeout(240_000); + await until(`the migration of ${imported} should finish`, 120_000, async () => { + const said = (await ctx.page.locator("[data-testid=migrationRows]").allInnerTexts()) + .join(" | ") + .replace(/\s+/g, " "); + if (/Failed/.test(said)) + throw new Error(`the migration of ${imported} failed, the page said: ${said}`); + return [/\bDone\b/.test(said), `the page said: ${said}`]; + }); +}; + +flow("importing a repo from a git url brings its files over") + .actor("owner") + .member() + .open(page.profile) + .step("pick a name", pickImportName) + .step("open the migrate page", goToMigrate) + .click(gitUrlMethod) + .fill(gitUrlField, stubRepo) + .click(fetchDetails) + .sees(fetched) + .fill(importName, () => imported) + .click(startMigration) + .click(authorizeInline) + .step("grant the migrator access", grantTheMigrator) + .click(gitUrlMethod) + .sees(startMigration) + .click(startMigration) + .sees(testid("migrationStartedText")) + .step("the migration finishes", migrationFinishes) + .step("bobbin knows the new repo", async (ctx) => repoOnBobbin(imported)(ctx)) + .step("the knot has the stub's branch", stubBranchOnKnot) + .step("bobbin's mirror lists it too", mirrorHas("main")) + .open(page.repo) + .sees(exactLink("README.md")) + .sees(exactLink("src")); + +const connectGitHub = testid("connectGithub"); +const authorizeMigrator = testid("authorize"); +const pickStubRepo = testid("repoRowCheckbox", { repo: "octostub/hello" }); +const migrateOne = testid("migrate"); + +flow("importing a repo from github brings its files over") + .actor("owner") + .member() + .open(page.profile) + .step("the repo to bring over", async () => { + imported = "hello"; + }) + .step("open the migrate page", goToMigrate) + .click(connectGitHub) + .sees(testid("githubConnectedStatus")) + .click(authorizeMigrator) + .step("grant the migrator access", grantTheMigrator) + .click(pickStubRepo) + .click(migrateOne) + .sees(testid("migrationStartedText")) + .checkpoint("the migration started") + .step("the migration finishes", migrationFinishes) + .step("bobbin knows the new repo", async (ctx) => repoOnBobbin(imported)(ctx)) + .step("the knot has the stub's branch", stubBranchOnKnot) + .step("bobbin's mirror lists it too", mirrorHas("main")) + .open(page.repo) + .sees(exactLink("README.md")) + .sees(exactLink("hello.txt")); + +const helloRepo = "https://github.tngl.boltless.dev/octostub/hello.git"; +const unfetchableRepo = "https://github.tngl.boltless.dev/octostub/unfetchable.git"; +const hiddenReason = "the import from the source didn't finish; the repo owner can see details"; + +const importSays = (wanted: string) => async (ctx: Ctx) => + until(`the knot should say the import is ${wanted}`, 60_000, async () => { + const { status, body } = await importStatus(ctx.repo.did); + return [ + status === 200 && body?.status === wanted, + `it answered ${status} ${JSON.stringify(body)}` + ]; + }); + +const reasons = new WeakMap(); + +const ownerReadsWhatWentWrong = async (ctx: Ctx) => { + const { status, body } = await importStatus(ctx.repo.did, ctx.actor); + expect(status, JSON.stringify(body)).toBe(200); + expect(body.status).toBe("failed"); + expect(body.reason, "the owner should read what went wrong").toBeTruthy(); + expect(body.reason).not.toBe(hiddenReason); + reasons.set(ctx, body.reason); +}; + +const everyoneElseReadsTheHiddenReason = async (ctx: Ctx) => { + const asked = { "an anonymous caller": undefined, "a stranger": seeded.charlie }; + for (const [who, account] of Object.entries(asked)) { + const { status, body } = await importStatus(ctx.repo.did, account); + expect(status, `${who}: ${JSON.stringify(body)}`).toBe(200); + expect(body, who).toEqual({ status: "failed", reason: hiddenReason }); + } +}; + +const unknownRepoIsNotFound = async () => { + const { status, body } = await importStatus("did:plc:e2enotarepoanywhereaaaaa"); + expect(status, JSON.stringify(body)).toBe(404); + expect(body.error).toBe("RepoNotFound"); +}; + +const importFailed = text(/The import from .* failed\./); +const askTheOperator = text("Ask your knot operator for support."); +const ownersReason = target("the reason the owner read from the knot", (tab, ctx) => + tab.getByText(reasons.get(ctx) ?? "no reason was read") +); + +const seesTheFixedBanner = (flow: Flow) => + flow + .sees(importFailed) + .sees(askTheOperator) + .sees(not(ownersReason)) + .sees(not(text(hiddenReason))); + +flow("an import that lands says so on the knot") + .actor("owner") + .repo({ importFrom: helloRepo }) + .step("the knot says the import is present", importSays("present")) + .step("an anonymous caller reads it the same, with no reason", async (ctx) => { + expect((await importStatus(ctx.repo.did)).body).toEqual({ status: "present" }); + }) + .step("bobbin's mirror lists main", mirrorHas("main")) + .open(page.repo) + .sees(exactLink("hello.txt")) + .sees(not(importFailed)); + +flow( + "a failed import tells only its owner what went wrong, the page asks everyone to ask their knot operator" +) + .actor("owner") + .repo({ importFrom: unfetchableRepo }) + .step("the knot gives up on the import", importSays("failed")) + .step("the owner reads what went wrong", ownerReadsWhatWentWrong) + .step( + "an anonymous caller and a stranger read the hidden reason", + everyoneElseReadsTheHiddenReason + ) + .step("the knot doesn't know an unknown repo", unknownRepoIsNotFound) + .open(page.repo) + .use(seesTheFixedBanner) + .actor(seeded.charlie) + .open(page.repo) + .use(seesTheFixedBanner) + .step("signs out", forgetTheLogin) + .open(page.repo) + .use(seesTheFixedBanner); diff --git a/e2e/specs/routes.spec.ts b/e2e/specs/routes.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..039e8dd2739be942a818e27e20d06f6cd6db53be --- /dev/null +++ b/e2e/specs/routes.spec.ts @@ -0,0 +1,371 @@ +import { expect } from "@playwright/test"; +import { fetch as undiciFetch } from "undici"; + +import { config } from "../harness/config"; +import { + type Ctx, + flow, + link, + not, + page, + starId, + target, + testid, + through, + pds, + where, + within +} from "../harness/flow"; +import { dispatcher, get, stamp, until, xrpc } from "../harness/net"; +import { bobbin, hydrant, spindle } from "../harness/services"; +import { serviceDo } from "../harness/api"; +import { base, pipelineFor, recall, remember, workflow, workflowPath } from "../harness/shared"; +import { delivers, dropsHook, hookId, hookOn } from "../harness/webhooks"; + +const shas = new WeakMap>(); +const markPush = (name: string) => async (ctx: Ctx) => { + shas.set(ctx, { ...shas.get(ctx), [name]: ctx.pushed.sha }); +}; +const shaOf = (ctx: Ctx, name: string) => { + const sha = shas.get(ctx)?.[name]; + if (!sha) throw new Error(`no push was marked ${name}`); + return sha; +}; + +const commitLink = (name: string) => + target(`the link to commit ${name}`, (tab, ctx) => + tab.locator(`main a[href$="/commit/${shaOf(ctx, name)}"]`).first() + ); +const linkedCommits = (ctx: Ctx) => + ctx.page + .locator('[data-testid="commitHref"]') + .evaluateAll((links) => links.map((link) => link.getAttribute("href") ?? "")); + +const answers = (status: number, path: (ctx: Ctx) => string) => async (ctx: Ctx) => { + const response = await ctx.page.goto(path(ctx)); + expect(response?.status(), `${path(ctx)} should answer ${status}`).toBe(status); +}; + +const diffed = (path: string) => testid("diffFileName", { path }); + +flow("a pushed commit's page lists the files it changed and nothing else") + .actor("owner") + .repo() + .push(base) + .push({ "e2e/one.txt": "one\n", "e2e/two.txt": "two\n" }) + .step("mark the commit", markPush("change")) + .mirrored() + .open(where("the commit page", (ctx) => `${ctx.repo.path}/commit/${shaOf(ctx, "change")}`)) + .sees(diffed("e2e/one.txt")) + .sees(diffed("e2e/two.txt")) + .sees(not(diffed("e2e/base.txt"))); + +flow("a commit's patch download is the commit as a git patch") + .actor("owner") + .repo() + .push(base) + .push({ "e2e/patched.txt": "patched\n" }) + .step("mark the commit", markPush("patched")) + .mirrored() + .open(where("the commit page", (ctx) => `${ctx.repo.path}/commit/${shaOf(ctx, "patched")}`)) + .sees(diffed("e2e/patched.txt")) + .step("the .patch link answers with the commit as a patch", async (ctx) => { + const sha = shaOf(ctx, "patched"); + const url = new URL(`${ctx.repo.path}/commit/${sha}.patch`, config.webUrl); + const served = await ctx.page.request.get(url.toString()); + expect(served.status()).toBe(200); + const patch = await served.text(); + expect(patch.startsWith(`From ${sha} `), patch).toBe(true); + expect(patch).toContain("\nSubject: [PATCH] e2e\n"); + expect(patch).toContain("\n+patched\n"); + }); + +flow("a commit page for a sha the repo doesn't have answers 404") + .actor("owner") + .repo() + .push(base) + .mirrored() + .open(page.repo) + .step( + "the commit page 404s", + answers(404, (ctx) => `${ctx.repo.path}/commit/${"0".repeat(64)}`) + ); + +flow("the commit log lists every push, newest first") + .actor("owner") + .repo() + .push({ "e2e/first.txt": "first\n" }) + .step("mark the first", markPush("first")) + .push({ "e2e/second.txt": "second\n" }) + .step("mark the second", markPush("second")) + .push({ "e2e/third.txt": "third\n" }) + .step("mark the third", markPush("third")) + .mirrored() + .open(where("the commit log", (ctx) => `${ctx.repo.path}/commits/main`)) + .sees(commitLink("first")) + .step("the log is newest first", async (ctx) => { + const order = ["third", "second", "first"].map((name) => shaOf(ctx, name)); + const listed = (await linkedCommits(ctx)) + .map((href) => href.split("/commit/")[1]) + .filter((sha) => order.includes(sha)); + expect([...new Set(listed)]).toEqual(order); + }); + +flow("a file's history lists only the commits that touched it") + .actor("owner") + .repo() + .push({ "e2e/tracked.txt": "one\n" }) + .step("mark the first change", markPush("first")) + .push({ "e2e/other.txt": "unrelated\n" }) + .step("mark the unrelated change", markPush("other")) + .push({ "e2e/tracked.txt": "two\n" }) + .step("mark the second change", markPush("second")) + .mirrored() + .open(where("the file's history", (ctx) => `${ctx.repo.path}/history/main/e2e/tracked.txt`)) + .sees(commitLink("first")) + .sees(commitLink("second")) + .sees(not(commitLink("other"))); + +const contents = "raw bytes, exactly\n"; + +flow("a file's raw link serves exactly the pushed bytes") + .actor("owner") + .repo() + .push({ "e2e/raw.txt": contents }) + .mirrored() + .open(where("the blob page", (ctx) => `${ctx.repo.path}/blob/main/e2e/raw.txt`)) + .sees(target("the raw link", (tab, ctx) => testid("rawHref").locate(tab, ctx).first())) + .step("the raw link answers with the pushed bytes", async (ctx) => { + const href = await testid("rawHref").locate(ctx.page, ctx).first().getAttribute("href"); + const served = await ctx.page.request.get(new URL(href!, config.webUrl).toString()); + expect(served.status()).toBe(200); + expect(await served.text()).toBe(contents); + }); + +flow("a branch or a path the repo doesn't have answers 404") + .actor("owner") + .repo() + .push(base) + .mirrored() + .open(page.repo) + .step( + "a missing branch 404s", + answers(404, (ctx) => `${ctx.repo.path}/tree/no-such-branch`) + ) + .step( + "a missing file 404s", + answers(404, (ctx) => `${ctx.repo.path}/blob/main/e2e/no-such-file.txt`) + ); + +flow("a user or a repo that doesn't exist answers 404") + .actor("owner") + .repo() + .open(page.repo) + .step( + "a missing user 404s", + answers(404, () => `/nobody-${stamp()}.pds.tngl.boltless.dev`) + ) + .step( + "a missing repo of a real user 404s", + answers(404, (ctx) => `/${ctx.actor.handle}/no-such-repo-${stamp()}`) + ); + +const permalink = (rest: string) => + where(`the repo's did permalink${rest}`, (ctx) => `/${ctx.repo.did}${rest}`); +const inRepo = (rest: string) => where(`the repo${rest}`, (ctx) => `${ctx.repo.path}${rest}`); + +const redirectsOnceToRepo = (rest: string) => async (ctx: Ctx) => { + const url = `${config.webUrl}${permalink(rest).path(ctx)}`; + const res = await undiciFetch(url, { dispatcher, redirect: "manual" }); + await res.arrayBuffer(); + expect([res.status, res.headers.get("location")]).toEqual([307, inRepo(rest).path(ctx)]); +}; + +flow("a repo's did permalink opens the repo, with any path after it") + .actor("owner") + .repo() + .push(base) + .mirrored() + .open(permalink("")) + .at(page.repo) + .open(permalink("/issues")) + .at(page.issues) + .open(permalink("/tree/main/e2e")) + .at(inRepo("/tree/main/e2e")) + .sees(link("base.txt")) + .step("the page names the repo url as canonical", async (ctx) => { + await expect(ctx.page.locator('link[rel="canonical"]')).toHaveAttribute( + "href", + `https://tangled.org${ctx.repo.path}/tree/main/e2e` + ); + }) + .step("the permalink is one redirect to the repo url", redirectsOnceToRepo("")) + .step("a path under it is one redirect too", redirectsOnceToRepo("/issues/1")); + +flow("an account's did opens its profile, and a did nobody has opens nothing") + .actor("owner") + .open(where("the actor's did", (ctx) => `/${ctx.actor.did}`)) + .at(page.profile) + .step("a did nobody has doesn't open a page", async (ctx) => { + const res = await ctx.page.goto("/did:plc:e2enotarepoanywhereaaaaa"); + expect(res?.status()).toBeGreaterThanOrEqual(400); + expect(ctx.page.url()).toBe(`${config.webUrl}/did:plc:e2enotarepoanywhereaaaaa`); + }); + +const finished = new WeakMap(); + +flow("the pipelines page shows the push's workflow in the status spindle ended it in") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ ...base, [workflowPath]: workflow }) + .step( + "spindle has a pipeline for the push", + pipelineFor("push", (ctx) => ctx.pushed.sha) + ) + .step("spindle finishes the workflow", async (ctx) => { + finished.set(ctx, await spindle.finishedWorkflow(ctx.repo.did, ctx.pushed.sha)); + }) + .open(where("the pipelines page", (ctx) => `${ctx.repo.path}/pipelines`)) + .sees( + target("the push's commit on a pipeline card", (tab, ctx) => + tab.locator(`[data-testid="commitHref"][href$="/commit/${ctx.pushed.sha}"]`).first() + ) + ) + .click( + target("the pipeline's workflows", (tab, ctx) => + testid("dropdown").locate(tab, ctx).first() + ) + ) + .sees( + target("the workflow in spindle's final status", (tab, ctx) => { + const done = finished.get(ctx)!; + return testid("workflowName", { name: done.name ?? done.id, status: done.status }) + .locate(tab, ctx) + .first(); + }) + ); + +flow("the deliveries page lists what the receiver got") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .step("a webhook for pushes", hookOn(["push"])) + .push(base) + .step( + "the receiver got the push and spindle logged it", + delivers( + "push", + (ctx) => `Tangled-Hook/${ctx.pushed.sha.slice(0, 7)}`, + () => {} + ) + ) + .open( + where( + "the hook's deliveries", + (ctx) => `${ctx.repo.path}/settings/hooks/${hookId(ctx)}/deliveries` + ) + ) + .sees(testid("delivery", { event: "push", success: "true" })) + .step("take the webhook away", dropsHook); + +flow("the stars page lists who starred the repo") + .actor("owner") + .repo() + .actor("fan") + .step("remember the fan", remember("fan")) + .open(page.repo) + .click(testid("star", { starred: "false" })) + .trace(starId, through(pds, hydrant, bobbin)) + .actor("owner") + .open(where("the stars page", (ctx) => `${ctx.repo.path}/stars`)) + .sees( + target("the fan on the stars page", (tab, ctx) => + tab.locator(`a[href="/${recall(ctx, "fan").handle}"]`).first() + ) + ); + +const deliberiAud = `did:web:${new URL(config.deliberiUrl).host}`; +const emailsOf = async (ctx: Ctx) => { + const reply = await serviceDo( + ctx.actor, + config.deliberiUrl, + deliberiAud, + "org.tangled.temp.account.listEmails" + ); + expect(reply.status, `deliberi answered ${JSON.stringify(reply.body)}`).toBe(200); + return (reply.body.emails ?? []) as { address: string; verified?: boolean }[]; +}; +const addresses = new WeakMap(); +const emailsNew = where("the add email form", () => "/settings/emails/new"); + +flow("an added email is kept unverified, and adding it again keeps one") + .actor("mailer") + .step("pick an address", async (ctx) => { + addresses.set(ctx, `e2e-${stamp()}@pds.tngl.boltless.dev`); + }) + .open(emailsNew) + .fill(testid("email"), (ctx) => addresses.get(ctx)!) + .click(testid("send")) + .step("deliberi keeps it unverified", async (ctx) => { + await until("deliberi should list the address", 10_000, async () => { + const kept = (await emailsOf(ctx)).filter((one) => one.address === addresses.get(ctx)); + return [kept.length === 1 && !kept[0].verified, `it lists ${JSON.stringify(kept)}`]; + }); + }) + .open(emailsNew) + .fill(testid("email"), (ctx) => addresses.get(ctx)!) + .click(testid("send")) + .step("deliberi still has it once", async (ctx) => { + const kept = (await emailsOf(ctx)).filter((one) => one.address === addresses.get(ctx)); + expect(kept).toHaveLength(1); + }); + +const orgs = new WeakMap(); +const orgHandle = (ctx: Ctx) => `${orgs.get(ctx)}.tranquil.tngl.boltless.dev`; +const resolves = async (handle: string) => + ( + await get( + xrpc("https://tranquil.tngl.boltless.dev", "com.atproto.identity.resolveHandle", { + handle + }) + ) + ).status === 200; + +flow("a new organization gets its own account, and deleting it takes the account away") + .actor("org") + .step("pick the org's name", async (ctx) => { + orgs.set(ctx, `org${stamp()}`.slice(0, 20)); + }) + .open(where("the new organization form", () => "/organizations/new")) + .fill(testid("organizationHandle"), (ctx) => orgs.get(ctx)!) + .click(testid("submit")) + .step("the form lands on the org's welcome", async (ctx) => { + await expect(ctx.page).toHaveURL(new RegExp(`/organizations/${orgHandle(ctx)}/welcome`), { + timeout: 15_000 + }); + }) + .step("the org's handle resolves", async (ctx) => { + expect(await resolves(orgHandle(ctx))).toBe(true); + }) + .open(where("the org's settings", (ctx) => `/${orgHandle(ctx)}/settings`)) + .click(testid("openDeleteConfirm")) + .click(within(testid("deleteConfirm"), testid("confirm"))) + .step("the org's handle no longer resolves", async (ctx) => { + await until("the org account should be gone", 15_000, async () => [ + !(await resolves(orgHandle(ctx))), + "it still resolves" + ]); + }); + +const needle = `zq${stamp()}`; + +flow("a repo's own search finds a word pushed to it") + .actor("owner") + .repo() + .push({ "e2e/needle.txt": `${needle}\n` }) + .mirrored() + .open(where("the repo's search", (ctx) => `${ctx.repo.path}/search?q=${needle}`)) + .sees( + target("the file with the word", (tab) => + tab.locator('[data-testid="fileHref"][href*="e2e/needle.txt"]').first() + ) + ); diff --git a/e2e/specs/search.spec.ts b/e2e/specs/search.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..33a5b93312d7893e35fee981e677dad2739ceb22 --- /dev/null +++ b/e2e/specs/search.spec.ts @@ -0,0 +1,106 @@ +import { + bobbin, + type Ctx, + exactLink, + flow, + git, + hydrant, + knotCopy, + manifestId, + not, + refsId, + target, + testid, + through, + where +} from "../harness/flow"; +import { stamp, until } from "../harness/net"; + +const reposOf = (query: string) => + where( + `the actor's repos tab${query && ` searched for ${query}`}`, + (ctx) => `/${ctx.actor.handle}?tab=repos${query && `&q=${encodeURIComponent(query)}`}` + ); + +let firstRepo = ""; + +const searchBox = testid("q"); +// search matches any word, so the stamp is what tells two flow repos apart +const stampOf = (ctx: Ctx) => ctx.repo.name.replace(/^e2e-/, ""); +const searchedFor = (name: (ctx: Ctx) => string) => + where("the repos tab searched for the new repo", (ctx) => reposOf(name(ctx)).path(ctx)); + +const twoRepos = (name: string) => + flow(name) + .actor("owner") + .repo() + .step("remember the first repo", async (ctx) => { + firstRepo = ctx.repo.name; + }) + .repo() + .follow(manifestId, through(knotCopy, hydrant, bobbin)); + +twoRepos("a profile's repo search finds a repo made today") + .open(searchedFor(stampOf)) + .sees(exactLink((ctx) => ctx.repo.name)) + .sees(not(exactLink(() => firstRepo))) + .open(reposOf("no-such-repo")) + .sees(testid("emptyState")); + +twoRepos("typing in the profile's repo search narrows the list") + .open(reposOf("")) + .sees(exactLink((ctx) => ctx.repo.name)) + .sees(exactLink(() => firstRepo)) + .fill(searchBox, stampOf) + .press("Enter") + .at(searchedFor(stampOf)) + .sees(exactLink((ctx) => ctx.repo.name)) + .sees(not(exactLink(() => firstRepo))); + +twoRepos("clearing the profile's repo search brings the whole list back") + .open(searchedFor(stampOf)) + .sees(not(exactLink(() => firstRepo))) + .click(testid("clear")) + .at(reposOf("")) + .sees(exactLink((ctx) => ctx.repo.name)) + .sees(exactLink(() => firstRepo)); + +const topbarSearch = testid("topbarSearch"); +const repoResult = target("the repo in the results", (tab, ctx) => + tab.getByRole("link", { name: ctx.repo.path.slice(1), exact: true }) +); + +flow("searching the top bar for a repo by name finds the repo") + .actor("owner") + .repo() + + .open(where("the home page", () => "/")) + .fill(topbarSearch, (ctx) => ctx.repo.name) + .press("Enter") + .at(where("the search page", (ctx) => `/search?q=${ctx.repo.name}`)) + .sees(repoResult); + +// one word per worker, so only this flow's repo can match it +const word = `zq${stamp()}`; +const codeSearch = where( + "the code search", + () => `/search?type=code&q=${encodeURIComponent(word)}` +); + +// the index builds after a push, so ask again until it knows the word +const searchUntilFound = async (ctx: Ctx) => { + await until(`code search should find ${word}`, 60_000, async () => { + await ctx.page.goto(codeSearch.path(ctx), { waitUntil: "networkidle" }); + const found = await repoResult.locate(ctx.page, ctx).isVisible({ timeout: 3000 }); + return [found, (await ctx.page.getByRole("main").innerText()).slice(0, 200)]; + }); +}; + +flow("code search finds a word that was pushed to a repo") + .actor("owner") + .repo() + .push({ "needle.txt": `${word}\n` }) + .follow(refsId, through(git, knotCopy, hydrant, bobbin)) + .open(where("the home page", () => "/")) + .step("search for it until the index has it", searchUntilFound) + .sees(repoResult); diff --git a/e2e/specs/session.spec.ts b/e2e/specs/session.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..b02323ba1f3db66ab7d76c89cc408ed2cbf5af22 --- /dev/null +++ b/e2e/specs/session.spec.ts @@ -0,0 +1,293 @@ +import { type BrowserContext, expect, test } from "@playwright/test"; +import { readFileSync } from "node:fs"; + +import { type Account, authFile, config, seeded } from "../harness/config"; +import { + bobbin, + component, + type Ctx, + flow, + hydrant, + knots, + loginThen, + page, + pds, + starId, + target, + testid, + through, + where, + within +} from "../harness/flow"; +import { parseBody, stamp } from "../harness/net"; +import { signInHere } from "../harness/signin"; +import { forgetTheLogin, recall, remember } from "../harness/shared"; + +const menuOf = (who: (ctx: Ctx) => Account) => + target("the account menu", (tab, ctx) => + tab.getByRole("banner").getByRole("button", { name: who(ctx).handle }) + ); +const actorMenu = menuOf((ctx) => ctx.actor); +const logout = testid("signOut"); + +const otherOf = (ctx: Ctx) => recall(ctx, "other"); + +const signInFromHere = (who: (ctx: Ctx) => Account) => (ctx: Ctx) => signInHere(ctx.page, who(ctx)); +const signInAsActor = signInFromHere((ctx) => ctx.actor); + +const browserState = (ctx: Ctx) => + ctx.page.evaluate(() => ({ + current: localStorage.getItem("tangled.currentDid"), + sessions: localStorage.getItem("atcute-oauth:sessions") ?? "", + accounts: localStorage.getItem("tangled.accounts") ?? "" + })); + +// dids become "did" so two logins compare equal +const dropDids = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(dropDids); + if (!value || typeof value !== "object") return typeof value; + return Object.fromEntries( + Object.entries(value).map(([key, inner]) => [ + key.startsWith("did:") ? "did" : key, + dropDids(inner) + ]) + ); +}; +const kept = (state: Awaited>) => ({ + cookies: state.cookies + .map( + (c) => + `${c.domain} ${c.name} ${c.httpOnly} ${c.secure} ${c.sameSite} ${c.expires === -1}` + ) + .sort(), + storage: state.origins.map(({ origin, localStorage }) => ({ + origin, + items: Object.fromEntries( + localStorage.map(({ name, value }) => [name, dropDids(parseBody(value))]) + ) + })) +}); + +const keptLikeALogin = async (ctx: Ctx) => { + const login = JSON.parse(readFileSync(authFile(seeded.alice), "utf8")); + expect(kept(await ctx.page.context().storageState())).toEqual(kept(login)); +}; + +const firstStar = (name: string, options: { login?: "ui" } = {}) => + flow(name) + .actor("owner") + .repo() + .actor("fan", options) + + .open(page.repo) + .sees(actorMenu) + .step("the browser keeps the session the way a login does", keptLikeALogin) + .click(testid("star", { starred: "false" })) + .sends( + starId, + pds, + (ctx) => ({ record: { subject: { identity: ctx.repo.did } } }), + "create" + ) + .trace(starId, through(pds, hydrant, bobbin)) + .reload() + .sees(testid("star", { starred: "true" })); + +firstStar("a new account logged in through the login page stars a repo", { login: "ui" }); +firstStar("a new account with a session minted from node stars a repo"); + +flow("logging out of one of two accounts in a browser keeps the other signed in") + .actor("second") + .step("remember the second account", remember("other")) + .actor("first", { login: "ui" }) + + .open(knots) + .click(actorMenu) + .click(testid("addAccount")) + .at(loginThen(() => "/settings/knots")) + .step("the second account signs in beside the first", signInFromHere(otherOf)) + .at(knots) + .sees(menuOf(otherOf)) + + .click(menuOf(otherOf)) + .sees(logout) + .click(logout) + .sees(actorMenu) + .step("the browser keeps the first account and has let go of the second", async (ctx) => { + const saw = await browserState(ctx); + expect(saw.current).toBe(ctx.actor.did); + expect(saw.sessions).toContain(ctx.actor.did); + expect(saw.sessions).not.toContain(otherOf(ctx).did); + expect(saw.accounts).not.toContain(otherOf(ctx).did); + }) + .reload() + .sees(actorMenu) + .open(knots) + .at(knots); + +flow("logging out ends the session and a settings page asks for a login again") + .actor("leaver", { login: "ui" }) + + .open(page.profile) + .step("the page is signed in", async (ctx) => { + expect((await browserState(ctx)).current).toBe(ctx.actor.did); + }) + .click(actorMenu) + .click(logout) + .sees(within(component("Topbar"), testid("login"))) + .step("the browser has forgotten the account", async (ctx) => { + const saw = await browserState(ctx); + expect(saw.current).toBeNull(); + expect(saw.sessions).not.toContain(ctx.actor.did); + const cookies = await ctx.page.context().cookies(); + expect(cookies.find((cookie) => cookie.name === "tangled.currentDid")).toBeUndefined(); + }) + .open(knots) + .at(loginThen(() => "/settings/knots")); + +flow("switching to the other account in the menu changes who the page is signed in as") + .actor("second") + .step("remember the second account", remember("other")) + .actor("first", { login: "ui" }) + + .open(knots) + .click(actorMenu) + .click(testid("addAccount")) + .step("the second account signs in beside the first", signInFromHere(otherOf)) + .at(knots) + .sees(menuOf(otherOf)) + + .click(menuOf(otherOf)) + .click( + target("the first account in the menu", (tab, ctx) => + tab.getByRole("menuitem").filter({ hasText: ctx.actor.handle }) + ) + ) + .sees(actorMenu) + .step( + "the browser says the first account is the one in use and still has both", + async (ctx) => { + const saw = await browserState(ctx); + expect(saw.current).toBe(ctx.actor.did); + expect(saw.sessions).toContain(otherOf(ctx).did); + const cookies = await ctx.page.context().cookies(); + expect(cookies.find((cookie) => cookie.name === "tangled.currentDid")?.value).toContain( + ctx.actor.did.replace(/:/g, "%3A") + ); + } + ) + .reload() + .sees(actorMenu); + +const followeeProfile = where("the followee's profile", (ctx) => `/${otherOf(ctx).handle}`); + +flow("a visitor who clicks Follow is sent to log in and comes back to that profile") + .actor("star") + .step("remember who is followed", remember("other")) + .actor("fan", { login: "ui" }) + + .open(followeeProfile) + .step("signs out", forgetTheLogin) + .open(followeeProfile) + .click(testid("follow", { following: "false" })) + .at(loginThen((ctx) => `/${otherOf(ctx).handle}`)) + .step("signs back in", signInAsActor) + .at(followeeProfile) + .sees(testid("follow", { following: "false" })); + +flow("a visitor who clicks Star is sent to log in and comes back to that repo") + .actor("owner") + .repo() + .actor("reader", { login: "ui" }) + + .open(page.repo) + .step("signs out", forgetTheLogin) + .open(page.repo) + .click(testid("star", { starred: "false" })) + .at(loginThen((ctx) => ctx.repo.path)) + .step("signs back in", signInAsActor) + .at(page.repo) + .sees(testid("star", { starred: "false" })); + +flow("a visitor who opens a repo's settings is sent to log in and comes back to them") + .actor("owner", { login: "ui" }) + .repo() + + .open(page.repo) + .step("signs out", forgetTheLogin) + .open(page.settings) + .at(loginThen((ctx) => `${ctx.repo.path}/settings`)) + .step("signs back in", signInAsActor) + .at(page.settings) + .sees(testid("settingsSectionTitle", { name: "danger" })); + +const knotsWithQuery = where("the knots settings with a query", () => "/settings/knots?from=email"); + +flow("a visitor who opens the knot settings is sent to log in and comes back to them") + .actor("reader", { login: "ui" }) + + .open(knots) + .step("signs out", forgetTheLogin) + .open(knotsWithQuery) + .at(loginThen(() => "/settings/knots?from=email")) + .step("signs back in", signInAsActor) + .at(knotsWithQuery) + .sees(testid("settingsHeaderTitle", { name: "knots" })); + +const inviteHost = "knot.invited.example.test"; +const invite = where("the invite page", (ctx) => `/invite?knot=${inviteHost}&code=${code(ctx)}`); +const codes = new WeakMap(); +// the page only reads the code, so a made up one is fine until Join +const pickCode = async (ctx: Ctx) => { + codes.set(ctx, `e2e-${stamp()}`); +}; +const code = (ctx: Ctx) => codes.get(ctx)!; + +flow("an invite link opened signed out logs you in and lands on the join page") + .actor("guest", { login: "ui" }) + .step("pick a code", pickCode) + + .open(page.profile) + .step("signs out", forgetTheLogin) + .open(invite) + .at(loginThen((ctx) => `/invite?knot=${inviteHost}&code=${code(ctx)}`)) + .step("signs in", signInAsActor) + .at(invite) + .sees(testid("joinHeading", { knot: inviteHost })) + .sees(testid("join")); + +test("the login page shows an error when it is opened with one", async ({ page: tab }) => { + await tab.goto("/login?error=session", { waitUntil: "networkidle" }); + await expect(tab.locator('[data-testid="loginErrorHeading"]')).toBeVisible(); +}); + +test("a handle that does not exist is told so on the login page and stays there", async ({ + page: tab +}) => { + await tab.goto("/login", { waitUntil: "networkidle" }); + const handle = tab.getByTestId("identifier"); + await handle.fill(`nobody-${stamp()}.pds.tngl.boltless.dev`); + await handle.press("Enter"); + await expect(tab.locator('[data-testid="loginErrorHeading"]')).toBeVisible(); + await expect(tab.locator('[data-testid="loginErrorMessage"]')).toBeVisible(); + await expect(tab).toHaveURL(`${config.webUrl}/login`); +}); + +const guarded = [ + "/welcome", + "/strings/new", + "/organizations/new", + "/settings/profile/delete?step=1", + "/settings/keys/new" +]; + +for (const path of guarded) { + test(`a visitor opening ${path} is sent to log in with that page to come back to`, async ({ + page: tab + }) => { + await tab.goto(path, { waitUntil: "networkidle" }); + await expect(tab).toHaveURL( + `${config.webUrl}/login?return_url=${encodeURIComponent(path)}` + ); + }); +} diff --git a/e2e/specs/signup.spec.ts b/e2e/specs/signup.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..1ee0bf94d4baa7c4ddda551ac04b2ae76ff3d43b --- /dev/null +++ b/e2e/specs/signup.spec.ts @@ -0,0 +1,140 @@ +import { expect } from "@playwright/test"; + +import { preauthorize } from "../harness/api"; +import { config, seeded, type Account } from "../harness/config"; +import { button, field, flow, page, testid, where, type Ctx } from "../harness/flow"; +import { request, stamp } from "../harness/net"; +import { signupToken } from "../harness/processes"; +import { forgetTheLogin } from "../harness/shared"; +import { pushedRequest } from "../harness/signin"; + +const COMPLETE_SIGNUP = "/xrpc/org.tangled.temp.account.completeSignup"; +const pdsOrigin = new URL(config.pdsUrl).origin; + +interface Signup { + email: string; + username: string; + made?: Account; + pdsPages: string[]; +} + +const signups = new WeakMap(); +const signupOf = (ctx: Ctx) => { + if (!signups.has(ctx)) { + const username = `signup-${stamp()}`; + signups.set(ctx, { email: `${username}@pds.tngl.boltless.dev`, username, pdsPages: [] }); + } + return signups.get(ctx)!; +}; +const madeBy = (ctx: Ctx) => { + const made = signupOf(ctx).made; + if (!made) throw new Error("the signup made no account yet"); + return made; +}; + +const signupPage = where("the sign up page", () => "/signup"); +const emailedLink = where( + "the link from the signup email", + (ctx) => `/signup/verify?token=${signupToken(signupOf(ctx).email)}` +); +const welcome = where("the welcome page", () => "/welcome"); + +const beginSignup = (name: string) => + flow(name) + .actor("guest", { login: "ui" }) + .open(page.profile) + .step("signs out", forgetTheLogin) + .open(signupPage) + .fill(testid("email"), (ctx) => signupOf(ctx).email) + .click(button("Sign up")) + .sees(testid("checkEmailTitle")) + .open(emailedLink) + .step("notes every page it opens on the pds", async (ctx) => { + ctx.page.on("request", (request) => { + if (request.isNavigationRequest() && new URL(request.url()).origin === pdsOrigin) + signupOf(ctx).pdsPages.push(request.url()); + }); + }) + .fill(field("Username"), (ctx) => signupOf(ctx).username) + .fill(field("Password"), config.password); + +const createAccount = async (ctx: Ctx) => { + const signup = signupOf(ctx); + const [response] = await Promise.all([ + ctx.page.waitForResponse((reply) => new URL(reply.url()).pathname === COMPLETE_SIGNUP), + button("Create account").locate(ctx.page, ctx).click() + ]); + expect(response.status()).toBe(200); + const { did, handle } = await response.json(); + signup.made = { name: signup.username, handle, did }; + ctx.owe(`account ${handle}`, { kind: "account", account: signup.made }); +}; + +beginSignup("signing up signs the new account in to tangled without the pds pages") + .step("creates the account", createAccount) + .at(welcome, 20_000) + .step("the browser holds a session for the new account", async (ctx) => { + const { did } = madeBy(ctx); + const stored = await ctx.page.evaluate(() => ({ + active: localStorage.getItem("tangled.currentDid"), + sessions: Object.keys(JSON.parse(localStorage.getItem("atcute-oauth:sessions") ?? "{}")) + })); + expect(stored).toEqual({ active: did, sessions: [did] }); + }) + .step("it never opened a pds page", async (ctx) => { + expect(signupOf(ctx).pdsPages).toEqual([]); + }); + +beginSignup("a signup the pds can't sign in goes to the pds sign in with the handle filled") + .step("deliberi gets no oauth request, like on a pds without the route", async (ctx) => { + await ctx.page.route(`**${COMPLETE_SIGNUP}`, async (route) => { + const { requestUri: _, ...input } = route.request().postDataJSON(); + await route.continue({ postData: JSON.stringify(input) }); + }); + }) + .step("creates the account", createAccount) + .step("lands on the pds sign in with the handle filled", async (ctx) => { + const { handle } = madeBy(ctx); + await ctx.page.waitForURL( + (url) => url.origin === pdsOrigin && url.pathname === "/oauth/authorize" + ); + await expect + .poll(() => + ctx.page + .locator("input") + .evaluateAll( + (inputs, value) => + inputs.some((input) => (input as HTMLInputElement).value === value), + handle + ) + ) + .toBe(true); + }); + +flow("the pds refuses to sign in an account that isn't new or a client it doesn't list") + .actor("fresh") + .step("an account that signed in before is refused", async () => { + const { status, body } = await preauthorize(seeded.alice.did, await pushedRequest()); + expect(status).toBe(403); + // on a stack up for under 5 minutes alice is still new, but setup already signed her in + expect(["account is not new", "account already authorized a client"]).toContain( + body.message + ); + }) + .step("a client outside the list is refused", async (ctx) => { + const loopback = { + client_id: "http://localhost", + redirect_uris: ["http://127.0.0.1/"], + scope: "atproto" + }; + const { status, body } = await preauthorize(ctx.actor.did, await pushedRequest(loopback)); + expect(status).toBe(403); + expect(body.message).toBe("client is not allowed"); + }) + .step("without the admin password the pds says nothing else", async (ctx) => { + const { status } = await request(`${config.pdsUrl}/_preauthorize`, { + method: "POST", + json: { did: ctx.actor.did, requestUri: await pushedRequest() } + }); + expect(status).toBe(401); + }); diff --git a/e2e/specs/social.spec.ts b/e2e/specs/social.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..821565c49e9f0bea14a5926b98e125d18044626c --- /dev/null +++ b/e2e/specs/social.spec.ts @@ -0,0 +1,786 @@ +import { expect } from "@playwright/test"; + +import { comments, pdsCall } from "../harness/api"; +import { type Account, config } from "../harness/config"; +import { + bobbin, + commentId, + type Ctx, + deliberi, + flow, + type Flow, + followId, + hydrant, + knot, + not, + page, + pds, + pullId, + record, + rkeyOf, + starCount, + starId, + subscriptionId, + type Slot, + target, + testid, + text, + through, + ticketId, + type Where, + where, + within +} from "../harness/flow"; +import { get, until, xrpc } from "../harness/net"; +import { aboutOf, type Notice, noticesMadeFrom, noticesOf } from "../harness/services"; +import { + deletesIssue, + fanOpened, + fanPulled, + made, + mergesThrough, + note as comment, + pdsRecordUrl, + recall, + remember, + theFan, + theOwner +} from "../harness/shared"; +import { type Follow } from "../harness/lexicons"; + +const noticeFor = async ( + ctx: Ctx, + type: string, + about: (notice: Notice) => boolean, + who: Account = ctx.actor +) => + until(`deliberi should list a ${type} notice for ${who.handle}`, 20_000, async () => { + const found = (await noticesOf(who)).filter( + (notice) => notice.type === type && about(notice) + ); + return [ + found.length > 0, + `it lists ${JSON.stringify((await noticesOf(who)).map((notice) => notice.type))}`, + found[0] + ]; + }); + +const notices = where("the notifications page", () => "/notifications"); +const bell = (unread: number) => + unread > 0 ? testid("unreadCount", { count: unread }) : not(testid("unreadCount")); +// the actor gets one notice per type, so the type alone finds the row +const noticeLink = (type: string) => + within(testid("notificationRow", { type }), testid("notificationPhraseLink")); + +const starsOwner = (ctx: Ctx) => ({ + record: { subject: { $type: "org.tangled.feed.star#repo", identity: ctx.repo.did } } +}); + +flow("a star notifies the repo owner and opening the notice marks it read") + .actor("fan") + .step("remember the fan", remember("fan")) + .actor("owner") + .repo() + .actor("fan") + .open(page.repo) + .click(testid("star", { starred: "false" })) + .sends(starId, pds, starsOwner, "create") + .trace(starId, through(pds, hydrant, bobbin)) + + .actor("owner") + .step("deliberi lists the star as unread", async (ctx) => { + const notice = await noticeFor( + ctx, + "repo_starred", + (n) => n.actorDid === recall(ctx, "fan").did + ); + expect(notice.read).toBe(false); + }) + .open(notices) + .sees(noticeLink("repo_starred")) + .sees(bell(1)) + .click(noticeLink("repo_starred")) + .at(page.repo) + .step("deliberi lists the star as read", async (ctx) => { + await until("deliberi should list the star as read", 10_000, async () => { + const found = (await noticesOf(ctx.actor)).find((n) => n.type === "repo_starred"); + return [found?.read === true, `it lists ${JSON.stringify(found)}`]; + }); + }) + .sees(bell(0)) + .open(notices) + .sees(noticeLink("repo_starred")) + .sees(bell(0)); + +const issueTitle = "somebody will answer"; + +// deliberi only tells subscribers, so the owner has to open the issue in the browser first +const commentedByAFan = (name: string) => + flow(name) + .actor("fan") + .step("remember the fan", remember("fan")) + .actor("owner") + .repo() + .open(page.newIssue) + .fill(testid("title"), issueTitle) + .click(testid("submit")) + .trace(ticketId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds, hydrant, bobbin)) + .actor("fan") + .open(page.issue(ticketId)) + .fill(testid("body"), "me too") + .click(testid("submit")) + .trace(commentId, through(knot, hydrant, bobbin)) + .trace(subscriptionId, through(pds)) + .actor("owner") + .step("deliberi lists the comment as unread", async (ctx) => { + const notice = await noticeFor( + ctx, + "issue_commented", + (n) => n.actorDid === recall(ctx, "fan").did + ); + expect(notice.issueAt).toBe(ctx.get(ticketId).uri); + }); + +commentedByAFan("the notice for a comment on an issue opens that issue") + .open(notices) + .click(noticeLink("issue_commented")) + .at(page.issue(ticketId)) + .sees(within(testid("ticketTitle"), text(issueTitle))); + +commentedByAFan("deleting an issue takes the notice about it away") + .open(page.issue(ticketId)) + .use(deletesIssue) + .step("deliberi no longer lists the comment", async (ctx) => { + await until("deliberi should drop the notice about the erased issue", 15_000, async () => { + const left = (await noticesOf(ctx.actor)).filter( + (n) => n.issueAt === ctx.get(ticketId).uri + ); + return [left.length === 0, `it still lists ${JSON.stringify(left)}`]; + }); + }) + .open(notices) + .sees(not(noticeLink("issue_commented"))); + +const stargazers = where("the stargazers page", (ctx) => `${ctx.repo.path}/stars`); +const handleLink = (who: (ctx: Ctx) => Account) => + target("the stargazer's link", (tab, ctx) => + tab.getByRole("link", { name: who(ctx).handle, exact: true }) + ); + +flow("a star shows on the repo's stargazers page") + .actor("owner") + .repo() + .actor("fan") + + .open(stargazers) + .sees(testid("emptyState")) + .open(page.repo) + .click(testid("star", { starred: "false" })) + .sends(starId, pds, starsOwner, "create") + .trace(starId, through(pds, hydrant, bobbin)) + .open(stargazers) + .sees(handleLink((ctx) => ctx.actor)) + .sees(not(testid("emptyState"))) + .reload() + .sees(handleLink((ctx) => ctx.actor)); + +const repoPaths = new WeakMap>(); +const rememberRepo = (label: string) => async (ctx: Ctx) => { + repoPaths.set(ctx, { ...repoPaths.get(ctx), [label]: ctx.repo.path }); +}; +const repoRow = (label: string) => + target(`the ${label}'s repo in the feed`, (tab, ctx) => + tab.locator(`a[href="${repoPaths.get(ctx)![label]}"]`).first() + ); + +const followingFeed = where("the following timeline", () => "/timeline?following=true"); + +flow("the following timeline shows a followed account's repo and not a stranger's") + .actor("maker") + .step("remember the maker", remember("maker")) + .repo() + .step("remember the maker's repo", rememberRepo("maker")) + .actor("rando") + .repo() + .step("remember the rando's repo", rememberRepo("rando")) + .actor("fan") + .step("the fan follows the maker", async (ctx) => { + const written = await pdsCall(ctx.actor, "com.atproto.repo.putRecord", { + collection: followId.nsid, + rkey: recall(ctx, "maker").did, + record: { $type: followId.nsid, createdAt: new Date().toISOString() } satisfies Follow + }); + await bobbin.indexed(written); + }) + + .open(followingFeed) + .sees(repoRow("maker")) + .sees(not(repoRow("rando"))) + .reload() + .sees(repoRow("maker")) + .sees(not(repoRow("rando"))); + +const peopleTab = (label: string, tab: "followers" | "following") => + where(`the ${label}'s ${tab} tab`, (ctx) => `/${recall(ctx, label).handle}?tab=${tab}`); +// a card bobbin didn't enrich still renders, as handle.invalid with no counts +const personCard = (label: string, counts: { followers: number; following: number }) => + target( + `the ${label}'s card, ${counts.followers} followers and ${counts.following} following`, + (tab, ctx) => + testid("person", { handle: recall(ctx, label).handle, ...counts }).locate(tab, ctx) + ); + +flow("the following and followers tabs name both ends of a follow") + .actor("star") + .step("remember the star", remember("star")) + .actor("fan") + .step("remember the fan", remember("fan")) + .step("the fan follows the star", async (ctx) => { + const written = await pdsCall(ctx.actor, "com.atproto.repo.putRecord", { + collection: followId.nsid, + rkey: recall(ctx, "star").did, + record: { $type: followId.nsid, createdAt: new Date().toISOString() } satisfies Follow + }); + await bobbin.indexed(written); + }) + + .open(peopleTab("fan", "following")) + .sees(personCard("star", { followers: 1, following: 0 })) + .open(peopleTab("star", "followers")) + .sees(personCard("fan", { followers: 0, following: 1 })); + +// one flow at a time per worker, so a module-level let is safe +let star: Account; + +const profile = where("the followee's profile", () => `/${star.handle}`); + +const followersAre = (count: number) => async () => { + await until(`bobbin should count ${count} followers of ${star.handle}`, 15_000, async () => { + const { status, body } = await get( + xrpc(config.bobbinUrl, "org.tangled.graph.countFollows", { subject: star.did }) + ); + return [ + status === 200 && body.count === count, + `countFollows answered ${status} ${JSON.stringify(body)}` + ]; + }); +}; + +// the follow record has no subject, the followee's did is the rkey +const followedBy = (rkey: () => string) => (ctx: { actor: Account }) => ({ + collection: "org.tangled.graph.follow", + repo: ctx.actor.did, + rkey: rkey() +}); + +flow("following someone and unfollowing them") + .actor("star") + .step("remember who is being followed", async (ctx) => { + star = ctx.actor; + }) + .actor("fan") + + .open(profile) + .sees(testid("followersCount", { count: 0 })) + .click(testid("follow", { following: "false" })) + .sends( + followId, + pds, + followedBy(() => star.did), + "put" + ) + // the button waits on the server, so the state doesn't flip yet + .sees(testid("follow", { following: "false" })) + .trace(followId, through(pds, hydrant, bobbin)) + .step("the follow record has no subject, its rkey is the followee", async (ctx) => { + const { body } = await get(pdsRecordUrl(ctx.actor.did, followId.nsid, star.did)); + expect(Object.keys(body.value).sort()).toEqual(["$type", "createdAt"]); + }) + .step("bobbin counts one follower", followersAre(1)) + .sees(testid("follow", { following: "true" })) + .sees(testid("followersCount", { count: 1 })) + .reload() + .sees(testid("follow", { following: "true" })) + .sees(testid("followersCount", { count: 1 })) + + .click(testid("follow", { following: "true" })) + .sends( + followId, + pds, + followedBy(() => star.did), + "delete" + ) + .sees(testid("follow", { following: "true" })) + .trace(followId, through(pds, hydrant, bobbin)) + .step("bobbin counts no followers", followersAre(0)) + .sees(testid("follow", { following: "false" })) + .reload() + .sees(testid("follow", { following: "false" })) + .sees(testid("followersCount", { count: 0 })); + +flow("starring and unstarring a repo") + .actor("owner") + .repo() + .actor("fan") + + .open(page.repo) + .click(testid("star", { starred: "false" })) + .sends(starId, pds, starsOwner, "create") + // optimistic ui, the server hasn't seen the write yet + .sees(testid("star", { starred: "true" })) + .sees(starCount, "1") + .step("bobbin has not counted it", async (ctx) => + expect(await bobbin.countStars(ctx.repo.did)).toBe(0) + ) + .trace(starId, through(pds, hydrant, bobbin)) + .step("bobbin counts one star", async (ctx) => + expect(await bobbin.countStars(ctx.repo.did)).toBe(1) + ) + .reload() + .sees(testid("star", { starred: "true" })) + .sees(starCount, "1") + + .click(testid("star", { starred: "true" })) + .sends(starId, pds, (ctx) => ({ rkey: rkeyOf(ctx, starId) }), "delete") + .sees(testid("star", { starred: "false" })) + .sees(starCount, "0") + .trace(starId, through(pds, hydrant, bobbin)) + .step("bobbin counts no stars", async (ctx) => + expect(await bobbin.countStars(ctx.repo.did)).toBe(0) + ) + .reload() + .sees(testid("star", { starred: "false" })) + .sees(starCount, "0"); + +const subscribed = + (subject: (ctx: Ctx) => string, state: boolean | "muted") => async (ctx: Ctx) => { + const { status, body } = await get( + xrpc(config.bobbinUrl, "org.tangled.feed.subscription.getForActor", { + actor: ctx.actor.did, + subject: subject(ctx) + }) + ); + const answered = `getForActor answered ${status} ${JSON.stringify(body)}`; + expect(status, answered).toBe(state === false ? 404 : 200); + if (state !== false) expect(body.ignore === true, answered).toBe(state === "muted"); + }; + +flow("watching a repo and giving it up again") + .actor("owner") + .repo() + .actor("fan") + + .open(page.repo) + .click(testid("watch", { subscribed: "false" })) + .sends( + subscriptionId, + pds, + (ctx) => ({ + record: { subject: { $type: "org.tangled.feed.subscription#repo", did: ctx.repo.did } } + }), + "create" + ) + // the watch button doesn't wait for the server + .sees(testid("watch", { subscribed: "true" })) + .trace(subscriptionId, through(pds, hydrant, bobbin)) + .step( + "bobbin lists the watch", + subscribed((ctx) => ctx.repo.did, true) + ) + .reload() + .sees(testid("watch", { subscribed: "true" })) + + .click(testid("watch", { subscribed: "true" })) + .sends(subscriptionId, pds, (ctx) => ({ rkey: rkeyOf(ctx, subscriptionId) }), "delete") + .sees(testid("watch", { subscribed: "false" })) + .trace(subscriptionId, through(pds, hydrant, bobbin)) + .step( + "bobbin no longer lists the watch", + subscribed((ctx) => ctx.repo.did, false) + ) + .reload() + .sees(testid("watch", { subscribed: "false" })); + +const issueUri = (ctx: Ctx) => ctx.get(ticketId).uri; + +flow("subscribing to an issue and unsubscribing again") + .actor("owner") + .repo() + .ticket(ticketId, { title: "follow along" }) + .actor("fan") + + .open(page.issue(ticketId)) + .click(testid("toggleSubscription", { subscribed: "false" })) + .sends( + subscriptionId, + pds, + (ctx) => ({ + record: { + subject: { $type: "org.tangled.feed.subscription#record", uri: issueUri(ctx) } + } + }), + "create" + ) + // the subscribe button flips before the server answers + .sees(testid("toggleSubscription", { subscribed: "true", pending: "true" })) + .trace(subscriptionId, through(pds, hydrant, bobbin)) + .step("bobbin lists the subscription", subscribed(issueUri, true)) + .sees(testid("toggleSubscription", { subscribed: "true" })) + .reload() + .sees(testid("toggleSubscription", { subscribed: "true" })) + + // muting puts ignore: true, deleting the record would bring the default notices back + .click(testid("toggleSubscription", { subscribed: "true" })) + .sends( + subscriptionId, + pds, + (ctx) => ({ rkey: rkeyOf(ctx, subscriptionId), record: { ignore: true } }), + "put" + ) + .sees(testid("toggleSubscription", { subscribed: "false", pending: "true" })) + .trace(subscriptionId, through(pds, hydrant, bobbin)) + .step("bobbin has the subscription as a mute", subscribed(issueUri, "muted")) + .sees(testid("toggleSubscription", { subscribed: "false" })) + .reload() + .sees(testid("toggleSubscription", { subscribed: "false" })); + +// state changes match by ticket, comments by their own uri +interface Told { + type: string; + slot: Slot; + from?: true; +} + +const isTold = (ctx: Ctx, told: Told) => (notice: Notice) => + notice.type === told.type && + (told.from ? notice.uri : aboutOf(notice)) === ctx.get(told.slot).uri; + +const q = JSON.stringify; +const noticeRow = (told: Told) => + target(`the ${told.type} notice`, (tab, ctx) => { + const uri = told.from ? `[data-uri=${q(ctx.get(told.slot).uri)}]` : ""; + return tab + .locator(`[data-testid="notificationRow"][data-type=${q(told.type)}]${uri}`) + .filter({ visible: true }) + .first(); + }); +const noticeOpener = (told: Told) => + target(`the link of the ${told.type} notice`, (tab, ctx) => + noticeRow(told).locate(tab, ctx).locator('[data-testid="notificationPhraseLink"]') + ); + +const toldTo = + (told: Told, who: (ctx: Ctx) => Account, actor: (ctx: Ctx) => Account) => async (ctx: Ctx) => { + const notice = await noticeFor(ctx, told.type, isTold(ctx, told), who(ctx)); + expect(notice.actorDid).toBe(actor(ctx).did); + expect(notice.read).toBe(false); + }; + +// a missing notice only proves something if someone else got one +const toldNobody = (told: Told, who: (ctx: Ctx) => Account) => async (ctx: Ctx) => { + const found = (await noticesOf(who(ctx))).filter(isTold(ctx, told)); + expect(found, `${who(ctx).name}'s ${told.type} notices`).toEqual([]); +}; + +const markedRead = (told: Told) => async (ctx: Ctx) => { + await until(`deliberi should list the ${told.type} notice as read`, 10_000, async () => { + const found = (await noticesOf(ctx.actor)).find(isTold(ctx, told)); + return [found?.read === true, `it lists ${JSON.stringify(found)}`]; + }); +}; + +const landsOn = (here: Where) => (flow: Flow) => flow.at(here).sees(testid("ticketTitle")); +// pull pages can have a version after the path +const landsUnder = (here: Where) => (flow: Flow) => + flow.step(`the notice opens ${here.describe}`, async (ctx) => { + const path = here.path(ctx).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + await expect(ctx.page).toHaveURL(new RegExp(`^${config.webUrl}${path}(/|$)`)); + await expect(testid("ticketTitle").locate(ctx.page, ctx)).toBeVisible(); + }); + +const opensNotice = (told: Told, unread: number, lands: (flow: Flow) => Flow) => (flow: Flow) => + flow + .open(notices) + .sees(noticeRow(told)) + .sees(bell(unread)) + .click(noticeOpener(told)) + .use(lands) + .step(`deliberi lists the ${told.type} notice as read`, markedRead(told)) + .sees(bell(unread - 1)); + +// a missing row only proves something once the control row shows the list loaded +const notAmongNotices = (control: Told, missing: Told) => (flow: Flow) => + flow + .open(notices) + .sees(noticeRow(control)) + .sees(not(noticeRow(missing))); + +const deliberiToo = through(knot, hydrant, bobbin, deliberi); + +const closed = { type: "issue_closed", slot: ticketId } satisfies Told; +const reopened = { type: "issue_reopen", slot: ticketId } satisfies Told; +const issueOpened = { type: "issue_created", slot: ticketId } satisfies Told; +const toggle = (subscribed: boolean) => + testid("toggleSubscription", { subscribed: String(subscribed) }); + +fanOpened("closing and reopening an issue tells whoever opened it and not whoever did it") + .open(page.issue(ticketId)) + .changeState( + ticketId, + "closed", + testid("requestStateChange", { state: "open" }), + testid("requestStateChange", { state: "closed" }), + deliberiToo + ) + .step( + "deliberi made the fan an issue_closed notice naming the owner", + toldTo(closed, theFan, theOwner) + ) + .step("deliberi made the owner none", toldNobody(closed, theOwner)) + .use(notAmongNotices(issueOpened, closed)) + .actor("fan") + .use(opensNotice(closed, 1, landsOn(page.issue(ticketId)))) + + .actor("owner") + .open(page.issue(ticketId)) + .changeState( + ticketId, + "open", + testid("requestStateChange", { state: "closed" }), + testid("requestStateChange", { state: "open" }), + deliberiToo + ) + .step( + "deliberi made the fan an issue_reopen notice naming the owner", + toldTo(reopened, theFan, theOwner) + ) + .step("deliberi made the owner none", toldNobody(reopened, theOwner)) + .use(notAmongNotices(issueOpened, reopened)) + .actor("fan") + .use(opensNotice(reopened, 1, landsOn(page.issue(ticketId)))); + +const merged = { type: "pull_merged", slot: pullId } satisfies Told; +const pullClosed = { type: "pull_closed", slot: pullId } satisfies Told; +const pullReopened = { type: "pull_reopen", slot: pullId } satisfies Told; +const pullOpened = { type: "pull_created", slot: pullId } satisfies Told; + +fanPulled("merging a pull tells whoever opened it and not whoever merged it") + .open(page.pull(pullId)) + .use(mergesThrough(deliberiToo)) + .step( + "deliberi made the fan a pull_merged notice naming the owner", + toldTo(merged, theFan, theOwner) + ) + .step("deliberi made the owner none", toldNobody(merged, theOwner)) + .use(notAmongNotices(pullOpened, merged)) + .actor("fan") + .use(opensNotice(merged, 1, landsUnder(page.pull(pullId)))); + +fanPulled("closing and reopening a pull tells whoever opened it and not whoever did it") + .open(page.pull(pullId)) + .changeState(pullId, "closed", testid("requestClose"), testid("changeState"), deliberiToo) + .step( + "deliberi made the fan a pull_closed notice naming the owner", + toldTo(pullClosed, theFan, theOwner) + ) + .step("deliberi made the owner none", toldNobody(pullClosed, theOwner)) + .use(notAmongNotices(pullOpened, pullClosed)) + .actor("fan") + .use(opensNotice(pullClosed, 1, landsUnder(page.pull(pullId)))) + + .actor("owner") + .open(page.pull(pullId)) + .changeState(pullId, "open", testid("changeState"), testid("requestClose"), deliberiToo) + .step( + "deliberi made the fan a pull_reopen notice naming the owner", + toldTo(pullReopened, theFan, theOwner) + ) + .step("deliberi made the owner none", toldNobody(pullReopened, theOwner)) + .use(notAmongNotices(pullOpened, pullReopened)) + .actor("fan") + .use(opensNotice(pullReopened, 1, landsUnder(page.pull(pullId)))); + +const aComment = record("org.tangled.feed.comment"); +const anotherComment = record("org.tangled.feed.comment"); + +// comments through the api, so the page sends nothing this flow could check +const commentsAs = (slot: Slot, words: string) => async (ctx: Ctx) => { + const written = made( + `${ctx.actor.name} commenting`, + await comments.add(ctx.actor, ctx.repo.did, ctx.get(ticketId), words) + ); + ctx.adopt(slot, written); + await bobbin.indexed(written); +}; + +const newComment = within(testid("commentEditor", { mode: "new" }), testid("body")); +const submitComment = within(testid("commentEditor", { mode: "new" }), testid("submit")); + +const hasIgnore = (muted: boolean) => async (ctx: Ctx) => { + const { body } = await get( + pdsRecordUrl(ctx.actor.did, subscriptionId.nsid, rkeyOf(ctx, subscriptionId)) + ); + expect(body.value.ignore === true, JSON.stringify(body.value)).toBe(muted); +}; + +const firstComment = { type: "issue_commented", slot: aComment, from: true } satisfies Told; +const secondComment = { type: "issue_commented", slot: anotherComment, from: true } satisfies Told; +const theAuthor = (ctx: Ctx) => recall(ctx, "author"); +const fanFollowsAuthor = async (ctx: Ctx) => { + const written = await pdsCall(ctx.actor, "com.atproto.repo.putRecord", { + collection: followId.nsid, + rkey: theAuthor(ctx).did, + record: { $type: followId.nsid, createdAt: new Date().toISOString() } satisfies Follow + }); + await bobbin.indexed(written); +}; +const followed = { type: "followed", slot: followId } satisfies Told; + +flow("an author who mutes their own issue hears no comments until they comment again") + .actor("owner") + .step("remember the owner", remember("owner")) + .repo() + .actor("author") + .step("remember the author", remember("author")) + .open(page.newIssue) + .fill(testid("title"), issueTitle) + .click(testid("submit")) + .trace(ticketId, deliberiToo) + // opening subscribes the author, so this record exists to mute + .trace(subscriptionId, through(pds, hydrant, bobbin, deliberi)) + .open(page.issue(ticketId)) + .sees(toggle(true)) + .click(toggle(true)) + .sends( + subscriptionId, + pds, + (ctx) => ({ rkey: rkeyOf(ctx, subscriptionId), record: { ignore: true } }), + "put" + ) + .sees(testid("toggleSubscription", { subscribed: "false", pending: "true" })) + .trace(subscriptionId, through(pds, hydrant, bobbin, deliberi)) + .step("bobbin has the subscription as a mute", subscribed(issueUri, "muted")) + .sees(toggle(false)) + + .actor("fan") + .step("remember the fan", remember("fan")) + .step("the fan comments", commentsAs(aComment, "while muted")) + .follow(aComment, through(deliberi)) + .step("the fan follows the author, a notice the mute doesn't touch", fanFollowsAuthor) + .actor("owner") + .step("deliberi told the owner about the comment", toldTo(firstComment, theOwner, theFan)) + .actor("author") + .step("deliberi told the author about the follow, so it had caught up", async (ctx) => { + await noticeFor(ctx, "followed", (n) => n.actorDid === recall(ctx, "fan").did); + }) + .step("deliberi made the muted author no notice from the comment", async (ctx) => { + expect(await noticesMadeFrom(ctx.get(aComment).uri, [ctx.actor])).toEqual([]); + }) + .open(notices) + .sees(noticeRow(followed)) + .sees(not(noticeRow(firstComment))) + .open(page.issue(ticketId)) + .fill(newComment, comment) + .click(submitComment) + .trace(commentId, deliberiToo) + .trace(subscriptionId, through(pds, hydrant, bobbin, deliberi)) + .step("the subscription record lost its ignore", hasIgnore(false)) + .step("bobbin lists the subscription as live", subscribed(issueUri, true)) + .sees(toggle(true)) + .reload() + .sees(toggle(true)) + + .actor("fan") + .step("the fan comments again", commentsAs(anotherComment, "now they are back")) + .follow(anotherComment, through(deliberi)) + .actor("author") + .step( + "deliberi told the author about the second comment", + toldTo(secondComment, theAuthor, theFan) + ) + .use(opensNotice(secondComment, 2, landsOn(page.issue(ticketId)))); + +const theOther = (ctx: Ctx) => recall(ctx, "other"); + +fanOpened("an owner who mutes an issue on their repo hears no comments until they unmute it") + .open(page.issue(ticketId)) + // no subscription row, but the owner still gets told + .step("bobbin has no subscription for the owner", subscribed(issueUri, false)) + .sees(toggle(true)) + .click(toggle(true)) + .sends( + subscriptionId, + pds, + (ctx) => ({ + record: { + ignore: true, + subject: { $type: "org.tangled.feed.subscription#record", uri: issueUri(ctx) } + } + }), + "create" + ) + .sees(testid("toggleSubscription", { subscribed: "false", pending: "true" })) + .trace(subscriptionId, through(pds, hydrant, bobbin, deliberi)) + .step("bobbin has the subscription as a mute", subscribed(issueUri, "muted")) + .reload() + .sees(toggle(false)) + + .actor("other") + .step("remember the other", remember("other")) + .step("the other comments", commentsAs(aComment, "while the owner is muted")) + .follow(aComment, through(deliberi)) + .actor("fan") + .step("deliberi told the fan who opened it", toldTo(firstComment, theFan, theOther)) + .actor("owner") + .step( + "deliberi made the muted owner no notice from the comment", + toldNobody(firstComment, theOwner) + ) + .use(notAmongNotices(issueOpened, firstComment)) + .actor("fan") + .use(opensNotice(firstComment, 1, landsOn(page.issue(ticketId)))) + + .actor("owner") + .open(page.issue(ticketId)) + .sees(toggle(false)) + .click(toggle(false)) + .sends(subscriptionId, pds, (ctx) => ({ rkey: rkeyOf(ctx, subscriptionId) }), "put") + .sees(testid("toggleSubscription", { subscribed: "true", pending: "true" })) + .trace(subscriptionId, through(pds, hydrant, bobbin, deliberi)) + .step("the subscription record lost its ignore", hasIgnore(false)) + .step("bobbin lists the subscription as live", subscribed(issueUri, true)) + + .actor("other") + .step("the other comments again", commentsAs(anotherComment, "now the owner is back")) + .follow(anotherComment, through(deliberi)) + .actor("owner") + .step( + "deliberi told the owner about the second comment", + toldTo(secondComment, theOwner, theOther) + ) + // the mute visit already marked the created notice read + .use(opensNotice(secondComment, 1, landsOn(page.issue(ticketId)))); + +flow( + "an issue's author and the repo's owner start out subscribed with no record, a stranger doesn't" +) + .actor("fan") + .step("remember the fan", remember("fan")) + .actor("owner") + .repo() + .actor("fan") + .ticket(ticketId, { title: "opened through the api" }) + + .actor("owner") + .open(page.issue(ticketId)) + .sees(toggle(true)) + .step("bobbin has no subscription for the owner", subscribed(issueUri, false)) + .actor("fan") + .open(page.issue(ticketId)) + .sees(toggle(true)) + .step("bobbin has no subscription for the author", subscribed(issueUri, false)) + .actor("other") + .open(page.issue(ticketId)) + .sees(toggle(false)) + .step("bobbin has no subscription for the stranger", subscribed(issueUri, false)); diff --git a/e2e/specs/strings.spec.ts b/e2e/specs/strings.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..b9959c91ea22df6c77c729f54b90b3c5e0768fe3 --- /dev/null +++ b/e2e/specs/strings.spec.ts @@ -0,0 +1,179 @@ +import { expect } from "@playwright/test"; + +import { getOk } from "../harness/api"; +import { config } from "../harness/config"; +import { + bobbin, + type Ctx, + flow, + hydrant, + link, + not, + pds, + record, + replacing, + rkeyOf, + starId, + target, + text, + testid, + through, + where, + within +} from "../harness/flow"; +import { get, until, xrpc } from "../harness/net"; +import { acceptConfirm, pdsRecordUrl, recall, remember } from "../harness/shared"; + +const string = record("org.tangled.snippet.string"); +const stringFilename = "notes.txt"; +const stringBody = "the needle is in the haystack"; +const newString = where("the new string form", () => "/strings/new"); +const stringPage = where( + "the string page", + (ctx) => `/strings/${recall(ctx, "writer").handle}/${rkeyOf(ctx, string)}` +); +const stringsTab = where( + "the writer's strings tab", + (ctx) => `/${recall(ctx, "writer").handle}?tab=strings` +); +// the editor is a web component, its contenteditable is in a shadow root +const stringEditor = target("the string editor", (tab) => + tab.locator("diffs-container").getByRole("textbox") +); +const typeInEditor = + (words: string, replace = false) => + async (ctx: Ctx) => { + const editor = stringEditor.locate(ctx.page, ctx); + const typed = ctx.page.locator( + `[data-testid="contentBytes"][data-bytes="${Buffer.byteLength(words)}"]` + ); + // the editor drops keys when it redraws after the filename changes + await until("the editor should hold what was typed", 30_000, async () => { + await editor.click(); + await ctx.page.waitForTimeout(500); + await ctx.page.keyboard.press("ControlOrMeta+a"); + if (!replace) await ctx.page.keyboard.press("Backspace"); + await editor.pressSequentially(words, { delay: 20 }); + const held = await typed.isVisible({ timeout: 1500 }).catch(() => false); + return [held, held ? "it does" : "it does not"]; + }); + }; + +const starredStringCard = target("the starred string's card", (tab, ctx) => + tab.locator( + `[data-component="StringCardContent"]:has(a[href="/strings/${recall(ctx, "writer").handle}/${rkeyOf(ctx, string)}"])` + ) +); + +const stringOnPds = async (ctx: Ctx) => { + const { value } = await getOk( + pdsRecordUrl(recall(ctx, "writer").did, string.nsid, rkeyOf(ctx, string)) + ); + const blob = await get( + xrpc(config.pdsUrl, "com.atproto.sync.getBlob", { + did: recall(ctx, "writer").did, + cid: value.snippet.content.ref.$link + }) + ); + return { value, contents: blob.body as string }; +}; + +const writtenString = (name: string) => + flow(name) + .actor("writer") + .step("remember the writer", remember("writer")) + .open(newString) + .fill(testid("filename"), stringFilename) + .fill(testid("description"), "my notes") + .step("types the content", typeInEditor(stringBody)) + .click(testid("submit")) + .sends( + string, + pds, + { + record: { + $type: string.nsid, + snippet: { name: stringFilename }, + description: "my notes" + } + }, + "put" + ) + .trace(string, through(pds, hydrant, bobbin)) + .at(stringPage) + .sees(text(stringBody)); + +writtenString("creating a string") + .step("the pds keeps the text in a blob", async (ctx) => { + const { value, contents } = await stringOnPds(ctx); + expect(value.snippet.name).toBe(stringFilename); + expect(contents).toBe(stringBody); + }) + .reload() + .sees(text(stringBody)) + .sees(within(testid("stringDescription"), text("my notes"))) + .open(stringsTab) + .sees(link(stringFilename)); + +const edited = "the needle moved to another haystack"; + +writtenString("editing a string") + .click(testid("edit")) + .fill(testid("description"), "my edited notes") + .step("types the new content", typeInEditor(edited, true)) + .click(testid("submit")) + .sends( + string, + pds, + replacing(string, { + snippet: { name: stringFilename }, + description: "my edited notes" + }), + "put" + ) + .trace(string, through(pds, hydrant, bobbin)) + .sees(text(edited)) + .sees(within(testid("stringDescription"), text("my edited notes"))) + .step("the pds keeps the new text", async (ctx) => { + expect((await stringOnPds(ctx)).contents).toBe(edited); + }) + .reload() + .sees(text(edited)) + .sees(not(text(stringBody))); + +writtenString("deleting a string") + .step("agree to the confirm", acceptConfirm) + .click(testid("moreMenu")) + .click(testid("delete")) + .sends(string, pds, replacing(string), "delete") + .trace(string, through(pds, hydrant, bobbin)) + .at(stringsTab) + .sees(not(link(stringFilename))) + .open(stringPage) + .sees(testid("errorTitle", { status: "404" })); + +writtenString("starring a string") + .actor("fan") + .open(stringPage) + .click(testid("star", { starred: "false" })) + .sends( + starId, + pds, + (ctx) => ({ + record: { + subject: { + $type: "org.tangled.feed.star#record", + record: { uri: ctx.get(string).uri, cid: ctx.get(string).cid } + } + } + }), + "create" + ) + .sees(testid("star", { starred: "true" })) + .trace(starId, through(pds, hydrant, bobbin)) + .reload() + .sees(testid("star", { starred: "true" })) + .open(where("the fan's starred tab", (ctx) => `/${ctx.actor.handle}?tab=starred`)) + .sees(starredStringCard) + .sees(within(starredStringCard, link(stringFilename))) + .sees(within(starredStringCard, text("my notes"))); diff --git a/e2e/specs/webhooks.spec.ts b/e2e/specs/webhooks.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..6fc69c26087a8e839d370d2129a61ea172fe7989 --- /dev/null +++ b/e2e/specs/webhooks.spec.ts @@ -0,0 +1,257 @@ +import { expect } from "@playwright/test"; + +import { spindleGet } from "../harness/api"; +import { config } from "../harness/config"; +import { + type Ctx, + bobbin, + flow, + hydrant, + knot, + page, + patchId, + pullId, + record, + rkeyOf, + settingsAt, + target, + testid, + through, + unique +} from "../harness/flow"; +import { + base, + branches, + finished, + merges, + onBranches, + pipelineFor, + renamesTo, + workflow, + workflowPath +} from "../harness/shared"; +import { WEBHOOKS, delivers, dropsHook, hookOn, hookUrl, nothingMore } from "../harness/webhooks"; + +const secret = "e2e-webhook-secret"; + +// the base push has to be through spindle before the hook exists, or it would deliver that push too +const onHookedRepo = (name: string, events: string[], options: { secret?: string } = {}) => + flow(name) + .actor("owner") + .repo({ spindle: config.spindleDid }) + .push({ ...base, [workflowPath]: workflow }) + .step( + "spindle has a pipeline for the first push", + pipelineFor("push", (ctx) => ctx.pushed.sha) + ) + .step("point a webhook at the receiver", hookOn(events, options)); + +const tips = new WeakMap(); +const rememberTip = async (ctx: Ctx) => { + tips.set(ctx, ctx.repo.heads[ctx.repo.defaultBranch]); +}; + +const pushDelivered = delivers( + "push", + (ctx) => `Tangled-Hook/${ctx.pushed.sha.slice(0, 7)}`, + (body, ctx) => { + expect(body).toMatchObject({ + ref: `refs/heads/${ctx.repo.defaultBranch}`, + before: tips.get(ctx), + after: ctx.pushed.sha, + pusher: { did: ctx.actor.did }, + repository: { name: ctx.repo.name } + }); + } +); + +onHookedRepo("pushing delivers one signed push webhook", ["push"], { secret }) + .step("remember the branch tip", rememberTip) + .push({ "e2e/more.txt": "more\n" }) + .step("the receiver got the push", pushDelivered) + .step("nothing else arrived", nothingMore) + .step("take the webhook away", dropsHook); + +onHookedRepo("a push that skips ci still delivers an unsigned push webhook", ["push"]) + .step("remember the branch tip", rememberTip) + .push({ "e2e/more.txt": "more\n" }, { options: ["skip-ci"] }) + .step("the receiver got the push", pushDelivered) + .step("nothing else arrived", nothingMore) + .step("take the webhook away", dropsHook); + +const hookSwitch = target("the webhook switch", (tab, ctx) => + testid("webhookSwitch", { url: hookUrl(ctx) }).locate(tab, ctx) +); + +onHookedRepo("a webhook switched off gets nothing", ["push"], { secret }) + .open(settingsAt("hooks")) + .click(hookSwitch) + .step("the switch goes off", async (ctx) => { + await expect(hookSwitch.locate(ctx.page, ctx)).not.toBeChecked(); + }) + .step("spindle lists it inactive", async (ctx) => { + const listed = await spindleGet(ctx.actor, `${WEBHOOKS}.listWebhooks`, { + repo: ctx.repo.did + }); + expect(listed.body.webhooks).toMatchObject([{ url: hookUrl(ctx), active: false }]); + }) + .push({ "e2e/more.txt": "more\n" }) + .step( + "spindle has a pipeline for the push", + pipelineFor("push", (ctx) => ctx.pushed.sha) + ) + .step("nothing arrived", nothingMore) + .step("take the webhook away", dropsHook); + +let renamedTo = ""; + +flow("renaming a repo delivers one repository:renamed webhook") + .actor("owner") + .repo({ spindle: config.spindleDid }) + .step("point a webhook at the receiver", hookOn(["repository:renamed"], { secret })) + .step("pick the new name", async (ctx) => { + renamedTo = `${ctx.repo.name}-renamed`; + }) + .use(renamesTo(() => renamedTo)) + .step("the rename finishes on the new settings page", (ctx) => + finished( + ctx, + "the rename did not finish", + ctx.page.waitForURL(`${config.webUrl}/${ctx.actor.handle}/${renamedTo}/settings`, { + timeout: 5000 + }) + ) + ) + .step( + "the receiver got the rename", + delivers("repository:renamed", "Tangled-Hook/rename", (body, ctx) => { + expect(body).toMatchObject({ + old_name: ctx.repo.name, + new_name: renamedTo, + sender: { did: ctx.actor.did }, + repository: { name: renamedTo, full_name: `${ctx.actor.did}/${renamedTo}` } + }); + }) + ) + .step("nothing else arrived", nothingMore) + .step("take the webhook away", dropsHook); + +const pullEvents = [ + "pull_request:created", + "pull_request:resubmitted", + "pull_request:merged", + "pull_request:closed", + "pull_request:reopened" +]; +const pullTitle = unique("a pull to hook"); + +// who the page acts as changes with the step, so a pull names its author and its sender apart +const pullDelivered = ( + action: string, + who: (ctx: Ctx) => string, + state: string, + round: number, + sha: (ctx: Ctx) => string +) => + delivers(`pull_request:${action}`, "Tangled-Hook/pull_request", (body, ctx) => { + expect(body).toMatchObject({ + action, + sender: { did: who(ctx) }, + pull_request: { + title: pullTitle, + state, + target_branch: branches.into, + round_number: round, + owner: { did: author }, + source: { sha: sha(ctx) } + } + }); + }); +let author = ""; + +const resubmittedId = record("org.tangled.review.patch"); + +// the author opens it so the payload's owner and sender can be told apart once the owner acts +const openedPull = (name: string) => + onBranches(name, { ci: true }) + .step( + "spindle has a pipeline for the first push", + pipelineFor("push", (ctx) => ctx.repo.heads[branches.into]) + ) + .step("point a webhook at the receiver", hookOn(pullEvents, { secret })) + .actor("author") + .step("remember the author", async (ctx) => { + author = ctx.actor.did; + }) + .pull(pullId, patchId, { ...branches, title: pullTitle }) + .step( + "the receiver got the new pull", + pullDelivered( + "created", + (ctx) => ctx.actor.did, + "open", + 1, + (ctx) => ctx.repo.heads[branches.from] + ) + ); + +openedPull("pull request webhooks follow a pull from opening to merging") + .actor("owner") + .open(page.pull(pullId)) + .changeState(pullId, "closed", testid("requestClose"), testid("changeState")) + .step( + "the receiver got the close", + pullDelivered( + "closed", + (ctx) => ctx.actor.did, + "closed", + 1, + (ctx) => ctx.repo.heads[branches.from] + ) + ) + .changeState(pullId, "open", testid("changeState"), testid("requestClose")) + .step( + "the receiver got the reopen", + pullDelivered( + "reopened", + (ctx) => ctx.actor.did, + "open", + 1, + (ctx) => ctx.repo.heads[branches.from] + ) + ) + .use(merges) + .step( + "the receiver got the merge", + pullDelivered( + "merged", + (ctx) => ctx.actor.did, + "merged", + 1, + (ctx) => ctx.repo.heads[branches.from] + ) + ) + .step("nothing else arrived", nothingMore) + .step("take the webhook away", dropsHook); + +openedPull("resubmitting a pull delivers pull_request:resubmitted") + .actor("owner") + .push({ "e2e/more.txt": "more\n" }, { branch: branches.from }) + .mirrored() + .actor("author") + .open(page.pull(pullId)) + .click(testid("resubmit")) + .sends(resubmittedId, knot, (ctx) => ({ patch: { ticket: rkeyOf(ctx, pullId) } }), "create") + .trace(resubmittedId, through(knot, hydrant, bobbin)) + .step( + "the receiver got the resubmit", + pullDelivered( + "resubmitted", + (ctx) => ctx.actor.did, + "open", + 2, + (ctx) => ctx.repo.heads[branches.from] + ) + ) + .step("nothing else arrived", nothingMore) + .step("take the webhook away", dropsHook); diff --git a/e2e/tools/sweep.spec.ts b/e2e/tools/sweep.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..1bc1a1c70ff8f4f0c97b39427ee37de27d73942a --- /dev/null +++ b/e2e/tools/sweep.spec.ts @@ -0,0 +1,63 @@ +import { test } from "@playwright/test"; + +import { accounts, getOk, repos } from "../harness/api"; +import { config, seeded } from "../harness/config"; +import { partsOf, stampedAt, xrpc } from "../harness/net"; + +const AGE_MINUTES = Number(process.env.E2E_SWEEP_MINUTES ?? 30); +if (!(AGE_MINUTES >= 0)) { + throw new Error( + `E2E_SWEEP_MINUTES is a number of minutes, saw ${process.env.E2E_SWEEP_MINUTES}` + ); +} + +// stopping at the first page would silently miss the rest +const listAll = async (nsid: string, params: Record, key: string) => { + const all: any[] = []; + let cursor: string | undefined; + do { + const page = await getOk( + xrpc(config.pdsUrl, nsid, { ...params, limit: "100", ...(cursor ? { cursor } : {}) }) + ); + all.push(...page[key]); + cursor = page.cursor; + } while (cursor); + return all; +}; + +test("sweep leaked accounts", async () => { + test.setTimeout(300_000); + const keep = new Set(Object.values(seeded).map((account) => account.did)); + const failed: string[] = []; + for (const { did } of await listAll("com.atproto.sync.listRepos", {}, "repos")) { + if (keep.has(did)) continue; + const info = await getOk( + xrpc(config.pdsUrl, "com.atproto.repo.describeRepo", { repo: did }) + ); + const name = info.handle.split(".")[0]; + const made = stampedAt(name); + if (!made || Date.now() - made < AGE_MINUTES * 60_000) continue; + + const account = { name, handle: info.handle, did }; + const declared = await listAll( + "com.atproto.repo.listRecords", + { repo: did, collection: "org.tangled.repo.declaration" }, + "records" + ); + let stuck = false; + for (const record of declared) { + await repos + .remove(account, { did: record.value.repo, name: partsOf(record.uri).rkey }) + .catch((cause) => { + stuck = true; + failed.push(`repo of ${name}: ${String(cause).slice(0, 200)}`); + }); + } + // the account is the only way back to its repos, so keep it while one fails + if (stuck) continue; + await accounts.removeMember(account).catch(() => {}); + await accounts.remove(account); + console.log("swept", info.handle); + } + if (failed.length) throw new Error(`could not sweep:\n ${failed.join("\n ")}`); +}); diff --git a/e2e/tools/tidy.spec.ts b/e2e/tools/tidy.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..45bf788effd5d5c18576b5514f1ab78dd795a08c --- /dev/null +++ b/e2e/tools/tidy.spec.ts @@ -0,0 +1,19 @@ +import { expect, test } from "@playwright/test"; +import { readdirSync } from "node:fs"; + +import { LEDGER, replay } from "../harness/chores"; + +// what a worker owed is replayed once it is gone, so this waits for its entries to clear +test("tidy what a stopped worker still owed", async () => { + const worker = process.env.E2E_TIDY_FOR; + test.setTimeout(90_000); + await expect + .poll( + async () => { + await replay("dead"); + return readdirSync(LEDGER).filter((name) => name.split("-")[1] === worker); + }, + { intervals: [1000], timeout: 80_000 } + ) + .toEqual([]); +}); diff --git a/e2e/tsconfig.json b/e2e/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..3080aa4a7b3ac60796d5f3b5a5d06f27f4098848 --- /dev/null +++ b/e2e/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "esnext", + "moduleResolution": "bundler", + "strict": true, + "noEmit": true, + "types": ["node"], + "skipLibCheck": true, + // the web's generated lexicon types import atcute, and web/ may have no node_modules, so + // they read ours. no extension, because playwright loads the .js through these paths too + "paths": { + "@atcute/lexicons": ["./node_modules/@atcute/lexicons/dist/index"], + "@atcute/lexicons/*": [ + "./node_modules/@atcute/lexicons/dist/*/index", + "./node_modules/@atcute/lexicons/dist/*" + ], + "@atcute/atproto/types/*": [ + "./node_modules/@atcute/atproto/dist/lexicons/types/com/atproto/*.d.ts" + ] + } + }, + "include": ["**/*.ts"] +} diff --git a/e2e/up b/e2e/up new file mode 100755 index 0000000000000000000000000000000000000000..4d4511b06ca68acb413d58639d4e530af831152b --- /dev/null +++ b/e2e/up @@ -0,0 +1,179 @@ +#!/bin/sh +# starts the localinfra stack the way e2e needs it, with the web as a production build. +# --check says whether it is up and its web build is current: 0 yes, 1 not up, 2 web out of date +# a live pc stack wins over compose, so this check has to come first +set -eu +here="$(cd "$(dirname "$0")" && pwd)" +cd "$here/.." + +E2E_WEB_SRC="$(node "$here/harness/web-hash.ts")" +export E2E_WEB_SRC + +pcdir="$PWD/localinfra/pc" +state="$(here="$pcdir" && . "$pcdir/bin/env.sh" && echo "$TNGL_STATE")" +sock="$state/run/pc.sock" +pc() { curl -sf --unix-socket "$sock" "$@"; } +pc_running() { pc http://localhost/live >/dev/null 2>&1; } + +stack="${E2E_STACK:-}" +if [ -z "$stack" ]; then + if pc_running; then + stack=pc + elif command -v podman >/dev/null 2>&1 || command -v docker >/dev/null 2>&1; then + stack=compose + else + stack=pc + fi +fi + +if [ "${1:-}" = "--stack" ]; then + echo "$stack" + exit +fi + +# only the browser knows these names, so curl needs --resolve to reach caddy +alice_resolves() { + curl -sf --cacert localinfra/certs/root.crt --resolve pds.tngl.boltless.dev:443:127.0.0.1 \ + "https://pds.tngl.boltless.dev/xrpc/com.atproto.identity.resolveHandle?handle=alice.pds.tngl.boltless.dev" >/dev/null +} + +if [ "$stack" = pc ]; then + log="$state/up.log" + launcher=0 + # a live stack only rebuilds the web. its other services have to come from this tree + backend="$(git log -1 --format=%H -- . ':(exclude)web' ':(exclude)e2e')" + # setup writes the source hash first, then the env checksum + built="$(cut -d ' ' -f 1 "$state/data/web/built" 2>/dev/null || true)" + + if [ "${1:-}" = "--check" ]; then + pc_running || exit 1 + [ "$built" = "$E2E_WEB_SRC" ] || exit 2 + exit + fi + + if pc_running; then + if [ "$(cat "$state/backend" 2>/dev/null)" != "$backend" ]; then + echo "the running stack's services are not from this tree, stop it with: curl --unix-socket $sock -X POST http://localhost/project/stop" >&2 + exit 1 + fi + if [ "$built" != "$E2E_WEB_SRC" ]; then + (here="$pcdir" && . "$pcdir/bin/env.sh" && sh "$pcdir/services/web/setup") + pc -X POST http://localhost/process/restart/web >/dev/null + # a restart keeps reporting the old exit code until the new process is up, so poll for ready + tries=0 + until pc http://localhost/process/web | grep -q '"is_ready":"Ready"'; do + tries=$((tries + 1)) + [ "$tries" -lt 300 ] || { echo "the web did not come back after its restart" >&2; exit 1; } + sleep 1 + done + fi + else + mkdir -p "$state" + echo "$backend" >"$state/backend" + # pipeline specs need a spindle, and compose always runs one. the sites specs need the worker and sitesd + for profile in spindle sites; do + case ",${TNGL_PROFILES:-}," in + *,"$profile",*) ;; + *) TNGL_PROFILES=${TNGL_PROFILES:+$TNGL_PROFILES,}$profile ;; + esac + done + export TNGL_PROFILES + # not -D, linux port forwarders and slirp die with the foreground process + # own session, so a caller killing its process group doesn't take the stack down + own_session() { + if command -v setsid >/dev/null 2>&1; then exec setsid "$@"; fi + exec perl -MPOSIX -e 'POSIX::setsid(); exec @ARGV' "$@" + } + if command -v nix >/dev/null 2>&1; then + own_session nix run .#localinfra -- -t=false >"$log" 2>&1 & + else + own_session "$pcdir/bin/up" -t=false >"$log" 2>&1 & + fi + launcher=$! + fi + + node -e ' + const http = require("node:http"); + const list = () => + new Promise((resolve, reject) => + http + .get({ socketPath: process.argv[1], path: "/processes" }, (res) => { + let body = ""; + res.on("data", (chunk) => (body += chunk)); + res.on("end", () => resolve(JSON.parse(body).data)); + }) + .on("error", reject) + ); + const failed = (p) => p.status === "Error" || (p.status === "Completed" && p.exit_code !== 0); + const launcher = Number(process.argv[2]); + const deadline = Date.now() + Number(process.argv[3]) * 1000; + const launcherGone = () => { + if (!launcher) return false; + try { + process.kill(launcher, 0); + return false; + } catch { + return true; + } + }; + (async () => { + for (;;) { + const all = await list().catch(() => []); + const bad = all.find(failed); + if (bad) { + console.error(`${bad.name} failed, its log is in the stack`); + process.exit(1); + } + if (all.length && all.every((p) => p.status === "Completed" || p.is_ready === "Ready")) return; + if (launcherGone() || Date.now() > deadline) { + console.error("the stack did not come up"); + process.exit(1); + } + await new Promise((resolve) => setTimeout(resolve, 2000)); + } + })(); + ' "$sock" "$launcher" "${E2E_UP_TIMEOUT:-3600}" || { + [ "$launcher" = 0 ] || tail -n 30 "$log" >&2 + exit 1 + } + + if ! alice_resolves; then + echo "alice doesn't resolve, the stack's init-data hasn't run" >&2 + exit 1 + fi + exit +fi + +# the web is a production build here, so no page waits on a module compiling when it is first asked for +web="-f localinfra/docker-compose.web-build.yml" +if command -v podman >/dev/null 2>&1; then + engine=podman + compose="podman compose -f docker-compose.yml $web" +else + # docker doesn't publish ports from an internal network, which caddy sits on + engine=docker + compose="docker compose -f docker-compose.yml -f localinfra/docker-compose.docker.yml $web" +fi + +if [ "${1:-}" = "--check" ]; then + id=$($compose ps --status running -q web 2>/dev/null || true) + [ -n "$id" ] || exit 1 + # the dev server answers too, but every first page waits on it compiling + $engine inspect --format '{{join .Config.Cmd " "}}' "$id" | grep -q "vite build" || exit 1 + built="$($engine inspect --format '{{index .Config.Labels "e2e.web-src"}}' "$id")" + [ "$built" = "$E2E_WEB_SRC" ] || exit 2 + exit +fi + +if [ ! -f localinfra/certs/root.crt ]; then + echo "no dev ca yet, run ./localinfra/scripts/make-certs.sh and trust localinfra/certs/root.crt" >&2 + exit 1 +fi + +# compose only recreates the web when this label value changes +$compose up -d --wait + +if ! alice_resolves; then + echo "alice doesn't resolve, the stack's init-data hasn't run" >&2 + exit 1 +fi diff --git a/email-did/package.json b/email-did/package.json new file mode 100644 index 0000000000000000000000000000000000000000..e81979284be0caee892e2ab6e527b97c845f8c39 --- /dev/null +++ b/email-did/package.json @@ -0,0 +1,18 @@ +{ + "name": "@tangled/email-did-worker", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "wrangler dev", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "deploy": "wrangler deploy" + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20260317.1", + "typescript": "^5.9.3", + "vitest": "~3.0.7", + "wrangler": "^4.75.0" + } +} diff --git a/email-did/pnpm-lock.yaml b/email-did/pnpm-lock.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebeed0e8bbc71fe970ae091a55b3620443c1206c --- /dev/null +++ b/email-did/pnpm-lock.yaml @@ -0,0 +1,1868 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + devDependencies: + '@cloudflare/workers-types': + specifier: ^4.20260317.1 + version: 4.20260702.1 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ~3.0.7 + version: 3.0.9(supports-color@10.2.2) + wrangler: + specifier: ^4.75.0 + version: 4.136.2(@cloudflare/workers-types@4.20260702.1) + +packages: + + '@cloudflare/kv-asset-handler@0.5.0': + resolution: {integrity: sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==} + engines: {node: '>=22.0.0'} + + '@cloudflare/unenv-preset@2.16.2': + resolution: {integrity: sha512-JBP1+Z7ZSNG/d4mRP+y8VC5dka3tZVMLEZRvS+rzQ4DGV1EoxRFQckcJTTkXbHSQiTj0DtNI01Zwb/V2fX0mvQ==} + peerDependencies: + unenv: 2.0.0-rc.24 + workerd: '>1.20260305.0 <2.0.0-0' + peerDependenciesMeta: + workerd: + optional: true + + '@cloudflare/workerd-darwin-64@1.20260921.1': + resolution: {integrity: sha512-3iB2WnYOlZ29T+1zhCwbHFExCBp6E9bgmDUMryATYwrIGEQ1YbvR78m4ydm56XKN/d/yF3803ivMGfZMYDtiMg==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@cloudflare/workerd-darwin-arm64@1.20260921.1': + resolution: {integrity: sha512-FpqVR7IQXVBmGtajyonEmhmb5UAsmV7dTaIkpemmHZXHEw7uYpkhkzKPjc4BOPhNQy8iwt2p+RZBPMY3Y7/bvQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@cloudflare/workerd-linux-64@1.20260921.1': + resolution: {integrity: sha512-riAJIohaVp5A8Sqy4yKlzHOaLPOICMf5oey+jC2rm45RVT+wK8+7UU0d31Dy/02Nc8YUkobAFwNVjX06P8WQ5g==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@cloudflare/workerd-linux-arm64@1.20260921.1': + resolution: {integrity: sha512-tnJu08tT7s0XWDqp3O0H/vCp0voy9OqVAzspb89biMo1dh8IiEpnyXnoPmdJ7H4qBnXCmXgy0kuEphuvpDPj9w==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@cloudflare/workerd-windows-64@1.20260921.1': + resolution: {integrity: sha512-VgNcRPstoZMb1G94JTrx+jU24GtkkazNfox0gnF/2fkuXpcfW/M0e0xvdMovYfwt8ZxG5AB2ZNvanD6ufBwiuQ==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + + '@cloudflare/workers-types@4.20260702.1': + resolution: {integrity: sha512-mOhf5TUEB1m2vPrxtqoIGfz0fUC9xyxRDx5gWHy5s+OCo6dcV+g7wI1R7gYCMFohhqF/2y2xeKVwMwCJjfn/WA==} + + '@cspotcode/source-map-support@0.8.1': + resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} + engines: {node: '>=12'} + + '@emnapi/runtime@1.11.3': + resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} + + '@esbuild/aix-ppc64@0.25.12': + resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.25.12': + resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.25.12': + resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.25.12': + resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.25.12': + resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.25.12': + resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.25.12': + resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.25.12': + resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.25.12': + resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.25.12': + resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.25.12': + resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.25.12': + resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.25.12': + resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.25.12': + resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.25.12': + resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.25.12': + resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.25.12': + resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.12': + resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.25.12': + resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.12': + resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.25.12': + resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.25.12': + resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.25.12': + resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.25.12': + resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.25.12': + resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.25.12': + resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.35.4': + resolution: {integrity: sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.35.4': + resolution: {integrity: sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [darwin] + + '@img/sharp-freebsd-wasm32@0.35.4': + resolution: {integrity: sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==} + engines: {node: '>=20.9.0'} + os: [freebsd] + + '@img/sharp-libvips-darwin-arm64@1.3.3': + resolution: {integrity: sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.3.3': + resolution: {integrity: sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.3.3': + resolution: {integrity: sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-arm@1.3.3': + resolution: {integrity: sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-ppc64@1.3.3': + resolution: {integrity: sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-riscv64@1.3.3': + resolution: {integrity: sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-s390x@1.3.3': + resolution: {integrity: sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-x64@1.3.3': + resolution: {integrity: sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linuxmusl-arm64@1.3.3': + resolution: {integrity: sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-libvips-linuxmusl-x64@1.3.3': + resolution: {integrity: sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-linux-arm64@0.35.4': + resolution: {integrity: sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-arm@0.35.4': + resolution: {integrity: sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==} + engines: {node: '>=20.9.0'} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-ppc64@0.35.4': + resolution: {integrity: sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==} + engines: {node: '>=20.9.0'} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-riscv64@0.35.4': + resolution: {integrity: sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==} + engines: {node: '>=20.9.0'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-s390x@0.35.4': + resolution: {integrity: sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==} + engines: {node: '>=20.9.0'} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-x64@0.35.4': + resolution: {integrity: sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-linuxmusl-arm64@0.35.4': + resolution: {integrity: sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-linuxmusl-x64@0.35.4': + resolution: {integrity: sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-wasm32@0.35.4': + resolution: {integrity: sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==} + engines: {node: '>=20.9.0'} + + '@img/sharp-webcontainers-wasm32@0.35.4': + resolution: {integrity: sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==} + engines: {node: '>=20.9.0'} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.35.4': + resolution: {integrity: sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.35.4': + resolution: {integrity: sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==} + engines: {node: ^20.9.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.35.4': + resolution: {integrity: sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [win32] + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + + '@jridgewell/trace-mapping@0.3.9': + resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + resolution: {integrity: sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@poppinss/colors@4.1.6': + resolution: {integrity: sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==} + + '@poppinss/dumper@0.6.5': + resolution: {integrity: sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==} + + '@poppinss/exception@1.2.3': + resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} + + '@rollup/rollup-android-arm-eabi@4.63.4': + resolution: {integrity: sha512-I+BSHzTAhKN2n7ZwGZsegGcZjDpLqFOMAtJz/u6uFGe0pUFbq56dEHjqJV/ZUdRJtNXNxA+hREUatZBvMR3Oiw==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.63.4': + resolution: {integrity: sha512-pu3BdjS2LtEzRu2elmGzS3fIeWSZy4BMDIaLNwjorO76+k2d0LMluijhsDx3KQyQBQ/lLUZCQA9/s6csvUfuhw==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.63.4': + resolution: {integrity: sha512-xfSrj9MHnWK9GaSqT9U0ImHtH/N8WZlHLx4cZHiuLcqs640hvZ3hLPd5UR2AZS57FaE8HrRUSpltbZdWRxHiDA==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.63.4': + resolution: {integrity: sha512-bqU99PLJb/dqb3S0GIMdeuyAEETSUgZBoqXYd3Sd+WCsV+MmPhnN6JrotWyir31+QgH7EvvE5/mwGJlEoci8Fw==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.63.4': + resolution: {integrity: sha512-JinsFZ5G40oXQb+sUuiA5x689vhr6dDYK0H0NL+rwKdL6CqnmYN8PE4ZwfRSoIjrCxqTQG/SLfTtSvHeGxoVlw==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.63.4': + resolution: {integrity: sha512-GAdA4UxpiNm27cLHr2GqXBpAD0x9FqwYBY7/YSP0Ss0/PNi4k8gbviqpIpYbVSRBaS2ZcegXEzgTQMbRNCwxCw==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.63.4': + resolution: {integrity: sha512-qDd6NoA1znaLjp4jR5U/KWCdLAKDJNB8W9ChbbDaKbo0xA+Atln5HK6LFCZ4oJQpemtRZA288DCirFRjrspptw==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm-musleabihf@4.63.4': + resolution: {integrity: sha512-WtB5Tz5KTNINb8ZA+8sQ7bmjuS1JrRT7YverYIhUGdWWDlpzVWmIwuZE+jidkEXUn1l0zrEkaIMa8dHF3NGcsA==} + cpu: [arm] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-arm64-gnu@4.63.4': + resolution: {integrity: sha512-VcQ3L1tjnkKzWjryAVaFhHEWcqOfICX9uxVVoDzm2t0DpgKRHd2zOpVrJc0xsWeBZcBFyYROCIBdyR/fS174pg==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm64-musl@4.63.4': + resolution: {integrity: sha512-6+ZQX6P5s0cMDN2Ypb8Lbm2+/sZYmZjdaYny992ujUU9UKi/4CWoJWsl1pNvjWJHNHGK51m+jKGLlh1ylb2ifQ==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-loong64-gnu@4.63.4': + resolution: {integrity: sha512-D72ZnvkFkBXOfzMMQLcwfPLyGkKb7HZ9/mf97B7v6/P5Lbv4oFOtSY/uHbS8lH6uKUOxoKiuokdb50XZSzzbJw==} + cpu: [loong64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-loong64-musl@4.63.4': + resolution: {integrity: sha512-piU6BxeqA3O9KSu3kRCIQQtNqFFaTu21SEV4FwaRZowpnj3bLaWPZHw+xFqCs0XlJ+aOH3PTRWGoglH+mKA/OA==} + cpu: [loong64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-ppc64-gnu@4.63.4': + resolution: {integrity: sha512-/5PGpHwqt2EEEOUs1XwzubE/ucr0dWDQ+to3zqi4Ds7EWpwtQ79wXc4JBoxqj/OwpawTsKWzJxHfSuBOq3DrWA==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-ppc64-musl@4.63.4': + resolution: {integrity: sha512-cX3beZDLWt7G2oJF+nhChiT+qtaihs+S2xi7ziGmVB+2pwPng6D0Ed0HmElQOgv2UsUmSJJLGwpBao/3TDx3VA==} + cpu: [ppc64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-riscv64-gnu@4.63.4': + resolution: {integrity: sha512-1uz2mGWHyptR7DgHHrlbdRAjXK7v7elGZ9lMja910/RP+ZYbX6xAmCiU9UZSX4hqmgtHMv6lr5l3kq1HIOpcag==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-riscv64-musl@4.63.4': + resolution: {integrity: sha512-nLS8topojxyz7SRpKR2IODRpQ0XPZ+xaOXvT3+hqK/Uy8Lo5HFgkkIBiIrCu5tL5YqzTvgovGw55PwpahTAGig==} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-s390x-gnu@4.63.4': + resolution: {integrity: sha512-gs7DRKotr3l3q+jGPQBjH0ng1FjlEDm5ueQrkw5JtQvtLyEIcLASqAEaor56BhkKRzk+IcQzrcanBdb/bBQn8g==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-gnu@4.63.4': + resolution: {integrity: sha512-791ET7W17NnScOZM7h4dX5hYspxE28htPFsb1awY/NRR8+PRNkS53e475rDdxXXDrP+kwnCcNWg9CX5ztn/Aqw==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-musl@4.63.4': + resolution: {integrity: sha512-iwZQRcmj7g88g3tzefIrQY7qvmuA/cfYwhrDtTBhsmukO4U2huVO5W+86XacUMRvdSFVAc6kZUZy21JaRwiB9w==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rollup/rollup-openbsd-x64@4.63.4': + resolution: {integrity: sha512-dVHFp9gRWrdTpnqQuGfCwd7hOQDatK1VCP2iWhLY/cGrOQs/ucFzJ6A5SRqbXX12ZDI8EUuejSM5kwg+ja7Png==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.63.4': + resolution: {integrity: sha512-t3NlauOW6gxZVVFcBEnO62Cb4wbyDFL416gTg1uFI/2tgqYQlf69FbSE115Ajre9I+c26Lk4mcmdFUsS/DGifQ==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.63.4': + resolution: {integrity: sha512-xWuIaSye5FWZF8+UYtVEcHtRJDN5kN9Kfgxx3Kq8XIov9KSKbc1fiqQCm90SKrgQbUXZelbnUhnlUJmfSE7P9A==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.63.4': + resolution: {integrity: sha512-9ALJJUOg/ZflMJepVo2PlgsGxSaxN7SQ4Z8GoZfVlarWr6r3rkHUNsd/zAio7p4YMtChSMXPionxej4Hkf6CXQ==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.63.4': + resolution: {integrity: sha512-blj9z5qx/Pv4WU0W1NMFDB97e0JH5ed+aZGywW8WCvp/NhWX/4PFAq5uu6Q0AebNn+Vo6KzUYDT++JzTT5ojlQ==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.63.4': + resolution: {integrity: sha512-Erx822VRBwLa124shbj+wNXe//BOgMEctDV0m1aqTQdNO1S69DgNUCFKC1RCeZfixs1J31l6igk1ziyXErbigQ==} + cpu: [x64] + os: [win32] + + '@sindresorhus/is@7.2.0': + resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} + engines: {node: '>=18'} + + '@speed-highlight/core@1.2.24': + resolution: {integrity: sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@vitest/expect@3.0.9': + resolution: {integrity: sha512-5eCqRItYgIML7NNVgJj6TVCmdzE7ZVgJhruW0ziSQV4V7PvLkDL1bBkBdcTs/VuIz0IxPb5da1IDSqc1TR9eig==} + + '@vitest/mocker@3.0.9': + resolution: {integrity: sha512-ryERPIBOnvevAkTq+L1lD+DTFBRcjueL9lOUfXsLfwP92h4e+Heb+PjiqS3/OURWPtywfafK0kj++yDFjWUmrA==} + peerDependencies: + msw: ^2.4.9 + vite: ^5.0.0 || ^6.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@3.0.9': + resolution: {integrity: sha512-OW9F8t2J3AwFEwENg3yMyKWweF7oRJlMyHOMIhO5F3n0+cgQAJZBjNgrF8dLwFTEXl5jUqBLXd9QyyKv8zEcmA==} + + '@vitest/pretty-format@3.2.7': + resolution: {integrity: sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==} + + '@vitest/runner@3.0.9': + resolution: {integrity: sha512-NX9oUXgF9HPfJSwl8tUZCMP1oGx2+Sf+ru6d05QjzQz4OwWg0psEzwY6VexP2tTHWdOkhKHUIZH+fS6nA7jfOw==} + + '@vitest/snapshot@3.0.9': + resolution: {integrity: sha512-AiLUiuZ0FuA+/8i19mTYd+re5jqjEc2jZbgJ2up0VY0Ddyyxg/uUtBDpIFAy4uzKaQxOW8gMgBdAJJ2ydhu39A==} + + '@vitest/spy@3.0.9': + resolution: {integrity: sha512-/CcK2UDl0aQ2wtkp3YVWldrpLRNCfVcIOFGlVGKO4R5eajsH393Z1yiXLVQ7vWsj26JOEjeZI0x5sm5P4OGUNQ==} + + '@vitest/utils@3.0.9': + resolution: {integrity: sha512-ilHM5fHhZ89MCp5aAaM9uhfl1c2JdxVxl3McqsdVyVNN6JffnEen8UMCdRTzOhGXNQGo5GNL9QugHrz727Wnng==} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + blake3-wasm@2.1.5: + resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} + + cac@6.7.14: + resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} + engines: {node: '>=8'} + + chai@5.3.3: + resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + engines: {node: '>=18'} + + check-error@2.1.3: + resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} + engines: {node: '>= 16'} + + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deep-eql@5.0.2: + resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} + engines: {node: '>=6'} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + error-stack-parser-es@1.0.5: + resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} + + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + + esbuild@0.25.12: + resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} + engines: {node: '>=18'} + hasBin: true + + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + kleur@4.1.5: + resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} + engines: {node: '>=6'} + + loupe@3.2.1: + resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + miniflare@5.20260921.0-alpha: + resolution: {integrity: sha512-vHH/unOYvV2jA1Q9SdkmzrQhhMoksdwg5jegu6ZeKaaRzgxZhVbt1NdTpQjHF2VTgiBjgP8SiUlUMfruB3N3SQ==} + engines: {node: '>=22.0.0'} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.19: + resolution: {integrity: sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + path-to-regexp@6.3.0: + resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + pathval@2.0.1: + resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} + engines: {node: '>= 14.16'} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.7: + resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} + engines: {node: '>=12'} + + postcss@8.5.28: + resolution: {integrity: sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==} + engines: {node: ^10 || ^12 || >=14} + + rollup@4.63.4: + resolution: {integrity: sha512-4U0liVayNIoLp3GFl1FcI8561WepLnZ1rqfraGh7S9B3Ur5F9S283y8Futii7RUU2C/97tOBmBy7nYvhoiOpbQ==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + sharp@0.35.4: + resolution: {integrity: sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==} + engines: {node: '>=20.9.0'} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@3.10.0: + resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tinypool@1.1.1: + resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} + engines: {node: ^18.0.0 || >=20.0.0} + + tinyrainbow@2.0.0: + resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} + engines: {node: '>=14.0.0'} + + tinyspy@3.0.2: + resolution: {integrity: sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==} + engines: {node: '>=14.0.0'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + + unenv@2.0.0-rc.24: + resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} + + vite-node@3.0.9: + resolution: {integrity: sha512-w3Gdx7jDcuT9cNn9jExXgOyKmf5UOTb6WMHz8LGAm54eS1Elf5OuBhCxl6zJxGhEeIkgsE1WbHuoL0mj/UXqXg==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + + vite@6.4.3: + resolution: {integrity: sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + peerDependencies: + '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + jiti: '>=1.21.0' + less: '*' + lightningcss: ^1.21.0 + sass: '*' + sass-embedded: '*' + stylus: '*' + sugarss: '*' + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + jiti: + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@3.0.9: + resolution: {integrity: sha512-BbcFDqNyBlfSpATmTtXOAOj71RNKDDvjBM/uPfnxxVGrG+FSH2RQIwgeEngTaTkuU/h0ScFvf+tRcKfYXzBybQ==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/debug': ^4.1.12 + '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + '@vitest/browser': 3.0.9 + '@vitest/ui': 3.0.9 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/debug': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + + workerd@1.20260921.1: + resolution: {integrity: sha512-4HyG7G1W4ksa6tUZ8bV2jxDRWuL5PXnHm9+Z1sjFPb9OZNoYtXz4y7QQRh4ibi0BF/lOmlAVjbhkUqsAVZuUKA==} + engines: {node: '>=16'} + hasBin: true + + wrangler@4.136.2: + resolution: {integrity: sha512-BeWkkU7leFyalKi++LDHUaWSumwLVNeBeY2Ma+CqylT/hxs2sKT5e/q88MvXUl4Ng4iPS90H9H1qlb5oK3uhhQ==} + engines: {node: '>=22.0.0'} + hasBin: true + peerDependencies: + '@cloudflare/workers-types': ^5.20260921.1 + peerDependenciesMeta: + '@cloudflare/workers-types': + optional: true + + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + youch-core@0.3.3: + resolution: {integrity: sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==} + + youch@4.1.0-beta.10: + resolution: {integrity: sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==} + +snapshots: + + '@cloudflare/kv-asset-handler@0.5.0': {} + + '@cloudflare/unenv-preset@2.16.2(unenv@2.0.0-rc.24)(workerd@1.20260921.1)': + dependencies: + unenv: 2.0.0-rc.24 + optionalDependencies: + workerd: 1.20260921.1 + + '@cloudflare/workerd-darwin-64@1.20260921.1': + optional: true + + '@cloudflare/workerd-darwin-arm64@1.20260921.1': + optional: true + + '@cloudflare/workerd-linux-64@1.20260921.1': + optional: true + + '@cloudflare/workerd-linux-arm64@1.20260921.1': + optional: true + + '@cloudflare/workerd-windows-64@1.20260921.1': + optional: true + + '@cloudflare/workers-types@4.20260702.1': {} + + '@cspotcode/source-map-support@0.8.1': + dependencies: + '@jridgewell/trace-mapping': 0.3.9 + + '@emnapi/runtime@1.11.3': + dependencies: + tslib: 2.8.1 + optional: true + + '@esbuild/aix-ppc64@0.25.12': + optional: true + + '@esbuild/aix-ppc64@0.28.1': + optional: true + + '@esbuild/android-arm64@0.25.12': + optional: true + + '@esbuild/android-arm64@0.28.1': + optional: true + + '@esbuild/android-arm@0.25.12': + optional: true + + '@esbuild/android-arm@0.28.1': + optional: true + + '@esbuild/android-x64@0.25.12': + optional: true + + '@esbuild/android-x64@0.28.1': + optional: true + + '@esbuild/darwin-arm64@0.25.12': + optional: true + + '@esbuild/darwin-arm64@0.28.1': + optional: true + + '@esbuild/darwin-x64@0.25.12': + optional: true + + '@esbuild/darwin-x64@0.28.1': + optional: true + + '@esbuild/freebsd-arm64@0.25.12': + optional: true + + '@esbuild/freebsd-arm64@0.28.1': + optional: true + + '@esbuild/freebsd-x64@0.25.12': + optional: true + + '@esbuild/freebsd-x64@0.28.1': + optional: true + + '@esbuild/linux-arm64@0.25.12': + optional: true + + '@esbuild/linux-arm64@0.28.1': + optional: true + + '@esbuild/linux-arm@0.25.12': + optional: true + + '@esbuild/linux-arm@0.28.1': + optional: true + + '@esbuild/linux-ia32@0.25.12': + optional: true + + '@esbuild/linux-ia32@0.28.1': + optional: true + + '@esbuild/linux-loong64@0.25.12': + optional: true + + '@esbuild/linux-loong64@0.28.1': + optional: true + + '@esbuild/linux-mips64el@0.25.12': + optional: true + + '@esbuild/linux-mips64el@0.28.1': + optional: true + + '@esbuild/linux-ppc64@0.25.12': + optional: true + + '@esbuild/linux-ppc64@0.28.1': + optional: true + + '@esbuild/linux-riscv64@0.25.12': + optional: true + + '@esbuild/linux-riscv64@0.28.1': + optional: true + + '@esbuild/linux-s390x@0.25.12': + optional: true + + '@esbuild/linux-s390x@0.28.1': + optional: true + + '@esbuild/linux-x64@0.25.12': + optional: true + + '@esbuild/linux-x64@0.28.1': + optional: true + + '@esbuild/netbsd-arm64@0.25.12': + optional: true + + '@esbuild/netbsd-arm64@0.28.1': + optional: true + + '@esbuild/netbsd-x64@0.25.12': + optional: true + + '@esbuild/netbsd-x64@0.28.1': + optional: true + + '@esbuild/openbsd-arm64@0.25.12': + optional: true + + '@esbuild/openbsd-arm64@0.28.1': + optional: true + + '@esbuild/openbsd-x64@0.25.12': + optional: true + + '@esbuild/openbsd-x64@0.28.1': + optional: true + + '@esbuild/openharmony-arm64@0.25.12': + optional: true + + '@esbuild/openharmony-arm64@0.28.1': + optional: true + + '@esbuild/sunos-x64@0.25.12': + optional: true + + '@esbuild/sunos-x64@0.28.1': + optional: true + + '@esbuild/win32-arm64@0.25.12': + optional: true + + '@esbuild/win32-arm64@0.28.1': + optional: true + + '@esbuild/win32-ia32@0.25.12': + optional: true + + '@esbuild/win32-ia32@0.28.1': + optional: true + + '@esbuild/win32-x64@0.25.12': + optional: true + + '@esbuild/win32-x64@0.28.1': + optional: true + + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.3.3 + optional: true + + '@img/sharp-darwin-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.3.3 + optional: true + + '@img/sharp-freebsd-wasm32@0.35.4': + dependencies: + '@img/sharp-wasm32': 0.35.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-darwin-x64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-arm@1.3.3': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-s390x@1.3.3': + optional: true + + '@img/sharp-libvips-linux-x64@1.3.3': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.3.3': + optional: true + + '@img/sharp-linux-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.3.3 + optional: true + + '@img/sharp-linux-arm@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.3.3 + optional: true + + '@img/sharp-linux-ppc64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.3.3 + optional: true + + '@img/sharp-linux-riscv64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.3.3 + optional: true + + '@img/sharp-linux-s390x@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.3.3 + optional: true + + '@img/sharp-linux-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.3.3 + optional: true + + '@img/sharp-linuxmusl-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 + optional: true + + '@img/sharp-linuxmusl-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.3.3 + optional: true + + '@img/sharp-wasm32@0.35.4': + dependencies: + '@emnapi/runtime': 1.11.3 + optional: true + + '@img/sharp-webcontainers-wasm32@0.35.4': + dependencies: + '@img/sharp-wasm32': 0.35.4 + optional: true + + '@img/sharp-win32-arm64@0.35.4': + optional: true + + '@img/sharp-win32-ia32@0.35.4': + optional: true + + '@img/sharp-win32-x64@0.35.4': + optional: true + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.6.0': {} + + '@jridgewell/trace-mapping@0.3.9': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.6.0 + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + optional: true + + '@poppinss/colors@4.1.6': + dependencies: + kleur: 4.1.5 + + '@poppinss/dumper@0.6.5': + dependencies: + '@poppinss/colors': 4.1.6 + '@sindresorhus/is': 7.2.0 + supports-color: 10.2.2 + + '@poppinss/exception@1.2.3': {} + + '@rollup/rollup-android-arm-eabi@4.63.4': + optional: true + + '@rollup/rollup-android-arm64@4.63.4': + optional: true + + '@rollup/rollup-darwin-arm64@4.63.4': + optional: true + + '@rollup/rollup-darwin-x64@4.63.4': + optional: true + + '@rollup/rollup-freebsd-arm64@4.63.4': + optional: true + + '@rollup/rollup-freebsd-x64@4.63.4': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.63.4': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.63.4': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.63.4': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.63.4': + optional: true + + '@rollup/rollup-linux-x64-musl@4.63.4': + optional: true + + '@rollup/rollup-openbsd-x64@4.63.4': + optional: true + + '@rollup/rollup-openharmony-arm64@4.63.4': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.63.4': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.63.4': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.63.4': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.63.4': + optional: true + + '@sindresorhus/is@7.2.0': {} + + '@speed-highlight/core@1.2.24': {} + + '@types/estree@1.0.9': {} + + '@vitest/expect@3.0.9': + dependencies: + '@vitest/spy': 3.0.9 + '@vitest/utils': 3.0.9 + chai: 5.3.3 + tinyrainbow: 2.0.0 + + '@vitest/mocker@3.0.9(vite@6.4.3)': + dependencies: + '@vitest/spy': 3.0.9 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 6.4.3 + + '@vitest/pretty-format@3.0.9': + dependencies: + tinyrainbow: 2.0.0 + + '@vitest/pretty-format@3.2.7': + dependencies: + tinyrainbow: 2.0.0 + + '@vitest/runner@3.0.9': + dependencies: + '@vitest/utils': 3.0.9 + pathe: 2.0.3 + + '@vitest/snapshot@3.0.9': + dependencies: + '@vitest/pretty-format': 3.0.9 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@3.0.9': + dependencies: + tinyspy: 3.0.2 + + '@vitest/utils@3.0.9': + dependencies: + '@vitest/pretty-format': 3.0.9 + loupe: 3.2.1 + tinyrainbow: 2.0.0 + + assertion-error@2.0.1: {} + + blake3-wasm@2.1.5: {} + + cac@6.7.14: {} + + chai@5.3.3: + dependencies: + assertion-error: 2.0.1 + check-error: 2.1.3 + deep-eql: 5.0.2 + loupe: 3.2.1 + pathval: 2.0.1 + + check-error@2.1.3: {} + + cookie@1.1.1: {} + + debug@4.4.3(supports-color@10.2.2): + dependencies: + ms: 2.1.3 + optionalDependencies: + supports-color: 10.2.2 + + deep-eql@5.0.2: {} + + detect-libc@2.1.2: {} + + error-stack-parser-es@1.0.5: {} + + es-module-lexer@1.7.0: {} + + esbuild@0.25.12: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.12 + '@esbuild/android-arm': 0.25.12 + '@esbuild/android-arm64': 0.25.12 + '@esbuild/android-x64': 0.25.12 + '@esbuild/darwin-arm64': 0.25.12 + '@esbuild/darwin-x64': 0.25.12 + '@esbuild/freebsd-arm64': 0.25.12 + '@esbuild/freebsd-x64': 0.25.12 + '@esbuild/linux-arm': 0.25.12 + '@esbuild/linux-arm64': 0.25.12 + '@esbuild/linux-ia32': 0.25.12 + '@esbuild/linux-loong64': 0.25.12 + '@esbuild/linux-mips64el': 0.25.12 + '@esbuild/linux-ppc64': 0.25.12 + '@esbuild/linux-riscv64': 0.25.12 + '@esbuild/linux-s390x': 0.25.12 + '@esbuild/linux-x64': 0.25.12 + '@esbuild/netbsd-arm64': 0.25.12 + '@esbuild/netbsd-x64': 0.25.12 + '@esbuild/openbsd-arm64': 0.25.12 + '@esbuild/openbsd-x64': 0.25.12 + '@esbuild/openharmony-arm64': 0.25.12 + '@esbuild/sunos-x64': 0.25.12 + '@esbuild/win32-arm64': 0.25.12 + '@esbuild/win32-ia32': 0.25.12 + '@esbuild/win32-x64': 0.25.12 + + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + expect-type@1.4.0: {} + + fdir@6.5.0(picomatch@4.0.7): + optionalDependencies: + picomatch: 4.0.7 + + fsevents@2.3.3: + optional: true + + kleur@4.1.5: {} + + loupe@3.2.1: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + + miniflare@5.20260921.0-alpha: + dependencies: + '@cspotcode/source-map-support': 0.8.1 + sharp: 0.35.4 + undici: 7.29.0 + workerd: 1.20260921.1 + ws: 8.21.0 + youch: 4.1.0-beta.10 + transitivePeerDependencies: + - '@types/node' + - bufferutil + - utf-8-validate + + ms@2.1.3: {} + + nanoid@3.3.19: {} + + path-to-regexp@6.3.0: {} + + pathe@2.0.3: {} + + pathval@2.0.1: {} + + picocolors@1.1.1: {} + + picomatch@4.0.7: {} + + postcss@8.5.28: + dependencies: + nanoid: 3.3.19 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + rollup@4.63.4: + dependencies: + '@types/estree': 1.0.9 + optionalDependencies: + '@napi-rs/lzma-linux-x64-gnu': 1.5.1 + '@rollup/rollup-android-arm-eabi': 4.63.4 + '@rollup/rollup-android-arm64': 4.63.4 + '@rollup/rollup-darwin-arm64': 4.63.4 + '@rollup/rollup-darwin-x64': 4.63.4 + '@rollup/rollup-freebsd-arm64': 4.63.4 + '@rollup/rollup-freebsd-x64': 4.63.4 + '@rollup/rollup-linux-arm-gnueabihf': 4.63.4 + '@rollup/rollup-linux-arm-musleabihf': 4.63.4 + '@rollup/rollup-linux-arm64-gnu': 4.63.4 + '@rollup/rollup-linux-arm64-musl': 4.63.4 + '@rollup/rollup-linux-loong64-gnu': 4.63.4 + '@rollup/rollup-linux-loong64-musl': 4.63.4 + '@rollup/rollup-linux-ppc64-gnu': 4.63.4 + '@rollup/rollup-linux-ppc64-musl': 4.63.4 + '@rollup/rollup-linux-riscv64-gnu': 4.63.4 + '@rollup/rollup-linux-riscv64-musl': 4.63.4 + '@rollup/rollup-linux-s390x-gnu': 4.63.4 + '@rollup/rollup-linux-x64-gnu': 4.63.4 + '@rollup/rollup-linux-x64-musl': 4.63.4 + '@rollup/rollup-openbsd-x64': 4.63.4 + '@rollup/rollup-openharmony-arm64': 4.63.4 + '@rollup/rollup-win32-arm64-msvc': 4.63.4 + '@rollup/rollup-win32-ia32-msvc': 4.63.4 + '@rollup/rollup-win32-x64-gnu': 4.63.4 + '@rollup/rollup-win32-x64-msvc': 4.63.4 + fsevents: 2.3.3 + + semver@7.8.5: {} + + sharp@0.35.4: + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.8.5 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.35.4 + '@img/sharp-darwin-x64': 0.35.4 + '@img/sharp-freebsd-wasm32': 0.35.4 + '@img/sharp-libvips-darwin-arm64': 1.3.3 + '@img/sharp-libvips-darwin-x64': 1.3.3 + '@img/sharp-libvips-linux-arm': 1.3.3 + '@img/sharp-libvips-linux-arm64': 1.3.3 + '@img/sharp-libvips-linux-ppc64': 1.3.3 + '@img/sharp-libvips-linux-riscv64': 1.3.3 + '@img/sharp-libvips-linux-s390x': 1.3.3 + '@img/sharp-libvips-linux-x64': 1.3.3 + '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 + '@img/sharp-libvips-linuxmusl-x64': 1.3.3 + '@img/sharp-linux-arm': 0.35.4 + '@img/sharp-linux-arm64': 0.35.4 + '@img/sharp-linux-ppc64': 0.35.4 + '@img/sharp-linux-riscv64': 0.35.4 + '@img/sharp-linux-s390x': 0.35.4 + '@img/sharp-linux-x64': 0.35.4 + '@img/sharp-linuxmusl-arm64': 0.35.4 + '@img/sharp-linuxmusl-x64': 0.35.4 + '@img/sharp-webcontainers-wasm32': 0.35.4 + '@img/sharp-win32-arm64': 0.35.4 + '@img/sharp-win32-ia32': 0.35.4 + '@img/sharp-win32-x64': 0.35.4 + + siginfo@2.0.0: {} + + source-map-js@1.2.1: {} + + stackback@0.0.2: {} + + std-env@3.10.0: {} + + supports-color@10.2.2: {} + + tinybench@2.9.0: {} + + tinyexec@0.3.2: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + + tinypool@1.1.1: {} + + tinyrainbow@2.0.0: {} + + tinyspy@3.0.2: {} + + tslib@2.8.1: + optional: true + + typescript@5.9.3: {} + + undici@7.29.0: {} + + unenv@2.0.0-rc.24: + dependencies: + pathe: 2.0.3 + + vite-node@3.0.9(supports-color@10.2.2): + dependencies: + cac: 6.7.14 + debug: 4.4.3(supports-color@10.2.2) + es-module-lexer: 1.7.0 + pathe: 2.0.3 + vite: 6.4.3 + transitivePeerDependencies: + - '@types/node' + - jiti + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + + vite@6.4.3: + dependencies: + esbuild: 0.25.12 + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + postcss: 8.5.28 + rollup: 4.63.4 + tinyglobby: 0.2.17 + optionalDependencies: + fsevents: 2.3.3 + + vitest@3.0.9(supports-color@10.2.2): + dependencies: + '@vitest/expect': 3.0.9 + '@vitest/mocker': 3.0.9(vite@6.4.3) + '@vitest/pretty-format': 3.2.7 + '@vitest/runner': 3.0.9 + '@vitest/snapshot': 3.0.9 + '@vitest/spy': 3.0.9 + '@vitest/utils': 3.0.9 + chai: 5.3.3 + debug: 4.4.3(supports-color@10.2.2) + expect-type: 1.4.0 + magic-string: 0.30.21 + pathe: 2.0.3 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinypool: 1.1.1 + tinyrainbow: 2.0.0 + vite: 6.4.3 + vite-node: 3.0.9(supports-color@10.2.2) + why-is-node-running: 2.3.0 + transitivePeerDependencies: + - jiti + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + + workerd@1.20260921.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20260921.1 + '@cloudflare/workerd-darwin-arm64': 1.20260921.1 + '@cloudflare/workerd-linux-64': 1.20260921.1 + '@cloudflare/workerd-linux-arm64': 1.20260921.1 + '@cloudflare/workerd-windows-64': 1.20260921.1 + + wrangler@4.136.2(@cloudflare/workers-types@4.20260702.1): + dependencies: + '@cloudflare/kv-asset-handler': 0.5.0 + '@cloudflare/unenv-preset': 2.16.2(unenv@2.0.0-rc.24)(workerd@1.20260921.1) + blake3-wasm: 2.1.5 + esbuild: 0.28.1 + miniflare: 5.20260921.0-alpha + path-to-regexp: 6.3.0 + unenv: 2.0.0-rc.24 + workerd: 1.20260921.1 + optionalDependencies: + '@cloudflare/workers-types': 4.20260702.1 + fsevents: 2.3.3 + transitivePeerDependencies: + - '@types/node' + - bufferutil + - utf-8-validate + + ws@8.21.0: {} + + youch-core@0.3.3: + dependencies: + '@poppinss/exception': 1.2.3 + error-stack-parser-es: 1.0.5 + + youch@4.1.0-beta.10: + dependencies: + '@poppinss/colors': 4.1.6 + '@poppinss/dumper': 0.6.5 + '@speed-highlight/core': 1.2.24 + cookie: 1.1.1 + youch-core: 0.3.3 diff --git a/email-did/pnpm-workspace.yaml b/email-did/pnpm-workspace.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1ffadc049b7a5e0ed20528ec5929bad7862781e --- /dev/null +++ b/email-did/pnpm-workspace.yaml @@ -0,0 +1,6 @@ +allowBuilds: + esbuild: true + workerd: true +onlyBuiltDependencies: + - esbuild + - workerd diff --git a/email-did/src/index.ts b/email-did/src/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..a93132981c8b5e945687fb151efbc4ea76a55737 --- /dev/null +++ b/email-did/src/index.ts @@ -0,0 +1,43 @@ +// the verified email → did mapping, loaded once at cutover by cmd/email-did-migrate. +// plain git emails resolve to the signup did. only svfe calls this, with the shared token +export interface Env { + EMAIL_DID: KVNamespace; + EMAIL_DID_TOKEN?: string; +} + +export interface EmailDidStore { + get(key: string): Promise; +} + +const MAX_LOOKUPS = 250; + +const encoder = new TextEncoder(); + +const authorized = (request: Request, token: string | undefined): boolean => { + if (!token) return false; + const given = encoder.encode(request.headers.get("authorization") ?? ""); + const want = encoder.encode(`Bearer ${token}`); + return given.byteLength === want.byteLength && crypto.subtle.timingSafeEqual(given, want); +}; + +export const lookup = async (url: URL, store: EmailDidStore): Promise> => { + // repeated params rather than a csv: emails may legally contain commas + const emails = url.searchParams.getAll("emails").filter(Boolean).slice(0, MAX_LOOKUPS); + + // keys are lowercased, so each address is read once however the caller cased it + const byLower = new Map( + await Promise.all( + [...new Set(emails.map((email) => email.toLowerCase()))].map( + async (key) => [key, await store.get(key).catch(() => null)] as const, + ), + ), + ); + return Object.fromEntries(emails.map((email) => [email, byLower.get(email.toLowerCase()) ?? null])); +}; + +export default { + fetch: async (request, env) => + authorized(request, env.EMAIL_DID_TOKEN) + ? Response.json(await lookup(new URL(request.url), env.EMAIL_DID)) + : Response.json({ error: "unauthorized" }, { status: 401 }), +} satisfies ExportedHandler; diff --git a/email-did/test/index.test.ts b/email-did/test/index.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..fe3e5c14488194a0605bb9d504535164675b2000 --- /dev/null +++ b/email-did/test/index.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it, vi } from "vitest"; +import worker, { type Env } from "../src/index"; + +const store = (entries: Record = {}) => ({ + get: vi.fn(async (key: string) => entries[key] ?? null), +}); + +const call = (query: string, { token = "s3cret", auth = "Bearer s3cret" as string | null, entries = {} } = {}) => { + const kv = store(entries); + const env = { EMAIL_DID: kv, EMAIL_DID_TOKEN: token } as unknown as Env; + const request = new Request(`https://email-did.example.workers.dev/${query}`, { + headers: auth === null ? {} : { authorization: auth }, + }) as Parameters[0]; + return { res: worker.fetch(request, env), kv }; +}; + +describe("email-did", () => { + it.each([ + ["no authorization", { auth: null }], + ["a wrong token", { auth: "Bearer nope" }], + ["a token of another length", { auth: "Bearer s3cret-and-more" }], + ["the token without the bearer scheme", { auth: "s3cret" }], + ])("rejects %s without touching the store", async (_, opts) => { + const { res, kv } = call("?emails=alice%40example.com", { ...opts, entries: { "alice@example.com": "did:plc:alice" } }); + expect((await res).status).toBe(401); + expect(kv.get).not.toHaveBeenCalled(); + }); + + it("rejects everything when no token is configured, even a guess at the placeholder", async () => { + const kv = store({ "alice@example.com": "did:plc:alice" }); + const request = new Request("https://x.test/?emails=alice%40example.com", { + headers: { authorization: "Bearer undefined" }, + }) as Parameters[0]; + const res = await worker.fetch(request, { EMAIL_DID: kv } as unknown as Env); + expect(res.status).toBe(401); + expect(kv.get).not.toHaveBeenCalled(); + }); + + it.each([ + ["lowercases keys and answers in the caller's case", "?emails=Alice%40Example.com", { "alice@example.com": "did:plc:alice" }, { "Alice@Example.com": "did:plc:alice" }], + ["maps missing keys to null", "?emails=nobody%40example.com", {}, { "nobody@example.com": null }], + ["keeps a plus-address through query decoding", "?emails=a%2Bb%40example.com", { "a+b@example.com": "did:plc:plus" }, { "a+b@example.com": "did:plc:plus" }], + ["answers an empty map for empty params", "", {}, {}], + ])("%s", async (_, query, entries, expected) => { + const res = await call(query, { entries }).res; + expect(res.status).toBe(200); + await expect(res.json()).resolves.toEqual(expected); + }); + + it("reads a mixed-case duplicate once and answers every case", async () => { + const { res, kv } = call("?emails=a%40b.com&emails=A%40B.com", { entries: { "a@b.com": "did:plc:a" } }); + await expect((await res).json()).resolves.toEqual({ "a@b.com": "did:plc:a", "A@B.com": "did:plc:a" }); + expect(kv.get).toHaveBeenCalledTimes(1); + }); + + it("answers null for a store error instead of failing the batch", async () => { + const env = { + EMAIL_DID: { get: async () => { throw new Error("kv down"); } }, + EMAIL_DID_TOKEN: "s3cret", + } as unknown as Env; + const request = new Request("https://x.test/?emails=a%40b.com&emails=c%40b.com", { + headers: { authorization: "Bearer s3cret" }, + }) as Parameters[0]; + const res = await worker.fetch(request, env); + await expect(res.json()).resolves.toEqual({ "a@b.com": null, "c@b.com": null }); + }); +}); diff --git a/email-did/test/setup.ts b/email-did/test/setup.ts new file mode 100644 index 0000000000000000000000000000000000000000..7df3f696acd2afafd3fb80b02e714b79cc1831cc --- /dev/null +++ b/email-did/test/setup.ts @@ -0,0 +1,10 @@ +// stand-in for workerd's webcrypto extension, which node lacks; tests only need equality +const bytes = (v: ArrayBufferView | ArrayBuffer) => + ArrayBuffer.isView(v) ? new Uint8Array(v.buffer, v.byteOffset, v.byteLength) : new Uint8Array(v); + +Object.defineProperty(crypto.subtle, "timingSafeEqual", { + value: (a: ArrayBufferView | ArrayBuffer, b: ArrayBufferView | ArrayBuffer) => { + const [x, y] = [bytes(a), bytes(b)]; + return x.length === y.length && x.every((v, i) => v === y[i]); + }, +}); diff --git a/email-did/tsconfig.json b/email-did/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..b34a0f03430f4309a22ca9c433adb226d251d4ff --- /dev/null +++ b/email-did/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "lib": ["ES2022"], + "moduleResolution": "bundler", + "types": ["@cloudflare/workers-types"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "noEmit": true + }, + "include": ["src/**/*", "test/**/*"], + "exclude": ["node_modules"] +} diff --git a/email-did/vitest.config.ts b/email-did/vitest.config.ts new file mode 100644 index 0000000000000000000000000000000000000000..3ecb3d4a29e6599ec05a0276c0bd43bbf7f6a862 --- /dev/null +++ b/email-did/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ test: { setupFiles: ["test/setup.ts"] } }); diff --git a/email-did/wrangler.jsonc b/email-did/wrangler.jsonc new file mode 100644 index 0000000000000000000000000000000000000000..d7c0666112aa06002aaac3a503993a68046cc0ea --- /dev/null +++ b/email-did/wrangler.jsonc @@ -0,0 +1,17 @@ +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "email-did", + "main": "src/index.ts", + "workers_dev": true, + "preview_urls": false, + "compatibility_date": "2026-09-01", + "observability": { + "enabled": true, + }, + "kv_namespaces": [ + { + "binding": "EMAIL_DID", + "id": "645c742c4c674357a9a7825bf4ba62ca", + }, + ], +} diff --git a/filetree/filetree.go b/filetree/filetree.go new file mode 100644 index 0000000000000000000000000000000000000000..03961a748676d2fb43af2600f954c0cd411034db --- /dev/null +++ b/filetree/filetree.go @@ -0,0 +1,65 @@ +package filetree + +import ( + "path/filepath" + "sort" + "strings" +) + +type FileTreeNode struct { + Name string + Path string + IsDirectory bool + Level int + Children map[string]*FileTreeNode +} + +// newNode creates a new node +func newNode(name, path string, isDir bool, level int) *FileTreeNode { + return &FileTreeNode{ + Name: name, + Path: path, + IsDirectory: isDir, + Level: level, + Children: make(map[string]*FileTreeNode), + } +} + +func FileTree(files []string) *FileTreeNode { + rootNode := newNode("", "", true, 0) + + sort.Strings(files) + + for _, file := range files { + if file == "" { + continue + } + + parts := strings.Split(filepath.Clean(file), "/") + if len(parts) == 0 { + continue + } + + currentNode := rootNode + currentPath := "" + + for i, part := range parts { + if currentPath == "" { + currentPath = part + } else { + currentPath = filepath.Join(currentPath, part) + } + + isDir := i < len(parts)-1 + level := i + 1 + + if _, exists := currentNode.Children[part]; !exists { + currentNode.Children[part] = newNode(part, currentPath, isDir, level) + } + + currentNode = currentNode.Children[part] + } + } + + return rootNode +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000000000000000000000000000000000..028e7a8286678ececa0b5ed2a3afe28648e770d8 --- /dev/null +++ b/flake.lock @@ -0,0 +1,305 @@ +{ + "nodes": { + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1780397302, + "narHash": "sha256-SQmrkj17xBdvc8lVMbsY7pIJCjKftgNh42R1keP3dLQ=", + "owner": "nix-community", + "repo": "fenix", + "rev": "f6670530f53e69cc284f7aef818eb0f08fe81905", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + "fetch-tangled": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1779479671, + "narHash": "sha256-8PR9nZnK0Fn4gDOikz+3sLdsEJYnKC3gaunBs4N+V9Q=", + "ref": "refs/heads/main", + "rev": "2e3744d31b990248f277d16a11ef81753b366f0f", + "revCount": 22, + "type": "git", + "url": "https://tangled.org/isabelroses.com/fetch-tangled" + }, + "original": { + "type": "git", + "url": "https://tangled.org/isabelroses.com/fetch-tangled" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1751685974, + "narHash": "sha256-NKw96t+BgHIYzHUjkTK95FqYRVKB8DHpVhefWSz/kTw=", + "rev": "549f2762aebeff29a2e5ece7a7dc0f955281a1d1", + "type": "tarball", + "url": "https://git.lix.systems/api/v1/repos/lix-project/flake-compat/archive/549f2762aebeff29a2e5ece7a7dc0f955281a1d1.tar.gz?rev=549f2762aebeff29a2e5ece7a7dc0f955281a1d1" + }, + "original": { + "type": "tarball", + "url": "https://git.lix.systems/lix-project/flake-compat/archive/main.tar.gz" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gomod2nix": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1763982521, + "narHash": "sha256-ur4QIAHwgFc0vXiaxn5No/FuZicxBr2p0gmT54xZkUQ=", + "owner": "nix-community", + "repo": "gomod2nix", + "rev": "02e63a239d6eabd595db56852535992c898eba72", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "gomod2nix", + "type": "github" + } + }, + "hydrant": { + "flake": false, + "locked": { + "lastModified": 1792544679, + "narHash": "sha256-fm0cuIyHjJTSvleJyLbT4E5KugtanuysL/oMOnGhN3I=", + "ref": "refs/heads/main", + "rev": "0128352f6b73d26e7ffaa5a79446ef556e452617", + "revCount": 811, + "type": "git", + "url": "https://tangled.org/ptr.pet/hydrant" + }, + "original": { + "type": "git", + "url": "https://tangled.org/ptr.pet/hydrant" + } + }, + "hydrant-dev": { + "flake": false, + "locked": { + "lastModified": 1792544679, + "narHash": "sha256-fm0cuIyHjJTSvleJyLbT4E5KugtanuysL/oMOnGhN3I=", + "ref": "refs/heads/main", + "rev": "0128352f6b73d26e7ffaa5a79446ef556e452617", + "revCount": 811, + "type": "git", + "url": "https://tangled.org/ptr.pet/hydrant" + }, + "original": { + "type": "git", + "url": "https://tangled.org/ptr.pet/hydrant" + } + }, + "ibm-plex-mono-src": { + "flake": false, + "locked": { + "lastModified": 1731402384, + "narHash": "sha256-OwUmrPfEehLDz0fl2ChYLK8FQM2p0G1+EMrGsYEq+6g=", + "type": "tarball", + "url": "https://github.com/IBM/plex/releases/download/@ibm%2Fplex-mono@1.1.0/ibm-plex-mono.zip" + }, + "original": { + "type": "tarball", + "url": "https://github.com/IBM/plex/releases/download/@ibm%2Fplex-mono@1.1.0/ibm-plex-mono.zip" + } + }, + "indigo": { + "flake": false, + "locked": { + "lastModified": 1753693716, + "narHash": "sha256-DMIKnCJRODQXEHUxA+7mLzRALmnZhkkbHlFT2rCQYrE=", + "owner": "oppiliappan", + "repo": "indigo", + "rev": "5f170569da9360f57add450a278d73538092d8ca", + "type": "github" + }, + "original": { + "owner": "oppiliappan", + "repo": "indigo", + "type": "github" + } + }, + "microvm": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "spectrum": "spectrum" + }, + "locked": { + "lastModified": 1779970379, + "narHash": "sha256-ZHsxoYXXnfJtMVh1/yY+1Eh9hHcPBhE28Qvinauh+BQ=", + "owner": "microvm-nix", + "repo": "microvm.nix", + "rev": "0d49083ba2d7419b22908ac392777c16df9a032e", + "type": "github" + }, + "original": { + "owner": "microvm-nix", + "repo": "microvm.nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1781216227, + "narHash": "sha256-9mUW6gNwoN2SWc/l0fW4svPNOulXLl8ijqKyeSOGgJE=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a0374025a863d007d98e3297f6aa46cc3141c2f0", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-26.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-pds": { + "locked": { + "lastModified": 1791734076, + "narHash": "sha256-i5qk1kLiit98rRbh1AXmWBNT9QXC/y4l5Wk/u5EdpDQ=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "7fd4004d0511991fff3a0524856a9be09e28c5f0", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "fenix": "fenix", + "fetch-tangled": "fetch-tangled", + "flake-compat": "flake-compat", + "gomod2nix": "gomod2nix", + "hydrant": "hydrant", + "hydrant-dev": "hydrant-dev", + "ibm-plex-mono-src": "ibm-plex-mono-src", + "indigo": "indigo", + "microvm": "microvm", + "nixpkgs": "nixpkgs", + "nixpkgs-pds": "nixpkgs-pds", + "sqlite-lib-src": "sqlite-lib-src", + "timeless-fonts-src": "timeless-fonts-src" + } + }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1780337665, + "narHash": "sha256-RS3F+/6dtBIwd5+47GVen3jAk62/KumFZ4q3RVYafDk=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "55af1774e9e840d3b28d12fcfaa72d2e2caa8ac9", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "spectrum": { + "flake": false, + "locked": { + "lastModified": 1778940603, + "narHash": "sha256-voSM8dZNlaOWN3kbYFky+FNY6fFQOEw0xF+ZMpZKkCQ=", + "ref": "refs/heads/main", + "rev": "367dd227f539267eae2b62770b4c17b88ac8c1f1", + "revCount": 1265, + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + }, + "original": { + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + } + }, + "sqlite-lib-src": { + "flake": false, + "locked": { + "lastModified": 1706631843, + "narHash": "sha256-bJoMjirsBjm2Qk9KPiy3yV3+8b/POlYe76/FQbciHro=", + "type": "tarball", + "url": "https://sqlite.org/2024/sqlite-amalgamation-3450100.zip" + }, + "original": { + "type": "tarball", + "url": "https://sqlite.org/2024/sqlite-amalgamation-3450100.zip" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "timeless-fonts-src": { + "flake": false, + "locked": { + "lastModified": 1791052084, + "narHash": "sha256-zYpoDkE/YrICzYaHF4/NSuINczFWB41YymTxaL6KqLY=", + "type": "tarball", + "url": "https://media.timeless.co/type/Timeless-Type-Family-1.094.zip" + }, + "original": { + "type": "tarball", + "url": "https://media.timeless.co/type/Timeless-Type-Family-1.094.zip" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000000000000000000000000000000000..ee941f773c2401217883dbc46bb30bf6e27b2028 --- /dev/null +++ b/flake.nix @@ -0,0 +1,859 @@ +{ + description = "atproto github"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-26.05"; + microvm = { + url = "github:microvm-nix/microvm.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + fenix = { + url = "github:nix-community/fenix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + gomod2nix = { + url = "github:nix-community/gomod2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + flake-compat = { + url = "https://git.lix.systems/lix-project/flake-compat/archive/main.tar.gz"; + flake = false; + }; + indigo = { + url = "github:oppiliappan/indigo"; + flake = false; + }; + timeless-fonts-src = { + url = "https://media.timeless.co/type/Timeless-Type-Family-1.094.zip"; + flake = false; + }; + ibm-plex-mono-src = { + url = "https://github.com/IBM/plex/releases/download/%40ibm%2Fplex-mono%401.1.0/ibm-plex-mono.zip"; + flake = false; + }; + sqlite-lib-src = { + url = "https://sqlite.org/2024/sqlite-amalgamation-3450100.zip"; + flake = false; + }; + fetch-tangled = { + url = "git+https://tangled.org/isabelroses.com/fetch-tangled"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + # the stable pds refuses oauth when it shares a site with the web, as dev hostnames do + nixpkgs-pds.url = "github:nixos/nixpkgs/nixos-unstable"; + # hydrant has to match go.mod, hydrant-dev follows main + hydrant-dev = { + url = "git+https://tangled.org/ptr.pet/hydrant"; + flake = false; + }; + hydrant = { + url = "git+https://tangled.org/ptr.pet/hydrant"; + flake = false; + }; + }; + + outputs = { + self, + nixpkgs, + fenix, + gomod2nix, + indigo, + timeless-fonts-src, + sqlite-lib-src, + ibm-plex-mono-src, + microvm, + fetch-tangled, + hydrant, + hydrant-dev, + nixpkgs-pds, + ... + }: let + supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + tangledPdsFor = system: let + pdsPkgs = nixpkgs-pds.legacyPackages.${system}; + in + pdsPkgs.bluesky-pds.overrideAttrs (old: { + postInstall = + (old.postInstall or "") + + '' + cp ${./pds/preauthorize.mjs} $out/lib/pds/tangled-preauthorize.mjs + substituteInPlace $out/lib/pds/index.ts \ + --replace-fail "import pkg from" "import { mountPreauthorize } from './tangled-preauthorize.mjs' + import pkg from" \ + --replace-fail "onCreated: (pds) => {" "onCreated: (pds) => { + mountPreauthorize(pds)" + ''; + # a pds bump that drops what the route calls fails the build, not the pds at startup + doInstallCheck = true; + installCheckPhase = '' + runHook preInstallCheck + cd $(mktemp -d) + cp ${./pds}/* . + ln -s $out/lib/pds/node_modules node_modules + TANGLED_PDS_BIN=$out/bin/pds ${pdsPkgs.nodejs_22}/bin/node --test preauthorize.test.mjs + runHook postInstallCheck + ''; + __darwinAllowLocalNetworking = true; + }); + pdsFor = system: + (tangledPdsFor system).overrideAttrs (old: { + postInstall = + old.postInstall + + '' + node ${./localinfra/pds-patch.mjs} $out/lib/pds ${./localinfra/pds-theme.css} ${timeless-fonts-src}/Sans-Grotesk/TimelessSansVF.woff2 + ''; + }); + # go 1.26.10 refuses chains through the leaked emudhra intermediates, and nixpkgs + # doesn't ship it yet. only our own builds get it, because swapping go for all of + # nixpkgs rebuilds the world (libcap builds with go) + goFor = pkgs: let + go = pkgs.go_1_26; + in + if nixpkgs.lib.versionOlder go.version "1.26.10" + then + go.overrideAttrs (_: rec { + version = "1.26.10"; + src = pkgs.fetchurl { + url = "https://go.dev/dl/go${version}.src.tar.gz"; + hash = "sha256-/WPog7ovK/ypPBZGsx6+9j4rHk0rMXyXidl2po2B828="; + }; + }) + else go; + # flake registry entries every spindle image gets: nixpkgs is the rev the images are built from + spindleRegistry = let + lock = builtins.fromJSON (builtins.readFile ./flake.lock); + locked = lock.nodes.${lock.nodes.root.inputs.nixpkgs}.locked; + in { + # a tarball, not github:, as nix 2.34 ignores narHash on github inputs + nixpkgs = { + type = "tarball"; + url = "https://github.com/${locked.owner}/${locked.repo}/archive/${locked.rev}.tar.gz"; + inherit (locked) rev lastModified narHash; + }; + unstable = { + type = "tarball"; + url = "https://channels.nixos.org/nixpkgs-unstable/nixexprs.tar.zst"; + }; + }; + nixpkgsFor = forAllSystems (system: + import nixpkgs { + inherit system; + overlays = [ + fetch-tangled.overlays.default + (final: prev: + prev.lib.optionalAttrs prev.stdenv.hostPlatform.isDarwin { + # tsup's dts worker races the cjs build on darwin (EBADF) + wrangler = prev.wrangler.overrideAttrs (old: { + preBuild = + (old.preBuild or "") + + '' + substituteInPlace packages/wrangler/tsup.config.ts --replace-fail "dts: true" "dts: false" + ''; + }); + }) + ]; + }); + + mkPackageSet = pkgs: + pkgs.lib.makeScope pkgs.newScope (self: { + # docs and blog render web/src's components and theme + mkSiteSrc = dir: + pkgs.lib.fileset.toSource { + root = ./.; + fileset = pkgs.lib.fileset.unions [(./. + "/${dir}") ./web/src ./web/tsconfig.json]; + }; + mkGoSrc = import ./nix/go-src.nix { + inherit (pkgs) lib; + root = ./.; + }; + go = goFor pkgs.buildPackages; + buildGoModule = pkgs.buildGoModule.override {inherit (self) go;}; + buildGoApplication = args: + (self.callPackage "${gomod2nix}/builder" { + gomod2nix = gomod2nix.legacyPackages.${pkgs.stdenv.hostPlatform.system}.gomod2nix; + # proxy.golang.org can't fetch from knot2 yet, so sum.golang.org has + # no entry for hydrant and the module fetch fails. the fixed output + # hash pins its content anyway. the builder only uses this stdenv for + # module fetches + stdenvNoCC = + pkgs.stdenvNoCC + // { + mkDerivation = attrs: pkgs.stdenvNoCC.mkDerivation (attrs // {GONOSUMDB = "tangled.org/ptr.pet";}); + }; + }).buildGoApplication ({inherit (self) go;} // args); + rustPlatform = pkgs.makeRustPlatform { + inherit (fenix.packages.${pkgs.stdenv.hostPlatform.system}.stable) rustc cargo; + }; + rustToolchainStatic = let + system = pkgs.stdenv.hostPlatform.system; + muslTarget = pkgs.pkgsStatic.stdenv.hostPlatform.rust.rustcTarget; + in + fenix.packages.${system}.combine [ + fenix.packages.${system}.stable.cargo + fenix.packages.${system}.stable.rustc + fenix.packages.${system}.targets.${muslTarget}.stable.rust-std + ]; + rustPlatformStatic = pkgs.pkgsStatic.makeRustPlatform { + cargo = self.rustToolchainStatic; + rustc = self.rustToolchainStatic; + }; + cargoNix = self.callPackage ./nix/cargo-nix.nix { + inherit (fenix.packages.${pkgs.stdenv.hostPlatform.system}.stable) rustc cargo; + }; + cargoNixStatic = pkgs.pkgsStatic.callPackage ./nix/cargo-nix.nix { + rustc = self.rustToolchainStatic; + cargo = self.rustToolchainStatic; + }; + crateBin = cargoNix: crate: + cargoNix.workspaceMembers.${crate}.build.overrideAttrs (old: { + meta = (old.meta or {}) // {mainProgram = crate;}; + }); + modules = ./nix/gomod2nix.toml; + sqlite-lib = self.callPackage ./nix/pkgs/sqlite-lib.nix { + inherit sqlite-lib-src; + }; + lexgen = self.callPackage ./nix/pkgs/lexgen.nix {inherit indigo;}; + goat = self.callPackage ./nix/pkgs/goat.nix {inherit indigo;}; + web-static-files = self.callPackage ./nix/pkgs/web-static-files.nix { + inherit timeless-fonts-src ibm-plex-mono-src; + }; + ogre-fonts = self.callPackage ./nix/pkgs/ogre-fonts.nix { + inherit timeless-fonts-src; + }; + blog = self.callPackage ./nix/pkgs/blog.nix { + src = self.mkSiteSrc "blog"; + }; + docs = self.callPackage ./nix/pkgs/docs.nix { + src = self.mkSiteSrc "docs"; + }; + hydrant-ffi = self.callPackage ./nix/pkgs/hydrant-ffi.nix { + inherit hydrant; + # spindle's static builds link this too, and fenix's default toolchain has no musl std + rustPlatform = + if pkgs.stdenv.hostPlatform.isStatic + then self.rustPlatformStatic + else self.rustPlatform; + }; + hydrant-bin = self.callPackage ./nix/pkgs/hydrant-bin.nix { + hydrant = hydrant-dev; + inherit (self) rustPlatform; + }; + spindle = self.callPackage ./nix/pkgs/spindle.nix { + src = self.mkGoSrc "cmd/spindle"; + }; + shuttle = self.crateBin self.cargoNix "shuttle"; + shuttle-static = self.crateBin self.cargoNixStatic "shuttle"; + knot-rs = self.crateBin self.cargoNix "knot-server"; + knot-migrate = self.crateBin self.cargoNix "knot-migrate"; + dolly = self.callPackage ./nix/pkgs/dolly.nix {}; + deliberi = self.callPackage ./nix/pkgs/deliberi.nix { + src = self.mkGoSrc "cmd/deliberi"; + }; + migrator = self.callPackage ./nix/pkgs/migrator.nix { + src = self.mkGoSrc "cmd/migrator"; + }; + sitesd = self.callPackage ./nix/pkgs/sitesd.nix { + src = self.mkGoSrc "cmd/sitesd"; + }; + cilogs = self.callPackage ./nix/pkgs/cilogs.nix { + src = self.mkGoSrc "cmd/cilogs"; + }; + knotmirror = self.cargoNix.workspaceMembers.knotmirror.build; + bobbin = self.crateBin self.cargoNix "bobbin"; + # no ci workflow runs these, so the package build does + tranquil-gate = (self.crateBin self.cargoNix "tranquil-gate").override { + runTests = true; + testPreRun = "export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + # these clear the env, so they need CA roots in /etc, which the sandbox lacks + testCrateFlags = ["--skip" "toml_and_environment_overrides" "--skip" "existing_environment_only_configuration_works"]; + }; + zoekt-webserver = self.callPackage ./nix/pkgs/zoekt-webserver.nix {}; + attic-server = pkgs.callPackage ./nix/pkgs/attic-server.nix {}; + zoekt-tngl-indexserver = self.callPackage ./nix/pkgs/zoekt-tngl-indexserver.nix { + src = self.mkGoSrc "cmd/zoekt-tngl-indexserver"; + }; + }); + + spindleNixosFor = system: + nixpkgs.lib.nixosSystem { + inherit system; + modules = [self.nixosModules.spindle-nixos]; + }; + + devTools = system: let + pkgs = nixpkgsFor.${system}; + packages' = self.packages.${system}; + in + [ + pkgs.go + pkgs.air + pkgs.gopls + pkgs.httpie + pkgs.litecli + pkgs.websocat + pkgs.tailwindcss + pkgs.nixos-shell + pkgs.redis + pkgs.worker-build + pkgs.cargo-generate + pkgs.qemu + pkgs.cdrkit + pkgs.buf + pkgs.protobuf + pkgs.protoc-gen-prost + pkgs.protoc-gen-prost-crate + pkgs.protoc-gen-prost-serde + pkgs.protoc-gen-go + pkgs.protoc-gen-go-grpc + (fenix.packages.${system}.combine [ + fenix.packages.${system}.stable.cargo + fenix.packages.${system}.stable.rustc + fenix.packages.${system}.stable.rust-src + fenix.packages.${system}.stable.clippy + fenix.packages.${system}.stable.rustfmt + fenix.packages.${system}.stable.rust-analyzer + fenix.packages.${system}.targets.wasm32-unknown-unknown.stable.rust-std + ]) + pkgs.coreutils # for those of us who are on systems that use busybox (alpine) + packages'.lexgen + packages'.goat + packages'.treefmt-wrapper + pkgs.e2fsprogs + pkgs.util-linux + pkgs.pnpm + # node dropped corepack in 25, and the web setup needs it + pkgs.corepack + # e2e needs node 26, whose fetch accepts undici's dispatcher + pkgs.nodejs_26 + # the blog's link check (pnpm check:links) + pkgs.lychee + ] + ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ + pkgs.parted + pkgs.slirp4netns + pkgs.iproute2 + # cargo wants a plain cc and the static shell's compiler is prefixed + pkgs.stdenv.cc + # jemalloc builds with make, which the static shell lacks + pkgs.gnumake + ]; + in { + overlays.default = final: prev: { + inherit + (mkPackageSet final) + lexgen + goat + sqlite-lib + spindle + shuttle + knot-rs + knot-migrate + docs + dolly + deliberi + migrator + sitesd + cilogs + knotmirror + bobbin + tranquil-gate + zoekt-webserver + zoekt-tngl-indexserver + ; + }; + + packages = forAllSystems (system: let + pkgs = nixpkgsFor.${system}; + linuxPkgs = nixpkgsFor."x86_64-linux"; + imageSystem = "${nixpkgs.lib.head (nixpkgs.lib.splitString "-" system)}-linux"; + imagePkgs = nixpkgsFor.${imageSystem}; + packages = mkPackageSet pkgs; + staticPackages = mkPackageSet pkgs.pkgsStatic; + crossPackages = mkPackageSet pkgs.pkgsCross.gnu64.pkgsStatic; + + spindle-image-helpers = let + shuttle = (mkPackageSet linuxPkgs).shuttle-static; + in + linuxPkgs.callPackage ./nix/pkgs/spindle-image-helpers.nix { + inherit shuttle spindleRegistry; + }; + in { + inherit + (packages) + web-static-files + ogre-fonts + blog + lexgen + goat + spindle + hydrant-ffi + hydrant-bin + knot-rs + knot-migrate + sqlite-lib + docs + shuttle + shuttle-static + dolly + deliberi + migrator + sitesd + cilogs + knotmirror + bobbin + tranquil-gate + zoekt-webserver + attic-server + zoekt-tngl-indexserver + ; + + tangled-pds = tangledPdsFor system; + + pkgsStatic-spindle = staticPackages.spindle; + pkgsStatic-sqlite-lib = staticPackages.sqlite-lib; + pkgsStatic-dolly = staticPackages.dolly; + + pkgsCross-gnu64-pkgsStatic-spindle = crossPackages.spindle; + pkgsCross-gnu64-pkgsStatic-dolly = crossPackages.dolly; + + treefmt-wrapper = pkgs.treefmt.withConfig { + settings.formatter = { + alejandra = { + command = pkgs.lib.getExe pkgs.alejandra; + includes = ["*.nix"]; + excludes = ["nix/Cargo.nix"]; + }; + + gofmt = { + command = pkgs.lib.getExe' pkgs.go "gofmt"; + options = ["-w"]; + includes = ["*.go"]; + }; + + # web formats via pnpm lint (pinned prettier, svelte-aware); treefmt + # prettier would fight it on union types + rustfmt = { + command = pkgs.lib.getExe' fenix.packages.${system}.stable.rustfmt "rustfmt"; + options = ["--edition" "2024"]; + includes = ["*.rs"]; + excludes = ["**/src/_lex/**" "**/src/gen/**" "knot2/third_party/gix-imara-diff/**"]; + }; + }; + }; + + spindle-nixos-image = imagePkgs.callPackage ./nix/pkgs/spindle-nixos-image.nix { + nixosSystem = spindleNixosFor imageSystem; + }; + spindle-nixos-large-image = self.packages.${system}.spindle-nixos-image.override { + vcpus = 6; + memoryMiB = 8192; + }; + spindle-nixos-image-tarball = imagePkgs.runCommand "spindle-nixos-image-tarball.tar.gz" {} '' + tar -S -C ${self.packages.${system}.spindle-nixos-image} -h -czf $out . + ''; + + spindle-alpine-image = let + branch = "3.24"; + version = "${branch}.0"; + arch = "x86_64"; + cdn = "https://dl-cdn.alpinelinux.org/alpine/v${branch}/releases/${arch}"; + in + linuxPkgs.callPackage ./nix/pkgs/spindle-alpine-image.nix { + inherit arch spindle-image-helpers; + repositories = [ + "https://dl-cdn.alpinelinux.org/alpine/v${branch}/main" + "https://dl-cdn.alpinelinux.org/alpine/v${branch}/community" + ]; + rootfs = linuxPkgs.fetchurl { + url = "${cdn}/alpine-minirootfs-${version}-${arch}.tar.gz"; + hash = "sha256-3poRwODn6clNs+2K97RQ6vwLE2h71+kZnVUFDyCqCok="; + }; + kernel = linuxPkgs.fetchurl { + url = "${cdn}/netboot-${version}/vmlinuz-virt"; + hash = "sha256-Hmv5Ancgx1w+0NeRcfIbV5HuQMqXldB8fG4E3F6irpA="; + }; + initramfs = linuxPkgs.fetchurl { + url = "${cdn}/netboot-${version}/initramfs-virt"; + hash = "sha256-ZCWGSaVMOYOmLz1Gwsf2RhYarMqk+tFVA6MMDWiHVJQ="; + }; + modloop = linuxPkgs.fetchurl { + url = "${cdn}/netboot-${version}/modloop-virt"; + hash = "sha256-p3yO7yU28k04iT01sOzhDmEYi+Yl7VZs5r3RYsWCBX0="; + }; + }; + spindle-alpine-image-tarball = linuxPkgs.runCommand "spindle-alpine-image-tarball.tar.gz" {} '' + tar -S -C ${self.packages.${system}.spindle-alpine-image} -h -czf $out . + ''; + + spindle-almalinux10-image = let + branch = "10"; + version = "${branch}.2"; + arch = "x86_64"; + goarch = "amd64"; + kver = "6.12.0-211.34.1.el10_2.${arch}"; + + # Every time a new point release comes out, and the previous one is deprecated, + # those repos get moved to the vault URL. However, for CI purposes, it + # would be annoying if the image completely stopped working from a new + # release being available, so we just fall back to the vault URL until + # the RPMs can be updated. + makeRpmUrls = rpm: [ + "https://repo.almalinux.org/almalinux/${version}/BaseOS/${arch}/os/Packages/${rpm}" + "https://vault.almalinux.org/almalinux/${version}/BaseOS/${arch}/os/Packages/${rpm}" + ]; + in + linuxPkgs.callPackage ./nix/pkgs/spindle-almalinux-image.nix { + inherit arch kver spindle-image-helpers; + rootfs = linuxPkgs.fetchurl { + url = "https://github.com/AlmaLinux/container-images/raw/cd5582c41aaf7f695b2473264b322f4080066796/default/${goarch}/almalinux-${branch}-default-${goarch}.tar.xz"; + hash = "sha256-60TIMFmaCoPv//XLAm6Cm/wkpmtbwH996zgM7CB/ypQ="; + }; + kernel-modules-core = linuxPkgs.fetchurl { + urls = makeRpmUrls "kernel-modules-core-${kver}.rpm"; + hash = "sha256-KYR/dnm/oCglQ4uvFTZlW6WBR1HBUNh/yS64PxyIQvk="; + }; + kernel-modules = linuxPkgs.fetchurl { + urls = makeRpmUrls "kernel-modules-${kver}.rpm"; + hash = "sha256-n2SPQuM+dmTQn4Ustkl9GyF8xH3godU0fjJO8oz4JDY="; + }; + uki = linuxPkgs.fetchurl { + urls = makeRpmUrls "kernel-uki-virt-${kver}.rpm"; + hash = "sha256-ds2+uSPpuHHoVyYaUrh8DSwWgVWYMijgGoneAE+JP9I="; + }; + # doas-sudo-shim 0.1.x in the repos requires util-linux, which isn't in the + # container and is annoying to get inside, so just get it ourselves. + doas-sudo-shim-src = linuxPkgs.fetchFromGitHub { + owner = "jirutka"; + repo = "doas-sudo-shim"; + rev = "v0.2.0"; + sha256 = "sha256-USSakVUzCbUY1DJLmDCiwdq/xjOwwnm3VtXBBeXeV1A="; + }; + }; + spindle-almalinux10-image-tarball = linuxPkgs.runCommand "spindle-almalinux10-image-tarball.tar.gz" {} '' + tar -S -C ${self.packages.${system}.spindle-almalinux10-image} -h -czf $out . + ''; + }); + checks = forAllSystems (system: { + tangled-pds = self.packages.${system}.tangled-pds; + }); + + devShells = forAllSystems (system: let + pkgs = nixpkgsFor.${system}; + packages' = self.packages.${system}; + staticShell = args: + (pkgs.mkShell.override { + stdenv = pkgs.pkgsStatic.stdenv; + }) (args + // { + nativeBuildInputs = + args.nativeBuildInputs + ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ + pkgs.darwin.cctools + ]; + }); + in { + default = staticShell { + nativeBuildInputs = devTools system; + shellHook = '' + export CC=${pkgs.stdenv.cc}/bin/cc + unset NIX_CFLAGS_LINK + export SQLX_OFFLINE=true + ( + cd "$(git rev-parse --show-toplevel 2>/dev/null || pwd)" + mkdir -p web/static + rm -rf web/static/fonts web/static/logos + cp -fr --no-preserve=ownership,mode ${packages'.web-static-files}/* web/static + # the docs and blog sites serve the same fonts and logos from their own public dirs + rm -rf docs/public/fonts blog/public/fonts blog/public/logos + mkdir -p docs/public blog/public + cp -fr --no-preserve=ownership,mode ${packages'.web-static-files}/fonts docs/public/fonts + cp -fr --no-preserve=ownership,mode ${packages'.web-static-files}/{fonts,logos} blog/public/ + rm -rf ogre/fonts + cp -fr --no-preserve=ownership,mode ${packages'.ogre-fonts} ogre/fonts + ) + export TANGLED_OAUTH_CLIENT_KID="$(date +%s)" + export TANGLED_OAUTH_CLIENT_SECRET="$(${packages'.goat}/bin/goat key generate -t P-256 | grep -A1 "Secret Key" | tail -n1 | awk '{print $1}')" + # Make xcrun (Nix stub) able to find ld from the system Command Line Tools. + # Without this, worker-build/cargo fails with "error: tool 'ld' not found". + # it goes last because its make is a shim that looks for gnumake inside + # DEVELOPER_DIR, which is a nix sdk without one, so it broke every C build. + if [ -d /Library/Developer/CommandLineTools/usr/bin ]; then + export PATH=$PATH:/Library/Developer/CommandLineTools/usr/bin + fi + ''; + env.CGO_ENABLED = 1; + # spindle embeds hydrant, so anything that links it needs the staticlib + env.CGO_LDFLAGS = "-L${packages'.hydrant-ffi}/lib"; + }; + }); + apps = forAllSystems (system: let + pkgs = nixpkgsFor."${system}"; + packages' = self.packages.${system}; + air-watcher = name: arg: + pkgs.writeShellScriptBin "run" + '' + export PATH=${pkgs.go}/bin:$PATH + ${pkgs.air}/bin/air -c ./.air/${name}.toml \ + -build.args_bin "${arg}" + ''; + in { + watch-spindle = { + type = "app"; + program = ''${air-watcher "spindle" ""}/bin/run''; + }; + watch-blog = { + type = "app"; + program = toString (pkgs.writeShellScript "watch-blog" '' + # astro reloads on post and component changes; drafts show up here only + cd blog + [ -d node_modules ] || ${pkgs.nodejs}/bin/npm install + exec ${pkgs.nodejs}/bin/npm run dev -- --host 0.0.0.0 --port 3001 + ''); + }; + serve-docs = { + type = "app"; + program = toString (pkgs.writeShellScript "serve-docs" '' + echo "building docs..." + docsOut=$(nix build --no-link --print-out-paths .#docs) + echo "serving docs at http://localhost:1414" + cd "$docsOut" + exec ${pkgs.python3}/bin/python3 -m http.server 1414 + ''); + }; + watch-docs = { + type = "app"; + program = toString (pkgs.writeShellScript "watch-docs" '' + export PATH=${pkgs.nodejs}/bin:$PATH + cd docs + [ -d node_modules ] || npm install + exec npm run dev + ''); + }; + regenerate-proto = { + type = "app"; + program = + (pkgs.writeShellApplication { + name = "regenerate-proto"; + runtimeInputs = with pkgs; [git buf coreutils]; + text = '' + rootDir=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + cd "$rootDir" + echo ">>> regenerating protobuf files.." + buf generate + echo ">>> generating file descriptor set for shuttle..." + buf build -o shuttle/src/gen/file_descriptor_set.bin + echo ">>> done" + ''; + }) + + "/bin/regenerate-proto"; + }; + cargo-nix = { + type = "app"; + program = + (pkgs.writeShellApplication { + name = "cargo-nix"; + runtimeInputs = [pkgs.crate2nix pkgs.git fenix.packages.${system}.stable.cargo]; + text = '' + cd "$(git rev-parse --show-toplevel)" + crate2nix generate -f ./Cargo.toml -o ./nix/Cargo.nix -h ./nix/crate-hashes.json + ''; + }) + + "/bin/cargo-nix"; + }; + gomod2nix = { + type = "app"; + program = toString (pkgs.writeShellScript "gomod2nix" '' + ${gomod2nix.legacyPackages.${system}.gomod2nix}/bin/gomod2nix generate --outdir ./nix + ''); + }; + localinfra = { + type = "app"; + program = + (pkgs.writeShellApplication { + name = "localinfra"; + runtimeInputs = with pkgs; + [process-compose postgresql_14 caddy dnsmasq dnsutils curl jq git nodejs_22 yarn openssl packages'.hydrant-bin (pdsFor system)] + ++ lib.optionals stdenv.hostPlatform.isLinux [socat slirp4netns util-linux iproute2 iptables procps gawk qemu_kvm e2fsprogs patchelf] + ++ lib.optionals stdenv.hostPlatform.isDarwin [stdenv.cc] + ++ lib.concatMap (name: let + file = ./localinfra/pc/services + "/${name}/packages"; + in + lib.optionals (builtins.pathExists file) + (map (attr: pkgs.${attr}) (lib.filter (attr: attr != "") (lib.splitString "\n" (builtins.readFile file))))) + (lib.attrNames (builtins.readDir ./localinfra/pc/services)) + ++ devTools system; + text = '' + rootDir=$(jj --ignore-working-copy root 2>/dev/null || git rev-parse --show-toplevel) || (echo "error: can't find repo root?"; exit 1) + # only macos ships the sqlite the libsqlite3 tag needs + export CGO_CFLAGS="-I${packages'.sqlite-lib}/include''${CGO_CFLAGS:+ $CGO_CFLAGS}" + export CGO_LDFLAGS="-L${packages'.sqlite-lib}/lib''${CGO_LDFLAGS:+ $CGO_LDFLAGS}" + ${pkgs.lib.optionalString pkgs.stdenv.hostPlatform.isDarwin '' + # .cargo/config.toml names the static shell's prefixed cc, which this app doesn't have + export CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=cc + # rusqlite links the system sqlite, which the nix toolchain can't see + export LIBRARY_PATH="${packages'.sqlite-lib}/lib''${LIBRARY_PATH:+:$LIBRARY_PATH}" + ''} + ${pkgs.lib.optionalString pkgs.stdenv.hostPlatform.isLinux '' + # the static shell leaves -static in NIX_CFLAGS_LINK, which breaks glibc links + unset NIX_CFLAGS_LINK + export OPENSSL_INCLUDE_DIR=${pkgs.openssl.dev}/include + export OPENSSL_LIB_DIR=${pkgs.openssl.out}/lib + # rusqlite would link the system sqlite + export SQLITE3_LIB_DIR=${packages'.sqlite-lib}/lib + export SQLITE3_STATIC=1 + # npm's workerd needs a nix loader, bin/patch-workerd reads these + export TNGL_DYNAMIC_LINKER=${pkgs.stdenv.cc.bintools.dynamicLinker} + export TNGL_NIX_LIBS=${pkgs.lib.makeLibraryPath [pkgs.stdenv.cc.cc.lib pkgs.stdenv.cc.libc]} + # e2e/run uses this chromium because playwright's own won't run on nixos + export E2E_CHROMIUM=${pkgs.chromium}/bin/chromium + ''} + exec "$rootDir/localinfra/pc/bin/up" "$@" + ''; + }) + + /bin/localinfra; + }; + lexgen = { + type = "app"; + program = + (pkgs.writeShellApplication { + name = "lexgen"; + runtimeInputs = [pkgs.stdenv.cc pkgs.gnused]; + text = '' + if ! command -v lexgen > /dev/null; then + echo "error: must be executed from devshell" + exit 1 + fi + + # pin CC to a glibc gcc so a stray musl cc cant mess up CGO + export CC=${pkgs.stdenv.cc}/bin/cc + + rootDir=$(jj --ignore-working-copy root || git rev-parse --show-toplevel) || (echo "error: can't find repo root?"; exit 1) + cd "$rootDir" + + # preserve hand-written extensions and tests + find api/tangled api/org_tangled -maxdepth 1 -type f -not -name '*_ext.go' -not -name '*_test.go' -delete + lexgen --build-file lexicon-build-config.json lexicons + + # disable type registration temporarily while running cborgen + find api/tangled api/org_tangled -maxdepth 1 -name '*.go' -not -name '*_ext.go' -exec \ + sed -i.bak 's/\tutil/\/\/\tutil/' {} + + # lexgen generates incomplete Marshaler/Unmarshaler for union types + find api/tangled api/org_tangled -maxdepth 1 -name '*.go' -not -name '*_ext.go' -not -name "cbor_gen.go" -exec \ + sed -i '/^func.*\(MarshalCBOR\|UnmarshalCBOR\)/,/^}/ s/^/\/\/ /' {} + + for f in api/tangled/*_ext.go api/org_tangled/*_ext.go; do [ -e "''$f" ] || continue; mv "''$f" "''$f.bak"; done + ${pkgs.gotools}/bin/goimports -w api/tangled/* api/org_tangled/* + CGO_ENABLED=1 go run ./cmd/cborgen/ + for f in api/tangled/*_ext.go.bak api/org_tangled/*_ext.go.bak; do [ -e "''$f" ] || continue; mv "''$f" "''${f%.bak}"; done + + lexgen --build-file lexicon-build-config.json lexicons + go run ./cmd/lexpostgen/ + rm -f api/tangled/*.bak api/org_tangled/*.bak + ''; + }) + + /bin/lexgen; + }; + }); + + nixosModules.deliberi = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/deliberi.nix]; + + services.tangled.deliberi.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.deliberi; + }; + nixosModules.tranquil-gate = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/tranquil-gate.nix]; + + services.tangled.tranquil-gate.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.tranquil-gate; + }; + nixosModules.migrator = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/migrator.nix]; + + services.tangled.migrator.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.migrator; + }; + nixosModules.sitesd = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/sitesd.nix]; + + services.tangled.sitesd.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.sitesd; + }; + nixosModules.cilogs = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/cilogs.nix]; + + services.tangled.cilogs.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.cilogs; + }; + nixosModules.knotmirror = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/knotmirror.nix]; + + services.tangled.knotmirror.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.knotmirror; + }; + nixosModules.zoekt-tnglserver = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/zoekt-tnglserver.nix]; + + services.tangled.zoekt.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.zoekt-tngl-indexserver; + services.tangled.zoekt.zoekt-webserver-package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.zoekt-webserver; + }; + nixosModules.knot-rs = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/knot-rs.nix]; + + services.tangled.knot-rs.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.knot-rs; + }; + nixosModules.spindle = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/spindle.nix]; + + services.tangled.spindle.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.spindle; + }; + nixosModules.shuttle = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/shuttle.nix]; + + services.tangled.shuttle.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.shuttle; + }; + + nixosModules.spindle-nixos = import ./nix/microvm/spindle-vm.nix {inherit self microvm spindleRegistry;} ./nix/microvm/qemu.nix; + + formatter = forAllSystems (system: self.packages.${system}.treefmt-wrapper); + + nixosConfigurations = { + spindle-nixos = spindleNixosFor "x86_64-linux"; + spindle-nixos-aarch64 = spindleNixosFor "aarch64-linux"; + }; + }; +} diff --git a/gitclient/gitclient.go b/gitclient/gitclient.go new file mode 100644 index 0000000000000000000000000000000000000000..d3577f8fff241f8eeed86681415ea822cac88bd9 --- /dev/null +++ b/gitclient/gitclient.go @@ -0,0 +1,444 @@ +// Package gitclient is a small git protocol v2 client over smart http, just +// enough to read refs, trees and blobs from a knot without cloning. +package gitclient + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "maps" + "net/http" + "path" + "slices" + "strings" + "sync" +) + +var ( + ErrRefNotFound = errors.New("branch or tag not found") + ErrPathNotFound = errors.New("path not found at the commit") + ErrNotDir = fmt.Errorf("%w: not a directory", ErrPathNotFound) + ErrTooLarge = errors.New("pack exceeds the read limit") + ErrUnsupported = errors.New("server lacks a required v2 capability") +) + +// hex object id, sha1 or sha256 depending on the repo's object-format +type Oid string + +type Client struct { + http *http.Client + url string // repo base, e.g. https://knot.example/did:plc:abc + maxBytes int64 // cap on one fetch's pack + + mu sync.Mutex + caps *Caps +} + +func New(httpClient *http.Client, repoURL string, maxBytes int64) *Client { + if httpClient == nil { + httpClient = http.DefaultClient + } + return &Client{http: httpClient, url: strings.TrimSuffix(repoURL, "/"), maxBytes: maxBytes} +} + +type Caps struct { + ObjectFormat string // "sha1" or "sha256" + LsRefs bool + Fetch []string // fetch= features: shallow, filter, ... +} + +func (c *Client) Caps(ctx context.Context) (Caps, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.caps != nil { + return *c.caps, nil + } + u := c.url + "/info/refs?service=git-upload-pack" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return Caps{}, err + } + req.Header.Set("Git-Protocol", "version=2") + resp, err := c.http.Do(req) + if err != nil { + return Caps{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return Caps{}, httpError(resp) + } + caps := Caps{ObjectFormat: "sha1"} + p := newPktReader(io.LimitReader(resp.Body, 1<<20)) + v2 := false + for { + kind, line, err := p.next() + if err != nil && !v2 { + return Caps{}, fmt.Errorf("%w: %s doesn't speak protocol v2", ErrUnsupported, u) + } + if err != nil { + return Caps{}, fmt.Errorf("reading %s: %w", u, err) + } + if kind != pktData { + // a v0-style "# service=" banner ends in a flush before the v2 advert + if v2 { + break + } + continue + } + key, val, _ := strings.Cut(strings.TrimSuffix(string(line), "\n"), "=") + switch key { + case "version 2": + v2 = true + case "ls-refs": + caps.LsRefs = true + case "fetch": + caps.Fetch = strings.Fields(val) + case "object-format": + caps.ObjectFormat = val + } + } + if caps.ObjectFormat != "sha1" && caps.ObjectFormat != "sha256" { + return Caps{}, fmt.Errorf("%w: object-format=%s", ErrUnsupported, caps.ObjectFormat) + } + c.caps = &caps + return caps, nil +} + +type Ref struct { + Name string + Oid Oid + Peeled Oid // set for annotated tags when Peel was asked + Symref string // target of HEAD when Symrefs was asked +} + +type LsRefsOpts struct { + Prefixes []string // ref-prefix lines + Peel bool + Symrefs bool +} + +// command=ls-refs +func (c *Client) LsRefs(ctx context.Context, opts LsRefsOpts) ([]Ref, error) { + caps, err := c.Caps(ctx) + if err != nil { + return nil, err + } + if !caps.LsRefs { + return nil, fmt.Errorf("%w: ls-refs", ErrUnsupported) + } + var args []string + if opts.Peel { + args = append(args, "peel") + } + if opts.Symrefs { + args = append(args, "symrefs") + } + for _, prefix := range opts.Prefixes { + args = append(args, "ref-prefix "+prefix) + } + resp, err := c.command(ctx, caps, "ls-refs", args) + if err != nil { + return nil, err + } + defer resp.Body.Close() + p := newPktReader(&capReader{r: resp.Body, n: c.maxBytes}) + var refs []Ref + for { + kind, line, err := p.next() + if err != nil { + return nil, fmt.Errorf("ls-refs: %w", err) + } + if kind != pktData { + return refs, nil + } + text := strings.TrimSuffix(string(line), "\n") + if msg, ok := strings.CutPrefix(text, "ERR "); ok { + return nil, fmt.Errorf("remote: %s", msg) + } + fields := strings.Split(text, " ") + if len(fields) < 2 || !isOid(fields[0]) { + return nil, fmt.Errorf("ls-refs: bad line %q", text) + } + ref := Ref{Oid: Oid(fields[0]), Name: fields[1]} + for _, attr := range fields[2:] { + if v, ok := strings.CutPrefix(attr, "peeled:"); ok { + ref.Peeled = Oid(v) + } else if v, ok := strings.CutPrefix(attr, "symref-target:"); ok { + ref.Symref = v + } + } + refs = append(refs, ref) + } +} + +type FetchOpts struct { + Wants []Oid + Deepen int // 0 means no deepen line + Filter string // e.g. "tree:0", "blob:none"; empty means none +} + +// command=fetch with done and no haves, since there's no local object store +func (c *Client) Fetch(ctx context.Context, opts FetchOpts) (Objects, error) { + caps, err := c.Caps(ctx) + if err != nil { + return nil, err + } + if opts.Filter != "" && !slices.Contains(caps.Fetch, "filter") { + return nil, fmt.Errorf("%w: fetch=filter", ErrUnsupported) + } + if opts.Deepen > 0 && !slices.Contains(caps.Fetch, "shallow") { + return nil, fmt.Errorf("%w: fetch=shallow", ErrUnsupported) + } + if len(opts.Wants) == 0 { + return Objects{}, nil + } + args := []string{"no-progress", "ofs-delta"} + for _, want := range opts.Wants { + if len(want) != oidLen(caps.ObjectFormat) || !isOid(string(want)) { + return nil, fmt.Errorf("bad object id %q", want) + } + args = append(args, "want "+string(want)) + } + if opts.Deepen > 0 { + args = append(args, fmt.Sprintf("deepen %d", opts.Deepen)) + } + if opts.Filter != "" { + args = append(args, "filter "+opts.Filter) + } + args = append(args, "done") + resp, err := c.command(ctx, caps, "fetch", args) + if err != nil { + return nil, err + } + defer resp.Body.Close() + pack, err := packfileSection(newPktReader(resp.Body)) + if err != nil { + return nil, fmt.Errorf("fetch: %w", err) + } + objs, err := readPack(&capReader{r: pack, n: c.maxBytes}, caps.ObjectFormat) + if err != nil { + return nil, fmt.Errorf("fetch: %w", err) + } + return objs, nil +} + +// branches before tags, like git; tags peeled +func (c *Client) Resolve(ctx context.Context, ref string) (Oid, error) { + if ref == "" || strings.ContainsAny(ref, "\x00\n ") { + return "", ErrRefNotFound + } + candidates := []string{ref} + if !strings.HasPrefix(ref, "refs/") { + candidates = []string{"refs/heads/" + ref, "refs/tags/" + ref} + } + refs, err := c.LsRefs(ctx, LsRefsOpts{Prefixes: candidates, Peel: true}) + if err != nil { + return "", err + } + for _, name := range candidates { + for _, r := range refs { + if r.Name != name { + continue + } + if r.Peeled != "" { + return r.Peeled, nil + } + return r.Oid, nil + } + } + return "", ErrRefNotFound +} + +type File struct { + Name string + Mode string // octal, as in the tree + Oid Oid + Data []byte // nil for subtrees and submodules +} + +// one level of dir, blobs filled in +func (c *Client) ReadDir(ctx context.Context, commit Oid, dir string) ([]File, error) { + caps, objs, oid, err := c.walk(ctx, commit, dir) + if err != nil { + return nil, err + } + entries, err := c.tree(ctx, caps, objs, oid, "tree:1") + if err != nil { + return nil, err + } + files := make([]File, 0, len(entries)) + for _, e := range entries { + files = append(files, File{Name: e.Name, Mode: e.Mode, Oid: e.Oid}) + } + if err := c.fillBlobs(ctx, objs, files); err != nil { + return nil, err + } + return files, nil +} + +// everything under dir, named relative to it; the dir tree is wanted unfiltered +// so one pack brings it all +func (c *Client) ReadTree(ctx context.Context, commit Oid, dir string) ([]File, error) { + caps, objs, oid, err := c.walk(ctx, commit, dir) + if err != nil { + return nil, err + } + var files []File + queue := []File{{Oid: oid}} + for len(queue) > 0 { + t := queue[0] + queue = queue[1:] + entries, err := c.tree(ctx, caps, objs, t.Oid, "") + if err != nil { + return nil, err + } + for _, e := range entries { + f := File{Name: path.Join(t.Name, e.Name), Mode: e.Mode, Oid: e.Oid} + if isTreeMode(e.Mode) { + queue = append(queue, f) + } + files = append(files, f) + } + } + if err := c.fillBlobs(ctx, objs, files); err != nil { + return nil, err + } + return files, nil +} + +// middle trees come alone (tree:0) so sibling blobs stay on the server; servers +// differ on filter depth for a wanted tree, hence fillBlobs +func (c *Client) walk(ctx context.Context, commit Oid, dir string) (Caps, Objects, Oid, error) { + caps, err := c.Caps(ctx) + if err != nil { + return Caps{}, nil, "", err + } + objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{commit}, Deepen: 1, Filter: "tree:1"}) + if err != nil { + return Caps{}, nil, "", err + } + obj, ok := objs[commit] + if !ok || obj.Kind != KindCommit { + return Caps{}, nil, "", fmt.Errorf("fetch of commit %s didn't return it", commit) + } + tree, err := commitTree(obj.Data) + if err != nil { + return Caps{}, nil, "", err + } + for _, name := range strings.Split(dir, "/") { + if name == "" { + continue + } + entries, err := c.tree(ctx, caps, objs, tree, "tree:0") + if err != nil { + return Caps{}, nil, "", err + } + i := slices.IndexFunc(entries, func(e treeEntry) bool { return e.Name == name }) + if i < 0 { + return Caps{}, nil, "", fmt.Errorf("%s: %w", dir, ErrPathNotFound) + } + if !isTreeMode(entries[i].Mode) { + return Caps{}, nil, "", fmt.Errorf("%s: %w", dir, ErrNotDir) + } + tree = entries[i].Oid + } + return caps, objs, tree, nil +} + +func (c *Client) fillBlobs(ctx context.Context, objs Objects, files []File) error { + var missing []Oid + for _, f := range files { + if _, ok := objs[f.Oid]; isBlobMode(f.Mode) && !ok && !slices.Contains(missing, f.Oid) { + missing = append(missing, f.Oid) + } + } + if len(missing) > 0 { + // wanted objects pass any filter; it keeps the server on its selective path + more, err := c.Fetch(ctx, FetchOpts{Wants: missing, Filter: "tree:0"}) + if err != nil { + return err + } + maps.Copy(objs, more) + } + for i, f := range files { + if !isBlobMode(f.Mode) { + continue + } + obj, ok := objs[f.Oid] + if !ok || obj.Kind != KindBlob { + return fmt.Errorf("fetch of blob %s didn't return it", f.Oid) + } + files[i].Data = obj.Data + } + return nil +} + +func (c *Client) tree(ctx context.Context, caps Caps, have Objects, oid Oid, filter string) ([]treeEntry, error) { + if _, ok := have[oid]; !ok { + objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{oid}, Filter: filter}) + if err != nil { + return nil, err + } + maps.Copy(have, objs) + } + obj, ok := have[oid] + if !ok || obj.Kind != KindTree { + return nil, fmt.Errorf("fetch of tree %s didn't return it", oid) + } + return parseTree(obj.Data, caps.ObjectFormat) +} + +func isBlobMode(mode string) bool { + mode = strings.TrimLeft(mode, "0") + return mode != "40000" && mode != "160000" +} + +func isTreeMode(mode string) bool { + return strings.TrimLeft(mode, "0") == "40000" +} + +func (c *Client) command(ctx context.Context, caps Caps, command string, args []string) (*http.Response, error) { + var body bytes.Buffer + writePkt(&body, "command="+command+"\n") + // git refuses non-sha1 unless the format is echoed back + if caps.ObjectFormat != "sha1" { + writePkt(&body, "object-format="+caps.ObjectFormat+"\n") + } + writeDelim(&body) + for _, arg := range args { + if err := writePkt(&body, arg+"\n"); err != nil { + return nil, err + } + } + writeFlush(&body) + + u := c.url + "/git-upload-pack" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, &body) + if err != nil { + return nil, err + } + req.Header.Set("Git-Protocol", "version=2") + req.Header.Set("Content-Type", "application/x-git-upload-pack-request") + req.Header.Set("Accept", "application/x-git-upload-pack-result") + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() + return nil, httpError(resp) + } + return resp, nil +} + +// knot2 says why in a plain text body, like a want it won't serve +func httpError(resp *http.Response) error { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + err := fmt.Errorf("%s %s: %s", resp.Request.Method, resp.Request.URL, resp.Status) + if msg := strings.TrimSpace(string(body)); msg != "" && !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") { + err = fmt.Errorf("%w: %s", err, strings.ToValidUTF8(msg, "?")) + } + return err +} diff --git a/gitclient/gitclient_test.go b/gitclient/gitclient_test.go new file mode 100644 index 0000000000000000000000000000000000000000..da36f909c6478da8a28c765e421c5b4226c0918d --- /dev/null +++ b/gitclient/gitclient_test.go @@ -0,0 +1,198 @@ +package gitclient + +import ( + "bytes" + "context" + "errors" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func gitIn(t *testing.T, dir string, stdin []byte, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t") + cmd.Stdin = bytes.NewReader(stdin) + out, err := cmd.Output() + if err != nil { + t.Fatalf("git %v: %v", args, err) + } + return strings.TrimSpace(string(out)) +} + +func TestStockGit(t *testing.T) { + gitPath, err := exec.LookPath("git") + if err != nil { + t.Skip("no git binary") + } + root := t.TempDir() + repo := filepath.Join(root, "repo") + git := func(args ...string) string { return gitIn(t, repo, nil, args...) } + write := func(name, data string, mode os.FileMode) { + p := filepath.Join(repo, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(data), mode); err != nil { + t.Fatal(err) + } + } + gitIn(t, root, nil, "init", "-q", "-b", "main", "repo") + git("config", "uploadpack.allowFilter", "true") + git("config", "uploadpack.allowAnySHA1InWant", "true") + write("README", "hi\n", 0o644) + git("add", ".") + git("commit", "-qm", "first") + first := git("rev-parse", "HEAD") + git("tag", "dup") + + write(".tangled/workflows/a.yml", "when: push\n", 0o644) + write(".tangled/workflows/b.yml", "when: pr\n", 0o755) + write(".tangled/workflows/sub/c.yml", "nested\n", 0o644) + if err := os.Symlink("a.yml", filepath.Join(repo, ".tangled/workflows/link")); err != nil { + t.Fatal(err) + } + git("add", ".") + git("commit", "-qm", "second") + second := git("rev-parse", "HEAD") + git("branch", "dup") + git("tag", "-am", "annotated", "v1") + git("tag", "light") + + srv := httptest.NewServer(&cgi.Handler{ + Path: gitPath, + Args: []string{"http-backend"}, + Env: []string{"GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1", "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null"}, + }) + defer srv.Close() + ctx := context.Background() + c := New(srv.Client(), srv.URL+"/repo", 4<<20) + + for ref, want := range map[string]string{ + "main": second, + "refs/heads/main": second, + "dup": second, // branch before tag + "refs/tags/dup": first, + "v1": second, // peeled + "light": second, + } { + got, err := c.Resolve(ctx, ref) + if err != nil || string(got) != want { + t.Errorf("Resolve(%s) = %s, %v; want %s", ref, got, err, want) + } + } + if _, err := c.Resolve(ctx, "nope"); !errors.Is(err, ErrRefNotFound) { + t.Errorf("Resolve(nope) = %v", err) + } + + files, err := c.ReadDir(ctx, Oid(second), ".tangled/workflows") + if err != nil { + t.Fatal(err) + } + var got []string + for _, f := range files { + got = append(got, f.Mode+" "+f.Name+" "+string(f.Data)) + } + want := []string{"100644 a.yml when: push\n", "100755 b.yml when: pr\n", "120000 link a.yml", "40000 sub "} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("ReadDir = %q", got) + } + + if _, err := c.ReadDir(ctx, Oid(first), ".tangled/workflows"); !errors.Is(err, ErrPathNotFound) { + t.Errorf("ReadDir(first) = %v", err) + } + for _, dir := range []string{".tangled/nope", ".tangled/workflows/a.yml"} { + if _, err := c.ReadDir(ctx, Oid(second), dir); !errors.Is(err, ErrPathNotFound) { + t.Errorf("ReadDir(%s) = %v", dir, err) + } + } + + if _, err := c.ReadDir(ctx, Oid(second), ".tangled/workflows/a.yml"); !errors.Is(err, ErrNotDir) { + t.Errorf("ReadDir(a.yml) = %v", err) + } + + files, err = c.ReadTree(ctx, Oid(second), "/.tangled") + if err != nil { + t.Fatal(err) + } + got = nil + for _, f := range files { + got = append(got, f.Mode+" "+f.Name+" "+string(f.Data)) + } + want = []string{"40000 workflows ", "100644 workflows/a.yml when: push\n", "100755 workflows/b.yml when: pr\n", + "120000 workflows/link a.yml", "40000 workflows/sub ", "100644 workflows/sub/c.yml nested\n"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("ReadTree = %q", got) + } + if _, err := c.ReadTree(ctx, Oid(second), "nope"); !errors.Is(err, ErrPathNotFound) || errors.Is(err, ErrNotDir) { + t.Errorf("ReadTree(nope) = %v", err) + } + if _, err := c.ReadTree(ctx, Oid(second), "README/x"); !errors.Is(err, ErrNotDir) { + t.Errorf("ReadTree(README/x) = %v", err) + } + + small := New(srv.Client(), srv.URL+"/repo", 64) + if _, err := small.ReadDir(ctx, Oid(second), ".tangled/workflows"); !errors.Is(err, ErrTooLarge) { + t.Errorf("small ReadDir = %v", err) + } + gone := New(srv.Client(), srv.URL+"/missing", 4<<20) + if _, err := gone.Resolve(ctx, "main"); err == nil || errors.Is(err, ErrRefNotFound) { + t.Errorf("Resolve on missing repo = %v", err) + } + + // ofs and ref deltas straight from pack-objects + base := strings.Repeat("line of text\n", 400) + b1 := gitIn(t, repo, []byte(base), "hash-object", "-w", "--stdin") + b2 := gitIn(t, repo, []byte(base+"tail\n"), "hash-object", "-w", "--stdin") + for _, flag := range []string{"--delta-base-offset", "--window=10"} { + cmd := exec.Command("git", "pack-objects", "--stdout", flag) + cmd.Dir = repo + cmd.Stdin = strings.NewReader(b1 + "\n" + b2 + "\n") + pack, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + objs, err := readPack(bytes.NewReader(pack), "sha1") + if err != nil { + t.Fatalf("%s: %v", flag, err) + } + if string(objs[Oid(b2)].Data) != base+"tail\n" || string(objs[Oid(b1)].Data) != base { + t.Errorf("%s: bad objects", flag) + } + } +} + +func TestApplyDelta(t *testing.T) { + // src 5, dst 8: copy 5 bytes from 0, insert "xyz" + out, err := applyDelta([]byte("hello"), []byte{5, 8, 0x90, 5, 3, 'x', 'y', 'z'}) + if err != nil || string(out) != "helloxyz" { + t.Fatalf("applyDelta = %q, %v", out, err) + } + if _, err := applyDelta([]byte("hello"), []byte{5, 8, 0x90, 6}); err == nil { + t.Fatal("copy past base accepted") + } +} + +func TestPkt(t *testing.T) { + var b bytes.Buffer + writePkt(&b, "a\n") + writeDelim(&b) + writeFlush(&b) + if b.String() != "0006a\n00010000" { + t.Fatalf("framing %q", b.String()) + } + p := newPktReader(&b) + for _, want := range []pktKind{pktData, pktDelim, pktFlush} { + if kind, _, err := p.next(); err != nil || kind != want { + t.Fatalf("next = %v, %v", kind, err) + } + } +} diff --git a/gitclient/pack.go b/gitclient/pack.go new file mode 100644 index 0000000000000000000000000000000000000000..9b82d69bdaabe1e3314d4a62cf5aa55f9d4ffcc8 --- /dev/null +++ b/gitclient/pack.go @@ -0,0 +1,365 @@ +package gitclient + +import ( + "bufio" + "bytes" + "compress/zlib" + "crypto/sha1" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "strconv" +) + +type Kind int + +const ( + KindCommit Kind = iota + 1 + KindTree + KindBlob + KindTag +) + +var kindNames = [...]string{KindCommit: "commit", KindTree: "tree", KindBlob: "blob", KindTag: "tag"} + +type Object struct { + Kind Kind + Data []byte +} + +type Objects map[Oid]Object + +const ( + maxObjects = 1 << 14 + maxInflated = 64 << 20 // per object and for the whole pack +) + +const ( + packOfsDelta = 6 + packRefDelta = 7 +) + +type packEntry struct { + kind Kind // 0 until a delta is resolved + data []byte + baseOff int64 // ofs delta + baseOid Oid // ref delta +} + +// counts consumed bytes for ofs deltas; flate takes the ByteReader as is, so +// it never reads past an entry's zlib stream +type countingReader struct { + r *bufio.Reader + off int64 +} + +func (c *countingReader) Read(b []byte) (int, error) { + n, err := c.r.Read(b) + c.off += int64(n) + return n, err +} + +func (c *countingReader) ReadByte() (byte, error) { + b, err := c.r.ReadByte() + if err == nil { + c.off++ + } + return b, err +} + +// whole pack into memory with ofs/ref deltas resolved; fine for the handful of +// trees and blobs a ReadDir asks for +func readPack(r io.Reader, objectFormat string) (Objects, error) { + hashLen := oidLen(objectFormat) / 2 + cr := &countingReader{r: bufio.NewReader(r)} + var hdr [12]byte + if _, err := io.ReadFull(cr, hdr[:]); err != nil { + return nil, fmt.Errorf("pack header: %w", err) + } + if string(hdr[:4]) != "PACK" { + return nil, errors.New("not a pack") + } + if v := binary.BigEndian.Uint32(hdr[4:8]); v != 2 && v != 3 { + return nil, fmt.Errorf("pack version %d", v) + } + count := binary.BigEndian.Uint32(hdr[8:]) + if count > maxObjects { + return nil, fmt.Errorf("%w: %d objects", ErrTooLarge, count) + } + + entries := make([]packEntry, count) + byOff := make(map[int64]int, count) + var budget int64 = maxInflated + for i := range entries { + off := cr.off + byOff[off] = i + c, err := cr.ReadByte() + if err != nil { + return nil, fmt.Errorf("pack entry: %w", err) + } + typ := int(c>>4) & 7 + size := int64(c & 15) + for shift := 4; c&0x80 != 0; shift += 7 { + if shift > 56 { + return nil, errors.New("pack entry size overflows") + } + if c, err = cr.ReadByte(); err != nil { + return nil, err + } + size |= int64(c&0x7f) << shift + } + e := &entries[i] + switch typ { + case 1, 2, 3, 4: + e.kind = Kind(typ) + case packOfsDelta: + if c, err = cr.ReadByte(); err != nil { + return nil, err + } + rel := int64(c & 0x7f) + for c&0x80 != 0 { + if rel > 1<<48 { + return nil, errors.New("ofs delta offset overflows") + } + if c, err = cr.ReadByte(); err != nil { + return nil, err + } + rel = (rel+1)<<7 | int64(c&0x7f) + } + e.baseOff = off - rel + if rel == 0 || e.baseOff < 0 { + return nil, errors.New("bad ofs delta offset") + } + case packRefDelta: + raw := make([]byte, hashLen) + if _, err := io.ReadFull(cr, raw); err != nil { + return nil, err + } + e.baseOid = Oid(hex.EncodeToString(raw)) + default: + return nil, fmt.Errorf("bad pack entry type %d", typ) + } + if size > budget { + return nil, fmt.Errorf("%w: inflated objects", ErrTooLarge) + } + budget -= size + if e.data, err = inflate(cr, size); err != nil { + return nil, err + } + } + + objs := make(Objects, count) + oidAt := make(map[int]Oid, count) + for i, e := range entries { + if e.kind != 0 { + oid := hashObject(e.kind, e.data, objectFormat) + objs[oid] = Object{e.kind, e.data} + oidAt[i] = oid + } + } + // ofs bases precede their deltas, so one pass in pack order resolves + // ofs chains; extra passes only for ref deltas pointing forward + for progress := true; progress; { + progress = false + for i := range entries { + e := &entries[i] + if e.kind != 0 { + continue + } + var base Object + if e.baseOid != "" { + var ok bool + if base, ok = objs[e.baseOid]; !ok { + continue + } + } else { + j, ok := byOff[e.baseOff] + if !ok { + return nil, errors.New("ofs delta base isn't an entry") + } + if entries[j].kind == 0 { + continue + } + base = Object{entries[j].kind, entries[j].data} + } + data, err := applyDelta(base.Data, e.data) + if err != nil { + return nil, err + } + if budget -= int64(len(data)); budget < 0 { + return nil, fmt.Errorf("%w: inflated objects", ErrTooLarge) + } + e.kind, e.data = base.Kind, data + oid := hashObject(e.kind, e.data, objectFormat) + objs[oid] = Object{e.kind, e.data} + oidAt[i] = oid + progress = true + } + } + if len(oidAt) != len(entries) { + return nil, errors.New("pack has deltas with missing bases") + } + return objs, nil +} + +func inflate(r *countingReader, size int64) ([]byte, error) { + zr, err := zlib.NewReader(r) + if err != nil { + return nil, fmt.Errorf("pack entry: %w", err) + } + // reading to EOF also checks the adler32 trailer + data, err := io.ReadAll(io.LimitReader(zr, size+1)) + if err != nil { + return nil, fmt.Errorf("pack entry: %w", err) + } + if int64(len(data)) != size { + return nil, errors.New("pack entry size mismatch") + } + return data, nil +} + +func deltaSize(d []byte) (int64, []byte, error) { + var n int64 + for shift := 0; ; shift += 7 { + if len(d) == 0 || shift > 56 { + return 0, nil, errors.New("bad delta header") + } + c := d[0] + d = d[1:] + n |= int64(c&0x7f) << shift + if c&0x80 == 0 { + return n, d, nil + } + } +} + +func applyDelta(base, delta []byte) ([]byte, error) { + srcSize, delta, err := deltaSize(delta) + if err != nil { + return nil, err + } + if srcSize != int64(len(base)) { + return nil, errors.New("delta base size mismatch") + } + dstSize, delta, err := deltaSize(delta) + if err != nil { + return nil, err + } + if dstSize > maxInflated { + return nil, fmt.Errorf("%w: delta result", ErrTooLarge) + } + out := make([]byte, 0, dstSize) + for len(delta) > 0 { + op := delta[0] + delta = delta[1:] + switch { + case op&0x80 != 0: + var off, n int64 + for bit := range 7 { + if op&(1< int64(len(base)) || int64(len(out))+n > dstSize { + return nil, errors.New("delta copy out of bounds") + } + out = append(out, base[off:off+n]...) + case op != 0: + if int(op) > len(delta) || int64(len(out))+int64(op) > dstSize { + return nil, errors.New("delta insert out of bounds") + } + out = append(out, delta[:op]...) + delta = delta[op:] + default: + return nil, errors.New("delta opcode 0") + } + } + if int64(len(out)) != dstSize { + return nil, errors.New("delta result size mismatch") + } + return out, nil +} + +func newHash(objectFormat string) hash.Hash { + if objectFormat == "sha256" { + return sha256.New() + } + return sha1.New() +} + +// hex length +func oidLen(objectFormat string) int { + return newHash(objectFormat).Size() * 2 +} + +func hashObject(kind Kind, data []byte, objectFormat string) Oid { + h := newHash(objectFormat) + h.Write([]byte(kindNames[kind] + " " + strconv.Itoa(len(data)) + "\x00")) + h.Write(data) + return Oid(hex.EncodeToString(h.Sum(nil))) +} + +type treeEntry struct { + Mode string + Name string + Oid Oid +} + +func parseTree(data []byte, objectFormat string) ([]treeEntry, error) { + hashLen := oidLen(objectFormat) / 2 + var entries []treeEntry + for len(data) > 0 { + mode, rest, ok := bytes.Cut(data, []byte(" ")) + if !ok { + return nil, errors.New("bad tree entry") + } + name, rest, ok := bytes.Cut(rest, []byte{0}) + if !ok || len(rest) < hashLen { + return nil, errors.New("bad tree entry") + } + entries = append(entries, treeEntry{ + Mode: string(mode), + Name: string(name), + Oid: Oid(hex.EncodeToString(rest[:hashLen])), + }) + data = rest[hashLen:] + } + return entries, nil +} + +func commitTree(data []byte) (Oid, error) { + line, _, _ := bytes.Cut(data, []byte("\n")) + tree, ok := bytes.CutPrefix(line, []byte("tree ")) + if !ok || !isOid(string(tree)) { + return "", errors.New("commit has no tree line") + } + return Oid(tree), nil +} + +func isOid(s string) bool { + if len(s) != 40 && len(s) != 64 { + return false + } + for _, c := range s { + if !('0' <= c && c <= '9' || 'a' <= c && c <= 'f') { + return false + } + } + return true +} diff --git a/gitclient/pkt.go b/gitclient/pkt.go new file mode 100644 index 0000000000000000000000000000000000000000..a03bb80e6069cfc209e0db421e5f0fc82d361e55 --- /dev/null +++ b/gitclient/pkt.go @@ -0,0 +1,148 @@ +package gitclient + +import ( + "bytes" + "errors" + "fmt" + "io" + "strconv" +) + +type pktKind int + +const ( + pktData pktKind = iota + pktFlush + pktDelim + pktResponseEnd +) + +const maxPktLen = 65520 + +func writePkt(w io.Writer, line string) error { + if len(line)+4 > maxPktLen { + return fmt.Errorf("pkt-line too long: %d bytes", len(line)) + } + _, err := fmt.Fprintf(w, "%04x%s", len(line)+4, line) + return err +} + +func writeFlush(w io.Writer) error { + _, err := io.WriteString(w, "0000") + return err +} + +func writeDelim(w io.Writer) error { + _, err := io.WriteString(w, "0001") + return err +} + +type pktReader struct { + r io.Reader + buf []byte +} + +func newPktReader(r io.Reader) *pktReader { + return &pktReader{r: r, buf: make([]byte, maxPktLen)} +} + +// payload is only valid until the next call +func (p *pktReader) next() (pktKind, []byte, error) { + if _, err := io.ReadFull(p.r, p.buf[:4]); err != nil { + if errors.Is(err, io.EOF) { + err = io.ErrUnexpectedEOF + } + return 0, nil, err + } + n, err := strconv.ParseUint(string(p.buf[:4]), 16, 16) + if err != nil { + return 0, nil, fmt.Errorf("bad pkt-line length %q", p.buf[:4]) + } + switch { + case n == 0: + return pktFlush, nil, nil + case n == 1: + return pktDelim, nil, nil + case n == 2: + return pktResponseEnd, nil, nil + case n < 4 || n > maxPktLen: + return 0, nil, fmt.Errorf("bad pkt-line length %d", n) + } + payload := p.buf[:n-4] + if _, err := io.ReadFull(p.r, payload); err != nil { + return 0, nil, err + } + return pktData, payload, nil +} + +// skips acknowledgments/shallow-info; the reader yields band 1 of the +// packfile section, band 3 becomes an error +func packfileSection(p *pktReader) (io.Reader, error) { + for { + kind, line, err := p.next() + if err != nil { + return nil, err + } + if kind == pktFlush || kind == pktResponseEnd { + return nil, errors.New("fetch response has no packfile section") + } + if msg, ok := bytes.CutPrefix(line, []byte("ERR ")); ok { + return nil, fmt.Errorf("remote: %s", bytes.TrimSpace(msg)) + } + if string(line) == "packfile\n" { + return &sideband{p: p}, nil + } + } +} + +type sideband struct { + p *pktReader + data []byte + done bool +} + +func (s *sideband) Read(b []byte) (int, error) { + for len(s.data) == 0 { + if s.done { + return 0, io.EOF + } + kind, line, err := s.p.next() + if err != nil { + return 0, err + } + if kind != pktData { + s.done = true + continue + } + if len(line) == 0 { + return 0, errors.New("empty sideband pkt-line") + } + switch line[0] { + case 1: + s.data = line[1:] + case 2: + case 3: + return 0, fmt.Errorf("remote: %s", bytes.TrimSpace(line[1:])) + default: + return 0, fmt.Errorf("bad sideband %d", line[0]) + } + } + n := copy(b, s.data) + s.data = s.data[n:] + return n, nil +} + +// errors with ErrTooLarge once more than n bytes come through +type capReader struct { + r io.Reader + n int64 +} + +func (c *capReader) Read(b []byte) (int, error) { + n, err := c.r.Read(b) + c.n -= int64(n) + if c.n < 0 { + return 0, ErrTooLarge + } + return n, err +} diff --git a/gitutil/archive.go b/gitutil/archive.go new file mode 100644 index 0000000000000000000000000000000000000000..806e06d4f6ecacd3134f4977050ca97c11c8ac6d --- /dev/null +++ b/gitutil/archive.go @@ -0,0 +1,278 @@ +package gitutil + +import ( + "bytes" + "cmp" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "slices" + "strings" + "unicode" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/samber/lo" +) + +type ArchiveFormat string + +const ( + ArchiveTarGz ArchiveFormat = "tar.gz" + ArchiveZip ArchiveFormat = "zip" +) + +var ArchiveFormats = []ArchiveFormat{ArchiveTarGz, ArchiveZip} + +func ParseArchiveFormat(raw string) (ArchiveFormat, error) { + if format := ArchiveFormat(raw); slices.Contains(ArchiveFormats, format) { + return format, nil + } + return "", fmt.Errorf("only tar.gz and zip formats are supported, got %q", raw) +} + +func (f ArchiveFormat) String() string { return string(f) } + +func (f ArchiveFormat) contentType() string { + return lo.Ternary(f == ArchiveZip, "application/zip", "application/gzip") +} + +var pathSeparators = strings.NewReplacer("/", "-", `\`, "-") + +type RepoName string + +type Rev string + +const RevHead Rev = "HEAD" + +func ParseRev(raw string) (Rev, error) { + switch { + case raw == "": + return "", fmt.Errorf("ref is empty") + case strings.ContainsFunc(raw, func(c rune) bool { return unicode.IsSpace(c) || unicode.IsControl(c) }): + return "", fmt.Errorf("ref contains whitespace or a control character: %q", raw) + case strings.HasPrefix(raw, "-"): + return "", fmt.Errorf("ref starts with a dash: %q", raw) + } + return Rev(raw), nil +} + +func RevFromHash(h plumbing.Hash) Rev { return Rev(h.String()) } + +const sha1HexLen, sha256HexLen, hexDigits = 40, 64, "0123456789abcdef" + +func (r Rev) IsObjectID() bool { + return slices.Contains([]int{sha1HexLen, sha256HexLen}, len(r)) && + strings.TrimLeft(string(r), hexDigits) == "" +} + +func (r Rev) String() string { return string(r) } + +func (r Rev) Slug() string { return pathSeparators.Replace(plumbing.ReferenceName(r).Short()) } + +func (r Rev) Or(fallback Rev) Rev { return lo.Ternary(r == "", fallback, r) } + +type ArchivePrefix string + +const MaxArchivePrefixLen = 255 + +func ParseArchivePrefix(raw string) (ArchivePrefix, error) { + switch { + case strings.ContainsFunc(raw, unicode.IsControl): + return "", fmt.Errorf("prefix contains a control character: %q", raw) + case strings.Contains(raw, `\`): + return "", fmt.Errorf("prefix contains a backslash: %q", raw) + } + components := lo.Filter(strings.Split(raw, "/"), func(component string, _ int) bool { + return component != "" && component != "." + }) + if slices.Contains(components, "..") { + return "", fmt.Errorf("prefix escapes the archive root: %q", raw) + } + prefix := ArchivePrefix(strings.Join(components, "/")) + if len(prefix) > MaxArchivePrefixLen { + return "", fmt.Errorf("prefix is %d bytes, over the %d byte limit", len(prefix), MaxArchivePrefixLen) + } + return prefix, nil +} + +func (p ArchivePrefix) String() string { return string(p) } + +func archiveStem(repo RepoName, rev Rev) ArchivePrefix { + stem := strings.ToValidUTF8(pathSeparators.Replace(string(repo))+"-"+rev.Slug(), "�") + return ArchivePrefix(strings.ToValidUTF8(stem[:min(len(stem), MaxArchivePrefixLen)], "")) +} + +type ArchiveParams struct { + Rev Rev + Format ArchiveFormat + Prefix ArchivePrefix +} + +func ParseArchiveParams(q url.Values) (ArchiveParams, error) { + p := ArchiveParams{Format: ArchiveTarGz} + var err error + if raw := q.Get("ref"); raw != "" { + if p.Rev, err = ParseRev(raw); err != nil { + return ArchiveParams{}, err + } + } + if raw := q.Get("format"); raw != "" { + if p.Format, err = ParseArchiveFormat(raw); err != nil { + return ArchiveParams{}, err + } + } + if p.Prefix, err = ParseArchivePrefix(q.Get("prefix")); err != nil { + return ArchiveParams{}, err + } + return p, nil +} + +func (p ArchiveParams) WithRev(rev Rev) ArchiveParams { + p.Rev = rev + return p +} + +type ServedArchive struct { + rev Rev + format ArchiveFormat + prefix ArchivePrefix +} + +func (p ArchiveParams) Serve(repo RepoName) ServedArchive { + return ServedArchive{p.Rev, p.Format, cmp.Or(p.Prefix, archiveStem(repo, p.Rev))} +} + +func (a ServedArchive) WithRev(rev Rev) ServedArchive { + a.rev = rev + return a +} + +func (a ServedArchive) Query(repo string) url.Values { + return url.Values{ + "repo": {repo}, + "ref": {a.rev.String()}, + "format": {a.format.String()}, + "prefix": {a.prefix.String()}, + } +} + +func (a ServedArchive) SuffixURL() string { + return fmt.Sprintf("%s.%s?%s", url.PathEscape(a.rev.String()), a.format, + url.Values{"prefix": {a.prefix.String()}}.Encode()) +} + +func (a ServedArchive) SetHeaders(h http.Header) { + h.Set("Content-Type", a.format.contentType()) + h.Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{ + "filename": pathSeparators.Replace(a.prefix.String()) + "." + a.format.String(), + })) + h.Set("X-Content-Type-Options", "nosniff") +} + +const archiveETagDomain = "gitutil.archive.v1" + +type RepoIdentity string + +func (a ServedArchive) ETag(repo RepoIdentity) string { + digest := sha256.Sum256([]byte(strings.Join( + []string{archiveETagDomain, string(repo), a.rev.String(), a.format.String(), a.prefix.String()}, "\x00", + ))) + return `"` + hex.EncodeToString(digest[:]) + `"` +} + +func (a ServedArchive) ServeNotModified(w http.ResponseWriter, r *http.Request, repo RepoIdentity) bool { + etag := a.ETag(repo) + w.Header().Set("Etag", etag) + w.Header().Set("Cache-Control", "no-cache") + if !ETagMatches(r.Header, etag) { + return false + } + w.WriteHeader(http.StatusNotModified) + return true +} + +func ETagMatches(h http.Header, etag string) bool { + offered := strings.Split(strings.Join(h.Values("If-None-Match"), ","), ",") + return lo.SomeBy(offered, func(candidate string) bool { + trimmed := strings.TrimSpace(candidate) + return trimmed == "*" || strings.TrimPrefix(trimmed, "W/") == etag + }) +} + +func ForwardHeaders(dst, src http.Header, keys ...string) { + lo.ForEach(keys, func(key string, _ int) { + if values := src.Values(key); len(values) > 0 { + dst[http.CanonicalHeaderKey(key)] = slices.Clone(values) + } + }) +} + +func ImmutableLink(target string) string { return `<` + target + `>; rel="immutable"` } + +func ParseImmutableLink(header string) (Rev, error) { + target := strings.TrimSuffix(strings.TrimPrefix(header, "<"), `>; rel="immutable"`) + parsed, err := url.Parse(target) + if err != nil { + return "", err + } + return ParseRev(parsed.Query().Get("ref")) +} + +type ResponseBody struct { + inner http.ResponseWriter + started bool +} + +func NewResponseBody(w http.ResponseWriter) *ResponseBody { return &ResponseBody{inner: w} } + +func (b *ResponseBody) Write(p []byte) (int, error) { + n, err := b.inner.Write(p) + b.started = b.started || n > 0 + return n, err +} + +func (b *ResponseBody) Fail() { + if b.started { + panic(http.ErrAbortHandler) + } + header := b.inner.Header() + lo.ForEach([]string{"Content-Length", "Content-Type", "Content-Disposition", "Etag", "Link"}, + func(key string, _ int) { header.Del(key) }) + header.Set("Cache-Control", "no-store") + b.inner.WriteHeader(http.StatusInternalServerError) +} + +func WriteArchive(ctx context.Context, w io.Writer, repoPath string, archive ServedArchive) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + cmd := Command(ctx, "archive", + "--format="+archive.format.String(), + "--prefix="+archive.prefix.String()+"/", + "--", archive.rev.String()) + cmd.Dir = repoPath + stderr := new(bytes.Buffer) + cmd.Stderr = stderr + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { + return err + } + if _, err := io.Copy(w, stdout); err != nil { + cancel() + _ = cmd.Wait() + return fmt.Errorf("writing the archive body: %w", err) + } + if err := cmd.Wait(); err != nil { + return fmt.Errorf("%w, stderr: %s", err, stderr.String()) + } + return nil +} diff --git a/gitutil/archive_test.go b/gitutil/archive_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2500aca50a7b9e43ffb00cb2c4b5d58238205668 --- /dev/null +++ b/gitutil/archive_test.go @@ -0,0 +1,281 @@ +package gitutil + +import ( + "archive/zip" + "bytes" + "context" + "errors" + "mime" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "unicode/utf8" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testArchiveEndpoint = "https://knot.nel.pet/xrpc/sh.tangled.repo.archive" + +func TestParseArchiveParams(t *testing.T) { + cases := []struct { + name string + query url.Values + repo RepoName + want ArchiveParams + wantStem ArchivePrefix + wantFilename string + wantType string + wantErr string + }{ + {"all empty", url.Values{}, "", ArchiveParams{Format: ArchiveTarGz}, "", "", "", ""}, + {"tar.gz", url.Values{"format": {"tar.gz"}}, "", ArchiveParams{Format: ArchiveTarGz}, "", "", "", ""}, + {"zip", url.Values{"format": {"zip"}}, "", ArchiveParams{Format: ArchiveZip}, "", "", "", ""}, + { + "every param", + url.Values{"ref": {"refs/tags/v1.0.0"}, "format": {"zip"}, "prefix": {"/kelp/"}}, + "", ArchiveParams{Rev: "refs/tags/v1.0.0", Format: ArchiveZip, Prefix: "kelp"}, + "kelp", "kelp.zip", "application/zip", "", + }, + {"branch", url.Values{"ref": {"main"}}, "", ArchiveParams{Rev: "main", Format: ArchiveTarGz}, "squid-main", "squid-main.tar.gz", "application/gzip", ""}, + {"nested prefix in the filename", url.Values{"ref": {"main"}, "prefix": {"kelp/uni"}}, "", ArchiveParams{Rev: "main", Format: ArchiveTarGz, Prefix: "kelp/uni"}, "kelp/uni", "kelp-uni.tar.gz", "application/gzip", ""}, + {"full ref", url.Values{"ref": {"refs/heads/feat/uni"}}, "", ArchiveParams{Rev: "refs/heads/feat/uni", Format: ArchiveTarGz}, "squid-feat-uni", "squid-feat-uni.tar.gz", "application/gzip", ""}, + {"head", url.Values{"ref": {"HEAD"}}, "", ArchiveParams{Rev: "HEAD", Format: ArchiveTarGz}, "squid-HEAD", "", "", ""}, + {"trailing slash", url.Values{"ref": {"refs/heads/main/"}}, "", ArchiveParams{Rev: "refs/heads/main/", Format: ArchiveTarGz}, "squid-main-", "", "", ""}, + {"traversal in a ref", url.Values{"ref": {"../../etc"}}, "", ArchiveParams{Rev: "../../etc", Format: ArchiveTarGz}, "squid-..-..-etc", "", "", ""}, + {"windows separator in a ref", url.Values{"ref": {`feat\uni`}}, "", ArchiveParams{Rev: `feat\uni`, Format: ArchiveTarGz}, "squid-feat-uni", "", "", ""}, + {"slash in a repo name", url.Values{"ref": {"main"}}, "kelp/limpet", ArchiveParams{Rev: "main", Format: ArchiveTarGz}, "kelp-limpet-main", "kelp-limpet-main.tar.gz", "application/gzip", ""}, + {"quote in a repo name", url.Values{"ref": {"main"}}, `squid-a"b`, ArchiveParams{Rev: "main", Format: ArchiveTarGz}, `squid-a"b-main`, `squid-a"b-main.tar.gz`, "application/gzip", ""}, + {"non-ascii repo name", url.Values{"ref": {"main"}, "format": {"zip"}}, "squid-über", ArchiveParams{Rev: "main", Format: ArchiveZip}, "squid-über-main", "squid-über-main.zip", "application/zip", ""}, + + {"bare slash prefix", url.Values{"prefix": {"/"}}, "", ArchiveParams{Format: ArchiveTarGz}, "", "", "", ""}, + {"did prefix", url.Values{"prefix": {"did:plc:boltless"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "did:plc:boltless"}, "", "", "", ""}, + {"nested prefix", url.Values{"prefix": {"squid/main"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "squid/main"}, "", "", "", ""}, + {"prefix wrapped in slashes", url.Values{"prefix": {"/squid/main/"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "squid/main"}, "", "", "", ""}, + {"space in a prefix", url.Values{"prefix": {"squid main"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "squid main"}, "", "", "", ""}, + {"bare dot prefix", url.Values{"prefix": {"."}}, "", ArchiveParams{Format: ArchiveTarGz}, "", "", "", ""}, + {"repeated separators and dot segments", url.Values{"prefix": {"/kelp//./uni/"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "kelp/uni"}, "", "", "", ""}, + + {"unsupported format", url.Values{"format": {"tar"}}, "", ArchiveParams{}, "", "", "", "only tar.gz and zip formats are supported"}, + {"space in a ref", url.Values{"ref": {"refs/tags/a b"}}, "", ArchiveParams{}, "", "", "", "ref contains whitespace"}, + {"control character in a ref", url.Values{"ref": {"refs/tags/a\nb"}}, "", ArchiveParams{}, "", "", "", "ref contains whitespace"}, + {"ref that git would read as an option", url.Values{"ref": {"--output=/tmp/evil"}}, "", ArchiveParams{}, "", "", "", "ref starts with a dash"}, + {"prefix escaping the root", url.Values{"prefix": {"../../evil"}}, "", ArchiveParams{}, "", "", "", "prefix escapes the archive root"}, + {"prefix escaping below the root", url.Values{"prefix": {"squid/../../evil"}}, "", ArchiveParams{}, "", "", "", "prefix escapes the archive root"}, + {"a dot-dot segment the knot would reject", url.Values{"prefix": {"squid/../limpet"}}, "", ArchiveParams{}, "", "", "", "prefix escapes the archive root"}, + {"control character in a prefix", url.Values{"prefix": {"squid\nmain"}}, "", ArchiveParams{}, "", "", "", "prefix contains a control character"}, + {"windows separator in a prefix", url.Values{"prefix": {`..\..\evil`}}, "", ArchiveParams{}, "", "", "", "prefix contains a backslash"}, + {"prefix over the length limit", url.Values{"prefix": {strings.Repeat("a", MaxArchivePrefixLen+1)}}, "", ArchiveParams{}, "", "", "", "over the 255 byte limit"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ParseArchiveParams(tc.query) + rejected := err != nil + if got != tc.want || rejected != (tc.wantErr != "") || (rejected && !strings.Contains(err.Error(), tc.wantErr)) { + t.Fatalf("params = %+v with err %v, want %+v and an error mentioning %q", got, err, tc.want, tc.wantErr) + } + if rejected { + return + } + + repo := RepoName("squid") + if tc.repo != "" { + repo = tc.repo + } + served := got.Serve(repo) + query := served.Query("did:plc:limpet") + if stem := ArchivePrefix(query.Get("prefix")); tc.wantStem != "" && stem != tc.wantStem { + t.Errorf("served prefix = %q, want %q", stem, tc.wantStem) + } + if tc.wantFilename != "" { + header := http.Header{} + served.SetHeaders(header) + mediatype, fields, err := mime.ParseMediaType(header.Get("Content-Disposition")) + if err != nil || mediatype != "attachment" || fields["filename"] != tc.wantFilename { + t.Errorf("Content-Disposition = %q (err %v), want an attachment with filename %q", header.Get("Content-Disposition"), err, tc.wantFilename) + } + if ct, sniff := header.Get("Content-Type"), header.Get("X-Content-Type-Options"); ct != tc.wantType || sniff != "nosniff" { + t.Errorf("Content-Type = %q with X-Content-Type-Options %q, want %q and nosniff", ct, sniff, tc.wantType) + } + } + + if back, err := ParseArchiveParams(query); err != nil || back.Serve(repo) != served { + t.Errorf("query round trip = %+v (err %v), want the served archive %+v", back, err, served) + } + back, err := ParseImmutableLink(ImmutableLink(testArchiveEndpoint + "?" + query.Encode())) + if got.Rev != "" && (err != nil || back != got.Rev) { + t.Errorf("Link round trip = %q (err %v), want %q", back, err, got.Rev) + } + }) + } +} + +func TestArchiveFallbacks(t *testing.T) { + hash := RevFromHash(plumbing.NewHash("6f1d3a2b4c5d6e7f8091a2b3c4d5e6f708192a3b")) + if kept, filled := Rev("refs/heads/main").Or(hash), Rev("").Or(hash); kept != "refs/heads/main" || filled != hash { + t.Errorf("Or kept %q and filled %q, want refs/heads/main and the hash %q", kept, filled, hash) + } + if got := Rev("").Or(RevHead); got != RevHead { + t.Errorf("empty rev = %q, want HEAD", got) + } + if _, err := ParseImmutableLink(""); err == nil { + t.Error("ParseImmutableLink must reject an empty header") + } + + params := ArchiveParams{Rev: "main", Format: ArchiveZip, Prefix: "kelp"} + if got := params.WithRev("6f1d3a2"); got != (ArchiveParams{Rev: "6f1d3a2", Format: ArchiveZip, Prefix: "kelp"}) || params.Rev != "main" { + t.Errorf("WithRev = %+v leaving the receiver at %q, want only the rev replaced", got, params.Rev) + } + + served := params.Serve("squid") + if got := served.WithRev("6f1d3a2"); got != (ArchiveParams{Rev: "6f1d3a2", Format: ArchiveZip, Prefix: "kelp"}).Serve("squid") || served.rev != "main" { + t.Errorf("WithRev = %+v leaving the receiver at %q, want only the rev replaced", got, served.rev) + } + + for _, stem := range []ArchivePrefix{ + archiveStem("squid", Rev("refs/heads/"+strings.Repeat("ü", 400))), + archiveStem(RepoName(strings.Repeat("\xff", 300)), "main"), + } { + _, err := ParseArchivePrefix(stem.String()) + if stem == "" || len(stem) > MaxArchivePrefixLen || !utf8.ValidString(stem.String()) || err != nil { + t.Errorf("default prefix is %d bytes %q (err %v), want a non-empty valid prefix within %d bytes ending on a rune boundary", + len(stem), stem, err, MaxArchivePrefixLen) + } + } +} + +func TestRevIsObjectID(t *testing.T) { + sha1Hex, sha256Hex := "6f1d3a2b4c5d6e7f8091a2b3c4d5e6f708192a3b", strings.Repeat("6f1d3a2b", 8) + for rev, want := range map[Rev]bool{ + Rev(sha1Hex): true, Rev(sha256Hex): true, + "": false, "main": false, "refs/heads/main": false, "6f1d3a2": false, RevHead: false, + Rev(strings.ToUpper(sha1Hex)): false, Rev(sha1Hex + "b"): false, + Rev(sha1Hex[:39] + "g"): false, Rev(strings.Repeat("6", 48)): false, + } { + assert.Equal(t, want, rev.IsObjectID(), "%q: only 40 or 64 characters of lowercase hex identify one commit forever", rev) + } +} + +func TestArchiveResponseHeaders(t *testing.T) { + upstream := http.Header{"Etag": {`"6f1d3a2b"`}, "Content-Length": {"4096"}, "Cache-Control": {"no-cache"}} + + got := http.Header{"Etag": {`"limpet"`, `"conch"`}} + ForwardHeaders(got, upstream, "Etag", "Content-Length", "Link") + assert.Equal(t, http.Header{"Etag": {`"6f1d3a2b"`}, "Content-Length": {"4096"}}, got, + "ForwardHeaders overwrites Etag, leaves the unlisted Cache-Control alone, and skips the Link that the knot never sent") + + revalidation := http.Header{} + ForwardHeaders(revalidation, upstream, "Etag") + assert.Equal(t, http.Header{"Etag": {`"6f1d3a2b"`}}, revalidation, "the 304 path forwards the validator without a length") + + validators := http.Header{} + ForwardHeaders(validators, http.Header{"If-None-Match": {`"kelp"`, `"uni"`}}, "if-none-match") + assert.Equal(t, []string{`"kelp"`, `"uni"`}, validators.Values("If-None-Match"), + "ForwardHeaders canonicalizes a lowercase key and sends every validator that a client offered") + + untouched := httptest.NewRecorder() + ArchiveParams{Rev: "main", Format: ArchiveTarGz}.Serve("squid").SetHeaders(untouched.Header()) + untouched.Header().Set("Content-Length", "4096") + untouched.Header().Set("Etag", `"6f1d3a2b"`) + untouched.Header().Set("Link", ImmutableLink(testArchiveEndpoint+"?ref=6f1d3a2b")) + untouched.Header().Set("Cache-Control", "public, max-age=31536000") + NewResponseBody(untouched).Fail() + assert.Equal(t, http.StatusInternalServerError, untouched.Code) + assert.Equal(t, http.Header{"Cache-Control": {"no-store"}, "X-Content-Type-Options": {"nosniff"}}, untouched.Header(), + "Fail deletes the length, the type, the filename, the validator and the link, because a 500 doesn't describe the archive they came from") + + body := NewResponseBody(httptest.NewRecorder()) + written, err := body.Write([]byte("PK\x03\x04")) + require.NoError(t, err) + assert.Equal(t, 4, written) + assert.PanicsWithError(t, http.ErrAbortHandler.Error(), body.Fail, + "a truncated body has to abort the connection, since a clean return reads as a complete archive") + + archive := ArchiveParams{Rev: "6f1d3a2b", Format: ArchiveTarGz}.Serve("squid") + etag := archive.ETag("did:plc:limpet") + assert.Regexp(t, `^"[0-9a-f]{64}"$`, etag) + assert.NotContains(t, []string{ + archive.ETag("did:plc:conch"), + archive.WithRev("main").ETag("did:plc:limpet"), + ArchiveParams{Rev: "6f1d3a2b", Format: ArchiveZip}.Serve("squid").ETag("did:plc:limpet"), + ArchiveParams{Rev: "6f1d3a2b", Format: ArchiveTarGz}.Serve("kelp").ETag("did:plc:limpet"), + }, etag, "a repo, rev, format or prefix that shapes different bytes gets a different validator") + + for offered, want := range map[string]bool{ + etag: true, "W/" + etag: true, "*": true, `"conch", ` + etag: true, `"conch"`: false, "": false, + } { + request := httptest.NewRequest(http.MethodGet, testArchiveEndpoint, nil) + if offered != "" { + request.Header.Set("If-None-Match", offered) + } + recorder := httptest.NewRecorder() + assert.Equal(t, want, archive.ServeNotModified(recorder, request, "did:plc:limpet"), "If-None-Match %q", offered) + assert.Equal(t, lo.Ternary(want, http.StatusNotModified, http.StatusOK), recorder.Code) + assert.Equal(t, etag, recorder.Header().Get("Etag"), "the knot sets the validator on every answer, so a client with a stale copy learns it too") + assert.Equal(t, "no-cache", recorder.Header().Get("Cache-Control")) + } +} + +func TestWriteArchive(t *testing.T) { + repoPath := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(repoPath, "README.md"), []byte("# squid\n"), 0644)) + for _, args := range [][]string{ + {"init", "-q", "-b", "main"}, + {"add", "README.md"}, + {"-c", "user.name=nel", "-c", "user.email=noreply@nel.pet", "commit", "-qm", "Initial commit"}, + {"branch", "feat/uni"}, + } { + cmd := exec.Command("git", args...) + cmd.Dir = repoPath + require.NoError(t, cmd.Run(), "git %v", args) + } + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + + head := ArchiveParams{Rev: RevHead, Format: ArchiveZip} + cases := []struct { + name string + ctx context.Context + archive ServedArchive + want []string + }{ + {"the defaulted prefix begins every entry", context.Background(), ArchiveParams{Rev: "refs/heads/feat/uni", Format: ArchiveZip}.Serve("squid"), []string{"squid-feat-uni/", "squid-feat-uni/README.md"}}, + {"a requested prefix replaces the default", context.Background(), ArchiveParams{Rev: RevHead, Format: ArchiveZip, Prefix: "kelp"}.Serve("squid"), []string{"kelp/", "kelp/README.md"}}, + {"canceled context", canceled, head.Serve("squid"), nil}, + {"a ref the repo doesn't have", context.Background(), ArchiveParams{Rev: "refs/heads/limpet", Format: ArchiveZip}.Serve("squid"), nil}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var body bytes.Buffer + err := WriteArchive(tc.ctx, &body, repoPath, tc.archive) + if tc.want == nil { + assert.Error(t, err) + return + } + require.NoError(t, err) + + entries, err := zip.NewReader(bytes.NewReader(body.Bytes()), int64(body.Len())) + require.NoError(t, err) + assert.Equal(t, tc.want, lo.Map(entries.File, func(f *zip.File, _ int) string { return f.Name })) + }) + } + + err := WriteArchive(context.Background(), refusingWriter{}, repoPath, head.Serve("squid")) + assert.ErrorIs(t, err, errClientGone, "WriteArchive returns the write error itself") +} + +var errClientGone = errors.New("the client stopped reading") + +type refusingWriter struct{} + +func (refusingWriter) Write(p []byte) (int, error) { return 0, errClientGone } diff --git a/gitutil/command.go b/gitutil/command.go new file mode 100644 index 0000000000000000000000000000000000000000..970d8ae7a89114b7b5925143df1076938da210f6 --- /dev/null +++ b/gitutil/command.go @@ -0,0 +1,77 @@ +package gitutil + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + "syscall" + "time" +) + +// git runs network work in child processes, so killing only git leaves them running + +// WaitDelay stops waits on output held open by git's children +const WaitDelay = 10 * time.Second + +// LC_ALL=C keeps git's messages in english, callers match on its stderr +func Command(ctx context.Context, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Env = append(os.Environ(), "LC_ALL=C") + return Contain(cmd) +} + +// Contain is for sandbox-rebuilt commands and direct git binaries +func Contain(cmd *exec.Cmd) *exec.Cmd { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.Setpgid = true + cmd.WaitDelay = WaitDelay + // exec rejects Cancel on commands not created with CommandContext + if cmd.Cancel != nil { + cmd.Cancel = func() error { + if cmd.Process == nil { + return nil + } + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if err == syscall.ESRCH { + return nil + } + return err + } + } + return cmd +} + +func Run(ctx context.Context, args ...string) error { + var stderr bytes.Buffer + cmd := Command(ctx, args...) + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String())) + } + return nil +} + +func Output(ctx context.Context, args ...string) ([]byte, error) { + return output(Command(ctx, args...)) +} + +func OutputOffline(ctx context.Context, args ...string) ([]byte, error) { + cmd := Command(ctx, args...) + cmd.Env = append(cmd.Env, "GIT_NO_LAZY_FETCH=1") + return output(cmd) +} + +func output(cmd *exec.Cmd) ([]byte, error) { + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String())) + } + return stdout.Bytes(), nil +} diff --git a/gitutil/command_test.go b/gitutil/command_test.go new file mode 100644 index 0000000000000000000000000000000000000000..17213a6b21f661ee5eaca0cc941839702a9eec9f --- /dev/null +++ b/gitutil/command_test.go @@ -0,0 +1,48 @@ +package gitutil + +import ( + "context" + "os/exec" + "strings" + "testing" +) + +// os/exec refuses to start a command that carries a Cancel it did not get from +// CommandContext, and the sandbox-wrapped commands in knotserver are exactly +// that shape, so containment has to degrade instead of breaking them +func TestContainContextlessCommand(t *testing.T) { + cmd := Contain(exec.Command("git", "version")) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v: %s", err, out) + } + if !cmd.SysProcAttr.Setpgid { + t.Fatal("a contained command has to lead its own process group") + } + if cmd.WaitDelay == 0 { + t.Fatal("a contained command has to bound its wait") + } +} + +func TestCommandSpeaksEnglishWhateverTheHostLocale(t *testing.T) { + t.Setenv("LC_ALL", "de_DE.UTF-8") + t.Setenv("LANG", "de_DE.UTF-8") + t.Setenv("LANGUAGE", "de") + + out, err := Command(context.Background(), "-C", t.TempDir(), "rev-parse", "HEAD").CombinedOutput() + if err == nil { + t.Fatalf("rev-parse outside a repo succeeded: %s", out) + } + if !strings.Contains(string(out), "not a git repository") { + t.Fatalf("git said %q, want its English message so the callers' matches hold", out) + } +} + +func TestCommandPinsTheLocaleInItsEnvironment(t *testing.T) { + t.Setenv("LC_ALL", "de_DE.UTF-8") + + env := Command(context.Background(), "version").Env + if len(env) == 0 || env[len(env)-1] != "LC_ALL=C" { + t.Fatalf("env ends with %q, want LC_ALL=C last so it beats the host's", env[max(0, len(env)-1):]) + } +} diff --git a/gitutil/keyed_mutex_test.go b/gitutil/keyed_mutex_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ccab995e29ca6f577b744ecbb83069218486834f --- /dev/null +++ b/gitutil/keyed_mutex_test.go @@ -0,0 +1,52 @@ +package gitutil + +import ( + "fmt" + "testing" + "time" +) + +func (k *keyedMutex) entries() int { + k.mu.Lock() + defer k.mu.Unlock() + return len(k.m) +} + +func TestKeyedMutexForgetsKeysNobodyHolds(t *testing.T) { + var k keyedMutex + for i := range 100 { + k.lock(fmt.Sprintf("/repos/did:web:made-up-%d", i))() + } + if got := k.entries(); got != 0 { + t.Fatalf("%d entries left after every lock was released, want 0", got) + } + + unlock := k.lock("same") + entered := make(chan struct{}) + go func() { + k.lock("same")() + close(entered) + }() + select { + case <-entered: + t.Fatal("a second holder got in while the first still held the key") + case <-time.After(50 * time.Millisecond): + } + if got := k.entries(); got != 1 { + t.Fatalf("%d entries while one holds and one waits, want 1", got) + } + unlock() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("the waiter never got the key after the holder let go") + } + // the waiter drops its entry just after signaling, wait for that + deadline := time.Now().Add(5 * time.Second) + for k.entries() != 0 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if got := k.entries(); got != 0 { + t.Fatalf("%d entries left after the waiter finished, want 0", got) + } +} diff --git a/gitutil/readme.go b/gitutil/readme.go new file mode 100644 index 0000000000000000000000000000000000000000..78dd5187368259c9f127437eed756fd0707352b2 --- /dev/null +++ b/gitutil/readme.go @@ -0,0 +1,22 @@ +package gitutil + +import ( + "regexp" + + "github.com/go-git/go-git/v5/plumbing/filemode" +) + +var readmePattern = regexp.MustCompile(`(?i)^readme(?:\.[^.]+)?$`) + +// IsReadmeFile reports whether name/mode identifies a readme blob. The git +// mode is checked so directories or symlinks named "readme" are filtered out. +func IsReadmeFile(name, mode string) bool { + if !readmePattern.MatchString(name) { + return false + } + m, err := filemode.New(mode) + if err != nil { + return false + } + return m == filemode.Regular || m == filemode.Executable +} diff --git a/gitutil/readme_test.go b/gitutil/readme_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ea0d335d4e143b33596eba97b0175738a5c900c2 --- /dev/null +++ b/gitutil/readme_test.go @@ -0,0 +1,48 @@ +package gitutil + +import "testing" + +func TestIsReadmeFile(t *testing.T) { + const ( + fileMode = "100644" + execMode = "100755" + dirMode = "040000" + ) + + cases := []struct { + name string + mode string + want bool + }{ + {"README.md", fileMode, true}, + {"ReadMe.MD", fileMode, true}, + {"readme", fileMode, true}, + {"README", execMode, true}, + + // regression: a directory named "readme" must not be picked up + // as the README blob; tree handlers used to fetch it and 503. + {"readme", dirMode, false}, + {"README.md", dirMode, false}, + + // readme is matched by convention, not by renderable format; + // unsupported markup falls through to plaintext in GetFormat. + {"README.rst", fileMode, true}, + {"README.foo", fileMode, true}, + + {"README.Music.md", fileMode, false}, + {"readme-old", fileMode, false}, + {"readme_legacy", fileMode, false}, + {"notreadme.md", fileMode, false}, + {"READMEISH", fileMode, false}, + {"README.md", "", false}, + {"README.md", "120000", false}, // symlink + } + + for _, c := range cases { + t.Run(c.name+"/"+c.mode, func(t *testing.T) { + if got := IsReadmeFile(c.name, c.mode); got != c.want { + t.Errorf("IsReadmeFile(%q, %q) = %v, want %v", c.name, c.mode, got, c.want) + } + }) + } +} diff --git a/gitutil/sync.go b/gitutil/sync.go new file mode 100644 index 0000000000000000000000000000000000000000..76626621d93eea6e3a7fc7137b3c0c5bd1795626 --- /dev/null +++ b/gitutil/sync.go @@ -0,0 +1,208 @@ +package gitutil + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + + "github.com/hashicorp/go-version" +) + +// don't let two requests mutate the same checkout at once +var repoLocks keyedMutex + +// entries drop once nobody holds them since callers pick the keys and nothing else bounds the map +type keyedMutex struct { + mu sync.Mutex + m map[string]*keyedLock +} + +type keyedLock struct { + mu sync.Mutex + users int +} + +func (k *keyedMutex) lock(key string) func() { + k.mu.Lock() + if k.m == nil { + k.m = make(map[string]*keyedLock) + } + l, ok := k.m[key] + if !ok { + l = &keyedLock{} + k.m[key] = l + } + l.users++ + k.mu.Unlock() + + l.mu.Lock() + return func() { + l.mu.Unlock() + k.mu.Lock() + defer k.mu.Unlock() + if l.users--; l.users == 0 { + delete(k.m, key) + } + } +} + +func Version() (*version.Version, error) { + out, err := Output(context.Background(), "version") + if err != nil { + return nil, err + } + fields := strings.Fields(string(out)) + if len(fields) < 3 { + return nil, fmt.Errorf("invalid git version: %s", out) + } + + // git version 2.29.3.windows.1 + versionString := fields[2] + if pos := strings.Index(versionString, "windows"); pos >= 1 { + versionString = versionString[:pos-1] + } + return version.NewVersion(versionString) +} + +// SparseSync updates a sparse checkout, fetching it first if needed +// this should only fail if the repo is completely empty! +func SparseSync(ctx context.Context, cloneUri, path, rev string, sparse ...string) error { + defer repoLocks.lock(path)() + + exist, err := isDir(path) + if err != nil { + return err + } + if exist { + gitDirExist, err := isDir(path + "/.git") + if err != nil { + return err + } + if !gitDirExist { + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("cleanup invalid git dir: %w", err) + } + exist = false + } + } + if rev == "" { + rev = "HEAD" + } + setup := func() error { + if err := os.MkdirAll(path, 0o755); err != nil { + return fmt.Errorf("mkdir repo dir: %w", err) + } + if err := Run(ctx, "init", "-q", "--object-format="+objectFormat(ctx, cloneUri, rev), path); err != nil { + return fmt.Errorf("git init: %w", err) + } + if err := Run(ctx, "-C", path, "remote", "add", "origin", cloneUri); err != nil { + return fmt.Errorf("git remote add: %w", err) + } + if err := Run(ctx, append([]string{"-C", path, "sparse-checkout", "set", "--no-cone"}, sparse...)...); err != nil { + return fmt.Errorf("git sparse-checkout set: %w", err) + } + return nil + } + fetch := func() error { + return Run(ctx, "-C", path, "fetch", "--depth=1", "--filter=tree:0", "origin", rev) + } + + if !exist { + if err := setup(); err != nil { + return err + } + if err := fetch(); err != nil { + os.RemoveAll(path) + return fmt.Errorf("git fetch: %w", err) + } + } else { + if err := fetch(); err != nil { + // a cancelled fetch can leave lock files behind + removeStaleLocks(path) + if retryErr := fetch(); retryErr != nil { + if rmErr := os.RemoveAll(path); rmErr != nil { + return fmt.Errorf("git fetch: %w (cleanup failed: %v)", retryErr, rmErr) + } + if setupErr := setup(); setupErr != nil { + return fmt.Errorf("git fetch: %w (re-init failed: %v)", retryErr, setupErr) + } + if cloneErr := fetch(); cloneErr != nil { + return fmt.Errorf("git fetch: %w (re-fetch failed: %v)", retryErr, cloneErr) + } + } + } + } + removeStaleLocks(path) + if err := Run(ctx, "-C", path, "checkout", "--detach", "FETCH_HEAD"); err != nil { + return fmt.Errorf("git checkout: %w", err) + } + return nil +} + +// a clone can't fetch objects of the other hash, so match the remote's format +// if ls-remote fails we guess sha1 and let the fetch report the real error +func objectFormat(ctx context.Context, cloneUri, rev string) string { + if !Rev(rev).IsObjectID() { + out, err := Output(ctx, "ls-remote", cloneUri, rev) + if err != nil { + return "sha1" + } + rev, _, _ = strings.Cut(string(out), "\t") + } + if len(rev) == sha256HexLen { + return "sha256" + } + return "sha1" +} + +const maxAncestryDepth = 100 + +// reports whether ref's recent history includes sha +// this doesnt use SparseSync bcz that is shallow and we need the partially-fully-graph +func RefContainsCommit(ctx context.Context, cloneUri, ref, sha string) (bool, error) { + path, err := os.MkdirTemp("", "ref-ancestry-") + if err != nil { + return false, fmt.Errorf("making a scratch dir: %w", err) + } + defer os.RemoveAll(path) + + if err := Run(ctx, "init", "-q", "--object-format="+objectFormat(ctx, cloneUri, ref), path); err != nil { + return false, fmt.Errorf("git init: %w", err) + } + if err := Run(ctx, "-C", path, "remote", "add", "origin", cloneUri); err != nil { + return false, fmt.Errorf("git remote add: %w", err) + } + if err := Run(ctx, "-C", path, "fetch", "--filter=tree:0", "--depth="+strconv.Itoa(maxAncestryDepth), "origin", ref); err != nil { + return false, fmt.Errorf("git fetch %s: %w", ref, err) + } + + // this exits 0 when the ref's history contains it + // a sha outside the fetched window fails the command + out, err := Output(ctx, "-C", path, "merge-base", sha, "FETCH_HEAD") + if err != nil { + return false, nil + } + return strings.TrimSpace(string(out)) == sha, nil +} + +func removeStaleLocks(path string) { + locks, _ := filepath.Glob(filepath.Join(path, ".git", "*.lock")) + for _, lock := range locks { + os.Remove(lock) + } +} + +func isDir(path string) (bool, error) { + info, err := os.Stat(path) + if err == nil && info.IsDir() { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} diff --git a/gitutil/sync_test.go b/gitutil/sync_test.go new file mode 100644 index 0000000000000000000000000000000000000000..15961c19d3f4fd3e943db53aca72dc1a8ebc7500 --- /dev/null +++ b/gitutil/sync_test.go @@ -0,0 +1,310 @@ +package gitutil + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +const testSparseDir = "/.tangled/workflows" + +func newGitRepo(t *testing.T, initArgs ...string) (string, func(args ...string) string) { + t.Helper() + // ignore the developer's git config + t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull) + t.Setenv("GIT_CONFIG_SYSTEM", os.DevNull) + t.Setenv("GIT_AUTHOR_NAME", "spindle") + t.Setenv("GIT_AUTHOR_EMAIL", "spindle@tangled.org") + t.Setenv("GIT_COMMITTER_NAME", "spindle") + t.Setenv("GIT_COMMITTER_EMAIL", "spindle@tangled.org") + + src := t.TempDir() + run := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", src}, args...)...) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + + run(append([]string{"init", "-b", "main"}, initArgs...)...) + return src, run +} + +func commitWorkflow(t *testing.T, src string, run func(args ...string) string) string { + t.Helper() + if err := os.MkdirAll(filepath.Join(src, ".tangled", "workflows"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, ".tangled", "workflows", "ci.yml"), []byte("when: []\n"), 0o644); err != nil { + t.Fatal(err) + } + run("add", ".") + run("commit", "-m", "workflows") + return run("rev-parse", "HEAD") +} + +func TestSparseSync(t *testing.T) { + src, run := newGitRepo(t) + + rev := commitWorkflow(t, src, run) + + path := filepath.Join(t.TempDir(), "repo") + for _, stage := range []string{"clone", "fetch"} { + if err := SparseSync(context.Background(), src, path, rev, testSparseDir); err != nil { + t.Fatalf("%s: %v", stage, err) + } + if _, err := os.Stat(filepath.Join(path, ".tangled", "workflows", "ci.yml")); err != nil { + t.Fatalf("%s: sparse file missing from the checkout: %v", stage, err) + } + } +} + +func TestSparseSync_DanglingRemoteHead(t *testing.T) { + src, run := newGitRepo(t) + + sha := commitWorkflow(t, src, run) + + // switch branch to master and point HEAD to unborn main + run("branch", "-m", "main", "master") + run("symbolic-ref", "HEAD", "refs/heads/main") + + path := filepath.Join(t.TempDir(), "repo") + // syncing by commit sha must succeed even when remote HEAD is dangling + if err := SparseSync(context.Background(), src, path, sha, testSparseDir); err != nil { + t.Fatalf("sync by sha failed on dangling HEAD remote: %v", err) + } + if _, err := os.Stat(filepath.Join(path, ".tangled", "workflows", "ci.yml")); err != nil { + t.Fatalf("sparse file missing: %v", err) + } + + // syncing by ref name must also succeed + path2 := filepath.Join(t.TempDir(), "repo2") + if err := SparseSync(context.Background(), src, path2, "refs/heads/master", testSparseDir); err != nil { + t.Fatalf("sync by ref failed on dangling HEAD remote: %v", err) + } +} + +func TestSparseSync_Sha256Remote(t *testing.T) { + src, run := newGitRepo(t, "--object-format=sha256") + sha := commitWorkflow(t, src, run) + if len(sha) != sha256HexLen { + t.Fatalf("fixture commit %q is not a sha256 id", sha) + } + + for name, rev := range map[string]string{"by sha": sha, "by HEAD": ""} { + path := filepath.Join(t.TempDir(), "repo") + if err := SparseSync(context.Background(), src, path, rev, testSparseDir); err != nil { + t.Fatalf("%s: %v", name, err) + } + if _, err := os.Stat(filepath.Join(path, ".tangled", "workflows", "ci.yml")); err != nil { + t.Fatalf("%s: sparse file missing: %v", name, err) + } + } +} + +func TestSparseSync_Sha256DanglingHeadByRef(t *testing.T) { + src, run := newGitRepo(t, "--object-format=sha256") + commitWorkflow(t, src, run) + run("branch", "-m", "main", "master") + run("symbolic-ref", "HEAD", "refs/heads/main") + + path := filepath.Join(t.TempDir(), "repo") + if err := SparseSync(context.Background(), src, path, "refs/heads/master", testSparseDir); err != nil { + t.Fatalf("sync by ref failed on a sha256 remote with a dangling HEAD: %v", err) + } +} + +func TestRefContainsCommit_Sha256Remote(t *testing.T) { + src, run := newGitRepo(t, "--object-format=sha256") + sha := commitWorkflow(t, src, run) + + got, err := RefContainsCommit(context.Background(), src, "refs/heads/main", sha) + if err != nil || !got { + t.Fatalf("RefContainsCommit() = %t, err = %v, want true on a sha256 remote", got, err) + } +} + +func TestRefContainsCommit(t *testing.T) { + src, run := newGitRepo(t) + + if err := os.WriteFile(filepath.Join(src, "f"), []byte("a"), 0o644); err != nil { + t.Fatal(err) + } + run("add", ".") + run("commit", "-m", "a") + olderThanWindow := run("rev-parse", "HEAD") + + // fill main past the ancestry window RefContainsCommit fetches + for i := 0; i < maxAncestryDepth+5; i++ { + if err := os.WriteFile(filepath.Join(src, "f"), []byte(fmt.Sprintf("f%d", i)), 0o644); err != nil { + t.Fatal(err) + } + run("add", ".") + run("commit", "-m", fmt.Sprintf("fill %d", i)) + } + tip := run("rev-parse", "HEAD") + ancestor := run("rev-parse", "HEAD~1") + + // a commit that exists in the repository but not on main + run("checkout", "-b", "dev") + if err := os.WriteFile(filepath.Join(src, "f"), []byte("c"), 0o644); err != nil { + t.Fatal(err) + } + run("add", ".") + run("commit", "-m", "c") + offBranch := run("rev-parse", "HEAD") + run("checkout", "main") + + for _, tt := range []struct { + name string + sha string + want bool + }{ + {"the ref tip", tip, true}, + {"an ancestor of the tip", ancestor, true}, + {"a commit older than the ancestry window", olderThanWindow, false}, + {"a commit on another branch", offBranch, false}, + {"a commit the repo never had", strings.Repeat("1", 40), false}, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := RefContainsCommit(context.Background(), src, "refs/heads/main", tt.sha) + if got != tt.want { + t.Fatalf("RefContainsCommit() = %t, err = %v, want %t", got, err, tt.want) + } + }) + } +} + +// hangingKnot keeps the child process alive until the client closes its socket +func hangingKnot(t *testing.T) (url string, asked, dropped <-chan struct{}) { + t.Helper() + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + + askedCh := make(chan struct{}) + droppedCh := make(chan struct{}) + var askOnce, dropOnce sync.Once + + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + br := bufio.NewReader(conn) + for { + line, err := br.ReadString('\n') + if err != nil { + return + } + if strings.TrimSpace(line) == "" { + break + } + } + io.WriteString(conn, "HTTP/1.1 200 OK\r\n"+ + "Content-Type: application/x-git-upload-pack-advertisement\r\n"+ + "Transfer-Encoding: chunked\r\n\r\n") + askOnce.Do(func() { close(askedCh) }) + // this ends only when the child process dies + io.Copy(io.Discard, br) + dropOnce.Do(func() { close(droppedCh) }) + }() + } + }() + + return "http://" + ln.Addr().String(), askedCh, droppedCh +} + +func TestRunCancelKillsRemoteHelper(t *testing.T) { + knot, asked, dropped := hangingKnot(t) + path := filepath.Join(t.TempDir(), "repo") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan error, 1) + go func() { done <- Run(ctx, "clone", "--no-checkout", "--depth=1", knot+"/repo.git", path) }() + + select { + case <-asked: + case err := <-done: + t.Fatalf("clone returned before it reached the knot: %v", err) + case <-time.After(30 * time.Second): + t.Fatal("git never asked the knot for refs") + } + + cancel() + + select { + case err := <-done: + if err == nil { + t.Fatal("a cancelled clone reported success") + } + case <-time.After(15 * time.Second): + t.Fatal("Run is still blocked after cancellation: a surviving helper holds the inherited stderr pipe open") + } + + select { + case <-dropped: + case <-time.After(15 * time.Second): + t.Fatal("git-remote-http outlived the cancelled clone") + } +} + +func TestSparseSyncCancelReleasesRepoLock(t *testing.T) { + knot, asked, _ := hangingKnot(t) + path := filepath.Join(t.TempDir(), "repo") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan error, 1) + go func() { done <- SparseSync(ctx, knot+"/repo.git", path, "HEAD", testSparseDir) }() + + select { + case <-asked: + case err := <-done: + t.Fatalf("sync returned before it reached the knot: %v", err) + case <-time.After(30 * time.Second): + t.Fatal("git never asked the knot for refs") + } + + cancel() + + select { + case <-done: + case <-time.After(15 * time.Second): + t.Fatal("SparseSync is still blocked after cancellation") + } + + // another request must be able to use this repo + dead, cancelDead := context.WithCancel(context.Background()) + cancelDead() + + next := make(chan error, 1) + go func() { next <- SparseSync(dead, knot+"/repo.git", path, "HEAD", testSparseDir) }() + + select { + case <-next: + case <-time.After(15 * time.Second): + t.Fatal("a cancelled sync never released the repo lock") + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..74a7c873315bf72a42fc64fd539fbf06a0cdc74e --- /dev/null +++ b/go.mod @@ -0,0 +1,283 @@ +module tangled.org/core + +go 1.26.0 + +require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 + buf.build/go/protovalidate v1.2.0 + github.com/Blank-Xu/sql-adapter v1.1.1 + github.com/adrg/frontmatter v0.2.0 + github.com/alecthomas/assert/v2 v2.11.0 + github.com/avast/retry-go/v4 v4.6.1 + github.com/aws/aws-sdk-go-v2 v1.41.4 + github.com/aws/aws-sdk-go-v2/config v1.32.12 + github.com/aws/aws-sdk-go-v2/credentials v1.19.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 + github.com/bluesky-social/indigo v0.0.0-20260220055544-bf41e2ee75ab + github.com/bmatcuk/doublestar/v4 v4.9.1 + github.com/carlmjohnson/versioninfo v0.22.5 + github.com/casbin/casbin/v2 v2.103.0 + github.com/charmbracelet/bubbles v1.0.0 + github.com/charmbracelet/bubbletea v1.3.10 + github.com/charmbracelet/lipgloss v1.1.0 + github.com/charmbracelet/log v0.4.2 + github.com/charmbracelet/ssh v0.0.0-20250128164007-98fd5ae11894 + github.com/charmbracelet/wish v1.4.7 + github.com/containerd/cgroups/v3 v3.1.3 + github.com/did-method-plc/go-didplc v0.2.2 + github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242 + github.com/docker/docker v28.2.2+incompatible + github.com/felixge/httpsnoop v1.0.4 + github.com/gliderlabs/ssh v0.3.8 + github.com/go-chi/chi/v5 v5.2.0 + github.com/go-chi/cors v1.2.2 + github.com/go-git/go-git/v5 v5.19.0 + github.com/google/uuid v1.6.0 + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 + github.com/hashicorp/go-version v1.8.0 + github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/hiddeco/sshsig v0.2.0 + github.com/hpcloud/tail v1.0.0 + github.com/ipfs/go-cid v0.6.2 + github.com/mattn/go-sqlite3 v1.14.34 + github.com/mdlayher/vsock v1.3.0 + github.com/miekg/dns v1.1.72 + github.com/multiformats/go-multihash v0.2.3 + github.com/openbao/openbao/api/v2 v2.3.0 + github.com/posthog/posthog-go v1.5.5 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 + github.com/prometheus/procfs v0.19.2 + github.com/redis/go-redis/v9 v9.7.3 + github.com/resend/resend-go/v3 v3.5.0 + github.com/robfig/cron/v3 v3.0.1 + github.com/samber/lo v1.38.1 + github.com/sethvargo/go-envconfig v1.1.0 + github.com/sourcegraph/zoekt v0.0.0-20260605122240-893a523804f1 + github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c + github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef + github.com/stretchr/testify v1.12.1 + github.com/urfave/cli/v3 v3.14.0 + github.com/whyrusleeping/cbor-gen v0.3.1 + github.com/yuin/goldmark v1.8.2 + gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b + go.opentelemetry.io/contrib/bridges/otelslog v0.18.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 + go.opentelemetry.io/otel v1.45.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 + go.opentelemetry.io/otel/log v0.19.0 + go.opentelemetry.io/otel/sdk v1.45.0 + go.opentelemetry.io/otel/sdk/log v0.19.0 + go.opentelemetry.io/otel/trace v1.45.0 + golang.org/x/crypto v0.57.0 + golang.org/x/image v0.31.0 + golang.org/x/sync v0.23.0 + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da + google.golang.org/protobuf v1.36.12 + gopkg.in/yaml.v3 v3.0.1 + tangled.org/ptr.pet/hydrant v0.0.0-20261021010439-0128352f6b73 +) + +require ( + github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-enry/go-enry/v2 v2.9.6 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/sys v0.48.0 // indirect +) + +require ( + cel.dev/expr v0.25.2 // indirect + dario.cat/mergo v1.0.1 // indirect + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.3.0 // indirect + github.com/RoaringBitmap/roaring v1.9.4 // indirect + github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b // indirect + github.com/alecthomas/repr v0.5.2 // indirect + github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect + github.com/aws/smithy-go v1.24.2 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.24.4 // indirect + github.com/bmatcuk/doublestar v1.3.4 // indirect + github.com/casbin/govaluate v1.3.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.4.1 // indirect + github.com/charmbracelet/keygen v0.5.3 // indirect + github.com/charmbracelet/x/ansi v0.11.6 // indirect + github.com/charmbracelet/x/cellbuf v0.0.15 // indirect + github.com/charmbracelet/x/conpty v0.1.0 // indirect + github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 // indirect + github.com/charmbracelet/x/input v0.3.4 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/charmbracelet/x/termios v0.1.0 // indirect + github.com/charmbracelet/x/windows v0.2.0 // indirect + github.com/cilium/ebpf v0.16.0 // indirect + github.com/clipperhouse/displaywidth v0.9.0 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.5.0 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/creack/pty v1.1.21 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/earthboundkid/versioninfo/v2 v2.24.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/go-enry/go-oniguruma v1.2.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect + github.com/go-jose/go-jose/v3 v3.0.4 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-redis/cache/v9 v9.0.0 // indirect + github.com/go-test/deep v1.1.1 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/mock v1.6.0 // indirect + github.com/google/cel-go v0.28.0 // indirect + github.com/google/go-querystring v1.2.0 // indirect + github.com/grafana/regexp v0.0.0-20240607082908-2cb410fa05da // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/hcl v1.0.1-vault-7 // indirect + github.com/hexops/gotextdiff v1.0.3 // indirect + github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/boxo v0.36.0 // indirect + github.com/ipfs/go-block-format v0.2.3 // indirect + github.com/ipfs/go-blockservice v0.5.2 // indirect + github.com/ipfs/go-datastore v0.9.0 // indirect + github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect + github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect + github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect + github.com/ipfs/go-ipfs-util v0.0.3 // indirect + github.com/ipfs/go-ipld-cbor v0.2.1 // indirect + github.com/ipfs/go-ipld-format v0.6.3 // indirect + github.com/ipfs/go-ipld-legacy v0.2.2 // indirect + github.com/ipfs/go-log v1.0.5 // indirect + github.com/ipfs/go-log/v2 v2.9.1 // indirect + github.com/ipfs/go-merkledag v0.11.0 // indirect + github.com/ipfs/go-metrics-interface v0.3.0 // indirect + github.com/ipfs/go-verifcid v0.0.3 // indirect + github.com/ipld/go-car v0.6.2 // indirect + github.com/ipld/go-codec-dagpb v1.7.0 // indirect + github.com/ipld/go-ipld-prime v0.24.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.18.6 + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mdlayher/socket v0.6.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/mr-tron/base58 v1.3.0 // indirect + github.com/mschoch/smat v0.2.0 // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multibase v0.3.0 // indirect + github.com/multiformats/go-varint v0.1.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/onsi/gomega v1.37.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/opencontainers/runtime-spec v1.3.0 // indirect + github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/polydawn/refmt v0.90.0 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sourcegraph/go-ctags v0.0.0-20250729094530-349a251d78d8 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/tetratelabs/wazero v1.9.0 // indirect + github.com/vmihailenco/go-tinylfu v0.2.2 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/wasilibs/go-re2 v1.10.0 // indirect + github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect + golang.org/x/mod v0.41.0 // indirect + golang.org/x/text v0.42.0 // indirect + golang.org/x/time v0.16.0 + golang.org/x/tools v0.51.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/grpc v1.83.2 // indirect + gopkg.in/fsnotify.v1 v1.4.7 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gorm.io/gorm v1.31.2 // indirect + gotest.tools/v3 v3.5.2 // indirect + lukechampine.com/blake3 v1.4.1 // indirect +) + +replace github.com/sergi/go-diff => github.com/sergi/go-diff v1.1.0 + +replace github.com/go-git/go-git/v5 => github.com/oppiliappan/go-git/v5 v5.17.0 + +replace github.com/bluesky-social/indigo => tangled.org/ptr.pet/indigo v0.0.0-20261021233448-bf24d47f0126 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000000000000000000000000000000000000..d559d631066364470a5c0545239ad10e4d82cbfe --- /dev/null +++ b/go.sum @@ -0,0 +1,988 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 h1:s6hzCXtND/ICdGPTMGk7C+/BFlr2Jg5GyH0NKf4XGXg= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= +buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= +buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Blank-Xu/sql-adapter v1.1.1 h1:+g7QXU9sl/qT6Po97teMpf3GjAO0X9aFaqgSePXvYko= +github.com/Blank-Xu/sql-adapter v1.1.1/go.mod h1:o2g8EZhZ3TudnYEGDkoU+3jCTCgDgx1o/Ig5ajKkaLY= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= +github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= +github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= +github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= +github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b h1:5/++qT1/z812ZqBvqQt6ToRswSuPZ/B33m6xVHRzADU= +github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b/go.mod h1:4+EPqMRApwwE/6yo6CxiHoSnBzjRr3jsqer7frxP8y4= +github.com/adrg/frontmatter v0.2.0 h1:/DgnNe82o03riBd1S+ZDjd43wAmC6W35q67NHeLkPd4= +github.com/adrg/frontmatter v0.2.0/go.mod h1:93rQCj3z3ZlwyxxpQioRKC1wDLto4aXHrbqIsnH9wmE= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= +github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk= +github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA= +github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= +github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 h1:3kGOqnh1pPeddVa/E37XNTaWJ8W6vrbYV9lJEkCnhuY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= +github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 h1:SwGMTMLIlvDNyhMteQ6r8IJSBPlRdXX5d4idhIGbkXA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21/go.mod h1:UUxgWxofmOdAMuqEsSppbDtGKLfR04HGsD0HXzvhI1k= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 h1:qtJZ70afD3ISKWnoX3xB0J2otEqu3LqicRcDBqsj0hQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12/go.mod h1:v2pNpJbRNl4vEUWEh5ytQok0zACAKfdmKS51Hotc3pQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 h1:siU1A6xjUZ2N8zjTHSXFhB9L/2OY8Dqs0xXiLjF30jA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20/go.mod h1:4TLZCmVJDM3FOu5P5TJP0zOlu9zWgDWU7aUxWbr+rcw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 h1:csi9NLpFZXb9fxY7rS1xVzgPRGMt7MSNWeQ6eo247kE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1/go.mod h1:qXVal5H0ChqXP63t6jze5LmFalc7+ZE7wOdLtZ0LCP0= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.24.4 h1:95H15Og1clikBrKr/DuzMXkQzECs1M6hhoGXLwLQOZE= +github.com/bits-and-blooms/bitset v1.24.4/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= +github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= +github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bmatcuk/doublestar/v4 v4.7.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= +github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc= +github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8= +github.com/casbin/casbin/v2 v2.100.0/go.mod h1:LO7YPez4dX3LgoTCqSQAleQDo0S0BeZBDxYnPUl95Ng= +github.com/casbin/casbin/v2 v2.103.0 h1:dHElatNXNrr8XcseUov0ZSiWjauwmZZE6YMV3eU1yic= +github.com/casbin/casbin/v2 v2.103.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/govaluate v1.2.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= +github.com/casbin/govaluate v1.3.0 h1:VA0eSY0M2lA86dYd5kPPuNZMUD9QkWnOCnavGrw9myc= +github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc= +github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= +github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk= +github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk= +github.com/charmbracelet/keygen v0.5.3 h1:2MSDC62OUbDy6VmjIE2jM24LuXUvKywLCmaJDmr/Z/4= +github.com/charmbracelet/keygen v0.5.3/go.mod h1:TcpNoMAO5GSmhx3SgcEMqCrtn8BahKhB8AlwnLjRUpk= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/log v0.4.2 h1:hYt8Qj6a8yLnvR+h7MwsJv/XvmBJXiueUcI3cIxsyig= +github.com/charmbracelet/log v0.4.2/go.mod h1:qifHGX/tc7eluv2R6pWIpyHDDrrb/AG71Pf2ysQu5nw= +github.com/charmbracelet/ssh v0.0.0-20250128164007-98fd5ae11894 h1:Ffon9TbltLGBsT6XE//YvNuu4OAaThXioqalhH11xEw= +github.com/charmbracelet/ssh v0.0.0-20250128164007-98fd5ae11894/go.mod h1:hg+I6gvlMl16nS9ZzQNgBIrrCasGwEw0QiLsDcP01Ko= +github.com/charmbracelet/wish v1.4.7 h1:O+jdLac3s6GaqkOHHSwezejNK04vl6VjO1A+hl8J8Yc= +github.com/charmbracelet/wish v1.4.7/go.mod h1:OBZ8vC62JC5cvbxJLh+bIWtG7Ctmct+ewziuUWK+G14= +github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8= +github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ= +github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= +github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= +github.com/charmbracelet/x/conpty v0.1.0 h1:4zc8KaIcbiL4mghEON8D72agYtSeIgq8FSThSPQIb+U= +github.com/charmbracelet/x/conpty v0.1.0/go.mod h1:rMFsDJoDwVmiYM10aD4bH2XiRgwI7NYJtQgl5yskjEQ= +github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9iqk37QUU2Rvb6DSBYRLtWqFqfxf8l5hOZUA= +github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86/go.mod h1:2P0UgXMEa6TsToMSuFqKFQR+fZTO9CNGUNokkPatT/0= +github.com/charmbracelet/x/input v0.3.4 h1:Mujmnv/4DaitU0p+kIsrlfZl/UlmeLKw1wAP3e1fMN0= +github.com/charmbracelet/x/input v0.3.4/go.mod h1:JI8RcvdZWQIhn09VzeK3hdp4lTz7+yhiEdpEQtZN+2c= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= +github.com/charmbracelet/x/termios v0.1.0 h1:y4rjAHeFksBAfGbkRDmVinMg7x7DELIGAFbdNvxg97k= +github.com/charmbracelet/x/termios v0.1.0/go.mod h1:H/EVv/KRnrYjz+fCYa9bsKdqF3S8ouDK0AZEbG7r+/U= +github.com/charmbracelet/x/windows v0.2.0 h1:ilXA1GJjTNkgOm94CLPeSz7rar54jtFatdmoiONPuEw= +github.com/charmbracelet/x/windows v0.2.0/go.mod h1:ZibNFR49ZFqCXgP76sYanisxRyC+EYrBE7TTknD8s1s= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= +github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA= +github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U= +github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= +github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILMqgNeV5jiqR4j+sTuvQNHdf2chuKj1M5k= +github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6dyLYXQ= +github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= +github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= +github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/did-method-plc/go-didplc v0.2.2 h1:53HFhTT8NCAeFmZ6fdIZCf3PGDvj7A3cDjzOOEqn5XM= +github.com/did-method-plc/go-didplc v0.2.2/go.mod h1:bKdJ21irnwNHgVLWWL32zUWqZueXYbJRUcxplZghByo= +github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e h1:SCnqm8SjSa0QqRxXbo5YY//S+OryeJioe17nK+iDZpg= +github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e/go.mod h1:o129ljs6alsIQTc8d6eweihqpmmrbxZ2g1jhgjhPykI= +github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242 h1:rh6rt8pF5U4iyQ86h6lRDenJoX4ht2wFnZXB9ogIrIM= +github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242/go.mod h1:LGHUtlhsY4vRGM6AHejEQKVI5e3eHbSylMHwTSpQtVw= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= +github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/earthboundkid/versioninfo/v2 v2.24.1 h1:SJTMHaoUx3GzjjnUO1QzP3ZXK6Ee/nbWyCm58eY3oUg= +github.com/earthboundkid/versioninfo/v2 v2.24.1/go.mod h1:VcWEooDEuyUJnMfbdTh0uFN4cfEIg+kHMuWB2CDCLjw= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/getsentry/sentry-go v0.31.1 h1:ELVc0h7gwyhnXHDouXkhqTFSO5oslsRDk0++eyE0KJ4= +github.com/getsentry/sentry-go v0.31.1/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-chi/chi/v5 v5.2.0 h1:Aj1EtB0qR2Rdo2dG4O94RIU35w2lvQSj6BRA4+qwFL0= +github.com/go-chi/chi/v5 v5.2.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= +github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-enry/go-enry/v2 v2.9.6 h1:np63eOtMV56zfYDHnFVgpEVOk8fr2kmylcMnAZUDbSs= +github.com/go-enry/go-enry/v2 v2.9.6/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= +github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= +github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git-fixtures/v5 v5.0.0-20241203230421-0753e18f8f03 h1:LumE+tQdnYW24a9RoO08w64LHTzkNkdUqBD/0QPtlEY= +github.com/go-git/go-git-fixtures/v5 v5.0.0-20241203230421-0753e18f8f03/go.mod h1:hMKrMnUE4W0SJ7bFyM00dyz/HoknZoptGWzrj6M+dEM= +github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= +github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= +github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= +github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.28.0 h1:KjSWstCpz/MN5t4a8gnGJNIYUsJRpdi/r97xWDphIQc= +github.com/google/cel-go v0.28.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= +github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/grafana/regexp v0.0.0-20240607082908-2cb410fa05da h1:BML5sNe+bw2uO8t8cQSwe5QhvoP04eHPF7bnaQma0Kw= +github.com/grafana/regexp v0.0.0-20240607082908-2cb410fa05da/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= +github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= +github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= +github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hexops/autogold v0.8.1/go.mod h1:97HLDXyG23akzAoRYJh/2OBs3kd80eHyKPvZw0S5ZBY= +github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo= +github.com/hexops/autogold v1.3.1/go.mod h1:sQO+mQUCVfxOKPht+ipDSkJ2SCJ7BNJVHZexsXqWMx4= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/hexops/valast v1.4.3 h1:oBoGERMJh6UZdRc6cduE1CTPK+VAdXA59Y1HFgu3sm0= +github.com/hexops/valast v1.4.3/go.mod h1:Iqx2kLj3Jn47wuXpj3wX40xn6F93QNFBHuiKBerkTGA= +github.com/hiddeco/sshsig v0.2.0 h1:gMWllgKCITXdydVkDL+Zro0PU96QI55LwUwebSwNTSw= +github.com/hiddeco/sshsig v0.2.0/go.mod h1:nJc98aGgiH6Yql2doqH4CTBVHexQA40Q+hMMLHP4EqE= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= +github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/boxo v0.36.0 h1:DarrMBM46xCs6GU6Vz+AL8VUyXykqHAqZYx8mR0Oics= +github.com/ipfs/boxo v0.36.0/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= +github.com/ipfs/go-bitswap v0.11.0/go.mod h1:05aE8H3XOU+LXpTedeAS0OZpcO1WFsj5niYQH9a1Tmk= +github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= +github.com/ipfs/go-block-format v0.2.3/go.mod h1:WJaQmPAKhD3LspLixqlqNFxiZ3BZ3xgqxxoSR/76pnA= +github.com/ipfs/go-blockservice v0.5.2 h1:in9Bc+QcXwd1apOVM7Un9t8tixPKdaHQFdLSUM1Xgk8= +github.com/ipfs/go-blockservice v0.5.2/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= +github.com/ipfs/go-cid v0.6.2 h1:VuGwJd+KJTaMJ4S4d5EEf9SXc17YUblS5axCbocn9YE= +github.com/ipfs/go-cid v0.6.2/go.mod h1:Xhwg8NzHeK9xPCEZkCw4idzPiuNMpX3fARuI5Iwj1Lo= +github.com/ipfs/go-datastore v0.9.0 h1:WocriPOayqalEsueHv6SdD4nPVl4rYMfYGLD4bqCZ+w= +github.com/ipfs/go-datastore v0.9.0/go.mod h1:uT77w/XEGrvJWwHgdrMr8bqCN6ZTW9gzmi+3uK+ouHg= +github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= +github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= +github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= +github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= +github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= +github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= +github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= +github.com/ipfs/go-ipfs-exchange-interface v0.2.1 h1:jMzo2VhLKSHbVe+mHNzYgs95n0+t0Q69GQ5WhRDZV/s= +github.com/ipfs/go-ipfs-exchange-interface v0.2.1/go.mod h1:MUsYn6rKbG6CTtsDp+lKJPmVt3ZrCViNyH3rfPGsZ2E= +github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= +github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s= +github.com/ipfs/go-ipfs-pq v0.0.4 h1:U7jjENWJd1jhcrR8X/xHTaph14PTAK9O+yaLJbjqgOw= +github.com/ipfs/go-ipfs-pq v0.0.4/go.mod h1:9UdLOIIb99IFrgT0Fc53pvbvlJBhpUb4GJuAQf3+O2A= +github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= +github.com/ipfs/go-ipfs-routing v0.3.0/go.mod h1:dKqtTFIql7e1zYsEuWLyuOU+E0WJWW8JjbTPLParDWo= +github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0= +github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs= +github.com/ipfs/go-ipld-cbor v0.2.1 h1:H05yEJbK/hxg0uf2AJhyerBDbjOuHX4yi+1U/ogRa7E= +github.com/ipfs/go-ipld-cbor v0.2.1/go.mod h1:x9Zbeq8CoE5R2WicYgBMcr/9mnkQ0lHddYWJP2sMV3A= +github.com/ipfs/go-ipld-format v0.6.3 h1:9/lurLDTotJpZSuL++gh3sTdmcFhVkCwsgx2+rAh4j8= +github.com/ipfs/go-ipld-format v0.6.3/go.mod h1:74ilVN12NXVMIV+SrBAyC05UJRk0jVvGqdmrcYZvCBk= +github.com/ipfs/go-ipld-legacy v0.2.2 h1:DThbqCPVLpWBcGtU23KDLiY2YRZZnTkXQyfz8aOfBkQ= +github.com/ipfs/go-ipld-legacy v0.2.2/go.mod h1:hhkj+b3kG9b2BcUNw8IFYAsfeNo8E3U7eYlWeAOPyDU= +github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= +github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= +github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= +github.com/ipfs/go-log/v2 v2.9.1 h1:3JXwHWU31dsCpvQ+7asz6/QsFJHqFr4gLgQ0FWteujk= +github.com/ipfs/go-log/v2 v2.9.1/go.mod h1:evFx7sBiohUN3AG12mXlZBw5hacBQld3ZPHrowlJYoo= +github.com/ipfs/go-merkledag v0.11.0 h1:DgzwK5hprESOzS4O1t/wi6JDpyVQdvm9Bs59N/jqfBY= +github.com/ipfs/go-merkledag v0.11.0/go.mod h1:Q4f/1ezvBiJV0YCIXvt51W/9/kqJGH4I1LsA7+djsM4= +github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6q/JR9V40TU= +github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY= +github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3eHfHM8w= +github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA= +github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs= +github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw= +github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= +github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= +github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= +github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= +github.com/ipld/go-ipld-prime v0.24.0 h1:6th8Z6Peh5bCWuRAVZcDO1sHzZdVF6F2cCCDG3681tg= +github.com/ipld/go-ipld-prime v0.24.0/go.mod h1:DYZxr/5caLNFbcuU6zLOgwSW7CgUEoC4wJiZMEU8Zhs= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= +github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM= +github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= +github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-libp2p v0.47.0 h1:qQpBjSCWNQFF0hjBbKirMXE9RHLtSuzTDkTfr1rw0yc= +github.com/libp2p/go-libp2p v0.47.0/go.mod h1:s8HPh7mMV933OtXzONaGFseCg/BE//m1V34p3x4EUOY= +github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= +github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= +github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= +github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= +github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= +github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= +github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= +github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= +github.com/libp2p/go-netroute v0.4.0 h1:sZZx9hyANYUx9PZyqcgE/E1GUG3iEtTZHUEvdtXT7/Q= +github.com/libp2p/go-netroute v0.4.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.16 h1:+2YjeAe7xP1OQbZW1jERPWWCDLlNqFNCvjiqGiAt84A= +github.com/mattn/go-colorable v0.1.16/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk= +github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= +github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= +github.com/mdlayher/socket v0.6.0 h1:ScZPaAGyO1icQnbFrhPM8mnXyMu9qukC1K4ZoM2IQKU= +github.com/mdlayher/socket v0.6.0/go.mod h1:q7vozUAnxSqnjHc12Fik5yUKIzfZ8ITCfMkhOtE9z18= +github.com/mdlayher/vsock v1.3.0 h1:bqQfZ1OznI03y6YiXp2sze05RVdzLn/zsfjnjd4+ivI= +github.com/mdlayher/vsock v1.3.0/go.mod h1:WsuksavOvwCnV5UqGHUkvAvCy+Dqy81y4goKQTzxxNY= +github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= +github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI= +github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.16.1 h1:fgJ0Pitow+wWXzN9do+1b8Pyjmo8m5WhGfzpL82MpCw= +github.com/multiformats/go-multiaddr v0.16.1/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= +github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= +github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= +github.com/multiformats/go-multibase v0.3.0 h1:8helZD2+4Db7NNWFiktk2NePbF0boolBe6bDQvM4r68= +github.com/multiformats/go-multibase v0.3.0/go.mod h1:MoBLQPCkRTOL3eveIPO81860j2AQY8JwcnNlRkGRUfI= +github.com/multiformats/go-multicodec v0.10.0 h1:UpP223cig/Cx8J76jWt91njpK3GTAO1w02sdcjZDSuc= +github.com/multiformats/go-multicodec v0.10.0/go.mod h1:wg88pM+s2kZJEQfRCKBNU+g32F5aWBEjyFHXvZLTcLI= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-multistream v0.6.1 h1:4aoX5v6T+yWmc2raBHsTvzmFhOI8WVOer28DeBBEYdQ= +github.com/multiformats/go-multistream v0.6.1/go.mod h1:ksQf6kqHAb6zIsyw7Zm+gAuVo57Qbq84E27YlYqavqw= +github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOoETFs5dI= +github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJhSkUlGQV9wgObdI= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA= +github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= +github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/openbao/openbao/api/v2 v2.3.0 h1:61FO3ILtpKoxbD9kTWeGaCq8pz1sdt4dv2cmTXsiaAc= +github.com/openbao/openbao/api/v2 v2.3.0/go.mod h1:T47WKHb7DqHa3Ms3xicQtl5EiPE+U8diKjb9888okWs= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= +github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= +github.com/oppiliappan/go-git/v5 v5.17.0 h1:CuJnpcIDxr0oiNaSHMconovSWnowHznVDG+AhjGuSEo= +github.com/oppiliappan/go-git/v5 v5.17.0/go.mod h1:q/FE8C3SPMoRN7LoH9vRFiBzidAOBWJPS1CqVS8DN+w= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/polydawn/refmt v0.90.0 h1:58BfEsP+G4uIRD9ApJTFsag+Mw+QQlZuH9uI/lPmjfY= +github.com/polydawn/refmt v0.90.0/go.mod h1:XAlDMOunevTYDsZtOKQd8itHXFMsX/QtDkPHaj6ZLxk= +github.com/posthog/posthog-go v1.5.5 h1:2o3j7IrHbTIfxRtj4MPaXKeimuTYg49onNzNBZbwksM= +github.com/posthog/posthog-go v1.5.5/go.mod h1:3RqUmSnPuwmeVj/GYrS75wNGqcAKdpODiwc83xZWgdE= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= +github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= +github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= +github.com/resend/resend-go/v3 v3.5.0 h1:yScYxHinY352Mj7Cn9rbWsR2gDqD2mtFPWwh2UyFMeE= +github.com/resend/resend-go/v3 v3.5.0/go.mod h1:iI7VA0NoGjWvsNii5iNC5Dy0llsI3HncXPejhniYzwE= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sethvargo/go-envconfig v1.1.0 h1:cWZiJxeTm7AlCvzGXrEXaSTCNgip5oJepekh/BOQuog= +github.com/sethvargo/go-envconfig v1.1.0/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/sourcegraph/go-ctags v0.0.0-20250729094530-349a251d78d8 h1:hpCD/FvbXLR7/034fKD0CQ8LmT4zoQfT2DzJIjqMzUI= +github.com/sourcegraph/go-ctags v0.0.0-20250729094530-349a251d78d8/go.mod h1:Or1cqbhDzkbH+hlwv5iW7uCTPEMKH9u/mTUh7otRQHY= +github.com/sourcegraph/log v0.0.0-20241024013702-574f7079c888 h1:9PUH8Hn8mVhPTtRKqot1HHsbLRDP0H2A+FSyuRumP2Q= +github.com/sourcegraph/log v0.0.0-20241024013702-574f7079c888/go.mod h1:IDp09QkoqS8Z3CyN2RW6vXjgABkNpDbyjLIHNQwQ8P8= +github.com/sourcegraph/zoekt v0.0.0-20260605122240-893a523804f1 h1:jdGYyvAjA5/oJEf8fsnUkzLcsJkiSR8+qK0zRGfSwO8= +github.com/sourcegraph/zoekt v0.0.0-20260605122240-893a523804f1/go.mod h1:Q5unQupAiJ4Gl7lVzDGaWmoF4yELtsyM2xzE7B18V34= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c h1:km8GpoQut05eY3GiYWEedbTT0qnSxrCjsVbb7yKY1KE= +github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c/go.mod h1:cNQ3dwVJtS5Hmnjxy6AgTPd0Inb3pW05ftPSX7NZO7Q= +github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ= +github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= +github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= +github.com/urfave/cli/v3 v3.14.0 h1:a8414NQlHJs0c/iBsulKLzlES0n/lEAskbL2LKpU4/s= +github.com/urfave/cli/v3 v3.14.0/go.mod h1:vXn6HxPNccJSzQr2QvwVncOKrgYGIHU0HY5h8B2nQj4= +github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI= +github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q= +github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= +github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= +github.com/wasilibs/go-re2 v1.10.0 h1:vQZEBYZOCA9jdBMmrO4+CvqyCj0x4OomXTJ4a5/urQ0= +github.com/wasilibs/go-re2 v1.10.0/go.mod h1:k+5XqO2bCJS+QpGOnqugyfwC04nw0jaglmjrrkG8U6o= +github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4= +github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY= +github.com/whyrusleeping/cbor-gen v0.3.1 h1:82ioxmhEYut7LBVGhGq8xoRkXPLElVuh5mV67AFfdv0= +github.com/whyrusleeping/cbor-gen v0.3.1/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= +github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA= +gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8= +gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q= +gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02/go.mod h1:JTnUj0mpYiAsuZLmKjTx/ex3AtMowcCgnE7YNyCEP0I= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/bridges/otelslog v0.18.0 h1:hhPGP3zvvy1xWT9RTy970wlniSxFttBIsAK1gvMguJM= +go.opentelemetry.io/contrib/bridges/otelslog v0.18.0/go.mod h1:twJF7inoMza6kxMcF8JOdL3mPmtOZu7GEr34CUNE6Dg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0 h1:HIBTQ3VO5aupLKjC90JgMqpezVXwFuq6Ryjn0/izoag= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0/go.mod h1:ji9vId85hMxqfvICA0Jt8JqEdrXaAkcpkI9HPXya0ro= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8= +go.opentelemetry.io/otel/log v0.19.0 h1:KUZs/GOsw79TBBMfDWsXS+KZ4g2Ckzksd1ymzsIEbo4= +go.opentelemetry.io/otel/log v0.19.0/go.mod h1:5DQYeGmxVIr4n0/BcJvF4upsraHjg6vudJJpnkL6Ipk= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/log v0.19.0 h1:scYVLqT22D2gqXItnWiocLUKGH9yvkkeql5dBDiXyko= +go.opentelemetry.io/otel/sdk/log v0.19.0/go.mod h1:vFBowwXGLlW9AvpuF7bMgnNI95LiW10szrOdvzBHlAg= +go.opentelemetry.io/otel/sdk/log/logtest v0.19.0 h1:BEbF7ZBB6qQloV/Ub1+3NQoOUnVtcGkU3XX4Ws3GQfk= +go.opentelemetry.io/otel/sdk/log/logtest v0.19.0/go.mod h1:Lua81/3yM0wOmoHTokLj9y9ADeA02v1naRrVrkAZuKk= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA= +golang.org/x/image v0.31.0/go.mod h1:R9ec5Lcp96v9FTF+ajwaH3uGxPH4fKfHHAVbUILxghA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 h1:utmqiUzZAEINbLmRyAt1QjVkMyLQ+lvIwCpLaB1rZfw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= +gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= +gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo= +gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= +lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= +mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM= +mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= +tangled.org/ptr.pet/hydrant v0.0.0-20261021010439-0128352f6b73 h1:BVvi19dkqfGnFZWuazeneWn9IXOWzb1NDYsXDNxvNQo= +tangled.org/ptr.pet/hydrant v0.0.0-20261021010439-0128352f6b73/go.mod h1:tcoB8X51Kkd7d+88iJ9vc7PvZTiPdGIdcXmINYXGMPs= +tangled.org/ptr.pet/indigo v0.0.0-20261021233448-bf24d47f0126 h1:DGNXAhkG+VOkfe28KJfGxuzaeGaFDXBEIl2jArBwwMs= +tangled.org/ptr.pet/indigo v0.0.0-20261021233448-bf24d47f0126/go.mod h1:2zj5D2cvh2bvhIFM5P0zmdy03q0vZjYO9I5xaXO2RwQ= diff --git a/hostutil/hostutil.go b/hostutil/hostutil.go new file mode 100644 index 0000000000000000000000000000000000000000..2038a06b3c53bb7e712e4d1734e99f6f55ed24f7 --- /dev/null +++ b/hostutil/hostutil.go @@ -0,0 +1,80 @@ +package hostutil + +import ( + "fmt" + "net/url" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +func ParseHostname(raw string) (hostname string, noSSL bool, err error) { + // handle case of bare hostname + if !strings.Contains(raw, "://") { + if strings.HasPrefix(raw, "localhost:") { + raw = "http://" + raw + } else { + raw = "https://" + raw + } + } + + u, err := url.Parse(raw) + if err != nil { + return "", false, fmt.Errorf("not a valid host URL: %w", err) + } + + switch u.Scheme { + case "https", "wss": + noSSL = false + case "http", "ws": + noSSL = true + default: + return "", false, fmt.Errorf("unsupported URL scheme: %s", u.Scheme) + } + + // 'localhost' (exact string) is allowed *with* a required port number; SSL is optional + if u.Hostname() == "localhost" { + if u.Port() == "" || !strings.HasPrefix(u.Host, "localhost:") { + return "", false, fmt.Errorf("port number is required for localhost") + } + return u.Host, noSSL, nil + } + + // port numbers not allowed otherwise + if u.Port() != "" { + return "", false, fmt.Errorf("port number not allowed for non-local names") + } + + // check it is a real hostname (eg, not IP address or single-word alias) + h, err := syntax.ParseHandle(u.Host) + if err != nil { + return "", false, fmt.Errorf("not a public hostname") + } + + // lower-case in response + return h.Normalize().String(), noSSL, nil +} + +func EnsureHttpScheme(host string) (string, error) { + hostname, noSSL, err := ParseHostname(host) + if err != nil { + return "", err + } + if noSSL { + return "http://" + hostname, nil + } else { + return "https://" + hostname, nil + } +} + +func EnsureWsScheme(host string) (string, error) { + hostname, noSSL, err := ParseHostname(host) + if err != nil { + return "", err + } + if noSSL { + return "ws://" + hostname, nil + } else { + return "wss://" + hostname, nil + } +} diff --git a/hostutil/safedial.go b/hostutil/safedial.go new file mode 100644 index 0000000000000000000000000000000000000000..bd5d625976a5755d63609891dfc910a0d1d33ed4 --- /dev/null +++ b/hostutil/safedial.go @@ -0,0 +1,76 @@ +package hostutil + +import ( + "fmt" + "net" + "net/http" + "net/url" + "syscall" + + "github.com/bluesky-social/indigo/util/ssrf" + "time" +) + +// safeDialer rejects dials to non-public addresses. the Control hook runs before +// connect, so it also covers rebinding and redirects, and inherits indigo's +// maintained blocklist (6to4, nat64 and the rest). disabled in dev. +func safeDialer(dev bool) *net.Dialer { + d := &net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + } + if dev { + return d + } + d.Control = func(_, address string, _ syscall.RawConn) error { + host, _, err := net.SplitHostPort(address) + if err != nil { + return fmt.Errorf("invalid dial address %q: %w", address, err) + } + ip := net.ParseIP(host) + if ip == nil { + return fmt.Errorf("dial address %q did not resolve to an IP", address) + } + if !ssrf.IsPublicIPAddress(ip) { + return fmt.Errorf("refusing to dial %s: reserved or private address", ip) + } + return nil + } + return d +} + +// ValidateExternalURL checks raw is a well-formed http(s) url and rejects +// ip-literal hosts in blocked ranges; dns hosts are re-checked at dial time. +func ValidateExternalURL(raw string, dev bool) error { + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("invalid URL: %w", err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("URL must use http or https") + } + if u.Hostname() == "" { + return fmt.Errorf("URL must include a host") + } + if dev { + return nil + } + if ip := net.ParseIP(u.Hostname()); ip != nil && !ssrf.IsPublicIPAddress(ip) { + return fmt.Errorf("URL host is a reserved or private address") + } + return nil +} + +// SafeClient returns an http.Client for fetching untrusted urls (e.g. +// webhooks): it blocks internal address ranges and won't follow redirects. +func SafeClient(dev bool, timeout time.Duration) *http.Client { + return &http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + DialContext: safeDialer(dev).DialContext, + }, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} diff --git a/ico/ico.go b/ico/ico.go new file mode 100644 index 0000000000000000000000000000000000000000..431387628b09fd3b8d9ccc06c2274e0f17162906 --- /dev/null +++ b/ico/ico.go @@ -0,0 +1,88 @@ +package ico + +import ( + "bytes" + "encoding/binary" + "fmt" + "image" + "image/png" +) + +type IconDir struct { + Reserved uint16 // must be 0 + Type uint16 // 1 for ICO, 2 for CUR + Count uint16 // number of images +} + +type IconDirEntry struct { + Width uint8 // 0 means 256 + Height uint8 // 0 means 256 + ColorCount uint8 + Reserved uint8 // must be 0 + ColorPlanes uint16 // 0 or 1 + BitsPerPixel uint16 + SizeInBytes uint32 + Offset uint32 +} + +func ImageToIco(img image.Image) ([]byte, error) { + // encode image as png + var pngBuf bytes.Buffer + if err := png.Encode(&pngBuf, img); err != nil { + return nil, fmt.Errorf("failed to encode PNG: %w", err) + } + pngData := pngBuf.Bytes() + + // get image dimensions + bounds := img.Bounds() + width := bounds.Dx() + height := bounds.Dy() + + // prepare output buffer + var icoBuf bytes.Buffer + + iconDir := IconDir{ + Reserved: 0, + Type: 1, // ICO format + Count: 1, // One image + } + + w := uint8(width) + h := uint8(height) + + // width/height of 256 should be stored as 0 + if width == 256 { + w = 0 + } + if height == 256 { + h = 0 + } + + iconDirEntry := IconDirEntry{ + Width: w, + Height: h, + ColorCount: 0, // 0 for PNG (32-bit) + Reserved: 0, + ColorPlanes: 1, + BitsPerPixel: 32, // PNG with alpha + SizeInBytes: uint32(len(pngData)), + Offset: 6 + 16, // Size of ICONDIR + ICONDIRENTRY + } + + // write IconDir + if err := binary.Write(&icoBuf, binary.LittleEndian, iconDir); err != nil { + return nil, fmt.Errorf("failed to write ICONDIR: %w", err) + } + + // write IconDirEntry + if err := binary.Write(&icoBuf, binary.LittleEndian, iconDirEntry); err != nil { + return nil, fmt.Errorf("failed to write ICONDIRENTRY: %w", err) + } + + // write PNG data directly + if _, err := icoBuf.Write(pngData); err != nil { + return nil, fmt.Errorf("failed to write PNG data: %w", err) + } + + return icoBuf.Bytes(), nil +} diff --git a/icons/bluesky.svg b/icons/bluesky.svg new file mode 100644 index 0000000000000000000000000000000000000000..856495830b772488ec7687873c1d0ef52be7bc9c --- /dev/null +++ b/icons/bluesky.svg @@ -0,0 +1,4 @@ + + Bluesky + + diff --git a/icons/user-round-arrow-forward.svg b/icons/user-round-arrow-forward.svg new file mode 100644 index 0000000000000000000000000000000000000000..4d117f9c5f648427c010987d982af020ec94a6b8 --- /dev/null +++ b/icons/user-round-arrow-forward.svg @@ -0,0 +1,17 @@ + + + + + + + diff --git a/idresolver/mock.go b/idresolver/mock.go new file mode 100644 index 0000000000000000000000000000000000000000..714456822e2dbeaff0ab969d72d0bc47b7793b02 --- /dev/null +++ b/idresolver/mock.go @@ -0,0 +1,35 @@ +package idresolver + +import ( + "context" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +func NewMockResolver(dir identity.Directory) *Resolver { + return &Resolver{ + directory: dir, + base: &identity.BaseDirectory{}, + } +} + +type MockDirectory struct { + Ident *identity.Identity +} + +func (m MockDirectory) LookupDID(context.Context, syntax.DID) (*identity.Identity, error) { + return m.Ident, nil +} + +func (m MockDirectory) LookupHandle(context.Context, syntax.Handle) (*identity.Identity, error) { + return m.Ident, nil +} + +func (m MockDirectory) Lookup(context.Context, syntax.AtIdentifier) (*identity.Identity, error) { + return m.Ident, nil +} + +func (m MockDirectory) Purge(context.Context, syntax.AtIdentifier) error { + return nil +} diff --git a/idresolver/resolver.go b/idresolver/resolver.go new file mode 100644 index 0000000000000000000000000000000000000000..ee764dc98914e41323d460c91028e1d446ecf3d7 --- /dev/null +++ b/idresolver/resolver.go @@ -0,0 +1,167 @@ +package idresolver + +import ( + "context" + "fmt" + "net" + "net/http" + "slices" + "strings" + "sync" + "time" + + "os" + + "tangled.org/core/netutil" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/identity/redisdir" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/carlmjohnson/versioninfo" +) + +type Resolver struct { + directory identity.Directory + base *identity.BaseDirectory +} + +func BaseDirectory(plcUrl string) *identity.BaseDirectory { + base := identity.BaseDirectory{ + PLCURL: plcUrl, + HTTPClient: http.Client{ + Timeout: time.Second * 10, + Transport: &http.Transport{ + // did documents are fetched for caller-supplied dids, so the transport refuses + // non-public addresses; IDRESOLVER_DEV=1 lifts the guard for local plc instances + DialContext: netutil.SSRFDialer(os.Getenv("IDRESOLVER_DEV") == "1").DialContext, + // would want this around 100ms for services doing lots of handle resolution. Impacts PLC connections as well, but not too bad. + IdleConnTimeout: time.Millisecond * 1000, + MaxIdleConns: 100, + }, + }, + Resolver: net.Resolver{ + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{Timeout: time.Second * 3} + return d.DialContext(ctx, network, address) + }, + }, + TryAuthoritativeDNS: true, + // primary Bluesky PDS instance only supports HTTP resolution method + SkipDNSDomainSuffixes: []string{".bsky.social"}, + UserAgent: "indigo-identity/" + versioninfo.Short(), + } + return &base +} + +func DefaultResolver(plcUrl string) *Resolver { + base := BaseDirectory(plcUrl) + cached := identity.NewCacheDirectory(base, 250_000, time.Hour*24, time.Minute*2, time.Minute*5) + return &Resolver{ + directory: cached, + base: base, + } +} + +func RedisDirectory(base *identity.BaseDirectory, url string) (identity.Directory, error) { + hitTTL := time.Hour * 24 + errTTL := time.Second * 30 + invalidHandleTTL := time.Minute * 5 + return redisdir.NewRedisDirectory(base, url, hitTTL, errTTL, invalidHandleTTL, 10000) +} + +func RedisResolver(redisUrl, plcUrl string) (*Resolver, error) { + base := BaseDirectory(plcUrl) + directory, err := RedisDirectory(base, redisUrl) + if err != nil { + return nil, err + } + return &Resolver{ + directory: directory, + base: base, + }, nil +} + +type handleResolver interface { + ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) +} + +func (r *Resolver) ResolveHandle(ctx context.Context, handle syntax.Handle) (syntax.DID, error) { + if hr, ok := r.directory.(handleResolver); ok { + return hr.ResolveHandle(ctx, handle) + } + return r.base.ResolveHandle(ctx, handle) +} + +func (r *Resolver) ResolveAtIdentifier(ctx context.Context, input string) (*identity.Identity, error) { + if did, err := syntax.ParseDID(input); err == nil { + return r.directory.LookupDID(ctx, did) + } + handle, err := syntax.ParseHandle(input) + if err != nil { + return nil, fmt.Errorf("not a did or handle: %w", err) + } + handle = handle.Normalize() + did, err := r.ResolveHandle(ctx, handle) + if err != nil { + return nil, fmt.Errorf("resolve handle %q: %w", handle, err) + } + ident, err := r.directory.LookupDID(ctx, did) + if err != nil { + return nil, fmt.Errorf("lookup did for %q: %w", handle, err) + } + aka := "at://" + handle.String() + if !slices.ContainsFunc(ident.AlsoKnownAs, func(s string) bool { return strings.EqualFold(s, aka) }) { + return nil, fmt.Errorf("handle %q not declared in alsoKnownAs for %s", handle, did) + } + return ident, nil +} + +func (r *Resolver) ResolveIdent(ctx context.Context, arg string) (*identity.Identity, error) { + id, err := syntax.ParseAtIdentifier(arg) + if err != nil { + return nil, err + } + + return r.directory.Lookup(ctx, id) +} + +func (r *Resolver) ResolveIdents(ctx context.Context, idents []string) []*identity.Identity { + results := make([]*identity.Identity, len(idents)) + var wg sync.WaitGroup + + done := make(chan struct{}) + defer close(done) + + for idx, ident := range idents { + wg.Add(1) + go func(index int, id string) { + defer wg.Done() + + select { + case <-ctx.Done(): + results[index] = nil + case <-done: + results[index] = nil + default: + identity, _ := r.ResolveIdent(ctx, id) + results[index] = identity + } + }(idx, ident) + } + + wg.Wait() + return results +} + +func (r *Resolver) InvalidateIdent(ctx context.Context, arg string) error { + id, err := syntax.ParseAtIdentifier(arg) + if err != nil { + return err + } + + return r.directory.Purge(ctx, id) +} + +func (r *Resolver) Directory() identity.Directory { + return r.directory +} diff --git a/input.css b/input.css new file mode 100644 index 0000000000000000000000000000000000000000..3f6004b74652ed79927f56a0974211e1d0d26cc3 --- /dev/null +++ b/input.css @@ -0,0 +1,1699 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; +@layer base { + @font-face { + font-family: "InterVariable"; + src: url("/static/fonts/InterVariable.woff2") format("woff2-variations"); + font-weight: 100 600; + font-style: normal; + font-display: swap; + } + + @font-face { + font-family: "InterVariable"; + src: url("/static/fonts/InterVariable-Italic.woff2") + format("woff2-variations"); + font-weight: 100 600; + font-style: italic; + font-display: swap; + } + + @font-face { + font-family: "InterVariable"; + src: url("/static/fonts/InterDisplay-Bold.woff2") format("woff2"); + font-weight: bold; + font-style: normal; + font-display: swap; + } + + @font-face { + font-family: "InterVariable"; + src: url("/static/fonts/InterDisplay-BoldItalic.woff2") format("woff2"); + font-weight: bold; + font-style: italic; + font-display: swap; + } + + @font-face { + font-family: "IBMPlexMono"; + src: url("/static/fonts/IBMPlexMono-Regular.woff2") format("woff2"); + font-weight: normal; + font-style: normal; + font-display: swap; + } + + @font-face { + font-family: "IBMPlexMono"; + src: url("/static/fonts/IBMPlexMono-Italic.woff2") format("woff2"); + font-weight: normal; + font-style: italic; + font-display: swap; + } + + @font-face { + font-family: "IBMPlexMono"; + src: url("/static/fonts/IBMPlexMono-Bold.woff2") format("woff2"); + font-weight: bold; + font-style: normal; + font-display: swap; + } + + @font-face { + font-family: "IBMPlexMono"; + src: url("/static/fonts/IBMPlexMono-BoldItalic.woff2") format("woff2"); + font-weight: bold; + font-style: italic; + font-display: swap; + } + + ::selection { + @apply bg-yellow-400 text-black bg-opacity-30 dark:bg-yellow-600 dark:bg-opacity-50 dark:text-white; + } + + @layer base { + :root { + font-size: 14px; + scrollbar-gutter: stable; + + color-scheme: light; + } + + :root[data-theme="dark"] { + color-scheme: dark; + } + + @supports (font-variation-settings: normal) { + html { + font-feature-settings: + "kern" 1, + "liga" 1, + "cv05" 1; + } + } + + a { + @apply no-underline text-black hover:underline hover:text-gray-800 dark:text-white dark:hover:text-gray-300; + } + + label { + @apply block text-gray-900 text-sm py-2 dark:text-gray-100; + } + + input, + textarea { + @apply block rounded p-3 + bg-white dark:bg-gray-800 dark:text-white + border border-gray-200 dark:border-gray-600 + focus:outline-none focus:ring-1 focus:ring-gray-400 dark:focus:ring-gray-500; + } + + input[type="checkbox"] { + @apply appearance-none size-4 p-0 rounded + bg-transparent border border-gray-200 + hover:bg-gray-100 + checked:bg-gray-900 checked:border-transparent + checked:hover:bg-gray-800 + indeterminate:bg-gray-900 indeterminate:border-transparent + disabled:cursor-not-allowed disabled:bg-gray-100 disabled:border-gray-200 + disabled:checked:bg-gray-300 disabled:checked:border-transparent + disabled:indeterminate:bg-gray-100 disabled:indeterminate:border-gray-200 + dark:border-gray-600 + dark:hover:bg-gray-700 + dark:checked:bg-gray-100 + dark:checked:hover:bg-gray-200 + dark:checked:border-gray-400 + dark:indeterminate:bg-gray-100 dark:indeterminate:border-transparent + dark:disabled:bg-gray-700 dark:disabled:border-gray-600 + dark:disabled:checked:bg-gray-500 dark:disabled:checked:border-transparent + dark:disabled:indeterminate:bg-gray-700 dark:disabled:indeterminate:border-gray-600 + focus:outline-none focus:ring-1 focus:ring-gray-400 dark:focus:ring-gray-500; + background-position: center; + background-repeat: no-repeat; + background-size: 12px; + } + input[type="checkbox"]:checked { + background-image: url("data:image/svg+xml,"); + } + input[type="checkbox"]:indeterminate { + background-image: url("data:image/svg+xml,"); + } + input[type="checkbox"]:disabled:indeterminate { + background-image: url("data:image/svg+xml,"); + } + [data-theme="dark"] input[type="checkbox"]:checked { + background-image: url("data:image/svg+xml,"); + } + [data-theme="dark"] input[type="checkbox"]:indeterminate { + background-image: url("data:image/svg+xml,"); + } + [data-theme="dark"] input[type="checkbox"]:disabled:indeterminate { + background-image: url("data:image/svg+xml,"); + } + + input[type="radio"] { + @apply appearance-none size-4 p-0 rounded-full bg-transparent border + + border-gray-400 hover:border-gray-900 + hover:bg-gray-200 + + disabled:cursor-not-allowed + disabled:border-gray-300 + disabled:checked:border-gray-300 + disabled:hover:bg-inherit + + dark:border-gray-400 dark:hover:border-gray-100 + + dark:hover:bg-gray-600 + + dark:checked:border-gray-400 + + dark:disabled:border-gray-500 + dark:disabled:checked:border-gray-500 + + focus:outline-none focus:ring-1 + focus:ring-gray-400 dark:focus:ring-gray-500; + + background-position: center; + background-repeat: no-repeat; + background-size: 8px; + } + input[type="radio"]:checked { + background-image: url("data:image/svg+xml,"); + } + input[type="radio"]:disabled:checked { + background-image: url("data:image/svg+xml,"); + } + [data-theme="dark"] input[type="radio"]:checked { + background-image: url("data:image/svg+xml,"); + } + [data-theme="dark"] input[type="radio"]:disabled:checked { + background-image: url("data:image/svg+xml,"); + } + + select { + @apply flex items-center rounded p-2 h-[32px] + border border-gray-200 + bg-white + dark:text-white + dark:bg-gray-800 dark:border-gray-700 + ; + } + + /* iOS Safari zooms the viewport when focusing a form control + whose font-size is under 16px; with the 14px root font size + even text-lg (1.125rem) falls short. !important so this also + wins over text-* utilities applied in templates. */ + @media (max-width: 639px) { + input:not([type="checkbox"]):not([type="radio"]), + textarea, + select { + font-size: 16px !important; + } + } + + details summary::-webkit-details-marker { + display: none; + } + + code { + @apply font-mono rounded bg-gray-100 dark:bg-gray-700 text-black dark:text-white; + } + } + + @layer components { + .btn { + @apply relative z-0 inline-flex overflow-hidden items-center justify-center + min-h-[32px] px-2 py-[6px] gap-1.5 + rounded border border-gray-200 dark:border-gray-700 + bg-white dark:bg-gray-800 + text-sm text-gray-900 dark:text-gray-100 + transition-colors duration-150 ease-in-out + outline-transparent cursor-pointer + focus-visible:outline focus-visible:outline-2 focus-visible:outline-gray-400 dark:focus-visible:outline-gray-600 + no-underline hover:no-underline; + + @apply before:absolute before:inset-0 before:-z-10 before:block before:rounded-sm before:content-[''] + before:transition-all before:duration-150 before:ease-in-out + before:shadow-[inset_0_-2px_0_0_theme(colors.gray.200)] dark:before:shadow-[inset_0_-2px_0_0_theme(colors.gray.900)]; + + @apply hover:bg-gray-50 dark:hover:bg-gray-700 + hover:before:shadow-[inset_0_-2px_0_0_theme(colors.gray.300)] dark:hover:before:shadow-[inset_0_-2px_0_0_theme(colors.gray.800)] + active:before:shadow-[inset_0_2px_2px_0_theme(colors.gray.200)] dark:active:before:shadow-[inset_0_2px_2px_0_theme(colors.gray.900)] + active:translate-y-0.5 + disabled:cursor-not-allowed disabled:opacity-50; + + @apply disabled:hover:bg-white dark:disabled:hover:bg-gray-800 + disabled:hover:before:shadow-[inset_0_-2px_0_0_theme(colors.gray.200)] dark:disabled:hover:before:shadow-[inset_0_-2px_0_0_theme(colors.gray.900)] + disabled:active:before:shadow-[inset_0_-2px_0_0_theme(colors.gray.200)] dark:disabled:active:before:shadow-[inset_0_-2px_0_0_theme(colors.gray.900)] + disabled:active:translate-y-0; + } + + .btn-flat { + @apply inline-flex items-center justify-center + min-h-[32px] px-2 py-[6px] gap-1.5 + rounded + text-sm text-gray-900 dark:text-gray-100 cursor-pointer no-underline hover:no-underline + transition-colors duration-150 ease-in-out + focus:outline-none + focus-visible:outline focus-visible:outline-2 focus-visible:outline-gray-400 + hover:bg-gray-100 + dark:hover:text-gray-100 dark:hover:bg-gray-700 + disabled:cursor-not-allowed disabled:opacity-50 disabled:hover:text-inherit disabled:hover:bg-inherit; + } + + .btn-create { + @apply btn text-white + bg-green-600 hover:bg-green-700 dark:bg-green-700 dark:hover:bg-green-800 + border border-green-700 hover:border-green-800 + focus-visible:outline-green-500 + disabled:bg-green-400 dark:disabled:bg-green-600 + before:shadow-[inset_0_-2px_0_0_theme(colors.gray.900/25%)] + hover:before:shadow-[inset_0_-2px_0_0_theme(colors.gray.900/25%)] + active:before:shadow-[inset_0_2px_2px_0_theme(colors.gray.900/50%)]; + } + + .btn-create-flat { + @apply btn-create + before:shadow-none + hover:before:shadow-none + active:before:shadow-none; + } + + .btn-primary { + @apply btn text-white + bg-indigo-600 hover:bg-indigo-700 dark:bg-indigo-500 dark:hover:bg-indigo-600 + border border-indigo-700 hover:border-indigo-800 dark:border-indigo-600 dark:hover:border-indigo-700 + focus-visible:outline-indigo-500 + disabled:bg-indigo-400 dark:disabled:bg-indigo-400 + before:shadow-[inset_0_-2px_0_0_theme(colors.gray.900/25%)] + hover:before:shadow-[inset_0_-2px_0_0_theme(colors.gray.900/25%)] + active:before:shadow-[inset_0_2px_2px_0_theme(colors.gray.900/50%)]; + } + + .btn-cancel { + @apply btn text-white + bg-red-600 hover:bg-red-700 dark:bg-red-700 dark:hover:bg-red-800 + border border-red-700 hover:border-red-800 + focus-visible:outline-red-500 + disabled:bg-red-400 dark:disabled:bg-red-600; + } + + .btn-cancel-flat { + @apply btn-cancel + before:shadow-none + hover:before:shadow-none + active:before:shadow-none; + } + + .btn-group { + @apply inline-flex items-stretch + max-h-[32px] + rounded border border-gray-200 dark:border-gray-700 + divide-x divide-gray-200 dark:divide-gray-700 + overflow-clip; + } + + .btn-group-item { + @apply btn-flat min-h-[30px] gap-1.5 + bg-gray-50 dark:bg-gray-900 + rounded-none border-y-0 + no-underline hover:no-underline + hover:text-gray-600 dark:hover:text-gray-400; + } + + .btn-group-item.active { + @apply btn min-h-[30px] rounded-none border-y-0 + before:border-none before:rounded-none before:bg-inherit; + + &:first-child { + @apply border-none before:rounded-l-sm; + } + &:last-child { + @apply border-y-0 border-r-0 border-l before:rounded-r-sm; + } + } + + .prose { + overflow-wrap: anywhere; + } + + .prose td, + .prose th { + overflow-wrap: normal; + } + + .prose hr { + @apply my-2; + } + + .prose li:has(input) { + @apply list-none; + } + + .prose ul:has(input) { + @apply pl-2; + } + + .prose .heading .anchor { + @apply no-underline mx-2 opacity-0; + } + + .prose .heading:hover .anchor { + @apply opacity-70; + } + + .prose .heading .anchor:hover { + @apply opacity-70; + } + + .prose h1:target, + .prose h2:target, + .prose h3:target, + .prose h4:target, + .prose h5:target, + .prose h6:target { + @apply bg-yellow-200/30 dark:bg-yellow-600/30; + } + + .prose a.footnote-backref { + @apply no-underline; + } + + .prose a.mention { + @apply no-underline hover:underline font-bold; + } + + .prose li { + @apply my-0 py-0; + } + + .prose ul, + .prose ol { + @apply my-1 py-0; + } + + .prose img { + display: inline; + margin: 0; + vertical-align: middle; + } + + .prose input { + @apply inline-block my-0 mt-1 mx-1; + } + + /* Mermaid diagrams */ + .prose pre.mermaid { + @apply flex justify-center my-4 overflow-x-auto bg-transparent border-0; + } + + /* Base callout */ + details[data-callout] { + @apply border-l-4 pl-3 py-2 text-gray-800 dark:text-gray-200 my-4; + } + + details[data-callout] > summary { + @apply font-bold cursor-pointer mb-1; + } + + details[data-callout] > .callout-content { + @apply text-sm leading-snug; + } + + /* Note (blue) */ + details[data-callout="note" i] { + @apply border-blue-400 dark:border-blue-500; + } + details[data-callout="note" i] > summary { + @apply text-blue-700 dark:text-blue-400; + } + + /* Important (purple) */ + details[data-callout="important" i] { + @apply border-purple-400 dark:border-purple-500; + } + details[data-callout="important" i] > summary { + @apply text-purple-700 dark:text-purple-400; + } + + /* Warning (yellow) */ + details[data-callout="warning" i] { + @apply border-yellow-400 dark:border-yellow-500; + } + details[data-callout="warning" i] > summary { + @apply text-yellow-700 dark:text-yellow-400; + } + + /* Caution (red) */ + details[data-callout="caution" i] { + @apply border-red-400 dark:border-red-500; + } + details[data-callout="caution" i] > summary { + @apply text-red-700 dark:text-red-400; + } + + /* Tip (green) */ + details[data-callout="tip" i] { + @apply border-green-400 dark:border-green-500; + } + details[data-callout="tip" i] > summary { + @apply text-green-700 dark:text-green-400; + } + + /* Optional: hide the disclosure arrow like GitHub */ + details[data-callout] > summary::-webkit-details-marker { + display: none; + } + } + @layer utilities { + .error { + @apply py-1 text-red-400 dark:text-red-300; + } + .success { + @apply py-1 text-gray-900 dark:text-gray-100; + } + + @keyframes scroll { + 0% { + transform: translateX(0); + } + 100% { + transform: translateX(-50%); + } + } + + .animate-marquee { + animation: scroll 60s linear infinite; + } + + .animate-marquee:hover { + animation-play-state: paused; + } + + @media (prefers-reduced-motion: reduce) { + .animate-marquee { + animation: none; + transform: none; + } + } + + @keyframes progress { + from { + width: 0%; + } + to { + width: 100%; + } + } + + .animate-progress { + animation: progress 10s linear forwards; + } + + @keyframes fadeIn { + from { + opacity: 0; + } + to { + opacity: 1; + } + } + + @keyframes fadeOut { + from { + opacity: 1; + } + to { + opacity: 0; + } + } + + .animate-fadein { + animation: fadeIn 0.25s ease-in forwards; + } + + .animate-fadeout { + animation: fadeOut 0.25s ease-out forwards; + } + } +} + +/* Background */ +.bg { + color: #4c4f69; + background-color: #eff1f5; +} +/* PreWrapper */ +.chroma { + color: #4c4f69; +} +/* Error */ +.chroma .err { + color: #d20f39; +} +/* LineLink */ +.chroma .lnlinks { + outline: none; + text-decoration: none; + color: inherit; +} +/* LineTableTD */ +.chroma .lntd { + vertical-align: top; + padding: 0; + margin: 0; + border: 0; +} +/* LineTable */ +.chroma .lntable { + border-spacing: 0; + padding: 0; + margin: 0; + border: 0; +} +/* LineHighlight */ +.chroma .hl { + @apply bg-amber-400/30 dark:bg-amber-500/20; +} + +.line-quote-hl, +.line-range-hl, +.comment-hl { + @apply !bg-yellow-200/30 dark:!bg-yellow-700/30; +} + +.chunk-match-hl { + @apply rounded-sm !bg-yellow-300/70 dark:!bg-yellow-600/60; +} + +.comment-hl { + @apply rounded-md; +} + +:is(.line-quote-hl, .line-range-hl) > .min-w-\[3\.5rem\] { + @apply !bg-yellow-200/30 dark:!bg-yellow-700/30; +} + +:is(.line-quote-hl, .line-range-hl) > .min-w-\[3\.5rem\] a { + @apply !text-black dark:!text-white; +} + +/* LineNumbersTable */ +.chroma .lnt { + white-space: pre; + -webkit-user-select: none; + user-select: none; + margin-right: 0.4em; + padding: 0 0.4em 0 0.4em; + color: #8c8fa1; +} +/* LineNumbers */ +.chroma .ln { + white-space: pre; + -webkit-user-select: none; + user-select: none; + margin-right: 0.4em; + padding: 0 0.4em 0 0.4em; + color: #8c8fa1; +} +/* Line */ +.chroma .line { + display: flex; +} +/* Keyword */ +.chroma .k { + color: #8839ef; +} +/* KeywordConstant */ +.chroma .kc { + color: #fe640b; +} +/* KeywordDeclaration */ +.chroma .kd { + color: #d20f39; +} +/* KeywordNamespace */ +.chroma .kn { + color: #179299; +} +/* KeywordPseudo */ +.chroma .kp { + color: #8839ef; +} +/* KeywordReserved */ +.chroma .kr { + color: #8839ef; +} +/* KeywordType */ +.chroma .kt { + color: #d20f39; +} +/* NameAttribute */ +.chroma .na { + color: #1e66f5; +} +/* NameBuiltin */ +.chroma .nb { + color: #04a5e5; +} +/* NameBuiltinPseudo */ +.chroma .bp { + color: #04a5e5; +} +/* NameClass */ +.chroma .nc { + color: #df8e1d; +} +/* NameConstant */ +.chroma .no { + color: #df8e1d; +} +/* NameDecorator */ +.chroma .nd { + color: #1e66f5; + font-weight: bold; +} +/* NameEntity */ +.chroma .ni { + color: #179299; +} +/* NameException */ +.chroma .ne { + color: #fe640b; +} +/* NameFunction */ +.chroma .nf { + color: #1e66f5; +} +/* NameFunctionMagic */ +.chroma .fm { + color: #1e66f5; +} +/* NameLabel */ +.chroma .nl { + color: #04a5e5; +} +/* NameNamespace */ +.chroma .nn { + color: #fe640b; +} +/* NameProperty */ +.chroma .py { + color: #fe640b; +} +/* NameTag */ +.chroma .nt { + color: #8839ef; +} +/* NameVariable */ +.chroma .nv { + color: #dc8a78; +} +/* NameVariableClass */ +.chroma .vc { + color: #dc8a78; +} +/* NameVariableGlobal */ +.chroma .vg { + color: #dc8a78; +} +/* NameVariableInstance */ +.chroma .vi { + color: #dc8a78; +} +/* NameVariableMagic */ +.chroma .vm { + color: #dc8a78; +} +/* LiteralString */ +.chroma .s { + color: #40a02b; +} +/* LiteralStringAffix */ +.chroma .sa { + color: #d20f39; +} +/* LiteralStringBacktick */ +.chroma .sb { + color: #40a02b; +} +/* LiteralStringChar */ +.chroma .sc { + color: #40a02b; +} +/* LiteralStringDelimiter */ +.chroma .dl { + color: #1e66f5; +} +/* LiteralStringDoc */ +.chroma .sd { + color: #9ca0b0; +} +/* LiteralStringDouble */ +.chroma .s2 { + color: #40a02b; +} +/* LiteralStringEscape */ +.chroma .se { + color: #1e66f5; +} +/* LiteralStringHeredoc */ +.chroma .sh { + color: #9ca0b0; +} +/* LiteralStringInterpol */ +.chroma .si { + color: #40a02b; +} +/* LiteralStringOther */ +.chroma .sx { + color: #40a02b; +} +/* LiteralStringRegex */ +.chroma .sr { + color: #179299; +} +/* LiteralStringSingle */ +.chroma .s1 { + color: #40a02b; +} +/* LiteralStringSymbol */ +.chroma .ss { + color: #40a02b; +} +/* LiteralNumber */ +.chroma .m { + color: #fe640b; +} +/* LiteralNumberBin */ +.chroma .mb { + color: #fe640b; +} +/* LiteralNumberFloat */ +.chroma .mf { + color: #fe640b; +} +/* LiteralNumberHex */ +.chroma .mh { + color: #fe640b; +} +/* LiteralNumberInteger */ +.chroma .mi { + color: #fe640b; +} +/* LiteralNumberIntegerLong */ +.chroma .il { + color: #fe640b; +} +/* LiteralNumberOct */ +.chroma .mo { + color: #fe640b; +} +/* Operator */ +.chroma .o { + color: #04a5e5; + font-weight: bold; +} +/* OperatorWord */ +.chroma .ow { + color: #04a5e5; + font-weight: bold; +} +/* Comment */ +.chroma .c { + color: #9ca0b0; + font-style: italic; +} +/* CommentHashbang */ +.chroma .ch { + color: #9ca0b0; + font-style: italic; +} +/* CommentMultiline */ +.chroma .cm { + color: #9ca0b0; + font-style: italic; +} +/* CommentSingle */ +.chroma .c1 { + color: #9ca0b0; + font-style: italic; +} +/* CommentSpecial */ +.chroma .cs { + color: #9ca0b0; + font-style: italic; +} +/* CommentPreproc */ +.chroma .cp { + color: #9ca0b0; + font-style: italic; +} +/* CommentPreprocFile */ +.chroma .cpf { + color: #9ca0b0; + font-weight: bold; + font-style: italic; +} +/* GenericDeleted */ +.chroma .gd { + color: #d20f39; + background-color: oklch(93.6% 0.032 17.717); +} +/* GenericEmph */ +.chroma .ge { + font-style: italic; +} +/* GenericError */ +.chroma .gr { + color: #d20f39; +} +/* GenericHeading */ +.chroma .gh { + color: #fe640b; + font-weight: bold; +} +/* GenericInserted */ +.chroma .gi { + color: #40a02b; + background-color: oklch(96.2% 0.044 156.743); +} +/* GenericStrong */ +.chroma .gs { + font-weight: bold; +} +/* GenericSubheading */ +.chroma .gu { + color: #fe640b; + font-weight: bold; +} +/* GenericTraceback */ +.chroma .gt { + color: #d20f39; +} +/* GenericUnderline */ +.chroma .gl { + text-decoration: underline; +} + +[data-theme="dark"] { + /* Background */ + .bg { + color: #cad3f5; + background-color: #24273a; + } + /* PreWrapper */ + .chroma { + color: #cad3f5; + } + /* Error */ + .chroma .err { + color: #ed8796; + } + /* LineLink */ + .chroma .lnlinks { + outline: none; + text-decoration: none; + color: inherit; + } + /* LineTableTD */ + .chroma .lntd { + vertical-align: top; + padding: 0; + margin: 0; + border: 0; + } + /* LineTable */ + .chroma .lntable { + border-spacing: 0; + padding: 0; + margin: 0; + border: 0; + } + /* LineHighlight */ + .chroma .hl { + background-color: #494d64; + } + /* LineNumbersTable */ + .chroma .lnt { + white-space: pre; + -webkit-user-select: none; + user-select: none; + margin-right: 0.4em; + padding: 0 0.4em 0 0.4em; + color: #8087a2; + } + /* LineNumbers */ + .chroma .ln { + white-space: pre; + -webkit-user-select: none; + user-select: none; + margin-right: 0.4em; + padding: 0 0.4em 0 0.4em; + color: #8087a2; + } + /* Line */ + .chroma .line { + display: flex; + } + /* Keyword */ + .chroma .k { + color: #c6a0f6; + } + /* KeywordConstant */ + .chroma .kc { + color: #f5a97f; + } + /* KeywordDeclaration */ + .chroma .kd { + color: #ed8796; + } + /* KeywordNamespace */ + .chroma .kn { + color: #8bd5ca; + } + /* KeywordPseudo */ + .chroma .kp { + color: #c6a0f6; + } + /* KeywordReserved */ + .chroma .kr { + color: #c6a0f6; + } + /* KeywordType */ + .chroma .kt { + color: #ed8796; + } + /* NameAttribute */ + .chroma .na { + color: #8aadf4; + } + /* NameBuiltin */ + .chroma .nb { + color: #91d7e3; + } + /* NameBuiltinPseudo */ + .chroma .bp { + color: #91d7e3; + } + /* NameClass */ + .chroma .nc { + color: #eed49f; + } + /* NameConstant */ + .chroma .no { + color: #eed49f; + } + /* NameDecorator */ + .chroma .nd { + color: #8aadf4; + font-weight: bold; + } + /* NameEntity */ + .chroma .ni { + color: #8bd5ca; + } + /* NameException */ + .chroma .ne { + color: #f5a97f; + } + /* NameFunction */ + .chroma .nf { + color: #8aadf4; + } + /* NameFunctionMagic */ + .chroma .fm { + color: #8aadf4; + } + /* NameLabel */ + .chroma .nl { + color: #91d7e3; + } + /* NameNamespace */ + .chroma .nn { + color: #f5a97f; + } + /* NameProperty */ + .chroma .py { + color: #f5a97f; + } + /* NameTag */ + .chroma .nt { + color: #c6a0f6; + } + /* NameVariable */ + .chroma .nv { + color: #f4dbd6; + } + /* NameVariableClass */ + .chroma .vc { + color: #f4dbd6; + } + /* NameVariableGlobal */ + .chroma .vg { + color: #f4dbd6; + } + /* NameVariableInstance */ + .chroma .vi { + color: #f4dbd6; + } + /* NameVariableMagic */ + .chroma .vm { + color: #f4dbd6; + } + /* LiteralString */ + .chroma .s { + color: #a6da95; + } + /* LiteralStringAffix */ + .chroma .sa { + color: #ed8796; + } + /* LiteralStringBacktick */ + .chroma .sb { + color: #a6da95; + } + /* LiteralStringChar */ + .chroma .sc { + color: #a6da95; + } + /* LiteralStringDelimiter */ + .chroma .dl { + color: #8aadf4; + } + /* LiteralStringDoc */ + .chroma .sd { + color: #6e738d; + } + /* LiteralStringDouble */ + .chroma .s2 { + color: #a6da95; + } + /* LiteralStringEscape */ + .chroma .se { + color: #8aadf4; + } + /* LiteralStringHeredoc */ + .chroma .sh { + color: #6e738d; + } + /* LiteralStringInterpol */ + .chroma .si { + color: #a6da95; + } + /* LiteralStringOther */ + .chroma .sx { + color: #a6da95; + } + /* LiteralStringRegex */ + .chroma .sr { + color: #8bd5ca; + } + /* LiteralStringSingle */ + .chroma .s1 { + color: #a6da95; + } + /* LiteralStringSymbol */ + .chroma .ss { + color: #a6da95; + } + /* LiteralNumber */ + .chroma .m { + color: #f5a97f; + } + /* LiteralNumberBin */ + .chroma .mb { + color: #f5a97f; + } + /* LiteralNumberFloat */ + .chroma .mf { + color: #f5a97f; + } + /* LiteralNumberHex */ + .chroma .mh { + color: #f5a97f; + } + /* LiteralNumberInteger */ + .chroma .mi { + color: #f5a97f; + } + /* LiteralNumberIntegerLong */ + .chroma .il { + color: #f5a97f; + } + /* LiteralNumberOct */ + .chroma .mo { + color: #f5a97f; + } + /* Operator */ + .chroma .o { + color: #91d7e3; + font-weight: bold; + } + /* OperatorWord */ + .chroma .ow { + color: #91d7e3; + font-weight: bold; + } + /* Comment */ + .chroma .c { + color: #6e738d; + font-style: italic; + } + /* CommentHashbang */ + .chroma .ch { + color: #6e738d; + font-style: italic; + } + /* CommentMultiline */ + .chroma .cm { + color: #6e738d; + font-style: italic; + } + /* CommentSingle */ + .chroma .c1 { + color: #6e738d; + font-style: italic; + } + /* CommentSpecial */ + .chroma .cs { + color: #6e738d; + font-style: italic; + } + /* CommentPreproc */ + .chroma .cp { + color: #6e738d; + font-style: italic; + } + /* CommentPreprocFile */ + .chroma .cpf { + color: #6e738d; + font-weight: bold; + font-style: italic; + } + /* GenericDeleted */ + .chroma .gd { + color: #ed8796; + background-color: oklch(44.4% 0.177 26.899 / 0.5); + } + /* GenericEmph */ + .chroma .ge { + font-style: italic; + } + /* GenericError */ + .chroma .gr { + color: #ed8796; + } + /* GenericHeading */ + .chroma .gh { + color: #f5a97f; + font-weight: bold; + } + /* GenericInserted */ + .chroma .gi { + color: #a6da95; + background-color: oklch(44.8% 0.119 151.328 / 0.5); + } + /* GenericStrong */ + .chroma .gs { + font-weight: bold; + } + /* GenericSubheading */ + .chroma .gu { + color: #f5a97f; + font-weight: bold; + } + /* GenericTraceback */ + .chroma .gt { + color: #ed8796; + } + /* GenericUnderline */ + .chroma .gl { + text-decoration: underline; + } +} + +actor-typeahead { + --color-background: #ffffff; + --color-border: #d1d5db; + --color-shadow: #000000; + --color-hover: #f9fafb; + --color-avatar-fallback: #e5e7eb; + --radius: 0; + --padding-menu: 0rem; + z-index: 1000; +} + +actor-typeahead::part(handle) { + color: #111827; +} + +actor-typeahead::part(menu) { + box-shadow: + 0 4px 6px -1px rgb(0 0 0 / 0.1), + 0 2px 4px -2px rgb(0 0 0 / 0.1); +} + +[data-theme="dark"] { + actor-typeahead { + --color-background: #1f2937; + --color-border: #4b5563; + --color-shadow: #000000; + --color-hover: #374151; + --color-avatar-fallback: #4b5563; + } + + actor-typeahead::part(handle) { + color: #f9fafb; + } +} + +markdown-editor.drag-hover textarea { + outline: dashed 2px #ccc; + outline-offset: -0.5em; +} +markdown-editor:not(.drag-hover) button > .condensed, +markdown-editor.drag-hover button > .spacious { + display: none; +} + +@layer utilities { + .hit-area { + position: relative; + } + + .hit-area::before { + content: ""; + position: absolute; + top: var(--hit-area-t, 0px); + right: var(--hit-area-r, 0px); + bottom: var(--hit-area-b, 0px); + left: var(--hit-area-l, 0px); + pointer-events: inherit; + } + + .hit-area-debug { + position: relative; + } + + .hit-area-debug::before { + content: ""; + position: absolute; + top: var(--hit-area-t, 0px); + right: var(--hit-area-r, 0px); + bottom: var(--hit-area-b, 0px); + left: var(--hit-area-l, 0px); + pointer-events: inherit; + border: 1px dashed #3b82f6; + background-color: rgba(59, 130, 246, 0.1); + } + + .hit-area-debug:hover::before { + border-color: #22c55e; + background-color: rgba(34, 197, 94, 0.1); + } + + .hit-area-1 { + --hit-area-t: -0.25rem; + --hit-area-r: -0.25rem; + --hit-area-b: -0.25rem; + --hit-area-l: -0.25rem; + } + .hit-area-2 { + --hit-area-t: -0.5rem; + --hit-area-r: -0.5rem; + --hit-area-b: -0.5rem; + --hit-area-l: -0.5rem; + } + .hit-area-4 { + --hit-area-t: -1rem; + --hit-area-r: -1rem; + --hit-area-b: -1rem; + --hit-area-l: -1rem; + } + .hit-area-6 { + --hit-area-t: -1.5rem; + --hit-area-r: -1.5rem; + --hit-area-b: -1.5rem; + --hit-area-l: -1.5rem; + } + .hit-area-8 { + --hit-area-t: -2rem; + --hit-area-r: -2rem; + --hit-area-b: -2rem; + --hit-area-l: -2rem; + } + .hit-area-10 { + --hit-area-t: -2.5rem; + --hit-area-r: -2.5rem; + --hit-area-b: -2.5rem; + --hit-area-l: -2.5rem; + } + .hit-area-12 { + --hit-area-t: -3rem; + --hit-area-r: -3rem; + --hit-area-b: -3rem; + --hit-area-l: -3rem; + } + + .hit-area-l-1 { + --hit-area-l: -0.25rem; + } + .hit-area-l-2 { + --hit-area-l: -0.5rem; + } + .hit-area-l-4 { + --hit-area-l: -1rem; + } + .hit-area-l-6 { + --hit-area-l: -1.5rem; + } + .hit-area-l-8 { + --hit-area-l: -2rem; + } + .hit-area-l-10 { + --hit-area-l: -2.5rem; + } + .hit-area-l-12 { + --hit-area-l: -3rem; + } + + .hit-area-r-1 { + --hit-area-r: -0.25rem; + } + .hit-area-r-2 { + --hit-area-r: -0.5rem; + } + .hit-area-r-4 { + --hit-area-r: -1rem; + } + .hit-area-r-6 { + --hit-area-r: -1.5rem; + } + .hit-area-r-8 { + --hit-area-r: -2rem; + } + .hit-area-r-10 { + --hit-area-r: -2.5rem; + } + .hit-area-r-12 { + --hit-area-r: -3rem; + } + + .hit-area-t-1 { + --hit-area-t: -0.25rem; + } + .hit-area-t-2 { + --hit-area-t: -0.5rem; + } + .hit-area-t-4 { + --hit-area-t: -1rem; + } + .hit-area-t-6 { + --hit-area-t: -1.5rem; + } + .hit-area-t-8 { + --hit-area-t: -2rem; + } + .hit-area-t-10 { + --hit-area-t: -2.5rem; + } + .hit-area-t-12 { + --hit-area-t: -3rem; + } + + .hit-area-b-1 { + --hit-area-b: -0.25rem; + } + .hit-area-b-2 { + --hit-area-b: -0.5rem; + } + .hit-area-b-4 { + --hit-area-b: -1rem; + } + .hit-area-b-6 { + --hit-area-b: -1.5rem; + } + .hit-area-b-8 { + --hit-area-b: -2rem; + } + .hit-area-b-10 { + --hit-area-b: -2.5rem; + } + .hit-area-b-12 { + --hit-area-b: -3rem; + } + + .hit-area-x-1 { + --hit-area-l: -0.25rem; + --hit-area-r: -0.25rem; + } + .hit-area-x-2 { + --hit-area-l: -0.5rem; + --hit-area-r: -0.5rem; + } + .hit-area-x-4 { + --hit-area-l: -1rem; + --hit-area-r: -1rem; + } + .hit-area-x-6 { + --hit-area-l: -1.5rem; + --hit-area-r: -1.5rem; + } + .hit-area-x-8 { + --hit-area-l: -2rem; + --hit-area-r: -2rem; + } + .hit-area-x-10 { + --hit-area-l: -2.5rem; + --hit-area-r: -2.5rem; + } + .hit-area-x-12 { + --hit-area-l: -3rem; + --hit-area-r: -3rem; + } + + .hit-area-y-1 { + --hit-area-t: -0.25rem; + --hit-area-b: -0.25rem; + } + .hit-area-y-2 { + --hit-area-t: -0.5rem; + --hit-area-b: -0.5rem; + } + .hit-area-y-4 { + --hit-area-t: -1rem; + --hit-area-b: -1rem; + } + .hit-area-y-6 { + --hit-area-t: -1.5rem; + --hit-area-b: -1.5rem; + } + .hit-area-y-8 { + --hit-area-t: -2rem; + --hit-area-b: -2rem; + } + .hit-area-y-10 { + --hit-area-t: -2.5rem; + --hit-area-b: -2.5rem; + } + .hit-area-y-12 { + --hit-area-t: -3rem; + --hit-area-b: -3rem; + } + + /* terminal-to-html: catpuccin colors */ + .term-fg30 { + color: #6c6f85; + } /* black */ + .term-fg31 { + color: #d20f39; + } /* red */ + .term-fg32 { + color: #40a02b; + } /* green */ + .term-fg33 { + color: #df8e1d; + } /* yellow */ + .term-fg34 { + color: #1e66f5; + } /* blue */ + .term-fg35 { + color: #8839ef; + } /* magenta */ + .term-fg36 { + color: #179299; + } /* cyan */ + .term-fg37 { + color: #4c4f69; + } /* white */ + .term-fgi90 { + color: #8c8fa1; + } /* bright-black */ + .term-fgi91 { + color: #e64553; + } /* bright-red */ + .term-fgi92 { + color: #40a02b; + } /* bright-green */ + .term-fgi93 { + color: #e5c890; + } /* bright-yellow */ + .term-fgi94 { + color: #209fb5; + } /* bright-blue */ + .term-fgi95 { + color: #ea76cb; + } /* bright-magenta */ + .term-fgi96 { + color: #04a5e5; + } /* bright-cyan */ + .term-fgi97 { + color: #bcc0cc; + } /* bright-white */ + .term-bg40 { + background-color: #6c6f85; + } /* black */ + .term-bg41 { + background-color: #d20f39; + } /* red */ + .term-bg42 { + background-color: #40a02b; + } /* green */ + .term-bg43 { + background-color: #df8e1d; + } /* yellow */ + .term-bg44 { + background-color: #1e66f5; + } /* blue */ + .term-bg45 { + background-color: #8839ef; + } /* magenta */ + .term-bg46 { + background-color: #179299; + } /* cyan */ + .term-bg47 { + background-color: #dce0e8; + } /* white */ + .term-fg1 { + @apply font-bold; + } + .term-fg2 { + @apply opacity-60; + } + .term-fg3 { + @apply italic; + } + .term-fg4 { + @apply underline; + } + .term-fg9 { + @apply line-through; + } + + [data-theme="dark"] { + .term-fg30 { + color: #a5adcb; + } /* black */ + .term-fg31 { + color: #ed8796; + } /* red */ + .term-fg32 { + color: #a6da95; + } /* green */ + .term-fg33 { + color: #eed49f; + } /* yellow */ + .term-fg34 { + color: #8aadf4; + } /* blue */ + .term-fg35 { + color: #c6a0f6; + } /* magenta */ + .term-fg36 { + color: #8bd5ca; + } /* cyan */ + .term-fg37 { + color: #cad3f5; + } /* white */ + .term-fgi90 { + color: #8087a2; + } /* bright-black */ + .term-fgi91 { + color: #f5bde6; + } /* bright-red */ + .term-fgi92 { + color: #a6da95; + } /* bright-green */ + .term-fgi93 { + color: #f5a97f; + } /* bright-yellow */ + .term-fgi94 { + color: #7dc4e4; + } /* bright-blue */ + .term-fgi95 { + color: #f4dbd6; + } /* bright-magenta */ + .term-fgi96 { + color: #91d7e3; + } /* bright-cyan */ + .term-fgi97 { + color: #f0c6c6; + } /* bright-white */ + .term-bg40 { + background-color: #a5adcb; + } /* black */ + .term-bg41 { + background-color: #ed8796; + } /* red */ + .term-bg42 { + background-color: #a6da95; + } /* green */ + .term-bg43 { + background-color: #eed49f; + } /* yellow */ + .term-bg44 { + background-color: #8aadf4; + } /* blue */ + .term-bg45 { + background-color: #c6a0f6; + } /* magenta */ + .term-bg46 { + background-color: #8bd5ca; + } /* cyan */ + .term-bg47 { + background-color: #494d64; + } /* white */ + } +} + +.focus-pill { + --r: 10px; + + &::before, + &::after { + @apply bg-indigo-600 absolute bottom-0; + content: ""; + width: var(--r); + height: var(--r); + -webkit-mask: radial-gradient(var(--r) at var(--r) 0, #0000 98%, #000); + mask: radial-gradient(var(--r) at var(--r) 0, #0000 98%, #000); + } + + &::before { + left: calc(-1 * var(--r)); + @apply -scale-x-100; + } + + &::after { + right: calc(-1 * var(--r)); + } +} + +/* Tailwind Preflight resets svg to display:block; MathJax v4 emits inline SVGs + (one or more per inline equation), so that forces each onto its own line. + Restore inline flow for MathJax output only, leaving icon svgs untouched. */ +mjx-container svg { + display: inline-block; +} diff --git a/jetstream/jetstream.go b/jetstream/jetstream.go new file mode 100644 index 0000000000000000000000000000000000000000..13bed06abe0570f7b5ea5ed346b604115ef07434 --- /dev/null +++ b/jetstream/jetstream.go @@ -0,0 +1,297 @@ +package jetstream + +import ( + "context" + "fmt" + "log/slog" + "sync" + "sync/atomic" + "time" + + "tangled.org/core/log" +) + +const ( + gapRewind = time.Second + // shutdown saves the cursor after this even if events are still held + heldGrace = 30 * time.Second +) + +type DB interface { + GetLastTimeUs() (int64, error) + SaveLastTimeUs(int64) error +} + +type Set[T comparable] map[T]struct{} + +type JetstreamClient struct { + cfg StreamConfig + l *slog.Logger + + logDids bool + wantedDids Set[string] + unfilteredNsids Set[string] + db DB + waitForDid bool + mu sync.RWMutex + + lastSeenUs atomic.Int64 + marks watermark + holding atomic.Int64 + + // the first event of each connection clears it, so the gap check runs once + anchorUs atomic.Int64 + onGap func(context.Context) + + done chan struct{} + + saveEvery time.Duration +} + +func (j *JetstreamClient) AddDid(did string) { + if did == "" { + return + } + + if j.logDids { + j.l.Info("adding did to in-memory filter", "did", did) + } + j.mu.Lock() + j.wantedDids[did] = struct{}{} + j.mu.Unlock() +} + +// call before StartJetstream, fn runs on the reader so don't block in it +func (j *JetstreamClient) SetGapHandler(fn func(context.Context)) { + j.onGap = fn +} + +func (j *JetstreamClient) ExemptCollection(nsid string) { + j.mu.Lock() + j.unfilteredNsids[nsid] = struct{}{} + j.mu.Unlock() +} + +func (j *JetstreamClient) RemoveDid(did string) { + if did == "" { + return + } + + if j.logDids { + j.l.Info("removing did from in-memory filter", "did", did) + } + j.mu.Lock() + delete(j.wantedDids, did) + j.mu.Unlock() +} + +type processor func(context.Context, *Event) error + +func (j *JetstreamClient) withDidFilter(processFunc processor) processor { + // since this closure references j.WantedDids; it should auto-update + // existing instances of the closure when j.WantedDids is mutated + return func(ctx context.Context, evt *Event) error { + if anchor := j.anchorUs.Swap(0); anchor != 0 && evt.TimeUS > anchor && j.onGap != nil { + j.l.Warn("jetstream history no longer reaches the last event seen", "last", anchor, "first", evt.TimeUS) + j.onGap(ctx) + } + + ticket := j.ticket(evt.TimeUS) + h := &hold{ticket: ticket} + ctx = context.WithValue(ctx, holdKey{}, h) + + j.mu.RLock() + // empty filter => all dids allowed + matches := len(j.wantedDids) == 0 + if !matches { + if _, ok := j.wantedDids[evt.Did]; ok { + matches = true + } + } + if !matches && evt.Commit != nil { + if _, ok := j.unfilteredNsids[evt.Commit.Collection]; ok { + matches = true + } + } + j.mu.RUnlock() + + var err error + if matches { + err = processFunc(ctx, evt) + } + if err != nil && ctx.Err() != nil { + // shutdown cut this event short, so the cursor stays before it and the next start replays it + return err + } + + if !h.held.Load() { + ticket.Done() + } + return err + } +} + +// StartJetstream replaces cfg.Cursor with the db one, so don't set it +func NewJetstreamClient(endpoint string, collections []string, cfg *StreamConfig, logger *slog.Logger, db DB, waitForDid, logDids bool) (*JetstreamClient, error) { + if cfg == nil { + cfg = &StreamConfig{Endpoint: endpoint, Collections: collections, Compress: true} + } + + return &JetstreamClient{ + cfg: *cfg, + db: db, + l: logger, + wantedDids: make(map[string]struct{}), + unfilteredNsids: make(map[string]struct{}), + + logDids: logDids, + + // This will make the goroutine in StartJetstream wait until + // j.wantedDids has been populated, typically using addDids. + waitForDid: waitForDid, + + done: make(chan struct{}), + + saveEvery: time.Minute, + }, nil +} + +// StartJetstream starts the jetstream client and processes events using the provided processFunc. +// The client persists the last time_us cursor itself via the DB it was constructed with. +func (j *JetstreamClient) StartJetstream(ctx context.Context, processFunc func(context.Context, *Event) error) error { + cfg := j.cfg + cfg.TailWhenNoCursor = true + cfg.Cursor = func() (int64, bool, error) { return j.cursor(ctx) } + stream, err := NewStream(cfg, j.l) + if err != nil { + return fmt.Errorf("failed to create jetstream client: %w", err) + } + + handle := j.withDidFilter(processFunc) + go func() { + defer close(j.done) + if j.waitForDid && !j.waitForWantedDid(ctx) { + return + } + j.l.Info("done waiting for did") + + ticked := make(chan struct{}) + go func() { + defer close(ticked) + j.periodicLastTimeSave(ctx) + }() + + // returning the error would replay it forever, so log and skip + stream.Run(ctx, func(ctx context.Context, e *Event) error { + if err := handle(ctx, e); err != nil { + if ctx.Err() != nil { + return err + } + j.l.Error("error processing event", "error", err, "time_us", e.TimeUS) + } + return nil + }) + <-ticked + j.waitForHeld() + // save last, nothing may move the cursor once it runs + j.saveLastSeen(ctx) + }() + + return nil +} + +func (j *JetstreamClient) waitForHeld() { + for deadline := time.Now().Add(heldGrace); j.holding.Load() > 0 && time.Now().Before(deadline); { + time.Sleep(10 * time.Millisecond) + } +} + +// one past the last finished event, not the event itself +func (j *JetstreamClient) LastSeenUs() int64 { + return j.lastSeenUs.Load() +} + +// wait on it before closing the db, the final cursor save needs it open +// it never closes if StartJetstream wasn't called +func (j *JetstreamClient) Done() <-chan struct{} { + return j.done +} + +func (j *JetstreamClient) waitForWantedDid(ctx context.Context) bool { + for { + j.mu.RLock() + hasDid := len(j.wantedDids) != 0 + j.mu.RUnlock() + if hasDid { + return true + } + select { + case <-ctx.Done(): + return false + case <-time.After(time.Second): + } + } +} + +// save cursor periodically +func (j *JetstreamClient) periodicLastTimeSave(ctx context.Context) { + ticker := time.NewTicker(j.saveEvery) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + j.saveLastSeen(ctx) + } + } +} + +func (j *JetstreamClient) resume(ctx context.Context) *int64 { + cursor := j.resumeCursor(ctx) + j.anchorUs.Store(0) + if cursor == nil || j.onGap == nil || *cursor <= 0 { + return cursor + } + // cursor sits one past its event, hence the -1 + j.anchorUs.Store(*cursor - 1) + rewound := max(*cursor-gapRewind.Microseconds(), 0) + return &rewound +} + +func (j *JetstreamClient) cursor(ctx context.Context) (int64, bool, error) { + cursor := j.resume(ctx) + if cursor == nil { + return 0, false, nil + } + return *cursor, true, nil +} + +func (j *JetstreamClient) resumeCursor(ctx context.Context) *int64 { + if seen := j.lastSeenUs.Load(); seen != 0 { + return &seen + } + return j.getLastTimeUs(ctx) +} + +// the saved cursor is always an event time_us, never a wall clock +func (j *JetstreamClient) getLastTimeUs(ctx context.Context) *int64 { + l := log.FromContext(ctx) + lastTimeUs, err := j.db.GetLastTimeUs() + if err != nil { + l.Warn("couldn't get last time us, starting from now", "error", err) + return nil + } + + l.Info("found last time_us", "time_us", lastTimeUs) + return &lastTimeUs +} + +func (j *JetstreamClient) saveLastSeen(ctx context.Context) { + if seen := j.lastSeenUs.Load(); seen != 0 { + if err := j.db.SaveLastTimeUs(seen); err != nil { + log.FromContext(ctx).Error("failed to save cursor", "error", err) + } + } +} diff --git a/jetstream/jetstream_test.go b/jetstream/jetstream_test.go new file mode 100644 index 0000000000000000000000000000000000000000..eefe160bb0df009f28a1a4c71850d248017eecae --- /dev/null +++ b/jetstream/jetstream_test.go @@ -0,0 +1,478 @@ +package jetstream + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/klauspost/compress/zstd" +) + +type fakeCursorDB struct { + saved int64 + savedErr error + + mu sync.Mutex + got int64 +} + +func (f *fakeCursorDB) GetLastTimeUs() (int64, error) { return f.saved, f.savedErr } +func (f *fakeCursorDB) SaveLastTimeUs(us int64) error { + f.mu.Lock() + defer f.mu.Unlock() + f.got = us + return nil +} + +func cursorFor(db DB) int64 { + j := &JetstreamClient{db: db, wantedDids: make(Set[string])} + return *j.getLastTimeUs(context.Background()) +} + +func newGapClient(db DB, gaps *int) *JetstreamClient { + j := &JetstreamClient{db: db, l: slog.Default(), wantedDids: make(Set[string])} + j.SetGapHandler(func(context.Context) { *gaps++ }) + return j +} + +const twoDaysUs = int64(2 * 24 * 60 * 60 * 1000 * 1000) + +func TestStaleCursorIsHonored(t *testing.T) { + old := time.Now().UnixMicro() - 5*twoDaysUs + if got := cursorFor(&fakeCursorDB{saved: old}); got != old { + t.Fatalf("stale cursor must be honored, not snapped forward: got %d, want %d", got, old) + } +} + +func TestZeroCursorIsHonored(t *testing.T) { + if got := cursorFor(&fakeCursorDB{saved: 0}); got != 0 { + t.Fatalf("cursor 0 must replay from the start: got %d, want 0", got) + } +} + +func TestMissingCursorConnectsLive(t *testing.T) { + var gaps int + j := newGapClient(&fakeCursorDB{savedErr: errors.New("no row")}, &gaps) + if got := j.resume(context.Background()); got != nil { + t.Fatalf("missing cursor must connect live, got %d", *got) + } + _ = j.withDidFilter(func(context.Context, *Event) error { return nil })(context.Background(), &Event{Did: "did:plc:a", TimeUS: 5}) + if gaps != 0 { + t.Fatal("a first boot has nothing to catch up on") + } +} + +func TestResumeRewindsASecondFromTheLastEvent(t *testing.T) { + var gaps int + j := newGapClient(&fakeCursorDB{saved: 10_000_001}, &gaps) + if got := j.resume(context.Background()); got == nil || *got != 9_000_001 { + t.Fatalf("cursor = %v, want 9000001", got) + } +} + +func TestResumeKeepsTheStoredCursorWithoutAGapHandler(t *testing.T) { + j := &JetstreamClient{db: &fakeCursorDB{saved: 10_000_001}, l: slog.Default(), wantedDids: make(Set[string])} + if got := j.resume(context.Background()); got == nil || *got != 10_000_001 { + t.Fatalf("cursor = %v, want the stored one", got) + } +} + +func firstEventAfterResume(t *testing.T, saved, first int64) int { + t.Helper() + var gaps int + j := newGapClient(&fakeCursorDB{saved: saved}, &gaps) + j.resume(context.Background()) + wrapped := j.withDidFilter(func(context.Context, *Event) error { return nil }) + _ = wrapped(context.Background(), &Event{Did: "did:plc:a", TimeUS: first}) + _ = wrapped(context.Background(), &Event{Did: "did:plc:a", TimeUS: first + 1_000_000}) + return gaps +} + +func TestAGapIsReportedWhenTheStreamStartsAfterTheLastEvent(t *testing.T) { + // last event was 10_000_000, so the stored cursor is 10_000_001 + if got := firstEventAfterResume(t, 10_000_001, 10_000_001); got != 1 { + t.Fatalf("gap handler ran %d times, want once", got) + } +} + +func TestNoGapWhenTheStreamReplaysTheLastEvent(t *testing.T) { + for name, first := range map[string]int64{ + "the last event itself": 10_000_000, + "an earlier event": 9_500_000, + } { + if got := firstEventAfterResume(t, 10_000_001, first); got != 0 { + t.Fatalf("%s: gap handler ran %d times, want none", name, got) + } + } +} + +func TestLastSeenTracksEveryEventPreFilter(t *testing.T) { + j := &JetstreamClient{wantedDids: Set[string]{"did:plc:boltless": {}}} + wrapped := j.withDidFilter(func(context.Context, *Event) error { return nil }) + + _ = wrapped(context.Background(), &Event{Did: "did:plc:akshay", TimeUS: 100}) + if got := j.lastSeenUs.Load(); got != 101 { + t.Fatalf("filtered-out event must still advance last-seen: got %d, want 101", got) + } + + _ = wrapped(context.Background(), &Event{Did: "did:plc:boltless", TimeUS: 200}) + if got := j.lastSeenUs.Load(); got != 201 { + t.Fatalf("matching event must advance last-seen: got %d, want 201", got) + } +} + +func TestExemptCollectionShouldBypassDidFilter(t *testing.T) { + j := &JetstreamClient{ + wantedDids: Set[string]{"did:plc:boltless": {}}, + unfilteredNsids: make(Set[string]), + } + j.ExemptCollection("sh.tangled.repo") + + var processed int + wrapped := j.withDidFilter(func(context.Context, *Event) error { + processed++ + return nil + }) + + _ = wrapped(context.Background(), &Event{ + Did: "did:plc:akshay", TimeUS: 100, + Commit: &Commit{Collection: "sh.tangled.repo"}, + }) + if processed != 1 { + t.Fatal("An exempt collection must reach this processor from an unwanted DID; enrollment is decided later, not by this filter.") + } + + _ = wrapped(context.Background(), &Event{ + Did: "did:plc:akshay", TimeUS: 200, + Commit: &Commit{Collection: "sh.tangled.repo.pull"}, + }) + if processed != 1 { + t.Fatal("A non-exempt collection must still be filtered by DID.") + } +} + +func TestLastSeenAdvancesAfterProcessing(t *testing.T) { + j := &JetstreamClient{wantedDids: make(Set[string])} + + var seenDuringProcess int64 + wrapped := j.withDidFilter(func(context.Context, *Event) error { + seenDuringProcess = j.lastSeenUs.Load() + return nil + }) + + _ = wrapped(context.Background(), &Event{Did: "did:plc:boltless", TimeUS: 500}) + + if seenDuringProcess == 501 { + t.Fatal("cursor advanced before the event finished processing; a crash mid-process would skip it") + } + if got := j.lastSeenUs.Load(); got != 501 { + t.Fatalf("cursor must advance once processing returns: got %d, want 501", got) + } +} + +func TestShutdownLeavesTheCursorBeforeTheCutEvent(t *testing.T) { + j := &JetstreamClient{wantedDids: make(Set[string])} + ctx, cancel := context.WithCancel(context.Background()) + wrapped := j.withDidFilter(func(ctx context.Context, _ *Event) error { + cancel() + return ctx.Err() + }) + + _ = wrapped(ctx, &Event{Did: "did:plc:boltless", TimeUS: 700}) + if got := j.lastSeenUs.Load(); got != 0 { + t.Fatalf("cursor moved past an event shutdown cut short: got %d, want it untouched", got) + } +} + +func TestAFailingEventStillAdvancesTheCursor(t *testing.T) { + j := &JetstreamClient{wantedDids: make(Set[string])} + wrapped := j.withDidFilter(func(context.Context, *Event) error { + return errors.New("bad record") + }) + + _ = wrapped(context.Background(), &Event{Did: "did:plc:boltless", TimeUS: 800}) + if got := j.lastSeenUs.Load(); got != 801 { + t.Fatalf("a record that fails on its own must not stall the cursor: got %d, want 801", got) + } +} + +// nothing listens on port 1, so every connect fails at once +const refusedEndpoint = "ws://127.0.0.1:1/subscribe" + +func startClient(t *testing.T, endpoint string, db DB, waitForDid bool) (*JetstreamClient, context.CancelFunc) { + t.Helper() + j, err := NewJetstreamClient(endpoint, nil, nil, slog.Default(), db, waitForDid, false) + if err != nil { + t.Fatal(err) + } + j.lastSeenUs.Store(42) + ctx, cancel := context.WithCancel(context.Background()) + if err := j.StartJetstream(ctx, func(context.Context, *Event) error { return nil }); err != nil { + t.Fatal(err) + } + return j, cancel +} + +func requireStopsWithSavedCursor(t *testing.T, j *JetstreamClient, db *fakeCursorDB, msg string) { + t.Helper() + select { + case <-j.Done(): + case <-time.After(5 * time.Second): + t.Fatal(msg) + } + db.mu.Lock() + defer db.mu.Unlock() + if db.got != 42 { + t.Fatalf("cursor saved at stop = %d, want 42", db.got) + } +} + +func TestDoneClosesOnlyAfterTheFinalCursorSave(t *testing.T) { + db := &fakeCursorDB{savedErr: errors.New("no row")} + j, cancel := startClient(t, refusedEndpoint, db, false) + // long enough to be waiting out a failed connect when the cancel lands + time.Sleep(50 * time.Millisecond) + cancel() + requireStopsWithSavedCursor(t, j, db, "the reconnect loop kept going after its context was cancelled") +} + +func TestDoneClosesWhenCancelledBeforeAnyDidArrives(t *testing.T) { + j, cancel := startClient(t, refusedEndpoint, &fakeCursorDB{}, true) + cancel() + select { + case <-j.Done(): + case <-time.After(5 * time.Second): + t.Fatal("a client still waiting for its first did ignored shutdown") + } +} + +func TestDoneClosesEvenWhenTheStreamIsQuiet(t *testing.T) { + connected := make(chan struct{}) + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + close(connected) + <-release + })) + defer srv.Close() + defer close(release) + + db := &fakeCursorDB{savedErr: errors.New("no row")} + j, cancel := startClient(t, "ws"+strings.TrimPrefix(srv.URL, "http")+"/subscribe", db, false) + <-connected + cancel() + requireStopsWithSavedCursor(t, j, db, "a stream with no traffic held shutdown hostage, so the final cursor save never ran") +} + +func frameServer(t *testing.T, frames ...string) *streamServer { + t.Helper() + enc, err := zstd.NewWriter(nil, zstd.WithEncoderDict(zstdDictionary)) + if err != nil { + t.Fatal(err) + } + var srv *streamServer + srv = newStreamServer(t, func(n int, con *websocket.Conn) { + srv.mu.Lock() + compressed := srv.headers[n].Get("Socket-Encoding") == "zstd" + srv.mu.Unlock() + for _, f := range frames { + if compressed { + con.WriteMessage(websocket.BinaryMessage, enc.EncodeAll([]byte(f), nil)) + } else { + con.WriteMessage(websocket.TextMessage, []byte(f)) + } + } + drain(con) + }) + return srv +} + +func commitFrame(did string, us int, collection string, live bool) string { + return fmt.Sprintf(`{"did":%q,"time_us":%d,"kind":"commit","commit":{"operation":"create","collection":%q,"rkey":"k","live":%t}}`, did, us, collection, live) +} + +func runClient(t *testing.T, j *JetstreamClient, stop func(*Event) bool) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := j.StartJetstream(ctx, func(_ context.Context, e *Event) error { + if stop(e) { + cancel() + } + return nil + }); err != nil { + t.Fatal(err) + } + <-j.Done() + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatal("the client never delivered what the test waited for") + } +} + +func newTestClient(t *testing.T, srv *streamServer, db DB, collections ...string) *JetstreamClient { + t.Helper() + j, err := NewJetstreamClient(srv.endpoint(), collections, nil, slog.Default(), db, false, false) + if err != nil { + t.Fatal(err) + } + return j +} + +func TestClientDeliversInOrderAndOneAtATime(t *testing.T) { + var frames []string + for i := 1; i <= 50; i++ { + frames = append(frames, commitFrame("did:plc:a", i, "c", true)) + } + j := newTestClient(t, frameServer(t, frames...), &fakeCursorDB{savedErr: errors.New("no row")}) + + var inflight atomic.Int32 + var got []int64 + runClient(t, j, func(e *Event) bool { + if inflight.Add(1) != 1 { + t.Error("two events were being handled at once") + } + time.Sleep(time.Millisecond) + inflight.Add(-1) + got = append(got, e.TimeUS) + return e.TimeUS == 50 + }) + for i, us := range got { + if us != int64(i+1) { + t.Fatalf("event %d arrived as time_us %d, want %d", i, us, i+1) + } + } +} + +func TestClientDidFilterChangesAtRuntime(t *testing.T) { + srv := frameServer(t, + commitFrame("did:plc:a", 1, "c", true), + commitFrame("did:plc:b", 2, "c", true), + commitFrame("did:plc:a", 3, "c", true), + commitFrame("did:plc:b", 4, "c", true), + commitFrame("did:plc:a", 5, "c", true), + ) + j := newTestClient(t, srv, &fakeCursorDB{savedErr: errors.New("no row")}) + j.AddDid("did:plc:b") + + var got []int64 + runClient(t, j, func(e *Event) bool { + got = append(got, e.TimeUS) + if e.TimeUS == 2 { + j.AddDid("did:plc:a") + j.RemoveDid("did:plc:b") + } + return e.TimeUS == 5 + }) + if want := []int64{2, 3, 5}; !slices.Equal(got, want) { + t.Fatalf("delivered %v, want %v", got, want) + } +} + +func TestClientUnfilteredCollectionIgnoresTheDidFilter(t *testing.T) { + srv := frameServer(t, + commitFrame("did:plc:a", 1, "other", true), + commitFrame("did:plc:a", 2, "open", true), + commitFrame("did:plc:b", 3, "other", true), + ) + j := newTestClient(t, srv, &fakeCursorDB{savedErr: errors.New("no row")}) + j.AddDid("did:plc:b") + j.ExemptCollection("open") + + var got []int64 + runClient(t, j, func(e *Event) bool { + got = append(got, e.TimeUS) + return e.TimeUS == 3 + }) + if len(got) != 2 || got[0] != 2 || got[1] != 3 { + t.Fatalf("delivered %v, want the exempt event 2 and the wanted did's event 3", got) + } +} + +// only deliberi sends wantedEventTypes, a plain jetstream client must not +func TestClientAsksForNoEventTypes(t *testing.T) { + srv := frameServer(t, commitFrame("did:plc:a", 1, "c", true)) + j := newTestClient(t, srv, &fakeCursorDB{savedErr: errors.New("no row")}, "c") + + runClient(t, j, func(e *Event) bool { return true }) + if q := srv.query(0); q.Has("wantedEventTypes") { + t.Fatalf("query = %v, want no wantedEventTypes", q) + } +} + +func TestClientSavesTheCursorOnTheTimer(t *testing.T) { + srv := frameServer(t, commitFrame("did:plc:a", 10, "c", true)) + db := &fakeCursorDB{savedErr: errors.New("no row")} + j := newTestClient(t, srv, db) + j.saveEvery = 10 * time.Millisecond + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := j.StartJetstream(ctx, func(context.Context, *Event) error { return nil }); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for { + db.mu.Lock() + got := db.got + db.mu.Unlock() + if got == 11 { + return + } + if time.Now().After(deadline) { + t.Fatalf("saved cursor = %d, want 11 from the timer before any shutdown", got) + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestClientReportsAGapAndResumesARewoundCursor(t *testing.T) { + // cursor is one past 10_000_000, but the server's history starts after it + srv := frameServer(t, commitFrame("did:plc:a", 20_000_000, "c", true)) + j := newTestClient(t, srv, &fakeCursorDB{saved: 10_000_001}) + gaps := make(chan struct{}, 2) + j.SetGapHandler(func(context.Context) { gaps <- struct{}{} }) + + runClient(t, j, func(e *Event) bool { return true }) + if len(gaps) != 1 { + t.Fatalf("gap handler ran %d times, want once", len(gaps)) + } + if got := srv.query(0).Get("cursor"); got != "9000001" { + t.Fatalf("connected with cursor %q, want the stored one rewound a second", got) + } +} + +func TestClientFailingEventsDontDropTheConnection(t *testing.T) { + srv := frameServer(t, + commitFrame("did:plc:a", 1, "c", true), + commitFrame("did:plc:a", 2, "c", true), + ) + j := newTestClient(t, srv, &fakeCursorDB{savedErr: errors.New("no row")}) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var seen atomic.Int32 + _ = j.StartJetstream(ctx, func(_ context.Context, e *Event) error { + if seen.Add(1) == 2 { + cancel() + } + return errors.New("bad record") + }) + <-j.Done() + if seen.Load() != 2 || srv.connects() != 1 { + t.Fatalf("saw %d events over %d connections, want 2 over 1", seen.Load(), srv.connects()) + } +} diff --git a/jetstream/stream.go b/jetstream/stream.go new file mode 100644 index 0000000000000000000000000000000000000000..2d575a908a869630e970c6870c1c1677e8d19f7e --- /dev/null +++ b/jetstream/stream.go @@ -0,0 +1,220 @@ +package jetstream + +import ( + "context" + _ "embed" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/url" + "strconv" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/gorilla/websocket" + "github.com/klauspost/compress/zstd" +) + +// bsky jetstream compresses with this dictionary, and hydrant's frames decode with it too +// +//go:embed zstd_dictionary +var zstdDictionary []byte + +const ( + EventKindCommit = "commit" + EventKindAccount = "account" + EventKindIdentity = "identity" + + CommitOperationCreate = "create" + CommitOperationUpdate = "update" + CommitOperationDelete = "delete" +) + +// hydrant pings every 30s, so 90s of silence means the connection is dead +const defaultReadTimeout = 90 * time.Second + +// refused dials fail instantly, without the wait this loop spins hot +const reconnectDelay = time.Second + +// indigo's jetstream event plus a live flag on commits, which is what hydrant's /subscribe sends +type Event struct { + Did string `json:"did"` + TimeUS int64 `json:"time_us"` + Kind string `json:"kind,omitempty"` + Commit *Commit `json:"commit,omitempty"` + Account *comatproto.SyncSubscribeRepos_Account `json:"account,omitempty"` + Identity *comatproto.SyncSubscribeRepos_Identity `json:"identity,omitempty"` +} + +type Commit struct { + Rev string `json:"rev,omitempty"` + Operation string `json:"operation,omitempty"` + Collection string `json:"collection,omitempty"` + RKey string `json:"rkey,omitempty"` + Record json.RawMessage `json:"record,omitempty"` + CID string `json:"cid,omitempty"` + // plain jetstream never sends this, so nil counts as live + Live *bool `json:"live,omitempty"` +} + +func (c *Commit) IsLive() bool { + return c.Live == nil || *c.Live +} + +type StreamConfig struct { + Endpoint string + Collections []string + // hydrant only backfills when asked here, plain jetstream doesn't know the param + EventTypes []string + Compress bool + + // read on every connect, ok false replays from 0 unless TailWhenNoCursor is set + Cursor func() (timeUs int64, ok bool, err error) + TailWhenNoCursor bool + // each hydrant stamps time_us its own way, so resume a little before the cursor + Rewind time.Duration + + ReadTimeout time.Duration + ReconnectDelay time.Duration + UserAgent string +} + +// the handler saves the cursor in the same transaction as the event, so a crash can't keep one without the other +type Stream struct { + cfg StreamConfig + l *slog.Logger + dec *zstd.Decoder +} + +func NewStream(cfg StreamConfig, logger *slog.Logger) (*Stream, error) { + if cfg.Cursor == nil { + return nil, fmt.Errorf("stream needs a cursor source") + } + if cfg.ReadTimeout == 0 { + cfg.ReadTimeout = defaultReadTimeout + } + if cfg.ReconnectDelay == 0 { + cfg.ReconnectDelay = reconnectDelay + } + s := &Stream{cfg: cfg, l: logger} + if cfg.Compress { + dec, err := zstd.NewReader(nil, zstd.WithDecoderDicts(zstdDictionary)) + if err != nil { + return nil, fmt.Errorf("creating zstd decoder: %w", err) + } + s.dec = dec + } + return s, nil +} + +// a handler error drops the connection, so the event comes back from the cursor +func (s *Stream) Run(ctx context.Context, handle func(context.Context, *Event) error) error { + for { + if err := s.readOnce(ctx, handle); err != nil && ctx.Err() == nil { + s.l.Error("reading stream", "err", err) + } + select { + case <-ctx.Done(): + return nil + case <-time.After(s.cfg.ReconnectDelay): + } + } +} + +// nil leaves the cursor param off, which starts at the live tail +func (s *Stream) resumeCursor() (*int64, error) { + us, ok, err := s.cfg.Cursor() + if err != nil { + return nil, err + } + if !ok { + if s.cfg.TailWhenNoCursor { + return nil, nil + } + us = 0 + } + cursor := max(us-s.cfg.Rewind.Microseconds(), 0) + return &cursor, nil +} + +func (s *Stream) url(cursor *int64) (string, error) { + u, err := url.Parse(s.cfg.Endpoint) + if err != nil { + return "", fmt.Errorf("parsing endpoint %q: %w", s.cfg.Endpoint, err) + } + q := u.Query() + q.Del("cursor") + if cursor != nil { + q.Set("cursor", strconv.FormatInt(*cursor, 10)) + } + q.Del("wantedCollections") + for _, nsid := range s.cfg.Collections { + q.Add("wantedCollections", nsid) + } + q.Del("wantedEventTypes") + for _, t := range s.cfg.EventTypes { + q.Add("wantedEventTypes", t) + } + u.RawQuery = q.Encode() + return u.String(), nil +} + +func (s *Stream) readOnce(ctx context.Context, handle func(context.Context, *Event) error) error { + cursor, err := s.resumeCursor() + if err != nil { + return fmt.Errorf("reading cursor: %w", err) + } + target, err := s.url(cursor) + if err != nil { + return err + } + + header := http.Header{} + if s.cfg.UserAgent != "" { + header.Set("User-Agent", s.cfg.UserAgent) + } + if s.dec != nil { + header.Set("Socket-Encoding", "zstd") + } + + s.l.Info("connecting to stream", "url", target) + con, _, err := websocket.DefaultDialer.DialContext(ctx, target, header) + if err != nil { + return err + } + defer con.Close() + // a quiet stream blocks the read and never checks ctx, so cancel closes the conn + defer context.AfterFunc(ctx, func() { con.Close() })() + + alive := func() error { return con.SetReadDeadline(time.Now().Add(s.cfg.ReadTimeout)) } + con.SetPingHandler(func(data string) error { + if err := alive(); err != nil { + return err + } + return con.WriteControl(websocket.PongMessage, []byte(data), time.Now().Add(time.Second)) + }) + + for { + if err := alive(); err != nil { + return err + } + _, msg, err := con.ReadMessage() + if err != nil { + return fmt.Errorf("reading message: %w", err) + } + if s.dec != nil { + if msg, err = s.dec.DecodeAll(msg, nil); err != nil { + return fmt.Errorf("decompressing message: %w", err) + } + } + var e Event + if err := json.Unmarshal(msg, &e); err != nil { + s.l.Warn("skipping undecodable frame", "err", err) + continue + } + if err := handle(ctx, &e); err != nil { + return fmt.Errorf("handling event at %d: %w", e.TimeUS, err) + } + } +} diff --git a/jetstream/stream_test.go b/jetstream/stream_test.go new file mode 100644 index 0000000000000000000000000000000000000000..362888b47e8ebad9e813bae2b4ad588e7dd5b9ab --- /dev/null +++ b/jetstream/stream_test.go @@ -0,0 +1,321 @@ +package jetstream + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/klauspost/compress/zstd" +) + +type streamServer struct { + *httptest.Server + + mu sync.Mutex + queries []url.Values + headers []http.Header +} + +func newStreamServer(t *testing.T, script func(n int, con *websocket.Conn)) *streamServer { + t.Helper() + s := &streamServer{} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + con, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err != nil { + return + } + defer con.Close() + s.mu.Lock() + s.queries = append(s.queries, r.URL.Query()) + s.headers = append(s.headers, r.Header) + n := len(s.queries) - 1 + s.mu.Unlock() + script(n, con) + })) + t.Cleanup(s.Close) + return s +} + +func (s *streamServer) endpoint() string { + return "ws" + strings.TrimPrefix(s.URL, "http") + "/subscribe" +} + +func (s *streamServer) query(n int) url.Values { + s.mu.Lock() + defer s.mu.Unlock() + return s.queries[n] +} + +func (s *streamServer) connects() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.queries) +} + +func drain(con *websocket.Conn) { + for { + if _, _, err := con.ReadMessage(); err != nil { + return + } + } +} + +func testStream(t *testing.T, cfg StreamConfig) *Stream { + t.Helper() + if cfg.Cursor == nil { + cfg.Cursor = func() (int64, bool, error) { return 0, false, nil } + } + if cfg.ReconnectDelay == 0 { + cfg.ReconnectDelay = 10 * time.Millisecond + } + s, err := NewStream(cfg, slog.New(slog.NewTextHandler(io.Discard, nil))) + if err != nil { + t.Fatal(err) + } + return s +} + +func run(t *testing.T, s *Stream, until func(*Event) bool) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err := make(chan error, 1) + go func() { + err <- s.Run(ctx, func(_ context.Context, e *Event) error { + if until(e) { + cancel() + } + return nil + }) + }() + if e := <-err; e != nil { + t.Fatal(e) + } + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatal("the stream never delivered what the test waited for") + } +} + +func TestStreamURLCarriesEachParamOnce(t *testing.T) { + s := testStream(t, StreamConfig{ + Endpoint: "ws://hydrant/subscribe?cursor=9&wantedEventTypes=old", + Collections: []string{"org.tangled.track.ticket", "org.tangled.feed.comment"}, + EventTypes: []string{"live", "historical"}, + }) + got, err := s.url(new(int64)) + if err != nil { + t.Fatal(err) + } + u, err := url.Parse(got) + if err != nil { + t.Fatal(err) + } + q := u.Query() + if strings.Count(got, "cursor=") != 1 || q.Get("cursor") != "0" { + t.Errorf("url %q should carry exactly one cursor=0", got) + } + if strings.Count(got, "?") != 1 { + t.Errorf("url %q has more than one query string", got) + } + if want := []string{"live", "historical"}; strings.Join(q["wantedEventTypes"], ",") != strings.Join(want, ",") { + t.Errorf("wantedEventTypes = %v, want %v", q["wantedEventTypes"], want) + } + if len(q["wantedCollections"]) != 2 { + t.Errorf("wantedCollections = %v, want both", q["wantedCollections"]) + } +} + +func TestStreamDecodesLive(t *testing.T) { + srv := newStreamServer(t, func(_ int, con *websocket.Conn) { + for _, frame := range []string{ + `{"did":"d","time_us":1,"kind":"commit","commit":{"rev":"r","operation":"create","live":true}}`, + `{"did":"d","time_us":2,"kind":"commit","commit":{"rev":"r","operation":"create","live":false}}`, + `{"did":"d","time_us":3,"kind":"commit","commit":{"rev":"r","operation":"create"}}`, + } { + con.WriteMessage(websocket.TextMessage, []byte(frame)) + } + drain(con) + }) + s := testStream(t, StreamConfig{Endpoint: srv.endpoint()}) + + live := map[int64]bool{} + run(t, s, func(e *Event) bool { + live[e.TimeUS] = e.Commit.IsLive() + return e.TimeUS == 3 + }) + if want := map[int64]bool{1: true, 2: false, 3: true}; len(live) != 3 || live[1] != want[1] || live[2] != want[2] || live[3] != want[3] { + t.Fatalf("live = %v, want %v (absent means live)", live, want) + } +} + +func TestStreamSkipsUndecodableFrames(t *testing.T) { + srv := newStreamServer(t, func(_ int, con *websocket.Conn) { + con.WriteMessage(websocket.TextMessage, []byte(`not json`)) + con.WriteMessage(websocket.TextMessage, []byte(`{"did":"d","time_us":7}`)) + drain(con) + }) + run(t, testStream(t, StreamConfig{Endpoint: srv.endpoint()}), func(e *Event) bool { return e.TimeUS == 7 }) +} + +func TestStreamResumesFromTheHandlersCursorRewound(t *testing.T) { + var mu sync.Mutex + cursor, have := int64(0), false + srv := newStreamServer(t, func(n int, con *websocket.Conn) { + if n == 0 { + con.WriteMessage(websocket.TextMessage, []byte(`{"did":"d","time_us":10000000}`)) + } + if n == 1 { + con.WriteMessage(websocket.TextMessage, []byte(`{"did":"d","time_us":10000001}`)) + } + drain(con) + }) + s := testStream(t, StreamConfig{ + Endpoint: srv.endpoint(), + Rewind: 2 * time.Second, + Cursor: func() (int64, bool, error) { + mu.Lock() + defer mu.Unlock() + return cursor, have, nil + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + s.Run(ctx, func(_ context.Context, e *Event) error { + mu.Lock() + cursor, have = e.TimeUS, true + mu.Unlock() + if e.TimeUS == 10_000_000 { + return errors.New("drop the connection") + } + cancel() + return nil + }) + }() + <-done + + if got := srv.query(0).Get("cursor"); got != "0" { + t.Errorf("empty db cursor = %q, want 0 so a fresh db replays", got) + } + if got := srv.query(1).Get("cursor"); got != "8000000" { + t.Errorf("reconnect cursor = %q, want the db cursor minus the rewind", got) + } +} + +func TestStreamOnlyReadsTheCursorOncePerConnect(t *testing.T) { + var reads int + srv := newStreamServer(t, func(_ int, con *websocket.Conn) { + con.WriteMessage(websocket.TextMessage, []byte(`{"did":"d","time_us":5}`)) + drain(con) + }) + s := testStream(t, StreamConfig{ + Endpoint: srv.endpoint(), + Cursor: func() (int64, bool, error) { reads++; return 0, false, nil }, + }) + run(t, s, func(e *Event) bool { return true }) + if reads != 1 { + t.Fatalf("cursor source read %d times for one connection, want 1", reads) + } +} + +func TestStreamCursorErrorStopsTheConnect(t *testing.T) { + srv := newStreamServer(t, func(int, *websocket.Conn) { t.Error("connected despite an unreadable cursor") }) + s := testStream(t, StreamConfig{ + Endpoint: srv.endpoint(), + Cursor: func() (int64, bool, error) { return 0, false, errors.New("db down") }, + }) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := s.Run(ctx, func(context.Context, *Event) error { return nil }); err != nil { + t.Fatal(err) + } +} + +func TestStreamReadTimeoutReconnects(t *testing.T) { + srv := newStreamServer(t, func(n int, con *websocket.Conn) { + if n == 0 { + drain(con) + return + } + con.WriteMessage(websocket.TextMessage, []byte(`{"did":"d","time_us":9}`)) + drain(con) + }) + s := testStream(t, StreamConfig{Endpoint: srv.endpoint(), ReadTimeout: 100 * time.Millisecond}) + run(t, s, func(e *Event) bool { return e.TimeUS == 9 }) +} + +func TestStreamPingsKeepAConnectionAlive(t *testing.T) { + srv := newStreamServer(t, func(n int, con *websocket.Conn) { + for range 6 { + time.Sleep(60 * time.Millisecond) + con.WriteControl(websocket.PingMessage, nil, time.Now().Add(time.Second)) + } + con.WriteMessage(websocket.TextMessage, []byte(`{"did":"d","time_us":4}`)) + drain(con) + }) + s := testStream(t, StreamConfig{Endpoint: srv.endpoint(), ReadTimeout: 150 * time.Millisecond}) + run(t, s, func(e *Event) bool { return e.TimeUS == 4 }) + if n := len(srv.queries); n != 1 { + t.Fatalf("connected %d times, pings should have kept the first alive", n) + } +} + +func TestStreamDecodesZstdWithTheJetstreamDictionary(t *testing.T) { + enc, err := zstd.NewWriter(nil, zstd.WithEncoderDict(zstdDictionary)) + if err != nil { + t.Fatal(err) + } + srv := newStreamServer(t, func(_ int, con *websocket.Conn) { + frame := enc.EncodeAll([]byte(`{"did":"d","time_us":6,"kind":"commit","commit":{"live":false}}`), nil) + con.WriteMessage(websocket.BinaryMessage, frame) + drain(con) + }) + s := testStream(t, StreamConfig{Endpoint: srv.endpoint(), Compress: true, UserAgent: "test"}) + run(t, s, func(e *Event) bool { return e.TimeUS == 6 && !e.Commit.IsLive() }) + if got := srv.headers[0].Get("Socket-Encoding"); got != "zstd" { + t.Errorf("Socket-Encoding = %q, want zstd", got) + } + if got := srv.headers[0].Get("User-Agent"); got != "test" { + t.Errorf("User-Agent = %q, want test", got) + } +} + +func TestStreamDecodesZstdWithoutADictionary(t *testing.T) { + enc, err := zstd.NewWriter(nil) + if err != nil { + t.Fatal(err) + } + srv := newStreamServer(t, func(_ int, con *websocket.Conn) { + frame := enc.EncodeAll([]byte(`{"did":"d","time_us":7,"kind":"commit","commit":{"live":false}}`), nil) + con.WriteMessage(websocket.BinaryMessage, frame) + drain(con) + }) + s := testStream(t, StreamConfig{Endpoint: srv.endpoint(), Compress: true}) + run(t, s, func(e *Event) bool { return e.TimeUS == 7 && !e.Commit.IsLive() }) +} + +func TestStreamWithoutACursorTailsWhenAsked(t *testing.T) { + srv := newStreamServer(t, func(_ int, con *websocket.Conn) { drain(con) }) + s := testStream(t, StreamConfig{Endpoint: srv.endpoint(), TailWhenNoCursor: true}) + ctx, cancel := context.WithCancel(context.Background()) + go s.Run(ctx, func(context.Context, *Event) error { return nil }) + for srv.connects() == 0 { + time.Sleep(time.Millisecond) + } + cancel() + if _, ok := srv.query(0)["cursor"]; ok { + t.Fatal("a stream with no cursor should connect without one") + } +} diff --git a/jetstream/watermark.go b/jetstream/watermark.go new file mode 100644 index 0000000000000000000000000000000000000000..41bb7c7b63a0d4403fea4c22b097588c860ee607 --- /dev/null +++ b/jetstream/watermark.go @@ -0,0 +1,96 @@ +package jetstream + +import ( + "context" + "sync" + "sync/atomic" +) + +// a crash replays whatever is still pending because the cursor only moves up to the front +type watermark struct { + mu sync.Mutex + front uint64 + pending []mark +} + +type mark struct { + us int64 + done bool +} + +// a ticket is a held event, the cursor can't pass it until Done. +// skipping Done pins the cursor here forever +type Ticket struct { + j *JetstreamClient + id uint64 + us int64 + // shutdown waits for held tickets before the last cursor save + holding atomic.Bool +} + +func (j *JetstreamClient) ticket(us int64) *Ticket { + w := &j.marks + w.mu.Lock() + defer w.mu.Unlock() + id := w.front + uint64(len(w.pending)) + w.pending = append(w.pending, mark{us: us}) + return &Ticket{j: j, id: id, us: us} +} + +func (t *Ticket) Done() { + if t == nil { + return + } + defer t.release() + w := &t.j.marks + w.mu.Lock() + defer w.mu.Unlock() + if t.id < w.front { + return + } + w.pending[t.id-w.front].done = true + var newest int64 + moved := false + for len(w.pending) > 0 && w.pending[0].done { + newest, moved = max(newest, w.pending[0].us), true + w.pending = w.pending[1:] + w.front++ + } + // never move the cursor back, since reconnect replays can be older than it + if moved && newest+1 > t.j.lastSeenUs.Load() { + t.j.lastSeenUs.Store(newest + 1) + } +} + +// the cursor stays before this event since dropping doesn't mark it done +func (t *Ticket) Drop() { + if t != nil { + t.release() + } +} + +func (t *Ticket) release() { + if t.holding.CompareAndSwap(true, false) { + t.j.holding.Add(-1) + } +} + +type holdKey struct{} + +type hold struct { + ticket *Ticket + held atomic.Bool +} + +// hold before the processor returns, or the cursor moves past it +func Hold(ctx context.Context) *Ticket { + h, ok := ctx.Value(holdKey{}).(*hold) + if !ok { + return nil + } + h.held.Store(true) + if h.ticket.holding.CompareAndSwap(false, true) { + h.ticket.j.holding.Add(1) + } + return h.ticket +} diff --git a/jetstream/watermark_test.go b/jetstream/watermark_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d8a58192160641a782115f45d7f48f258e4cc02a --- /dev/null +++ b/jetstream/watermark_test.go @@ -0,0 +1,120 @@ +package jetstream + +import ( + "context" + "errors" + "math/rand/v2" + "sync" + "testing" + "time" +) + +func TestTheCursorNeverPassesAnUnfinishedEvent(t *testing.T) { + const n = 64 + for range 200 { + j := &JetstreamClient{} + tickets := make([]*Ticket, n) + for i := range tickets { + tickets[i] = j.ticket(int64(100 + i*10)) + } + rand.Shuffle(n, func(a, b int) { tickets[a], tickets[b] = tickets[b], tickets[a] }) + + done := map[int64]bool{} + for _, tk := range tickets { + done[tk.us] = true + tk.Done() + for i := range n { + us := int64(100 + i*10) + if !done[us] { + if got := j.lastSeenUs.Load(); got > us { + t.Fatalf("cursor %d passed the unfinished event at %d", got, us) + } + break + } + } + } + if got, want := j.lastSeenUs.Load(), int64(100+(n-1)*10+1); got != want { + t.Fatalf("cursor = %d once everything is done, want %d", got, want) + } + } +} + +func TestAHeldEventKeepsTheCursorBehindIt(t *testing.T) { + j := &JetstreamClient{wantedDids: make(Set[string])} + var held *Ticket + wrapped := j.withDidFilter(func(ctx context.Context, e *Event) error { + if e.TimeUS == 100 { + held = Hold(ctx) + } + return nil + }) + + _ = wrapped(context.Background(), &Event{Did: "did:plc:a", TimeUS: 100}) + _ = wrapped(context.Background(), &Event{Did: "did:plc:a", TimeUS: 200}) + if got := j.lastSeenUs.Load(); got != 0 { + t.Fatalf("cursor = %d with an earlier event still held, want it untouched", got) + } + + held.Done() + if got := j.lastSeenUs.Load(); got != 201 { + t.Fatalf("cursor = %d after the held event finished, want 201", got) + } +} + +func TestHoldOutsideTheClientIsANoOp(t *testing.T) { + if tk := Hold(context.Background()); tk != nil { + t.Fatalf("Hold = %+v with no event in the context", tk) + } + var none *Ticket + none.Done() +} + +func TestTicketsFinishingAtOnceStillLandOnTheNewestEvent(t *testing.T) { + const n = 500 + j := &JetstreamClient{} + tickets := make([]*Ticket, n) + for i := range tickets { + tickets[i] = j.ticket(int64(i + 1)) + } + var wg sync.WaitGroup + for _, tk := range tickets { + wg.Go(tk.Done) + } + wg.Wait() + if got := j.lastSeenUs.Load(); got != n+1 { + t.Fatalf("cursor = %d, want %d", got, n+1) + } + if len(j.marks.pending) != 0 { + t.Fatalf("%d events still pending", len(j.marks.pending)) + } +} + +func TestShutdownWaitsForHeldEventsBeforeTheLastSave(t *testing.T) { + db := &fakeCursorDB{savedErr: errors.New("no row")} + j, cancel := startClient(t, refusedEndpoint, db, false) + ticket := j.ticket(7) + ticket.holding.Store(true) + j.holding.Add(1) + + cancel() + select { + case <-j.Done(): + t.Fatal("the client finished while a processor still held an event") + case <-time.After(200 * time.Millisecond): + } + + ticket.Drop() + requireStopsWithSavedCursor(t, j, db, "the client never finished after the held event was dropped") +} + +// replays come out of time order, and the cursor tracks the max, not the last one +func TestTheCursorLandsOnTheNewestEventReleasedTogether(t *testing.T) { + j := &JetstreamClient{} + a, b, replayed := j.ticket(100), j.ticket(200), j.ticket(100) + b.Done() + replayed.Done() + a.Done() + if got := j.lastSeenUs.Load(); got != 201 { + t.Fatalf("cursor = %d, want 201 once the event at 200 is done too", got) + } +} diff --git a/jetstream/zstd_dictionary b/jetstream/zstd_dictionary new file mode 100644 index 0000000000000000000000000000000000000000..106847e7007b4ac409fd58912fed85d69aedb352 GIT binary patch literal 112640 zcmb@vTdZwMdLCpuo!B}=kcWUGc$wZNJe|GHT2(cw*4i)a(`_4B@h#mpkwPOb7H$Vn~pQWP(7A;}FM&NC;y^fQ7*O0VIUPBhNl40tpE}@&;dxwa?mX z%{AuS^Yl8q)n{MUoHfU+%U}QH`@UcNgX{mh{F#6LZ~QBN;eY$+H~-oHFZ}wozxQ*0 z{6GK0pZZ%r_c#CUZ~n!<_y7KL|Eu^b<2V1S|K@-Hp?~II`eFKKe+a++_QDaCU;G#T=AZlR z|MG9#e5L>M|JRS);XTR!^5Z}JKmPFBA2q-Fm%j4DKlDSN{-wje_!qwWXYd33<%hn4 z|Nj{Nf0fEu_dikUs<*y+_3^93Ca>Ov|GueR{HB{$BJ_k8Wm8&=wi)6&u6g(BJO9SV zuWXg*`PG|mzfw!Df4ai^2oWD|fBR-H+a7Vjz0rr*=(upU>#zQ;kG?+i`hZ7#bNERmJL}4!(^sl177QL+=?))#^rzqd+y4kZ zcCznvH{sK4_4pO9zVq>`c$-lr)1}qc%qF*N?PaPZzKbAbIro&Yv8&fH6N0ERs$w@K zB`u_h2j)ul>O0^0=-WpNW9K?!e5qGQi$TjtPIR2jtg(W~S{?fq8{^xr{^fVW?Y=v1u_!wEH0%oO8TR`6?lrmj-dmvP&pnW4EJPG<=|>Z<2Z^)CW-Z8?75~Sn>!(vZqRD=%BECcwcVPI z6Jw5HxS_T9JlVhb*4-yhAG`Kl|H=J)v9w7NQelmb zRyxg<$5|l*sZvhpoK-EpW;+qKm%%n7LLG9>)1WH#!-! zY*@B2UcS+zHxaqICTBOg>9b}eV??W*8N~$YYFoJyn>-zjAdO*7pHD-VXMNk3C6}xZ zCw*t5&uts!#1|E;J6Xqg>gJ)NnL4%6hs0F2aAbVG(PP17RIrhhUX_ILX&E}#xuHuM zA1@*@)F`s>qRw13jM?0}e0QVsxnnwC455uJd{}BTil`ZPRvH$KlWTf?^M3J8ZuDV^ zN*+0~ykMm{pOmzA>xWsa_(v3rkn~NyL?#d zQcu#ssK&h?8`bA|9mj?=sY`~aTGSBdmW*rG>_3g37O|2hnWIrJb3s?qPU*0YC->c~ z>!Hmg&)}7q5Fs@J}Eql>$ z>D^cor`$xRboVM1H{#}w-Cks2Xl{6Gh=0Mp8)=>`8wb#j>TLpOGgD9iDfV_7(H(jLzgw?E}F*^SLrgPS>pU$jhe`OBu>DUJ&6Exoy+_}@hqW|&zvk?l_U&|Q+XvNl(#cg|1P8ISy?VHu{m z@Y^tF7W-&`Gp-YR!|2sDeR`ujxb}Hl_EQyHtmqBBhCMoK0Sv2^OeVRgA}iOpSB*B& zxDdF`bt}5EVM!~8T#Zdq=S?}3w6yyc088wKNHf{0bSYTV_(UgGpWGMqA`05WW<1{) zy3eu^=7f;C&}21HVj@$t5o!8rn$~_uvOMOzY4eivFE<2?ycJ%vAgg?eu`-`3FFb1=78{^ETc5kX2aUN3dsbz=7>fBBK-_IAoC>|W|)VJ zv&7e>+PC$jl|}4{7|Xo2apg!_6`hX4Ry|&6L2sj*J3`6(#XIR^jKM808R=iJ(qRmR zZy&~Z`AXwV^D7=j=TiwbBSTcIwuu_jy4~w?>FTM^ZKU(#P#Gb#NA^A|Z`$%4A&a-dPp!FnL7tHm>_3 zX4#rHEgavFx;S1z9`S3~27Y(J3c`(F)@3H`#VZ(Mz_3*B1K%))SNCIlXa(QB`^>z< zON_1vr(p#j@+Y}`oIm;AuYUHuf5fgo`_+H)*{}W4XTSCbfAZab_}TaV@U!pz`u%^O ztwCC-HtQPG^0Ic{`_!b%^-zNJ&{NtK;5RwC+hl`($?wWG38V8|h`e=454k zGka%RvbP45-rk7l?xK5iPA91(3HRa;8j}rk3g^>qV|-{0-lZ3MD2++<9~iL^^6BSv%2hc@eB>y4AN0U7}sY$7Jxfx{D@B#4+)w>ZUEAwft zE912+s=1MBGAXI}uJw;1c0=y2FH5DYBUJ?n%lg56d*r@7G9M#Zhu+)|b7t!v4RB`n zBBMaBZs^n7{=sq&Gc9=Hr=g}xZsLwLQz=WupIME>t9xPMWaHxF$z5h?!hw zYFnvHOzmRkGVjU=(b{gDg;UZ<*%+63TU3)^m77Xq_v5TPI9W(#y`Rw2cQ0*BVPeu< zGW(DHq{mZ_F|_NpG0sf=<9DB#cf?t@S9k3CoYhDM)4I&WEMah+#X^;12Hw`fXeugQ zRteQ)VV%lokJQOp6fz_*cH(=3OEtBg;zV{aV`W#W%=-D1)hHUibaPRVVLNA09ZBI! z7pY3dq)|muaF&mmsp<+z?kc4UPV=dc_8GoJi@Z-Yh*|N*l}a-iCv zWmB(ZjO3o<5po{1SA@)(q_ON=l8TkW4<-5Ph}BWf;qcL2S6=9ZcXg#(zw~p`OM zj%+SR(L`xmATKOfXNM}PQ&o|CDM(BP!}%InRYiT_W;cwx=7bAfmW`{8sma_l&Cu6A?ZuZ3C1XwdK=^A<^--hxr%O|I^ahYkc_~7NtC=yl4X9% z5MSoAs}t7}*_E!r@{uBPJz~kBMpbF-qfkH?kcxj??pwN|fya;}97PbD~QV>M29iTDE?4(->J&Kb`B&+)H_6y;`Gs7;wA>ALsRmz|+^kafVyeT4L+2poHZ;N|&zJ2y@$T2A`!oyoM3T z5YXqgGK(CeB8nu7UD)-iw62IW|gfSY`ZezUs^sqP4%@q+( zlVC@u$B3%X9V==sVufr|8^MtDU03r_pf)zQ(Ar%Pf;Qj^yAr;=Ms!-lF6Yo2F% z&8H>nds5F0J+aa*!PZ!l4uyqhE_@=pDd|v*7T(*ea#1w&GN%pe4T|t(n~rR)Xxe#A zS+lx0OY_~Nr^a^4DWzr8E>_}&&JOmzl!28#Dx~_vB4a(x$x^PUT=ms}Xb9GjXaBznn~#~4Q6kMZ&?595Z64Lf^bq=KR1K_qj)S=443$`yHO zgoTslQpYwfvT4>8(q(RPs;!H7OF{1VpLuEu&5AB(@VI(v#yF;B-3}*LdYWfAyKsY^ zt#sm-Ts0_34!IURS4xh{h@| zxEvD{H+PJi;Peia#ORXz1cJP|^AX(F_kINaxS!8G3ez6lWMLTBr`s@R7W-&`GnW}$ zL3+jc(=5e9oHz9V!uTp7ZA7KEGB%B!)+oP^^N#QjG8z0V#}Tm^U_m}&eKycEZ%;1f zTBF8Ka4SDsOp`Xtq&wG-ZH+9pi3*`-?yXD^{pqw zXFd94>?}%@5iP1ZHmX3;s~aegM=dVh4990KQP;24*iR-lsDP&HiL!9Bcx1liqDclA zM`l@u6E`RGiIvVKRwZyN-i3`%;Wwr$JGub8Rb~>d%`wW{vE|CG zB|*%8@6kWQG<7vYX{%WkUGpW%Tro5@GikD|^gYBXVpoKotre(N#br9XOp$@nVKJht zC%)1lpw; zl81VBt>78zLpvMq_L@aEghUr102r`^L?Q_O!8hKBF_f^|7%$&={0(EkdYoP9xQ!KH zT~-&?l$bFht9nF$W7^pdHfp%x425IQW}o_W7|TM10_V<3>#-ASwZJ_}+qxrjE~U6NWjn*hd4Lxl-tx8`LlOi45>Gvit*Z zQ0B7=_>&E6_#tG&3DY@Z3`@*5#+fhm_}yo|5f8rY&F$ITs%*LxC!Q-RDXXxPle|qb zlC88R5uj=`8%jgFSyI>7G%H&nPh64<@Ks16`jusQF1jp*NzR)59d;@Jfw#wgfP)T0 zmdr3bI|)u@RnZOJ3JaVy(84WH5=-G}BVCL1u2<8do1(87pvQAjBUl=CqnMoZ!uUJ9EB|G@>T0P#r7=ywY3}d`}r4hK3D|TQ1 zJO%tsfhuZO51>LqsfcoKDlV72pPQt$^AIgH?v~W37MXExfvooZi4;vD5mBmKQIpYJ z8kSd)^t|CG+@V7E4b4cX7(8=)r4uA|q_2@joV#JrUBddFvvwS$U_*f#RjCUj^FA*d z0N9z_!vAL}n#HqxZhTrGGrlM_R=O+eB-`AW<9PM%7O89@F1eM46{*}YCtKAEpov(% z{oS{};}7obZwJgT^6m2DgYG(cz|9hZp#d@h)%z@Ky&dz))k{&mSp_53oN1QXO7|FV^FGmiWBU zI^#R}tydo(S`60v;p(Grz54XsS3&CR5F=_DRtsdb(&fiMLzlk8qi(c;L)~HxQ2IK+ z=g7j3OWB3dfs5`A5+4V0L;zJ+cvg$A8~%&AY+H~J@bw@jsu~X# zW~;O%UYf{5`Sy42pC6{C^Y}vV<_teU{tggse3|X5;F)7L*x~B%(c#msgE5hBKKeP2 z$B~Dh#Vq(^rt9ll{hZdbJyxxim+E!Ydv{YvWqi#L5=cR3GjNrez6=S=Q`f~c66bGj z7@%$2b)iw5+NUCHKe+-aMYs6=81dncd&GsOul|XL*Md8vJM8cJ9Z%w;pW1eM8zA=B za50t`+oIFk_Qx7DgAF@9><%n-ioIh;Y+(FRI=mLEj>+?F|Mu@!nAStsHSdmIVC&$G z@=$Bc2!_W2-Od2EY*{;igmHG?{yz3$h0TbMO0kdd{u2N8ari?xdQnW0G1UbIJ&}bH=gLhU9 zzKH8-9x7~xnG;?vOQcal%)zio^QhYe0u-jA9Wv}7d#I#uG7dy?=lx!!iEIMu(6@J& zqyk53`)rjjs~uqs6*XT{?Fgx%TR?s{5YeeiR4#N)+bNMKCCrKzg@G?36>d*6qQE;F zAT0F)ihzrNs%9Ag+$Y?nBu8SnNO>8}^+H=(;*N7k?@nBzB_Lq{jkSB(Bjg=9o~CzM zj&OHEeOoZYT><2w)w4jo~g^)=4VyQri63*1hsoApL&mO&13l^k`(yRfjG49zI zp%kBQ2({z;pA&JzZZ1Nr42v?kp<6n)U#y)75I1nxynjxtN;(`a(Zdm`B%(hO8#n4M zr82K*Pur#70$6_!>IrGeUJtE_Pt*fIQX3m&-Mgw8KzgF9jrut0PN5WV&P)03+3;}h z2!o$yKKKbRLK?RuY~aU-aH*HUWZ?J1h-4ltNS~8yJG|4 z33p7egW*kq{S^3wUFo0M8`Vy_uj+NyQz>ksi-s5vOOeW|Y&%Jk(YGcdDN81mIxUhi z+1-UhG1RV6BXX&$ofobrSJx`qT}mR5V4x9u*jO-^+tzx{)*EIqq#T&{_q#CUEWX$- zWC)?fz!Tv8@ZK|i`xgm2nLv>CG?V#|kY166p5_%e>xLt9GwT9i7~aLtXFt}3#*<}O zgSJ@bWCCb$=EByamY7LB6IbB6>QzNFcw6gK~9i2N}7SuK3W=SdENDcpFSg*ATzt0%;5 zIBB+bAt&(7x4#X%zX#K=#VG<2@PI2BJ~S9yft-sohZ{Ul2H`;-{?*{r!cX;wZw|qI zy3YlD?mPm zx-{Gtc-dWzs+V*jromUi2OowKZj@~Cx8Se@4-OtatPi7<3ZuS{yOR0B!y>wnXYfsCU4twy&P6sU?-#wF z#L8RROtNoUx-8jPTWbw-EtV}T-&5`{zoz7dirdTfvqdfWKGpbx=gGmo756F6Gn@Hw z0^yYR4$qXlfgNGd**tlQ>`t{R5|vrgExzP&1dyv7>(mT&X^k~QhUAJ%ERxQ%6%i0H z4_ovJ_cxA{ij-3_qVTxR!hn%Y$AOYl=C(b`UMJ-p^I zUptU^O@>`u)ppF65WEhHGW_!sV!5G-Tw|AFRM1a6! zMD)Q00BQT%uMmFYY(1k@n=()Hu?KwD*?CQ-rh)NPa0QbBq5&DMnT7ItLh-J$9i4g` zWhb&LoW}xja*7KPD_lb#9kdJ|YiN`Ww->S%4IMmAHw zj?2gdSjax|bBdncDX|9-sBc$wHinp{}Nmg`DUQmI&xnLHMAF{1=8imZ)nnRa|CL_+py zF&qUr@NU*MD&nfe~q)}-- zmq{-t)c%l3wMCweh$=ylP`HFAyC#5Qpq^Qcw#~)}*j8-Bod~dn5ms^EVt!c*4TU)c z?A58jS$lR{-VlDn=|%W~0c40?$1f`Gg)zJ(Us7?82An9lJsayQo~zz2m14<)aAl5{ zzU@TC$GYv|WIGV5G^}a`sFZccvXmx}gb5Rj2)PwNiN7fE^l8iu72xEN6+P!=C8NZ0Xfro#0 zw*kB!yxS+#u>a$~`&<9?xBj#L=a2q_fBN^p5&O^o&p-SppMCG&`|Nw5hyB)P-}~=B z``+(-_Py_a_PyWu>{s9Em<$pW?J9T#&sHZ+#u)cU(gALmg2V%(P;>aj)i9+LKqdfl zS&MrnrRxAFo~uyq*jpXnQjyXE@WrLV&I7+`W>aeSGBL~v!NzF$7`_Srqm^2mz;vSstbLfsFEv$xk4>yf!ig;0}PUj?=qq*t&g-zp-=OYS_ z3!ES^#MS7B6N#0Sn`F+id1g&&^Oqkh*#aZ#Gm1{D`TFPi}s$YE&WwAj+?$9n|s$&KKbCD!I)ML>BO0QD%# z{>Xmyfp?wln{F#$}4;uMp0 z3XQKk?eoZs$^&5~J6ib}3?m1)Sr4idmI#M(|JJQjzouDC0U68c1&VAyZjOt*hyNll zewOyr1#}1K)O4CmP=FEHN#MVHRNL= zRGDf6%eC0kMZbFLA@j5+0Hf#(DqE81CJ||t_SI@&3>Huokp$>nJW&@z01_yI-11e| z5jVTpfCA`^CNGd7j2wt2yp7;5Lt|&>!UsS24!k#ZQAQO+tbs0La@j@QI`^@3q_`#5Z44dpBMtp+mC}fMWP8N!%=I!sH;^Dvg>}-Smp`rn`lq6sI zI{0PEBx=&AIhOeWZ1dqg=@AqdF@Xn9E0 zMX+^rW$g?wV1W>B&f>L4I$I$}j_F7CAK5E^;(YruiRY%yl6=Z5PU$>QJ2n+N6#(*@ zW+3mqlPr}>t!Bg2PVp#Teqx+gSlAlva5jkhi;N(s8{ttAcU22Exn9(ju3#@ zcQ-K8m#pP5FDvE4V^8XIF?(B!9XmZQs7~lF32O|0h+Z*vHoOCJaQ0^4Icw!AVrC(O zP@wh`l9LneOgBR4xAma>3M~-wED6LfccUW!8q%q1aW< zYD$o$qqqbVO51e?LQ>60wYI1k+*)#XqDokHwY=yAsz= zE!Trb2re>&n2`r!pnEEa2~2I%q!7wHo&l&!Kr=*)3cPhdGI*!i8T_hk!aCeDEI5L2jVSn!t z&_7uDDvZI(`!QbZ5#HRqAs;MX#cG%kBGpk}b|Yg`;!M>=0#-jAbLbxCO-CkFzXcIV zI{D6}?Fmb>pnkKaP-{#o+g2QjU5^`nIw|HMW)z`-b`lOUTG*sQ$lVp zgXqj`lw!V&m-F;~e{5)GhFn_xAzIGU_n)ydH0W}Y^Bfk_0&6*DWQ|=~At-7=UGGPt z$9jk|*2X!jC;`wmTfqoz8pbkpBF=Wq*4bK+$ydnaHA~wt12*kDN-z8|A(u#W!5@G- zx`zTxR|jB44{hd!Lmsw-l0^)Zd#jpmN*SPRs;MWQG|+uE-l_o_11EeHm@;8;7ZHN& z90Vr$x|F6!ixYQTvBb3l1C6ZP&U;sD|AoL-sB|1S3Sbii9J9VUeEl0@Lk{1=^J5F1 z>^;8k7m#0-?L9vK^A|yf5=fnZ{=5I_bH_LK*s+jtVXf)Zx-P`Cv?aX~WI~p#Et9Dz zTZ5S1B%lgR9^u)p<7*dy1yayhEWAAECHo3AMXk1U(NRi4uqc9L|3zshXoF-^;sMJw&(UXS!4M&#)nMUhig*N8wR<;+byVlh&S%rJ;3L7=Wr&tR^^KQ4p?usK-w{i z6>4Rx_03SD{!V)(CRXY1EE z9+#4k$9whLQdc)+DRd=s52lELYNPY*v2L!dz|I81(U3Fs<^Ziqu?3?-p8+X9{s zhfB5#xBw&2`}ZKf?bhD9J`9VfP;1&GEMC47z$duALLGf4^8CN@TzO93MXD1u5rW0CNM)5^qLK^Pl+g7k!?5d(Cifa+mB1)xJg3o)Nl(bp#|4KT1t zmjMF{eftwTK`RAKG~B)f7hYq zq@Br1!Kgxk&uXBLnv-0!vRr$yjuUuwp^)=A;^@+&89!MgX#yY{pKOn#aoFZk=PTYkpm$-vFh;)?_z-C9sxQNhopxX05IlV!$1UK`0 zdvpf2tZ9Sr<0Aj-$bEluEo)k;`(e)P+M@x^-^-|x+?{|egXi3t1z@uwp2rFIu3cBC zWuRnn?Ek|iD1|OBV2zC}xaGd8O)*J zLwf)IFVEVyv^F+jE52(W1XN}96y7GA-+LwErRYKj}0KM?HVz$n~+yj^e?Hqsw}ymII^ z{v9|~9iVw2V+9a1_-`3P0*n|K1V8cQ`HvP?XIU%(GYO3gOacYVs8_DcAfLRn>so5z zvULHv2je>AFDGHQ7~=DK7xP20Vi4!;$^q(i6XlB2b(jK`r( z$NLxKk6#R#hdj(|f+>V0{4)CY+(JV4EGj7zz?MSRLPyDhe5}p}ZZnOvcLd0RlA}zE zv_m>}O*H_DMO>Bs1S&~0sNLipM)0c#Q7wa36a;VMQ)VBD2nZnW(s)zIj2DmvZ4iev z?J7!ChSIEn;3;gXaRXQ!F|fsSX=pS>D;oXmi_Wfqrd7=h#!cg;)R~35&U3q)aR$-s zCc3%41Py^s9LWTEzeg*5y72fkGJ8rwCfLIxdpj}A|N8rfh$<;tYdf-DL01kxe2c&$`GL{q?gTg=`c4*R3TWByl zuhD=g9qL`I1ZWmj`IJ~QVD$uR+dac6F+ACI4zJfU5Bhi$vI9Ew8&YZaNRhXLzyXw&>m|y zu22ga%}fcp-1Kc63+MwD#Yp3m=d6{s7stqZ01_QzX%h41GEy4(^2q zv|~XZAv|RZQLi3Km{S2v667eSG-?3ZwF-=~Q_uSayFx%eY?RQSK@XhgFeY8Gno|Ze zQY?!M$l$bfJ_4)EIyFPyYRyJ3TZ$~1XdP)NgVy(sT_7DK8YQXQF>)e=2MgVv*d=%`1R;xPkh;MM`axG3W6bODLHYNK zSDJ!4b#wMi$B^g@Egbi1q6k%FLF`2$tPk%$v-2Q%c6}wz*F;S<%Dj$dx@erg-Ws@0 z0&E4Bryd|OUX&%M{(z^qfK$pWuF;Tm-zM4yiIWvJNyPE60tHy3y16DN1$TQ;f3zs} zkcO#@S~6E!S=2BT#K<6xJ_&Ti$-5#nNC@raG?Jh(_~k0gib*xQwTv=qT0r0^dR+Aj z3btvn6qKnw8-u2u(G3Wkmw>oXe}(#@0&?!so%gU>k6n*tU7

    jq)KY^YH#Ns|~av z>sN(1$Di=}aNJ-916H5>2SysEO zhB7bdQiXgP zw0)4enM2YTe?XNY$lKtxf%RHK_~Tvz1jG4zF%-nhtYYtvBpfB$q89F?}0XlO*S_R)uTY!cS=pnX^Y@zTIUh0A-3z&VN1_%CgIpE>0 z4!?xI2d$neoQjDuLhxpr_w`*Z^%?4 zAN_S?WqSX~_6n?5bw7RgGg12?4k9_}+kYHp6HSvXw2$6TV3=uawD1j(975aLcM~2a zE=&$454?X!HQjGo?4mGZVO6o54+>V>!ae=jwjH*A-}b~d+vr05ZV!i9!9-w9`nUfz z`c9*nVA!VRR#}p5n2K*`>Fm1wbKgo#846WMOo2;|9fjWjqBHOQv3&b`iH(DR4z_Ms zZF#?2@tD}W;G4q>#XsI}_1CX)V1qnP_&}^SjxbcO`XDoe;SJQfp=JeEJ0=Oo0GdG# z1BdC@zRX5%5BgDf0^kr5Kuk=9`X-uX+OBQ-84i&opdn1?-de5>5T)Et5jI5$b-GiX zK*kgrAS%u^G#StfN@X17-eo@B))P!T2IwiS$`k2!DJtOxM<_%i0m#qMtwN!ncL}v~ z$O`~VHgl+=pmc=Z%8f74RQOcy3Up9%t!FY#;z?MjQ$t(EF@Lh$-PVz7AVYb7J-kb| zqoo!*aYSBe=3y>;$kwb0=)p*6rb=QyRjceI#F$@yJQJcUQ0nee@ zdS)HRD*O)Q|J%#RH4w~3{pt&l+@8lO5JM&aa9u=ld-v`$KEx(nbVtvkWD#>tE1LPC zSvR0iQ6#v{xq~?FQlRZ1xUw@a%;MqHYy?WkEZTb;iH}nq?kaF>1Hhun}(K2I@7x= zJ=XB+TIdy&XgtwDs5ur3)SdS*lYz4e{Lf`trQpn<8SXnY`kRM*Py|(#DeG#r=57Jy zxSVEz!v3yG${iRI==*m`XoM1nGuanmPd?vRVGN+jzod-?=k)H1u%{rWqaZk&=_Hdh znp$qPAQ8wS5FUiRY0#*1482knIyWfvFF-?FTwpl0 zC!EvJIh7^Q0}0{lQ6f$uV2ipHjZmD`FcTKFxTYSkEhaSAbrmKF4WoFE${s39Kx+Y! zznAWy5L-`~s1&?yQD?cx30g3(yFoB;WkE~jG7NOD=@)8u`VdMy3PnaV9h@#~c>3`E zGiC(11dv*8&W0jTEUU{J$blA}T+rf$aZ-_u06^mWxM%NP6=T;h6ekBhMv!wuPnITuhGVm^nq++icGy~<*zs;&!Os%VCAS5fU-apK ztU_0xBKEj(u%&5Zuy3F3cuA;CE8LiIX!E(cxkKlG7Scd;h3q$j?CmOYutstTnu#os z|15UXKDJr_Suc=-DhmV=t(rv5Rkx?VC2kOW^VhhjS=9{xs6cMzTg1@4);zy4F%-XdLUb!|~QR z=fuDeCoLGf5EMkkkfM77dX1B2NNMSTEsV=uf5|6FytF~4fR|tktyK|se2i&0oJXE9 zO6v6H`tCAiY)}e%UVeejz`KVWqd0v(%9nA-;l%K(yQ8km$4(3~ChLUitumeFJ+kMf zRlY+%1y{7`oGwG3@yY>)RDwpkBB3>&KvPL@$ly|=8G&VKvesxtWhB(fPdYJkQ#O18 z&HmYHCli#I%CeFW$)*yhvwQ$fcxDPBvoWd07=h#*JC}j8hR)+aleYVZ+~sr51PWig zM01b21$=9u{bq`fR$HLS&&@5rzC;~+q*XVf2)<#=KXk|+=i(n3^e_gbF1Im0wAPoz zK{xygSJJuaTmpU`1hbH^?_7lz&P7KcVGe07lvC~ddev2+2*B4d)!Ni8bz>8`>1IB=JwvF-f z9RmO+nj7Ant@0VSLO3^POlMFVgd!&7-MY1}Do7G8C6uOS>rxA~Cy#B!(Gi+#aK4?$ zdFGX_puDLÐKZgA5t;dQKrN9Wdi)thnLLk8c9g(Ht6ixc1Sh+{}^BXz#j}BZO{9 z=)stFIWTa5-L?y4sEdia-J1aLFhf!Nl>o4b3a}8~E}&HHxqA@K1w)NoWE+qqgM@Vq z2Koi=-qHR2$UKB$I22gIFlQF~Xn->z4)_GH5J&BV9vpyz*CnC4Eul1-!P7HjLDFx1 z*XMaJKFsGXLhTH7xS8SGRzOstOSai3V*s-btnVf9;Q&BG&8sg!@OzF~4`bw@{D#ou z%-n5;`NNoXOk8w@{8%up5194(vGVzri7quV?8vnu`5v;(A@vvX=*YmLXfmh+xjy`r z<6qD6fAIid0cr;g7zJLNDNY@_#HXt)nz&i>tPUC_3ySL_w?=dxGL^e5`C*L_v&&=! zPg)S!qoF`q6uV8b#5K7FBkH2yeOR+8ZRLx)XooT6uzX2fw4*oIH&HFCJEYVbpbP;Z1W&V= zhS{CODj0~kw%{<{W3r!Iq9C;Bd^GVejzn64rtUGQBtGe_;w*Dzf@s=6YRQ@=!d)!K z6-bjER0;e6Z0LY!7B5Y1w zh-iH}i~?pRFQs$n`+-YhK%xMHzuU3{D4Re1hG5_UOo(fM#-i5y_lp z1&r0UiGEPaFk%e5+>h~c%dnLbudkmr$vsY(M6T8hqO#MBPCdwdJMkWC3xp<|q}Qxe6d29A#Etf@6_ZNK8X}M(kdG=qo;?Wdm|M$d|a~fm0p| zXcFxiv+HXJ07EJ7{oH22+I(?k8jPX)WEqtv&j&?|SM0jxHvqry|XO9XC3-7RR>>k8T+vjLRq=Xz>ME z`^)NXXD&tz1CH^BT+gEc&iFYvfOl8dUoKKr4^ z@(98Av0#t~2W7hh?jAK;MMhafKvNj>w*Zo{u?5;G%ozfc-2l>O$kU^%C3+K;aVkZT zM~&~6-GT_r2K#D62w7%9X85)QEg&`H~9h2o%PW-Wp1Gpnwkq(mXEx%+K6U*f7- zQgG@!x9};Md4rtTMy@!DVZ<%=E#VjCA4l3nC8E`f%=povf8PIAb;V zrk6mTdZrIn1K37r)6D}eS*yEga4TBE#4!LPNA42pa-chmXlGbBFbpDemOxR656dVP zGORqy-YrbFm<8=$r-AkvWu{&^WZ45-I^dibfI{+XZt=BLkA!vJA}Zl3Kj zqyut*C5ej;>L$JrsYh=o1DIm!Ry36cUkygB zTQjo_x3m**)uA7ja3Y2lbKP_muyV4q^prOW0&{_mexFN?zFU5joc29*c@1Ik0tR_Vbw*>K_5EBB;;cd`ReWdSaGYwrrx0uTg#AruZ^LSatms}yY*OtJdDE0@ql6|D>tS4q;K>G=lJc$O=e?1I7`T3SzX zG&Nd-mQ+|;?sjlMiUTMrdQr(8;-F=fy{LeSq!qZ!2w-q7K4cj_oJB;w3Wyr^w1eJJ zt^i87tb`;~Qz63OdVm-Wl%dKR0*}g|0I?v=noE$>%M2KN)OPf#4i4C;SU}c_&|{}A ztU(!nM(^5F#|jO+(LJ$*ZqqY=B8B@vhoz^HgKi@L0Z12mR_Zc}upfwFifYFVNQfoU`39X@^q0+9aY`JlzZGb0Z( z!>3qPRzs%RWl<&zny*nb!Q+EAzOHgHK#LC-(x!5NvgHVx>SsLZL zh#r(a0ZYCpPcsreCw#uFYJUpRgE4qqf2d^t_}yn5z9`@`K=?Zst|V+8GM28I^06NvbzRi6G69lkX1AgghbcZzT!<(_NQFZCgA;5+mLA=L z(Zjsobng=U%X^~5nXwIQ>ehZ$f+|*74{V(E-~~N4kinwB3qq;^EhMmdV2IHnd^ZW#5C+AT*qh!rQCjxKvL2L!4dH0_b`7~fE5-AIO$5p@gF*lE zv29}yuUs>CNpqmdkk}YjAf`s(G)H?vy<}$flow5nMiZ#LLbhM_e4$-= z%K0g?zCe8sj^4Agujq)SsnVp)xX#9cwg6B#K%a-Uio_6nFyyIg%c~gzK#*_B_U8+V zn8012VMhf<7%GzJfz$bM-SYg$=?B1wp%Nr6fj^BDeQe5?!JonyP}}*E;7>f%3)uZi zqb-7lk$UeUp-t(%)!BY>1;VlR*z%{mF5DjvUKeP1aB}Y7@o2=ePM49n*9zd(h*d_7 z5KTe8847!dOov#G3-ApB$EraOYOj+K4QC>3_9sr)_JN<2bvT*#J{O086S=($J7wSH zdgjZZyQ<4e$g$<5@MtUqOQh=M@XF1=7ITO5Io>7DV6#&4uPoZ-U z8O0KsX^ns=s9lsZIl7;uLpFp~e#aTbfW-&tg_KE+#!SGWHA3ez)HQqxs+=_(cp_00 z{R&?qH;zri?(D096+Xk5=vX)zKAeWacu`s$cp z(WbYC5<9v8ox0kGnoU(5qa63_YS&{QvX(36O)LXHVq$WnAR zfhXr11N7G3T*O@p_aXz2aY(@r2eZ++r8c{{ZI}Qo;^tkyOIM#P5Cdg37gc>(2J2`e zJ>H2JMh*8&f-l7mAdeNHzVoz9{IC-dT!8;Ot7WnlEL8N#TBk%t*hEJ=5wi~~mqWN5 z3SfZ6swIY~#!RM!mU>E3Q;5fqn7v{Oow|=WUELQ-5KAPv1n)Jf%-9z|lENU0*>| zSr1TpSSFzhTS1i>;4xExr_vW_A(Ga$Ko*<;-LpVub~q7opyOTDPOX`gBo&}KU>z?d zOHStx0VU3*52EMa`S?ECzrMW^AdGW2oMK^Ii3=yV&?GA`|gdTEe!_*sU8d@|k;p ziq{PwvKP4r=t|s76(;&68#A=d1(1C6)?(!7cb)W$Hzq@D7oBAWX#(8<#J&=Y^JrN! z2hp<5K>rcOp0|*+mJUe?h{AYdscSoLLBiS1Jkc3g&$t^4usKoMhWrJ*Vo0)|x&S=y z3x!s3JNkb>zb*q)6NGRKQjCD2bQME0j2I0S1hqh+a51+GPDoKL(^M3b*v&CQZUgP> z(Gg>8T$`{M%J7*icNNVA0QC^LxVcD|3wOEa|MBbI70%Gp@W+RJQf-W^8l@&B0^+_rUQ{BpmeT7401C%gA6^~ z?EPHTC{A+}8`a9YKZ zCI3Hb@77yOn%?)_>r+sg07>9!ZnEk41g*t zw~_42*7(z9M42;X0hiHH^%_^^=mKX8)YYH%{^fYZ5!>%x#UNFBOgvQt)!0 zlm$*+V;L}FRJ+u9J`~8ds}PYVt2hS^rn6=W;U`HYR66+~8EVn&3GYw)r^RmFqBW<>IVh{GPXjnb%8S(fGjt)L-`C+uoR0!cWu44QFhkry5%!>~XG|eJSu<|_m zRXIb3Dk2%1Aei9nkRzB8KiX+~hX@tm($MOjR8a}2Hkv*nn;8%9z5N}3sH9k&t(6@3 zq^W=t5cl2703JZGG$k9pngqnlk=M*wWCmdYi8|y7dm$2RYb95G3O*Q<{_ABh6ufk; zrovJ4{7z1CR=};kV^6*hEm5?VRKxE-sAjAc??vPhBIzSy;yG z?%Bi?FAA0RhOG?%0)QBxHz7#;>~)V!>^9()3)|9eRoAj((}onV^V(FcCC28HR(R*t zMGKiVuMO&CCnaA)ZF;%u)yo~5)QfhFrI^y%q&2zaZ1X>y1rst|2V3)IQUu zhY|6eeJDhZKv;Y-&gVkW&@?sze`}taq8f4=&I!>DjbE-d@>96f5HEsS0;!MyUs(2@ zB+n<08vbkQ*cc#SIj>1z=~Wu-?NmK>(;iV1SOD%Ymt4Q5?F=!{Y4P>#E9&8Hu!bV^ zXNuz`b+BrJONNepT&?MHMb-Im$0-h8h@ZMH^(dw|EBh=3rOe5@cdBv!v#r+Lcr+Z^ zw-0GRD^-FL1oqN{&m?>a>~%?=Irydi;?JREFxdY`e{p(vwUDCD*? zM&uoL|3d-0G5lv>%JAR%Mk}@c&Fjsfo(7~xSx|Bq<>O{b%&p(_T>&1~rn}%_7j;!^kxRV4Vy#1aCB4yAfz%MVb29WAgR?z` zn^9NT!GuVzi7fWYWNs}p?#wI~dmIVt^{leeVFH3v3<3FmyE9&fQL5>Z$C>3L*Kp+p z!tl8o4tvZCSi_%0LKbF)1Ds^s>&tj-#niOLQaE#~`UMjIz&JBY#zwEv#YJ2@cbW<289B8GPL9i}1E3C6>orroMO%Ogw`+d#-`s_iNIN=ihy{ zp}8Tz;(W%C50NeGtT4e1!}2B@2tFWCezJ062?(gifJlYddRjUscaYt#cXl@H8X^F= z=bBVN(^9z%$-oi(aQ865y-YSW=IST7Hn^EiB2SU>kx^Amb%XDu2#`C0mFt#jIdq8F z9JJk4!k;^4W0QLP8jB+~cV9}O+uM8fC_sHmVc1Y+FJ);3 z{zY}b>okl*86%Fu)s8@hNCvK)Taq}E#zjd`Sjxr(&LWO{Vc5_sB4*Hvg~O!AP-G#m z;f^bv@9PDX8#MO7rpC;%VimP-?aL5H*Cl_GvhU;>g#mV*P&>;R=g)eucgoRY0i2IfiZc0E1`>?n4 zt#YfVBa@{bTqZV2$0l9wtj69*Qq=GWg4q@k$%xFra^$jWLO1!@7lG=3SlvM*za!Whv6l*5W#%fFwyT{h0$*nf)EL;tVor20U zZPh~SoI~GVMj)}$IBYK@kQ|{w&$iydN(NxlM-EAMP~k!aPL_7_rchqD?NSXs=Z@F4 zOB;&}*%>-c*Y53JSYv`$7#4D?VH1RSoZ&Zx3}fesIm5Z2#}}n6#%^`LX%0hzlLGPX z9=6$q1tP-yE}cIKV;{H%oQ2PXu~Ri`pTL*}V#9N|I|XFNPX z*6!sT7nSK`9ZtnaYn$)XB5zDn%7JekjjUoL`y$BWDi&kgL3>$(Se1@j_&KYq8+2~G znV3VyG?PoxsqE7z#DI1a&ohY8T%FjN>a|D!{yP~)Wwd1}fV(gNyM1@bMmLBVcl1wm zDCZhXFF%u!v|jPNyYpoSU1y1{?h=%f9dHf|&Iuk&o1_bn_wJP)t=86(;d*;|q+y`dy5lqiW&6bcamNs1K@HJ> z$$mX_1NvGRa@cKlOG_7HGma$tVKgQZgI|nd-fL*z>)NGMm zfspj=y-=5E+WM(+D?7Yw^V#2#*UO|@y|(`xUEoYT4A4v#x4i@KJ6+c>m!o#7dv|cz za*$R<@uMN!%2a-|g!X$U83OwG)od4!BH+}j#Q7|ckl^N8wSG}2b(18aZ3l2=Yunvq zT~w9Z%^1t4&Nz@LQHmM@^OA3Q(%jvkVY@J1Z6M%SQ&%6()z>PYqnr2qJQOew8{!#Q zc4vy8qYIq5`)YNS6ZT8M?g<-+7jkiOe{Nz7SEx^`IA%c79TX%lx7HvY%!1_4}Fu+0|IcnKD zDkQwaa}sDpo1fNJ9^OCnQ2F70pt@%mMO6_!ebD3tHBXt5GID{v>_O2Yd z&P@|vu0nd(;VNfdm6zXt=1qBkcGcNRISR6*Y{JZ)3q&3eJ&#5u6i7@rHrlm`>-sWxRsb^Lc;{~&KfOH%XZLyM#PkN#NhPGe1#p)E) z*YG8XD0D}(VnTXJgq6N9N#qme)UV0yft(X}vu||={5}@+%!-09uWOy+OVM^)xYZp^ zD=7FH9Q~oq*&Wvm0)u=(yLU-XbhNNv>JWGVB=m&e3WOAd&z#JQ4;bh`?P?fTgL+K_Eh61*A0Q0gJ~MZ;+0P3Bg}Z=0P; z)y10Is?!*uZiITY$xK;75Z;GPf(q+{aLX*sP{I{c*aU7Y^B@Q@S6ckAY8hOsK5Vl( zhgLwIaNq$U)abBty?HQo4T9LV#!ADm^>t#2{S-%>Er1Nnl#ZN8+KXPl)>1s71F`L4 zv-u{zU15*~VT~E{)~>$tJu``<$u5i%B8yc zFDIS0$F!=NyhThDeE8kTVTcGCq^+qt)Wn;VrC8OIT35Q)BmwqZNlPVAZSIlOP6U<; zaoM0#3cNzzy+fS#WtDkf4xSD~NlF$JLNv(yxY?SjWj7|PM~P^NBiWkqE(Id`D4%Lr z=9nq-K3wC>+3@_k&%ERu4A%AS zc|Rw_FvfaFIu^~gwawCCg|1$ee#kAgA?S+ExaUNc&!Hr*leQU>mp#3Inm zv7YdJ<;76~e3xbGLflXsgVwiJvyfEhK#Q!)Nz60Ixwltn_M~;JEyBvpJEES$%uA_h z6@YsSeo`9v=6OKKyvCf+AHzo3kV&dSPf6NNc9xO~z4d+-O!si57k%4NAsqr}m4DdZ+}P&uch7CE z-aY?k=ZfPy_|-rBm8k2g_RDwgKoi#F+v2(`u+1>qqk5ef%#EU-|O0zN;R-i`z*m ztRK!lg7`!XYWBNL#^c!g#QzmwS3`dIyIX1oWDjEH^x{<`m#^6d=b|S3_%nanq2bb-{Su8E$iRE||1iUXDV1s>;C4kkVG z91TR`bxlpmqF6+NH&+m+i>-<#60HRcoi=5g%;Q|lJG@ypEupmqUBRpe@BJJuW|QT_sEb9hZb3ulPM$@Gd*$uc{_kMO!dOXV+|mHA@*@D`ysu? zhX;D({egH%UqhBWcY;Jii@gDO6dCE6H4f9vt&;WJ$4QNgE0Zi+MFsxbj%>gSBbn|Q zB?DYRrx`dltW*phJA2CE6YmY`vs`wg=RlY{trdW)+bAnqZ>hqY zY2Ks7!BH^G9ayj)rR9%rprWO@9pH5xpnpl`sI2j@4ADhWD+qFp9f4wT~{!{b!M98Y9xpR>t~D$eDuGM znIo9-^B$4^!A^Sc?Za#jKg!gNtn8zcw~8{bSV;o57U@11r`om z;&7*tSa(=j+9#x8^+73NH}62)SaV{NRAcu;lmn7Z;+oQuJcHeOw2#$N+FR3b2MbfH z&u8w9!1O)qB2fkNIHj@Iil{&kJ-e8`0v2hK@=4Do7csro^j?Ru^Ma zZVxm7aPXwQ=zV66MiclPsN2UF5B2EN)B53btAV!;bfRBso3^?x;sIM;r=XWY;UhN{w+QT)@EW_vDeP#z#U)tjO{_arM zJ(UjCb6pNjB=`qYMN|1+<^5{T=B>84hg2xHL4w1-Ar}e;$12llV&|&9>U#bqBX$Hi z)vD_uQs8vgj7Auz#J^G#OwimkdLz%(oUDE5!3QMnl9^dX?lcCVK2qBfS=|PH63xL> zR8Xp#aF!_XiWpgBI~-IDM`JU|fb|xu?FRzj)2!En9~#O+#G!5QBUEel0{-8lEJ(sA zNF-@j8=`h9Y#7PDj;CU$Csdl%8Wp(EDu7X&BsNmvR9ft=!6kFGiXO8fA6u92XTSK% z^)BLU7k~TeC#K*YL16#KKgXHw|2w^lYN#t90q8|_;iXv>Lyd^KD3An*LMH17w9QF% z=YWcxv^11P9nW>h%$ISL`ds^)&fsKTg1TI5Aj<43g%+Cvzu z!*I&-brIW57HAQg6xS?>0=f*uvO3P6l2~1vi4D1Co+MGGiYi|aPEeOy(-@3>h$7)I zyI{ZR$EHa^&X-G0{=<&3L6GDcrWg*_ICJZtfA^XF2JIGTuR41sV(ynP32h)u_;C?1 zwN)`Ced64;jzMIX78;U zfDcoECcqPPv}J(B=rpW}5F_>-Euh*FGUr01*S=J)k48LBD)`lvh+Er*`!$8C#?S`J zV?>mDow`Gw4dkh8J@q!U%f;1WF~bfIyHf~% z*nI5RU+BliR+~ItA3);d^x;cVLeJ96?s+@(U6S{YX#1-F=N!1ZW}bAeM)%h%tBSgSWbUrVvRp{03IK(9s?Q@>I#m zt&)Iar-cMVj;2+4ARI_KW|WZq97CxvF|4U&g2;_k#MT}(>(E`H8-EBKr!?N^bt&8K zrUD(rn8QcEh8ZI_PQirq<1xh_--a18wzjc4dT?ty@Bi_L8Nb9rzXFJ4e}BMF zVT4?@jrR=BZLKdTSo3QLM*PPW#e4VrUzlIBeqFo2qyFgk|K9K$MfscMZ%*I)f%vMp z`PFyvb!$KW;^Fq)cYgP)-}oc?)v?8 zQmrWR3wZh?Tz$3v^}W?MwZS~S&#~!mR}nq6^=d|qs(uZw$ba!~TCA-R)>EW=xKDrk zYGApbo^;Bq?~-#7XDBCs@t5t@t+;Xs0>CplaF&PQ3kv)n(;VYYef4AXl>8$?RIIAY ztN+5X{>wEKfDg=P-~%JRUlDiY_Uq-HU%vXzhl|p-gt#TwoI`eB1#{L}Tv z@zMEUp2_%({5I<~4zU~jI8N>LXY$|qu@Tnu&ibY6Z_I~jI`hTTueE-cln+0J{==d7 zXt^3+#|K+I0$_=K5fEYfVD>Anv9Ub6{2}d$0-JU(Hw*O$F*L$O3tGyi>0sl!^JJJJ z)qd(`LD2vO?wJM*xOCWQkmteJ&FSbMb1TDC?McbuxHtrwv|GxYV0JRu%N!ASGFrNv zqtX%R1oLpU>?I^4V~Q-Ayp42>rkl){WmB&K=RQKB37>fAL&_;l2x#wzTfX+la%i9K z#m&bRz1!&a6JG3Wkcal+HNfwA>G-ih$4EIqTJdm=Pd}IdW2|cm=W|aQ$b?j+nF_-i zV9{kkKlOo(#;LY=s%{+^s=LS!kjz6*Go|engY^VZ!jDUSTHzzELAb_FxAr;C}3@wAxKt?o;-i zX2pw6UJJKf`q&?mvBa8xwUwMpQ&n6us0z0ODs@Wyllx(8lw3xHWz<_+=l>Ww2@#)t zU_-F7D+;p#>ZN~>)E&JR^4a+>|va8{W)?)UX+L@LarsHM246<{Bko?+196j0aLaEOn zbl4s+>INk8ICd*2lSidu$ow!6)3~g|AxRu_$6R`Sqa%nl$b!MzW@f@cP~dkf`hmFa zO-*&PK%k#;5+|D)gXo;3@G;cPVtK?zugV&hN~YTFMIgn}r)nAD5nk}B93ebU$AwML z9*)Y+4xHJOw*^;0lZ&oXBS{&=Wt85q6fey`Jw$zN2;7(T3t*bpg%*HW>l;uVyC$D_ z8t0e)fi5*W*V;yg=rTeMN zS3lx{hKlQe=zr5=o@)ei^BT1Gyhu2+TI?^y4Zid0>z*N{U0Yp?|gXQ`T+pBGHi)jO51ZWH=(jdv8$>446^`9m|-RtCANJC^( zUtzay(VupgVk_{4Wdm zCq$3cfvuW97kSDFfpNc%R5$dG6Wm9sSF~8fxQ|QYWhgjiX7`|ru1yN!;^RG9;rGiy z9lMdeFkv?>w%0qa!{plWtjb2YbtsmpU8KoL*xsw}^LljrAcXaGSon|d|BtSI^4+h# zeQ-P@2CL5rV$0X)t+i7!gxzFjh+AT`PPTb>EYd89lDu_*N^t|E-9U5Sx@xr{p54`E zhpq+fr3E>$kch=M3q_UtKJ_6L*)YNZKwwj~UtNpo+%$oghj*e~?y$zVD=K$Aa! zn8MoE9bu_?)%W{KX=M3_s*Mw;K=n1__W*<|tz{n*KmFT3`Rl*>2k(FSr~m2S{txf}_#eLi`p@2f{g2=O@lXHe zAO7jz{K?<{k6-_f@7MqRllNc$i}zpuukU}#7yN_uzyADxd;j%c{muXUzpVfG_22&w z|AN2&>EHabKmYgt5C8t>|Nf8v<@>Ms!~gXDYyR+0-+%qj-+#?F{u93OWOoWmMf$QY zTG;Rn-Y;@i6TWDogCR$RMmqPEGlLTjv{r5)&g?Do_;(kr*BN1f|+r zZ23EqiEeFqe@PKKkl9eA!SjCQ%|_bmHR4hpuJPL5^tx#b3j3RDbw1JD$F`|gK7zwC zvSU~nq_eh@DshZmY~suNaEgX$%)JB|O`m3I%OK79>bUsRfi@wcZoY62S0v+ zlXl~d4i9_Oiv*H(55_%`IPuW`-`oNVon`#_ft8UUWW&6VQ@gV|CVKrWZG_HotqLwdz_ z61jwPQ4xk6x|4(fDTeeTssRU<<%~f<9}r<4j{P`n_WvcB&~@Xs`0B~fofJMRb%v;H z>n__jz$?~84zz>i7MP74+$O&H{_hq=M8X4~<+b&6$^MTw3E|SN<>K_*@)&%~jCl3L% zw+hH!qycS2FiKnANy(Ry%XUav`_9$x#oYtZl|R&)?~wNq@)v{vR~{Z~ndSe%1cn-yVK^V?)j32Rvhkzlr<3NwofkAA=+E zlrX-=+B~CT+u|kn4>R}995|A%y=#}$mDo_I>)V@`%+jMdr~!smxGb02$5E*9+X>To zT(#l>L*RPCu^->k5-g%9G3``_5$Ddn;v1ROPG%=o5lmPb)OezGnZ4^67$=^e;+eQ5 z-E7!L2I@&{44}OTyU2U#3Sei$1~)r%sG$ERl$J?l&M6&&Ora|OQXyZ>DG z@Dsye;rnIZATQWa6|4dF&)#IsvitA8xzJ*r>%|3Ay@wg#I+@)0xq?dS+0V7aqn6=? zoj0rFth24xU`P`r*scyG)^-3LRlLMy9+hL>$9dYjnoK2Barj>Yl)D&h?Wp^t7lc@b zFO?zFH%nCOl?2$m;F>w5(_57lYhrn0< zkYX}HtR7nVDZwRnA@Xx)%>Pzc$RhYp&1Q>=8;TBV=$QQgq2M_9VGxIk^lED}*Q*2-4b;zF!Y2}OA{IP8XA97{xFiM?_A z$cM z%vd}WOrHWvOcNHeVB8s27HV*S*+RO(%=Q(w6mI7x@y@yTR$G@o3IX}2gRR83e`PcN z7lAUC<9BE0#6S{mnbw}|wfN5Xv98xaapVhYg>S<#kv6VdLNN+lW5raJ`1){|J5 z0~3%S>|ilJBK&~RyL)i=jG|*q!B@Kd(Ud3184KHLXo@(umV~@iw9_?uXB&njG_^cd z^al8qSp8fkUOCs)fm>A6Nx8Na8GvIgmSMzily|J$Jh+(%GUd6*FKi&B#EakJuCcXK zNcd0>*Ldw!z2x8~sp{tbP@-+`6u$!*2g-?a^>+zQCZD2rV=~FfmVniMP?lDJvMUP( z>igl}HvIR&iLan>Lb5s8P28lgT`#ABP#x|Lqsj89XL?6ZTqAXX0Y?;56o*+eh#~0R z7FEdBBZ^i}=zs{kwikZiTM6urfwyOCH@SoJetWkfTp!x7{b|lTrISd}mwo(V{&5o&2ZFTC4h2=a zfnc^IR0Fdc%g?@!`?$|{qkU`F0+MH3bGokGGHQ{d^e|DD4!V@p6hO-uq^k|x$@ zTZCk_A@R(kO#}i%tj$KhpA{zX%!0C57?hl{#jU(U2DkyI*bD(lnZFF^B`AFMFFS&c z<~mQ)+chsJN((&9Upa-2E^rpuC+J7ry^Q;M8Vot!kRTe7=*o~d1|Upo>~T#xU9uP` zBk{*V(M}END0ti(W4h`=;% zIq|8q#<4!(;{uHGQ#Xb~Wjtj9QV;&msZYpfLGIiV?2CumaUPnbwzAVbBNT8R8#F+J zbPE~SHX-gw+4`PR>>32x+IZp^O7K$Yxb*UnEFDGlw)l z`kV*So(-+4d%M-8Z(`&I$1AExqskFS6m-6gcR}OY~QGKAnpHO>9dJ0HWB^z;&q!^&U zAu+7Sjdf#ZRY)sfWIswZaS_)2fwy=()?R-@1UDCr6Re1BDt!Y5<_SK0_8W5iVeZpj zM1pzx{xdrl&T)*j#aSARameQxbr=%1%!^`& zTeS*`O=aZ`LI-xO$q{ul$8EHE-j)#@(b_0LlPq_VmLYq|Z?ZMkP$+JA+84dgJzf%d z@s_d1R*4+?D4H%BYg|$yGuBY*?2~rzxSdIzt&P){#9@hWK*t+3L|Qtd>f$iVvE`01co6tjo36R&xP3gW)QDyel0)kC-q4{I*j)dsQy$&W z(`Iz!|A8OGE$NVFr-4yDi*CUc%#&De0SVTH!;rrefr|vJC6v{W$$*P}kmCuR0446J zg*B;}y~8aMb{bbA7beiN)%82YOBPqlLOLNY&Pns{LCMTOQi8@3IHWoP=4^~^t_wmb zut2l0l=7-9CnPi@#R{LRX!9888!{Vs3_jM6g{+4N-(@C(n>ZC#)oUK`C7UEc9hjFa zmst=W)%eeLNiLIh0uTB1c-SY_9$nzfHi=q|RLUdo!KWr+!O13W7SB~KqHx%OI$srj z?MagmRsmRQGLr0<&N4vzjj6)UEs{!r@Y1= z<0z;1>F!&)YdCXS7|Ur@lhNo{Fw%y@tn-CAk(o!21`MeMTI`Z2cZqMmExp|Ln1zwy zlU0tx@{%riq7I~;Ki>V30q)7OG~nn}s^R&TE}m;uu;2y|O1F>KVwx@keLT6O#B-Jy z`kEdn%baBtmBf$Xx71bI_HLv-A9*mTm-I6B@!_j%fu_8GT>9R~|W$(Lg^TQFK2 z^v1pjqj?=hQ)@=!ojfoww_!8@u@xpeNWGAy4e?d_;jh!%FC}I>z!FYWR{%I;-=u&X zrI!7SA#}JnVJM;zgY}_fMr7)CJ8{S(;(7&gXwvHi8F$}CL@Cspf_eek!MNj+c?WR? zHM+{@YL{;SxHmO<;)iRT*~OmsJ!cltA@4w|v;8&}PPEo}NTx+nBbcz-2m{DQFX9c%?UukpnkJQee0mP=O7nSZ3$RiL5$^Mbw$&oR&$A=2|Uv1u)3Vs zI*U9Ev%H;W!qUhW(>Up>?a@kV=S|zk_<({W%s$}^aE+RFOO>GX9Z7FmJyP2Z+R$?= zO*5xV(Gz@Jmi+NeXy`zuVUv@~#5aEWqh_U%Ykod+1;cYv4DhA&XAAV9z^+_R8H4w9 zK_+A;ejW`lSOQ`gGM}kbuKbJ5%N+yJvZj2^|RFHoy1Y601JC7-I zs1s$eiF-ryTwO9Qf+H5UmEH7)J$0j~Uod}P+54a6NsE=0Eallnji*wNcd8E!jYJJ3 z>QPF=S9G;Ulz<9_Xlo41gqPH<*A9tvS8IapMg7*mINYHUg+}tU4mE_3A>eOyIx~eb zV??ldQjr!nJ8sov-iTK+y}HFN?oA^BFYyoZ3j-q5c*9h5VUiWY?uvo_Cn_RG)_47Am=|eGk8?arY6~^UG#9NL ztP~U9g4DbjMrAV65l4ZLqfYR`7~Oueq9iD&nvn;Mg}Y#E3w$yPYg6 z#~&1I-fIbm`)mdXdUOjt;^%6}kiCRg6%uJ$mrY+W-x8`vY~~;$mlbBEGIP3R%z_3v z=GLZ@%yw&tV+ryazc5%AQEU%(xQQg#tE=kmMMS2a11lZgVg2QsJwAOW9pWebJ zPUD4@#9?%Ay+T=&v~gGxCIYLLz#?{VVa{degt4|kX~R*Q)6!$qMdXb&ixM}g9VEd` zf~l;Swo!7ps8w97?UGRMW*3qQry~>y8&p`2Bf;%P#CdgSBCo_UJQ5x`SH-OBu~Q;m z4TKy5zI@cAfru^pU{}`-#VWYw-j(I4=cyq=18rLnj#KV7s#>DLSz{U#`1?VsqG0r? zO_5S|4VL7cy8uS<95#S`^tEzI!!E@}&dqg&h!Q{HZZBBD4E9W*_TjVf;U2mE-y?k| zzWSwa9q3^{ET2eQC+~gWS7o@8f`=vPt!j$dNll&z4QLHGqKNEr@u;-51z;?Dzem`< z_1>e7NGPN@n)QqXT?DdhvmxI)Y#A?$l^A^4)^XS(KQVy7v^HeT?XBaj?>!Q-8yc9i z*uTO0&`olIMM7m6tk#>Eh^nTxK__DbK%-1a3h+kIl_C~19uLqr#!RiWl}H@UfjfT3DZU;pC#EG*_y!V1vPpF;SHgn>rX?J zE*-A*8)cQw3M6?;Z0o5edO92s!FgoH@e3Ruwj|YomD=_t*nba{3|M*@GJb>_Jck{) zjH-jr2RpzJyS=8{{)1G@G!Z=~IvJo5B`Jj{;Q*?dbc&C6+vg3@-P@0`Sduo;=>po| zn68Rqrdci#WBb){U7LtJu+y_XENidHnb?&&?)xc3j@1i2XutPjB=jNN1#D!Jx>pf? z7^cSWY;(aXUFZJLOT1^h6yBg7orNs+Ew@dSpICX|=kc+%*&WsmBGQ&lF(LW*#r*Nb zS--jDo;^dZp*piF`9H@&by%?0-C55Dc3Z^Kz8hdNjlD^&8OKI#gi5A^($P&E7_o;U z9(%7&>Z~1?cCQce143{oQYPXl6~W;7v#RW}+-#U^%sp^~*v!yAo2k^n1 z{Nv8oIn0#`&oewOdNOy(I0s~-HG+CF#X%!Bt7B}T;%dZzxh))ks@jf=%u6?|Jmme& zJysr<@qb1!A#!|!_rXRWozIv5j zHAB}@1mlCedForUS4XRbM6@8a#cnCJRXVom_9h~7yjy*5#$ayaNe_%?Cs!=G3fFh6 z?}<_7eN;KRVuw{78C&^9NJNVu&hU8!zdxctc}+w)eo=vnV&=$D#;D_@4y%m_IiCr@u$m@ z^oiSRd!5T|lrJ^;SQrKZ7W-XC929gQYstOMUxXAp;^3?Ytl*{phXe7}@}qIlMr1>0 z&w{=$#SJEy?14FSmCn%&pk&CP!u)Y;-DRi5TM*jMKBQ_gK?x^x@+NBx9wK`WyBVU= zP1U28avMYc&JTX@15AMb(O(?@-)a3nfuj;05|@5b*^~iTD@Pub^NKAiP(ekKYUM|$ zM5jS?yx_q$$uuFZHYfTv!se=(in0hs(*)IFh;+=CSTo3YK`t9$G^iRsQD|#7v&Yj~ z_wmoM6Wqf1KF|G$B%wSQJ%wHEMv|Dlti+HdW_{IQsn|j$TKdZ{T(_U4%S026}aoIoh$c84z`kEttHJPP{l#l{(M4Kd;P6 zPDfQe*GLPN-33qj^-&eDDj_!3Vb5oof&IZgoQqESZCE6V`1cN5K-Y^V?Tjtzwu+9Q ze$NwFA~Z1h<4JAaL2GX8nnQww`803qc43!Kie_`Ih&xzvbl;bpnpP)m2at&0{sViP zKfbN>wmP`vc+OvN2VXD9AV}O@Tly^9Bn)|zI+E8iYOZ1m25IRU%1K%PQxbkL)6eK3 zh;&aq072R!ikA zyfBv03cDcAaKQVNCmnWP+8wU(>ANBfGSID!cr=c{m+aa&nE^|rk*ElR}{OGRoz}oT> z^L^wlQ9wdo$g3rLmx_c}aJEWhPK}$BF3@N}+~}shBhEA)#SA@fu;CHnn` z`sjJ~Cu%W3XL83szBju?I7+WD*Q@iv$1moO+uO-GK!%~9_i&jr@7d7>&Wy0}?7F|! z;-yylsd|Jm&zt2EpObbf9(c_H!a0)kr10^5V2B*J-q&eeG}$!iV05C+R~BPXYVXui zq)m9OVtk$oiJ<8XV=4&AeZ#HC~32{?qRMQh92H=eW#gx%+bMu`Li> z`}S|$+%RJ1!YG}NyzxEX>F~pI2Ml@a1&AYywAUA>8Yf5wjdbf?*WJ=tzSmf?X#Hs( z5|S{ZEK*F?MkfxkidkF{Rhfx5p8_PclyH$@P2^2V$e;Kox$d}WOna{cRsChgw2K?G zq$IE78*)fr`i31*`(59I)&=!=uoKTpTV=W$FD|e;YaVKqskJ*)UC<2^D*ZtZNJzQt z#`ZR-sUTTn5LxZUgEdXcnk=lr_71X~nPdR>z&qVz3~WowRxwilPINF^VLWDalf?_# z0mA6&A$G9_*YG`EM~_6;pq<$0ntUR4M;zsFxvd=Hv!^yEg|ge43~G6QZ!s9U#F2hr zFLG4>e2-Zhu{Z89X*kXkG+;pAjgM~nxi17HEsvO0@*ldzArbb_jUD6FcMy9}C0w*U z(hbFvOb$Ba%c`v>CiV-IqQm92Ib7w;`aED9Pv3uLb0i)a4T^PEkLnH!^%MlBL714h zJR%FSrKw$!kSeu~;~82er6N;=l=nsw2R`oT*7{EJYMG&L@ru+j5|*+i*~Hmf#topDtp>KNub4sfzk}xF#kWF>73>apw0@RQe z5=0MQEXUX)lGv*8aP!{$rl_%J6PO$4VV#!YrOWBzur|GIBaM$ z&Y`Y^PcxCu%2kgx4=d$lF1cSgF8|=-^m_4$NsjU*e zY{5H~3EPt45Wn+|XFaN20)4hL*X|s8oe5}}fnDKynkdRWMVpdMMLkb`f`4((G;ZC_ zc(Z^;xt}ls)+EeTbvCtA z!}dhNSU!;}hG1cAhMgA+gHTNFa0$mA9?2-2#f&u{@28Q>Gm1fN*}M`Qkq6B5(Ua71 zh~%B>DV$+#_FlxGXpHhj=GijD&{`}8%(2^dVN@gq6eg@5H;>BIi@ZS8OvX^ucIXlE zg~aft*?6&(f*fhxsf#)ZM$9%X)!{l9!TB4|&P=0@W?{X?YcJL%(2k|a^Ev<%pTmXc ztHv3S^dC}nx6GtgtopeX&K-~N6Fhi+o76hz@0i~!ym&6?yC63pa1P&{-y79IL)pJe}b%rCF0{vh@nz3FZ`(03Q#w z{bqPZ_`34{9Hei3QYN|!k#I8sXmPjlbX?U#IAoI7Kx)!8s=Rau9Cjiho6X7Dl1Zz| z#u;mdnoQ5M^T$*q{?<~{7N1kxh)dc=Y&hzXh(K%oeJ@&Sr$gEG8ZFsV>os0mYOkNx z@=Hrx^9!6it)n$N7wmL`fk)8wnDYS?mUYA24@<7=398l{8*$vWLsjqPLJEGSo7QWIjW$}u5c(jOq5*TJgS8apN?OV)(W;0HtFm7A?; z#;KpjDL#E^ScJA65dM@fn%#IKwW4*pxxXmPYp`*d00TU{+28a@bB#Lte5W%!=OmQa*{HUsXx|pxsbN&$td=5zu2iC{_j9dk_@8K%1t^Kd1*iog36FTWy9SY#lG8GXZ(3=P`AcNBi>EEN#%fr5} zGOb6M6$BZ%MHgpnvmGcG55zfOMN?rtn@y{EUf$uBzRe(EW_(Wq_{Zh5dtMJhh1$qZ zlj9(ZWx;-s;~++$|HdRa#^NPU0~W+IC%7-+pSg*d5vzbff-JlU=(7rYIG-iL!_DE( zJt?jHW*3dbg`{-|=mS392mbg*%#3UW8KS2+T;{X9SWskOUFesoolh^8wVIh=$(#>U zKD*j;0Ej!@k&_pTv?RAsfjux<;e1P+rCv&>!;Gx9h8fmVYC2iDL*ix#zG({Nb|F}g z2`uwijJBz57*4eghq}!Uh0!q(WMf8hZ`&V*-e+`P*gYyNo)6QNZ+h%nslZa{GZFC_ zy^(2g9u9FhEPaR2ih-TiI66i;Z>@qzC?{hwcSDEQ>M%5L6`cySL_-hT-oW2lVD%)T zKp=ng(0kFfMj^9iP~;G1oIOkt$==MbGc;eBu}R*^W)YGmk%>NbECR9#qu5+$8H^#O zc+xG^7(!gk1p^ z(B{NB5GLV-&O;O-VUe^z4$fi?uRVWSzmm?W`o*8qDe^CHl>EhC645~xA`MhX=-}!x znO~t1F!;t5_&~$q`9J^p$r1_cIYIwGXOxq%Aey)(=*1Qd8H;UECG%Xs76T+Dl*1u6 ztR-V&vj@>hTP3ogbS;~pP5oLGGwBpt0&=ZwaSL4fk<)c%iP$OpHC#77af2dOCCp## z_%A>CU~(hhzj@h-JZ(^w6cz0qDr|{NyCVR`w(#zXPVPS3$%4qqaADjU5vgX)6P~gt zP{HxQID7)3MiL+eT;c!<{V;^{yRlPznb2?fuO~ za|t4cB{A`Ds4`DG!1WrDdAP=zv*9@gc&6gOHevM~d3i7#p?H1({SOFXbeC3;?u36X z0vhx}uA#sqy*Z9WkS+mEdOa*%cXanWzEG+z86bKY#0y<)y1LgcI^E??hJ9QpGW)92 z4iM8BPgRBh0xUyPhI;G7;Rv?~p(*H~9Op3TjEt)=HDnzubHZ`cvF(j3C11rBDZqIR z*_rg{&AG*hU2vGNtKe)hYE#!(GKntRd>iyV;yZDf0(~Z|4bQ)e3;>Rn(z7bG=Q1_} z+E?_dM;AEzPOK`LyPdT-s5k=+nIcA%SPG>T3K`bYX%!l%VYbsO9}CWr0efRi%#wIa zYl_%9UpV~1VSUJreE}8I!<}e*c2c1KT6m{IJj^@X<}hpx3bIr;)}p?lRI~3MNulRO z;r$ZjhGD>`@O0K|NC;r-H9p5$#KlzJysZ9@)}rG$YIZ`E>xrcptvP?ywC?G8>XS^M zNQ+umW~PyWc5zOKWV16Lw5A0w%DD*_TCilcCO^uYG0b*mCwVgxJe|8+JRKh*enlRs zTqHtaDN>?}e5tf&FCwpq2yaIl8x)L+h*(lp-MMme*G)`hkr=FWIM?n7LXKIGcHB*; z2T4J0--wU>#s|1Pf%-L+PB>RSjZ&UllApTZ^_h0~_X$~bu!rJ0B;t1oeGUhJ;mvk< zDi{*VYK7$G!ntaC zCWl8Gm=|QdK=Yae##)dHXO2s{>=cutZVJ>*ZN+(5*)DUTJY=xYB0Jh@qnVRQg$oeB z%8YwPHOYBy=V03s4$DeK+`-2NX1|fMLIT&sCz<~ZRtV!V|4in8)Z2p0z_V(5@(#{>G(CklaD6aa_1CUdQoSUh<^-XOio)rWqYd643ca)Jx1RiEpIp#$sEVT^R4VgzpqjD5~<}U zAtpD*!vl$CX?D^sgfA!ja+>94eKJ%{G-U8SJbIVD9xFTtLjJsULf)+`8mzJ}S>E>8 z=fj9z_2U=w$89jPu^7vYyv;4GpnJlG_17(R35GDUnX; zWD^UyV3G}Wv5qfBcGc9RQ{oea!iU>f-n3mBuZF}zsgxpg$eK2C0yXWNkl>R2VznLB z{b?my7m4OwqY#aNu&cqvgUT3M8K9IgD|{R93XHQHBVV@R(t~1cE$=i=q!I0G^1AU^%xehJ3vJJtYe$?`ho~X^c`AmF9`n=( zLx{HY2<+gxALTzv!C-m#Jg4%X!v=kOw=MtS-M3t`xE^+_s54<2dS)gheB-xMg09QH z{lE|UY7GE8fL!AIK7a#pn2lisQBIDjhYwrfC&nBw4E9_PQk^Aa{XY5?zoZCwkh{!K*z=-M_f!1%6^B&d z_n5$@g|nbo@aVcKj}Tfljd0X#+&?YB3?TSj#66)l zq}RF()I971rxXF0l0+P@1c}%8f!8Sl1VCap_n=+)ECFEO`K_Rnu>dKmOqt)l`0mzrOblJ5ne29Huy?`Qqh2Q2kzqaqzdJ z=vQy0{=@#}#!}<>tBkx3d+1XUZHi}EnXK0JVh znPxmuiVEL)y+cdR>;-?bVXkJ|>+xCVCGz1S?z*|s$aFGaE8CN^%9~3q(AZ=SlpY=; z^rbrUAWg(pouD`_i5Q%cNcjz^U`cG)DmV2te$i`PX@Yc?IGM3h<@6-GZrRw4?r!j+ zVCTMUb{7@a(wsFcdNMJ1@GGhsj!Dihww}~*@^M)0copq_BV0A2K`u`Ii%9aFb33pS z#{_d$SfNQ9^5FtecXSmIV0tA=v&6WK#9=kII6S<*q=6ZwT$i-#6a}Fq9O3`j;|qT_ zhtTgZM6%#`LW|af%^L^4ZeF+~SN2_#IGt~BVx#vDsgb?BO9~FXS2q|-3w5LTveQLy z_l0%;)$RRyBttNm@Aim z52)X@*H=71KW^!Q=N~!;ZE;3~_R4F5^%wa}l#AIbbTHwULyz@Pg1l7NP%I6*aLP@xEQ`vxOF?CwfmDce3q4zpj2W^eGR!0uEM;nSs%3^8~EK;3BlVv;td zA|29A72qu^zGPNWQ`(|b?4 zBOAqHgd|Uoj-Z}h;U@H7shcloYEZAQZVL&>NET5qmnaD=8l}#|jW_T*@(#AzoVXyW z&kKl7m6@1*yQ=X63FmHIi$`q^jQ~JkYlhvUuFD#$izPlLCr2G4p0J--9-}qk)U0ct zVFLpDI>ez~B$2@>Dsmj3oYcucB(puSdDLO=Q(8!~d-y*-##(7AMy!>#jyilxZAu++ z2{(7JC%<)#bs#rgRM?#ELF+a0pg&yWwX){*J?PMY>+^dMhuh5LjHzV~k-$N`iH5>c z;f(k?sr{aWNF1}0Kvm;x;gOQGuunGM%(q(95)_b(GJ=nD@}#Lwr?Kf83&~1}L;g@l z0_2jIDXY##tW`u3#sdkde$aQ|;7KeupLEWg!pdC-i(Nlw4%D=(Da~+*KRJFgaoVsf zr9VQ{`(xRD1poQe(}ruv&vx2aE7Wp#b9ON31&LFSi-RRfgZ<&^hU9uv@4^Kjszhhr zM5&IGX{?uS))8Jd z+}A}a+nk^|WYGqBORHyBF?pF?p)eOT77$ZVt6`atM+U2wZ|#-JUDfb^u*^_ zPOpuX&j6pfM7zG_^vIwTof%`^kM-^Dpr>E3)3iI5OO zoy2j_#S8GT=rC{ThtAuz27*#nj0AygWPG^^DJeBJ8u$~(#nSuq=H!UleVmrF&igLM z3@VvJ!RZ-TWj)39CfRB<7)K(lTH*<}0dI`fv~g$N9P`OL9JB0;3gp3Fz2@D6(E2ib z<@D*V*NE2m#?SKfS?7wN?t?^Vw?KyeUv(zIYra|&QdFJQ*Fk?L!cXlQIPPjE_?>Hg|cz$;oH8 zOPn?=&$mZR1&jxu*sJV{C~OJonKI)rPCiM>npJ56$d-LP3{0J!upI)6D{d6X`ffFM zlQ$O`MB2-Y<fEf+BkTGUeV8 z`{(%?%w?E^zb4*0y1pHlh4!W~n` zE#ZsE)LnHv30on(g4Yurlu9zU1FOa$LFJ1=$Qj^rxOMl*|0OQ(J^2B|@vmwt;m#nHPD-xRPX<0!Pp0>d9B)dSJ48K&6mR3A~+wt{$R3n&&maw;Plp%14G%=ImJ_Buhs7iu?HnU&9A?!Rk59hj`I*M$*dE zq=ecL6b7b0d9^A#ADS&5#11ENi@Vb1{h=MTO_fZ67fmS0 zDjSBy0K7$0Zo(n9MmSqFY8K^{V4FL>jbWB(vy6m9mBebghM}skjM**Ddaj+}+`19V z8OEdA1_XWunPVwy!S++tjTbKP%B(zfdA9{|5%k{W97r#uDcU>*7Atjz(n5BQGi;=x zjd+?vVCxjPyG6^yaP|`9U=SoJx@%U;H-FOWHIfi#lF#y_u{l9%cwSU$$mB*zT9}d* zMAIDgfGT2=F1o?U>cT3duHCD_9_ z?a}Ao^sxKNs|TO|@0Wd(Ga)Yc7f+7=-))>0nJCrE$p-A9_4k^B*WEXB!S+TLpzBN+ zlY^tdU{2ge#*Jij8CCmTkBW$@2RIRjw@tgp+_2E%E9gigy# z#v_9r)IQ#ymphMaRf@E>N?znVa!IAg!AJYEvcR6^#?^Bn{a3&9{i`VE0Td2+EvesE zzH{}xGW*5%R&FT!J+6KpFOB#NepUB{XtSrlzd@@UY`YzsgvNeV%W@gI|n z0*js44t&ei6(rqFsQQh;Ab2`IeKtPa#{!iRwQIyB81G(^A)>STD5@c-r3<-3np7^d z)49ounC{%7MCOvdlg5yPi2H#$1Bmqb2k5gI@Rp2p6V`;;lO#iGni37o4EA8q_(h}w z*$f3B{WgXuE^R1zz|lYUmW<3O@!b`K!#ljQtnP3X>HsqwQ9)WW( zm0I?qfK^o4ut=Mef(*=br^)h5laN-!hy;Qj@)pHmyjq>ahf`6N#qsEXb|i@?ls>uC zpCG&%d4xF-^^al80fo> zV=UbIvw>!Y7M32oB)cbw$nAZ9gztMl+fVR$-z*S~BSt{lB{=4c>KVs+$_kN0wq@tz z&Fil}$c|YF2|g1o`Qh%n6*sd(fcQ&NzJgT~x5#9ud03YF-43fg9)5YAMG#IT)h28G z5eG^8S*wQL>|sZQ*gMyvhKth*JevVROF%5r5`d3h;WEk(PGZm7zPC$yRm7-Iqp#%z#@z9r_hqfyinN?10p?2P{kO@wA|YNOfl5h^xw8fUH5RFry2OTA zbjUnn{l*NOcM^B2UEvWkj>I#QJrLb`2PFX1Rs2%ai4n7g%63vw!Eb2Bx)!~Le!p{7qx$zz!0>X2ZX zr~4lh?O|@8*X@70k^{uEt7m#@Zuv(QeMI9J3(NkYxqbfeVzmN!JpsM7=5{UP+F$=n BUhe<^ literal 0 HcmV?d00001 diff --git a/jobs/.gitignore b/jobs/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..5990e8821f400a5b93c1e644c5d5d77f4a8b7464 --- /dev/null +++ b/jobs/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +.dev.vars +.wrangler/ +static/fonts/ diff --git a/jobs/package-lock.json b/jobs/package-lock.json new file mode 100644 index 0000000000000000000000000000000000000000..19194702d76c4143310f111449c65a205d67340d --- /dev/null +++ b/jobs/package-lock.json @@ -0,0 +1,1552 @@ +{ + "name": "tangled-jobs", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tangled-jobs", + "version": "1.0.0", + "dependencies": { + "countries-list": "^3.1.1", + "marked": "^15.0.0" + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20250428.0", + "typescript": "^5.8.3", + "wrangler": "^4.13.2" + } + }, + "node_modules/@cloudflare/kv-asset-handler": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.4.2.tgz", + "integrity": "sha512-SIOD2DxrRRwQ+jgzlXCqoEFiKOFqaPjhnNTGKXSRLvp1HiOvapLaFG2kEr9dYQTYe8rKrd9uvDUzmAITeNyaHQ==", + "dev": true, + "license": "MIT OR Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@cloudflare/unenv-preset": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", + "integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==", + "dev": true, + "license": "MIT OR Apache-2.0", + "peerDependencies": { + "unenv": "2.0.0-rc.24", + "workerd": ">1.20260305.0 <2.0.0-0" + }, + "peerDependenciesMeta": { + "workerd": { + "optional": true + } + } + }, + "node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20260426.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260426.1.tgz", + "integrity": "sha512-Ch7DqsmYzSQRTY87pZpsGsFVz9VVBnLPnCBOHxKt1HH25a7oMu1w1PbPWqVmE0VerCLsj/TScX7Ob3v6E14TZw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-darwin-arm64": { + "version": "1.20260426.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260426.1.tgz", + "integrity": "sha512-0m0U8vaPRH25SpKjbSyRql6gmPe4rCsETRV2WW0qBnuMdKNr5Vh5/Uez80xVrfiCCRMTULGeg63Nqg2vg6CDOA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20260426.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260426.1.tgz", + "integrity": "sha512-C8LlC8uSYzg49y51n++75esxZmMp+Uz1OKHHA/4lkv6rjOTbcHQJuEwSLppjybVIXpv7A8MBhbu9iyCTvyv1mw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20260426.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260426.1.tgz", + "integrity": "sha512-ESVp/OIFMAqjQsa8BOP2BQQz5Vpfv6ncN6lNnIuNeOgsISQBdYk+LA60bwQHMud9tvmnSYtONp1zkZ8OQz+x6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20260426.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260426.1.tgz", + "integrity": "sha512-d3Xj/IjINRgNVwH+eKhpUn4xkkcEewbWXbOvBlapiirKWh5zl9m0Epi3qOqmjyRYK6MICqIGXg4qZBEt0lxudw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workers-types": { + "version": "4.20260426.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260426.1.tgz", + "integrity": "sha512-cBYeQaWwv/jFV8ualmwp6wIxmAf0rDe2DPPQwPbslKmPHqgv861YpAvm45r05K40QboZgxNQVIPgNkmtHqZeJQ==", + "dev": true, + "license": "MIT OR Apache-2.0", + "peer": true + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", + "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", + "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", + "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", + "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", + "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", + "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", + "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", + "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", + "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", + "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", + "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", + "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", + "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", + "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", + "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", + "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", + "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", + "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", + "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", + "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", + "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", + "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", + "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", + "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", + "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", + "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@poppinss/colors": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", + "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^4.1.5" + } + }, + "node_modules/@poppinss/dumper": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", + "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@sindresorhus/is": "^7.0.2", + "supports-color": "^10.0.0" + } + }, + "node_modules/@poppinss/exception": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", + "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", + "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@speed-highlight/core": { + "version": "1.2.15", + "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.15.tgz", + "integrity": "sha512-BMq1K3DsElxDWawkX6eLg9+CKJrTVGCBAWVuHXVUV2u0s2711qiChLSId6ikYPfxhdYocLNt3wWwSvDiTvFabw==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/blake3-wasm": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", + "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/countries-list": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/countries-list/-/countries-list-3.3.0.tgz", + "integrity": "sha512-XRUjS+dcZuNh/fg3+mka3bXgcg4TbQZ1gaK5IJqO6qulerBANl1bmrd20P2dgmPkBpP+5FnejiSF1gd7bgAg+g==", + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/error-stack-parser-es": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", + "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/esbuild": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", + "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.3", + "@esbuild/android-arm": "0.27.3", + "@esbuild/android-arm64": "0.27.3", + "@esbuild/android-x64": "0.27.3", + "@esbuild/darwin-arm64": "0.27.3", + "@esbuild/darwin-x64": "0.27.3", + "@esbuild/freebsd-arm64": "0.27.3", + "@esbuild/freebsd-x64": "0.27.3", + "@esbuild/linux-arm": "0.27.3", + "@esbuild/linux-arm64": "0.27.3", + "@esbuild/linux-ia32": "0.27.3", + "@esbuild/linux-loong64": "0.27.3", + "@esbuild/linux-mips64el": "0.27.3", + "@esbuild/linux-ppc64": "0.27.3", + "@esbuild/linux-riscv64": "0.27.3", + "@esbuild/linux-s390x": "0.27.3", + "@esbuild/linux-x64": "0.27.3", + "@esbuild/netbsd-arm64": "0.27.3", + "@esbuild/netbsd-x64": "0.27.3", + "@esbuild/openbsd-arm64": "0.27.3", + "@esbuild/openbsd-x64": "0.27.3", + "@esbuild/openharmony-arm64": "0.27.3", + "@esbuild/sunos-x64": "0.27.3", + "@esbuild/win32-arm64": "0.27.3", + "@esbuild/win32-ia32": "0.27.3", + "@esbuild/win32-x64": "0.27.3" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/marked": { + "version": "15.0.12", + "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.12.tgz", + "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/miniflare": { + "version": "4.20260426.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260426.0.tgz", + "integrity": "sha512-KM+v76d04qT+NsPfVKVQEgnnuLNE3uzCCl2QKMTJ5OXor5JbBm1vpkQwQ+l7o5ELCrZ74RnyKhJKLiJyUA39Tw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "sharp": "^0.34.5", + "undici": "7.24.8", + "workerd": "1.20260426.1", + "ws": "8.18.0", + "youch": "4.1.0-beta.10" + }, + "bin": { + "miniflare": "bootstrap.js" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici": { + "version": "7.24.8", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.8.tgz", + "integrity": "sha512-6KQ/+QxK49Z/p3HO6E5ZCZWNnCasyZLa5ExaVYyvPxUwKtbCPMKELJOqh7EqOle0t9cH/7d2TaaTRRa6Nhs4YQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/unenv": { + "version": "2.0.0-rc.24", + "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.24.tgz", + "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "pathe": "^2.0.3" + } + }, + "node_modules/workerd": { + "version": "1.20260426.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260426.1.tgz", + "integrity": "sha512-ELvGgN8c9oo+E6EPyecxk1TEf6/eAK4TxxQTW5mQ87C7jbjCzhMbg0P2ije49UBHV0dkBYPJcJvcklUltipl2A==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "peer": true, + "bin": { + "workerd": "bin/workerd" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@cloudflare/workerd-darwin-64": "1.20260426.1", + "@cloudflare/workerd-darwin-arm64": "1.20260426.1", + "@cloudflare/workerd-linux-64": "1.20260426.1", + "@cloudflare/workerd-linux-arm64": "1.20260426.1", + "@cloudflare/workerd-windows-64": "1.20260426.1" + } + }, + "node_modules/wrangler": { + "version": "4.86.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.86.0.tgz", + "integrity": "sha512-9aa/gbF/HiUeeUEwyQpW5LDPBEzyt7iaE6xHwm0vk2Ly8A6J+jh03pzchqVnCCWR832mNyA28MD8oAYt0Kfvlw==", + "dev": true, + "license": "MIT OR Apache-2.0", + "dependencies": { + "@cloudflare/kv-asset-handler": "0.4.2", + "@cloudflare/unenv-preset": "2.16.1", + "blake3-wasm": "2.1.5", + "esbuild": "0.27.3", + "miniflare": "4.20260426.0", + "path-to-regexp": "6.3.0", + "unenv": "2.0.0-rc.24", + "workerd": "1.20260426.1" + }, + "bin": { + "wrangler": "bin/wrangler.js", + "wrangler2": "bin/wrangler.js" + }, + "engines": { + "node": ">=20.3.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + }, + "peerDependencies": { + "@cloudflare/workers-types": "^4.20260426.1" + }, + "peerDependenciesMeta": { + "@cloudflare/workers-types": { + "optional": true + } + } + }, + "node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/youch": { + "version": "4.1.0-beta.10", + "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", + "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@poppinss/dumper": "^0.6.4", + "@speed-highlight/core": "^1.2.7", + "cookie": "^1.0.2", + "youch-core": "^0.3.3" + } + }, + "node_modules/youch-core": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", + "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/exception": "^1.2.2", + "error-stack-parser-es": "^1.0.5" + } + } + } +} diff --git a/jobs/package.json b/jobs/package.json new file mode 100644 index 0000000000000000000000000000000000000000..dcd52907f3ba11caa942e2688ebe13ca3016b36d --- /dev/null +++ b/jobs/package.json @@ -0,0 +1,20 @@ +{ + "name": "tangled-jobs", + "version": "1.0.0", + "private": true, + "scripts": { + "predev": "mkdir -p static/fonts && cp -f ../web/static/fonts/TimelessSansVF.woff2 static/fonts/", + "dev": "wrangler dev", + "predeploy": "mkdir -p static/fonts && cp -f ../web/static/fonts/TimelessSansVF.woff2 static/fonts/", + "deploy": "wrangler deploy" + }, + "dependencies": { + "countries-list": "^3.1.1", + "marked": "^15.0.0" + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20250428.0", + "typescript": "^5.8.3", + "wrangler": "^4.13.2" + } +} diff --git a/jobs/src/index.ts b/jobs/src/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..8a02e64ebc0cd12b218d32da7fd5defd0a7c193a --- /dev/null +++ b/jobs/src/index.ts @@ -0,0 +1,617 @@ +import { marked } from 'marked'; +import { getEmojiFlag, countries } from 'countries-list'; +import timelessSans from '../static/fonts/TimelessSansVF.woff2'; + +export interface Env { + LINEAR_API_KEY: string; + LINEAR_TEAM_ID: string; +} + +// --------------------------------------------------------------------------- +// Job postings — add a new markdown file in src/postings/ and import it here +// --------------------------------------------------------------------------- +// import softwareEngineer from './postings/software-engineer.md'; + +interface Posting { + slug: string; + title: string; + location: string; + type: string; + salary: string; + body: string; // raw markdown (frontmatter stripped) +} + +function parsePosting(slug: string, raw: string): Posting { + const fm = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/); + if (!fm) throw new Error(`Posting ${slug} is missing frontmatter`); + + const meta: Record = {}; + for (const line of fm[1].split('\n')) { + const [k, ...rest] = line.split(':'); + if (k && rest.length) meta[k.trim()] = rest.join(':').trim(); + } + + return { + slug, + title: meta['title'] ?? slug, + location: meta['location'] ?? 'Remote', + type: meta['type'] ?? 'Full-time', + salary: meta['salary'] ?? '', + body: fm[2].trim(), + }; +} + +const POSTINGS: Posting[] = [ + // parsePosting('software-engineer', softwareEngineer as string), + // add more here as you create markdown files +]; + +const POSTINGS_BY_SLUG = new Map(POSTINGS.map((p) => [p.slug, p])); + +// --------------------------------------------------------------------------- +// Country list (sorted alphabetically by name, from countries-list) +// --------------------------------------------------------------------------- +const COUNTRIES: { code: string; name: string }[] = Object.entries(countries) + .map(([code, c]) => ({ code, name: c.name })) + .sort((a, b) => a.name.localeCompare(b.name)); + +// --------------------------------------------------------------------------- +// SVG logo +// --------------------------------------------------------------------------- +const DOLLY_SVG = ` + + + + +`; + +// --------------------------------------------------------------------------- +// HTML shell +// --------------------------------------------------------------------------- +function page(title: string, body: string): string { + return ` + + + + + ${escapeHtml(title)} · jobs at tangled + + + + + + + + + + +

    +
    + + + + ${body} + + + + +`; +} + +// --------------------------------------------------------------------------- +// Pages +// --------------------------------------------------------------------------- +function listingsPage(): string { + const rows = POSTINGS.map( + (p) => ` + +
    +
    ${escapeHtml(p.title)}
    +
    ${escapeHtml(p.location)} · ${escapeHtml(p.type)}${p.salary ? ` · ${escapeHtml(p.salary)}` : ''}
    +
    + → +     `, + ).join(''); + + const body = ` +
    +
    +

    Open positions

    +

    + We're a lean, globally distributed team working on building the next-generation of social coding. + We work remotely and try to meet once a year in-person. +

    +
    + + ${ + POSTINGS.length === 0 + ? `

    No open positions right now; check back soon.

    ` + : `
    ${rows}
    ` + } +
    `; + + return page('open positions', body); +} + +function jobPage(posting: Posting): string { + const bodyHtml = marked.parse(posting.body) as string; + + const body = ` +
    +
    + ← All positions +
    + +
    +

    ${escapeHtml(posting.location)} · ${escapeHtml(posting.type)}${posting.salary ? ` · ${escapeHtml(posting.salary)}` : ''}

    +

    ${escapeHtml(posting.title)}

    +
    + +
    + ${bodyHtml} +
    + +
    +

    Apply for this role

    +

    We read every application; if there's a fit, we'll be in touch via email.

    + ${applyForm(posting.slug)} +
    +
    `; + + return page(posting.title, body); +} + +function applyForm(slug: string, error?: string): string { + const errorHtml = error + ? `
    ${escapeHtml(error)}
    ` + : ''; + + const inputClass = 'block w-full rounded p-3 bg-gray-50 dark:bg-gray-800 dark:text-white border border-gray-300 dark:border-gray-600 focus:outline-none focus:ring-1 focus:ring-gray-400 dark:focus:ring-gray-500'; + const selectClass = `${inputClass} appearance-none`; + + return ` + ${errorHtml} +
    +
    +
    + + +
    +
    + + +
    +
    + +
    +
    + + +
    +
    + + +
    +
    + +
    + + +
    + +
    +
    + + +
    +
    + + +
    +
    + +
    + + +
    + +
    + + +
    + +
    + +
    +

    By submitting this form you agree to your data being processed by our subprocessors: Cloudflare and Linear.

    +
    + `; +} + +function successPage(firstName: string, posting: Posting): string { + const body = ` +
    +
    🎉
    +

    Application received

    +

    + Thanks, ${escapeHtml(firstName)}! We've received your application for ${escapeHtml(posting.title)} and will be in touch soon. +

    + ← View all positions +
    `; + + return page('application received', body); +} + +// --------------------------------------------------------------------------- +// Linear +// --------------------------------------------------------------------------- + +// Upload a file to Linear's asset storage and return the public asset URL. +async function uploadToLinear(env: Env, file: File): Promise { + const query = ` + mutation FileUpload($contentType: String!, $filename: String!, $size: Int!) { + fileUpload(contentType: $contentType, filename: $filename, size: $size) { + uploadFile { + uploadUrl + assetUrl + headers { key value } + } + } + }`; + + const metaResp = await fetch('https://api.linear.app/graphql', { + method: 'POST', + headers: { Authorization: env.LINEAR_API_KEY, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + query, + variables: { contentType: file.type, filename: file.name, size: file.size }, + }), + }); + + const meta = (await metaResp.json()) as { + data?: { fileUpload?: { uploadFile?: { uploadUrl: string; assetUrl: string; headers: { key: string; value: string }[] } } }; + errors?: unknown[]; + }; + + const uploadFile = meta.data?.fileUpload?.uploadFile; + if (!uploadFile || meta.errors) { + throw new Error(`Linear fileUpload error: ${JSON.stringify(meta.errors ?? meta)}`); + } + + const uploadHeaders: Record = { 'Content-Type': file.type }; + for (const { key, value } of uploadFile.headers) uploadHeaders[key] = value; + + const uploadResp = await fetch(uploadFile.uploadUrl, { + method: 'PUT', + headers: uploadHeaders, + body: await file.arrayBuffer(), + }); + + if (!uploadResp.ok) { + throw new Error(`Resume upload failed: ${uploadResp.status} ${uploadResp.statusText}`); + } + + return uploadFile.assetUrl; +} + +async function getOrCreateLabel(env: Env, name: string): Promise { + try { + const searchQuery = ` + query Labels($teamId: ID!) { + issueLabels(filter: { team: { id: { eq: $teamId } } }) { + nodes { id name } + } + }`; + + const searchResp = await fetch('https://api.linear.app/graphql', { + method: 'POST', + headers: { Authorization: env.LINEAR_API_KEY, 'Content-Type': 'application/json' }, + body: JSON.stringify({ query: searchQuery, variables: { teamId: env.LINEAR_TEAM_ID } }), + }); + + const searchResult = (await searchResp.json()) as { + data?: { issueLabels?: { nodes: { id: string; name: string }[] } }; + }; + + const existing = searchResult.data?.issueLabels?.nodes.find((l) => l.name === name); + if (existing) return existing.id; + + const createQuery = ` + mutation LabelCreate($input: IssueLabelCreateInput!) { + issueLabelCreate(input: $input) { + success + issueLabel { id } + } + }`; + + const createResp = await fetch('https://api.linear.app/graphql', { + method: 'POST', + headers: { Authorization: env.LINEAR_API_KEY, 'Content-Type': 'application/json' }, + body: JSON.stringify({ query: createQuery, variables: { input: { name, teamId: env.LINEAR_TEAM_ID } } }), + }); + + const created = (await createResp.json()) as { + data?: { issueLabelCreate?: { success: boolean; issueLabel: { id: string } } }; + }; + + return created.data?.issueLabelCreate?.issueLabel?.id ?? null; + } catch (err) { + console.error('Failed to get/create label:', err); + return null; + } +} + +async function createLinearIssue( + env: Env, + posting: Posting, + data: { + firstName: string; + lastName: string; + email: string; + country: string; + city: string; + portfolio: string; + linkedin: string; + cover: string; + resume: File | null; + }, +): Promise { + const fullName = `${data.firstName} ${data.lastName}`; + const title = fullName; + + const [resumeUrl, labelId] = await Promise.all([ + data.resume ? uploadToLinear(env, data.resume) : Promise.resolve(null), + getOrCreateLabel(env, posting.title), + ]); + + const description = [ + `**Name:** ${fullName}`, + `**Email:** ${data.email}`, + `**Location:** ${data.city}, ${data.country}`, + `**Role:** ${posting.title}`, + data.portfolio ? `**Portfolio:** ${data.portfolio}` : null, + data.linkedin ? `**LinkedIn:** ${data.linkedin}` : null, + resumeUrl ? `**Résumé:** [${data.resume!.name}](${resumeUrl})` : null, + '', + '---', + '', + '## Cover letter', + '', + data.cover, + ] + .filter((l): l is string => l !== null) + .join('\n'); + + const query = ` + mutation CreateIssue($input: IssueCreateInput!) { + issueCreate(input: $input) { + success + issue { id identifier } + } + }`; + + const input: Record = { title, description, teamId: env.LINEAR_TEAM_ID }; + if (labelId) input.labelIds = [labelId]; + + const resp = await fetch('https://api.linear.app/graphql', { + method: 'POST', + headers: { Authorization: env.LINEAR_API_KEY, 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables: { input } }), + }); + + const result = (await resp.json()) as { data?: { issueCreate?: { success: boolean } }; errors?: unknown[] }; + + if (result.errors || !result.data?.issueCreate?.success) { + throw new Error(`Linear error: ${JSON.stringify(result.errors ?? result)}`); + } +} + +// --------------------------------------------------------------------------- +// Router +// --------------------------------------------------------------------------- +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + const parts = url.pathname.replace(/^\//, '').split('/'); + + // GET /favicon.svg + if (request.method === 'GET' && url.pathname === '/favicon.svg') { + return new Response(DOLLY_SVG, { headers: { 'Content-Type': 'image/svg+xml' } }); + } + + if (request.method === 'GET' && url.pathname === '/static/fonts/TimelessSansVF.woff2') { + return new Response(timelessSans, { + headers: { 'Content-Type': 'font/woff2', 'Cache-Control': 'public, max-age=86400' }, + }); + } + + // GET / — listings + if (request.method === 'GET' && url.pathname === '/') { + return html(listingsPage()); + } + + // GET /:slug — job posting + if (request.method === 'GET' && parts.length === 1 && parts[0]) { + const posting = POSTINGS_BY_SLUG.get(parts[0]); + if (!posting) return notFound(); + return html(jobPage(posting)); + } + + // POST /:slug/apply — submit application + if (request.method === 'POST' && parts.length === 2 && parts[1] === 'apply') { + const posting = POSTINGS_BY_SLUG.get(parts[0]); + if (!posting) return notFound(); + + let formData: FormData; + try { + formData = await request.formData(); + } catch { + return html(jobPage(posting), 400); + } + + const firstName = (formData.get('first_name') as string | null)?.trim() ?? ''; + const lastName = (formData.get('last_name') as string | null)?.trim() ?? ''; + const email = (formData.get('email') as string | null)?.trim() ?? ''; + const country = (formData.get('country') as string | null)?.trim() ?? ''; + const city = (formData.get('city') as string | null)?.trim() ?? ''; + const portfolio = (formData.get('portfolio') as string | null)?.trim() ?? ''; + const linkedin = (formData.get('linkedin') as string | null)?.trim() ?? ''; + const cover = (formData.get('cover') as string | null)?.trim() ?? ''; + const resumeEntry = formData.get('resume'); + const resume = + resumeEntry instanceof File && resumeEntry.size > 0 && resumeEntry.type === 'application/pdf' + ? resumeEntry + : null; + + if (!firstName || !lastName || !email || !country || !city || !cover || !resume) { + const bodyHtml = marked.parse(posting.body) as string; + const body = ` +
    +
    + ← All positions +
    +
    +

    ${escapeHtml(posting.location)} · ${escapeHtml(posting.type)}${posting.salary ? ` · ${escapeHtml(posting.salary)}` : ''}

    +

    ${escapeHtml(posting.title)}

    +
    +
    ${bodyHtml}
    +
    +

    Apply for this role

    +

    We read every application; if there's a fit, we'll be in touch via email.

    + ${applyForm(posting.slug, 'Please fill in all required fields.')} +
    +
    `; + return html(page(posting.title, body), 400); + } + + try { + await createLinearIssue(env, posting, { firstName, lastName, email, country, city, portfolio, linkedin, cover, resume }); + return html(successPage(firstName, posting)); + } catch (err) { + console.error('Linear issue creation failed:', err); + const bodyHtml = marked.parse(posting.body) as string; + const body = ` +
    +
    + ← All positions +
    +
    +

    ${escapeHtml(posting.location)} · ${escapeHtml(posting.type)}${posting.salary ? ` · ${escapeHtml(posting.salary)}` : ''}

    +

    ${escapeHtml(posting.title)}

    +
    +
    ${bodyHtml}
    +
    +

    Apply for this role

    +

    We read every application; if there's a fit, we'll be in touch via email.

    + ${applyForm(posting.slug, 'Something went wrong submitting your application. Please try again.')} +
    +
    `; + return html(page(posting.title, body), 500); + } + } + + return notFound(); + }, +} satisfies ExportedHandler; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- +function html(body: string, status = 200): Response { + return new Response(body, { status, headers: { 'Content-Type': 'text/html; charset=utf-8' } }); +} + +function notFound(): Response { + return new Response('not found', { status: 404 }); +} + +function escapeHtml(str: string): string { + return str.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"').replace(/'/g, '''); +} diff --git a/jobs/src/postings/archive/software-engineer.md b/jobs/src/postings/archive/software-engineer.md new file mode 100644 index 0000000000000000000000000000000000000000..fe6e9409550f675a346a7edcb9f2fae3b0b2cad3 --- /dev/null +++ b/jobs/src/postings/archive/software-engineer.md @@ -0,0 +1,16 @@ +--- +title: Software Engineer, Platform +location: Remote +type: Full-time +salary: €100k–€120k +--- + +We're looking for a Software Engineer to join us in building the +next-generation code forge. + +**What you'll work on**: Primarily, our new CI platform: Go, Firecracker/QEMU, +Nix and general distributed systems expertise. Ideally you check all these +boxes. + +**What we offer**: Competitive (for Europe) salary, equity, benefits. We hire +from anywhere in the world and don't location-adjust pay. diff --git a/jobs/src/woff2.d.ts b/jobs/src/woff2.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d51f005eefbb2983466e17972edcbbc884b18524 --- /dev/null +++ b/jobs/src/woff2.d.ts @@ -0,0 +1,4 @@ +declare module '*.woff2' { + const data: ArrayBuffer; + export default data; +} diff --git a/jobs/tsconfig.json b/jobs/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..d2da03f19b14d5403f0c10b3abfde8a249404940 --- /dev/null +++ b/jobs/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "ES2022", + "moduleResolution": "bundler", + "types": ["@cloudflare/workers-types"], + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true + }, + "include": ["src/**/*.ts"] +} diff --git a/jobs/wrangler.toml b/jobs/wrangler.toml new file mode 100644 index 0000000000000000000000000000000000000000..7a12a1c4cfe60c7ac7ffd285b1cdb59d8589af33 --- /dev/null +++ b/jobs/wrangler.toml @@ -0,0 +1,28 @@ +name = "tangled-jobs" +main = "src/index.ts" +compatibility_date = "2026-04-28" + +[[rules]] +type = "Text" +globs = ["**/*.md"] + +[[rules]] +type = "Data" +globs = ["**/*.woff2"] +fallthrough = true + +[vars] +LINEAR_TEAM_ID = "4be92e1b-1c19-4d37-ab9f-38c7484a00af" + +# Set secrets via: wrangler secret put LINEAR_API_KEY +# LINEAR_API_KEY = "lin_api_..." + +[[routes]] +pattern = "jobs.tangled.org" +zone_name = "tangled.org" +custom_domain = true + +[observability] +[observability.logs] +enabled = true +invocation_logs = true diff --git a/knot2/.gitignore b/knot2/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b1ba243b209bb708c4e67df53e2a0038bbe2ab04 --- /dev/null +++ b/knot2/.gitignore @@ -0,0 +1,9 @@ +/target +**/target + +crates/knot-pack/fuzz/corpus/ +crates/knot-pack/fuzz/artifacts/ + +config.toml + +*.swp diff --git a/knot2/Containerfile b/knot2/Containerfile new file mode 100644 index 0000000000000000000000000000000000000000..131d1b4e347e74d18025fbd56e916a352c83daa9 --- /dev/null +++ b/knot2/Containerfile @@ -0,0 +1,23 @@ +FROM docker.io/library/rust:1.96-slim-trixie AS builder +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates build-essential cmake perl pkg-config clang mold \ + && rm -rf /var/lib/apt/lists/* +ENV RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=mold" +WORKDIR /src +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY crates ./crates +COPY bobbin/crates ./bobbin/crates +COPY shuttle ./shuttle +COPY knotmirror/crates ./knotmirror/crates +COPY lexicons ./lexicons +COPY knot2/crates ./knot2/crates +COPY knot2/third_party ./knot2/third_party +RUN cargo build --release --package knot-server --package knot-migrate +RUN strip target/release/knot-server target/release/knot-migrate + +FROM gcr.io/distroless/cc-debian13:latest@sha256:d74d144baa124dd9defd01d71872bb56c49d4441dba6153ccf790364f97e64b1 +COPY --from=builder /src/target/release/knot-server /usr/local/bin/knot-server +COPY --from=builder /src/target/release/knot-migrate /usr/local/bin/knot-migrate +EXPOSE 5555 2222 +ENTRYPOINT ["/usr/local/bin/knot-server"] +CMD ["--config", "/etc/knot/config.toml"] diff --git a/knot2/README.md b/knot2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7878c32020559ed4ddeea2b23b2e86bdf0f6f43a --- /dev/null +++ b/knot2/README.md @@ -0,0 +1,24 @@ +# Knot 2 + +In Tangled, a "knot" is simply a git server that does its auth layer over the AT Protocol. +Essentially: + +1. Atproto users declare their public SSH key for themselves, in their PDS. +2. A knot admin (defined by atproto DID in the knot config) makes a request to the knot to allow membership to a given atproto user. +3. Said user can now create/update/delete a git repo on the knot, and do normal git things to that git repo over SSH. +4. Said user makes a request to the knot to allow more atproto users as collaborators on their specific git repo. + +- **It has no database, or should I say, git is the only database, along with a secret-key-file.** +- It aims to be small, fast, and modern - for example: http3/quic support, SHA256 by default, and no need for unix `git` user, to name a couple of features. + +# Running a knot + +The operator documentation: [the docs](https://tangled.org/docs): + +- [Run a knot](https://tangled.org/docs/self-hosting/knot/) - a knot on NixOS or in a container. +- [Self-hosting a knot](https://tangled.org/docs/knots/self-hosting/) - more detail: ports, systemd, TLS, members, LFS, backups. +- [Knot configuration](https://tangled.org/docs/reference/knot-config/) - list of keys, defaults, and `KNOT_*` env vars. +- [Knot](https://tangled.org/docs/knots/) - what a knot does and how it talks to the rest of Tangled. +- If running the old golang knot and want this instead: [Migrate](https://tangled.org/docs/upgrading/knot-2/) with `knot-migrate` tool - afterwards the knot will serve the same hostname with the same repos, owners, members and collaborators. + +Happy knotting! diff --git a/knot2/_typos.toml b/knot2/_typos.toml new file mode 100644 index 0000000000000000000000000000000000000000..15e51d143d2c641ec5b6bc89ad6bef7267dd36c2 --- /dev/null +++ b/knot2/_typos.toml @@ -0,0 +1,4 @@ +[files] +extend-exclude = ["lexicons/", "**/_lex/", "Cargo.lock", "pack-spike/"] + +[default.extend-words] diff --git a/knot2/crates/knot-acl/Cargo.toml b/knot2/crates/knot-acl/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..433aa7307443d8f90efdd988599733da92feca53 --- /dev/null +++ b/knot2/crates/knot-acl/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "knot-acl" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-index = { workspace = true } + +[dev-dependencies] +knot-index = { workspace = true, features = ["test-support"] } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-git = { workspace = true } +knot-runtime = { workspace = true } +tokio = { workspace = true } +tempfile = { workspace = true } diff --git a/knot2/crates/knot-acl/src/lib.rs b/knot2/crates/knot-acl/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..f21d07280dfb80ae0349d5122831257ef31717c7 --- /dev/null +++ b/knot2/crates/knot-acl/src/lib.rs @@ -0,0 +1,923 @@ +use std::collections::BTreeSet; + +use knot_index::{CollaborationConsent, Index, MemberStatus, Resolved}; +use knot_types::{ + AccountDid, AdmissionPolicy, Blocked, ContributionPermission, ContributionPolicy, Membership, + OwnerDid, RepoDid, RepoPolicy, RepoRole, +}; + +pub use knot_types::Decision; + +pub trait Acl { + fn is_admin(&self, who: &AccountDid) -> bool; + fn admission(&self) -> AdmissionPolicy; + fn is_blocked(&self, who: &AccountDid) -> Resolved; + fn repo_owner(&self, repo: &RepoDid) -> Resolved>; +} + +fn confirmed(resolved: Resolved) -> bool { + matches!(resolved, Resolved::Ready(true)) +} + +fn ownership(acl: &impl Acl, who: &AccountDid, repo: &RepoDid) -> Resolved { + acl.repo_owner(repo) + .map(|owner| owner.is_some_and(|owner| owner.is(who))) +} + +fn owns_repo(acl: &impl Acl, who: &AccountDid, repo: &RepoDid) -> bool { + confirmed(ownership(acl, who, repo)) +} + +fn not_blocked(acl: &impl Acl, who: &AccountDid) -> bool { + acl.is_admin(who) || matches!(acl.is_blocked(who), Resolved::Ready(false)) +} + +pub fn can_admin_knot(acl: &impl Acl, who: &AccountDid) -> Decision { + Decision::allow_if(acl.is_admin(who)) +} + +fn closed_to_outsiders(acl: &impl Acl) -> bool { + matches!(acl.admission(), AdmissionPolicy::Closed) +} + +pub fn needs_membership(acl: &impl Acl, who: &AccountDid) -> bool { + !acl.is_admin(who) && closed_to_outsiders(acl) +} + +pub fn can_create_repo(acl: &impl Acl, membership: &MemberStatus<'_>) -> Decision { + let who = membership.subject(); + Decision::allow_if( + acl.is_admin(who) + || (not_blocked(acl, who) && (!closed_to_outsiders(acl) || membership.granted())), + ) +} + +pub fn can_push(acl: &impl Acl, consent: &CollaborationConsent<'_>) -> Decision { + let who = consent.subject(); + Decision::allow_if( + not_blocked(acl, who) && (owns_repo(acl, who, consent.repo()) || consent.granted()), + ) +} + +pub fn can_manage_collaborators(acl: &impl Acl, who: &AccountDid, repo: &RepoDid) -> Decision { + Decision::allow_if(not_blocked(acl, who) && owns_repo(acl, who, repo)) +} + +pub fn can_set_contribution_policy(acl: &impl Acl, who: &AccountDid, repo: &RepoDid) -> Decision { + Decision::allow_if(not_blocked(acl, who) && (acl.is_admin(who) || owns_repo(acl, who, repo))) +} + +pub fn can_delete_repo(acl: &impl Acl, who: &AccountDid, repo: &RepoDid) -> Decision { + Decision::allow_if(not_blocked(acl, who) && (acl.is_admin(who) || owns_repo(acl, who, repo))) +} + +pub struct RepoAccess<'a> { + collaboration: &'a CollaborationConsent<'a>, + membership: &'a MemberStatus<'a>, + policy: &'a RepoPolicy, +} + +impl<'a> RepoAccess<'a> { + pub fn of( + collaboration: &'a CollaborationConsent<'a>, + membership: &'a MemberStatus<'a>, + policy: &'a RepoPolicy, + ) -> Option { + let one_subject = collaboration.subject() == membership.subject(); + let one_repo = collaboration.repo() == policy.repo(); + (one_subject && one_repo).then_some(Self { + collaboration, + membership, + policy, + }) + } + + pub const fn subject(&self) -> &'a AccountDid { + self.collaboration.subject() + } + + pub const fn repo(&self) -> &'a RepoDid { + self.collaboration.repo() + } + + pub const fn policy(&self) -> ContributionPolicy { + self.policy.policy() + } +} + +pub fn repo_role(acl: &impl Acl, access: &RepoAccess) -> Resolved { + let who = access.subject(); + ownership(acl, who, access.repo()).map(|owns| { + match ( + owns, + access.collaboration.granted(), + acl.is_admin(who), + access.membership.granted(), + ) { + (true, ..) => RepoRole::Owner, + (_, true, ..) => RepoRole::Collaborator, + (.., true, member) => RepoRole::KnotAdmin(Membership::of(member)), + (.., true) => RepoRole::Member, + _ => RepoRole::Stranger, + } + }) +} + +pub fn permission_of(acl: &impl Acl, access: &RepoAccess) -> Resolved { + let blocked = if not_blocked(acl, access.subject()) { + Blocked::No + } else { + Blocked::Yes + }; + repo_role(acl, access).map(|role| ContributionPermission::new(blocked, role, access.policy())) +} + +pub fn can_contribute(acl: &impl Acl, access: &RepoAccess) -> Resolved { + permission_of(acl, access).map(ContributionPermission::contributes) +} + +pub fn can_moderate(acl: &impl Acl, access: &RepoAccess) -> Resolved { + permission_of(acl, access).map(ContributionPermission::moderates) +} + +pub struct KnotAcl<'a> { + admins: &'a BTreeSet, + policy: AdmissionPolicy, + index: &'a Index, +} + +impl<'a> KnotAcl<'a> { + pub fn new( + admins: &'a BTreeSet, + policy: AdmissionPolicy, + index: &'a Index, + ) -> Self { + Self { + admins, + policy, + index, + } + } +} + +impl Acl for KnotAcl<'_> { + fn is_admin(&self, who: &AccountDid) -> bool { + self.admins.contains(who) + } + + fn admission(&self) -> AdmissionPolicy { + self.policy + } + + fn is_blocked(&self, who: &AccountDid) -> Resolved { + self.index.is_blocked(who) + } + + fn repo_owner(&self, repo: &RepoDid) -> Resolved> { + self.index.owner_of(repo) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn acc(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn owner(suffix: &str) -> OwnerDid { + OwnerDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn repo(suffix: &str) -> RepoDid { + RepoDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn joined(who: &AccountDid) -> MemberStatus<'_> { + MemberStatus::assumed((), who, true) + } + + fn outside(who: &AccountDid) -> MemberStatus<'_> { + MemberStatus::assumed((), who, false) + } + + fn collaborating<'a>(repo: &'a RepoDid, who: &'a AccountDid) -> CollaborationConsent<'a> { + CollaborationConsent::assumed(repo, who, true) + } + + fn ungranted<'a>(repo: &'a RepoDid, who: &'a AccountDid) -> CollaborationConsent<'a> { + CollaborationConsent::assumed(repo, who, false) + } + + struct Fake { + admins: BTreeSet, + admission: AdmissionPolicy, + blocked: Resolved, + owner: Resolved>, + } + + impl Fake { + fn new() -> Self { + Self { + admins: BTreeSet::new(), + admission: AdmissionPolicy::Closed, + blocked: Resolved::Ready(false), + owner: Resolved::Warming, + } + } + + fn admin(mut self, who: &str) -> Self { + self.admins.insert(acc(who)); + self + } + + fn open(mut self) -> Self { + self.admission = AdmissionPolicy::Open; + self + } + + fn blocked(mut self, resolved: Resolved) -> Self { + self.blocked = resolved; + self + } + + fn owner(mut self, resolved: Resolved>) -> Self { + self.owner = resolved; + self + } + } + + impl Acl for Fake { + fn is_admin(&self, who: &AccountDid) -> bool { + self.admins.contains(who) + } + + fn admission(&self) -> AdmissionPolicy { + self.admission + } + + fn is_blocked(&self, _who: &AccountDid) -> Resolved { + self.blocked.clone() + } + + fn repo_owner(&self, _repo: &RepoDid) -> Resolved> { + self.owner.clone() + } + } + + fn access_of<'a>( + collaboration: &'a CollaborationConsent<'a>, + membership: &'a MemberStatus<'a>, + policy: &'a RepoPolicy, + ) -> RepoAccess<'a> { + RepoAccess::of(collaboration, membership, policy) + .expect("one access is one subject on one repo") + } + + fn policy_for(repo: &RepoDid, policy: ContributionPolicy) -> RepoPolicy { + RepoPolicy::new(repo.clone(), policy) + } + + fn allowed(decision: Resolved) -> bool { + matches!(decision, Resolved::Ready(Decision::Allow)) + } + + #[test] + fn an_admin_who_doesnt_own_and_doesnt_collaborate_gets_exactly_the_moderation_verbs() { + let acl = Fake::new() + .admin("nel") + .owner(Resolved::Ready(Some(owner("olaren")))); + let who = acc("nel"); + let target = repo("anemone"); + let collaboration = ungranted(&target, &who); + let membership = outside(&who); + let anyone = policy_for(&target, ContributionPolicy::Anyone); + let members = policy_for(&target, ContributionPolicy::Members); + let collaborators = policy_for(&target, ContributionPolicy::Collaborators); + + assert_eq!( + repo_role(&acl, &access_of(&collaboration, &membership, &anyone)), + Resolved::Ready(RepoRole::KnotAdmin(Membership::Outsider)) + ); + assert!( + allowed(can_moderate( + &acl, + &access_of(&collaboration, &membership, &anyone) + )), + "Operator may take down what it publishes" + ); + assert!( + !allowed(can_contribute( + &acl, + &access_of(&collaboration, &membership, &collaborators) + )), + "moderating somebody's repo doesn't make admin one of its authors" + ); + assert!(!allowed(can_contribute( + &acl, + &access_of(&collaboration, &membership, &members) + ))); + assert!( + allowed(can_contribute( + &acl, + &access_of(&collaboration, &membership, &anyone) + )), + "under Anyone admin contributes on the same footing as strangers" + ); + assert!( + !can_push(&acl, &collaboration).is_allowed(), + "Admin carve-out stops at moderation. It never reaches git" + ); + assert!(!can_manage_collaborators(&acl, &who, &target).is_allowed()); + + let membership = joined(&who); + assert_eq!( + repo_role(&acl, &access_of(&collaboration, &membership, &members)), + Resolved::Ready(RepoRole::KnotAdmin(Membership::Member)), + "Admin set doesn't strip somebody of membership they also have" + ); + } + + #[test] + fn ownership_outranks_collaboration_outranks_the_admin_set_outranks_membership() { + let target = repo("anemone"); + let who = acc("nel"); + type Case = (&'static str, Fake, bool, bool, Resolved); + let cases: Vec = vec![ + ( + "an_owner_who_is_also_a_collaborator_reads_as_the_owner", + Fake::new() + .admin("nel") + .owner(Resolved::Ready(Some(owner("nel")))), + true, + true, + Resolved::Ready(RepoRole::Owner), + ), + ( + "a_collaborator_in_the_admin_set_reads_as_a_collaborator", + Fake::new() + .admin("nel") + .owner(Resolved::Ready(Some(owner("olaren")))), + true, + false, + Resolved::Ready(RepoRole::Collaborator), + ), + ( + "a_member_who_never_collaborated_reads_as_a_member", + Fake::new().owner(Resolved::Ready(Some(owner("olaren")))), + false, + true, + Resolved::Ready(RepoRole::Member), + ), + ( + "nobody_at_all_reads_as_a_stranger", + Fake::new().owner(Resolved::Ready(Some(owner("olaren")))), + false, + false, + Resolved::Ready(RepoRole::Stranger), + ), + ( + "warming_ownership_answers_warming_rather_than_a_demoted_role", + Fake::new(), + false, + false, + Resolved::Warming, + ), + ]; + let anyone = policy_for(&target, ContributionPolicy::Anyone); + cases + .iter() + .for_each(|(label, acl, collaborates, member, expected)| { + let collaboration = CollaborationConsent::assumed(&target, &who, *collaborates); + let membership = MemberStatus::assumed((), &who, *member); + let access = access_of(&collaboration, &membership, &anyone); + assert_eq!(repo_role(acl, &access), *expected, "{label}"); + }); + } + + #[test] + fn the_blocklist_decides_admission_not_identity() { + let target = repo("anemone"); + let who = acc("nel"); + let collaboration = collaborating(&target, &who); + let membership = joined(&who); + [Resolved::Ready(true), Resolved::Warming] + .into_iter() + .for_each(|blocked| { + let acl = Fake::new() + .blocked(blocked) + .owner(Resolved::Ready(Some(owner("olaren")))); + let anyone = policy_for(&target, ContributionPolicy::Anyone); + let access = access_of(&collaboration, &membership, &anyone); + assert_eq!( + repo_role(&acl, &access), + Resolved::Ready(RepoRole::Collaborator), + "Blocklist decides admission, not who somebody is" + ); + assert!( + !allowed(can_contribute(&acl, &access)), + "Blocked subject doesn't write, whatever policy runs underneath" + ); + }); + } + + #[test] + fn one_access_is_one_subject_on_one_repo() { + let target = repo("anemone"); + let elsewhere = repo("barnacle"); + let (nel, olaren) = (acc("nel"), acc("olaren")); + let collaboration = collaborating(&target, &nel); + let here = policy_for(&target, ContributionPolicy::Anyone); + let there = policy_for(&elsewhere, ContributionPolicy::Anyone); + assert!( + RepoAccess::of(&collaboration, &joined(&olaren), &here).is_none(), + "Role read off two subjects' consents would authorize account nobody consented to" + ); + assert!( + RepoAccess::of(&collaboration, &joined(&nel), &there).is_none(), + "Access built from another repo's policy would authorize repository policy never named" + ); + assert!(RepoAccess::of(&collaboration, &joined(&nel), &here).is_some()); + } + + #[test] + fn an_admin_administers_and_creates_but_doesnt_push_arbitrary_repos() { + let acl = Fake::new() + .admin("nel") + .owner(Resolved::Ready(Some(owner("olaren")))); + let (nel, squid) = (acc("nel"), repo("squid")); + assert_eq!(can_admin_knot(&acl, &nel), Decision::Allow); + assert_eq!(can_create_repo(&acl, &outside(&nel)), Decision::Allow); + assert_eq!( + can_push(&acl, &ungranted(&squid, &nel)), + Decision::Deny, + "Knot admin can't push to somebody else's repo" + ); + } + + #[test] + fn decisions() { + type Case = (&'static str, Fake, fn(&Fake) -> Decision, Decision); + let cases: Vec = vec![ + ( + "a_member_creates_repos", + Fake::new(), + |acl| can_create_repo(acl, &joined(&acc("olaren"))), + Decision::Allow, + ), + ( + "a_member_cant_administer_the_knot", + Fake::new(), + |acl| can_admin_knot(acl, &acc("olaren")), + Decision::Deny, + ), + ( + "an_open_knot_lets_an_outsider_create", + Fake::new().open(), + |acl| can_create_repo(acl, &outside(&acc("teq"))), + Decision::Allow, + ), + ( + "an_open_knot_doesnt_widen_push", + Fake::new() + .open() + .owner(Resolved::Ready(Some(owner("nel")))), + |acl| can_push(acl, &ungranted(&repo("squid"), &acc("teq"))), + Decision::Deny, + ), + ( + "a_blocked_account_cant_create", + Fake::new().open().blocked(Resolved::Ready(true)), + |acl| can_create_repo(acl, &joined(&acc("squid"))), + Decision::Deny, + ), + ( + "an_admin_is_immune_to_the_blocklist", + Fake::new() + .open() + .admin("nel") + .blocked(Resolved::Ready(true)), + |acl| can_create_repo(acl, &outside(&acc("nel"))), + Decision::Allow, + ), + ( + "the_repo_owner_pushes", + Fake::new().owner(Resolved::Ready(Some(owner("nel")))), + |acl| can_push(acl, &ungranted(&repo("squid"), &acc("nel"))), + Decision::Allow, + ), + ( + "a_collaborator_pushes_without_owning", + Fake::new().owner(Resolved::Ready(Some(owner("nel")))), + |acl| can_push(acl, &collaborating(&repo("squid"), &acc("olaren"))), + Decision::Allow, + ), + ( + "push_allows_on_a_confirmed_collaborator_while_the_registry_warms", + Fake::new().owner(Resolved::Warming), + |acl| can_push(acl, &collaborating(&repo("squid"), &acc("olaren"))), + Decision::Allow, + ), + ( + "push_allows_a_confirmed_owner_with_no_acceptance_behind_them", + Fake::new().owner(Resolved::Ready(Some(owner("nel")))), + |acl| can_push(acl, &ungranted(&repo("squid"), &acc("nel"))), + Decision::Allow, + ), + ( + "push_denies_when_ownership_is_warming_and_not_a_collaborator", + Fake::new().owner(Resolved::Warming), + |acl| can_push(acl, &ungranted(&repo("squid"), &acc("nel"))), + Decision::Deny, + ), + ( + "push_denies_an_unregistered_repo", + Fake::new().owner(Resolved::Ready(None)), + |acl| can_push(acl, &ungranted(&repo("squid"), &acc("nel"))), + Decision::Deny, + ), + ( + "push_matches_a_did_web_owner_across_authority_case", + Fake::new().owner(Resolved::Ready(Some( + OwnerDid::new("did:web:OYSTER.cafe").unwrap(), + ))), + |acl| { + can_push( + acl, + &ungranted( + &repo("squid"), + &AccountDid::new("did:web:oyster.cafe").unwrap(), + ), + ) + }, + Decision::Allow, + ), + ( + "push_denies_a_did_plc_owner_whose_case_differs", + Fake::new().owner(Resolved::Ready(Some(OwnerDid::new("did:plc:ABC").unwrap()))), + |acl| { + can_push( + acl, + &ungranted(&repo("squid"), &AccountDid::new("did:plc:abc").unwrap()), + ) + }, + Decision::Deny, + ), + ( + "manage_collaborators_fails_closed_while_ownership_is_warming", + Fake::new().owner(Resolved::Warming), + |acl| can_manage_collaborators(acl, &acc("olaren"), &repo("squid")), + Decision::Deny, + ), + ( + "an_admin_is_authorized_before_the_projection_warms", + Fake::new().admin("nel"), + |acl| can_admin_knot(acl, &acc("nel")), + Decision::Allow, + ), + ( + "an_admin_creates_before_the_projection_warms", + Fake::new().admin("nel"), + |acl| can_create_repo(acl, &outside(&acc("nel"))), + Decision::Allow, + ), + ( + "an_admin_deletes_a_repo_before_the_registry_warms", + Fake::new().admin("nel").owner(Resolved::Warming), + |acl| can_delete_repo(acl, &acc("nel"), &repo("squid")), + Decision::Allow, + ), + ]; + cases.iter().for_each(|(label, acl, eval, expected)| { + assert_eq!(eval(acl), *expected, "{label}"); + }); + } + + #[test] + fn a_blocked_owner_cannot_push_or_invite() { + let acl = Fake::new() + .owner(Resolved::Ready(Some(owner("squid")))) + .blocked(Resolved::Ready(true)); + let (squid, anemone) = (acc("squid"), repo("anemone")); + assert_eq!( + can_push(&acl, &ungranted(&anemone, &squid)), + Decision::Deny, + "ban overrides ownership on write path" + ); + assert_eq!( + can_manage_collaborators(&acl, &squid, &anemone), + Decision::Deny + ); + } + + #[test] + fn a_warming_blocklist_fails_create_and_push_closed() { + let acl = Fake::new() + .open() + .blocked(Resolved::Warming) + .owner(Resolved::Ready(Some(owner("squid")))); + let (squid, anemone) = (acc("squid"), repo("anemone")); + assert_eq!( + can_create_repo(&acl, &joined(&squid)), + Decision::Deny, + "an unresolved blocklist could contain the ban, so create must fail closed" + ); + assert_eq!(can_push(&acl, &ungranted(&anemone, &squid)), Decision::Deny); + } + + #[test] + fn a_stranger_is_denied_everything() { + let acl = Fake::new().owner(Resolved::Ready(Some(owner("nel")))); + let (teq, squid) = (acc("teq"), repo("squid")); + assert_eq!(can_admin_knot(&acl, &teq), Decision::Deny); + assert_eq!(can_create_repo(&acl, &outside(&teq)), Decision::Deny); + assert_eq!(can_push(&acl, &ungranted(&squid, &teq)), Decision::Deny); + } + + #[test] + fn a_fully_warming_index_denies_every_index_backed_decision() { + let acl = Fake::new(); + let (olaren, nel, squid) = (acc("olaren"), acc("nel"), repo("squid")); + assert_eq!(can_create_repo(&acl, &outside(&olaren)), Decision::Deny); + assert_eq!(can_push(&acl, &ungranted(&squid, &nel)), Decision::Deny); + } + + #[test] + fn is_allowed_reports_the_verdict() { + assert!(Decision::Allow.is_allowed()); + assert!(!Decision::Deny.is_allowed()); + } + + #[test] + fn only_the_repo_owner_manages_collaborators() { + let acl = Fake::new() + .admin("nel") + .owner(Resolved::Ready(Some(owner("olaren")))); + assert_eq!( + can_manage_collaborators(&acl, &acc("olaren"), &repo("squid")), + Decision::Allow, + "repo owner manages its own collaborators" + ); + assert_eq!( + can_manage_collaborators(&acl, &acc("lyna"), &repo("squid")), + Decision::Deny, + "collaborator cannot manage collaborator set" + ); + assert_eq!( + can_manage_collaborators(&acl, &acc("nel"), &repo("squid")), + Decision::Deny, + "a knot admin's authority stops at the knot, and this repo answers to olaren" + ); + } + + #[test] + fn repo_deletion_is_the_owner_or_a_knot_admin() { + let acl = Fake::new() + .admin("nel") + .owner(Resolved::Ready(Some(owner("olaren")))); + assert_eq!( + can_delete_repo(&acl, &acc("olaren"), &repo("squid")), + Decision::Allow, + "repo owner deletes its own repo" + ); + assert_eq!( + can_delete_repo(&acl, &acc("nel"), &repo("squid")), + Decision::Allow, + "knot admin deletes any repo" + ); + assert_eq!( + can_delete_repo(&acl, &acc("lyna"), &repo("squid")), + Decision::Deny, + "collaborator cannot delete the repo" + ); + } + + mod integration { + use super::*; + use knot_cob::{CobHome, CobStore, KnotAuthored}; + use knot_cobs::{CollaboratorsChange, Grant, MembersChange, Registration, RegistryChange}; + use knot_git::{Layout, Repo}; + use knot_runtime::{K256Signer, SeededEntropy}; + use knot_types::{KnotId, RepoName, RepoRkey, UnixSeconds}; + use std::path::PathBuf; + + fn knot_home() -> CobHome { + CobHome::from(&KnotId::new("did:web:knot.nel.pet").unwrap()) + } + + fn grant(subject: &str, at: i64) -> Grant { + Grant { + subject: acc(subject), + added_by: acc("nel"), + created_at: UnixSeconds::new(at), + } + } + + fn registration( + owner_id: &str, + key: &str, + repo_did: &knot_types::RepoDid, + at: i64, + ) -> Registration { + Registration { + owner: owner(owner_id), + rkey: RepoRkey::new(key).unwrap(), + name: RepoName::new(key).unwrap(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(at), + policy: ContributionPolicy::default(), + fostered: false, + + retired_at: None, + } + } + + fn world() -> (tempfile::TempDir, PathBuf, Layout, K256Signer) { + let dir = tempfile::tempdir().unwrap(); + let meta_path = dir.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(dir.path().join("repos")); + let signer = K256Signer::generate(&SeededEntropy::new(1)); + (dir, meta_path, layout, signer) + } + + fn seed( + store: &CobStore, + home: &CobHome, + change: &P, + signer: &K256Signer, + at: UnixSeconds, + ) { + store.create(home, change, signer, at).unwrap(); + } + + async fn creates(index: &Index, acl: &KnotAcl<'_>, who: &AccountDid) -> Decision { + can_create_repo(acl, &index.member_status(who)) + } + + async fn pushes( + index: &Index, + acl: &KnotAcl<'_>, + repo: &RepoDid, + who: &AccountDid, + ) -> Decision { + can_push(acl, &index.collaboration_consent(repo, who)) + } + + #[tokio::test] + async fn the_enforcer_decides_over_a_real_rebuilt_index() { + let (_dir, meta_path, layout, signer) = world(); + let at = UnixSeconds::new; + + let meta = Repo::open(&meta_path).unwrap(); + let store = CobStore::new(&meta); + let squid = repo("squid"); + seed( + &store, + &knot_home(), + &MembersChange::Add(grant("olaren", 1)), + &signer, + at(1), + ); + seed( + &store, + &knot_home(), + &RegistryChange::Register(registration("nel", "anemone", &squid, 1)), + &signer, + at(1), + ); + let git = layout.create(&squid).unwrap(); + seed( + &CobStore::new(&git), + &CobHome::from(&squid), + &CollaboratorsChange::Add(grant("lyna", 1)), + &signer, + at(1), + ); + + let index = Index::new(&meta_path, layout.clone()); + index.rebuild().unwrap(); + index.ensure_collaborators(&squid).unwrap(); + let admins = BTreeSet::from([acc("nel")]); + let acl = KnotAcl::new(&admins, AdmissionPolicy::Closed, &index); + + assert_eq!(can_admin_knot(&acl, &acc("nel")), Decision::Allow); + assert_eq!(creates(&index, &acl, &acc("nel")).await, Decision::Allow); + assert_eq!( + pushes(&index, &acl, &squid, &acc("nel")).await, + Decision::Allow, + "nel owns squid in the registry" + ); + + assert_eq!(can_admin_knot(&acl, &acc("olaren")), Decision::Deny); + assert_eq!(creates(&index, &acl, &acc("olaren")).await, Decision::Allow); + assert_eq!( + pushes(&index, &acl, &squid, &acc("olaren")).await, + Decision::Deny, + "member who is neither owner nor collaborator cannot push" + ); + + assert_eq!( + pushes(&index, &acl, &squid, &acc("lyna")).await, + Decision::Allow, + "lyna collaborates on squid" + ); + assert_eq!(creates(&index, &acl, &acc("lyna")).await, Decision::Deny); + + assert_eq!( + pushes(&index, &acl, &squid, &acc("teq")).await, + Decision::Deny + ); + assert_eq!(creates(&index, &acl, &acc("teq")).await, Decision::Deny); + + let cold = Index::new(&meta_path, layout); + let cold_acl = KnotAcl::new(&admins, AdmissionPolicy::Closed, &cold); + assert_eq!( + can_admin_knot(&cold_acl, &acc("nel")), + Decision::Allow, + "admin is config, answered before any rebuild" + ); + assert_eq!( + pushes(&cold, &cold_acl, &squid, &acc("nel")).await, + Decision::Deny, + "before rebuild owner lookup is warming, so push fails closed" + ); + assert_eq!( + creates(&cold, &cold_acl, &acc("olaren")).await, + Decision::Deny + ); + } + + #[tokio::test] + async fn a_repo_re_registered_under_a_second_owner_grants_push_only_to_the_later_owner() { + let (_dir, meta_path, layout, signer) = world(); + let at = UnixSeconds::new; + + let squid = repo("squid"); + layout.create(&squid).unwrap(); + + let meta = Repo::open(&meta_path).unwrap(); + let store = CobStore::new(&meta); + let created = store + .create( + &knot_home(), + &RegistryChange::Register(registration("nel", "anemone", &squid, 1)), + &signer, + at(1), + ) + .unwrap(); + store + .update( + &knot_home(), + created.object, + &RegistryChange::Register(registration("olaren", "fork", &squid, 2)), + &signer, + at(2), + ) + .unwrap(); + + let index = Index::new(&meta_path, layout); + index.rebuild().unwrap(); + let admins = BTreeSet::new(); + let acl = KnotAcl::new(&admins, AdmissionPolicy::Closed, &index); + + assert_eq!( + pushes(&index, &acl, &squid, &acc("nel")).await, + Decision::Deny, + "re-register moves repo wholesale, so displaced owner loses push" + ); + assert_eq!( + pushes(&index, &acl, &squid, &acc("olaren")).await, + Decision::Allow, + "linear causal order gives later registrant deterministic ownership" + ); + } + + #[tokio::test] + async fn a_collaborator_on_an_unregistered_repo_cant_push_after_a_real_rebuild() { + let (_dir, meta_path, layout, signer) = world(); + let at = UnixSeconds::new; + + let squid = repo("squid"); + let git = layout.create(&squid).unwrap(); + seed( + &CobStore::new(&git), + &CobHome::from(&squid), + &CollaboratorsChange::Add(grant("lyna", 1)), + &signer, + at(1), + ); + + let index = Index::new(&meta_path, layout); + index.rebuild().unwrap(); + let admins = BTreeSet::new(); + let acl = KnotAcl::new(&admins, AdmissionPolicy::Closed, &index); + assert_eq!( + pushes(&index, &acl, &squid, &acc("lyna")).await, + Decision::Deny, + "rebuild materializes collaborators only for registered repos, so a collaborator COB on an unregistered repo never warms and never grants a push" + ); + } + } +} diff --git a/knot2/crates/knot-atproto/Cargo.toml b/knot2/crates/knot-atproto/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..45b37d9691a6d9d3641b37cc45f33823d624854b --- /dev/null +++ b/knot2/crates/knot-atproto/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "knot-atproto" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-runtime = { workspace = true } +knot-cache = { workspace = true } +cid = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +sha2 = { workspace = true } +base32 = { workspace = true } +bs58 = { workspace = true } +thiserror = { workspace = true } +url = { workspace = true } +base64 = { workspace = true } +bytes = { workspace = true } +http = { workspace = true } +k256 = { workspace = true } + +[dev-dependencies] +knot-lexicons = { workspace = true } +tokio = { workspace = true } +proptest = { workspace = true } diff --git a/knot2/crates/knot-atproto/fuzz/.gitignore b/knot2/crates/knot-atproto/fuzz/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1a45eee7760d240bfa8ac1989088e0349169f264 --- /dev/null +++ b/knot2/crates/knot-atproto/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-atproto/fuzz/Cargo.lock b/knot2/crates/knot-atproto/fuzz/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..1e17fcac85a73d632e0f1a631afc79b2a394098a --- /dev/null +++ b/knot2/crates/knot-atproto/fuzz/Cargo.lock @@ -0,0 +1,4225 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base32" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "022dfe9eb35f19ebbcb51e0b40a5ab759f46ad60cadf7297e0bd085afb50e076" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +dependencies = [ + "serde", +] + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "gix-trace", + "libc", + "prodash", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest 0.12.28", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-atproto" +version = "0.1.0" +dependencies = [ + "base32", + "base64", + "bs58", + "bytes", + "futures", + "http", + "knot-runtime", + "knot-types", + "moka", + "scc", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.18", + "url", +] + +[[package]] +name = "knot-atproto-fuzz" +version = "0.0.0" +dependencies = [ + "knot-atproto", + "libfuzzer-sys", +] + +[[package]] +name = "knot-runtime" +version = "0.1.0" +dependencies = [ + "bytes", + "futures", + "getrandom 0.4.3", + "http", + "k256", + "reqwest 0.13.1", + "thiserror 2.0.18", + "tokio", + "url", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multibase" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" +dependencies = [ + "base-x", + "base256emoji", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "reqwest" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "saa" +version = "5.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f5acb362a0e75c2a963532fa7fabf13dff81626dc494df16488d30befcbea0" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scc" +version = "3.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5581cd5dd2cb79cbe9d8137071d67f62f0db926e83a07b4d14561c5b7d423776" +dependencies = [ + "saa", + "sdd", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdd" +version = "4.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f0e40a01b94e35d1dacbcfbe5bfd3d31e37d9590b2e6d86a82b0e87bd4f551" +dependencies = [ + "saa", +] + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/knot2/crates/knot-atproto/fuzz/Cargo.toml b/knot2/crates/knot-atproto/fuzz/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..86a16201ebc45d91108e8a830773bbb2bb4d436d --- /dev/null +++ b/knot2/crates/knot-atproto/fuzz/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "knot-atproto-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-atproto] +path = ".." + +[[bin]] +name = "pubkey" +path = "fuzz_targets/pubkey.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "did_document" +path = "fuzz_targets/did_document.rs" +test = false +doc = false +bench = false diff --git a/knot2/crates/knot-atproto/fuzz/fuzz_targets/did_document.rs b/knot2/crates/knot-atproto/fuzz/fuzz_targets/did_document.rs new file mode 100644 index 0000000000000000000000000000000000000000..6bd86efbc23864a49047ce212fc93bf379d778aa --- /dev/null +++ b/knot2/crates/knot-atproto/fuzz/fuzz_targets/did_document.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_atproto::fuzz::did_document(data); +}); diff --git a/knot2/crates/knot-atproto/fuzz/fuzz_targets/pubkey.rs b/knot2/crates/knot-atproto/fuzz/fuzz_targets/pubkey.rs new file mode 100644 index 0000000000000000000000000000000000000000..4ad03d3f4775d92d12dfeb964a878c032f3ec37c --- /dev/null +++ b/knot2/crates/knot-atproto/fuzz/fuzz_targets/pubkey.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_atproto::fuzz::pubkey(data); +}); diff --git a/knot2/crates/knot-atproto/src/auth.rs b/knot2/crates/knot-atproto/src/auth.rs new file mode 100644 index 0000000000000000000000000000000000000000..e936afd64d44f2fff026fadd3beb131cd276fb52 --- /dev/null +++ b/knot2/crates/knot-atproto/src/auth.rs @@ -0,0 +1,60 @@ +use http::HeaderValue; +use knot_runtime::{Entropy, HttpRequest, Signer}; +use knot_types::{KnotId, Nsid, ServiceDid, UnixSeconds}; + +use crate::AtprotoError; +use crate::jwt; +use crate::jwt::JwtNonce; + +const POINTER_NONCE_BYTES: usize = 16; + +pub struct PointerAuth<'a> { + pub issuer: &'a KnotId, + pub audience: &'a ServiceDid, + pub lxm: &'a Nsid, + pub now_unix: UnixSeconds, +} + +// a sender holding one across an await only stays Send with this +pub trait PointerAuthorizer: Sync { + fn authorize( + &self, + request: &mut HttpRequest, + ctx: &PointerAuth<'_>, + ) -> Result<(), AtprotoError>; +} + +pub struct ServiceAuth<'a> { + signer: &'a dyn Signer, + entropy: &'a dyn Entropy, +} + +impl<'a> ServiceAuth<'a> { + pub fn new(signer: &'a dyn Signer, entropy: &'a dyn Entropy) -> Self { + Self { signer, entropy } + } +} + +impl PointerAuthorizer for ServiceAuth<'_> { + fn authorize( + &self, + request: &mut HttpRequest, + ctx: &PointerAuth<'_>, + ) -> Result<(), AtprotoError> { + let mut bytes = [0u8; POINTER_NONCE_BYTES]; + self.entropy.fill(&mut bytes); + let nonce = JwtNonce::new(knot_types::lowercase_hex(&bytes))?; + let token = jwt::mint( + self.signer, + ctx.issuer, + ctx.audience, + ctx.lxm, + nonce, + ctx.now_unix, + ); + let header = HeaderValue::from_str(&format!("Bearer {token}")) + .expect("base64url jwt is valid header value"); + request.headers.insert(http::header::AUTHORIZATION, header); + Ok(()) + } +} diff --git a/knot2/crates/knot-atproto/src/identity.rs b/knot2/crates/knot-atproto/src/identity.rs new file mode 100644 index 0000000000000000000000000000000000000000..2b63c507bc74d948d0fa4ccf5aaf4077e27f9dc8 --- /dev/null +++ b/knot2/crates/knot-atproto/src/identity.rs @@ -0,0 +1,454 @@ +use std::collections::BTreeMap; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use knot_runtime::{Entropy, PublicKeyBytes, Signer}; +use knot_types::{ActorId, KnotId, KnotServiceUrl, OwnerDid, RepoDid, RepoRkey}; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +const PLC_OP_TYPE: &str = "plc_operation"; +pub const ATPROTO_METHOD: &str = "atproto"; +pub(crate) const KNOT_HOME_SERVICE: &str = "tangled_knot"; +pub(crate) const KNOT_HOME_TYPE: &str = "TangledKnot"; +const PDS_SERVICE_ID: &str = crate::plc::ATPROTO_PDS_SERVICE; +const DID_PLC_PREFIX: &str = "did:plc:"; +const DID_SUFFIX_LEN: usize = 24; +const MINT_NONCE_LEN: usize = 16; + +#[derive(Debug, thiserror::Error)] +pub enum IdentityError { + #[error("plc operation couldn't be encoded: {0}")] + Encode(String), + #[error("plc operation shape unsupported: {0}")] + Op(String), + #[error("derived did:plc isn't valid DID: {0}")] + Did(#[from] knot_types::ParseError), +} + +#[derive(Serialize)] +struct PlcService { + r#type: &'static str, + endpoint: String, +} + +#[derive(Serialize)] +struct PlcOperation { + #[serde(rename = "type")] + op_type: &'static str, + #[serde(rename = "rotationKeys")] + rotation_keys: Vec, + #[serde(rename = "verificationMethods")] + verification_methods: BTreeMap<&'static str, String>, + #[serde(rename = "alsoKnownAs")] + also_known_as: Vec, + services: BTreeMap<&'static str, PlcService>, + prev: Option, + #[serde(skip_serializing_if = "Option::is_none")] + sig: Option, +} + +pub struct PreparedRepoDid { + pub did: RepoDid, + operation: serde_json::Value, +} + +impl PreparedRepoDid { + pub fn operation(&self) -> &serde_json::Value { + &self.operation + } +} + +pub(crate) fn did_key_for(public: &PublicKeyBytes) -> String { + format!("did:key:{}", multikey_secp256k1(public)) +} + +fn multikey_secp256k1(public: &PublicKeyBytes) -> ActorId { + ActorId::from_secp256k1(public.as_bytes()) +} + +fn encode_cbor(operation: &PlcOperation) -> Result, IdentityError> { + serde_ipld_dagcbor::to_vec(operation).map_err(|error| IdentityError::Encode(error.to_string())) +} + +pub struct MintNonce([u8; MINT_NONCE_LEN]); + +impl MintNonce { + pub fn mint(entropy: &dyn Entropy, owner: &OwnerDid, rkey: &RepoRkey) -> Self { + let mut bytes = [0u8; MINT_NONCE_LEN]; + entropy.derive(mint_label(owner, rkey)).fill(&mut bytes); + Self(bytes) + } + + fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +fn mint_label(owner: &OwnerDid, rkey: &RepoRkey) -> u64 { + [owner.as_str(), rkey.as_str()] + .iter() + .flat_map(|part| part.bytes().chain(std::iter::once(0u8))) + .fold(0xcbf2_9ce4_8422_2325u64, |hash, byte| { + (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) +} + +pub fn prepare_repo_did( + signer: &dyn Signer, + knot_service_url: &KnotServiceUrl, + mint_nonce: &MintNonce, +) -> Result { + let tag = base32::encode( + base32::Alphabet::Rfc4648 { padding: false }, + mint_nonce.as_bytes(), + ) + .to_lowercase(); + let base = knot_service_url.as_str(); + let knot_key = did_key_for(&signer.public_key()); + let mut operation = PlcOperation { + op_type: PLC_OP_TYPE, + rotation_keys: vec![knot_key.clone()], + verification_methods: BTreeMap::from([(ATPROTO_METHOD, knot_key)]), + also_known_as: Vec::new(), + services: BTreeMap::from([ + ( + KNOT_HOME_SERVICE, + PlcService { + r#type: KNOT_HOME_TYPE, + endpoint: format!("{base}/repo/{tag}"), + }, + ), + ( + PDS_SERVICE_ID, + PlcService { + r#type: crate::plc::PDS_SERVICE_TYPE, + endpoint: base.to_owned(), + }, + ), + ]), + prev: None, + sig: None, + }; + + let unsigned = encode_cbor(&operation)?; + operation.sig = Some(URL_SAFE_NO_PAD.encode(signer.sign(&unsigned).as_bytes())); + + let signed = encode_cbor(&operation)?; + let did = derive_did_plc(&signed)?; + let operation = serde_json::to_value(operation) + .map_err(|error| IdentityError::Encode(error.to_string()))?; + Ok(PreparedRepoDid { did, operation }) +} + +fn derive_did_plc(signed_cbor: &[u8]) -> Result { + let digest = Sha256::digest(signed_cbor); + let encoded = + base32::encode(base32::Alphabet::Rfc4648 { padding: false }, &digest).to_lowercase(); + let suffix: String = encoded.chars().take(DID_SUFFIX_LEN).collect(); + Ok(RepoDid::new(format!("{DID_PLC_PREFIX}{suffix}"))?) +} + +pub fn knot_did_document( + id: &KnotId, + signing_key: &PublicKeyBytes, + service_url: &KnotServiceUrl, +) -> serde_json::Value { + serde_json::json!({ + "@context": [ + "https://www.w3.org/ns/did/v1", + "https://w3id.org/security/multikey/v1", + "https://w3id.org/security/suites/secp256k1-2019/v1" + ], + "id": id, + "verificationMethod": [{ + "id": format!("{id}#{ATPROTO_METHOD}"), + "type": "Multikey", + "controller": id, + "publicKeyMultibase": multikey_secp256k1(signing_key) + }], + "service": [{ + "id": format!("#{KNOT_HOME_SERVICE}"), + "type": KNOT_HOME_TYPE, + "serviceEndpoint": service_url + }, { + "id": format!("#{PDS_SERVICE_ID}"), + "type": crate::plc::PDS_SERVICE_TYPE, + "serviceEndpoint": service_url + }] + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::*; + use knot_runtime::verify; + + #[test] + fn did_derivation_is_deterministic_and_nonce_sensitive() { + let key = runtime_signer(2); + let first = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://nel.pet").unwrap(), + &repo_nonce(101), + ) + .unwrap(); + let again = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://nel.pet").unwrap(), + &repo_nonce(101), + ) + .unwrap(); + assert_eq!(first.did, again.did); + assert_eq!(first.operation(), again.operation()); + + let slashed = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://nel.pet/").unwrap(), + &repo_nonce(101), + ) + .unwrap(); + assert_eq!( + first.did, slashed.did, + "a trailing slash on the knot url changes neither the endpoint nor the did" + ); + + let other_nonce = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://nel.pet").unwrap(), + &repo_nonce(102), + ) + .unwrap(); + assert_ne!( + first.did, other_nonce.did, + "shared knot key no longer distinguishes repos; mint nonce must" + ); + } + + #[test] + fn the_derived_did_plc_is_pinned_and_well_formed() { + let prepared = prepare_repo_did( + &runtime_signer(1), + &KnotServiceUrl::new("https://knot.oyster.cafe").unwrap(), + &repo_nonce(104), + ) + .unwrap(); + let did = prepared.did.as_str(); + assert_eq!( + did, "did:plc:uvctvz2ohwroy3n7hdau6tu5", + "any change to this value means did:plc derivation no longer matches the PLC directory" + ); + let suffix = did.strip_prefix(DID_PLC_PREFIX).unwrap(); + assert_eq!(suffix.len(), DID_SUFFIX_LEN); + assert!( + suffix + .chars() + .all(|c| c.is_ascii_lowercase() || ('2'..='7').contains(&c)), + "the did:plc suffix is lowercase base32" + ); + } + + #[test] + fn the_operation_signature_verifies_against_the_repo_key_over_the_unsigned_cbor() { + let key = runtime_signer(5); + let prepared = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://nel.pet").unwrap(), + &repo_nonce(106), + ) + .unwrap(); + let operation = prepared.operation(); + + let signature_b64 = operation["sig"].as_str().unwrap(); + let signature = + knot_runtime::Signature::from_bytes(URL_SAFE_NO_PAD.decode(signature_b64).unwrap()); + + let mut unsigned = operation.clone(); + unsigned.as_object_mut().unwrap().remove("sig"); + let unsigned_cbor = serde_ipld_dagcbor::to_vec(&unsigned).unwrap(); + + assert!( + verify(&key.public_key(), &unsigned_cbor, &signature), + "genesis op is self-signed by repo rotation key over its unsigned dag-cbor" + ); + } + + #[test] + fn the_genesis_op_marks_the_home_knot_and_publishes_the_knot_as_pds() { + let key = runtime_signer(6); + let prepared = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://knot.oyster.cafe").unwrap(), + &repo_nonce(107), + ) + .unwrap(); + let operation = prepared.operation(); + + assert_eq!(operation["type"], "plc_operation"); + assert_eq!(operation["prev"], serde_json::Value::Null); + assert_eq!(operation["alsoKnownAs"], serde_json::json!([])); + assert!( + operation["services"][KNOT_HOME_SERVICE]["endpoint"] + .as_str() + .unwrap() + .starts_with("https://knot.oyster.cafe/repo/") + ); + assert_eq!( + operation["services"][KNOT_HOME_SERVICE]["type"], + KNOT_HOME_TYPE + ); + let expected_key = did_key_for(&key.public_key()); + assert_eq!( + operation["services"]["atproto_pds"], + serde_json::json!({ + "type": crate::plc::PDS_SERVICE_TYPE, + "endpoint": "https://knot.oyster.cafe" + }) + ); + assert_eq!( + operation["verificationMethods"], + serde_json::json!({ "atproto": expected_key }) + ); + assert_eq!(operation["rotationKeys"][0], expected_key); + assert_eq!(operation["rotationKeys"].as_array().unwrap().len(), 1); + } + + #[test] + fn the_knot_document_declares_its_signing_key_and_tangled_knot_service() { + let key = runtime_signer(7); + let knot = KnotId::new("did:web:knot.oyster.cafe").unwrap(); + let document = knot_did_document( + &knot, + &key.public_key(), + &KnotServiceUrl::new("https://knot.oyster.cafe").unwrap(), + ); + + assert_eq!( + document["verificationMethod"][0]["id"], "did:web:knot.oyster.cafe#atproto", + "knot publishes its signing key under the atproto method" + ); + assert_eq!( + document["verificationMethod"][0]["publicKeyMultibase"], + serde_json::json!(multikey_secp256k1(&key.public_key())), + "published verification method is the knot's own signing key" + ); + assert_eq!( + document["service"][0], + serde_json::json!({ + "id": "#tangled_knot", + "type": "TangledKnot", + "serviceEndpoint": "https://knot.oyster.cafe" + }), + "knot self-declares its tangled_knot service at the knot root" + ); + assert_eq!( + document["service"][1], + serde_json::json!({ + "id": "#atproto_pds", + "type": crate::plc::PDS_SERVICE_TYPE, + "serviceEndpoint": "https://knot.oyster.cafe" + }), + "knot publishes itself as its own PDS beside the service entry" + ); + } + + #[test] + fn the_update_operation_reaches_the_desired_shape_and_chains_prev() { + let key = runtime_signer(8); + let target = crate::plc::RepoTarget::new( + &key.public_key(), + &KnotServiceUrl::new("https://knot.oyster.cafe").unwrap(), + ); + let genesis = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://knot.oyster.cafe").unwrap(), + &repo_nonce(108), + ) + .unwrap(); + let last = genesis.operation(); + + assert!( + target.satisfied_by(last).unwrap(), + "a fresh genesis op is already in the desired account shape" + ); + let update = crate::plc::update_operation(&key, last, &target).unwrap(); + assert_eq!( + update["rotationKeys"], + serde_json::json!([did_key_for(&key.public_key())]), + "the update pins rotation to the knot key" + ); + assert_eq!( + update["verificationMethods"]["atproto"], + did_key_for(&key.public_key()) + ); + let pds = &update["services"]["atproto_pds"]; + assert_eq!(pds["type"], crate::plc::PDS_SERVICE_TYPE); + assert_eq!(pds["endpoint"], "https://knot.oyster.cafe"); + assert!( + update["services"][KNOT_HOME_SERVICE]["endpoint"] + .as_str() + .unwrap() + .starts_with("https://knot.oyster.cafe/repo/"), + "the tangled_knot service survives the snapshot with its repo path" + ); + assert!( + update["prev"] + .as_str() + .expect("a chained update carries prev") + .starts_with("bafyre"), + "prev must be a CIDv1 dag-cbor string the way the directory serves them" + ); + assert_ne!(update["sig"], serde_json::Value::Null); + assert!( + target.satisfied_by(&update).unwrap(), + "the signed update lands in the desired shape" + ); + + let mut tampered = update.clone(); + tampered["verificationMethods"]["atproto"] = + serde_json::Value::String(did_key_for(&runtime_signer(9).public_key())); + assert!( + !target.satisfied_by(&tampered).unwrap(), + "a substituted signing key no longer satisfies" + ); + } + + #[test] + fn legacy_knot1_document_is_rewritten_to_desired_shape() { + let per_repo = runtime_signer(10); + let knot = runtime_signer(11); + let legacy = serde_json::json!({ + "type": "plc_operation", + "rotationKeys": [did_key_for(&per_repo.public_key())], + "verificationMethods": {"atproto": did_key_for(&per_repo.public_key())}, + "alsoKnownAs": [], + "services": {"atproto_pds": { + "type": crate::plc::PDS_SERVICE_TYPE, + "endpoint": "https://old.knot.example" + }}, + "prev": null, + "sig": "AAAA" + }); + let target = crate::plc::RepoTarget::new( + &knot.public_key(), + &KnotServiceUrl::new("https://new.knot.example").unwrap(), + ); + assert!(!target.satisfied_by(&legacy).unwrap()); + let replacement = crate::plc::update_operation(&per_repo, &legacy, &target).unwrap(); + assert_eq!( + replacement["rotationKeys"], + serde_json::json!([did_key_for(&knot.public_key())]), + "the replacement moves rotation to the knot key in the same operation" + ); + assert_eq!( + replacement["services"]["tangled_knot"], + serde_json::json!({ + "type": "TangledKnot", + "endpoint": "https://new.knot.example", + }), + "a knot1 document without a home service gains one pointing here" + ); + assert!(target.satisfied_by(&replacement).unwrap()); + } +} diff --git a/knot2/crates/knot-atproto/src/jwt.rs b/knot2/crates/knot-atproto/src/jwt.rs new file mode 100644 index 0000000000000000000000000000000000000000..756175b7d4f862579f9e04fd1fbfd15fb97c78d6 --- /dev/null +++ b/knot2/crates/knot-atproto/src/jwt.rs @@ -0,0 +1,669 @@ +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use knot_runtime::{Entropy, OsEntropy, SignatureScheme, Signer}; +use knot_types::crypto::{KeyCodec, PublicKey as CryptoKey}; +use knot_types::service_auth::{ + JwtHeader, ParsedJwt, PublicKey as VerifyKey, ServiceAuthClaims, ServiceAuthError, parse_jwt, +}; +use knot_types::{ + AccountDid, CowStr, Did, DidService, KnotId, Nsid, RepoDid, ServiceDid, UnixSeconds, + lowercase_hex, +}; +use serde::Serialize; + +pub(crate) const CLOCK_SKEW_SECS: i64 = 60; +pub(crate) const SERVICE_TOKEN_LIFETIME_SECS: i64 = 60; +const REPO_TOKEN_MAX_LIFETIME_SECS: i64 = 60; +const MAX_TOKEN_LIFETIME_SECS: i64 = 300; +const MAX_NONCE_BYTES: usize = 256; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct JwtNonce(String); + +#[derive(Clone, PartialEq, Eq, Serialize)] +#[serde(transparent)] +pub struct ServiceJwt(String); + +impl std::fmt::Debug for ServiceJwt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("ServiceJwt").finish_non_exhaustive() + } +} + +impl ServiceJwt { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let three_segments = + value.split('.').count() == 3 && value.split('.').all(|segment| !segment.is_empty()); + match three_segments { + true => Ok(Self(value)), + false => Err(JwtError::NotAJwt), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for ServiceJwt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl JwtNonce { + pub fn from_entropy(bytes: [u8; 16]) -> Self { + Self(lowercase_hex(&bytes)) + } + + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let len = value.len(); + (len <= MAX_NONCE_BYTES) + .then_some(Self(value)) + .ok_or(JwtError::OversizedNonce { len }) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug, thiserror::Error)] +pub enum JwtError { + #[error("malformed token: {0}")] + Parse(#[from] ServiceAuthError), + #[error("token type {typ:?} isn't JWT")] + UnexpectedType { typ: String }, + #[error("token isn't three dot-separated JWT segments")] + NotAJwt, + #[error("issuer {value:?} isn't valid account DID")] + MalformedIssuer { value: String }, + #[error("Audience {value:?} isn't a repo DID.")] + MalformedAudience { value: String }, + #[error("token has no jti, refusing write without replay protection")] + MissingNonce, + #[error("{len}-byte jti exceeds {MAX_NONCE_BYTES}-byte nonce limit")] + OversizedNonce { len: usize }, + #[error("issuer key codec isn't a signing algorithm")] + UnsupportedKeyCodec, + #[error("issuer key isn't valid verifying key: {0}")] + MalformedKey(String), + #[error("signature doesn't verify against issuer key")] + InvalidSignature, + #[error("audience mismatch: token addressed {actual}, expected {expected}")] + AudienceMismatch { expected: String, actual: String }, + #[error("token expired at {exp}, now {now}")] + Expired { exp: UnixSeconds, now: UnixSeconds }, + #[error("token issued in future: iat {iat}, now {now}")] + IssuedInFuture { iat: UnixSeconds, now: UnixSeconds }, + #[error("token lifetime is too long: iat {iat}, exp {exp}, limit {max}s")] + LifetimeTooLong { + exp: UnixSeconds, + iat: UnixSeconds, + max: i64, + }, + #[error("token expires at {exp}, before its issue at {iat}")] + ExpiresBeforeIssued { exp: UnixSeconds, iat: UnixSeconds }, + #[error("method binding mismatch: token bound to {actual:?}, expected {expected}")] + MethodMismatch { + expected: String, + actual: Option, + }, +} + +fn alg_of(signer: &dyn Signer) -> CowStr<'static> { + CowStr::new_static(match signer.scheme() { + SignatureScheme::Secp256k1 => "ES256K", + SignatureScheme::P256 => "ES256", + }) +} + +fn sign_jwt(signer: &dyn Signer, claims: &T) -> ServiceJwt { + let header = JwtHeader { + alg: alg_of(signer), + typ: CowStr::new_static("JWT"), + }; + let header_b64 = + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header).expect("jwt header serializes")); + let payload_b64 = + URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims).expect("service-auth claims serialize")); + let signing_input = format!("{header_b64}.{payload_b64}"); + let signature = signer.sign(signing_input.as_bytes()); + ServiceJwt(format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.as_bytes()) + )) +} + +pub(crate) fn mint( + signer: &dyn Signer, + issuer: &KnotId, + audience: &ServiceDid, + method: &Nsid, + nonce: JwtNonce, + now_unix: UnixSeconds, +) -> ServiceJwt { + let claims = ServiceAuthClaims { + iss: Did::new_owned(issuer.as_str()).expect("knot DID parses as a DID"), + aud: DidService::new_owned(audience.as_str()).expect("service DID parses as a DID"), + exp: now_unix + .saturating_add_secs(SERVICE_TOKEN_LIFETIME_SECS) + .get(), + iat: now_unix.get(), + jti: Some(nonce.as_str().into()), + lxm: Some(method.clone()), + }; + sign_jwt(signer, &claims) +} +pub fn mint_service_jwt( + signer: &dyn Signer, + issuer: &KnotId, + audience: &ServiceDid, + method: &Nsid, + now_unix: UnixSeconds, +) -> ServiceJwt { + let mut entropy = [0u8; 16]; + OsEntropy.fill(&mut entropy); + mint( + signer, + issuer, + audience, + method, + JwtNonce::from_entropy(entropy), + now_unix, + ) +} + +#[derive(Serialize)] +struct ControllerClaim<'a> { + sub: &'a AccountDid, +} + +#[derive(Serialize)] +struct RepoTokenClaims<'a> { + iss: &'a RepoDid, + aud: &'a DidService, + exp: UnixSeconds, + iat: UnixSeconds, + jti: &'a str, + lxm: &'a Nsid, + act: ControllerClaim<'a>, +} + +#[derive(Debug, thiserror::Error)] +pub enum MintExpiryError { + #[error("Exp must be in the future.")] + NotAfterMint, + #[error( + "Exp is {lifespan} seconds after minting, past the {REPO_TOKEN_MAX_LIFETIME_SECS}-second limit." + )] + TooLong { lifespan: i64 }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RepoTokenExpiry(UnixSeconds); + +impl RepoTokenExpiry { + pub fn mint(exp: i64, now: UnixSeconds) -> Result { + let requested = UnixSeconds::new(exp); + match requested.get().saturating_sub(now.get()) { + lifespan if lifespan <= 0 => Err(MintExpiryError::NotAfterMint), + lifespan if lifespan > REPO_TOKEN_MAX_LIFETIME_SECS => { + Err(MintExpiryError::TooLong { lifespan }) + } + _ => Ok(Self(requested)), + } + } + + pub fn get(self) -> UnixSeconds { + self.0 + } +} + +pub struct RepoTokenRequest<'a> { + pub repo: &'a RepoDid, + pub audience: &'a DidService, + pub method: &'a Nsid, + pub nonce: &'a JwtNonce, + pub controller: &'a AccountDid, + pub now: UnixSeconds, + pub expires_at: RepoTokenExpiry, +} + +pub fn mint_repo_token(signer: &dyn Signer, request: RepoTokenRequest<'_>) -> ServiceJwt { + let claims = RepoTokenClaims { + iss: request.repo, + aud: request.audience, + exp: request.expires_at.get(), + iat: request.now, + jti: request.nonce.as_str(), + lxm: request.method, + act: ControllerClaim { + sub: request.controller, + }, + }; + sign_jwt(signer, &claims) +} + +pub fn parse(token: &ServiceJwt) -> Result { + let parsed = parse_jwt(token.as_str())?; + let typ = parsed.header().typ.as_str(); + if !typ.eq_ignore_ascii_case("JWT") { + return Err(JwtError::UnexpectedType { + typ: typ.to_string(), + }); + } + Ok(parsed) +} + +pub fn issuer(parsed: &ParsedJwt) -> Result { + let iss = parsed.claims().iss.as_str(); + AccountDid::new(iss).map_err(|_| JwtError::MalformedIssuer { + value: iss.to_string(), + }) +} + +fn repo_audience(parsed: &ParsedJwt) -> Result { + let aud = parsed.claims().aud.audience().as_str().to_owned(); + RepoDid::new(&aud).map_err(|_| JwtError::MalformedAudience { value: aud }) +} + +pub fn token_audience(token: &ServiceJwt) -> Result { + repo_audience(&parse(token)?) +} + +fn verifying_key(key: &CryptoKey<'_>) -> Result { + match key.codec { + KeyCodec::Secp256k1 => VerifyKey::from_k256_bytes(&key.bytes) + .map_err(|e| JwtError::MalformedKey(e.to_string())), + KeyCodec::P256 => VerifyKey::from_p256_bytes(&key.bytes) + .map_err(|e| JwtError::MalformedKey(e.to_string())), + KeyCodec::Ed25519 | KeyCodec::Unknown(_) => Err(JwtError::UnsupportedKeyCodec), + } +} + +pub trait TokenAudience { + fn as_str(&self) -> &str; + fn canonicalizes(&self, claimed: &str) -> bool; +} + +macro_rules! token_audience { + ($($id:ty),+ $(,)?) => { + $(impl TokenAudience for $id { + fn as_str(&self) -> &str { + <$id>::as_str(self) + } + + fn canonicalizes(&self, claimed: &str) -> bool { + <$id>::new(claimed).is_ok_and(|aud| <$id>::as_str(&aud) == <$id>::as_str(self)) + } + })+ + }; +} + +token_audience!(KnotId, ServiceDid, RepoDid); + +pub fn check_claims( + parsed: &ParsedJwt, + audience: &impl TokenAudience, + method: &Nsid, + now_unix: UnixSeconds, +) -> Result<(), JwtError> { + let claims = parsed.claims(); + let exp = UnixSeconds::new(claims.exp); + let iat = UnixSeconds::new(claims.iat); + + if !audience.canonicalizes(claims.aud.audience().as_str()) { + return Err(JwtError::AudienceMismatch { + expected: audience.as_str().to_string(), + actual: claims.aud.as_str().to_string(), + }); + } + + if exp.saturating_add_secs(CLOCK_SKEW_SECS) < now_unix { + return Err(JwtError::Expired { exp, now: now_unix }); + } + + if iat.saturating_sub_secs(CLOCK_SKEW_SECS) > now_unix { + return Err(JwtError::IssuedInFuture { iat, now: now_unix }); + } + + if exp < iat { + return Err(JwtError::ExpiresBeforeIssued { exp, iat }); + } + + if exp.get().saturating_sub(iat.get()) > MAX_TOKEN_LIFETIME_SECS { + return Err(JwtError::LifetimeTooLong { + exp, + iat, + max: MAX_TOKEN_LIFETIME_SECS, + }); + } + + let bound = claims.lxm.as_ref().map(|lxm| lxm.as_str()); + if bound != Some(method.as_str()) { + return Err(JwtError::MethodMismatch { + expected: method.as_str().to_string(), + actual: bound.map(str::to_string), + }); + } + + Ok(()) +} + +pub(crate) fn nonce(parsed: &ParsedJwt) -> Result { + let jti: &str = parsed + .claims() + .jti + .as_ref() + .map(|jti| jti.as_ref()) + .ok_or(JwtError::MissingNonce)?; + JwtNonce::new(jti) +} + +pub fn verify_signature(parsed: &ParsedJwt, issuer_key: &CryptoKey<'_>) -> Result<(), JwtError> { + let key = verifying_key(issuer_key)?; + knot_types::service_auth::verify_signature(parsed, &key).map_err(|error| match error { + ServiceAuthError::InvalidSignature => JwtError::InvalidSignature, + other => JwtError::Parse(other), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::mint as mint_claims; + use crate::test_support::*; + use std::borrow::Cow; + + fn claims(iss: &str, aud: &str, exp: UnixSeconds, lxm: &str) -> serde_json::Value { + serde_json::json!({ + "iss": iss, + "aud": aud, + "exp": exp.get(), + "iat": exp.saturating_sub_secs(60).get(), + "lxm": lxm, + }) + } + + #[test] + fn a_well_formed_token_authenticates_its_issuer() { + let signing = signer(1); + let public = k256_public(&signing); + let token = mint_claims( + &signing, + &claims(SQUID, KNOT, UnixSeconds::new(1_000), METHOD), + ); + let parsed = parse(&token).unwrap(); + check_claims( + &parsed, + &knot_did(KNOT), + &member_method(), + UnixSeconds::new(900), + ) + .unwrap(); + verify_signature(&parsed, &public).unwrap(); + assert_eq!(issuer(&parsed).unwrap(), AccountDid::new(SQUID).unwrap()); + } + + #[test] + fn a_repository_audience_takes_the_same_service_reference_as_the_knot() { + let signing = signer(1); + let repo = RepoDid::new("did:plc:3fwecdnvtcscjnrx2p4n7alz").unwrap(); + let checked = |aud: &str| { + let token = mint_claims( + &signing, + &claims(SQUID, aud, UnixSeconds::new(1_000), METHOD), + ); + check_claims( + &parse(&token).unwrap(), + &repo, + &member_method(), + UnixSeconds::new(900), + ) + }; + + checked(repo.as_str()).unwrap(); + checked(&format!("{}#tangled_knot", repo.as_str())).unwrap(); + assert!( + matches!( + checked("did:plc:qxx2hmw5gv6ghjvcxl63q5d3#tangled_knot"), + Err(JwtError::AudienceMismatch { .. }) + ), + "a fragment let another DID pass as this repository" + ); + } + + struct ClaimsCase { + name: &'static str, + claims: fn() -> serde_json::Value, + now: i64, + expect: fn(&Result<(), JwtError>) -> bool, + } + + const CLAIMS_CASES: &[ClaimsCase] = &[ + ClaimsCase { + name: "expired past the skew window", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_000, "iat": 940, "lxm": METHOD }), + now: 1_100, + expect: |r| { + matches!(r, Err(JwtError::Expired { exp, now }) + if *exp == UnixSeconds::new(1_000) && *now == UnixSeconds::new(1_100)) + }, + }, + ClaimsCase { + name: "within the skew window past exp", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_000, "iat": 940, "lxm": METHOD }), + now: 1_030, + expect: |r| r.is_ok(), + }, + ClaimsCase { + name: "addressed to another knot", + claims: || serde_json::json!({ "iss": SQUID, "aud": "did:web:oyster.cafe", "exp": 1_000, "iat": 940, "lxm": METHOD }), + now: 900, + expect: |r| matches!(r, Err(JwtError::AudienceMismatch { .. })), + }, + ClaimsCase { + name: "bound to another method", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_000, "iat": 940, "lxm": "org.tangled.repo.deleteRepo" }), + now: 900, + expect: |r| matches!(r, Err(JwtError::MethodMismatch { .. })), + }, + ClaimsCase { + name: "has no method binding", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_000, "iat": 940 }), + now: 900, + expect: |r| matches!(r, Err(JwtError::MethodMismatch { actual: None, .. })), + }, + ClaimsCase { + name: "issued in the future", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 2_001, "iat": 2_000, "lxm": METHOD }), + now: 900, + expect: |r| { + matches!(r, Err(JwtError::IssuedInFuture { iat, now }) + if *iat == UnixSeconds::new(2_000) && *now == UnixSeconds::new(900)) + }, + }, + ClaimsCase { + name: "lifetime exceeds the limit", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_400, "iat": 1_000, "lxm": METHOD }), + now: 1_000, + expect: |r| { + matches!(r, Err(JwtError::LifetimeTooLong { exp, iat, max }) + if *exp == UnixSeconds::new(1_400) && *iat == UnixSeconds::new(1_000) && *max == 300) + }, + }, + ClaimsCase { + name: "expires before it was issued", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_040, "iat": 1_050, "lxm": METHOD }), + now: 1_000, + expect: |r| { + matches!(r, Err(JwtError::ExpiresBeforeIssued { exp, iat }) + if *exp == UnixSeconds::new(1_040) && *iat == UnixSeconds::new(1_050)) + }, + }, + ClaimsCase { + name: "audience in a different case still matches", + claims: || serde_json::json!({ "iss": SQUID, "aud": "did:web:NEL.PET", "exp": 1_000, "iat": 940, "lxm": METHOD }), + now: 900, + expect: |r| r.is_ok(), + }, + ClaimsCase { + name: "a service reference matches on the DID alone", + claims: || serde_json::json!({ "iss": SQUID, "aud": "did:web:nel.pet#tangled_knot", "exp": 1_000, "iat": 940, "lxm": METHOD }), + now: 900, + expect: |r| r.is_ok(), + }, + ClaimsCase { + name: "a fragment on another knot's DID", + claims: || serde_json::json!({ "iss": SQUID, "aud": "did:web:oyster.cafe#tangled_knot", "exp": 1_000, "iat": 940, "lxm": METHOD }), + now: 900, + expect: |r| matches!(r, Err(JwtError::AudienceMismatch { .. })), + }, + ]; + + #[test] + fn check_claims_enforces_the_audience_window_and_method_binding() { + let signing = signer(1); + CLAIMS_CASES.iter().for_each(|case| { + let token = mint_claims(&signing, &(case.claims)()); + let parsed = parse(&token).unwrap(); + let result = check_claims( + &parsed, + &knot_did(KNOT), + &member_method(), + UnixSeconds::new(case.now), + ); + assert!( + (case.expect)(&result), + "case {:?} got {result:?}", + case.name + ); + }); + } + + #[test] + fn nonce_extraction_enforces_presence_and_limit() { + let signing = signer(1); + let with = |jti: Option| { + let mut body = claims(SQUID, KNOT, UnixSeconds::new(1_000), METHOD); + if let Some(jti) = jti { + body["jti"] = serde_json::json!(jti); + } + parse(&mint_claims(&signing, &body)).unwrap() + }; + + assert!(matches!(nonce(&with(None)), Err(JwtError::MissingNonce))); + assert_eq!( + nonce(&with(Some("nonce-1".to_string()))).unwrap().as_str(), + "nonce-1" + ); + assert!(matches!( + nonce(&with(Some("n".repeat(MAX_NONCE_BYTES + 1)))), + Err(JwtError::OversizedNonce { len }) if len == MAX_NONCE_BYTES + 1 + )); + assert!(nonce(&with(Some("n".repeat(MAX_NONCE_BYTES)))).is_ok()); + } + + #[test] + fn a_minted_token_round_trips_through_the_verify_half_and_honors_its_lifetime() { + let key = runtime_signer(8); + let knot_issuer = knot_did(KNOT); + let audience = ServiceDid::new("did:web:pds.oyster.cafe").unwrap(); + let bound = Nsid::new_owned("com.atproto.repo.putRecord").unwrap(); + let token = super::mint( + &key, + &knot_issuer, + &audience, + &bound, + JwtNonce::new("nonce-minted").unwrap(), + UnixSeconds::new(1_000), + ); + let parsed = parse(&token).unwrap(); + + assert_eq!(parsed.claims().iat, 1_000); + assert_eq!(parsed.claims().exp, 1_000 + SERVICE_TOKEN_LIFETIME_SECS); + assert_eq!( + parsed.claims().aud.as_str(), + audience.as_str(), + "the reference PDS compares its configured DID byte for byte, so a fragment would 401" + ); + check_claims(&parsed, &audience, &bound, UnixSeconds::new(1_005)).unwrap(); + assert_eq!(nonce(&parsed).unwrap().as_str(), "nonce-minted"); + assert_eq!(issuer(&parsed).unwrap(), AccountDid::new(KNOT).unwrap()); + + let public = CryptoKey { + codec: KeyCodec::Secp256k1, + bytes: Cow::Owned(knot_runtime::Signer::public_key(&key).as_bytes().to_vec()), + }; + verify_signature(&parsed, &public).unwrap(); + + let stranger = k256_public(&signer(4)); + assert!(matches!( + verify_signature(&parsed, &stranger).unwrap_err(), + JwtError::InvalidSignature + )); + } + + #[test] + fn a_jwt_nonce_preserves_its_string_and_compares_by_value() { + let nonce = JwtNonce::new("nonce-value").unwrap(); + assert_eq!(nonce.as_str(), "nonce-value"); + assert_eq!(nonce, JwtNonce::new("nonce-value".to_string()).unwrap()); + assert_ne!(nonce, JwtNonce::new("other").unwrap()); + } + + #[test] + fn repo_token_expiry_should_be_bounded_to_given_mint() { + let now = UnixSeconds::new(1_000); + assert_eq!( + RepoTokenExpiry::mint(1_060, now).unwrap().get(), + UnixSeconds::new(1_060) + ); + assert_eq!( + RepoTokenExpiry::mint(1_001, now).unwrap().get(), + UnixSeconds::new(1_001) + ); + assert!(matches!( + RepoTokenExpiry::mint(1_000, now), + Err(MintExpiryError::NotAfterMint) + )); + assert!(matches!( + RepoTokenExpiry::mint(999, now), + Err(MintExpiryError::NotAfterMint) + )); + assert!(matches!( + RepoTokenExpiry::mint(1_061, now), + Err(MintExpiryError::TooLong { lifespan: 61 }) + )); + } + + #[test] + fn entropy_ought_to_mint_hex_nonce_without_revalidating() { + let mut bytes = [0u8; 16]; + bytes[0] = 0xde; + bytes[15] = 0xff; + assert_eq!( + JwtNonce::from_entropy(bytes).as_str(), + "de0000000000000000000000000000ff" + ); + } + + #[test] + fn an_ed25519_issuer_key_is_unsupported() { + let signing = signer(1); + let token = mint_claims( + &signing, + &claims(SQUID, KNOT, UnixSeconds::new(1_000), METHOD), + ); + let parsed = parse(&token).unwrap(); + let ed = CryptoKey { + codec: KeyCodec::Ed25519, + bytes: Cow::Owned(vec![0u8; 32]), + }; + let error = verify_signature(&parsed, &ed).unwrap_err(); + assert!(matches!(error, JwtError::UnsupportedKeyCodec)); + } +} diff --git a/knot2/crates/knot-atproto/src/lib.rs b/knot2/crates/knot-atproto/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..b9518f436cd41834a4a4990ba74a427cc8c2ae93 --- /dev/null +++ b/knot2/crates/knot-atproto/src/lib.rs @@ -0,0 +1,2609 @@ +mod auth; +mod identity; +mod jwt; +mod plc; +mod pointer; +mod pubkeys; +mod resolve; +#[cfg(test)] +mod test_support; + +pub use auth::{PointerAuth, PointerAuthorizer, ServiceAuth}; +pub use identity::{ + ATPROTO_METHOD, IdentityError, MintNonce, PreparedRepoDid, knot_did_document, prepare_repo_did, +}; +pub use jwt::{ + JwtError, JwtNonce, MintExpiryError, RepoTokenExpiry, RepoTokenRequest, ServiceJwt, + mint_repo_token, mint_service_jwt, token_audience, +}; +pub use plc::{PDS_SERVICE_TYPE, RepoTarget, update_operation}; +pub use pointer::PointerReceipt; +pub use pubkeys::{KeyParseError, parse_authorized_key}; +pub use resolve::{Identity, PdsEndpoint, PlcDirectory, ResolveError}; + +#[doc(hidden)] +pub mod fuzz { + pub fn pubkey(data: &[u8]) { + let _ = crate::pubkeys::offered_page(data, 100); + let _ = crate::parse_authorized_key(&String::from_utf8_lossy(data)); + } + + pub fn did_document(data: &[u8]) { + let did = knot_types::AccountDid::new("did:plc:nel").expect("Constant test DID is valid"); + let _ = crate::resolve::identity_from_document(&did, data); + let repo_did = knot_types::RepoDid::new("did:plc:nel").expect("Constant test DID is valid"); + let pds = knot_types::KnotServiceUrl::new("https://knot.oyster.cafe") + .expect("Constant test URL is valid"); + let _ = crate::resolve::document_publishes_account_shape( + &repo_did, + data, + &knot_runtime::PublicKeyBytes::from_bytes(data.to_vec()), + &pds, + ); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordPresence { + Present, + Absent, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReplayGuard { + SingleUse, + ReusableUntilExpiry, +} + +use std::sync::Arc; +use std::time::Duration; + +use futures::stream::{self, TryStreamExt}; +use http::StatusCode; +use knot_cache::{ + Admitted, AsyncCache, EntryCount, Expiring, Filled, GroupQuota, MokaFuture, Quotas, Rejected, + TotalQuota, Weight, +}; +use knot_runtime::{ + Clock, DnsTxtResolver, HttpRequest, HttpTransport, NetworkError, PublicKeyBytes, SystemDns, +}; +use knot_types::{ + AccountDid, Collection, Handle, HttpStatus, KnotId, Nsid, OfferedKey, RepoDid, RepoRkey, Rkey, + ServiceDid, UnixMicros, UnixSeconds, +}; +use pubkeys::Cursor; +use serde::{Deserialize, Serialize}; +use url::Url; + +const DEFAULT_TTL: Duration = Duration::from_secs(300); +const NEGATIVE_TTL: Duration = Duration::from_secs(30); +const STALE_TTL: Duration = Duration::from_secs(30); +const PUBKEY_PAGE_LIMIT: u16 = 100; +fn repo_collection() -> Nsid { + Nsid::new_static("org.tangled.repo.declaration").expect("Literal NSID parses") +} +fn legacy_repo_collection() -> Nsid { + Nsid::new_static("sh.tangled.repo").expect("Literal NSID parses") +} +const PUBKEY_MAX_PAGES: usize = 8; +const PUBKEY_TTL: Duration = Duration::from_secs(30); +const MAX_PUBKEY_CACHE_BYTES: u64 = 1 << 20; +const PUBKEY_CACHE_ENTRY_OVERHEAD: u64 = 128; +const PUBKEY_CACHE_KEY_OVERHEAD: u64 = 192; +const MAX_IDENTITY_CACHE: usize = 4096; +const MAX_SEEN_JTI: usize = 8192; + +const MAX_JTI_PER_ISSUER: usize = MAX_SEEN_JTI / 16; + +#[derive(Debug, thiserror::Error)] +pub enum AtprotoError { + #[error(transparent)] + Resolve(#[from] ResolveError), + #[error(transparent)] + Jwt(#[from] JwtError), + #[error("Network failure: {0}")] + Network(#[from] NetworkError), + #[error("listRecords for {did} returned HTTP {status}")] + ListRecords { did: AccountDid, status: HttpStatus }, + #[error("listRecords response wasn't valid JSON: {0}")] + MalformedRecords(String), + #[error("PDS endpoint {pds:?} isn't a usable base URL")] + BadPdsEndpoint { pds: String }, + #[error("Service token {jti:?} has already been presented")] + Replay { jti: JwtNonce }, + #[error("Replay-protection store is full and can't accept another nonce")] + ReplayStoreSaturated, + #[error("Issuer {issuer} has too many live replay nonces")] + ReplayShareExhausted { issuer: AccountDid }, + #[error(transparent)] + Identity(#[from] IdentityError), + #[error("PLC submission for {did} returned HTTP {status}")] + PlcSubmit { did: RepoDid, status: HttpStatus }, + #[error("PLC log fetch for {did} returned HTTP {status}")] + PlcFetch { did: RepoDid, status: HttpStatus }, + #[error("putRecord for {subject} returned HTTP {status}")] + PutRecord { + subject: AccountDid, + status: HttpStatus, + }, + #[error("getRecord for {owner} returned HTTP {status}")] + GetRecord { + owner: AccountDid, + status: HttpStatus, + }, + #[error("Pointer record couldn't be encoded: {0}")] + PointerEncode(String), + #[error("putRecord response isn't a valid receipt: {0}")] + MalformedReceipt(String), + #[error("notifyEvent to {spindle} returned HTTP {status}")] + NotifyEvent { + spindle: ServiceDid, + status: HttpStatus, + }, +} + +impl AtprotoError { + pub fn is_transient(&self) -> bool { + match self { + AtprotoError::Network(_) + | AtprotoError::ReplayStoreSaturated + | AtprotoError::ReplayShareExhausted { .. } => true, + AtprotoError::Resolve(error) => error.is_transient(), + AtprotoError::ListRecords { status, .. } + | AtprotoError::PlcSubmit { status, .. } + | AtprotoError::PlcFetch { status, .. } + | AtprotoError::PutRecord { status, .. } + | AtprotoError::GetRecord { status, .. } + | AtprotoError::NotifyEvent { status, .. } => status.is_transient(), + _ => false, + } + } + + pub fn is_gone(&self) -> bool { + match self { + AtprotoError::Resolve(error) => error.is_gone(), + AtprotoError::Jwt(_) + | AtprotoError::Network(_) + | AtprotoError::ListRecords { .. } + | AtprotoError::MalformedRecords(_) + | AtprotoError::BadPdsEndpoint { .. } + | AtprotoError::Replay { .. } + | AtprotoError::ReplayStoreSaturated + | AtprotoError::ReplayShareExhausted { .. } + | AtprotoError::Identity(_) + | AtprotoError::PlcSubmit { .. } + | AtprotoError::PlcFetch { .. } + | AtprotoError::PutRecord { .. } + | AtprotoError::GetRecord { .. } + | AtprotoError::PointerEncode(_) + | AtprotoError::MalformedReceipt(_) + | AtprotoError::NotifyEvent { .. } => false, + } + } +} + +/// What a spindle said back to `notifyEvent`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SpindleAnswer { + /// The spindle-local ids of the pipelines it scheduled, and any text it has for the pusher. + Scheduled { + pipelines: Vec, + message: Option, + }, + Nothing, + Malformed(String), +} + +const MAX_PIPELINE_ID_LEN: usize = 256; + +fn is_pipeline_id(id: &str) -> bool { + !id.is_empty() && id.len() <= MAX_PIPELINE_ID_LEN && !id.chars().any(char::is_control) +} + +#[derive(Clone)] +enum Resolution { + Found(Identity), + Failed(ResolveError), + Transient(ResolveError), +} + +#[derive(Clone)] +enum HandleResolution { + Bound(AccountDid), + Unbound(ResolveError), + Transient(ResolveError), +} + +#[derive(Clone)] +struct Cached { + resolution: R, + expires_at: UnixMicros, +} + +#[derive(Clone)] +pub enum ClaimedKeys { + Published(Vec), + Unread(Arc), +} + +fn claimed_weight(cached: &Cached) -> Weight { + Weight::new(match &cached.resolution { + ClaimedKeys::Published(keys) => keys + .iter() + .map(|key| key.as_bytes().len() as u64 + PUBKEY_CACHE_KEY_OVERHEAD) + .sum::() + .saturating_add(PUBKEY_CACHE_ENTRY_OVERHEAD), + ClaimedKeys::Unread(_) => PUBKEY_CACHE_ENTRY_OVERHEAD, + }) +} + +pub struct Atproto { + http: H, + clock: C, + knot_did: KnotId, + plc_directory: PlcDirectory, + dns: Arc, + identities: MokaFuture>, + handles: MokaFuture>, + claimed: MokaFuture>, + seen_jti: Expiring<(AccountDid, JwtNonce), AccountDid, ()>, +} + +impl Atproto { + pub fn new(http: H, clock: C, knot_did: KnotId, plc_directory: PlcDirectory) -> Self { + Self { + http, + clock, + knot_did, + plc_directory, + dns: Arc::new(SystemDns::new()), + identities: MokaFuture::by_count(EntryCount::new(MAX_IDENTITY_CACHE as u64)), + handles: MokaFuture::by_count(EntryCount::new(MAX_IDENTITY_CACHE as u64)), + claimed: MokaFuture::by_weight(Weight::new(MAX_PUBKEY_CACHE_BYTES), claimed_weight), + seen_jti: Expiring::new(Quotas { + per_group: GroupQuota::new(MAX_JTI_PER_ISSUER), + total: TotalQuota::new(MAX_SEEN_JTI), + }), + } + } + + pub fn with_dns(mut self, dns: Arc) -> Self { + self.dns = dns; + self + } + + pub fn now(&self) -> UnixMicros { + self.clock.now_unix_micros() + } + + pub async fn resolve_identity(&self, did: &AccountDid) -> Result { + Ok(self.identity(did).await?.value) + } + + pub async fn resolve_knot_endpoint(&self, did: &RepoDid) -> Result { + let url = resolve::document_url(&AccountDid::from(did.clone()), &self.plc_directory)?; + resolve::guard_fetch_url(&url)?; + let response = self.http.execute(HttpRequest::get(url)).await?; + if !response.status.is_success() { + return Err(AtprotoError::Resolve(ResolveError::Status { + status: HttpStatus::from(response.status), + })); + } + resolve::knot_endpoint(did, &response.body).map_err(AtprotoError::Resolve) + } + + async fn identity(&self, did: &AccountDid) -> Result, ResolveError> { + let now = self.clock.now_unix_micros(); + let filled = self + .identities + .get_or_fill_if( + did.clone(), + |cached: &Cached| cached.expires_at <= now, + self.fill_identity(did, now), + ) + .await; + let fresh = filled.fresh; + match filled.value.resolution { + Resolution::Found(value) => Ok(Filled { value, fresh }), + Resolution::Transient(error) => Err(error), + Resolution::Failed(error) if fresh || error.is_gone() => Err(error), + Resolution::Failed(_) => Err(ResolveError::RecentlyFailed { did: did.clone() }), + } + } + + pub async fn resolve_handle_to_did(&self, handle: &Handle) -> Result { + let now = self.clock.now_unix_micros(); + let filled = self + .handles + .get_or_fill_if( + handle.clone(), + |cached: &Cached| cached.expires_at <= now, + self.fill_handle(handle, now), + ) + .await; + let fresh = filled.fresh; + match filled.value.resolution { + HandleResolution::Bound(did) => Ok(did), + HandleResolution::Transient(error) => Err(error.into()), + HandleResolution::Unbound(error) if fresh => Err(error.into()), + HandleResolution::Unbound(_) => Err(ResolveError::HandleRecentlyFailed { + handle: handle.clone(), + } + .into()), + } + } + + async fn stale_handle_did(&self, handle: &Handle) -> Option { + // `fill_handle` calls this from inside its own + // `or_insert_with_if` init for this key. + // Moka will keep the prior entry readable until init returns, + // so this `get` will yield the last resolved DID to re-serve, + // when there's a temporary outage. + match self.handles.get(handle).await?.resolution { + HandleResolution::Bound(did) => Some(did), + _ => None, + } + } + + async fn fill_handle(&self, handle: &Handle, now: UnixMicros) -> Cached { + match self.verify_handle(handle).await { + Ok(did) => Cached { + resolution: HandleResolution::Bound(did), + expires_at: expires(now, DEFAULT_TTL), + }, + Err(error) if error.is_transient() => match self.stale_handle_did(handle).await { + Some(did) => Cached { + resolution: HandleResolution::Bound(did), + expires_at: expires(now, STALE_TTL), + }, + None => Cached { + resolution: HandleResolution::Transient(error), + expires_at: now, + }, + }, + Err(error) => Cached { + resolution: HandleResolution::Unbound(error), + expires_at: expires(now, NEGATIVE_TTL), + }, + } + } + + async fn verify_handle(&self, handle: &Handle) -> Result { + let candidate = match self.dns_txt_did(handle).await { + Ok(Some(did)) => did, + Ok(None) => self.wellknown_did(handle).await?, + Err(dns_error) if dns_error.is_transient() => match self.wellknown_did(handle).await { + Ok(did) => did, + Err(_) => return Err(dns_error), + }, + Err(dns_error) => return Err(dns_error), + }; + let identity = self.identity(&candidate).await?.value; + if identity.claims_handle(handle) { + Ok(candidate) + } else { + Err(ResolveError::HandleMismatch { + handle: handle.clone(), + resolved: candidate, + claimed: identity.primary_handle().cloned(), + }) + } + } + + async fn dns_txt_did(&self, handle: &Handle) -> Result, ResolveError> { + let records = self + .dns + .lookup_txt(format!("_atproto.{}", handle.as_str())) + .await?; + let dids = records + .iter() + .filter_map(|record| record.trim().strip_prefix("did=").map(str::trim)) + .map(|value| { + AccountDid::new(value).map_err(|_| ResolveError::HandleForwardMalformed { + handle: handle.clone(), + value: value.to_string(), + }) + }) + .collect::, ResolveError>>()?; + let distinct = dids + .iter() + .map(AccountDid::as_str) + .collect::>() + .len(); + match distinct { + 0 => Ok(None), + 1 => Ok(dids.into_iter().next()), + _ => Err(ResolveError::HandleAmbiguous { + handle: handle.clone(), + }), + } + } + + async fn wellknown_did(&self, handle: &Handle) -> Result { + let url = Url::parse(&format!( + "https://{}/.well-known/atproto-did", + handle.as_str() + )) + .map_err(|_| ResolveError::HandleUnresolvable { + handle: handle.clone(), + })?; + resolve::guard_fetch_url(&url)?; + let response = self + .http + .execute(HttpRequest::get(url)) + .await + .map_err(ResolveError::from)?; + if !response.status.is_success() { + let status = HttpStatus::from(response.status); + if status.is_transient() { + return Err(ResolveError::Status { status }); + } + return Err(ResolveError::HandleUnresolvable { + handle: handle.clone(), + }); + } + let value = std::str::from_utf8(&response.body) + .map_err(|_| ResolveError::HandleUnresolvable { + handle: handle.clone(), + })? + .trim(); + AccountDid::new(value).map_err(|_| ResolveError::HandleForwardMalformed { + handle: handle.clone(), + value: value.to_string(), + }) + } + + async fn fill_identity(&self, did: &AccountDid, now: UnixMicros) -> Cached { + match self.fetch_identity(did).await { + Ok(identity) => Cached { + resolution: Resolution::Found(identity), + expires_at: expires(now, DEFAULT_TTL), + }, + Err(error) if warrants_negative_cache(&error) => Cached { + resolution: Resolution::Failed(error), + expires_at: expires(now, NEGATIVE_TTL), + }, + Err(error) => Cached { + resolution: Resolution::Transient(error), + expires_at: now, + }, + } + } + + async fn fetch_identity(&self, did: &AccountDid) -> Result { + let url = resolve::document_url(did, &self.plc_directory)?; + resolve::guard_fetch_url(&url)?; + let response = self + .http + .execute(HttpRequest::get(url)) + .await + .map_err(ResolveError::from)?; + if !response.status.is_success() { + let status = HttpStatus::from(response.status); + return match status.get() { + 404 | 410 => Err(ResolveError::Gone { + did: did.clone(), + status, + }), + _ => Err(ResolveError::Status { status }), + }; + } + resolve::identity_from_document(did, &response.body) + } + + pub fn document_url(&self, did: &AccountDid) -> Result { + resolve::document_url(did, &self.plc_directory) + } + + pub async fn resolve_pubkeys(&self, did: &AccountDid) -> Result, AtprotoError> { + let identity = self.resolve_identity(did).await?; + self.pubkeys_at(&identity, did).await + } + + pub async fn claimed_pubkeys(&self, did: &AccountDid) -> ClaimedKeys { + let now = self.clock.now_unix_micros(); + let filled = self + .claimed + .get_or_fill_if( + did.clone(), + |cached: &Cached| cached.expires_at <= now, + self.fill_claimed(did, now), + ) + .await; + filled.value.resolution + } + + // what `claimed_pubkeys` has cached, without fetching anything + pub async fn cached_claimed_pubkeys(&self, did: &AccountDid) -> Option { + let now = self.clock.now_unix_micros(); + let cached = self.claimed.get(did).await?; + (cached.expires_at > now).then_some(cached.resolution) + } + + // what `resolve_handle_to_did` has cached, without fetching anything + pub async fn cached_handle_did(&self, handle: &Handle) -> Option { + let now = self.clock.now_unix_micros(); + let cached = self.handles.get(handle).await?; + match cached.resolution { + HandleResolution::Bound(did) if cached.expires_at > now => Some(did), + _ => None, + } + } + + async fn fill_claimed(&self, did: &AccountDid, now: UnixMicros) -> Cached { + match self.resolve_pubkeys(did).await { + Ok(keys) => Cached { + resolution: ClaimedKeys::Published(keys), + expires_at: expires(now, PUBKEY_TTL), + }, + Err(error) => Cached { + expires_at: match error.is_transient() { + true => now, + false => expires(now, NEGATIVE_TTL), + }, + resolution: ClaimedKeys::Unread(Arc::new(error)), + }, + } + } + + pub async fn pubkeys_at( + &self, + identity: &Identity, + did: &AccountDid, + ) -> Result, AtprotoError> { + let http = &self.http; + let pds = &identity.pds; + let pages = stream::try_unfold(Page::First(PUBKEY_MAX_PAGES), move |state| async move { + let (cursor, budget) = match state { + Page::Done | Page::First(0) | Page::Next(_, 0) => { + return Ok::<_, AtprotoError>(None); + } + Page::First(budget) => (None, budget), + Page::Next(cursor, budget) => (Some(cursor), budget), + }; + let url = list_records_url(pds, did, cursor.as_ref())?; + resolve::guard_fetch_url(&url)?; + let response = http.execute(HttpRequest::get(url)).await?; + if !response.status.is_success() { + return Err(AtprotoError::ListRecords { + did: did.clone(), + status: HttpStatus::from(response.status), + }); + } + let page = pubkeys::offered_page(&response.body, PUBKEY_PAGE_LIMIT as usize) + .map_err(|error| AtprotoError::MalformedRecords(error.to_string()))?; + let next = match page.cursor { + Some(cursor) => Page::Next(cursor, budget - 1), + None => Page::Done, + }; + Ok(Some((page.keys, next))) + }); + pages + .try_fold(Vec::new(), |mut acc, keys| async move { + acc.extend(keys); + Ok(acc) + }) + .await + } + + pub async fn repo_record_present( + &self, + owner: &AccountDid, + rkey: &RepoRkey, + ) -> Result { + let cached = self.identity(owner).await?; + let read = self.read_declared(&cached.value, owner, rkey).await?; + match read { + RecordPresence::Absent if !cached.fresh => { + let current = self.fetch_identity(owner).await?; + match current.pds.url() == cached.value.pds.url() { + true => Ok(RecordPresence::Absent), + false => self.read_declared(¤t, owner, rkey).await, + } + } + settled => Ok(settled), + } + } + async fn read_declared( + &self, + identity: &Identity, + owner: &AccountDid, + rkey: &RepoRkey, + ) -> Result { + match self + .read_record(identity, owner, &repo_collection(), rkey) + .await? + { + RecordPresence::Present => Ok(RecordPresence::Present), + RecordPresence::Absent => { + self.read_record(identity, owner, &legacy_repo_collection(), rkey) + .await + } + } + } + + async fn read_record( + &self, + identity: &Identity, + owner: &AccountDid, + collection: &Nsid, + rkey: &RepoRkey, + ) -> Result { + let url = get_record_url(&identity.pds, owner, collection, rkey.as_str())?; + resolve::guard_fetch_url(&url)?; + let response = self.http.execute(HttpRequest::get(url)).await?; + match response.status { + status if status.is_success() => Ok(RecordPresence::Present), + StatusCode::NOT_FOUND => Ok(RecordPresence::Absent), + StatusCode::BAD_REQUEST if record_not_found(response.body.as_ref()) => { + Ok(RecordPresence::Absent) + } + status => Err(AtprotoError::GetRecord { + owner: owner.clone(), + status: HttpStatus::from(status), + }), + } + } + + pub async fn verify_service_jwt( + &self, + token: &ServiceJwt, + method: &Nsid, + ) -> Result { + self.verify_service_jwt_guarded(token, method, ReplayGuard::SingleUse) + .await + } + + pub async fn verify_service_jwt_guarded( + &self, + token: &ServiceJwt, + method: &Nsid, + replay: ReplayGuard, + ) -> Result { + self.verify_addressed(token, method, &self.knot_did, replay) + .await + } + + pub async fn verify_service_jwt_for_repo( + &self, + token: &ServiceJwt, + method: &Nsid, + repo: &RepoDid, + ) -> Result { + self.verify_addressed(token, method, repo, ReplayGuard::SingleUse) + .await + } + + async fn verify_addressed( + &self, + token: &ServiceJwt, + method: &Nsid, + audience: &impl jwt::TokenAudience, + replay: ReplayGuard, + ) -> Result { + let parsed = jwt::parse(token)?; + let issuer = jwt::issuer(&parsed)?; + let now_micros = self.clock.now_unix_micros(); + let now = UnixSeconds::new((now_micros.get() / 1_000_000) as i64); + + jwt::check_claims(&parsed, audience, method, now)?; + let jti = jwt::nonce(&parsed)?; + + let identity = self.resolve_identity(&issuer).await?; + jwt::verify_signature(&parsed, &identity.signing_key)?; + + match replay { + ReplayGuard::SingleUse => { + let exp = UnixSeconds::new(parsed.claims().exp); + self.record_jti(&issuer, jti, exp, now_micros)?; + } + ReplayGuard::ReusableUntilExpiry => drop(jti), + } + Ok(issuer) + } + + pub async fn submit_plc_operation( + &self, + did: &RepoDid, + operation: &serde_json::Value, + ) -> Result<(), AtprotoError> { + let url = self.document_url(&AccountDid::from(did.clone()))?; + let body = serde_json::to_vec(operation) + .map_err(|error| AtprotoError::PointerEncode(error.to_string()))?; + resolve::guard_fetch_url(&url)?; + let request = json_post(url, bytes::Bytes::from(body)); + let response = self.http.execute(request).await?; + if response.status.is_success() { + Ok(()) + } else { + Err(AtprotoError::PlcSubmit { + did: did.clone(), + status: HttpStatus::from(response.status), + }) + } + } + + pub async fn last_plc_operation( + &self, + did: &RepoDid, + ) -> Result, AtprotoError> { + let mut url = self.document_url(&AccountDid::from(did.clone()))?; + url.set_path(&format!("{}/log/last", url.path().trim_end_matches('/'))); + resolve::guard_fetch_url(&url)?; + let response = self.http.execute(HttpRequest::get(url)).await?; + let status = HttpStatus::from(response.status); + if status.get() == 404 { + return Ok(None); + } + if !response.status.is_success() { + return Err(AtprotoError::PlcFetch { + did: did.clone(), + status, + }); + } + serde_json::from_slice(&response.body) + .map(Some) + .map_err(|error| AtprotoError::MalformedRecords(error.to_string())) + } + + pub async fn publish_pointer( + &self, + authorizer: &dyn PointerAuthorizer, + subject: &AccountDid, + rkey: &Rkey, + record: &R, + ) -> Result { + let identity = self.resolve_identity(subject).await?; + let method = pointer::put_record_method(); + let url = xrpc_url(&identity.pds, &method)?; + resolve::guard_fetch_url(&url)?; + let audience = pointer::pds_service_did(&identity.pds)?; + let now = UnixSeconds::new((self.clock.now_unix_micros().get() / 1_000_000) as i64); + let body = pointer::put_record_body(subject, rkey, record)?; + let mut request = json_post(url, bytes::Bytes::from(body)); + authorizer.authorize( + &mut request, + &PointerAuth { + issuer: &self.knot_did, + audience: &audience, + lxm: &method, + now_unix: now, + }, + )?; + let response = self.http.execute(request).await?; + if !response.status.is_success() { + return Err(AtprotoError::PutRecord { + subject: subject.clone(), + status: HttpStatus::from(response.status), + }); + } + pointer::receipt_from_response(&response.body) + } + + /// Calls `org.tangled.event.notifyEvent` on `spindle` under this knot's + /// service auth. The spindle has already acted by the time it answers, so + /// an answer we can't read is reported, never failed. + pub async fn notify_spindle( + &self, + authorizer: &dyn PointerAuthorizer, + spindle: &ServiceDid, + body: bytes::Bytes, + ) -> Result { + let method = + Nsid::new_static("org.tangled.event.notifyEvent").expect("Literal NSID parses"); + let url = self.spindle_xrpc_url(spindle, &method).await?; + let now = UnixSeconds::new((self.clock.now_unix_micros().get() / 1_000_000) as i64); + let mut request = json_post(url, body); + authorizer.authorize( + &mut request, + &PointerAuth { + issuer: &self.knot_did, + audience: spindle, + lxm: &method, + now_unix: now, + }, + )?; + let response = self.http.execute(request).await?; + if !response.status.is_success() { + return Err(AtprotoError::NotifyEvent { + spindle: spindle.clone(), + status: HttpStatus::from(response.status), + }); + } + #[derive(Deserialize)] + struct NotifyOutput { + ci: Option, + } + #[derive(Deserialize)] + struct CiOutput { + #[serde(default)] + pipelines: Vec, + message: Option, + } + let ci = match serde_json::from_slice(&response.body) { + Ok(NotifyOutput { ci }) => ci, + Err(error) => return Ok(SpindleAnswer::Malformed(error.to_string())), + }; + Ok(match ci { + None => SpindleAnswer::Nothing, + Some(CiOutput { pipelines, .. }) if pipelines.is_empty() => SpindleAnswer::Nothing, + Some(CiOutput { pipelines, .. }) if !pipelines.iter().all(|id| is_pipeline_id(id)) => { + SpindleAnswer::Malformed(format!("pipeline ids {pipelines:?} aren't usable")) + } + Some(CiOutput { pipelines, message }) => { + SpindleAnswer::Scheduled { pipelines, message } + } + }) + } + + async fn spindle_xrpc_url( + &self, + spindle: &ServiceDid, + method: &Nsid, + ) -> Result { + let did = AccountDid::from(spindle.clone()); + let url = self.document_url(&did)?; + resolve::guard_fetch_url(&url)?; + let response = self + .http + .execute(HttpRequest::get(url)) + .await + .map_err(ResolveError::from)?; + if !response.status.is_success() { + return Err(ResolveError::Status { + status: HttpStatus::from(response.status), + } + .into()); + } + let endpoint = resolve::spindle_endpoint(spindle, &response.body)?; + let base = PdsEndpoint::new(endpoint.clone()).map_err(|_| ResolveError::BadSpindle { + value: endpoint.to_string(), + })?; + let url = xrpc_url(&base, method)?; + resolve::guard_fetch_url(&url)?; + Ok(url) + } + + pub async fn verify_account_document( + &self, + did: &RepoDid, + expected: &PublicKeyBytes, + expected_pds: &knot_types::KnotServiceUrl, + ) -> Result<(), AtprotoError> { + let url = self.document_url(&AccountDid::from(did.clone()))?; + resolve::guard_fetch_url(&url)?; + let response = self + .http + .execute(HttpRequest::get(url)) + .await + .map_err(ResolveError::from)?; + if !response.status.is_success() { + return Err(ResolveError::Status { + status: HttpStatus::from(response.status), + } + .into()); + } + resolve::document_publishes_account_shape(did, &response.body, expected, expected_pds) + .map_err(Into::into) + } + + fn record_jti( + &self, + issuer: &AccountDid, + jti: JwtNonce, + exp: UnixSeconds, + now: UnixMicros, + ) -> Result<(), AtprotoError> { + let horizon = exp.saturating_add_secs(jwt::CLOCK_SKEW_SECS).get().max(0) as u64; + let expires_at = UnixMicros::new(horizon.saturating_mul(1_000_000)); + match self.seen_jti.admit( + (issuer.clone(), jti.clone()), + issuer.clone(), + (), + expires_at, + now, + ) { + Ok(Admitted::Inserted) => Ok(()), + Ok(Admitted::Occupied(())) => Err(AtprotoError::Replay { jti }), + Err(Rejected::Total) => Err(AtprotoError::ReplayStoreSaturated), + Err(Rejected::Group) => Err(AtprotoError::ReplayShareExhausted { + issuer: issuer.clone(), + }), + } + } +} + +fn expires(now: UnixMicros, ttl: Duration) -> UnixMicros { + let micros = u64::try_from(ttl.as_micros()).unwrap_or(u64::MAX); + UnixMicros::new(now.get().saturating_add(micros)) +} + +fn warrants_negative_cache(error: &ResolveError) -> bool { + match error { + ResolveError::Status { status } => { + (400..500).contains(&status.get()) && status.get() != 429 + } + ResolveError::Gone { .. } + | ResolveError::Malformed(_) + | ResolveError::IdMismatch { .. } + | ResolveError::BadSigningKey(_) + | ResolveError::BadPds { .. } => true, + _ => false, + } +} + +enum Page { + First(usize), + Next(Cursor, usize), + Done, +} + +fn json_post(url: Url, body: bytes::Bytes) -> HttpRequest { + let mut request = HttpRequest::post(url, body); + request.headers.insert( + http::header::CONTENT_TYPE, + http::HeaderValue::from_static("application/json"), + ); + request +} + +fn xrpc_url(pds: &PdsEndpoint, method: &Nsid) -> Result { + let mut url = pds.url().clone(); + url.path_segments_mut() + .map_err(|_| AtprotoError::BadPdsEndpoint { + pds: pds.url().as_str().to_string(), + })? + .pop_if_empty() + .extend(["xrpc", method.as_str()]); + Ok(url) +} + +fn list_records_url( + pds: &PdsEndpoint, + did: &AccountDid, + cursor: Option<&Cursor>, +) -> Result { + let method: Nsid = + Nsid::new_static("com.atproto.repo.listRecords").expect("Literal NSID parses"); + let collection: Nsid = Nsid::new_static("org.tangled.key.sshKey").expect("Literal NSID parses"); + let mut url = xrpc_url(pds, &method)?; + url.query_pairs_mut() + .append_pair("repo", did.as_str()) + .append_pair("collection", collection.as_str()) + .append_pair("limit", &PUBKEY_PAGE_LIMIT.to_string()); + if let Some(cursor) = cursor { + url.query_pairs_mut().append_pair("cursor", cursor.as_str()); + } + Ok(url) +} + +#[derive(Deserialize)] +struct XrpcErrorBody { + error: String, +} + +fn record_not_found(body: &[u8]) -> bool { + serde_json::from_slice::(body) + .is_ok_and(|parsed| parsed.error == "RecordNotFound") +} + +fn get_record_url( + pds: &PdsEndpoint, + owner: &AccountDid, + collection: &Nsid, + rkey: &str, +) -> Result { + let method: Nsid = Nsid::new_static("com.atproto.repo.getRecord").expect("Literal NSID parses"); + let mut url = xrpc_url(pds, &method)?; + url.query_pairs_mut() + .append_pair("repo", owner.as_str()) + .append_pair("collection", collection.as_str()) + .append_pair("rkey", rkey); + Ok(url) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::*; + use bytes::Bytes; + use futures::StreamExt; + use http::StatusCode; + use knot_runtime::{DnsTxtResolver, FakeDns, FakeHttp, ManualClock, NetworkError}; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + + const POINTER_RKEY: &str = "3jzfcijpj2z2a"; + const POINTER_CID: &str = "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"; + + fn squid_doc(signing: &k256::ecdsa::SigningKey) -> Bytes { + did_doc(DocSpec { + id: SQUID, + signing, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }) + } + + fn resolver(dns: impl DnsTxtResolver, http: T) -> Atproto { + Atproto::new(http, clock(), knot_did(KNOT), plc()).with_dns(Arc::new(dns)) + } + + #[tokio::test] + async fn an_identity_is_resolved_from_a_did_document() { + let signing = signer(9); + let http = FakeHttp::new(move |request| { + assert_eq!(request.url.as_str(), "https://plc.directory/did:plc:squid"); + Ok(ok(squid_doc(&signing))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let identity = atproto.resolve_identity(&did(SQUID)).await.unwrap(); + assert_eq!(identity.pds.url().as_str(), "https://pds.oyster.cafe/"); + assert_eq!(identity.primary_handle().unwrap().as_str(), "nel.pet"); + } + + #[tokio::test] + async fn a_second_resolution_is_served_from_cache() { + let signing = signer(9); + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let http = FakeHttp::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(ok(squid_doc(&signing))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + atproto.resolve_identity(&did(SQUID)).await.unwrap(); + atproto.resolve_identity(&did(SQUID)).await.unwrap(); + assert_eq!(hits.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn an_expired_cache_entry_is_refetched() { + let signing = signer(9); + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let http = FakeHttp::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(ok(squid_doc(&signing))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + atproto.resolve_identity(&did(SQUID)).await.unwrap(); + atproto + .clock + .advance(DEFAULT_TTL + Duration::from_micros(1)); + atproto.resolve_identity(&did(SQUID)).await.unwrap(); + assert_eq!(hits.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn a_handle_resolves_bidirectionally_via_dns_and_is_cached() { + let signing = signer(9); + let dns_hits = Arc::new(AtomicUsize::new(0)); + let counter = dns_hits.clone(); + let dns = FakeDns::new(move |name: &str| { + assert_eq!(name, "_atproto.nel.pet"); + counter.fetch_add(1, Ordering::SeqCst); + Ok(vec!["did=did:plc:squid".to_string()]) + }); + let http = FakeHttp::new(move |request| { + assert_eq!(request.url.as_str(), "https://plc.directory/did:plc:squid"); + let mut doc: serde_json::Value = serde_json::from_slice(&squid_doc(&signing)).unwrap(); + doc["alsoKnownAs"] = serde_json::json!(["at://olaren.dev", "at://nel.pet"]); + Ok(ok(Bytes::from(serde_json::to_vec(&doc).unwrap()))) + }); + let atproto = resolver(dns, http); + let owner = handle("nel.pet"); + assert_eq!( + atproto + .resolve_handle_to_did(&owner) + .await + .unwrap() + .as_str(), + "did:plc:squid", + "a handle listed anywhere in alsoKnownAs must resolve, even when it isn't first" + ); + assert_eq!( + atproto + .resolve_handle_to_did(&owner) + .await + .unwrap() + .as_str(), + "did:plc:squid" + ); + assert_eq!( + dns_hits.load(Ordering::SeqCst), + 1, + "A resolved handle is served from cache" + ); + } + + #[tokio::test] + async fn a_handle_falls_back_to_well_known_when_dns_is_empty_or_transient() { + let well_known = || { + let signing = signer(9); + FakeHttp::new(move |request| match request.url.as_str() { + "https://nel.pet/.well-known/atproto-did" => { + Ok(ok(Bytes::from_static(b"did:plc:squid\n"))) + } + "https://plc.directory/did:plc:squid" => Ok(ok(squid_doc(&signing))), + other => panic!("Unexpected URL {other}"), + }) + }; + let empty = resolver(FakeDns::new(|_| Ok(Vec::new())), well_known()); + let transient = resolver( + FakeDns::new(|_| Err(NetworkError::Request("dns unreachable".to_string()))), + well_known(), + ); + for atproto in [empty, transient] { + assert_eq!( + atproto + .resolve_handle_to_did(&handle("nel.pet")) + .await + .unwrap() + .as_str(), + "did:plc:squid" + ); + } + } + + #[tokio::test] + async fn a_handle_is_rejected_when_ambiguous_or_disowned() { + let ambiguous = resolver( + FakeDns::new(|_| { + Ok(vec![ + "did=did:plc:squid".to_string(), + "did=did:plc:limpet".to_string(), + ]) + }), + FakeHttp::new(|_| panic!("Resolution must stop before any fetch")), + ); + assert!(matches!( + ambiguous + .resolve_handle_to_did(&handle("nel.pet")) + .await + .unwrap_err(), + AtprotoError::Resolve(ResolveError::HandleAmbiguous { .. }) + )); + + let signing = signer(9); + let disowned = resolver( + FakeDns::new(|_| Ok(vec!["did=did:plc:squid".to_string()])), + FakeHttp::new(move |_| Ok(ok(squid_doc(&signing)))), + ); + assert!(matches!( + disowned + .resolve_handle_to_did(&handle("olaren.dev")) + .await + .unwrap_err(), + AtprotoError::Resolve(ResolveError::HandleMismatch { .. }) + )); + } + + #[tokio::test] + async fn handle_failures_cache_by_class() { + let owner = handle("nel.pet"); + + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let unresolvable = resolver( + FakeDns::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(Vec::new()) + }), + FakeHttp::new(|_| Ok(status(StatusCode::NOT_FOUND, Bytes::new()))), + ); + assert!(matches!( + unresolvable + .resolve_handle_to_did(&owner) + .await + .unwrap_err(), + AtprotoError::Resolve(ResolveError::HandleUnresolvable { .. }) + )); + assert!(matches!( + unresolvable + .resolve_handle_to_did(&owner) + .await + .unwrap_err(), + AtprotoError::Resolve(ResolveError::HandleRecentlyFailed { .. }) + )); + assert_eq!( + hits.load(Ordering::SeqCst), + 1, + "An unresolvable handle is resolved once, then served from the negative cache" + ); + + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let transient = resolver( + FakeDns::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(Vec::new()) + }), + FakeHttp::new(|_| Ok(status(StatusCode::SERVICE_UNAVAILABLE, Bytes::new()))), + ); + transient.resolve_handle_to_did(&owner).await.unwrap_err(); + transient.resolve_handle_to_did(&owner).await.unwrap_err(); + assert_eq!( + hits.load(Ordering::SeqCst), + 2, + "A transient well-known status mustn't be negatively cached" + ); + } + + #[tokio::test] + async fn a_transient_outage_serves_the_last_good_did() { + let signing = signer(9); + let outage = Arc::new(AtomicUsize::new(0)); + let switch = outage.clone(); + let dns = FakeDns::new(move |_| match switch.load(Ordering::SeqCst) { + 0 => Ok(vec!["did=did:plc:squid".to_string()]), + _ => Err(NetworkError::Request("dns unreachable".to_string())), + }); + let atproto = resolver(dns, FakeHttp::new(move |_| Ok(ok(squid_doc(&signing))))); + let owner = handle("nel.pet"); + assert_eq!( + atproto + .resolve_handle_to_did(&owner) + .await + .unwrap() + .as_str(), + "did:plc:squid" + ); + + atproto.clock.advance(DEFAULT_TTL + Duration::from_secs(1)); + outage.store(1, Ordering::SeqCst); + assert_eq!( + atproto + .resolve_handle_to_did(&owner) + .await + .unwrap() + .as_str(), + "did:plc:squid", + "a transient outage must serve the last resolved DID" + ); + + atproto.clock.advance(STALE_TTL + Duration::from_secs(1)); + outage.store(0, Ordering::SeqCst); + assert_eq!( + atproto + .resolve_handle_to_did(&owner) + .await + .unwrap() + .as_str(), + "did:plc:squid" + ); + } + + #[tokio::test] + async fn a_404_identity_is_negatively_cached() { + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let http = FakeHttp::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(status(StatusCode::NOT_FOUND, Bytes::new())) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let first = atproto.resolve_identity(&did(SQUID)).await.unwrap_err(); + assert!( + matches!(first, AtprotoError::Resolve(ResolveError::Gone { status, .. }) if status.get() == 404), + "got {first:?}" + ); + let second = atproto.resolve_identity(&did(SQUID)).await.unwrap_err(); + assert!( + matches!(second, AtprotoError::Resolve(ResolveError::Gone { .. })), + "a caller that acts on a missing account must see the same answer from the cache as \ + from the fetch, or it acts on the first read only: got {second:?}" + ); + assert_eq!( + hits.load(Ordering::SeqCst), + 1, + "404 is served from the negative cache" + ); + } + + #[tokio::test] + async fn a_transient_5xx_identity_is_not_cached() { + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let http = FakeHttp::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(status(StatusCode::SERVICE_UNAVAILABLE, Bytes::new())) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let _ = atproto.resolve_identity(&did(SQUID)).await.unwrap_err(); + let _ = atproto.resolve_identity(&did(SQUID)).await.unwrap_err(); + assert_eq!( + hits.load(Ordering::SeqCst), + 2, + "a transient 503 is re-fetched instead of negatively cached" + ); + } + + #[tokio::test] + async fn a_document_describing_another_did_is_rejected() { + let doc_key = signer(7); + let http = FakeHttp::new(move |_| { + Ok(ok(did_doc(DocSpec { + id: LIMPET, + signing: &doc_key, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let error = atproto.resolve_identity(&did(SQUID)).await.unwrap_err(); + assert!( + matches!( + error, + AtprotoError::Resolve(ResolveError::IdMismatch { .. }) + ), + "got {error:?}" + ); + } + + #[tokio::test] + async fn a_claimed_handle_is_returned_unverified() { + let signing = signer(9); + let http = FakeHttp::new(move |_| { + Ok(ok(did_doc(DocSpec { + id: SQUID, + signing: &signing, + handle: "olaren.dev", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let identity = atproto.resolve_identity(&did(SQUID)).await.unwrap(); + assert_eq!( + identity.primary_handle().unwrap().as_str(), + "olaren.dev", + "alsoKnownAs handle is taken at face value with no bidirectional verification" + ); + } + + #[tokio::test] + async fn an_internal_ip_with_a_port_is_refused() { + let (sink, urls) = recorder(); + let http = FakeHttp::new(move |request| { + sink.lock().unwrap().push(request.url.clone()); + Ok(ok(Bytes::new())) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let error = atproto + .resolve_identity(&did("did:web:169.254.169.254%3A6379")) + .await + .unwrap_err(); + assert!( + matches!( + error, + AtprotoError::Resolve(ResolveError::BlockedHost { .. }) + ), + "got {error:?}" + ); + assert!(urls.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn repo_record_presence_maps_pds_status_to_a_verdict() { + let signing = signer(9); + let http = FakeHttp::new(move |request| { + if request.url.path().ends_with("did.json") + || request.url.host_str() == Some("plc.directory") + { + return Ok(ok(squid_doc(&signing))); + } + assert!(request.url.path().ends_with("com.atproto.repo.getRecord")); + assert!( + request.url.query().unwrap().contains("repo.declaration") + || request.url.query().unwrap().contains("sh.tangled.repo") + ); + let collection = request + .url + .query_pairs() + .find(|(key, _)| key == "collection") + .map(|(_, value)| value.into_owned()) + .unwrap_or_default(); + let rkey = request + .url + .query_pairs() + .find(|(key, _)| key == "rkey") + .map(|(_, value)| value.into_owned()) + .unwrap_or_default(); + let (st, body) = match (collection.as_str(), rkey.as_str()) { + (_, "boom") => (StatusCode::INTERNAL_SERVER_ERROR, Bytes::new()), + ("sh.tangled.repo", "legacy") => (StatusCode::OK, Bytes::new()), + ("org.tangled.repo.declaration", "present") => (StatusCode::OK, Bytes::new()), + _ => ( + StatusCode::BAD_REQUEST, + Bytes::from_static(b"{\"error\":\"RecordNotFound\"}"), + ), + }; + Ok(status(st, body)) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let owner = did(SQUID); + assert_eq!( + atproto + .repo_record_present(&owner, &RepoRkey::new("present").unwrap()) + .await + .unwrap(), + RecordPresence::Present + ); + assert_eq!( + atproto + .repo_record_present(&owner, &RepoRkey::new("missing").unwrap()) + .await + .unwrap(), + RecordPresence::Absent + ); + assert_eq!( + atproto + .repo_record_present(&owner, &RepoRkey::new("legacy").unwrap()) + .await + .unwrap(), + RecordPresence::Present, + "a declaration the previous release wrote still counts as declared" + ); + assert!( + atproto + .repo_record_present(&owner, &RepoRkey::new("boom").unwrap()) + .await + .is_err(), + "5xx from the PDS surfaces as an error so the caller can fall back to best-effort" + ); + } + + fn asking() -> (AccountDid, RepoRkey) { + (did(SQUID), RepoRkey::new("anemone").unwrap()) + } + + #[derive(Clone, Copy)] + enum Hosting { + Moved, + Blind, + } + + fn serving(then: Hosting) -> (Atproto, Arc) { + let (signing, lookups) = (signer(9), Arc::new(AtomicUsize::new(0))); + let counted = Arc::clone(&lookups); + let missing = Bytes::from_static(b"{\"error\":\"RecordNotFound\"}"); + let http = FakeHttp::new(move |request| match request.url.host_str() { + Some("plc.directory") => match counted.fetch_add(1, Ordering::SeqCst) { + 0 => Ok(ok(squid_doc(&signing))), + _ => match then { + Hosting::Blind => Ok(status(StatusCode::BAD_REQUEST, Bytes::new())), + Hosting::Moved => Ok(ok(did_doc(DocSpec { + id: SQUID, + signing: &signing, + handle: "nel.pet", + pds: "https://pds.anemone.town", + method: MethodKind::Multikey, + }))), + }, + }, + Some("pds.anemone.town") => Ok(ok(Bytes::new())), + _ => Ok(status(StatusCode::BAD_REQUEST, missing.clone())), + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + (atproto, lookups) + } + + #[tokio::test] + async fn an_absence_is_re_read_against_the_pds_the_account_moved_to() { + let (atproto, lookups) = serving(Hosting::Moved); + let (owner, rkey) = asking(); + let sealed = atproto.repo_record_present(&owner, &rkey).await; + assert_eq!( + (sealed.unwrap(), lookups.load(Ordering::SeqCst)), + (RecordPresence::Absent, 1), + "nothing can have moved between the document this call read and the record it asked \ + for, so a document this call resolved needs no second look" + ); + let moved = atproto.repo_record_present(&owner, &rkey).await; + assert_eq!( + (moved.unwrap(), lookups.load(Ordering::SeqCst)), + (RecordPresence::Present, 2), + "a 404 from the pds the account left is no evidence the record was deleted, and the \ + document sat inside its ttl, so only an absence buys the second read" + ); + } + + #[tokio::test] + async fn directory_400_fails_read_and_leaves_document_intact() { + let (atproto, lookups) = serving(Hosting::Blind); + let (owner, rkey) = asking(); + atproto.resolve_identity(&owner).await.unwrap(); + let unproven = atproto.repo_record_present(&owner, &rkey).await; + let intact = atproto.resolve_identity(&owner).await.unwrap(); + assert!( + unproven.is_err(), + "the 404 came off a pds the knot can no longer confirm is the current one, so \ + calling the record deleted would revoke on a guess" + ); + assert_eq!( + (intact.pds.url().as_str(), lookups.load(Ordering::SeqCst)), + ("https://pds.oyster.cafe/", 2), + "a confirmation that fails leaves the document every other caller reads alone: a 400 \ + from the directory mustn't cost this account its ssh keys or its service auth, and \ + the surviving document answers this third read off one fill and one confirmation" + ); + } + + #[tokio::test] + async fn pubkey_resolution_follows_the_cursor() { + let doc_key = signer(9); + let list_hits = Arc::new(AtomicUsize::new(0)); + let counter = list_hits.clone(); + let page_one = list_body( + &[ssh_line("ssh-ed25519", &[1u8; 32], "one")], + Some("page-2"), + ); + let page_two = list_body(&[ssh_line("ssh-ed25519", &[2u8; 32], "two")], None); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + return Ok(ok(squid_doc(&doc_key))); + } + counter.fetch_add(1, Ordering::SeqCst); + let on_second = request.url.query().unwrap().contains("cursor=page-2"); + Ok(ok(if on_second { + page_two.clone() + } else { + page_one.clone() + })) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let keys = atproto.resolve_pubkeys(&did(SQUID)).await.unwrap(); + assert_eq!(keys.len(), 2); + assert_eq!(list_hits.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn a_repeated_claim_is_served_from_the_cache_instead_of_the_pds() { + let signing = signer(9); + let line = ssh_line("ssh-ed25519", &[4u8; 32], "nel@oyster.cafe"); + let expected = parse_authorized_key(&line).unwrap(); + let listings = Arc::new(AtomicUsize::new(0)); + let counter = listings.clone(); + let body = list_body(&[line], None); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + return Ok(ok(squid_doc(&signing))); + } + assert_eq!(request.url.host_str(), Some("pds.oyster.cafe")); + assert!(request.url.path().ends_with("com.atproto.repo.listRecords")); + assert!( + request + .url + .query() + .unwrap() + .contains("org.tangled.key.sshKey") + ); + counter.fetch_add(1, Ordering::SeqCst); + Ok(ok(body.clone())) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + + let first = atproto.claimed_pubkeys(&did(SQUID)).await; + assert!( + matches!(&first, ClaimedKeys::Published(keys) if keys == std::slice::from_ref(&expected)) + ); + let second = atproto.claimed_pubkeys(&did(SQUID)).await; + assert!(matches!(&second, ClaimedKeys::Published(keys) if keys == &[expected])); + assert_eq!( + listings.load(Ordering::SeqCst), + 1, + "an ssh handshake asserting a login name reads the claimed account's records, so a \ + repeat inside the cache turn mustn't read them again, or any stranger with an ssh \ + client can make the knot fetch from that account's PDS at will" + ); + + atproto.clock.advance(PUBKEY_TTL + Duration::from_micros(1)); + let _ = atproto.claimed_pubkeys(&did(SQUID)).await; + assert_eq!( + listings.load(Ordering::SeqCst), + 2, + "A key published after the last read will be read once the cache turn is over" + ); + } + + #[tokio::test] + async fn a_refused_claim_read_is_cached_and_an_outage_is_retried() { + let refusal = Arc::new(Mutex::new(StatusCode::BAD_REQUEST)); + let answered = Arc::clone(&refusal); + let signing = signer(9); + let listings = Arc::new(AtomicUsize::new(0)); + let counter = listings.clone(); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + return Ok(ok(squid_doc(&signing))); + } + counter.fetch_add(1, Ordering::SeqCst); + Ok(status(*answered.lock().unwrap(), Bytes::new())) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + + assert!(matches!( + atproto.claimed_pubkeys(&did(SQUID)).await, + ClaimedKeys::Unread(_) + )); + let _ = atproto.claimed_pubkeys(&did(SQUID)).await; + assert_eq!( + listings.load(Ordering::SeqCst), + 1, + "a listing the PDS refuses outright is served from the negative cache, so \ + repeating the claim mustn't read the PDS again" + ); + + atproto + .clock + .advance(NEGATIVE_TTL + Duration::from_micros(1)); + *refusal.lock().unwrap() = StatusCode::SERVICE_UNAVAILABLE; + let _ = atproto.claimed_pubkeys(&did(SQUID)).await; + let _ = atproto.claimed_pubkeys(&did(SQUID)).await; + assert_eq!( + listings.load(Ordering::SeqCst), + 3, + "a transient failure is cached with no lifetime, so the account's next claim will \ + read the PDS again instead of waiting out a negative turn" + ); + } + + #[tokio::test] + async fn a_plain_http_pds_is_refused() { + let doc_key = signer(9); + let (sink, urls) = recorder(); + let http = FakeHttp::new(move |request| { + sink.lock().unwrap().push(request.url.clone()); + if request.url.host_str() == Some("plc.directory") { + Ok(ok(did_doc(DocSpec { + id: SQUID, + signing: &doc_key, + handle: "nel.pet", + pds: "http://127.0.0.1:6379", + method: MethodKind::Multikey, + }))) + } else { + Ok(ok(list_body(&[], None))) + } + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let error = atproto.resolve_pubkeys(&did(SQUID)).await.unwrap_err(); + assert!( + matches!( + error, + AtprotoError::Resolve(ResolveError::InsecureScheme { .. }) + ), + "got {error:?}" + ); + assert_eq!(urls.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn a_single_page_is_bounded_by_the_pubkey_page_limit() { + let signing = signer(9); + let lines: Vec = (0..3_000u32) + .map(|seed| { + let mut material = [0u8; 32]; + material[..4].copy_from_slice(&seed.to_be_bytes()); + ssh_line("ssh-ed25519", &material, "k") + }) + .collect(); + let page = list_body(&lines, None); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + Ok(ok(squid_doc(&signing))) + } else { + Ok(ok(page.clone())) + } + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let keys = atproto.resolve_pubkeys(&did(SQUID)).await.unwrap(); + assert_eq!( + keys.len(), + PUBKEY_PAGE_LIMIT as usize, + "single page yields at most the page limit even when the PDS floods it" + ); + } + + #[tokio::test] + async fn a_service_jwt_authenticates_against_the_resolved_issuer_key() { + let signing = signer(9); + let doc = signing.clone(); + let http = FakeHttp::new(move |_| Ok(ok(squid_doc(&doc)))); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + let claims = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-authenticates", "lxm": METHOD, + }); + let token = mint(&signing, &claims); + let authed = atproto.verify_service_jwt(&token, &method).await.unwrap(); + assert_eq!(authed, did(SQUID)); + } + + #[tokio::test] + async fn a_service_jwt_signed_by_an_impostor_is_rejected() { + let real = signer(9); + let impostor = signer(3); + let http = FakeHttp::new(move |_| Ok(ok(squid_doc(&real)))); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + let claims = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-impostor", "lxm": METHOD, + }); + let token = mint(&impostor, &claims); + let error = atproto + .verify_service_jwt(&token, &method) + .await + .unwrap_err(); + assert!(matches!( + error, + AtprotoError::Jwt(JwtError::InvalidSignature) + )); + } + + #[tokio::test] + async fn a_replayed_token_is_rejected() { + let key = signer(9); + let http = FakeHttp::new(move |_| Ok(ok(squid_doc(&key)))); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + let claims = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "single-use-nonce", "lxm": METHOD, + }); + let token = mint(&signer(9), &claims); + assert_eq!( + atproto.verify_service_jwt(&token, &method).await.unwrap(), + did(SQUID) + ); + let replay = atproto + .verify_service_jwt(&token, &method) + .await + .unwrap_err(); + assert!( + matches!(replay, AtprotoError::Replay { .. }), + "got {replay:?}" + ); + } + + #[tokio::test] + async fn two_issuers_may_share_a_nonce() { + let squid_key = signer(1); + let limpet_key = signer(2); + let sq = squid_key.clone(); + let li = limpet_key.clone(); + let http = FakeHttp::new(move |request| { + if request.url.as_str().ends_with(LIMPET) { + Ok(ok(did_doc(DocSpec { + id: LIMPET, + signing: &li, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + } else { + Ok(ok(squid_doc(&sq))) + } + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + let squid = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "shared-nonce", "lxm": METHOD, + }); + let limpet = serde_json::json!({ + "iss": LIMPET, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "shared-nonce", "lxm": METHOD, + }); + assert_eq!( + atproto + .verify_service_jwt(&mint(&squid_key, &squid), &method) + .await + .unwrap(), + did(SQUID) + ); + assert_eq!( + atproto + .verify_service_jwt(&mint(&limpet_key, &limpet), &method) + .await + .unwrap(), + did(LIMPET) + ); + } + + #[test] + fn knot_id_lowercases_its_host() { + assert_eq!(knot_did("did:web:NEL.PET").as_str(), "did:web:nel.pet"); + } + + #[test] + fn list_records_url_is_well_formed() { + let url = list_records_url( + &PdsEndpoint::new(Url::parse("https://pds.oyster.cafe").unwrap()).unwrap(), + &did(SQUID), + None, + ) + .unwrap(); + assert_eq!(url.host_str(), Some("pds.oyster.cafe")); + assert_eq!(url.path(), "/xrpc/com.atproto.repo.listRecords"); + let query = url.query().unwrap(); + assert!(query.contains("repo=did%3Aplc%3Asquid")); + assert!(query.contains("collection=org.tangled.key.sshKey")); + } + + #[test] + fn list_records_url_preserves_a_pds_base_path() { + let url = list_records_url( + &PdsEndpoint::new(Url::parse("https://shared.host/account-pds").unwrap()).unwrap(), + &did(SQUID), + Some(&Cursor::new("page2")), + ) + .unwrap(); + assert_eq!(url.path(), "/account-pds/xrpc/com.atproto.repo.listRecords"); + assert!(url.query().unwrap().contains("cursor=page2")); + } + + struct JwtCase { + name: &'static str, + header: &'static [u8], + mutate: fn(&mut serde_json::Value), + expect: fn(&Result) -> bool, + zero_network: bool, + } + + const JWT_HEADER: &[u8] = br#"{"alg":"ES256K","typ":"JWT"}"#; + + const JWT_CASES: &[JwtCase] = &[ + JwtCase { + name: "internal-ip issuer is blocked before any fetch", + header: JWT_HEADER, + mutate: |c| c["iss"] = serde_json::json!("did:web:169.254.169.254"), + expect: |r| { + matches!( + r, + Err(AtprotoError::Resolve(ResolveError::BlockedHost { .. })) + ) + }, + zero_network: true, + }, + JwtCase { + name: "decade-long token exceeds the lifetime limit", + header: JWT_HEADER, + mutate: |c| { + c["exp"] = serde_json::json!(1000 + 315_360_000i64); + c["iat"] = serde_json::json!(1000); + }, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::LifetimeTooLong { .. }))), + zero_network: true, + }, + JwtCase { + name: "audience in a different case is accepted", + header: JWT_HEADER, + mutate: |c| c["aud"] = serde_json::json!("did:web:NEL.PET"), + expect: |r| r.is_ok(), + zero_network: false, + }, + JwtCase { + name: "legacy-typed issuer key verifies", + header: JWT_HEADER, + mutate: |c| c["iss"] = serde_json::json!(LIMPET), + expect: |r| matches!(r, Ok(did) if did.as_str() == LIMPET), + zero_network: false, + }, + JwtCase { + name: "token addressed to another knot is refused", + header: JWT_HEADER, + mutate: |c| c["aud"] = serde_json::json!("did:web:somewhere.else"), + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::AudienceMismatch { .. }))), + zero_network: true, + }, + JwtCase { + name: "stale token is expired before resolution", + header: JWT_HEADER, + mutate: |c| { + c["exp"] = serde_json::json!(1); + c["iat"] = serde_json::json!(0); + }, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::Expired { .. }))), + zero_network: true, + }, + JwtCase { + name: "nonceless token is refused before resolution", + header: JWT_HEADER, + mutate: |c| { + c.as_object_mut().unwrap().remove("jti"); + }, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::MissingNonce))), + zero_network: true, + }, + JwtCase { + name: "alg none is refused", + header: br#"{"alg":"none","typ":"JWT"}"#, + mutate: |_| {}, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::Parse(_)))), + zero_network: false, + }, + JwtCase { + name: "es256 header against a k256 doc key is refused", + header: br#"{"alg":"ES256","typ":"JWT"}"#, + mutate: |_| {}, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::Parse(_)))), + zero_network: false, + }, + JwtCase { + name: "token type that isn't JWT is refused", + header: br#"{"alg":"ES256K","typ":"secevent+jwt"}"#, + mutate: |_| {}, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::UnexpectedType { .. }))), + zero_network: true, + }, + JwtCase { + name: "method mismatch is refused before resolution", + header: JWT_HEADER, + mutate: |c| c["lxm"] = serde_json::json!("org.tangled.repo.deleteRepo"), + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::MethodMismatch { .. }))), + zero_network: true, + }, + JwtCase { + name: "oversized nonce is refused before resolution", + header: JWT_HEADER, + mutate: |c| c["jti"] = serde_json::json!("n".repeat(100_000)), + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::OversizedNonce { .. }))), + zero_network: true, + }, + ]; + + #[tokio::test] + async fn verify_service_jwt_rejects_every_malformed_or_adversarial_token() { + let method = member_method(); + stream::iter(JWT_CASES) + .for_each(|case| { + let method = &method; + async move { + let doc_key = signer(1); + let calls = Arc::new(AtomicUsize::new(0)); + let counter = calls.clone(); + let http = FakeHttp::new(move |request| { + counter.fetch_add(1, Ordering::SeqCst); + if request.url.as_str().ends_with(LIMPET) { + Ok(ok(did_doc(DocSpec { + id: LIMPET, + signing: &doc_key, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::LegacyK256, + }))) + } else { + Ok(ok(squid_doc(&doc_key))) + } + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let mut claims = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-3a", "lxm": METHOD, + }); + (case.mutate)(&mut claims); + let token = mint_with_header(&signer(1), case.header, &claims); + let result = atproto.verify_service_jwt(&token, method).await; + assert!( + (case.expect)(&result), + "case {:?} got {result:?}", + case.name + ); + if case.zero_network { + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "case {:?} must be refused before any network resolution", + case.name + ); + } + } + }) + .await; + } + + #[tokio::test] + async fn the_jti_replay_store_is_bounded_and_fails_closed_when_saturated() { + let signing = signer(9); + let doc = signing.clone(); + let http = FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap().to_string(); + let id = format!("did:web:{host}"); + Ok(ok(did_doc(DocSpec { + id: &id, + signing: &doc, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + let issuers = MAX_SEEN_JTI / MAX_JTI_PER_ISSUER; + stream::iter( + (0..issuers).flat_map(|issuer| (0..MAX_JTI_PER_ISSUER).map(move |i| (issuer, i))), + ) + .for_each(|(issuer, i)| { + let atproto = &atproto; + let method = &method; + let signing = &signing; + async move { + let claims = serde_json::json!({ + "iss": format!("did:web:i{issuer}.oyster.cafe"), "aud": KNOT, + "exp": 1_001, "iat": 999, + "jti": format!("nonce-{issuer}-{i}"), "lxm": METHOD, + }); + atproto + .verify_service_jwt(&mint(signing, &claims), method) + .await + .unwrap(); + } + }) + .await; + assert_eq!(atproto.seen_jti.len(), MAX_SEEN_JTI); + let overflow = serde_json::json!({ + "iss": "did:web:fresh.oyster.cafe", "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-overflow", "lxm": METHOD, + }); + let error = atproto + .verify_service_jwt(&mint(&signing, &overflow), &method) + .await + .unwrap_err(); + assert!( + matches!(error, AtprotoError::ReplayStoreSaturated), + "every verify past the global limit fails closed, got {error:?}" + ); + assert!( + atproto.seen_jti.len() <= MAX_SEEN_JTI, + "replay store must stay bounded, held {}", + atproto.seen_jti.len() + ); + } + + #[tokio::test] + async fn one_issuer_cannot_hog_the_replay_store() { + let signing = signer(9); + let doc = signing.clone(); + let http = FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap().to_string(); + if host == "plc.directory" { + Ok(ok(squid_doc(&doc))) + } else { + let id = format!("did:web:{host}"); + Ok(ok(did_doc(DocSpec { + id: &id, + signing: &doc, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + } + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + stream::iter(0..MAX_JTI_PER_ISSUER) + .for_each(|i| { + let atproto = &atproto; + let method = &method; + let signing = &signing; + async move { + let claims = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": format!("nonce-{i}"), "lxm": METHOD, + }); + atproto + .verify_service_jwt(&mint(signing, &claims), method) + .await + .unwrap(); + } + }) + .await; + + let hogged = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-over-budget", "lxm": METHOD, + }); + let error = atproto + .verify_service_jwt(&mint(&signing, &hogged), &method) + .await + .unwrap_err(); + assert!( + matches!(error, AtprotoError::ReplayShareExhausted { .. }), + "issuer past its share fails closed, got {error:?}" + ); + + let bystander = serde_json::json!({ + "iss": "did:web:bystander.oyster.cafe", "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-bystander", "lxm": METHOD, + }); + atproto + .verify_service_jwt(&mint(&signing, &bystander), &method) + .await + .expect("Unrelated issuer is unaffected by the hog"); + + atproto.clock.advance(Duration::from_secs(62)); + let after_expiry = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_100, "iat": 1_050, + "jti": "nonce-after-expiry", "lxm": METHOD, + }); + atproto + .verify_service_jwt(&mint(&signing, &after_expiry), &method) + .await + .expect("Hog recovers once its nonces expire from the store"); + } + + #[tokio::test] + async fn the_identity_cache_stays_bounded_and_retains_hot_entries() { + let signing = signer(9); + let hot_hits = Arc::new(AtomicUsize::new(0)); + let sentinel_hits = Arc::new(AtomicUsize::new(0)); + let hot = hot_hits.clone(); + let sentinel = sentinel_hits.clone(); + let http = FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap().to_string(); + if host == "hot.oyster.cafe" { + hot.fetch_add(1, Ordering::SeqCst); + } + // yeah so what + if host == "sentinel.oyster.cafe" { + sentinel.fetch_add(1, Ordering::SeqCst); + } + let id = format!("did:web:{host}"); + Ok(ok(did_doc(DocSpec { + id: &id, + signing: &signing, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let hot_did = did("did:web:hot.oyster.cafe"); + stream::iter(0..64) + .for_each(|_| { + let atproto = &atproto; + let hot_did = hot_did.clone(); + async move { + atproto.resolve_identity(&hot_did).await.unwrap(); + } + }) + .await; + stream::iter(0..MAX_IDENTITY_CACHE * 2) + .for_each(|i| { + let atproto = &atproto; + let hot_did = hot_did.clone(); + async move { + let filler = AccountDid::new(format!("did:web:c{i}.oyster.cafe")).unwrap(); + atproto.resolve_identity(&filler).await.unwrap(); + if i.is_multiple_of(8) { + atproto.resolve_identity(&hot_did).await.unwrap(); + } + } + }) + .await; + atproto.identities.run_pending_tasks().await; + assert!( + atproto.identities.entry_count().get() <= MAX_IDENTITY_CACHE as u64, + "cache stays bounded under a cold-DID flood" + ); + let before = hot_hits.load(Ordering::SeqCst); + atproto.resolve_identity(&hot_did).await.unwrap(); + assert_eq!( + hot_hits.load(Ordering::SeqCst), + before, + "Frequently resolved identity is retained through the flood" + ); + let novel = did("did:web:sentinel.oyster.cafe"); + atproto.resolve_identity(&novel).await.unwrap(); + atproto.resolve_identity(&novel).await.unwrap(); + assert_eq!( + sentinel_hits.load(Ordering::SeqCst), + 1, + "Saturated cache still accepts a new entry and reuses it without a re-fetch" + ); + atproto.identities.run_pending_tasks().await; + assert!( + atproto.identities.entry_count().get() <= MAX_IDENTITY_CACHE as u64, + "Cache stays bounded after eviction" + ); + } + + struct GatedHttp { + status: StatusCode, + body: Bytes, + calls: Arc, + gate: Arc, + } + + impl HttpTransport for GatedHttp { + fn execute(&self, _request: HttpRequest) -> knot_runtime::HttpFuture { + self.calls.fetch_add(1, Ordering::SeqCst); + let gate = self.gate.clone(); + let response = status(self.status, self.body.clone()); + Box::pin(async move { + gate.notified().await; + Ok(response) + }) + } + } + + async fn gated_wave( + st: StatusCode, + body: Bytes, + ) -> (Vec>, usize) { + let calls = Arc::new(AtomicUsize::new(0)); + let gate = Arc::new(tokio::sync::Notify::new()); + let http = GatedHttp { + status: st, + body, + calls: calls.clone(), + gate: gate.clone(), + }; + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let subject = did(SQUID); + let resolves = + futures::future::join_all((0..2000).map(|_| atproto.resolve_identity(&subject))); + let releaser = async { + stream::iter(0..16) + .for_each(|_| async { + tokio::task::yield_now().await; + }) + .await; + gate.notify_one(); + }; + let (results, ()) = futures::join!(resolves, releaser); + (results, calls.load(Ordering::SeqCst)) + } + + #[tokio::test] + async fn concurrent_cold_resolves_coalesce_into_one_fetch() { + let (results, calls) = gated_wave(StatusCode::OK, squid_doc(&signer(9))).await; + assert_eq!(results.len(), 2000); + assert!(results.iter().all(|outcome| outcome.is_ok())); + assert_eq!( + calls, 1, + "2000 concurrent resolves of one cold DID issue exactly one outbound fetch" + ); + } + + #[tokio::test] + async fn a_concurrent_429_wave_gets_the_real_error_and_never_poisons_the_cache() { + let (results, calls) = gated_wave(StatusCode::TOO_MANY_REQUESTS, Bytes::new()).await; + assert_eq!(calls, 1, "Failing wave coalesces into one outbound fetch"); + assert!( + results.iter().all(|outcome| matches!( + outcome, + Err(AtprotoError::Resolve(ResolveError::Status { status })) if status.get() == 429 + )), + "every caller in the wave receives a real 429, never a poisoned RecentlyFailed" + ); + } + + #[tokio::test] + async fn every_caller_in_a_coalesced_404_wave_learns_the_account_is_missing() { + let (results, calls) = gated_wave(StatusCode::NOT_FOUND, Bytes::new()).await; + assert_eq!(calls, 1, "404 wave coalesces into one outbound fetch"); + let gone = results + .iter() + .filter(|outcome| { + matches!( + outcome, + Err(AtprotoError::Resolve(ResolveError::Gone { status, .. })) if status.get() == 404 + ) + }) + .count(); + assert_eq!( + gone, 2000, + "Coalescing mustn't decide which callers learn the account is missing; \ + every caller sees the same answer" + ); + } + + #[tokio::test] + async fn a_prepared_plc_operation_is_posted_and_a_rejection_is_typed() { + let prepared = prepare_repo_did( + &runtime_signer(11), + &knot_types::KnotServiceUrl::new("https://knot.nel.pet").unwrap(), + &repo_nonce(111), + ) + .unwrap(); + let operation = prepared.operation(); + let expected_path = format!("/{}", prepared.did.as_str()); + let http = FakeHttp::new(move |request| { + assert_eq!(request.method, http::Method::POST); + assert_eq!(request.url.path(), expected_path); + assert!(request.body.as_ref().is_some_and(|body| !body.is_empty())); + Ok(ok(Bytes::new())) + }); + Atproto::new(http, clock(), knot_did(KNOT), plc()) + .submit_plc_operation(&prepared.did.clone(), operation) + .await + .unwrap(); + + let rejected = prepare_repo_did( + &runtime_signer(12), + &knot_types::KnotServiceUrl::new("https://knot.nel.pet").unwrap(), + &repo_nonce(112), + ) + .unwrap(); + let rejected_op = rejected.operation(); + let http = FakeHttp::new(move |_| Ok(status(StatusCode::BAD_REQUEST, Bytes::new()))); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + assert!(matches!( + atproto.submit_plc_operation(&rejected.did.clone(), rejected_op).await, + Err(AtprotoError::PlcSubmit { status, .. }) if status.get() == 400 + )); + } + + #[tokio::test] + async fn a_did_web_document_verification_demands_atproto_and_pds_entries() { + let signing = signer(9); + let expected_pds = knot_types::KnotServiceUrl::new("https://knot.oyster.cafe").unwrap(); + let doc = signing.clone(); + let http = FakeHttp::new(move |request| { + assert_eq!( + request.url.as_str(), + "https://limpet.olaren.dev/.well-known/did.json" + ); + Ok(ok(did_doc(DocSpec { + id: "did:web:limpet.olaren.dev", + signing: &doc, + handle: "nel.pet", + pds: "https://knot.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + atproto + .verify_account_document( + &repo_did("did:web:limpet.olaren.dev"), + &PublicKeyBytes::from_bytes(sec1(&signing)), + &expected_pds, + ) + .await + .unwrap(); + + let published = signer(9); + let http = FakeHttp::new(move |_| { + Ok(ok(did_doc(DocSpec { + id: "did:web:limpet.olaren.dev", + signing: &published, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let error = atproto + .verify_account_document( + &repo_did("did:web:limpet.olaren.dev"), + &PublicKeyBytes::from_bytes(sec1(&signer(3))), + &expected_pds, + ) + .await + .unwrap_err(); + assert!(matches!( + error, + AtprotoError::Resolve(ResolveError::ExpectedKeyAbsent { .. }) + )); + + let stray_pds = knot_types::KnotServiceUrl::new("https://elsewhere.example").unwrap(); + let doc = signing.clone(); + let http = FakeHttp::new(move |_| { + Ok(ok(did_doc(DocSpec { + id: "did:web:limpet.olaren.dev", + signing: &doc, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let error = atproto + .verify_account_document( + &repo_did("did:web:limpet.olaren.dev"), + &PublicKeyBytes::from_bytes(sec1(&signing)), + &stray_pds, + ) + .await + .unwrap_err(); + assert!(matches!( + error, + AtprotoError::Resolve(ResolveError::BadPds { .. }) + )); + + let http = FakeHttp::new(|_| Ok(status(StatusCode::NOT_FOUND, Bytes::new()))); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let error = atproto + .verify_account_document( + &repo_did("did:web:limpet.olaren.dev"), + &PublicKeyBytes::from_bytes(vec![1, 2, 3]), + &expected_pds, + ) + .await + .unwrap_err(); + assert!(matches!( + error, + AtprotoError::Resolve(ResolveError::Status { status }) if status.get() == 404 + )); + } + + #[tokio::test] + async fn a_pointer_record_is_published_to_the_subjects_pds_over_service_auth() { + let doc_signer = signer(9); + let knot_key = runtime_signer(21); + let knot_public = knot_runtime::Signer::public_key(&knot_key); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + return Ok(ok(squid_doc(&doc_signer))); + } + assert_eq!(request.method, http::Method::POST); + assert_eq!( + request.url.as_str(), + "https://pds.oyster.cafe/xrpc/com.atproto.repo.putRecord" + ); + let bearer = request + .headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + .expect("Request includes a bearer service token"); + let parsed = knot_types::service_auth::parse_jwt(bearer).unwrap(); + assert_eq!(parsed.claims().iss.as_str(), KNOT); + assert_eq!(parsed.claims().aud.as_str(), "did:web:pds.oyster.cafe"); + assert_eq!( + parsed.claims().lxm.as_ref().unwrap().as_str(), + "com.atproto.repo.putRecord" + ); + assert!(parsed.claims().jti.is_some()); + let key = knot_types::service_auth::PublicKey::from_k256_bytes(knot_public.as_bytes()) + .unwrap(); + knot_types::service_auth::verify_signature(&parsed, &key) + .expect("Token is signed by the knot key"); + let body: serde_json::Value = + serde_json::from_slice(request.body.as_ref().unwrap()).unwrap(); + assert_eq!(body["repo"], SQUID); + assert_eq!(body["collection"], "org.tangled.knot.member"); + assert_eq!(body["rkey"], POINTER_RKEY); + assert_eq!(body["record"]["$type"], "org.tangled.knot.member"); + assert_eq!(body["record"]["createdAt"], "2026-06-11T00:00:00Z"); + let receipt = serde_json::json!({ + "uri": format!("at://{SQUID}/org.tangled.knot.member/{POINTER_RKEY}"), + "cid": POINTER_CID, + }); + Ok(ok(Bytes::from(serde_json::to_vec(&receipt).unwrap()))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let rkey = Rkey::new_owned(POINTER_RKEY).unwrap(); + let receipt = atproto + .publish_pointer( + &ServiceAuth::new(&knot_key, &entropy(31)), + &did(SQUID), + &rkey, + &member_pointer(), + ) + .await + .unwrap(); + assert_eq!( + receipt.uri.as_str(), + format!("at://{SQUID}/org.tangled.knot.member/{POINTER_RKEY}") + ); + assert_eq!(receipt.cid.as_str(), POINTER_CID); + } + + #[test] + fn a_pipeline_id_is_a_short_printable_string() { + assert!(is_pipeline_id("3lubrptx5zzzz")); + assert!(!is_pipeline_id("")); + assert!(!is_pipeline_id("two\nlines")); + assert!(!is_pipeline_id(&"x".repeat(MAX_PIPELINE_ID_LEN + 1))); + } + + async fn rejected_put_record(st: StatusCode) -> AtprotoError { + let doc_signer = signer(9); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + return Ok(ok(squid_doc(&doc_signer))); + } + Ok(status(st, Bytes::new())) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let rkey = Rkey::new_owned(POINTER_RKEY).unwrap(); + atproto + .publish_pointer( + &ServiceAuth::new(&runtime_signer(22), &entropy(31)), + &did(SQUID), + &rkey, + &member_pointer(), + ) + .await + .unwrap_err() + } + + #[tokio::test] + async fn a_rejected_put_record_is_a_typed_error_and_transient_only_on_server_failure() { + let server_failure = rejected_put_record(StatusCode::BAD_GATEWAY).await; + assert!(matches!( + server_failure, + AtprotoError::PutRecord { status, .. } if status.get() == 502 + )); + assert!(server_failure.is_transient()); + + let rate_limited = rejected_put_record(StatusCode::TOO_MANY_REQUESTS).await; + assert!(matches!( + rate_limited, + AtprotoError::PutRecord { status, .. } if status.get() == 429 + )); + assert!(rate_limited.is_transient()); + + let client_failure = rejected_put_record(StatusCode::BAD_REQUEST).await; + assert!(matches!( + client_failure, + AtprotoError::PutRecord { status, .. } if status.get() == 400 + )); + assert!(!client_failure.is_transient()); + } + + #[tokio::test] + async fn a_malformed_put_record_receipt_is_a_typed_error() { + let doc_signer = signer(9); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + return Ok(ok(squid_doc(&doc_signer))); + } + Ok(ok(Bytes::from_static(b"not json"))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let rkey = Rkey::new_owned(POINTER_RKEY).unwrap(); + let error = atproto + .publish_pointer( + &ServiceAuth::new(&runtime_signer(23), &entropy(31)), + &did(SQUID), + &rkey, + &member_pointer(), + ) + .await + .unwrap_err(); + assert!(matches!(error, AtprotoError::MalformedReceipt(_))); + } + + #[tokio::test] + async fn a_pointer_to_an_insecure_pds_endpoint_is_refused() { + let doc_signer = signer(9); + let http = FakeHttp::new(move |request| { + assert_eq!( + request.url.host_str(), + Some("plc.directory"), + "no request may reach the insecure PDS" + ); + Ok(ok(did_doc(DocSpec { + id: SQUID, + signing: &doc_signer, + handle: "nel.pet", + pds: "http://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let rkey = Rkey::new_owned(POINTER_RKEY).unwrap(); + let error = atproto + .publish_pointer( + &ServiceAuth::new(&runtime_signer(24), &entropy(31)), + &did(SQUID), + &rkey, + &member_pointer(), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + AtprotoError::Resolve(ResolveError::InsecureScheme { .. }) + )); + } +} diff --git a/knot2/crates/knot-atproto/src/plc.rs b/knot2/crates/knot-atproto/src/plc.rs new file mode 100644 index 0000000000000000000000000000000000000000..a841592891deae7834bd93bef2b3df6ebb558181 --- /dev/null +++ b/knot2/crates/knot-atproto/src/plc.rs @@ -0,0 +1,307 @@ +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value, json}; +use sha2::Digest; + +use knot_runtime::PublicKeyBytes; +use knot_runtime::Signer; +use knot_types::KnotServiceUrl; + +use crate::identity::IdentityError; +use crate::identity::KNOT_HOME_SERVICE; +use crate::identity::KNOT_HOME_TYPE; +use crate::identity::did_key_for; + +pub const ATPROTO_PDS_SERVICE: &str = "atproto_pds"; +pub const PDS_SERVICE_TYPE: &str = "AtprotoPersonalDataServer"; + +const PLC_OP_TYPE: &str = "plc_operation"; +const CID_DAG_CBOR_CODEC: u64 = 0x71; +const MULTIHASH_SHA2_256_CODE: u64 = 0x12; +const PLC_MAX_OPERATION_BYTES: usize = 7500; + +pub struct RepoTarget { + knot_did_key: String, + service_url: KnotServiceUrl, + home_origin: String, +} + +impl RepoTarget { + pub fn new(knot_public: &PublicKeyBytes, service_url: &KnotServiceUrl) -> Self { + let home_origin = url::Url::parse(service_url.as_str()) + .map(|url| url.origin().ascii_serialization()) + .unwrap_or_else(|_| service_url.as_str().to_owned()); + Self { + knot_did_key: did_key_for(knot_public), + service_url: service_url.clone(), + home_origin, + } + } + + pub fn knot_did_key(&self) -> &str { + &self.knot_did_key + } + + pub fn satisfied_by(&self, last_op: &Value) -> Result { + let current = Operation::parse(last_op).ok_or_else(unsupported_shape)?; + let desired = desired_operation(self, ¤t); + Ok(current.rotation_keys == desired.rotation_keys + && current.verification_methods == desired.verification_methods + && current.services == desired.services) + } +} + +fn unsupported_shape() -> IdentityError { + IdentityError::Op("previous operation isn't a supported plc_operation shape".into()) +} + +#[derive(Serialize, Deserialize)] +struct Operation { + #[serde(rename = "type")] + op_type: String, + #[serde(rename = "rotationKeys")] + rotation_keys: Vec, + #[serde(rename = "verificationMethods")] + verification_methods: Map, + #[serde(rename = "alsoKnownAs")] + also_known_as: Vec, + services: Map, +} + +impl Operation { + fn parse(op: &Value) -> Option { + let parsed: Operation = serde_json::from_value(op.clone()).ok()?; + (parsed.op_type == PLC_OP_TYPE).then_some(parsed) + } +} + +#[derive(Deserialize)] +struct ServiceEndpoint { + endpoint: String, +} + +fn points_at_origin(endpoint: &str, origin: &str) -> bool { + url::Url::parse(endpoint).is_ok_and(|parsed| parsed.origin().ascii_serialization() == origin) +} + +fn desired_operation(target: &RepoTarget, current: &Operation) -> Operation { + let home = current + .services + .get(KNOT_HOME_SERVICE) + .and_then(|entry| serde_json::from_value::(entry.clone()).ok()) + .map(|home| { + if points_at_origin(&home.endpoint, &target.home_origin) { + json!({ + "type": KNOT_HOME_TYPE, + "endpoint": home.endpoint, + }) + } else { + fresh_home(target) + } + }) + .unwrap_or_else(|| fresh_home(target)); + let mut desired = Operation { + op_type: PLC_OP_TYPE.to_owned(), + rotation_keys: vec![target.knot_did_key.clone()], + verification_methods: current.verification_methods.clone(), + also_known_as: current.also_known_as.clone(), + services: current.services.clone(), + }; + desired.verification_methods.insert( + crate::identity::ATPROTO_METHOD.to_owned(), + json!(target.knot_did_key), + ); + desired.services.insert( + ATPROTO_PDS_SERVICE.to_owned(), + json!({ "type": PDS_SERVICE_TYPE, "endpoint": target.service_url.as_str() }), + ); + desired.services.insert(KNOT_HOME_SERVICE.to_owned(), home); + desired +} + +fn fresh_home(target: &RepoTarget) -> Value { + json!({ "type": KNOT_HOME_TYPE, "endpoint": target.service_url.as_str() }) +} + +fn encode_unsigned_cbor(op: &Value) -> Result, IdentityError> { + let mut unsigned = op.clone(); + if let Some(object) = unsigned.as_object_mut() { + object.remove("sig"); + } + serde_ipld_dagcbor::to_vec(&unsigned).map_err(|error| IdentityError::Encode(error.to_string())) +} + +fn operation_cid(op: &Value) -> Result { + let bytes = + serde_ipld_dagcbor::to_vec(op).map_err(|error| IdentityError::Encode(error.to_string()))?; + let digest = sha2::Sha256::digest(&bytes); + let multihash = cid::multihash::Multihash::wrap(MULTIHASH_SHA2_256_CODE, &digest) + .map_err(|error| IdentityError::Encode(error.to_string()))?; + Ok(cid::Cid::new_v1(CID_DAG_CBOR_CODEC, multihash).to_string()) +} + +fn signed(mut op: Value, signer: &dyn Signer) -> Result { + let unsigned = encode_unsigned_cbor(&op)?; + let sig = URL_SAFE_NO_PAD.encode(signer.sign(&unsigned).as_bytes()); + match op.as_object_mut() { + Some(object) => { + object.insert("sig".into(), Value::String(sig)); + } + None => return Err(IdentityError::Op("operation isn't a JSON object".into())), + } + let bytes = serde_ipld_dagcbor::to_vec(&op) + .map_err(|error| IdentityError::Encode(error.to_string()))?; + if bytes.len() > PLC_MAX_OPERATION_BYTES { + return Err(IdentityError::Op(format!( + "encoded operation is {} bytes over the {PLC_MAX_OPERATION_BYTES}-byte limit", + bytes.len() + ))); + } + Ok(op) +} + +pub fn update_operation( + signer: &dyn Signer, + last_op: &Value, + target: &RepoTarget, +) -> Result { + let current = Operation::parse(last_op).ok_or_else(unsupported_shape)?; + let desired = desired_operation(target, ¤t); + let mut op = + serde_json::to_value(desired).map_err(|error| IdentityError::Encode(error.to_string()))?; + op["prev"] = json!(operation_cid(last_op)?); + signed(op, signer) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn test_signer(seed: u8) -> knot_runtime::K256Signer { + knot_runtime::K256Signer::from_slice(&[seed; 32]).expect("a fixed scalar signs") + } + + fn cafe_url() -> KnotServiceUrl { + KnotServiceUrl::new("https://knot.oyster.cafe").expect("constant test url is valid") + } + + #[test] + fn a_second_rotation_key_defeats_satisfaction_until_collapsed() { + let knot = test_signer(3); + let stranger = test_signer(4); + let target = RepoTarget::new(&knot.public_key(), &cafe_url()); + let last = json!({ + "type": PLC_OP_TYPE, + "rotationKeys": [did_key_for(&knot.public_key()), did_key_for(&stranger.public_key())], + "verificationMethods": {"atproto": did_key_for(&knot.public_key())}, + "alsoKnownAs": [], + "services": {}, + }); + assert!( + !target.satisfied_by(&last).unwrap(), + "a document keeping a second rotation authority isn't converged" + ); + let collapsed = update_operation(&stranger, &last, &target).unwrap(); + assert_eq!( + collapsed["rotationKeys"], + json!([did_key_for(&knot.public_key())]), + "the update collapses rotation onto the knot key alone" + ); + assert!(target.satisfied_by(&collapsed).unwrap()); + } + + #[test] + fn an_oversized_operation_refuses_to_encode() { + let signer = test_signer(6); + let target = RepoTarget::new(&signer.public_key(), &cafe_url()); + let filler = "p".repeat(9000); + let last = json!({ + "type": PLC_OP_TYPE, + "rotationKeys": [did_key_for(&signer.public_key())], + "verificationMethods": {"atproto": did_key_for(&signer.public_key())}, + "alsoKnownAs": [format!("at://{filler}")], + "services": {"atproto_pds": { + "type": PDS_SERVICE_TYPE, + "endpoint": format!("https://pad.example/{filler}"), + }}, + }); + let error = update_operation(&signer, &last, &target).unwrap_err(); + assert!(matches!(&error, IdentityError::Op(message) if message.contains("byte limit"))); + } + + #[test] + fn operation_cids_reproduce_the_directory_over_a_captured_operation() { + let captured = json!({ + "sig": "ZznbxHinpBI3NgEYWzXUXLA65s2U1ezJooreZlscHYQRfd4EMlijqhpikGMabs-81Tsy4Mt9Iscpmk7Uz13aAg", + "prev": null, + "type": PLC_OP_TYPE, + "services": {}, + "alsoKnownAs": ["at://op0"], + "rotationKeys": [ + "did:key:zQ3shmWf4f6ZwzNyjUDYw4oFQjgHWZoYZDjJYtz75YfYYrphB", + "did:key:zQ3shr2PwdoF6kbuYYymyZq6YbGWShiTXAkdMioCPdSdPL9NV" + ], + "verificationMethods": {}, + }); + let stored_cid = "bafyreihqa4o4gsyai22ydms6j4cua6yr6tuc7trq2temvnycadk2daniru"; + assert_eq!( + operation_cid(&captured).unwrap(), + stored_cid, + "the directory hashes signed operations including sig" + ); + } + + #[test] + fn a_foreign_tangled_knot_origin_is_replaced_instead_of_kept() { + let knot = test_signer(7); + let target = RepoTarget::new(&knot.public_key(), &cafe_url()); + let last = json!({ + "type": PLC_OP_TYPE, + "rotationKeys": [did_key_for(&knot.public_key())], + "verificationMethods": {"atproto": did_key_for(&knot.public_key())}, + "alsoKnownAs": [], + "services": {"tangled_knot": { + "type": KNOT_HOME_TYPE, + "endpoint": "https://knot.oyster.cafe.evil.example/repo/lure", + }}, + }); + assert!( + !target.satisfied_by(&last).unwrap(), + "a home service pointing at a stranger's origin isn't converged" + ); + let replaced = update_operation(&knot, &last, &target).unwrap(); + assert_eq!( + replaced["services"]["tangled_knot"]["endpoint"], + "https://knot.oyster.cafe" + ); + assert!(target.satisfied_by(&replaced).unwrap()); + } + + #[test] + fn a_same_origin_home_keeps_its_path_and_gains_the_canonical_type() { + let knot = test_signer(8); + let target = RepoTarget::new(&knot.public_key(), &cafe_url()); + let last = json!({ + "type": PLC_OP_TYPE, + "rotationKeys": [did_key_for(&knot.public_key())], + "verificationMethods": {"atproto": did_key_for(&knot.public_key())}, + "alsoKnownAs": [], + "services": {"tangled_knot": { + "type": "TangledKnotV1", + "endpoint": "https://knot.oyster.cafe/repo/tagged", + }}, + }); + let update = update_operation(&knot, &last, &target).unwrap(); + assert_eq!( + update["services"]["tangled_knot"], + json!({ + "type": "TangledKnot", + "endpoint": "https://knot.oyster.cafe/repo/tagged", + }), + "same-origin paths survive; only the type moves to the canonical one" + ); + assert!(target.satisfied_by(&update).unwrap()); + } +} diff --git a/knot2/crates/knot-atproto/src/pointer.rs b/knot2/crates/knot-atproto/src/pointer.rs new file mode 100644 index 0000000000000000000000000000000000000000..aa93bd1d2fe591928a57a2933b6dfdefe35602da --- /dev/null +++ b/knot2/crates/knot-atproto/src/pointer.rs @@ -0,0 +1,166 @@ +use knot_types::{AccountDid, AtUri, Cid, Collection, Nsid, Rkey, ServiceDid}; +use serde::{Deserialize, Serialize}; + +use crate::AtprotoError; +use crate::resolve::PdsEndpoint; + +pub fn put_record_method() -> Nsid { + Nsid::new_static("com.atproto.repo.putRecord").expect("literal method nsid parses") +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PointerReceipt { + pub uri: AtUri, + pub cid: Cid, +} + +pub(crate) fn pds_service_did(pds: &PdsEndpoint) -> Result { + let bad = || AtprotoError::BadPdsEndpoint { + pds: pds.url().as_str().to_string(), + }; + let host = pds + .url() + .host_str() + .filter(|host| !host.is_empty()) + .ok_or_else(bad)?; + let authority = pds + .url() + .port() + .map_or_else(|| host.to_string(), |port| format!("{host}%3A{port}")); + let msid = std::iter::once(authority) + .chain( + pds.url() + .path_segments() + .into_iter() + .flatten() + .filter(|segment| !segment.is_empty()) + .map(str::to_string), + ) + .collect::>() + .join(":"); + ServiceDid::new(format!("did:web:{msid}")).map_err(|_| bad()) +} + +#[derive(Serialize)] +struct PutRecordInput<'a, R: Serialize> { + repo: &'a AccountDid, + collection: &'static str, + rkey: &'a Rkey, + record: &'a R, +} + +pub(crate) fn put_record_body( + subject: &AccountDid, + rkey: &Rkey, + record: &R, +) -> Result, AtprotoError> { + serde_json::to_vec(&PutRecordInput { + repo: subject, + collection: R::NSID, + rkey, + record, + }) + .map_err(|error| AtprotoError::PointerEncode(error.to_string())) +} + +#[derive(Deserialize)] +struct PutRecordOutput { + uri: String, + cid: String, +} + +pub(crate) fn receipt_from_response(body: &[u8]) -> Result { + let output: PutRecordOutput = serde_json::from_slice(body) + .map_err(|error| AtprotoError::MalformedReceipt(error.to_string()))?; + let uri = AtUri::new_owned(&output.uri) + .map_err(|error| AtprotoError::MalformedReceipt(error.to_string()))?; + let cid = Cid::new_owned(output.cid.as_bytes()) + .map_err(|error| AtprotoError::MalformedReceipt(error.to_string())) + .and_then(|cid: Cid| { + cid.is_valid().then_some(cid).ok_or_else(|| { + AtprotoError::MalformedReceipt(format!("cid {:?} doesn't parse", output.cid)) + }) + })?; + Ok(PointerReceipt { uri, cid }) +} + +#[cfg(test)] +mod tests { + use super::*; + use url::Url; + + fn service_did_of(value: &str) -> Result { + PdsEndpoint::new(Url::parse(value).unwrap()) + .map_err(|_| AtprotoError::BadPdsEndpoint { + pds: value.to_string(), + }) + .and_then(|pds| pds_service_did(&pds)) + } + + struct PdsCase { + url: &'static str, + expect: fn(&Result) -> bool, + } + + const PDS_CASES: &[PdsCase] = &[ + PdsCase { + url: "https://pds.oyster.cafe", + expect: |r| matches!(r, Ok(did) if did.as_str() == "did:web:pds.oyster.cafe"), + }, + PdsCase { + url: "https://pds.oyster.cafe:8443", + expect: |r| matches!(r, Ok(did) if did.as_str() == "did:web:pds.oyster.cafe%3A8443"), + }, + PdsCase { + url: "https://shared.host/account-pds", + expect: |r| matches!(r, Ok(did) if did.as_str() == "did:web:shared.host:account-pds"), + }, + PdsCase { + url: "unix:/run/pds.sock", + expect: |r| matches!(r, Err(AtprotoError::BadPdsEndpoint { .. })), + }, + ]; + + #[test] + fn pds_service_did_maps_each_endpoint_shape_to_its_web_did() { + PDS_CASES.iter().for_each(|case| { + let result = service_did_of(case.url); + assert!((case.expect)(&result), "case {:?} got {result:?}", case.url); + }); + } + + #[test] + fn a_receipt_round_trips_its_uri_and_cid() { + let body = serde_json::json!({ + "uri": "at://did:plc:squid/org.tangled.knot.member/3jzfcijpj2z2a", + "cid": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a" + }); + let receipt = receipt_from_response(&serde_json::to_vec(&body).unwrap()).unwrap(); + assert_eq!( + receipt.uri.as_str(), + "at://did:plc:squid/org.tangled.knot.member/3jzfcijpj2z2a" + ); + assert_eq!( + receipt.cid.as_str(), + "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a" + ); + } + + #[test] + fn a_garbage_receipt_is_a_typed_error() { + assert!(matches!( + receipt_from_response(b"not json"), + Err(AtprotoError::MalformedReceipt(_)) + )); + let bad_uri = serde_json::json!({ "uri": "http://nope", "cid": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a" }); + assert!(matches!( + receipt_from_response(&serde_json::to_vec(&bad_uri).unwrap()), + Err(AtprotoError::MalformedReceipt(_)) + )); + let bad_cid = serde_json::json!({ "uri": "at://did:plc:squid/org.tangled.knot.member/3jzfcijpj2z2a", "cid": "not-a-cid" }); + assert!(matches!( + receipt_from_response(&serde_json::to_vec(&bad_cid).unwrap()), + Err(AtprotoError::MalformedReceipt(_)) + )); + } +} diff --git a/knot2/crates/knot-atproto/src/pubkeys.rs b/knot2/crates/knot-atproto/src/pubkeys.rs new file mode 100644 index 0000000000000000000000000000000000000000..704c4879c1d482f2c980280c5864e680ee01801c --- /dev/null +++ b/knot2/crates/knot-atproto/src/pubkeys.rs @@ -0,0 +1,245 @@ +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use knot_types::OfferedKey; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub(crate) struct Cursor(String); + +impl Cursor { + #[cfg(test)] + pub(crate) fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug, thiserror::Error)] +pub enum KeyParseError { + #[error("public key line is missing algorithm and blob")] + Incomplete, + #[error("public key blob isn't valid base64: {0}")] + Base64(String), + #[error("public key blob is truncated")] + Truncated, + #[error("declared algorithm {declared:?} doesn't match blob's {embedded:?}")] + AlgorithmMismatch { declared: String, embedded: String }, +} + +pub fn parse_authorized_key(line: &str) -> Result { + let parts: Vec<&str> = line.split_whitespace().take(2).collect(); + let [algo, blob_b64] = parts.as_slice() else { + return Err(KeyParseError::Incomplete); + }; + let blob = STANDARD + .decode(blob_b64) + .map_err(|error| KeyParseError::Base64(error.to_string()))?; + let embedded = embedded_algorithm(&blob)?; + if embedded != algo.as_bytes() { + return Err(KeyParseError::AlgorithmMismatch { + declared: (*algo).to_string(), + embedded: String::from_utf8_lossy(embedded).into_owned(), + }); + } + Ok(OfferedKey::from_bytes(blob)) +} + +fn embedded_algorithm(blob: &[u8]) -> Result<&[u8], KeyParseError> { + let length = blob + .get(..4) + .map(|head| u32::from_be_bytes(head.try_into().expect("four bytes")) as usize) + .ok_or(KeyParseError::Truncated)?; + let end = length.checked_add(4).ok_or(KeyParseError::Truncated)?; + blob.get(4..end).ok_or(KeyParseError::Truncated) +} + +#[derive(Deserialize)] +struct ListRecords { + records: Vec, + #[serde(default)] + cursor: Option, +} + +#[derive(Deserialize)] +struct Envelope { + value: KeyRecord, +} + +#[derive(Deserialize)] +struct KeyRecord { + #[serde(rename = "publicKey")] + public_key: String, + #[serde(default)] + usages: Option>, +} + +impl KeyRecord { + fn authenticates(&self) -> bool { + self.usages + .as_ref() + .is_none_or(|usages| usages.iter().any(|usage| usage == "auth")) + } +} + +pub(crate) struct PubkeyPage { + pub keys: Vec, + pub cursor: Option, +} + +pub(crate) fn offered_page(body: &[u8], max_keys: usize) -> Result { + let listing: ListRecords = serde_json::from_slice(body)?; + let keys = listing + .records + .iter() + .take(max_keys) + .filter(|envelope| envelope.value.authenticates()) + .filter_map(|envelope| parse_authorized_key(&envelope.value.public_key).ok()) + .collect(); + Ok(PubkeyPage { + keys, + cursor: listing.cursor.filter(|cursor| !cursor.as_str().is_empty()), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::*; + + fn ssh_string(bytes: &[u8]) -> Vec { + [&(bytes.len() as u32).to_be_bytes()[..], bytes].concat() + } + + fn wire_blob(material: &[u8]) -> OfferedKey { + OfferedKey::from_bytes([ssh_string(b"ssh-ed25519"), ssh_string(material)].concat()) + } + + struct KeyCase { + name: &'static str, + line: fn() -> String, + expect: fn(&Result) -> bool, + } + + const KEY_CASES: &[KeyCase] = &[ + KeyCase { + name: "genuine ed25519 key with a trailing comment", + line: || ssh_line("ssh-ed25519", &[7u8; 32], "nel@oyster.cafe"), + expect: |r| matches!(r, Ok(key) if *key == wire_blob(&[7u8; 32])), + }, + KeyCase { + name: "the same key material with no comment", + line: || ssh_line("ssh-ed25519", &[7u8; 32], ""), + expect: |r| matches!(r, Ok(key) if *key == wire_blob(&[7u8; 32])), + }, + KeyCase { + name: "bare algorithm with no blob", + line: || "ssh-ed25519".to_string(), + expect: |r| matches!(r, Err(KeyParseError::Incomplete)), + }, + KeyCase { + name: "whitespace-only line", + line: || " ".to_string(), + expect: |r| matches!(r, Err(KeyParseError::Incomplete)), + }, + KeyCase { + name: "blob that isn't base64", + line: || "ssh-ed25519 not-base64!!!".to_string(), + expect: |r| matches!(r, Err(KeyParseError::Base64(_))), + }, + KeyCase { + name: "declared algorithm lying about the blob", + line: || { + let blob = [ssh_string(b"ssh-ed25519"), ssh_string(&[1u8; 32])].concat(); + format!("ssh-rsa {}", STANDARD.encode(blob)) + }, + expect: |r| matches!(r, Err(KeyParseError::AlgorithmMismatch { .. })), + }, + KeyCase { + name: "authorized_keys options prefix", + line: || { + let blob = [ssh_string(b"ssh-ed25519"), ssh_string(&[7u8; 32])].concat(); + format!( + "command=\"true\",no-pty ssh-ed25519 {} nel@oyster.cafe", + STANDARD.encode(blob) + ) + }, + expect: |r| r.is_err(), + }, + KeyCase { + name: "overlong length prefix", + line: || { + let lying = [&u32::MAX.to_be_bytes()[..], b"short"].concat(); + format!("ssh-ed25519 {}", STANDARD.encode(lying)) + }, + expect: |r| matches!(r, Err(KeyParseError::Truncated)), + }, + ]; + + #[test] + fn parse_authorized_key_accepts_genuine_lines_and_rejects_malformed_ones() { + KEY_CASES.iter().for_each(|case| { + let result = parse_authorized_key(&(case.line)()); + assert!( + (case.expect)(&result), + "case {:?} got {result:?}", + case.name + ); + }); + } + + #[test] + fn list_records_yields_every_well_formed_key_and_skips_the_rest() { + let good_one = ssh_line("ssh-ed25519", &[1u8; 32], "one"); + let good_two = ssh_line("ssh-ed25519", &[2u8; 32], "two"); + let signing_only = ssh_line("ssh-ed25519", &[3u8; 32], "three"); + let body = serde_json::json!({ + "records": [ + { "uri": "at://did:plc:squid/org.tangled.key.sshKey/a", "value": { "$type": "org.tangled.key.sshKey", "publicKey": good_one, "name": "laptop", "createdAt": "2026-06-08T00:00:00Z" } }, + { "uri": "at://did:plc:squid/org.tangled.key.sshKey/b", "value": { "$type": "org.tangled.key.sshKey", "publicKey": "garbage line", "name": "broken", "createdAt": "2026-06-08T00:00:00Z" } }, + { "uri": "at://did:plc:squid/org.tangled.key.sshKey/c", "value": { "$type": "org.tangled.key.sshKey", "publicKey": good_two, "name": "desktop", "usages": ["sign", "auth"], "createdAt": "2026-06-08T00:00:00Z" } }, + { "uri": "at://did:plc:squid/org.tangled.key.sshKey/d", "value": { "$type": "org.tangled.key.sshKey", "publicKey": signing_only, "name": "signer", "usages": ["sign"], "createdAt": "2026-06-08T00:00:00Z" } } + ], + "cursor": "c" + }); + let page = offered_page(serde_json::to_vec(&body).unwrap().as_slice(), 100).unwrap(); + assert_eq!(page.keys.len(), 2); + assert_eq!(page.keys[0], parse_authorized_key(&good_one).unwrap()); + assert_eq!(page.keys[1], parse_authorized_key(&good_two).unwrap()); + assert_eq!(page.cursor.as_ref().map(Cursor::as_str), Some("c")); + } + + #[test] + fn a_cursor_serializes_as_a_plain_json_string() { + let cursor = Cursor::new("page-token"); + assert_eq!(cursor.as_str(), "page-token"); + assert_eq!(serde_json::to_string(&cursor).unwrap(), "\"page-token\""); + let parsed: Cursor = serde_json::from_str("\"page-token\"").unwrap(); + assert_eq!(parsed, cursor); + } + + #[test] + fn a_page_is_bounded_by_the_requested_record_limit() { + let records: Vec<_> = (0u32..50) + .map(|seed| { + let mut material = [0u8; 32]; + material[..4].copy_from_slice(&seed.to_be_bytes()); + let line = ssh_line("ssh-ed25519", &material, "k"); + serde_json::json!({ + "uri": "at://did:plc:squid/org.tangled.key.sshKey/x", + "value": { "$type": "org.tangled.key.sshKey", "publicKey": line, "name": "k", "createdAt": "2026-06-08T00:00:00Z" } + }) + }) + .collect(); + let body = serde_json::json!({ "records": records }); + let page = offered_page(serde_json::to_vec(&body).unwrap().as_slice(), 10).unwrap(); + assert_eq!( + page.keys.len(), + 10, + "page yields at most requested record limit, no matter how many the PDS returns" + ); + } +} diff --git a/knot2/crates/knot-atproto/src/resolve.rs b/knot2/crates/knot-atproto/src/resolve.rs new file mode 100644 index 0000000000000000000000000000000000000000..4e8cb49e0b950ac0136aa801a31883dcfbe27af9 --- /dev/null +++ b/knot2/crates/knot-atproto/src/resolve.rs @@ -0,0 +1,720 @@ +use std::borrow::Cow; + +use knot_runtime::PublicKeyBytes; +use knot_types::crypto::{KeyCodec, PublicKey as CryptoKey}; +use knot_types::did_doc::DidDocument; +use knot_types::{AccountDid, Handle, HttpStatus, RepoDid, ServiceDid}; +use url::{Host, Url}; + +const LEGACY_K256_KIND: &str = "EcdsaSecp256k1VerificationKey2019"; +const LEGACY_P256_KIND: &str = "EcdsaSecp256r1VerificationKey2019"; +const SPINDLE_SERVICE_ID: &str = "#tangled_spindle"; +const SPINDLE_SERVICE_TYPE: &str = "TangledSpindle"; +const KNOT_SERVICE_ID: &str = "#tangled_knot"; +const KNOT_SERVICE_TYPE: &str = "TangledKnot"; + +#[derive(Debug, Clone, thiserror::Error)] +pub enum ResolveError { + #[error("unsupported DID method in {value:?}")] + UnsupportedMethod { value: String }, + #[error("DID {value:?} doesn't form a resolvable document location")] + Unresolvable { value: String }, + #[error("DID document fetch returned HTTP {status}")] + Status { status: HttpStatus }, + #[error("network failure resolving DID document: {0}")] + Network(#[from] knot_runtime::NetworkError), + #[error("DID document isn't valid JSON: {0}")] + Malformed(String), + #[error("DID document for {requested:?} claims to describe {document:?}")] + IdMismatch { requested: String, document: String }, + #[error("refusing to resolve over non-https endpoint: {url}")] + InsecureScheme { url: String }, + #[error("refusing to resolve non-public address {host}")] + BlockedHost { host: String }, + #[error("DID document declares no atproto signing key")] + MissingSigningKey, + #[error("DID document signing key is unusable: {0}")] + BadSigningKey(String), + #[error("DID document declares no atproto_pds service endpoint")] + MissingPds, + #[error("atproto_pds endpoint {value:?} isn't valid URL")] + BadPds { value: String }, + #[error("DID document declares no tangled_spindle service endpoint")] + MissingSpindle, + #[error("tangled_spindle endpoint {value:?} isn't valid URL")] + BadSpindle { value: String }, + #[error("DID document declares no tangled_knot service endpoint")] + MissingKnot, + #[error("tangled_knot endpoint {value:?} isn't valid URL")] + BadKnot { value: String }, + #[error("PLC directory {value:?} isn't valid http(s) base URL")] + BadPlcDirectory { value: String }, + #[error("identity {did} recently failed to resolve and is negatively cached")] + RecentlyFailed { did: AccountDid }, + #[error("{did} doesn't have a DID document, the directory answered HTTP {status}")] + Gone { did: AccountDid, status: HttpStatus }, + #[error("did:web document for {did} doesn't publish expected signing key")] + ExpectedKeyAbsent { did: RepoDid }, + #[error("handle {handle} has no atproto DNS or well-known record")] + HandleUnresolvable { handle: Handle }, + #[error("handle {handle} resolves to more than one distinct DID")] + HandleAmbiguous { handle: Handle }, + #[error("handle {handle} points at {value:?}, which isn't a valid DID")] + HandleForwardMalformed { handle: Handle, value: String }, + #[error("handle {handle} resolved to {resolved} but that document claims {claimed:?}")] + HandleMismatch { + handle: Handle, + resolved: AccountDid, + claimed: Option, + }, + #[error("handle {handle} recently failed to resolve and is negatively cached")] + HandleRecentlyFailed { handle: Handle }, +} + +impl ResolveError { + pub fn is_transient(&self) -> bool { + match self { + ResolveError::Network(_) => true, + ResolveError::Status { status } => status.is_transient(), + _ => false, + } + } + + pub fn is_gone(&self) -> bool { + match self { + ResolveError::Gone { .. } | ResolveError::Unresolvable { .. } => true, + ResolveError::UnsupportedMethod { .. } + | ResolveError::Status { .. } + | ResolveError::Network(_) + | ResolveError::Malformed(_) + | ResolveError::IdMismatch { .. } + | ResolveError::InsecureScheme { .. } + | ResolveError::BlockedHost { .. } + | ResolveError::MissingSigningKey + | ResolveError::BadSigningKey(_) + | ResolveError::MissingPds + | ResolveError::BadPds { .. } + | ResolveError::MissingSpindle + | ResolveError::BadSpindle { .. } + | ResolveError::MissingKnot + | ResolveError::BadKnot { .. } + | ResolveError::BadPlcDirectory { .. } + | ResolveError::RecentlyFailed { .. } + | ResolveError::ExpectedKeyAbsent { .. } + | ResolveError::HandleUnresolvable { .. } + | ResolveError::HandleAmbiguous { .. } + | ResolveError::HandleForwardMalformed { .. } + | ResolveError::HandleMismatch { .. } + | ResolveError::HandleRecentlyFailed { .. } => false, + } + } +} + +#[derive(Debug, Clone)] +pub struct PdsEndpoint(Url); + +impl PdsEndpoint { + pub fn new(url: Url) -> Result { + match http_base(&url) { + true => Ok(Self(url)), + false => Err(ResolveError::BadPds { + value: url.as_str().to_string(), + }), + } + } + + pub fn url(&self) -> &Url { + &self.0 + } +} + +#[derive(Debug, Clone)] +pub struct PlcDirectory(Url); + +impl PlcDirectory { + pub fn new(url: Url) -> Result { + match http_base(&url) { + true => Ok(Self(url)), + false => Err(ResolveError::BadPlcDirectory { + value: url.as_str().to_string(), + }), + } + } +} + +fn http_base(url: &Url) -> bool { + matches!(url.scheme(), "http" | "https") + && url.has_host() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() +} + +#[derive(Debug, Clone)] +pub struct Identity { + pub did: AccountDid, + pub handles: Vec, + pub signing_key: CryptoKey<'static>, + pub pds: PdsEndpoint, +} + +impl Identity { + pub fn primary_handle(&self) -> Option<&Handle> { + self.handles.first() + } + + pub fn claims_handle(&self, handle: &Handle) -> bool { + self.handles.iter().any(|known| known == handle) + } +} + +pub(crate) fn document_url( + did: &AccountDid, + plc_directory: &PlcDirectory, +) -> Result { + let value = did.as_str(); + if value.strip_prefix("did:plc:").is_some() { + let base = plc_directory.0.as_str().trim_end_matches('/'); + Url::parse(&format!("{base}/{value}")).map_err(|_| ResolveError::Unresolvable { + value: value.to_string(), + }) + } else if let Some(rest) = value.strip_prefix("did:web:") { + web_document_url(rest).ok_or(ResolveError::Unresolvable { + value: value.to_string(), + }) + } else { + Err(ResolveError::UnsupportedMethod { + value: value.to_string(), + }) + } +} + +pub(crate) fn guard_fetch_url(url: &Url) -> Result<(), ResolveError> { + if url.scheme() != "https" { + return Err(ResolveError::InsecureScheme { + url: url.as_str().to_string(), + }); + } + let blocked = match url.host() { + Some(Host::Ipv4(ip)) => knot_runtime::is_blocked_ip(ip.into()).then(|| ip.to_string()), + Some(Host::Ipv6(ip)) => knot_runtime::is_blocked_ip(ip.into()).then(|| ip.to_string()), + _ => None, + }; + match blocked { + Some(host) => Err(ResolveError::BlockedHost { host }), + None => Ok(()), + } +} + +fn web_document_url(rest: &str) -> Option { + let mut segments = rest.split(':'); + let authority = segments.next().filter(|head| !head.is_empty())?; + let host = authority.replace("%3A", ":").replace("%3a", ":"); + let path: Vec<&str> = segments.collect(); + let tail = if path.is_empty() { + ".well-known/did.json".to_string() + } else if path.iter().any(|segment| segment.is_empty()) { + return None; + } else { + format!("{}/did.json", path.join("/")) + }; + Url::parse(&format!("https://{host}/{tail}")).ok() +} + +pub(crate) fn identity_from_document( + did: &AccountDid, + body: &[u8], +) -> Result { + let document: DidDocument = + serde_json::from_slice(body).map_err(|error| ResolveError::Malformed(error.to_string()))?; + if AccountDid::new(document.id.as_str()).ok().as_ref() != Some(did) { + return Err(ResolveError::IdMismatch { + requested: did.as_str().to_string(), + document: document.id.as_str().to_string(), + }); + } + let signing_key = atproto_signing_key(&document)?; + let pds_endpoint = document.pds_endpoint().ok_or(ResolveError::MissingPds)?; + let pds = Url::parse(pds_endpoint.as_str()) + .map_err(|_| ResolveError::BadPds { + value: pds_endpoint.as_str().to_string(), + }) + .and_then(PdsEndpoint::new)?; + let handles = document + .handles() + .iter() + .filter_map(|found| Handle::new_owned(found.as_str()).ok()) + .collect(); + Ok(Identity { + did: did.clone(), + handles, + signing_key, + pds, + }) +} + +/// The `#tangled_spindle` service endpoint of a spindle's DID document. +pub(crate) fn spindle_endpoint(did: &ServiceDid, body: &[u8]) -> Result { + let document: DidDocument = + serde_json::from_slice(body).map_err(|error| ResolveError::Malformed(error.to_string()))?; + if document.id.as_str() != did.as_str() { + return Err(ResolveError::IdMismatch { + requested: did.as_str().to_string(), + document: document.id.as_str().to_string(), + }); + } + let endpoint = document + .service + .iter() + .flatten() + .find(|service| { + service.id.as_str().ends_with(SPINDLE_SERVICE_ID) + && service.r#type.as_str() == SPINDLE_SERVICE_TYPE + }) + .and_then(|service| service.service_endpoint.as_ref()?.as_str()) + .ok_or(ResolveError::MissingSpindle)?; + Url::parse(endpoint).map_err(|_| ResolveError::BadSpindle { + value: endpoint.to_string(), + }) +} + +pub(crate) fn knot_endpoint(did: &RepoDid, body: &[u8]) -> Result { + let document: DidDocument = + serde_json::from_slice(body).map_err(|error| ResolveError::Malformed(error.to_string()))?; + if document.id.as_str() != did.as_str() { + return Err(ResolveError::IdMismatch { + requested: did.as_str().to_string(), + document: document.id.as_str().to_string(), + }); + } + let absolute = format!("{}{KNOT_SERVICE_ID}", did.as_str()); + let endpoint = document + .service + .iter() + .flatten() + .find(|service| { + let id = service.id.as_str(); + (id == KNOT_SERVICE_ID || id == absolute) + && service.r#type.as_str() == KNOT_SERVICE_TYPE + }) + .and_then(|service| service.service_endpoint.as_ref()?.as_str()) + .ok_or(ResolveError::MissingKnot)?; + Url::parse(endpoint).map_err(|_| ResolveError::BadKnot { + value: endpoint.to_string(), + }) +} + +pub(crate) fn document_publishes_account_shape( + did: &RepoDid, + body: &[u8], + expected: &PublicKeyBytes, + expected_pds: &knot_types::KnotServiceUrl, +) -> Result<(), ResolveError> { + let document: DidDocument = + serde_json::from_slice(body).map_err(|error| ResolveError::Malformed(error.to_string()))?; + if document.id.as_str() != did.as_str() { + return Err(ResolveError::IdMismatch { + requested: did.as_str().to_string(), + document: document.id.as_str().to_string(), + }); + } + let methods = document + .verification_method + .as_ref() + .ok_or(ResolveError::MissingSigningKey)?; + let published = methods.iter().any(|method| { + method.id.as_str().ends_with("#atproto") + && method_key(method).is_ok_and(|key| { + matches!(key.codec, KeyCodec::Secp256k1) + && key.bytes.as_ref() == expected.as_bytes() + }) + }); + if !published { + return Err(ResolveError::ExpectedKeyAbsent { did: did.clone() }); + } + let sent = document + .pds_endpoint() + .ok_or(ResolveError::MissingPds)? + .as_str(); + if knot_types::KnotServiceUrl::new(sent).is_ok_and(|endpoint| endpoint == *expected_pds) { + Ok(()) + } else { + Err(ResolveError::BadPds { + value: sent.to_string(), + }) + } +} + +fn supported_kind(kind: &str) -> bool { + matches!(kind, "Multikey" | LEGACY_K256_KIND | LEGACY_P256_KIND) +} + +fn method_key( + method: &knot_types::did_doc::VerificationMethod, +) -> Result, String> { + let multibase = method + .public_key_multibase + .as_ref() + .ok_or_else(|| "verification method lacks publicKeyMultibase".to_string())? + .as_ref(); + match method.r#type.as_ref() { + "Multikey" => CryptoKey::decode_owned(multibase).map_err(|error| error.to_string()), + LEGACY_K256_KIND => legacy_key(KeyCodec::Secp256k1, multibase), + LEGACY_P256_KIND => legacy_key(KeyCodec::P256, multibase), + other => Err(format!("unsupported verification method type {other:?}")), + } +} + +fn legacy_key(codec: KeyCodec, multibase: &str) -> Result, String> { + let encoded = multibase + .strip_prefix('z') + .ok_or_else(|| format!("legacy key {multibase:?} isn't base58btc multibase"))?; + let bytes = bs58::decode(encoded) + .into_vec() + .map_err(|error| error.to_string())?; + Ok(CryptoKey { + codec, + bytes: Cow::Owned(bytes), + }) +} + +fn atproto_signing_key(document: &DidDocument) -> Result, ResolveError> { + let method = document + .verification_method + .as_ref() + .and_then(|methods| { + methods.iter().find(|method| { + let id: &str = method.id.as_ref(); + id.ends_with("#atproto") + && supported_kind(method.r#type.as_ref()) + && method.public_key_multibase.is_some() + }) + }) + .ok_or(ResolveError::MissingSigningKey)?; + method_key(method).map_err(ResolveError::BadSigningKey) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::*; + use bytes::Bytes; + + struct UrlCase { + did: &'static str, + expect: fn(&Result) -> bool, + } + + const URL_CASES: &[UrlCase] = &[ + UrlCase { + did: "did:plc:squid", + expect: |r| matches!(r, Ok(u) if u.as_str() == "https://plc.directory/did:plc:squid"), + }, + UrlCase { + did: "did:web:nel.pet", + expect: |r| matches!(r, Ok(u) if u.as_str() == "https://nel.pet/.well-known/did.json"), + }, + UrlCase { + did: "did:web:nel.pet:repos:squid", + expect: |r| matches!(r, Ok(u) if u.as_str() == "https://nel.pet/repos/squid/did.json"), + }, + UrlCase { + did: "did:web:nel.pet%3A8443", + expect: |r| matches!(r, Ok(u) if u.as_str() == "https://nel.pet:8443/.well-known/did.json"), + }, + UrlCase { + did: "did:key:zabc", + expect: |r| matches!(r, Err(ResolveError::UnsupportedMethod { .. })), + }, + ]; + + #[test] + fn document_url_maps_each_did_method_to_its_document_location() { + URL_CASES.iter().for_each(|case| { + let result = document_url(&did(case.did), &plc()); + assert!((case.expect)(&result), "case {:?} got {result:?}", case.did); + }); + } + + fn body(value: serde_json::Value) -> Bytes { + Bytes::from(serde_json::to_vec(&value).unwrap()) + } + + fn sample_key() -> String { + knot_types::crypto::multikey(0xe7, &sec1(&signer(5))) + } + + fn legacy_document(kind: &str, multibase: &str) -> Bytes { + body(serde_json::json!({ + "id": SQUID, + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [{ + "id": format!("{SQUID}#atproto"), + "type": kind, + "controller": SQUID, + "publicKeyMultibase": multibase + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.oyster.cafe" + }] + })) + } + + #[test] + fn a_complete_document_yields_a_full_identity() { + let body = did_doc(DocSpec { + id: SQUID, + signing: &signer(5), + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }); + let identity = identity_from_document(&did(SQUID), &body).unwrap(); + assert_eq!(identity.pds.url().as_str(), "https://pds.oyster.cafe/"); + assert_eq!(identity.primary_handle().unwrap().as_str(), "nel.pet"); + } + + #[test] + fn the_atproto_verification_method_wins_over_a_decoy_first_key() { + let decoy = sec1(&signer(3)); + let real = sec1(&signer(5)); + let document = body(serde_json::json!({ + "id": SQUID, + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [ + { + "id": format!("{SQUID}#extra"), + "type": "Multikey", + "controller": SQUID, + "publicKeyMultibase": knot_types::crypto::multikey(0xe7, &decoy) + }, + { + "id": format!("{SQUID}#atproto"), + "type": "Multikey", + "controller": SQUID, + "publicKeyMultibase": knot_types::crypto::multikey(0xe7, &real) + } + ], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.oyster.cafe" + }] + })); + let identity = identity_from_document(&did(SQUID), &document).unwrap(); + assert_eq!(identity.signing_key.bytes.as_ref(), real.as_slice()); + assert_ne!(identity.signing_key.bytes.as_ref(), decoy.as_slice()); + } + + #[test] + fn a_legacy_secp256k1_verification_method_resolves() { + let body = did_doc(DocSpec { + id: SQUID, + signing: &signer(5), + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::LegacyK256, + }); + let identity = identity_from_document(&did(SQUID), &body).unwrap(); + assert_eq!( + identity.signing_key.bytes.as_ref(), + sec1(&signer(5)).as_slice() + ); + assert!(matches!(identity.signing_key.codec, KeyCodec::Secp256k1)); + } + + #[test] + fn an_account_shape_rejects_the_same_bytes_under_a_foreign_codec() { + let published = did_doc(DocSpec { + id: SQUID, + signing: &signer(5), + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::LegacyP256, + }); + let error = crate::resolve::document_publishes_account_shape( + &RepoDid::new(SQUID).unwrap(), + &published, + &PublicKeyBytes::from_bytes(sec1(&signer(5))), + &knot_types::KnotServiceUrl::new("https://pds.oyster.cafe").unwrap(), + ) + .unwrap_err(); + assert!( + matches!(error, ResolveError::ExpectedKeyAbsent { .. }), + "the same bytes under a P-256 codec mustn't satisfy the account shape, got {error:?}" + ); + } + + struct DocCase { + name: &'static str, + requested: &'static str, + body: fn() -> Bytes, + expect: fn(&Result) -> bool, + } + + const DOC_CASES: &[DocCase] = &[ + DocCase { + name: "document declares no atproto_pds service", + requested: SQUID, + body: || { + body(serde_json::json!({ + "id": SQUID, + "verificationMethod": [{ + "id": format!("{SQUID}#atproto"), + "type": "Multikey", + "controller": SQUID, + "publicKeyMultibase": sample_key() + }] + })) + }, + expect: |r| matches!(r, Err(ResolveError::MissingPds)), + }, + DocCase { + name: "document declares no signing key", + requested: SQUID, + body: || { + body(serde_json::json!({ + "id": SQUID, + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.oyster.cafe" + }] + })) + }, + expect: |r| matches!(r, Err(ResolveError::MissingSigningKey)), + }, + DocCase { + name: "body isn't valid json", + requested: SQUID, + body: || Bytes::from_static(b"not json"), + expect: |r| matches!(r, Err(ResolveError::Malformed(_))), + }, + DocCase { + name: "legacy key without a multibase prefix", + requested: SQUID, + body: || { + legacy_document( + "EcdsaSecp256k1VerificationKey2019", + &bs58::encode(sec1(&signer(5))).into_string(), + ) + }, + expect: |r| matches!(r, Err(ResolveError::BadSigningKey(_))), + }, + DocCase { + name: "unsupported verification method type", + requested: SQUID, + body: || { + legacy_document( + "JsonWebKey2020", + &format!("z{}", bs58::encode(sec1(&signer(5))).into_string()), + ) + }, + expect: |r| matches!(r, Err(ResolveError::MissingSigningKey)), + }, + DocCase { + name: "document has only a non-atproto method", + requested: SQUID, + body: || { + body(serde_json::json!({ + "id": SQUID, + "verificationMethod": [{ + "id": format!("{SQUID}#extra"), + "type": "Multikey", + "controller": SQUID, + "publicKeyMultibase": knot_types::crypto::multikey(0xe7, &sec1(&signer(3))) + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.oyster.cafe" + }] + })) + }, + expect: |r| matches!(r, Err(ResolveError::MissingSigningKey)), + }, + DocCase { + name: "knot repo did:plc isn't resolvable as an account", + requested: "did:plc:anemone", + body: || { + did_doc(DocSpec { + id: "did:plc:anemone", + signing: &signer(5), + handle: "nel.pet", + pds: "https://knot.oyster.cafe/repo/anemone", + method: MethodKind::None, + }) + }, + expect: |r| matches!(r, Err(ResolveError::MissingSigningKey)), + }, + ]; + + #[test] + fn identity_from_document_rejects_every_underspecified_document() { + DOC_CASES.iter().for_each(|case| { + let result = identity_from_document(&did(case.requested), &(case.body)()); + assert!( + (case.expect)(&result), + "case {:?} got {result:?}", + case.name + ); + }); + } + + #[test] + fn only_a_missing_document_reads_as_an_account_that_no_longer_exists() { + assert!( + ResolveError::Gone { + did: did("did:plc:squid"), + status: HttpStatus::new(404) + } + .is_gone() + ); + assert!( + ResolveError::Unresolvable { + value: "did:web:nel.pet/../..".to_string() + } + .is_gone(), + "a DID that doesn't form a document location can never have published a key" + ); + assert!( + !ResolveError::Malformed("{".to_string()).is_gone(), + "a knot that read a broken document as proof the account stopped publishing keys \ + would delete the keys of everyone behind one bad PDS migration" + ); + assert!( + !ResolveError::RecentlyFailed { + did: did("did:plc:squid") + } + .is_gone(), + "the negative cache stores broken documents as well as missing documents, so \ + reading it back mustn't stand in for either" + ); + assert!( + !ResolveError::Status { + status: HttpStatus::new(404) + } + .is_gone(), + "the handle and pubkey fetches answer 404 too, so the knot can't read a bare 404 \ + as an account that stopped existing" + ); + } + + #[test] + fn a_spindle_answers_at_its_tangled_spindle_service() { + let did = ServiceDid::new("did:web:crow.tngl.boltless.dev").unwrap(); + let crow = br##"{"@context":["https://www.w3.org/ns/did/v1"],"id":"did:web:crow.tngl.boltless.dev","service":[{"id":"#tangled_spindle","serviceEndpoint":"https://crow.tngl.boltless.dev","type":"TangledSpindle"}]}"##; + assert_eq!( + spindle_endpoint(&did, crow).unwrap().as_str(), + "https://crow.tngl.boltless.dev/" + ); + let knot = br##"{"id":"did:web:crow.tngl.boltless.dev","service":[{"id":"#tangled_knot","serviceEndpoint":"https://crow.tngl.boltless.dev","type":"TangledKnot"}]}"##; + assert!(matches!( + spindle_endpoint(&did, knot), + Err(ResolveError::MissingSpindle) + )); + } +} diff --git a/knot2/crates/knot-atproto/src/test_support.rs b/knot2/crates/knot-atproto/src/test_support.rs new file mode 100644 index 0000000000000000000000000000000000000000..45895fc95792ee6387fa4023efb10b65f1ec6e9d --- /dev/null +++ b/knot2/crates/knot-atproto/src/test_support.rs @@ -0,0 +1,197 @@ +use std::borrow::Cow; +use std::sync::{Arc, Mutex}; + +use base64::Engine; +use base64::engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD}; +use bytes::Bytes; +use http::StatusCode; +use k256::ecdsa::{Signature, SigningKey, signature::Signer}; +use knot_runtime::{HttpResponse, K256Signer, ManualClock, SeededEntropy}; +use knot_types::crypto::{KeyCodec, PublicKey as CryptoKey}; +use knot_types::{AccountDid, KnotId, Nsid, OwnerDid, RepoDid, RepoRkey, UnixMicros}; +use serde_json::Value; +use url::Url; + +use crate::{MintNonce, ServiceJwt}; + +pub(crate) const SQUID: &str = "did:plc:squid"; +pub(crate) const LIMPET: &str = "did:plc:limpet"; +pub(crate) const KNOT: &str = "did:web:nel.pet"; +pub(crate) const METHOD: &str = "org.tangled.knot.member"; + +pub(crate) fn did(value: &str) -> AccountDid { + AccountDid::new(value).unwrap() +} + +pub(crate) fn repo_did(value: &str) -> RepoDid { + RepoDid::new(value).unwrap() +} + +pub(crate) fn owner_did(value: &str) -> OwnerDid { + OwnerDid::new(value).unwrap() +} + +pub(crate) fn knot_did(value: &str) -> KnotId { + KnotId::new(value).unwrap() +} + +pub(crate) fn handle(value: &str) -> knot_types::Handle { + knot_types::Handle::new_owned(value).unwrap() +} + +pub(crate) fn member_method() -> Nsid { + Nsid::new_owned(METHOD).unwrap() +} + +pub(crate) fn plc() -> crate::PlcDirectory { + crate::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap() +} + +pub(crate) fn clock() -> ManualClock { + ManualClock::new(UnixMicros::new(1_000_000_000)) +} + +pub(crate) fn signer(seed: u8) -> SigningKey { + SigningKey::from_bytes(&[seed; 32].into()).unwrap() +} + +pub(crate) fn sec1(signing: &SigningKey) -> Vec { + signing + .verifying_key() + .to_encoded_point(true) + .as_bytes() + .to_vec() +} + +pub(crate) fn k256_public(signing: &SigningKey) -> CryptoKey<'static> { + CryptoKey { + codec: KeyCodec::Secp256k1, + bytes: Cow::Owned(sec1(signing)), + } +} + +pub(crate) enum MethodKind { + Multikey, + LegacyK256, + LegacyP256, + None, +} + +pub(crate) struct DocSpec<'a> { + pub id: &'a str, + pub signing: &'a SigningKey, + pub handle: &'a str, + pub pds: &'a str, + pub method: MethodKind, +} + +pub(crate) fn did_doc(spec: DocSpec) -> Bytes { + let raw = sec1(spec.signing); + let legacy = || format!("z{}", bs58::encode(&raw).into_string()); + let method_entry: Option<(&'static str, String)> = match spec.method { + MethodKind::Multikey => Some(("Multikey", knot_types::crypto::multikey(0xe7, &raw))), + MethodKind::LegacyK256 => Some(("EcdsaSecp256k1VerificationKey2019", legacy())), + MethodKind::LegacyP256 => Some(("EcdsaSecp256r1VerificationKey2019", legacy())), + MethodKind::None => None, + }; + let verification_method: Value = match method_entry { + Some((kind, multibase)) => serde_json::json!([{ + "id": format!("{}#atproto", spec.id), + "type": kind, + "controller": spec.id, + "publicKeyMultibase": multibase, + }]), + None => serde_json::json!([]), + }; + let body = serde_json::json!({ + "id": spec.id, + "alsoKnownAs": [format!("at://{}", spec.handle)], + "verificationMethod": verification_method, + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": spec.pds, + }] + }); + Bytes::from(serde_json::to_vec(&body).unwrap()) +} + +pub(crate) fn ssh_line(algo: &str, material: &[u8], comment: &str) -> String { + let ssh_string = |bytes: &[u8]| [&(bytes.len() as u32).to_be_bytes()[..], bytes].concat(); + let blob = [ssh_string(algo.as_bytes()), ssh_string(material)].concat(); + format!("{algo} {} {comment}", STANDARD.encode(blob)) +} + +pub(crate) fn list_body(lines: &[String], cursor: Option<&str>) -> Bytes { + let records: Vec<_> = lines + .iter() + .enumerate() + .map(|(index, line)| { + serde_json::json!({ + "uri": format!("at://{SQUID}/org.tangled.key.sshKey/{index}"), + "value": { "$type": "org.tangled.key.sshKey", "publicKey": line, "name": "k", "createdAt": "2026-06-08T00:00:00Z" } + }) + }) + .collect(); + let cursor_field: Value = cursor.map_or(Value::Null, |value| serde_json::json!(value)); + let body = serde_json::json!({ "records": records, "cursor": cursor_field }); + Bytes::from(serde_json::to_vec(&body).unwrap()) +} + +pub(crate) fn ok(body: Bytes) -> HttpResponse { + status(StatusCode::OK, body) +} + +pub(crate) fn status(status: StatusCode, body: Bytes) -> HttpResponse { + HttpResponse { + status, + headers: http::HeaderMap::new(), + body, + } +} + +pub(crate) fn mint(signing: &SigningKey, claims: &Value) -> ServiceJwt { + mint_with_header(signing, br#"{"alg":"ES256K","typ":"JWT"}"#, claims) +} + +pub(crate) fn mint_with_header(signing: &SigningKey, header: &[u8], claims: &Value) -> ServiceJwt { + let header_b64 = URL_SAFE_NO_PAD.encode(header); + let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims).unwrap()); + let signing_input = format!("{header_b64}.{payload}"); + let signature: Signature = signing.sign(signing_input.as_bytes()); + ServiceJwt::new(format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.to_bytes()) + )) + .expect("Minted token is structurally a JWT") +} + +pub(crate) fn runtime_signer(seed: u64) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed)) +} + +pub(crate) fn entropy(seed: u64) -> SeededEntropy { + SeededEntropy::new(seed) +} + +pub(crate) fn repo_nonce(seed: u64) -> MintNonce { + MintNonce::mint( + &entropy(seed), + &owner_did("did:plc:nel"), + &RepoRkey::new("anemone").unwrap(), + ) +} + +pub(crate) fn member_pointer() -> knot_lexicons::org_tangled::knot::member::Member { + knot_lexicons::org_tangled::knot::member::Member { + created_at: knot_types::Datetime::raw_str("2026-06-11T00:00:00Z"), + extra_data: None, + } +} + +pub(crate) type UrlLog = Arc>>; + +pub(crate) fn recorder() -> (UrlLog, UrlLog) { + let urls: UrlLog = Arc::new(Mutex::new(Vec::new())); + (urls.clone(), urls) +} diff --git a/knot2/crates/knot-atproto/tests/fuzz_smoke.rs b/knot2/crates/knot-atproto/tests/fuzz_smoke.rs new file mode 100644 index 0000000000000000000000000000000000000000..ff710b2f505311f98cfa4d2248812cdb73af9e07 --- /dev/null +++ b/knot2/crates/knot-atproto/tests/fuzz_smoke.rs @@ -0,0 +1,15 @@ +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn the_pubkey_parsers_never_panic(data in proptest::collection::vec(any::(), 0..4096)) { + knot_atproto::fuzz::pubkey(&data); + } + + #[test] + fn the_did_document_decoder_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_atproto::fuzz::did_document(&data); + } +} diff --git a/knot2/crates/knot-atproto/tests/security.rs b/knot2/crates/knot-atproto/tests/security.rs new file mode 100644 index 0000000000000000000000000000000000000000..620ef45837cd6308b63ff551181cb31b06110b08 --- /dev/null +++ b/knot2/crates/knot-atproto/tests/security.rs @@ -0,0 +1,62 @@ +use std::io::{Read, Write}; +use std::net::TcpListener; + +use knot_runtime::{HttpLimits, HttpRequest, HttpTransport, NetworkError, ReqwestHttp}; +use url::Url; + +#[tokio::test] +async fn the_transport_refuses_a_loopback_target() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let http = ReqwestHttp::new(HttpLimits::default()).unwrap(); + let url = Url::parse(&format!("http://127.0.0.1:{port}/")).unwrap(); + let error = http.execute(HttpRequest::get(url)).await.unwrap_err(); + assert!( + matches!(error, NetworkError::Blocked { .. }), + "got {error:?}" + ); + drop(listener); +} + +#[tokio::test] +async fn a_hostname_resolving_only_to_loopback_is_refused_at_dns() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let http = ReqwestHttp::new(HttpLimits::default()).unwrap(); + let url = Url::parse(&format!("https://localhost:{port}/")).unwrap(); + let error = http.execute(HttpRequest::get(url)).await.unwrap_err(); + assert!( + matches!( + error, + NetworkError::Connect(_) | NetworkError::Request(_) | NetworkError::Timeout(_) + ), + "hostname whose only addresses are loopback must fail to connect, got {error:?}" + ); + drop(listener); +} + +#[tokio::test] +async fn a_redirect_to_an_internal_host_is_not_followed() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let _ = stream.read(&mut [0u8; 1024]); + let _ = stream.write_all( + b"HTTP/1.1 302 Found\r\nLocation: http://169.254.169.254/\r\nContent-Length: 0\r\n\r\n", + ); + } + }); + let limits = HttpLimits { + block_private_addresses: false, + ..HttpLimits::default() + }; + let http = ReqwestHttp::new(limits).unwrap(); + let url = Url::parse(&format!("http://{addr}/")).unwrap(); + let response = http.execute(HttpRequest::get(url)).await.unwrap(); + assert_eq!( + response.status.as_u16(), + 302, + "302 is surfaced verbatim, redirect to internal host is never followed" + ); +} diff --git a/knot2/crates/knot-bench/Cargo.toml b/knot2/crates/knot-bench/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..ea1c371ce72472edde45e823867837f412bf0693 --- /dev/null +++ b/knot2/crates/knot-bench/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "knot-bench" +version = "1.20.0" +edition = { workspace = true } +rust-version = { workspace = true } +license = { workspace = true } +publish = false + +[features] +instrument = ["knot-git/instrument", "knot-cob/instrument", "knot-pack/instrument"] + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-pack = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-record = { workspace = true } +knot-index = { workspace = true } +knot-maintenance = { workspace = true } +knot-runtime = { workspace = true } +gix = { workspace = true } +tempfile = { workspace = true } + +[dev-dependencies] +divan = { workspace = true } + +[[bench]] +name = "pack" +harness = false + +[[bench]] +name = "cob" +harness = false + +[[bench]] +name = "coldstart" +harness = false + +[[bench]] +name = "advert" +harness = false + +[[bench]] +name = "projection" +harness = false + +[[bench]] +name = "blame" +harness = false diff --git a/knot2/crates/knot-bench/benches/advert.rs b/knot2/crates/knot-bench/benches/advert.rs new file mode 100644 index 0000000000000000000000000000000000000000..1fb1e8c2cc4746aa218bbc70feb3683f033bac28 --- /dev/null +++ b/knot2/crates/knot-bench/benches/advert.rs @@ -0,0 +1,30 @@ +use divan::Bencher; +use divan::counter::ItemsCount; +use knot_bench::{RefCount, build_many_refs}; + +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); + +const GRADES: &[u32] = &[16, 256, 4096]; + +fn main() { + divan::main(); +} + +#[divan::bench(args = GRADES)] +fn advert_uncached(bencher: Bencher, refs: u32) { + let built = build_many_refs(RefCount::new(refs)); + let count = built.repo().references().unwrap().len(); + bencher + .counter(ItemsCount::new(count)) + .bench_local(|| built.repo().references().unwrap()); +} + +#[divan::bench(args = GRADES)] +fn advert_cached(bencher: Bencher, refs: u32) { + let built = build_many_refs(RefCount::new(refs)); + let count = built.repo().advertised_refs().unwrap().len(); + bencher + .counter(ItemsCount::new(count)) + .bench_local(|| built.repo().advertised_refs().unwrap()); +} diff --git a/knot2/crates/knot-bench/benches/blame.rs b/knot2/crates/knot-bench/benches/blame.rs new file mode 100644 index 0000000000000000000000000000000000000000..c8bec5fbbc9708b1123d4a56fb2998b7d9c23177 --- /dev/null +++ b/knot2/crates/knot-bench/benches/blame.rs @@ -0,0 +1,289 @@ +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +use divan::Bencher; +use divan::counter::{BytesCount, ItemsCount}; +use knot_git::{ + BlameHunk, EntryKind, GitError, Identity, NewCommit, Repo, StagedAction, StagedChange, +}; +use knot_types::{AuthorName, Email, Oid, RepoPath, UnixSeconds}; +use tempfile::TempDir; + +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); + +const FLAT_BYTES: &[usize] = &[64 * 1024, 256 * 1024, 1024 * 1024, 2 * 1024 * 1024]; +const FLAT_LINES: &[usize] = &[10_000, 50_000, 100_000, 150_000]; +const SHORT_LINES: &[usize] = &[100_000, 500_000, 1_000_000]; +const FRAGMENTED_RUNS: &[usize] = &[10_000, 50_000, 100_000]; +const HISTORY_DEPTHS: &[usize] = &[64, 256, 1024, 4096, 16_384]; +const HISTORY_RUNS: &[usize] = &[64, 256, 1024, 4096]; +const PACKED_HISTORY_RUNS: &[usize] = &[1024, 4096]; +const BLAME_DEADLINE: Duration = Duration::from_millis(10); +const CONCURRENT_WORKERS: &[usize] = &[1, 2, 4, 8, 12]; +const LARGE_ANCESTOR_BYTES: &[usize] = &[1024 * 1024, 4 * 1024 * 1024, 16 * 1024 * 1024]; +const BLAME_OBJECT_BYTES: u64 = 1024 * 1024; +const INDEXED_LINE_BYTES: usize = 7; +const PATH: &str = "bench.txt"; + +fn main() { + divan::main(); +} + +struct Fixture { + _dir: TempDir, + repo_path: PathBuf, + tip: Oid, + path: RepoPath, +} + +impl Fixture { + fn blame(&self) -> Vec { + Repo::open(&self.repo_path) + .expect("reopen fixture") + .blame_file(self.tip, &self.path) + .expect("blame fixture") + } + + fn blame_bounded( + &self, + max_object_bytes: u64, + duration: Duration, + ) -> Result, GitError> { + Repo::open(&self.repo_path) + .expect("reopen fixture") + .blame_file_bounded( + self.tip, + &self.path, + max_object_bytes, + Instant::now() + duration, + ) + } + + fn blame_limited(&self, max_object_bytes: u64) -> Result, GitError> { + self.blame_bounded(max_object_bytes, Duration::from_secs(60)) + } + + fn measure(&self, bencher: Bencher) { + bencher.bench(|| self.blame()); + } + + fn assert_runs(&self, expected: usize) { + assert_eq!(self.blame().len(), expected); + } + + fn pack(&self) { + let status = std::process::Command::new("git") + .args([ + "-C", + self.repo_path.to_str().expect("fixture path"), + "gc", + "--quiet", + ]) + .status() + .expect("run git gc"); + assert!(status.success(), "pack fixture"); + } +} + +fn identity(revision: usize) -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000 + revision as i64), + offset_seconds: 0, + } +} + +fn append( + repo: &Repo, + tree: Oid, + parent: Option, + revision: usize, + content: Vec, +) -> (Oid, Oid) { + let path = RepoPath::new(PATH).expect("fixture path"); + let tree = repo + .write_staged_tree( + tree, + &[StagedChange { + path, + action: StagedAction::Put { + content, + kind: EntryKind::Blob, + }, + }], + ) + .expect("write fixture tree"); + let commit = repo + .write_commit(&NewCommit { + tree, + parents: parent.into_iter().collect(), + author: identity(revision), + committer: identity(revision), + message: format!("revision {revision}"), + extra_headers: Vec::new(), + }) + .expect("write fixture commit"); + (tree, commit) +} + +fn fixture_from(contents: impl IntoIterator>) -> Fixture { + let dir = tempfile::tempdir().expect("tempdir"); + let repo_path = dir.path().join("repo.git"); + let repo = Repo::create(&repo_path).expect("create fixture"); + let initial = Oid::from(repo.git().empty_tree().id().detach()); + let (_, tip, _) = contents.into_iter().fold( + (initial, None, 0_usize), + |(tree, parent, revision), content| { + let (tree, commit) = append(&repo, tree, parent, revision, content); + (tree, Some(commit), revision + 1) + }, + ); + Fixture { + _dir: dir, + repo_path, + tip: tip.expect("at least one fixture revision"), + path: RepoPath::new(PATH).expect("fixture path"), + } +} + +fn indexed_lines(lines: usize, changed: impl Fn(usize) -> bool) -> Vec { + (0..lines) + .flat_map(|index| { + let marker = if changed(index) { 'b' } else { 'a' }; + format!("{marker}{index:05x}\n").into_bytes() + }) + .collect() +} + +#[divan::bench(args = FLAT_BYTES)] +fn flat_bytes(bencher: Bencher, bytes: usize) { + let mut content = vec![b'x'; bytes]; + content[bytes - 1] = b'\n'; + let fixture = fixture_from([content]); + fixture.assert_runs(1); + fixture.measure(bencher.counter(BytesCount::new(bytes))); +} + +#[divan::bench(args = FLAT_LINES)] +fn flat_lines(bencher: Bencher, lines: usize) { + let fixture = fixture_from([indexed_lines(lines, |_| false)]); + fixture.assert_runs(1); + fixture.measure(bencher.counter(ItemsCount::new(lines))); +} + +#[divan::bench(args = SHORT_LINES)] +fn short_lines(bencher: Bencher, lines: usize) { + let fixture = fixture_from([vec![b'\n'; lines]]); + fixture.assert_runs(1); + fixture.measure(bencher.counter(ItemsCount::new(lines))); +} + +#[divan::bench(args = FRAGMENTED_RUNS)] +fn fragmented_runs(bencher: Bencher, lines: usize) { + let fixture = fixture_from([ + indexed_lines(lines, |_| false), + indexed_lines(lines, |index| index % 2 == 0), + ]); + fixture.assert_runs(lines); + fixture.measure(bencher.counter(ItemsCount::new(lines))); +} + +fn large_ancestor_fixture(bytes: usize) -> Fixture { + fixture_from([ + "x\n".repeat(bytes / 2).into_bytes(), + b"x\nsentinel\n".to_vec(), + ]) +} + +#[divan::bench(args = LARGE_ANCESTOR_BYTES)] +fn large_ancestor(bencher: Bencher, bytes: usize) { + let fixture = large_ancestor_fixture(bytes); + fixture.assert_runs(2); + fixture.measure(bencher.counter(BytesCount::new(bytes))); +} + +#[divan::bench(args = LARGE_ANCESTOR_BYTES)] +fn limited_large_ancestor(bencher: Bencher, bytes: usize) { + let fixture = large_ancestor_fixture(bytes); + let expect_error = bytes as u64 > BLAME_OBJECT_BYTES; + assert_eq!( + fixture.blame_limited(BLAME_OBJECT_BYTES).is_err(), + expect_error + ); + bencher + .counter(BytesCount::new(bytes)) + .bench(|| fixture.blame_limited(BLAME_OBJECT_BYTES)); +} + +#[divan::bench(args = HISTORY_DEPTHS)] +fn history_depth(bencher: Bencher, commits: usize) { + let revisions = (0..commits).map(|revision| format!("{revision:07x}\nsentinel\n").into_bytes()); + let fixture = fixture_from(revisions); + fixture.assert_runs(2); + fixture.measure(bencher.counter(ItemsCount::new(commits))); +} + +fn history_and_runs_fixture(commits: usize) -> Fixture { + let lines = commits * 2 + 1; + let revisions = std::iter::successors( + Some((0_usize, indexed_lines(lines, |_| false))), + move |(revision, content)| { + if *revision == commits { + return None; + } + let mut next = content.clone(); + next[*revision * 2 * INDEXED_LINE_BYTES] = b'b'; + Some((revision + 1, next)) + }, + ) + .map(|(_, content)| content); + fixture_from(revisions) +} + +#[divan::bench(args = HISTORY_RUNS)] +fn history_and_runs(bencher: Bencher, commits: usize) { + let fixture = history_and_runs_fixture(commits); + fixture.assert_runs(commits * 2); + fixture.measure(bencher.counter(ItemsCount::new(commits))); +} + +#[divan::bench(args = PACKED_HISTORY_RUNS)] +fn packed_history_and_runs(bencher: Bencher, commits: usize) { + let fixture = history_and_runs_fixture(commits); + fixture.pack(); + fixture.assert_runs(commits * 2); + fixture.measure(bencher.counter(ItemsCount::new(commits))); +} + +#[divan::bench(args = PACKED_HISTORY_RUNS)] +fn deadline_packed_history_and_runs(bencher: Bencher, commits: usize) { + let fixture = history_and_runs_fixture(commits); + fixture.pack(); + assert!(matches!( + fixture.blame_bounded(BLAME_OBJECT_BYTES, BLAME_DEADLINE), + Err(GitError::BlameTimedOut) + )); + bencher + .counter(ItemsCount::new(commits)) + .bench(|| fixture.blame_bounded(BLAME_OBJECT_BYTES, BLAME_DEADLINE)); +} + +#[divan::bench(args = CONCURRENT_WORKERS)] +fn concurrent_history_and_runs(bencher: Bencher, workers: usize) { + let commits = 1024; + let fixture = history_and_runs_fixture(commits); + fixture.assert_runs(commits * 2); + bencher.counter(ItemsCount::new(workers)).bench_local(|| { + std::thread::scope(|scope| { + let handles: Vec<_> = (0..workers) + .map(|_| scope.spawn(|| fixture.blame())) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().expect("blame worker").len()) + .sum::() + }) + }); +} diff --git a/knot2/crates/knot-bench/benches/cob.rs b/knot2/crates/knot-bench/benches/cob.rs new file mode 100644 index 0000000000000000000000000000000000000000..d915215e72bf3fc896443307ac51b1680b32cb95 --- /dev/null +++ b/knot2/crates/knot-bench/benches/cob.rs @@ -0,0 +1,27 @@ +use divan::Bencher; +use divan::counter::ItemsCount; +use knot_bench::{ChangeCount, RepoCount, build_linear_cob, build_registry_checkpointed}; + +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); + +const GRADES: &[u32] = &[64, 512, 4096]; +const REPO_GRADES: &[u64] = &[256, 1024, 4096]; + +fn main() { + divan::main(); +} + +#[divan::bench(args = GRADES)] +fn replay(bencher: Bencher, changes: u32) { + let cob = build_linear_cob(ChangeCount::new(changes)); + bencher + .counter(ItemsCount::new(changes as usize)) + .bench_local(|| cob.replay()); +} + +#[divan::bench(args = REPO_GRADES)] +fn registry_write_checkpointed(bencher: Bencher, repos: u64) { + let writer = build_registry_checkpointed(RepoCount::new(repos)); + bencher.bench_local(|| writer.probe()); +} diff --git a/knot2/crates/knot-bench/benches/coldstart.rs b/knot2/crates/knot-bench/benches/coldstart.rs new file mode 100644 index 0000000000000000000000000000000000000000..44cb3d1a30f5a580373cfe627a4547b6e57497d6 --- /dev/null +++ b/knot2/crates/knot-bench/benches/coldstart.rs @@ -0,0 +1,66 @@ +use divan::Bencher; +use divan::counter::ItemsCount; +use knot_bench::{OpenLatency, RepoCount, build_registry, replay_boot}; +use knot_types::RepoDid; + +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); + +const FABRIC_OPEN_MICROS: u64 = 200; + +fn repo_counts() -> Vec { + let Ok(raw) = std::env::var("KNOT_BENCH_REPOS") else { + return vec![1, 1000]; + }; + let counts: Vec = raw + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .map(|entry| { + entry + .parse::() + .unwrap_or_else(|_| panic!("KNOT_BENCH_REPOS entry {entry:?} is not a repo count")) + }) + .collect(); + assert!( + !counts.is_empty(), + "KNOT_BENCH_REPOS is set but lists no repo counts" + ); + counts +} + +fn main() { + divan::main(); +} + +#[divan::bench(args = repo_counts())] +fn boot(bencher: Bencher, repos: u64) { + let registry = build_registry(RepoCount::new(repos)); + bencher + .counter(ItemsCount::new(registry.dids().len())) + .bench_local(|| { + registry.clear_registry_checkpoint(); + registry.index().rebuild().unwrap(); + }); +} + +#[divan::bench(args = repo_counts())] +fn boot_checkpointed(bencher: Bencher, repos: u64) { + let registry = build_registry(RepoCount::new(repos)); + registry.index().rebuild().unwrap(); + bencher + .counter(ItemsCount::new(registry.dids().len())) + .bench_local(|| registry.index().rebuild().unwrap()); +} + +#[divan::bench(args = repo_counts())] +fn boot_fabric(bencher: Bencher, repos: u64) { + let registry = build_registry(RepoCount::new(repos)); + let dids: Vec = registry.dids().to_vec(); + bencher + .counter(ItemsCount::new(dids.len())) + .bench_local(|| { + let index = registry.index(); + replay_boot(&index, &dids, OpenLatency::micros(FABRIC_OPEN_MICROS)); + }); +} diff --git a/knot2/crates/knot-bench/benches/pack.rs b/knot2/crates/knot-bench/benches/pack.rs new file mode 100644 index 0000000000000000000000000000000000000000..b9f95464c74301c14fa9bc715f3f68062b22a9f2 --- /dev/null +++ b/knot2/crates/knot-bench/benches/pack.rs @@ -0,0 +1,297 @@ +use std::time::Duration; + +use divan::Bencher; +use divan::counter::{BytesCount, ItemsCount}; +use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, build_history}; +use knot_git::{Filter, Haves, PackBudget, Repo, Wants}; +use knot_pack::{ + HaveOids, MaxWireBytes, PackLimits, ReceiveCommand, ReceiveGuard, RefDecision, WantOids, + count_expanded, local_pack, receive_pack_guarded, upload_archive, write_expanded, write_pack, +}; +use knot_types::{ObjectCount, Oid}; + +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); + +const GRADES: &[u32] = &[64, 256, 1024]; +const CAP: MaxWireBytes = MaxWireBytes::new(512 * 1024 * 1024); + +fn spec_for(commits: u32) -> HistorySpec { + HistorySpec { + commits: CommitCount::new(commits), + paths: PathCount::new(commits.max(64)), + churn: ChurnCount::new(8), + } +} + +fn pkt(payload: &[u8]) -> Vec { + let mut out = format!("{:04x}", payload.len() + 4).into_bytes(); + out.extend_from_slice(payload); + out +} + +struct AllowAll; + +impl ReceiveGuard for AllowAll { + fn authorize(&self, _staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands.iter().map(|_| RefDecision::Allow).collect() + } +} + +fn main() { + divan::main(); +} + +#[divan::bench(args = GRADES)] +fn select(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let tips = history.tips(); + let count = history + .repo() + .select_pack_objects_filtered( + Wants::new(&tips), + Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send + .len(); + bencher.counter(ItemsCount::new(count)).bench_local(|| { + history + .repo() + .select_pack_objects_filtered( + Wants::new(&tips), + Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + }); +} + +#[divan::bench(args = GRADES)] +fn clone(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let wants = WantOids::new(history.tips()); + let no_haves = HaveOids::default(); + let bytes = local_pack(history.repo(), &wants, &no_haves, CAP) + .unwrap() + .len(); + bencher + .counter(BytesCount::new(bytes)) + .bench_local(|| local_pack(history.repo(), &wants, &no_haves, CAP).unwrap()); +} + +#[divan::bench(args = GRADES)] +fn full_clone_manual(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let tips = history.tips(); + let dir = history.repo().objects_dir(); + let count = history + .repo() + .select_pack_objects_filtered( + Wants::new(&tips), + Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send + .len(); + bencher.counter(ItemsCount::new(count)).bench_local(|| { + let send = history + .repo() + .select_pack_objects_filtered( + Wants::new(&tips), + Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send; + write_pack( + &dir, + send, + None, + &mut std::io::sink(), + history.repo().object_format().kind(), + ) + .unwrap(); + }); +} + +#[divan::bench(args = GRADES)] +fn full_clone_expanding(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let tips = history.tips(); + let dir = history.repo().objects_dir(); + let far = Duration::from_secs(3600); + let roots = history + .repo() + .clone_roots(&tips, PackBudget::unbounded()) + .unwrap(); + let kind = history.repo().object_format().kind(); + let count = count_expanded(&dir, roots, ObjectCount::new(usize::MAX), far, kind) + .unwrap() + .len(); + bencher.counter(ItemsCount::new(count)).bench_local(|| { + let roots = history + .repo() + .clone_roots(&tips, PackBudget::unbounded()) + .unwrap(); + let pack = count_expanded(&dir, roots, ObjectCount::new(usize::MAX), far, kind).unwrap(); + write_expanded(pack, &mut std::io::sink()).unwrap(); + }); +} + +#[divan::bench(args = GRADES)] +fn fetch(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let tips = history.tips(); + let walk = history + .repo() + .rev_walk(Wants::new(&tips), Haves::new(&[])) + .unwrap(); + let haves = HaveOids::new(vec![walk[walk.len() / 2]]); + let wants = WantOids::new(tips); + let bytes = local_pack(history.repo(), &wants, &haves, CAP) + .unwrap() + .len(); + bencher + .counter(BytesCount::new(bytes)) + .bench_local(|| local_pack(history.repo(), &wants, &haves, CAP).unwrap()); +} + +#[divan::bench(args = GRADES)] +fn push(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let tips = history.tips(); + let pack = local_pack( + history.repo(), + &WantOids::new(tips.clone()), + &HaveOids::default(), + CAP, + ) + .unwrap(); + let walk = history + .repo() + .rev_walk(Wants::new(&tips), Haves::new(&[])) + .unwrap(); + let stride = walk.len().max(1) / 8 + 1; + let branch_tips: Vec = walk.iter().step_by(stride).copied().collect(); + let request = build_receive_request(&branch_tips, &pack); + let limits = PackLimits::default(); + bencher + .counter(ItemsCount::new(branch_tips.len())) + .with_inputs(fresh_target) + .bench_local_values(|target| { + receive_pack_guarded( + target.repo(), + &request, + &limits, + &AllowAll, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + }); +} + +#[divan::bench(args = GRADES)] +fn archive(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let request = build_archive_request(history.tip()); + let bytes = upload_archive(history.repo(), &request, knot_git::ArchiveLimit::default()) + .unwrap() + .len(); + bencher.counter(BytesCount::new(bytes)).bench_local(|| { + upload_archive(history.repo(), &request, knot_git::ArchiveLimit::default()).unwrap() + }); +} + +// a tree want isn't advertised or a commit, so ensure_wanted has to prove it reachable: +// `false` takes the object walk, `true` repacks first and checks against the bitmap +#[divan::bench(args = GRADES, consts = [false, true])] +fn tree_want(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + if BITMAP { + let opts = knot_maintenance::Options { + repack_max_objects: ObjectCount::new(1_000_000), + geometric_factor: knot_maintenance::GeometricFactor::full_repack(), + loose_threshold: ObjectCount::new(0), + prune_grace: knot_maintenance::PruneGrace::from_secs(0), + full_repack_interval: knot_maintenance::FullRepackInterval::from_secs(432_000), + reflog_floor: knot_maintenance::ReflogRetention::from_secs(i64::MAX as u64 / 4), + commit_graph: false, + multi_pack_index: false, + bitmap: true, + }; + let now = knot_types::UnixSeconds::new(1_700_000_000); + let report = knot_maintenance::run_repo(history.repo(), now, &opts).unwrap(); + assert!(report.bitmap, "the repack writes a bitmap"); + } + let tree = history + .repo() + .git() + .find_commit(history.tip().object_id()) + .unwrap() + .tree_id() + .unwrap(); + let want = format!("want {}\n", Oid::from(tree.detach())); + let mut body = pkt(b"command=fetch\n"); + body.extend_from_slice(b"0001"); + body.extend(pkt(want.as_bytes())); + body.extend(pkt(b"filter tree:0\n")); + body.extend(pkt(b"done\n")); + body.extend_from_slice(b"0000"); + bencher.bench_local(|| knot_pack::upload_pack(history.repo(), &body).unwrap()); +} + +struct FreshTarget { + _dir: tempfile::TempDir, + repo: Repo, +} + +impl FreshTarget { + fn repo(&self) -> &Repo { + &self.repo + } +} + +fn fresh_target() -> FreshTarget { + let dir = tempfile::tempdir().expect("tempdir"); + let repo = Repo::create(dir.path().join("target.git")).expect("create target"); + FreshTarget { _dir: dir, repo } +} + +fn build_receive_request(branch_tips: &[Oid], pack: &[u8]) -> Vec { + let null = Oid::null().to_hex(); + let commands: Vec> = branch_tips + .iter() + .enumerate() + .map(|(index, tip)| { + let line = format!("{null} {} refs/heads/b{index}", tip.to_hex()); + match index { + 0 => { + let mut payload = line.into_bytes(); + payload.push(0); + payload.extend_from_slice(b"report-status\n"); + pkt(&payload) + } + _ => pkt(format!("{line}\n").as_bytes()), + } + }) + .collect(); + let mut request: Vec = commands.concat(); + request.extend_from_slice(b"0000"); + request.extend_from_slice(pack); + request +} + +fn build_archive_request(tip: Oid) -> Vec { + let mut request = pkt(b"argument --format=tar"); + request.extend_from_slice(&pkt(format!("argument {}", tip.to_hex()).as_bytes())); + request.extend_from_slice(b"0000"); + request +} diff --git a/knot2/crates/knot-bench/benches/projection.rs b/knot2/crates/knot-bench/benches/projection.rs new file mode 100644 index 0000000000000000000000000000000000000000..ef58ab37275204663d8148a82181f86fcdb465e2 --- /dev/null +++ b/knot2/crates/knot-bench/benches/projection.rs @@ -0,0 +1,73 @@ +use divan::Bencher; +use divan::counter::ItemsCount; +use knot_bench::{ + GrantListCount, IssueCount, RepoCount, RepoIndex, build_collaborator_grant_list, + build_issue_tracker, build_registry, +}; +use knot_cob::checkpoint_ref_name; +use knot_types::TypeName; + +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); + +const GRANT_LIST_GRADES: &[u32] = &[64, 512, 4096]; +const REGISTRY_GRADES: &[u64] = &[64, 4096]; +const TRACKER_GRADES: &[u32] = &[100, 1000]; + +fn main() { + divan::main(); +} + +#[divan::bench(args = GRANT_LIST_GRADES)] +fn collaborator_refresh(bencher: Bencher, collaborators: u32) { + let built = build_collaborator_grant_list(GrantListCount::new(collaborators)); + let index = built.index(); + index.rebuild().expect("rebuild"); + index + .ensure_collaborators(built.repo()) + .expect("first materialize"); + bencher + .counter(ItemsCount::new(collaborators as usize)) + .bench_local(|| index.refresh_collaborators(built.repo()).expect("refresh")); +} + +#[divan::bench(args = REGISTRY_GRADES)] +fn resolve(bencher: Bencher, repos: u64) { + let registry = build_registry(RepoCount::new(repos)); + let index = registry.index(); + index.rebuild().expect("rebuild"); + let (owner, rkey) = registry.alias(RepoIndex::new(repos / 2)); + bencher + .counter(ItemsCount::new(1usize)) + .bench_local(|| index.resolve_repo(&owner, &rkey)); +} + +#[divan::bench(args = TRACKER_GRADES)] +fn social_boot(bencher: Bencher, issues: u32) { + let built = build_issue_tracker(IssueCount::new(issues)); + assert!( + built.checkpoints().is_empty(), + "a one-change object should never write a checkpoint" + ); + let edited = built.edit_first_issue_past_stride(); + assert_eq!( + built.checkpoints(), + vec![ + checkpoint_ref_name( + &TypeName::new(knot_record::ticket::TICKET_COLLECTION).expect("ticket type name"), + edited + ) + .expect("checkpoint ref name") + ], + "the checkpoint path oughtta apply per object: the one edited past the stride, and no other" + ); + let index = built.index(); + index.rebuild().expect("rebuild"); + bencher + .counter(ItemsCount::new(2 * issues as usize)) + .bench_local(|| { + index + .refresh_social(built.repo()) + .expect("cold social rebuild") + }); +} diff --git a/knot2/crates/knot-bench/maint-bench.sh b/knot2/crates/knot-bench/maint-bench.sh new file mode 100755 index 0000000000000000000000000000000000000000..3e61fd6146923cd6ab24f25e736f08ce50f830b0 --- /dev/null +++ b/knot2/crates/knot-bench/maint-bench.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# usage: maint-bench.sh [workdir] +# BIN= compares another revision's build; the source repo is never modified. +set -euo pipefail + +src=$(realpath "$1") +work=${2:-$(mktemp -d)} +bin=${BIN:-} +if [ -z "$bin" ]; then + root=$(cd "$(dirname "$0")" && git rev-parse --show-toplevel) + cargo build -q --profile profiling --manifest-path "$root/Cargo.toml" -p knot-bench --bin maint-bench + bin=$root/target/profiling/maint-bench +fi +repo=$work/repo +mkdir -p "$work" + +cow_copy() { + rm -rf "$2" + if [ "$(uname)" = Darwin ]; then /bin/cp -cR "$1" "$2"; else cp -a --reflink=auto "$1" "$2"; fi +} + +run() { + local scenario=$1 err out mem + shift + err=$(mktemp) + if [ "$(uname)" = Darwin ]; then + out=$(/usr/bin/time -l "$bin" "$@" 2>"$err") || out="failed=$? $(grep -v '^ ' "$err" | tail -1)" + mem="footprint=$(awk '/peak memory footprint/ {print $1}' "$err") maxrss=$(awk '/maximum resident/ {print $1}' "$err")" + else + out=$(/usr/bin/env time -f 'maxrss_kb=%M' "$bin" "$@" 2>"$err") || out="failed=$? $(tail -2 "$err" | head -1)" + mem=$(tail -1 "$err") + fi + rm -f "$err" + echo "scenario=$scenario $out $mem" +} + +add_commit() { + local blob tree commit + blob=$(echo "bench $RANDOM $RANDOM $(date +%s)" | git --git-dir="$repo" hash-object -w --stdin) + tree=$(printf '100644 blob %s\tbench\n' "$blob" | git --git-dir="$repo" mktree) + commit=$(GIT_AUTHOR_NAME=bench GIT_AUTHOR_EMAIL=bench@localhost \ + GIT_COMMITTER_NAME=bench GIT_COMMITTER_EMAIL=bench@localhost \ + git --git-dir="$repo" commit-tree "$tree" -p HEAD -m bench) + git --git-dir="$repo" update-ref HEAD "$commit" +} + +cow_copy "$src" "$repo" +rm -rf "$repo"/objects/pack/*.bitmap "$repo"/objects/pack/multi-pack-index* "$repo"/objects/info/commit-graph* +run fresh maintain "$repo" +run settled maintain "$repo" + +run walk clone "$repo" --walk +for threads in 1 4 8; do + run bitmap clone "$repo" --threads "$threads" +done +for behind in 10 1000; do + run walk fetch "$repo" --behind "$behind" + run bitmap fetch "$repo" --behind "$behind" --bitmap +done + +add_commit +run loose maintain "$repo" + +add_commit +git --git-dir="$repo" repack -q +run second-pack maintain "$repo" +run second-pack-bitmap clone "$repo" diff --git a/knot2/crates/knot-bench/src/bin/maint-bench.rs b/knot2/crates/knot-bench/src/bin/maint-bench.rs new file mode 100644 index 0000000000000000000000000000000000000000..9dde19028275bc6ed331ad934ba04f997b0ca77f --- /dev/null +++ b/knot2/crates/knot-bench/src/bin/maint-bench.rs @@ -0,0 +1,154 @@ +use std::error::Error; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use knot_git::{Haves, Repo, Wants}; +use knot_maintenance::{ + FullRepackInterval, GeometricFactor, Options, PruneGrace, ReflogRetention, run_repo, +}; +use knot_types::{ObjectCount, Oid, UnixSeconds}; + +const USAGE: &str = "usage: + maint-bench maintain [--no-bitmap] [--no-commit-graph] [--no-midx] + maint-bench clone [--walk] [--threads N] + maint-bench fetch --behind N [--bitmap] [--threads N] + maint-bench ingest "; + +fn main() -> Result<(), Box> { + let args: Vec = std::env::args().skip(1).collect(); + let (Some(case), Some(path)) = (args.first(), args.get(1)) else { + return Err(USAGE.into()); + }; + let flag = |name: &str| args.iter().any(|arg| arg == name); + let value = |name: &str| -> Result, Box> { + match args.iter().position(|arg| arg == name) { + Some(at) => Ok(Some(args.get(at + 1).ok_or(USAGE)?.parse()?)), + None => Ok(None), + } + }; + let threads = value("--threads")?.unwrap_or(1); + + let started = Instant::now(); + let detail = match case.as_str() { + "maintain" => maintain( + path, + !flag("--no-bitmap"), + !flag("--no-commit-graph"), + !flag("--no-midx"), + )?, + "clone" => select_concurrently(path, threads, |repo| { + let wants: Vec = repo + .references()? + .into_iter() + .map(|record| record.target) + .collect(); + select(repo, &wants, &[], !flag("--walk")) + })?, + "fetch" => { + let behind = value("--behind")?.ok_or(USAGE)?; + select_concurrently(path, threads, |repo| { + let head = Oid::from(repo.git().head_id()?.detach()); + let have = first_parent_ancestor(repo, head, behind)?; + select(repo, &[head], &[have], flag("--bitmap")) + })? + } + "ingest" => { + let repo = Repo::open(path)?; + let pack = std::fs::read(args.get(2).ok_or(USAGE)?)?; + knot_pack::ingest_pack( + &repo.objects_dir(), + &pack, + &knot_pack::PackLimits::default(), + repo.object_format().kind(), + )?; + String::new() + } + _ => return Err(USAGE.into()), + }; + println!( + "case={case} threads={threads} elapsed_ms={} {detail}", + started.elapsed().as_millis() + ); + Ok(()) +} + +fn maintain( + path: &str, + bitmap: bool, + commit_graph: bool, + multi_pack_index: bool, +) -> Result> { + let options = Options { + repack_max_objects: ObjectCount::new(16_000_000), + geometric_factor: GeometricFactor::new(2), + loose_threshold: ObjectCount::new(6700), + prune_grace: PruneGrace::from_secs(1_209_600), + full_repack_interval: FullRepackInterval::from_secs(432_000), + reflog_floor: ReflogRetention::from_secs(7_776_000), + commit_graph, + multi_pack_index, + bitmap, + }; + let now = UnixSeconds::new(SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() as i64); + let report = run_repo(&Repo::open(path)?, now, &options)?; + Ok(format!("report={report:?}")) +} + +fn select_concurrently( + path: &str, + threads: usize, + run: impl Fn(&Repo) -> Result> + Sync, +) -> Result> { + let results: Vec> = std::thread::scope(|scope| { + let handles: Vec<_> = (0..threads) + .map(|_| { + scope.spawn(|| { + Repo::open(path) + .map_err(Into::into) + .and_then(|repo| run(&repo)) + .map_err(|error| error.to_string()) + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().expect("selection thread panicked")) + .collect() + }); + results + .into_iter() + .next() + .expect("at least one thread") + .map_err(Into::into) +} + +fn select( + repo: &Repo, + wants: &[Oid], + haves: &[Oid], + bitmap: bool, +) -> Result> { + if bitmap + && let Some(objects) = + knot_git::reachable_via_bitmap(repo, Wants::new(wants), Haves::new(haves))? + { + return Ok(format!("path=bitmap objects={}", objects.len())); + } + let objects = repo.select_pack_objects(Wants::new(wants), Haves::new(haves))?; + Ok(format!("path=walk objects={}", objects.len())) +} + +fn first_parent_ancestor( + repo: &Repo, + mut commit: Oid, + steps: usize, +) -> Result> { + for _ in 0..steps { + let parent = repo + .git() + .find_commit(commit.object_id())? + .parent_ids() + .next(); + commit = Oid::from(parent.ok_or("history is shorter than --behind")?.detach()); + } + Ok(commit) +} diff --git a/knot2/crates/knot-bench/src/fixtures.rs b/knot2/crates/knot-bench/src/fixtures.rs new file mode 100644 index 0000000000000000000000000000000000000000..61a802b6896668be4dd13593138b8eb5613f81ee --- /dev/null +++ b/knot2/crates/knot-bench/src/fixtures.rs @@ -0,0 +1,808 @@ +use std::path::{Path, PathBuf}; + +use knot_cob::{ChangePayload, CobError, CobHome, CobStore}; +use knot_cobs::{ + CollaboratorsChange, Grant, MembersChange, MembersCob, Registration, RegistryChange, + RepoRegistryCob, register_repo, +}; +use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, +}; +use knot_index::Index; +use knot_runtime::{K256Signer, SeededEntropy}; +use knot_types::{ + AccountDid, AuthorName, CobId, Email, KnotId, Oid, OwnerDid, RecordCollection, RefName, + RepoDid, RepoName, RepoRkey, UnixSeconds, +}; +use tempfile::TempDir; + +macro_rules! at_least_one { + ($($name:ident of $value:ty)+) => {$( + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct $name($value); + + impl $name { + pub fn new(value: $value) -> Self { + Self(value.max(1)) + } + + pub fn get(self) -> $value { + self.0 + } + } + )+}; +} + +at_least_one! { + CommitCount of u32 + PathCount of u32 + ChurnCount of u32 + ChangeCount of u32 + RepoCount of u64 + RefCount of u32 + GrantListCount of u32 + IssueCount of u32 +} + +#[derive(Debug, Clone, Copy)] +pub struct HistorySpec { + pub commits: CommitCount, + pub paths: PathCount, + pub churn: ChurnCount, +} + +const CONTENT_BYTES: usize = 128; +const GENESIS_SECONDS: i64 = 1_700_000_000; + +fn splitmix(seed: u64) -> u64 { + let z = seed.wrapping_add(0x9e37_79b9_7f4a_7c15); + let z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + let z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +knot_types::scalar_newtype! { + struct PathIndex(u32); + struct Revision(u32); + pub struct RepoIndex(u64); +} + +fn blob_content(path_index: PathIndex, revision: Revision) -> Vec { + let seed = splitmix(u64::from(path_index.get()) ^ u64::from(revision.get()).rotate_left(32)); + (0..CONTENT_BYTES) + .scan(seed, |state, _| { + *state = splitmix(*state); + Some((*state & 0xff) as u8) + }) + .collect() +} + +fn path_at(path_index: PathIndex) -> knot_types::RepoPath { + let path_index = path_index.get(); + knot_types::RepoPath::new(format!( + "src/m{:04}/f{:06}.dat", + path_index / 256, + path_index + )) + .expect("generated fixture path is well-formed") +} + +fn identity(revision: Revision) -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(GENESIS_SECONDS + i64::from(revision.get())), + offset_seconds: 0, + } +} + +fn put(path_index: PathIndex, revision: Revision) -> StagedChange { + StagedChange { + path: path_at(path_index), + action: StagedAction::Put { + content: blob_content(path_index, revision), + kind: EntryKind::Blob, + }, + } +} + +fn churn_indices(revision: Revision, spec: HistorySpec) -> impl Iterator { + let span = u64::from(spec.paths.get()); + let base = u64::from(revision.get()).wrapping_mul(u64::from(spec.churn.get())); + (0..spec.churn.get()) + .map(move |offset| PathIndex::new(((base + u64::from(offset)) % span) as u32)) +} + +pub struct BuiltHistory { + _dir: TempDir, + repo: Repo, + tip: Oid, +} + +impl BuiltHistory { + pub fn repo(&self) -> &Repo { + &self.repo + } + pub fn tip(&self) -> Oid { + self.tip + } + pub fn tips(&self) -> Vec { + vec![self.tip] + } +} + +pub fn build_history(spec: HistorySpec) -> BuiltHistory { + let dir = tempfile::tempdir().expect("tempdir"); + let repo = Repo::create(dir.path().join("repo.git")).expect("create repo"); + let tip = write_history(&repo, spec); + BuiltHistory { + _dir: dir, + repo, + tip, + } +} + +pub fn write_history(repo: &Repo, spec: HistorySpec) -> Oid { + let empty_tree = Oid::from(repo.git().empty_tree().id().detach()); + + let genesis_changes: Vec = (0..spec.paths.get()) + .map(|index| put(PathIndex::new(index), Revision::new(0))) + .collect(); + let genesis_tree = repo + .write_staged_tree(empty_tree, &genesis_changes) + .expect("genesis tree"); + let genesis_commit = repo + .write_commit(&NewCommit { + tree: genesis_tree, + parents: Vec::new(), + author: identity(Revision::new(0)), + committer: identity(Revision::new(0)), + message: "genesis".to_string(), + extra_headers: Vec::new(), + }) + .expect("genesis commit"); + + let (_, tip) = (1..spec.commits.get()) + .try_fold( + (genesis_tree, genesis_commit), + |(prev_tree, prev_commit), revision| -> Result<(Oid, Oid), knot_git::GitError> { + let revision = Revision::new(revision); + let changes: Vec = churn_indices(revision, spec) + .map(|index| put(index, revision)) + .collect(); + let tree = repo.write_staged_tree(prev_tree, &changes)?; + let commit = repo.write_commit(&NewCommit { + tree, + parents: vec![prev_commit], + author: identity(revision), + committer: identity(revision), + message: format!("revision {}", revision.get()), + extra_headers: Vec::new(), + })?; + Ok((tree, commit)) + }, + ) + .expect("commit chain"); + + let main = RefName::new("refs/heads/main").expect("main ref name"); + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: tip, + }) + .expect("create main"); + repo.set_head(&main).expect("set head"); + + tip +} + +pub struct BuiltRefs { + _dir: TempDir, + repo: Repo, +} + +impl BuiltRefs { + pub fn repo(&self) -> &Repo { + &self.repo + } +} + +pub fn build_many_refs(count: RefCount) -> BuiltRefs { + let dir = tempfile::tempdir().expect("tempdir"); + let repo = Repo::create(dir.path().join("repo.git")).expect("create repo"); + let tip = write_history( + &repo, + HistorySpec { + commits: CommitCount::new(1), + paths: PathCount::new(1), + churn: ChurnCount::new(1), + }, + ); + (0..count.get()).for_each(|index| { + let name = RefName::new(format!("refs/heads/branch{index:06}")).expect("ref name"); + repo.update_ref(&RefUpdate::Create { name, new: tip }) + .expect("create ref"); + }); + BuiltRefs { _dir: dir, repo } +} + +fn synthetic_account(seed: u64) -> AccountDid { + AccountDid::new(format!("did:plc:acct{seed:012}")).expect("account did") +} + +fn synthetic_repo_did(index: RepoIndex) -> RepoDid { + RepoDid::new(format!("did:plc:repo{:012}", index.get())).expect("repo did") +} + +fn registry_owner() -> OwnerDid { + OwnerDid::new("did:plc:nel").expect("owner did") +} + +fn knot_home() -> CobHome { + CobHome::from(&KnotId::new("did:web:knot.nel.pet").expect("knot did")) +} + +fn registration(index: RepoIndex) -> Registration { + let rkey = format!("repo{:012}", index.get()); + Registration { + owner: registry_owner(), + rkey: RepoRkey::new(&rkey).expect("rkey"), + name: RepoName::new(&rkey).expect("repo name"), + repo: synthetic_repo_did(index), + created_at: UnixSeconds::new(GENESIS_SECONDS + index.get() as i64), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + + retired_at: None, + } +} + +fn grant(seed: u64) -> Grant { + Grant { + subject: synthetic_account(seed), + added_by: synthetic_account(0), + created_at: UnixSeconds::new(GENESIS_SECONDS), + } +} + +pub struct BuiltRegistry { + _dir: TempDir, + meta_path: PathBuf, + layout: Layout, + registry: CobId, + dids: Vec, +} + +impl BuiltRegistry { + pub fn index(&self) -> Index { + Index::new(&self.meta_path, self.layout.clone()) + } + pub fn dids(&self) -> &[RepoDid] { + &self.dids + } + pub fn alias(&self, index: RepoIndex) -> (OwnerDid, RepoRkey) { + let reg = registration(index); + (reg.owner, reg.rkey) + } + pub fn clear_registry_checkpoint(&self) { + let meta = Repo::open(&self.meta_path).expect("openable meta path"); + let name = knot_cob::checkpoint_ref_name(&RegistryChange::type_name(), self.registry) + .expect("checkpoint ref name"); + if let Some(old) = meta.find_ref(&name).expect("able to read checkpoint ref") { + meta.update_ref(&RefUpdate::Delete { name, old }) + .expect("able to delete checkpoint ref"); + } + } +} + +fn seed_members(meta_path: &Path, signer: &dyn knot_runtime::Signer) { + let meta = Repo::open(meta_path).expect("open meta"); + let store = CobStore::new(&meta); + store + .create( + &knot_home(), + &knot_cobs::MembersChange::Add(grant(1)), + signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("seed members"); +} + +pub fn build_registry(repos: RepoCount) -> BuiltRegistry { + let dir = tempfile::tempdir().expect("tempdir"); + let meta_path = dir.path().join("meta.git"); + Repo::create(&meta_path).expect("create meta"); + let layout = Layout::new(dir.path().join("repos")); + let signer = K256Signer::generate(&SeededEntropy::new(7)); + + seed_members(&meta_path, &signer); + + let meta = Repo::open(&meta_path).expect("open meta"); + let store = CobStore::new(&meta); + let registry_object = store + .create( + &knot_home(), + &RegistryChange::Register(registration(RepoIndex::new(0))), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("create registry") + .object; + (1..repos.get()).for_each(|index| { + let index = RepoIndex::new(index); + store + .update( + &knot_home(), + registry_object, + &RegistryChange::Register(registration(index)), + &signer, + UnixSeconds::new(GENESIS_SECONDS + index.get() as i64), + ) + .expect("register repo"); + }); + + let dids: Vec = (0..repos.get()) + .map(|index| synthetic_repo_did(RepoIndex::new(index))) + .collect(); + + dids.iter().enumerate().for_each(|(index, did)| { + let git = layout.create(did).expect("create repo dir"); + let collab = CobStore::new(&git); + collab + .create( + &CobHome::from(did), + &CollaboratorsChange::Add(grant(index as u64 + 2)), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("seed collaborator"); + }); + + BuiltRegistry { + _dir: dir, + meta_path, + layout, + registry: registry_object, + dids, + } +} + +pub struct BuiltGrantList { + _dir: TempDir, + meta_path: PathBuf, + layout: Layout, + repo: RepoDid, +} + +impl BuiltGrantList { + pub fn index(&self) -> Index { + Index::new(&self.meta_path, self.layout.clone()) + } + pub fn repo(&self) -> &RepoDid { + &self.repo + } +} + +pub fn build_collaborator_grant_list(collaborators: GrantListCount) -> BuiltGrantList { + let dir = tempfile::tempdir().expect("tempdir"); + let meta_path = dir.path().join("meta.git"); + Repo::create(&meta_path).expect("create meta"); + let layout = Layout::new(dir.path().join("repos")); + let repo = synthetic_repo_did(RepoIndex::new(0)); + let signer = K256Signer::generate(&SeededEntropy::new(9)); + + seed_members(&meta_path, &signer); + + let git = layout.create(&repo).expect("create repo dir"); + let store = CobStore::new(&git); + let home = CobHome::from(&repo); + let object = store + .create( + &home, + &CollaboratorsChange::Add(grant(2)), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("seed collaborator") + .object; + (1..collaborators.get()).for_each(|index| { + store + .update( + &home, + object, + &CollaboratorsChange::Add(grant(u64::from(index) + 2)), + &signer, + UnixSeconds::new(GENESIS_SECONDS + i64::from(index)), + ) + .expect("add collaborator"); + }); + + BuiltGrantList { + _dir: dir, + meta_path, + layout, + repo, + } +} + +pub struct BuiltTracker { + _dir: TempDir, + meta_path: PathBuf, + layout: Layout, + repo: RepoDid, + first: CobId, + signer: K256Signer, +} + +impl BuiltTracker { + pub fn index(&self) -> Index { + Index::new(&self.meta_path, self.layout.clone()) + } + pub fn repo(&self) -> &RepoDid { + &self.repo + } + pub fn checkpoints(&self) -> Vec { + let git = self.layout.open(&self.repo).expect("open repo"); + git.git() + .references() + .expect("refs") + .prefixed(knot_cob::CHECKPOINTS_PREFIX) + .expect("checkpoint refs") + .filter_map(Result::ok) + .map(|reference| { + RefName::new(reference.name().as_bstr().to_string()).expect("checkpoint ref name") + }) + .collect() + } + pub fn edit_first_issue_past_stride(&self) -> CobId { + let git = self.layout.open(&self.repo).expect("open repo"); + let store = CobStore::new(&git); + let owner = synthetic_account(0); + let cx = owner_context(&owner, &self.repo); + let stride = ::SNAPSHOT_STRIDE.get() as u32; + (1..=stride).fold(ticket_body(&owner, Revision::new(0)).cid(), |from, step| { + let produces = ticket_body(&owner, Revision::new(step)).cid(); + let change = knot_cobs::TicketChange::Edit { + editing: knot_cobs::Editing { + author: owner.clone(), + from, + produces, + edited_at: UnixSeconds::new(GENESIS_SECONDS + i64::from(step)), + }, + body: ticket(knot_record::ticket::TicketState::Open), + }; + let approved = knot_cob::Approved::signed_by(&cx, knot_cob::Materialized::bare(change)) + .expect("the owner signs its own change"); + store + .append_approved::( + &CobHome::from(&self.repo), + self.first, + &approved, + &self.signer, + UnixSeconds::new(GENESIS_SECONDS + i64::from(step)), + ) + .expect("edit the first issue"); + produces + }); + self.first + } +} + +fn owner_context<'a>(owner: &'a AccountDid, repo: &'a RepoDid) -> knot_cob::WriteContext<'a> { + knot_cob::WriteContext::new( + owner, + repo, + knot_types::ContributionPermission::new( + knot_types::Blocked::No, + knot_types::RepoRole::Owner, + knot_types::ContributionPolicy::Collaborators, + ), + ) +} + +fn ticket_body(owner: &AccountDid, revision: Revision) -> knot_types::RecordBody { + use knot_record::Prose; + use knot_record::ticket::{TicketRecord, TicketState, TicketTitle}; + let prose = Prose::new( + TicketTitle::new(format!("revision {}", revision.get())).expect("a numbered title"), + None, + ); + TicketRecord::new( + prose, + TicketState::Open, + None, + None, + None, + None, + UnixSeconds::new(GENESIS_SECONDS), + owner.clone(), + ) + .expect("ticket record") + .body() + .expect("ticket record is dag-cbor") +} + +fn ticket_address(collection: RecordCollection, index: u32) -> knot_types::RecordAddress { + knot_types::RecordAddress::new( + collection, + knot_types::RecordRkey::minted(knot_types::Tid::from_time( + 1_700_000_000_000_000 + u64::from(index), + 0, + )), + ) +} + +fn ticket(state: knot_record::ticket::TicketState) -> knot_cobs::Ticket { + knot_cobs::Ticket { + state, + tangled_labels: None, + assignees: None, + } +} + +fn open_social( + store: &CobStore, + cx: &knot_cob::WriteContext<'_>, + signer: &dyn knot_runtime::Signer, + creation: knot_cobs::Creation, + body: B, + record: knot_cob::Record, +) -> CobId { + let change = knot_cobs::SocialChange::Open { creation, body }; + let approved = knot_cob::Approved::signed_by(cx, knot_cob::Materialized { change, record }) + .expect("the owner signs its own change"); + store + .create_approved::>( + &CobHome::from(cx.repo()), + &approved, + signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("open social object") + .object +} + +pub fn build_issue_tracker(issues: IssueCount) -> BuiltTracker { + let dir = tempfile::tempdir().expect("tempdir"); + let meta_path = dir.path().join("meta.git"); + let repo = synthetic_repo_did(RepoIndex::new(0)); + let layout = Layout::new(dir.path().join("repos")); + let signer = K256Signer::generate(&SeededEntropy::new(9)); + seed_members(&meta_path, &signer); + + let git = layout.create(&repo).expect("create repo dir"); + let store = CobStore::new(&git); + let owner = synthetic_account(0); + let cx = owner_context(&owner, &repo); + let ticket_body = ticket_body(&owner, Revision::new(0)); + let opened: Vec = (0..issues.get()) + .map(|index| { + let number = knot_types::SubjectNumber::new(index + 1).expect("nonzero"); + let creation = |address: knot_types::RecordAddress, + parent: Option, + produces| { + knot_cobs::Creation { + address, + author: owner.clone(), + number, + parent, + created_at: UnixSeconds::new(GENESIS_SECONDS), + produces, + } + }; + let address = ticket_address(knot_record::ticket::ticket_collection(), index); + open_social( + &store, + &cx, + &signer, + creation(address.clone(), None, ticket_body.cid()), + ticket(knot_record::ticket::TicketState::Closed), + knot_cob::Record::Bare, + ) + }) + .collect(); + + BuiltTracker { + _dir: dir, + meta_path, + layout, + repo, + first: opened[0], + signer, + } +} + +pub struct BuiltRegistryWriter { + _dir: TempDir, + meta_path: PathBuf, + object: CobId, + signer: K256Signer, +} + +impl BuiltRegistryWriter { + pub fn probe(&self) { + let meta = Repo::open(&self.meta_path).expect("reopen meta"); + let store = CobStore::new(&meta); + register_repo( + &store, + &knot_home(), + self.object, + registration(RepoIndex::new(0)), + &self.signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("idempotent re-register refreshes the registry"); + } + + pub fn full_replay(&self) { + let meta = Repo::open(&self.meta_path).expect("reopen meta"); + let store = CobStore::new(&meta); + store + .get::(self.object) + .expect("full replay of the registry"); + } +} + +pub fn build_registry_checkpointed(repos: RepoCount) -> BuiltRegistryWriter { + let dir = tempfile::tempdir().expect("tempdir"); + let meta_path = dir.path().join("meta.git"); + Repo::create(&meta_path).expect("create meta"); + let signer = K256Signer::generate(&SeededEntropy::new(13)); + + let meta = Repo::open(&meta_path).expect("open meta"); + let store = CobStore::new(&meta); + let object = store + .create( + &knot_home(), + &RegistryChange::Register(registration(RepoIndex::new(0))), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("create registry") + .object; + (1..repos.get()).for_each(|index| { + let index = RepoIndex::new(index); + register_repo( + &store, + &knot_home(), + object, + registration(index), + &signer, + UnixSeconds::new(GENESIS_SECONDS + index.get() as i64), + ) + .expect("register repo"); + }); + + BuiltRegistryWriter { + _dir: dir, + meta_path, + object, + signer, + } +} + +pub struct BuiltMembersWriter { + _dir: TempDir, + meta_path: PathBuf, + object: CobId, + signer: K256Signer, +} + +impl BuiltMembersWriter { + pub fn probe(&self) { + let meta = Repo::open(&self.meta_path).expect("reopen meta"); + let store = CobStore::new(&meta); + store + .update_maybe_checkpointed::( + &knot_home(), + self.object, + &self.signer, + UnixSeconds::new(GENESIS_SECONDS), + |list| { + Ok(if list.contains(&grant(0).subject) { + None + } else { + Some(MembersChange::Add(grant(0))) + }) + }, + ) + .expect("idempotent re-add refreshes the bounded suffix"); + } + + pub fn full_replay(&self) { + let meta = Repo::open(&self.meta_path).expect("reopen meta"); + let store = CobStore::new(&meta); + store + .get::(self.object) + .expect("full replay of members"); + } +} + +pub fn build_members_checkpointed(members: GrantListCount) -> BuiltMembersWriter { + let dir = tempfile::tempdir().expect("tempdir"); + let meta_path = dir.path().join("meta.git"); + Repo::create(&meta_path).expect("create meta"); + let signer = K256Signer::generate(&SeededEntropy::new(17)); + + let meta = Repo::open(&meta_path).expect("open meta"); + let store = CobStore::new(&meta); + let object = store + .create( + &knot_home(), + &MembersChange::Add(grant(0)), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("create members") + .object; + (1..members.get()).for_each(|index| { + store + .update_with_checkpointed::( + &knot_home(), + object, + &signer, + UnixSeconds::new(GENESIS_SECONDS + i64::from(index)), + |_| Ok(MembersChange::Add(grant(u64::from(index) + 1))), + ) + .expect("add member"); + }); + + BuiltMembersWriter { + _dir: dir, + meta_path, + object, + signer, + } +} + +pub struct BuiltLinearCob { + _dir: TempDir, + repo: Repo, + object: CobId, +} + +impl BuiltLinearCob { + pub fn replay(&self) -> usize { + let store = CobStore::new(&self.repo); + store + .get::(self.object) + .expect("full replay of registry") + .state() + .len() + } +} + +pub fn build_linear_cob(changes: ChangeCount) -> BuiltLinearCob { + let dir = tempfile::tempdir().expect("tempdir"); + let meta_path = dir.path().join("meta.git"); + Repo::create(&meta_path).expect("create meta"); + let signer = K256Signer::generate(&SeededEntropy::new(11)); + + let meta = Repo::open(&meta_path).expect("open meta"); + let store = CobStore::new(&meta); + let object = store + .create( + &knot_home(), + &RegistryChange::Register(registration(RepoIndex::new(0))), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("create registry") + .object; + (1..changes.get()).for_each(|index| { + store + .update( + &knot_home(), + object, + &RegistryChange::Register(registration(RepoIndex::new(u64::from(index)))), + &signer, + UnixSeconds::new(GENESIS_SECONDS + i64::from(index)), + ) + .expect("append change"); + }); + + BuiltLinearCob { + _dir: dir, + repo: Repo::open(&meta_path).expect("reopen meta"), + object, + } +} diff --git a/knot2/crates/knot-bench/src/latency.rs b/knot2/crates/knot-bench/src/latency.rs new file mode 100644 index 0000000000000000000000000000000000000000..3de004efc9742528030aea112f77bd4b801c9d2a --- /dev/null +++ b/knot2/crates/knot-bench/src/latency.rs @@ -0,0 +1,32 @@ +use std::time::Duration; + +use knot_index::Index; +use knot_types::RepoDid; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OpenLatency(Duration); + +impl OpenLatency { + pub fn micros(value: u64) -> Self { + Self(Duration::from_micros(value)) + } + pub fn zero() -> Self { + Self(Duration::ZERO) + } + fn stall(self) { + if !self.0.is_zero() { + std::thread::sleep(self.0); + } + } +} + +pub fn replay_boot(index: &Index, repos: &[RepoDid], per_open: OpenLatency) { + index.refresh_members().expect("refresh members"); + index.refresh_registry().expect("refresh registry"); + repos.iter().for_each(|repo| { + per_open.stall(); + index + .refresh_collaborators(repo) + .expect("fabric-boot repo must materialize, or the bench measures garbage"); + }); +} diff --git a/knot2/crates/knot-bench/src/lib.rs b/knot2/crates/knot-bench/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..01756c406b2207937818f1447ca0435c4eb32ee0 --- /dev/null +++ b/knot2/crates/knot-bench/src/lib.rs @@ -0,0 +1,12 @@ +mod fixtures; +mod latency; + +pub use fixtures::{ + BuiltGrantList, BuiltHistory, BuiltLinearCob, BuiltMembersWriter, BuiltRefs, BuiltRegistry, + BuiltRegistryWriter, BuiltTracker, ChangeCount, ChurnCount, CommitCount, GrantListCount, + HistorySpec, IssueCount, PathCount, RefCount, RepoCount, RepoIndex, + build_collaborator_grant_list, build_history, build_issue_tracker, build_linear_cob, + build_many_refs, build_members_checkpointed, build_registry, build_registry_checkpointed, + write_history, +}; +pub use latency::{OpenLatency, replay_boot}; diff --git a/knot2/crates/knot-bench/tests/boot.rs b/knot2/crates/knot-bench/tests/boot.rs new file mode 100644 index 0000000000000000000000000000000000000000..aba35ab1df381692912e58f67f6f657e8aa89f47 --- /dev/null +++ b/knot2/crates/knot-bench/tests/boot.rs @@ -0,0 +1,26 @@ +use knot_bench::{RepoCount, build_registry}; +use knot_index::Resolved; + +#[test] +fn the_registry_fixture_boots_without_materializing_collaborators() { + let registry = build_registry(RepoCount::new(8)); + let index = registry.index(); + index.rebuild().unwrap(); + + registry.dids().iter().for_each(|repo| { + assert_eq!( + index.effective_collaborator(repo, &knot_types::AccountDid::new("did:plc:nel").unwrap()), + Resolved::Warming, + "rebuild mustn't materialize any collaborator COB, so every repo reads warming until first access" + ); + }); + + let first = ®istry.dids()[0]; + index.ensure_collaborators(first).unwrap(); + assert!( + !index + .effective_collaborator(first, &knot_types::AccountDid::new("did:plc:nel").unwrap()) + .is_warming(), + "a repo materializes on first access" + ); +} diff --git a/knot2/crates/knot-bench/tests/gate.rs b/knot2/crates/knot-bench/tests/gate.rs new file mode 100644 index 0000000000000000000000000000000000000000..d168fce7e462dd39d2452fb41567ac75c4b6ef2c --- /dev/null +++ b/knot2/crates/knot-bench/tests/gate.rs @@ -0,0 +1,86 @@ +#![cfg(feature = "instrument")] + +use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, build_history}; +use knot_git::instrument::measure; +use knot_git::{Filter, Haves, PackBudget, Wants}; +use knot_pack::upload_pack; +use knot_types::Oid; + +fn gate_spec() -> HistorySpec { + HistorySpec { + commits: CommitCount::new(32), + paths: PathCount::new(64), + churn: ChurnCount::new(4), + } +} + +const SELECTION_ODB_READS: u64 = 129; +const SERVER_FETCH_ODB_READS: u64 = 2; + +fn pkt(payload: &[u8]) -> Vec { + let mut out = format!("{:04x}", payload.len() + 4).into_bytes(); + out.extend_from_slice(payload); + out +} + +fn fetch_request(want: Oid) -> Vec { + let mut request = pkt(b"command=fetch\n"); + request.extend_from_slice(b"0001"); + request.extend(pkt(format!("want {}\n", want.to_hex()).as_bytes())); + request.extend(pkt(b"done\n")); + request.extend_from_slice(b"0000"); + request +} + +#[test] +fn a_single_selection_walk_has_an_exact_odb_read_count() { + let history = build_history(gate_spec()); + let tips = history.tips(); + let (_selection, reads) = measure(|| { + history + .repo() + .select_pack_objects_filtered( + Wants::new(&tips), + Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + }); + assert_eq!( + reads.get(), + SELECTION_ODB_READS, + "the selection walk made {} explicit object loads through Repo::load_object. \ + The gate pins this at {SELECTION_ODB_READS}. A lower count means the single-pass \ + commit-walk fix landed. A higher count is a regression. The counter records loads \ + on the calling thread only. gix's internal rev-walk decodes never reach load_object \ + and stay uncounted. Update the constant only when the change is deliberate", + reads.get() + ); +} + +#[test] +fn the_upload_pack_server_path_has_an_exact_odb_read_count() { + let history = build_history(gate_spec()); + let tips = history.tips(); + let walk = history + .repo() + .rev_walk(Wants::new(&tips), Haves::new(&[])) + .unwrap(); + let hidden = walk + .iter() + .copied() + .find(|commit| *commit != history.tip()) + .expect("multi-commit history has a non-tip commit to want"); + let request = fetch_request(hidden); + let (_response, reads) = measure(|| upload_pack(history.repo(), &request).unwrap()); + assert_eq!( + reads.get(), + SERVER_FETCH_ODB_READS, + "server fetch made {} Repo::load_object calls, gate pins {SERVER_FETCH_ODB_READS}. \ + No-haves fetch enumerates inside gix, so only the want check and root peel reach \ + load_object. A count near the old manual-walk figure means the full-clone fast path \ + stopped firing", + reads.get() + ); +} diff --git a/knot2/crates/knot-bench/tests/members_gate.rs b/knot2/crates/knot-bench/tests/members_gate.rs new file mode 100644 index 0000000000000000000000000000000000000000..01f60f8382555788dde99feab49d6b5cca7604f1 --- /dev/null +++ b/knot2/crates/knot-bench/tests/members_gate.rs @@ -0,0 +1,44 @@ +#![cfg(feature = "instrument")] + +use knot_bench::{GrantListCount, build_members_checkpointed}; +use knot_cob::instrument::measure; + +#[test] +fn a_checkpointed_member_write_reads_a_bounded_object_set() { + let small = build_members_checkpointed(GrantListCount::new(512)); + let large = build_members_checkpointed(GrantListCount::new(1024)); + + let (_, full_small) = measure(|| small.full_replay()); + let (_, full_large) = measure(|| large.full_replay()); + let (_, write_small) = measure(|| small.probe()); + let (_, write_large) = measure(|| large.probe()); + + assert_eq!( + full_small.get(), + 2 * 512, + "a full replay reads every change's commit and payload, so twice the member count" + ); + assert_eq!( + full_large.get(), + 2 * 1024, + "the full replay the checkpoint replaces is linear in member count" + ); + assert_eq!( + write_small.get(), + write_large.get(), + "a checkpointed member write reads only the bounded suffix, the same object count at \ + 512 members as at 1024" + ); + assert!( + write_large.get() <= 2 * 256, + "per-write read set is bounded by twice the snapshot stride, not the member count, \ + was {}", + write_large.get() + ); + assert!( + write_large.get() < full_large.get() / 3, + "checkpoint cuts per-write object reads far below the full replay, {} vs {}", + write_large.get(), + full_large.get() + ); +} diff --git a/knot2/crates/knot-bench/tests/registry_gate.rs b/knot2/crates/knot-bench/tests/registry_gate.rs new file mode 100644 index 0000000000000000000000000000000000000000..fe2b83e57db5503ef71ec76d5512506640166823 --- /dev/null +++ b/knot2/crates/knot-bench/tests/registry_gate.rs @@ -0,0 +1,44 @@ +#![cfg(feature = "instrument")] + +use knot_bench::{RepoCount, build_registry_checkpointed}; +use knot_cob::instrument::measure; + +#[test] +fn a_checkpointed_registry_write_reads_a_bounded_object_set() { + let small = build_registry_checkpointed(RepoCount::new(512)); + let large = build_registry_checkpointed(RepoCount::new(1024)); + + let (_, full_small) = measure(|| small.full_replay()); + let (_, full_large) = measure(|| large.full_replay()); + let (_, write_small) = measure(|| small.probe()); + let (_, write_large) = measure(|| large.probe()); + + assert_eq!( + full_small.get(), + 2 * 512, + "a full replay reads every change's commit and payload, so twice the change count" + ); + assert_eq!( + full_large.get(), + 2 * 1024, + "the full replay the checkpoint replaces is linear in repo count" + ); + assert_eq!( + write_small.get(), + write_large.get(), + "checkpointed write reads only the bounded suffix, the same object count at 512 \ + repos as at 1024" + ); + assert!( + write_large.get() <= 2 * 256, + "per-write read set is bounded by twice the snapshot stride, not the repo count, \ + was {}", + write_large.get() + ); + assert!( + write_large.get() < full_large.get() / 3, + "checkpoint cuts per-write object reads far below the full replay, {} vs {}", + write_large.get(), + full_large.get() + ); +} diff --git a/knot2/crates/knot-cache/Cargo.toml b/knot2/crates/knot-cache/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..e2b467d0e85a49cd036c4d75b5e4f9ed461423a7 --- /dev/null +++ b/knot2/crates/knot-cache/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "knot-cache" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-runtime = { workspace = true } +knot-types = { workspace = true } +moka = { workspace = true } diff --git a/knot2/crates/knot-cache/src/expiring.rs b/knot2/crates/knot-cache/src/expiring.rs new file mode 100644 index 0000000000000000000000000000000000000000..c86725f41a5edb09e16e9f5b11a38c34f89ab746 --- /dev/null +++ b/knot2/crates/knot-cache/src/expiring.rs @@ -0,0 +1,388 @@ +use std::collections::HashMap; +use std::collections::hash_map::Entry as Slot; +use std::hash::Hash; +use std::sync::Mutex; + +use knot_types::UnixMicros; + +knot_types::scalar_newtype! { + pub struct GroupQuota(usize); + pub struct TotalQuota(usize); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Quotas { + pub per_group: GroupQuota, + pub total: TotalQuota, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Rejected { + Total, + Group, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Admitted { + Inserted, + Occupied(V), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Occupancy { + Keep, + Extend, +} + +struct Entry { + group: G, + value: V, + expires_at: UnixMicros, +} + +struct Inner { + entries: HashMap>, + group_counts: HashMap, +} + +pub struct Expiring { + inner: Mutex>, + quotas: Quotas, +} + +impl Expiring +where + K: Eq + Hash + Clone, + G: Eq + Hash + Clone, +{ + pub fn new(quotas: Quotas) -> Self { + Self { + inner: Mutex::new(Inner { + entries: HashMap::new(), + group_counts: HashMap::new(), + }), + quotas, + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub fn prune(&self, now: UnixMicros) -> Vec { + prune_locked(&mut self.lock(), now) + } + + pub fn admit( + &self, + key: K, + group: G, + value: V, + expires_at: UnixMicros, + now: UnixMicros, + ) -> Result, Rejected> + where + V: Clone + PartialEq, + { + self.enter(key, group, value, expires_at, now, Occupancy::Keep) + } + + pub fn admit_or_renew( + &self, + key: K, + group: G, + value: V, + expires_at: UnixMicros, + now: UnixMicros, + ) -> Result, Rejected> + where + V: Clone + PartialEq, + { + self.enter(key, group, value, expires_at, now, Occupancy::Extend) + } + + fn enter( + &self, + key: K, + group: G, + value: V, + expires_at: UnixMicros, + now: UnixMicros, + occupied: Occupancy, + ) -> Result, Rejected> + where + V: Clone + PartialEq, + { + let mut inner = self.lock(); + let key = match inner.entries.entry(key) { + Slot::Occupied(mut held) if held.get().expires_at > now => { + let entry = held.get_mut(); + if occupied == Occupancy::Extend && entry.value == value { + entry.expires_at = expires_at; + } + return Ok(Admitted::Occupied(entry.value.clone())); + } + Slot::Occupied(held) => { + let (key, entry) = held.remove_entry(); + release_group(&mut inner.group_counts, &entry.group); + key + } + Slot::Vacant(free) => free.into_key(), + }; + // This is lazy on purpose! + // Pruning traverses every entry so we shouldn't do it on every + // admission, better to do this O(1) quota check. + if self.rejection(&inner, &group).is_some() { + prune_locked(&mut inner, now); + } + match self.rejection(&inner, &group) { + Some(rejected) => Err(rejected), + None => { + *inner.group_counts.entry(group.clone()).or_insert(0) += 1; + inner.entries.insert( + key, + Entry { + group, + value, + expires_at, + }, + ); + Ok(Admitted::Inserted) + } + } + } + + fn rejection(&self, inner: &Inner, group: &G) -> Option { + let held = inner.group_counts.get(group).copied().unwrap_or(0); + match ( + held >= self.quotas.per_group.get(), + inner.entries.len() >= self.quotas.total.get(), + ) { + (true, _) => Some(Rejected::Group), + (_, true) => Some(Rejected::Total), + (false, false) => None, + } + } + + pub fn remove(&self, key: &K) -> Option { + remove_locked(&mut self.lock(), key) + } + + pub fn get(&self, key: &K, now: UnixMicros) -> Option + where + V: Clone, + { + let inner = self.lock(); + inner + .entries + .get(key) + .filter(|entry| entry.expires_at > now) + .map(|entry| entry.value.clone()) + } + + pub fn len(&self) -> usize { + self.lock().entries.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn group_len(&self, group: &G) -> usize { + self.lock().group_counts.get(group).copied().unwrap_or(0) + } +} + +fn prune_locked(inner: &mut Inner, now: UnixMicros) -> Vec +where + K: Eq + Hash + Clone, + G: Eq + Hash, +{ + let expired: Vec = inner + .entries + .iter() + .filter(|(_, entry)| entry.expires_at <= now) + .map(|(key, _)| key.clone()) + .collect(); + expired.iter().for_each(|key| { + remove_locked(inner, key); + }); + expired +} + +fn remove_locked(inner: &mut Inner, key: &K) -> Option +where + K: Eq + Hash, + G: Eq + Hash, +{ + let entry = inner.entries.remove(key)?; + release_group(&mut inner.group_counts, &entry.group); + Some(entry.value) +} + +fn release_group(counts: &mut HashMap, group: &G) { + if let Some(count) = counts.get_mut(group) { + *count = count.saturating_sub(1); + if *count == 0 { + counts.remove(group); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at(micros: u64) -> UnixMicros { + UnixMicros::new(micros) + } + + fn store(per_group: usize, total: usize) -> Expiring<&'static str, &'static str, u8> { + Expiring::new(Quotas { + per_group: GroupQuota::new(per_group), + total: TotalQuota::new(total), + }) + } + + #[test] + fn a_slot_stays_occupied_until_it_expires_and_is_then_free_to_take_over() { + let store = store(8, 8); + assert_eq!( + store.admit("uni", "nel", 1, at(100), at(0)), + Ok(Admitted::Inserted) + ); + assert_eq!( + store.admit("uni", "olaren", 2, at(900), at(50)), + Ok(Admitted::Occupied(1)), + "the caller decides whether an occupied slot is a replay or a renewal" + ); + assert_eq!(store.get(&"uni", at(50)), Some(1)); + assert_eq!( + store.get(&"uni", at(100)), + None, + "a replay guard must keep the original expiry or a replayed token renews its own window" + ); + assert_eq!( + store.admit("uni", "olaren", 2, at(300), at(100)), + Ok(Admitted::Inserted), + "expiry is inclusive so a slot expiring exactly now is available" + ); + assert_eq!( + store.group_len(&"nel"), + 0, + "the takeover releases the old group's count" + ); + assert_eq!(store.group_len(&"olaren"), 1); + } + + #[test] + fn admit_or_renew_extends_only_the_current_holder() { + let store = store(8, 8); + store + .admit_or_renew("uni", "nel", 1, at(100), at(0)) + .unwrap(); + assert_eq!( + store.admit_or_renew("uni", "olaren", 2, at(900), at(50)), + Ok(Admitted::Occupied(1)) + ); + assert_eq!( + store.get(&"uni", at(150)), + None, + "a caller that isn't the holder mustn't extend the lease" + ); + store + .admit_or_renew("uni", "nel", 1, at(300), at(200)) + .unwrap(); + assert_eq!( + store.admit_or_renew("uni", "nel", 1, at(600), at(250)), + Ok(Admitted::Occupied(1)) + ); + assert_eq!( + store.get(&"uni", at(500)), + Some(1), + "renewing under the lock that read the entry leaves no window for a release \ + to drop the slot between the read and the renewal" + ); + } + + #[test] + fn each_quota_bounds_its_own_scope_and_names_itself_when_it_refuses() { + let store = store(1, 1); + store.admit("uni", "nel", 1, at(100), at(0)).unwrap(); + assert_eq!( + store.admit("kelp", "nel", 2, at(100), at(0)), + Err(Rejected::Group), + "a rejection names the group limit first because the caller can act on its own quota" + ); + assert_eq!( + store.admit("kelp", "olaren", 2, at(100), at(0)), + Err(Rejected::Total), + "a distinct group has its own budget but still shares the total" + ); + assert_eq!( + store.admit("kelp", "olaren", 2, at(400), at(200)), + Ok(Admitted::Inserted), + "admit prunes the expired entry and takes the slot it freed" + ); + } + + #[test] + fn releasing_a_slot_by_expiry_or_by_hand_frees_its_group_budget() { + let store = store(2, 8); + store.admit("uni", "nel", 7, at(100), at(0)).unwrap(); + store.admit("kelp", "nel", 8, at(500), at(0)).unwrap(); + assert_eq!(store.prune(at(200)), vec!["uni"]); + assert_eq!(store.len(), 1); + assert_eq!( + store.group_len(&"nel"), + 1, + "pruning decrements the group count rather than rebuilding it" + ); + assert_eq!(store.remove(&"kelp"), Some(8)); + assert_eq!(store.remove(&"kelp"), None); + assert_eq!( + store.group_len(&"nel"), + 0, + "removing the last entry of a group frees its budget" + ); + assert_eq!( + store.admit("whelk", "nel", 9, at(600), at(200)), + Ok(Admitted::Inserted) + ); + } + + #[test] + fn concurrent_admissions_at_the_quota_keep_group_counts_equal_to_what_is_stored() { + let store = std::sync::Arc::new(Expiring::new(Quotas { + per_group: GroupQuota::new(2), + total: TotalQuota::new(3), + })); + let keys = ["uni", "kelp", "whelk", "clam", "conch", "limpet"]; + let groups = ["nel", "olaren"]; + let threads: Vec<_> = (0..8u64) + .map(|thread| { + let store = std::sync::Arc::clone(&store); + std::thread::spawn(move || { + (0..2_000u64).for_each(|round| { + let key = keys[(thread + round) as usize % keys.len()]; + let group = groups[(thread + round) as usize % groups.len()]; + let now = at(round * 10); + let _ = store.admit(key, group, 1u8, at(round * 10 + 40), now); + store.prune(now); + }); + }) + }) + .collect(); + threads + .into_iter() + .for_each(|thread| thread.join().unwrap()); + let counted: usize = groups.iter().map(|group| store.group_len(group)).sum(); + assert_eq!( + counted, + store.len(), + "a group count higher than what is stored locks its group out of every later admission" + ); + } +} diff --git a/knot2/crates/knot-cache/src/lib.rs b/knot2/crates/knot-cache/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..73eede639c8d2c9177584ca3658c5c7a0dd33a4b --- /dev/null +++ b/knot2/crates/knot-cache/src/lib.rs @@ -0,0 +1,647 @@ +mod expiring; + +pub use expiring::{Admitted, Expiring, GroupQuota, Quotas, Rejected, TotalQuota}; + +use std::collections::{BTreeMap, HashMap}; +use std::future::Future; +use std::hash::Hash; +use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::time::Duration; + +use knot_runtime::Clock; +use knot_types::UnixMicros; + +knot_types::scalar_newtype! { + pub struct EntryCount(u64); + pub struct Weight(u64); +} + +pub trait Cache: Send + Sync { + fn get(&self, key: &K) -> Option; + fn insert(&self, key: K, value: V); + fn invalidate(&self, key: &K); + fn invalidate_all(&self); + fn entry_count(&self) -> EntryCount; + fn weighted_size(&self) -> Weight; +} + +pub struct Untimed; + +impl Clock for Untimed { + fn now_unix_micros(&self) -> UnixMicros { + UnixMicros::new(0) + } +} + +pub struct Moka { + inner: moka::sync::Cache, +} + +impl Moka +where + K: Hash + Eq + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, +{ + pub fn by_count(max_entries: EntryCount) -> Self { + Self { + inner: moka::sync::Cache::builder() + .max_capacity(max_entries.get()) + .build(), + } + } + + pub fn by_weight(max_weight: Weight, weigh: F) -> Self + where + F: Fn(&V) -> Weight + Send + Sync + 'static, + { + Self { + inner: moka::sync::Cache::builder() + .max_capacity(max_weight.get()) + .weigher(move |_key: &K, value: &V| weigh(value).get().min(u32::MAX as u64) as u32) + .build(), + } + } + + pub fn get_or_try_insert_with(&self, key: K, init: F) -> Result> + where + F: FnOnce() -> Result, + E: Send + Sync + 'static, + { + self.inner.try_get_with(key, init) + } +} + +impl Cache for Moka +where + K: Hash + Eq + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, +{ + fn get(&self, key: &K) -> Option { + self.inner.get(key) + } + + fn insert(&self, key: K, value: V) { + self.inner.insert(key, value); + } + + fn invalidate(&self, key: &K) { + self.inner.invalidate(key); + } + + fn invalidate_all(&self) { + self.inner.invalidate_all(); + } + + fn entry_count(&self) -> EntryCount { + EntryCount::new(self.inner.entry_count()) + } + + fn weighted_size(&self) -> Weight { + Weight::new(self.inner.weighted_size()) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct Tick(u64); + +impl Tick { + fn issue(&mut self) -> Tick { + let issued = *self; + self.0 = self.0.saturating_add(1); + issued + } +} + +struct Node { + value: V, + tick: Tick, + weight: u64, + expires_at: Option, +} + +struct LruInner { + by_key: HashMap>, + order: BTreeMap, + next: Tick, + total_weight: u64, +} + +type Weigh = Arc Weight + Send + Sync>; + +pub struct Lru { + inner: Mutex>, + max_entries: Option, + max_weight: Option, + weigh: Option>, + ttl: Option, + clock: C, +} + +impl Lru +where + K: Hash + Eq + Clone, + V: Clone, +{ + pub fn by_count(max_entries: EntryCount) -> Self { + Self::build(Some(max_entries), None, None, None, Untimed) + } + + pub fn by_weight(max_weight: Weight, weigh: F) -> Self + where + F: Fn(&V) -> Weight + Send + Sync + 'static, + { + Self::build(None, Some(max_weight), Some(Arc::new(weigh)), None, Untimed) + } +} + +impl Lru +where + K: Hash + Eq + Clone, + V: Clone, + C: Clock, +{ + pub fn by_count_with_ttl(max_entries: EntryCount, ttl: Duration, clock: C) -> Self { + Self::build(Some(max_entries), None, None, Some(ttl), clock) + } + + pub fn by_weight_with_ttl(max_weight: Weight, ttl: Duration, clock: C, weigh: F) -> Self + where + F: Fn(&V) -> Weight + Send + Sync + 'static, + { + Self::build( + None, + Some(max_weight), + Some(Arc::new(weigh)), + Some(ttl), + clock, + ) + } + + pub fn with_entry_cap(mut self, max_entries: EntryCount) -> Self { + self.max_entries = Some(max_entries); + self + } + + fn build( + max_entries: Option, + max_weight: Option, + weigh: Option>, + ttl: Option, + clock: C, + ) -> Self { + Self { + inner: Mutex::new(LruInner { + by_key: HashMap::new(), + order: BTreeMap::new(), + next: Tick(0), + total_weight: 0, + }), + max_entries, + max_weight, + weigh, + ttl, + clock, + } + } + + fn weight_of(&self, value: &V) -> u64 { + self.weigh.as_ref().map_or(1, |weigh| weigh(value).get()) + } + + fn lock(&self) -> std::sync::MutexGuard<'_, LruInner> { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} + +impl Cache for Lru +where + K: Hash + Eq + Clone + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, + C: Clock, +{ + fn get(&self, key: &K) -> Option { + let mut guard = self.lock(); + let inner = &mut *guard; + let (stale, expires_at) = inner + .by_key + .get(key) + .map(|node| (node.tick, node.expires_at))?; + if expires_at.is_some_and(|at| at.get() <= self.clock.now_unix_micros().get()) { + if let Some(node) = inner.by_key.remove(key) { + inner.total_weight = inner.total_weight.saturating_sub(node.weight); + } + inner.order.remove(&stale); + return None; + } + let fresh = inner.next.issue(); + inner.order.remove(&stale); + inner.order.insert(fresh, key.clone()); + let node = inner.by_key.get_mut(key).expect("hit is still present"); + node.tick = fresh; + Some(node.value.clone()) + } + + fn insert(&self, key: K, value: V) { + let expires_at = self.ttl.map(|ttl| { + UnixMicros::new( + self.clock + .now_unix_micros() + .get() + .saturating_add(ttl.as_micros() as u64), + ) + }); + let weight = self.weight_of(&value); + let mut guard = self.lock(); + let inner = &mut *guard; + if let Some(previous) = inner.by_key.remove(&key) { + inner.order.remove(&previous.tick); + inner.total_weight = inner.total_weight.saturating_sub(previous.weight); + } + let fresh = inner.next.issue(); + inner.order.insert(fresh, key.clone()); + inner.total_weight = inner.total_weight.saturating_add(weight); + inner.by_key.insert( + key, + Node { + value, + tick: fresh, + weight, + expires_at, + }, + ); + while self + .max_entries + .is_some_and(|max| inner.by_key.len() as u64 > max.get()) + || self + .max_weight + .is_some_and(|max| inner.total_weight > max.get()) + { + let Some((_, evicted)) = inner.order.pop_first() else { + break; + }; + if let Some(node) = inner.by_key.remove(&evicted) { + inner.total_weight = inner.total_weight.saturating_sub(node.weight); + } + } + } + + fn invalidate(&self, key: &K) { + let mut guard = self.lock(); + if let Some(node) = guard.by_key.remove(key) { + guard.order.remove(&node.tick); + guard.total_weight = guard.total_weight.saturating_sub(node.weight); + } + } + + fn invalidate_all(&self) { + let mut guard = self.lock(); + guard.by_key.clear(); + guard.order.clear(); + guard.total_weight = 0; + } + + fn entry_count(&self) -> EntryCount { + EntryCount::new(self.lock().by_key.len() as u64) + } + + fn weighted_size(&self) -> Weight { + Weight::new(self.lock().total_weight) + } +} + +pub struct Noop; + +impl Cache for Noop +where + K: Send + Sync + 'static, + V: Send + Sync + 'static, +{ + fn get(&self, _key: &K) -> Option { + None + } + + fn insert(&self, _key: K, _value: V) {} + + fn invalidate(&self, _key: &K) {} + + fn invalidate_all(&self) {} + + fn entry_count(&self) -> EntryCount { + EntryCount::new(0) + } + + fn weighted_size(&self) -> Weight { + Weight::new(0) + } +} + +pub struct Filled { + pub value: V, + pub fresh: bool, +} + +type DeterministicHasher = std::hash::BuildHasherDefault; + +pub trait AsyncCache: Send + Sync { + fn get(&self, key: &K) -> impl Future> + Send; + fn entry_count(&self) -> EntryCount; +} + +pub struct MokaFuture { + inner: moka::future::Cache, +} + +impl MokaFuture +where + K: Hash + Eq + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, +{ + pub fn by_count(max_entries: EntryCount) -> Self { + Self { + inner: moka::future::Cache::builder() + .max_capacity(max_entries.get()) + .build_with_hasher(DeterministicHasher::default()), + } + } + + pub fn by_weight(max_weight: Weight, weigh: F) -> Self + where + F: Fn(&V) -> Weight + Send + Sync + 'static, + { + Self { + inner: moka::future::Cache::builder() + .max_capacity(max_weight.get()) + .weigher(move |_key: &K, value: &V| weigh(value).get().min(u32::MAX as u64) as u32) + .build_with_hasher(DeterministicHasher::default()), + } + } + + pub async fn get_or_fill_if(&self, key: K, refill_if: P, fill: Fut) -> Filled + where + Fut: Future + Send, + P: FnMut(&V) -> bool + Send, + { + let entry = self + .inner + .entry(key) + .or_insert_with_if(fill, refill_if) + .await; + Filled { + fresh: entry.is_fresh(), + value: entry.into_value(), + } + } + + pub async fn run_pending_tasks(&self) { + self.inner.run_pending_tasks().await; + } +} + +impl AsyncCache for MokaFuture +where + K: Hash + Eq + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, +{ + fn get(&self, key: &K) -> impl Future> + Send { + self.inner.get(key) + } + + fn entry_count(&self) -> EntryCount { + EntryCount::new(self.inner.entry_count()) + } +} + +pub trait Reclaimable: Send + Sync { + fn footprint(&self) -> Weight; + fn reclaim(&self); +} + +impl Reclaimable for Moka +where + K: Hash + Eq + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, +{ + fn footprint(&self) -> Weight { + self.weighted_size() + } + + fn reclaim(&self) { + self.invalidate_all(); + } +} + +impl Reclaimable for Lru +where + K: Hash + Eq + Clone + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, + C: Clock, +{ + fn footprint(&self) -> Weight { + self.weighted_size() + } + + fn reclaim(&self) { + self.invalidate_all(); + } +} + +#[derive(Default)] +struct Registry { + caches: Mutex>>, +} + +impl Registry { + fn lock(&self) -> std::sync::MutexGuard<'_, Vec>> { + self.caches + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn register(&self, cache: &Arc) { + let erased: Arc = cache.clone(); + let weak = Arc::downgrade(&erased); + let mut caches = self.lock(); + caches.retain(|entry| entry.strong_count() > 0); + caches.push(weak); + } + + fn reclaim_largest(&self) -> Option { + let largest = self + .lock() + .iter() + .filter_map(Weak::upgrade) + .max_by_key(|cache| cache.footprint().get())?; + let freed = largest.footprint(); + (freed.get() > 0).then(|| { + largest.reclaim(); + freed + }) + } +} + +fn global_registry() -> &'static Registry { + static REGISTRY: OnceLock = OnceLock::new(); + REGISTRY.get_or_init(Registry::default) +} + +pub fn register(cache: &Arc) { + global_registry().register(cache); +} + +pub fn reclaim_largest() -> Option { + global_registry().reclaim_largest() +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use knot_runtime::ManualClock; + + use super::*; + + #[test] + fn a_no_ttl_cache_retains_until_invalidated() { + let cache: Moka = Moka::by_count(EntryCount::new(16)); + cache.insert(1, 9); + cache.insert(2, 10); + assert_eq!(cache.get(&1), Some(9)); + cache.invalidate(&1); + assert_eq!(cache.get(&1), None); + assert_eq!(cache.get(&2), Some(10)); + } + + #[test] + fn the_same_clock_sequence_yields_the_same_hits_and_misses() { + let run = || { + let clock = Arc::new(ManualClock::new(UnixMicros::new(0))); + let cache: Lru = Lru::by_count_with_ttl( + EntryCount::new(16), + Duration::from_secs(5), + Arc::clone(&clock), + ); + cache.insert(1, 100); + let before = cache.get(&1); + clock.advance(Duration::from_secs(6)); + let after = cache.get(&1); + (before, after) + }; + assert_eq!(run(), run()); + } + + #[test] + fn get_or_try_insert_with_serves_the_first_value_without_recomputing() { + let cache: Moka = Moka::by_count(EntryCount::new(16)); + let first = cache.get_or_try_insert_with(1, || Ok::(7)); + assert_eq!(first.unwrap(), 7); + let second = cache.get_or_try_insert_with(1, || Ok::(99)); + assert_eq!(second.unwrap(), 7); + } + + #[test] + fn get_or_try_insert_with_propagates_the_error_and_stores_nothing() { + let cache: Moka = Moka::by_count(EntryCount::new(16)); + let failed = cache.get_or_try_insert_with(1, || Err::(9)); + assert_eq!(*failed.unwrap_err(), 9); + assert_eq!(cache.get(&1), None); + } + + #[test] + fn the_lru_evicts_the_least_recently_used_key() { + let cache: Lru = Lru::by_count(EntryCount::new(3)); + cache.insert(0, 0); + cache.insert(1, 1); + cache.insert(2, 2); + assert_eq!(cache.get(&0), Some(0)); + cache.insert(3, 3); + assert_eq!( + cache.get(&1), + None, + "the least recently used key is evicted" + ); + assert_eq!(cache.get(&0), Some(0), "the touched key survives"); + assert_eq!(cache.get(&3), Some(3)); + assert_eq!(cache.entry_count().get(), 3); + } + + #[test] + fn the_weighted_lru_evicts_oldest_until_it_fits_the_byte_budget() { + let cache: Lru> = Lru::by_weight(Weight::new(8), |value: &Vec| { + Weight::new(value.len() as u64) + }); + cache.insert(0, vec![0u8; 5]); + cache.insert(1, vec![0u8; 5]); + assert_eq!(cache.get(&0), None, "the oldest entry is evicted to fit"); + assert_eq!(cache.get(&1), Some(vec![0u8; 5])); + assert_eq!(cache.weighted_size().get(), 5); + } + + #[test] + fn the_entry_cap_bounds_a_flood_of_zero_weight_entries() { + let cache: Lru> = Lru::by_weight(Weight::new(1_000_000), |value: &Vec| { + Weight::new(value.len() as u64) + }) + .with_entry_cap(EntryCount::new(4)); + (0..64).for_each(|nonce| cache.insert(nonce, Vec::new())); + assert!(cache.entry_count().get() <= 4); + } + + #[test] + fn the_lru_expires_entries_on_the_injected_clock() { + let clock = Arc::new(ManualClock::new(UnixMicros::new(0))); + let cache: Lru = Lru::by_count_with_ttl( + EntryCount::new(8), + Duration::from_secs(1), + Arc::clone(&clock), + ); + cache.insert(1, 5); + assert_eq!(cache.get(&1), Some(5)); + clock.advance(Duration::from_secs(2)); + assert_eq!(cache.get(&1), None); + assert_eq!(cache.entry_count().get(), 0); + } + + #[test] + fn the_governor_sheds_the_largest_registered_cache() { + let weigh = |value: &Vec| Weight::new(value.len() as u64); + let small: Arc>> = Arc::new(Lru::by_weight(Weight::new(1_000_000), weigh)); + let big: Arc>> = Arc::new(Lru::by_weight(Weight::new(1_000_000), weigh)); + small.insert(0, vec![0u8; 10]); + big.insert(0, vec![0u8; 100]); + let registry = Registry::default(); + registry.register(&small); + registry.register(&big); + let freed = registry + .reclaim_largest() + .expect("a registered cache is shed"); + assert_eq!(freed.get(), 100, "the largest footprint is reclaimed"); + assert_eq!(big.entry_count().get(), 0, "the largest cache is emptied"); + assert_eq!( + small.entry_count().get(), + 1, + "the smaller cache is untouched" + ); + } + + #[test] + fn a_registered_cache_with_nothing_to_reclaim_is_not_shed() { + let registry = Registry::default(); + let cache: Arc>> = Arc::new(Lru::by_count(EntryCount::new(8))); + registry.register(&cache); + assert_eq!( + registry.reclaim_largest(), + None, + "an empty registered cache reports nothing to shed" + ); + } + + #[test] + fn the_noop_cache_never_retains() { + let cache: &dyn Cache = &Noop; + cache.insert(1, 2); + assert_eq!(cache.get(&1), None); + assert_eq!(cache.entry_count().get(), 0); + } +} diff --git a/knot2/crates/knot-cob/Cargo.toml b/knot2/crates/knot-cob/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..d4463093f0c442d742022174809b28a1d4592934 --- /dev/null +++ b/knot2/crates/knot-cob/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "knot-cob" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[features] +instrument = [] + +[dependencies] +knot-types = { workspace = true } +knot-runtime = { workspace = true } +knot-git = { workspace = true } +knot-resource = { workspace = true } +tracing = { workspace = true } +gix = { workspace = true } +gix-hash = { workspace = true } +k256 = { workspace = true } +serde = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +proptest = { workspace = true } diff --git a/knot2/crates/knot-cob/src/backend.rs b/knot2/crates/knot-cob/src/backend.rs new file mode 100644 index 0000000000000000000000000000000000000000..4fa9d001b236827a20b17c424bb134a4d7469473 --- /dev/null +++ b/knot2/crates/knot-cob/src/backend.rs @@ -0,0 +1,541 @@ +use std::collections::BTreeMap; + +use gix::bstr::{BStr, ByteSlice as _}; +use gix::date::Time; +use gix::objs::FindHeader as _; +use knot_git::Repo; +use knot_runtime::{Signature, Signer}; +use knot_types::{ + ActorId, ChangeId, CobId, MAX_RECORD_BYTES, Oid, RecordBody, RefName, TypeName, UnixSeconds, +}; + +use crate::change::{Change, CobHome, Payload, Record}; +use crate::error::CobError; +use crate::graph::ChangeGraph; +use crate::object::HistoryModel; + +pub const COBS_PREFIX: &str = "refs/cobs/"; +pub const CHECKPOINTS_PREFIX: &str = "refs/cob-checkpoints/"; +const TYPE_HEADER: &str = "cob-type"; +const SIG_HEADER: &str = "cob-sig"; +const AUTHOR_HEADER: &str = "cob-author"; +const PAYLOAD_BLOB: &str = "payload"; +const RECORD_BLOB: &str = "record"; +const TOMBSTONE_BLOB: &str = "tombstone"; +pub(crate) const MAX_GRAPH_CHANGES: usize = 100_000; +const REBUILD_CHANGE_BYTES: u64 = 2560; +const REBUILD_FOLD_DIVISOR: u64 = 4; +// two bazillion +const REBUILD_UNMEASURED_CHANGES: usize = 2_000_000; + +pub(crate) fn rebuild_change_limit_for(available: Option) -> usize { + let derived = match available { + Some(available) => { + usize::try_from(available.get() / REBUILD_FOLD_DIVISOR / REBUILD_CHANGE_BYTES) + .unwrap_or(usize::MAX) + } + None => REBUILD_UNMEASURED_CHANGES, + }; + derived.max(MAX_GRAPH_CHANGES) +} + +pub(crate) fn rebuild_graph_limit() -> usize { + rebuild_change_limit_for(knot_resource::available_bytes()) +} + +pub fn cob_ref_name(type_name: &TypeName, object: CobId) -> Result { + prefixed_ref_name(COBS_PREFIX, type_name, object) +} + +pub fn checkpoint_ref_name(type_name: &TypeName, object: CobId) -> Result { + prefixed_ref_name(CHECKPOINTS_PREFIX, type_name, object) +} + +fn prefixed_ref_name( + prefix: &str, + type_name: &TypeName, + object: CobId, +) -> Result { + let raw = format!("{prefix}{}/{}", type_name.as_str(), object.oid().to_hex()); + RefName::new(raw.as_str()).map_err(|_| CobError::RefName(raw)) +} + +pub fn parse_cob_ref(refname: &RefName) -> Option<(TypeName, CobId)> { + let (nsid, oid) = refname + .as_str() + .strip_prefix(COBS_PREFIX)? + .rsplit_once('/')?; + let type_name = TypeName::new(nsid).ok()?; + let object = Oid::from_hex(oid).ok().map(CobId::new)?; + Some((type_name, object)) +} + +pub(crate) fn resolve_tip( + repo: &Repo, + type_name: &TypeName, + object: CobId, +) -> Result, CobError> { + let name = cob_ref_name(type_name, object)?; + Ok(repo.find_ref(&name)?) +} + +pub(crate) fn list_objects(repo: &Repo, type_name: &TypeName) -> Result, CobError> { + Ok(repo + .references()? + .into_iter() + .filter_map(|record| parse_cob_ref(&record.name)) + .filter(|(found, _)| found == type_name) + .map(|(_, object)| object) + .collect()) +} + +pub(crate) fn list_cobs(repo: &Repo) -> Result, CobError> { + Ok(repo + .references()? + .into_iter() + .filter_map(|record| parse_cob_ref(&record.name)) + .collect()) +} + +pub fn migrate_legacy_refs(repo: &Repo) -> Result { + let renamed = repo + .references()? + .into_iter() + .filter_map(|record| { + let refname = record.name.as_str(); + let modern = [COBS_PREFIX, CHECKPOINTS_PREFIX] + .into_iter() + .find_map(|prefix| { + let rest = refname.strip_prefix(prefix)?; + let (legacy, oid) = rest.rsplit_once('/')?; + let modern = crate::modernized_type_of(&TypeName::new(legacy).ok()?)?; + RefName::new(format!("{prefix}{}/{}", modern.as_str(), oid)).ok() + })?; + Some((record.name, modern, record.target)) + }) + .collect::>(); + let count = renamed.len(); + repo.rename_references(renamed).map_err(CobError::from)?; + Ok(count) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Blob { + Payload, + Record, +} + +impl Blob { + const fn name(self) -> &'static str { + match self { + Self::Payload => PAYLOAD_BLOB, + Self::Record => RECORD_BLOB, + } + } +} + +pub(crate) fn check_blob_size(bytes: usize, blob: Blob) -> Result<(), CobError> { + if bytes > MAX_RECORD_BYTES { + Err(CobError::ChangeTooLarge { + blob: blob.name(), + bytes, + limit: MAX_RECORD_BYTES, + }) + } else { + Ok(()) + } +} + +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) fn write_change( + home: &CobHome, + repo: &Repo, + type_name: &TypeName, + payload: &[u8], + parents: &[ChangeId], + object: Option, + signer: &dyn Signer, + timestamp: UnixSeconds, +) -> Result { + write_change_with_record( + home, + repo, + type_name, + payload, + &Record::Bare, + parents, + object, + signer, + timestamp, + ) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn write_change_with_record( + home: &CobHome, + repo: &Repo, + type_name: &TypeName, + payload: &[u8], + record: &Record, + parents: &[ChangeId], + object: Option, + signer: &dyn Signer, + timestamp: UnixSeconds, +) -> Result { + check_blob_size(payload.len(), Blob::Payload)?; + if let Some(bytes) = record.bytes() { + check_blob_size(bytes.len(), Blob::Record)?; + } + let git = repo.git(); + let payload_oid = git + .write_blob(payload) + .map_err(|error| CobError::Write(error.to_string()))? + .detach(); + let record_oid = record + .bytes() + .map(|bytes| { + git.write_blob(bytes) + .map(|oid| oid.detach()) + .map_err(|error| CobError::Write(error.to_string())) + }) + .transpose()?; + let tombstone_oid = record + .is_tombstone() + .then(|| { + git.write_blob([]) + .map(|oid| oid.detach()) + .map_err(|error| CobError::Write(error.to_string())) + }) + .transpose()?; + let revision = git + .write_object(build_tree(payload_oid, record_oid, tombstone_oid)) + .map_err(|error| CobError::Write(error.to_string()))? + .detach(); + let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); + let revision_oid = Oid::from(revision); + let binding = crate::change::object_binding(parents, object); + let signing = crate::change::signing_bytes( + home, + revision_oid, + parents, + type_name, + &author, + timestamp, + binding, + ); + let signature = signer.sign(&signing); + if !crate::change::verify_signature( + home, + revision_oid, + parents, + type_name, + &author, + timestamp, + object, + signature.as_bytes(), + ) { + return Err(CobError::SelfCheck(type_name.clone())); + } + let commit = gix::objs::Commit { + tree: revision, + parents: parents + .iter() + .map(|parent| parent.oid().object_id()) + .collect(), + author: knot_identity(timestamp), + committer: knot_identity(timestamp), + encoding: None, + message: Vec::new().into(), + extra_headers: vec![ + (TYPE_HEADER.into(), type_name.as_str().into()), + (AUTHOR_HEADER.into(), author.as_str().into()), + ( + SIG_HEADER.into(), + knot_types::lowercase_hex(signature.as_bytes()).into(), + ), + ], + }; + let id = git + .write_object(commit) + .map_err(|error| CobError::Write(error.to_string()))? + .detach(); + Ok(ChangeId::new(Oid::from(id))) +} + +pub(crate) fn load_graph( + repo: &Repo, + type_name: &TypeName, + object: CobId, + history: HistoryModel, + limit: usize, +) -> Result<(ChangeGraph, ChangeId), CobError> { + let tip = resolve_tip(repo, type_name, object)?.ok_or(CobError::NoSuchObject(object))?; + let changes = collect(repo, ChangeId::new(tip), object, limit, None)?; + check_full_shape(&changes, object, history)?; + Ok((ChangeGraph::new(object, changes), ChangeId::new(tip))) +} + +pub(crate) fn check_full_shape( + changes: &BTreeMap, + object: CobId, + history: HistoryModel, +) -> Result<(), CobError> { + let root_id = ChangeId::new(object.oid()); + let root = changes.get(&root_id).ok_or(CobError::DetachedTip(object))?; + if !root.parents.is_empty() { + return Err(CobError::RootNotGenesis(object)); + } + if let Some(stray) = changes + .values() + .find(|change| change.id != root_id && change.parents.is_empty()) + { + return Err(CobError::MultipleRoots { + object, + stray: stray.id, + }); + } + check_no_forbidden_merge(changes, object, history) +} + +pub(crate) fn check_delta_shape( + changes: &BTreeMap, + object: CobId, + since: ChangeId, + history: HistoryModel, +) -> Result<(), CobError> { + check_no_forbidden_merge(changes, object, history)?; + let descends = changes + .values() + .any(|change| change.parents.contains(&since)); + if !changes.is_empty() && !descends { + return Err(CobError::DivergedTip { object, since }); + } + // The descends check isn't good enough because it only proves that + // 1 given collected change has `since` as a parent - what might not have that? A merge, for example. + let detached = changes.values().find(|change| { + change.parents.is_empty() + || change + .parents + .iter() + .any(|parent| *parent != since && !changes.contains_key(parent)) + }); + match detached { + Some(change) => Err(CobError::DeltaNotDescended { + object, + change: change.id, + }), + None => Ok(()), + } +} + +fn check_no_forbidden_merge( + changes: &BTreeMap, + object: CobId, + history: HistoryModel, +) -> Result<(), CobError> { + let forbidden_merge = (history == HistoryModel::Linear) + .then(|| changes.values().find(|change| change.parents.len() > 1)) + .flatten(); + match forbidden_merge { + Some(merge) => Err(CobError::ForkedHistory { + object, + change: merge.id, + }), + None => Ok(()), + } +} + +pub(crate) fn collect( + repo: &Repo, + tip: ChangeId, + object: CobId, + limit: usize, + stop: Option, +) -> Result, CobError> { + let mut frontier = vec![tip]; + let mut seen: BTreeMap = BTreeMap::new(); + let mut overflowed = false; + let walk = { + let mut step = || -> Option> { + let head = frontier.pop()?; + if Some(head) == stop || seen.contains_key(&head) { + return Some(Ok(())); + } + if seen.len() >= limit { + overflowed = true; + return None; + } + match read_change(repo, head) { + Ok(change) => { + frontier.extend(change.parents.iter().copied()); + seen.insert(head, change); + Some(Ok(())) + } + Err(error) => Some(Err(error)), + } + }; + std::iter::from_fn(&mut step).try_for_each(|outcome| outcome) + }; + walk?; + if overflowed { + return Err(CobError::HistoryTooLong(object)); + } + Ok(seen) +} + +pub(crate) fn read_change(repo: &Repo, id: ChangeId) -> Result { + let oid = id.oid(); + let malformed = |reason: String| CobError::MalformedChange { oid, reason }; + #[cfg(feature = "instrument")] + crate::instrument::record_read(); + let data = repo + .git() + .find_object(oid.object_id()) + .map_err(|error| malformed(error.to_string()))? + .detach() + .data; + let commit = gix::objs::CommitRef::from_bytes(&data, repo.git().object_hash()) + .map_err(|error| malformed(error.to_string()))?; + let revision = Oid::from(commit.tree()); + let parents = commit + .parents() + .map(|parent| ChangeId::new(Oid::from(parent))) + .collect(); + let timestamp = UnixSeconds::new( + commit + .time() + .map_err(|error| malformed(error.to_string()))? + .seconds, + ); + let type_raw = commit + .extra_headers() + .find(TYPE_HEADER) + .ok_or_else(|| malformed("missing cob-type header".into()))?; + let type_name = TypeName::new( + type_raw + .to_str() + .map_err(|error| malformed(error.to_string()))?, + ) + .map_err(|error| malformed(error.to_string()))?; + let author_raw = commit + .extra_headers() + .find(AUTHOR_HEADER) + .ok_or_else(|| malformed("missing cob-author header".into()))?; + let author = ActorId::new( + author_raw + .to_str() + .map_err(|error| malformed(error.to_string()))?, + ) + .map_err(|error| malformed(error.to_string()))?; + let signature = commit + .extra_headers() + .find(SIG_HEADER) + .ok_or_else(|| malformed("missing cob-sig header".into())) + .and_then(|raw| { + knot_types::decode_hex(raw).ok_or_else(|| malformed("cob-sig isn't valid hex".into())) + })?; + let (payload, record) = read_tree_blobs(repo, revision)?; + Ok(Change { + id, + revision, + parents, + type_name, + author, + signature: Signature::from_bytes(signature), + payload: Payload::new(payload), + record, + timestamp, + }) +} + +fn read_tree_blobs(repo: &Repo, revision: Oid) -> Result<(Vec, Record), CobError> { + let malformed = |reason: String| CobError::MalformedChange { + oid: revision, + reason, + }; + #[cfg(feature = "instrument")] + crate::instrument::record_read(); + let data = repo + .git() + .find_object(revision.object_id()) + .map_err(|error| malformed(error.to_string()))? + .detach() + .data; + let tree = gix::objs::TreeRef::from_bytes(&data, repo.git().object_hash()) + .map_err(|error| malformed(error.to_string()))?; + let read = |blob: Blob| { + entry_oid(&tree, blob.name()).map(|oid| { + let header = repo + .git() + .objects + .try_header(&oid) + .map_err(|error| malformed(error.to_string()))? + .ok_or_else(|| malformed(format!("missing {} blob", blob.name())))?; + check_blob_size(header.size as usize, blob)?; + repo.git() + .find_object(oid) + .map(|object| object.detach().data) + .map_err(|error| malformed(error.to_string())) + }) + }; + let payload = read(Blob::Payload) + .transpose()? + .ok_or_else(|| malformed("missing payload blob".into()))?; + let record = match ( + read(Blob::Record).transpose()?, + entry_oid(&tree, TOMBSTONE_BLOB), + ) { + (Some(_), Some(_)) => { + return Err(malformed( + "Change has both record and tombstone for said record".into(), + )); + } + (Some(bytes), None) => { + Record::version(RecordBody::new(bytes).map_err(|error| malformed(error.to_string()))?) + } + (None, Some(_)) => Record::Tombstone, + (None, None) => Record::Bare, + }; + Ok((payload, record)) +} + +fn entry_oid(tree: &gix::objs::TreeRef<'_>, name: &str) -> Option { + tree.entries + .iter() + .find(|entry| entry.filename == BStr::new(name)) + .map(|entry| entry.oid.to_owned()) +} + +fn build_tree( + payload_oid: gix::ObjectId, + record_oid: Option, + tombstone_oid: Option, +) -> gix::objs::Tree { + gix::objs::Tree { + entries: [ + Some(blob_entry(PAYLOAD_BLOB, payload_oid)), + record_oid.map(|oid| blob_entry(RECORD_BLOB, oid)), + tombstone_oid.map(|oid| blob_entry(TOMBSTONE_BLOB, oid)), + ] + .into_iter() + .flatten() + .collect(), + } +} + +fn blob_entry(name: &str, oid: gix::ObjectId) -> gix::objs::tree::Entry { + gix::objs::tree::Entry { + mode: gix::objs::tree::EntryKind::Blob.into(), + filename: name.into(), + oid, + } +} + +fn knot_identity(time: UnixSeconds) -> gix::actor::Signature { + gix::actor::Signature { + name: "knot".into(), + email: "noreply@knot".into(), + time: Time::new(time.get(), 0), + } +} diff --git a/knot2/crates/knot-cob/src/change.rs b/knot2/crates/knot-cob/src/change.rs new file mode 100644 index 0000000000000000000000000000000000000000..9cd72e1af482dc49bc3fb1fb66e7cf3d0f78733f --- /dev/null +++ b/knot2/crates/knot-cob/src/change.rs @@ -0,0 +1,523 @@ +use k256::ecdsa::signature::Verifier as _; +use k256::ecdsa::{Signature as K256Signature, VerifyingKey}; +use knot_runtime::Signature; +use knot_types::crypto::PublicKey; +use knot_types::{ + ActorId, ChangeId, CobId, KnotId, Oid, RecordAddress, RecordBody, RecordCid, RecordCollection, + RepoDid, TypeName, UnixSeconds, +}; +use serde::Serialize; +use serde::de::DeserializeOwned; + +use crate::error::PayloadError; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CobHome { + Repo(RepoDid), + Knot(KnotId), +} + +impl CobHome { + pub fn as_str(&self) -> &str { + match self { + CobHome::Repo(did) => did.as_str(), + CobHome::Knot(knot) => knot.as_str(), + } + } + + fn kind(&self) -> &'static str { + match self { + CobHome::Repo(_) => "repo", + CobHome::Knot(_) => "knot", + } + } +} + +impl From<&RepoDid> for CobHome { + fn from(did: &RepoDid) -> Self { + CobHome::Repo(did.clone()) + } +} + +impl From<&KnotId> for CobHome { + fn from(knot: &KnotId) -> Self { + CobHome::Knot(knot.clone()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Payload(Vec); + +impl Payload { + pub fn new(bytes: Vec) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum Record { + #[default] + Bare, + Version(RecordBody), + Tombstone, +} + +impl Record { + pub fn version(body: RecordBody) -> Self { + Self::Version(body) + } + + pub fn into_body(self) -> Option { + match self { + Self::Version(body) => Some(body), + Self::Bare | Self::Tombstone => None, + } + } + + pub fn bytes(&self) -> Option<&[u8]> { + match self { + Self::Version(body) => Some(body.as_bytes()), + Self::Bare | Self::Tombstone => None, + } + } + + pub const fn is_tombstone(&self) -> bool { + matches!(self, Self::Tombstone) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Emits { + #[default] + Unclaimed, + Version(RecordCid), + Tombstone, +} + +pub trait ChangePayload: Serialize + DeserializeOwned + Sized { + const TYPE: &'static str; + + fn address(&self) -> Option { + None + } + + fn emits(&self) -> Emits { + Emits::Unclaimed + } + + fn collection() -> Option { + None + } + + fn stamped_at(&self) -> Option { + None + } + + fn type_name() -> TypeName { + TypeName::new(Self::TYPE).expect("ChangePayload::TYPE must be valid nsid") + } + + fn encode(&self) -> Result, PayloadError> { + serde_ipld_dagcbor::to_vec(self).map_err(|error| PayloadError::Encode(error.to_string())) + } + + fn decode(bytes: &[u8]) -> Result { + decode_payload(bytes) + } +} + +pub fn decode_payload(bytes: &[u8]) -> Result { + serde_ipld_dagcbor::from_slice(bytes).map_err(|error| PayloadError::Decode(error.to_string())) +} + +const SIGNING_CONTEXT: SigningContext = SigningContext::MODERN; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct SigningContext(&'static str); + +impl SigningContext { + pub(crate) const MODERN: Self = Self("org.tangled.knot.cob.change.v1"); + pub(crate) const LEGACY: Self = Self("sh.tangled.knot.cob.change.v1"); + + pub(crate) const fn as_str(self) -> &'static str { + self.0 + } +} + +#[derive(Serialize)] +struct SignedChange<'a> { + context: &'static str, + #[serde(rename = "homeKind")] + home_kind: &'static str, + home: &'a str, + revision: Oid, + parents: &'a [ChangeId], + #[serde(rename = "typeName")] + type_name: &'a TypeName, + author: &'a ActorId, + timestamp: UnixSeconds, + #[serde(skip_serializing_if = "Option::is_none")] + object: Option, +} + +pub(crate) fn signing_bytes( + home: &CobHome, + revision: Oid, + parents: &[ChangeId], + type_name: &TypeName, + author: &ActorId, + timestamp: UnixSeconds, + object: Option, +) -> Vec { + signing_bytes_in( + SIGNING_CONTEXT, + home, + revision, + parents, + type_name, + author, + timestamp, + object, + ) +} + +#[allow(clippy::too_many_arguments)] +fn signing_bytes_in( + context: SigningContext, + home: &CobHome, + revision: Oid, + parents: &[ChangeId], + type_name: &TypeName, + author: &ActorId, + timestamp: UnixSeconds, + object: Option, +) -> Vec { + let view = SignedChange { + context: context.as_str(), + home_kind: home.kind(), + home: home.as_str(), + revision, + parents, + type_name, + author, + timestamp, + object, + }; + serde_ipld_dagcbor::to_vec(&view).expect("change signing view always encodes") +} + +pub(crate) fn object_binding(parents: &[ChangeId], object: Option) -> Option { + if parents.is_empty() { None } else { object } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn verify_signature( + home: &CobHome, + revision: Oid, + parents: &[ChangeId], + type_name: &TypeName, + author: &ActorId, + timestamp: UnixSeconds, + object: Option, + signature: &[u8], +) -> bool { + let Ok(public) = PublicKey::decode(author.as_str()) else { + return false; + }; + let Ok(verifying) = public.to_k256() else { + return false; + }; + let Ok(signature) = K256Signature::from_slice(signature) else { + return false; + }; + let message = signing_bytes( + home, + revision, + parents, + type_name, + author, + timestamp, + object_binding(parents, object), + ); + let verifying = VerifyingKey::from(&verifying); + verifying.verify(&message, &signature).is_ok() + || verifying + .verify( + &signing_bytes_in( + SigningContext::LEGACY, + home, + revision, + parents, + type_name, + author, + timestamp, + object_binding(parents, object), + ), + &signature, + ) + .is_ok() +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Change { + pub id: ChangeId, + pub revision: Oid, + pub parents: Vec, + pub type_name: TypeName, + pub author: ActorId, + pub signature: Signature, + pub payload: Payload, + pub record: Record, + pub timestamp: UnixSeconds, +} + +impl Change { + pub fn payload(&self) -> &[u8] { + self.payload.as_bytes() + } + + pub fn sort_key(&self) -> (UnixSeconds, ChangeId) { + (self.timestamp, self.id) + } + + pub fn verify(&self, home: &CobHome, expected_author: &ActorId, object: Option) -> bool { + &self.author == expected_author + && verify_signature( + home, + self.revision, + &self.parents, + &self.type_name, + &self.author, + self.timestamp, + object, + self.signature.as_bytes(), + ) + } +} + +#[cfg(test)] +mod tests { + use knot_runtime::{K256Signer, SeededEntropy, Signer}; + + use super::*; + + fn type_name() -> TypeName { + TypeName::new("org.tangled.test.tag").unwrap() + } + + fn cob_home() -> CobHome { + CobHome::from(&RepoDid::new("did:plc:squid").unwrap()) + } + + fn signed_change( + signer: &K256Signer, + revision: Oid, + parents: Vec, + timestamp: i64, + ) -> Change { + let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); + let timestamp = UnixSeconds::new(timestamp); + let bytes = signing_bytes( + &cob_home(), + revision, + &parents, + &type_name(), + &author, + timestamp, + object_binding(&parents, None), + ); + Change { + id: ChangeId::new(Oid::null()), + revision, + parents, + type_name: type_name(), + author, + signature: signer.sign(&bytes), + payload: Payload::new(Vec::new()), + record: Record::Bare, + timestamp, + } + } + + #[test] + fn a_change_signed_under_the_legacy_context_still_verifies() { + let signer = K256Signer::generate(&SeededEntropy::new(21)); + let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); + let revision = Oid::from_hex("7777777777777777777777777777777777777777").unwrap(); + let timestamp = UnixSeconds::new(1); + let legacy_type = TypeName::new("sh.tangled.knot.member").unwrap(); + let bytes = signing_bytes_in( + SigningContext::LEGACY, + &cob_home(), + revision, + &[], + &legacy_type, + &author, + timestamp, + None, + ); + let signature = signer.sign(&bytes); + assert!( + verify_signature( + &cob_home(), + revision, + &[], + &legacy_type, + &author, + timestamp, + None, + signature.as_bytes() + ), + "a change the previous release signed still verifies" + ); + let mut tampered = signature.as_bytes().to_vec(); + tampered[0] ^= 0xff; + assert!( + !verify_signature( + &cob_home(), + revision, + &[], + &legacy_type, + &author, + timestamp, + None, + &tampered + ), + "the legacy context is an accepted signing domain, not a bypass" + ); + } + + #[test] + fn signing_bytes_stay_byte_stable() { + let author = { + let mut compressed = [0u8; 33]; + compressed[0] = 0x02; + compressed[1] = 0x09; + ActorId::from_secp256k1(&compressed) + }; + let parents = vec![ChangeId::new( + Oid::from_hex("2222222222222222222222222222222222222222").unwrap(), + )]; + let object = Some(CobId::new( + Oid::from_hex("3333333333333333333333333333333333333333").unwrap(), + )); + let bytes = signing_bytes( + &cob_home(), + Oid::from_hex("1111111111111111111111111111111111111111").unwrap(), + &parents, + &type_name(), + &author, + UnixSeconds::new(1_700_000_000), + object_binding(&parents, object), + ); + assert_eq!( + knot_types::lowercase_hex(&bytes), + "a964686f6d656d6469643a706c633a737175696466617574686f7278317a513373684e317652664257527847397234564c3251796466474e675955715a5a385836743971535359774b636a644a50666f626a65637478283333333333333333333333333333333333333333333333333333333333333333333333333333333367636f6e74657874781e6f72672e74616e676c65642e6b6e6f742e636f622e6368616e67652e763167706172656e74738178283232323232323232323232323232323232323232323232323232323232323232323232323232323268686f6d654b696e64647265706f687265766973696f6e78283131313131313131313131313131313131313131313131313131313131313131313131313131313168747970654e616d65746f72672e74616e676c65642e746573742e7461676974696d657374616d701a6553f100" + ); + } + + #[test] + fn verify_binds_the_author() { + let signer = K256Signer::generate(&SeededEntropy::new(13)); + let stranger = K256Signer::generate(&SeededEntropy::new(14)); + let revision = Oid::from_hex("6666666666666666666666666666666666666666").unwrap(); + let change = signed_change(&signer, revision, Vec::new(), 1); + let stranger_actor = ActorId::from_secp256k1(stranger.public_key().as_bytes()); + assert!(change.verify(&cob_home(), &change.author, None)); + assert!( + !change.verify(&cob_home(), &stranger_actor, None), + "a valid signature under an unexpected expected-author is refused" + ); + let foreign = Change { + author: stranger_actor, + ..change + }; + assert!( + !foreign.verify(&cob_home(), &foreign.author, None), + "an author swapped to a stranger fails its own signature check" + ); + } + + #[test] + fn verify_rejects_a_tampered_transplanted_or_rehomed_change() { + let signer = K256Signer::generate(&SeededEntropy::new(10)); + let revision = Oid::from_hex("1111111111111111111111111111111111111111").unwrap(); + let genuine = signed_change(&signer, revision, Vec::new(), 1); + assert!( + genuine.verify(&cob_home(), &genuine.author, None), + "genuine root change verifies" + ); + + let mutators: Vec Change> = vec![ + |change| { + let mut bytes = change.signature.as_bytes().to_vec(); + bytes[0] ^= 0xff; + Change { + signature: Signature::from_bytes(bytes), + ..change + } + }, + |change| Change { + parents: vec![ChangeId::new( + Oid::from_hex("3333333333333333333333333333333333333333").unwrap(), + )], + ..change + }, + |change| Change { + timestamp: UnixSeconds::new(9_999_999), + ..change + }, + ]; + mutators.into_iter().for_each(|mutate| { + let broken = mutate(genuine.clone()); + assert!( + !broken.verify(&cob_home(), &broken.author, None), + "a tampered or transplanted change fails verification" + ); + }); + + let other_home = CobHome::from(&RepoDid::new("did:plc:limpet").unwrap()); + assert!( + !genuine.verify(&other_home, &genuine.author, None), + "change signed for one repo mustn't verify under another repo's home" + ); + + let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); + let parent = + ChangeId::new(Oid::from_hex("5555555555555555555555555555555555555555").unwrap()); + let home = CobId::new(Oid::from_hex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap()); + let elsewhere = + CobId::new(Oid::from_hex("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb").unwrap()); + let timestamp = UnixSeconds::new(1); + let bytes = signing_bytes( + &cob_home(), + revision, + &[parent], + &type_name(), + &author, + timestamp, + object_binding(&[parent], Some(home)), + ); + let bound = Change { + id: ChangeId::new(Oid::null()), + revision, + parents: vec![parent], + type_name: type_name(), + author, + signature: signer.sign(&bytes), + payload: Payload::new(Vec::new()), + record: Record::Bare, + timestamp, + }; + assert!(bound.verify(&cob_home(), &bound.author, Some(home))); + assert!( + !bound.verify(&cob_home(), &bound.author, Some(elsewhere)), + "a non-root change is bound to its object" + ); + assert!(!bound.verify(&cob_home(), &bound.author, None)); + } +} diff --git a/knot2/crates/knot-cob/src/error.rs b/knot2/crates/knot-cob/src/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..2140c2607f8cb6919ee59e0a20a9d06249b735c8 --- /dev/null +++ b/knot2/crates/knot-cob/src/error.rs @@ -0,0 +1,130 @@ +use knot_types::{ + AccountDid, AtUri, ChangeId, CobId, ContributionPermission, Oid, RecordAddress, RecordCid, + TypeName, UnixSeconds, +}; + +#[derive(Debug, thiserror::Error)] +pub enum PayloadError { + #[error("dag-cbor encode failed: {0}")] + Encode(String), + #[error("dag-cbor decode failed: {0}")] + Decode(String), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AddressMove { + pub object: CobId, + pub expected: RecordAddress, + pub found: RecordAddress, +} + +#[derive(Debug, thiserror::Error)] +pub enum CobError { + #[error(transparent)] + Git(#[from] knot_git::GitError), + #[error(transparent)] + Payload(#[from] PayloadError), + #[error("object {0} doesn't exist")] + NoSuchObject(CobId), + #[error("object {0} tip doesn't descend from its root change")] + DetachedTip(CobId), + #[error("object {object} tip doesn't descend from indexed tip {since}")] + DivergedTip { object: CobId, since: ChangeId }, + #[error( + "applying change {change}, of object {object}, to that mr tip, which it doesn't descend from, would count changes the index already has" + )] + DeltaNotDescended { object: CobId, change: ChangeId }, + #[error("change's {blob} blob, of {bytes} bytes, is over {limit} byte limit")] + ChangeTooLarge { + blob: &'static str, + bytes: usize, + limit: usize, + }, + #[error("object {0} isn't rooted at its genesis change")] + RootNotGenesis(CobId), + #[error("object {object} contains second parentless change {stray}")] + MultipleRoots { object: CobId, stray: ChangeId }, + #[error( + "object {} materializes into {}, and change can't move it to {}", + .0.object, + .0.expected, + .0.found + )] + AddressMoved(Box), + #[error("object {object} doesn't emit records. Change can't address one at {found}")] + AddressAdded { object: CobId, found: RecordAddress }, + #[error("object {object} with no record versions can't go stale")] + Unaddressed { object: CobId }, + #[error("{actor} may not write change authored by {author}, please sign it as its author")] + Impersonation { + actor: AccountDid, + author: AccountDid, + }, + #[error("{actor} may not open {type_name} object here as {permission}")] + CreateRejected { + actor: AccountDid, + type_name: TypeName, + permission: ContributionPermission, + }, + #[error("{actor} may not append to object {object} as {permission}")] + AppendRejected { + actor: AccountDid, + object: CobId, + permission: ContributionPermission, + }, + #[error( + "{uri} is at {current}, and change was made from another version, please rebase from {current}" + )] + StaleRecord { + object: CobId, + uri: AtUri, + current: RecordCid, + }, + #[error("Change declares record {declared} and stores {found}")] + RecordMisclaimed { + declared: RecordCid, + found: RecordCid, + }, + #[error("change declares record {declared}. Change won't store said record's version")] + RecordUnwritten { declared: RecordCid }, + #[error("change erases its record, and stores said record's version instead of tombstone")] + TombstoneUnwritten, + #[error("Change that leaves its record live can't store its tombstone")] + TombstoneUnclaimed, + #[error("{type_name} change can't address the record at {found}, which is another collection")] + AddressForeign { + type_name: TypeName, + found: RecordAddress, + }, + #[error("Change is stamped {stamped} and signed at {signed}")] + StampMoved { + stamped: UnixSeconds, + signed: UnixSeconds, + }, + #[error("authoritative object {object} has forked history at merge change {change}")] + ForkedHistory { object: CobId, change: ChangeId }, + #[error("change {change} isn't validly signed by its owning identity")] + UnverifiedChange { change: ChangeId }, + #[error("concurrent write moved object {object} past expected tip {expected}")] + StaleTip { object: CobId, expected: ChangeId }, + #[error("object {0} exceeded its compare-and-swap retry budget under contention")] + Contended(CobId), + #[error("change {oid} is malformed: {reason}")] + MalformedChange { oid: Oid, reason: String }, + #[error("change {change} payload does not decode: {reason}")] + UndecodableChange { change: ChangeId, reason: String }, + #[error("change {change} is a {found} change in {expected} object")] + UnexpectedChangeType { + change: ChangeId, + expected: TypeName, + found: TypeName, + }, + #[error("produced unverifiable signature for {0} change")] + SelfCheck(TypeName), + #[error("change graph for {0} exceeds load bound")] + HistoryTooLong(CobId), + #[error("'{0}' is not usable collaborative-object ref name")] + RefName(String), + #[error("writing git object failed: {0}")] + Write(String), +} diff --git a/knot2/crates/knot-cob/src/graph.rs b/knot2/crates/knot-cob/src/graph.rs new file mode 100644 index 0000000000000000000000000000000000000000..64e1ffd78ad948311819197aa100a58dcb2cefbb --- /dev/null +++ b/knot2/crates/knot-cob/src/graph.rs @@ -0,0 +1,141 @@ +use std::cmp::Reverse; +#[cfg(test)] +use std::collections::BTreeSet; +use std::collections::{BTreeMap, BinaryHeap}; + +use knot_types::{ChangeId, CobId, UnixSeconds}; + +use crate::change::Change; + +#[derive(Debug)] +pub struct ChangeGraph { + root: CobId, + changes: BTreeMap, +} + +impl ChangeGraph { + pub(crate) fn new(root: CobId, changes: BTreeMap) -> Self { + Self { root, changes } + } + + pub fn root(&self) -> CobId { + self.root + } + + pub fn len(&self) -> usize { + self.changes.len() + } + + pub fn is_empty(&self) -> bool { + self.changes.is_empty() + } + + #[cfg(test)] + pub fn causal_order(&self) -> Vec { + order(&self.changes) + } + + pub(crate) fn into_ordered(self) -> Vec { + let ordered = order(&self.changes); + let mut changes = self.changes; + ordered + .into_iter() + .map(|id| changes.remove(&id).expect("ordered id is in graph")) + .collect() + } +} + +fn order(changes: &BTreeMap) -> Vec { + let mut indegree: BTreeMap = changes + .values() + .map(|change| { + let present = change + .parents + .iter() + .filter(|parent| changes.contains_key(*parent)) + .count(); + (change.id, present) + }) + .collect(); + let children: BTreeMap> = + changes.values().fold(BTreeMap::new(), |mut acc, change| { + change + .parents + .iter() + .filter(|parent| changes.contains_key(*parent)) + .for_each(|parent| acc.entry(*parent).or_default().push(change.id)); + acc + }); + let mut ready: BinaryHeap> = changes + .values() + .filter(|change| indegree[&change.id] == 0) + .map(|change| Reverse(change.sort_key())) + .collect(); + let ordered: Vec = std::iter::from_fn(move || { + let Reverse((_, id)) = ready.pop()?; + children.get(&id).into_iter().flatten().for_each(|child| { + let degree = indegree + .get_mut(child) + .expect("every child has an indegree entry"); + *degree -= 1; + if *degree == 0 { + ready.push(Reverse(changes[child].sort_key())); + } + }); + Some(id) + }) + .collect(); + debug_assert_eq!( + ordered.len(), + changes.len(), + "the graph out of git is acyclic, which lets sorting reach every change" + ); + ordered +} + +#[derive(Debug)] +pub struct History { + root: ChangeId, + changes: Vec, +} + +impl History { + pub(crate) fn new(root: ChangeId, changes: Vec) -> Self { + Self { root, changes } + } + + pub fn root(&self) -> ChangeId { + self.root + } + + pub fn changes(&self) -> &[Change] { + &self.changes + } + + pub fn len(&self) -> usize { + self.changes.len() + } + + pub fn is_empty(&self) -> bool { + self.changes.is_empty() + } + + #[cfg(test)] + pub fn traverse A>(&self, init: A, f: F) -> A { + self.changes.iter().fold(init, f) + } + + #[cfg(test)] + pub fn tips(&self) -> Vec { + let referenced: BTreeSet = self + .changes + .iter() + .flat_map(|change| change.parents.iter().copied()) + .collect(); + self.changes + .iter() + .map(|change| change.id) + .filter(|id| !referenced.contains(id)) + .collect() + } +} diff --git a/knot2/crates/knot-cob/src/instrument.rs b/knot2/crates/knot-cob/src/instrument.rs new file mode 100644 index 0000000000000000000000000000000000000000..96d4f2aaba62f2bf0ba7fd8c65f7b2973b6f0246 --- /dev/null +++ b/knot2/crates/knot-cob/src/instrument.rs @@ -0,0 +1,8 @@ +knot_types::read_counter!( + CobReads, + record_read, + cob_reads, + reset_cob_reads, + add_cob_reads, + measure +); diff --git a/knot2/crates/knot-cob/src/lib.rs b/knot2/crates/knot-cob/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..7a978247fc50671f6c0a4d07447df9834f903b29 --- /dev/null +++ b/knot2/crates/knot-cob/src/lib.rs @@ -0,0 +1,3543 @@ +mod backend; +mod change; +mod error; +mod graph; +#[cfg(feature = "instrument")] +pub mod instrument; +mod object; + +pub use change::{Change, ChangePayload, CobHome, Emits, Payload, Record, decode_payload}; +pub use error::{AddressMove, CobError, PayloadError}; +pub use graph::{ChangeGraph, History}; +use knot_types::{ActorId, ChangeId, CobId, TypeName}; +pub use object::{ + Append, Authored, Authorize, Checkpoint, Evaluate, HistoryModel, KnotAuthored, Object, + SnapshotStride, StateSize, WriteContext, +}; + +pub use backend::{ + CHECKPOINTS_PREFIX, COBS_PREFIX, checkpoint_ref_name, cob_ref_name, migrate_legacy_refs, + parse_cob_ref, +}; + +const LEGACY_TYPE_ALIASES: [(&str, &str); 4] = [ + ("sh.tangled.knot.member", "org.tangled.knot.member"), + ("sh.tangled.knot.block", "org.tangled.knot.block"), + ( + "sh.tangled.knot.repoRegistry", + "org.tangled.knot.repoRegistry", + ), + ( + "sh.tangled.repo.collaborator", + "org.tangled.graph.collaborator", + ), +]; + +pub(crate) fn legacy_type_of(modern: &TypeName) -> Option { + LEGACY_TYPE_ALIASES + .iter() + .find(|(_, modern_name)| modern.as_str() == *modern_name) + .and_then(|(legacy, _)| TypeName::new(*legacy).ok()) +} + +pub fn modernized_type_of(legacy: &TypeName) -> Option { + LEGACY_TYPE_ALIASES + .iter() + .find(|(legacy_name, _)| legacy.as_str() == *legacy_name) + .and_then(|(_, modern)| TypeName::new(*modern).ok()) +} + +pub(crate) fn accepts_change_type(expected: &TypeName, found: &TypeName) -> bool { + found == expected || legacy_type_of(expected).is_some_and(|legacy| &legacy == found) +} + +use std::collections::BTreeSet; + +use knot_git::{Haves, RefUpdate, Repo, Wants}; +use knot_runtime::Signer; +use knot_types::{Decision, Oid, RecordAddress, UnixSeconds}; +use serde::Serialize; +use serde::de::DeserializeOwned; + +const MAX_CAS_RETRIES: usize = 16; +const CHECKPOINT_GROWTH_DIVISOR: usize = 16; +// don't ask + +knot_types::scalar_newtype! { + pub(crate) struct Tail(usize); +} + +impl Tail { + fn after_append(self) -> Self { + Self(self.0.saturating_add(1)) + } + + fn reaches(self, stride: SnapshotStride) -> bool { + self.0 >= stride.get() + } +} + +fn checkpoint_stride(snapshot_stride: SnapshotStride, size: StateSize) -> SnapshotStride { + SnapshotStride::new( + snapshot_stride + .get() + .max(size.get() / CHECKPOINT_GROWTH_DIVISOR) + .clamp(1, backend::MAX_GRAPH_CHANGES), + ) +} + +struct Recovered { + state: S, + tip: ChangeId, + tail: Tail, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Created { + pub object: CobId, + pub tip: ChangeId, +} + +#[derive(Debug, Clone)] +pub struct Materialized

    { + pub change: P, + pub record: Record, +} + +impl

    Materialized

    { + pub fn bare(change: P) -> Self { + Self { + change, + record: Record::Bare, + } + } +} + +pub struct Approved<'cx, C> { + cx: &'cx WriteContext<'cx>, + materialized: Materialized, +} + +impl<'cx, C: Authored> Approved<'cx, C> { + pub fn signed_by( + cx: &'cx WriteContext<'cx>, + materialized: Materialized, + ) -> Result { + let author = materialized.change.author(); + if author == cx.actor() { + Ok(Self { cx, materialized }) + } else { + Err(CobError::Impersonation { + actor: cx.actor().clone(), + author: author.clone(), + }) + } + } + + pub const fn context(&self) -> &'cx WriteContext<'cx> { + self.cx + } +} + +#[derive(Debug)] +pub struct Delta { + pub changes: Vec, + pub tip: ChangeId, +} + +const CHECKPOINT_FORMAT: u16 = 5; + +#[derive(Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct CheckpointDigest(Oid); + +fn checkpoint_digest( + object: CobId, + tip: ChangeId, + state: &S, +) -> Result { + let bytes = serde_ipld_dagcbor::to_vec(&(object, tip, state)) + .map_err(|error| CobError::Write(error.to_string()))?; + let mut hasher = gix_hash::hasher(gix_hash::Kind::Sha1); + hasher.update(&bytes); + hasher + .try_finalize() + .map(|id| CheckpointDigest(Oid::from(id))) + .map_err(|error| CobError::Write(error.to_string())) +} + +fn encode_checkpoint( + object: CobId, + tip: ChangeId, + state: &S, +) -> Result, CobError> { + let digest = checkpoint_digest(object, tip, state)?; + serde_ipld_dagcbor::to_vec(&(CHECKPOINT_FORMAT, object, tip, digest, state)) + .map_err(|error| CobError::Write(error.to_string())) +} + +fn decode_checkpoint( + object: CobId, + bytes: &[u8], +) -> Option<(ChangeId, S)> { + let (version, decoded_object, tip, digest, state): (u16, CobId, ChangeId, CheckpointDigest, S) = + serde_ipld_dagcbor::from_slice(bytes).ok()?; + (version == CHECKPOINT_FORMAT).then_some(())?; + (decoded_object == object).then_some(())?; + (checkpoint_digest(object, tip, &state).ok()? == digest).then_some(())?; + Some((tip, state)) +} + +pub struct CobStore<'r> { + repo: &'r Repo, +} + +impl<'r> CobStore<'r> { + pub fn new(repo: &'r Repo) -> Self { + Self { repo } + } + + pub fn create( + &self, + home: &CobHome, + payload: &P, + signer: &dyn Signer, + timestamp: UnixSeconds, + ) -> Result { + let bytes = payload.encode()?; + Self::check_payload(payload, &Record::Bare, timestamp)?; + self.create_from_bytes( + home, + &P::type_name(), + &bytes, + &Record::Bare, + signer, + timestamp, + ) + } + + pub fn create_materialized( + &self, + home: &CobHome, + payload: &Materialized

    , + signer: &dyn Signer, + timestamp: UnixSeconds, + ) -> Result { + let bytes = payload.change.encode()?; + Self::check_payload(&payload.change, &payload.record, timestamp)?; + self.create_from_bytes( + home, + &P::type_name(), + &bytes, + &payload.record, + signer, + timestamp, + ) + } + + pub fn create_approved( + &self, + home: &CobHome, + approved: &Approved<'_, E::Change>, + signer: &dyn Signer, + timestamp: UnixSeconds, + ) -> Result + where + E: Authorize, + E::Change: ChangePayload + Authored, + { + let cx = approved.context(); + match E::may_create(cx) { + Decision::Allow => { + self.create_materialized(home, &approved.materialized, signer, timestamp) + } + Decision::Deny => Err(CobError::CreateRejected { + actor: cx.actor().clone(), + type_name: E::Change::type_name(), + permission: cx.permission(), + }), + } + } + + pub fn append_approved( + &self, + home: &CobHome, + object: CobId, + approved: &Approved<'_, E::Change>, + signer: &dyn Signer, + timestamp: UnixSeconds, + ) -> Result + where + E: Authorize + Checkpoint, + E::State: Serialize + DeserializeOwned, + E::Change: ChangePayload + Authored + Clone, + { + let cx = approved.context(); + self.retrying_append::(home, object, signer, timestamp, |state| { + match E::may_append(cx, state, &approved.materialized.change) { + Append::Allow => Ok(Some(approved.materialized.clone())), + Append::Deny => Err(CobError::AppendRejected { + actor: cx.actor().clone(), + object, + permission: cx.permission(), + }), + Append::Stale { current } => Err(CobError::StaleRecord { + object, + uri: self + .root_address::(object)? + .ok_or(CobError::Unaddressed { object })? + .at_uri(cx.repo()), + current, + }), + } + }) + .map(|tip| tip.expect("Approved append always yields its change")) + } + + fn create_from_bytes( + &self, + home: &CobHome, + type_name: &TypeName, + bytes: &[u8], + record: &Record, + signer: &dyn Signer, + timestamp: UnixSeconds, + ) -> Result { + let tip = backend::write_change_with_record( + home, + self.repo, + type_name, + bytes, + record, + &[], + None, + signer, + timestamp, + )?; + let object = CobId::new(tip.oid()); + let name = backend::cob_ref_name(type_name, object)?; + match backend::resolve_tip(self.repo, type_name, object)? { + Some(current) if current == tip.oid() => Ok(Created { object, tip }), + Some(_) => Err(CobError::StaleTip { + object, + expected: tip, + }), + None => self + .repo + .update_ref(&RefUpdate::Create { + name, + new: tip.oid(), + }) + .map(|()| Created { object, tip }) + .map_err(CobError::from), + } + } + + fn current_tip(&self, type_name: &TypeName, object: CobId) -> Result { + backend::resolve_tip(self.repo, type_name, object)? + .map(ChangeId::new) + .ok_or(CobError::NoSuchObject(object)) + } + + pub fn update( + &self, + home: &CobHome, + object: CobId, + payload: &P, + signer: &dyn Signer, + timestamp: UnixSeconds, + ) -> Result { + let type_name = P::type_name(); + let expected = self.current_tip(&type_name, object)?; + self.chain( + home, + object, + &type_name, + expected, + std::iter::once((payload, &Record::Bare, timestamp)), + signer, + ) + .map(|tip| tip.expect("one change chains onto one tip")) + } + pub fn update_materialized( + &self, + home: &CobHome, + object: CobId, + payload: &Materialized

    , + signer: &dyn Signer, + timestamp: UnixSeconds, + ) -> Result { + let type_name = P::type_name(); + let expected = self.current_tip(&type_name, object)?; + self.append_materialized( + home, object, &type_name, expected, payload, signer, timestamp, + ) + } + + pub fn extend<'p, P: KnotAuthored + 'p>( + &self, + home: &CobHome, + object: CobId, + changes: impl IntoIterator, + signer: &dyn Signer, + ) -> Result, CobError> { + let type_name = P::type_name(); + let expected = self.current_tip(&type_name, object)?; + self.chain( + home, + object, + &type_name, + expected, + changes + .into_iter() + .map(|(payload, timestamp)| (payload, &Record::Bare, timestamp)), + signer, + ) + } + + #[allow(clippy::too_many_arguments)] + fn append_materialized( + &self, + home: &CobHome, + object: CobId, + type_name: &TypeName, + expected: ChangeId, + payload: &Materialized

    , + signer: &dyn Signer, + timestamp: UnixSeconds, + ) -> Result { + self.chain( + home, + object, + type_name, + expected, + std::iter::once((&payload.change, &payload.record, timestamp)), + signer, + ) + .map(|tip| tip.expect("one change chains onto one tip")) + } + + fn check_payload( + payload: &P, + record: &Record, + timestamp: UnixSeconds, + ) -> Result<(), CobError> { + Self::check_collection(payload)?; + Self::check_stamp(payload, timestamp)?; + Self::check_claim(payload, record) + } + + fn check_collection(payload: &P) -> Result<(), CobError> { + match (payload.address(), P::collection()) { + (Some(found), Some(collection)) if *found.collection() != collection => { + Err(CobError::AddressForeign { + type_name: P::type_name(), + found, + }) + } + _ => Ok(()), + } + } + + fn check_stamp(payload: &P, timestamp: UnixSeconds) -> Result<(), CobError> { + match payload.stamped_at() { + Some(stamped) if stamped != timestamp => Err(CobError::StampMoved { + stamped, + signed: timestamp, + }), + _ => Ok(()), + } + } + + fn check_claim(payload: &P, record: &Record) -> Result<(), CobError> { + match (payload.emits(), record) { + (Emits::Version(declared), Record::Version(body)) => { + let found = body.cid(); + if found == declared { + Ok(()) + } else { + Err(CobError::RecordMisclaimed { declared, found }) + } + } + (Emits::Version(declared), Record::Bare | Record::Tombstone) => { + Err(CobError::RecordUnwritten { declared }) + } + (Emits::Tombstone, Record::Tombstone) => Ok(()), + (Emits::Tombstone, Record::Bare | Record::Version(_)) => { + Err(CobError::TombstoneUnwritten) + } + (Emits::Unclaimed, Record::Tombstone) => Err(CobError::TombstoneUnclaimed), + (Emits::Unclaimed, Record::Bare | Record::Version(_)) => Ok(()), + } + } + + pub fn root_address( + &self, + object: CobId, + ) -> Result, CobError> { + let root = backend::read_change(self.repo, ChangeId::new(object.oid()))?; + Ok(P::decode(root.payload())?.address()) + } + + fn check_address(&self, object: CobId, payload: &P) -> Result<(), CobError> { + let Some(found) = payload.address() else { + return Ok(()); + }; + match self.root_address::

    (object)? { + Some(expected) if expected == found => Ok(()), + Some(expected) => Err(CobError::AddressMoved(Box::new(error::AddressMove { + object, + expected, + found, + }))), + None => Err(CobError::AddressAdded { object, found }), + } + } + + fn chain<'p, P: ChangePayload + 'p>( + &self, + home: &CobHome, + object: CobId, + type_name: &TypeName, + expected: ChangeId, + changes: impl IntoIterator, + signer: &dyn Signer, + ) -> Result, CobError> { + let written = match changes.into_iter().try_fold( + Vec::::new(), + |mut written, (payload, record, timestamp)| { + let parent = written.last().copied().unwrap_or(expected); + let write = || -> Result { + Self::check_payload(payload, record, timestamp)?; + self.check_address::

    (object, payload)?; + let bytes = payload.encode()?; + backend::write_change_with_record( + home, + self.repo, + type_name, + &bytes, + record, + &[parent], + Some(object), + signer, + timestamp, + ) + }; + match write() { + Ok(tip) => { + written.push(tip); + Ok(written) + } + Err(error) => Err(Box::new((written, error))), + } + }, + ) { + Ok(written) => written, + Err(boxed) => { + let (partial, error) = *boxed; + self.discard(type_name, object, expected, &partial)?; + return Err(error); + } + }; + written.last().copied().map_or(Ok(None), |tip| { + self.publish(type_name, object, expected, tip, &written) + .map(Some) + }) + } + + fn publish( + &self, + type_name: &TypeName, + object: CobId, + expected: ChangeId, + tip: ChangeId, + written: &[ChangeId], + ) -> Result { + let name = backend::cob_ref_name(type_name, object)?; + match self.repo.update_ref(&RefUpdate::Update { + name, + old: expected.oid(), + new: tip.oid(), + }) { + Ok(()) => Ok(tip), + Err(error) => { + let actual = backend::resolve_tip(self.repo, type_name, object)?; + self.discard(type_name, object, expected, written)?; + match actual { + actual if actual != Some(expected.oid()) => { + Err(CobError::StaleTip { object, expected }) + } + _ => Err(CobError::Git(error)), + } + } + } + } + + fn discard( + &self, + type_name: &TypeName, + object: CobId, + expected: ChangeId, + written: &[ChangeId], + ) -> Result<(), CobError> { + if written.is_empty() { + return Ok(()); + } + // Originally I was gonna just delete `written` entirely but + // a simulatenous writing loser has the same literal OId(s) as the winner by nature, + // so let's not rip those out of what *is* actually valid history. + let published = self.published_since(type_name, object, expected)?; + Ok(written + .iter() + .filter(|change| !published.contains(&change.oid())) + .try_for_each(|change| self.repo.remove_loose_object(change.oid()))?) + } + + fn published_since( + &self, + type_name: &TypeName, + object: CobId, + expected: ChangeId, + ) -> Result, CobError> { + let Some(current) = backend::resolve_tip(self.repo, type_name, object)? else { + return Ok(BTreeSet::new()); + }; + if current == expected.oid() { + return Ok(BTreeSet::new()); + } + let have = [expected.oid()]; + Ok(self + .repo + .rev_walk(Wants::new(&[current]), Haves::new(&have))? + .into_iter() + .collect()) + } + + #[cfg(test)] + pub fn update_with( + &self, + home: &CobHome, + object: CobId, + signer: &dyn Signer, + timestamp: UnixSeconds, + decide: impl Fn(&E::State) -> Result, + ) -> Result + where + E: Evaluate, + E::Change: KnotAuthored, + D: From, + { + self.update_maybe::(home, object, signer, timestamp, |state| { + decide(state).map(Some) + }) + .map(|tip| tip.expect("update_with always yields change to append")) + } + + #[cfg(test)] + pub fn update_maybe( + &self, + home: &CobHome, + object: CobId, + signer: &dyn Signer, + timestamp: UnixSeconds, + decide: impl Fn(&E::State) -> Result, D>, + ) -> Result, D> + where + E: Evaluate, + E::Change: KnotAuthored, + D: From, + { + let type_name = E::Change::type_name(); + let attempt = || -> Result>, D> { + let (graph, expected) = backend::load_graph( + self.repo, + &type_name, + object, + E::HISTORY, + backend::MAX_GRAPH_CHANGES, + )?; + let (state, _history) = object::evaluate::(graph, &type_name)?; + match decide(&state)? { + None => Ok(Some(None)), + Some(change) => { + match self.append_materialized( + home, + object, + &type_name, + expected, + &Materialized::bare(change), + signer, + timestamp, + ) { + Ok(id) => Ok(Some(Some(id))), + Err(CobError::StaleTip { .. }) => Ok(None), + Err(other) => Err(D::from(other)), + } + } + } + }; + (0..MAX_CAS_RETRIES) + .find_map(|_| attempt().transpose()) + .unwrap_or_else(|| Err(D::from(CobError::Contended(object)))) + } + + #[cfg(test)] + pub fn graph(&self, object: CobId) -> Result { + Ok(backend::load_graph( + self.repo, + &E::Change::type_name(), + object, + E::HISTORY, + backend::MAX_GRAPH_CHANGES, + )? + .0) + } + + pub fn get(&self, object: CobId) -> Result, CobError> { + let type_name = E::Change::type_name(); + let (graph, _tip) = backend::load_graph( + self.repo, + &type_name, + object, + E::HISTORY, + backend::MAX_GRAPH_CHANGES, + )?; + let (state, history) = object::evaluate::(graph, &type_name)?; + Ok(Object::new(object, type_name, state, history)) + } + + pub fn verify( + &self, + home: &CobHome, + object: CobId, + owner: &ActorId, + ) -> Result<(), CobError> { + let type_name = E::Change::type_name(); + let tip = self.current_tip(&type_name, object)?; + self.verify_type_from_tip(home, &type_name, object, tip, E::HISTORY, owner) + } + + pub fn verify_type_from_tip( + &self, + home: &CobHome, + type_name: &TypeName, + object: CobId, + tip: ChangeId, + history: HistoryModel, + owner: &ActorId, + ) -> Result<(), CobError> { + let changes = backend::collect(self.repo, tip, object, backend::MAX_GRAPH_CHANGES, None)?; + backend::check_full_shape(&changes, object, history)?; + changes.values().try_for_each(|change| { + if !accepts_change_type(type_name, &change.type_name) { + return Err(CobError::UnexpectedChangeType { + change: change.id, + expected: type_name.clone(), + found: change.type_name.clone(), + }); + } + change + .verify(home, owner, Some(object)) + .then_some(()) + .ok_or(CobError::UnverifiedChange { change: change.id }) + }) + } + + pub fn list(&self) -> Result, CobError> { + backend::list_objects(self.repo, &E::Change::type_name()) + } + + pub fn cobs(&self) -> Result, CobError> { + backend::list_cobs(self.repo) + } + + pub fn changes_of( + &self, + type_name: &TypeName, + object: CobId, + history: HistoryModel, + since: Option, + ) -> Result { + let tip = self.current_tip(type_name, object)?; + let collected = + backend::collect(self.repo, tip, object, backend::MAX_GRAPH_CHANGES, since)?; + match since { + None => backend::check_full_shape(&collected, object, history)?, + Some(since) => backend::check_delta_shape(&collected, object, since, history)?, + } + Ok(Delta { + changes: ChangeGraph::new(object, collected).into_ordered(), + tip, + }) + } + + pub fn changes_since( + &self, + object: CobId, + since: Option, + ) -> Result { + self.changes_of(&E::Change::type_name(), object, E::HISTORY, since) + } + + pub fn update_with_checkpointed( + &self, + home: &CobHome, + object: CobId, + signer: &dyn Signer, + timestamp: UnixSeconds, + decide: impl Fn(&E::State) -> Result, + ) -> Result + where + E: Checkpoint, + E::Change: KnotAuthored, + E::State: Serialize + DeserializeOwned, + D: From, + { + self.update_maybe_checkpointed::(home, object, signer, timestamp, |state| { + decide(state).map(Some) + }) + .map(|tip| tip.expect("update_with_checkpointed always yields change to append")) + } + + pub fn update_maybe_checkpointed( + &self, + home: &CobHome, + object: CobId, + signer: &dyn Signer, + timestamp: UnixSeconds, + decide: impl Fn(&E::State) -> Result, D>, + ) -> Result, D> + where + E: Checkpoint, + E::Change: KnotAuthored, + E::State: Serialize + DeserializeOwned, + D: From, + { + self.retrying_append::(home, object, signer, timestamp, |state| { + decide(state).map(|change| change.map(Materialized::bare)) + }) + } + + fn retrying_append( + &self, + home: &CobHome, + object: CobId, + signer: &dyn Signer, + timestamp: UnixSeconds, + decide: impl Fn(&E::State) -> Result>, D>, + ) -> Result, D> + where + E: Checkpoint, + E::State: Serialize + DeserializeOwned, + D: From, + { + let type_name = E::Change::type_name(); + let attempt = || -> Result>, D> { + let Recovered { + state, + tip: expected, + tail, + } = self.checkpointed_state::(object)?; + let Some(materialized) = decide(&state)? else { + return Ok(Some(None)); + }; + match self.append_materialized( + home, + object, + &type_name, + expected, + &materialized, + signer, + timestamp, + ) { + Ok(tip) => { + let stride = checkpoint_stride(E::SNAPSHOT_STRIDE, E::checkpoint_size(&state)); + if tail.after_append().reaches(stride) { + let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); + let applied = E::apply(state, materialized.change, &author); + if let Err(error) = self.write_checkpoint::(object, tip, &applied) { + tracing::warn!( + cob = type_name.as_str(), + object = %object.oid().to_hex(), + %error, + "checkpoint write failed" + ); + } + } + Ok(Some(Some(tip))) + } + Err(CobError::StaleTip { .. }) => Ok(None), + Err(other) => Err(D::from(other)), + } + }; + (0..MAX_CAS_RETRIES) + .find_map(|_| attempt().transpose()) + .unwrap_or_else(|| Err(D::from(CobError::Contended(object)))) + } + + pub fn materialize(&self, object: CobId) -> Result<(E::State, ChangeId), CobError> + where + E: Checkpoint, + E::State: Serialize + DeserializeOwned, + { + self.checkpointed_state::(object) + .map(|Recovered { state, tip, .. }| (state, tip)) + } + + fn checkpointed_state(&self, object: CobId) -> Result, CobError> + where + E: Checkpoint, + E::State: Serialize + DeserializeOwned, + { + let type_name = E::Change::type_name(); + let tip = self.current_tip(&type_name, object)?; + match self.load_checkpoint::(object)? { + Some((checkpoint_tip, state)) if checkpoint_tip == tip => Ok(Recovered { + state, + tip, + tail: Tail(0), + }), + Some((checkpoint_tip, state)) => { + match self.changes_since::(object, Some(checkpoint_tip)) { + Ok(delta) => { + let applied = + object::apply_changes::(state, &delta.changes, &type_name)?; + Ok(Recovered { + state: applied, + tip: delta.tip, + tail: Tail(delta.changes.len()), + }) + } + Err( + unusable @ (CobError::DivergedTip { .. } + | CobError::DeltaNotDescended { .. }), + ) => { + tracing::warn!( + %object, + error = %unusable, + "checkpoint no longer bounds this object's history, the replay starts over from its root change" + ); + self.recovered_state::(object) + } + Err(error) => Err(error), + } + } + None => self.recovered_state::(object), + } + } + + fn recovered_state(&self, object: CobId) -> Result, CobError> + where + E: Checkpoint, + E::State: Serialize + DeserializeOwned, + { + let type_name = E::Change::type_name(); + let (graph, tip) = backend::load_graph( + self.repo, + &type_name, + object, + E::HISTORY, + backend::rebuild_graph_limit(), + )?; + let replay = Recovered { + tail: Tail(graph.len()), + tip, + state: object::evaluate::(graph, &type_name)?.0, + }; + let stride = checkpoint_stride(E::SNAPSHOT_STRIDE, E::checkpoint_size(&replay.state)); + if !replay.tail.reaches(stride) { + return Ok(replay); + } + match self.write_checkpoint::(object, replay.tip, &replay.state) { + Ok(()) => { + tracing::info!( + cob = type_name.as_str(), + object = %object.oid().to_hex(), + replayed = replay.tail.get(), + stride = stride.get(), + "replayed thru all state and saved a checkpoint!" + ); + Ok(Recovered { + tail: Tail(0), + ..replay + }) + } + Err(error) => { + tracing::warn!( + cob = type_name.as_str(), + object = %object.oid().to_hex(), + %error, + "failure to save a replay checkpoint" + ); + Ok(replay) + } + } + } + + fn load_checkpoint(&self, object: CobId) -> Result, CobError> + where + E: Checkpoint, + E::State: DeserializeOwned + Serialize, + { + let name = backend::checkpoint_ref_name(&E::Change::type_name(), object)?; + let Some(blob) = self.repo.find_ref(&name)? else { + return Ok(None); + }; + let Ok(bytes) = self.repo.read_blob(blob) else { + return Ok(None); + }; + Ok(decode_checkpoint::(object, &bytes)) + } + + fn write_checkpoint( + &self, + object: CobId, + tip: ChangeId, + state: &E::State, + ) -> Result<(), CobError> + where + E: Checkpoint, + E::State: Serialize, + { + let bytes = encode_checkpoint(object, tip, state)?; + let blob = Oid::from( + self.repo + .git() + .write_blob(&bytes) + .map_err(|error| CobError::Write(error.to_string()))? + .detach(), + ); + let name = backend::checkpoint_ref_name(&E::Change::type_name(), object)?; + let update = match self.repo.find_ref(&name)? { + Some(old) => RefUpdate::Update { + name, + old, + new: blob, + }, + None => RefUpdate::Create { name, new: blob }, + }; + self.repo.update_ref(&update).map_err(CobError::from) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use knot_git::{Layout, Repo}; + use knot_runtime::{K256Signer, SeededEntropy, Signature, Signer}; + use knot_types::{ + Blocked, ContributionPermission, ContributionPolicy, RecordBody, RecordCid, RefName, + RepoDid, RepoRole, UnixSeconds, + }; + use proptest::prelude::*; + use serde::{Deserialize, Serialize}; + use tempfile::TempDir; + + use super::*; + use crate::backend::{checkpoint_ref_name, cob_ref_name}; + + #[derive(Debug, Serialize, Deserialize)] + #[serde(tag = "op", content = "subject")] + enum Tag { + Add(String), + Remove(String), + } + + impl ChangePayload for Tag { + const TYPE: &'static str = "org.tangled.test.tag"; + } + + impl KnotAuthored for Tag {} + + #[test] + fn checkpoint_encoding_stays_byte_stable() { + let object = CobId::new(Oid::from_hex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap()); + let tip = ChangeId::new(Oid::from_hex("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb").unwrap()); + let mut state = BTreeSet::new(); + state.insert("kelp".to_string()); + state.insert("squid".to_string()); + let bytes = encode_checkpoint(object, tip, &state).unwrap(); + assert_eq!( + knot_types::lowercase_hex(&bytes), + "850578286161616161616161616161616161616161616161616161616161616161616161616161616161616178286262626262626262626262626262626262626262626262626262626262626262626262626262626278283861623531376164633234616530313138383962643634343930663836633562646333333632333782646b656c70657371756964" + ); + } + + struct Tags; + + impl Evaluate for Tags { + type State = BTreeSet; + type Change = Tag; + + const HISTORY: HistoryModel = HistoryModel::Convergent; + + fn initial() -> Self::State { + BTreeSet::new() + } + + fn apply(mut state: Self::State, change: Self::Change, _author: &ActorId) -> Self::State { + match change { + Tag::Add(subject) => { + state.insert(subject); + } + Tag::Remove(subject) => { + state.remove(&subject); + } + } + state + } + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(tag = "op", content = "subject")] + enum NameChange { + Claim(String), + } + + impl ChangePayload for NameChange { + const TYPE: &'static str = "org.tangled.test.name"; + } + + impl KnotAuthored for NameChange {} + + struct Names; + + impl Evaluate for Names { + type State = BTreeSet; + type Change = NameChange; + + const HISTORY: HistoryModel = HistoryModel::Linear; + + fn initial() -> Self::State { + BTreeSet::new() + } + + fn apply(mut state: Self::State, change: Self::Change, _author: &ActorId) -> Self::State { + match change { + NameChange::Claim(subject) => { + state.insert(subject); + } + } + state + } + } + + impl Checkpoint for Names { + const SNAPSHOT_STRIDE: SnapshotStride = SnapshotStride::new(4); + fn checkpoint_size(state: &Self::State) -> StateSize { + StateSize::new(state.len()) + } + } + + #[derive(Debug)] + enum NameError { + Taken, + Cob(CobError), + } + + impl From for NameError { + fn from(error: CobError) -> Self { + NameError::Cob(error) + } + } + + fn claim_unless_taken( + who: &str, + ) -> impl Fn(&BTreeSet) -> Result, NameError> + '_ { + move |state| match state.contains(who) { + true => Err(NameError::Taken), + false => Ok(Some(NameChange::Claim(who.to_string()))), + } + } + + fn claim( + store: &CobStore, + object: CobId, + key: &K256Signer, + who: &str, + when: UnixSeconds, + ) -> Result, NameError> { + store.update_maybe_checkpointed::( + &cob_home(), + object, + key, + when, + claim_unless_taken(who), + ) + } + + fn appended(result: Result, NameError>) -> ChangeId { + match result { + Ok(Some(id)) => id, + Ok(None) => panic!("distinct claim should append a change"), + Err(NameError::Taken) => panic!("name was unexpectedly already taken"), + Err(NameError::Cob(error)) => panic!("checkpointed write failed: {error:?}"), + } + } + + fn plain_claim( + store: &CobStore, + object: CobId, + key: &K256Signer, + who: &str, + when: UnixSeconds, + ) -> ChangeId { + appended(store.update_maybe::( + &cob_home(), + object, + key, + when, + claim_unless_taken(who), + )) + } + + fn plain_history(store: &CobStore, key: &K256Signer, changes: usize) -> CobId { + let object = store + .create( + &cob_home(), + &NameChange::Claim("name0000".into()), + key, + at(0), + ) + .unwrap() + .object; + (1..changes).for_each(|index| { + plain_claim( + store, + object, + key, + &format!("name{index:04}"), + at(index as i64), + ); + }); + object + } + + fn fixture() -> (TempDir, Repo) { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let repo = layout + .create(&RepoDid::new("did:plc:squid").unwrap()) + .unwrap(); + (dir, repo) + } + + fn signer(seed: u64) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed)) + } + + fn cob_home() -> CobHome { + CobHome::from(&RepoDid::new("did:plc:squid").unwrap()) + } + + fn at(seconds: i64) -> UnixSeconds { + UnixSeconds::new(seconds) + } + + fn tag_root(repo: &Repo, key: &K256Signer, subject: &str, when: UnixSeconds) -> ChangeId { + backend::write_change( + &cob_home(), + repo, + &Tag::type_name(), + &Tag::Add(subject.into()).encode().unwrap(), + &[], + None, + key, + when, + ) + .unwrap() + } + + fn tag_child( + repo: &Repo, + key: &K256Signer, + subject: &str, + parents: &[ChangeId], + object: CobId, + when: UnixSeconds, + ) -> ChangeId { + backend::write_change( + &cob_home(), + repo, + &Tag::type_name(), + &Tag::Add(subject.into()).encode().unwrap(), + parents, + Some(object), + key, + when, + ) + .unwrap() + } + + fn publish(repo: &Repo, object: CobId, tip: ChangeId) { + let name = backend::cob_ref_name(&Tag::type_name(), object).unwrap(); + repo.update_ref(&RefUpdate::Create { + name, + new: tip.oid(), + }) + .unwrap(); + } + + fn forked_tag_object( + repo: &Repo, + key: &K256Signer, + left: &str, + right: &str, + merge: &str, + ) -> CobId { + let root = tag_root(repo, key, "base", at(1)); + let object = CobId::new(root.oid()); + let left = tag_child(repo, key, left, &[root], object, at(2)); + let right = tag_child(repo, key, right, &[root], object, at(3)); + let merge = tag_child(repo, key, merge, &[left, right], object, at(4)); + publish(repo, object, merge); + object + } + + fn linear_chain(repo: &Repo, key: &K256Signer, subjects: &[&str]) -> CobId { + let (first, rest) = subjects.split_first().expect("chain needs a root subject"); + let root = tag_root(repo, key, first, at(1)); + let object = CobId::new(root.oid()); + let tip = rest + .iter() + .enumerate() + .fold(root, |parent, (index, subject)| { + tag_child(repo, key, subject, &[parent], object, at(index as i64 + 2)) + }); + publish(repo, object, tip); + object + } + + #[test] + fn extend_chains_the_same_tip_as_sequential_updates() { + let (_sequential_dir, sequential_repo) = fixture(); + let (_batched_dir, batched_repo) = fixture(); + let key = signer(3); + let changes = [ + NameChange::Claim("kelp".into()), + NameChange::Claim("squid".into()), + NameChange::Claim("whelk".into()), + ]; + let sequential = CobStore::new(&sequential_repo); + let batched = CobStore::new(&batched_repo); + let root = |store: &CobStore| { + store + .create(&cob_home(), &NameChange::Claim("uni".into()), &key, at(0)) + .unwrap() + .object + }; + let one_by_one = root(&sequential); + changes.iter().zip(1..).for_each(|(change, when)| { + sequential + .update(&cob_home(), one_by_one, change, &key, at(when)) + .unwrap(); + }); + let in_one_go = root(&batched); + let tip = batched + .extend( + &cob_home(), + in_one_go, + changes.iter().zip((1..).map(at)), + &key, + ) + .unwrap(); + + let resolved = |repo: &Repo, object| { + backend::resolve_tip(repo, &NameChange::type_name(), object) + .unwrap() + .map(ChangeId::new) + }; + assert_eq!(one_by_one, in_one_go); + assert_eq!(tip, resolved(&batched_repo, in_one_go)); + assert_eq!(tip, resolved(&sequential_repo, one_by_one)); + assert_eq!( + sequential.materialize::(one_by_one).unwrap().0, + batched.materialize::(in_one_go).unwrap().0 + ); + assert_eq!( + batched + .extend::(&cob_home(), in_one_go, std::iter::empty(), &key) + .unwrap(), + None + ); + assert_eq!(tip, resolved(&batched_repo, in_one_go)); + } + + #[test] + fn checkpointed_writes_match_a_full_replay_and_leave_a_snapshot() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(1); + let object = store + .create( + &cob_home(), + &NameChange::Claim("name0000".into()), + &key, + at(0), + ) + .unwrap() + .object; + + let total = Names::SNAPSHOT_STRIDE.get() * 3; + (1..total).for_each(|index| { + appended(claim( + &store, + object, + &key, + &format!("name{index:04}"), + at(index as i64), + )); + }); + + let replayed = store.get::(object).unwrap(); + assert_eq!( + replayed.state().len(), + total, + "a full replay counts every checkpointed write" + ); + let snapshot = backend::checkpoint_ref_name(&NameChange::type_name(), object).unwrap(); + assert!( + repo.find_ref(&snapshot).unwrap().is_some(), + "snapshot ref was written once stride was crossed" + ); + } + + #[test] + fn checkpoint_stride_grows_with_state_so_large_cobs_snapshot_less_often() { + assert_eq!( + checkpoint_stride(SnapshotStride::new(256), StateSize::new(0)), + SnapshotStride::new(256) + ); + assert_eq!( + checkpoint_stride( + SnapshotStride::new(256), + StateSize::new(256 * CHECKPOINT_GROWTH_DIVISOR) + ), + SnapshotStride::new(256), + "up to stride*divisor entries the fixed floor governs, so small cobs snapshot exactly as before" + ); + assert_eq!( + checkpoint_stride(SnapshotStride::new(256), StateSize::new(1_600_000)), + SnapshotStride::new(100_000), + "a large cob snapshots at a fixed fraction of its size, so total snapshot bytes stay linear in the change count" + ); + assert_eq!( + checkpoint_stride(SnapshotStride::new(256), StateSize::new(4_000_000)), + SnapshotStride::new(backend::MAX_GRAPH_CHANGES), + "the stride stops at the serving limit so the boot tail always replays incrementally, never overflowing into a full rebuild" + ); + } + + #[test] + fn a_large_checkpointed_cob_bounds_its_boot_suffix_to_a_fraction_of_its_size() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(2); + let object = store + .create( + &cob_home(), + &NameChange::Claim("name0000".into()), + &key, + at(0), + ) + .unwrap() + .object; + + let total = Names::SNAPSHOT_STRIDE.get() * CHECKPOINT_GROWTH_DIVISOR * 4; + (1..total).for_each(|index| { + claim( + &store, + object, + &key, + &format!("name{index:04}"), + at(index as i64), + ) + .unwrap(); + }); + + assert_eq!( + store.get::(object).unwrap().state().len(), + total, + "a full replay still counts every write after adaptive snapshotting" + ); + + let Recovered { tail, .. } = store.checkpointed_state::(object).unwrap(); + let bound = checkpoint_stride(Names::SNAPSHOT_STRIDE, StateSize::new(total)); + assert!( + tail.get() <= bound.get() && bound.get() < total, + "at boot time we'll only replay from the {:?} changes since last snapshot, bounded by stride of {:?}, not thru history of all {total}", + tail, + bound + ); + } + + #[test] + fn a_corrupt_checkpoint_never_serves_a_wrong_answer_and_heals_at_the_live_tip() { + type Forge = fn(CobId, Oid, &BTreeSet) -> Vec; + let cases: &[Forge] = &[ + |_object, _tip, _state| b"not a valid checkpoint envelope".to_vec(), + |object, tip, state| { + serde_ipld_dagcbor::to_vec(&( + CHECKPOINT_FORMAT + 1, + object.oid().to_hex(), + tip.to_hex(), + checkpoint_digest(object, ChangeId::new(tip), state).unwrap(), + state, + )) + .unwrap() + }, + |object, tip, state| { + let mut tampered = state.clone(); + tampered.insert("intruder".to_string()); + serde_ipld_dagcbor::to_vec(&( + CHECKPOINT_FORMAT, + object.oid().to_hex(), + tip.to_hex(), + checkpoint_digest(object, ChangeId::new(tip), state).unwrap(), + tampered, + )) + .unwrap() + }, + ]; + + cases.iter().enumerate().for_each(|(index, forge)| { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(200 + index as u64); + let object = store + .create(&cob_home(), &NameChange::Claim("squid".into()), &key, at(0)) + .unwrap() + .object; + (1..=Names::SNAPSHOT_STRIDE.get()).for_each(|step| { + claim( + &store, + object, + &key, + &format!("name{step:04}"), + at(step as i64), + ) + .unwrap(); + }); + + let real_state = store.get::(object).unwrap().state().clone(); + let real_tip = backend::resolve_tip(&repo, &NameChange::type_name(), object) + .unwrap() + .unwrap(); + let forged = forge(object, real_tip, &real_state); + + let snapshot = backend::checkpoint_ref_name(&NameChange::type_name(), object).unwrap(); + let live = repo.find_ref(&snapshot).unwrap().expect("snapshot exists"); + let blob = Oid::from(repo.git().write_blob(&forged).unwrap().detach()); + repo.update_ref(&RefUpdate::Update { + name: snapshot.clone(), + old: live, + new: blob, + }) + .unwrap(); + + let duplicate = claim(&store, object, &key, "squid", at(100)); + assert!( + matches!(duplicate, Err(NameError::Taken)), + "corrupt checkpoint falls back to real state instead of waving a duplicate through" + ); + + let healed = repo.find_ref(&snapshot).unwrap().unwrap(); + let bytes = repo.read_blob(healed).unwrap(); + let (tip, state) = decode_checkpoint::>(object, &bytes) + .expect("healed snapshot decodes"); + let live_tip = backend::resolve_tip(&repo, &NameChange::type_name(), object) + .unwrap() + .unwrap(); + assert_eq!( + tip, + ChangeId::new(live_tip), + "healed snapshot sits at the live tip" + ); + assert_eq!( + &state, + store.get::(object).unwrap().state(), + "healed snapshot matches the full replay" + ); + + appended(claim(&store, object, &key, "intruder", at(101))); + let Recovered { tail, .. } = store.checkpointed_state::(object).unwrap(); + assert_eq!( + tail, + Tail(1), + "healed checkpoint should only be as bounded as tail after boot" + ); + }); + } + + #[test] + fn replay_saves_a_single_checkpoint() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(7); + let brief_key = signer(71); + + let brief = plain_history(&store, &brief_key, Names::SNAPSHOT_STRIDE.get() - 1); + let (short, _) = store.materialize::(brief).unwrap(); + assert_eq!(short.len(), Names::SNAPSHOT_STRIDE.get() - 1); + assert!( + repo.find_ref(&backend::checkpoint_ref_name(&NameChange::type_name(), brief).unwrap()) + .unwrap() + .is_none(), + "if replay of refs count is smaller than num-to-stride, don't bother checkpointing yet" + ); + + let total = Names::SNAPSHOT_STRIDE.get() * 2; + let object = plain_history(&store, &key, total); + let snapshot = backend::checkpoint_ref_name(&NameChange::type_name(), object).unwrap(); + let (state, tip) = store.materialize::(object).unwrap(); + assert_eq!(state.len(), total, "replayed-state given back from store"); + let written = repo + .find_ref(&snapshot) + .unwrap() + .expect("replay persisted a checkpoint"); + assert_eq!( + decode_checkpoint::>(object, &repo.read_blob(written).unwrap()), + Some((tip, state)), + "checkpoint commits to tip, returning count replayable" + ); + + let Recovered { tail, .. } = store.checkpointed_state::(object).unwrap(); + assert_eq!(tail, Tail(0), "next boot will load from saved checkpoint"); + + plain_claim(&store, object, &key, "tailname", at(total as i64)); + let (moved, _new_tip) = store.materialize::(object).unwrap(); + assert_eq!( + moved.len(), + total + 1, + "even next read will replay new tail onto checkpoint" + ); + assert_eq!( + repo.find_ref(&snapshot).unwrap().unwrap(), + written, + "no touchy checkpoint if tail num smaller than stride num" + ); + } + + #[test] + fn checkpoint_then_append_will_only_write_when_bigger_than_stride_num() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(14); + + let brief = plain_history(&store, &signer(141), 2); + store + .update_with_checkpointed::(&cob_home(), brief, &key, at(2), |_| { + Ok(NameChange::Claim("name0002".into())) + }) + .unwrap(); + assert!( + repo.find_ref(&backend::checkpoint_ref_name(&NameChange::type_name(), brief).unwrap()) + .unwrap() + .is_none(), + "don't do another checkpoint if less than stride num" + ); + + let total = Names::SNAPSHOT_STRIDE.get() * 2; + let object = plain_history(&store, &key, total); + let before = ChangeId::new( + backend::resolve_tip(&repo, &NameChange::type_name(), object) + .unwrap() + .expect("tip before appending"), + ); + store + .update_with_checkpointed::( + &cob_home(), + object, + &key, + at(total as i64), + |state| Ok(NameChange::Claim(format!("tail{}", state.len()))), + ) + .unwrap(); + let snapshot = backend::checkpoint_ref_name(&NameChange::type_name(), object).unwrap(); + let written = repo + .find_ref(&snapshot) + .unwrap() + .expect("replay should've persisted a checkpoint"); + let (tip, state) = + decode_checkpoint::>(object, &repo.read_blob(written).unwrap()) + .expect("the should checkpoint decode"); + assert_eq!(tip, before, "append doesn't cost another checkpoint write"); + assert_eq!(state.len(), total, "checkpoint matches state pre-append"); + let Recovered { tail, .. } = store.checkpointed_state::(object).unwrap(); + assert_eq!(tail, Tail(1), "next boot should only replay added change"); + } + + #[test] + fn read_that_also_reads_should_heal_a_checkpoint_which_isnt_bounding_history() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(9); + let total = Names::SNAPSHOT_STRIDE.get() * 2; + let object = plain_history(&store, &key, total); + + let foreign = Oid::from(repo.git().write_blob(b"not a change").unwrap().detach()); + let forged = + encode_checkpoint(object, ChangeId::new(foreign), &BTreeSet::::new()).unwrap(); + let snapshot = backend::checkpoint_ref_name(&NameChange::type_name(), object).unwrap(); + let blob = Oid::from(repo.git().write_blob(&forged).unwrap().detach()); + repo.update_ref(&RefUpdate::Create { + name: snapshot.clone(), + new: blob, + }) + .unwrap(); + + let (healed_state, live_tip) = store.materialize::(object).unwrap(); + assert_eq!( + healed_state.len(), + total, + "we definitely don't serve forged checkpoint data" + ); + let replaced = repo + .find_ref(&snapshot) + .unwrap() + .expect("stale/incorrect checkpoint was nuked & replaced"); + let bytes = repo.read_blob(replaced).unwrap(); + let (tip, checkpoint_state) = decode_checkpoint::>(object, &bytes) + .expect("healed checkpoint should decode correctly"); + assert_eq!( + tip, live_tip, + "healed checkpoint is correctly bound to current tip" + ); + assert_eq!( + checkpoint_state, healed_state, + "healed checkpoint matches replay" + ); + } + + #[test] + fn a_checkpoint_envelope_is_bound_to_its_object_tip_and_format() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(6); + let a = store + .create(&cob_home(), &NameChange::Claim("squid".into()), &key, at(0)) + .unwrap(); + let b = store + .create( + &cob_home(), + &NameChange::Claim("anemone".into()), + &key, + at(1), + ) + .unwrap(); + let state = BTreeSet::from(["squid".to_string()]); + + let bytes = encode_checkpoint(a.object, a.tip, &state).unwrap(); + assert!( + decode_checkpoint::>(b.object, &bytes).is_none(), + "checkpoint minted for one object mustn't decode under another" + ); + assert_eq!( + decode_checkpoint::>(a.object, &bytes), + Some((a.tip, state.clone())), + "checkpoint decodes to the tip and state its digest commits to under its own object" + ); + + let wrong_tip = ChangeId::new(Oid::from_hex(&"b".repeat(40)).unwrap()); + let swapped_tip = serde_ipld_dagcbor::to_vec(&( + CHECKPOINT_FORMAT, + a.object.oid().to_hex(), + wrong_tip.oid().to_hex(), + checkpoint_digest(a.object, a.tip, &state).unwrap(), + state.clone(), + )) + .unwrap(); + assert!( + decode_checkpoint::>(a.object, &swapped_tip).is_none(), + "checkpoint whose tip is swapped away from the one its digest covers mustn't decode" + ); + + let future_version = serde_ipld_dagcbor::to_vec(&( + CHECKPOINT_FORMAT + 1, + a.object.oid().to_hex(), + a.tip.oid().to_hex(), + checkpoint_digest(a.object, a.tip, &state).unwrap(), + state, + )) + .unwrap(); + assert!( + decode_checkpoint::>(a.object, &future_version).is_none(), + "envelope whose format version isn't the current one mustn't decode" + ); + } + + #[test] + fn the_rebuild_walk_floors_at_the_serving_limit_and_scales_above_it_with_headroom() { + assert_eq!( + backend::rebuild_change_limit_for(None), + 2_000_000, + "an unmeasurable host rebuilds up to the generous fixed ceiling" + ); + let available = |bytes| Some(knot_resource::AvailableBytes::new(bytes)); + assert_eq!( + backend::rebuild_change_limit_for(available(64 * 1024 * 1024)), + backend::MAX_GRAPH_CHANGES, + "a squeezed host floors the rebuild walk at the serving limit, never beneath it" + ); + assert_eq!( + backend::rebuild_change_limit_for(available(0)), + backend::MAX_GRAPH_CHANGES, + "a host with no measured headroom still heals at least as deep as it serves" + ); + let one_gib = backend::rebuild_change_limit_for(available(1024 * 1024 * 1024)); + assert!( + (100_000..150_000).contains(&one_gib), + "on a 1 GiB host the rebuild budget tracks the measured per-change cost near the serving limit, was {one_gib}" + ); + let roomy = backend::rebuild_change_limit_for(available(64 * 1024 * 1024 * 1024)); + assert!( + roomy > backend::MAX_GRAPH_CHANGES, + "a roomy host rebuilds far past the serving limit, was {roomy}" + ); + } + + #[test] + fn tag_lifecycle_roundtrips_appends_reloads_and_keeps_signatures() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(1); + + let created = store + .create(&cob_home(), &Tag::Add("nel".into()), &key, at(1)) + .unwrap(); + let fresh = store.get::(created.object).unwrap(); + assert_eq!(fresh.id(), created.object); + assert_eq!(fresh.state(), &BTreeSet::from(["nel".to_string()])); + assert_eq!(fresh.history().len(), 1); + assert_eq!(fresh.history().root(), created.tip); + let root = backend::read_change(&repo, ChangeId::new(created.object.oid())).unwrap(); + assert!(root.verify(&cob_home(), &root.author, None)); + + store + .update( + &cob_home(), + created.object, + &Tag::Add("olaren".into()), + &key, + at(2), + ) + .unwrap(); + let tip = store + .update( + &cob_home(), + created.object, + &Tag::Remove("nel".into()), + &key, + at(3), + ) + .unwrap(); + + let object = store.get::(created.object).unwrap(); + assert_eq!(object.state(), &BTreeSet::from(["olaren".to_string()])); + assert_eq!(object.history().len(), 3); + assert_eq!(store.list::().unwrap(), vec![created.object]); + let order = object.history().traverse(Vec::new(), |mut acc, change| { + acc.push(change.timestamp.get()); + acc + }); + assert_eq!(order, vec![1, 2, 3]); + assert_eq!(object.history().tips(), vec![tip]); + + let reloaded = store.get::(created.object).unwrap(); + assert_eq!(object.state(), reloaded.state()); + + let head = backend::read_change(&repo, tip).unwrap(); + assert!(head.verify(&cob_home(), &head.author, Some(created.object))); + assert!(!head.verify(&cob_home(), &head.author, None)); + assert!(!head.verify( + &cob_home(), + &head.author, + Some(CobId::new(knot_types::Oid::null())) + )); + + let absent = CobId::new(knot_types::Oid::from_hex(&"0".repeat(40)).unwrap()); + assert!(matches!( + store.get::(absent), + Err(CobError::NoSuchObject(_)) + )); + } + + #[test] + fn undecodable_change_poisons_the_object() { + let (_dir, repo) = fixture(); + let key = signer(5); + let nsid = Tag::type_name(); + + let root = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("nel".into()).encode().unwrap(), + &[], + None, + &key, + at(1), + ) + .unwrap(); + let object = CobId::new(root.oid()); + let garbage = backend::write_change( + &cob_home(), + &repo, + &nsid, + &[0xff, 0xff, 0xff], + &[root], + Some(object), + &key, + at(2), + ) + .unwrap(); + let name = backend::cob_ref_name(&nsid, object).unwrap(); + repo.update_ref(&RefUpdate::Create { + name, + new: garbage.oid(), + }) + .unwrap(); + + assert!(matches!( + CobStore::new(&repo).get::(object), + Err(CobError::UndecodableChange { .. }) + )); + } + + #[test] + fn merge_is_last_writer_wins_with_no_tombstone() { + let key = signer(40); + let nsid = Tag::type_name(); + let resolve = |add_seconds: i64, remove_seconds: i64| -> bool { + let (_dir, repo) = fixture(); + let root = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("seed".into()).encode().unwrap(), + &[], + None, + &key, + at(1), + ) + .unwrap(); + let object = CobId::new(root.oid()); + let add = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("nel".into()).encode().unwrap(), + &[root], + Some(object), + &key, + at(add_seconds), + ) + .unwrap(); + let remove = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Remove("nel".into()).encode().unwrap(), + &[root], + Some(object), + &key, + at(remove_seconds), + ) + .unwrap(); + let merge = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("merged".into()).encode().unwrap(), + &[add, remove], + Some(object), + &key, + at(100), + ) + .unwrap(); + let name = backend::cob_ref_name(&nsid, object).unwrap(); + repo.update_ref(&RefUpdate::Create { + name, + new: merge.oid(), + }) + .unwrap(); + CobStore::new(&repo) + .get::(object) + .unwrap() + .into_state() + .contains("nel") + }; + + assert!(resolve(3, 2), "later add resurrects removed element"); + assert!(!resolve(2, 3), "later remove wins under last-writer-wins"); + } + + #[test] + fn tip_not_descending_from_root_is_detached() { + let (_dir, repo) = fixture(); + let key = signer(22); + let nsid = Tag::type_name(); + let root = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("nel".into()).encode().unwrap(), + &[], + None, + &key, + at(1), + ) + .unwrap(); + let object = CobId::new(root.oid()); + let unrelated = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("olaren".into()).encode().unwrap(), + &[], + None, + &key, + at(2), + ) + .unwrap(); + let name = backend::cob_ref_name(&nsid, object).unwrap(); + repo.update_ref(&RefUpdate::Create { + name, + new: unrelated.oid(), + }) + .unwrap(); + + assert!(matches!( + CobStore::new(&repo).get::(object), + Err(CobError::DetachedTip(_)) + )); + } + + #[test] + fn deep_history_loads_and_orders_without_overflow() { + let count: i64 = 8_000; + let (_dir, repo) = fixture(); + let key = signer(23); + let nsid = Tag::type_name(); + let payload = Tag::Add("nel".into()).encode().unwrap(); + let root = + backend::write_change(&cob_home(), &repo, &nsid, &payload, &[], None, &key, at(0)) + .unwrap(); + let object = CobId::new(root.oid()); + let tip = (1..count).fold(root, |parent, i| { + backend::write_change( + &cob_home(), + &repo, + &nsid, + &payload, + &[parent], + Some(object), + &key, + at(i), + ) + .unwrap() + }); + publish(&repo, object, tip); + let graph = CobStore::new(&repo).graph::(object).unwrap(); + assert_eq!(graph.len(), count as usize); + assert_eq!(graph.causal_order().len(), count as usize); + + let synthetic: u64 = 100_000; + let oid = |index: u64| knot_types::Oid::from_hex(&format!("{index:040x}")).unwrap(); + let actor = ActorId::from_secp256k1(&[0x02; 33]); + let changes: std::collections::BTreeMap = (1..=synthetic) + .map(|index| { + let id = ChangeId::new(oid(index)); + let parents = if index == 1 { + Vec::new() + } else { + vec![ChangeId::new(oid(index - 1))] + }; + let change = Change { + id, + revision: oid(index), + parents, + type_name: Tag::type_name(), + author: actor.clone(), + signature: Signature::from_bytes(Vec::new()), + payload: Payload::new(Vec::new()), + record: Record::Bare, + timestamp: UnixSeconds::new(index as i64), + }; + (id, change) + }) + .collect(); + let synthetic_order = ChangeGraph::new(CobId::new(oid(1)), changes).causal_order(); + assert_eq!(synthetic_order.len(), synthetic as usize); + assert_eq!(synthetic_order.first(), Some(&ChangeId::new(oid(1)))); + assert_eq!(synthetic_order.last(), Some(&ChangeId::new(oid(synthetic)))); + } + + #[test] + fn grafted_foreign_genesis_is_refused() { + let (_dir, repo) = fixture(); + let key = signer(31); + let nsid = Tag::type_name(); + let root = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("nel".into()).encode().unwrap(), + &[], + None, + &key, + at(1), + ) + .unwrap(); + let object = CobId::new(root.oid()); + let child = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("olaren".into()).encode().unwrap(), + &[root], + Some(object), + &key, + at(2), + ) + .unwrap(); + let foreign = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("evil".into()).encode().unwrap(), + &[], + None, + &key, + at(3), + ) + .unwrap(); + let merge = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("merge".into()).encode().unwrap(), + &[child, foreign], + Some(object), + &key, + at(4), + ) + .unwrap(); + let name = backend::cob_ref_name(&nsid, object).unwrap(); + repo.update_ref(&RefUpdate::Create { + name, + new: merge.oid(), + }) + .unwrap(); + + assert!(matches!( + CobStore::new(&repo).get::(object), + Err(CobError::MultipleRoots { .. }) + )); + } + + #[test] + fn cob_ref_name_and_parse_cob_ref_are_inverses() { + let nsid = Tag::type_name(); + let object = CobId::new(knot_types::Oid::from_hex(&"a".repeat(40)).unwrap()); + let name = backend::cob_ref_name(&nsid, object).unwrap(); + + assert_eq!(parse_cob_ref(&name), Some((nsid, object))); + let branch = knot_types::RefName::new("refs/heads/main").unwrap(); + assert_eq!(parse_cob_ref(&branch), None); + let stray = knot_types::RefName::new("refs/cobs/org.tangled.test.tag/not-an-oid").unwrap(); + assert_eq!(parse_cob_ref(&stray), None); + } + + fn loose_commits(repo: &Repo) -> BTreeSet { + std::fs::read_dir(repo.path().join("objects")) + .unwrap() + .filter_map(Result::ok) + .filter(|shard| { + shard + .file_name() + .to_str() + .map(|s| s.len() == 2) + .unwrap_or(false) + }) + .flat_map(|shard| { + let prefix = shard.file_name().to_str().unwrap().to_string(); + std::fs::read_dir(shard.path()) + .unwrap() + .filter_map(Result::ok) + .filter_map(move |entry| { + let rest = entry.file_name().to_str()?.to_string(); + knot_types::Oid::from_hex(&format!("{prefix}{rest}")).ok() + }) + .collect::>() + }) + .filter(|oid| { + repo.git() + .find_object(oid.object_id()) + .map(|object| object.kind == gix::object::Kind::Commit) + .unwrap_or(false) + }) + .collect() + } + + #[test] + fn a_contended_retry_keeps_both_writes_and_leaves_no_dangling_commit() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(124); + let created = store + .create(&cob_home(), &Tag::Add("base".into()), &key, at(1)) + .unwrap(); + + let injected = std::cell::Cell::new(false); + let mine = store.update_with::( + &cob_home(), + created.object, + &key, + at(3), + |_state| { + if !injected.replace(true) { + store + .update( + &cob_home(), + created.object, + &Tag::Add("intruder".into()), + &key, + at(2), + ) + .unwrap(); + } + Ok(Tag::Add("mine".into())) + }, + ); + assert!( + mine.is_ok(), + "handler retried instead of surfacing StaleTip" + ); + + let state = store.get::(created.object).unwrap().into_state(); + assert!( + state.contains("intruder"), + "concurrent append survives retry" + ); + assert!( + state.contains("mine"), + "retried append isn't lost to stale tip" + ); + + let reachable: BTreeSet = store + .graph::(created.object) + .unwrap() + .causal_order() + .into_iter() + .map(|change| change.oid()) + .collect(); + assert_eq!( + reachable.len(), + 3, + "live chain is base, intruder, then mine" + ); + assert_eq!( + loose_commits(&repo), + reachable, + "abandoned compare-and-swap attempt left no dangling commit behind" + ); + } + + #[test] + fn exhausting_the_retry_budget_is_contended_and_leaves_no_dangling_commits() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(126); + let created = store + .create(&cob_home(), &Tag::Add("base".into()), &key, at(1)) + .unwrap(); + let calls = std::cell::Cell::new(0i64); + + let result = store.update_with::( + &cob_home(), + created.object, + &key, + at(1000), + |_state| { + let i = calls.get(); + calls.set(i + 1); + store + .update( + &cob_home(), + created.object, + &Tag::Add(format!("intruder{i}")), + &key, + at(100 + i), + ) + .unwrap(); + Ok(Tag::Add("mine".into())) + }, + ); + + assert!(matches!(result, Err(CobError::Contended(_)))); + assert_eq!( + calls.get(), + MAX_CAS_RETRIES as i64, + "decision ran exactly the retry budget before failing closed" + ); + + let reachable: BTreeSet = store + .graph::(created.object) + .unwrap() + .causal_order() + .into_iter() + .map(|change| change.oid()) + .collect(); + assert_eq!( + reachable.len(), + 1 + MAX_CAS_RETRIES, + "base plus one landed commit per injected intruder" + ); + assert_eq!( + loose_commits(&repo), + reachable, + "every failed attempt across whole budget cleaned up its own orphan" + ); + } + + #[test] + fn an_identical_concurrent_change_is_not_cleaned_up_as_the_live_tip() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(125); + let nsid = Tag::type_name(); + let created = store + .create(&cob_home(), &Tag::Add("base".into()), &key, at(1)) + .unwrap(); + let dup = Tag::Add("dup".into()); + let bytes = dup.encode().unwrap(); + + let first = backend::write_change( + &cob_home(), + &repo, + &nsid, + &bytes, + &[created.tip], + Some(created.object), + &key, + at(2), + ) + .unwrap(); + let second = backend::write_change( + &cob_home(), + &repo, + &nsid, + &bytes, + &[created.tip], + Some(created.object), + &key, + at(2), + ) + .unwrap(); + assert_eq!( + first, second, + "byte-identical changes are content-addressed to one commit" + ); + + let winner = store + .append_materialized( + &cob_home(), + created.object, + &nsid, + created.tip, + &Materialized::bare(Tag::Add("dup".into())), + &key, + at(2), + ) + .unwrap(); + assert_eq!(winner, first); + + let loser = store.append_materialized( + &cob_home(), + created.object, + &nsid, + created.tip, + &Materialized::bare(Tag::Add("dup".into())), + &key, + at(2), + ); + assert!(matches!(loser, Err(CobError::StaleTip { .. }))); + + assert_eq!( + backend::resolve_tip(&repo, &nsid, created.object).unwrap(), + Some(winner.oid()), + "shared live tip survived identical-write loser's cleanup" + ); + assert!( + store.get::(created.object).is_ok(), + "live tip wasn't deleted out from under the object" + ); + } + + #[test] + fn collect_refuses_a_graph_past_its_limit() { + let (_dir, repo) = fixture(); + let key = signer(34); + let nsid = Tag::type_name(); + let payload = Tag::Add("nel".into()).encode().unwrap(); + let root = + backend::write_change(&cob_home(), &repo, &nsid, &payload, &[], None, &key, at(0)) + .unwrap(); + let object = CobId::new(root.oid()); + let tip = (1..4).fold(root, |parent, i| { + backend::write_change( + &cob_home(), + &repo, + &nsid, + &payload, + &[parent], + Some(object), + &key, + at(i), + ) + .unwrap() + }); + + assert!(matches!( + backend::collect(&repo, tip, object, 2, None), + Err(CobError::HistoryTooLong(_)) + )); + assert_eq!( + backend::collect(&repo, tip, object, 10, None) + .unwrap() + .len(), + 4 + ); + } + + #[test] + fn changes_since_returns_a_suffix_enforces_shape_and_rejects_a_non_ancestor() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(60); + + let created = store + .create(&cob_home(), &Tag::Add("nel".into()), &key, at(1)) + .unwrap(); + let full = store.changes_since::(created.object, None).unwrap(); + assert_eq!(full.tip, created.tip); + assert_eq!( + full.changes.iter().map(|c| c.id).collect::>(), + vec![created.tip] + ); + + let second = store + .update( + &cob_home(), + created.object, + &Tag::Add("olaren".into()), + &key, + at(2), + ) + .unwrap(); + let third = store + .update( + &cob_home(), + created.object, + &Tag::Add("teq".into()), + &key, + at(3), + ) + .unwrap(); + + let delta = store + .changes_since::(created.object, Some(created.tip)) + .unwrap(); + assert_eq!(delta.tip, third); + assert_eq!( + delta.changes.iter().map(|c| c.id).collect::>(), + vec![second, third], + "delta is the appended changes in causal order instead of whole graph" + ); + + let caught_up = store + .changes_since::(created.object, Some(third)) + .unwrap(); + assert!(caught_up.changes.is_empty()); + assert_eq!(caught_up.tip, third); + + let (_diverged_dir, diverged_repo) = fixture(); + let diverged = linear_chain(&diverged_repo, &signer(71), &["nel", "olaren"]); + let stranger = ChangeId::new(knot_types::Oid::from_hex(&"a".repeat(40)).unwrap()); + assert!( + matches!( + CobStore::new(&diverged_repo).changes_since::(diverged, Some(stranger)), + Err(CobError::DivergedTip { .. }) + ), + "a since that doesn't descend from the indexed tip is refused, not applied onto stale state" + ); + + let (_forked_dir, forked_repo) = fixture(); + let forked = forked_tag_object(&forked_repo, &signer(70), "nel", "olaren", "teq"); + let forked_store = CobStore::new(&forked_repo); + assert!( + matches!( + forked_store.get::(forked), + Err(CobError::ForkedHistory { .. }) + ), + "materialization fails closed on forked linear history" + ); + assert!( + matches!( + forked_store.changes_since::(forked, None), + Err(CobError::ForkedHistory { .. }) + ), + "changes_since rejects the same fork instead of replaying it for the index" + ); + assert!( + forked_store.changes_since::(forked, None).is_ok(), + "convergent class still replays the same graph" + ); + } + + struct LinearTags; + + impl Evaluate for LinearTags { + type State = BTreeSet; + type Change = Tag; + + const HISTORY: HistoryModel = HistoryModel::Linear; + + fn initial() -> Self::State { + BTreeSet::new() + } + + fn apply(state: Self::State, change: Self::Change, author: &ActorId) -> Self::State { + Tags::apply(state, change, author) + } + } + + #[test] + fn verify_accepts_the_owner_and_rejects_a_foreign_signer_or_mismatched_type() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(51); + let object = linear_chain(&repo, &key, &["nel", "olaren"]); + let owner = ActorId::from_secp256k1(key.public_key().as_bytes()); + assert!(store.verify::(&cob_home(), object, &owner).is_ok()); + + let stranger = ActorId::from_secp256k1(signer(52).public_key().as_bytes()); + assert!(matches!( + store.verify::(&cob_home(), object, &stranger), + Err(CobError::UnverifiedChange { .. }) + )); + + let (_other_dir, other_repo) = fixture(); + let key = signer(53); + let foreign = TypeName::new("org.tangled.test.other").unwrap(); + let root = tag_root(&other_repo, &key, "nel", at(1)); + let object = CobId::new(root.oid()); + let child = backend::write_change( + &cob_home(), + &other_repo, + &foreign, + &Tag::Add("olaren".into()).encode().unwrap(), + &[root], + Some(object), + &key, + at(2), + ) + .unwrap(); + publish(&other_repo, object, child); + let owner = ActorId::from_secp256k1(key.public_key().as_bytes()); + assert!( + matches!( + CobStore::new(&other_repo).verify::(&cob_home(), object, &owner), + Err(CobError::UnexpectedChangeType { .. }) + ), + "owner-signed change whose type doesn't match namespace is refused at import" + ); + } + + fn ops_and_perm() -> impl Strategy, Vec)> { + prop::collection::vec((any::(), 0u8..4u8), 1..8usize).prop_flat_map(|ops| { + let len = ops.len(); + (Just(ops), prop::collection::vec(any::(), len)) + }) + } + + fn permutation(keys: &[u32]) -> Vec { + let mut order: Vec = (0..keys.len()).collect(); + order.sort_by_key(|&index| keys[index]); + order + } + + fn model_state(ops: &[(bool, u8)]) -> BTreeSet { + ops.iter().fold( + BTreeSet::from(["base".to_string()]), + |mut state, (is_add, subject)| { + let name = format!("s{subject}"); + if *is_add { + state.insert(name); + } else { + state.remove(&name); + } + state + }, + ) + } + + fn build_repo(ops: &[(bool, u8)], creation_order: &[usize]) -> (TempDir, Repo, CobId) { + let (dir, repo) = fixture(); + let key = signer(500); + let nsid = Tag::type_name(); + let root = tag_root(&repo, &key, "base", at(1)); + let object = CobId::new(root.oid()); + let child = |index: usize| { + let (is_add, subject) = ops[index]; + let name = format!("s{subject}"); + let payload = if is_add { + Tag::Add(name) + } else { + Tag::Remove(name) + }; + backend::write_change( + &cob_home(), + &repo, + &nsid, + &payload.encode().unwrap(), + &[root], + Some(object), + &key, + at(index as i64 + 2), + ) + .unwrap() + }; + creation_order.iter().for_each(|&index| { + child(index); + }); + let parents: Vec = (0..ops.len()).map(child).collect(); + let merge = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Remove("absent".into()).encode().unwrap(), + &parents, + Some(object), + &key, + at(ops.len() as i64 + 2), + ) + .unwrap(); + publish(&repo, object, merge); + (dir, repo, object) + } + + proptest! { + #![proptest_config(ProptestConfig { cases: 32, ..ProptestConfig::default() })] + + #[test] + fn prop_concurrent_changes_converge_to_the_causal_order((ops, keys) in ops_and_perm()) { + let identity: Vec = (0..ops.len()).collect(); + let model = model_state(&ops); + + let (_canon_dir, canon_repo, canon_object) = build_repo(&ops, &identity); + let canonical = CobStore::new(&canon_repo) + .get::(canon_object) + .unwrap() + .into_state(); + + let (_perm_dir, perm_repo, perm_object) = build_repo(&ops, &permutation(&keys)); + let permuted = CobStore::new(&perm_repo) + .get::(perm_object) + .unwrap() + .into_state(); + + prop_assert_eq!(&canonical, &model); + prop_assert_eq!(&permuted, &model); + } + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + struct Opening { + address: knot_types::RecordAddress, + author: knot_types::AccountDid, + produces: RecordCid, + text: String, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + struct Editing { + author: knot_types::AccountDid, + from: RecordCid, + produces: RecordCid, + text: String, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + struct Erasure { + author: knot_types::AccountDid, + from: RecordCid, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "op", content = "data", rename_all = "snake_case")] + enum NoteChange { + Open(Opening), + Edit(Editing), + Erase(Erasure), + } + + impl ChangePayload for NoteChange { + const TYPE: &'static str = "org.tangled.test.note"; + + fn address(&self) -> Option { + match self { + Self::Open(opening) => Some(opening.address.clone()), + Self::Edit(_) | Self::Erase(_) => None, + } + } + + fn emits(&self) -> Emits { + match self { + Self::Open(opening) => Emits::Version(opening.produces), + Self::Edit(editing) => Emits::Version(editing.produces), + Self::Erase(_) => Emits::Tombstone, + } + } + + fn collection() -> Option { + knot_types::RecordCollection::new(Self::TYPE).ok() + } + } + + impl Authored for NoteChange { + fn author(&self) -> &knot_types::AccountDid { + match self { + Self::Open(opening) => &opening.author, + Self::Edit(editing) => &editing.author, + Self::Erase(erasure) => &erasure.author, + } + } + } + + #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] + struct Note { + author: Option, + version: Option, + text: Vec, + } + + struct Notes; + + impl Evaluate for Notes { + type State = Note; + type Change = NoteChange; + + const HISTORY: HistoryModel = HistoryModel::Convergent; + + fn initial() -> Self::State { + Note::default() + } + + fn apply(mut state: Self::State, change: Self::Change, _author: &ActorId) -> Self::State { + match change { + NoteChange::Open(opening) => { + state.author = Some(opening.author); + state.version = Some(opening.produces); + state.text.push(opening.text); + } + NoteChange::Edit(editing) => { + state.version = Some(editing.produces); + state.text.push(editing.text); + } + NoteChange::Erase(_) => { + state.version = None; + state.text.clear(); + } + } + state + } + } + + impl Checkpoint for Notes { + const SNAPSHOT_STRIDE: SnapshotStride = SnapshotStride::new(4); + fn checkpoint_size(state: &Self::State) -> StateSize { + StateSize::new(state.text.len()) + } + } + + impl Authorize for Notes { + fn may_create(cx: &WriteContext<'_>) -> Decision { + cx.permission().contributes() + } + + fn may_append(cx: &WriteContext<'_>, state: &Self::State, change: &Self::Change) -> Append { + let owns = state.author.as_ref() == Some(cx.actor()); + match ( + change, + owns || cx.permission().moderates().is_allowed(), + state.version, + ) { + (NoteChange::Open(_), ..) | (_, false, _) => Append::Deny, + (_, true, None) => Append::Deny, + ( + NoteChange::Edit(Editing { from, .. }) + | NoteChange::Erase(Erasure { from, .. }), + true, + Some(current), + ) if *from != current => Append::Stale { current }, + (NoteChange::Edit(_) | NoteChange::Erase(_), true, Some(_)) => Append::Allow, + } + } + } + + fn note_address(rkey: &str) -> knot_types::RecordAddress { + knot_types::RecordAddress::new( + knot_types::RecordCollection::new("org.tangled.test.note").unwrap(), + knot_types::RecordRkey::new(rkey).unwrap(), + ) + } + + fn who(suffix: &str) -> knot_types::AccountDid { + knot_types::AccountDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn note_record(text: &str) -> (RecordBody, RecordCid) { + let body = RecordBody::new( + serde_ipld_dagcbor::to_vec(&std::collections::BTreeMap::from([( + "text", + text.to_owned(), + )])) + .unwrap(), + ) + .unwrap(); + let cid = body.cid(); + (body, cid) + } + + fn opened(author: &str, rkey: &str, text: &str) -> Materialized { + let (body, produces) = note_record(text); + Materialized { + change: NoteChange::Open(Opening { + address: note_address(rkey), + author: who(author), + produces, + text: text.to_owned(), + }), + record: Record::version(body), + } + } + + fn edited(author: &str, from: RecordCid, text: &str) -> Materialized { + let (body, produces) = note_record(text); + Materialized { + change: NoteChange::Edit(Editing { + author: who(author), + from, + produces, + text: text.to_owned(), + }), + record: Record::version(body), + } + } + + fn erased(author: &str, from: RecordCid) -> Materialized { + Materialized { + change: NoteChange::Erase(Erasure { + author: who(author), + from, + }), + record: Record::Tombstone, + } + } + fn with_record( + materialized: Materialized, + record: Record, + ) -> Materialized { + Materialized { + change: materialized.change, + record, + } + } + + fn squid() -> RepoDid { + RepoDid::new("did:plc:squid").unwrap() + } + + fn context<'a>( + actor: &'a knot_types::AccountDid, + repo: &'a RepoDid, + role: RepoRole, + ) -> WriteContext<'a> { + WriteContext::new( + actor, + repo, + ContributionPermission::new(Blocked::No, role, ContributionPolicy::Anyone), + ) + } + + fn note_append( + store: &CobStore, + object: CobId, + key: &K256Signer, + materialized: Materialized, + when: UnixSeconds, + ) -> Result, CobError> { + store.retrying_append::(&cob_home(), object, key, when, |_state| { + Ok(Some(materialized.clone())) + }) + } + fn opened_note(seed: u64) -> (TempDir, Repo, Created, RecordCid) { + let (dir, repo) = fixture(); + let first = opened("nel", "3lubrptx57d22", "kelp"); + let Emits::Version(live) = first.change.emits() else { + panic!("an opened note emits a version") + }; + let created = CobStore::new(&repo) + .create_materialized(&cob_home(), &first, &signer(seed), at(1)) + .unwrap(); + (dir, repo, created, live) + } + + #[test] + fn a_bare_change_and_a_tombstoned_record_stay_apart_through_git() { + let (_dir, repo, created, opened_cid) = opened_note(11); + let key = signer(11); + let store = CobStore::new(&repo); + note_append( + &store, + created.object, + &key, + erased("nel", opened_cid), + at(2), + ) + .unwrap(); + + let records: Vec = store + .changes_since::(created.object, None) + .unwrap() + .changes + .into_iter() + .map(|change| change.record) + .collect(); + assert_eq!( + records, + vec![Record::version(note_record("kelp").0), Record::Tombstone,], + "version this change wrote, and tombstone that erased it, live apart in git" + ); + + let tags = linear_chain(&repo, &key, &["base"]); + let bare: Vec = store + .changes_since::(tags, None) + .unwrap() + .changes + .into_iter() + .map(|change| change.record) + .collect(); + assert_eq!( + bare, + vec![Record::Bare], + "change that doesn't emit records, and change that erases one, are different facts" + ); + } + + #[test] + fn a_change_cant_move_or_invent_the_address_its_object_was_opened_at() { + let (_dir, repo, created, live) = opened_note(12); + let key = signer(12); + let store = CobStore::new(&repo); + + let moved = note_append( + &store, + created.object, + &key, + opened("nel", "3lubrptx57d33", "elsewhere"), + at(2), + ); + assert!( + matches!(moved, Err(CobError::AddressMoved { .. })), + "Object materializes at one address for its whole life, got {moved:?}" + ); + assert!( + note_append( + &store, + created.object, + &key, + opened("nel", "3lubrptx57d22", "same place"), + at(3), + ) + .is_ok(), + "Root's address is the one later change may name" + ); + + let bare = store + .create_materialized(&cob_home(), &erased("nel", live), &key, at(1)) + .unwrap(); + let invented = note_append( + &store, + bare.object, + &key, + opened("nel", "3lubrptx57d44", "late"), + at(2), + ); + assert!( + matches!(invented, Err(CobError::AddressAdded { .. })), + "Object with no record can't grow one after its root, got {invented:?}" + ); + } + + #[test] + fn a_declared_record_and_the_stored_one_are_one_or_the_write_stops() { + let (_dir, repo) = fixture(); + let key = signer(13); + let store = CobStore::new(&repo); + let (_, claimed) = note_record("kelp"); + let mismatched = store.create_materialized( + &cob_home(), + &with_record( + opened("nel", "3lubrptx57d22", "kelp"), + Record::version(note_record("someone else's words").0), + ), + &key, + at(1), + ); + assert!( + matches!(mismatched, Err(CobError::RecordMisclaimed { .. })), + "named version and stored version differ, got {mismatched:?}" + ); + let unwritten = store.create_materialized( + &cob_home(), + &with_record(opened("nel", "3lubrptx57d22", "kelp"), Record::Bare), + &key, + at(1), + ); + assert!( + matches!(unwritten, Err(CobError::RecordUnwritten { .. })), + "Change that names its record has to store said record, got {unwritten:?}" + ); + let erasing = store.create_materialized( + &cob_home(), + &with_record( + erased("nel", claimed), + Record::version(note_record("kelp").0), + ), + &key, + at(1), + ); + assert!( + matches!(erasing, Err(CobError::TombstoneUnwritten)), + "Change that erases its record stores tombstone, not version, got {erasing:?}" + ); + } + + #[test] + fn impersonated_bylines_never_reach_git() { + let nel = who("nel"); + let repo_did = squid(); + let cx = context(&nel, &repo_did, RepoRole::Stranger); + let impersonation = Approved::signed_by(&cx, opened("olaren", "3lubrptx57d22", "kelp")); + assert!( + matches!(impersonation, Err(CobError::Impersonation { .. })), + "Engine, not handler, is where stranger's byline gets turned away" + ); + assert!(Approved::signed_by(&cx, opened("nel", "3lubrptx57d22", "kelp")).is_ok()); + } + + #[test] + fn the_policy_opens_an_object_and_the_author_or_a_moderator_edits_it() { + let (_dir, repo) = fixture(); + let key = signer(14); + let store = CobStore::new(&repo); + let (nel, olaren) = (who("nel"), who("olaren")); + + let repo_did = squid(); + let closed = WriteContext::new( + &nel, + &repo_did, + ContributionPermission::new( + Blocked::No, + RepoRole::Stranger, + ContributionPolicy::Collaborators, + ), + ); + let refused = store.create_approved::( + &cob_home(), + &Approved::signed_by(&closed, opened("nel", "3lubrptx57d22", "kelp")).unwrap(), + &key, + at(1), + ); + let sentence = refused.as_ref().unwrap_err().to_string(); + assert!( + matches!(refused, Err(CobError::CreateRejected { .. })) + && sentence.contains("stranger, where contribution policy is collaborators"), + "Stranger can't open objects on a collaborator-only repository, \ + and 403 sentence names policy, got {sentence}" + ); + + let author = context(&nel, &repo_did, RepoRole::Stranger); + let first = opened("nel", "3lubrptx57d22", "kelp"); + let Emits::Version(live) = first.change.emits() else { + panic!("an opened note emits a version") + }; + let created = store + .create_approved::( + &cob_home(), + &Approved::signed_by(&author, first).unwrap(), + &key, + at(1), + ) + .unwrap(); + + let stranger = context(&olaren, &repo_did, RepoRole::Stranger); + let intruded = store.append_approved::( + &cob_home(), + created.object, + &Approved::signed_by(&stranger, edited("olaren", live, "not yours")).unwrap(), + &key, + at(2), + ); + assert!( + matches!(intruded, Err(CobError::AppendRejected { .. })), + "Edit is author's, got {intruded:?}" + ); + + let maintainer = context(&olaren, &repo_did, RepoRole::Collaborator); + assert!( + store + .append_approved::( + &cob_home(), + created.object, + &Approved::signed_by(&maintainer, erased("olaren", live)).unwrap(), + &key, + at(3), + ) + .is_ok(), + "Maintainer override is what makes the operator able to moderate" + ); + } + + #[test] + fn two_edits_from_one_materialized_state_end_as_one_write_and_one_conflict() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let did = squid(); + let repo = layout.create(&did).unwrap(); + let key = signer(15); + let nel = who("nel"); + let cx = context(&nel, &did, RepoRole::Stranger); + let first = opened("nel", "3lubrptx57d22", "kelp"); + let Emits::Version(live) = first.change.emits() else { + panic!("an opened note emits a version") + }; + let object = CobStore::new(&repo) + .create_approved::( + &cob_home(), + &Approved::signed_by(&cx, first).unwrap(), + &key, + at(1), + ) + .unwrap() + .object; + drop(repo); + + let outcomes: Vec> = std::thread::scope(|scope| { + let handles: Vec<_> = ["one", "two"] + .into_iter() + .map(|text| { + let layout = layout.clone(); + let did = &did; + let key = &key; + let nel = &nel; + scope.spawn(move || { + let repo = layout.open(did).unwrap(); + let cx = context(nel, did, RepoRole::Stranger); + CobStore::new(&repo).append_approved::( + &cob_home(), + object, + &Approved::signed_by(&cx, edited("nel", live, text)).unwrap(), + key, + at(2), + ) + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().expect("Edit thread runs to its answer")) + .collect() + }); + + assert_eq!( + outcomes.iter().filter(|outcome| outcome.is_ok()).count(), + 1, + "one edit from a shared version is stored, got {outcomes:?}" + ); + let expected_uri = note_address("3lubrptx57d22").at_uri(&did); + assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!( + outcome, + Err(CobError::StaleRecord { current, uri, .. }) + if *current != live && *uri == expected_uri + )) + .count(), + 1, + "loser reads at-uri and current version, for rebasing, got {outcomes:?}" + ); + } + + fn note_change( + repo: &Repo, + key: &K256Signer, + materialized: &Materialized, + parents: &[ChangeId], + object: Option, + when: UnixSeconds, + ) -> ChangeId { + backend::write_change_with_record( + &cob_home(), + repo, + &NoteChange::type_name(), + &materialized.change.encode().unwrap(), + &materialized.record, + parents, + object, + key, + when, + ) + .unwrap() + } + + #[test] + fn a_forked_social_history_replays_through_the_convergent_arm() { + let (_dir, repo) = fixture(); + let key = signer(16); + let store = CobStore::new(&repo); + let root_change = opened("nel", "3lubrptx57d22", "kelp"); + let Emits::Version(live) = root_change.change.emits() else { + panic!("an opened note emits a version") + }; + let root = note_change(&repo, &key, &root_change, &[], None, at(1)); + let object = CobId::new(root.oid()); + + let left_change = edited("nel", live, "left"); + let right_change = edited("nel", live, "right"); + let left = note_change(&repo, &key, &left_change, &[root], Some(object), at(2)); + let right = note_change(&repo, &key, &right_change, &[root], Some(object), at(3)); + let merge_change = edited("nel", live, "merged"); + let merge = note_change( + &repo, + &key, + &merge_change, + &[left, right], + Some(object), + at(4), + ); + repo.update_ref(&RefUpdate::Create { + name: backend::cob_ref_name(&NoteChange::type_name(), object).unwrap(), + new: merge.oid(), + }) + .unwrap(); + + let replayed = store.get::(object).unwrap(); + assert_eq!( + replayed.state().text, + vec!["kelp", "left", "right", "merged"], + "Causal order puts fork's branches in timestamp order under merge" + ); + let Emits::Version(merged) = merge_change.change.emits() else { + panic!("an edit emits a version") + }; + assert_eq!( + replayed.state().version, + Some(merged), + "the current version is the merge's, which replay order reaches last" + ); + let again = store.get::(object).unwrap(); + assert_eq!( + serde_ipld_dagcbor::to_vec(replayed.state()).unwrap(), + serde_ipld_dagcbor::to_vec(again.state()).unwrap(), + "a second replay of one forked history reaches byte-identical state" + ); + } + + fn note_fan(creation_order: &[usize]) -> (TempDir, Repo, CobId) { + let (dir, repo) = fixture(); + let key = signer(600); + let root_change = opened("nel", "3lubrptx57d22", "kelp"); + let Emits::Version(live) = root_change.change.emits() else { + panic!("an opened note emits a version") + }; + let root = note_change(&repo, &key, &root_change, &[], None, at(1)); + let object = CobId::new(root.oid()); + let edit = |index: usize| { + let change = edited("nel", live, &format!("edit{index}")); + note_change( + &repo, + &key, + &change, + &[root], + Some(object), + at(index as i64 + 2), + ) + }; + creation_order.iter().for_each(|&index| { + edit(index); + }); + let parents: Vec = (0..creation_order.len()).map(edit).collect(); + let merge = note_change( + &repo, + &key, + &edited("nel", live, "merged"), + &parents, + Some(object), + at(creation_order.len() as i64 + 2), + ); + repo.update_ref(&RefUpdate::Create { + name: backend::cob_ref_name(&NoteChange::type_name(), object).unwrap(), + new: merge.oid(), + }) + .unwrap(); + (dir, repo, object) + } + + proptest! { + #![proptest_config(ProptestConfig { cases: 32, ..ProptestConfig::default() })] + + #[test] + fn prop_a_social_fork_replays_the_same_whatever_order_its_changes_arrived( + keys in prop::collection::vec(any::(), 1..8usize) + ) { + let identity: Vec = (0..keys.len()).collect(); + let (_canon_dir, canon_repo, canon_object) = note_fan(&identity); + let canonical = CobStore::new(&canon_repo).get::(canon_object).unwrap(); + let (_perm_dir, perm_repo, perm_object) = note_fan(&permutation(&keys)); + let permuted = CobStore::new(&perm_repo).get::(perm_object).unwrap(); + + let expected: Vec = std::iter::once("kelp".to_owned()) + .chain((0..keys.len()).map(|index| format!("edit{index}"))) + .chain(std::iter::once("merged".to_owned())) + .collect(); + prop_assert_eq!(&canonical.state().text, &expected); + prop_assert_eq!( + serde_ipld_dagcbor::to_vec(canonical.state()).unwrap(), + serde_ipld_dagcbor::to_vec(permuted.state()).unwrap(), + "Fold order is causal sort, not the order git took objects in" + ); + } + } + + #[test] + fn a_fork_of_one_edit_replays_root_edit_and_merge_once_each() { + let (_dir, repo, object) = note_fan(&[0]); + let replayed = CobStore::new(&repo).get::(object).unwrap(); + assert_eq!( + replayed.state().text, + vec!["kelp", "edit0", "merged"], + "replay order is causal sort, not the order git took objects in" + ); + } + + #[test] + fn a_merge_that_forks_below_a_checkpoint_replays_from_the_root_instead_of_double_counting() { + let (_dir, repo, created, mut live) = opened_note(17); + let key = signer(17); + let store = CobStore::new(&repo); + let object = created.object; + let below = (2..=Notes::SNAPSHOT_STRIDE.get() as i64).fold(created.tip, |_, step| { + let edit = edited("nel", live, &format!("step{step}")); + let Emits::Version(next) = edit.change.emits() else { + panic!("an edit emits a version") + }; + live = next; + note_append(&store, object, &key, edit, at(step)) + .unwrap() + .expect("Edit is stored") + }); + let snapshot = backend::checkpoint_ref_name(&NoteChange::type_name(), object).unwrap(); + assert!( + repo.find_ref(&snapshot).unwrap().is_some(), + "stride has to have written its checkpoint, or this test doesn't prove anything" + ); + + let root = ChangeId::new(object.oid()); + let side_change = edited("nel", live, "side"); + let side = note_change(&repo, &key, &side_change, &[root], Some(object), at(20)); + let merge_change = edited("nel", live, "merged"); + let merge = note_change( + &repo, + &key, + &merge_change, + &[below, side], + Some(object), + at(21), + ); + repo.update_ref(&RefUpdate::Update { + name: backend::cob_ref_name(&NoteChange::type_name(), object).unwrap(), + old: below.oid(), + new: merge.oid(), + }) + .unwrap(); + + let (materialized, tip) = store.materialize::(object).unwrap(); + let replayed = store.get::(object).unwrap(); + assert_eq!(tip, merge); + assert_eq!( + materialized, + *replayed.state(), + "checkpoint is abandoned, rather than trusted, delta under that mr checkpoint would count its prefix twice" + ); + assert_eq!( + materialized.text.len(), + Notes::SNAPSHOT_STRIDE.get() + 2, + "Fold counts each change once: root, its edits, side and merge" + ); + } + + #[test] + fn a_failed_chain_leaves_the_prefix_another_writer_already_published() { + let (_dir, repo, created, live) = opened_note(18); + let key = signer(18); + let store = CobStore::new(&repo); + + let shared = edited("nel", live, "shared"); + let published = note_append(&store, created.object, &key, shared.clone(), at(2)) + .unwrap() + .expect("First writer stores its edit"); + + let refused = store.chain( + &cob_home(), + created.object, + &NoteChange::type_name(), + created.tip, + [ + (&shared.change, &shared.record, at(2)), + ( + &edited("nel", live, "unwritten").change, + &Record::Bare, + at(3), + ), + ], + &key, + ); + assert!( + matches!(refused, Err(CobError::RecordUnwritten { .. })), + "Second change of chain names record, which said change doesn't store, got {refused:?}" + ); + + assert_eq!( + backend::resolve_tip(&repo, &NoteChange::type_name(), created.object).unwrap(), + Some(published.oid()), + "Loser's cleanup left winner's tip alone" + ); + assert!( + store.get::(created.object).is_ok(), + "two chains sharing one prefix are content-addressed to one commit. Loser \ + mustn't pull it out from under winner" + ); + } + + #[test] + fn a_change_past_the_byte_limit_doesnt_reach_git() { + let (_dir, repo) = fixture(); + let key = signer(19); + let store = CobStore::new(&repo); + let fat = "x".repeat(knot_types::MAX_RECORD_BYTES + 1); + let refused = store.create(&cob_home(), &Tag::Add(fat), &key, at(1)); + assert!( + matches!( + refused, + Err(CobError::ChangeTooLarge { + blob: "payload", + .. + }) + ), + "Payload past limit is refused before blob is written, got {refused:?}" + ); + assert!( + loose_commits(&repo).is_empty(), + "Change store turned away didn't reach any commit" + ); + } + + #[test] + fn a_replayed_create_answers_with_the_object_it_already_made() { + let (_dir, repo) = fixture(); + let key = signer(20); + let store = CobStore::new(&repo); + let first = store + .create(&cob_home(), &Tag::Add("base".into()), &key, at(1)) + .unwrap(); + let replayed = store + .create(&cob_home(), &Tag::Add("base".into()), &key, at(1)) + .unwrap(); + assert_eq!( + replayed, first, + "content-addressed create makes client's retry, after timeout, return same object, rather than error about ref that already exists" + ); + + store + .update( + &cob_home(), + first.object, + &Tag::Add("later".into()), + &key, + at(2), + ) + .unwrap(); + let grown = store.create(&cob_home(), &Tag::Add("base".into()), &key, at(1)); + assert!( + matches!(grown, Err(CobError::StaleTip { .. })), + "Replay of root whose object has moved on conflicts, got {grown:?}" + ); + } + + #[test] + fn the_alias_table_maps_both_ways() { + let legacy = TypeName::new("sh.tangled.repo.collaborator").unwrap(); + let modern = TypeName::new("org.tangled.graph.collaborator").unwrap(); + assert_eq!(modernized_type_of(&legacy), Some(modern.clone())); + assert_eq!(legacy_type_of(&modern), Some(legacy.clone())); + assert!(accepts_change_type(&modern, &legacy)); + assert!(accepts_change_type(&modern, &modern)); + let stranger = TypeName::new("org.tangled.feed.comment").unwrap(); + assert!(!accepts_change_type(&modern, &stranger)); + assert_eq!(legacy_type_of(&stranger), None); + assert_eq!(modernized_type_of(&stranger), None); + } + + #[test] + fn legacy_cob_refs_rename_to_their_modern_type_once() { + let (dir, repo) = fixture(); + let object = CobId::new(Oid::from_hex("4444444444444444444444444444444444444444").unwrap()); + let legacy = RefName::new(format!( + "refs/cobs/sh.tangled.knot.member/{}", + object.oid().to_hex() + )) + .unwrap(); + let tip = Oid::from_hex("5555555555555555555555555555555555555555").unwrap(); + repo.set_reference(&legacy, tip).unwrap(); + let legacy_checkpoint = RefName::new(format!( + "refs/cob-checkpoints/sh.tangled.repo.collaborator/{}", + object.oid().to_hex() + )) + .unwrap(); + repo.set_reference(&legacy_checkpoint, tip).unwrap(); + + assert_eq!(migrate_legacy_refs(&repo).unwrap(), 2); + let modern = + cob_ref_name(&TypeName::new("org.tangled.knot.member").unwrap(), object).unwrap(); + assert_eq!(repo.find_ref(&modern).unwrap(), Some(tip)); + assert_eq!(repo.find_ref(&legacy).unwrap(), None); + let modern_checkpoint = checkpoint_ref_name( + &TypeName::new("org.tangled.graph.collaborator").unwrap(), + object, + ) + .unwrap(); + assert_eq!(repo.find_ref(&modern_checkpoint).unwrap(), Some(tip)); + assert_eq!(repo.find_ref(&legacy_checkpoint).unwrap(), None); + assert_eq!( + migrate_legacy_refs(&repo).unwrap(), + 0, + "a second sweep finds no refs left to rename" + ); + drop(dir); + } +} diff --git a/knot2/crates/knot-cob/src/object.rs b/knot2/crates/knot-cob/src/object.rs new file mode 100644 index 0000000000000000000000000000000000000000..15299bad55d6522b6fe9fb4d58959f8ff07ba33e --- /dev/null +++ b/knot2/crates/knot-cob/src/object.rs @@ -0,0 +1,159 @@ +use knot_types::{ + AccountDid, ActorId, ChangeId, CobId, ContributionPermission, Decision, RecordCid, RepoDid, + TypeName, +}; + +use crate::change::{Change, ChangePayload}; +use crate::error::CobError; +use crate::graph::{ChangeGraph, History}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HistoryModel { + Linear, + Convergent, +} + +pub trait Evaluate { + type State; + type Change: ChangePayload; + + const HISTORY: HistoryModel; + + fn initial() -> Self::State; + fn apply(state: Self::State, change: Self::Change, author: &ActorId) -> Self::State; +} + +pub struct WriteContext<'a> { + actor: &'a AccountDid, + repo: &'a RepoDid, + permission: ContributionPermission, +} + +impl<'a> WriteContext<'a> { + pub const fn new( + actor: &'a AccountDid, + repo: &'a RepoDid, + permission: ContributionPermission, + ) -> Self { + Self { + actor, + repo, + permission, + } + } + + pub const fn actor(&self) -> &'a AccountDid { + self.actor + } + + pub const fn repo(&self) -> &'a RepoDid { + self.repo + } + + pub const fn permission(&self) -> ContributionPermission { + self.permission + } +} + +pub trait Authored { + fn author(&self) -> &AccountDid; +} + +pub trait KnotAuthored: ChangePayload {} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[must_use] +pub enum Append { + Allow, + Deny, + Stale { current: RecordCid }, +} + +pub trait Authorize: Evaluate +where + Self::Change: Authored, +{ + fn may_create(cx: &WriteContext<'_>) -> Decision; + fn may_append(cx: &WriteContext<'_>, state: &Self::State, change: &Self::Change) -> Append; +} + +knot_types::scalar_newtype! { + pub struct SnapshotStride(usize); + pub struct StateSize(usize); +} + +pub trait Checkpoint: Evaluate { + const SNAPSHOT_STRIDE: SnapshotStride; + fn checkpoint_size(state: &Self::State) -> StateSize; +} + +pub(crate) fn apply_changes( + state: E::State, + changes: &[Change], + expected: &TypeName, +) -> Result { + changes.iter().try_fold(state, |state, change| { + if !crate::accepts_change_type(expected, &change.type_name) { + return Err(CobError::UnexpectedChangeType { + change: change.id, + expected: expected.clone(), + found: change.type_name.clone(), + }); + } + let payload = + E::Change::decode(change.payload()).map_err(|error| CobError::UndecodableChange { + change: change.id, + reason: error.to_string(), + })?; + Ok(E::apply(state, payload, &change.author)) + }) +} + +pub(crate) fn evaluate( + graph: ChangeGraph, + expected: &TypeName, +) -> Result<(E::State, History), CobError> { + let root = ChangeId::new(graph.root().oid()); + let ordered = graph.into_ordered(); + let state = apply_changes::(E::initial(), &ordered, expected)?; + Ok((state, History::new(root, ordered))) +} + +#[derive(Debug)] +pub struct Object { + id: CobId, + type_name: TypeName, + state: S, + history: History, +} + +impl Object { + pub(crate) fn new(id: CobId, type_name: TypeName, state: S, history: History) -> Self { + Self { + id, + type_name, + state, + history, + } + } + + pub fn id(&self) -> CobId { + self.id + } + + pub fn type_name(&self) -> &TypeName { + &self.type_name + } + + pub fn state(&self) -> &S { + &self.state + } + + pub fn into_state(self) -> S { + self.state + } + + pub fn history(&self) -> &History { + &self.history + } +} diff --git a/knot2/crates/knot-cobs/Cargo.toml b/knot2/crates/knot-cobs/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..38d31f48079eb66a7c672171bc864e510aec5147 --- /dev/null +++ b/knot2/crates/knot-cobs/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "knot-cobs" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-cob = { workspace = true } +knot-record = { workspace = true } +knot-runtime = { workspace = true } +serde = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +knot-git = { workspace = true } +knot-runtime = { workspace = true } +tempfile = { workspace = true } +gix = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +proptest = { workspace = true } diff --git a/knot2/crates/knot-cobs/fuzz/.gitignore b/knot2/crates/knot-cobs/fuzz/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1a45eee7760d240bfa8ac1989088e0349169f264 --- /dev/null +++ b/knot2/crates/knot-cobs/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-cobs/fuzz/Cargo.lock b/knot2/crates/knot-cobs/fuzz/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..35ebe74c36744b087bb587cb091e1b03e7371299 --- /dev/null +++ b/knot2/crates/knot-cobs/fuzz/Cargo.lock @@ -0,0 +1,5398 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +dependencies = [ + "serde", +] + +[[package]] +name = "bytesize" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e78e506b9d7633710dab98996f22f95f3d0f488e8f1aa162830556ed9fc14d" + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "clru" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "gix" +version = "0.84.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae54ae0ebd1a5a3c3f8d95dd3b5ca6e63f4fed9bfd585e13801a97d7bde8f9ce" +dependencies = [ + "gix-actor", + "gix-archive", + "gix-attributes", + "gix-blame", + "gix-command", + "gix-commitgraph", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-ignore", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-negotiate", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-pathspec", + "gix-prompt", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-shallow", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "gix-worktree", + "gix-worktree-state", + "gix-worktree-stream", + "nonempty", + "parking_lot", + "regex", + "signal-hook", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-actor" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc998b8f746dda8565450d08a63b792ced9165d8c27a1ed3f02799ec6a7820f" +dependencies = [ + "bstr", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-archive" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16909cacc78936ab96f6c3be08379d0a2e88bfa3a7527972d2ed75c7517ef31e" +dependencies = [ + "bstr", + "flate2", + "gix-date", + "gix-error", + "gix-object", + "gix-path", + "gix-worktree-stream", + "rawzip", + "tar", +] + +[[package]] +name = "gix-attributes" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d43f12e246d3bf7ec624c8fc15ac4a4b62b7c4c6f586cb82be6c90bf84c9d02" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-bitmap" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ebef0c26ad305747649e727bbcd56a7b7910754eb7cea88f6dff6f93c51283" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-blame" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d39a0c14af94c2edaa5eefe06d5ef2cdea55316ae9a9321314288e3f55fa4c0" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-diff", + "gix-error", + "gix-hash", + "gix-object", + "gix-revwalk", + "gix-trace", + "gix-traverse", + "gix-worktree", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-chunk" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9faee47943b638e58ddd5e275a4906ad3e4b6c8584f1d41bd18ab9032ec52afb" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-command" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00706d4fef135ef4b01680d5218c6ee40cda8baf697b864296cbc887d19118f6" +dependencies = [ + "bstr", + "gix-path", + "gix-quote", + "gix-trace", + "shell-words", +] + +[[package]] +name = "gix-commitgraph" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" +dependencies = [ + "bstr", + "gix-chunk", + "gix-error", + "gix-hash", + "memmap2", + "nonempty", +] + +[[package]] +name = "gix-config" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2372d4b49ca28431e7d150cab9d25edc1890f0184bd57eb0e917c7799e63de" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-config-value" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" +dependencies = [ + "bitflags", + "bstr", + "gix-path", + "libc", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-credentials" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40cd22f0dd71988be12d6e78b1709de2370e1957c5f107ff31e56caeba3745d" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-date", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-trace", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-date" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ecab64a98bbac9f8e02990a9ea5e3c974a7d49b95f2bd70ad94ad22fa6b48c" +dependencies = [ + "bstr", + "gix-error", + "itoa", + "jiff", +] + +[[package]] +name = "gix-diff" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6d9528f32d94cef2edf39a1ac01fe5a0fc44ddbb18d9e44099936047c3302b" +dependencies = [ + "bstr", + "gix-attributes", + "gix-command", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-imara-diff", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-dir" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bb2a53a6fd917ec499ed0bfb5b6887de7a15bd79197dcea7c987938749a9f1" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-discover" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77bacdd12b7879d2178a80c58c2f319995e4654e1a7a23e3181e5c8a12b824f7" +dependencies = [ + "bstr", + "dunce", + "gix-fs", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-error" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57831e199be480af90dcd7e459abed8a174c09ec9a6e2cc8f7ca6c54598b06b" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "bytes", + "bytesize", + "crc32fast", + "crossbeam-channel", + "gix-path", + "gix-trace", + "gix-utils", + "libc", + "once_cell", + "parking_lot", + "prodash", + "thiserror 2.0.18", + "walkdir", + "zlib-rs", +] + +[[package]] +name = "gix-filter" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf74b7d16f6694ce4a3049074c41be0c7987105743674f1671807bd6dce09fa" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash", + "gix-object", + "gix-packetline", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-fs" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" +dependencies = [ + "bstr", + "fastrand", + "gix-features", + "gix-path", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-glob" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" +dependencies = [ + "bitflags", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-hashtable" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0e30b93eea8718baf7d8153fcb938e2926175bbf18097c09f1c01b6f0be0563" +dependencies = [ + "gix-hash", + "hashbrown 0.17.1", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-imara-diff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19753d40da53d0ec41604750eeb969097a90fb2d7f7992730d904541c04e2c19" +dependencies = [ + "bstr", + "hashbrown 0.15.5", +] + +[[package]] +name = "gix-index" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6b28cc592dc753adb58302bb14a64e412ee591a3bec77aa4df87bff74fa80d" +dependencies = [ + "bitflags", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils", + "gix-validate", + "hashbrown 0.17.1", + "itoa", + "libc", + "memmap2", + "rustix", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-lock" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-mailmap" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195fd20808055824531be2fd0d34136d900e5fbca3ffb0a3c07e8beeefb9c828" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-negotiate" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890c936a215bae25818c076cb881cb2e54d2c66ba947ba58b8dd47cff921bf55" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-object", + "gix-revwalk", +] + +[[package]] +name = "gix-object" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5cd857e29429c7213bdef3f5aef83f8cc124774fe8ae0d27b1607d218d6d525" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-odb" +version = "0.81.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d004c32858b1556f2d7874405edb3c97dc78fc09beaa87d57bb077ee2858a7d" +dependencies = [ + "arc-swap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "memmap2", + "parking_lot", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pack" +version = "0.71.0" +dependencies = [ + "clru", + "gix-chunk", + "gix-diff", + "gix-error", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-traverse", + "parking_lot", + "smallvec", + "thiserror 2.0.18", + "uluru", +] + +[[package]] +name = "gix-packetline" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b217dd0ee0c4021ecf169a4a519b1b4f80d15e3f3765f3dc466223dc0ac891d7" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-path" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa6ac14cd14939ea94a496ce7460daa6511c09f5b84757e9cfc6f9c8d0f93a6" +dependencies = [ + "bstr", + "gix-trace", + "gix-validate", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pathspec" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" +dependencies = [ + "bitflags", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-prompt" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee604d7746080ae7e1023bf47204bcc2c5f307bfbe2306a3c90b1bfd1a2c6d8" +dependencies = [ + "gix-command", + "gix-config-value", + "parking_lot", + "rustix", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-protocol" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51dea3acb390707ab868f1f9584f18449eb95d869deffae96768e47d303595ee" +dependencies = [ + "bstr", + "gix-date", + "gix-features", + "gix-hash", + "gix-ref", + "gix-shallow", + "gix-transport", + "gix-utils", + "maybe-async", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-quote" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" +dependencies = [ + "bstr", + "gix-error", + "gix-utils", +] + +[[package]] +name = "gix-ref" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c04f64c37eb7e6feb73c7060f8dc6f381cc5de5d53249bfd450bc48a86b2e8b" +dependencies = [ + "gix-actor", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils", + "gix-validate", + "memmap2", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-refspec" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b216ae06ec74b5f24ad0142026a997fb0a935b7410eaf9c1616fc3f0e6c5a6d3" +dependencies = [ + "bstr", + "gix-error", + "gix-glob", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-revision" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b47c88884dd3c1a19a39da19d10211fcdea2809aadc86869b6e824a1774340f" +dependencies = [ + "bitflags", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "nonempty", +] + +[[package]] +name = "gix-revwalk" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f5756abffe0917827aac683b13684ed99875bc398fa1f9b8f479b0681ef9e6" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-sec" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8519976e4c7e486270740a5400369f37940779b80bd1377d94cfa1125d01b3" +dependencies = [ + "bitflags", + "gix-path", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "gix-shallow" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" +dependencies = [ + "bstr", + "gix-hash", + "gix-lock", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-status" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22042e385d28a34275e029d98f4970285045be14b9073658ca897923f2ed8700" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-submodule" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3059890ef054066c22a94bfc6a3eaba0d806aedcd630a0bc9e5783fd88884781" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-tempfile" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27850097e1ff9515f46a0dad0f5f9c9d020e972727772dabab9450690c4adb22" +dependencies = [ + "dashmap", + "gix-fs", + "libc", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "gix-transport" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd0e34995b1aab0fa8dff2af8db726a0bfad3e119c89302604463264046e7ff" +dependencies = [ + "bstr", + "gix-command", + "gix-features", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-traverse" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8de590ecc86a3b2870665f2288324fa9f7f8672c7fc2d4e020fdd81cd1f7aed" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-url" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bb01ec69d55e82ccb7a19e264501ead4e6aac38463a8cebfdd81e22bb67ab2" +dependencies = [ + "bstr", + "gix-path", + "percent-encoding", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-utils" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c50966184123caf580ffa64e28031a878597f1c7fceb8fe19566c38eb1b771" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc6fc771c4063ba7cd2f47b91fb6076251c6a823b64b7fe7b8874b0fe4afae3" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-worktree" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef414ed275e8407cd5d53d301e83be19700b0dd3f859d2434417b58f454a2d1" +dependencies = [ + "bstr", + "gix-attributes", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate", +] + +[[package]] +name = "gix-worktree-state" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bffae8b3ca258fdd50370cd51f06deb4c76a3b43db3868bc28dde45ffa77d69" +dependencies = [ + "bstr", + "gix-features", + "gix-filter", + "gix-fs", + "gix-index", + "gix-object", + "gix-path", + "gix-worktree", + "io-close", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-worktree-stream" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25e9ed30100c63f7590bc581c225e53f731a53e06aa79a245739c07f7dcc557" +dependencies = [ + "gix-attributes", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-traverse", + "parking_lot", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "human_format" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaec953f16e5bcf6b8a3cb3aa959b17e5577dbd2693e94554c462c08be22624b" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest 0.12.28", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jiff" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-static" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-cob" +version = "0.1.0" +dependencies = [ + "gix", + "gix-hash", + "k256", + "knot-git", + "knot-resource", + "knot-runtime", + "knot-types", + "serde", + "serde_ipld_dagcbor", + "thiserror 2.0.18", + "tracing", +] + +[[package]] +name = "knot-cobs" +version = "0.1.0" +dependencies = [ + "knot-cob", + "knot-runtime", + "knot-types", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "knot-cobs-fuzz" +version = "0.0.0" +dependencies = [ + "knot-cobs", + "libfuzzer-sys", +] + +[[package]] +name = "knot-git" +version = "0.1.0" +dependencies = [ + "base64", + "dashmap", + "flate2", + "gix", + "gix-archive", + "gix-bitmap", + "gix-hash", + "gix-pack", + "knot-resource", + "knot-types", + "moka", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "knot-resource" +version = "0.1.0" +dependencies = [ + "rustix", +] + +[[package]] +name = "knot-runtime" +version = "0.1.0" +dependencies = [ + "bytes", + "futures", + "getrandom 0.4.3", + "http", + "k256", + "reqwest 0.13.1", + "thiserror 2.0.18", + "tokio", + "url", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multibase" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" +dependencies = [ + "base-x", + "base256emoji", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nonempty" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "bytesize", + "human_format", + "parking_lot", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "reqwest" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uluru" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8a2469e56e6e5095c82ccd3afb98dad95f7af7929aab6d8ba8d6e0f73657da" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/knot2/crates/knot-cobs/fuzz/Cargo.toml b/knot2/crates/knot-cobs/fuzz/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..3ab67e5f6a4140a884f755d71524e94c4034505b --- /dev/null +++ b/knot2/crates/knot-cobs/fuzz/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "knot-cobs-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-cobs] +path = ".." + +[[bin]] +name = "cob_change" +path = "fuzz_targets/cob_change.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "cob_ref" +path = "fuzz_targets/cob_ref.rs" +test = false +doc = false +bench = false + +[patch.crates-io] +gix-pack = { path = "../../../third_party/gix-pack" } diff --git a/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_change.rs b/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_change.rs new file mode 100644 index 0000000000000000000000000000000000000000..bf91822178c66cd10321e8b2d1aae763d886802c --- /dev/null +++ b/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_change.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_cobs::fuzz::change_decode(data); +}); diff --git a/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_ref.rs b/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_ref.rs new file mode 100644 index 0000000000000000000000000000000000000000..24cdd34baeb1a8e09630698d6116fab0c4acec32 --- /dev/null +++ b/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_ref.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_cobs::fuzz::ref_parse(data); +}); diff --git a/knot2/crates/knot-cobs/src/artifact.rs b/knot2/crates/knot-cobs/src/artifact.rs new file mode 100644 index 0000000000000000000000000000000000000000..1eec46483b7b12bd4e42998b34e7126ca7c76d4c --- /dev/null +++ b/knot2/crates/knot-cobs/src/artifact.rs @@ -0,0 +1,44 @@ +use knot_types::{BlobCid, RecordBody}; +use serde::{Deserialize, Serialize}; + +use knot_record::artifact::ARTIFACT_COLLECTION; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct Artifact; + +#[derive(Deserialize)] +struct StoredOid { + #[serde(rename = "oid")] + _oid: String, +} + +#[derive(Deserialize)] +struct StoredBlob { + #[serde(rename = "ref")] + _cid: BlobCid, +} + +#[derive(Deserialize)] +struct StoredArtifact { + #[serde(rename = "subject")] + _subject: StoredOid, + #[serde(rename = "name")] + _name: String, + #[serde(rename = "artifact")] + _artifact: StoredBlob, +} + +fn decode_stored(body: &RecordBody) -> Result { + serde_ipld_dagcbor::from_slice::(body.as_bytes()) + .map(|_| Artifact) + .map_err(|error| error.to_string()) +} + +crate::social_cob! { + body = Artifact, + change = ArtifactChange, + cob = ArtifactCob, + collection = ARTIFACT_COLLECTION, + opening = crate::Opening::Triage, + stored = decode_stored, +} diff --git a/knot2/crates/knot-cobs/src/blocklist.rs b/knot2/crates/knot-cobs/src/blocklist.rs new file mode 100644 index 0000000000000000000000000000000000000000..61c73ed3902b6c5651733ea950d86786627987ed --- /dev/null +++ b/knot2/crates/knot-cobs/src/blocklist.rs @@ -0,0 +1,9 @@ +use crate::grant::grant_cob; + +grant_cob! { + change = BlocklistChange, + ops = { Add(Grant), Remove(Removal) }, + cob = BlocklistCob, + state = Blocklist, + type_name = "org.tangled.knot.block", +} diff --git a/knot2/crates/knot-cobs/src/collaborators.rs b/knot2/crates/knot-cobs/src/collaborators.rs new file mode 100644 index 0000000000000000000000000000000000000000..4e5e18b88601d7ef8c91484c2f0c50eff4217488 --- /dev/null +++ b/knot2/crates/knot-cobs/src/collaborators.rs @@ -0,0 +1,9 @@ +use crate::grant::grant_cob; + +grant_cob! { + change = CollaboratorsChange, + ops = { Add(Grant), Invite(Invite), Accept(Accept), Remove(Removal) }, + cob = CollaboratorsCob, + state = Collaborators, + type_name = "org.tangled.graph.collaborator", +} diff --git a/knot2/crates/knot-cobs/src/comment.rs b/knot2/crates/knot-cobs/src/comment.rs new file mode 100644 index 0000000000000000000000000000000000000000..1f4d10ed333607ec5752db93e561301ebb593e36 --- /dev/null +++ b/knot2/crates/knot-cobs/src/comment.rs @@ -0,0 +1,161 @@ +use serde::{Deserialize, Serialize}; + +use knot_record::comment::CommentRef; +use knot_types::{AtUri, RecordAddress, RecordBody, RecordCid}; + +use crate::Opening; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Comment { + pub subject: RecordCid, + pub reply: Option, +} + +#[derive(Deserialize)] +pub(crate) struct StoredRef { + pub(crate) uri: AtUri, + pub(crate) cid: StoredCid, +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub(crate) enum StoredCid { + Link(RecordCid), + Text(String), +} + +impl StoredCid { + pub(crate) fn resolve(self) -> Result { + match self { + StoredCid::Link(cid) => Ok(cid), + StoredCid::Text(sent) => RecordCid::parse(&sent) + .ok_or_else(|| format!("`cid` must be a CID, and this one doesn't parse: {sent}")), + } + } +} + +fn stored_ref(reference: StoredRef) -> Result { + let (_, address) = + RecordAddress::parse_at_uri(&reference.uri).map_err(|error| error.to_string())?; + Ok(CommentRef::new(address, reference.cid.resolve()?)) +} + +#[derive(Deserialize)] +struct StoredSubject { + subject: StoredRef, +} + +pub fn stored_subject(body: &RecordBody) -> Result { + let stored: StoredSubject = + serde_ipld_dagcbor::from_slice(body.as_bytes()).map_err(|error| error.to_string())?; + stored_ref(stored.subject) +} + +#[derive(Deserialize)] +struct StoredComment { + subject: StoredRef, + #[serde(rename = "replyTo", default)] + reply_to: Option, +} + +fn decode_stored(body: &RecordBody) -> Result { + let stored: StoredComment = + serde_ipld_dagcbor::from_slice(body.as_bytes()).map_err(|error| error.to_string())?; + Ok(Comment { + subject: stored.subject.cid.resolve()?, + reply: stored.reply_to.map(stored_ref).transpose()?, + }) +} + +crate::social_cob! { + body = Comment, + change = CommentChange, + cob = CommentCob, + collection = knot_record::comment::COMMENT_COLLECTION, + opening = Opening::Contribution, + stored = decode_stored, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::SocialChange; + use crate::social::seeded; + use knot_cob::ChangePayload; + use knot_record::comment::comment_collection; + use knot_record::ticket::ticket_collection; + use knot_types::{RecordAddress, RecordRkey, dag_cbor_cid}; + + fn open(body: Comment) -> SocialChange { + SocialChange::Open { + creation: seeded( + RecordAddress::new( + comment_collection(), + RecordRkey::new("3lubrptx57d22").unwrap(), + ), + Some(RecordAddress::new( + ticket_collection(), + RecordRkey::new("3lubrptx57d33").unwrap(), + )), + ), + body, + } + } + + #[test] + fn comment_change_round_trips_with_its_refs() { + let reply = CommentRef::new( + RecordAddress::new( + comment_collection(), + RecordRkey::new("3lubrptx57d44").unwrap(), + ), + RecordCid::new(dag_cbor_cid(b"reply")).unwrap(), + ); + let change = open(Comment { + subject: RecordCid::new(dag_cbor_cid(b"subject")).unwrap(), + reply: Some(reply), + }); + assert_eq!( + CommentChange::decode(&change.encode().unwrap()).unwrap(), + change, + "returned refs ought to be the exact same ones the write path signed" + ); + } +} + +#[cfg(test)] +mod stored_subject_tests { + use super::*; + use knot_record::ticket::ticket_collection; + use knot_types::{AccountDid, RecordAddress, RecordRkey, RepoDid, dag_cbor_cid}; + + #[test] + fn stored_subject_reads_the_address_a_comment_was_opened_on() { + let repo = RepoDid::new("did:plc:squid").unwrap(); + let subject = CommentRef::new( + RecordAddress::new(ticket_collection(), RecordRkey::new("288").unwrap()), + RecordCid::new(dag_cbor_cid(b"subject")).unwrap(), + ); + let markup = knot_record::comment::CommentMarkup::new( + knot_record::ticket::MarkdownBody::new("kelp".to_owned()).unwrap(), + None, + ); + let record = knot_record::comment::CommentRecord::new( + repo, + subject.clone(), + markup, + None, + None, + knot_types::UnixSeconds::new(1), + AccountDid::new("did:plc:nel").unwrap(), + ) + .unwrap() + .body() + .unwrap(); + assert_eq!( + stored_subject(&record).unwrap().address(), + subject.address(), + "the subject the foster stored is the subject the converter recovers" + ); + } +} diff --git a/knot2/crates/knot-cobs/src/grant.rs b/knot2/crates/knot-cobs/src/grant.rs new file mode 100644 index 0000000000000000000000000000000000000000..562ac18aa15cfa224c15151cd42c4139e181fae9 --- /dev/null +++ b/knot2/crates/knot-cobs/src/grant.rs @@ -0,0 +1,432 @@ +use std::collections::BTreeMap; + +use knot_types::{AccountDid, UnixSeconds}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Grant { + pub subject: AccountDid, + pub added_by: AccountDid, + pub created_at: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Invite(pub Grant); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Removal { + pub subject: AccountDid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Accept { + pub subject: AccountDid, + pub verified_at: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Entry { + pub added_by: AccountDid, + pub created_at: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GrantList { + entries: BTreeMap, +} + +impl GrantList { + pub fn empty() -> Self { + Self { + entries: BTreeMap::new(), + } + } + + pub fn apply_change(mut self, change: &impl GrantChange) -> Self { + match change.effect() { + Effect::Admit(grant) => { + self.entries + .entry(grant.subject.clone()) + .or_insert_with(|| Entry { + added_by: grant.added_by.clone(), + created_at: grant.created_at, + }); + } + Effect::Revoke(subject) => { + self.entries.remove(subject); + } + Effect::Noop(_) => {} + } + self + } + + pub fn would_change(&self, change: &impl GrantChange) -> bool { + match change.effect() { + Effect::Admit(grant) => !self.entries.contains_key(&grant.subject), + Effect::Revoke(subject) => self.entries.contains_key(subject), + Effect::Noop(_) => false, + } + } + + pub fn contains(&self, subject: &AccountDid) -> bool { + self.entries.contains_key(subject) + } + + pub fn get(&self, subject: &AccountDid) -> Option<&Entry> { + self.entries.get(subject) + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn entries(&self) -> impl Iterator { + self.entries.iter() + } +} + +#[derive(Debug)] +pub enum Effect<'a> { + Admit(&'a Grant), + Revoke(&'a AccountDid), + Noop(&'a AccountDid), +} + +pub trait GrantChange { + fn effect(&self) -> Effect<'_>; + + fn subject(&self) -> &AccountDid { + match self.effect() { + Effect::Admit(grant) => &grant.subject, + Effect::Revoke(subject) | Effect::Noop(subject) => subject, + } + } +} + +macro_rules! grant_change { + ($($payload:ident($binding:ident) => $effect:expr),+ $(,)?) => { + $(impl GrantChange for $payload { + fn effect(&self) -> Effect<'_> { + let $binding = self; + $effect + } + })+ + }; +} + +grant_change! { + Grant(grant) => Effect::Admit(grant), + Invite(invite) => Effect::Admit(&invite.0), + Accept(accept) => Effect::Noop(&accept.subject), + Removal(removal) => Effect::Revoke(&removal.subject), +} + +macro_rules! grant_cob { + ( + change = $change:ident, + ops = { $($variant:ident($payload:ident)),+ $(,)? }, + cob = $cob:ident, + state = $state:ident, + type_name = $type_name:literal $(,)? + ) => { + #[derive(Debug, Clone, PartialEq, Eq, ::serde::Serialize, ::serde::Deserialize)] + #[serde(tag = "op", content = "data", rename_all = "snake_case")] + pub enum $change { + $($variant($crate::grant::$payload)),+ + } + + impl $crate::grant::GrantChange for $change { + fn effect(&self) -> $crate::grant::Effect<'_> { + match self { + $($change::$variant(op) => $crate::grant::GrantChange::effect(op)),+ + } + } + } + + impl ::knot_cob::ChangePayload for $change { + const TYPE: &'static str = $type_name; + } + + impl ::knot_cob::KnotAuthored for $change {} + + pub type $state = $crate::grant::GrantList; + + pub struct $cob; + + impl ::knot_cob::Evaluate for $cob { + type State = $state; + type Change = $change; + + const HISTORY: ::knot_cob::HistoryModel = ::knot_cob::HistoryModel::Linear; + + fn initial() -> Self::State { + $crate::grant::GrantList::empty() + } + + fn apply( + state: Self::State, + change: Self::Change, + _author: &::knot_types::ActorId, + ) -> Self::State { + state.apply_change(&change) + } + } + + impl ::knot_cob::Checkpoint for $cob { + const SNAPSHOT_STRIDE: ::knot_cob::SnapshotStride = + ::knot_cob::SnapshotStride::new(256); + fn checkpoint_size(state: &Self::State) -> ::knot_cob::StateSize { + ::knot_cob::StateSize::new(state.len()) + } + } + + #[cfg(test)] + #[test] + fn the_change_declares_its_lexicon_type_and_applies_through_the_cob() { + use ::knot_cob::{ChangePayload, Evaluate}; + + let did = |value: &str| ::knot_types::AccountDid::new(value).unwrap(); + let change = $change::Add($crate::grant::Grant { + subject: did("did:plc:nel"), + added_by: did("did:plc:olaren"), + created_at: ::knot_types::UnixSeconds::new(9), + }); + let actor = ::knot_types::ActorId::from_secp256k1(&[0x02; 33]); + assert_eq!($change::TYPE, $type_name); + assert_eq!( + $change::decode(&change.encode().unwrap()).unwrap(), + change, + "Change must decode from dag-cbor as itself, or its replay builds another grant list" + ); + assert_eq!( + $cob::apply($cob::initial(), change, &actor).len(), + 1, + "Cob's `apply` has to write the change's subject into its grant list" + ); + } + }; +} + +pub(crate) use grant_cob; + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + use crate::MembersChange; + + const ALPHABET: &[u8; 4] = b"giar"; + + fn did(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn change(op: u8, subject: &str, added_by: &str, at: i64) -> MembersChange { + let grant = Grant { + subject: did(subject), + added_by: did(added_by), + created_at: UnixSeconds::new(at), + }; + match op { + b'g' => MembersChange::Add(grant), + b'i' => MembersChange::Invite(Invite(grant)), + b'a' => MembersChange::Accept(Accept { + subject: grant.subject, + verified_at: grant.created_at, + }), + _ => MembersChange::Remove(Removal { + subject: grant.subject, + }), + } + } + + fn fold(changes: Vec) -> GrantList { + changes + .iter() + .fold(GrantList::empty(), GrantList::apply_change) + } + + fn entry(added_by: &str, at: i64) -> Option { + Some(Entry { + added_by: did(added_by), + created_at: UnixSeconds::new(at), + }) + } + + fn history() -> GrantList { + fold(vec![ + change(b'g', "granted", "olaren", 1), + change(b'i', "invited", "olaren", 1), + change(b'g', "regranted", "olaren", 1), + change(b'g', "regranted", "teq", 5), + change(b'i', "reinvited", "olaren", 1), + change(b'i', "reinvited", "teq", 5), + change(b'i', "granted_over_invite", "olaren", 1), + change(b'g', "granted_over_invite", "teq", 5), + change(b'g', "invited_over_grant", "olaren", 1), + change(b'i', "invited_over_grant", "teq", 5), + change(b'a', "unoffered", "olaren", 7), + change(b'a', "accepted_early", "olaren", 7), + change(b'i', "accepted_early", "olaren", 9), + change(b'i', "invite_then_accept", "olaren", 1), + change(b'a', "invite_then_accept", "olaren", 7), + change(b'g', "grant_then_accept", "olaren", 1), + change(b'a', "grant_then_accept", "olaren", 7), + change(b'i', "accepted_twice", "olaren", 1), + change(b'a', "accepted_twice", "olaren", 7), + change(b'a', "accepted_twice", "olaren", 9), + change(b'i', "removed", "olaren", 1), + change(b'a', "removed", "olaren", 7), + change(b'r', "removed", "olaren", 0), + change(b'i', "reinvite_after_remove", "olaren", 1), + change(b'a', "reinvite_after_remove", "olaren", 7), + change(b'r', "reinvite_after_remove", "olaren", 0), + change(b'i', "reinvite_after_remove", "teq", 11), + change(b'g', "readd_after_remove", "olaren", 1), + change(b'a', "readd_after_remove", "olaren", 7), + change(b'r', "readd_after_remove", "olaren", 0), + change(b'g', "readd_after_remove", "olaren", 11), + ]) + } + + #[test] + fn replay_keeps_the_first_admission_until_a_removal_clears_it() { + let list = history(); + [ + ("nobody", None), + ("granted", entry("olaren", 1)), + ("invited", entry("olaren", 1)), + ("regranted", entry("olaren", 1)), + ("reinvited", entry("olaren", 1)), + ("granted_over_invite", entry("olaren", 1)), + ("invited_over_grant", entry("olaren", 1)), + ("unoffered", None), + ("accepted_early", entry("olaren", 9)), + ("invite_then_accept", entry("olaren", 1)), + ("grant_then_accept", entry("olaren", 1)), + ("accepted_twice", entry("olaren", 1)), + ("removed", None), + ("reinvite_after_remove", entry("teq", 11)), + ("readd_after_remove", entry("olaren", 11)), + ] + .into_iter() + .for_each(|(subject, expected)| { + assert_eq!( + list.get(&did(subject)).cloned(), + expected, + "the first admission sets an entry's provenance, and only a removal clears it: \ + {subject}" + ); + }); + assert!( + list.entries().all(|(subject, _)| list.contains(subject)), + "every entry on the grant list is a member, and pending isn't a state an entry can end up in" + ); + assert!( + !fold(vec![]).contains(&did("nel")), + "an empty change log doesn't admit anybody" + ); + } + + #[test] + fn a_legacy_invite_admits_and_apply_skips_its_acceptance() { + let invited = GrantList::empty().apply_change(&change(b'i', "nel", "olaren", 1)); + assert_eq!( + invited.get(&did("nel")).cloned(), + entry("olaren", 1), + "an invite replayed out of a live change log is a membership now, not an offer" + ); + let accepted = invited + .clone() + .apply_change(&change(b'a', "nel", "olaren", 7)); + assert_eq!( + ( + accepted, + invited.would_change(&change(b'a', "nel", "olaren", 7)) + ), + (invited.clone(), false), + "an acceptance from the consent model mustn't change the grant list or append a \ + change" + ); + } + + #[derive(Serialize)] + struct EntryBeforeConsent { + added_by: AccountDid, + created_at: UnixSeconds, + } + + #[derive(Serialize)] + struct EntryUnderConsent { + offer: &'static str, + added_by: AccountDid, + created_at: UnixSeconds, + verified_at: UnixSeconds, + } + + #[test] + fn a_checkpoint_stays_byte_stable_and_still_reads_the_consent_era_fields() { + let before = serde_ipld_dagcbor::to_vec(&EntryBeforeConsent { + added_by: did("olaren"), + created_at: UnixSeconds::new(7), + }) + .unwrap(); + assert_eq!( + before, + serde_ipld_dagcbor::to_vec(&entry("olaren", 7).unwrap()).unwrap(), + "a checkpoint written before the consent model must still match its own digest" + ); + let under_consent = serde_ipld_dagcbor::to_vec(&EntryUnderConsent { + offer: "invited", + added_by: did("olaren"), + created_at: UnixSeconds::new(7), + verified_at: UnixSeconds::new(9), + }) + .unwrap(); + assert_eq!( + serde_ipld_dagcbor::from_slice::(&under_consent).unwrap(), + entry("olaren", 7).unwrap(), + "a checkpoint written under the consent model has to keep decoding, or a live knot \ + loses its grant list on upgrade" + ); + let mixed = history(); + let reread: GrantList = + serde_ipld_dagcbor::from_slice(&serde_ipld_dagcbor::to_vec(&mixed).unwrap()).unwrap(); + assert_eq!( + reread, mixed, + "the checkpoint is the grant list's encoding, so it has to decode back into the same list" + ); + } + + proptest! { + #![proptest_config(ProptestConfig { cases: 256, ..ProptestConfig::default() })] + + #[test] + fn would_change_agrees_with_replay_over_the_whole_alphabet( + ops in prop::collection::vec((0u8..4, 0u8..3), 0..12), + (op, who) in (0u8..4, 0u8..3), + ) { + let list = fold( + ops.iter() + .enumerate() + .map(|(index, (op, who))| { + let at = index as i64 + 1; + change(ALPHABET[*op as usize], &format!("s{who}"), "olaren", at) + }) + .collect(), + ); + let next = change(ALPHABET[op as usize], &format!("s{who}"), "olaren", 99); + prop_assert_eq!( + list.would_change(&next), + list.clone().apply_change(&next) != list, + "the append-time predicate and the replay disagree on {:?}", + next + ); + } + } +} diff --git a/knot2/crates/knot-cobs/src/import.rs b/knot2/crates/knot-cobs/src/import.rs new file mode 100644 index 0000000000000000000000000000000000000000..74d17fb402b8cbeec1958e420d99f21f4817119a --- /dev/null +++ b/knot2/crates/knot-cobs/src/import.rs @@ -0,0 +1,54 @@ +use knot_cob::{ChangePayload, CobError, CobHome, CobStore, parse_cob_ref}; +use knot_types::{ActorId, CobId, RefName, TypeName}; + +use crate::collaborators::{CollaboratorsChange, CollaboratorsCob}; +use crate::members::{MembersChange, MembersCob}; +use crate::registry::{RegistryChange, RepoRegistryCob}; + +#[derive(Debug, thiserror::Error)] +pub enum ImportError { + #[error("'{0}' isn't refs/cobs// ref")] + NotCobRef(String), + #[error("no collaborative object type is registered for namespace '{0}'")] + UnknownType(TypeName), + #[error(transparent)] + Cob(#[from] CobError), +} + +type Verifier = fn(&CobStore<'_>, &CobHome, CobId, &ActorId) -> Result<(), CobError>; + +fn verifier_for(type_name: &TypeName) -> Option { + [ + (MembersChange::type_name(), { + |store: &CobStore<'_>, home, object, owner| { + store.verify::(home, object, owner) + } + } as Verifier), + (CollaboratorsChange::type_name(), { + |store: &CobStore<'_>, home, object, owner| { + store.verify::(home, object, owner) + } + } as Verifier), + (RegistryChange::type_name(), { + |store: &CobStore<'_>, home, object, owner| { + store.verify::(home, object, owner) + } + } as Verifier), + ] + .into_iter() + .find_map(|(name, verifier)| (name == *type_name).then_some(verifier)) +} + +pub fn verify_cob_ref( + store: &CobStore, + home: &CobHome, + refname: &RefName, + owner: &ActorId, +) -> Result { + let (type_name, object) = parse_cob_ref(refname) + .ok_or_else(|| ImportError::NotCobRef(refname.as_str().to_string()))?; + let type_name = knot_cob::modernized_type_of(&type_name).unwrap_or(type_name); + let verifier = verifier_for(&type_name).ok_or(ImportError::UnknownType(type_name))?; + verifier(store, home, object, owner)?; + Ok(object) +} diff --git a/knot2/crates/knot-cobs/src/label.rs b/knot2/crates/knot-cobs/src/label.rs new file mode 100644 index 0000000000000000000000000000000000000000..d08e1b063fe84072ffdb5066253145ce3a18599f --- /dev/null +++ b/knot2/crates/knot-cobs/src/label.rs @@ -0,0 +1,49 @@ +use knot_types::RecordBody; +use serde::{Deserialize, Serialize}; + +use knot_record::label::LABEL_DEF_COLLECTION; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct LabelDef; + +fn decode_stored(_: &RecordBody) -> Result { + Ok(LabelDef) +} + +crate::social_cob! { + body = LabelDef, + change = LabelDefChange, + cob = LabelDefCob, + collection = LABEL_DEF_COLLECTION, + opening = crate::Opening::Ownership, + stored = decode_stored, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::SocialChange; + use crate::social::seeded; + use knot_cob::ChangePayload; + use knot_record::label::label_def_collection; + use knot_types::{LabelKey, RecordAddress, RecordRkey}; + + #[test] + fn label_def_change_round_trips() { + let change = SocialChange::Open { + creation: seeded( + RecordAddress::new( + label_def_collection(), + RecordRkey::composed(LabelKey::new("wontfix").unwrap()), + ), + None, + ), + body: LabelDef, + }; + assert_eq!( + LabelDefChange::decode(&change.encode().unwrap()).unwrap(), + change, + "a label def change round trips through its own encoding" + ); + } +} diff --git a/knot2/crates/knot-cobs/src/lib.rs b/knot2/crates/knot-cobs/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..70d496898d3fa0e9d53ca6771538e406ace53e99 --- /dev/null +++ b/knot2/crates/knot-cobs/src/lib.rs @@ -0,0 +1,66 @@ +mod artifact; +mod blocklist; +mod collaborators; +mod comment; +mod grant; +mod import; +mod label; +mod manifest; +mod members; +mod pull; +mod reaction; +mod registry; +mod social; +mod ticket; +pub use artifact::{Artifact, ArtifactChange, ArtifactCob}; +pub use blocklist::{Blocklist, BlocklistChange, BlocklistCob}; +pub use label::{LabelDef, LabelDefChange, LabelDefCob}; +pub use manifest::{Manifest, ManifestChange, ManifestCob}; + +pub use collaborators::{Collaborators, CollaboratorsChange, CollaboratorsCob}; +pub use comment::{Comment, CommentChange, CommentCob, stored_subject}; +pub use grant::{Accept, Effect, Entry, Grant, GrantChange, GrantList, Invite, Removal}; +pub use import::{ImportError, verify_cob_ref}; +pub use members::{Members, MembersChange, MembersCob}; +pub use pull::{Patch, PatchChange, PatchCob}; +pub use reaction::{Reaction, ReactionChange, ReactionCob, ReactionKind}; +pub use registry::{ + Policy, Registration, RegistrationField, Registry, RegistryChange, RegistryError, Rename, + RepoRecord, RepoRef, RepoRegistryCob, Retired, deregister_repo, register_repo, rename_repo, + retire_repo, set_repo_policy, +}; +pub use social::{ + AnyBody, AnySocialChange, Creation, Editing, Emission, Erasure, Opening, SettleError, Settled, + Social, SocialBody, SocialChange, SocialCob, SocialObject, SocialState, decode_social, +}; +pub use ticket::{ + Ticket, TicketChange, TicketCob, history_model_of, settle_stored, social_collections, + social_root_address, social_state, validate_stored, +}; + +#[doc(hidden)] +pub mod fuzz { + use knot_cob::ChangePayload; + + use crate::{BlocklistChange, CollaboratorsChange, MembersChange, RegistryChange}; + + pub fn change_decode(data: &[u8]) { + let _ = RegistryChange::decode(data); + let _ = MembersChange::decode(data); + let _ = BlocklistChange::decode(data); + let _ = CollaboratorsChange::decode(data); + let _ = crate::decode_social(data); + let _ = crate::CommentChange::decode(data); + let _ = crate::ReactionChange::decode(data); + let _ = crate::LabelDefChange::decode(data); + let _ = crate::PatchChange::decode(data); + let _ = crate::ManifestChange::decode(data); + let _ = crate::ArtifactChange::decode(data); + } + + pub fn ref_parse(data: &[u8]) { + let _ = knot_types::RefName::new(String::from_utf8_lossy(data)) + .ok() + .and_then(|name| knot_cob::parse_cob_ref(&name)); + } +} diff --git a/knot2/crates/knot-cobs/src/manifest.rs b/knot2/crates/knot-cobs/src/manifest.rs new file mode 100644 index 0000000000000000000000000000000000000000..bf8dbbc839582bd00871b36d8a9a768d7adfe516 --- /dev/null +++ b/knot2/crates/knot-cobs/src/manifest.rs @@ -0,0 +1,49 @@ +use knot_types::RecordBody; +use serde::{Deserialize, Serialize}; + +use knot_record::manifest::MANIFEST_COLLECTION; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct Manifest; + +fn decode_stored(_: &RecordBody) -> Result { + Ok(Manifest) +} + +crate::social_cob! { + body = Manifest, + change = ManifestChange, + cob = ManifestCob, + collection = MANIFEST_COLLECTION, + opening = crate::Opening::Ownership, + stored = decode_stored, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::SocialChange; + use crate::social::seeded; + use knot_cob::ChangePayload; + use knot_record::manifest::{MANIFEST_RKEY, manifest_collection}; + use knot_types::{RecordAddress, RecordRkey}; + + #[test] + fn manifest_change_round_trips() { + let change = SocialChange::Open { + creation: seeded( + RecordAddress::new( + manifest_collection(), + RecordRkey::new(MANIFEST_RKEY).unwrap(), + ), + None, + ), + body: Manifest, + }; + assert_eq!( + ManifestChange::decode(&change.encode().unwrap()).unwrap(), + change, + "a manifest change round trips through its own encoding" + ); + } +} diff --git a/knot2/crates/knot-cobs/src/members.rs b/knot2/crates/knot-cobs/src/members.rs new file mode 100644 index 0000000000000000000000000000000000000000..bee1ba781e3983da052129d47d6b9775a1f339ce --- /dev/null +++ b/knot2/crates/knot-cobs/src/members.rs @@ -0,0 +1,9 @@ +use crate::grant::grant_cob; + +grant_cob! { + change = MembersChange, + ops = { Add(Grant), Invite(Invite), Accept(Accept), Remove(Removal) }, + cob = MembersCob, + state = Members, + type_name = "org.tangled.knot.member", +} diff --git a/knot2/crates/knot-cobs/src/pull.rs b/knot2/crates/knot-cobs/src/pull.rs new file mode 100644 index 0000000000000000000000000000000000000000..fe07c13cf1a2e7565b613d848512bc584f0ef7f7 --- /dev/null +++ b/knot2/crates/knot-cobs/src/pull.rs @@ -0,0 +1,53 @@ +use knot_types::RecordBody; +use serde::{Deserialize, Serialize}; + +use crate::Opening; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct Patch; + +fn decode_stored(_: &RecordBody) -> Result { + Ok(Patch) +} + +crate::social_cob! { + body = Patch, + change = PatchChange, + cob = PatchCob, + collection = knot_record::pull::PATCH_COLLECTION, + opening = Opening::Contribution, + stored = decode_stored, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::SocialChange; + use crate::social::seeded; + use knot_cob::ChangePayload; + use knot_record::pull::patch_collection; + use knot_record::ticket::ticket_collection; + use knot_types::{RecordAddress, RecordRkey}; + + #[test] + fn patch_change_round_trips() { + let change = SocialChange::Open { + creation: seeded( + RecordAddress::new( + patch_collection(), + RecordRkey::new("3lubrptx57d22").unwrap(), + ), + Some(RecordAddress::new( + ticket_collection(), + RecordRkey::new("3lubrptx57d33").unwrap(), + )), + ), + body: Patch, + }; + assert_eq!( + PatchChange::decode(&change.encode().unwrap()).unwrap(), + change, + "a patch change round trips through its own encoding" + ); + } +} diff --git a/knot2/crates/knot-cobs/src/reaction.rs b/knot2/crates/knot-cobs/src/reaction.rs new file mode 100644 index 0000000000000000000000000000000000000000..46ab2505f255b5ca98af7663f8890daf8df88ee6 --- /dev/null +++ b/knot2/crates/knot-cobs/src/reaction.rs @@ -0,0 +1,98 @@ +use serde::{Deserialize, Serialize}; + +pub use knot_record::reaction::ReactionKind; +use knot_types::{RecordBody, RecordCid}; + +use crate::Opening; +use crate::comment::StoredRef; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct Reaction { + pub subject: RecordCid, + pub kind: ReactionKind, +} + +#[derive(Deserialize)] +struct StoredReaction { + subject: StoredRef, + reaction: ReactionKind, +} + +fn decode_stored(body: &RecordBody) -> Result { + let stored: StoredReaction = + serde_ipld_dagcbor::from_slice(body.as_bytes()).map_err(|error| error.to_string())?; + Ok(Reaction { + subject: stored.subject.cid.resolve()?, + kind: stored.reaction, + }) +} + +crate::social_cob! { + body = Reaction, + change = ReactionChange, + cob = ReactionCob, + collection = knot_record::reaction::REACTION_COLLECTION, + opening = Opening::Reaction, + stored = decode_stored, +} +#[cfg(test)] +mod tests { + use super::*; + use crate::social::seeded; + use crate::{Erasure, SocialChange}; + use knot_cob::ChangePayload; + use knot_record::reaction::reaction_collection; + use knot_record::ticket::ticket_collection; + use knot_types::{AccountDid, RecordAddress, RecordCid, RecordRkey, UnixSeconds, dag_cbor_cid}; + + fn open(kind: ReactionKind) -> SocialChange { + SocialChange::Open { + creation: seeded( + RecordAddress::new( + reaction_collection(), + RecordRkey::new("3lubrptx57d22").unwrap(), + ), + Some(RecordAddress::new( + ticket_collection(), + RecordRkey::new("3lubrptx57d33").unwrap(), + )), + ), + body: Reaction { + subject: RecordCid::new(dag_cbor_cid(b"subject")).unwrap(), + kind, + }, + } + } + + #[test] + fn reaction_changes_round_trip_with_their_kind() { + for kind in [ + ReactionKind::new("👍").unwrap(), + ReactionKind::new("👎").unwrap(), + ReactionKind::new("😆").unwrap(), + ReactionKind::new("🎉").unwrap(), + ReactionKind::new("🫤").unwrap(), + ReactionKind::new("❤️").unwrap(), + ReactionKind::new("🚀").unwrap(), + ReactionKind::new("👀").unwrap(), + ] { + let change = open(kind); + assert_eq!( + ReactionChange::decode(&change.encode().unwrap()).unwrap(), + change, + "every emoji probably oughtta encode and decode back to itself" + ); + } + let erased = SocialChange::Erase { + erasure: Erasure { + author: AccountDid::new("did:plc:nel").unwrap(), + from: RecordCid::new(dag_cbor_cid(b"kelp")).unwrap(), + erased_at: UnixSeconds::new(2), + }, + }; + assert_eq!( + ReactionChange::decode(&erased.encode().unwrap()).unwrap(), + erased + ); + } +} diff --git a/knot2/crates/knot-cobs/src/registry.rs b/knot2/crates/knot-cobs/src/registry.rs new file mode 100644 index 0000000000000000000000000000000000000000..f64b2ee65c694245fe76595a338fc9e977fe60f0 --- /dev/null +++ b/knot2/crates/knot-cobs/src/registry.rs @@ -0,0 +1,895 @@ +use std::collections::BTreeMap; +use std::fmt; + +use knot_cob::{ + ChangePayload, Checkpoint, CobError, CobHome, CobStore, Evaluate, HistoryModel, KnotAuthored, + SnapshotStride, StateSize, +}; +use knot_runtime::Signer; +use knot_types::{ + ActorId, ChangeId, CobId, ContributionPolicy, OwnerDid, RepoDid, RepoName, RepoRkey, + UnixSeconds, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Registration { + pub owner: OwnerDid, + pub rkey: RepoRkey, + pub name: RepoName, + pub repo: RepoDid, + pub created_at: UnixSeconds, + #[serde(default)] + pub policy: ContributionPolicy, + #[serde(default)] + pub fostered: bool, + #[serde(default)] + pub retired_at: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RegistrationField { + Owner, + Rkey, + Name, + CreatedAt, + Policy, + Fostered, +} + +impl fmt::Display for RegistrationField { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(match self { + Self::Owner => "owner", + Self::Rkey => "rkey", + Self::Name => "name", + Self::CreatedAt => "createdAt", + Self::Policy => "contributionPolicy", + Self::Fostered => "fostered", + }) + } +} + +impl Registration { + fn diverges_from(&self, record: &RepoRecord) -> Option { + [ + (record.owner != self.owner, RegistrationField::Owner), + (record.rkey != self.rkey, RegistrationField::Rkey), + (record.name != self.name, RegistrationField::Name), + ( + record.created_at != self.created_at, + RegistrationField::CreatedAt, + ), + (record.policy != self.policy, RegistrationField::Policy), + ( + record.fostered != self.fostered, + RegistrationField::Fostered, + ), + ] + .into_iter() + .find_map(|(moved, field)| moved.then_some(field)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Policy { + pub owner: OwnerDid, + pub repo: RepoDid, + pub policy: ContributionPolicy, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Rename { + pub owner: OwnerDid, + pub rkey: RepoRkey, + pub name: RepoName, + pub repo: RepoDid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RepoRef { + pub owner: OwnerDid, + pub rkey: RepoRkey, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Retired { + pub owner: OwnerDid, + pub rkey: RepoRkey, + pub repo: RepoDid, + pub at: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "op", content = "data", rename_all = "snake_case")] +pub enum RegistryChange { + Register(Registration), + Rename(Rename), + Deregister(RepoRef), + SetPolicy(Policy), + Retired(Retired), +} + +impl ChangePayload for RegistryChange { + const TYPE: &'static str = "org.tangled.knot.repoRegistry"; +} + +impl KnotAuthored for RegistryChange {} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RepoRecord { + pub owner: OwnerDid, + pub rkey: RepoRkey, + pub name: RepoName, + pub created_at: UnixSeconds, + #[serde(default)] + pub policy: ContributionPolicy, + #[serde(default)] + pub fostered: bool, + #[serde(default)] + pub retired_at: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct Registry { + records: BTreeMap, + aliases: BTreeMap>, +} + +impl Registry { + pub fn resolve(&self, owner: &OwnerDid, rkey: &RepoRkey) -> Option<&RepoDid> { + self.aliases.get(owner)?.get(rkey) + } + + pub fn record_of(&self, repo: &RepoDid) -> Option<&RepoRecord> { + self.records.get(repo) + } + + pub fn owner_of(&self, repo: &RepoDid) -> Option { + self.records.get(repo).map(|record| record.owner.clone()) + } + + pub fn len(&self) -> usize { + self.records.len() + } + + pub fn is_empty(&self) -> bool { + self.records.is_empty() + } + + pub fn records(&self) -> impl Iterator { + self.records.iter() + } + + pub fn aliases(&self) -> impl Iterator { + self.aliases + .iter() + .flat_map(|(owner, names)| names.iter().map(move |(rkey, repo)| (owner, rkey, repo))) + } + + fn canonical_holder(&self, owner: &OwnerDid, rkey: &RepoRkey) -> Option<&RepoDid> { + let holder = self.resolve(owner, rkey)?; + self.records + .get(holder) + .filter(|record| record.rkey == *rkey) + .map(|_| holder) + } + + fn register(mut self, registration: Registration) -> Self { + self = self.drop_repo(®istration.repo); + self = self.steal_alias(®istration.owner, ®istration.rkey, ®istration.repo); + self.aliases + .entry(registration.owner.clone()) + .or_default() + .insert(registration.rkey.clone(), registration.repo.clone()); + self.records.insert( + registration.repo, + RepoRecord { + owner: registration.owner, + rkey: registration.rkey, + name: registration.name, + created_at: registration.created_at, + policy: registration.policy, + fostered: registration.fostered, + retired_at: registration.retired_at, + }, + ); + self + } + + fn rename(mut self, rename: Rename) -> Self { + match self.records.get(&rename.repo) { + Some(record) if record.owner == rename.owner => {} + _ => return self, + } + self = self.steal_alias(&rename.owner, &rename.rkey, &rename.repo); + self.aliases + .entry(rename.owner.clone()) + .or_default() + .insert(rename.rkey.clone(), rename.repo.clone()); + if let Some(record) = self.records.get_mut(&rename.repo) { + record.rkey = rename.rkey; + record.name = rename.name; + } + self + } + + fn deregister(self, target: RepoRef) -> Self { + match self.resolve(&target.owner, &target.rkey).cloned() { + Some(repo) => self.drop_repo(&repo), + None => self, + } + } + + fn set_policy(mut self, change: Policy) -> Self { + if let Some(record) = self + .records + .get_mut(&change.repo) + .filter(|record| record.owner == change.owner) + { + record.policy = change.policy; + } + self + } + + fn retire(mut self, change: Retired) -> Self { + if let Some(record) = self + .records + .get_mut(&change.repo) + .filter(|record| record.owner == change.owner) + { + let at = match record.retired_at { + Some(existing) => existing.max(change.at), + None => change.at, + }; + record.retired_at = Some(at); + } + self + } + + fn steal_alias(mut self, owner: &OwnerDid, rkey: &RepoRkey, target: &RepoDid) -> Self { + match self.resolve(owner, rkey).cloned() { + Some(holder) if holder != *target => { + let canonical = self + .records + .get(&holder) + .is_some_and(|record| record.rkey == *rkey); + if canonical { + self.drop_repo(&holder) + } else { + if let Some(names) = self.aliases.get_mut(owner) { + names.remove(rkey); + } + self.prune_empty_owners() + } + } + _ => self, + } + } + + fn drop_repo(mut self, repo: &RepoDid) -> Self { + self.records.remove(repo); + self.aliases + .values_mut() + .for_each(|names| names.retain(|_, holder| holder != repo)); + self.prune_empty_owners() + } + + fn prune_empty_owners(mut self) -> Self { + self.aliases.retain(|_, names| !names.is_empty()); + self + } +} + +pub struct RepoRegistryCob; + +impl Evaluate for RepoRegistryCob { + type State = Registry; + type Change = RegistryChange; + + const HISTORY: HistoryModel = HistoryModel::Linear; + + fn initial() -> Self::State { + Registry::default() + } + + fn apply(state: Self::State, change: Self::Change, _author: &ActorId) -> Self::State { + match change { + RegistryChange::Register(registration) => state.register(registration), + RegistryChange::Rename(rename) => state.rename(rename), + RegistryChange::Deregister(target) => state.deregister(target), + RegistryChange::SetPolicy(policy) => state.set_policy(policy), + RegistryChange::Retired(change) => state.retire(change), + } + } +} + +impl Checkpoint for RepoRegistryCob { + const SNAPSHOT_STRIDE: SnapshotStride = SnapshotStride::new(256); + fn checkpoint_size(state: &Self::State) -> StateSize { + StateSize::new(state.len()) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum RegistryError { + #[error(transparent)] + Cob(#[from] CobError), + #[error("no repo is registered at {owner}/{rkey}")] + NotRegistered { owner: OwnerDid, rkey: RepoRkey }, + #[error("record key {owner}/{rkey} resolves to {found}, expected {expected}")] + RepoMismatch { + owner: OwnerDid, + rkey: RepoRkey, + expected: RepoDid, + found: RepoDid, + }, + #[error("repo {repo} is already registered as {owner}/{rkey} with another {field}")] + AlreadyRegistered { + repo: RepoDid, + owner: OwnerDid, + rkey: RepoRkey, + field: RegistrationField, + }, + #[error("record key {owner}/{rkey} is canonical key of {existing}")] + RkeyTaken { + owner: OwnerDid, + rkey: RepoRkey, + existing: RepoDid, + }, + #[error("repo {repo} isn't hosted on this knot")] + NotHosted { repo: RepoDid }, + #[error("repo {repo} is no longer registered to {expected}")] + OwnerMoved { repo: RepoDid, expected: OwnerDid }, +} + +fn owned_record<'a>( + registry: &'a Registry, + repo: &RepoDid, + owner: &OwnerDid, +) -> Result<&'a RepoRecord, RegistryError> { + let record = registry + .record_of(repo) + .ok_or_else(|| RegistryError::NotHosted { repo: repo.clone() })?; + (record.owner == *owner) + .then_some(record) + .ok_or_else(|| RegistryError::OwnerMoved { + repo: repo.clone(), + expected: owner.clone(), + }) +} + +fn unclaimed_rkey( + registry: &Registry, + owner: &OwnerDid, + rkey: &RepoRkey, + repo: &RepoDid, +) -> Result<(), RegistryError> { + match registry.canonical_holder(owner, rkey) { + Some(holder) if holder != repo => Err(RegistryError::RkeyTaken { + owner: owner.clone(), + rkey: rkey.clone(), + existing: holder.clone(), + }), + _ => Ok(()), + } +} + +pub fn register_repo( + store: &CobStore, + home: &CobHome, + object: CobId, + registration: Registration, + signer: &dyn Signer, + timestamp: UnixSeconds, +) -> Result, RegistryError> { + store.update_maybe_checkpointed::( + home, + object, + signer, + timestamp, + |registry| { + unclaimed_rkey( + registry, + ®istration.owner, + ®istration.rkey, + ®istration.repo, + )?; + match registry + .record_of(®istration.repo) + .map(|record| (record, registration.diverges_from(record))) + { + Some((record, Some(field))) => Err(RegistryError::AlreadyRegistered { + repo: registration.repo.clone(), + owner: record.owner.clone(), + rkey: record.rkey.clone(), + field, + }), + Some((_, None)) => Ok(None), + None => Ok(Some(RegistryChange::Register(registration.clone()))), + } + }, + ) +} + +pub fn rename_repo( + store: &CobStore, + home: &CobHome, + object: CobId, + rename: Rename, + signer: &dyn Signer, + timestamp: UnixSeconds, +) -> Result, RegistryError> { + store.update_maybe_checkpointed::( + home, + object, + signer, + timestamp, + |registry| { + let record = owned_record(registry, &rename.repo, &rename.owner)?; + if record.rkey == rename.rkey && record.name == rename.name { + return Ok(None); + } + unclaimed_rkey(registry, &rename.owner, &rename.rkey, &rename.repo)?; + Ok(Some(RegistryChange::Rename(rename.clone()))) + }, + ) +} + +pub fn deregister_repo( + store: &CobStore, + home: &CobHome, + object: CobId, + target: RepoRef, + expected: RepoDid, + signer: &dyn Signer, + timestamp: UnixSeconds, +) -> Result { + store.update_with_checkpointed::( + home, + object, + signer, + timestamp, + |registry| match registry.resolve(&target.owner, &target.rkey) { + None => Err(RegistryError::NotRegistered { + owner: target.owner.clone(), + rkey: target.rkey.clone(), + }), + Some(found) if found != &expected => Err(RegistryError::RepoMismatch { + owner: target.owner.clone(), + rkey: target.rkey.clone(), + expected: expected.clone(), + found: found.clone(), + }), + Some(_) => Ok(RegistryChange::Deregister(target.clone())), + }, + ) +} + +pub fn set_repo_policy( + store: &CobStore, + home: &CobHome, + object: CobId, + change: Policy, + signer: &dyn Signer, + timestamp: UnixSeconds, +) -> Result, RegistryError> { + store.update_maybe_checkpointed::( + home, + object, + signer, + timestamp, + |registry| { + let record = owned_record(registry, &change.repo, &change.owner)?; + Ok((record.policy != change.policy).then(|| RegistryChange::SetPolicy(change.clone()))) + }, + ) +} + +pub fn retire_repo( + store: &CobStore, + home: &CobHome, + object: CobId, + change: Retired, + signer: &dyn Signer, + timestamp: UnixSeconds, +) -> Result, RegistryError> { + store.update_maybe_checkpointed::( + home, + object, + signer, + timestamp, + |registry| { + let record = owned_record(registry, &change.repo, &change.owner)?; + match record.retired_at { + Some(existing) if existing >= change.at => Ok(None), + _ => Ok(Some(RegistryChange::Retired(change.clone()))), + } + }, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn owner(suffix: &str) -> OwnerDid { + OwnerDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn repo(suffix: &str) -> RepoDid { + RepoDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn rkey(value: &str) -> RepoRkey { + RepoRkey::new(value).unwrap() + } + + fn name(value: &str) -> RepoName { + RepoName::new(value).unwrap() + } + + fn register(owner_id: &str, key: &str, repo_id: &str, at: i64) -> RegistryChange { + RegistryChange::Register(Registration { + owner: owner(owner_id), + rkey: rkey(key), + name: name(key), + repo: repo(repo_id), + created_at: UnixSeconds::new(at), + policy: ContributionPolicy::default(), + fostered: false, + retired_at: None, + }) + } + + fn rename(owner_id: &str, key: &str, repo_id: &str) -> RegistryChange { + RegistryChange::Rename(Rename { + owner: owner(owner_id), + rkey: rkey(key), + name: name(key), + repo: repo(repo_id), + }) + } + + fn deregister(owner_id: &str, key: &str) -> RegistryChange { + RegistryChange::Deregister(RepoRef { + owner: owner(owner_id), + rkey: rkey(key), + }) + } + + fn retired(owner_id: &str, key: &str, repo_id: &str, at: i64) -> RegistryChange { + RegistryChange::Retired(Retired { + owner: owner(owner_id), + rkey: rkey(key), + repo: repo(repo_id), + at: UnixSeconds::new(at), + }) + } + + fn fold(changes: Vec) -> Registry { + let author = ActorId::from_secp256k1(&[0x02; 33]); + changes + .into_iter() + .fold(RepoRegistryCob::initial(), |state, change| { + RepoRegistryCob::apply(state, change, &author) + }) + } + + #[test] + fn register_maps_owner_and_rkey_to_a_repo() { + let state = fold(vec![register("nel", "anemone", "squid", 5)]); + assert_eq!( + state.resolve(&owner("nel"), &rkey("anemone")), + Some(&repo("squid")) + ); + let record = state.record_of(&repo("squid")).unwrap(); + assert_eq!(record.owner, owner("nel")); + assert_eq!(record.rkey, rkey("anemone")); + assert_eq!(record.name, name("anemone")); + assert_eq!(record.created_at, UnixSeconds::new(5)); + } + + #[test] + fn re_register_replaces_the_repo_under_an_rkey() { + let state = fold(vec![ + register("nel", "anemone", "squid", 1), + register("nel", "anemone", "limpet", 2), + ]); + assert_eq!( + state.resolve(&owner("nel"), &rkey("anemone")), + Some(&repo("limpet")) + ); + assert!( + state.record_of(&repo("squid")).is_none(), + "repo whose canonical rkey is taken by later register is dropped wholesale" + ); + assert_eq!(state.len(), 1); + } + + #[test] + fn rename_retains_the_prior_rkey_as_an_alias() { + let state = fold(vec![ + register("nel", "anemone", "squid", 1), + rename("nel", "barnacle", "squid"), + ]); + assert_eq!( + state.resolve(&owner("nel"), &rkey("barnacle")), + Some(&repo("squid")), + "new rkey resolves" + ); + assert_eq!( + state.resolve(&owner("nel"), &rkey("anemone")), + Some(&repo("squid")), + "prior rkey keeps resolving as an alias" + ); + let record = state.record_of(&repo("squid")).unwrap(); + assert_eq!(record.rkey, rkey("barnacle")); + assert_eq!(record.name, name("barnacle")); + assert_eq!(state.len(), 1); + } + + #[test] + fn rename_of_an_unregistered_repo_is_a_no_op() { + let registered = fold(vec![register("nel", "anemone", "squid", 1)]); + let after = fold(vec![ + register("nel", "anemone", "squid", 1), + rename("nel", "barnacle", "conch"), + ]); + assert_eq!(after, registered); + } + + #[test] + fn rename_under_a_mismatched_owner_is_a_no_op() { + let registered = fold(vec![register("nel", "anemone", "squid", 1)]); + let after = fold(vec![ + register("nel", "anemone", "squid", 1), + rename("olaren", "barnacle", "squid"), + ]); + assert_eq!(after, registered); + } + + #[test] + fn deregister_by_any_alias_removes_the_repo_and_every_alias() { + let state = fold(vec![ + register("nel", "anemone", "squid", 1), + rename("nel", "barnacle", "squid"), + deregister("nel", "anemone"), + deregister("nel", "anemone"), + ]); + assert_eq!(state.resolve(&owner("nel"), &rkey("anemone")), None); + assert_eq!(state.resolve(&owner("nel"), &rkey("barnacle")), None); + assert!(state.is_empty(), "replaying a deregister is a no-op"); + assert_eq!(state, Registry::default()); + } + + #[test] + fn a_later_change_steals_a_stale_alias_but_keeps_the_victim_canonical() { + let state = fold(vec![ + register("nel", "anemone", "squid", 1), + rename("nel", "barnacle", "squid"), + register("nel", "anemone", "whelk", 2), + ]); + assert_eq!( + state.resolve(&owner("nel"), &rkey("anemone")), + Some(&repo("whelk")), + "later register wins stale alias" + ); + assert_eq!( + state.resolve(&owner("nel"), &rkey("barnacle")), + Some(&repo("squid")), + "victim keeps its canonical rkey" + ); + assert_eq!(state.len(), 2); + } + + #[test] + fn owner_of_resolves_through_the_record_with_later_register_precedence() { + let unique = fold(vec![ + register("nel", "anemone", "squid", 1), + register("nel", "barnacle", "whelk", 2), + ]); + assert_eq!(unique.owner_of(&repo("squid")), Some(owner("nel"))); + assert_eq!( + unique.record_of(&repo("squid")).unwrap().rkey, + rkey("anemone") + ); + assert_eq!(unique.owner_of(&repo("conch")), None); + + let moved = fold(vec![ + register("nel", "anemone", "squid", 1), + register("olaren", "fork", "squid", 2), + ]); + assert_eq!( + moved.owner_of(&repo("squid")), + Some(owner("olaren")), + "linear causal order gives later register deterministic precedence" + ); + assert_eq!( + moved.resolve(&owner("nel"), &rkey("anemone")), + None, + "re-register under a new owner drops old owner's aliases" + ); + } + + #[test] + fn change_payload_roundtrips_through_dag_cbor() { + [ + register("nel", "anemone", "squid", 5), + rename("nel", "barnacle", "squid"), + deregister("nel", "anemone"), + RegistryChange::SetPolicy(Policy { + owner: owner("nel"), + repo: repo("squid"), + policy: ContributionPolicy::Anyone, + }), + ] + .into_iter() + .for_each(|change| { + let bytes = change.encode().unwrap(); + assert_eq!(RegistryChange::decode(&bytes).unwrap(), change); + }); + } + + #[test] + fn a_registration_written_before_the_policy_field_still_applies() { + #[derive(Serialize)] + struct Before<'a> { + owner: &'a str, + rkey: &'a str, + name: &'a str, + repo: &'a str, + created_at: i64, + } + + #[derive(Serialize)] + #[serde(tag = "op", content = "data", rename_all = "snake_case")] + enum BeforeChange<'a> { + Register(Before<'a>), + } + + let bytes = serde_ipld_dagcbor::to_vec(&BeforeChange::Register(Before { + owner: "did:plc:nel", + rkey: "anemone", + name: "anemone", + repo: "did:plc:squid", + created_at: 5, + })) + .unwrap(); + let decoded = RegistryChange::decode(&bytes) + .expect("Change from before this field existed still applies"); + let state = fold(vec![decoded]); + assert_eq!( + state.record_of(&repo("squid")).unwrap().policy, + ContributionPolicy::Anyone, + "repositories registered before this field existed take Anyone, like every policy nobody chose" + ); + } + + #[test] + fn only_the_owner_moves_the_policy_and_a_repeat_doesnt_write_a_change() { + let registered = || { + RegistryChange::Register(Registration { + owner: owner("nel"), + rkey: rkey("anemone"), + name: name("anemone"), + repo: repo("squid"), + created_at: UnixSeconds::new(5), + policy: ContributionPolicy::Members, + fostered: false, + retired_at: None, + }) + }; + let opened = fold(vec![ + registered(), + RegistryChange::SetPolicy(Policy { + owner: owner("nel"), + repo: repo("squid"), + policy: ContributionPolicy::Anyone, + }), + ]); + assert_eq!( + opened.record_of(&repo("squid")).unwrap().policy, + ContributionPolicy::Anyone + ); + let stranger = fold(vec![ + registered(), + RegistryChange::SetPolicy(Policy { + owner: owner("olaren"), + repo: repo("squid"), + policy: ContributionPolicy::Anyone, + }), + ]); + assert_eq!( + stranger.record_of(&repo("squid")).unwrap().policy, + ContributionPolicy::Members, + "Fold answers to the owner the registry has on file, not to what the change claims" + ); + } + + #[test] + fn retire_records_acked_at_and_keeps_latest() { + let state = fold(vec![ + register("nel", "anemone", "squid", 1), + retired("nel", "anemone", "squid", 10), + ]); + assert_eq!( + state.record_of(&repo("squid")).unwrap().retired_at, + Some(UnixSeconds::new(10)) + ); + + let later = fold(vec![ + register("nel", "anemone", "squid", 1), + retired("nel", "anemone", "squid", 10), + retired("nel", "anemone", "squid", 20), + ]); + assert_eq!( + later.record_of(&repo("squid")).unwrap().retired_at, + Some(UnixSeconds::new(20)), + "a later ack wins" + ); + + let stale = fold(vec![ + register("nel", "anemone", "squid", 1), + retired("nel", "anemone", "squid", 20), + retired("nel", "anemone", "squid", 5), + ]); + assert_eq!( + stale.record_of(&repo("squid")).unwrap().retired_at, + Some(UnixSeconds::new(20)), + "an old ack can't retire a fresh shadow" + ); + } + + #[test] + fn retire_under_mismatched_owner_is_no_op() { + let state = fold(vec![ + register("nel", "anemone", "squid", 1), + retired("olaren", "anemone", "squid", 10), + ]); + assert_eq!( + state.record_of(&repo("squid")).unwrap().retired_at, + None, + "only the owner on file can retire" + ); + } + + #[test] + fn registration_before_retired_field_still_applies() { + #[derive(Serialize)] + struct Before<'a> { + owner: &'a str, + rkey: &'a str, + name: &'a str, + repo: &'a str, + created_at: i64, + policy: ContributionPolicy, + fostered: bool, + } + + #[derive(Serialize)] + #[serde(tag = "op", content = "data", rename_all = "snake_case")] + enum BeforeChange<'a> { + Register(Before<'a>), + } + + let bytes = serde_ipld_dagcbor::to_vec(&BeforeChange::Register(Before { + owner: "did:plc:nel", + rkey: "anemone", + name: "anemone", + repo: "did:plc:squid", + created_at: 5, + policy: ContributionPolicy::Collaborators, + fostered: false, + })) + .unwrap(); + let decoded = RegistryChange::decode(&bytes) + .expect("Change from before this field existed still applies"); + let state = fold(vec![decoded]); + assert_eq!( + state.record_of(&repo("squid")).unwrap().retired_at, + None, + "a repo registered before this field existed isn't retired" + ); + } +} diff --git a/knot2/crates/knot-cobs/src/social.rs b/knot2/crates/knot-cobs/src/social.rs new file mode 100644 index 0000000000000000000000000000000000000000..c1bef1c45c23e1fb311355622b940991ea9a308f --- /dev/null +++ b/knot2/crates/knot-cobs/src/social.rs @@ -0,0 +1,1241 @@ +use std::fmt::Debug; +use std::marker::PhantomData; + +use knot_cob::{ + Append, Approved, Authored, Authorize, ChangePayload, Checkpoint, CobHome, CobStore, Emits, + Evaluate, HistoryModel, Materialized, Record, SnapshotStride, StateSize, WriteContext, +}; +use knot_runtime::Signer; +use knot_types::{ + AccountDid, ActorId, Blocked, CobId, ContributionPermission, ContributionPolicy, Decision, + RecordAddress, RecordBody, RecordCid, RecordCollection, RepoDid, RepoRole, SubjectNumber, + UnixSeconds, +}; +use serde::de::{DeserializeOwned, DeserializeSeed, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Creation { + pub address: RecordAddress, + pub author: AccountDid, + pub number: SubjectNumber, + pub parent: Option, + pub created_at: UnixSeconds, + pub produces: RecordCid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Editing { + pub author: AccountDid, + pub from: RecordCid, + pub produces: RecordCid, + pub edited_at: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Erasure { + pub author: AccountDid, + pub from: RecordCid, + pub erased_at: UnixSeconds, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Opening { + Contribution, + Reaction, + Moderation, + Ownership, + Triage, +} + +impl Opening { + pub fn decide(self, permission: ContributionPermission) -> Decision { + match self { + Self::Contribution | Self::Reaction => permission.contributes(), + Self::Moderation => Decision::allow_if( + permission.contributes().is_allowed() || permission.moderates().is_allowed(), + ), + Self::Ownership => Decision::allow_if( + permission.blocked() == Blocked::No && matches!(permission.role(), RepoRole::Owner), + ), + Self::Triage => Decision::allow_if( + permission.blocked() == Blocked::No + && matches!(permission.role(), RepoRole::Owner | RepoRole::Collaborator), + ), + } + } + + fn admits_edit(self, permission: ContributionPermission, own: bool) -> bool { + match self { + Self::Contribution | Self::Reaction | Self::Moderation => { + (own && permission.contributes().is_allowed()) + || permission.moderates().is_allowed() + } + Self::Ownership => self.decide(permission).is_allowed(), + Self::Triage => false, + } + } + + fn admits_erase(self, permission: ContributionPermission, own: bool) -> bool { + match self { + Self::Contribution => permission.moderates().is_allowed(), + Self::Reaction | Self::Moderation | Self::Triage => { + own || permission.moderates().is_allowed() + } + Self::Ownership => self.decide(permission).is_allowed(), + } + } +} + +pub trait SocialBody: + Serialize + DeserializeOwned + Debug + Clone + PartialEq + Eq + Sized +{ + const COLLECTION: &'static str; + const OPENING: Opening; + fn from_stored(body: &RecordBody) -> Result; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Settled { + AtSeed, + Superseded, +} + +#[derive(Debug, thiserror::Error)] +pub enum SettleError { + #[error(transparent)] + Cob(#[from] knot_cob::CobError), + #[error("{0}")] + Stored(String), +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn settle( + store: &CobStore, + home: &CobHome, + repo: &RepoDid, + signer: &dyn Signer, + existing: Option, + creation: Creation, + record: RecordBody, + edits: Vec<(Editing, RecordBody)>, +) -> Result { + let created_at = creation.created_at; + let body = B::from_stored(&record).map_err(SettleError::Stored)?; + let permission = + ContributionPermission::new(Blocked::No, RepoRole::Owner, ContributionPolicy::Anyone); + let author = creation.author.clone(); + let cx = WriteContext::new(&author, repo, permission); + let versions: Vec = std::iter::once(creation.produces) + .chain(edits.iter().map(|(editing, _)| editing.produces)) + .collect(); + let object = match existing { + None => { + let materialized = Materialized { + change: SocialChange::Open { creation, body }, + record: Record::version(record), + }; + let approved = Approved::signed_by(&cx, materialized)?; + store + .create_approved::>(home, &approved, signer, created_at)? + .object + } + Some(object) => object, + }; + let current = store + .get::>(object)? + .state() + .object() + .and_then(|live| live.version()); + let Some(at) = versions + .iter() + .rposition(|version| Some(*version) == current) + else { + return Ok(Settled::Superseded); + }; + for (editing, record) in edits.into_iter().skip(at) { + let edited_at = editing.edited_at; + let editor = editing.author.clone(); + let cx = WriteContext::new(&editor, repo, permission); + let materialized = Materialized { + change: SocialChange::Edit { + editing, + body: B::from_stored(&record).map_err(SettleError::Stored)?, + }, + record: Record::version(record), + }; + let approved = Approved::signed_by(&cx, materialized)?; + store.append_approved::>(home, object, &approved, signer, edited_at)?; + } + Ok(Settled::AtSeed) +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "op", content = "data", rename_all = "snake_case")] +pub enum SocialChange { + Open { creation: Creation, body: B }, + Edit { editing: Editing, body: B }, + Erase { erasure: Erasure }, +} + +pub trait Social { + fn emission(&self) -> Emission<'_>; +} + +impl Social for SocialChange { + fn emission(&self) -> Emission<'_> { + match self { + Self::Open { creation, .. } => Emission::Open(creation), + Self::Edit { editing, .. } => Emission::Edit(editing), + Self::Erase { erasure } => Emission::Erase(erasure), + } + } +} + +impl Authored for SocialChange { + fn author(&self) -> &AccountDid { + self.emission().author() + } +} + +impl ChangePayload for SocialChange { + const TYPE: &'static str = B::COLLECTION; + + fn address(&self) -> Option { + match self { + Self::Open { creation, .. } => Some(creation.address.clone()), + Self::Edit { .. } | Self::Erase { .. } => None, + } + } + + fn emits(&self) -> Emits { + match self.emission().produces() { + Some(produces) => Emits::Version(produces), + None => Emits::Tombstone, + } + } + + fn collection() -> Option { + Some(collection_of::()) + } + + fn stamped_at(&self) -> Option { + Some(self.emission().at()) + } +} + +pub struct SocialCob(PhantomData); + +impl SocialCob { + pub fn collection() -> RecordCollection { + collection_of::() + } +} + +fn collection_of() -> RecordCollection { + RecordCollection::new(B::COLLECTION).expect("Social collection is valid NSID") +} + +impl Evaluate for SocialCob { + type State = SocialState; + type Change = SocialChange; + + const HISTORY: HistoryModel = HistoryModel::Convergent; + + fn initial() -> Self::State { + SocialState::Unopened + } + + fn apply(state: Self::State, change: Self::Change, _author: &ActorId) -> Self::State { + state.apply(&change) + } +} + +impl Checkpoint for SocialCob { + const SNAPSHOT_STRIDE: SnapshotStride = SnapshotStride::new(64); + + fn checkpoint_size(_state: &Self::State) -> StateSize { + StateSize::new(1) + } +} + +impl Authorize for SocialCob { + fn may_create(cx: &WriteContext<'_>) -> Decision { + B::OPENING.decide(cx.permission()) + } + + fn may_append(cx: &WriteContext<'_>, state: &Self::State, change: &Self::Change) -> Append { + state.decide(B::OPENING, cx, change) + } +} + +#[macro_export] +macro_rules! social_cob { + ( + body = $body:ty, + change = $change:ident, + cob = $cob:ident, + collection = $collection:expr, + opening = $opening:expr, + stored = $stored:path $(,)? + ) => { + impl $crate::SocialBody for $body { + const COLLECTION: &'static str = $collection; + const OPENING: $crate::Opening = $opening; + + fn from_stored(body: &knot_types::RecordBody) -> Result { + $stored(body) + } + } + + pub type $change = $crate::SocialChange<$body>; + pub type $cob = $crate::SocialCob<$body>; + }; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Emission<'a> { + Open(&'a Creation), + Edit(&'a Editing), + Erase(&'a Erasure), +} + +impl<'a> Emission<'a> { + pub const fn author(self) -> &'a AccountDid { + match self { + Self::Open(creation) => &creation.author, + Self::Edit(editing) => &editing.author, + Self::Erase(erasure) => &erasure.author, + } + } + + pub const fn from(self) -> Option { + match self { + Self::Open(_) => None, + Self::Edit(editing) => Some(editing.from), + Self::Erase(erasure) => Some(erasure.from), + } + } + + pub const fn produces(self) -> Option { + match self { + Self::Open(creation) => Some(creation.produces), + Self::Edit(editing) => Some(editing.produces), + Self::Erase(_) => None, + } + } + + pub const fn at(self) -> UnixSeconds { + match self { + Self::Open(creation) => creation.created_at, + Self::Edit(editing) => editing.edited_at, + Self::Erase(erasure) => erasure.erased_at, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SocialObject { + address: RecordAddress, + author: AccountDid, + number: SubjectNumber, + parent: Option, + created_at: UnixSeconds, + editor: AccountDid, + updated_at: UnixSeconds, + version: Option, +} + +impl SocialObject { + pub const fn address(&self) -> &RecordAddress { + &self.address + } + + pub const fn author(&self) -> &AccountDid { + &self.author + } + + pub const fn number(&self) -> SubjectNumber { + self.number + } + + pub const fn parent(&self) -> Option<&RecordAddress> { + self.parent.as_ref() + } + + pub const fn created_at(&self) -> UnixSeconds { + self.created_at + } + + pub const fn editor(&self) -> &AccountDid { + &self.editor + } + + pub const fn updated_at(&self) -> UnixSeconds { + self.updated_at + } + + pub const fn version(&self) -> Option { + self.version + } + + pub const fn is_erased(&self) -> bool { + self.version.is_none() + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum SocialState { + #[default] + Unopened, + Live(Box), +} +impl SocialState { + pub fn object(&self) -> Option<&SocialObject> { + match self { + Self::Unopened => None, + Self::Live(object) => Some(object.as_ref()), + } + } + + pub fn apply(self, change: &impl Social) -> Self { + match (self, change.emission()) { + (Self::Unopened, Emission::Open(creation)) => Self::Live(Box::new(SocialObject { + address: creation.address.clone(), + author: creation.author.clone(), + number: creation.number, + parent: creation.parent.clone(), + created_at: creation.created_at, + editor: creation.author.clone(), + updated_at: creation.created_at, + version: Some(creation.produces), + })), + (Self::Unopened, Emission::Edit(_) | Emission::Erase(_)) => Self::Unopened, + (Self::Live(object), Emission::Open(_)) => Self::Live(object), + (Self::Live(object), emission) if emission.from() != object.version => { + Self::Live(object) + } + (Self::Live(object), emission) => Self::Live(Box::new(SocialObject { + editor: emission.author().clone(), + updated_at: emission.at(), + version: emission.produces(), + ..*object + })), + } + } + + pub fn decide(&self, opening: Opening, cx: &WriteContext<'_>, change: &impl Social) -> Append { + let Self::Live(object) = self else { + return Append::Deny; + }; + let emission = change.emission(); + let permission = cx.permission(); + let own = object.author == *cx.actor() && matches!(permission.blocked(), Blocked::No); + let permitted = match emission { + Emission::Open(_) => false, + Emission::Edit(_) => opening.admits_edit(permission, own), + Emission::Erase(_) => opening.admits_erase(permission, own), + }; + match (permitted, object.version) { + (false, _) | (true, None) => Append::Deny, + (true, Some(current)) if emission.from() != Some(current) => Append::Stale { current }, + (true, Some(_)) => Append::Allow, + } + } +} + +const MAX_BODY_DEPTH: usize = 32; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AnyBody; + +// I know, I know, `IgnoredAny` exists. +// However it traverses nested values, and this is user input, and I don't want to traverse +// forever from malicious input. +impl<'de> Deserialize<'de> for AnyBody { + fn deserialize>(deserializer: D) -> Result { + Bounded(MAX_BODY_DEPTH) + .deserialize(deserializer) + .map(|()| Self) + } +} + +struct Bounded(usize); + +impl Bounded { + fn deeper(self) -> Result { + self.0 + .checked_sub(1) + .map(Self) + .ok_or_else(|| E::custom(format!("Body nests deeper than {MAX_BODY_DEPTH} levels"))) + } +} + +impl<'de> DeserializeSeed<'de> for Bounded { + type Value = (); + + fn deserialize>(self, deserializer: D) -> Result<(), D::Error> { + deserializer.deserialize_any(self) + } +} + +impl<'de> Visitor<'de> for Bounded { + type Value = (); + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Record body nested at most {MAX_BODY_DEPTH} levels deep") + } + + fn visit_bool(self, _: bool) -> Result<(), E> { + Ok(()) + } + + fn visit_i64(self, _: i64) -> Result<(), E> { + Ok(()) + } + + fn visit_i128(self, _: i128) -> Result<(), E> { + Ok(()) + } + + fn visit_u64(self, _: u64) -> Result<(), E> { + Ok(()) + } + + fn visit_u128(self, _: u128) -> Result<(), E> { + Ok(()) + } + + fn visit_f64(self, _: f64) -> Result<(), E> { + Ok(()) + } + + fn visit_str(self, _: &str) -> Result<(), E> { + Ok(()) + } + + fn visit_bytes(self, _: &[u8]) -> Result<(), E> { + Ok(()) + } + + fn visit_none(self) -> Result<(), E> { + Ok(()) + } + + fn visit_unit(self) -> Result<(), E> { + Ok(()) + } + + fn visit_some>(self, deserializer: D) -> Result<(), D::Error> { + self.deserialize(deserializer) + } + + fn visit_newtype_struct>( + self, + deserializer: D, + ) -> Result<(), D::Error> { + self.deserialize(deserializer) + } + + fn visit_seq>(self, mut seq: A) -> Result<(), A::Error> { + let inner = self.deeper::()?.0; + std::iter::from_fn(|| seq.next_element_seed(Bounded(inner)).transpose()) + .try_for_each(|element| element) + } + + fn visit_map>(self, mut map: A) -> Result<(), A::Error> { + let inner = self.deeper::()?.0; + std::iter::from_fn(|| { + map.next_entry_seed(Bounded(inner), Bounded(inner)) + .transpose() + }) + .try_for_each(|entry| entry.map(|((), ())| ())) + } +} + +pub type AnySocialChange = SocialChange; + +pub fn decode_social(payload: &[u8]) -> Result { + knot_cob::decode_payload(payload) +} + +#[cfg(test)] +pub(crate) fn seeded(address: RecordAddress, parent: Option) -> Creation { + Creation { + address, + author: AccountDid::new("did:plc:nel").unwrap(), + number: SubjectNumber::FIRST, + parent, + created_at: UnixSeconds::new(1), + produces: knot_types::RecordBody::new(vec![0x64, b'k', b'e', b'l', b'p']) + .unwrap() + .cid(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use knot_types::{Blocked, RecordBody, RecordRkey, RepoRole}; + use proptest::prelude::*; + + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + pub struct NoteBody { + text: String, + } + + fn note_stored(_: &RecordBody) -> Result { + Ok(NoteBody { + text: String::new(), + }) + } + + social_cob! { + body = NoteBody, + change = NoteChange, + cob = NoteCob, + collection = "org.tangled.test.note", + opening = Opening::Contribution, + stored = note_stored, + } + + fn who(suffix: u8) -> AccountDid { + AccountDid::new(format!("did:plc:a{suffix}")).unwrap() + } + + fn version(seed: u8) -> RecordCid { + RecordBody::new(vec![seed]).unwrap().cid() + } + + fn address() -> RecordAddress { + RecordAddress::new( + NoteCob::collection(), + RecordRkey::new("3lubrptx57d22").unwrap(), + ) + } + + fn creation(author: u8, seed: u8) -> Creation { + Creation { + address: address(), + author: who(author), + number: SubjectNumber::FIRST, + parent: None, + created_at: UnixSeconds::new(1), + produces: version(seed), + } + } + + fn opened(author: u8, seed: u8) -> AnySocialChange { + AnySocialChange::Open { + creation: creation(author, seed), + body: AnyBody, + } + } + + fn edited(author: u8, from: u8, seed: u8, at: i64) -> AnySocialChange { + AnySocialChange::Edit { + editing: Editing { + author: who(author), + from: version(from), + produces: version(seed), + edited_at: UnixSeconds::new(at), + }, + body: AnyBody, + } + } + + fn erased(author: u8, from: u8, at: i64) -> AnySocialChange { + AnySocialChange::Erase { + erasure: Erasure { + author: who(author), + from: version(from), + erased_at: UnixSeconds::new(at), + }, + } + } + + fn replayed(changes: &[AnySocialChange]) -> SocialState { + changes + .iter() + .fold(SocialState::Unopened, |state, change| state.apply(change)) + } + + #[test] + fn a_change_before_the_root_leaves_the_object_unopened() { + assert_eq!(replayed(&[edited(1, 1, 2, 2)]), SocialState::Unopened); + assert_eq!(replayed(&[erased(1, 1, 2)]), SocialState::Unopened); + assert!( + replayed(&[opened(1, 1), edited(1, 1, 2, 2)]) + .object() + .is_some_and(|object| object.version() == Some(version(2))), + "the state follows the version the latest change produced" + ); + } + + #[test] + fn a_second_root_never_moves_an_object_that_is_already_open() { + let once = replayed(&[opened(1, 1)]); + let twice = replayed(&[opened(1, 1), opened(2, 9)]); + assert_eq!( + once, twice, + "The address the root fixed, and the author that mr root named, both outlive any later root" + ); + } + + #[test] + fn an_erasure_clears_the_version_and_keeps_the_authorship() { + let state = replayed(&[opened(1, 1), erased(1, 1, 5)]); + let object = state.object().expect("Erased object still exists"); + assert!(object.is_erased()); + assert_eq!(object.author(), &who(1)); + assert_eq!(object.number(), SubjectNumber::FIRST); + } + + #[test] + fn an_edit_made_from_an_older_version_never_reaches_the_state() { + let live = replayed(&[opened(1, 1), edited(1, 1, 2, 2)]); + let stale = replayed(&[opened(1, 1), edited(1, 1, 2, 2), edited(1, 1, 3, 3)]); + assert_eq!( + live, stale, + "the projection ought to pin the version Edit states it was made from" + ); + } + + #[test] + fn only_one_of_two_changes_from_a_shared_version_is_stored() { + assert!( + replayed(&[opened(1, 1), erased(1, 1, 2), edited(1, 1, 3, 3)]) + .object() + .is_some_and(SocialObject::is_erased), + "Edit from version erasure retired doesn't republish the record" + ); + assert_eq!( + replayed(&[opened(1, 1), edited(1, 1, 3, 3), erased(1, 1, 2)]) + .object() + .and_then(SocialObject::version), + Some(version(3)), + "Erasure chasing version that already moved is stale" + ); + } + + #[test] + fn the_wire_mirror_decodes_what_a_typed_change_encodes() { + let typed = NoteChange::Open { + creation: creation(1, 1), + body: NoteBody { + text: "kelp".to_owned(), + }, + }; + let bytes = typed.encode().unwrap(); + assert_eq!( + decode_social(&bytes).unwrap().emission(), + typed.emission(), + "Indexer reads the same schema the writer wrote, not some copy of it" + ); + assert_eq!( + NoteChange::decode(&bytes).unwrap(), + typed, + "Typed round trip keeps the body the wire mirror drops" + ); + } + + #[test] + fn a_social_change_stamps_itself_with_the_time_it_declares() { + let erasure = NoteChange::Erase { + erasure: Erasure { + author: who(1), + from: version(1), + erased_at: UnixSeconds::new(9), + }, + }; + assert_eq!( + erasure.stamped_at(), + Some(UnixSeconds::new(9)), + "Engine stamps the erasure with the signed clock, not the payload's" + ); + assert_eq!(erasure.emits(), Emits::Tombstone); + assert_eq!(NoteChange::collection(), Some(NoteCob::collection())); + assert_eq!(NoteChange::type_name().as_str(), NoteBody::COLLECTION); + } + + fn permission(role: RepoRole, blocked: Blocked) -> knot_types::ContributionPermission { + knot_types::ContributionPermission::new( + blocked, + role, + knot_types::ContributionPolicy::Anyone, + ) + } + + fn decide( + opening: Opening, + state: &SocialState, + actor: u8, + permission: knot_types::ContributionPermission, + change: &AnySocialChange, + ) -> Append { + let actor = who(actor); + let repo = knot_types::RepoDid::new("did:plc:squid").unwrap(); + state.decide( + opening, + &WriteContext::new(&actor, &repo, permission), + change, + ) + } + + #[test] + fn the_blocklist_outranks_an_edit_of_an_account_s_own_object() { + let live = replayed(&[opened(1, 1)]); + let edit = edited(1, 1, 2, 2); + let erase = erased(1, 1, 2); + assert_eq!( + decide( + Opening::Contribution, + &live, + 1, + permission(RepoRole::Stranger, Blocked::No), + &edit + ), + Append::Allow, + "Author who isn't blocked edits the text said author wrote" + ); + assert_eq!( + decide( + Opening::Contribution, + &live, + 1, + permission(RepoRole::Stranger, Blocked::Yes), + &edit + ), + Append::Deny, + "Banned account doesn't rewrite the text this knot publishes for the account" + ); + assert_eq!( + decide( + Opening::Contribution, + &live, + 1, + permission(RepoRole::Stranger, Blocked::Yes), + &erase + ), + Append::Deny, + "Banned account doesn't erase text either; erasure is moderation action" + ); + assert_eq!( + decide( + Opening::Contribution, + &live, + 1, + permission(RepoRole::Owner, Blocked::Yes), + &erase + ), + Append::Deny, + "Blocklist denies owner's retraction too" + ); + } + + #[test] + fn only_a_moderator_erases_contributed_text() { + use knot_types::{ContributionPermission, ContributionPolicy}; + let live = replayed(&[opened(1, 1)]); + let erase = erased(1, 1, 2); + let open = ContributionPermission::new( + Blocked::No, + RepoRole::Stranger, + ContributionPolicy::Anyone, + ); + assert_eq!( + decide(Opening::Contribution, &live, 1, open, &edited(1, 1, 2, 2)), + Append::Allow, + "Stranger still edits the text they wrote" + ); + assert_eq!( + decide(Opening::Contribution, &live, 1, open, &erase), + Append::Deny, + "But erasing it is a collaborator's call, even under an open policy" + ); + assert_eq!( + decide(Opening::Reaction, &live, 1, open, &erase), + Append::Allow, + "A reaction is still the stranger's to take back" + ); + assert_eq!( + decide(Opening::Reaction, &live, 2, open, &erase), + Append::Deny, + "Not somebody else's reaction, though" + ); + let collaborator = ContributionPermission::new( + Blocked::No, + RepoRole::Collaborator, + ContributionPolicy::Collaborators, + ); + assert_eq!( + decide(Opening::Contribution, &live, 2, collaborator, &erase), + Append::Allow, + "Collaborator erases anyone's text" + ); + let banned = ContributionPermission::new( + Blocked::Yes, + RepoRole::Stranger, + ContributionPolicy::Anyone, + ); + assert_eq!( + decide(Opening::Contribution, &live, 1, banned, &erase), + Append::Deny, + "A blocked account mustn't write anything, retraction included" + ); + } + + #[test] + fn openings_decide_from_role_policy_and_block() { + use knot_types::{ContributionPolicy, Membership}; + [ + ( + Opening::Contribution, + RepoRole::KnotAdmin(Membership::Outsider), + ContributionPolicy::Collaborators, + Blocked::No, + Decision::Deny, + "an admin can't open an issue on a collaborators-only repo", + ), + ( + Opening::Moderation, + RepoRole::KnotAdmin(Membership::Outsider), + ContributionPolicy::Collaborators, + Blocked::No, + Decision::Allow, + "But closes one", + ), + ( + Opening::Moderation, + RepoRole::Stranger, + ContributionPolicy::Collaborators, + Blocked::No, + Decision::Deny, + "Policy that shut the author out of editing shuts them out of transitions too", + ), + ( + Opening::Moderation, + RepoRole::Stranger, + ContributionPolicy::Anyone, + Blocked::No, + Decision::Allow, + "And a contributor opens one", + ), + ( + Opening::Moderation, + RepoRole::Owner, + ContributionPolicy::Anyone, + Blocked::Yes, + Decision::Deny, + "Blocklist outranks both roles", + ), + ( + Opening::Ownership, + RepoRole::Owner, + ContributionPolicy::Anyone, + Blocked::No, + Decision::Allow, + "Owner defines repository labels", + ), + ( + Opening::Ownership, + RepoRole::Owner, + ContributionPolicy::Anyone, + Blocked::Yes, + Decision::Deny, + "Blocklist outranks the owner's label defs too", + ), + ( + Opening::Ownership, + RepoRole::KnotAdmin(Membership::Outsider), + ContributionPolicy::Anyone, + Blocked::No, + Decision::Deny, + "an admin moderates labels and leaves authoring to the owner", + ), + ( + Opening::Ownership, + RepoRole::Collaborator, + ContributionPolicy::Anyone, + Blocked::No, + Decision::Deny, + "a collaborator writes code and issues, and the owner writes the label set", + ), + ( + Opening::Triage, + RepoRole::Collaborator, + ContributionPolicy::Collaborators, + Blocked::No, + Decision::Allow, + "a collaborator applies labels, the appview's repo:push rule", + ), + ( + Opening::Triage, + RepoRole::Stranger, + ContributionPolicy::Anyone, + Blocked::No, + Decision::Deny, + "an open contribution policy accepts issues and rejects label ops", + ), + ( + Opening::Triage, + RepoRole::KnotAdmin(Membership::Outsider), + ContributionPolicy::Anyone, + Blocked::No, + Decision::Deny, + "repo:push is owner or collaborator, and a knot admin is neither", + ), + ] + .into_iter() + .for_each(|(opening, role, policy, blocked, expected, why)| { + let permission = knot_types::ContributionPermission::new(blocked, role, policy); + assert_eq!(opening.decide(permission), expected, "{why}"); + }); + } + + #[test] + fn edit_and_erasure_belong_to_author_or_moderator() { + let live = replayed(&[opened(1, 1)]); + let open = opened(2, 5); + let good = permission(RepoRole::Stranger, Blocked::No); + let collaborator = permission(RepoRole::Collaborator, Blocked::No); + [ + ( + good, + edited(2, 1, 2, 2), + Append::Deny, + "Stranger doesn't edit somebody else's text", + ), + ( + good, + erased(2, 1, 2), + Append::Deny, + "Stranger doesn't erase text either; erasure needs moderator", + ), + ( + collaborator, + erased(2, 1, 2), + Append::Allow, + "Collaborator takes down the text the repository publishes", + ), + ( + collaborator, + edited(2, 1, 2, 2), + Append::Allow, + "And rewrites it, with the collaborator as the new version's editor", + ), + ] + .into_iter() + .for_each(|(permission, change, expected, why)| { + assert_eq!( + decide(Opening::Contribution, &live, 2, permission, &change), + expected, + "{why}" + ); + }); + assert_eq!( + decide(Opening::Contribution, &live, 1, good, &open), + Append::Deny, + "Root, and only root, opens object" + ); + assert_eq!( + decide( + Opening::Contribution, + &SocialState::Unopened, + 1, + good, + &edited(1, 1, 2, 2) + ), + Append::Deny, + "An object no one opened doesn't take appends" + ); + } + + #[test] + fn label_openings_decide_who_edits_and_erases() { + use knot_types::Membership; + let live = replayed(&[opened(1, 1)]); + let owner = permission(RepoRole::Owner, Blocked::No); + let admin = permission(RepoRole::KnotAdmin(Membership::Outsider), Blocked::No); + let collaborator = permission(RepoRole::Collaborator, Blocked::No); + let stranger = permission(RepoRole::Stranger, Blocked::No); + [ + ( + Opening::Ownership, + 1, + owner, + edited(1, 1, 2, 2), + Append::Allow, + "Owner rewrites repository label defs", + ), + ( + Opening::Ownership, + 1, + admin, + edited(1, 1, 2, 2), + Append::Deny, + "an admin moderates labels and leaves authoring to the owner", + ), + ( + Opening::Ownership, + 1, + owner, + erased(1, 1, 2), + Append::Allow, + "Owner deletes a def and accompanying slug", + ), + ( + Opening::Ownership, + 2, + owner, + edited(1, 1, 2, 2), + Append::Allow, + "Whoever owns the repository edits, and a former owner who authored the \ + def doesn't", + ), + ( + Opening::Triage, + 1, + collaborator, + edited(1, 1, 2, 2), + Append::Deny, + "A label op stays as written, author included", + ), + ( + Opening::Triage, + 1, + collaborator, + erased(1, 1, 2), + Append::Allow, + "Author retracts the label op", + ), + ( + Opening::Triage, + 1, + admin, + erased(1, 1, 2), + Append::Allow, + "An admin retracts a label op too, moderating the label set", + ), + ( + Opening::Triage, + 2, + stranger, + erased(1, 1, 2), + Append::Deny, + "A stranger doesn't retract a label op", + ), + ] + .into_iter() + .for_each(|(opening, actor, permission, change, expected, why)| { + assert_eq!( + decide(opening, &live, actor, permission, &change), + expected, + "{why}" + ); + }); + } + + #[test] + fn an_edit_from_a_version_that_moved_on_reads_the_current_one() { + let live = replayed(&[opened(1, 1), edited(1, 1, 2, 2)]); + let good = permission(RepoRole::Stranger, Blocked::No); + assert_eq!( + decide(Opening::Contribution, &live, 1, good, &edited(1, 1, 3, 3)), + Append::Stale { + current: version(2) + }, + "Loser of the race reads the version it has to rebase onto" + ); + assert_eq!( + decide(Opening::Contribution, &live, 1, good, &edited(1, 2, 3, 3)), + Append::Allow + ); + let erased_state = replayed(&[opened(1, 1), erased(1, 1, 2)]); + assert_eq!( + decide( + Opening::Contribution, + &erased_state, + 1, + good, + &edited(1, 1, 3, 3) + ), + Append::Deny, + "Erased object doesn't leave any version to edit from" + ); + } + + fn changes() -> impl Strategy> { + proptest::collection::vec((0u8..3, 0u8..4, 1u8..5, 1u8..5), 0..8).prop_map(|ops| { + std::iter::once(opened(0, 0)) + .chain( + ops.into_iter() + .enumerate() + .map(|(index, (op, author, from, seed))| { + let at = index as i64 + 2; + match op { + 0 => opened(author, seed), + 1 => edited(author, from, seed, at), + _ => erased(author, from, at), + } + }), + ) + .collect() + }) + } + + #[derive(Serialize)] + struct Nest(Vec); + + fn nested(depth: usize) -> Nest { + (0..depth).fold(Nest(Vec::new()), |inner, _| Nest(vec![inner])) + } + + #[test] + fn the_wire_mirror_rejects_a_body_nested_past_the_depth_it_will_walk() { + let wire = |depth: usize| { + serde_ipld_dagcbor::to_vec(&SocialChange::Open { + creation: creation(1, 1), + body: nested(depth), + }) + .unwrap() + }; + assert!( + decode_social(&wire(MAX_BODY_DEPTH - 1)).is_ok(), + "Body within the bound decodes. Its nesting is the indexer's to ignore" + ); + assert!( + decode_social(&wire(MAX_BODY_DEPTH + 8)).is_err(), + "Body past bound is refused instead of walked recursively" + ); + } + + proptest! { + #![proptest_config(ProptestConfig { cases: 64, ..ProptestConfig::default() })] + + #[test] + fn replayed_changes_match_the_first_pass_state(changes in changes()) { + let once = replayed(&changes); + let doubled: Vec = changes + .iter() + .flat_map(|change| [change.clone(), change.clone()]) + .collect(); + prop_assert_eq!( + once, + replayed(&doubled), + "Replayed change set gives the state the first pass reached" + ); + } + + #[test] + fn author_is_fixed_at_root_whatever_follows(changes in changes()) { + prop_assume!(changes.len() > 1); + let whole = replayed(&changes); + let prefix = replayed(&changes[..changes.len().saturating_sub(1)]); + prop_assert_eq!( + prefix.object().map(SocialObject::author), + whole.object().map(SocialObject::author), + "author is fixed at the root. Later changes don't move it" + ); + } + } +} diff --git a/knot2/crates/knot-cobs/src/ticket.rs b/knot2/crates/knot-cobs/src/ticket.rs new file mode 100644 index 0000000000000000000000000000000000000000..2c71e9bae2fdc9a3a2a9bc3618093c5d39332b7f --- /dev/null +++ b/knot2/crates/knot-cobs/src/ticket.rs @@ -0,0 +1,164 @@ +use knot_record::label::LabelToken; +use knot_record::ticket::{TICKET_COLLECTION, TicketState}; +use knot_types::{AccountDid, LabelKey, RecordBody}; +use serde::{Deserialize, Serialize}; + +use crate::Opening; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Ticket { + pub state: TicketState, + #[serde(default)] + pub tangled_labels: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub assignees: Option>, +} + +#[derive(Deserialize)] +struct StoredTicket { + state: TicketState, + #[serde(rename = "tangledLabels", default)] + tangled_labels: Option>, + #[serde(default)] + assignees: Option>, +} + +fn decode_stored(body: &RecordBody) -> Result { + let stored: StoredTicket = + serde_ipld_dagcbor::from_slice(body.as_bytes()).map_err(|error| error.to_string())?; + Ok(Ticket { + state: stored.state, + tangled_labels: stored.tangled_labels.map(|tokens| { + tokens + .into_iter() + .map(|token| token.key().clone()) + .collect() + }), + assignees: stored.assignees.filter(|assignees| !assignees.is_empty()), + }) +} + +crate::social_cob! { + body = Ticket, + change = TicketChange, + cob = TicketCob, + collection = TICKET_COLLECTION, + opening = Opening::Contribution, + stored = decode_stored, +} + +macro_rules! social_kinds { + ($($body:ty),+ $(,)?) => { + const SOCIAL_KINDS: usize = [$(stringify!($body)),+].len(); + + pub fn social_collections() -> [knot_types::RecordCollection; SOCIAL_KINDS] { + [$( + knot_types::RecordCollection::new(<$body as $crate::SocialBody>::COLLECTION) + .expect("a social collection is a valid nsid"), + )+] + } + + pub fn social_state( + store: &knot_cob::CobStore, + type_name: &knot_types::TypeName, + object: knot_types::CobId, + ) -> Result, knot_cob::CobError> { + match type_name.as_str() { + $(<$body as $crate::SocialBody>::COLLECTION => + store.materialize::<$crate::SocialCob<$body>>(object).map(Some),)+ + _ => Ok(None), + } + } + + pub fn history_model_of( + type_name: &knot_types::TypeName, + ) -> Option { + match type_name.as_str() { + $(<$body as $crate::SocialBody>::COLLECTION => + Some(<$crate::SocialCob<$body> as knot_cob::Evaluate>::HISTORY),)+ + _ => None, + } + } + + pub fn social_root_address( + store: &knot_cob::CobStore, + type_name: &knot_types::TypeName, + object: knot_types::CobId, + ) -> Result, knot_cob::CobError> { + match type_name.as_str() { + $(<$body as $crate::SocialBody>::COLLECTION => + store.root_address::<$crate::SocialChange<$body>>(object),)+ + _ => Ok(None), + } + } + + pub fn validate_stored( + collection: &knot_types::RecordCollection, + record: &knot_types::RecordBody, + ) -> Option> { + match collection.as_str() { + $(<$body as $crate::SocialBody>::COLLECTION => + Some(<$body as $crate::SocialBody>::from_stored(record).map(|_| ())),)+ + _ => None, + } + } + + #[allow(clippy::too_many_arguments)] + pub fn settle_stored( + store: &knot_cob::CobStore, + home: &knot_cob::CobHome, + repo: &knot_types::RepoDid, + signer: &dyn knot_runtime::Signer, + collection: &knot_types::RecordCollection, + existing: Option, + creation: $crate::Creation, + record: knot_types::RecordBody, + edits: Vec<($crate::Editing, knot_types::RecordBody)>, + ) -> Result, $crate::SettleError> { + match collection.as_str() { + $(<$body as $crate::SocialBody>::COLLECTION => $crate::social::settle::<$body>( + store, home, repo, signer, existing, creation, record, edits, + ) + .map(Some),)+ + _ => Ok(None), + } + } + }; +} + +social_kinds!( + crate::Ticket, + crate::Comment, + crate::Reaction, + crate::LabelDef, + crate::Patch, + crate::Manifest, + crate::Artifact +); + +#[cfg(test)] +mod tests { + use super::*; + use crate::{SocialChange, social::seeded}; + use knot_cob::ChangePayload; + use knot_record::ticket::ticket_collection; + use knot_types::{RecordAddress, RecordRkey}; + + #[test] + fn ticket_change_round_trips_with_its_state() { + let rkey = RecordRkey::new("3lubrptx57d22").unwrap(); + let change = SocialChange::Open { + creation: seeded(RecordAddress::new(ticket_collection(), rkey), None), + body: Ticket { + state: TicketState::Open, + tangled_labels: None, + assignees: None, + }, + }; + assert_eq!( + TicketChange::decode(&change.encode().unwrap()).unwrap(), + change, + "the state the write path signed is the state the replay reads back" + ); + } +} diff --git a/knot2/crates/knot-cobs/tests/cobs.rs b/knot2/crates/knot-cobs/tests/cobs.rs new file mode 100644 index 0000000000000000000000000000000000000000..cfff67f79a74402ec0acbf4f61b1126db3b1e5d2 --- /dev/null +++ b/knot2/crates/knot-cobs/tests/cobs.rs @@ -0,0 +1,363 @@ +mod common; + +use common::{account, at, did, fixture, grant, home, registration, rename, reopen, rkey, signer}; +use knot_cob::CobStore; +use knot_cobs::{ + CollaboratorsChange, MembersChange, MembersCob, RegistrationField, RegistryChange, + RegistryError, Removal, RepoRegistryCob, register_repo, rename_repo, +}; +use knot_types::{OwnerDid, RepoDid}; + +#[test] +fn members_roundtrip_and_reload_is_identical() { + let (_dir, repo) = fixture(); + let key = signer(1); + let store = CobStore::new(&repo); + + let created = store + .create( + &home(), + &MembersChange::Add(grant("nel", "nel", 1)), + &key, + at(1), + ) + .unwrap(); + store + .update( + &home(), + created.object, + &MembersChange::Add(grant("olaren", "nel", 2)), + &key, + at(2), + ) + .unwrap(); + store + .update( + &home(), + created.object, + &MembersChange::Add(grant("teq", "nel", 3)), + &key, + at(3), + ) + .unwrap(); + store + .update( + &home(), + created.object, + &MembersChange::Remove(Removal { + subject: account("teq"), + }), + &key, + at(4), + ) + .unwrap(); + + let object = store.get::(created.object).unwrap(); + let list = object.state(); + assert!(list.contains(&account("nel"))); + assert_eq!( + list.get(&account("olaren")).unwrap().added_by, + account("nel") + ); + assert!(!list.contains(&account("teq"))); + assert_eq!(list.len(), 2); + + let reopened = reopen(repo); + let reloaded = CobStore::new(&reopened) + .get::(created.object) + .unwrap(); + assert_eq!(list, reloaded.state()); + + let (_dup_dir, dup_repo) = fixture(); + let dup_store = CobStore::new(&dup_repo); + let dup = dup_store + .create( + &home(), + &MembersChange::Add(grant("nel", "olaren", 7)), + &key, + at(1), + ) + .unwrap(); + dup_store + .update( + &home(), + dup.object, + &MembersChange::Add(grant("nel", "olaren", 7)), + &key, + at(2), + ) + .unwrap(); + let replayed = dup_store + .get::(dup.object) + .unwrap() + .into_state(); + assert_eq!(replayed.len(), 1, "replaying an identical add is a no-op"); + let entry = replayed.get(&account("nel")).unwrap(); + assert_eq!(entry.added_by, account("olaren")); + assert_eq!(entry.created_at, at(7)); + + let converge = |order: [(&str, &str, i64); 2]| { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let [first, second] = order; + let created = store + .create( + &home(), + &MembersChange::Add(grant(first.0, first.1, first.2)), + &key, + at(1), + ) + .unwrap(); + store + .update( + &home(), + created.object, + &MembersChange::Add(grant(second.0, second.1, second.2)), + &key, + at(2), + ) + .unwrap(); + store + .get::(created.object) + .unwrap() + .into_state() + }; + assert_eq!( + converge([("nel", "nel", 10), ("olaren", "nel", 20)]), + converge([("olaren", "nel", 20), ("nel", "nel", 10)]), + "independent grants converge across write order" + ); +} + +#[test] +fn collaborators_live_in_a_per_repo_cob_namespace() { + let (_dir, repo) = fixture(); + let key = signer(2); + let store = CobStore::new(&repo); + store + .create( + &home(), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &key, + at(1), + ) + .unwrap(); + + let cob_ref = repo.references().unwrap().into_iter().find(|record| { + record + .name + .as_str() + .contains("org.tangled.graph.collaborator") + }); + assert!(cob_ref.is_some(), "collaborators live under refs/cobs"); + assert!(repo.advertised_refs().unwrap().is_empty()); +} + +#[test] +fn registry_handler_semantics() { + let (_dir, repo) = fixture(); + let key = signer(21); + let store = CobStore::new(&repo); + let nel = did::("nel"); + + let created = store + .create( + &home(), + &RegistryChange::Register(registration("nel", "anemone", "squid", 1)), + &key, + at(1), + ) + .unwrap(); + let object = created.object; + + let clash = register_repo( + &store, + &home(), + object, + registration("nel", "anemone", "whelk", 2), + &key, + at(2), + ); + assert!( + matches!(clash, Err(RegistryError::RkeyTaken { .. })), + "register can't claim the canonical rkey, of live repo, under the same owner" + ); + assert_eq!( + register_repo( + &store, + &home(), + object, + registration("nel", "anemone", "squid", 1), + &key, + at(2), + ) + .unwrap(), + None, + "re-registering record registry already has doesn't append change" + ); + let moved = register_repo( + &store, + &home(), + object, + registration("nel", "anemone", "squid", 2), + &key, + at(2), + ); + assert!( + matches!( + moved, + Err(RegistryError::AlreadyRegistered { + field: RegistrationField::CreatedAt, + .. + }) + ), + "Register that would move stored field answers with field's name instead of \ + dropping change, got {moved:?}" + ); + + let renamed = rename_repo( + &store, + &home(), + object, + rename("nel", "barnacle", "squid"), + &key, + at(2), + ) + .unwrap(); + assert!(renamed.is_some(), "real rename appends a change"); + let state = store.get::(object).unwrap().into_state(); + assert_eq!( + state.resolve(&nel, &rkey("anemone")), + Some(&did::("squid")), + "prior rkey keeps resolving as an alias" + ); + assert_eq!( + state.resolve(&nel, &rkey("barnacle")), + Some(&did::("squid")) + ); + assert_eq!( + state.record_of(&did("squid")).unwrap().rkey, + rkey("barnacle") + ); + + let redundant = register_repo( + &store, + &home(), + object, + registration("nel", "barnacle", "squid", 1), + &key, + at(3), + ) + .unwrap(); + assert_eq!( + redundant, None, + "re-register matching whole stored record doesn't append change" + ); + assert_eq!( + store + .get::(object) + .unwrap() + .into_state() + .resolve(&nel, &rkey("anemone")), + Some(&did::("squid")), + "retained alias survives a redundant re-register" + ); + + store + .update( + &home(), + object, + &RegistryChange::Register(registration("nel", "kelp", "whelk", 4)), + &key, + at(4), + ) + .unwrap(); + + let taken = rename_repo( + &store, + &home(), + object, + rename("nel", "kelp", "squid"), + &key, + at(5), + ); + assert!( + matches!(taken, Err(RegistryError::RkeyTaken { .. })), + "rename cannot take canonical rkey of another live repo" + ); + + let unhosted = rename_repo( + &store, + &home(), + object, + rename("nel", "limpet", "conch"), + &key, + at(6), + ); + assert!( + matches!(unhosted, Err(RegistryError::NotHosted { .. })), + "rename of a repo with no registration is refused" + ); + + let moved = rename_repo( + &store, + &home(), + object, + rename("olaren", "uni", "squid"), + &key, + at(7), + ); + assert!( + matches!(moved, Err(RegistryError::OwnerMoved { .. })), + "rename under an owner the repo no longer belongs to is refused" + ); + + assert_eq!( + store + .get::(object) + .unwrap() + .into_state() + .record_of(&did("squid")) + .unwrap() + .rkey, + rkey("barnacle"), + "refused renames left the canonical rkey untouched" + ); + + let revived = register_repo( + &store, + &home(), + object, + registration("nel", "anemone", "cuttlefish", 8), + &key, + at(8), + ) + .unwrap(); + assert!( + revived.is_some(), + "rkey that renaming made free now, is now claimable by a new repo" + ); + assert_eq!( + store + .get::(object) + .unwrap() + .into_state() + .resolve(&nel, &rkey("anemone")), + Some(&did::("cuttlefish")), + "newly claimed rkey lets the new repo live under it" + ); + + let mut flipped = registration("nel", "anemone", "cuttlefish", 8); + flipped.fostered = true; + let flip = register_repo(&store, &home(), object, flipped, &key, at(9)); + assert!( + matches!( + flip, + Err(RegistryError::AlreadyRegistered { + field: RegistrationField::Fostered, + .. + }) + ), + "a re-register can't mess with a repo's fostered... birthright? forgive the term" + ); +} diff --git a/knot2/crates/knot-cobs/tests/common/mod.rs b/knot2/crates/knot-cobs/tests/common/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..ff970b14440aa3b6a71b5121687e861af22c506e --- /dev/null +++ b/knot2/crates/knot-cobs/tests/common/mod.rs @@ -0,0 +1,217 @@ +#![allow(dead_code)] + +use knot_cob::{ChangePayload, CobError, CobHome, CobStore}; +use knot_cobs::{Grant, Members, MembersChange, MembersCob, Registration, RegistryChange, Rename}; +use knot_git::{Layout, RefUpdate, Repo}; +use knot_runtime::{K256Signer, SeededEntropy, Signer}; +use knot_types::{ + AccountDid, ActorId, CobId, Oid, OwnerDid, RefName, RepoDid, RepoName, RepoRkey, TypeName, + UnixSeconds, +}; +use tempfile::TempDir; + +pub fn fixture() -> (TempDir, Repo) { + let dir = tempfile::tempdir().unwrap(); + let repo = Layout::new(dir.path()) + .create(&RepoDid::new("did:plc:squid").unwrap()) + .unwrap(); + (dir, repo) +} + +pub fn signer(seed: u64) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed)) +} + +pub fn home() -> CobHome { + CobHome::from(&RepoDid::new("did:plc:squid").unwrap()) +} + +pub fn account(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{suffix}")).unwrap() +} + +pub fn grant(subject: &str, added_by: &str, at: i64) -> Grant { + Grant { + subject: account(subject), + added_by: account(added_by), + created_at: UnixSeconds::new(at), + } +} + +pub fn at(seconds: i64) -> UnixSeconds { + UnixSeconds::new(seconds) +} + +pub fn owner_of(seed: u64) -> ActorId { + ActorId::from_secp256k1(signer(seed).public_key().as_bytes()) +} + +pub fn rkey(value: &str) -> RepoRkey { + RepoRkey::new(value).unwrap() +} + +pub fn did(suffix: &str) -> T +where + T::Err: std::fmt::Debug, +{ + format!("did:plc:{suffix}").parse().unwrap() +} + +pub fn registration(owner: &str, key: &str, repo_id: &str, ts: i64) -> Registration { + Registration { + owner: OwnerDid::new(format!("did:plc:{owner}")).unwrap(), + rkey: rkey(key), + name: RepoName::new(key).unwrap(), + repo: RepoDid::new(format!("did:plc:{repo_id}")).unwrap(), + created_at: at(ts), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + } +} + +pub fn rename(owner_id: &str, key: &str, repo_id: &str) -> Rename { + Rename { + owner: did::(owner_id), + rkey: rkey(key), + name: RepoName::new(key).unwrap(), + repo: did::(repo_id), + } +} + +pub fn reopen(repo: Repo) -> Repo { + let path = repo.path().to_path_buf(); + drop(repo); + Repo::open(path).unwrap() +} + +pub fn members_store( + seed: u64, + steps: &[(MembersChange, i64)], +) -> (TempDir, Repo, K256Signer, CobId) { + let (dir, repo) = fixture(); + let key = signer(seed); + let store = CobStore::new(&repo); + let (first, first_at) = &steps[0]; + let created = store.create(&home(), first, &key, at(*first_at)).unwrap(); + steps[1..].iter().for_each(|(change, ts)| { + store + .update(&home(), created.object, change, &key, at(*ts)) + .unwrap(); + }); + (dir, repo, key, created.object) +} + +pub fn build_members(seed: u64, steps: &[(MembersChange, i64)]) -> Members { + let (_dir, repo, _key, object) = members_store(seed, steps); + CobStore::new(&repo) + .get::(object) + .unwrap() + .into_state() +} + +pub fn write_cob_commit( + repo: &Repo, + type_name: &TypeName, + payload: &[u8], + parents: &[Oid], + author: &ActorId, + timestamp: i64, +) -> Oid { + let git = repo.git(); + let payload_oid = git.write_blob(payload).unwrap().detach(); + let tree = gix::objs::Tree { + entries: vec![gix::objs::tree::Entry { + mode: gix::objs::tree::EntryKind::Blob.into(), + filename: "payload".into(), + oid: payload_oid, + }], + }; + let revision = git.write_object(tree).unwrap().detach(); + let identity = gix::actor::Signature { + name: "knot".into(), + email: "noreply@knot".into(), + time: gix::date::Time::new(timestamp, 0), + }; + let commit = gix::objs::Commit { + tree: revision, + parents: parents.iter().map(|oid| oid.object_id()).collect(), + author: identity.clone(), + committer: identity, + encoding: None, + message: Vec::new().into(), + extra_headers: vec![ + ("cob-type".into(), type_name.as_str().into()), + ("cob-author".into(), author.as_str().into()), + ("cob-sig".into(), "00".into()), + ], + }; + Oid::from(git.write_object(commit).unwrap().detach()) +} + +pub fn cob_ref(type_name: &TypeName, object: CobId) -> RefName { + RefName::new(format!( + "refs/cobs/{}/{}", + type_name.as_str(), + object.oid().to_hex() + )) + .unwrap() +} + +pub fn forked_members_object( + seed: u64, + root: (MembersChange, i64), + left: (MembersChange, i64), + right: (MembersChange, i64), + merge: (MembersChange, i64), +) -> (TempDir, Repo, CobId) { + let (dir, repo) = fixture(); + let nsid = MembersChange::type_name(); + let author = ActorId::from_secp256k1(signer(seed).public_key().as_bytes()); + let enc = |c: &MembersChange| c.encode().unwrap(); + + let root_oid = write_cob_commit(&repo, &nsid, &enc(&root.0), &[], &author, root.1); + let object = CobId::new(root_oid); + let left_oid = write_cob_commit(&repo, &nsid, &enc(&left.0), &[root_oid], &author, left.1); + let right_oid = write_cob_commit(&repo, &nsid, &enc(&right.0), &[root_oid], &author, right.1); + let merge_oid = write_cob_commit( + &repo, + &nsid, + &enc(&merge.0), + &[left_oid, right_oid], + &author, + merge.1, + ); + repo.update_ref(&RefUpdate::Create { + name: cob_ref(&nsid, object), + new: merge_oid, + }) + .unwrap(); + (dir, repo, object) +} + +pub fn forked_members( + seed: u64, + root: (MembersChange, i64), + left: (MembersChange, i64), + right: (MembersChange, i64), + merge: (MembersChange, i64), +) -> Result { + let (_dir, repo, object) = forked_members_object(seed, root, left, right, merge); + CobStore::new(&repo) + .get::(object) + .map(|object| object.into_state()) +} + +pub fn registry_with(repo: &Repo, key: &K256Signer, name: &str, repo_id: &str) -> CobId { + let store = CobStore::new(repo); + store + .create( + &home(), + &RegistryChange::Register(registration("nel", name, repo_id, 1)), + key, + at(1), + ) + .unwrap() + .object +} diff --git a/knot2/crates/knot-cobs/tests/fuzz_smoke.rs b/knot2/crates/knot-cobs/tests/fuzz_smoke.rs new file mode 100644 index 0000000000000000000000000000000000000000..e25149c565b3c5268e7699b52f98160e85086812 --- /dev/null +++ b/knot2/crates/knot-cobs/tests/fuzz_smoke.rs @@ -0,0 +1,15 @@ +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn the_cob_change_decoder_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_cobs::fuzz::change_decode(&data); + } + + #[test] + fn the_cob_ref_parser_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_cobs::fuzz::ref_parse(&data); + } +} diff --git a/knot2/crates/knot-cobs/tests/invariants.rs b/knot2/crates/knot-cobs/tests/invariants.rs new file mode 100644 index 0000000000000000000000000000000000000000..93056694547fcc05efccc95a0c35396f8766acbc --- /dev/null +++ b/knot2/crates/knot-cobs/tests/invariants.rs @@ -0,0 +1,504 @@ +mod common; + +use common::{ + account, at, build_members, cob_ref, fixture, forked_members, forked_members_object, grant, + home, members_store, owner_of, registration, registry_with, rkey, signer, write_cob_commit, +}; +use knot_cob::{ChangePayload, CobError, CobHome, CobStore, Evaluate}; +use knot_cobs::{ + Accept, CollaboratorsChange, ImportError, Invite, MembersChange, MembersCob, RegistryChange, + RegistryError, Removal, RepoRef, RepoRegistryCob, deregister_repo, register_repo, + verify_cob_ref, +}; +use knot_git::RefUpdate; +use knot_runtime::Signer; +use knot_types::{ActorId, CobId, OwnerDid, RepoDid, TypeName}; +use proptest::prelude::*; +use serde::Serialize; + +#[test] +fn forked_acl_is_rejected_not_merged() { + let forked = forked_members( + 1, + (MembersChange::Add(grant("seed", "seed", 1)), 1), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + (MembersChange::Add(grant("nel", "olaren", 3)), 3), + (MembersChange::Add(grant("teq", "teq", 4)), 4), + ); + assert!( + matches!(forked, Err(CobError::ForkedHistory { .. })), + "forked ACL is refused regardless of which branch a merge would favor" + ); +} + +fn accepted(subject: &str, seconds: i64) -> MembersChange { + MembersChange::Accept(Accept { + subject: account(subject), + verified_at: at(seconds), + }) +} + +#[test] +fn an_acceptance_without_an_admission_stays_off_the_grant_list() { + let uninvited = build_members( + 3, + &[ + (MembersChange::Add(grant("olaren", "olaren", 1)), 1), + (accepted("nel", 2), 2), + ], + ); + assert!( + !uninvited.contains(&account("nel")), + "an acceptance without an admission doesn't put nel on the grant list" + ); + + let admitted = build_members( + 4, + &[ + (MembersChange::Add(grant("nel", "olaren", 1)), 1), + (accepted("nel", 2), 2), + (accepted("nel", 9), 9), + ], + ); + assert_eq!( + admitted.get(&account("nel")), + build_members(4, &[(MembersChange::Add(grant("nel", "olaren", 1)), 1)]) + .get(&account("nel")), + "replaying the consent era's acceptances has to leave the admission exactly as it was" + ); +} + +#[test] +fn linear_member_semantics() { + let readd = build_members( + 2, + &[ + (MembersChange::Add(grant("nel", "olaren", 1)), 1), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + (MembersChange::Add(grant("nel", "teq", 3)), 3), + ], + ); + assert!( + readd.contains(&account("nel")), + "linear re-add after a remove is a legitimate decision and takes effect" + ); + + let stale_remove = build_members( + 20, + &[ + (MembersChange::Add(grant("nel", "olaren", 5)), 5), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + ], + ); + assert!( + !stale_remove.contains(&account("nel")), + "in a linear chain Remove is Add's child, so it applies last even with an older timestamp" + ); + + let signed_by_one = build_members(1, &[(MembersChange::Add(grant("nel", "olaren", 9)), 1)]); + assert_eq!( + signed_by_one.get(&account("nel")).unwrap().added_by, + account("olaren"), + "added_by is whatever the payload claims, unrelated to who signed" + ); + let signed_by_another = + build_members(99, &[(MembersChange::Add(grant("nel", "olaren", 9)), 1)]); + assert_eq!( + signed_by_one, signed_by_another, + "a different signing key over an identical payload yields identical state" + ); + + let once = build_members( + 10, + &[ + (MembersChange::Add(grant("nel", "nel", 1)), 1), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + ], + ); + let twice = build_members( + 10, + &[ + (MembersChange::Add(grant("nel", "nel", 1)), 1), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 3, + ), + ], + ); + assert_eq!(once, twice, "replaying a remove is idempotent"); + assert!(once.is_empty()); + + let created_at = build_members( + 13, + &[ + (MembersChange::Add(grant("nel", "olaren", 100)), 1), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + (MembersChange::Add(grant("nel", "teq", 50)), 3), + ], + ); + let entry = created_at.get(&account("nel")).unwrap(); + assert_eq!(entry.added_by, account("teq"), "last linear Add wins"); + assert_eq!( + entry.created_at, + at(50), + "the later Add's created_at takes effect even though it is older than an earlier entry's" + ); +} + +#[test] +fn verify_rejects_a_change_with_a_forged_signature() { + let (_dir, repo) = fixture(); + let nsid = MembersChange::type_name(); + let owner = owner_of(32); + let payload = MembersChange::Add(grant("nel", "nel", 1)).encode().unwrap(); + let root = write_cob_commit(&repo, &nsid, &payload, &[], &owner, 1); + let object = CobId::new(root); + repo.update_ref(&RefUpdate::Create { + name: cob_ref(&nsid, object), + new: root, + }) + .unwrap(); + + let store = CobStore::new(&repo); + assert!( + store.get::(object).is_ok(), + "read path materializes without checking signatures, by design" + ); + assert!( + matches!( + store.verify::(&home(), object, &owner), + Err(CobError::UnverifiedChange { .. }) + ), + "import verification catches forged signature the read path trusts" + ); +} + +#[derive(Serialize)] +struct WireRegister<'a> { + op: &'a str, + data: WireRegistration<'a>, +} + +#[derive(Serialize)] +struct WireRegistration<'a> { + owner: &'a str, + rkey: &'a str, + name: &'a str, + repo: &'a str, + created_at: i64, +} + +fn encode_register(rkey: &str, name: &str) -> Vec { + serde_ipld_dagcbor::to_vec(&WireRegister { + op: "register", + data: WireRegistration { + owner: "did:plc:nel", + rkey, + name, + repo: "did:plc:squid", + created_at: 1, + }, + }) + .unwrap() +} + +#[test] +fn malformed_repo_name_or_rkey_is_rejected_at_decode() { + assert!( + RegistryChange::decode(&encode_register("anemone", "anemone")).is_ok(), + "control: well-formed wire payload decodes" + ); + assert!( + RegistryChange::decode(&encode_register("anemone", "../../etc/passwd")).is_err(), + "traversal repo name fails newtype validation during decode, never reaching a ref" + ); + assert!( + RegistryChange::decode(&encode_register("anemone", "refs/heads/main")).is_err(), + "name with path separators is rejected at decode" + ); + assert!( + RegistryChange::decode(&encode_register("not a record key", "anemone")).is_err(), + "rkey outside record-key grammar is rejected at decode" + ); + assert!( + RegistryChange::decode(&encode_register("..", "anemone")).is_err(), + "reserved '..' rkey is rejected at decode" + ); +} + +#[test] +fn registry_handler_guards() { + let (_dir, repo) = fixture(); + let key = signer(70); + let store = CobStore::new(&repo); + let object = registry_with(&repo, &key, "anemone", "squid"); + let nel = || OwnerDid::new("did:plc:nel").unwrap(); + let squid = || RepoDid::new("did:plc:squid").unwrap(); + + let already = register_repo( + &store, + &home(), + object, + registration("olaren", "fork", "squid", 2), + &key, + at(2), + ); + assert!( + matches!(already, Err(RegistryError::AlreadyRegistered { .. })), + "a repo DID already registered elsewhere cannot be claimed again" + ); + + let unregistered = deregister_repo( + &store, + &home(), + object, + RepoRef { + owner: nel(), + rkey: rkey("barnacle"), + }, + squid(), + &key, + at(3), + ); + assert!(matches!( + unregistered, + Err(RegistryError::NotRegistered { .. }) + )); + + let mismatch = deregister_repo( + &store, + &home(), + object, + RepoRef { + owner: nel(), + rkey: rkey("anemone"), + }, + RepoDid::new("did:plc:whelk").unwrap(), + &key, + at(4), + ); + assert!( + matches!(mismatch, Err(RegistryError::RepoMismatch { .. })), + "deregister whose expected repo doesn't match the keyed one is refused" + ); + assert_eq!( + store + .get::(object) + .unwrap() + .into_state() + .resolve(&nel(), &rkey("anemone")), + Some(&squid()), + "a refused deregister left the registration intact" + ); + + register_repo( + &store, + &home(), + object, + registration("nel", "barnacle", "whelk", 5), + &key, + at(5), + ) + .unwrap(); + assert_eq!( + store + .get::(object) + .unwrap() + .into_state() + .owner_of(&RepoDid::new("did:plc:whelk").unwrap()), + Some(nel()), + "a fresh repo DID lands" + ); + + deregister_repo( + &store, + &home(), + object, + RepoRef { + owner: nel(), + rkey: rkey("anemone"), + }, + squid(), + &key, + at(6), + ) + .unwrap(); + let after_deregister = store.get::(object).unwrap().into_state(); + assert!( + after_deregister.resolve(&nel(), &rkey("anemone")).is_none(), + "a matching deregister removes the keyed repo" + ); + assert_eq!( + after_deregister.resolve(&nel(), &rkey("barnacle")), + Some(&RepoDid::new("did:plc:whelk").unwrap()), + "deregistering one repo leaves its sibling resolving" + ); +} +#[test] +fn verify_cob_ref_boundary_cases() { + let (_dir, repo, key, object) = members_store( + 90, + &[ + (MembersChange::Add(grant("nel", "nel", 1)), 1), + (MembersChange::Add(grant("olaren", "nel", 2)), 2), + ], + ); + let store = CobStore::new(&repo); + let owner = ActorId::from_secp256k1(key.public_key().as_bytes()); + let refname = cob_ref(&MembersChange::type_name(), object); + + assert!( + store.verify::(&home(), object, &owner).is_ok(), + "every change is validly signed by the owning key" + ); + assert!( + matches!( + store.verify::(&home(), object, &owner_of(31)), + Err(CobError::UnverifiedChange { .. }) + ), + "a change not authored by the claimed owner is refused at import" + ); + + assert_eq!( + verify_cob_ref(&store, &home(), &refname, &owner).unwrap(), + object, + "a genuine object verifies through the namespace dispatcher" + ); + assert!(matches!( + verify_cob_ref(&store, &home(), &refname, &owner_of(91)), + Err(ImportError::Cob(CobError::UnverifiedChange { .. })) + )); + + let elsewhere = CobHome::from(&RepoDid::new("did:plc:limpet").unwrap()); + assert!( + matches!( + verify_cob_ref(&store, &elsewhere, &refname, &owner), + Err(ImportError::Cob(CobError::UnverifiedChange { .. })) + ), + "an object pushed under a different repo home is refused at import" + ); + + assert!(matches!( + verify_cob_ref( + &store, + &home(), + &knot_types::RefName::new("refs/heads/main").unwrap(), + &owner, + ), + Err(ImportError::NotCobRef(_)) + )); + + let stray = cob_ref(&TypeName::new("org.tangled.test.unknown").unwrap(), object); + assert!(matches!( + verify_cob_ref(&store, &home(), &stray, &owner), + Err(ImportError::UnknownType(_)) + )); + + let collaborators = store + .create( + &home(), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &key, + at(1), + ) + .unwrap() + .object; + let collab_ref = cob_ref(&CollaboratorsChange::type_name(), collaborators); + assert_eq!( + verify_cob_ref(&store, &home(), &collab_ref, &owner).unwrap(), + collaborators, + "the dispatcher routes a second namespace to its own resolver, not a hardcoded type" + ); + + let (_forked_dir, forked_repo, forked) = forked_members_object( + 95, + (MembersChange::Add(grant("seed", "seed", 1)), 1), + (MembersChange::Add(grant("nel", "olaren", 2)), 2), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 3, + ), + (MembersChange::Add(grant("teq", "teq", 4)), 4), + ); + let forked_store = CobStore::new(&forked_repo); + let forked_ref = cob_ref(&MembersChange::type_name(), forked); + assert!( + matches!( + verify_cob_ref(&forked_store, &home(), &forked_ref, &owner_of(95)), + Err(ImportError::Cob(CobError::ForkedHistory { .. })) + ), + "forked linear history is refused at import alongside the signature check" + ); +} + +fn knot_signed(op: u8, subject: &str, when: i64) -> MembersChange { + match op { + 0 => MembersChange::Invite(Invite(grant(subject, "olaren", when))), + 1 => MembersChange::Remove(Removal { + subject: account(subject), + }), + _ => MembersChange::Add(grant(subject, "olaren", when)), + } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 96, ..ProptestConfig::default() })] + + #[test] + fn a_subject_is_on_the_grant_list_only_because_an_admission_put_it_there( + ops in prop::collection::vec((0u8..3, 0u8..3), 1..24) + ) { + let author = ActorId::from_secp256k1(&[0x02; 33]); + let list = ops + .iter() + .enumerate() + .fold(MembersCob::initial(), |state, (step, (op, who))| { + let change = knot_signed(*op, &format!("s{who}"), step as i64 + 1); + MembersCob::apply(state, change, &author) + }); + + let unearned: Vec = (0u8..3) + .filter(|who| list.contains(&account(&format!("s{who}")))) + .filter(|who| !ops.iter().any(|(op, subject)| *op != 1 && subject == who)) + .collect(); + prop_assert!( + unearned.is_empty(), + "subjects on the grant list that an admission didn't put there: {unearned:?}" + ); + } +} diff --git a/knot2/crates/knot-cobs/tests/social.rs b/knot2/crates/knot-cobs/tests/social.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb5ff17b3123f520d79c052377dd6b06ca607ab7 --- /dev/null +++ b/knot2/crates/knot-cobs/tests/social.rs @@ -0,0 +1,332 @@ +mod common; + +use common::{account, at, cob_ref, fixture, home, owner_of, signer, write_cob_commit}; +use knot_cob::{ + Approved, ChangePayload, Checkpoint, CobError, CobStore, Materialized, Record, WriteContext, +}; +use knot_cobs::{Creation, Editing, Erasure, SocialObject, SocialState, social_cob}; +use knot_git::RefUpdate; +use knot_types::{ + AccountDid, Blocked, ContributionPermission, ContributionPolicy, RecordAddress, RecordBody, + RecordCid, RecordCollection, RecordRkey, RepoDid, RepoRole, SubjectNumber, UnixSeconds, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct NoteBody { + text: String, +} + +fn note_stored(_: &RecordBody) -> Result { + Ok(NoteBody { + text: String::new(), + }) +} + +social_cob! { + body = NoteBody, + change = NoteChange, + cob = NoteCob, + collection = "org.tangled.test.note", + opening = knot_cobs::Opening::Contribution, + stored = note_stored, +} + +fn squid() -> RepoDid { + RepoDid::new("did:plc:squid").unwrap() +} + +fn address() -> RecordAddress { + RecordAddress::new( + RecordCollection::new("org.tangled.test.note").unwrap(), + RecordRkey::new("3lubrptx57d22").unwrap(), + ) +} + +fn record(text: &str) -> (RecordBody, RecordCid) { + let body = RecordBody::new( + serde_ipld_dagcbor::to_vec(&std::collections::BTreeMap::from([("text", text)])).unwrap(), + ) + .unwrap(); + let cid = body.cid(); + (body, cid) +} + +fn opened(author: &AccountDid, text: &str, when: i64) -> Materialized { + let (body, produces) = record(text); + Materialized { + change: NoteChange::Open { + creation: Creation { + address: address(), + author: author.clone(), + number: SubjectNumber::FIRST, + parent: None, + created_at: at(when), + produces, + }, + body: NoteBody { + text: text.to_owned(), + }, + }, + record: Record::version(body), + } +} + +fn edited(author: &AccountDid, from: RecordCid, text: &str, when: i64) -> Materialized { + let (body, produces) = record(text); + Materialized { + change: NoteChange::Edit { + editing: Editing { + author: author.clone(), + from, + produces, + edited_at: at(when), + }, + body: NoteBody { + text: text.to_owned(), + }, + }, + record: Record::version(body), + } +} + +fn erased(author: &AccountDid, from: RecordCid, when: i64) -> Materialized { + Materialized { + change: NoteChange::Erase { + erasure: Erasure { + author: author.clone(), + from, + erased_at: at(when), + }, + }, + record: Record::Tombstone, + } +} + +fn context<'a>(actor: &'a AccountDid, repo: &'a RepoDid, role: RepoRole) -> WriteContext<'a> { + WriteContext::new( + actor, + repo, + ContributionPermission::new(Blocked::No, role, ContributionPolicy::Anyone), + ) +} + +fn live(state: &SocialState) -> RecordCid { + state + .object() + .and_then(SocialObject::version) + .expect("Object has current version") +} + +#[test] +fn a_social_object_opens_edits_and_erases_through_the_approved_doors_only() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(31); + let (nel, olaren, squid) = (account("nel"), account("olaren"), squid()); + let author = context(&nel, &squid, RepoRole::Stranger); + + let first = opened(&nel, "kelp", 1); + let created = store + .create_approved::( + &home(), + &Approved::signed_by(&author, first).unwrap(), + &key, + at(1), + ) + .unwrap(); + let state = store.get::(created.object).unwrap(); + let opened_at = live(state.state()); + + let edit = edited(&nel, opened_at, "kelp forest", 2); + let knot_cob::Emits::Version(produced) = edit.change.emits() else { + panic!("an edit emits a version") + }; + store + .append_approved::( + &home(), + created.object, + &Approved::signed_by(&author, edit).unwrap(), + &key, + at(2), + ) + .unwrap(); + assert_eq!( + live(store.get::(created.object).unwrap().state()), + produced, + "Version edit named is version the replay ends on" + ); + + let stranger = context(&olaren, &squid, RepoRole::Stranger); + let intruded = store.append_approved::( + &home(), + created.object, + &Approved::signed_by(&stranger, erased(&olaren, produced, 3)).unwrap(), + &key, + at(3), + ); + assert!( + matches!(intruded, Err(CobError::AppendRejected { .. })), + "somebody else's text isn't stranger's to erase, and stranger doesn't moderate: {intruded:?}" + ); + + let stale = store.append_approved::( + &home(), + created.object, + &Approved::signed_by(&author, edited(&nel, opened_at, "from the past", 3)).unwrap(), + &key, + at(3), + ); + let expected_uri = address().at_uri(&squid); + assert!( + matches!(&stale, Err(CobError::StaleRecord { uri, current, .. }) if *uri == expected_uri && *current == produced), + "edit, from version that moved on, reads at-uri and current version: {stale:?}" + ); + + let retracted = store.append_approved::( + &home(), + created.object, + &Approved::signed_by(&author, erased(&nel, produced, 4)).unwrap(), + &key, + at(4), + ); + assert!( + matches!(retracted, Err(CobError::AppendRejected { .. })), + "erasing is a collaborator's call, not the stranger author's: {retracted:?}" + ); + + let collaborator = context(&olaren, &squid, RepoRole::Collaborator); + store + .append_approved::( + &home(), + created.object, + &Approved::signed_by(&collaborator, erased(&olaren, produced, 4)).unwrap(), + &key, + at(4), + ) + .unwrap(); + let records: Vec = store + .changes_since::(created.object, None) + .unwrap() + .changes + .into_iter() + .map(|change| change.record) + .collect(); + assert!( + matches!( + records.as_slice(), + [Record::Version(_), Record::Version(_), Record::Tombstone] + ), + "every change of social object has version or tombstone, got {records:?}" + ); + assert!( + store + .get::(created.object) + .unwrap() + .state() + .object() + .is_some_and(SocialObject::is_erased), + "Erased object keeps its authorship and loses version" + ); +} + +#[test] +fn a_social_object_past_the_snapshot_stride_writes_a_checkpoint_that_replays_to_the_same_state() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(32); + let (nel, squid) = (account("nel"), squid()); + let author = context(&nel, &squid, RepoRole::Stranger); + let object = store + .create_approved::( + &home(), + &Approved::signed_by(&author, opened(&nel, "kelp", 1)).unwrap(), + &key, + at(1), + ) + .unwrap() + .object; + + (2..=NoteCob::SNAPSHOT_STRIDE.get() as i64 + 1).for_each(|step| { + let from = live(store.get::(object).unwrap().state()); + store + .append_approved::( + &home(), + object, + &Approved::signed_by(&author, edited(&nel, from, &format!("step{step}"), step)) + .unwrap(), + &key, + at(step), + ) + .unwrap(); + }); + + let checkpoints = repo + .references() + .unwrap() + .into_iter() + .filter(|reference| reference.name.as_str().starts_with("refs/cob-checkpoints/")) + .count(); + assert_eq!(checkpoints, 1, "Stride wrote exactly one checkpoint"); + let (materialized, _) = store.materialize::(object).unwrap(); + assert_eq!( + serde_ipld_dagcbor::to_vec(&materialized).unwrap(), + serde_ipld_dagcbor::to_vec(store.get::(object).unwrap().state()).unwrap(), + "Checkpointed replay and replay from root are byte-identical" + ); +} + +#[test] +fn a_forked_social_history_written_outside_the_knot_replays_through_the_convergent_arm() { + let (_dir, repo) = fixture(); + let nsid = NoteChange::type_name(); + let actor = owner_of(33); + let nel = account("nel"); + let root_change = opened(&nel, "kelp", 1); + let knot_cob::Emits::Version(opened_at) = root_change.change.emits() else { + panic!("an opened note emits a version") + }; + let enc = |change: &NoteChange| change.encode().unwrap(); + + let root = write_cob_commit(&repo, &nsid, &enc(&root_change.change), &[], &actor, 1); + let object = knot_types::CobId::new(root); + let left = write_cob_commit( + &repo, + &nsid, + &enc(&edited(&nel, opened_at, "left", 2).change), + &[root], + &actor, + 2, + ); + let right_change = edited(&nel, opened_at, "right", 3); + let right = write_cob_commit(&repo, &nsid, &enc(&right_change.change), &[root], &actor, 3); + let merge = write_cob_commit( + &repo, + &nsid, + &enc(&edited(&nel, opened_at, "merged", 4).change), + &[left, right], + &actor, + 4, + ); + repo.update_ref(&RefUpdate::Create { + name: cob_ref(&nsid, object), + new: merge, + }) + .unwrap(); + + let replayed = CobStore::new(&repo).get::(object).unwrap(); + let knot_cob::Emits::Version(left_version) = edited(&nel, opened_at, "left", 2).change.emits() + else { + panic!("an edit emits a version") + }; + assert_eq!( + live(replayed.state()), + left_version, + "First edit in causal order takes version, and two that named root's \ + version after it are stale" + ); + assert_eq!( + replayed.state().object().map(SocialObject::updated_at), + Some(UnixSeconds::new(2)) + ); +} diff --git a/knot2/crates/knot-config/Cargo.toml b/knot2/crates/knot-config/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..414d6e8a06e942e18894ea6b9f3173f1a62ce27a --- /dev/null +++ b/knot2/crates/knot-config/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "knot-config" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-runtime = { workspace = true } +knot-messages = { workspace = true } +confique = { workspace = true } +thiserror = { workspace = true } +base64 = { workspace = true } +tempfile = { workspace = true } +url = { workspace = true } diff --git a/knot2/crates/knot-config/src/lib.rs b/knot2/crates/knot-config/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..39a4a5cc605370667111d37d504d5666a7d897bb --- /dev/null +++ b/knot2/crates/knot-config/src/lib.rs @@ -0,0 +1,2130 @@ +use std::fmt; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use base64::Engine; +use confique::Config; +use knot_runtime::HttpLimits; +use knot_types::{ + AccountDid, AdmissionPolicy, AppviewEndpoint, BranchName, ContributionPolicy, KnotHostname, + KnotServiceUrl, ObjectFormat, TrustedProxies, comma_separated, +}; +use url::Url; + +#[derive(Debug, Config)] +pub struct KnotConfig { + #[config(nested)] + pub server: ServerConfig, + #[config(nested)] + pub tls: TlsConfig, + #[config(nested)] + pub acl: AclConfig, + #[config(nested)] + pub repo: RepoConfig, + #[config(nested)] + pub git: GitConfig, + #[config(nested)] + pub secrets: SecretsConfig, + #[config(nested)] + pub http: HttpConfig, + #[config(nested)] + pub atproto: AtprotoConfig, + #[config(nested)] + pub xrpc: XrpcConfig, + #[config(nested)] + pub maintenance: MaintenanceConfig, + #[config(nested)] + pub pack_cache: PackCacheConfig, + #[config(nested)] + pub pack: PackConfig, + #[config(nested)] + pub lfs: LfsConfig, + #[config(nested)] + pub social: SocialConfig, + #[config(nested)] + pub keyfill: KeyfillConfig, + #[config(nested)] + pub resources: ResourcesConfig, + #[config(nested)] + pub homepage: HomepageConfig, + #[config(nested)] + pub links: LinksConfig, + #[config(nested)] + pub contact: ContactConfig, + + #[config(nested)] + pub messages: knot_messages::MessagesConfig, +} + +#[derive(Debug, Config)] +pub struct AclConfig { + #[config(env = "KNOT_ADMISSION", default = "closed")] + pub admission: AdmissionPolicy, + + #[config(env = "KNOT_LEGACY_ADMIN_SECRET_ENV")] + pub legacy_admin_secret_env: Option, +} + +#[derive(Debug, Config)] +pub struct TlsConfig { + #[config(env = "KNOT_TLS_CERT_PATH")] + pub cert_path: Option, + + #[config(env = "KNOT_TLS_KEY_PATH")] + pub key_path: Option, + + #[config(env = "KNOT_TLS_HTTP3", default = true)] + pub http3: bool, + + #[config(env = "KNOT_TLS_ACME_ENABLED", default = false)] + pub acme_enabled: bool, + + #[config(env = "KNOT_TLS_ACME_CACHE_DIR")] + pub acme_cache_dir: Option, + + #[config(env = "KNOT_TLS_ACME_CONTACT")] + pub acme_contact: Option, + + #[config(env = "KNOT_TLS_ACME_STAGING", default = false)] + pub acme_staging: bool, + + #[config(env = "KNOT_TLS_MTLS_ENABLED", default = false)] + pub mtls_enabled: bool, + + #[config(env = "KNOT_TLS_MTLS_CLIENT_CA_PATH")] + pub mtls_client_ca_path: Option, + + #[config(env = "KNOT_TLS_MTLS_ADMIN_SPKI_PIN")] + pub mtls_admin_spki_pin: Option, +} + +#[derive(Debug, Config)] +pub struct ServerConfig { + #[config(env = "KNOT_HOSTNAME")] + pub hostname: KnotHostname, + + #[config(env = "KNOT_ADMINS", parse_env = parse_admins)] + pub admins: Vec, + + #[config(env = "KNOT_LISTEN_ADDR", default = "[::]:5555")] + pub listen_addr: SocketAddr, + + #[config(env = "KNOT_LISTEN_HEADER_TIMEOUT_MS", default = 10_000)] + pub listen_header_timeout_ms: u64, + + #[config(env = "KNOT_LISTEN_IDLE_TIMEOUT_MS", default = 60_000)] + pub listen_idle_timeout_ms: u64, + + #[config(env = "KNOT_LISTEN_MAX_CONNECTIONS", default = 1_024)] + pub listen_max_connections: u32, + + #[config(env = "KNOT_LISTEN_RATE_LIMIT_PER_SECOND", default = 50)] + pub listen_rate_limit_per_second: u32, + + #[config(env = "KNOT_LISTEN_RATE_LIMIT_BURST", default = 200)] + pub listen_rate_limit_burst: u32, + + #[config(env = "KNOT_LISTEN_MAX_INFLIGHT_REQUESTS", default = 1_024)] + pub listen_max_inflight_requests: u32, + + #[config(env = "KNOT_LISTEN_REQUEST_TIMEOUT_MS", default = 60_000)] + pub listen_request_timeout_ms: u64, + + #[config(env = "KNOT_LISTEN_BODY_TIMEOUT_MS", default = 30_000)] + pub listen_body_timeout_ms: u64, + + #[config(env = "KNOT_LISTEN_WRITE_REQUEST_TIMEOUT_MS", default = 1_800_000)] + pub listen_write_request_timeout_ms: u64, + + #[config(env = "KNOT_INTERNAL_LISTEN_ADDR", default = "[::1]:5444")] + pub internal_listen_addr: SocketAddr, + + #[config(env = "KNOT_SSH_LISTEN_ADDR", default = "[::]:2222")] + pub ssh_listen_addr: SocketAddr, + + #[config(env = "KNOT_SSH_HOST_KEY_FILE")] + pub ssh_host_key_file: PathBuf, + + #[config(env = "KNOT_SSH_MAX_PACK_BYTES", default = 8_589_934_592u64)] + pub ssh_max_pack_bytes: u64, + + /// Maximum number of SSH git operations that one client address can run at the same time. + #[config(env = "KNOT_SSH_MAX_INFLIGHT_PER_PEER", default = 4)] + pub ssh_max_inflight_per_peer: u32, + + /// Addresses allowed to prefix an ssh connection with a PROXY protocol v1 header, + /// each a bare IP without a port or a CIDR block, + /// for ex the address of a TCP proxy such as nginx `stream` with `proxy_protocol on`. + /// The knot will rate-limit the connection by the source address in the header. + /// The knot never reads the header from any other address. + #[config( + env = "KNOT_SSH_PROXY_PROTOCOL_FROM", + parse_env = comma_separated, + default = [] + )] + pub ssh_proxy_protocol_from: Vec, + + #[config(env = "KNOT_APPVIEW_ENDPOINT", default = "https://tangled.org")] + pub appview_endpoint: AppviewEndpoint, +} + +#[derive(Debug, Config)] +pub struct RepoConfig { + #[config(env = "KNOT_SCAN_PATH")] + pub scan_path: PathBuf, + + #[config(env = "KNOT_DEFAULT_BRANCH", default = "main")] + pub default_branch: BranchName, + + #[config(env = "KNOT_CONTRIBUTION_POLICY", default = "anyone")] + pub contribution_policy: ContributionPolicy, + + #[config(env = "KNOT_REPO_WRITE_BURST", default = 30)] + pub write_burst: u32, + #[config(env = "KNOT_REPO_WRITE_REFILL_MS", default = 500)] + pub write_refill_ms: u64, +} + +#[derive(Debug, Config)] +pub struct HomepageConfig { + #[config(env = "KNOT_HOMEPAGE_ENABLED", default = true)] + pub enabled: bool, + + #[config(env = "KNOT_HOMEPAGE_PATH")] + pub path: Option, +} + +/// Policy documents `org.tangled.knot.describeKnot` points to. +#[derive(Debug, Config)] +pub struct LinksConfig { + #[config(env = "KNOT_LINKS_PRIVACY_POLICY")] + pub privacy_policy: Option, + + #[config(env = "KNOT_LINKS_TERMS_OF_SERVICE")] + pub terms_of_service: Option, +} + +/// How to reach the knot operator, served by `org.tangled.knot.describeKnot`. +#[derive(Debug, Config)] +pub struct ContactConfig { + #[config(env = "KNOT_CONTACT_EMAIL")] + pub email: String, +} + +#[derive(Debug)] +pub enum HomepageSource { + Disabled, + Default, + File(PathBuf), +} + +impl HomepageConfig { + pub fn source(&self) -> HomepageSource { + match (self.enabled, self.path.as_ref()) { + (false, _) => HomepageSource::Disabled, + (true, None) => HomepageSource::Default, + (true, Some(path)) => HomepageSource::File(path.clone()), + } + } +} + +#[derive(Debug, Config)] +pub struct GitConfig { + #[config(env = "KNOT_GIT_OBJECT_FORMAT", default = "sha256")] + pub object_format: ObjectFormat, +} + +#[derive(Debug, Config)] +pub struct SecretsConfig { + #[config(env = "KNOT_SEALED_KEY_FILE")] + pub sealed_key_file: PathBuf, + + #[config(env = "KNOT_MASTER_KEY_ENV")] + pub master_key_env: String, +} + +#[derive(Debug, Config)] +pub struct HttpConfig { + #[config(env = "KNOT_HTTP_CONNECT_TIMEOUT_MS", default = 5_000)] + pub connect_timeout_ms: u64, + + #[config(env = "KNOT_HTTP_READ_TIMEOUT_MS", default = 30_000)] + pub read_timeout_ms: u64, + + #[config(env = "KNOT_HTTP_REQUEST_TIMEOUT_MS", default = 60_000)] + pub request_timeout_ms: u64, + + #[config(env = "KNOT_HTTP_MAX_RESPONSE_BYTES", default = 16_777_216)] + pub max_response_bytes: u64, + + #[config(env = "KNOT_HTTP_BLOCK_PRIVATE_ADDRESSES", default = true)] + pub block_private_addresses: bool, +} + +#[derive(Debug, Config)] +pub struct AtprotoConfig { + #[config(env = "KNOT_PLC_DIRECTORY")] + pub plc_directory: Url, + /// How many PLC documents the reconciler sweep checks against the directory concurrently. + #[config(env = "KNOT_PLC_SWEEP_WIDTH", default = 1)] + pub plc_sweep_width: u32, + + /// An archive of the per-repo signing keys, + /// that gotta get squished into the canonical key store here + /// at knot start time. + /// If unset, the knot will attempt just `repo-signing-keys.json` + /// next to `secrets.sealed_key_file` if that file exists too. + #[config(env = "KNOT_REPO_SIGNING_KEYS")] + pub repo_signing_keys: Option, + + /// Relays that get a com.atproto.sync.requestCrawl for this knot on startup. + #[config(env = "KNOT_CRAWLERS", parse_env = comma_separated, default = [ + "https://index.tangled.network", + "https://relay.fire.hose.cam", + "https://relay3.fr.hose.cam", + "https://bsky.network", + "https://northamerica.firehose.network", + "https://europe.firehose.network", + "https://asia.firehose.network", + "https://atproto.africa", + "https://relay.upcloud.world", + ])] + pub crawlers: Vec, +} + +#[derive(Debug, Config)] +pub struct XrpcConfig { + #[config(env = "KNOT_XRPC_MAX_BODY_BYTES", default = 65_536)] + pub max_body_bytes: u64, + + #[config(env = "KNOT_XRPC_MAX_RESPONSE_BYTES", default = 5_242_880)] + pub max_response_bytes: u64, + + /// Upper bound on bytes that a single archive spools, + /// across all our surfaces: the org.tangled.git.getArchive query, + /// `git archive --remote` over SSH, + /// and the smart HTTP archive route. + /// The knot will refuse writing smth that would blast an archive past this bound. + #[config(env = "KNOT_XRPC_MAX_ARCHIVE_BYTES", default = 1_073_741_824)] + pub max_archive_bytes: u64, + + #[config(env = "KNOT_XRPC_TREE_LAST_COMMIT_BUDGET_MS", default = 300)] + pub tree_last_commit_budget_ms: u64, + + #[config(env = "KNOT_XRPC_BLOB_LAST_COMMIT_BUDGET_MS", default = 2_000)] + pub blob_last_commit_budget_ms: u64, + + #[config(env = "KNOT_XRPC_LANGUAGES_BUDGET_MS", default = 1_000)] + pub languages_budget_ms: u64, + + /// Body limit for the seedRepo procedure, whose record batches routinely + /// exceed the general XRPC body limit. + #[config(env = "KNOT_XRPC_MAX_PATCH_BYTES", default = 16_777_216)] + pub max_patch_bytes: u64, + + /// Limit on the total decompressed size of a patch the merge procedures parse, + /// bounding binary-delta inflation and hunk expansion apart from the + /// compressed body limit above. + #[config(env = "KNOT_XRPC_MAX_PATCH_DECOMPRESSED_BYTES", default = 134_217_728)] + pub max_patch_decompressed_bytes: u64, + + #[config(env = "KNOT_XRPC_PREAUTH_BURST", default = 20)] + pub preauth_burst: u32, + + #[config(env = "KNOT_XRPC_PREAUTH_REFILL_MS", default = 100)] + pub preauth_refill_ms: u64, + + #[config(env = "KNOT_XRPC_PER_PEER_INFLIGHT", default = 8)] + pub per_peer_inflight: u32, + + #[config(env = "KNOT_XRPC_GLOBAL_INFLIGHT", default = 64)] + pub global_inflight: u32, + + #[config(env = "KNOT_XRPC_MAX_PENDING_RESERVATIONS", default = 256)] + pub max_pending_reservations: u32, + + /// Per-account limit on reserved repository keys awaiting creation, so one + /// account cannot consume the whole pending-reservation budget. + #[config(env = "KNOT_XRPC_PER_ACTOR_RESERVATIONS", default = 32)] + pub per_actor_reservations: u32, + + /// How long a reserved repository key is held before it lapses and its + /// sealed key is reclaimed, in seconds. + #[config(env = "KNOT_XRPC_RESERVATION_TTL_SECS", default = 3600)] + pub reservation_ttl_secs: u64, + + #[config(env = "KNOT_XRPC_FORK_MAX_PACK_BYTES", default = 1_073_741_824)] + pub fork_max_pack_bytes: u64, + + #[config(env = "KNOT_XRPC_IMPORT_MAX_PACK_BYTES", default = 268_435_456)] + pub import_max_pack_bytes: u64, + + #[config(env = "KNOT_XRPC_FORK_FETCH_TIMEOUT_MS", default = 600_000)] + pub fork_fetch_timeout_ms: u64, + + /// When the knot runs behind a trusted reverse proxy that terminates TLS, + /// set this to the header the proxy appends the client address to, + /// for example x-forwarded-for. + /// The knot will read the chain right -> left + /// and take the first entry that `trusted_proxies` doesn't cover. + /// Leave unset when the knot is directly exposed so the socket peer address is used. + /// Only set this when a trusted proxy overwrites or appends the header, + /// since a client can forge it otherwise. + #[config(env = "KNOT_XRPC_TRUSTED_PROXY_HEADER")] + pub trusted_proxy_header: Option, + + /// Addresses whose `trusted_proxy_header` the knot honors, + /// each a bare IP without a port or a CIDR block such as 173.245.48.0/20, + /// for ex the loopback address of a reverse proxy on the same host. + /// The knot will rate-limit a request from any other address + /// by its own socket address and ignore the header. + /// These same addresses are hops the knot will iterate over when it reads + /// the header, so list every proxy you control in the path. + /// A proxy that the knot doesn't know about becomes the entry it keys on, + /// and everyone that proxy serves will then share one rate-limit bucket. + /// The knot will read the last 32 entries of the chain, at most. + /// When the list covers all 32, the knot + /// will rate-limit by the address the request connected from. + /// Leave empty to honor the header from every peer and take its rightmost + /// entry, which is safe *only* while every route to this knot passes + /// through the proxy. + #[config( + env = "KNOT_XRPC_TRUSTED_PROXIES", + parse_env = comma_separated, + default = [] + )] + pub trusted_proxies: Vec, + + #[config(env = "KNOT_XRPC_EVENTS_REPLAY_BUFFER", default = 4096)] + pub events_replay_buffer: u32, + + #[config(env = "KNOT_XRPC_EVENTS_REPLAY_BYTES", default = 67_108_864)] + pub events_replay_bytes: u64, + + #[config(env = "KNOT_XRPC_EVENTS_MAX_SUBSCRIBERS", default = 256)] + pub events_max_subscribers: u32, + + #[config(env = "KNOT_XRPC_EVENTS_MAX_PER_PEER", default = 8)] + pub events_max_per_peer: u32, +} + +#[derive(Debug, Config)] +pub struct MaintenanceConfig { + #[config(env = "KNOT_MAINTENANCE_ENABLED", default = true)] + pub enabled: bool, + + #[config(env = "KNOT_MAINTENANCE_COMMIT_GRAPH", default = true)] + pub commit_graph: bool, + + #[config(env = "KNOT_MAINTENANCE_MULTI_PACK_INDEX", default = true)] + pub multi_pack_index: bool, + + #[config(env = "KNOT_MAINTENANCE_BITMAP", default = true)] + pub bitmap: bool, + + #[config(env = "KNOT_MAINTENANCE_INTERVAL_SECS", default = 21_600)] + pub interval_secs: u64, + + #[config(env = "KNOT_MAINTENANCE_REPACK_MAX_OBJECTS", default = 16_000_000)] + pub repack_max_objects: u64, + + #[config(env = "KNOT_MAINTENANCE_REPACK_GEOMETRIC_FACTOR", default = 2)] + pub repack_geometric_factor: u64, + + #[config(env = "KNOT_MAINTENANCE_REPACK_LOOSE_THRESHOLD", default = 6700)] + pub repack_loose_threshold: u64, + + #[config(env = "KNOT_MAINTENANCE_PRUNE_GRACE_SECS", default = 1_209_600)] + pub prune_grace_secs: u64, + + #[config(env = "KNOT_MAINTENANCE_FULL_REPACK_INTERVAL_SECS", default = 432_000)] + pub full_repack_interval_secs: u64, + + #[config(env = "KNOT_MAINTENANCE_REFLOG_EXPIRE_SECS", default = 7_776_000)] + pub reflog_expire_secs: u64, + + #[config(env = "KNOT_MAINTENANCE_LARGE_PUSH_BYTES", default = 52_428_800)] + pub large_push_bytes: u64, +} + +#[derive(Debug, Config)] +pub struct PackCacheConfig { + #[config(env = "KNOT_PACK_CACHE_ENABLED", default = true)] + pub enabled: bool, + + #[config(env = "KNOT_PACK_CACHE_TTL_SECS", default = 60)] + pub ttl_secs: u64, + + #[config(env = "KNOT_PACK_CACHE_MAX_ENTRY_BYTES", default = 67_108_864)] + pub max_entry_bytes: u64, + + #[config(env = "KNOT_PACK_CACHE_MAX_TOTAL_BYTES", default = 2_147_483_648u64)] + pub max_total_bytes: u64, +} + +#[derive(Debug, Config)] +pub struct PackConfig { + #[config(env = "KNOT_PACK_MAX_OBJECTS", default = 16_000_000)] + pub max_objects: u32, + + #[config(env = "KNOT_PACK_MAX_TOTAL_BYTES", default = 68_719_476_736u64)] + pub max_total_bytes: u64, + + #[config(env = "KNOT_PACK_SELECTION_MAX_OBJECTS", default = 16_000_000)] + pub selection_max_objects: u32, + + #[config(env = "KNOT_PACK_SELECTION_TIME_BUDGET_SECS", default = 600)] + pub selection_time_budget_secs: u64, +} + +#[derive(Debug, Config)] +pub struct LfsConfig { + #[config(env = "KNOT_LFS_STORE_PATH")] + pub store_path: PathBuf, + + #[config(env = "KNOT_LFS_MAX_OBJECT_BYTES", default = 5_368_709_120u64)] + pub max_object_bytes: u64, + + #[config(env = "KNOT_LFS_FREE_SPACE_FLOOR_BYTES", default = 1_073_741_824u64)] + pub free_space_floor_bytes: u64, + + #[config(env = "KNOT_LFS_GC_GRACE_SECS", default = 1_209_600)] + pub gc_grace_secs: u64, + + #[config(env = "KNOT_LFS_GC_INTERVAL_SECS", default = 21_600)] + pub gc_interval_secs: u64, + + #[config(env = "KNOT_LFS_MAX_SSH_TRANSFERS", default = 16)] + pub max_ssh_transfers: u32, + + #[config(env = "KNOT_LFS_MAX_HTTP_DOWNLOADS", default = 64)] + pub max_http_downloads: u32, +} + +#[derive(Debug, Config)] +pub struct SocialConfig { + /// Byte budget over the live record versions of one repository, summed from every atproto collection in the projection. + /// An edit swaps the old version's bytes for the new one's, and a delete frees the record's bytes back out of the budget. + /// A write past the budget 429s. + #[config(env = "KNOT_SOCIAL_REPO_BUDGET_BYTES", default = 1_073_741_824u64)] + pub repo_budget_bytes: u64, + /// The "Foster Knot" from whom this knot will pull "shadowed" records from during a one-time upgrade, at boot time. + /// Defaults to the "Tangled production foster knot", + /// so there's not much use of it outside this one-time upgrade required + /// on 2026-10-28. + /// We are sorry for designing our distributed system + /// in such a way that warrants what Lewis would call + /// "phoning home" but the alternative is that each knot pings + /// the entire atproto dataset, trying to hunt down the social records + /// that it will be taking on as COBs starting 2026-10-28. + /// We have this "Foster Knot" that has not only done this for you + /// already, but also shadow-serves these social records on your knot's + /// behalf until you upgrade, so there's no perceived incompatibility-time + /// for reading on https://tangled.org . + /// If you're uncomfortable with this enough that you want to unset this, + /// please ping all of atproto to hunt down those social records + /// regarding the git repos on this knot, and add them here. + /// If this knot is being spun up for the first time and the current + /// day is *after* 2026-10-28, please disregard. This + /// setting will be gutted once all knots are upgraded. + #[config( + env = "KNOT_SOCIAL_FOSTER_URL", + default = "https://knot2.mock.tangled.org" + )] + pub foster_url: String, + /// How many handovers that the upgrade of a knot2 at boot-time from a foster knot pull will run concurrently. + #[config(env = "KNOT_SOCIAL_FOSTER_BOOT_WIDTH", default = 4)] + pub foster_boot_width: u32, + + /// The following are for the "foster knot", you don't need to touch. + #[config(env = "KNOT_SOCIAL_FOSTER_ACK_SKIP_KNOT_SERVICE", default = false)] + pub foster_ack_skip_knot_service: bool, + #[config(env = "KNOT_SOCIAL_FOSTER_ACK_SKIP_ORIGIN_CHECKS", default = false)] + pub foster_ack_skip_origin_checks: bool, + #[config(env = "KNOT_SOCIAL_FOSTER_LIAR_HYDRANT_URL", default = "")] + pub foster_liar_hydrant_url: String, +} + +#[derive(Debug, Config)] +pub struct KeyfillConfig { + #[config(env = "KNOT_KEYFILL_KEY_BUDGET_MIB", default = 64)] + pub key_budget_mib: u32, + + #[config(env = "KNOT_KEYFILL_TTL_SECS", default = 3_600)] + pub ttl_secs: u64, + + #[config(env = "KNOT_KEYFILL_REPRIEVE_RETRY_SECS", default = 300)] + pub reprieve_retry_secs: u64, + + #[config(env = "KNOT_KEYFILL_REPRIEVE_BUDGET_SECS", default = 21_600)] + pub reprieve_budget_secs: u64, +} + +#[derive(Debug, Config)] +pub struct ResourcesConfig { + #[config(env = "KNOT_MAX_THREADS", default = 0)] + pub max_threads: u32, + + #[config(env = "KNOT_MAX_MEMORY_BYTES", default = 0)] + pub max_memory_bytes: u64, +} + +fn parse_admins(raw: &str) -> Result, knot_types::ParseError> { + raw.split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(AccountDid::new) + .collect() +} + +impl KnotConfig { + pub fn tls_enabled(&self) -> bool { + self.static_cert_enabled() || self.tls.acme_enabled + } + + pub fn static_cert_enabled(&self) -> bool { + self.tls.cert_path.is_some() && self.tls.key_path.is_some() + } + + pub fn http_limits(&self) -> HttpLimits { + HttpLimits { + connect_timeout: Duration::from_millis(self.http.connect_timeout_ms), + read_timeout: Duration::from_millis(self.http.read_timeout_ms), + request_timeout: Duration::from_millis(self.http.request_timeout_ms), + max_response_bytes: self.http.max_response_bytes, + block_private_addresses: self.http.block_private_addresses, + } + } + + pub fn fork_http_limits(&self) -> HttpLimits { + HttpLimits { + connect_timeout: Duration::from_millis(self.http.connect_timeout_ms), + read_timeout: Duration::from_millis(self.http.read_timeout_ms), + request_timeout: Duration::from_millis(self.xrpc.fork_fetch_timeout_ms), + max_response_bytes: self + .xrpc + .fork_max_pack_bytes + .saturating_add(self.xrpc.fork_max_pack_bytes / 64) + .saturating_add(1_048_576), + block_private_addresses: self.http.block_private_addresses, + } + } + + pub fn repo_signing_keys(&self) -> Option { + if let Some(archive) = &self.atproto.repo_signing_keys { + return Some(archive.clone()); + } + let archive = self + .secrets + .sealed_key_file + .parent()? + .join("repo-signing-keys.json"); + archive.exists().then_some(archive) + } + + pub fn validate(&self) -> Result<(), ConfigError> { + let errors: Vec = [ + check( + !self.server.admins.is_empty(), + "server.admins must list at least one DID", + ), + absolute(Field::RepoScanPath, Some(&self.repo.scan_path)), + check( + self.repo.write_burst > 0, + "repo.write_burst must be at least 1", + ), + check( + (1..=MAX_WRITE_REFILL_MS).contains(&self.repo.write_refill_ms), + "repo.write_refill_ms must be between one millisecond and one minute", + ), + absolute(Field::SealedKeyFile, Some(&self.secrets.sealed_key_file)), + absolute(Field::SshHostKeyFile, Some(&self.server.ssh_host_key_file)), + check( + self.tls.cert_path.is_some() == self.tls.key_path.is_some(), + "tls.cert_path and tls.key_path must both be set or both unset", + ), + absolute(Field::TlsCertPath, self.tls.cert_path.as_deref()), + absolute(Field::TlsKeyPath, self.tls.key_path.as_deref()), + check( + !(self.tls.acme_enabled && self.static_cert_enabled()), + "tls.acme_enabled cannot combine with a static tls.cert_path and tls.key_path", + ), + check( + !self.tls.acme_enabled || self.tls.acme_cache_dir.is_some(), + "tls.acme_cache_dir is required when tls.acme_enabled is set", + ), + absolute(Field::TlsAcmeCacheDir, self.tls.acme_cache_dir.as_deref()), + check( + !self.tls.acme_enabled + || self + .tls + .acme_contact + .as_deref() + .is_some_and(is_contact_email), + "tls.acme_contact must be a contact email when tls.acme_enabled is set", + ), + check( + !self.tls.mtls_enabled || self.tls_enabled(), + "tls.mtls_enabled requires a server certificate via static paths or ACME", + ), + check( + !self.tls.mtls_enabled || self.tls.mtls_client_ca_path.is_some(), + "tls.mtls_client_ca_path is required when tls.mtls_enabled is set", + ), + absolute(Field::TlsMtlsClientCaPath, self.tls.mtls_client_ca_path.as_deref()), + check( + !self.tls.mtls_enabled + || self + .tls + .mtls_admin_spki_pin + .as_deref() + .is_some_and(is_spki_pin), + "tls.mtls_admin_spki_pin must be a base64 SHA-256 pin when tls.mtls_enabled is set", + ), + check( + is_env_var_name(&self.secrets.master_key_env), + "secrets.master_key_env must be valid environment variable name", + ), + check( + self.server.ssh_max_pack_bytes > 0, + "server.ssh_max_pack_bytes must be greater than zero", + ), + check( + self.server.ssh_max_inflight_per_peer > 0, + "server.ssh_max_inflight_per_peer must be greater than zero", + ), + check( + self.server.listen_header_timeout_ms > 0, + "server.listen_header_timeout_ms must be greater than zero", + ), + check( + self.server.listen_idle_timeout_ms > 0, + "server.listen_idle_timeout_ms must be greater than zero", + ), + check( + self.server.listen_idle_timeout_ms >= self.server.listen_header_timeout_ms, + "server.listen_idle_timeout_ms must be at least server.listen_header_timeout_ms", + ), + check( + self.server.listen_max_connections > 0, + "server.listen_max_connections must be greater than zero", + ), + check( + self.server.listen_rate_limit_per_second > 0, + "server.listen_rate_limit_per_second must be greater than zero", + ), + check( + self.server.listen_rate_limit_burst > 0, + "server.listen_rate_limit_burst must be greater than zero", + ), + check( + self.server.listen_max_inflight_requests > 0, + "server.listen_max_inflight_requests must be greater than zero", + ), + check( + self.server.listen_request_timeout_ms > 0, + "server.listen_request_timeout_ms must be greater than zero", + ), + check( + self.server.listen_body_timeout_ms > 0, + "server.listen_body_timeout_ms must be greater than zero", + ), + check( + self.server.listen_write_request_timeout_ms > 0, + "server.listen_write_request_timeout_ms must be greater than zero", + ), + check( + self.http.connect_timeout_ms > 0, + "http.connect_timeout_ms must be greater than zero", + ), + check( + self.http.read_timeout_ms > 0, + "http.read_timeout_ms must be greater than zero", + ), + check( + self.http.request_timeout_ms > 0, + "http.request_timeout_ms must be greater than zero", + ), + check( + self.http.max_response_bytes > 0, + "http.max_response_bytes must be greater than zero", + ), + check( + self.atproto.plc_directory.scheme() == "https", + "atproto.plc_directory must be https URL", + ), + check( + self.atproto.plc_directory.host().is_some(), + "atproto.plc_directory must have host", + ), + check( + (1..=MAX_PLC_SWEEP_WIDTH).contains(&self.atproto.plc_sweep_width), + "atproto.plc_sweep_width must be between 1 and 64", + ), + check( + self.atproto + .repo_signing_keys + .as_ref() + .is_none_or(|path| !path.as_os_str().is_empty()), + "atproto.repo_signing_keys must name the migration key archive", + ), + check( + self.xrpc.max_body_bytes > 0, + "xrpc.max_body_bytes must be greater than zero", + ), + check( + self.xrpc.max_response_bytes > 0, + "xrpc.max_response_bytes must be greater than zero", + ), + check( + self.xrpc.max_archive_bytes > 0, + "xrpc.max_archive_bytes must be greater than zero", + ), + check( + self.xrpc.tree_last_commit_budget_ms > 0, + "xrpc.tree_last_commit_budget_ms must be greater than zero", + ), + check( + self.xrpc.blob_last_commit_budget_ms > 0, + "xrpc.blob_last_commit_budget_ms must be greater than zero", + ), + check( + self.xrpc.languages_budget_ms > 0, + "xrpc.languages_budget_ms must be greater than zero", + ), + check( + self.xrpc.max_patch_bytes > 0, + "xrpc.max_patch_bytes must be greater than zero", + ), + check( + self.xrpc.max_patch_decompressed_bytes > 0, + "xrpc.max_patch_decompressed_bytes must be greater than zero", + ), + check( + self.xrpc.fork_max_pack_bytes > 0, + "xrpc.fork_max_pack_bytes must be greater than zero", + ), + check( + self.xrpc.fork_fetch_timeout_ms > 0, + "xrpc.fork_fetch_timeout_ms must be greater than zero", + ), + check( + self.xrpc.preauth_burst > 0, + "xrpc.preauth_burst must be greater than zero", + ), + check( + self.xrpc.preauth_refill_ms > 0, + "xrpc.preauth_refill_ms must be greater than zero", + ), + check( + self.xrpc.per_peer_inflight > 0, + "xrpc.per_peer_inflight must be greater than zero", + ), + check( + self.xrpc.global_inflight >= self.xrpc.per_peer_inflight, + "xrpc.global_inflight must be at least xrpc.per_peer_inflight", + ), + check( + self.xrpc.per_actor_reservations > 0, + "xrpc.per_actor_reservations must be greater than zero", + ), + check( + self.xrpc.max_pending_reservations >= self.xrpc.per_actor_reservations, + "xrpc.max_pending_reservations must be at least xrpc.per_actor_reservations", + ), + check( + self.xrpc.reservation_ttl_secs > 0, + "xrpc.reservation_ttl_secs must be greater than zero", + ), + check( + self.xrpc.events_replay_buffer > 0, + "xrpc.events_replay_buffer must be greater than zero", + ), + check( + self.xrpc.events_replay_bytes > 0, + "xrpc.events_replay_bytes must be greater than zero", + ), + check( + self.xrpc.events_max_subscribers > 0, + "xrpc.events_max_subscribers must be greater than zero", + ), + check( + self.xrpc.events_max_per_peer > 0, + "xrpc.events_max_per_peer must be greater than zero", + ), + check( + self.xrpc.events_max_subscribers >= self.xrpc.events_max_per_peer, + "xrpc.events_max_subscribers must be at least xrpc.events_max_per_peer", + ), + check( + (1..=MAX_KEYFILL_BUDGET_MIB).contains(&self.keyfill.key_budget_mib), + "keyfill.key_budget_mib must be between one mebibyte and one tebibyte", + ), + check( + (1..=MAX_SPAN_SECS).contains(&self.keyfill.ttl_secs), + "keyfill.ttl_secs must be between one second and one year", + ), + check( + (1..=MAX_SPAN_SECS).contains(&self.keyfill.reprieve_retry_secs), + "keyfill.reprieve_retry_secs must be between one second and one year", + ), + check( + (1..=MAX_SPAN_SECS).contains(&self.keyfill.reprieve_budget_secs), + "keyfill.reprieve_budget_secs must be between one second and one year", + ), + check( + self.keyfill.reprieve_budget_secs >= self.keyfill.reprieve_retry_secs, + "keyfill.reprieve_budget_secs must be at least keyfill.reprieve_retry_secs", + ), + check( + self.maintenance.interval_secs > 0, + "maintenance.interval_secs must be greater than zero", + ), + check( + self.maintenance.repack_max_objects > 0, + "maintenance.repack_max_objects must be greater than zero", + ), + check( + self.maintenance.repack_geometric_factor >= 2, + "maintenance.repack_geometric_factor must be at least 2", + ), + check( + self.maintenance.large_push_bytes > 0, + "maintenance.large_push_bytes must be greater than zero", + ), + check( + self.pack_cache.ttl_secs > 0, + "pack_cache.ttl_secs must be greater than zero", + ), + check( + self.pack_cache.max_entry_bytes > 0, + "pack_cache.max_entry_bytes must be greater than zero", + ), + check( + self.pack_cache.max_total_bytes > 0, + "pack_cache.max_total_bytes must be greater than zero", + ), + check( + self.pack_cache.max_total_bytes >= self.pack_cache.max_entry_bytes, + "pack_cache.max_total_bytes must be at least pack_cache.max_entry_bytes", + ), + check( + self.pack.max_objects > 0, + "pack.max_objects must be greater than zero", + ), + check( + self.pack.max_total_bytes > 0, + "pack.max_total_bytes must be greater than zero", + ), + check( + self.pack.selection_max_objects > 0, + "pack.selection_max_objects must be greater than zero", + ), + check( + self.pack.selection_time_budget_secs > 0, + "pack.selection_time_budget_secs must be greater than zero", + ), + absolute(Field::LfsStorePath, Some(&self.lfs.store_path)), + disjoint( + Field::LfsStorePath, + &self.lfs.store_path, + Field::RepoScanPath, + &self.repo.scan_path, + ), + check( + self.social.repo_budget_bytes > 0, + "social.repo_budget_bytes must be at least 1", + ), + check( + (1..=MAX_FOSTER_BOOT_WIDTH).contains(&self.social.foster_boot_width), + "social.foster_boot_width must be between 1 and 256", + ), + check( + self.lfs.max_object_bytes > 0, + "lfs.max_object_bytes must be greater than zero", + ), + check( + self.lfs.gc_interval_secs > 0, + "lfs.gc_interval_secs must be greater than zero", + ), + check( + self.lfs.max_ssh_transfers > 0, + "lfs.max_ssh_transfers must be greater than zero", + ), + check( + self.lfs.max_http_downloads > 0, + "lfs.max_http_downloads must be greater than zero", + ), + self.xrpc + .trusted_proxy_header + .as_ref() + .filter(|header| !is_http_token(header)) + .map(|_| "xrpc.trusted_proxy_header isn't valid HTTP header name".to_string()), + check( + self.xrpc.trusted_proxy_header.is_some() || self.xrpc.trusted_proxies.is_empty(), + "xrpc.trusted_proxies needs xrpc.trusted_proxy_header, the header the knot honors from those addresses", + ), + self.acl + .legacy_admin_secret_env + .as_deref() + .filter(|name| !is_env_var_name(name)) + .map(|_| { + "acl.legacy_admin_secret_env must be valid environment variable name" + .to_string() + }), + check( + is_contact_email(&self.contact.email), + "contact.email must be a contact email", + ), + match self.homepage.source() { + HomepageSource::File(path) => absolute(Field::HomepagePath, Some(&path)), + _ => None, + }, + ] + .into_iter() + .flatten() + .chain(self.port_collisions()) + .collect(); + + if errors.is_empty() { + Ok(()) + } else { + Err(ConfigError { errors }) + } + } + + fn port_collisions(&self) -> Vec { + let binds = [ + ("server.listen_addr", self.server.listen_addr), + ( + "server.internal_listen_addr", + self.server.internal_listen_addr, + ), + ("server.ssh_listen_addr", self.server.ssh_listen_addr), + ]; + [(0, 1), (0, 2), (1, 2)] + .into_iter() + .filter(|&(a, b)| binds[a].1.port() == binds[b].1.port()) + .map(|(a, b)| { + format!( + "{} and {} cannot bind same port {}", + binds[a].0, + binds[b].0, + binds[a].1.port() + ) + }) + .collect() + } +} + +fn check(ok: bool, message: &str) -> Option { + (!ok).then(|| message.to_string()) +} + +#[derive(Clone, Copy)] +enum Field { + RepoScanPath, + SealedKeyFile, + SshHostKeyFile, + TlsCertPath, + TlsKeyPath, + TlsAcmeCacheDir, + TlsMtlsClientCaPath, + LfsStorePath, + HomepagePath, +} + +impl Field { + const fn as_str(self) -> &'static str { + match self { + Self::RepoScanPath => "repo.scan_path", + Self::SealedKeyFile => "secrets.sealed_key_file", + Self::SshHostKeyFile => "server.ssh_host_key_file", + Self::TlsCertPath => "tls.cert_path", + Self::TlsKeyPath => "tls.key_path", + Self::TlsAcmeCacheDir => "tls.acme_cache_dir", + Self::TlsMtlsClientCaPath => "tls.mtls_client_ca_path", + Self::LfsStorePath => "lfs.store_path", + Self::HomepagePath => "homepage.path", + } + } +} + +fn absolute(field: Field, path: Option<&Path>) -> Option { + path.filter(|path| !path.is_absolute()) + .map(|_| format!("{} must be absolute path", field.as_str())) +} + +fn disjoint(field: Field, a: &Path, other: Field, b: &Path) -> Option { + (a.starts_with(b) || b.starts_with(a)) + .then(|| format!("{} mustn't overlap {}", field.as_str(), other.as_str())) +} + +fn is_http_token(value: &str) -> bool { + !value.is_empty() + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&byte)) +} + +pub fn is_contact_email(value: &str) -> bool { + let mut parts = value.splitn(2, '@'); + matches!( + (parts.next(), parts.next()), + (Some(local), Some(domain)) + if !local.is_empty() + && domain.contains('.') + && !domain.starts_with('.') + && !domain.ends_with('.') + && !value.chars().any(char::is_whitespace) + ) +} + +fn is_spki_pin(value: &str) -> bool { + base64::engine::general_purpose::STANDARD + .decode(value) + .is_ok_and(|bytes| bytes.len() == 32) +} + +fn is_env_var_name(value: &str) -> bool { + !value.is_empty() + && !value.starts_with(|c: char| c.is_ascii_digit()) + && value + .chars() + .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_') +} + +#[derive(Debug, thiserror::Error)] +pub struct ConfigError { + pub errors: Vec, +} + +impl fmt::Display for ConfigError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{} configuration problem(s):", self.errors.len())?; + self.errors + .iter() + .try_for_each(|error| writeln!(f, " - {error}")) + } +} + +pub struct Validated { + config: KnotConfig, + crawlers: Vec, + catalog: Arc, + trusted_proxies: TrustedProxies, + ssh_proxy_protocol_from: TrustedProxies, +} + +impl Validated { + fn new(config: KnotConfig) -> Result { + config.validate()?; + let catalog = + knot_messages::Catalog::parse(&config.messages).map_err(|error| ConfigError { + errors: vec![error.to_string()], + })?; + let trusted_proxies = TrustedProxies::parse( + config.xrpc.trusted_proxies.iter().map(String::as_str), + ) + .map_err(|error| ConfigError { + errors: vec![format!("xrpc.trusted_proxies: {error}")], + })?; + let ssh_proxy_protocol_from = TrustedProxies::parse( + config + .server + .ssh_proxy_protocol_from + .iter() + .map(String::as_str), + ) + .map_err(|error| ConfigError { + errors: vec![format!("server.ssh_proxy_protocol_from: {error}")], + })?; + let crawlers = config + .atproto + .crawlers + .iter() + .map(|raw| KnotServiceUrl::new(raw.as_str())) + .collect::, _>>() + .map_err(|error| ConfigError { + errors: vec![format!("atproto.crawlers: {error}")], + })?; + Ok(Self { + config, + crawlers, + catalog: Arc::new(catalog), + trusted_proxies, + ssh_proxy_protocol_from, + }) + } + + pub fn knot_hostname(&self) -> &KnotHostname { + &self.config.server.hostname + } + + pub fn crawlers(&self) -> &[KnotServiceUrl] { + &self.crawlers + } + + pub fn catalog(&self) -> Arc { + Arc::clone(&self.catalog) + } + + pub fn trusted_proxies(&self) -> TrustedProxies { + self.trusted_proxies.clone() + } + + pub fn ssh_proxy_protocol_from(&self) -> TrustedProxies { + self.ssh_proxy_protocol_from.clone() + } + + pub fn verify_environment(&self) -> Result<(), EnvError> { + verify_writable_dir("repo.scan_path", &self.config.repo.scan_path)?; + verify_writable_dir("lfs.store_path", &self.config.lfs.store_path)?; + verify_homepage(self.config.homepage.source())?; + verify_master_key(&self.config.secrets.master_key_env) + } +} + +impl std::ops::Deref for Validated { + type Target = KnotConfig; + + fn deref(&self) -> &Self::Target { + &self.config + } +} + +#[derive(Debug, thiserror::Error)] +pub enum EnvError { + #[error("{field} {path} isn't accessible")] + DirInaccessible { + field: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("{field} {path} isn't directory")] + DirNotDir { field: &'static str, path: PathBuf }, + #[error("{field} {path} isn't writable")] + DirNotWritable { + field: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("homepage.path {path} isn't accessible")] + HomepageInaccessible { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("homepage.path {path} isn't a regular file")] + HomepageNotFile { path: PathBuf }, + #[error("master key env var {name} isn't set")] + MasterKeyUnset { name: String }, + #[error("master key env var {name} is empty")] + MasterKeyEmpty { name: String }, + #[error("master key env var {name} isn't valid base64")] + MasterKeyNotBase64 { + name: String, + #[source] + source: base64::DecodeError, + }, + #[error( + "master key env var {name} decodes to only {len} of minimum {MASTER_KEY_MIN_BYTES} bytes" + )] + MasterKeyTooShort { name: String, len: usize }, +} + +const MASTER_KEY_MIN_BYTES: usize = 32; + +const MAX_SPAN_SECS: u64 = 365 * 24 * 60 * 60; + +const MAX_KEYFILL_BUDGET_MIB: u32 = 1024 * 1024; + +const MAX_WRITE_REFILL_MS: u64 = 60_000; + +const MAX_FOSTER_BOOT_WIDTH: u32 = 256; + +const MAX_PLC_SWEEP_WIDTH: u32 = 64; + +fn verify_writable_dir(field: &'static str, path: &Path) -> Result<(), EnvError> { + let metadata = std::fs::metadata(path).map_err(|source| EnvError::DirInaccessible { + field, + path: path.to_path_buf(), + source, + })?; + if !metadata.is_dir() { + return Err(EnvError::DirNotDir { + field, + path: path.to_path_buf(), + }); + } + tempfile::Builder::new() + .prefix(".knot-write-probe") + .tempfile_in(path) + .map(drop) + .map_err(|source| EnvError::DirNotWritable { + field, + path: path.to_path_buf(), + source, + }) +} + +fn verify_homepage(source: HomepageSource) -> Result<(), EnvError> { + let HomepageSource::File(path) = source else { + return Ok(()); + }; + let file = std::fs::File::open(&path).map_err(|source| EnvError::HomepageInaccessible { + path: path.clone(), + source, + })?; + let is_file = file + .metadata() + .map_err(|source| EnvError::HomepageInaccessible { + path: path.clone(), + source, + })? + .is_file(); + is_file + .then_some(()) + .ok_or(EnvError::HomepageNotFile { path }) +} + +fn verify_master_key(name: &str) -> Result<(), EnvError> { + let value = std::env::var(name).ok(); + validate_master_key(name, value.as_deref()) +} + +fn validate_master_key(name: &str, value: Option<&str>) -> Result<(), EnvError> { + let raw = value.ok_or_else(|| EnvError::MasterKeyUnset { + name: name.to_string(), + })?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Err(EnvError::MasterKeyEmpty { + name: name.to_string(), + }); + } + let decoded = base64::engine::general_purpose::STANDARD + .decode(trimmed) + .map_err(|source| EnvError::MasterKeyNotBase64 { + name: name.to_string(), + source, + })?; + (decoded.len() >= MASTER_KEY_MIN_BYTES) + .then_some(()) + .ok_or(EnvError::MasterKeyTooShort { + name: name.to_string(), + len: decoded.len(), + }) +} + +#[derive(Debug, thiserror::Error)] +pub enum LoadError { + #[error("config file not found: {0}")] + Missing(PathBuf), + #[error(transparent)] + Confique(#[from] confique::Error), + #[error(transparent)] + Invalid(#[from] ConfigError), +} + +pub fn load(path: Option<&Path>) -> Result { + if let Some(path) = path + && !path.exists() + { + return Err(LoadError::Missing(path.to_path_buf())); + } + let mut builder = KnotConfig::builder().env(); + if let Some(path) = path { + builder = builder.file(path); + } + let config = builder.file("/etc/knot/config.toml").load()?; + Ok(Validated::new(config)?) +} + +pub fn template() -> String { + confique::toml::template::(confique::toml::FormatOptions::default()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn example_toml_is_the_generated_template() { + assert_eq!( + template(), + include_str!("../../../example.toml"), + "regenerate example.toml with `just gen-config` after changing config" + ); + } + + fn sample() -> KnotConfig { + KnotConfig { + server: ServerConfig { + hostname: KnotHostname::new("oyster.cafe").unwrap(), + admins: vec![AccountDid::new("did:plc:nel").unwrap()], + listen_addr: "[::]:5555".parse().unwrap(), + listen_header_timeout_ms: 10_000, + listen_idle_timeout_ms: 60_000, + listen_max_connections: 1_024, + listen_rate_limit_per_second: 50, + listen_rate_limit_burst: 200, + listen_max_inflight_requests: 1_024, + listen_request_timeout_ms: 60_000, + listen_body_timeout_ms: 30_000, + listen_write_request_timeout_ms: 1_800_000, + internal_listen_addr: "[::1]:5444".parse().unwrap(), + ssh_listen_addr: "[::]:2222".parse().unwrap(), + ssh_host_key_file: PathBuf::from("/var/lib/knot/ssh_host_key"), + ssh_max_pack_bytes: 8_589_934_592, + ssh_max_inflight_per_peer: 4, + ssh_proxy_protocol_from: vec![], + appview_endpoint: AppviewEndpoint::new("https://tangled.org").unwrap(), + }, + tls: TlsConfig { + cert_path: None, + key_path: None, + http3: true, + acme_enabled: false, + acme_cache_dir: None, + acme_contact: None, + acme_staging: false, + mtls_enabled: false, + mtls_client_ca_path: None, + mtls_admin_spki_pin: None, + }, + acl: AclConfig { + admission: AdmissionPolicy::Closed, + legacy_admin_secret_env: None, + }, + repo: RepoConfig { + scan_path: PathBuf::from("/srv/git"), + default_branch: BranchName::new("main").unwrap(), + contribution_policy: ContributionPolicy::Collaborators, + write_burst: 30, + write_refill_ms: 500, + }, + git: GitConfig { + object_format: ObjectFormat::SHA1, + }, + secrets: SecretsConfig { + sealed_key_file: PathBuf::from("/var/lib/knot/keys.sealed"), + master_key_env: "KNOT_MASTER_KEY".to_string(), + }, + http: HttpConfig { + connect_timeout_ms: 5_000, + read_timeout_ms: 30_000, + request_timeout_ms: 60_000, + max_response_bytes: 16_777_216, + block_private_addresses: true, + }, + atproto: AtprotoConfig { + plc_directory: Url::parse("https://plc.nel.pet/").unwrap(), + plc_sweep_width: 1, + repo_signing_keys: None, + crawlers: Vec::new(), + }, + xrpc: XrpcConfig { + max_body_bytes: 65_536, + max_response_bytes: 5_242_880, + max_archive_bytes: 1_073_741_824, + tree_last_commit_budget_ms: 300, + blob_last_commit_budget_ms: 2_000, + languages_budget_ms: 1_000, + max_patch_bytes: 16_777_216, + max_patch_decompressed_bytes: 134_217_728, + preauth_burst: 20, + preauth_refill_ms: 100, + per_peer_inflight: 8, + global_inflight: 64, + max_pending_reservations: 256, + per_actor_reservations: 32, + reservation_ttl_secs: 3_600, + fork_max_pack_bytes: 1_073_741_824, + import_max_pack_bytes: 268_435_456, + fork_fetch_timeout_ms: 600_000, + trusted_proxy_header: None, + trusted_proxies: Vec::new(), + events_replay_buffer: 4_096, + events_replay_bytes: 67_108_864, + events_max_subscribers: 256, + events_max_per_peer: 8, + }, + maintenance: MaintenanceConfig { + enabled: true, + commit_graph: true, + multi_pack_index: true, + bitmap: true, + interval_secs: 21_600, + repack_max_objects: 16_000_000, + repack_geometric_factor: 2, + repack_loose_threshold: 6700, + prune_grace_secs: 1_209_600, + full_repack_interval_secs: 432_000, + reflog_expire_secs: 7_776_000, + large_push_bytes: 52_428_800, + }, + pack_cache: PackCacheConfig { + enabled: true, + ttl_secs: 60, + max_entry_bytes: 67_108_864, + max_total_bytes: 536_870_912, + }, + pack: PackConfig { + max_objects: 16_000_000, + max_total_bytes: 68_719_476_736, + selection_max_objects: 16_000_000, + selection_time_budget_secs: 600, + }, + lfs: LfsConfig { + store_path: PathBuf::from("/srv/lfs"), + max_object_bytes: 5_368_709_120, + free_space_floor_bytes: 1_073_741_824, + gc_grace_secs: 1_209_600, + gc_interval_secs: 21_600, + max_ssh_transfers: 16, + max_http_downloads: 64, + }, + social: SocialConfig { + repo_budget_bytes: 1_073_741_824, + foster_url: String::new(), + foster_boot_width: 4, + foster_ack_skip_knot_service: false, + foster_ack_skip_origin_checks: false, + foster_liar_hydrant_url: String::new(), + }, + keyfill: KeyfillConfig { + key_budget_mib: 64, + ttl_secs: 3_600, + reprieve_retry_secs: 300, + reprieve_budget_secs: 21_600, + }, + resources: ResourcesConfig { + max_threads: 0, + max_memory_bytes: 0, + }, + messages: knot_messages::MessagesConfig::defaults(), + homepage: HomepageConfig { + enabled: true, + path: None, + }, + links: LinksConfig { + privacy_policy: None, + terms_of_service: None, + }, + contact: ContactConfig { + email: "knot@nel.pet".to_string(), + }, + } + } + + fn apply_acme(config: &mut KnotConfig) { + config.tls.acme_enabled = true; + config.tls.acme_cache_dir = Some(PathBuf::from("/var/lib/knot/acme")); + config.tls.acme_contact = Some("nel@oyster.cafe".to_string()); + } + + fn apply_mtls(config: &mut KnotConfig) { + config.tls.cert_path = Some(PathBuf::from("/etc/knot/tls/cert.pem")); + config.tls.key_path = Some(PathBuf::from("/etc/knot/tls/key.pem")); + config.tls.mtls_enabled = true; + config.tls.mtls_client_ca_path = Some(PathBuf::from("/etc/knot/tls/admin-ca.pem")); + config.tls.mtls_admin_spki_pin = + Some(base64::engine::general_purpose::STANDARD.encode([7u8; 32])); + } + + #[test] + fn accepts_valid_config() { + type Case = (&'static str, fn(&mut KnotConfig), bool, bool); + let cases: &[Case] = &[ + ("valid_config_passes", |_| {}, false, false), + ( + "matched_absolute_tls_paths", + |config| { + config.tls.cert_path = Some(PathBuf::from("/etc/knot/tls/cert.pem")); + config.tls.key_path = Some(PathBuf::from("/etc/knot/tls/key.pem")); + }, + true, + true, + ), + ("acme_enables_tls", apply_acme, true, false), + ("mtls_with_server_cert_and_pin", apply_mtls, true, true), + ( + "an_immediate_prune_grace", + |config| config.maintenance.prune_grace_secs = 0, + false, + false, + ), + ( + "one_byte_social_repo_budget", + |config| config.social.repo_budget_bytes = 1, + false, + false, + ), + ( + "the_widest_foster_boot_width", + |config| config.social.foster_boot_width = MAX_FOSTER_BOOT_WIDTH, + false, + false, + ), + ( + "the_widest_plc_sweep_width", + |config| config.atproto.plc_sweep_width = MAX_PLC_SWEEP_WIDTH, + false, + false, + ), + ( + "a_legacy_admin_secret_env_var", + |config| { + config.acl.legacy_admin_secret_env = + Some("KNOT_LEGACY_ADMIN_SECRET".to_string()) + }, + false, + false, + ), + ]; + cases + .iter() + .for_each(|(label, mutate, tls_enabled, static_cert)| { + let mut config = sample(); + mutate(&mut config); + assert!(config.validate().is_ok(), "{label}"); + assert_eq!(config.tls_enabled(), *tls_enabled, "{label} tls_enabled"); + assert_eq!( + config.static_cert_enabled(), + *static_cert, + "{label} static_cert_enabled" + ); + }); + } + + #[test] + fn rejects_invalid_config() { + type Case = (&'static str, fn(&mut KnotConfig), &'static str); + let cases: &[Case] = &[ + ( + "a_cert_without_a_key", + |config| config.tls.cert_path = Some(PathBuf::from("/etc/knot/tls/cert.pem")), + "both be set or both unset", + ), + ( + "a_relative_cert_path", + |config| { + config.tls.cert_path = Some(PathBuf::from("tls/cert.pem")); + config.tls.key_path = Some(PathBuf::from("tls/key.pem")); + }, + "tls.cert_path", + ), + ( + "acme_cannot_combine_with_static", + |config| { + apply_acme(config); + config.tls.cert_path = Some(PathBuf::from("/etc/knot/tls/cert.pem")); + config.tls.key_path = Some(PathBuf::from("/etc/knot/tls/key.pem")); + }, + "cannot combine", + ), + ( + "acme_without_a_cache_dir", + |config| { + apply_acme(config); + config.tls.acme_cache_dir = None; + }, + "tls.acme_cache_dir is required", + ), + ( + "acme_without_a_valid_contact", + |config| { + apply_acme(config); + config.tls.acme_contact = Some("not-an-email".to_string()); + }, + "tls.acme_contact", + ), + ( + "mtls_without_a_server_certificate", + |config| { + apply_mtls(config); + config.tls.cert_path = None; + config.tls.key_path = None; + }, + "tls.mtls_enabled requires a server certificate", + ), + ( + "mtls_with_a_malformed_pin", + |config| { + apply_mtls(config); + config.tls.mtls_admin_spki_pin = + Some(base64::engine::general_purpose::STANDARD.encode([0u8; 16])); + }, + "tls.mtls_admin_spki_pin", + ), + ( + "empty_admin_list", + |config| config.server.admins = Vec::new(), + "admins", + ), + ( + "a_malformed_legacy_admin_secret_env_var_name", + |config| config.acl.legacy_admin_secret_env = Some("9_NOT_A_VAR".to_string()), + "acl.legacy_admin_secret_env", + ), + ( + "a_zero_maintenance_interval", + |config| config.maintenance.interval_secs = 0, + "maintenance.interval_secs", + ), + ( + "a_zero_repack_object_limit", + |config| config.maintenance.repack_max_objects = 0, + "maintenance.repack_max_objects", + ), + ( + "a_geometric_factor_below_two", + |config| config.maintenance.repack_geometric_factor = 1, + "maintenance.repack_geometric_factor", + ), + ( + "a_zero_large_push_threshold", + |config| config.maintenance.large_push_bytes = 0, + "maintenance.large_push_bytes", + ), + ( + "a_zero_pack_cache_ttl", + |config| config.pack_cache.ttl_secs = 0, + "pack_cache.ttl_secs", + ), + ( + "a_zero_write_burst", + |config| config.repo.write_burst = 0, + "repo.write_burst", + ), + ( + "a_zero_write_refill", + |config| config.repo.write_refill_ms = 0, + "repo.write_refill_ms", + ), + ( + "a_write_refill_past_a_minute", + |config| config.repo.write_refill_ms = 60_001, + "repo.write_refill_ms", + ), + ( + "zero_social_repo_budget", + |config| config.social.repo_budget_bytes = 0, + "social.repo_budget_bytes", + ), + ( + "a_zero_pack_cache_entry_limit", + |config| config.pack_cache.max_entry_bytes = 0, + "pack_cache.max_entry_bytes", + ), + ( + "a_zero_pack_cache_total_limit", + |config| config.pack_cache.max_total_bytes = 0, + "pack_cache.max_total_bytes", + ), + ( + "a_pack_cache_total_below_one_entry", + |config| { + config.pack_cache.max_entry_bytes = 1_000; + config.pack_cache.max_total_bytes = 500; + }, + "at least pack_cache.max_entry_bytes", + ), + ( + "relative_scan_path", + |config| config.repo.scan_path = PathBuf::from("relative/git"), + "scan_path", + ), + ( + "a_relative_lfs_store_path", + |config| config.lfs.store_path = PathBuf::from("relative/lfs"), + "lfs.store_path must be absolute path", + ), + ( + "an_lfs_store_inside_the_scan_path", + |config| config.lfs.store_path = config.repo.scan_path.join("lfs"), + "lfs.store_path mustn't overlap repo.scan_path", + ), + ( + "a_scan_path_inside_the_lfs_store", + |config| { + config.lfs.store_path = PathBuf::from("/srv/media"); + config.repo.scan_path = PathBuf::from("/srv/media/git"); + }, + "lfs.store_path mustn't overlap repo.scan_path", + ), + ( + "a_zero_ssh_inflight_per_peer_limit", + |config| config.server.ssh_max_inflight_per_peer = 0, + "server.ssh_max_inflight_per_peer", + ), + ( + "a_zero_lfs_object_limit", + |config| config.lfs.max_object_bytes = 0, + "lfs.max_object_bytes", + ), + ( + "a_zero_lfs_gc_interval", + |config| config.lfs.gc_interval_secs = 0, + "lfs.gc_interval_secs", + ), + ( + "a_zero_lfs_ssh_transfer_limit", + |config| config.lfs.max_ssh_transfers = 0, + "lfs.max_ssh_transfers", + ), + ( + "a_zero_lfs_http_download_limit", + |config| config.lfs.max_http_downloads = 0, + "lfs.max_http_downloads", + ), + ( + "bad_master_key_env_name", + |config| config.secrets.master_key_env = "9 bad name".to_string(), + "master_key_env", + ), + ( + "zero_http_timeout", + |config| config.http.request_timeout_ms = 0, + "request_timeout_ms", + ), + ( + "zero_tree_last_commit_budget", + |config| config.xrpc.tree_last_commit_budget_ms = 0, + "tree_last_commit_budget_ms", + ), + ( + "zero_blob_last_commit_budget", + |config| config.xrpc.blob_last_commit_budget_ms = 0, + "blob_last_commit_budget_ms", + ), + ( + "zero_languages_budget", + |config| config.xrpc.languages_budget_ms = 0, + "languages_budget_ms", + ), + ( + "zero_events_replay_buffer", + |config| config.xrpc.events_replay_buffer = 0, + "events_replay_buffer", + ), + ( + "zero_events_replay_bytes", + |config| config.xrpc.events_replay_bytes = 0, + "events_replay_bytes", + ), + ( + "zero_events_max_subscribers", + |config| config.xrpc.events_max_subscribers = 0, + "events_max_subscribers", + ), + ( + "zero_events_max_per_peer", + |config| config.xrpc.events_max_per_peer = 0, + "events_max_per_peer", + ), + ( + "a_per_peer_limit_above_the_global_limit", + |config| { + config.xrpc.events_max_subscribers = 4; + config.xrpc.events_max_per_peer = 8; + }, + "events_max_subscribers must be at least", + ), + ( + "a_key_budget_too_small_for_any_key", + |config| config.keyfill.key_budget_mib = 0, + "keyfill.key_budget_mib", + ), + ( + "a_key_budget_past_what_any_machine_has", + |config| config.keyfill.key_budget_mib = u32::MAX, + "keyfill.key_budget_mib", + ), + ( + "a_key_ttl_that_expires_on_the_read", + |config| config.keyfill.ttl_secs = 0, + "keyfill.ttl_secs", + ), + ( + "a_key_ttl_past_what_a_unix_timestamp_can_represent", + |config| config.keyfill.ttl_secs = u64::MAX, + "keyfill.ttl_secs", + ), + ( + "a_zero_second_reprieve_retry", + |config| config.keyfill.reprieve_retry_secs = 0, + "keyfill.reprieve_retry_secs", + ), + ( + "a_reprieve_budget_under_one_retry", + |config| { + config.keyfill.reprieve_retry_secs = 600; + config.keyfill.reprieve_budget_secs = 300; + }, + "keyfill.reprieve_budget_secs must be at least", + ), + ( + "a_non_https_plc_directory", + |config| { + config.atproto.plc_directory = Url::parse("http://plc.nel.pet/").unwrap(); + }, + "plc_directory", + ), + ( + "a_zero_plc_sweep_width", + |config| config.atproto.plc_sweep_width = 0, + "atproto.plc_sweep_width must be between 1 and", + ), + ( + "an_overly_wide_plc_sweep_width", + |config| config.atproto.plc_sweep_width = MAX_PLC_SWEEP_WIDTH + 1, + "atproto.plc_sweep_width must be between 1 and", + ), + ( + "an_idle_timeout_below_the_header_timeout", + |config| { + config.server.listen_header_timeout_ms = 10_000; + config.server.listen_idle_timeout_ms = 5_000; + }, + "listen_idle_timeout_ms must be at least", + ), + ( + "colliding_bind_ports", + |config| config.server.internal_listen_addr = config.server.listen_addr, + "same port", + ), + ( + "a_relative_homepage_path", + |config| config.homepage.path = Some(PathBuf::from("homepage.html")), + "homepage.path must be absolute path", + ), + ( + "a_blank_contact_email", + |config| config.contact.email = " ".to_string(), + "contact.email must be a contact email", + ), + ( + "a_zero_foster_boot_width", + |config| config.social.foster_boot_width = 0, + "social.foster_boot_width must be between 1 and", + ), + ( + "an_overly_wide_foster_boot_width", + |config| config.social.foster_boot_width = MAX_FOSTER_BOOT_WIDTH + 1, + "social.foster_boot_width must be between 1 and", + ), + ( + "trusted_proxies_without_the_header_the_knot_honors", + |config| config.xrpc.trusted_proxies = vec!["127.0.0.1".to_owned()], + "needs xrpc.trusted_proxy_header", + ), + ]; + cases.iter().for_each(|(label, mutate, expected)| { + let mut config = sample(); + mutate(&mut config); + let errors = config.validate().unwrap_err().errors; + assert!( + errors.iter().any(|error| error.contains(expected)), + "{label}: expected an error containing {expected}, got {errors:?}" + ); + }); + } + + #[test] + fn homepage_source_resolves_states() { + let disabled = HomepageConfig { + enabled: false, + path: Some(PathBuf::from("/etc/knot/home.html")), + }; + assert!(matches!(disabled.source(), HomepageSource::Disabled)); + + let default = HomepageConfig { + enabled: true, + path: None, + }; + assert!(matches!(default.source(), HomepageSource::Default)); + + let file = HomepageConfig { + enabled: true, + path: Some(PathBuf::from("/etc/knot/home.html")), + }; + match file.source() { + HomepageSource::File(path) => assert_eq!(path, PathBuf::from("/etc/knot/home.html")), + other => panic!("expected File, got {other:?}"), + } + } + + #[test] + fn validates_master_key() { + let short = base64::engine::general_purpose::STANDARD.encode([0u8; 16]); + let key = base64::engine::general_purpose::STANDARD.encode([7u8; 32]); + type Case<'a> = (Option<&'a str>, fn(&Result<(), EnvError>) -> bool); + let cases: Vec> = vec![ + (None, |result| { + matches!(result, Err(EnvError::MasterKeyUnset { .. })) + }), + (Some(" "), |result| { + matches!(result, Err(EnvError::MasterKeyEmpty { .. })) + }), + (Some("not base64 *** value"), |result| { + matches!(result, Err(EnvError::MasterKeyNotBase64 { .. })) + }), + (Some(short.as_str()), |result| { + matches!(result, Err(EnvError::MasterKeyTooShort { .. })) + }), + (Some(key.as_str()), |result| result.is_ok()), + ]; + cases.iter().for_each(|(input, expect)| { + assert!(expect(&validate_master_key("KNOT_MASTER_KEY", *input))); + }); + } + + #[test] + fn admins_parse_from_comma_separated_env() { + let parsed = parse_admins("did:plc:nel, did:plc:olaren").unwrap(); + assert_eq!(parsed.len(), 2); + assert!(parse_admins("not-a-did").is_err()); + } + + #[test] + fn a_trusted_proxy_entry_takes_an_address_or_a_cidr_block() { + let listing = |entries: &[&str]| { + let mut config = sample(); + config.xrpc.trusted_proxy_header = Some("x-forwarded-for".to_owned()); + config.xrpc.trusted_proxies = entries.iter().map(|&e| e.to_owned()).collect(); + config + }; + let proxies = Validated::new(listing(&["127.0.0.1", "173.245.48.0/20", "2400:cb00::/32"])) + .unwrap() + .trusted_proxies(); + assert!(proxies.contains("173.245.48.7".parse().unwrap())); + assert!(proxies.contains("2400:cb00::1".parse().unwrap())); + + [ + ( + "127.0.0.1:5555", + "127.0.0.1:5555", + "xrpc.trusted_proxies takes a bare address or a CIDR block, so the failure must quote the rejected entry", + ), + ( + " ", + "blank entry", + "parse refuses a blank in the file instead of reading a list the operator filled in as empty, because the knot honors the header from every peer while the list is empty. `comma_separated` discards the same blank from the env var, since it can't tell that blank from the gap a trailing separator leaves", + ), + ] + .iter() + .for_each(|&(entry, quoted, why)| { + let report = Validated::new(listing(&[entry])).err().unwrap().to_string(); + assert!(report.contains(quoted), "{why}: {report}"); + }); + } + + #[test] + fn http_limits_map_from_config() { + let limits = sample().http_limits(); + assert_eq!(limits.connect_timeout, Duration::from_millis(5_000)); + assert_eq!(limits.request_timeout, Duration::from_millis(60_000)); + assert_eq!(limits.max_response_bytes, 16_777_216); + assert!(limits.block_private_addresses); + } + + #[test] + fn private_addresses_can_be_allowed_for_every_outbound_client() { + let mut config = sample(); + config.http.block_private_addresses = false; + assert!(!config.http_limits().block_private_addresses); + assert!(!config.fork_http_limits().block_private_addresses); + } + + #[test] + fn missing_dir_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let absent = dir.path().join("no-such-dir"); + assert!(matches!( + verify_writable_dir("repo.scan_path", &absent), + Err(EnvError::DirInaccessible { .. }) + )); + } + + #[test] + fn file_in_place_of_dir_is_rejected() { + let file = tempfile::NamedTempFile::new().unwrap(); + assert!(matches!( + verify_writable_dir("lfs.store_path", file.path()), + Err(EnvError::DirNotDir { .. }) + )); + } + + #[test] + fn writable_dir_passes() { + let dir = tempfile::tempdir().unwrap(); + assert!(verify_writable_dir("repo.scan_path", dir.path()).is_ok()); + } + + #[test] + fn verify_homepage_accepts_absent_and_default_sources() { + assert!(verify_homepage(HomepageSource::Disabled).is_ok()); + assert!(verify_homepage(HomepageSource::Default).is_ok()); + } + + #[test] + fn verify_homepage_rejects_missing_file() { + let dir = tempfile::tempdir().unwrap(); + let absent = dir.path().join("no-such-page.html"); + assert!(matches!( + verify_homepage(HomepageSource::File(absent)), + Err(EnvError::HomepageInaccessible { .. }) + )); + } + + #[test] + fn verify_homepage_rejects_directory() { + let dir = tempfile::tempdir().unwrap(); + assert!(matches!( + verify_homepage(HomepageSource::File(dir.path().to_path_buf())), + Err(EnvError::HomepageNotFile { .. }) + )); + } + + #[test] + fn verify_homepage_accepts_readable_file() { + let file = tempfile::NamedTempFile::new().unwrap(); + assert!(verify_homepage(HomepageSource::File(file.path().to_path_buf())).is_ok()); + } + + #[test] + fn disabled_homepage_ignores_relative_path() { + let mut config = sample(); + config.homepage.enabled = false; + config.homepage.path = Some(PathBuf::from("homepage.html")); + assert!(config.validate().is_ok()); + } + + #[test] + fn migration_key_archive_auto_discovered_beside_sealed_key_store_if_unset() { + let dir = tempfile::tempdir().unwrap(); + let mut config = sample(); + config.secrets.sealed_key_file = dir.path().join("sealed-keys"); + assert_eq!( + config.repo_signing_keys(), + None, + "knots that never needed an archive in the first place won't have the signing keys set anyway" + ); + + let archive = dir.path().join("repo-signing-keys.json"); + std::fs::write(&archive, "[]").unwrap(); + assert_eq!(config.repo_signing_keys(), Some(archive)); + + config.atproto.repo_signing_keys = Some(dir.path().join("elsewhere.json")); + assert_eq!( + config.repo_signing_keys(), + Some(dir.path().join("elsewhere.json")) + ); + } +} diff --git a/knot2/crates/knot-edge/Cargo.toml b/knot2/crates/knot-edge/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..a7610bc322755878ded32ddc37023f260e1d578b --- /dev/null +++ b/knot2/crates/knot-edge/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "knot-edge" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +axum = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } +tower = { workspace = true, features = ["limit", "load-shed"] } +tower-http = { workspace = true, features = ["compression-zstd", "compression-br", "compression-gzip", "timeout", "map-request-body", "cors", "set-header"] } +tower_governor = { workspace = true } +governor = { workspace = true } +http-body = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tokio-rustls = { workspace = true } +rustls = { workspace = true } +rustls-pemfile = { workspace = true } +rustls-acme = { workspace = true } +x509-parser = { workspace = true } +sha2 = { workspace = true } +base64 = { workspace = true } +quinn = { workspace = true } +h3 = { workspace = true } +h3-quinn = { workspace = true } +arc-swap = { workspace = true } +bytes = { workspace = true } +http = { workspace = true } +futures = { workspace = true } +async-trait = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } +rcgen = { workspace = true } +tempfile = { workspace = true } diff --git a/knot2/crates/knot-edge/fuzz/.gitignore b/knot2/crates/knot-edge/fuzz/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1a45eee7760d240bfa8ac1989088e0349169f264 --- /dev/null +++ b/knot2/crates/knot-edge/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-edge/fuzz/Cargo.lock b/knot2/crates/knot-edge/fuzz/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..4011cb1d78b0510fc373dbc5ee429e3bf85a38c1 --- /dev/null +++ b/knot2/crates/knot-edge/fuzz/Cargo.lock @@ -0,0 +1,4654 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive 0.5.1", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive 0.6.0", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-http-codec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "096146020b08dbc4587685b0730a7ba905625af13c65f8028035cdfd69573c91" +dependencies = [ + "anyhow", + "futures", + "http", + "httparse", + "log", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-net" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" +dependencies = [ + "async-io", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-web-client" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8caf502b44d6d4be6154ac33af012cbb5fef11e6066edcfb42834217fbaf501b" +dependencies = [ + "async-http-codec", + "async-net", + "futures", + "futures-rustls", + "http", + "lazy_static", + "log", + "rustls-pki-types", + "serde", + "thiserror 1.0.69", + "webpki-roots 0.26.11", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base45" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240e56f4d3c453c36faacb695c535a4d5f8c7d23dac175014f32eb0a71012a03" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +dependencies = [ + "serde", +] + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs 0.6.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs 0.7.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "forwarded-header-value" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" +dependencies = [ + "nonempty", + "thiserror 1.0.69", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls", + "rustls-pki-types", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "gix-trace", + "libc", + "prodash", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "dashmap", + "futures-sink", + "futures-timer", + "futures-util", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand 0.9.4", + "smallvec", + "spinning_top", + "web-time", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h3" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10872b55cfb02a821b69dc7cf8dc6a71d6af25eb9a79662bec4a9d016056b3be" +dependencies = [ + "bytes", + "fastrand", + "futures-util", + "http", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "h3-quinn" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2e732c8d91a74731663ac8479ab505042fbf547b9a207213ab7fbcbfc4f8b4" +dependencies = [ + "bytes", + "futures", + "h3", + "quinn", + "tokio", + "tokio-util", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.8", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-edge" +version = "0.1.0" +dependencies = [ + "arc-swap", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "governor", + "h3", + "h3-quinn", + "http", + "http-body", + "hyper", + "hyper-util", + "knot-types", + "quinn", + "rustls", + "rustls-acme", + "rustls-pemfile", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.7.0", + "tower_governor", + "tracing", + "x509-parser 0.18.1", +] + +[[package]] +name = "knot-edge-fuzz" +version = "0.0.0" +dependencies = [ + "knot-edge", + "libfuzzer-sys", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multibase" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e0e4a371cbf1dfd666b658ba137763edb23c45beb43cfe369b5593cd6b437b6" +dependencies = [ + "base-x", + "base256emoji", + "base45", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nonempty" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs 0.6.2", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs 0.7.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "futures-io", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.8", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-acme" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c70a17ecb067d5067565a16a2e0f26a4a2ea0924f49739d558c45186facc75" +dependencies = [ + "async-io", + "async-trait", + "async-web-client", + "aws-lc-rs", + "base64", + "blocking", + "chrono", + "futures", + "futures-rustls", + "http", + "log", + "pem", + "rcgen", + "serde", + "serde_json", + "thiserror 2.0.18", + "webpki-roots 1.0.8", + "x509-parser 0.16.0", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tower_governor" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44de9b94d849d3c46e06a883d72d408c2de6403367b39df2b1c9d9e7b6736fe6" +dependencies = [ + "axum", + "forwarded-header-value", + "governor", + "http", + "pin-project", + "thiserror 2.0.18", + "tower", + "tracing", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.8", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs 0.6.2", + "data-encoding", + "der-parser 9.0.0", + "lazy_static", + "nom", + "oid-registry 0.7.1", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs 0.7.2", + "data-encoding", + "der-parser 10.0.0", + "lazy_static", + "nom", + "oid-registry 0.8.1", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/knot2/crates/knot-edge/fuzz/Cargo.toml b/knot2/crates/knot-edge/fuzz/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..9856d93734e6e88c70c3a60145429ce2ac085dbb --- /dev/null +++ b/knot2/crates/knot-edge/fuzz/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "knot-edge-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-edge] +path = ".." + +[[bin]] +name = "spki" +path = "fuzz_targets/spki.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "spki_pin" +path = "fuzz_targets/spki_pin.rs" +test = false +doc = false +bench = false diff --git a/knot2/crates/knot-edge/fuzz/fuzz_targets/spki.rs b/knot2/crates/knot-edge/fuzz/fuzz_targets/spki.rs new file mode 100644 index 0000000000000000000000000000000000000000..62db520249bc35be418610d485236397bcf65153 --- /dev/null +++ b/knot2/crates/knot-edge/fuzz/fuzz_targets/spki.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_edge::fuzz::spki_of_certificate(data); +}); diff --git a/knot2/crates/knot-edge/fuzz/fuzz_targets/spki_pin.rs b/knot2/crates/knot-edge/fuzz/fuzz_targets/spki_pin.rs new file mode 100644 index 0000000000000000000000000000000000000000..1a0d94c80ad7d69276c6264ae04fb07acb4d4502 --- /dev/null +++ b/knot2/crates/knot-edge/fuzz/fuzz_targets/spki_pin.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_edge::fuzz::spki_pin(data); +}); diff --git a/knot2/crates/knot-edge/src/acme.rs b/knot2/crates/knot-edge/src/acme.rs new file mode 100644 index 0000000000000000000000000000000000000000..4f171511c8fb27eba5cb0ab3bc30f3852b0273e3 --- /dev/null +++ b/knot2/crates/knot-edge/src/acme.rs @@ -0,0 +1,276 @@ +use std::convert::Infallible; +use std::path::PathBuf; +use std::sync::Arc; + +use async_trait::async_trait; +use futures::StreamExt; +use knot_types::KnotHostname; +use rustls::server::ResolvesServerCert; +use rustls_acme::caches::DirCache; +use rustls_acme::{AccountCache, AcmeConfig, CertCache}; +use tokio_util::sync::CancellationToken; + +#[derive(Debug, thiserror::Error)] +pub enum AcmeError { + #[error("acme cache {path}: {source}")] + Cache { + path: String, + #[source] + source: std::io::Error, + }, +} + +struct RestrictedDirCache { + dir: PathBuf, + inner: DirCache, +} + +impl RestrictedDirCache { + fn new(dir: PathBuf) -> Self { + let inner = DirCache::new(dir.clone()); + Self { dir, inner } + } +} + +#[async_trait] +impl CertCache for RestrictedDirCache { + type EC = std::io::Error; + + async fn load_cert( + &self, + domains: &[String], + directory_url: &str, + ) -> Result>, Self::EC> { + self.inner.load_cert(domains, directory_url).await + } + + async fn store_cert( + &self, + domains: &[String], + directory_url: &str, + cert: &[u8], + ) -> Result<(), Self::EC> { + self.inner.store_cert(domains, directory_url, cert).await?; + restrict_cache_dir(&self.dir).map_err(std::io::Error::other) + } +} + +#[async_trait] +impl AccountCache for RestrictedDirCache { + type EA = std::io::Error; + + async fn load_account( + &self, + contact: &[String], + directory_url: &str, + ) -> Result>, Self::EA> { + self.inner.load_account(contact, directory_url).await + } + + async fn store_account( + &self, + contact: &[String], + directory_url: &str, + account: &[u8], + ) -> Result<(), Self::EA> { + self.inner + .store_account(contact, directory_url, account) + .await?; + restrict_cache_dir(&self.dir).map_err(std::io::Error::other) + } +} + +#[derive(Debug, thiserror::Error)] +#[error("acme contact {value:?} isn't a bare email address")] +pub struct AcmeContactError { + value: String, +} + +pub struct AcmeContact(String); + +impl AcmeContact { + // `mailto()` already puts on the scheme for us, + // so someone that came with one + // already went out to the ACME account as mailto:mailto:. + // Hence no colons. + pub fn new(contact: impl Into) -> Result { + let contact = contact.into(); + let mut halves = contact.split('@'); + let well_formed = matches!((halves.next(), halves.next(), halves.next()), (Some(local), Some(domain), None) if !local.is_empty() && domain.contains('.')) + && !contact.contains(':') + && !contact.chars().any(|c| c.is_whitespace() || c.is_control()); + match well_formed { + true => Ok(Self(contact)), + false => Err(AcmeContactError { value: contact }), + } + } + + pub fn mailto(&self) -> String { + format!("mailto:{}", self.0) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AcmeCacheDir(PathBuf); + +impl AcmeCacheDir { + pub fn new(path: impl Into) -> Self { + Self(path.into()) + } + + pub fn as_path(&self) -> &std::path::Path { + &self.0 + } +} + +pub struct AcmeParams { + pub domains: Vec, + pub contact: AcmeContact, + pub cache_dir: AcmeCacheDir, + pub production: bool, +} + +pub fn start( + params: AcmeParams, + shutdown: CancellationToken, +) -> Result, AcmeError> { + std::fs::create_dir_all(params.cache_dir.as_path()).map_err(|source| AcmeError::Cache { + path: params.cache_dir.as_path().display().to_string(), + source, + })?; + restrict_cache_dir(params.cache_dir.as_path())?; + + let mut state = AcmeConfig::::new(params.domains) + .contact_push(params.contact.mailto()) + .cache(RestrictedDirCache::new(params.cache_dir.0)) + .directory_lets_encrypt(params.production) + .state(); + let resolver = state.resolver(); + + tokio::spawn(async move { + loop { + tokio::select! { + () = shutdown.cancelled() => break, + event = state.next() => match event { + Some(Ok(ok)) => tracing::info!("acme: {ok:?}"), + Some(Err(error)) => tracing::warn!("acme: {error:?}"), + None => { + tracing::warn!("acme renewal stream ended, certificates will no longer renew"); + break; + } + }, + } + } + }); + + Ok(resolver) +} + +#[cfg(unix)] +fn restrict_cache_dir(path: &std::path::Path) -> Result<(), AcmeError> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).map_err(|source| { + AcmeError::Cache { + path: path.display().to_string(), + source, + } + })?; + std::fs::read_dir(path) + .map_err(|source| AcmeError::Cache { + path: path.display().to_string(), + source, + })? + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_type() + .map(|kind| kind.is_file()) + .unwrap_or(false) + }) + .try_for_each(|entry| { + std::fs::set_permissions(entry.path(), std::fs::Permissions::from_mode(0o600)).map_err( + |source| AcmeError::Cache { + path: entry.path().display().to_string(), + source, + }, + ) + }) +} + +#[cfg(not(unix))] +fn restrict_cache_dir(_path: &std::path::Path) -> Result<(), AcmeError> { + Ok(()) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + #[test] + fn acme_contact_accepts_a_bare_email_and_rejects_everything_else() { + assert_eq!( + AcmeContact::new("ops@oyster.cafe").unwrap().mailto(), + "mailto:ops@oyster.cafe" + ); + assert!(AcmeContact::new("").is_err()); + assert!(AcmeContact::new("ops").is_err()); + assert!(AcmeContact::new("ops@localhost").is_err()); + assert!(AcmeContact::new("mailto:ops@oyster.cafe").is_err()); + assert!(AcmeContact::new("ops@nel.pet@extra.dev").is_err()); + assert!(AcmeContact::new("ops @oyster.cafe").is_err()); + assert!(AcmeContact::new("@oyster.cafe").is_err()); + } + + #[test] + fn the_cache_dir_and_its_files_are_tightened_to_owner_only() { + let dir = tempfile::tempdir().unwrap(); + let key = dir.path().join("account.key"); + std::fs::write(&key, b"private material").unwrap(); + std::fs::set_permissions(&key, std::fs::Permissions::from_mode(0o644)).unwrap(); + + restrict_cache_dir(dir.path()).unwrap(); + + assert_eq!( + std::fs::metadata(dir.path()).unwrap().permissions().mode() & 0o777, + 0o700, + "the cache directory must be traversable only by its owner" + ); + assert_eq!( + std::fs::metadata(&key).unwrap().permissions().mode() & 0o777, + 0o600, + "a cached private key must be readable only by its owner" + ); + } + + #[tokio::test] + async fn a_cert_stored_after_boot_is_tightened_to_owner_only() { + let dir = tempfile::tempdir().unwrap(); + let cache = RestrictedDirCache::new(dir.path().to_path_buf()); + cache + .store_cert( + &["anemone.knot".to_string()], + "https://acme.test/directory", + b"private cert material", + ) + .await + .unwrap(); + + let modes: Vec = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(Result::ok) + .map(|entry| { + std::fs::metadata(entry.path()) + .unwrap() + .permissions() + .mode() + & 0o777 + }) + .collect(); + assert_eq!( + modes, + vec![0o600], + "a certificate written after boot must be the only entry and owner-only" + ); + } +} diff --git a/knot2/crates/knot-edge/src/altsvc.rs b/knot2/crates/knot-edge/src/altsvc.rs new file mode 100644 index 0000000000000000000000000000000000000000..ba005cf7633591666f4fd0f32e53c718dcac6bf6 --- /dev/null +++ b/knot2/crates/knot-edge/src/altsvc.rs @@ -0,0 +1,103 @@ +use axum::Router; +use axum::body::Body; +use http::header::ALT_SVC; +use http::{HeaderValue, Request, Response, StatusCode}; + +const ALT_SVC_MAX_AGE_SECS: u32 = 86_400; + +knot_types::scalar_newtype! { + pub struct Port(u16); +} + +pub fn alt_svc_header(port: Port) -> HeaderValue { + let port = port.get(); + HeaderValue::from_str(&format!("h3=\":{port}\"; ma={ALT_SVC_MAX_AGE_SECS}")) + .expect("alt-svc header value is valid ascii") +} + +pub fn with_alt_svc(app: Router, port: Port) -> Router { + let value = alt_svc_header(port); + app.layer(axum::middleware::map_response( + move |mut response: Response| { + let value = value.clone(); + async move { + if response.status() != StatusCode::SWITCHING_PROTOCOLS { + response.headers_mut().insert(ALT_SVC, value); + } + response + } + }, + )) +} + +pub fn with_host_from_authority(app: Router) -> Router { + app.layer(axum::middleware::map_request( + |mut request: Request| async move { + let authority = request + .uri() + .authority() + .map(|authority| HeaderValue::from_str(authority.as_str())); + match ( + request.headers().contains_key(http::header::HOST), + authority, + ) { + (false, Some(Ok(value))) => { + request.headers_mut().insert(http::header::HOST, value); + request + } + _ => request, + } + }, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + use axum::routing::get; + use tower::ServiceExt; + + #[test] + fn alt_svc_header_advertises_h3() { + assert_eq!( + alt_svc_header(Port::new(443)).to_str().unwrap(), + "h3=\":443\"; ma=86400" + ); + } + + #[tokio::test] + async fn alt_svc_added_to_responses_except_switching_protocols() { + let app = with_alt_svc( + Router::new().route("/ok", get(|| async { "ok" })).route( + "/upgrade", + get(|| async { + Response::builder() + .status(StatusCode::SWITCHING_PROTOCOLS) + .body(Body::empty()) + .unwrap() + }), + ), + Port::new(443), + ); + + let normal = app + .clone() + .oneshot(Request::get("/ok").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!( + normal.headers().get(ALT_SVC).and_then(|v| v.to_str().ok()), + Some("h3=\":443\"; ma=86400") + ); + + let upgrade = app + .oneshot(Request::get("/upgrade").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert!( + upgrade.headers().get(ALT_SVC).is_none(), + "101 responses mustn't include Alt-Svc" + ); + } +} diff --git a/knot2/crates/knot-edge/src/compression.rs b/knot2/crates/knot-edge/src/compression.rs new file mode 100644 index 0000000000000000000000000000000000000000..122c5bc53d1e1af8177c718d1729c57f69a5f3c9 --- /dev/null +++ b/knot2/crates/knot-edge/src/compression.rs @@ -0,0 +1,228 @@ +use http::Response; +use http::header::CONTENT_TYPE; +use http_body::Body; +use tower_http::compression::CompressionLayer; +use tower_http::compression::predicate::{And, Predicate, SizeAbove}; + +const MIN_COMPRESS_BYTES: u64 = 256; + +#[derive(Clone, Copy)] +pub(crate) struct CompressibleResponse; + +impl Predicate for CompressibleResponse { + fn should_compress(&self, response: &Response) -> bool + where + B: Body, + { + response + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(|value| { + value + .split(';') + .next() + .unwrap_or(value) + .trim() + .to_ascii_lowercase() + }) + .is_some_and(|base| { + matches!( + base.as_str(), + "application/json" | "application/x-git-upload-pack-advertisement" + ) + }) + } +} + +pub(crate) fn layer() -> CompressionLayer> { + CompressionLayer::new() + .compress_when(SizeAbove::new(MIN_COMPRESS_BYTES).and(CompressibleResponse)) +} + +#[cfg(test)] +mod tests { + use axum::Router; + use axum::response::IntoResponse; + use axum::routing::get; + use http::StatusCode; + use http::header::{ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_RANGE, CONTENT_TYPE, RANGE}; + use tower::ServiceExt; + + use super::layer; + + async fn json_body() -> impl IntoResponse { + ([(CONTENT_TYPE, "application/json")], "x".repeat(4096)) + } + + async fn advertisement() -> impl IntoResponse { + ( + [(CONTENT_TYPE, "application/x-git-upload-pack-advertisement")], + "x".repeat(4096), + ) + } + + async fn pack_result() -> impl IntoResponse { + ( + [(CONTENT_TYPE, "application/x-git-upload-pack-result")], + "x".repeat(4096), + ) + } + + async fn targz_archive() -> impl IntoResponse { + ([(CONTENT_TYPE, "application/gzip")], "x".repeat(4096)) + } + + async fn zip_archive() -> impl IntoResponse { + ([(CONTENT_TYPE, "application/zip")], "x".repeat(4096)) + } + + async fn small_json() -> impl IntoResponse { + ([(CONTENT_TYPE, "application/json")], "{}") + } + + async fn cased_json() -> impl IntoResponse { + ( + [(CONTENT_TYPE, "Application/JSON; charset=utf-8")], + "x".repeat(4096), + ) + } + + async fn partial_archive() -> impl IntoResponse { + ( + StatusCode::PARTIAL_CONTENT, + [ + (CONTENT_TYPE, "application/gzip"), + (CONTENT_RANGE, "bytes 0-3/4096"), + ], + "xxxx", + ) + } + + fn app() -> Router { + Router::new() + .route("/json", get(json_body)) + .route("/adv", get(advertisement)) + .route("/pack", get(pack_result)) + .route("/targz", get(targz_archive)) + .route("/zip", get(zip_archive)) + .route("/small", get(small_json)) + .route("/cased", get(cased_json)) + .route("/partial", get(partial_archive)) + .layer(layer()) + } + + async fn content_encoding(path: &str, accept: Option<&str>) -> Option { + let mut builder = http::Request::builder().method("GET").uri(path); + if let Some(value) = accept { + builder = builder.header(ACCEPT_ENCODING, value); + } + let request = builder.body(axum::body::Body::empty()).unwrap(); + app() + .oneshot(request) + .await + .unwrap() + .headers() + .get(CONTENT_ENCODING) + .map(|value| value.to_str().unwrap().to_string()) + } + + #[tokio::test] + async fn json_and_advertisement_compress_but_pack_bytes_pass_through() { + let negotiated = ["zstd", "br", "gzip"]; + let json = content_encoding("/json", Some("zstd, br, gzip")).await; + assert!( + json.as_deref().is_some_and(|enc| negotiated.contains(&enc)), + "json negotiates an encoding, got {json:?}" + ); + let adv = content_encoding("/adv", Some("zstd, br, gzip")).await; + assert!( + adv.as_deref().is_some_and(|enc| negotiated.contains(&enc)), + "the ref advertisement negotiates an encoding, got {adv:?}" + ); + assert_eq!( + content_encoding("/pack", Some("zstd, br, gzip")).await, + None, + "an already-compressed pack stream is never re-encoded" + ); + } + + #[tokio::test] + async fn already_compressed_archives_pass_through() { + assert_eq!( + content_encoding("/targz", Some("zstd, br, gzip")).await, + None, + "a gzip archive is never re-encoded" + ); + assert_eq!( + content_encoding("/zip", Some("zstd, br, gzip")).await, + None, + "a zip archive is never re-encoded" + ); + } + + #[tokio::test] + async fn zstd_outranks_brotli_outranks_gzip_on_equal_quality() { + assert_eq!( + content_encoding("/json", Some("gzip, br, zstd")) + .await + .as_deref(), + Some("zstd"), + "zstd wins over brotli and gzip at equal q" + ); + assert_eq!( + content_encoding("/json", Some("gzip, br")).await.as_deref(), + Some("br"), + "brotli wins over gzip at equal q" + ); + assert_eq!( + content_encoding("/json", Some("gzip")).await.as_deref(), + Some("gzip"), + "gzip serves the client that offers only gzip" + ); + } + + #[tokio::test] + async fn nothing_compresses_without_accept_encoding_or_below_the_floor() { + assert_eq!(content_encoding("/json", None).await, None); + assert_eq!( + content_encoding("/small", Some("zstd, br, gzip")).await, + None, + "a body below the size floor is left alone" + ); + } + + #[tokio::test] + async fn a_mixed_case_content_type_still_compresses() { + let negotiated = ["zstd", "br", "gzip"]; + let cased = content_encoding("/cased", Some("zstd, br, gzip")).await; + assert!( + cased + .as_deref() + .is_some_and(|enc| negotiated.contains(&enc)), + "content-type matching is case insensitive, got {cased:?}" + ); + } + + #[tokio::test] + async fn a_partial_archive_passes_through_with_its_range_intact() { + let mut builder = http::Request::builder().method("GET").uri("/partial"); + builder = builder.header(ACCEPT_ENCODING, "zstd, br, gzip"); + builder = builder.header(RANGE, "bytes=0-3"); + let response = app() + .oneshot(builder.body(axum::body::Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + response.headers().get(CONTENT_RANGE).unwrap(), + "bytes 0-3/4096", + "the range survives the compression layer untouched" + ); + assert_eq!( + response.headers().get(CONTENT_ENCODING), + None, + "a partial archive is never re-encoded" + ); + } +} diff --git a/knot2/crates/knot-edge/src/cors.rs b/knot2/crates/knot-edge/src/cors.rs new file mode 100644 index 0000000000000000000000000000000000000000..489ef5c64ca99c1068d02164648a665ef3cb14de --- /dev/null +++ b/knot2/crates/knot-edge/src/cors.rs @@ -0,0 +1,30 @@ +//! Preflight is answered under the guards, so a burst of them is rate +//! limited like any other request. The allow-origin header alone is set +//! outside the governor, which keeps a 429 or a shed 503 readable to a browser. + +use std::time::Duration; + +use axum::Router; +use http::header::{ACCESS_CONTROL_ALLOW_ORIGIN, AUTHORIZATION, CONTENT_TYPE}; +use http::{HeaderValue, Method}; +use tower_http::cors::{Any, CorsLayer}; +use tower_http::set_header::SetResponseHeaderLayer; + +const PREFLIGHT_CACHE: Duration = Duration::from_secs(86_400); + +pub(crate) fn preflight(router: Router) -> Router { + router.layer( + CorsLayer::new() + .allow_origin(Any) + .allow_methods([Method::GET, Method::POST, Method::OPTIONS]) + .allow_headers([AUTHORIZATION, CONTENT_TYPE]) + .max_age(PREFLIGHT_CACHE), + ) +} + +pub(crate) fn origin_header(router: Router) -> Router { + router.layer(SetResponseHeaderLayer::if_not_present( + ACCESS_CONTROL_ALLOW_ORIGIN, + HeaderValue::from_static("*"), + )) +} diff --git a/knot2/crates/knot-edge/src/lib.rs b/knot2/crates/knot-edge/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..914243f3af82ffc9702036b79f9b251a45b055bd --- /dev/null +++ b/knot2/crates/knot-edge/src/lib.rs @@ -0,0 +1,381 @@ +mod acme; +mod altsvc; +mod compression; +mod cors; +mod limits; +mod peer; +mod protocol; +mod quic; +mod robustness; +mod tcp; +mod tls; +mod zerortt; + +use std::future::Future; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::Arc; + +use axum::Router; +use axum::middleware::from_fn; +use rustls::server::ResolvesServerCert; +use tokio::net::TcpListener; +use tokio_util::sync::CancellationToken; + +pub use acme::{AcmeCacheDir, AcmeContact, AcmeContactError, AcmeError, AcmeParams}; +pub use limits::{ + ConnectionBudget, HeaderTimeout, IdleTimeout, ListenLimits, MaxConcurrentStreams, +}; +pub use peer::SocketPeer; +pub use protocol::NegotiatedProtocol; +pub use quic::EndpointError; +pub use robustness::{ + BodyInactivityTimeout, BurstSize, EdgeGuards, MaxInflightRequests, RequestTimeout, + RequestsPerSecond, WriteRequestTimeout, +}; +pub use tls::{ReloadableCertResolver, SpkiPin, TlsError, load_certified_key}; +pub use zerortt::{EarlyData, RequiresFullHandshake, ZeroRttRoutes, ZeroRttSafe}; + +pub mod fuzz { + pub fn spki_of_certificate(data: &[u8]) { + crate::tls::fuzz_of_certificate(data); + } + + pub fn spki_pin(data: &[u8]) { + let _ = crate::SpkiPin::from_base64(&String::from_utf8_lossy(data)); + } +} + +macro_rules! path_wrapper { + ($($name:ident),+ $(,)?) => {$( + #[derive(Debug, Clone, PartialEq, Eq)] + pub struct $name(PathBuf); + + impl $name { + pub fn new(path: impl Into) -> Self { + Self(path.into()) + } + + pub fn as_path(&self) -> &Path { + &self.0 + } + } + )+}; +} + +path_wrapper!(CertChainPath, ClientCaPath, PrivateKeyPath); + +macro_rules! bind_wrapper { + ($($name:ident),+ $(,)?) => {$( + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct $name(SocketAddr); + + impl $name { + pub const fn new(addr: SocketAddr) -> Self { + Self(addr) + } + + pub const fn get(self) -> SocketAddr { + self.0 + } + } + )+}; +} + +bind_wrapper!(InternalBind, PublicBind); + +pub struct StaticCertPaths { + pub cert_path: CertChainPath, + pub key_path: PrivateKeyPath, +} + +pub enum CertSource { + Static(StaticCertPaths), + Acme(AcmeParams), +} + +pub struct InternalTls { + pub addr: InternalBind, + pub client_ca_path: ClientCaPath, + pub spki_pin: SpkiPin, +} + +pub struct TlsSetup { + pub source: CertSource, + pub http3: bool, + pub internal: Option, +} + +pub struct EdgeConfig { + pub http_addr: PublicBind, + pub limits: ListenLimits, + pub guards: EdgeGuards, + pub tls: Option, +} + +#[derive(Debug, thiserror::Error)] +pub enum EdgeError { + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Tls(#[from] TlsError), + #[error(transparent)] + Acme(#[from] AcmeError), + #[error(transparent)] + Endpoint(#[from] EndpointError), +} + +type Served = Pin> + Send>>; + +fn base_router(app: RequiresFullHandshake, early_data_safe: ZeroRttRoutes) -> Router { + early_data_safe + .into_router() + .merge(app.into_router()) + .layer(compression::layer()) + .layer(from_fn(zerortt::tag_from_header)) +} + +fn finish(router: Router) -> Router { + altsvc::with_host_from_authority(router.layer(from_fn(protocol::tag))) +} + +pub async fn serve( + config: EdgeConfig, + app: RequiresFullHandshake, + early_data_safe: ZeroRttRoutes, + shutdown: CancellationToken, +) -> Result<(), EdgeError> { + let EdgeConfig { + http_addr, + limits, + guards, + tls, + } = config; + let layers = guards.prepare(&shutdown); + let early_data = early_data_safe.early_data_policy(); + let wants_internal = tls + .as_ref() + .and_then(|setup| setup.internal.as_ref()) + .is_some(); + let base = cors::preflight(base_router(app, early_data_safe)); + let internal_router = wants_internal.then(|| finish(base.clone())); + let router = finish(cors::origin_header(robustness::apply(base, layers))); + let listener = TcpListener::bind(http_addr.get()).await?; + + let Some(setup) = tls else { + return Ok(tcp::serve_plaintext(listener, router, limits, shutdown).await?); + }; + + let (resolver, acme): (Arc, bool) = match setup.source { + CertSource::Static(paths) => { + let certified = tls::load_certified_key(&paths)?; + let reloadable = Arc::new(ReloadableCertResolver::new(certified)); + tls::spawn_cert_reload(Arc::clone(&reloadable), paths, shutdown.clone()); + (reloadable, false) + } + CertSource::Acme(params) => (acme::start(params, shutdown.clone())?, true), + }; + + let extra_alpn: &[&[u8]] = if acme { &[tls::ACME_TLS_ALPN] } else { &[] }; + let tcp_config = Arc::new(tls::build_tls_server_config( + Arc::clone(&resolver), + extra_alpn, + )?); + let port = http_addr.get().port(); + + let mut servers: Vec = Vec::new(); + + servers.push({ + let app = match setup.http3 { + true => altsvc::with_alt_svc(router.clone(), altsvc::Port::new(port)), + false => router.clone(), + }; + let shutdown = shutdown.clone(); + Box::pin(async move { + let result = tcp::serve_tls(listener, app, tcp_config, limits, shutdown.clone()).await; + shutdown.cancel(); + Ok(result?) + }) + }); + + if setup.http3 { + let endpoint = + quic::build_endpoint(http_addr.get(), Arc::clone(&resolver), limits, early_data)?; + let app = router.clone(); + let shutdown = shutdown.clone(); + servers.push(Box::pin(async move { + quic::serve_http3(endpoint, app, limits, shutdown.clone()).await; + shutdown.cancel(); + Ok(()) + })); + } + + if let Some(internal) = setup.internal { + let internal_listener = TcpListener::bind(internal.addr.get()).await?; + let mtls_config = Arc::new(tls::build_mtls_server_config( + Arc::clone(&resolver), + &internal.client_ca_path, + internal.spki_pin, + )?); + let app = internal_router + .expect("an internal router is built whenever an internal bind is configured"); + let shutdown = shutdown.clone(); + servers.push(Box::pin(async move { + let result = tcp::serve_tls( + internal_listener, + app, + mtls_config, + limits, + shutdown.clone(), + ) + .await; + shutdown.cancel(); + Ok(result?) + })); + } + + futures::future::join_all(servers) + .await + .into_iter() + .collect::, EdgeError>>() + .map(drop) +} + +#[cfg(test)] +mod tests { + use super::*; + + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::routing::post; + use http::{Request, StatusCode}; + use tower::ServiceExt; + + use std::num::{NonZeroU32, NonZeroU64}; + + use zerortt::ZeroRttSafe; + + fn test_layers() -> robustness::GuardLayers { + EdgeGuards::new( + RequestsPerSecond::new(NonZeroU32::new(10_000).unwrap()), + BurstSize::new(NonZeroU32::new(10_000).unwrap()), + MaxInflightRequests::new(NonZeroU32::new(1_024).unwrap()), + RequestTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + BodyInactivityTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + WriteRequestTimeout::from_millis(NonZeroU64::new(1_800_000).unwrap()), + knot_types::ProxyTrust::default(), + ) + .prepare(&CancellationToken::new()) + } + + fn wired() -> Router { + let safe = + ZeroRttRoutes::new().get("/info/refs", ZeroRttSafe::new(|| async { "advertisement" })); + let full = RequiresFullHandshake::new( + Router::new().route("/git-upload-pack", post(|| async { "pack" })), + ); + let base = cors::preflight(base_router(full, safe)); + finish(cors::origin_header(robustness::apply(base, test_layers()))) + } + + fn tight_layers() -> robustness::GuardLayers { + EdgeGuards::new( + RequestsPerSecond::new(NonZeroU32::new(1).unwrap()), + BurstSize::new(NonZeroU32::new(2).unwrap()), + MaxInflightRequests::new(NonZeroU32::new(1_024).unwrap()), + RequestTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + BodyInactivityTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + WriteRequestTimeout::from_millis(NonZeroU64::new(1_800_000).unwrap()), + knot_types::ProxyTrust::default(), + ) + .prepare(&CancellationToken::new()) + } + + async fn status_of(mut request: Request) -> StatusCode { + request + .extensions_mut() + .insert(ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 41001)))); + wired().oneshot(request).await.unwrap().status() + } + + #[tokio::test] + async fn a_write_in_early_data_is_refused_with_425() { + let request = Request::post("/git-upload-pack") + .header("early-data", "1") + .body(Body::empty()) + .unwrap(); + assert_eq!(status_of(request).await, StatusCode::TOO_EARLY); + } + + #[tokio::test] + async fn a_write_after_the_handshake_is_served() { + let request = Request::post("/git-upload-pack") + .body(Body::empty()) + .unwrap(); + assert_eq!(status_of(request).await, StatusCode::OK); + } + + #[tokio::test] + async fn the_advertisement_is_served_even_in_early_data() { + let request = Request::get("/info/refs") + .header("early-data", "1") + .body(Body::empty()) + .unwrap(); + assert_eq!(status_of(request).await, StatusCode::OK); + } + + #[tokio::test] + async fn a_preflight_in_early_data_is_answered_and_not_deferred_to_425() { + let request = Request::options("/git-upload-pack") + .header("early-data", "1") + .header(http::header::ORIGIN, "https://tangled.org") + .header(http::header::ACCESS_CONTROL_REQUEST_METHOD, "POST") + .body(Body::empty()) + .unwrap(); + let answer = status_of(request).await; + assert!( + answer.is_success(), + "a preflight answer is a constant, so replaying one doesn't get an attacker a write: got {answer}" + ); + } + + #[tokio::test] + async fn the_internal_admin_router_shares_no_rate_limit_budget_with_the_data_plane() { + let safe = ZeroRttRoutes::new().get("/info/refs", ZeroRttSafe::new(|| async { "ok" })); + let full = RequiresFullHandshake::new( + Router::new().route("/git-upload-pack", post(|| async { "pack" })), + ); + let base = base_router(full, safe); + let public = finish(robustness::apply(base.clone(), tight_layers())); + let internal = finish(base); + + let request = || { + let mut request = Request::get("/info/refs").body(Body::empty()).unwrap(); + request + .extensions_mut() + .insert(ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 41001)))); + request + }; + + let p1 = public.clone().oneshot(request()).await.unwrap().status(); + let p2 = public.clone().oneshot(request()).await.unwrap().status(); + let p3 = public.clone().oneshot(request()).await.unwrap().status(); + assert_eq!([p1, p2], [StatusCode::OK, StatusCode::OK]); + assert_eq!( + p3, + StatusCode::TOO_MANY_REQUESTS, + "the public edge still enforces the per-IP burst" + ); + + let i1 = internal.clone().oneshot(request()).await.unwrap().status(); + let i2 = internal.clone().oneshot(request()).await.unwrap().status(); + let i3 = internal.clone().oneshot(request()).await.unwrap().status(); + let i4 = internal.clone().oneshot(request()).await.unwrap().status(); + assert_eq!( + [i1, i2, i3, i4], + [StatusCode::OK; 4], + "the internal admin bind is unguarded, so a public flood never sheds admin requests" + ); + } +} diff --git a/knot2/crates/knot-edge/src/limits.rs b/knot2/crates/knot-edge/src/limits.rs new file mode 100644 index 0000000000000000000000000000000000000000..7c73691bf77a90d00a1e690b94061f9c596d3746 --- /dev/null +++ b/knot2/crates/knot-edge/src/limits.rs @@ -0,0 +1,113 @@ +use std::num::{NonZeroU32, NonZeroU64}; +use std::time::Duration; + +const MAX_CONCURRENT_STREAMS: u32 = 256; +const STREAM_RECEIVE_WINDOW: u32 = 8 * 1024 * 1024; +const CONNECTION_RECEIVE_WINDOW: u32 = 32 * 1024 * 1024; +// hyper's default on h2, so a request is cut off at about the same size over either protocol. +// h3 has no limit at all unless this is set +const MAX_FIELD_SECTION_SIZE: u64 = 16 * 1024; + +knot_types::scalar_newtype! { + pub struct MaxConcurrentStreams(u32) => sealed; +} + +#[derive(Debug, Clone, Copy)] +pub struct ConnectionBudget { + max_concurrent_streams: MaxConcurrentStreams, + stream_receive_window: u32, + connection_receive_window: u32, + max_field_section_size: u64, +} + +impl ConnectionBudget { + const DEFAULT: Self = Self { + max_concurrent_streams: MaxConcurrentStreams::new(MAX_CONCURRENT_STREAMS), + stream_receive_window: STREAM_RECEIVE_WINDOW, + connection_receive_window: CONNECTION_RECEIVE_WINDOW, + max_field_section_size: MAX_FIELD_SECTION_SIZE, + }; + + pub fn max_concurrent_streams(self) -> MaxConcurrentStreams { + self.max_concurrent_streams + } + + pub fn stream_receive_window(self) -> u32 { + self.stream_receive_window + } + + pub fn connection_receive_window(self) -> u32 { + self.connection_receive_window + } + + pub fn max_field_section_size(self) -> u64 { + self.max_field_section_size + } +} + +// Separate types that `ListenLimits::new` used to take as +// a bunch of `NonZeroU64` in a row. +// Swapping them would incorrectly compile and +// gave the conn the wrong deadline. +#[derive(Debug, Clone, Copy)] +pub struct HeaderTimeout(Duration); + +impl HeaderTimeout { + pub fn from_millis(millis: NonZeroU64) -> Self { + Self(Duration::from_millis(millis.get())) + } + + pub fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct IdleTimeout(Duration); + +impl IdleTimeout { + pub fn from_millis(millis: NonZeroU64) -> Self { + Self(Duration::from_millis(millis.get())) + } + + pub fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct ListenLimits { + header_timeout: HeaderTimeout, + idle_timeout: IdleTimeout, + max_connections: usize, +} + +impl ListenLimits { + pub fn new( + header_timeout: HeaderTimeout, + idle_timeout: IdleTimeout, + max_connections: NonZeroU32, + ) -> Self { + Self { + header_timeout, + idle_timeout, + max_connections: max_connections.get() as usize, + } + } + + pub fn header_timeout(&self) -> HeaderTimeout { + self.header_timeout + } + + pub fn idle_timeout(&self) -> IdleTimeout { + self.idle_timeout + } + + pub fn max_connections(&self) -> usize { + self.max_connections + } + + pub fn connection_budget(&self) -> ConnectionBudget { + ConnectionBudget::DEFAULT + } +} diff --git a/knot2/crates/knot-edge/src/peer.rs b/knot2/crates/knot-edge/src/peer.rs new file mode 100644 index 0000000000000000000000000000000000000000..cd07cd8f8c2fe8f3b926ee692a46d1b9f3dd4c9e --- /dev/null +++ b/knot2/crates/knot-edge/src/peer.rs @@ -0,0 +1,58 @@ +use std::convert::Infallible; +use std::net::{IpAddr, SocketAddr}; + +use axum::extract::{ConnectInfo, FromRequestParts}; +use http::request::Parts; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SocketPeer(Option); + +impl SocketPeer { + pub fn ip(self) -> Option { + self.0 + } +} + +impl FromRequestParts for SocketPeer { + type Rejection = Infallible; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + Ok(Self( + parts + .extensions + .get::>() + .map(|connect| connect.0.ip()), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use std::net::Ipv4Addr; + + #[tokio::test] + async fn the_extractor_reads_connect_info_and_tolerates_its_absence() { + let with = { + let mut request = http::Request::builder().body(Body::empty()).unwrap(); + request + .extensions_mut() + .insert(ConnectInfo(SocketAddr::from(([203, 0, 113, 7], 443)))); + let (mut parts, _) = request.into_parts(); + SocketPeer::from_request_parts(&mut parts, &()) + .await + .unwrap() + }; + assert_eq!(with.ip(), Some(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 7)))); + + let without = { + let request = http::Request::builder().body(Body::empty()).unwrap(); + let (mut parts, _) = request.into_parts(); + SocketPeer::from_request_parts(&mut parts, &()) + .await + .unwrap() + }; + assert_eq!(without.ip(), None); + } +} diff --git a/knot2/crates/knot-edge/src/protocol.rs b/knot2/crates/knot-edge/src/protocol.rs new file mode 100644 index 0000000000000000000000000000000000000000..126aa17eeb79dd0fab0d4d541dee7165abdff912 --- /dev/null +++ b/knot2/crates/knot-edge/src/protocol.rs @@ -0,0 +1,78 @@ +use axum::extract::Request; +use axum::middleware::Next; +use axum::response::Response; +use http::Version; +use tracing::Instrument; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NegotiatedProtocol { + H1, + H2, + H3, +} + +impl NegotiatedProtocol { + pub fn as_str(self) -> &'static str { + match self { + NegotiatedProtocol::H1 => "h1", + NegotiatedProtocol::H2 => "h2", + NegotiatedProtocol::H3 => "h3", + } + } + + pub fn from_version(version: Version) -> Option { + match version { + Version::HTTP_10 | Version::HTTP_11 => Some(NegotiatedProtocol::H1), + Version::HTTP_2 => Some(NegotiatedProtocol::H2), + Version::HTTP_3 => Some(NegotiatedProtocol::H3), + _ => None, + } + } +} + +pub(crate) async fn tag(mut request: Request, next: Next) -> Response { + let protocol = NegotiatedProtocol::from_version(request.version()); + if let Some(protocol) = protocol { + request.extensions_mut().insert(protocol); + } + let span = tracing::debug_span!( + "request", + protocol = protocol + .map(NegotiatedProtocol::as_str) + .unwrap_or("unknown") + ); + next.run(request).instrument(span).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn each_http_version_maps_to_its_negotiated_protocol() { + assert_eq!( + NegotiatedProtocol::from_version(Version::HTTP_11), + Some(NegotiatedProtocol::H1) + ); + assert_eq!( + NegotiatedProtocol::from_version(Version::HTTP_10), + Some(NegotiatedProtocol::H1) + ); + assert_eq!( + NegotiatedProtocol::from_version(Version::HTTP_2), + Some(NegotiatedProtocol::H2) + ); + assert_eq!( + NegotiatedProtocol::from_version(Version::HTTP_3), + Some(NegotiatedProtocol::H3) + ); + assert_eq!(NegotiatedProtocol::from_version(Version::HTTP_09), None); + } + + #[test] + fn the_label_is_stable_for_logging_and_metrics() { + assert_eq!(NegotiatedProtocol::H1.as_str(), "h1"); + assert_eq!(NegotiatedProtocol::H2.as_str(), "h2"); + assert_eq!(NegotiatedProtocol::H3.as_str(), "h3"); + } +} diff --git a/knot2/crates/knot-edge/src/quic.rs b/knot2/crates/knot-edge/src/quic.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb1acfca8a6eca1d64d3e7ab690ffb34a5b12e24 --- /dev/null +++ b/knot2/crates/knot-edge/src/quic.rs @@ -0,0 +1,609 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use axum::Router; +use axum::body::Body; +use axum::extract::ConnectInfo; +use bytes::{Buf, Bytes}; +use futures::{FutureExt, StreamExt}; +use http::{Request, Response, Version}; +use quinn::{Endpoint, Incoming}; +use tokio::sync::Semaphore; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; +use tower::ServiceExt; + +use rustls::server::ResolvesServerCert; + +use crate::limits::ListenLimits; +use crate::tls::{self, TlsError}; +use crate::zerortt::{EarlyData, EarlyDataPolicy}; + +fn early_data(confirmed: bool) -> EarlyData { + match confirmed { + true => EarlyData::No, + false => EarlyData::Yes, + } +} + +const LISTENER_DRAIN_GRACE: Duration = Duration::from_secs(30); +const ENDPOINT_DRAIN_GRACE: Duration = Duration::from_secs(5); +const CONNECTION_DRAIN_GRACE: Duration = Duration::from_secs(10); + +pub fn build_endpoint( + addr: SocketAddr, + resolver: Arc, + limits: ListenLimits, + early_data: EarlyDataPolicy, +) -> Result { + let server_config = tls::build_quic_server_config(resolver, limits, early_data)?; + let endpoint = Endpoint::server(server_config, addr)?; + Ok(endpoint) +} + +#[derive(Debug, thiserror::Error)] +pub enum EndpointError { + #[error(transparent)] + Tls(#[from] TlsError), + #[error("binding quic socket: {0}")] + Bind(#[from] std::io::Error), +} + +pub async fn serve_http3( + endpoint: Endpoint, + app: Router, + limits: ListenLimits, + shutdown: CancellationToken, +) { + let tracker = TaskTracker::new(); + let connections = Arc::new(Semaphore::new(limits.max_connections())); + loop { + let incoming = tokio::select! { + () = shutdown.cancelled() => break, + incoming = endpoint.accept() => incoming, + }; + let Some(incoming) = incoming else { break }; + let Ok(permit) = Arc::clone(&connections).try_acquire_owned() else { + incoming.refuse(); + continue; + }; + let app = app.clone(); + let conn_shutdown = shutdown.clone(); + let conn_tracker = tracker.clone(); + let max_field_section_size = limits.connection_budget().max_field_section_size(); + tracker.spawn(async move { + let _permit = permit; + if let Err(error) = serve_connection( + incoming, + app, + max_field_section_size, + conn_shutdown, + conn_tracker, + ) + .await + { + tracing::debug!("h3 connection ended: {error}"); + } + }); + } + tracker.close(); + let _ = tokio::time::timeout(LISTENER_DRAIN_GRACE, tracker.wait()).await; + endpoint.close(0u32.into(), b"shutdown"); + let _ = tokio::time::timeout(ENDPOINT_DRAIN_GRACE, endpoint.wait_idle()).await; +} + +async fn serve_connection( + incoming: Incoming, + app: Router, + max_field_section_size: u64, + shutdown: CancellationToken, + tracker: TaskTracker, +) -> Result<(), Box> { + // the client's limits on how many streams we may open can arrive after we accept, and the h3 + // setup opens streams right away. quinn never wakes it when they land late, so the + // connection hung. waiting for the client hello first avoids that, + // see https://github.com/moq-dev/moq/pull/5774 + let mut connecting = incoming.accept()?; + connecting.handshake_data().await?; + let (conn, confirmation, mut confirmed) = match connecting.into_0rtt() { + Ok((conn, accepted)) => (conn, accepted.map(|_| ()).left_future(), false), + Err(connecting) => ( + connecting.await?, + std::future::pending::<()>().right_future(), + true, + ), + }; + tokio::pin!(confirmation); + let remote = conn.remote_address(); + let quic = conn.clone(); + tracing::trace!("h3 connection from {remote} accepted"); + let mut h3_conn = h3::server::builder() + .max_field_section_size(max_field_section_size) + .build::<_, Bytes>(h3_quinn::Connection::new(conn)) + .await?; + tracing::trace!("h3 connection from {remote} established"); + + let drain_deadline = tokio::time::sleep(CONNECTION_DRAIN_GRACE); + tokio::pin!(drain_deadline); + let mut draining = false; + loop { + tokio::select! { + biased; + () = shutdown.cancelled(), if !draining => { + draining = true; + drain_deadline + .as_mut() + .reset(tokio::time::Instant::now() + CONNECTION_DRAIN_GRACE); + let _ = h3_conn.shutdown(0).await; + } + () = &mut drain_deadline, if draining => { + tracing::debug!("h3 connection from {remote} drain timed out, closing"); + quic.close(0u32.into(), b"drain timeout"); + break; + } + resolved = h3_conn.accept() => match resolved { + Ok(Some(resolver)) => { + // checked here and not in the select, because when both were ready accept won + // and the first request on a new connection was marked early after all + if !confirmed { + confirmed = confirmation.as_mut().now_or_never().is_some(); + } + let app = app.clone(); + let early = early_data(confirmed); + tracker.spawn(async move { + if let Err(error) = serve_request(resolver, app, remote, early).await { + tracing::debug!("h3 request from {remote} failed: {error}"); + } + }); + } + Ok(None) => { + tracing::debug!("h3 connection from {remote} closed by the client"); + break; + } + Err(error) => { + tracing::debug!("h3 accept from {remote} error: {error}"); + break; + } + }, + } + } + Ok(()) +} + +async fn serve_request( + resolver: h3::server::RequestResolver, + app: Router, + remote: SocketAddr, + early: EarlyData, +) -> Result<(), Box> { + let (request, stream) = resolver.resolve_request().await?; + let (mut send, recv) = stream.split(); + + let (mut parts, ()) = request.into_parts(); + parts.version = Version::HTTP_3; + parts.extensions.insert(ConnectInfo(remote)); + parts.extensions.insert(early); + let request = Request::from_parts(parts, request_body(recv)); + + let response = match app.oneshot(request).await { + Ok(response) => response, + Err(infallible) => match infallible {}, + }; + + let (parts, body) = response.into_parts(); + send.send_response(Response::from_parts(parts, ())).await?; + + let mut data = body.into_data_stream(); + while let Some(chunk) = data.next().await { + match chunk { + Ok(bytes) if bytes.has_remaining() => send.send_data(bytes).await?, + Ok(_) => {} + Err(error) => { + tracing::debug!("h3 response body to {remote} errored: {error}"); + send.stop_stream(h3::error::Code::H3_INTERNAL_ERROR); + return Ok(()); + } + } + } + send.finish().await?; + Ok(()) +} + +struct RecvGuard { + stream: h3::server::RequestStream, + ended: bool, +} + +impl Drop for RecvGuard { + fn drop(&mut self) { + if !self.ended { + self.stream.stop_sending(h3::error::Code::H3_NO_ERROR); + } + } +} + +fn request_body(recv: h3::server::RequestStream) -> Body { + let guard = RecvGuard { + stream: recv, + ended: false, + }; + let stream = futures::stream::unfold(Some(guard), |state| async move { + let mut guard = state?; + match guard.stream.recv_data().await { + Ok(Some(mut buf)) => { + let bytes = buf.copy_to_bytes(buf.remaining()); + Some((Ok::(bytes), Some(guard))) + } + Ok(None) => { + guard.ended = true; + None + } + Err(error) => { + guard.ended = true; + Some((Err(std::io::Error::other(error.to_string())), None)) + } + } + }); + Body::from_stream(stream) +} + +#[cfg(test)] +mod tests { + use super::*; + + use axum::routing::get; + use rustls::crypto::aws_lc_rs; + + use crate::tls; + + #[test] + fn an_unconfirmed_handshake_is_early_data_and_a_confirmed_one_is_not() { + assert_eq!( + early_data(false), + EarlyData::Yes, + "data before handshake confirmation is early data, fail closed" + ); + assert_eq!(early_data(true), EarlyData::No); + } + + fn client_endpoint() -> Endpoint { + client_endpoint_with_provider(aws_lc_rs::default_provider()) + } + + fn client_endpoint_with_provider(provider: rustls::crypto::CryptoProvider) -> Endpoint { + let mut crypto = rustls::ClientConfig::builder_with_provider(Arc::new(provider)) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(tls::test_support::AcceptAnyServerCert)) + .with_no_client_auth(); + crypto.alpn_protocols = vec![b"h3".to_vec()]; + let quic = quinn::crypto::rustls::QuicClientConfig::try_from(crypto).unwrap(); + let mut endpoint = Endpoint::client("[::1]:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(quic))); + endpoint + } + + fn spawn_h3_server( + app: Router, + limits: ListenLimits, + early_data: EarlyDataPolicy, + ) -> (SocketAddr, CancellationToken) { + let endpoint = build_endpoint( + "[::1]:0".parse().unwrap(), + tls::test_support::resolver(), + limits, + early_data, + ) + .unwrap(); + let addr = endpoint.local_addr().unwrap(); + let shutdown = CancellationToken::new(); + tokio::spawn(serve_http3( + endpoint, + crate::altsvc::with_host_from_authority(app), + limits, + shutdown.clone(), + )); + (addr, shutdown) + } + + type H3Driver = h3::client::Connection; + + async fn h3_client_handshake( + client: &Endpoint, + addr: SocketAddr, + ) -> ( + H3Driver, + h3::client::SendRequest, + ) { + let conn = client.connect(addr, "localhost").unwrap().await.unwrap(); + h3::client::new(h3_quinn::Connection::new(conn)) + .await + .unwrap() + } + + fn drive(mut driver: H3Driver) { + tokio::spawn(async move { std::future::poll_fn(|cx| driver.poll_close(cx)).await }); + } + + async fn h3_client_connect( + client: &Endpoint, + addr: SocketAddr, + ) -> h3::client::SendRequest { + let (driver, send_request) = h3_client_handshake(client, addr).await; + drive(driver); + send_request + } + + async fn h3_request( + send_request: &mut h3::client::SendRequest, + path: &str, + ) -> (http::StatusCode, Vec) { + let request = http::Request::get(format!("https://localhost{path}")) + .body(()) + .unwrap(); + let mut stream = send_request.send_request(request).await.unwrap(); + // a server may stop reading a request it doesn't need with H3_NO_ERROR, rfc 9114 section 4.1 + match stream.finish().await { + Err(h3::error::StreamError::RemoteTerminate { code, .. }) + if code == h3::error::Code::H3_NO_ERROR => {} + other => other.unwrap(), + } + let status = stream.recv_response().await.unwrap().status(); + let chunks: Vec = futures::stream::unfold(stream, |mut stream| async move { + stream + .recv_data() + .await + .unwrap() + .map(|mut buf| (buf.copy_to_bytes(buf.remaining()), stream)) + }) + .collect() + .await; + let body = chunks + .iter() + .flat_map(|chunk| chunk.iter().copied()) + .collect(); + (status, body) + } + + #[tokio::test] + async fn an_h3_get_roundtrips_through_the_router() { + let app = Router::new().route( + "/", + get(|ConnectInfo(peer): ConnectInfo| async move { peer.to_string() }), + ); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Disabled); + + let client = client_endpoint(); + let client_port = client.local_addr().unwrap().port(); + let mut send_request = h3_client_connect(&client, addr).await; + let (status, body) = h3_request(&mut send_request, "/").await; + assert_eq!(status, 200); + let reported: SocketAddr = String::from_utf8(body).unwrap().parse().unwrap(); + assert_eq!( + reported.port(), + client_port, + "the h3 handler must see the QUIC remote address via ConnectInfo" + ); + shutdown.cancel(); + } + + #[tokio::test] + async fn an_h3_request_is_tagged_with_the_h3_protocol() { + use axum::middleware::from_fn; + + let app = Router::new() + .route( + "/proto", + get(|req: axum::extract::Request| async move { + req.extensions() + .get::() + .map(|protocol| protocol.as_str()) + .unwrap_or("missing") + .to_string() + }), + ) + .layer(from_fn(crate::protocol::tag)); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Disabled); + + let client = client_endpoint(); + let mut send_request = h3_client_connect(&client, addr).await; + let (status, body) = h3_request(&mut send_request, "/proto").await; + assert_eq!(status, 200); + assert_eq!( + String::from_utf8(body).unwrap(), + "h3", + "a request served over QUIC must be tagged as the h3 negotiated protocol" + ); + shutdown.cancel(); + } + + #[tokio::test] + async fn an_h3_request_past_the_field_section_limit_gets_a_431() { + let app = Router::new().route("/", get(|| async { "ok" })); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Disabled); + + let client = client_endpoint(); + // the client only learns the server's limit once its driver reads SETTINGS, and a + // hostile one never looks, so send before driving it + let (driver, mut send_request) = h3_client_handshake(&client, addr).await; + let mut request = http::Request::get("https://localhost/"); + for i in 0..1_000 { + request = request.header(format!("x-{i}"), "v"); + } + let mut stream = send_request + .send_request(request.body(()).unwrap()) + .await + .unwrap(); + stream.finish().await.unwrap(); + let status = stream.recv_response().await.unwrap().status(); + assert_eq!( + status, + http::StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE, + "an h3 request past the field section limit must be refused, not served" + ); + + drive(driver); + let (status, _) = h3_request(&mut send_request, "/").await; + assert_eq!( + status, 200, + "the connection must outlive the refused request" + ); + shutdown.cancel(); + } + + #[tokio::test] + async fn an_early_data_enabled_endpoint_still_serves_through_the_zero_rtt_path() { + let app = Router::new().route("/", get(|| async { "ok" })); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Enabled); + + let client = client_endpoint(); + let mut send_request = h3_client_connect(&client, addr).await; + let (status, _) = h3_request(&mut send_request, "/").await; + assert_eq!( + status, 200, + "a server with early data enabled must serve requests through the 0-RTT acceptance path" + ); + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn the_first_request_on_a_completed_handshake_is_not_early_data() { + let safe = crate::zerortt::ZeroRttRoutes::new().get( + "/safe", + crate::zerortt::ZeroRttSafe::new(|| async { "safe" }), + ); + let full = crate::zerortt::RequiresFullHandshake::new( + Router::new().route("/write", get(|| async { "written" })), + ); + let (addr, shutdown) = spawn_h3_server( + crate::base_router(full, safe), + tls::test_support::limits(), + EarlyDataPolicy::Enabled, + ); + + for round in 0..150 { + let client = client_endpoint(); + let mut send_request = h3_client_connect(&client, addr).await; + let (status, _) = tokio::time::timeout( + Duration::from_secs(10), + h3_request(&mut send_request, "/write"), + ) + .await + .unwrap_or_else(|_| panic!("round {round}: the request was never answered")); + assert_eq!( + status, 200, + "round {round}: a fresh connection's first request came after its handshake, so it is not early data" + ); + } + shutdown.cancel(); + } + + #[tokio::test] + async fn a_classical_only_h3_client_completes_over_x25519() { + let app = Router::new().route("/", get(|| async { "ok" })); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Disabled); + + let client = client_endpoint_with_provider(tls::test_support::classical_only_provider()); + let mut send_request = h3_client_connect(&client, addr).await; + let (status, _) = h3_request(&mut send_request, "/").await; + assert_eq!( + status, 200, + "a QUIC client without ML-KEM must still complete the h3 handshake over classical X25519" + ); + shutdown.cancel(); + } + + #[tokio::test] + async fn an_in_flight_h3_request_finishes_during_drain() { + let app = Router::new().route( + "/slow", + get(|| async { + tokio::time::sleep(Duration::from_millis(300)).await; + "drained-clean" + }), + ); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Disabled); + + let client = client_endpoint(); + let mut send_request = h3_client_connect(&client, addr).await; + let ((status, body), ()) = tokio::join!(h3_request(&mut send_request, "/slow"), async { + tokio::time::sleep(Duration::from_millis(100)).await; + shutdown.cancel(); + }); + assert_eq!( + status, 200, + "an in-flight h3 request must complete through the graceful drain" + ); + assert_eq!(body, b"drained-clean"); + shutdown.cancel(); + } + + #[tokio::test] + async fn an_h3_connection_survives_repeated_client_path_migration() { + let app = Router::new().route("/echo", get(|| async { "migrated-clean" })); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Disabled); + + let client = client_endpoint(); + let send_request = h3_client_connect(&client, addr).await; + futures::stream::iter(0u8..3) + .fold( + (client, send_request), + |(client, mut send_request), round| async move { + client + .rebind(std::net::UdpSocket::bind("[::1]:0").unwrap()) + .unwrap(); + let (status, body) = h3_request(&mut send_request, "/echo").await; + assert_eq!( + status, 200, + "round {round}: a request after path migration must still be served" + ); + assert_eq!( + body, b"migrated-clean", + "round {round}: the migrated connection must deliver the response uncorrupted" + ); + (client, send_request) + }, + ) + .await; + + shutdown.cancel(); + } + + #[tokio::test] + async fn a_slow_h3_response_survives_the_idle_timeout() { + use std::num::{NonZeroU32, NonZeroU64}; + + let limits = ListenLimits::new( + crate::limits::HeaderTimeout::from_millis(NonZeroU64::new(1_000).unwrap()), + crate::limits::IdleTimeout::from_millis(NonZeroU64::new(2_000).unwrap()), + NonZeroU32::new(64).unwrap(), + ); + let app = Router::new().route( + "/", + get(|| async { + tokio::time::sleep(Duration::from_secs(3)).await; + "ok" + }), + ); + let (addr, shutdown) = spawn_h3_server(app, limits, EarlyDataPolicy::Disabled); + + let client = client_endpoint(); + let mut send_request = h3_client_connect(&client, addr).await; + let (status, body) = h3_request(&mut send_request, "/").await; + assert_eq!( + status, 200, + "keep-alive must hold the connection through a response slower than the idle timeout" + ); + assert_eq!(body, b"ok"); + shutdown.cancel(); + } +} diff --git a/knot2/crates/knot-edge/src/robustness.rs b/knot2/crates/knot-edge/src/robustness.rs new file mode 100644 index 0000000000000000000000000000000000000000..be8e3c7c0d0ec811c5c826e255a2fcd471f046bc --- /dev/null +++ b/knot2/crates/knot-edge/src/robustness.rs @@ -0,0 +1,763 @@ +use std::net::{IpAddr, SocketAddr}; +use std::num::{NonZeroU32, NonZeroU64}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use axum::Router; +use axum::body::Body; +use axum::error_handling::HandleErrorLayer; +use axum::extract::{ConnectInfo, State}; +use axum::middleware::{Next, from_fn_with_state}; +use axum::response::{IntoResponse, Response}; +use governor::middleware::NoOpMiddleware; +use http::{Method, Request, StatusCode}; +use knot_types::ProxyTrust; +use tokio_util::sync::CancellationToken; +use tower::limit::GlobalConcurrencyLimitLayer; +use tower::load_shed::LoadShedLayer; +use tower::{BoxError, ServiceBuilder}; +use tower_governor::GovernorLayer; +use tower_governor::errors::GovernorError; +use tower_governor::governor::{GovernorConfig, GovernorConfigBuilder}; +use tower_governor::key_extractor::KeyExtractor; +use tower_http::map_request_body::MapRequestBodyLayer; +use tower_http::timeout::{RequestBodyTimeoutLayer, TimeoutBody}; + +const NANOS_PER_SECOND: u64 = 1_000_000_000; +const CLEANUP_INTERVAL: Duration = Duration::from_secs(60); + +#[derive(Debug, Clone, Copy)] +pub struct RequestsPerSecond(NonZeroU32); + +impl RequestsPerSecond { + pub fn new(value: NonZeroU32) -> Self { + Self(value) + } + + fn period(self) -> Duration { + Duration::from_nanos((NANOS_PER_SECOND / u64::from(self.0.get())).max(1)) + } +} + +knot_types::scalar_newtype! { + pub struct BurstSize(NonZeroU32); + pub struct MaxInflightRequests(NonZeroU32); +} + +#[derive(Debug, Clone, Copy)] +pub struct RequestTimeout(Duration); + +impl RequestTimeout { + pub fn from_millis(millis: NonZeroU64) -> Self { + Self(Duration::from_millis(millis.get())) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct BodyInactivityTimeout(Duration); + +impl BodyInactivityTimeout { + pub fn from_millis(millis: NonZeroU64) -> Self { + Self(Duration::from_millis(millis.get())) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct WriteRequestTimeout(Duration); + +impl WriteRequestTimeout { + pub fn from_millis(millis: NonZeroU64) -> Self { + Self(Duration::from_millis(millis.get())) + } +} + +pub struct EdgeGuards { + rate: RequestsPerSecond, + burst: BurstSize, + max_inflight: MaxInflightRequests, + request_timeout: RequestTimeout, + body_timeout: BodyInactivityTimeout, + write_request_timeout: WriteRequestTimeout, + proxy_trust: ProxyTrust, +} + +impl EdgeGuards { + pub fn new( + rate: RequestsPerSecond, + burst: BurstSize, + max_inflight: MaxInflightRequests, + request_timeout: RequestTimeout, + body_timeout: BodyInactivityTimeout, + write_request_timeout: WriteRequestTimeout, + proxy_trust: ProxyTrust, + ) -> Self { + Self { + rate, + burst, + max_inflight, + request_timeout, + body_timeout, + write_request_timeout, + proxy_trust, + } + } + + pub(crate) fn prepare(self, shutdown: &CancellationToken) -> GuardLayers { + let governor = build_governor(self.rate, self.burst, self.proxy_trust); + spawn_state_cleanup(Arc::clone(&governor), shutdown.clone()); + GuardLayers { + governor, + max_inflight: self.max_inflight.0.get() as usize, + body_timeout: self.body_timeout.0, + timeout: TimeoutBudget { + standard: self.request_timeout.0, + extended: self.write_request_timeout.0, + }, + } + } +} + +type GuardGovernor = GovernorConfig; + +pub(crate) struct GuardLayers { + governor: Arc, + max_inflight: usize, + body_timeout: Duration, + timeout: TimeoutBudget, +} + +#[derive(Clone, Copy)] +struct TimeoutBudget { + standard: Duration, + extended: Duration, +} + +#[derive(Clone, Default)] +struct IgnoredHeaderNotice(Arc>); + +impl IgnoredHeaderNotice { + fn report(&self, peer: IpAddr) { + if self.0.set(peer).is_ok() { + tracing::warn!( + %peer, + "a peer outside xrpc.trusted_proxies sent xrpc.trusted_proxy_header, so the knot ignored the header and rate-limits that peer by the address it connected from. Add this address to xrpc.trusted_proxies if it is the reverse proxy, since a proxy reaches the knot over one address family and listing the other one silently loses the header. This warning reports the first such peer only." + ); + } + } +} + +#[derive(Clone)] +struct ProxyAwareIp { + trust: ProxyTrust, + ignored_header: IgnoredHeaderNotice, +} + +impl KeyExtractor for ProxyAwareIp { + type Key = IpAddr; + + fn extract(&self, request: &Request) -> Result { + let socket = request + .extensions() + .get::>() + .map(|info| info.0.ip()); + let key = self.trust.peer_key(request.headers(), socket); + if let Some(peer) = key.ignored_header() { + self.ignored_header.report(peer); + } + key.address().ok_or(GovernorError::UnableToExtractKey) + } +} + +fn build_governor( + rate: RequestsPerSecond, + burst: BurstSize, + proxy_trust: ProxyTrust, +) -> Arc { + let mut builder = GovernorConfigBuilder::default(); + builder.period(rate.period()).burst_size(burst.0.get()); + let config = builder + .key_extractor(ProxyAwareIp { + trust: proxy_trust, + ignored_header: IgnoredHeaderNotice::default(), + }) + .finish() + .expect("a non-zero rate period and burst size always yield a governor config"); + Arc::new(config) +} + +fn spawn_state_cleanup(governor: Arc, shutdown: CancellationToken) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(CLEANUP_INTERVAL); + ticker.tick().await; + loop { + tokio::select! { + () = shutdown.cancelled() => break, + _ = ticker.tick() => governor.limiter().retain_recent(), + } + } + }); +} + +async fn shed_overloaded(_error: BoxError) -> StatusCode { + StatusCode::SERVICE_UNAVAILABLE +} + +async fn apply_request_timeout( + State(budget): State, + request: Request, + next: Next, +) -> Response { + let limit = match is_streaming_write(&request) { + true => budget.extended, + false => budget.standard, + }; + match tokio::time::timeout(limit, next.run(request)).await { + Ok(response) => response, + Err(_) => StatusCode::REQUEST_TIMEOUT.into_response(), + } +} + +fn is_streaming_write(request: &Request) -> bool { + let path = request.uri().path(); + match *request.method() { + Method::POST => path.ends_with("/git-receive-pack"), + Method::PUT => path.contains("/info/lfs/objects/"), + _ => false, + } +} + +fn rewrap_body(body: TimeoutBody) -> axum::body::Body { + axum::body::Body::new(body) +} + +pub(crate) fn apply(router: Router, layers: GuardLayers) -> Router { + let GuardLayers { + governor, + max_inflight, + body_timeout, + timeout, + } = layers; + let rate_limit: GovernorLayer = + GovernorLayer::new(governor); + router + .layer( + ServiceBuilder::new() + .layer(RequestBodyTimeoutLayer::new(body_timeout)) + .layer(MapRequestBodyLayer::new(rewrap_body)), + ) + .layer(from_fn_with_state(timeout, apply_request_timeout)) + .layer( + ServiceBuilder::new() + .layer(HandleErrorLayer::new(shed_overloaded)) + .layer(LoadShedLayer::new()) + .layer(GlobalConcurrencyLimitLayer::new(max_inflight)), + ) + .layer(rate_limit) +} + +#[cfg(test)] +mod tests { + use super::*; + + use axum::body::{Body, Bytes}; + use axum::routing::{get, post}; + use futures::StreamExt; + use http::StatusCode; + use tower::ServiceExt; + + fn guards( + rate: u32, + burst: u32, + inflight: u32, + request_timeout_ms: u64, + body_timeout_ms: u64, + proxy_trust: ProxyTrust, + ) -> EdgeGuards { + guards_with_write( + rate, + burst, + inflight, + request_timeout_ms, + body_timeout_ms, + request_timeout_ms, + proxy_trust, + ) + } + + #[allow(clippy::too_many_arguments)] + fn guards_with_write( + rate: u32, + burst: u32, + inflight: u32, + request_timeout_ms: u64, + body_timeout_ms: u64, + write_request_timeout_ms: u64, + proxy_trust: ProxyTrust, + ) -> EdgeGuards { + EdgeGuards::new( + RequestsPerSecond::new(NonZeroU32::new(rate).unwrap()), + BurstSize::new(NonZeroU32::new(burst).unwrap()), + MaxInflightRequests::new(NonZeroU32::new(inflight).unwrap()), + RequestTimeout::from_millis(NonZeroU64::new(request_timeout_ms).unwrap()), + BodyInactivityTimeout::from_millis(NonZeroU64::new(body_timeout_ms).unwrap()), + WriteRequestTimeout::from_millis(NonZeroU64::new(write_request_timeout_ms).unwrap()), + proxy_trust, + ) + } + + fn forwarded_for() -> http::HeaderName { + http::HeaderName::from_static("x-forwarded-for") + } + + fn trusting_any_peer() -> ProxyTrust { + ProxyTrust::new(Some(forwarded_for()), knot_types::TrustedProxies::default()) + } + + fn trusting_loopback() -> ProxyTrust { + ProxyTrust::new( + Some(forwarded_for()), + knot_types::TrustedProxies::parse(["127.0.0.1"]).unwrap(), + ) + } + + fn extractor(trust: ProxyTrust) -> ProxyAwareIp { + ProxyAwareIp { + trust, + ignored_header: IgnoredHeaderNotice::default(), + } + } + + fn guarded_router(router: Router, guards: EdgeGuards) -> Router { + apply(router, guards.prepare(&CancellationToken::new())) + } + + fn from_peer(request: Request, host: u8) -> Request { + let mut request = request; + request + .extensions_mut() + .insert(ConnectInfo(SocketAddr::from(([127, 0, 0, host], 47000)))); + request + } + + fn get_request() -> Request { + Request::get("/").body(Body::empty()).unwrap() + } + + #[test] + fn the_extractor_keys_on_the_trusted_proxy_header_when_configured() { + let extractor = extractor(trusting_any_peer()); + let request = Request::get("/") + .header("x-forwarded-for", "203.0.113.7, 198.51.100.4") + .extension(ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 5000)))) + .body(()) + .unwrap(); + assert_eq!( + extractor.extract(&request).unwrap(), + "198.51.100.4".parse::().unwrap(), + "the rightmost forwarded entry is the client the proxy appended" + ); + } + + #[test] + fn the_extractor_ignores_a_forgeable_header_when_no_proxy_is_trusted() { + let extractor = extractor(ProxyTrust::default()); + let request = Request::get("/") + .header("x-forwarded-for", "203.0.113.7") + .extension(ConnectInfo(SocketAddr::from(([10, 0, 0, 9], 5000)))) + .body(()) + .unwrap(); + assert_eq!( + extractor.extract(&request).unwrap(), + "10.0.0.9".parse::().unwrap(), + "with no trusted proxy the socket peer wins over a client-forgeable header" + ); + } + + #[test] + fn the_extractor_keys_an_unlisted_peer_on_its_socket_however_it_fills_the_header() { + let extractor = extractor(trusting_loopback()); + let forged = |host| { + Request::get("/") + .header("x-forwarded-for", "198.51.100.4") + .extension(ConnectInfo(SocketAddr::from(([127, 0, 0, host], 5000)))) + .body(()) + .unwrap() + }; + assert_eq!( + extractor.extract(&forged(1)).unwrap(), + "198.51.100.4".parse::().unwrap(), + "the listed proxy relayed this one, so the extractor keys on the header address" + ); + assert_eq!( + extractor.extract(&forged(9)).unwrap(), + "127.0.0.9".parse::().unwrap(), + "an unlisted peer picked its own token bucket by forging the header" + ); + } + + #[test] + fn the_extractor_falls_back_to_the_peer_when_the_trusted_header_is_absent() { + let extractor = extractor(trusting_any_peer()); + let request = Request::get("/") + .extension(ConnectInfo(SocketAddr::from(([10, 0, 0, 9], 5000)))) + .body(()) + .unwrap(); + assert_eq!( + extractor.extract(&request).unwrap(), + "10.0.0.9".parse::().unwrap() + ); + } + + #[test] + fn the_extractor_fails_when_no_peer_can_be_identified() { + let extractor = extractor(ProxyTrust::default()); + let request = Request::get("/").body(()).unwrap(); + assert!(matches!( + extractor.extract(&request), + Err(GovernorError::UnableToExtractKey) + )); + } + + #[test] + fn a_missing_connect_info_fails_closed_rather_than_taking_the_header_on_trust() { + let listed = extractor(trusting_loopback()); + let headed = || { + Request::get("/") + .header("x-forwarded-for", "198.51.100.4") + .body(()) + .unwrap() + }; + assert!( + matches!( + listed.extract(&headed()), + Err(GovernorError::UnableToExtractKey) + ), + "with no socket to match against the allowlist the extractor identifies no client" + ); + assert_eq!( + extractor(trusting_any_peer()).extract(&headed()).unwrap(), + "198.51.100.4".parse::().unwrap(), + "an operator who lists no proxy already told the knot to take the header from anyone" + ); + } + + #[test] + fn the_first_unlisted_peer_sending_the_header_is_reported_once() { + let extractor = extractor(trusting_loopback()); + let forged = |host| { + Request::get("/") + .header("x-forwarded-for", "198.51.100.4") + .extension(ConnectInfo(SocketAddr::from(([203, 0, 113, host], 5000)))) + .body(()) + .unwrap() + }; + extractor.extract(&forged(7)).unwrap(); + extractor.extract(&forged(9)).unwrap(); + assert_eq!( + extractor.ignored_header.0.get(), + Some(&"203.0.113.7".parse::().unwrap()), + "a wrong-family allowlist sends every request down this path, so only the first peer is reported" + ); + } + + #[test] + fn a_listed_proxy_and_a_headerless_request_report_nothing() { + let listed = extractor(trusting_loopback()); + listed + .extract( + &Request::get("/") + .header("x-forwarded-for", "198.51.100.4") + .extension(ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 5000)))) + .body(()) + .unwrap(), + ) + .unwrap(); + listed + .extract( + &Request::get("/") + .extension(ConnectInfo(SocketAddr::from(([203, 0, 113, 7], 5000)))) + .body(()) + .unwrap(), + ) + .unwrap(); + assert_eq!( + listed.ignored_header.0.get(), + None, + "neither a relayed request or a request without the header says anything about the allowlist" + ); + } + + #[tokio::test] + async fn a_well_behaved_request_passes_every_guard() { + let app = guarded_router( + Router::new().route("/", get(|| async { "ok" })), + guards(50, 200, 1_024, 60_000, 30_000, ProxyTrust::default()), + ); + let status = app + .oneshot(from_peer(get_request(), 1)) + .await + .unwrap() + .status(); + assert_eq!(status, StatusCode::OK); + } + + #[tokio::test] + async fn a_burst_beyond_the_per_ip_limit_is_rejected_with_429() { + let app = guarded_router( + Router::new().route("/", get(|| async { "ok" })), + guards(1, 2, 1_024, 60_000, 30_000, ProxyTrust::default()), + ); + let first = app + .clone() + .oneshot(from_peer(get_request(), 7)) + .await + .unwrap() + .status(); + let second = app + .clone() + .oneshot(from_peer(get_request(), 7)) + .await + .unwrap() + .status(); + let third = app + .clone() + .oneshot(from_peer(get_request(), 7)) + .await + .unwrap() + .status(); + let other_ip = app + .clone() + .oneshot(from_peer(get_request(), 8)) + .await + .unwrap() + .status(); + assert_eq!(first, StatusCode::OK); + assert_eq!(second, StatusCode::OK); + assert_eq!( + third, + StatusCode::TOO_MANY_REQUESTS, + "a third request inside the window exhausts the burst for this IP" + ); + assert_eq!( + other_ip, + StatusCode::OK, + "a different IP keeps its own independent quota" + ); + } + + fn browser_edge(router: Router, guards: EdgeGuards) -> Router { + crate::cors::origin_header(guarded_router(crate::cors::preflight(router), guards)) + } + + fn preflight(method: &str, headers: &str) -> Request { + Request::options("/") + .header(http::header::ORIGIN, "https://tangled.org") + .header(http::header::ACCESS_CONTROL_REQUEST_METHOD, method) + .header(http::header::ACCESS_CONTROL_REQUEST_HEADERS, headers) + .body(Body::empty()) + .unwrap() + } + + fn from_tangled_org() -> Request { + Request::get("/") + .header(http::header::ORIGIN, "https://tangled.org") + .body(Body::empty()) + .unwrap() + } + + #[tokio::test] + async fn the_cors_layer_answers_a_full_preflight_from_under_the_governor() { + let app = browser_edge( + Router::new().route("/", get(|| async { "ok" })), + guards(1, 1, 1_024, 60_000, 30_000, ProxyTrust::default()), + ); + let ask = || from_peer(preflight("POST", "authorization,content-type"), 12); + let answer = app.clone().oneshot(ask()).await.unwrap(); + assert!(answer.status().is_success(), "got {:?}", answer.status()); + let allowed = answer.headers()[http::header::ACCESS_CONTROL_ALLOW_HEADERS] + .to_str() + .map(str::to_ascii_lowercase) + .unwrap(); + assert!( + allowed.contains("authorization") && allowed.contains("content-type"), + "every authed xrpc call sends both headers: allowed {allowed}" + ); + assert_eq!( + answer.headers()[http::header::ACCESS_CONTROL_MAX_AGE], + "86400", + "without a max-age a browser re-preflights and every call costs two tokens" + ); + assert_eq!( + app.oneshot(ask()).await.unwrap().status(), + StatusCode::TOO_MANY_REQUESTS, + "an unmetered OPTIONS would bypass every guard" + ); + } + + #[tokio::test] + async fn the_allow_origin_header_outlives_a_handler_403_and_a_governor_429() { + let app = browser_edge( + Router::new().route("/", get(|| async { StatusCode::FORBIDDEN })), + guards(1, 1, 1_024, 60_000, 30_000, ProxyTrust::default()), + ); + let ask = || from_peer(from_tangled_org(), 13); + let denied = app.clone().oneshot(ask()).await.unwrap(); + let refused = app.oneshot(ask()).await.unwrap(); + assert_eq!(denied.status(), StatusCode::FORBIDDEN); + assert_eq!(refused.status(), StatusCode::TOO_MANY_REQUESTS); + [denied, refused].into_iter().for_each(|answer| { + assert_eq!( + answer.headers()[http::header::ACCESS_CONTROL_ALLOW_ORIGIN], + "*", + "the frontend prints the {} itself, since this header is set outside the governor", + answer.status() + ); + }); + } + + #[tokio::test] + async fn requests_beyond_the_inflight_limit_are_shed_with_503() { + let app = guarded_router( + Router::new().route( + "/slow", + get(|| async { + tokio::time::sleep(Duration::from_millis(300)).await; + "ok" + }), + ), + guards(10_000, 10_000, 1, 60_000, 30_000, ProxyTrust::default()), + ); + let holder = { + let app = app.clone(); + tokio::spawn(async move { + let request = from_peer(Request::get("/slow").body(Body::empty()).unwrap(), 1); + app.oneshot(request).await.unwrap().status() + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + let shed = app + .clone() + .oneshot(from_peer( + Request::get("/slow").body(Body::empty()).unwrap(), + 2, + )) + .await + .unwrap() + .status(); + assert_eq!( + shed, + StatusCode::SERVICE_UNAVAILABLE, + "with the single inflight slot held, the next request sheds rather than queues" + ); + assert_eq!(holder.await.unwrap(), StatusCode::OK); + } + + #[tokio::test] + async fn a_request_slower_than_the_timeout_is_cut_with_408() { + let app = guarded_router( + Router::new().route( + "/slow", + get(|| async { + tokio::time::sleep(Duration::from_millis(500)).await; + "ok" + }), + ), + guards(10_000, 10_000, 1_024, 80, 30_000, ProxyTrust::default()), + ); + let status = app + .oneshot(from_peer( + Request::get("/slow").body(Body::empty()).unwrap(), + 1, + )) + .await + .unwrap() + .status(); + assert_eq!(status, StatusCode::REQUEST_TIMEOUT); + } + + #[tokio::test] + async fn a_streaming_write_runs_under_the_extended_budget_while_reads_keep_the_standard_one() { + let app = guarded_router( + Router::new() + .route( + "/did/name/git-receive-pack", + post(|| async { + tokio::time::sleep(Duration::from_millis(200)).await; + "ok" + }), + ) + .route( + "/did/name/git-upload-pack", + post(|| async { + tokio::time::sleep(Duration::from_millis(200)).await; + "ok" + }), + ), + guards_with_write( + 10_000, + 10_000, + 1_024, + 80, + 30_000, + 5_000, + ProxyTrust::default(), + ), + ); + let push = app + .clone() + .oneshot(from_peer( + Request::post("/did/name/git-receive-pack") + .body(Body::empty()) + .unwrap(), + 1, + )) + .await + .unwrap() + .status(); + assert_eq!( + push, + StatusCode::OK, + "a push slower than the standard timeout survives on the extended write budget" + ); + let fetch = app + .oneshot(from_peer( + Request::post("/did/name/git-upload-pack") + .body(Body::empty()) + .unwrap(), + 1, + )) + .await + .unwrap() + .status(); + assert_eq!( + fetch, + StatusCode::REQUEST_TIMEOUT, + "a non-write request past the standard timeout is still cut" + ); + } + + #[tokio::test] + async fn a_stalled_request_body_is_cut_and_never_hangs() { + let app = guarded_router( + Router::new().route("/upload", post(|_body: Bytes| async { "ok" })), + guards(10_000, 10_000, 1_024, 60_000, 80, ProxyTrust::default()), + ); + let body = Body::from_stream( + futures::stream::once(async { + Ok::<_, std::io::Error>(Bytes::from_static(b"partial")) + }) + .chain(futures::stream::pending::>()), + ); + let request = from_peer(Request::post("/upload").body(body).unwrap(), 1); + let status = tokio::time::timeout(Duration::from_secs(5), app.oneshot(request)) + .await + .expect("the body inactivity timeout must cut a stalled upload instead of hanging") + .unwrap() + .status(); + assert_ne!( + status, + StatusCode::OK, + "a body that stalls past the inactivity timeout is never accepted" + ); + } +} diff --git a/knot2/crates/knot-edge/src/tcp.rs b/knot2/crates/knot-edge/src/tcp.rs new file mode 100644 index 0000000000000000000000000000000000000000..bbe4ed8c71c94cb698508bed5616498b22b4d79a --- /dev/null +++ b/knot2/crates/knot-edge/src/tcp.rs @@ -0,0 +1,521 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use axum::Router; +use axum::body::Body; +use hyper::body::Incoming; +use hyper::service::service_fn; +use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer}; +use hyper_util::server::conn::auto::Builder; +use rustls::ServerConfig; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::Semaphore; +use tokio_rustls::TlsAcceptor; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; +use tower::{Service, ServiceExt}; + +use crate::limits::ListenLimits; + +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); +const CONNECTION_DRAIN_GRACE: Duration = Duration::from_secs(10); +const LISTENER_DRAIN_GRACE: Duration = Duration::from_secs(30); +const ACCEPT_BACKOFF: Duration = Duration::from_millis(250); + +pub async fn serve_plaintext( + listener: TcpListener, + router: Router, + limits: ListenLimits, + shutdown: CancellationToken, +) -> std::io::Result<()> { + run_listener(listener, router, limits, shutdown, |stream| async move { + Some(stream) + }) + .await +} + +pub async fn serve_tls( + listener: TcpListener, + router: Router, + server_config: Arc, + limits: ListenLimits, + shutdown: CancellationToken, +) -> std::io::Result<()> { + let acceptor = TlsAcceptor::from(server_config); + run_listener(listener, router, limits, shutdown, move |stream| { + let acceptor = acceptor.clone(); + async move { + match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await { + Ok(Ok(tls_stream)) => Some(tls_stream), + Ok(Err(error)) => { + tracing::debug!("tls handshake failed: {error}"); + None + } + Err(_) => { + tracing::debug!("tls handshake timed out after {HANDSHAKE_TIMEOUT:?}"); + None + } + } + } + }) + .await +} + +async fn run_listener( + listener: TcpListener, + router: Router, + limits: ListenLimits, + shutdown: CancellationToken, + upgrade: Upgrade, +) -> std::io::Result<()> +where + IO: AsyncRead + AsyncWrite + Unpin + Send + 'static, + Upgrade: Fn(TcpStream) -> Fut + Send + Sync + 'static, + Fut: std::future::Future> + Send + 'static, +{ + let slots = Arc::new(Semaphore::new(limits.max_connections())); + let mut make_service = router.into_make_service_with_connect_info::(); + let tracker = TaskTracker::new(); + let upgrade = Arc::new(upgrade); + + loop { + let accepted = tokio::select! { + () = shutdown.cancelled() => break, + accepted = listener.accept() => accepted, + }; + let (stream, peer) = match accepted { + Ok(pair) => pair, + Err(error) if is_connection_error(&error) => continue, + Err(error) => { + tracing::warn!("tcp accept failed, backing off: {error}"); + tokio::select! { + () = shutdown.cancelled() => break, + () = tokio::time::sleep(ACCEPT_BACKOFF) => {} + } + continue; + } + }; + let Ok(slot) = Arc::clone(&slots).try_acquire_owned() else { + continue; + }; + let service = match make_service.call(peer).await { + Ok(service) => service, + Err(never) => match never {}, + }; + let header_timeout = limits.header_timeout().get(); + let conn_shutdown = shutdown.clone(); + let upgrade = Arc::clone(&upgrade); + tracker.spawn(async move { + let _slot = slot; + let Some(io) = upgrade(stream).await else { + return; + }; + let hyper_service = service_fn(move |request: hyper::Request| { + service.clone().oneshot(request.map(Body::new)) + }); + let budget = limits.connection_budget(); + let mut builder = Builder::new(TokioExecutor::new()); + builder + .http1() + .timer(TokioTimer::new()) + .header_read_timeout(header_timeout); + builder + .http2() + .timer(TokioTimer::new()) + .max_concurrent_streams(budget.max_concurrent_streams().get()) + .initial_stream_window_size(budget.stream_receive_window()) + .initial_connection_window_size(budget.connection_receive_window()) + .keep_alive_interval(Some(header_timeout)) + .keep_alive_timeout(header_timeout); + let connection = + builder.serve_connection_with_upgrades(TokioIo::new(io), hyper_service); + tokio::pin!(connection); + + tokio::select! { + served = connection.as_mut() => drop(served), + () = conn_shutdown.cancelled() => { + connection.as_mut().graceful_shutdown(); + let _ = tokio::time::timeout(CONNECTION_DRAIN_GRACE, connection.as_mut()).await; + } + } + }); + } + + tracker.close(); + let _ = tokio::time::timeout(LISTENER_DRAIN_GRACE, tracker.wait()).await; + Ok(()) +} + +fn is_connection_error(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::ConnectionReset + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::num::{NonZeroU32, NonZeroU64}; + + use axum::routing::get; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpStream; + + fn limits(header_timeout_ms: u64, max_connections: u32) -> ListenLimits { + ListenLimits::new( + crate::limits::HeaderTimeout::from_millis(NonZeroU64::new(header_timeout_ms).unwrap()), + crate::limits::IdleTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + NonZeroU32::new(max_connections).unwrap(), + ) + } + + async fn bind_and_serve(limits: ListenLimits) -> (SocketAddr, CancellationToken) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = Router::new().route("/", get(|| async { "ok" })); + let shutdown = CancellationToken::new(); + tokio::spawn(serve_plaintext(listener, router, limits, shutdown.clone())); + (addr, shutdown) + } + + async fn read_to_close(stream: &mut TcpStream) -> Vec { + let mut collected = Vec::new(); + stream.read_to_end(&mut collected).await.unwrap(); + collected + } + + #[tokio::test] + async fn a_full_request_is_answered() { + let (addr, _shutdown) = bind_and_serve(limits(5_000, 4)).await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream + .write_all(b"GET / HTTP/1.1\r\nhost: oyster.cafe\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + let answer = read_to_close(&mut stream).await; + let head = String::from_utf8_lossy(&answer); + assert!(head.starts_with("HTTP/1.1 200"), "got: {head}"); + } + + #[tokio::test] + async fn a_slowloris_connection_is_cut_at_the_header_timeout() { + let (addr, _shutdown) = bind_and_serve(limits(200, 4)).await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream.write_all(b"GET / HTT").await.unwrap(); + let closed = tokio::time::timeout(Duration::from_secs(5), read_to_close(&mut stream)) + .await + .expect("server cuts connection instead of waiting forever"); + let head = String::from_utf8_lossy(&closed); + assert!( + !head.contains("200"), + "half-sent request must never be answered, got: {head}" + ); + } + + #[tokio::test] + async fn an_idle_keep_alive_connection_is_cut_at_the_header_timeout() { + let (addr, _shutdown) = bind_and_serve(limits(200, 4)).await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream + .write_all(b"GET / HTTP/1.1\r\nhost: oyster.cafe\r\n\r\n") + .await + .unwrap(); + let answer = tokio::time::timeout(Duration::from_secs(5), read_to_close(&mut stream)) + .await + .expect("idle keep-alive connection is cut after answered request"); + let head = String::from_utf8_lossy(&answer); + assert!(head.starts_with("HTTP/1.1 200"), "got: {head}"); + } + + #[tokio::test] + async fn a_connection_beyond_the_limit_is_refused() { + let (addr, _shutdown) = bind_and_serve(limits(5_000, 1)).await; + let mut held = TcpStream::connect(addr).await.unwrap(); + held.write_all(b"GET / HTT").await.unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + + let mut refused = TcpStream::connect(addr).await.unwrap(); + refused.write_all(b"GET / HTT").await.unwrap(); + let mut answer = Vec::new(); + let outcome = + tokio::time::timeout(Duration::from_secs(1), refused.read_to_end(&mut answer)) + .await + .expect("over-limit connection is dropped at accept instead of held to timeout"); + match outcome { + Ok(_) => assert!( + answer.is_empty(), + "over-limit connection gets no bytes, got: {}", + String::from_utf8_lossy(&answer) + ), + Err(reset) => assert_eq!(reset.kind(), std::io::ErrorKind::ConnectionReset), + } + } + + #[tokio::test] + async fn an_in_flight_request_finishes_during_drain() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = Router::new().route( + "/slow", + get(|| async { + tokio::time::sleep(Duration::from_millis(300)).await; + "drained-clean" + }), + ); + let shutdown = CancellationToken::new(); + tokio::spawn(serve_plaintext( + listener, + router, + limits(5_000, 4), + shutdown.clone(), + )); + + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream + .write_all(b"GET /slow HTTP/1.1\r\nhost: oyster.cafe\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + + tokio::time::sleep(Duration::from_millis(100)).await; + shutdown.cancel(); + + let answer = tokio::time::timeout(Duration::from_secs(5), read_to_close(&mut stream)) + .await + .expect("an in-flight request is answered through the graceful drain"); + let text = String::from_utf8_lossy(&answer); + assert!(text.starts_with("HTTP/1.1 200"), "got: {text}"); + assert!( + text.trim_end().ends_with("drained-clean"), + "the in-flight response must complete during drain, got: {text}" + ); + } + + #[tokio::test] + async fn cancelling_the_token_stops_the_listener() { + let (addr, shutdown) = bind_and_serve(limits(5_000, 4)).await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream + .write_all(b"GET / HTTP/1.1\r\nhost: oyster.cafe\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + let _ = read_to_close(&mut stream).await; + + shutdown.cancel(); + tokio::time::sleep(Duration::from_millis(100)).await; + let refused = TcpStream::connect(addr).await; + if let Ok(mut late) = refused { + late.write_all(b"GET / HTTP/1.1\r\nhost: oyster.cafe\r\nconnection: close\r\n\r\n") + .await + .ok(); + let mut answer = Vec::new(); + let _ = + tokio::time::timeout(Duration::from_secs(1), late.read_to_end(&mut answer)).await; + assert!( + !String::from_utf8_lossy(&answer).contains("200"), + "a drained listener mustn't answer new requests" + ); + } + } +} + +#[cfg(test)] +mod tls_tests { + use super::*; + + use std::num::{NonZeroU32, NonZeroU64}; + + use axum::routing::get; + use futures::StreamExt; + use rustls::NamedGroup; + use rustls::crypto::aws_lc_rs; + use rustls::pki_types::ServerName; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpStream; + use tokio_rustls::TlsConnector; + + use crate::tls; + + fn client(alpn: &[&[u8]]) -> TlsConnector { + client_with_provider(aws_lc_rs::default_provider(), alpn) + } + + fn client_with_provider( + provider: rustls::crypto::CryptoProvider, + alpn: &[&[u8]], + ) -> TlsConnector { + let mut config = rustls::ClientConfig::builder_with_provider(Arc::new(provider)) + .with_safe_default_protocol_versions() + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(tls::test_support::AcceptAnyServerCert)) + .with_no_client_auth(); + config.alpn_protocols = alpn.iter().map(|p| p.to_vec()).collect(); + TlsConnector::from(Arc::new(config)) + } + + async fn spawn_tls_server( + resolver: Arc, + limits: ListenLimits, + ) -> (SocketAddr, CancellationToken) { + let server_config = Arc::new(tls::build_tls_server_config(resolver, &[]).unwrap()); + let listener = TcpListener::bind("[::1]:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = Router::new().route("/", get(|| async { "ok" })); + let shutdown = CancellationToken::new(); + tokio::spawn(serve_tls( + listener, + router, + server_config, + limits, + shutdown.clone(), + )); + (addr, shutdown) + } + + async fn serve( + alpn_offer: &[&[u8]], + ) -> ( + SocketAddr, + CancellationToken, + tokio_rustls::client::TlsStream, + ) { + let (addr, shutdown) = + spawn_tls_server(tls::test_support::resolver(), tls::test_support::limits()).await; + let tcp = TcpStream::connect(addr).await.unwrap(); + let name = ServerName::try_from("localhost").unwrap(); + let tls = client(alpn_offer).connect(name, tcp).await.unwrap(); + (addr, shutdown, tls) + } + + #[tokio::test] + async fn it_terminates_tls_over_http1_and_negotiates_post_quantum() { + let (_addr, shutdown, mut tls) = serve(&[b"http/1.1"]).await; + + let group = tls.get_ref().1.negotiated_key_exchange_group().unwrap(); + assert_eq!( + group.name(), + NamedGroup::X25519MLKEM768, + "prefer-post-quantum must select X25519MLKEM768 with a capable client" + ); + assert_eq!( + tls.get_ref().1.alpn_protocol(), + Some(b"http/1.1".as_slice()) + ); + + tls.write_all(b"GET / HTTP/1.1\r\nhost: localhost\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + let mut response = Vec::new(); + tls.read_to_end(&mut response).await.unwrap(); + let text = String::from_utf8_lossy(&response); + assert!(text.starts_with("HTTP/1.1 200"), "got: {text}"); + assert!(text.trim_end().ends_with("ok"), "got: {text}"); + + shutdown.cancel(); + } + + #[tokio::test] + async fn it_negotiates_h2_when_the_client_offers_only_h2() { + let (_addr, shutdown, tls) = serve(&[b"h2"]).await; + assert_eq!(tls.get_ref().1.alpn_protocol(), Some(b"h2".as_slice())); + shutdown.cancel(); + } + + #[tokio::test] + async fn a_classical_only_client_completes_over_x25519() { + let (addr, shutdown) = + spawn_tls_server(tls::test_support::resolver(), tls::test_support::limits()).await; + + let tcp = TcpStream::connect(addr).await.unwrap(); + let name = ServerName::try_from("localhost").unwrap(); + let tls = + client_with_provider(tls::test_support::classical_only_provider(), &[b"http/1.1"]) + .connect(name, tcp) + .await + .unwrap(); + + let group = tls.get_ref().1.negotiated_key_exchange_group().unwrap(); + assert_eq!( + group.name(), + NamedGroup::X25519, + "a client without ML-KEM must still complete the handshake over classical X25519" + ); + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread")] + async fn concurrent_handshakes_survive_a_cert_reload_under_load() { + let high_limit = ListenLimits::new( + crate::limits::HeaderTimeout::from_millis(NonZeroU64::new(5_000).unwrap()), + crate::limits::IdleTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + NonZeroU32::new(512).unwrap(), + ); + let resolver = tls::test_support::resolver(); + let (addr, shutdown) = spawn_tls_server(resolver.clone(), high_limit).await; + + let churn = { + let resolver = resolver.clone(); + let stop = shutdown.clone(); + tokio::spawn(async move { + futures::stream::unfold(0u32, move |swaps| { + let resolver = resolver.clone(); + let stop = stop.clone(); + async move { + match stop.is_cancelled() { + true => None, + false => { + resolver.store(tls::test_support::self_signed()); + tokio::time::sleep(Duration::from_millis(1)).await; + Some((swaps + 1, swaps + 1)) + } + } + } + }) + .fold(0u32, |_, swaps| async move { swaps }) + .await + }) + }; + + let clients: Vec<_> = (0..48) + .map(|_| { + tokio::spawn(async move { + let tcp = TcpStream::connect(addr).await.unwrap(); + let name = ServerName::try_from("localhost").unwrap(); + let mut tls = client(&[b"http/1.1"]).connect(name, tcp).await.unwrap(); + tls.write_all( + b"GET / HTTP/1.1\r\nhost: localhost\r\nconnection: close\r\n\r\n", + ) + .await + .unwrap(); + let mut response = Vec::new(); + tls.read_to_end(&mut response).await.unwrap(); + let text = String::from_utf8_lossy(&response).into_owned(); + text.starts_with("HTTP/1.1 200") && text.trim_end().ends_with("ok") + }) + }) + .collect(); + + let outcomes: Vec = futures::future::join_all(clients) + .await + .into_iter() + .map(|joined| joined.unwrap()) + .collect(); + assert!( + outcomes.iter().all(|served| *served), + "every handshake racing a cert reload must complete and serve the router uncorrupted" + ); + + shutdown.cancel(); + assert!( + churn.await.unwrap() > 1, + "the resolver must have reloaded the certificate during the load" + ); + } +} diff --git a/knot2/crates/knot-edge/src/tls.rs b/knot2/crates/knot-edge/src/tls.rs new file mode 100644 index 0000000000000000000000000000000000000000..e1e7d5af28fd9335f2c6ce712aa1e6fc4cb35d00 --- /dev/null +++ b/knot2/crates/knot-edge/src/tls.rs @@ -0,0 +1,679 @@ +use std::io::BufReader; +use std::path::Path; +use std::sync::Arc; + +use arc_swap::ArcSwap; +use base64::Engine; +use quinn::crypto::rustls::QuicServerConfig; +use rustls::RootCertStore; +use rustls::ServerConfig; +use rustls::crypto::aws_lc_rs; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, UnixTime}; +use rustls::server::danger::{ClientCertVerified, ClientCertVerifier}; +use rustls::server::{ClientHello, ResolvesServerCert, WebPkiClientVerifier}; +use rustls::sign::CertifiedKey; +use sha2::{Digest, Sha256}; +use tokio_util::sync::CancellationToken; + +use crate::limits::ListenLimits; +use crate::zerortt::EarlyDataPolicy; + +pub const ACME_TLS_ALPN: &[u8] = rustls_acme::acme::ACME_TLS_ALPN_NAME; + +#[derive(Debug, thiserror::Error)] +pub enum TlsError { + #[error("reading {path}: {source}")] + Read { + path: String, + source: std::io::Error, + }, + #[error("parsing {path}: {message}")] + Parse { path: String, message: String }, + #[error("no certificates found in {0}")] + NoCertificates(String), + #[error("no private key found in {0}")] + NoPrivateKey(String), + #[error("unusable private key: {0}")] + SigningKey(String), + #[error("building server config: {0}")] + Config(String), + #[error("certificate and private key don't match: {0}")] + KeyMismatch(String), + #[error("session ticketer: {0}")] + Ticketer(String), + #[error("client certificate verifier for {path}: {message}")] + ClientVerifier { path: String, message: String }, + #[error("admin SPKI pin: {0}")] + SpkiPin(String), +} + +#[derive(Clone)] +pub struct SpkiPin([u8; 32]); + +impl PartialEq for SpkiPin { + fn eq(&self, other: &Self) -> bool { + self.0 + .iter() + .zip(other.0.iter()) + .fold(0u8, |acc, (left, right)| acc | (left ^ right)) + == 0 + } +} + +impl Eq for SpkiPin {} + +impl SpkiPin { + pub fn from_base64(encoded: &str) -> Result { + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded.trim()) + .map_err(|error| TlsError::SpkiPin(error.to_string()))?; + let array: [u8; 32] = bytes.try_into().map_err(|bytes: Vec| { + TlsError::SpkiPin(format!("expected 32 bytes, got {}", bytes.len())) + })?; + Ok(Self(array)) + } + + fn of_certificate(cert: &CertificateDer<'_>) -> Result { + let (_, parsed) = x509_parser::parse_x509_certificate(cert.as_ref()).map_err(|error| { + rustls::Error::General(format!("parse client certificate: {error}")) + })?; + Ok(Self( + Sha256::digest(parsed.tbs_certificate.subject_pki.raw).into(), + )) + } +} + +impl std::fmt::Debug for SpkiPin { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("SpkiPin").finish_non_exhaustive() + } +} + +pub(crate) fn fuzz_of_certificate(data: &[u8]) { + let cert = CertificateDer::from(data.to_vec()); + let _ = SpkiPin::of_certificate(&cert); +} + +pub struct ReloadableCertResolver { + current: ArcSwap, +} + +impl std::fmt::Debug for ReloadableCertResolver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReloadableCertResolver") + .finish_non_exhaustive() + } +} + +impl ReloadableCertResolver { + pub fn new(initial: CertifiedKey) -> Self { + Self { + current: ArcSwap::from_pointee(initial), + } + } + + pub fn store(&self, key: CertifiedKey) { + self.current.store(Arc::new(key)); + } +} + +impl ResolvesServerCert for ReloadableCertResolver { + fn resolve(&self, _client_hello: ClientHello<'_>) -> Option> { + Some(self.current.load_full()) + } +} + +pub fn spawn_cert_reload( + resolver: Arc, + paths: crate::StaticCertPaths, + shutdown: CancellationToken, +) { + #[cfg(unix)] + tokio::spawn(async move { + use tokio::signal::unix::{SignalKind, signal}; + let mut hangup = match signal(SignalKind::hangup()) { + Ok(stream) => stream, + Err(error) => { + tracing::error!("install SIGHUP handler: {error}"); + return; + } + }; + loop { + tokio::select! { + () = shutdown.cancelled() => break, + received = hangup.recv() => { + if received.is_none() { + break; + } + match load_certified_key(&paths) { + Ok(key) => { + resolver.store(key); + tracing::info!("reloaded TLS certificate on SIGHUP"); + } + Err(error) => { + tracing::warn!("SIGHUP reload kept the existing certificate: {error}"); + } + } + } + } + } + }); + + #[cfg(not(unix))] + let _ = (resolver, paths, shutdown); +} + +pub fn load_certified_key(paths: &crate::StaticCertPaths) -> Result { + let certs = load_certs(paths.cert_path.as_path())?; + let key = load_private_key(paths.key_path.as_path())?; + let signing_key = aws_lc_rs::sign::any_supported_type(&key) + .map_err(|error| TlsError::SigningKey(error.to_string()))?; + let certified = CertifiedKey::new(certs, signing_key); + certified + .keys_match() + .map_err(|error| TlsError::KeyMismatch(error.to_string()))?; + Ok(certified) +} + +fn load_certs(path: &Path) -> Result>, TlsError> { + let bytes = std::fs::read(path).map_err(|source| TlsError::Read { + path: path.display().to_string(), + source, + })?; + let mut reader = BufReader::new(bytes.as_slice()); + let certs = rustls_pemfile::certs(&mut reader) + .collect::, _>>() + .map_err(|error| TlsError::Parse { + path: path.display().to_string(), + message: error.to_string(), + })?; + match certs.is_empty() { + true => Err(TlsError::NoCertificates(path.display().to_string())), + false => Ok(certs), + } +} + +fn load_private_key(path: &Path) -> Result, TlsError> { + let bytes = std::fs::read(path).map_err(|source| TlsError::Read { + path: path.display().to_string(), + source, + })?; + let mut reader = BufReader::new(bytes.as_slice()); + rustls_pemfile::private_key(&mut reader) + .map_err(|error| TlsError::Parse { + path: path.display().to_string(), + message: error.to_string(), + })? + .ok_or_else(|| TlsError::NoPrivateKey(path.display().to_string())) +} + +fn ticketer() -> Result, TlsError> { + aws_lc_rs::Ticketer::new().map_err(|error| TlsError::Ticketer(error.to_string())) +} + +fn tcp_alpn(extra: &[&[u8]]) -> Vec> { + [b"h2".as_slice(), b"http/1.1".as_slice()] + .into_iter() + .chain(extra.iter().copied()) + .map(<[u8]>::to_vec) + .collect() +} + +pub fn build_tls_server_config( + resolver: Arc, + extra_alpn: &[&[u8]], +) -> Result { + let provider = Arc::new(aws_lc_rs::default_provider()); + let mut config = ServerConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|error| TlsError::Config(error.to_string()))? + .with_no_client_auth() + .with_cert_resolver(resolver); + config.alpn_protocols = tcp_alpn(extra_alpn); + config.ticketer = ticketer()?; + Ok(config) +} + +pub fn build_mtls_server_config( + resolver: Arc, + client_ca: &crate::ClientCaPath, + pin: SpkiPin, +) -> Result { + let provider = Arc::new(aws_lc_rs::default_provider()); + let roots = load_client_ca(client_ca.as_path())?; + let webpki = + WebPkiClientVerifier::builder_with_provider(Arc::new(roots), Arc::clone(&provider)) + .build() + .map_err(|error| TlsError::ClientVerifier { + path: client_ca.as_path().display().to_string(), + message: error.to_string(), + })?; + let verifier = Arc::new(PinnedClientVerifier { inner: webpki, pin }); + let mut config = ServerConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|error| TlsError::Config(error.to_string()))? + .with_client_cert_verifier(verifier) + .with_cert_resolver(resolver); + config.alpn_protocols = tcp_alpn(&[]); + config.ticketer = ticketer()?; + Ok(config) +} + +fn load_client_ca(path: &Path) -> Result { + let certs = load_certs(path)?; + let mut roots = RootCertStore::empty(); + let (added, _) = roots.add_parsable_certificates(certs); + match added { + 0 => Err(TlsError::NoCertificates(path.display().to_string())), + _ => Ok(roots), + } +} + +#[derive(Debug)] +struct PinnedClientVerifier { + inner: Arc, + pin: SpkiPin, +} + +impl ClientCertVerifier for PinnedClientVerifier { + fn root_hint_subjects(&self) -> &[rustls::DistinguishedName] { + self.inner.root_hint_subjects() + } + + fn offer_client_auth(&self) -> bool { + self.inner.offer_client_auth() + } + + fn client_auth_mandatory(&self) -> bool { + self.inner.client_auth_mandatory() + } + + fn verify_client_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + now: UnixTime, + ) -> Result { + let verified = self + .inner + .verify_client_cert(end_entity, intermediates, now)?; + match SpkiPin::of_certificate(end_entity)? == self.pin { + true => Ok(verified), + false => Err(rustls::Error::General( + "client certificate SPKI doesn't match the pinned admin identity".to_string(), + )), + } + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + self.inner.verify_tls12_signature(message, cert, dss) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + self.inner.verify_tls13_signature(message, cert, dss) + } + + fn supported_verify_schemes(&self) -> Vec { + self.inner.supported_verify_schemes() + } +} + +pub fn build_quic_server_config( + resolver: Arc, + limits: ListenLimits, + early_data: EarlyDataPolicy, +) -> Result { + let provider = Arc::new(aws_lc_rs::default_provider()); + let mut crypto = ServerConfig::builder_with_provider(provider) + .with_protocol_versions(&[&rustls::version::TLS13]) + .map_err(|error| TlsError::Config(error.to_string()))? + .with_no_client_auth() + .with_cert_resolver(resolver); + crypto.alpn_protocols = vec![b"h3".to_vec()]; + crypto.max_early_data_size = early_data.max_early_data_size(); + + let quic_crypto = + QuicServerConfig::try_from(crypto).map_err(|error| TlsError::Config(error.to_string()))?; + let mut config = quinn::ServerConfig::with_crypto(Arc::new(quic_crypto)); + + let budget = limits.connection_budget(); + let mut transport = quinn::TransportConfig::default(); + transport.max_concurrent_bidi_streams(quinn::VarInt::from_u32( + budget.max_concurrent_streams().get(), + )); + transport.stream_receive_window(quinn::VarInt::from_u32(budget.stream_receive_window())); + transport.receive_window(quinn::VarInt::from_u32(budget.connection_receive_window())); + let idle = limits.idle_timeout().get(); + transport.max_idle_timeout(Some( + quinn::IdleTimeout::try_from(idle).map_err(|error| TlsError::Config(error.to_string()))?, + )); + transport.keep_alive_interval(Some(idle / 2)); + config.transport_config(Arc::new(transport)); + Ok(config) +} + +#[cfg(test)] +pub(crate) mod test_support { + use std::num::{NonZeroU32, NonZeroU64}; + + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + use rustls::sign::CertifiedKey; + + use super::*; + use crate::limits::ListenLimits; + + pub(crate) fn self_signed() -> CertifiedKey { + let generated = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).unwrap(); + let cert_der = generated.cert.der().clone(); + let key_der = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( + generated.signing_key.serialize_der(), + )); + let signing_key = aws_lc_rs::sign::any_supported_type(&key_der).unwrap(); + CertifiedKey::new(vec![cert_der], signing_key) + } + + pub(crate) fn resolver() -> Arc { + Arc::new(ReloadableCertResolver::new(self_signed())) + } + + pub(crate) fn limits() -> ListenLimits { + ListenLimits::new( + crate::limits::HeaderTimeout::from_millis(NonZeroU64::new(5_000).unwrap()), + crate::limits::IdleTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + NonZeroU32::new(64).unwrap(), + ) + } + + pub(crate) fn classical_only_provider() -> rustls::crypto::CryptoProvider { + let mut provider = aws_lc_rs::default_provider(); + provider.kx_groups = vec![aws_lc_rs::kx_group::X25519]; + provider + } + + #[derive(Debug)] + pub(crate) struct AcceptAnyServerCert; + + impl rustls::client::danger::ServerCertVerifier for AcceptAnyServerCert { + fn verify_server_cert( + &self, + _end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + aws_lc_rs::default_provider() + .signature_verification_algorithms + .supported_schemes() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_tcp_alpn_set_offers_h2_and_http1_but_not_h3() { + let config = build_tls_server_config(test_support::resolver(), &[]).unwrap(); + assert_eq!( + config.alpn_protocols, + vec![b"h2".to_vec(), b"http/1.1".to_vec()], + "h3 is QUIC-only and must never appear in the TCP ALPN set" + ); + } + + #[test] + fn the_acme_challenge_alpn_joins_only_when_requested() { + let config = build_tls_server_config(test_support::resolver(), &[ACME_TLS_ALPN]).unwrap(); + assert_eq!( + config.alpn_protocols, + vec![b"h2".to_vec(), b"http/1.1".to_vec(), ACME_TLS_ALPN.to_vec()], + "acme-tls/1 must trail h2 and http/1.1 so normal clients never select it" + ); + } + + #[test] + fn the_tcp_config_installs_an_enabled_session_ticketer() { + let config = build_tls_server_config(test_support::resolver(), &[]).unwrap(); + assert!( + config.ticketer.enabled(), + "session resumption requires an enabled ticketer" + ); + } + + #[test] + fn an_spki_pin_round_trips_through_base64() { + let encoded = base64::engine::general_purpose::STANDARD.encode([9u8; 32]); + assert_eq!(SpkiPin::from_base64(&encoded).unwrap(), SpkiPin([9u8; 32])); + } + + #[test] + fn the_spki_pin_matches_the_standard_openssl_recipe() { + const CERT_PEM: &str = "-----BEGIN CERTIFICATE-----\n\ +MIIBhDCCASugAwIBAgIUSkWE4CZvV8B8z9phedKo1PbahDUwCgYIKoZIzj0EAwIw\n\ +GDEWMBQGA1UEAwwNYW5lbW9uZS5hZG1pbjAeFw0yNjA2MjExOTA5MDdaFw0zNjA2\n\ +MTgxOTA5MDdaMBgxFjAUBgNVBAMMDWFuZW1vbmUuYWRtaW4wWTATBgcqhkjOPQIB\n\ +BggqhkjOPQMBBwNCAASFLKd70MtSGSyI2UjdpQyjaJrvXLofac41nI346wK0lC9G\n\ +PjZH/NKqo1iwQn+UfZB7gotfezWrDmAUz5OgT6Rlo1MwUTAdBgNVHQ4EFgQUis7S\n\ +XEFGpe4gQwWnzX/uzjpt274wHwYDVR0jBBgwFoAUis7SXEFGpe4gQwWnzX/uzjpt\n\ +274wDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNHADBEAiBhJgE4cMP5/FJw\n\ +imc3fYQxOhm5nO59cfG06+0vuDIV1QIgMsKZjFjsch8rbLRNiJL5+bDmlgO7MD14\n\ +0PAyPOyjb+w=\n\ +-----END CERTIFICATE-----\n"; + const OPENSSL_PIN: &str = "EhmM1HyWzC54br06EDvoAaqt1q1h+je3vVFJTcZ9e1U="; + + let der = rustls_pemfile::certs(&mut CERT_PEM.as_bytes()) + .next() + .unwrap() + .unwrap(); + assert_eq!( + SpkiPin::of_certificate(&der).unwrap(), + SpkiPin::from_base64(OPENSSL_PIN).unwrap(), + "of_certificate must hash the same SubjectPublicKeyInfo bytes as openssl pkey -pubin -outform DER | dgst -sha256" + ); + } + + #[test] + fn an_spki_pin_of_the_wrong_length_is_rejected() { + let encoded = base64::engine::general_purpose::STANDARD.encode([9u8; 16]); + assert!(matches!( + SpkiPin::from_base64(&encoded), + Err(TlsError::SpkiPin(_)) + )); + } + + #[test] + fn the_quic_alpn_set_offers_only_h3() { + let quic = build_quic_server_config( + test_support::resolver(), + test_support::limits(), + EarlyDataPolicy::Disabled, + ); + assert!(quic.is_ok()); + } + + #[test] + fn the_default_provider_prefers_post_quantum_key_exchange() { + use rustls::NamedGroup; + + let provider = aws_lc_rs::default_provider(); + let first = provider.kx_groups.first().expect("a key exchange group"); + assert_eq!( + first.name(), + NamedGroup::X25519MLKEM768, + "prefer-post-quantum must order X25519MLKEM768 first for both the TCP and QUIC configs" + ); + } + + #[test] + fn a_mismatched_certificate_and_key_is_rejected() { + let first = test_support::self_signed(); + let second = test_support::self_signed(); + let mismatched = CertifiedKey::new(first.cert.clone(), second.key.clone()); + assert!(mismatched.keys_match().is_err()); + } + + struct ClientIdentity { + ca_path: crate::ClientCaPath, + chain: Vec>, + key_der: Vec, + pin: SpkiPin, + } + + fn issue_client_identity() -> ClientIdentity { + let ca_key = rcgen::KeyPair::generate().unwrap(); + let mut ca_params = rcgen::CertificateParams::new(Vec::::new()).unwrap(); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let ca_cert = ca_params.self_signed(&ca_key).unwrap(); + let issuer = rcgen::Issuer::new(ca_params, ca_key); + + let client_key = rcgen::KeyPair::generate().unwrap(); + let client_params = rcgen::CertificateParams::new(vec!["admin.knot".to_string()]).unwrap(); + let client_cert = client_params.signed_by(&client_key, &issuer).unwrap(); + let client_der = client_cert.der().clone(); + + let ca_path = std::env::temp_dir().join(format!( + "knot_edge_mtls_ca_{}_{:p}.pem", + std::process::id(), + &client_der as *const _ + )); + std::fs::write(&ca_path, ca_cert.pem()).unwrap(); + + ClientIdentity { + ca_path: crate::ClientCaPath::new(ca_path), + pin: SpkiPin::of_certificate(&client_der).unwrap(), + chain: vec![client_der], + key_der: client_key.serialize_der(), + } + } + + async fn mtls_handshake( + server_config: ServerConfig, + identity: Option<&ClientIdentity>, + ) -> bool { + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer, ServerName}; + use tokio::net::{TcpListener, TcpStream}; + use tokio_rustls::{TlsAcceptor, TlsConnector}; + + let acceptor = TlsAcceptor::from(Arc::new(server_config)); + let listener = TcpListener::bind("[::1]:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (tcp, _) = listener.accept().await.unwrap(); + acceptor.accept(tcp).await.is_ok() + }); + + let verifier = + rustls::ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(test_support::AcceptAnyServerCert)); + let mut client_config = match identity { + Some(identity) => verifier + .with_client_auth_cert( + identity.chain.clone(), + PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(identity.key_der.clone())), + ) + .unwrap(), + None => verifier.with_no_client_auth(), + }; + client_config.alpn_protocols = vec![b"h2".to_vec()]; + let connector = TlsConnector::from(Arc::new(client_config)); + let tcp = TcpStream::connect(addr).await.unwrap(); + let client_ok = connector + .connect(ServerName::try_from("localhost").unwrap(), tcp) + .await + .is_ok(); + let server_ok = server.await.unwrap(); + client_ok && server_ok + } + + #[tokio::test] + async fn mtls_rejects_a_client_presenting_no_certificate() { + let identity = issue_client_identity(); + let config = build_mtls_server_config( + test_support::resolver(), + &identity.ca_path, + identity.pin.clone(), + ) + .unwrap(); + assert!( + !mtls_handshake(config, None).await, + "the mandatory mTLS verifier must reject a client that presents no certificate" + ); + } + + #[tokio::test] + async fn mtls_admits_the_pinned_admin_certificate() { + let identity = issue_client_identity(); + let config = build_mtls_server_config( + test_support::resolver(), + &identity.ca_path, + identity.pin.clone(), + ) + .unwrap(); + assert!( + mtls_handshake(config, Some(&identity)).await, + "a client presenting the pinned admin certificate must complete the mTLS handshake" + ); + } + + #[tokio::test] + async fn mtls_rejects_a_ca_trusted_client_whose_spki_is_not_pinned() { + let identity = issue_client_identity(); + let config = build_mtls_server_config( + test_support::resolver(), + &identity.ca_path, + SpkiPin([0u8; 32]), + ) + .unwrap(); + assert!( + !mtls_handshake(config, Some(&identity)).await, + "a client trusted by the CA but failing the SPKI pin must be rejected" + ); + } +} diff --git a/knot2/crates/knot-edge/src/zerortt.rs b/knot2/crates/knot-edge/src/zerortt.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb76495b9c1d3b0edb11ad301f9417db83462e37 --- /dev/null +++ b/knot2/crates/knot-edge/src/zerortt.rs @@ -0,0 +1,194 @@ +use axum::Router; +use axum::extract::Request; +use axum::handler::Handler; +use axum::middleware::{Next, from_fn}; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use http::StatusCode; + +use crate::protocol::NegotiatedProtocol; + +pub struct ZeroRttSafe { + handler: H, +} + +impl ZeroRttSafe { + pub fn new(handler: H) -> Self { + Self { handler } + } +} + +pub struct RequiresFullHandshake { + router: Router, +} + +impl RequiresFullHandshake { + pub fn new(router: Router) -> Self { + Self { + router: router.layer(from_fn(reject_early_writes)), + } + } + + pub(crate) fn into_router(self) -> Router { + self.router + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EarlyData { + Yes, + No, +} + +impl EarlyData { + pub fn is_early(self) -> bool { + matches!(self, EarlyData::Yes) + } +} + +const EARLY_DATA_HEADER: &str = "early-data"; + +pub struct ZeroRttRoutes { + router: Router, + count: usize, +} + +impl ZeroRttRoutes { + pub fn new() -> Self { + Self { + router: Router::new(), + count: 0, + } + } + + pub fn get(mut self, path: &str, handler: ZeroRttSafe) -> Self + where + H: Handler, + T: 'static, + { + self.router = self.router.route(path, get(handler.handler)); + self.count += 1; + self + } + + pub fn into_router(self) -> Router { + self.router + } + + pub(crate) fn early_data_policy(&self) -> EarlyDataPolicy { + match self.count { + 0 => EarlyDataPolicy::Disabled, + _ => EarlyDataPolicy::Enabled, + } + } +} + +impl Default for ZeroRttRoutes { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EarlyDataPolicy { + Disabled, + Enabled, +} + +impl EarlyDataPolicy { + pub(crate) fn max_early_data_size(self) -> u32 { + match self { + EarlyDataPolicy::Disabled => 0, + // rustls takes 0 or u32::MAX, + // and quinn unwraps errors, quite unfortunate. + // + // The 0 above is hard-off marker, for when no route opted in. + EarlyDataPolicy::Enabled => u32::MAX, + } + } +} + +pub(crate) async fn tag_from_header(mut request: Request, next: Next) -> Response { + if request.extensions().get::().is_none() { + let early = request + .headers() + .get(EARLY_DATA_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim() == "1"); + request + .extensions_mut() + .insert(if early { EarlyData::Yes } else { EarlyData::No }); + } + next.run(request).await +} + +async fn reject_early_writes(request: Request, next: Next) -> Response { + let early = request + .extensions() + .get::() + .copied() + .unwrap_or(EarlyData::Yes); + if early.is_early() { + let protocol = request + .extensions() + .get::() + .map(|protocol| protocol.as_str()) + .unwrap_or("unknown"); + tracing::debug!( + protocol, + path = %request.uri().path(), + "refused an early-data request to a full-handshake route with 425" + ); + return too_early(); + } + next.run(request).await +} + +fn too_early() -> Response { + ( + StatusCode::TOO_EARLY, + "this route refuses early data and needs a completed handshake", + ) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_empty_safe_set_keeps_early_data_disabled() { + assert_eq!( + ZeroRttRoutes::new().early_data_policy(), + EarlyDataPolicy::Disabled + ); + assert_eq!(EarlyDataPolicy::Disabled.max_early_data_size(), 0); + } + + #[test] + fn a_proven_safe_set_unlocks_a_nonzero_early_data_size() { + let routes = ZeroRttRoutes::new().get("/info/refs", ZeroRttSafe::new(|| async { "ok" })); + assert_eq!(routes.early_data_policy(), EarlyDataPolicy::Enabled); + assert_eq!(EarlyDataPolicy::Enabled.max_early_data_size(), u32::MAX); + } + + #[tokio::test] + async fn a_full_handshake_route_fails_closed_when_the_request_is_unclassified() { + use axum::body::Body; + use axum::routing::post; + use http::{Request, StatusCode}; + use tower::ServiceExt; + + let router = Router::new() + .route("/git-upload-pack", post(|| async { "pack" })) + .layer(from_fn(reject_early_writes)); + let request = Request::post("/git-upload-pack") + .body(Body::empty()) + .unwrap(); + assert_eq!( + router.oneshot(request).await.unwrap().status(), + StatusCode::TOO_EARLY, + "a write whose early-data status was never tagged must fail closed with 425" + ); + } +} diff --git a/knot2/crates/knot-edge/tests/fuzz_smoke.rs b/knot2/crates/knot-edge/tests/fuzz_smoke.rs new file mode 100644 index 0000000000000000000000000000000000000000..5b88b109ef1b260fe7925c7fd9efd9a81d1b7be6 --- /dev/null +++ b/knot2/crates/knot-edge/tests/fuzz_smoke.rs @@ -0,0 +1,11 @@ +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn tls_parsers_never_panic(data in proptest::collection::vec(any::(), 0..4096)) { + knot_edge::fuzz::spki_of_certificate(&data); + knot_edge::fuzz::spki_pin(&data); + } +} diff --git a/knot2/crates/knot-events/Cargo.toml b/knot2/crates/knot-events/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..fc4e5021badbaa5e86eb814ab45f1091330c665e --- /dev/null +++ b/knot2/crates/knot-events/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "knot-events" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +bytes = { workspace = true } +knot-types = { workspace = true } +knot-runtime = { workspace = true } +tokio = { workspace = true } diff --git a/knot2/crates/knot-events/src/lib.rs b/knot2/crates/knot-events/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..9ff1e7504ef46c6e7c13af248355edf7590ced11 --- /dev/null +++ b/knot2/crates/knot-events/src/lib.rs @@ -0,0 +1,756 @@ +use std::collections::{BTreeSet, HashMap, VecDeque}; +use std::net::IpAddr; +use std::sync::{Arc, Mutex}; + +use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch}; + +use knot_runtime::Clock; +use knot_types::UnixMicros; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct EventCursor(i64); + +impl EventCursor { + pub const START: Self = Self(0); + + pub fn new(nanos: i64) -> Self { + Self(nanos) + } + + pub fn get(self) -> i64 { + self.0 + } + + pub fn of_micros(micros: UnixMicros) -> Self { + Self((micros.get() as i64).saturating_mul(1_000)) + } + + pub fn micros(self) -> UnixMicros { + UnixMicros::new((self.0 / 1_000) as u64) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReplayEvents(std::num::NonZeroUsize); + +impl ReplayEvents { + pub fn new(value: usize) -> Option { + std::num::NonZeroUsize::new(value).map(Self) + } + + pub fn get(self) -> usize { + self.0.get() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReplayBytes(std::num::NonZeroUsize); + +impl ReplayBytes { + pub fn new(value: usize) -> Option { + std::num::NonZeroUsize::new(value).map(Self) + } + + pub fn get(self) -> usize { + self.0.get() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReplayBounds { + events: ReplayEvents, + bytes: ReplayBytes, +} + +impl ReplayBounds { + pub fn new(events: ReplayEvents, bytes: ReplayBytes) -> Self { + Self { events, bytes } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BatchEnd { + CaughtUp, + Bounded, +} + +pub struct Replayed { + pub events: Vec>, + pub end: BatchEnd, +} + +struct Entry { + event: Arc, + bytes: usize, +} + +struct Ring { + entries: VecDeque, + bytes: usize, + last_micros: UnixMicros, + pending: BTreeSet, + evicted_through: Option, +} + +struct Inner { + bounds: ReplayBounds, + ring: Mutex, + head: watch::Sender, +} + +impl Inner { + fn lock(&self) -> std::sync::MutexGuard<'_, Ring> { + self.ring + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + // leave that guinness be, boy! it needs to + fn settle(&self, ring: std::sync::MutexGuard<'_, Ring>) { + let head = stable_head(&ring); + drop(ring); + self.head.send_replace(head); + } +} + +fn stable_head(ring: &Ring) -> EventCursor { + let stable = match ring.pending.iter().next().copied() { + Some(horizon) => ring + .entries + .partition_point(|entry| entry.event.created < horizon), + None => ring.entries.len(), + }; + stable + .checked_sub(1) + .and_then(|index| ring.entries.get(index)) + .map(|entry| entry.event.created) + .unwrap_or(EventCursor::START) +} + +fn insert_sorted(ring: &mut Ring, event: Event, bounds: ReplayBounds) { + let bytes = std::mem::size_of::() + event.frame.len(); + let position = ring + .entries + .partition_point(|existing| existing.event.created < event.created); + ring.entries.insert( + position, + Entry { + event: Arc::new(event), + bytes, + }, + ); + ring.bytes += bytes; + evict_oldest(ring, bounds); +} + +fn evict_oldest(ring: &mut Ring, bounds: ReplayBounds) { + while ring.entries.len() > bounds.events.get() + || (ring.bytes > bounds.bytes.get() && ring.entries.len() > 1) + { + let Some(evicted) = ring.entries.pop_front() else { + break; + }; + ring.bytes -= evicted.bytes; + ring.evicted_through = Some(evicted.event.created); + } +} + +pub struct EventLog { + clock: C, + inner: Arc, +} + +impl EventLog { + pub fn new(clock: C, bounds: ReplayBounds) -> Self { + Self { + clock, + inner: Arc::new(Inner { + bounds, + ring: Mutex::new(Ring { + entries: VecDeque::new(), + bytes: 0, + last_micros: UnixMicros::new(0), + pending: BTreeSet::new(), + evicted_through: None, + }), + head: watch::Sender::new(EventCursor::START), + }), + } + } + + pub fn with_boot_floor(self, floor: UnixMicros) -> Self { + { + let mut ring = self.inner.lock(); + ring.last_micros = ring.last_micros.max(floor); + let booted = (ring.last_micros > UnixMicros::new(0)) + .then(|| EventCursor::of_micros(ring.last_micros)); + ring.evicted_through = ring.evicted_through.max(booted); + } + self + } + + pub fn raise_floor(&self, floor: UnixMicros) { + let mut ring = self.inner.lock(); + ring.last_micros = ring.last_micros.max(floor); + } + + pub fn last_issued(&self) -> UnixMicros { + self.inner.lock().last_micros + } + + pub fn pending_horizon(&self) -> Option { + self.inner + .lock() + .pending + .range(EventCursor::START..) + .next() + .map(|cursor| cursor.micros()) + } + + fn next_cursor(&self, ring: &mut Ring) -> (UnixMicros, EventCursor) { + let micros = self.clock.now_unix_micros().max(ring.last_micros.next()); + ring.last_micros = micros; + (micros, EventCursor::of_micros(micros)) + } + + pub fn reserve(&self) -> Reservation { + let mut ring = self.inner.lock(); + let (micros, cursor) = self.next_cursor(&mut ring); + ring.pending.insert(cursor); + drop(ring); + Reservation { + inner: Arc::clone(&self.inner), + cursor, + micros, + fulfilled: false, + } + } + + pub fn replay(&self, after: EventCursor, bounds: ReplayBounds) -> Replayed { + let ring = self.inner.lock(); + // The corresponding read side guarantee of `reserve`. + let horizon = ring.pending.iter().next().copied(); + let visible = |entry: &&Entry| { + entry.event.created > after + && horizon.is_none_or(|horizon| entry.event.created < horizon) + }; + let events: Vec> = ring + .entries + .iter() + .filter(visible) + .take(bounds.events.get()) + // Why the first event gets to ignore the byte bound? + // Imagine a consumer whose next event is by itself wider + // than the entire bound, right - + // every batch it requests would come back empty, + // its cursor would never advance, + // it would ask again, repeat. + // Sending that one event alone over the bound + // is the only way. + .scan(0usize, |spent, entry| { + let first = *spent == 0; + *spent += entry.bytes; + (first || *spent <= bounds.bytes.get()).then(|| Arc::clone(&entry.event)) + }) + .collect(); + let end = match ring.entries.iter().filter(visible).nth(events.len()) { + Some(_) => BatchEnd::Bounded, + None => BatchEnd::CaughtUp, + }; + Replayed { events, end } + } + + pub fn lost_before(&self, after: EventCursor) -> bool { + let ring = self.inner.lock(); + matches!(ring.evicted_through, Some(frontier) if after < frontier) + } + + pub fn subscribe(&self) -> watch::Receiver { + self.inner.head.subscribe() + } +} + +#[derive(Debug, Clone)] +pub struct Event { + pub created: EventCursor, + pub frame: bytes::Bytes, +} + +pub struct Reservation { + inner: Arc, + cursor: EventCursor, + micros: UnixMicros, + fulfilled: bool, +} + +impl Reservation { + pub fn cursor(&self) -> EventCursor { + self.cursor + } + + pub fn micros(&self) -> UnixMicros { + self.micros + } + + pub fn fulfill_frame(mut self, frame: bytes::Bytes) { + let mut ring = self.inner.lock(); + insert_sorted( + &mut ring, + Event { + created: self.cursor, + frame, + }, + self.inner.bounds, + ); + ring.pending.remove(&self.cursor); + self.inner.settle(ring); + self.fulfilled = true; + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + if self.fulfilled { + return; + } + let mut ring = self.inner.lock(); + ring.pending.remove(&self.cursor); + self.inner.settle(ring); + } +} + +knot_types::scalar_newtype! { + pub struct GlobalSubscriberLimit(usize); + pub struct PerPeerSubscriberLimit(usize); +} + +pub struct SubscriberGate { + global: Arc, + per_peer_max: usize, + peers: Mutex>, +} + +impl SubscriberGate { + pub fn new(global_max: GlobalSubscriberLimit, per_peer_max: PerPeerSubscriberLimit) -> Self { + Self { + global: Arc::new(Semaphore::new(global_max.get().max(1))), + per_peer_max: per_peer_max.get().max(1), + peers: Mutex::new(HashMap::new()), + } + } + + pub fn try_admit(self: &Arc, peer: IpAddr) -> Option { + let global = Arc::clone(&self.global).try_acquire_owned().ok()?; + let mut peers = self + .peers + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let current = peers.get(&peer).copied().unwrap_or(0); + if current >= self.per_peer_max { + return None; + } + peers.insert(peer, current + 1); + Some(SubscriberPermit { + _global: global, + gate: Arc::clone(self), + peer, + }) + } +} + +pub struct SubscriberPermit { + _global: OwnedSemaphorePermit, + gate: Arc, + peer: IpAddr, +} + +impl Drop for SubscriberPermit { + fn drop(&mut self) { + let mut peers = self + .gate + .peers + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(count) = peers.get_mut(&self.peer) { + *count -= 1; + if *count == 0 { + peers.remove(&self.peer); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use knot_runtime::ManualClock; + use knot_types::UnixMicros; + + fn bounds(events: usize, bytes: usize) -> ReplayBounds { + ReplayBounds::new( + ReplayEvents::new(events).unwrap(), + ReplayBytes::new(bytes).unwrap(), + ) + } + + fn log(capacity: usize) -> EventLog { + EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), + bounds(capacity, 1 << 20), + ) + } + + fn replay(log: &EventLog, after: EventCursor, limit: usize) -> Vec> { + log.replay(after, bounds(limit, 1 << 30)).events + } + + fn stamp(log: &EventLog, frame_bytes: usize) -> EventCursor { + let reservation = log.reserve(); + let cursor = reservation.cursor(); + reservation.fulfill_frame(bytes::Bytes::from(vec![0xa5u8; frame_bytes])); + cursor + } + + #[test] + fn a_frozen_clock_still_yields_strictly_increasing_cursors() { + let log = log(8); + let cursors: Vec<_> = (0..3).map(|_| stamp(&log, 16)).collect(); + assert!(cursors.windows(2).all(|pair| pair[0] < pair[1])); + } + + #[test] + fn the_ring_evicts_the_oldest_event_past_capacity() { + let log = log(2); + let first = stamp(&log, 16); + stamp(&log, 16); + stamp(&log, 16); + let replayed = replay(&log, EventCursor::START, 8); + assert_eq!(replayed.len(), 2); + assert!(replayed.iter().all(|event| event.created > first)); + } + + #[test] + fn an_evicting_ring_names_the_cursors_whose_events_are_gone() { + let log = log(2); + let first = stamp(&log, 16); + assert!(!log.lost_before(EventCursor::START)); + let second = stamp(&log, 16); + let third = stamp(&log, 16); + assert!(log.lost_before(EventCursor::START)); + assert!( + !log.lost_before(first), + "a consumer at the evicted cursor still has every event after it" + ); + assert!(!log.lost_before(second)); + assert!(!log.lost_before(third)); + let fourth = stamp(&log, 16); + assert!( + log.lost_before(first), + "the second eviction takes an event the first consumer still needs" + ); + assert!(!log.lost_before(fourth)); + } + + #[test] + fn a_boot_floor_loses_every_cursor_below_it_and_survives_a_lower_floor() { + let floor = UnixMicros::new(1_700_000_000_000_000); + let low = UnixMicros::new(1_600_000_000_000_000); + let at_floor = EventCursor::of_micros(floor); + let booted = log(8).with_boot_floor(floor); + assert!( + booted.lost_before(EventCursor::START) + && booted.lost_before(EventCursor::new(at_floor.get() - 1)), + "a just-booted ring can't replay an event from below its floor" + ); + assert!( + !booted.lost_before(at_floor), + "the floor is the highest seq issued, so a cursor there didn't miss an event" + ); + let unbooted = log(8).with_boot_floor(UnixMicros::new(0)); + assert!( + !unbooted.lost_before(EventCursor::START), + "a knot that hasn't issued an event hasn't evicted one either" + ); + let lowered = booted.with_boot_floor(low); + assert!( + lowered.lost_before(EventCursor::of_micros(low)), + "lowering the floor after boot doesn't bring back events from before it" + ); + assert_eq!( + lowered.last_issued(), + floor, + "the higher floor is still the highest seq issued" + ); + } + + #[test] + fn a_wide_event_evicts_by_bytes_long_before_the_ring_fills() { + let log = EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), + bounds(1_024, 64 * 1_024), + ); + (0..16).for_each(|_| { + stamp(&log, 8 * 1_024); + }); + let replayed = replay(&log, EventCursor::START, 1_024); + assert!( + (1..16).contains(&replayed.len()), + "the byte maximum evicts before the event maximum does: {}", + replayed.len() + ); + + let one = EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), + bounds(1_024, 1), + ); + let only = stamp(&one, 8 * 1_024); + assert_eq!( + replay(&one, EventCursor::START, 8) + .iter() + .map(|event| event.created) + .collect::>(), + vec![only], + "the ring keeps the one event wider than the whole byte maximum" + ); + } + + #[test] + fn a_frame_event_replays_its_bytes_and_advances_the_head() { + let log = log(8); + let head = log.subscribe(); + let reservation = log.reserve(); + let seq = reservation.micros(); + reservation.fulfill_frame(bytes::Bytes::from_static(b"header-and-payload")); + let replayed = replay(&log, EventCursor::START, 8); + assert_eq!(replayed.len(), 1); + assert_eq!( + replayed[0].frame.as_ref(), + b"header-and-payload", + "the ring holds the frame bytes verbatim, not a re-encoding" + ); + assert!(head.has_changed().unwrap()); + assert_eq!(replayed[0].created.get() as u64, seq.get() * 1_000); + } + + #[test] + fn replay_honors_the_cursor_and_the_limit() { + let log = log(8); + let cursors: Vec<_> = (0..4).map(|_| stamp(&log, 16)).collect(); + let after_second = replay(&log, cursors[1], 8); + assert_eq!( + after_second + .iter() + .map(|event| event.created) + .collect::>(), + cursors[2..].to_vec() + ); + assert_eq!(replay(&log, EventCursor::START, 2).len(), 2); + assert!(replay(&log, cursors[3], 8).is_empty()); + } + + #[test] + fn a_replay_batch_stops_at_the_byte_maximum_and_reports_whether_more_remains() { + let log = EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), + bounds(1_024, 1 << 20), + ); + assert_eq!( + log.replay(EventCursor::START, bounds(8, 1 << 20)).end, + BatchEnd::CaughtUp, + "an empty ring has nothing left to send" + ); + let cursors: Vec = (0..8).map(|_| stamp(&log, 8 * 1_024)).collect(); + + let batch = log.replay(EventCursor::START, bounds(1_024, 16 * 1_024)); + assert!( + (1..8).contains(&batch.events.len()), + "the byte maximum stops the batch before the event maximum does: {}", + batch.events.len() + ); + assert_eq!(batch.end, BatchEnd::Bounded); + + let rest = log.replay( + batch.events.last().expect("the batch is nonempty").created, + bounds(1_024, 1 << 30), + ); + assert_eq!(rest.end, BatchEnd::CaughtUp); + assert_eq!( + batch.events.len() + rest.events.len(), + cursors.len(), + "the two batches together are every event, with none repeated or skipped" + ); + let head = log.replay(cursors[7], bounds(8, 1 << 20)); + assert!(head.events.is_empty() && head.end == BatchEnd::CaughtUp); + + let single = log.replay(EventCursor::START, bounds(1_024, 1)); + assert_eq!( + single.events.len(), + 1, + "an event wider than the whole batch maximum is sent alone" + ); + assert_eq!(single.end, BatchEnd::Bounded); + } + + #[test] + fn a_subscriber_observes_the_head_advance() { + let log = log(8); + let mut head = log.subscribe(); + assert_eq!(*head.borrow_and_update(), EventCursor::START); + let created = stamp(&log, 16); + assert!(head.has_changed().unwrap()); + assert_eq!(*head.borrow_and_update(), created); + } + + fn peer(last: u8) -> IpAddr { + IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, last)) + } + + #[test] + fn gate_enforces_limits_and_prunes() { + let global = Arc::new(SubscriberGate::new( + GlobalSubscriberLimit::new(2), + PerPeerSubscriberLimit::new(8), + )); + let first = global + .try_admit(peer(1)) + .expect("first subscriber is admitted"); + let _second = global + .try_admit(peer(2)) + .expect("second subscriber is admitted"); + assert!( + global.try_admit(peer(3)).is_none(), + "third subscriber is refused once the global limit is reached" + ); + drop(first); + assert!( + global.try_admit(peer(3)).is_some(), + "freeing global slot admits waiting subscriber" + ); + + let per_peer = Arc::new(SubscriberGate::new( + GlobalSubscriberLimit::new(16), + PerPeerSubscriberLimit::new(2), + )); + let _socket = per_peer + .try_admit(peer(1)) + .expect("first socket is admitted"); + let second = per_peer + .try_admit(peer(1)) + .expect("second socket is admitted"); + assert!( + per_peer.try_admit(peer(1)).is_none(), + "third socket from same peer is refused at the per-peer limit" + ); + assert!( + per_peer.try_admit(peer(2)).is_some(), + "different peer keeps its own budget" + ); + drop(second); + assert!( + per_peer.try_admit(peer(1)).is_some(), + "freed per-peer slot is reusable" + ); + + let prune = Arc::new(SubscriberGate::new( + GlobalSubscriberLimit::new(16), + PerPeerSubscriberLimit::new(2), + )); + let permit = prune.try_admit(peer(1)).expect("admitted"); + drop(permit); + assert!( + prune + .peers + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .is_empty(), + "peer map prunes peer once its last socket closes" + ); + } + + fn cursors(log: &EventLog) -> Vec { + replay(log, EventCursor::START, 64) + .iter() + .map(|event| event.created) + .collect() + } + + #[test] + fn a_reservation_holds_back_later_events_until_it_is_fulfilled() { + let log = log(8); + let early = stamp(&log, 16); + let reservation = log.reserve(); + let later = stamp(&log, 16); + assert!(reservation.cursor() > early && reservation.cursor() < later); + assert_eq!( + cursors(&log), + vec![early], + "event published after reservation waits behind it" + ); + let mid = reservation.cursor(); + reservation.fulfill_frame(bytes::Bytes::from_static(b"mid")); + assert_eq!( + cursors(&log), + vec![early, mid, later], + "fulfilling reservation releases it and event queued behind it, in cursor order" + ); + } + + #[test] + fn out_of_order_fulfillment_still_replays_in_cursor_order() { + let log = log(8); + let first = log.reserve(); + let second = log.reserve(); + let (c1, c2) = (first.cursor(), second.cursor()); + assert!(c1 < c2); + second.fulfill_frame(bytes::Bytes::from_static(b"second")); + assert!( + cursors(&log).is_empty(), + "later reservation stays hidden while earlier one is outstanding" + ); + first.fulfill_frame(bytes::Bytes::from_static(b"first")); + assert_eq!( + cursors(&log), + vec![c1, c2], + "both surface in cursor order regardless of fulfillment order" + ); + } + + #[test] + fn a_dropped_reservation_unblocks_the_horizon_without_an_event() { + let log = log(8); + let reservation = log.reserve(); + let later = stamp(&log, 16); + assert!( + cursors(&log).is_empty(), + "later event waits behind unfulfilled reservation" + ); + drop(reservation); + assert_eq!( + cursors(&log), + vec![later], + "dropping reservation surfaces queued event and leaves no gap" + ); + } + + #[test] + fn the_head_holds_at_the_last_stable_event_until_a_reservation_is_fulfilled() { + let log = log(8); + let mut head = log.subscribe(); + let early = stamp(&log, 16); + assert_eq!(*head.borrow_and_update(), early); + let reservation = log.reserve(); + let later = stamp(&log, 16); + assert_eq!( + *head.borrow_and_update(), + early, + "head holds while lower-cursor reservation is pending" + ); + reservation.fulfill_frame(bytes::Bytes::from_static(b"fulfilled")); + assert_eq!( + *head.borrow_and_update(), + later, + "fulfilling reservation advances head past released events" + ); + } +} diff --git a/knot2/crates/knot-fixtures/Cargo.toml b/knot2/crates/knot-fixtures/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..66e05170436b2d03712bad7c72ea73b1dbe549f1 --- /dev/null +++ b/knot2/crates/knot-fixtures/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "knot-fixtures" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/knot2/crates/knot-fixtures/src/lib.rs b/knot2/crates/knot-fixtures/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..0c3fb0e0a29a7ae73347ee444378e408a3864c58 --- /dev/null +++ b/knot2/crates/knot-fixtures/src/lib.rs @@ -0,0 +1,142 @@ +use std::io::Write; +use std::path::Path; +use std::process::{Command, Output, Stdio}; + +pub const AUTHOR_NAME: &str = "nel"; +pub const AUTHOR_EMAIL: &str = "nel@oyster.cafe"; +pub const PINNED_DATE: &str = "2026-01-01T00:00:00 +0000"; + +pub const UNSHARED_SSH: &str = "-o IdentitiesOnly=yes -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=publickey -o BatchMode=yes \ + -o ControlMaster=no -o ControlPath=none"; + +pub fn command(cwd: &Path) -> Command { + let mut command = Command::new("git"); + command + .current_dir(cwd) + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_SYSTEM", "/dev/null") + .env("GIT_CONFIG_COUNT", "2") + .env("GIT_CONFIG_KEY_0", "maintenance.auto") + .env("GIT_CONFIG_VALUE_0", "false") + .env("GIT_CONFIG_KEY_1", "gc.autoDetach") + .env("GIT_CONFIG_VALUE_1", "false") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_ASKPASS", "true") + .env("GIT_AUTHOR_NAME", AUTHOR_NAME) + .env("GIT_AUTHOR_EMAIL", AUTHOR_EMAIL) + .env("GIT_COMMITTER_NAME", AUTHOR_NAME) + .env("GIT_COMMITTER_EMAIL", AUTHOR_EMAIL); + command +} + +pub fn command_at(cwd: &Path, stamp: &str) -> Command { + let mut command = command(cwd); + command + .env("GIT_AUTHOR_DATE", stamp) + .env("GIT_COMMITTER_DATE", stamp); + command +} + +pub fn available() -> bool { + Command::new("git") + .arg("--version") + .output() + .map(|out| out.status.success()) + .unwrap_or(false) +} + +fn combined(out: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) +} + +pub fn run(cwd: &Path, args: &[&str]) -> (bool, String) { + let out = command_at(cwd, PINNED_DATE) + .args(args) + .output() + .expect("git is available"); + (out.status.success(), combined(&out)) +} + +pub fn must(cwd: &Path, args: &[&str]) -> String { + let out = command_at(cwd, PINNED_DATE) + .args(args) + .output() + .expect("git is available"); + assert!(out.status.success(), "git {args:?}:\n{}", combined(&out)); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +pub fn feed(cwd: &Path, args: &[&str], stdin: &[u8]) -> (bool, String) { + let mut child = command_at(cwd, PINNED_DATE) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("git is available"); + child + .stdin + .take() + .expect("stdin was piped") + .write_all(stdin) + .expect("the write to git's stdin succeeds"); + let out = child.wait_with_output().expect("git exits"); + (out.status.success(), combined(&out)) +} + +pub fn fsck(bare: &Path) -> Result<(), String> { + match run( + bare, + &["fsck", "--no-dangling", "--no-reflogs", "--no-progress"], + ) { + (true, _) => Ok(()), + (false, report) => Err(report), + } +} + +pub fn commit(work: &Path, file: &str, contents: &str, message: &str) { + std::fs::write(work.join(file), contents).expect("fixture file is writable"); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", message]); +} + +pub fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_helper_pins_its_dates_so_the_same_sequence_yields_the_same_oid() { + if !available() { + return; + } + let oids = || { + let dir = tempfile::tempdir().unwrap(); + must(dir.path(), &["init", "-q", "-b", "main"]); + commit(dir.path(), "README.md", "kelp\n", "initial"); + let tree = must(dir.path(), &["hash-object", "-t", "tree", "-w", "--stdin"]); + let (ok, from_stdin) = feed(dir.path(), &["commit-tree", &tree, "-F", "-"], b"empty\n"); + assert!(ok, "{from_stdin}"); + ( + must(dir.path(), &["rev-parse", "HEAD"]), + from_stdin.trim().to_string(), + ) + }; + assert_eq!( + oids(), + oids(), + "an unpinned committer date would make every differential run disagree, \ + so a fixture writing through stdin pins the same dates as one that doesn't" + ); + } +} diff --git a/knot2/crates/knot-git/Cargo.toml b/knot2/crates/knot-git/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..f7c9748da0e60cc3b2b7321b32fbbc6b156bcd0e --- /dev/null +++ b/knot2/crates/knot-git/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "knot-git" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[features] +instrument = [] + +[dependencies] +knot-types = { workspace = true } +knot-storage = { workspace = true } +knot-resource = { workspace = true } +gix = { workspace = true } +gix-archive = { workspace = true } +gix-pack = { workspace = true } +gix-odb = { workspace = true } +gix-hash = { workspace = true } +flate2 = { workspace = true } +knot-cache = { workspace = true } +scc = { workspace = true } +thiserror = { workspace = true } +base64 = { workspace = true } +walkdir = { workspace = true } +memmap2 = { workspace = true } + +[dev-dependencies] +knot-fixtures = { workspace = true } +tempfile = { workspace = true } +proptest = { workspace = true } diff --git a/knot2/crates/knot-git/fuzz/.gitignore b/knot2/crates/knot-git/fuzz/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1a45eee7760d240bfa8ac1989088e0349169f264 --- /dev/null +++ b/knot2/crates/knot-git/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-git/fuzz/Cargo.lock b/knot2/crates/knot-git/fuzz/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..a298a96b5045ec5126893dd37ff4d99a2ec9734a --- /dev/null +++ b/knot2/crates/knot-git/fuzz/Cargo.lock @@ -0,0 +1,5247 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f04f6fef12d70d42a77b1433c9e0f065238479a6cefc4f5bab105e9873a3c3" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d0bd4c2f75335ad98052a37efb54f428b492f64340257143b3429c8a508fa7b" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] + +[[package]] +name = "bytesize" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "clru" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "gix" +version = "0.84.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae54ae0ebd1a5a3c3f8d95dd3b5ca6e63f4fed9bfd585e13801a97d7bde8f9ce" +dependencies = [ + "gix-actor", + "gix-archive", + "gix-attributes", + "gix-blame", + "gix-command", + "gix-commitgraph", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-ignore", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-negotiate", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-pathspec", + "gix-prompt", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-shallow", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "gix-worktree", + "gix-worktree-state", + "gix-worktree-stream", + "nonempty", + "parking_lot", + "regex", + "signal-hook", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-actor" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc998b8f746dda8565450d08a63b792ced9165d8c27a1ed3f02799ec6a7820f" +dependencies = [ + "bstr", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-archive" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16909cacc78936ab96f6c3be08379d0a2e88bfa3a7527972d2ed75c7517ef31e" +dependencies = [ + "bstr", + "flate2", + "gix-date", + "gix-error", + "gix-object", + "gix-path", + "gix-worktree-stream", + "rawzip", + "tar", +] + +[[package]] +name = "gix-attributes" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d43f12e246d3bf7ec624c8fc15ac4a4b62b7c4c6f586cb82be6c90bf84c9d02" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-bitmap" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ebef0c26ad305747649e727bbcd56a7b7910754eb7cea88f6dff6f93c51283" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-blame" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d39a0c14af94c2edaa5eefe06d5ef2cdea55316ae9a9321314288e3f55fa4c0" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-diff", + "gix-error", + "gix-hash", + "gix-object", + "gix-revwalk", + "gix-trace", + "gix-traverse", + "gix-worktree", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-chunk" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9faee47943b638e58ddd5e275a4906ad3e4b6c8584f1d41bd18ab9032ec52afb" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-command" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00706d4fef135ef4b01680d5218c6ee40cda8baf697b864296cbc887d19118f6" +dependencies = [ + "bstr", + "gix-path", + "gix-quote", + "gix-trace", + "shell-words", +] + +[[package]] +name = "gix-commitgraph" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" +dependencies = [ + "bstr", + "gix-chunk", + "gix-error", + "gix-hash", + "memmap2", + "nonempty", +] + +[[package]] +name = "gix-config" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2372d4b49ca28431e7d150cab9d25edc1890f0184bd57eb0e917c7799e63de" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-config-value" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" +dependencies = [ + "bitflags", + "bstr", + "gix-path", + "libc", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-credentials" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40cd22f0dd71988be12d6e78b1709de2370e1957c5f107ff31e56caeba3745d" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-date", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-trace", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-date" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ecab64a98bbac9f8e02990a9ea5e3c974a7d49b95f2bd70ad94ad22fa6b48c" +dependencies = [ + "bstr", + "gix-error", + "itoa", + "jiff", +] + +[[package]] +name = "gix-diff" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6d9528f32d94cef2edf39a1ac01fe5a0fc44ddbb18d9e44099936047c3302b" +dependencies = [ + "bstr", + "gix-attributes", + "gix-command", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-imara-diff", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-dir" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bb2a53a6fd917ec499ed0bfb5b6887de7a15bd79197dcea7c987938749a9f1" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-discover" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77bacdd12b7879d2178a80c58c2f319995e4654e1a7a23e3181e5c8a12b824f7" +dependencies = [ + "bstr", + "dunce", + "gix-fs", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-error" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57831e199be480af90dcd7e459abed8a174c09ec9a6e2cc8f7ca6c54598b06b" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "bytes", + "bytesize", + "crc32fast", + "crossbeam-channel", + "gix-path", + "gix-trace", + "gix-utils", + "libc", + "once_cell", + "parking_lot", + "prodash", + "thiserror 2.0.18", + "walkdir", + "zlib-rs", +] + +[[package]] +name = "gix-filter" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf74b7d16f6694ce4a3049074c41be0c7987105743674f1671807bd6dce09fa" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash", + "gix-object", + "gix-packetline", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-fs" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" +dependencies = [ + "bstr", + "fastrand", + "gix-features", + "gix-path", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-glob" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" +dependencies = [ + "bitflags", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-hashtable" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0e30b93eea8718baf7d8153fcb938e2926175bbf18097c09f1c01b6f0be0563" +dependencies = [ + "gix-hash", + "hashbrown 0.17.1", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-imara-diff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19753d40da53d0ec41604750eeb969097a90fb2d7f7992730d904541c04e2c19" +dependencies = [ + "bstr", + "hashbrown 0.17.1", +] + +[[package]] +name = "gix-index" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6b28cc592dc753adb58302bb14a64e412ee591a3bec77aa4df87bff74fa80d" +dependencies = [ + "bitflags", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils", + "gix-validate", + "hashbrown 0.17.1", + "itoa", + "libc", + "memmap2", + "rustix", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-lock" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-mailmap" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195fd20808055824531be2fd0d34136d900e5fbca3ffb0a3c07e8beeefb9c828" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-negotiate" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890c936a215bae25818c076cb881cb2e54d2c66ba947ba58b8dd47cff921bf55" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-object", + "gix-revwalk", +] + +[[package]] +name = "gix-object" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5cd857e29429c7213bdef3f5aef83f8cc124774fe8ae0d27b1607d218d6d525" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-odb" +version = "0.81.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d004c32858b1556f2d7874405edb3c97dc78fc09beaa87d57bb077ee2858a7d" +dependencies = [ + "arc-swap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "memmap2", + "parking_lot", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pack" +version = "0.71.0" +dependencies = [ + "clru", + "gix-chunk", + "gix-diff", + "gix-error", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-traverse", + "parking_lot", + "smallvec", + "thiserror 2.0.18", + "uluru", +] + +[[package]] +name = "gix-packetline" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb18337ba2830bb43367d1af43819c8c78f31337f079fc76d0f1f1750a173126" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-path" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa6ac14cd14939ea94a496ce7460daa6511c09f5b84757e9cfc6f9c8d0f93a6" +dependencies = [ + "bstr", + "gix-trace", + "gix-validate", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pathspec" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" +dependencies = [ + "bitflags", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-prompt" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee604d7746080ae7e1023bf47204bcc2c5f307bfbe2306a3c90b1bfd1a2c6d8" +dependencies = [ + "gix-command", + "gix-config-value", + "parking_lot", + "rustix", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-protocol" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51dea3acb390707ab868f1f9584f18449eb95d869deffae96768e47d303595ee" +dependencies = [ + "bstr", + "gix-date", + "gix-features", + "gix-hash", + "gix-ref", + "gix-shallow", + "gix-transport", + "gix-utils", + "maybe-async", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-quote" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" +dependencies = [ + "bstr", + "gix-error", + "gix-utils", +] + +[[package]] +name = "gix-ref" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c04f64c37eb7e6feb73c7060f8dc6f381cc5de5d53249bfd450bc48a86b2e8b" +dependencies = [ + "gix-actor", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils", + "gix-validate", + "memmap2", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-refspec" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b216ae06ec74b5f24ad0142026a997fb0a935b7410eaf9c1616fc3f0e6c5a6d3" +dependencies = [ + "bstr", + "gix-error", + "gix-glob", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-revision" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b47c88884dd3c1a19a39da19d10211fcdea2809aadc86869b6e824a1774340f" +dependencies = [ + "bitflags", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "nonempty", +] + +[[package]] +name = "gix-revwalk" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f5756abffe0917827aac683b13684ed99875bc398fa1f9b8f479b0681ef9e6" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-sec" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8519976e4c7e486270740a5400369f37940779b80bd1377d94cfa1125d01b3" +dependencies = [ + "bitflags", + "gix-path", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "gix-shallow" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" +dependencies = [ + "bstr", + "gix-hash", + "gix-lock", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-status" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22042e385d28a34275e029d98f4970285045be14b9073658ca897923f2ed8700" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-submodule" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3059890ef054066c22a94bfc6a3eaba0d806aedcd630a0bc9e5783fd88884781" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-tempfile" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27850097e1ff9515f46a0dad0f5f9c9d020e972727772dabab9450690c4adb22" +dependencies = [ + "dashmap", + "gix-fs", + "libc", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "gix-transport" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd0e34995b1aab0fa8dff2af8db726a0bfad3e119c89302604463264046e7ff" +dependencies = [ + "bstr", + "gix-command", + "gix-features", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-traverse" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8de590ecc86a3b2870665f2288324fa9f7f8672c7fc2d4e020fdd81cd1f7aed" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-url" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bb01ec69d55e82ccb7a19e264501ead4e6aac38463a8cebfdd81e22bb67ab2" +dependencies = [ + "bstr", + "gix-path", + "percent-encoding", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-utils" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c50966184123caf580ffa64e28031a878597f1c7fceb8fe19566c38eb1b771" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc6fc771c4063ba7cd2f47b91fb6076251c6a823b64b7fe7b8874b0fe4afae3" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-worktree" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef414ed275e8407cd5d53d301e83be19700b0dd3f859d2434417b58f454a2d1" +dependencies = [ + "bstr", + "gix-attributes", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate", +] + +[[package]] +name = "gix-worktree-state" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bffae8b3ca258fdd50370cd51f06deb4c76a3b43db3868bc28dde45ffa77d69" +dependencies = [ + "bstr", + "gix-features", + "gix-filter", + "gix-fs", + "gix-index", + "gix-object", + "gix-path", + "gix-worktree", + "io-close", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-worktree-stream" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25e9ed30100c63f7590bc581c225e53f731a53e06aa79a245739c07f7dcc557" +dependencies = [ + "gix-attributes", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-traverse", + "parking_lot", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "human_format" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaec953f16e5bcf6b8a3cb3aa959b17e5577dbd2693e94554c462c08be22624b" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jiff" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-static" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-git" +version = "0.1.0" +dependencies = [ + "base64", + "dashmap", + "flate2", + "gix", + "gix-archive", + "gix-bitmap", + "gix-hash", + "gix-pack", + "knot-resource", + "knot-types", + "moka", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "knot-git-fuzz" +version = "0.0.0" +dependencies = [ + "knot-git", + "libfuzzer-sys", +] + +[[package]] +name = "knot-resource" +version = "0.1.0" +dependencies = [ + "rustix", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multibase" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" +dependencies = [ + "base-x", + "base256emoji", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nonempty" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "bytesize", + "human_format", + "parking_lot", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uluru" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8a2469e56e6e5095c82ccd3afb98dad95f7af7929aab6d8ba8d6e0f73657da" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/knot2/crates/knot-git/fuzz/Cargo.toml b/knot2/crates/knot-git/fuzz/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..894023653dc760cc7817db9b5ca0163a446694dc --- /dev/null +++ b/knot2/crates/knot-git/fuzz/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "knot-git-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-git] +path = ".." + +[[bin]] +name = "patch" +path = "fuzz_targets/patch.rs" +test = false +doc = false +bench = false + +[patch.crates-io] +gix-pack = { path = "../../../third_party/gix-pack" } diff --git a/knot2/crates/knot-git/fuzz/fuzz_targets/patch.rs b/knot2/crates/knot-git/fuzz/fuzz_targets/patch.rs new file mode 100644 index 0000000000000000000000000000000000000000..ed1b1d00711859ca833e78725c8959f15820cb2b --- /dev/null +++ b/knot2/crates/knot-git/fuzz/fuzz_targets/patch.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_git::fuzz::patch(data); +}); diff --git a/knot2/crates/knot-git/src/archive.rs b/knot2/crates/knot-git/src/archive.rs new file mode 100644 index 0000000000000000000000000000000000000000..0dd218df3c910869d0d55195ea430523399bd1bc --- /dev/null +++ b/knot2/crates/knot-git/src/archive.rs @@ -0,0 +1,332 @@ +use std::io::{Seek, SeekFrom, Write}; +use std::sync::atomic::AtomicBool; + +use gix::bstr::BString; +use knot_types::{Oid, ParseError}; + +use crate::error::{GitError, backend}; +use crate::objects::MAX_TREE_DEPTH; +use crate::repo::Repo; + +const TAR_BLOCK: u64 = 512; + +knot_types::scalar_newtype! { + pub struct ArchiveLimit(u64); +} + +impl Default for ArchiveLimit { + fn default() -> Self { + Self::new(1024 * 1024 * 1024) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ArchiveFormat { + Tar, + TarGz, + Zip, +} + +impl ArchiveFormat { + fn gix(self) -> gix_archive::Format { + match self { + ArchiveFormat::Tar => gix_archive::Format::Tar, + ArchiveFormat::TarGz => gix_archive::Format::TarGz { + compression_level: None, + }, + ArchiveFormat::Zip => gix_archive::Format::Zip { + compression_level: None, + }, + } + } +} + +fn inside_root(value: String, kind: &'static str, limit: usize) -> Result { + let safe = value.len() <= limit + && !value.contains('\0') + && !value.starts_with(['/', '\\']) + && value.split(['/', '\\']).all(|component| component != ".."); + match safe { + true => Ok(value), + false => Err(ParseError::Invalid { kind, value }), + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArchivePrefix(String); + +impl ArchivePrefix { + pub const MAX_BYTES: usize = 255; + + pub fn new(value: impl Into) -> Result { + inside_root(value.into(), "archive prefix", Self::MAX_BYTES).map(Self) + } + + pub fn stem(repo_name: &str, safe_ref: &str) -> Self { + let joined = format!("{repo_name}-{safe_ref}").replace(['/', '\\', '\0'], "-"); + Self( + joined + .char_indices() + .take_while(|(offset, character)| offset + character.len_utf8() <= Self::MAX_BYTES) + .map(|(_, character)| character) + .collect(), + ) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_tree_prefix(self) -> TreePrefix { + TreePrefix(format!("{}/", self.0)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TreePrefix(String); + +impl TreePrefix { + pub const MAX_BYTES: usize = ArchivePrefix::MAX_BYTES + 1; + + pub fn new(value: impl Into) -> Result { + inside_root(value.into(), "archive tree prefix", Self::MAX_BYTES).map(Self) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Repo { + pub fn peel_to_tree(&self, oid: Oid) -> Result { + self.git() + .find_object(oid.object_id()) + .map_err(backend)? + .peel_to_tree() + .map(|tree| Oid::from(tree.id)) + .map_err(backend) + } + + pub fn write_archive( + &self, + tree: Oid, + format: ArchiveFormat, + prefix: Option<&TreePrefix>, + limit: ArchiveLimit, + out: impl std::io::Write + std::io::Seek, + ) -> Result<(), GitError> { + self.bound_archive_source(tree.object_id(), limit, MAX_TREE_DEPTH, &mut 0)?; + let (stream, _index) = self + .git() + .worktree_stream(tree.object_id()) + .map_err(backend)?; + let interrupt = AtomicBool::new(false); + let mut spool = BoundedSpool { + inner: out, + position: 0, + limit, + overflowed: false, + }; + let written = self.git().worktree_archive( + stream, + &mut spool, + gix::progress::Discard, + &interrupt, + gix_archive::Options { + format: format.gix(), + tree_prefix: prefix.map(|prefix| BString::from(prefix.as_str())), + modification_time: 0, + }, + ); + match (written, spool.overflowed) { + (_, true) => Err(GitError::ArchiveTooLarge { limit }), + (Ok(()), false) => Ok(()), + (Err(error), false) => Err(backend(error)), + } + } + + fn bound_archive_source( + &self, + tree: gix::ObjectId, + limit: ArchiveLimit, + nesting: usize, + spooled: &mut u64, + ) -> Result<(), GitError> { + if nesting == 0 { + return Err(GitError::DepthExceeded("tree nesting")); + } + if tree == gix::ObjectId::empty_tree(self.git().object_hash()) { + return Ok(()); + } + let object = self.git().find_tree(tree).map_err(backend)?; + let decoded = object + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + decoded.entries.iter().try_for_each(|entry| { + let oid = entry.oid.to_owned(); + *spooled = spooled.saturating_add(TAR_BLOCK); + match entry.mode.kind() { + _ if *spooled > limit.get() => Err(GitError::ArchiveTooLarge { limit }), + gix::objs::tree::EntryKind::Commit => Ok(()), + gix::objs::tree::EntryKind::Tree => { + self.bound_archive_source(oid, limit, nesting - 1, spooled) + } + _ => { + let content = self.blob_size(Oid::from(oid))?; + *spooled = spooled.saturating_add(content.next_multiple_of(TAR_BLOCK)); + match *spooled > limit.get() { + true => Err(GitError::ArchiveTooLarge { limit }), + false => Ok(()), + } + } + } + }) + } +} + +struct BoundedSpool { + inner: W, + position: u64, + limit: ArchiveLimit, + overflowed: bool, +} + +impl Write for BoundedSpool { + fn write(&mut self, data: &[u8]) -> std::io::Result { + let remaining = self.limit.get().saturating_sub(self.position); + if data.len() as u64 > remaining { + self.overflowed = true; + return Err(std::io::Error::from(std::io::ErrorKind::WriteZero)); + } + let written = self.inner.write(data)?; + self.position = self.position.saturating_add(written as u64); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +impl Seek for BoundedSpool { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + let position = self.inner.seek(pos)?; + self.position = position; + Ok(position) + } +} + +#[cfg(test)] +mod tests { + use super::{ArchiveLimit, ArchivePrefix, BoundedSpool, TreePrefix}; + use std::io::{Seek, SeekFrom, Write}; + + fn spool(limit: u64) -> BoundedSpool>> { + BoundedSpool { + inner: std::io::Cursor::new(Vec::new()), + position: 0, + limit: ArchiveLimit::new(limit), + overflowed: false, + } + } + + #[test] + fn the_spool_refuses_the_write_that_would_pass_the_limit() { + let mut spool = spool(8); + assert!(spool.write_all(b"12345678").is_ok()); + assert!(!spool.overflowed); + assert!(spool.write_all(b"9").is_err()); + assert!(spool.overflowed); + assert_eq!( + spool.inner.into_inner(), + b"12345678", + "the refused write never reaches the inner writer" + ); + } + + #[test] + fn a_seek_backwards_re_credits_the_budget_the_zip_writer_rewinds_over() { + let mut spool = spool(8); + spool.write_all(b"12345678").unwrap(); + spool.seek(SeekFrom::Start(4)).unwrap(); + assert_eq!(spool.position, 4); + spool + .write_all(b"abcd") + .expect("rewriting bytes already counted stays within the limit"); + assert!(!spool.overflowed); + } + + #[test] + fn a_write_whose_length_would_overflow_the_position_is_refused() { + let mut spool = spool(u64::MAX); + spool.position = u64::MAX; + assert!( + spool.write_all(b"1").is_err(), + "the position saturates at u64::MAX, so the spool must refuse the write" + ); + assert!(spool.overflowed); + } + + #[test] + fn a_plain_nested_prefix_is_accepted() { + assert!(ArchivePrefix::new("squid-main").is_ok()); + assert!(ArchivePrefix::new("nested/path").is_ok()); + } + + #[test] + fn traversal_is_rejected_across_both_separators() { + assert!(ArchivePrefix::new("../escape").is_err()); + assert!(ArchivePrefix::new("nested/../escape").is_err()); + assert!(ArchivePrefix::new("..\\escape").is_err()); + assert!(ArchivePrefix::new("nested\\..\\escape").is_err()); + assert!( + ArchivePrefix::new("dotted-..-name/").is_ok(), + "a component that merely contains dot-dot is not a traversal" + ); + assert!( + TreePrefix::new("dotted-..-name//").is_ok(), + "TreePrefix accepts the empty final component that a trailing separator adds" + ); + } + + #[test] + fn absolute_and_null_bearing_prefixes_are_rejected() { + assert!(ArchivePrefix::new("/etc").is_err()); + assert!(ArchivePrefix::new("\\windows").is_err()); + assert!(ArchivePrefix::new("good\0bad").is_err()); + } + + #[test] + fn a_prefix_is_bounded_and_a_stem_always_parses() { + let longest = "u".repeat(ArchivePrefix::MAX_BYTES); + assert!(ArchivePrefix::new(format!("{longest}u")).is_err()); + + let tree = ArchivePrefix::new(longest).unwrap().into_tree_prefix(); + assert_eq!(tree.as_str().len(), TreePrefix::MAX_BYTES); + assert!(TreePrefix::new(tree.as_str()).is_ok()); + assert!(TreePrefix::new("u".repeat(TreePrefix::MAX_BYTES + 1)).is_err()); + + let (long_name, truncated) = ("ü".repeat(400), "ü".repeat(127)); + for (repo_name, safe_ref, want) in [ + ("squid", "main", "squid-main"), + ("did:plc:limpet", "feat/uni", "did:plc:limpet-feat-uni"), + ("kelp/../conch", "..", "kelp-..-conch-.."), + ("/etc", "\0", "-etc--"), + ("", "", "-"), + (long_name.as_str(), "main", truncated.as_str()), + ] { + let stem = ArchivePrefix::stem(repo_name, safe_ref); + assert_eq!(stem.as_str(), want); + assert!( + ArchivePrefix::new(stem.as_str()).is_ok(), + "a stem passes the check that its own constructor skips, within {} bytes and on a char boundary", + ArchivePrefix::MAX_BYTES + ); + assert_eq!( + stem.into_tree_prefix().as_str(), + format!("{want}/"), + "into_tree_prefix appends the separator that puts a stem at the archive root" + ); + } + } +} diff --git a/knot2/crates/knot-git/src/base85.rs b/knot2/crates/knot-git/src/base85.rs new file mode 100644 index 0000000000000000000000000000000000000000..218b377d5d214c11156af9c0547015a4f4aeaca4 --- /dev/null +++ b/knot2/crates/knot-git/src/base85.rs @@ -0,0 +1,154 @@ +use std::sync::LazyLock; + +const MARKERS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +const BYTES_PER_LINE: usize = MARKERS.len(); + +const ALPHABET: &[u8] = + b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; + +static DIGITS: LazyLock<[Option; 256]> = LazyLock::new(|| { + std::array::from_fn(|byte| { + ALPHABET + .iter() + .position(|&candidate| candidate as usize == byte) + .map(|digit| digit as u8) + }) +}); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Malformed; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct LineLength(u8); + +impl LineLength { + fn of(bytes: usize) -> Option { + bytes + .checked_sub(1) + .filter(|index| *index < MARKERS.len()) + .map(|index| Self(index as u8)) + } + + fn from_marker(marker: u8) -> Option { + MARKERS + .iter() + .position(|&candidate| candidate == marker) + .map(|index| Self(index as u8)) + } + + fn marker(self) -> char { + MARKERS[self.0 as usize] as char + } + + fn get(self) -> usize { + self.0 as usize + 1 + } +} + +pub(crate) fn encode(packed: &[u8], out: &mut String) { + out.reserve(encoded_len(packed.len() as u64) as usize); + packed.chunks(BYTES_PER_LINE).for_each(|chunk| { + let length = LineLength::of(chunk.len()) + .expect("chunking by the line width yields 1..=BYTES_PER_LINE bytes"); + out.push(length.marker()); + chunk.chunks(4).for_each(|group| { + let word = group + .iter() + .fold(0u32, |acc, &byte| (acc << 8) | byte as u32) + << (8 * (4 - group.len())); + (0..5).rev().for_each(|power| { + let digit = (word / 85u32.pow(power)) % 85; + out.push(ALPHABET[digit as usize] as char); + }); + }); + out.push('\n'); + }); +} + +pub(crate) fn encoded_len(packed: u64) -> u64 { + packed.div_ceil(4) * 5 + packed.div_ceil(BYTES_PER_LINE as u64) * 2 +} + +pub(crate) fn decode_line(line: &str, out: &mut Vec) -> Result<(), Malformed> { + let (marker, data) = line.as_bytes().split_first().ok_or(Malformed)?; + let length = LineLength::from_marker(*marker).ok_or(Malformed)?; + if data.len() != length.get().div_ceil(4) * 5 { + return Err(Malformed); + } + data.chunks(5).enumerate().try_for_each(|(group, chunk)| { + let word = chunk + .iter() + .try_fold(0u64, |acc, &character| { + DIGITS[character as usize].map(|digit| acc * 85 + digit as u64) + }) + .filter(|&word| word <= u32::MAX as u64) + .ok_or(Malformed)?; + let take = (length.get() - group * 4).min(4); + out.extend_from_slice(&(word as u32).to_be_bytes()[..take]); + Ok(()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encoding_round_trips_at_every_payload_length() { + (0..=260usize).for_each(|len| { + let bytes: Vec = (0..len) + .map(|index| (index as u8).wrapping_mul(37) ^ 0x5a) + .collect(); + let mut encoded = String::new(); + encode(&bytes, &mut encoded); + assert_eq!( + encoded_len(len as u64), + encoded.len() as u64, + "encoded length of a {len} byte payload" + ); + let decoded = encoded.lines().fold(Vec::new(), |mut out, line| { + decode_line(line, &mut out).unwrap(); + out + }); + assert_eq!(decoded, bytes, "round trip of a {len} byte payload"); + }); + } + + #[test] + fn line_markers_map_both_ways() { + (1..=BYTES_PER_LINE).for_each(|len| { + let length = LineLength::of(len).unwrap(); + assert_eq!( + LineLength::from_marker(length.marker() as u8), + Some(length), + "marker for a line of {len} bytes" + ); + }); + assert_eq!(LineLength::of(0), None); + assert_eq!(LineLength::of(BYTES_PER_LINE + 1), None); + assert_eq!(LineLength::of(1).unwrap().marker(), 'A'); + assert_eq!(LineLength::of(26).unwrap().marker(), 'Z'); + assert_eq!(LineLength::of(27).unwrap().marker(), 'a'); + assert_eq!(LineLength::of(BYTES_PER_LINE).unwrap().marker(), 'z'); + let mut encoded = String::new(); + encode(&[0u8; BYTES_PER_LINE + 1], &mut encoded); + assert_eq!( + encoded.lines().map(|line| &line[..1]).collect::>(), + vec!["z", "A"], + "a payload past the line width gets a second marker" + ); + } + + #[test] + fn the_marker_sets_the_line_length_and_anything_else_is_malformed() { + let mut out = Vec::new(); + decode_line("D00000", &mut out).unwrap(); + decode_line("B00000", &mut out).unwrap(); + assert_eq!(out, vec![0, 0, 0, 0, 0, 0]); + assert_eq!(decode_line("D0000", &mut Vec::new()), Err(Malformed)); + assert_eq!(decode_line("D0\"000", &mut Vec::new()), Err(Malformed)); + assert_eq!(decode_line("?00000", &mut Vec::new()), Err(Malformed)); + assert_eq!(decode_line("", &mut Vec::new()), Err(Malformed)); + } +} diff --git a/knot2/crates/knot-git/src/bitmap/bitset.rs b/knot2/crates/knot-git/src/bitmap/bitset.rs new file mode 100644 index 0000000000000000000000000000000000000000..f755ef13f29de63f8253d2042856e36f640c53e0 --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/bitset.rs @@ -0,0 +1,188 @@ +use super::BitPosition; +use super::ewah::Ewah; +use crate::error::GitError; + +#[derive(Clone)] +pub(crate) struct Bitset { + words: Vec, +} + +impl Bitset { + pub(crate) fn zeros(num_bits: usize) -> Self { + Self { + words: vec![0u64; num_bits.div_ceil(64)], + } + } + + pub(crate) fn set(&mut self, index: BitPosition) { + let index = index.get() as usize; + self.words[index / 64] |= 1u64 << (index % 64); + } + + pub(crate) fn get(&self, index: BitPosition) -> bool { + let index = index.get() as usize; + self.words[index / 64] & (1u64 << (index % 64)) != 0 + } + + pub(crate) fn union_ewah(&mut self, vector: &Ewah, num_bits: usize) -> Result<(), GitError> { + self.apply_ewah(vector, num_bits, |word, mask| *word |= mask) + } + + pub(crate) fn xor_ewah(&mut self, vector: &Ewah, num_bits: usize) -> Result<(), GitError> { + self.apply_ewah(vector, num_bits, |word, mask| *word ^= mask) + } + + fn apply_ewah( + &mut self, + vector: &Ewah, + num_bits: usize, + op: impl Fn(&mut u64, u64), + ) -> Result<(), GitError> { + vector.for_each_word(|index, word| { + if word == 0 { + return Ok(()); + } + let valid = match index.cmp(&(num_bits / 64)) { + std::cmp::Ordering::Less => !0, + std::cmp::Ordering::Equal => (1u64 << (num_bits % 64)) - 1, + std::cmp::Ordering::Greater => 0, + }; + if word & !valid != 0 { + return Err(GitError::Backend( + "bitmap entry is wider than the object count".to_string(), + )); + } + op(&mut self.words[index], word); + Ok(()) + }) + } + + pub(crate) fn xor(&self, other: &Bitset) -> Bitset { + Bitset { + words: self + .words + .iter() + .zip(&other.words) + .map(|(a, b)| a ^ b) + .collect(), + } + } + + pub(crate) fn to_ewah(&self, num_bits: usize) -> Result { + Ewah::compress(&self.words, num_bits) + } + + pub(crate) fn union_with(&mut self, other: &Bitset) { + self.words + .iter_mut() + .zip(&other.words) + .for_each(|(slot, bits)| *slot |= *bits); + } + + pub(crate) fn count_common(&self, other: &Bitset) -> u64 { + self.words + .iter() + .zip(&other.words) + .map(|(a, b)| u64::from((a & b).count_ones())) + .sum() + } + + pub(crate) fn ones(&self) -> impl Iterator + '_ { + self.words + .iter() + .enumerate() + .flat_map(|(word, bits)| WordBits { + remaining: *bits, + base: (word as u32) * 64, + }) + } + + pub(crate) fn difference_indices<'a>( + &'a self, + other: &'a Bitset, + ) -> impl Iterator + 'a { + self.words + .iter() + .zip(&other.words) + .enumerate() + .flat_map(|(word, (present, absent))| WordBits { + remaining: present & !absent, + base: (word as u32) * 64, + }) + } +} + +struct WordBits { + remaining: u64, + base: u32, +} + +impl Iterator for WordBits { + type Item = BitPosition; + + fn next(&mut self) -> Option { + (self.remaining != 0).then(|| { + let offset = self.remaining.trailing_zeros(); + self.remaining &= self.remaining - 1; + BitPosition::new(self.base + offset) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn set_bits(bits: &Bitset, other: &Bitset) -> Vec { + bits.difference_indices(other) + .map(BitPosition::get) + .collect() + } + + #[test] + fn difference_yields_ascending_set_minus_set() { + let mut want = Bitset::zeros(130); + [1u32, 64, 65, 129] + .into_iter() + .for_each(|bit| want.set(BitPosition::new(bit))); + let mut have = Bitset::zeros(130); + [64u32, 129] + .into_iter() + .for_each(|bit| have.set(BitPosition::new(bit))); + assert_eq!(set_bits(&want, &have), vec![1, 65]); + } + + #[test] + fn union_accumulates_both_operands() { + let mut acc = Bitset::zeros(70); + let mut other = Bitset::zeros(70); + acc.set(BitPosition::new(3)); + other.set(BitPosition::new(3)); + other.set(BitPosition::new(69)); + acc.union_with(&other); + let empty = Bitset::zeros(70); + assert_eq!(set_bits(&acc, &empty), vec![3, 69]); + } + + #[test] + fn ewah_round_trips_runs_literals_and_a_partial_tail() { + let num_bits = 64 * 200 + 10; + let mut bits = Bitset::zeros(num_bits); + (64 * 50..64 * 120) + .chain([3, 64 * 130 + 7, num_bits - 1]) + .for_each(|bit| bits.set(BitPosition::new(bit as u32))); + let mut encoded = Vec::new(); + bits.to_ewah(num_bits).unwrap().write_to(&mut encoded); + assert!( + encoded.len() < 100, + "runs collapse: {} bytes", + encoded.len() + ); + let (vector, rest) = Ewah::decode(&encoded).unwrap(); + assert!(rest.is_empty()); + let mut decoded = Bitset::zeros(num_bits); + decoded.xor_ewah(&vector, num_bits).unwrap(); + let empty = Bitset::zeros(num_bits); + assert_eq!(set_bits(&decoded, &empty), set_bits(&bits, &empty)); + } +} diff --git a/knot2/crates/knot-git/src/bitmap/count.rs b/knot2/crates/knot-git/src/bitmap/count.rs new file mode 100644 index 0000000000000000000000000000000000000000..c8e2f1adfb8cf8c41ed42bf5b5492a671f725abb --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/count.rs @@ -0,0 +1,66 @@ +use std::collections::HashSet; +use std::time::Instant; + +use knot_types::Oid; + +use super::bitset::Bitset; +use super::reader::Bitmaps; +use super::revindex::RevIndex; +use crate::error::GitError; +use crate::repo::Repo; + +pub(super) struct Indexed<'a> { + pub(super) rev: &'a RevIndex, + pub(super) maps: &'a Bitmaps<'a>, +} + +/// Counts commits reachable from `tip`, unioning stored bitmaps where they exist and walking +/// everything else. `None` once `deadline` passes. +pub(super) fn count( + repo: &Repo, + tip: Oid, + deadline: Instant, + indexed: Option>, +) -> Result, GitError> { + let mut bits = indexed.as_ref().map(|index| Bitset::zeros(index.rev.len())); + let mut outside = HashSet::new(); + let mut commits = vec![tip]; + while let Some(commit) = commits.pop() { + if Instant::now() >= deadline { + return Ok(None); + } + let located = match &indexed { + Some(index) => index.rev.locate(commit)?, + None => None, + }; + match (located, &indexed, bits.as_mut()) { + (Some((position, bit)), Some(index), Some(bits)) => { + if bits.get(bit) { + continue; + } + if let Some(stored) = index.maps.bitmap(position)? { + bits.union_with(&stored); + continue; + } + bits.set(bit); + } + _ => { + if !outside.insert(commit) { + continue; + } + } + } + let (_, parents) = repo.commit_tree_and_parents(commit)?; + commits.extend(parents.into_iter().map(Oid::from)); + } + let in_bitmap = match (indexed, bits) { + (Some(index), Some(bits)) => { + let [commit_type, ..] = index.maps.types()?; + let mut mask = Bitset::zeros(index.rev.len()); + mask.union_ewah(&commit_type, index.rev.len())?; + bits.count_common(&mask) + } + _ => 0, + }; + Ok(Some(in_bitmap + outside.len() as u64)) +} diff --git a/knot2/crates/knot-git/src/bitmap/ewah.rs b/knot2/crates/knot-git/src/bitmap/ewah.rs new file mode 100644 index 0000000000000000000000000000000000000000..894b0e69d78f438b6a4cc99952c9fba8b0270fef --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/ewah.rs @@ -0,0 +1,170 @@ +// Adapted from gitoxide's gix-bitmap (MIT OR Apache-2.0). + +use crate::error::GitError; + +const RUN_LEN_MAX: u64 = (1 << 32) - 1; +const LITERALS_MAX: u64 = (1 << 31) - 1; + +#[derive(Clone)] +pub(crate) struct Ewah { + num_bits: u32, + words: Vec, + rlw: u32, +} + +impl Ewah { + pub(crate) fn compress(input: &[u64], num_bits: usize) -> Result { + let input = &input[..num_bits.div_ceil(64)]; + let num_bits = u32::try_from(num_bits) + .map_err(|_| GitError::Backend("ewah bit count exceeds u32".to_string()))?; + let full = num_bits as usize / 64; + let clean = |at: usize| at < full && (input[at] == 0 || input[at] == !0); + let mut words = Vec::new(); + let mut rlw; + let mut at = 0; + loop { + rlw = words.len(); + words.push(0u64); + let run_bit = clean(at) && input[at] == !0; + let mut run = 0u64; + while clean(at) && (input[at] == !0) == run_bit && run < RUN_LEN_MAX { + run += 1; + at += 1; + } + let mut literals = 0u64; + while at < input.len() && !clean(at) && literals < LITERALS_MAX { + words.push(input[at]); + literals += 1; + at += 1; + } + words[rlw] = u64::from(run_bit) | run << 1 | literals << 33; + if at == input.len() { + break; + } + } + Ok(Self { + num_bits, + words, + rlw: rlw as u32, + }) + } + + pub(crate) fn decode(data: &[u8]) -> Result<(Self, &[u8]), GitError> { + let (num_bits, data) = take_u32(data)?; + let (len, data) = take_u32(data)?; + let (body, data) = (len as usize) + .checked_mul(8) + .and_then(|size| data.split_at_checked(size)) + .ok_or_else(truncated)?; + let words = body + .as_chunks::<8>() + .0 + .iter() + .map(|word| u64::from_be_bytes(*word)) + .collect(); + let (rlw, data) = take_u32(data)?; + Ok(( + Self { + num_bits, + words, + rlw, + }, + data, + )) + } + + pub(crate) fn write_to(&self, out: &mut Vec) { + out.extend_from_slice(&self.num_bits.to_be_bytes()); + out.extend_from_slice(&(self.words.len() as u32).to_be_bytes()); + self.words + .iter() + .for_each(|word| out.extend_from_slice(&word.to_be_bytes())); + out.extend_from_slice(&self.rlw.to_be_bytes()); + } + + pub(crate) fn word_count(&self) -> usize { + self.words.len() + } + + pub(crate) fn for_each_word( + &self, + mut f: impl FnMut(usize, u64) -> Result<(), GitError>, + ) -> Result<(), GitError> { + let limit = (self.num_bits as usize).div_ceil(64); + let malformed = || GitError::Backend("malformed ewah bitmap".to_string()); + let mut at = 0usize; + let mut words = self.words.iter(); + while let Some(marker) = words.next() { + let run = ((marker >> 1) & RUN_LEN_MAX) as usize; + let end = at + .checked_add(run) + .filter(|end| *end <= limit) + .ok_or_else(malformed)?; + if marker & 1 == 1 { + (at..end).try_for_each(|index| f(index, !0))?; + } + at = end; + for _ in 0..marker >> 33 { + let word = words.next().filter(|_| at < limit).ok_or_else(malformed)?; + f(at, *word)?; + at += 1; + } + } + Ok(()) + } +} + +fn take_u32(data: &[u8]) -> Result<(u32, &[u8]), GitError> { + data.split_at_checked(4) + .map(|(num, rest)| (u32::from_be_bytes(num.try_into().unwrap()), rest)) + .ok_or_else(truncated) +} + +fn truncated() -> GitError { + GitError::Backend("truncated ewah bitmap".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encode(num_bits: u32, words: &[u64]) -> Vec { + let mut out = Vec::new(); + Ewah { + num_bits, + words: words.to_vec(), + rlw: 0, + } + .write_to(&mut out); + out + } + + fn walk(bytes: &[u8]) -> Result, GitError> { + let (vector, _) = Ewah::decode(bytes)?; + let mut seen = Vec::new(); + vector.for_each_word(|index, word| { + seen.push((index, word)); + Ok(()) + })?; + Ok(seen) + } + + #[test] + fn rejects_truncation_and_words_past_the_bit_count() { + assert_eq!( + walk(&encode(128, &[1 | 1 << 1 | 1 << 33, 5])).unwrap(), + vec![(0, !0), (1, 5)] + ); + let whole = encode(64, &[1 << 33, 5]); + assert!(walk(&whole[..whole.len() - 1]).is_err()); + assert!(walk(&encode(64, &[2 << 1])).is_err(), "run past num_bits"); + assert!( + walk(&encode(64, &[2 << 33, 1, 2])).is_err(), + "literal past num_bits" + ); + assert!( + walk(&encode(64, &[1 << 33])).is_err(), + "missing literal word" + ); + } +} diff --git a/knot2/crates/knot-git/src/bitmap/midx.rs b/knot2/crates/knot-git/src/bitmap/midx.rs new file mode 100644 index 0000000000000000000000000000000000000000..f190a215f2f81eb61b5c8450466e2b339b3a1680 --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/midx.rs @@ -0,0 +1,77 @@ +use std::path::{Path, PathBuf}; + +use super::BitmapIndex; +use super::revindex::{self, Order, OrderTable, Positions, RevIndex}; +use super::writer; +use crate::error::GitError; +use crate::repo::Repo; + +const MIDX_ALLOC_LIMIT_BYTES: usize = 16 * 1024 * 1024; + +fn midx_path(objects_dir: &Path) -> PathBuf { + objects_dir.join("pack").join("multi-pack-index") +} + +fn sidecar(objects_dir: &Path, checksum: &gix_hash::ObjectId, ext: &str) -> PathBuf { + objects_dir + .join("pack") + .join(format!("multi-pack-index-{}.{ext}", checksum.to_hex())) +} + +pub(super) fn write(repo: &Repo, previous: Option<&BitmapIndex>) -> Result { + let kind = repo.object_format().kind(); + let objects_dir = repo.objects_dir(); + let file = match gix_pack::multi_index::File::at( + midx_path(&objects_dir), + Some(MIDX_ALLOC_LIMIT_BYTES), + ) { + Ok(file) => file, + Err(_) => return Ok(false), + }; + let order = OrderTable::from_file(&file); + if order.len() == 0 { + return Ok(false); + } + + let _boost = knot_resource::saturate(); + let previous = previous.and_then(|index| writer::Previous::new(index, kind, &order)); + let categories = writer::categories_in_bit_order(repo, &order, previous.as_ref())?; + let types = writer::type_index_bits(&categories)?; + let selected = writer::selected_entries(repo, &order, &categories, previous.as_ref())?; + if selected.is_empty() { + return Ok(false); + } + + let checksum = file.checksum(); + let bytes = writer::assemble(kind, &checksum, &types, &selected)?; + + revindex::write( + &sidecar(&objects_dir, &checksum, "rev"), + &order, + kind, + &checksum, + )?; + writer::install(&sidecar(&objects_dir, &checksum, "bitmap"), &bytes)?; + Ok(true) +} + +pub(super) fn open(repo: &Repo) -> Result, GitError> { + let kind = repo.object_format().kind(); + let objects_dir = repo.objects_dir(); + let file = match gix_pack::multi_index::File::at( + midx_path(&objects_dir), + Some(MIDX_ALLOC_LIMIT_BYTES), + ) { + Ok(file) => file, + Err(_) => return Ok(None), + }; + let checksum = file.checksum(); + let Some(bytes) = super::map_file(&sidecar(&objects_dir, &checksum, "bitmap")) else { + return Ok(None); + }; + let rev_path = sidecar(&objects_dir, &checksum, "rev"); + Ok( + RevIndex::open(&rev_path, Positions::Midx(file), kind, &checksum)? + .map(|rev| BitmapIndex { rev, bytes }), + ) +} diff --git a/knot2/crates/knot-git/src/bitmap/mod.rs b/knot2/crates/knot-git/src/bitmap/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..5096cc2a52ae498e9a137a16c15e24022c4e49d0 --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/mod.rs @@ -0,0 +1,180 @@ +use std::path::{Path, PathBuf}; + +use knot_types::Oid; + +use crate::error::GitError; +use crate::objects::{Haves, Wants}; +use crate::repo::Repo; + +mod bitset; +mod count; +mod ewah; +mod midx; +mod reach; +mod reader; +mod revindex; +mod writer; + +use bitset::Bitset; +use revindex::{Positions, RevIndex}; + +knot_types::scalar_newtype! { + pub(crate) struct BitPosition(u32); + pub(crate) struct IndexPosition(u32) => ordered; + pub(crate) struct BitmapEntryOffset(u64); +} + +pub fn write_bitmap( + repo: &Repo, + pack_idx: &Path, + previous: Option<&BitmapIndex>, +) -> Result { + writer::write(repo, pack_idx, previous) +} + +pub fn write_midx_bitmap(repo: &Repo, previous: Option<&BitmapIndex>) -> Result { + midx::write(repo, previous) +} + +pub fn open_bitmap_index(repo: &Repo) -> Result, GitError> { + match single_pack(repo)? { + Some(loaded) => Ok(Some(loaded)), + None => midx::open(repo), + } +} + +pub fn reachable_via_bitmap( + repo: &Repo, + wants: Wants<'_>, + haves: Haves<'_>, +) -> Result>, GitError> { + let Some(loaded) = open_bitmap_index(repo)? else { + return Ok(None); + }; + Ok(loaded.resolve(repo, wants, haves)?.map(|mut resolved| { + resolved.in_bitmap.extend(resolved.outside); + resolved.in_bitmap + })) +} + +pub fn unreachable_via_bitmap( + repo: &Repo, + roots: &[Oid], + candidates: &[Oid], +) -> Result>, GitError> { + let Some(loaded) = open_bitmap_index(repo)? else { + return Ok(None); + }; + let maps = reader::parse(&loaded.bytes, repo.object_format().kind(), loaded.rev.len())?; + let reached = reach::reach(repo, &loaded.rev, &maps, roots, None)?; + let mut unreachable = Vec::new(); + for &candidate in candidates { + let bit = loaded.rev.locate(candidate)?.map(|(_, bit)| bit); + if !reached.has(bit, candidate) { + unreachable.push(candidate); + } + } + Ok(Some(unreachable)) +} + +/// Commits reachable from `tip` (which must be a commit), or `None` if `deadline` passes first. +pub fn count_commits( + repo: &Repo, + tip: Oid, + deadline: std::time::Instant, +) -> Result, GitError> { + let Some(loaded) = open_bitmap_index(repo)? else { + return count::count(repo, tip, deadline, None); + }; + let maps = reader::parse(&loaded.bytes, repo.object_format().kind(), loaded.rev.len())?; + let indexed = count::Indexed { + rev: &loaded.rev, + maps: &maps, + }; + count::count(repo, tip, deadline, Some(indexed)) +} + +pub struct BitmapIndex { + rev: RevIndex, + bytes: memmap2::Mmap, +} + +impl BitmapIndex { + fn resolve( + &self, + repo: &Repo, + wants: Wants<'_>, + haves: Haves<'_>, + ) -> Result, GitError> { + if !wants.as_slice().iter().all(|want| repo.contains(*want)) { + return Ok(None); + } + let maps = reader::parse(&self.bytes, repo.object_format().kind(), self.rev.len())?; + let haves: Vec = haves + .as_slice() + .iter() + .copied() + .filter(|have| repo.contains(*have)) + .collect(); + let have = reach::reach(repo, &self.rev, &maps, &haves, None)?; + let want = reach::reach(repo, &self.rev, &maps, wants.as_slice(), Some(&have))?; + want.minus(&have, &self.rev).map(Some) + } +} + +fn single_pack(repo: &Repo) -> Result, GitError> { + let kind = repo.object_format().kind(); + let Some(pack_idx) = bitmapped_pack(&repo.objects_dir()) else { + return Ok(None); + }; + let Some(bytes) = map_file(&pack_idx.with_extension("bitmap")) else { + return Ok(None); + }; + let index = gix_pack::index::File::at(&pack_idx, kind) + .map_err(|error| GitError::Backend(format!("open pack index: {error}")))?; + let checksum = index.pack_checksum(); + Ok(RevIndex::open( + &pack_idx.with_extension("rev"), + Positions::Pack(index), + kind, + &checksum, + )? + .map(|rev| BitmapIndex { rev, bytes })) +} + +fn map_file(path: &Path) -> Option { + let file = std::fs::File::open(path).ok()?; + unsafe { memmap2::Mmap::map(&file) }.ok() +} + +pub fn verbatim_clone_pack( + repo: &Repo, + wants: Wants<'_>, +) -> Result, GitError> { + let Some(pack_idx) = bitmapped_pack(&repo.objects_dir()) else { + return Ok(None); + }; + let Some(loaded) = single_pack(repo)? else { + return Ok(None); + }; + let Some(resolved) = loaded.resolve(repo, wants, Haves::new(&[]))? else { + return Ok(None); + }; + if !resolved.outside.is_empty() || resolved.in_bitmap.len() != loaded.rev.len() { + return Ok(None); + } + Ok(std::fs::File::open(pack_idx.with_extension("pack")).ok()) +} + +fn bitmapped_pack(objects_dir: &Path) -> Option { + let pack_dir = objects_dir.join("pack"); + let mut idxs: Vec = std::fs::read_dir(&pack_dir) + .ok()? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "idx")) + .collect(); + idxs.sort(); + idxs.into_iter() + .find(|idx| idx.with_extension("bitmap").exists()) +} diff --git a/knot2/crates/knot-git/src/bitmap/reach.rs b/knot2/crates/knot-git/src/bitmap/reach.rs new file mode 100644 index 0000000000000000000000000000000000000000..b6ae8d4fadcc3df19f08f28af2930e18334ebb45 --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/reach.rs @@ -0,0 +1,179 @@ +use std::collections::HashSet; + +use knot_types::Oid; + +use super::bitset::Bitset; +use super::reader::Bitmaps; +use super::revindex::RevIndex; +use super::{BitPosition, IndexPosition}; +use crate::error::GitError; +use crate::objects::{MAX_TAG_DEPTH, MAX_TREE_DEPTH, Peeled}; +use crate::repo::Repo; + +pub(super) struct Resolved { + pub(super) in_bitmap: Vec, + pub(super) outside: Vec, +} + +pub(super) struct Reach { + bits: Bitset, + outside: HashSet, +} + +impl Reach { + pub(super) fn has(&self, bit: Option, oid: Oid) -> bool { + match bit { + Some(bit) => self.bits.get(bit), + None => self.outside.contains(&oid), + } + } + + pub(super) fn minus(self, haves: &Reach, rev: &RevIndex) -> Result { + let in_bitmap = self + .bits + .difference_indices(&haves.bits) + .map(|bit| rev.oid_at_bit(bit)) + .collect::>()?; + Ok(Resolved { + in_bitmap, + outside: self.outside.into_iter().collect(), + }) + } +} + +struct Walk<'a> { + repo: &'a Repo, + rev: &'a RevIndex, + maps: &'a Bitmaps<'a>, + seen: Option<&'a Reach>, + out: Reach, +} + +pub(super) fn reach( + repo: &Repo, + rev: &RevIndex, + maps: &Bitmaps<'_>, + roots: &[Oid], + seen: Option<&Reach>, +) -> Result { + let mut walk = Walk { + repo, + rev, + maps, + seen, + out: Reach { + bits: Bitset::zeros(rev.len()), + outside: HashSet::new(), + }, + }; + let mut commits = Vec::new(); + let mut trees = Vec::new(); + roots.iter().try_for_each(|root| { + let mut tags = Vec::new(); + let peeled = repo.peel(root.object_id(), &mut tags, MAX_TAG_DEPTH)?; + tags.into_iter() + .try_for_each(|tag| walk.mark(tag).map(drop))?; + match peeled { + Peeled::Commit(commit) => commits.push(Oid::from(commit)), + Peeled::Direct(direct) => match object_kind(repo, direct)? { + gix::object::Kind::Tree => trees.push(direct), + _ => { + walk.mark(Oid::from(direct))?; + } + }, + } + Ok::<_, GitError>(()) + })?; + while let Some(commit) = commits.pop() { + let located = rev.locate(commit)?; + if walk.marked(located.map(|(_, bit)| bit), commit) { + continue; + } + if let Some((position, _)) = located + && walk.union_bitmap(position)? + { + continue; + } + walk.set(located.map(|(_, bit)| bit), commit); + let (tree, parents) = repo.commit_tree_and_parents(commit)?; + trees.push(tree); + commits.extend(parents.into_iter().map(Oid::from)); + } + trees + .into_iter() + .try_for_each(|tree| walk.fill_tree(tree, MAX_TREE_DEPTH))?; + Ok(walk.out) +} + +impl Walk<'_> { + fn marked(&self, bit: Option, oid: Oid) -> bool { + self.out.has(bit, oid) || self.seen.is_some_and(|seen| seen.has(bit, oid)) + } + + fn set(&mut self, bit: Option, oid: Oid) { + match bit { + Some(bit) => self.out.bits.set(bit), + None => { + self.out.outside.insert(oid); + } + } + } + + fn mark(&mut self, oid: Oid) -> Result { + let bit = self.rev.locate(oid)?.map(|(_, bit)| bit); + if self.marked(bit, oid) { + return Ok(false); + } + self.set(bit, oid); + Ok(true) + } + + fn union_bitmap(&mut self, position: IndexPosition) -> Result { + let Some(bits) = self.maps.bitmap(position)? else { + return Ok(false); + }; + self.out.bits.union_with(&bits); + Ok(true) + } + + fn fill_tree(&mut self, tree: gix::ObjectId, nesting: usize) -> Result<(), GitError> { + if nesting == 0 { + return Err(GitError::DepthExceeded("tree nesting")); + } + let oid = Oid::from(tree); + let bit = self.rev.locate(oid)?.map(|(_, bit)| bit); + if bit.is_none() && tree == gix::ObjectId::empty_tree(tree.kind()) { + return Ok(()); + } + if self.marked(bit, oid) { + return Ok(()); + } + self.set(bit, oid); + let object = self + .repo + .git() + .find_tree(tree) + .map_err(|error| GitError::Corrupt { + oid, + message: error.to_string(), + })?; + object.iter().try_for_each(|entry| { + let entry = entry.map_err(|error| GitError::Decode(error.to_string()))?; + match entry.mode().kind() { + gix::objs::tree::EntryKind::Tree => self.fill_tree(entry.object_id(), nesting - 1), + gix::objs::tree::EntryKind::Commit => Ok(()), + _ => self.mark(Oid::from(entry.object_id())).map(drop), + } + }) + } +} + +fn object_kind(repo: &Repo, oid: gix::ObjectId) -> Result { + repo.git() + .find_header(oid) + .map(|header| header.kind()) + .map_err(|error| GitError::Corrupt { + oid: Oid::from(oid), + message: error.to_string(), + }) +} diff --git a/knot2/crates/knot-git/src/bitmap/reader.rs b/knot2/crates/knot-git/src/bitmap/reader.rs new file mode 100644 index 0000000000000000000000000000000000000000..576576f657a7b8d482c64934b7679a8371355b82 --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/reader.rs @@ -0,0 +1,141 @@ +use super::bitset::Bitset; +use super::ewah::Ewah; +use super::{BitmapEntryOffset, IndexPosition}; +use crate::error::GitError; + +const OPT_LOOKUP_TABLE: u16 = 0x10; +const TRIPLET_LEN: usize = 16; + +pub(crate) struct Bitmaps<'a> { + body: &'a [u8], + types: &'a [u8], + table: Vec<(IndexPosition, BitmapEntryOffset, u32)>, + num_objects: usize, +} + +impl Bitmaps<'_> { + pub(crate) fn types(&self) -> Result<[Ewah; 4], GitError> { + let (commits, rest) = Ewah::decode(self.types)?; + let (trees, rest) = Ewah::decode(rest)?; + let (blobs, rest) = Ewah::decode(rest)?; + let (tags, _) = Ewah::decode(rest)?; + Ok([commits, trees, blobs, tags]) + } + + pub(crate) fn bitmap(&self, position: IndexPosition) -> Result, GitError> { + match self + .table + .binary_search_by_key(&position, |(pos, _, _)| *pos) + { + Ok(row) => self.decode_chain(row).map(Some), + Err(_) => Ok(None), + } + } + + fn decode_chain(&self, mut row: usize) -> Result { + let mut bits = Bitset::zeros(self.num_objects); + for _ in 0..self.table.len() { + let (commit_pos, offset, xor_row) = self.table[row]; + let (xor_offset, vector) = self.decode_at(commit_pos, offset)?; + bits.xor_ewah(&vector, self.num_objects)?; + if xor_offset == 0 { + return Ok(bits); + } + row = usize::try_from(xor_row) + .ok() + .filter(|row| *row < self.table.len()) + .ok_or_else(|| GitError::Backend("bitmap xor row out of range".to_string()))?; + } + Err(GitError::Backend("bitmap xor chain loops".to_string())) + } + + fn decode_at( + &self, + commit_pos: IndexPosition, + offset: BitmapEntryOffset, + ) -> Result<(u8, Ewah), GitError> { + let start = usize::try_from(offset.get()) + .ok() + .filter(|start| *start <= self.body.len()) + .ok_or_else(|| GitError::Backend("bitmap entry offset out of range".to_string()))?; + let entry = &self.body[start..]; + if entry.len() < 6 { + return Err(GitError::Backend( + "truncated bitmap entry header".to_string(), + )); + } + if u32::from_be_bytes([entry[0], entry[1], entry[2], entry[3]]) != commit_pos.get() { + return Err(GitError::Backend( + "bitmap lookup points at the wrong commit".to_string(), + )); + } + let (vector, _) = Ewah::decode(&entry[6..])?; + Ok((entry[4], vector)) + } +} + +pub(crate) fn parse( + bytes: &[u8], + kind: gix::hash::Kind, + num_objects: usize, +) -> Result, GitError> { + let raw = kind.len_in_bytes(); + let header_len = 12 + raw; + if bytes.len() < header_len + raw || &bytes[..4] != b"BITM" { + return Err(GitError::Backend("bitmap header isn't BITM".to_string())); + } + let version = u16::from_be_bytes([bytes[4], bytes[5]]); + if version != 1 { + return Err(GitError::Backend(format!( + "unsupported bitmap version {version}" + ))); + } + let flags = u16::from_be_bytes([bytes[6], bytes[7]]); + if flags & OPT_LOOKUP_TABLE == 0 { + return Err(GitError::Backend( + "bitmap lacks the lookup table extension".to_string(), + )); + } + let entry_count = u32::from_be_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize; + + let (body, trailer) = bytes.split_at(bytes.len() - raw); + let mut hasher = gix_hash::hasher(kind); + hasher.update(body); + let digest = hasher + .try_finalize() + .map_err(|error| GitError::Backend(format!("bitmap checksum: {error}")))?; + if digest.as_slice() != trailer { + return Err(GitError::Backend("bitmap checksum mismatch".to_string())); + } + + let table_len = entry_count + .checked_mul(TRIPLET_LEN) + .filter(|len| header_len + len <= body.len()) + .ok_or_else(|| GitError::Backend("bitmap lookup table overflows the file".to_string()))?; + let region = &body[body.len() - table_len..]; + let table: Vec<(IndexPosition, BitmapEntryOffset, u32)> = (0..entry_count) + .map(|index| { + let base = index * TRIPLET_LEN; + let commit_pos = u32::from_be_bytes(region[base..base + 4].try_into().unwrap()); + let offset = u64::from_be_bytes(region[base + 4..base + 12].try_into().unwrap()); + let xor_row = u32::from_be_bytes(region[base + 12..base + 16].try_into().unwrap()); + ( + IndexPosition::new(commit_pos), + BitmapEntryOffset::new(offset), + xor_row, + ) + }) + .collect(); + if !table.is_sorted_by_key(|(commit_pos, _, _)| *commit_pos) { + return Err(GitError::Backend( + "bitmap lookup table isn't sorted".to_string(), + )); + } + + Ok(Bitmaps { + body, + types: &body[header_len..], + table, + num_objects, + }) +} diff --git a/knot2/crates/knot-git/src/bitmap/revindex.rs b/knot2/crates/knot-git/src/bitmap/revindex.rs new file mode 100644 index 0000000000000000000000000000000000000000..b7373aac16474b4fba170908968d087fb6a64e3c --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/revindex.rs @@ -0,0 +1,246 @@ +use std::collections::HashMap; +use std::path::Path; + +use knot_types::Oid; + +use super::{BitPosition, IndexPosition}; +use crate::error::GitError; + +// hashtag easter egg +const RIDX_SIGNATURE: u32 = 0x5249_4458; +const RIDX_HEADER_LEN: usize = 12; + +pub(crate) trait Order { + fn len(&self) -> usize; + fn index_of(&self, oid: Oid) -> Option; + fn bit_at_index(&self, position: IndexPosition) -> BitPosition; + fn oid_at_bit(&self, bit: BitPosition) -> Oid; +} + +pub(crate) struct OrderTable { + oids: Vec, + bit_to_index: Vec, + index_to_bit: Vec, + by_oid: HashMap, +} + +impl OrderTable { + fn build(oids: Vec, placement: impl Fn(u32) -> K) -> Self { + let count = oids.len() as u32; + let mut order: Vec = (0..count).collect(); + order.sort_by_key(|position| placement(*position)); + let bit_to_index: Vec = order + .iter() + .map(|position| IndexPosition::new(*position)) + .collect(); + let index_to_bit = order.iter().enumerate().fold( + vec![BitPosition::new(0); oids.len()], + |mut table, (bit, position)| { + table[*position as usize] = BitPosition::new(bit as u32); + table + }, + ); + let by_oid: HashMap = oids + .iter() + .enumerate() + .map(|(position, oid)| (*oid, IndexPosition::new(position as u32))) + .collect(); + Self { + oids, + bit_to_index, + index_to_bit, + by_oid, + } + } + + pub(crate) fn from_index(index: &gix_pack::index::File) -> Self { + let count = index.num_objects(); + let oids: Vec = (0..count) + .map(|position| Oid::from(index.oid_at_index(position).to_owned())) + .collect(); + let offsets: Vec = (0..count) + .map(|position| index.pack_offset_at_index(position)) + .collect(); + Self::build(oids, |position| offsets[position as usize]) + } + + pub(crate) fn from_file(file: &gix_pack::multi_index::File) -> Self { + let count = file.num_objects(); + let oids: Vec = (0..count) + .map(|position| Oid::from(file.oid_at_index(position).to_owned())) + .collect(); + let placement: Vec<(u32, u64)> = (0..count) + .map(|position| file.pack_id_and_pack_offset_at_index(position)) + .collect(); + Self::build(oids, |position| placement[position as usize]) + } + + pub(crate) fn index_positions_in_bit_order(&self) -> &[IndexPosition] { + &self.bit_to_index + } +} + +impl Order for OrderTable { + fn len(&self) -> usize { + self.oids.len() + } + + fn index_of(&self, oid: Oid) -> Option { + self.by_oid.get(&oid).copied() + } + + fn bit_at_index(&self, position: IndexPosition) -> BitPosition { + self.index_to_bit[position.get() as usize] + } + + fn oid_at_bit(&self, bit: BitPosition) -> Oid { + self.oids[self.bit_to_index[bit.get() as usize].get() as usize] + } +} + +pub(crate) fn write( + path: &Path, + order: &OrderTable, + kind: gix::hash::Kind, + checksum: &gix_hash::ObjectId, +) -> Result<(), GitError> { + let mut out = Vec::new(); + out.extend_from_slice(&RIDX_SIGNATURE.to_be_bytes()); + out.extend_from_slice(&1u32.to_be_bytes()); + out.extend_from_slice(&hash_id(kind).to_be_bytes()); + order + .index_positions_in_bit_order() + .iter() + .for_each(|position| out.extend_from_slice(&position.get().to_be_bytes())); + out.extend_from_slice(checksum.as_slice()); + let mut hasher = gix_hash::hasher(kind); + hasher.update(&out); + let digest = hasher + .try_finalize() + .map_err(|error| GitError::Backend(format!("revindex checksum: {error}")))?; + out.extend_from_slice(digest.as_slice()); + super::writer::install(path, &out) +} + +fn hash_id(kind: gix::hash::Kind) -> u32 { + match kind { + gix::hash::Kind::Sha256 => 2, + _ => 1, + } +} + +pub(crate) enum Positions { + Pack(gix_pack::index::File), + Midx(gix_pack::multi_index::File), +} + +impl Positions { + fn len(&self) -> usize { + match self { + Positions::Pack(index) => index.num_objects() as usize, + Positions::Midx(file) => file.num_objects() as usize, + } + } + + fn lookup(&self, oid: Oid) -> Option { + match self { + Positions::Pack(index) => index.lookup(oid.object_id()), + Positions::Midx(file) => file.lookup(oid.object_id()), + } + .map(IndexPosition::new) + } + + fn oid_at(&self, position: IndexPosition) -> Oid { + match self { + Positions::Pack(index) => Oid::from(index.oid_at_index(position.get()).to_owned()), + Positions::Midx(file) => Oid::from(file.oid_at_index(position.get()).to_owned()), + } + } + + fn placement(&self, position: IndexPosition) -> (u32, u64) { + match self { + Positions::Pack(index) => (0, index.pack_offset_at_index(position.get())), + Positions::Midx(file) => file.pack_id_and_pack_offset_at_index(position.get()), + } + } +} + +pub(crate) struct RevIndex { + positions: Positions, + map: memmap2::Mmap, +} + +impl RevIndex { + pub(crate) fn open( + path: &Path, + positions: Positions, + kind: gix::hash::Kind, + checksum: &gix_hash::ObjectId, + ) -> Result, GitError> { + let Ok(file) = std::fs::File::open(path) else { + return Ok(None); + }; + let map = unsafe { memmap2::Mmap::map(&file) } + .map_err(|error| GitError::Backend(format!("map revindex: {error}")))?; + let raw = kind.len_in_bytes(); + let entries_end = RIDX_HEADER_LEN + 4 * positions.len(); + let valid = map.len() == entries_end + 2 * raw + && map[..4] == RIDX_SIGNATURE.to_be_bytes() + && map[4..8] == 1u32.to_be_bytes() + && map[8..12] == hash_id(kind).to_be_bytes() + && &map[entries_end..entries_end + raw] == checksum.as_slice(); + if !valid { + return Err(GitError::Backend(format!( + "revindex {} doesn't match its pack", + path.display() + ))); + } + Ok(Some(Self { positions, map })) + } + + pub(crate) fn len(&self) -> usize { + self.positions.len() + } + + fn index_at_bit(&self, bit: BitPosition) -> Result { + let at = RIDX_HEADER_LEN + 4 * bit.get() as usize; + let position = u32::from_be_bytes(self.map[at..at + 4].try_into().unwrap()); + if position as usize >= self.len() { + return Err(GitError::Backend("revindex entry out of range".to_string())); + } + Ok(IndexPosition::new(position)) + } + + pub(crate) fn index_of(&self, oid: Oid) -> Option { + self.positions.lookup(oid) + } + + pub(crate) fn oid_at_bit(&self, bit: BitPosition) -> Result { + Ok(self.positions.oid_at(self.index_at_bit(bit)?)) + } + + pub(crate) fn locate( + &self, + oid: Oid, + ) -> Result, GitError> { + let Some(position) = self.positions.lookup(oid) else { + return Ok(None); + }; + let target = self.positions.placement(position); + let (mut low, mut high) = (0u32, self.len() as u32); + while low < high { + let mid = low + (high - low) / 2; + let at = self + .positions + .placement(self.index_at_bit(BitPosition::new(mid))?); + match at.cmp(&target) { + std::cmp::Ordering::Less => low = mid + 1, + std::cmp::Ordering::Greater => high = mid, + std::cmp::Ordering::Equal => return Ok(Some((position, BitPosition::new(mid)))), + } + } + Err(GitError::Backend(format!( + "{oid} is missing from the revindex" + ))) + } +} diff --git a/knot2/crates/knot-git/src/bitmap/writer.rs b/knot2/crates/knot-git/src/bitmap/writer.rs new file mode 100644 index 0000000000000000000000000000000000000000..9c8aaed9ef5828900a70c794a0225e3ac77aa43d --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/writer.rs @@ -0,0 +1,435 @@ +use std::collections::{HashMap, HashSet, VecDeque}; +use std::path::Path; + +use knot_types::{Oid, UnixSeconds}; + +use super::ewah::Ewah; +use super::reader::{self, Bitmaps}; +use super::revindex::{self, Order, OrderTable}; +use super::{BitPosition, BitmapEntryOffset, BitmapIndex, Bitset, IndexPosition}; +use crate::error::GitError; +use crate::objects::MAX_TREE_DEPTH; +use crate::repo::Repo; + +const OPT_FULL_DAG: u16 = 0x1; +const OPT_LOOKUP_TABLE: u16 = 0x10; + +const CAT_COMMIT: u8 = 0; +const CAT_TREE: u8 = 1; +const CAT_BLOB: u8 = 2; +const CAT_TAG: u8 = 3; +const CAT_UNKNOWN: u8 = u8::MAX; + +const MAX_XOR_OFFSET_SEARCH: usize = 10; + +const MUST_REGION: usize = 100; +const MIN_REGION: usize = 20_000; +const MIN_COMMITS: usize = 100; +const MAX_COMMITS: usize = 5_000; + +pub(super) struct TypeBits { + commits: Ewah, + trees: Ewah, + blobs: Ewah, + tags: Ewah, +} + +pub(super) struct Selected { + commit_pos: IndexPosition, + xor_offset: u8, + bits: Ewah, +} + +pub(super) struct Previous<'a> { + index: &'a BitmapIndex, + maps: Bitmaps<'a>, + to_new: Vec>, +} + +impl<'a> Previous<'a> { + pub(super) fn new( + index: &'a BitmapIndex, + kind: gix::hash::Kind, + rev: &R, + ) -> Option { + let maps = reader::parse(&index.bytes, kind, index.rev.len()).ok()?; + let to_new = (0..index.rev.len()) + .map(|bit| { + let oid = index.rev.oid_at_bit(BitPosition::new(bit as u32))?; + Ok(rev.index_of(oid).map(|position| rev.bit_at_index(position))) + }) + .collect::>() + .ok()?; + Some(Self { + index, + maps, + to_new, + }) + } + + fn categories(&self, num_bits: usize) -> Result, GitError> { + let mut categories = vec![CAT_UNKNOWN; num_bits]; + let old_len = self.index.rev.len(); + for (category, vector) in (CAT_COMMIT..).zip(self.maps.types()?) { + let mut old = Bitset::zeros(old_len); + old.union_ewah(&vector, old_len)?; + old.ones() + .filter_map(|bit| self.to_new[bit.get() as usize]) + .for_each(|bit| categories[bit.get() as usize] = category); + } + Ok(categories) + } + + fn bits(&self, commit: Oid, num_bits: usize) -> Option { + let old = self.maps.bitmap(self.index.rev.index_of(commit)?).ok()??; + let mut bits = Bitset::zeros(num_bits); + for bit in old.ones() { + bits.set(self.to_new[bit.get() as usize]?); + } + Some(bits) + } +} + +pub(crate) fn write( + repo: &Repo, + pack_idx: &Path, + previous: Option<&BitmapIndex>, +) -> Result { + let kind = repo.object_format().kind(); + let index = gix_pack::index::File::at(pack_idx, kind) + .map_err(|error| GitError::Backend(format!("open pack index: {error}")))?; + let rev = OrderTable::from_index(&index); + if rev.len() == 0 { + return Ok(false); + } + + let _boost = knot_resource::saturate(); + let previous = previous.and_then(|index| Previous::new(index, kind, &rev)); + let categories = categories_in_bit_order(repo, &rev, previous.as_ref())?; + let types = type_index_bits(&categories)?; + let selected = selected_entries(repo, &rev, &categories, previous.as_ref())?; + if selected.is_empty() { + return Ok(false); + } + + let checksum = index.pack_checksum(); + let bytes = assemble(kind, &checksum, &types, &selected)?; + revindex::write(&pack_idx.with_extension("rev"), &rev, kind, &checksum)?; + install(&pack_idx.with_extension("bitmap"), &bytes)?; + Ok(true) +} + +pub(super) fn type_index_bits(categories: &[u8]) -> Result { + let select = |target: u8| { + let mut bits = Bitset::zeros(categories.len()); + categories + .iter() + .enumerate() + .filter(|(_, value)| **value == target) + .for_each(|(bit, _)| bits.set(BitPosition::new(bit as u32))); + bits.to_ewah(categories.len()) + }; + Ok(TypeBits { + commits: select(CAT_COMMIT)?, + trees: select(CAT_TREE)?, + blobs: select(CAT_BLOB)?, + tags: select(CAT_TAG)?, + }) +} + +pub(super) fn categories_in_bit_order( + repo: &Repo, + rev: &R, + previous: Option<&Previous>, +) -> Result, GitError> { + let known = previous + .and_then(|previous| previous.categories(rev.len()).ok()) + .unwrap_or_else(|| vec![CAT_UNKNOWN; rev.len()]); + let path = repo.path().to_owned(); + knot_resource::map_spans(rev.len(), |start, end| { + let local = Repo::open(&path)?; + (start..end) + .map(|bit| match known[bit] { + CAT_UNKNOWN => object_category(&local, rev.oid_at_bit(BitPosition(bit as u32))), + category => Ok(category), + }) + .collect::, GitError>>() + }) +} + +fn object_category(repo: &Repo, oid: Oid) -> Result { + match repo.git().try_find_header(oid.object_id()) { + Ok(Some(header)) => Ok(match header.kind() { + gix::object::Kind::Commit => CAT_COMMIT, + gix::object::Kind::Tree => CAT_TREE, + gix::object::Kind::Blob => CAT_BLOB, + gix::object::Kind::Tag => CAT_TAG, + }), + Ok(None) => Err(GitError::ObjectNotFound(oid)), + Err(error) => Err(GitError::Corrupt { + oid, + message: error.to_string(), + }), + } +} + +pub(super) fn selected_entries( + repo: &Repo, + rev: &R, + categories: &[u8], + previous: Option<&Previous>, +) -> Result, GitError> { + let tips: HashSet = repo + .references()? + .into_iter() + .filter_map(|record| repo.peel_to_commit(record.target).ok()) + .filter(|commit| rev.index_of(*commit).is_some()) + .collect(); + let commits: Vec = categories + .iter() + .enumerate() + .filter(|(_, category)| **category == CAT_COMMIT) + .map(|(bit, _)| rev.oid_at_bit(BitPosition::new(bit as u32))) + .collect(); + let chosen = sample_history(repo, commits, &tips)?; + let mut tips: Vec<(UnixSeconds, IndexPosition, Oid)> = chosen + .into_iter() + .filter_map(|commit| rev.index_of(commit).map(|position| (position, commit))) + .map(|(position, commit)| Ok((repo.commit_time(commit.object_id())?, position, commit))) + .collect::>()?; + tips.sort_by_key(|(time, position, _)| (*time, *position)); + + let mut built: HashMap = HashMap::new(); + let mut recent: VecDeque = VecDeque::new(); + let mut entries = Vec::new(); + for (_, commit_pos, commit) in tips { + let bits = closure_bits(repo, rev, commit, &built, previous)?; + let full = bits.to_ewah(rev.len())?; + let (xor_offset, write_as) = recent.iter().rev().zip(1u8..).try_fold( + (0, full.clone()), + |best, (base, offset)| { + let candidate = bits.xor(base).to_ewah(rev.len())?; + Ok::<_, GitError>(if candidate.word_count() < best.1.word_count() { + (offset, candidate) + } else { + best + }) + }, + )?; + entries.push(Selected { + commit_pos, + xor_offset, + bits: write_as, + }); + built.insert(commit, full); + recent.push_back(bits); + if recent.len() > MAX_XOR_OFFSET_SEARCH { + recent.pop_front(); + } + } + Ok(entries) +} + +fn sample_history( + repo: &Repo, + commits: Vec, + tips: &HashSet, +) -> Result, GitError> { + let mut dated: Vec<(UnixSeconds, Oid)> = commits + .into_iter() + .map(|commit| Ok((repo.commit_time(commit.object_id())?, commit))) + .collect::>()?; + dated.sort_by_key(|(time, _)| std::cmp::Reverse(*time)); + if dated.len() < MUST_REGION { + return Ok(dated.into_iter().map(|(_, commit)| commit).collect()); + } + let mut chosen = Vec::new(); + let mut at = 0; + loop { + let next = next_commit_index(at); + if at + next >= dated.len() { + break; + } + let pick = match next { + 0 => dated[at].1, + _ => preferred_in(repo, &dated[at..=at + next], tips)?, + }; + chosen.push(pick); + at += next + 1; + } + Ok(chosen) +} + +fn preferred_in( + repo: &Repo, + window: &[(UnixSeconds, Oid)], + tips: &HashSet, +) -> Result { + let mut pick = window[window.len() - 1].1; + for (_, commit) in window { + if tips.contains(commit) { + return Ok(*commit); + } + if repo.commit_tree_and_parents(*commit)?.1.len() > 1 { + pick = *commit; + } + } + Ok(pick) +} + +fn next_commit_index(at: usize) -> usize { + if at <= MUST_REGION { + return 0; + } + if at <= MIN_REGION { + return (at - MUST_REGION).min(MIN_COMMITS); + } + (at - MIN_REGION).clamp(MIN_COMMITS, MAX_COMMITS) +} + +fn closure_bits( + repo: &Repo, + rev: &R, + tip: Oid, + built: &HashMap, + previous: Option<&Previous>, +) -> Result { + let mut bits = Bitset::zeros(rev.len()); + let mut commits = vec![tip]; + let mut trees = Vec::new(); + while let Some(commit) = commits.pop() { + let bit = bit_of(rev, commit)?; + if bits.get(bit) { + continue; + } + if let Some(ancestor) = built.get(&commit) { + bits.union_ewah(ancestor, rev.len())?; + continue; + } + if let Some(old) = previous.and_then(|previous| previous.bits(commit, rev.len())) { + bits.union_with(&old); + continue; + } + bits.set(bit); + let (tree, parents) = repo.commit_tree_and_parents(commit)?; + trees.push(tree); + commits.extend(parents.into_iter().map(Oid::from)); + } + trees + .into_iter() + .try_for_each(|tree| fill_tree(repo, rev, tree, &mut bits, MAX_TREE_DEPTH))?; + Ok(bits) +} + +fn fill_tree( + repo: &Repo, + rev: &R, + tree: gix::ObjectId, + bits: &mut Bitset, + nesting: usize, +) -> Result<(), GitError> { + if nesting == 0 { + return Err(GitError::DepthExceeded("tree nesting")); + } + let oid = Oid::from(tree); + if tree == gix::ObjectId::empty_tree(tree.kind()) && rev.index_of(oid).is_none() { + return Ok(()); + } + let bit = bit_of(rev, oid)?; + if bits.get(bit) { + return Ok(()); + } + bits.set(bit); + let object = repo + .git() + .find_tree(tree) + .map_err(|error| GitError::Corrupt { + oid, + message: error.to_string(), + })?; + object.iter().try_for_each(|entry| { + let entry = entry.map_err(|error| GitError::Decode(error.to_string()))?; + match entry.mode().kind() { + gix::objs::tree::EntryKind::Tree => { + fill_tree(repo, rev, entry.object_id(), bits, nesting - 1) + } + gix::objs::tree::EntryKind::Commit => Ok(()), + _ => { + bits.set(bit_of(rev, Oid::from(entry.object_id()))?); + Ok(()) + } + } + }) +} + +fn bit_of(rev: &impl Order, oid: Oid) -> Result { + rev.index_of(oid) + .map(|position| rev.bit_at_index(position)) + .ok_or_else(|| { + GitError::Backend(format!( + "closure object {oid} is absent from the pack bitmap" + )) + }) +} + +pub(super) fn assemble( + kind: gix::hash::Kind, + checksum: &gix_hash::ObjectId, + types: &TypeBits, + selected: &[Selected], +) -> Result, GitError> { + let flags: u16 = OPT_FULL_DAG | OPT_LOOKUP_TABLE; + let mut out = Vec::new(); + out.extend_from_slice(b"BITM"); + out.extend_from_slice(&1u16.to_be_bytes()); + out.extend_from_slice(&flags.to_be_bytes()); + out.extend_from_slice(&(selected.len() as u32).to_be_bytes()); + out.extend_from_slice(checksum.as_slice()); + + [&types.commits, &types.trees, &types.blobs, &types.tags] + .into_iter() + .for_each(|bits| bits.write_to(&mut out)); + + let offsets: Vec = selected + .iter() + .map(|entry| { + let at = BitmapEntryOffset::new(out.len() as u64); + out.extend_from_slice(&entry.commit_pos.get().to_be_bytes()); + out.push(entry.xor_offset); + out.push(0); + entry.bits.write_to(&mut out); + at + }) + .collect(); + + let mut rows: Vec = (0..selected.len()).collect(); + rows.sort_by_key(|index| selected[*index].commit_pos); + let mut row_of = vec![0u32; selected.len()]; + rows.iter() + .enumerate() + .for_each(|(row, index)| row_of[*index] = row as u32); + rows.iter().for_each(|index| { + let entry = &selected[*index]; + let xor_row = match entry.xor_offset { + 0 => u32::MAX, + offset => row_of[index - offset as usize], + }; + out.extend_from_slice(&entry.commit_pos.get().to_be_bytes()); + out.extend_from_slice(&offsets[*index].get().to_be_bytes()); + out.extend_from_slice(&xor_row.to_be_bytes()); + }); + + let mut hasher = gix_hash::hasher(kind); + hasher.update(&out); + let digest = hasher + .try_finalize() + .map_err(|error| GitError::Backend(format!("bitmap checksum: {error}")))?; + out.extend_from_slice(digest.as_slice()); + Ok(out) +} + +pub(super) fn install(path: &Path, bytes: &[u8]) -> Result<(), GitError> { + knot_resource::atomic_write_bytes(path, bytes, knot_resource::FileMode::Inherited).map_err( + |error| GitError::Maintenance(format!("write bitmap {}: {}", path.display(), error.source)), + ) +} diff --git a/knot2/crates/knot-git/src/error.rs b/knot2/crates/knot-git/src/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..2ab5a0f06caceeaa3b9c7c165d8e345c0aba3cea --- /dev/null +++ b/knot2/crates/knot-git/src/error.rs @@ -0,0 +1,92 @@ +use std::path::PathBuf; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SelectionLimit { + Objects, + Time, +} + +impl std::fmt::Display for SelectionLimit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + SelectionLimit::Objects => "object-set limit", + SelectionLimit::Time => "wall-clock budget", + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum GitError { + #[error("repository already exists at {0}")] + AlreadyExists(PathBuf), + #[error("open repository at {path}: {message}")] + Open { path: PathBuf, message: String }, + #[error("create repository at {path}: {message}")] + Create { path: PathBuf, message: String }, + #[error("remove repository at {path}: {message}")] + Remove { path: PathBuf, message: String }, + #[error("reference {name}: {message}")] + Reference { name: String, message: String }, + #[error("atomic ref transaction: {0}")] + AtomicRefs(String), + #[error("fsync {path}: {message}")] + Fsync { path: PathBuf, message: String }, + #[error("atomic write to {path}: {message}")] + Write { path: PathBuf, message: String }, + #[error("repo DID {0} maps to unsafe on-disk path component")] + UnsafeRepoDid(String), + #[error("repo DID {0} is reserved for knot meta-repo and is never served as user repo")] + ReservedDid(String), + #[error("{0} exceeds maximum supported depth")] + DepthExceeded(&'static str), + #[error("archive exceeds the {} byte limit", limit.get())] + ArchiveTooLarge { limit: crate::ArchiveLimit }, + #[error("blame object exceeds the {limit} byte limit")] + BlameObjectTooLarge { limit: u64 }, + #[error("blame exceeded its computation deadline")] + BlameTimedOut, + #[error("revision walk: {0}")] + RevWalk(String), + #[error("upload-pack selection exceeded its {0}")] + Selection(SelectionLimit), + #[error("object not found: {0}")] + ObjectNotFound(knot_types::Oid), + #[error("remove loose object {oid}: {message}")] + RemoveObject { + oid: knot_types::Oid, + message: String, + }, + #[error("object {oid} is corrupt: {message}")] + Corrupt { + oid: knot_types::Oid, + message: String, + }, + #[error("object {oid} isn't {expected}")] + ObjectType { + oid: knot_types::Oid, + expected: &'static str, + }, + #[error("decode object: {0}")] + Decode(String), + #[error("git backend: {0}")] + Backend(String), + #[error("object staging: {0}")] + Staging(String), + #[error("repository config at {path}: {message}")] + Config { path: PathBuf, message: String }, + #[error("repository maintenance: {0}")] + Maintenance(String), +} + +impl From for GitError { + fn from(error: knot_resource::FsError) -> Self { + GitError::Write { + path: error.path, + message: error.source.to_string(), + } + } +} + +pub(crate) fn backend(error: impl std::fmt::Display) -> GitError { + GitError::Backend(error.to_string()) +} diff --git a/knot2/crates/knot-git/src/instrument.rs b/knot2/crates/knot-git/src/instrument.rs new file mode 100644 index 0000000000000000000000000000000000000000..0a207af90c3de2ce99f708ce579562ba6b782363 --- /dev/null +++ b/knot2/crates/knot-git/src/instrument.rs @@ -0,0 +1,8 @@ +knot_types::read_counter!( + OdbReads, + record_read, + odb_reads, + reset_odb_reads, + add_odb_reads, + measure +); diff --git a/knot2/crates/knot-git/src/lib.rs b/knot2/crates/knot-git/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..5848001aa24d4dcea376b78013ed9ddc67d5f582 --- /dev/null +++ b/knot2/crates/knot-git/src/lib.rs @@ -0,0 +1,62 @@ +mod archive; +mod base85; +mod bitmap; +mod error; +#[cfg(feature = "instrument")] +pub mod instrument; +mod maintenance; +mod objects; +mod patch; +mod patch_apply; +mod patch_parse; +mod reads; +mod repo; +mod staging; + +pub use archive::{ArchiveFormat, ArchiveLimit, ArchivePrefix, TreePrefix}; +pub use bitmap::{ + BitmapIndex, count_commits, open_bitmap_index, reachable_via_bitmap, unreachable_via_bitmap, + verbatim_clone_pack, write_bitmap, write_midx_bitmap, +}; +pub use error::{GitError, SelectionLimit}; +pub use knot_storage::{ + AdvertScope, Filter, HeadRef, PackBudget, PackHash, RefClass, RefEpoch, RefGeneration, + RefRecord, RefUpdate, ReflogUpdate, TreeDepth, +}; +pub use maintenance::{PackRefsReport, ReflogReport}; +pub use objects::{ + BlobReader, Commit, CommitChangeId, CommitDepth, CommitRange, Comparison, Deepen, EntryKind, + FileChange, Haves, Identity, MAX_TREE_DEPTH, PackSelection, ShallowCommits, ShallowPlan, Tree, + TreeEntry, Wants, +}; +pub use patch::{ + BinaryBudget, BinaryDiff, BinarySizes, FilePatch, Hunk, HunkLine, LineCount, LineNumber, + LineOp, MAX_DIFF_BLOB_BYTES, PatchBody, PatchRange, PatchStatus, is_binary, literal_block, + subproject_line, +}; +pub use patch_apply::{ + ApplyError, ApplyOutcome, Conflict, ConflictReason, NewCommit, PatchApplier, StagedAction, + StagedChange, +}; +pub use patch_parse::{ + FileIntent, MailPatch, ParsedFile, PatchParseError, PatchPayload, is_format_patch, + parse_mailbox, parse_mailbox_bounded, parse_patch, parse_patch_bounded, +}; +pub use reads::{ + AnnotatedTag, BlameHunk, BranchInfo, BranchTip, LastCommit, LogLimit, LogSkip, PathEntry, + SizedEntry, Submodule, TagInfo, +}; +pub use repo::{Layout, PackfileUri, PackfileUrl, RefTxn, Repo, is_branch, repo_shard}; +pub use staging::{INCOMING_PREFIX, Staging}; + +#[doc(hidden)] +pub mod fuzz { + pub fn patch(data: &[u8]) { + let text = String::from_utf8_lossy(data); + let _ = crate::is_format_patch(&text); + let _ = crate::parse_patch(&text); + let _ = crate::parse_mailbox(&text); + let mid = data.len() / 2; + let _ = crate::patch_apply::apply_delta(&data[..mid], &data[mid..]); + } +} diff --git a/knot2/crates/knot-git/src/maintenance.rs b/knot2/crates/knot-git/src/maintenance.rs new file mode 100644 index 0000000000000000000000000000000000000000..e5560ce5d79d2527a6e4bed7ad47a266f040d90b --- /dev/null +++ b/knot2/crates/knot-git/src/maintenance.rs @@ -0,0 +1,355 @@ +use std::path::Path; + +use gix::lock::acquire::Fail; +use gix::refs::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}; +use gix::refs::{FullName, Target, file::transaction::PackedRefs}; +use knot_types::UnixSeconds; + +use crate::error::{GitError, backend}; +use crate::repo::{Repo, fsync_if_present}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PackRefsReport { + pub packed: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReflogReport { + pub files: usize, + pub dropped: usize, +} + +impl Repo { + pub fn pack_refs(&self) -> Result { + self.locked(|| self.pack_refs_locked()) + } + + fn pack_refs_locked(&self) -> Result { + clear_stale_ref_locks(self.git().git_dir()); + let references = self.git().references().map_err(backend)?; + let edits: Vec = references + .all() + .map_err(backend)? + .filter_map(Result::ok) + .filter_map(|reference| { + let oid = reference.target().try_id()?.to_owned(); + let name: FullName = reference.name().to_owned(); + Some(RefEdit { + change: Change::Update { + log: LogChange { + mode: RefLog::AndReference, + force_create_reflog: false, + message: "knot pack-refs".into(), + }, + expected: PreviousValue::Any, + new: Target::Object(oid), + }, + name, + deref: false, + }) + }) + .collect(); + let packed = edits.len(); + if packed == 0 { + return Ok(PackRefsReport { packed }); + } + let committer: Option> = None; + self.git() + .refs + .transaction() + .packed_refs( + PackedRefs::DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference(Box::new( + &self.git().objects, + )), + ) + .prepare(edits, Fail::Immediately, Fail::Immediately) + .map_err(backend)? + .commit(committer) + .map_err(backend)?; + let git_dir = self.git().git_dir(); + fsync_if_present(&git_dir.join("packed-refs"))?; + fsync_if_present(&git_dir.join("refs"))?; + fsync_if_present(git_dir)?; + Ok(PackRefsReport { packed }) + } + + pub fn expire_reflogs(&self, floor_seconds: UnixSeconds) -> Result { + self.with_ref_lock(|| self.expire_reflogs_locked(floor_seconds)) + } + + fn expire_reflogs_locked(&self, floor_seconds: UnixSeconds) -> Result { + let logs_dir = self.git().git_dir().join("logs"); + if !logs_dir.exists() { + return Ok(ReflogReport { + files: 0, + dropped: 0, + }); + } + let touched_dirs = walkdir::WalkDir::new(&logs_dir) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .map(|entry| entry.into_path()) + .filter(|path| { + if is_maintenance_temp(path) { + let _ = std::fs::remove_file(path); + false + } else { + true + } + }) + .try_fold( + (0usize, 0usize, std::collections::BTreeSet::new()), + |(files, dropped, mut dirs), path| { + let removed = expire_reflog_file(&path, floor_seconds)?; + if let Some(parent) = path.parent() { + dirs.insert(parent.to_path_buf()); + } + Ok::<_, GitError>((files + 1, dropped + removed, dirs)) + }, + )?; + let (files, dropped, dirs) = touched_dirs; + dirs.iter().try_for_each(|dir| fsync_if_present(dir))?; + Ok(ReflogReport { files, dropped }) + } +} + +fn clear_stale_ref_locks(git_dir: &Path) { + let _ = std::fs::remove_file(git_dir.join("packed-refs.lock")); + walkdir::WalkDir::new(git_dir.join("refs")) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "lock")) + .for_each(|entry| { + let _ = std::fs::remove_file(entry.path()); + }); +} + +fn is_maintenance_temp(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.contains(".knot-tmp.")) +} + +fn expire_reflog_file(path: &Path, floor_seconds: UnixSeconds) -> Result { + let raw = std::fs::read(path).map_err(|error| GitError::Maintenance(error.to_string()))?; + if raw.is_empty() { + return Ok(0); + } + let lines: Vec<&[u8]> = split_keep_lines(&raw); + let total = lines.len(); + let last_index = total - 1; + let kept: Vec<&[u8]> = lines + .iter() + .enumerate() + .filter(|(index, line)| { + *index == last_index + || reflog_line_seconds(line).is_none_or(|secs| secs >= floor_seconds) + }) + .map(|(_, line)| *line) + .collect(); + let dropped = total - kept.len(); + if dropped == 0 { + return Ok(0); + } + let rewritten: Vec = kept.concat(); + rewrite_atomic(path, &rewritten)?; + Ok(dropped) +} + +fn split_keep_lines(raw: &[u8]) -> Vec<&[u8]> { + let mut out = Vec::new(); + let mut start = 0usize; + raw.iter().enumerate().for_each(|(index, byte)| { + if *byte == b'\n' { + out.push(&raw[start..=index]); + start = index + 1; + } + }); + if start < raw.len() { + out.push(&raw[start..]); + } + out +} + +fn reflog_line_seconds(line: &[u8]) -> Option { + let tab = line.iter().position(|byte| *byte == b'\t')?; + let before = &line[..tab]; + let text = std::str::from_utf8(before).ok()?; + let mut tokens = text.split_whitespace().rev(); + let _tz = tokens.next()?; + tokens.next()?.parse::().ok().map(UnixSeconds::new) +} + +fn rewrite_atomic(path: &Path, contents: &[u8]) -> Result<(), GitError> { + knot_resource::atomic_write_bytes(path, contents, knot_resource::FileMode::Inherited)?; + fsync_if_present(path) +} + +#[cfg(test)] +mod tests { + use knot_types::{AuthorName, BranchName, Email, Oid, RefName, RepoDid, UnixSeconds}; + + use crate::{EntryKind, Identity, Layout, NewCommit, RefUpdate, StagedAction, StagedChange}; + + // ah yes, of course, little johnny 4b + const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + + fn identity() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } + } + + fn commit_on(repo: &crate::Repo, body: u8, parent: Option) -> Oid { + let tree = repo + .write_staged_tree( + Oid::from_hex(EMPTY_TREE).unwrap(), + &[StagedChange { + path: knot_types::RepoPath::new(format!("file{body}.txt")).unwrap(), + action: StagedAction::Put { + content: vec![body], + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + repo.write_commit(&NewCommit { + tree, + parents: parent.into_iter().collect(), + author: identity(), + committer: identity(), + message: format!("commit {body}"), + extra_headers: Vec::new(), + }) + .unwrap() + } + + #[test] + fn pack_refs_moves_loose_refs_into_packed_refs_and_resolves() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + let main = RefName::new("refs/heads/main").unwrap(); + let side = RefName::new("refs/heads/side").unwrap(); + + let base = commit_on(&repo, 0, None); + let tip = commit_on(&repo, 1, Some(base)); + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: tip, + }) + .unwrap(); + repo.update_ref(&RefUpdate::Create { + name: side.clone(), + new: base, + }) + .unwrap(); + + let git_dir = repo.git().git_dir().to_path_buf(); + assert!(git_dir.join("refs/heads/main").exists()); + + let report = repo.pack_refs().unwrap(); + assert!(report.packed >= 2, "both branches are packed"); + assert!( + git_dir.join("packed-refs").exists(), + "packed-refs file is written" + ); + assert!( + !git_dir.join("refs/heads/main").exists(), + "loose ref file is removed once packed" + ); + assert_eq!(repo.find_ref(&main).unwrap(), Some(tip)); + assert_eq!(repo.find_ref(&side).unwrap(), Some(base)); + } + + #[test] + fn pack_refs_clears_stale_lock_files_left_by_a_crash() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:barnacle").unwrap(); + let repo = layout.create(&did).unwrap(); + let main = RefName::new("refs/heads/main").unwrap(); + + let tip = commit_on(&repo, 0, None); + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: tip, + }) + .unwrap(); + + let git_dir = repo.git().git_dir().to_path_buf(); + std::fs::write(git_dir.join("packed-refs.lock"), b"").unwrap(); + std::fs::write(git_dir.join("refs/heads/main.lock"), b"").unwrap(); + + let report = repo + .pack_refs() + .expect("crashed prior run's stale locks mustn't wedge next pack-refs"); + assert!(report.packed >= 1); + assert!( + !git_dir.join("packed-refs.lock").exists(), + "stale packed-refs lock is cleared" + ); + assert!( + !git_dir.join("refs/heads/main.lock").exists(), + "stale per-ref lock is cleared" + ); + assert_eq!(repo.find_ref(&main).unwrap(), Some(tip)); + } + + #[test] + fn expire_reflogs_drops_old_entries_but_keeps_the_newest() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + let repo = layout.create(&did).unwrap(); + let main = RefName::new("refs/heads/main").unwrap(); + + let c0 = commit_on(&repo, 0, None); + let c1 = commit_on(&repo, 1, Some(c0)); + let c2 = commit_on(&repo, 2, Some(c1)); + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: c0, + }) + .unwrap(); + repo.update_ref(&RefUpdate::Update { + name: main.clone(), + old: c0, + new: c1, + }) + .unwrap(); + repo.update_ref(&RefUpdate::Update { + name: main.clone(), + old: c1, + new: c2, + }) + .unwrap(); + + let log_path = repo.git().git_dir().join("logs/refs/heads/main"); + let before = std::fs::read(&log_path).unwrap(); + let line_count = before.iter().filter(|byte| **byte == b'\n').count(); + assert_eq!(line_count, 3, "three ref updates leave three reflog lines"); + + let report = repo.expire_reflogs(UnixSeconds::new(i64::MAX / 2)).unwrap(); + assert!( + report.files >= 2, + "main branch reflog and HEAD reflog are both rewritten" + ); + assert_eq!( + report.dropped, 4, + "future floor drops all but newest line of each of two reflogs" + ); + let after = std::fs::read(&log_path).unwrap(); + assert_eq!(after.iter().filter(|byte| **byte == b'\n').count(), 1); + assert_eq!(repo.find_ref(&main).unwrap(), Some(c2)); + + let untouched = repo.expire_reflogs(UnixSeconds::new(0)).unwrap(); + assert_eq!(untouched.dropped, 0, "zero floor keeps everything"); + } +} diff --git a/knot2/crates/knot-git/src/objects.rs b/knot2/crates/knot-git/src/objects.rs new file mode 100644 index 0000000000000000000000000000000000000000..4f7c0e16d9714ee2207dc2fdb7a9a4ec3d22c4cf --- /dev/null +++ b/knot2/crates/knot-git/src/objects.rs @@ -0,0 +1,1555 @@ +use std::collections::{HashMap, HashSet, VecDeque}; +use std::io::{BufRead, Read}; +use std::ops::ControlFlow; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use knot_storage::{Filter, PackBudget, TreeDepth}; +use knot_types::{AuthorName, Email, Oid, ParseError, RepoPath, UnixSeconds}; + +use crate::error::{GitError, SelectionLimit}; +use crate::repo::Repo; + +// why? idk. should we let this be deeper +pub const MAX_TREE_DEPTH: usize = 1024; +pub(crate) const MAX_TAG_DEPTH: usize = 32; + +#[derive(Debug, Clone, Copy)] +pub struct Wants<'a>(&'a [Oid]); + +#[derive(Debug, Clone, Copy)] +pub struct Haves<'a>(&'a [Oid]); + +#[derive(Debug, Clone, Copy)] +pub struct ShallowCommits<'a>(&'a [Oid]); + +impl<'a> Wants<'a> { + pub fn new(oids: &'a [Oid]) -> Self { + Self(oids) + } + + pub fn as_slice(self) -> &'a [Oid] { + self.0 + } +} + +impl<'a> Haves<'a> { + pub fn new(oids: &'a [Oid]) -> Self { + Self(oids) + } + + pub fn as_slice(self) -> &'a [Oid] { + self.0 + } +} + +impl<'a> ShallowCommits<'a> { + pub fn new(oids: &'a [Oid]) -> Self { + Self(oids) + } + + pub fn as_slice(self) -> &'a [Oid] { + self.0 + } +} + +pub(crate) enum Peeled { + Commit(gix::ObjectId), + Direct(gix::ObjectId), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Identity { + pub name: AuthorName, + pub email: Email, + pub time: UnixSeconds, + pub offset_seconds: i32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Commit { + pub id: Oid, + pub tree: Oid, + pub parents: Vec, + pub author: Identity, + pub committer: Identity, + pub message: String, + pub pgp_signature: Option, + pub merge_tag: Option, + pub extra_headers: Vec<(String, Vec)>, +} + +impl Commit { + pub fn change_id(&self) -> Option { + self.extra_headers + .iter() + .find(|(name, _)| name == "change-id") + .and_then(|(_, value)| std::str::from_utf8(value).ok()) + .and_then(|value| CommitChangeId::new(value).ok()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommitChangeId(String); + +impl CommitChangeId { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let valid = + !value.is_empty() && value.len() <= 100 && value.chars().all(|c| c.is_ascii_graphic()); + match valid { + true => Ok(Self(value)), + false => Err(ParseError::Invalid { + kind: "commit change-id", + value, + }), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for CommitChangeId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.pad(&self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EntryKind { + Tree, + Blob, + BlobExecutable, + Link, + Commit, +} + +impl EntryKind { + pub fn mode_octal(self) -> &'static str { + match self { + EntryKind::Tree => "0040000", + EntryKind::Blob => "0100644", + EntryKind::BlobExecutable => "0100755", + EntryKind::Link => "0120000", + EntryKind::Commit => "0160000", + } + } + + pub fn is_file(self) -> bool { + matches!(self, EntryKind::Blob | EntryKind::BlobExecutable) + } + + pub fn from_git_mode(mode: &str) -> Option { + match mode.trim() { + "100644" | "100664" => Some(EntryKind::Blob), + "100755" => Some(EntryKind::BlobExecutable), + "120000" => Some(EntryKind::Link), + "160000" => Some(EntryKind::Commit), + "040000" | "40000" => Some(EntryKind::Tree), + _ => None, + } + } +} + +impl From for gix::objs::tree::EntryKind { + fn from(kind: EntryKind) -> Self { + match kind { + EntryKind::Tree => Self::Tree, + EntryKind::Blob => Self::Blob, + EntryKind::BlobExecutable => Self::BlobExecutable, + EntryKind::Link => Self::Link, + EntryKind::Commit => Self::Commit, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TreeEntry { + pub name: String, + pub oid: Oid, + pub kind: EntryKind, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Tree { + pub entries: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileChange { + Added { + path: RepoPath, + oid: Oid, + }, + Deleted { + path: RepoPath, + oid: Oid, + }, + Modified { + path: RepoPath, + old: Oid, + new: Oid, + }, + Renamed { + from: RepoPath, + to: RepoPath, + old: Oid, + new: Oid, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Comparison { + pub commits: Vec, + pub changes: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct CommitRange { + pub base: Oid, + pub head: Oid, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct CommitDepth(u32); + +impl CommitDepth { + pub const fn new(depth: u32) -> Self { + Self(depth) + } + + const fn deeper(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Deepen { + pub depth: Option, + pub since: Option, + pub not: Vec, + pub relative: bool, +} + +impl Deepen { + pub fn is_shallow_request(&self) -> bool { + self.depth.is_some() || self.since.is_some() || !self.not.is_empty() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShallowPlan { + pub commits: Vec, + pub shallow: Vec, + pub unshallow: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackSelection { + pub send: Vec, + pub client_has: HashSet, +} + +#[derive(Clone, Copy)] +pub(crate) struct Walked { + budget: PackBudget, + count: usize, + deadline: Option, +} + +impl Walked { + pub(crate) fn new(budget: PackBudget) -> Self { + Self { + budget, + count: 0, + deadline: budget.stall().map(|stall| Instant::now() + stall), + } + } + + pub(crate) fn tick(&mut self) -> Result<(), GitError> { + self.count += 1; + if self.count > self.budget.max_objects().get() { + return Err(GitError::Selection(SelectionLimit::Objects)); + } + advance_stall(&mut self.deadline, self.budget.stall()) + } +} + +fn advance_stall(deadline: &mut Option, stall: Option) -> Result<(), GitError> { + if let (Some(deadline), Some(stall)) = (deadline.as_mut(), stall) { + let now = Instant::now(); + if now >= *deadline { + return Err(GitError::Selection(SelectionLimit::Time)); + } + *deadline = now + stall; + } + Ok(()) +} + +const MAX_LOOSE_HEADER: usize = 64; + +const PARALLEL_SELECT_MIN: usize = 4096; + +struct SharedWalk<'a> { + counter: &'a AtomicUsize, + max_objects: usize, + stall: Option, + deadline: Option, +} + +impl<'a> SharedWalk<'a> { + fn new(counter: &'a AtomicUsize, budget: &PackBudget) -> Self { + Self { + counter, + max_objects: budget.max_objects().get(), + stall: budget.stall(), + deadline: budget.stall().map(|stall| Instant::now() + stall), + } + } + + fn tick(&mut self) -> Result<(), GitError> { + let count = self.counter.fetch_add(1, Ordering::Relaxed) + 1; + if count > self.max_objects { + return Err(GitError::Selection(SelectionLimit::Objects)); + } + advance_stall(&mut self.deadline, self.stall) + } +} + +pub enum BlobReader { + Loose(std::io::BufReader>), + Packed(std::io::Cursor>), +} + +impl Read for BlobReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + BlobReader::Loose(reader) => reader.read(buf), + BlobReader::Packed(reader) => reader.read(buf), + } + } +} + +fn skip_loose_header(reader: &mut impl std::io::BufRead, oid: Oid) -> Result<(), GitError> { + let corrupt = |message: String| GitError::Corrupt { oid, message }; + let mut header = Vec::with_capacity(MAX_LOOSE_HEADER); + reader + .take(MAX_LOOSE_HEADER as u64) + .read_until(0, &mut header) + .map_err(|error| corrupt(error.to_string()))?; + if header.last() != Some(&0) || !header.starts_with(b"blob ") { + return Err(corrupt("loose object header isn't blob".to_string())); + } + Ok(()) +} + +pub(crate) fn identity(signature: gix::actor::SignatureRef<'_>) -> Result { + let time = signature + .time() + .map_err(|error| GitError::Decode(error.to_string()))?; + Ok(Identity { + name: AuthorName::new(signature.name.to_string()), + email: Email::new(signature.email.to_string()), + time: UnixSeconds::new(time.seconds), + offset_seconds: time.offset, + }) +} + +pub(crate) fn signature(identity: &Identity) -> gix::actor::Signature { + let clean = |raw: &str| raw.replace(['<', '>'], "").trim().to_string(); + gix::actor::Signature { + name: clean(identity.name.as_str()).into(), + email: clean(identity.email.as_str()).into(), + time: gix::date::Time { + seconds: identity.time.get(), + offset: identity.offset_seconds, + }, + } +} + +pub(crate) fn map_kind(kind: gix::objs::tree::EntryKind) -> EntryKind { + use gix::objs::tree::EntryKind as Source; + match kind { + Source::Tree => EntryKind::Tree, + Source::Blob => EntryKind::Blob, + Source::BlobExecutable => EntryKind::BlobExecutable, + Source::Link => EntryKind::Link, + Source::Commit => EntryKind::Commit, + } +} + +impl Repo { + fn load_object(&self, oid: Oid) -> Result, GitError> { + #[cfg(feature = "instrument")] + crate::instrument::record_read(); + match self.git().try_find_object(oid.object_id()) { + Ok(Some(object)) => Ok(object), + Ok(None) => Err(GitError::ObjectNotFound(oid)), + Err(error) => Err(GitError::Corrupt { + oid, + message: error.to_string(), + }), + } + } + + pub fn find_commit(&self, oid: Oid) -> Result { + let object = self.load_object(oid)?; + let commit = object.try_into_commit().map_err(|_| GitError::ObjectType { + oid, + expected: "commit", + })?; + let tree = Oid::from( + commit + .tree_id() + .map_err(|error| GitError::Decode(error.to_string()))? + .detach(), + ); + let parents = commit + .parent_ids() + .map(|id| Oid::from(id.detach())) + .collect(); + let author = identity( + commit + .author() + .map_err(|error| GitError::Decode(error.to_string()))?, + )?; + let committer = identity( + commit + .committer() + .map_err(|error| GitError::Decode(error.to_string()))?, + )?; + let message = commit + .message_raw() + .map_err(|error| GitError::Decode(error.to_string()))? + .to_string(); + let decoded = commit + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + let (mut pgp_signature, mut merge_tag) = (None, None); + let extra_headers = decoded + .extra_headers + .iter() + .filter_map(|(name, value)| match name.to_string().as_str() { + "gpgsig" => { + pgp_signature = Some(value.to_string()); + None + } + "mergetag" => { + merge_tag = Some(value.to_string()); + None + } + other => Some((other.to_string(), value.to_vec())), + }) + .collect(); + Ok(Commit { + id: oid, + tree, + parents, + author, + committer, + message, + pgp_signature, + merge_tag, + extra_headers, + }) + } + + pub fn find_tree(&self, oid: Oid) -> Result { + let object = self.load_object(oid)?; + let tree = object.try_into_tree().map_err(|_| GitError::ObjectType { + oid, + expected: "tree", + })?; + let decoded = tree + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + let entries = decoded + .entries + .iter() + .map(|entry| TreeEntry { + name: entry.filename.to_string(), + oid: Oid::from(entry.oid.to_owned()), + kind: map_kind(entry.mode.kind()), + }) + .collect(); + Ok(Tree { entries }) + } + + pub fn blob_size(&self, oid: Oid) -> Result { + match self.git().try_find_header(oid.object_id()) { + Ok(Some(header)) if header.kind() == gix::object::Kind::Blob => Ok(header.size()), + Ok(Some(_)) => Err(GitError::ObjectType { + oid, + expected: "blob", + }), + Ok(None) => Err(GitError::ObjectNotFound(oid)), + Err(error) => Err(GitError::Corrupt { + oid, + message: error.to_string(), + }), + } + } + + pub fn read_blob(&self, oid: Oid) -> Result, GitError> { + let object = self.load_object(oid)?; + let mut blob = object.try_into_blob().map_err(|_| GitError::ObjectType { + oid, + expected: "blob", + })?; + Ok(blob.take_data()) + } + + fn loose_object_path(&self, oid: Oid) -> PathBuf { + let hex = oid.to_hex(); + let (shard, rest) = hex.split_at(2); + self.git().git_dir().join("objects").join(shard).join(rest) + } + + pub fn remove_loose_object(&self, oid: Oid) -> Result<(), GitError> { + match std::fs::remove_file(self.loose_object_path(oid)) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(GitError::RemoveObject { + oid, + message: error.to_string(), + }), + } + } + + pub fn open_blob(&self, oid: Oid) -> Result<(u64, BlobReader), GitError> { + let header = match self.git().try_find_header(oid.object_id()) { + Ok(Some(header)) => header, + Ok(None) => return Err(GitError::ObjectNotFound(oid)), + Err(error) => { + return Err(GitError::Corrupt { + oid, + message: error.to_string(), + }); + } + }; + if header.kind() != gix::object::Kind::Blob { + return Err(GitError::ObjectType { + oid, + expected: "blob", + }); + } + let size = header.size(); + match std::fs::File::open(self.loose_object_path(oid)) { + Ok(file) => { + let mut reader = std::io::BufReader::new(flate2::read::ZlibDecoder::new(file)); + skip_loose_header(&mut reader, oid)?; + Ok((size, BlobReader::Loose(reader))) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(( + size, + BlobReader::Packed(std::io::Cursor::new(self.read_blob(oid)?)), + )), + Err(error) => Err(GitError::Corrupt { + oid, + message: error.to_string(), + }), + } + } + + fn graph_tree_and_parents(&self, commit: Oid) -> Option<(gix::ObjectId, Vec)> { + let graph = self.commit_graph()?; + let node = graph.commit_by_id(commit.object_id())?; + let tree = node.root_tree_id().to_owned(); + let parents = node + .iter_parents() + .filter_map(Result::ok) + .map(|position| graph.commit_at(position).id().to_owned()) + .collect(); + Some((tree, parents)) + } + + pub(crate) fn commit_tree(&self, commit: Oid) -> Result { + if let Some(node) = self + .commit_graph() + .and_then(|graph| graph.commit_by_id(commit.object_id())) + { + return Ok(node.root_tree_id().to_owned()); + } + let object = self.load_object(commit)?; + let commit_object = object.try_into_commit().map_err(|_| GitError::ObjectType { + oid: commit, + expected: "commit", + })?; + Ok(commit_object + .tree_id() + .map_err(|error| GitError::Decode(error.to_string()))? + .detach()) + } + + pub fn diff(&self, range: CommitRange) -> Result, GitError> { + let old_tree_oid = self.commit_tree(range.base)?; + let new_tree_oid = self.commit_tree(range.head)?; + let old_tree = self + .load_object(Oid::from(old_tree_oid))? + .try_into_tree() + .map_err(|error| GitError::Backend(error.to_string()))?; + let new_tree = self + .load_object(Oid::from(new_tree_oid))? + .try_into_tree() + .map_err(|error| GitError::Backend(error.to_string()))?; + + let mut changes = Vec::new(); + let mut emptied = std::collections::HashMap::new(); + old_tree + .changes() + .map_err(|error| GitError::Backend(error.to_string()))? + .for_each_to_obtain_tree(&new_tree, |change| { + use gix::object::tree::diff::Change; + let tree_path = |location: &gix::bstr::BStr| { + RepoPath::new(location.to_string()) + .map_err(|error| GitError::Decode(error.to_string())) + }; + // folders show up as their own entries, but only files are changes, the walk + // reports what's inside them separately + let only_folders = match &change { + Change::Addition { entry_mode, .. } + | Change::Deletion { entry_mode, .. } + | Change::Rewrite { entry_mode, .. } => entry_mode.is_tree(), + Change::Modification { + previous_entry_mode, + entry_mode, + .. + } => previous_entry_mode.is_tree() && entry_mode.is_tree(), + }; + if only_folders { + if let Change::Deletion { location, id, .. } = &change { + emptied.insert(tree_path(location)?, Oid::from(id.detach())); + } + return Ok(ControlFlow::Continue(())); + } + let mapped = match change { + Change::Addition { location, id, .. } => FileChange::Added { + path: tree_path(location)?, + oid: Oid::from(id.detach()), + }, + Change::Deletion { location, id, .. } => FileChange::Deleted { + path: tree_path(location)?, + oid: Oid::from(id.detach()), + }, + // a file that became a folder is gone, what's in the folder comes as additions + Change::Modification { + location, + entry_mode, + previous_id, + .. + } if entry_mode.is_tree() => FileChange::Deleted { + path: tree_path(location)?, + oid: Oid::from(previous_id.detach()), + }, + Change::Modification { + location, + previous_id, + id, + .. + } => FileChange::Modified { + path: tree_path(location)?, + old: Oid::from(previous_id.detach()), + new: Oid::from(id.detach()), + }, + Change::Rewrite { + source_location, + location, + source_id, + id, + .. + } => FileChange::Renamed { + from: tree_path(source_location)?, + to: tree_path(location)?, + old: Oid::from(source_id.detach()), + new: Oid::from(id.detach()), + }, + }; + changes.push(mapped); + Ok::<_, GitError>(ControlFlow::Continue(())) + }) + .map_err(|error| GitError::Backend(error.to_string()))?; + // a folder that became a file comes out as the folder's deletion plus the file's + // addition, which only makes sense together as one change to that path + Ok(changes + .into_iter() + .map(|change| match change { + FileChange::Added { path, oid } => match emptied.get(&path) { + Some(folder) => FileChange::Modified { + path, + old: *folder, + new: oid, + }, + None => FileChange::Added { path, oid }, + }, + change => change, + }) + .collect()) + } + + pub fn compare(&self, range: CommitRange) -> Result { + let commits = self.rev_walk(Wants::new(&[range.head]), Haves::new(&[range.base]))?; + let changes = self.diff(range)?; + Ok(Comparison { commits, changes }) + } + + pub(crate) fn peel( + &self, + oid: gix::ObjectId, + tags: &mut Vec, + depth: usize, + ) -> Result { + if depth == 0 { + return Err(GitError::DepthExceeded("annotated tag chain")); + } + let object = self.load_object(Oid::from(oid))?; + match object.kind { + gix::object::Kind::Commit => Ok(Peeled::Commit(oid)), + gix::object::Kind::Tree | gix::object::Kind::Blob => Ok(Peeled::Direct(oid)), + gix::object::Kind::Tag => { + tags.push(Oid::from(oid)); + let target = object + .try_into_tag() + .map_err(|error| GitError::Decode(error.to_string()))? + .target_id() + .map_err(|error| GitError::Decode(error.to_string()))? + .detach(); + self.peel(target, tags, depth - 1) + } + } + } + + pub fn peeled_target(&self, oid: Oid) -> Result, GitError> { + let object = self.load_object(oid)?; + match object.kind { + gix::object::Kind::Tag => { + let mut tags = Vec::new(); + let peeled = match self.peel(oid.object_id(), &mut tags, MAX_TAG_DEPTH)? { + Peeled::Commit(id) | Peeled::Direct(id) => Oid::from(id), + }; + Ok(Some(peeled)) + } + _ => Ok(None), + } + } + + pub(crate) fn commit_tree_and_parents( + &self, + commit: Oid, + ) -> Result<(gix::ObjectId, Vec), GitError> { + if let Some(found) = self.graph_tree_and_parents(commit) { + return Ok(found); + } + let object = self.load_object(commit)?; + let commit_object = object.try_into_commit().map_err(|_| GitError::ObjectType { + oid: commit, + expected: "commit", + })?; + let tree = commit_object + .tree_id() + .map_err(|error| GitError::Decode(error.to_string()))? + .detach(); + let parents = commit_object.parent_ids().map(|id| id.detach()).collect(); + Ok((tree, parents)) + } + + fn blob_kept(&self, oid: gix::ObjectId, filter: Filter) -> Result { + match filter { + Filter::None | Filter::TreeDepth(_) => Ok(true), + Filter::BlobNone => Ok(false), + Filter::BlobLimit(limit) => { + let header = self + .git() + .try_find_header(oid) + .map_err(|error| GitError::Corrupt { + oid: Oid::from(oid), + message: error.to_string(), + })? + .ok_or(GitError::ObjectNotFound(Oid::from(oid)))?; + Ok(header.size() < limit) + } + } + } + + fn walk_tree( + &self, + tree: gix::ObjectId, + seen: &mut HashSet, + visit: &mut dyn FnMut(Oid), + filter: Filter, + nesting: usize, + walked: &mut Walked, + ) -> Result<(), GitError> { + if nesting == 0 { + return Err(GitError::DepthExceeded("tree nesting")); + } + if tree == gix::ObjectId::empty_tree(self.git().object_hash()) { + return Ok(()); + } + if !seen.insert(tree) { + return Ok(()); + } + visit(Oid::from(tree)); + walked.tick()?; + let object = self.load_object(Oid::from(tree))?; + let decoded = object + .try_into_tree() + .map_err(|error| GitError::Decode(error.to_string()))?; + let decoded = decoded + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + decoded.entries.iter().try_for_each(|entry| { + let oid = entry.oid.to_owned(); + match entry.mode.kind() { + gix::objs::tree::EntryKind::Tree => { + self.walk_tree(oid, seen, visit, filter, nesting - 1, walked) + } + gix::objs::tree::EntryKind::Commit => Ok(()), + _ => { + if !seen.contains(&oid) && self.blob_kept(oid, filter)? { + seen.insert(oid); + visit(Oid::from(oid)); + walked.tick()?; + } + Ok(()) + } + } + }) + } + + #[allow(clippy::too_many_arguments)] + fn walk_tree_depth( + &self, + tree: gix::ObjectId, + remaining: TreeDepth, + seen: &mut HashSet, + expanded: &mut HashMap, + visit: &mut dyn FnMut(Oid), + nesting: usize, + walked: &mut Walked, + ) -> Result<(), GitError> { + if nesting == 0 { + return Err(GitError::DepthExceeded("tree nesting")); + } + if remaining.is_exhausted() || tree == gix::ObjectId::empty_tree(self.git().object_hash()) { + return Ok(()); + } + if expanded + .get(&tree) + .is_some_and(|deepest| *deepest >= remaining) + { + return Ok(()); + } + expanded.insert(tree, remaining); + if seen.insert(tree) { + visit(Oid::from(tree)); + walked.tick()?; + } + let object = self.load_object(Oid::from(tree))?; + let decoded = object + .try_into_tree() + .map_err(|error| GitError::Decode(error.to_string()))?; + let decoded = decoded + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + let next = remaining.shallower(); + decoded.entries.iter().try_for_each(|entry| { + let oid = entry.oid.to_owned(); + match entry.mode.kind() { + gix::objs::tree::EntryKind::Tree => { + self.walk_tree_depth(oid, next, seen, expanded, visit, nesting - 1, walked) + } + gix::objs::tree::EntryKind::Commit => Ok(()), + _ => { + if !next.is_exhausted() && seen.insert(oid) { + visit(Oid::from(oid)); + walked.tick()?; + } + Ok(()) + } + } + }) + } + + fn walk_root_tree( + &self, + tree: gix::ObjectId, + seen: &mut HashSet, + expanded: &mut HashMap, + visit: &mut dyn FnMut(Oid), + filter: Filter, + walked: &mut Walked, + ) -> Result<(), GitError> { + match filter { + Filter::TreeDepth(max) => { + self.walk_tree_depth(tree, max, seen, expanded, visit, MAX_TREE_DEPTH, walked) + } + _ => self.walk_tree(tree, seen, visit, filter, MAX_TREE_DEPTH, walked), + } + } + + fn walk_tree_shared( + &self, + tree: gix::ObjectId, + seen: &scc::HashSet, + out: &mut Vec, + nesting: usize, + walk: &mut SharedWalk, + ) -> Result<(), GitError> { + if nesting == 0 { + return Err(GitError::DepthExceeded("tree nesting")); + } + if tree == gix::ObjectId::empty_tree(self.git().object_hash()) { + return Ok(()); + } + if seen.insert_sync(tree).is_err() { + return Ok(()); + } + out.push(Oid::from(tree)); + walk.tick()?; + let object = self.load_object(Oid::from(tree))?; + let decoded = object + .try_into_tree() + .map_err(|error| GitError::Decode(error.to_string()))?; + let decoded = decoded + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + decoded.entries.iter().try_for_each(|entry| { + let oid = entry.oid.to_owned(); + match entry.mode.kind() { + gix::objs::tree::EntryKind::Tree => { + self.walk_tree_shared(oid, seen, out, nesting - 1, walk) + } + gix::objs::tree::EntryKind::Commit => Ok(()), + _ => { + if seen.insert_sync(oid).is_ok() { + out.push(Oid::from(oid)); + walk.tick()?; + } + Ok(()) + } + } + }) + } + + fn walk_send_trees( + &self, + send: &[(Oid, gix::ObjectId, Vec)], + seen: HashSet, + mut out: Vec, + budget: PackBudget, + ) -> Result, GitError> { + let seen: scc::HashSet = seen.into_iter().collect(); + let counter = AtomicUsize::new(out.len()); + let path = self.path().to_owned(); + let walk = + |batch: &[(Oid, gix::ObjectId, Vec)]| -> Result, GitError> { + let local = Repo::open(&path)?; + let mut walk = SharedWalk::new(&counter, &budget); + batch.iter().try_fold( + Vec::new(), + |mut acc, (commit, tree, _)| -> Result, GitError> { + if seen.insert_sync(commit.object_id()).is_ok() { + acc.push(*commit); + } + local.walk_tree_shared( + *tree, + &seen, + &mut acc, + MAX_TREE_DEPTH, + &mut walk, + )?; + Ok(acc) + }, + ) + }; + out.extend(knot_resource::map_chunks(send, walk)?); + Ok(out) + } + + fn collect_direct( + &self, + oid: Oid, + seen: &mut HashSet, + expanded: &mut HashMap, + visit: &mut dyn FnMut(Oid), + filter: Filter, + walked: &mut Walked, + ) -> Result<(), GitError> { + let object = self.load_object(oid)?; + match (object.kind, filter) { + (gix::object::Kind::Tree, Filter::TreeDepth(max)) => self.walk_tree_depth( + oid.object_id(), + max.deeper(), + seen, + expanded, + visit, + MAX_TREE_DEPTH, + walked, + ), + (gix::object::Kind::Tree, _) => { + self.walk_tree(oid.object_id(), seen, visit, filter, MAX_TREE_DEPTH, walked) + } + _ => { + if seen.insert(oid.object_id()) { + visit(oid); + walked.tick()?; + } + Ok(()) + } + } + } + + pub(crate) fn commit_time(&self, commit: gix::ObjectId) -> Result { + if let Some(node) = self + .commit_graph() + .and_then(|graph| graph.commit_by_id(commit)) + { + return Ok(UnixSeconds::new(node.committer_timestamp() as i64)); + } + let object = self.load_object(Oid::from(commit))?; + let commit = object.try_into_commit().map_err(|_| GitError::ObjectType { + oid: Oid::from(commit), + expected: "commit", + })?; + let time = commit + .committer() + .map_err(|error| GitError::Decode(error.to_string()))? + .time() + .map_err(|error| GitError::Decode(error.to_string()))?; + Ok(UnixSeconds::new(time.seconds)) + } + + pub fn shallow_walk( + &self, + wants: Wants<'_>, + deepen: &Deepen, + client_shallow: ShallowCommits<'_>, + ) -> Result { + let want_commits: Vec = wants + .as_slice() + .iter() + .map(|want| self.peel(want.object_id(), &mut Vec::new(), MAX_TAG_DEPTH)) + .collect::, _>>()? + .into_iter() + .filter_map(|peeled| match peeled { + Peeled::Commit(commit) => Some(commit), + Peeled::Direct(_) => None, + }) + .collect(); + + let excluded: HashSet = if deepen.not.is_empty() { + HashSet::new() + } else { + self.rev_walk(Wants::new(&deepen.not), Haves::new(&[]))? + .into_iter() + .map(Oid::object_id) + .collect() + }; + + let drop = |oid: gix::ObjectId, depth: CommitDepth| -> Result { + if excluded.contains(&oid) { + return Ok(true); + } + if let Some(max) = deepen.depth + && depth > max + { + return Ok(true); + } + if let Some(since) = deepen.since + && self.commit_time(oid)? < since + { + return Ok(true); + } + Ok(false) + }; + + let grafts = self.shallow_grafts()?; + let mut min_depth: HashMap = HashMap::new(); + let mut parents_of: HashMap> = HashMap::new(); + let mut queue: VecDeque<(gix::ObjectId, CommitDepth)> = want_commits + .iter() + .map(|commit| (*commit, CommitDepth::new(1))) + .collect(); + if deepen.relative { + client_shallow + .as_slice() + .iter() + .for_each(|oid| queue.push_back((oid.object_id(), CommitDepth::new(0)))); + } + while let Some((commit, depth)) = queue.pop_front() { + if drop(commit, depth)? { + continue; + } + if min_depth.get(&commit).is_some_and(|seen| *seen <= depth) { + continue; + } + min_depth.insert(commit, depth); + let (_, parents) = self.commit_tree_and_parents(Oid::from(commit))?; + if !grafts.contains(&commit) { + parents + .iter() + .for_each(|parent| queue.push_back((*parent, depth.deeper()))); + } + parents_of.insert(commit, parents); + } + + let included: HashSet = min_depth.keys().copied().collect(); + let boundary: HashSet = included + .iter() + .filter(|commit| { + parents_of + .get(*commit) + .is_some_and(|parents| parents.iter().any(|parent| !included.contains(parent))) + }) + .copied() + .collect(); + + let commits: Vec = min_depth.keys().map(|oid| Oid::from(*oid)).collect(); + let shallow: Vec = boundary.iter().map(|oid| Oid::from(*oid)).collect(); + let unshallow: Vec = client_shallow + .as_slice() + .iter() + .filter(|oid| { + min_depth.contains_key(&oid.object_id()) && !boundary.contains(&oid.object_id()) + }) + .copied() + .collect(); + Ok(ShallowPlan { + commits, + shallow, + unshallow, + }) + } + + pub fn select_shallow_objects( + &self, + wants: Wants, + commits: ShallowCommits, + haves: Haves, + filter: Filter, + budget: PackBudget, + ) -> Result { + let wants = wants.as_slice(); + let commits = commits.as_slice(); + let haves = haves.as_slice(); + let mut walked = Walked::new(budget); + let mut seen: HashSet = HashSet::new(); + let mut expanded: HashMap = HashMap::new(); + let mut have_commits: Vec = Vec::new(); + haves + .iter() + .filter(|have| self.contains(**have)) + .try_for_each(|have| -> Result<(), GitError> { + match self.peel(have.object_id(), &mut Vec::new(), MAX_TAG_DEPTH)? { + Peeled::Commit(commit) => { + have_commits.push(commit); + let tree = self.commit_tree(Oid::from(commit))?; + self.walk_tree( + tree, + &mut seen, + &mut |_| {}, + Filter::None, + MAX_TREE_DEPTH, + &mut walked, + ) + } + Peeled::Direct(direct) => self.collect_direct( + Oid::from(direct), + &mut seen, + &mut expanded, + &mut |_| {}, + Filter::None, + &mut walked, + ), + } + })?; + let client_has: HashSet = seen + .iter() + .copied() + .chain(have_commits) + .map(Oid::from) + .collect(); + + let mut want_tags = Vec::new(); + wants.iter().try_for_each(|want| -> Result<(), GitError> { + self.peel(want.object_id(), &mut want_tags, MAX_TAG_DEPTH) + .map(|_| ()) + })?; + + let mut out: Vec = Vec::new(); + want_tags + .iter() + .try_for_each(|tag| -> Result<(), GitError> { + if seen.insert(tag.object_id()) { + out.push(*tag); + walked.tick()?; + } + Ok(()) + })?; + commits + .iter() + .try_for_each(|commit| -> Result<(), GitError> { + if seen.insert(commit.object_id()) { + out.push(*commit); + walked.tick()?; + } + let tree = self.commit_tree(*commit)?; + self.walk_root_tree( + tree, + &mut seen, + &mut expanded, + &mut |oid| out.push(oid), + filter, + &mut walked, + ) + })?; + Ok(PackSelection { + send: out, + client_has, + }) + } + + pub fn select_pack_objects(&self, wants: Wants, haves: Haves) -> Result, GitError> { + self.select_pack_objects_filtered(wants, haves, Filter::None, PackBudget::unbounded()) + .map(|selection| selection.send) + } + + pub fn clone_roots(&self, wants: &[Oid], budget: PackBudget) -> Result, GitError> { + let mut walked = Walked::new(budget); + let mut want_tags = Vec::new(); + let mut want_commits = Vec::new(); + let mut want_direct = Vec::new(); + wants.iter().try_for_each(|want| -> Result<(), GitError> { + match self.peel(want.object_id(), &mut want_tags, MAX_TAG_DEPTH)? { + Peeled::Commit(commit) => want_commits.push(Oid::from(commit)), + Peeled::Direct(direct) => want_direct.push(Oid::from(direct)), + } + Ok(()) + })?; + let commits = + self.rev_walk_each(Wants::new(&want_commits), Haves::new(&[]), &mut walked)?; + Ok(want_tags + .into_iter() + .chain(commits) + .chain(want_direct) + .collect()) + } + + pub fn reachable_commits( + &self, + tips: &[Oid], + budget: PackBudget, + ) -> Result, GitError> { + let mut walked = Walked::new(budget); + let commits: Vec = self + .peel_to_commits(tips.iter().copied())? + .into_iter() + .map(Oid::from) + .collect(); + Ok(self + .rev_walk_each(Wants::new(&commits), Haves::new(&[]), &mut walked)? + .into_iter() + .collect()) + } + + fn peel_to_commits( + &self, + oids: impl Iterator, + ) -> Result, GitError> { + oids.map(|oid| self.peel(oid.object_id(), &mut Vec::new(), MAX_TAG_DEPTH)) + .filter_map(|peeled| match peeled { + Ok(Peeled::Commit(commit)) => Some(Ok(commit)), + Ok(Peeled::Direct(_)) => None, + Err(error) => Some(Err(error)), + }) + .collect() + } + + pub fn wants_satisfied_by(&self, wants: Wants, haves: Haves) -> Result { + let wants = wants.as_slice(); + let haves = haves.as_slice(); + let commons: HashSet = self + .peel_to_commits(haves.iter().copied().filter(|have| self.contains(*have)))? + .into_iter() + .collect(); + if commons.is_empty() { + return Ok(false); + } + let oldest = commons + .iter() + .map(|commit| self.commit_time(*commit)) + .collect::, _>>()? + .into_iter() + .min() + .unwrap_or(UnixSeconds::new(i64::MIN)); + let grafts = self.shallow_grafts()?; + + let reaches_a_common = |want: gix::ObjectId| -> Result { + let mut seen: HashSet = HashSet::new(); + let mut queue: VecDeque = VecDeque::from([want]); + while let Some(commit) = queue.pop_front() { + if commons.contains(&commit) { + return Ok(true); + } + if !seen.insert(commit) || grafts.contains(&commit) { + continue; + } + if self.commit_time(commit)? < oldest { + continue; + } + let (_, parents) = self.commit_tree_and_parents(Oid::from(commit))?; + queue.extend(parents); + } + Ok(false) + }; + + self.peel_to_commits(wants.iter().copied())? + .into_iter() + .try_fold(true, |all, want| Ok(all && reaches_a_common(want)?)) + } + + pub fn select_pack_objects_filtered( + &self, + wants: Wants, + haves: Haves, + filter: Filter, + budget: PackBudget, + ) -> Result { + let wants = wants.as_slice(); + let haves = haves.as_slice(); + let mut walked = Walked::new(budget); + let mut expanded: HashMap = HashMap::new(); + let mut want_tags = Vec::new(); + let mut want_commits = Vec::new(); + let mut want_direct = Vec::new(); + wants.iter().try_for_each(|want| -> Result<(), GitError> { + match self.peel(want.object_id(), &mut want_tags, MAX_TAG_DEPTH)? { + Peeled::Commit(commit) => want_commits.push(Oid::from(commit)), + Peeled::Direct(direct) => want_direct.push(Oid::from(direct)), + } + Ok(()) + })?; + + let mut have_tags = Vec::new(); + let mut have_commits = Vec::new(); + let mut have_direct = Vec::new(); + haves + .iter() + .filter(|have| self.contains(**have)) + .try_for_each(|have| -> Result<(), GitError> { + match self.peel(have.object_id(), &mut have_tags, MAX_TAG_DEPTH)? { + Peeled::Commit(commit) => have_commits.push(Oid::from(commit)), + Peeled::Direct(direct) => have_direct.push(Oid::from(direct)), + } + Ok(()) + })?; + + let grafts = self.shallow_grafts()?; + let send: Vec<(Oid, gix::ObjectId, Vec)> = self + .rev_walk_each( + Wants::new(&want_commits), + Haves::new(&have_commits), + &mut walked, + )? + .into_iter() + .map(|commit| { + let (tree, parents) = self.commit_tree_and_parents(commit)?; + let parents = match grafts.contains(&commit.object_id()) { + true => Vec::new(), + false => parents, + }; + Ok::<_, GitError>((commit, tree, parents)) + }) + .collect::>()?; + let send_set: HashSet = send + .iter() + .map(|(commit, _, _)| commit.object_id()) + .collect(); + + let mut uninteresting: HashSet = + have_tags.iter().map(|tag| tag.object_id()).collect(); + let boundary_commits: Vec = have_commits + .iter() + .map(|commit| commit.object_id()) + .chain( + send.iter() + .flat_map(|(_, _, parents)| parents.iter().copied()) + .filter(|parent| !send_set.contains(parent)), + ) + .collect(); + boundary_commits + .iter() + .try_for_each(|commit| -> Result<(), GitError> { + let tree = self.commit_tree(Oid::from(*commit))?; + self.walk_tree( + tree, + &mut uninteresting, + &mut |_| {}, + Filter::None, + MAX_TREE_DEPTH, + &mut walked, + ) + })?; + have_direct.iter().try_for_each(|direct| { + self.collect_direct( + *direct, + &mut uninteresting, + &mut expanded, + &mut |_| {}, + Filter::None, + &mut walked, + ) + })?; + let client_has: HashSet = uninteresting + .iter() + .copied() + .chain(boundary_commits) + .map(Oid::from) + .collect(); + + let mut seen = uninteresting; + let mut out: Vec = Vec::new(); + want_tags + .iter() + .try_for_each(|tag| -> Result<(), GitError> { + if seen.insert(tag.object_id()) { + out.push(*tag); + walked.tick()?; + } + Ok(()) + })?; + if matches!(filter, Filter::None) + && want_direct.is_empty() + && send.len() >= PARALLEL_SELECT_MIN + { + out = self.walk_send_trees(&send, seen, out, walked.budget)?; + } else { + send.iter() + .try_for_each(|(commit, tree, _)| -> Result<(), GitError> { + if seen.insert(commit.object_id()) { + out.push(*commit); + } + self.walk_root_tree( + *tree, + &mut seen, + &mut expanded, + &mut |oid| out.push(oid), + filter, + &mut walked, + ) + })?; + want_direct.iter().try_for_each(|direct| { + self.collect_direct( + *direct, + &mut seen, + &mut expanded, + &mut |oid| out.push(oid), + filter, + &mut walked, + ) + })?; + } + Ok(PackSelection { + send: out, + client_has, + }) + } +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::PermissionsExt; + + use knot_types::RepoDid; + + use super::*; + use crate::Layout; + + fn seeded() -> (tempfile::TempDir, Layout, RepoDid) { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + (dir, layout, did) + } + + #[test] + fn open_blob_streams_a_loose_blob() { + let (_dir, layout, did) = seeded(); + let content: Vec = (0..8192u32).map(|byte| byte as u8).collect(); + let repo = layout.open(&did).unwrap(); + let oid = Oid::from(repo.git().write_blob(&content).unwrap().detach()); + + let reread = layout.open(&did).unwrap(); + let (size, mut reader) = reread.open_blob(oid).unwrap(); + assert_eq!(size, content.len() as u64); + assert!(matches!(reader, BlobReader::Loose(_))); + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).unwrap(); + assert_eq!(buf, content); + } + + #[test] + fn open_blob_rejects_a_non_blob() { + let (_dir, layout, did) = seeded(); + let repo = layout.open(&did).unwrap(); + let tree = repo + .git() + .write_object(gix::objs::Tree { + entries: Vec::new(), + }) + .unwrap() + .detach(); + assert!(matches!( + repo.open_blob(Oid::from(tree)), + Err(GitError::ObjectType { + expected: "blob", + .. + }) + )); + } + + #[test] + fn corrupt_loose_object_is_a_typed_error_not_a_panic() { + let (_dir, layout, did) = seeded(); + let repo = layout.open(&did).unwrap(); + let oid = Oid::from( + repo.git() + .write_blob(b"hello streaming world\n") + .unwrap() + .detach(), + ); + let loose = repo.loose_object_path(oid); + + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o644)).unwrap(); + std::fs::write(&loose, b"this isn't a valid zlib object").unwrap(); + + let reopened = layout.open(&did).unwrap(); + assert!(matches!( + reopened.read_blob(oid), + Err(GitError::Corrupt { .. }) + )); + assert!(matches!( + reopened.open_blob(oid), + Err(GitError::Corrupt { .. }) + )); + } + + #[test] + fn missing_object_is_not_found() { + let (_dir, layout, did) = seeded(); + let repo = layout.open(&did).unwrap(); + let absent = Oid::from_hex("dead00000000000000000000000000000000beef").unwrap(); + assert!(matches!( + repo.read_blob(absent), + Err(GitError::ObjectNotFound(_)) + )); + assert!(matches!( + repo.open_blob(absent), + Err(GitError::ObjectNotFound(_)) + )); + } +} diff --git a/knot2/crates/knot-git/src/patch.rs b/knot2/crates/knot-git/src/patch.rs new file mode 100644 index 0000000000000000000000000000000000000000..28af930212632a28dff096b09262089beba8e6e1 --- /dev/null +++ b/knot2/crates/knot-git/src/patch.rs @@ -0,0 +1,596 @@ +use std::convert::Infallible; +use std::fmt::Write as _; +use std::io::Write; +use std::ops::ControlFlow; + +use flate2::Compression; +use flate2::write::ZlibEncoder; +use gix::diff::blob::unified_diff::{ConsumeHunk, ContextSize, DiffLineKind, HunkHeader}; +use gix::diff::blob::{Algorithm, Diff, InternedInput, UnifiedDiff}; +use knot_types::{ChangedFiles, ChangedFilesBudget, Listing, Oid, RepoPath}; + +use crate::base85; +use crate::error::{GitError, backend}; +use crate::objects::EntryKind; +use crate::repo::Repo; + +const BINARY_SNIFF_BYTES: usize = 8000; +const BLOCK_HEADER_MAX: usize = "literal 18446744073709551615\n".len(); +const BINARY_PATCH_HEADER: &str = "GIT binary patch\n"; +pub const MAX_DIFF_BLOB_BYTES: u64 = 25 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LineOp { + Context, + Delete, + Add, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HunkLine { + pub op: LineOp, + pub text: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PatchRange { + pub base: Option, + pub head: Oid, +} + +// Just making sure a count in a start slot doesn't even compile. +knot_types::scalar_newtype! { + pub struct LineNumber(u32); + pub struct LineCount(u32); +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Hunk { + pub old_start: LineNumber, + pub old_lines: LineCount, + pub new_start: LineNumber, + pub new_lines: LineCount, + pub lines: Vec, +} + +impl Hunk { + pub fn added(&self) -> LineCount { + self.count(LineOp::Add) + } + + pub fn deleted(&self) -> LineCount { + self.count(LineOp::Delete) + } + + fn count(&self, op: LineOp) -> LineCount { + LineCount::new( + self.lines + .iter() + .filter(|line| line.op == op) + .count() + .try_into() + .unwrap_or(u32::MAX), + ) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PatchStatus { + Added, + Deleted, + Modified, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BinarySizes { + pub old: u64, + pub new: u64, +} + +impl BinarySizes { + fn of(old: &[u8], new: &[u8]) -> Self { + Self { + old: old.len() as u64, + new: new.len() as u64, + } + } + + fn wire_bound(self) -> u64 { + let block = |inflated: u64| { + let deflated = inflated + .saturating_add(inflated.div_ceil(8)) + .saturating_add(inflated.div_ceil(64)) + .saturating_add(11); + base85::encoded_len(deflated).saturating_add(BLOCK_HEADER_MAX as u64 + 1) + }; + block(self.old) + .saturating_add(block(self.new)) + .saturating_add(BINARY_PATCH_HEADER.len() as u64) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BinaryBudget { + Omit, + Spend { remaining: u64, omitted: bool }, +} + +impl BinaryBudget { + pub fn new(bytes: u64) -> Self { + Self::Spend { + remaining: bytes, + omitted: false, + } + } + + pub fn omitted(self) -> bool { + matches!(self, Self::Spend { omitted: true, .. }) + } + + fn admit(&mut self, sizes: BinarySizes) -> bool { + match self { + Self::Omit => false, + Self::Spend { remaining, omitted } => match remaining.checked_sub(sizes.wire_bound()) { + Some(rest) => { + *remaining = rest; + true + } + None => { + *omitted = true; + false + } + }, + } + } +} + +pub fn literal_block(content: &[u8], out: &mut String) -> Result<(), GitError> { + // git's default zlib level for binary patches + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::fast()); + encoder.write_all(content).map_err(backend)?; + writeln!(out, "literal {}", content.len()).expect("formatting into a String never fails"); + base85::encode(&encoder.finish().map_err(backend)?, out); + out.push('\n'); + Ok(()) +} + +fn encode_binary(old: &[u8], new: &[u8]) -> Result { + let mut text = String::from(BINARY_PATCH_HEADER); + literal_block(new, &mut text)?; + literal_block(old, &mut text)?; + Ok(BinaryDiff::Encoded { + sizes: BinarySizes::of(old, new), + text, + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BinaryDiff { + Encoded { sizes: BinarySizes, text: String }, + Omitted(BinarySizes), + Unchanged(u64), +} + +impl BinaryDiff { + pub fn sizes(&self) -> BinarySizes { + match self { + Self::Encoded { sizes, .. } | Self::Omitted(sizes) => *sizes, + Self::Unchanged(bytes) => BinarySizes { + old: *bytes, + new: *bytes, + }, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PatchBody { + Text(Vec), + Binary(BinaryDiff), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FilePatch { + pub status: PatchStatus, + pub path: RepoPath, + pub old_oid: Oid, + pub new_oid: Oid, + pub old_kind: Option, + pub new_kind: Option, + pub body: PatchBody, +} + +impl FilePatch { + pub fn is_binary(&self) -> bool { + matches!(self.body, PatchBody::Binary(_)) + } + + pub fn hunks(&self) -> &[Hunk] { + match &self.body { + PatchBody::Text(hunks) => hunks, + PatchBody::Binary(_) => &[], + } + } +} + +pub fn is_binary(content: &[u8]) -> bool { + content[..content.len().min(BINARY_SNIFF_BYTES)].contains(&0) +} + +struct CollectHunks { + hunks: Vec, +} + +impl ConsumeHunk for CollectHunks { + type Out = Vec; + + fn consume_hunk( + &mut self, + header: HunkHeader, + lines: &[(DiffLineKind, &[u8])], + ) -> std::io::Result<()> { + let map_op = |kind: DiffLineKind| match kind { + DiffLineKind::Context => LineOp::Context, + DiffLineKind::Remove => LineOp::Delete, + DiffLineKind::Add => LineOp::Add, + }; + let adjust = |start: u32, len: u32| { + if len == 0 { + start.saturating_sub(1) + } else { + start + } + }; + self.hunks.push(Hunk { + old_start: LineNumber::new(adjust(header.before_hunk_start, header.before_hunk_len)), + old_lines: LineCount::new(header.before_hunk_len), + new_start: LineNumber::new(adjust(header.after_hunk_start, header.after_hunk_len)), + new_lines: LineCount::new(header.after_hunk_len), + lines: lines + .iter() + .map(|(kind, text)| HunkLine { + op: map_op(*kind), + text: text.to_vec(), + }) + .collect(), + }); + Ok(()) + } + + fn finish(self) -> Self::Out { + self.hunks + } +} + +fn text_hunks(old: &[u8], new: &[u8]) -> Result, GitError> { + let input = InternedInput::new(old, new); + let diff = Diff::compute(Algorithm::Histogram, &input); + UnifiedDiff::new( + &diff, + &input, + CollectHunks { hunks: Vec::new() }, + ContextSize::symmetrical(3), + ) + .consume() + .map_err(backend) +} + +enum Side { + Absent, + Present { oid: Oid, kind: EntryKind }, +} + +impl Side { + fn oid(&self, absent: Oid) -> Oid { + match self { + Side::Absent => absent, + Side::Present { oid, .. } => *oid, + } + } + + fn kind(&self) -> Option { + match self { + Side::Absent => None, + Side::Present { kind, .. } => Some(*kind), + } + } +} + +pub fn subproject_line(oid: Oid) -> Vec { + format!("Subproject commit {}\n", oid.to_hex()).into_bytes() +} + +impl Repo { + fn patch_content(&self, side: &Side) -> Result, GitError> { + match side { + Side::Absent => Ok(Vec::new()), + Side::Present { oid, kind } => match kind { + EntryKind::Commit => Ok(subproject_line(*oid)), + EntryKind::Tree => Ok(Vec::new()), + _ => self.read_blob(*oid), + }, + } + } + + fn sides_past_diff_budget( + &self, + old: &Side, + new: &Side, + ) -> Result, GitError> { + let size = |side: &Side| match side { + Side::Absent + | Side::Present { + kind: EntryKind::Tree, + .. + } => Ok(None), + Side::Present { + oid, + kind: EntryKind::Commit, + } => Ok(Some(subproject_line(*oid).len() as u64)), + Side::Present { oid, .. } => self.blob_size(*oid).map(Some), + }; + let (old, new) = (size(old)?, size(new)?); + let past = |bytes: Option| bytes.is_some_and(|bytes| bytes > MAX_DIFF_BLOB_BYTES); + Ok((past(old) || past(new)).then(|| BinarySizes { + old: old.unwrap_or(0), + new: new.unwrap_or(0), + })) + } + + fn file_patch( + &self, + status: PatchStatus, + path: RepoPath, + old: Side, + new: Side, + budget: &mut BinaryBudget, + ) -> Result { + let same_content = matches!( + (&old, &new), + (Side::Present { oid: before, .. }, Side::Present { oid: after, .. }) + if before == after + ); + let body = match self.sides_past_diff_budget(&old, &new)? { + Some(sizes) => PatchBody::Binary(BinaryDiff::Omitted(sizes)), + None => { + let old_content = self.patch_content(&old)?; + let new_content = self.patch_content(&new)?; + match is_binary(&old_content) || is_binary(&new_content) { + true => { + let sizes = BinarySizes::of(&old_content, &new_content); + PatchBody::Binary(match same_content { + true => BinaryDiff::Unchanged(sizes.new), + false => match budget.admit(sizes) { + true => encode_binary(&old_content, &new_content)?, + false => BinaryDiff::Omitted(sizes), + }, + }) + } + false => PatchBody::Text(text_hunks(&old_content, &new_content)?), + } + } + }; + Ok(FilePatch { + status, + path, + old_oid: old.oid(self.object_format().null_oid()), + new_oid: new.oid(self.object_format().null_oid()), + old_kind: old.kind(), + new_kind: new.kind(), + body, + }) + } + + fn diff_trees(&self, range: PatchRange) -> Result<(gix::Tree<'_>, gix::Tree<'_>), GitError> { + let PatchRange { + base: old_commit, + head: new_commit, + } = range; + let new_tree = self.root_tree(self.peel_to_commit(new_commit)?)?; + let old_tree = match old_commit { + Some(commit) => self.root_tree(self.peel_to_commit(commit)?)?, + None => self.git().empty_tree(), + }; + Ok((old_tree, new_tree)) + } + + pub fn changed_paths(&self, range: PatchRange) -> Result { + let (old_tree, new_tree) = self.diff_trees(range)?; + let mut budget = ChangedFilesBudget::new(); + let walked = old_tree + .changes() + .map_err(backend)? + .options(|options| { + options.track_rewrites(None); + }) + .for_each_to_obtain_tree(&new_tree, |change| -> Result, Infallible> { + use gix::object::tree::diff::Change; + let (location, is_tree) = match change { + Change::Addition { + location, + entry_mode, + .. + } + | Change::Deletion { + location, + entry_mode, + .. + } => (location, entry_mode.is_tree()), + Change::Modification { + location, + previous_entry_mode, + entry_mode, + .. + } => ( + location, + previous_entry_mode.is_tree() || entry_mode.is_tree(), + ), + Change::Rewrite { .. } => return Ok(ControlFlow::Continue(())), + }; + match (is_tree, RepoPath::new(location.to_string())) { + (true, _) => Ok(ControlFlow::Continue(())), + (false, Ok(path)) => Ok(budget.admit(path)), + (false, Err(_)) => Ok(budget.truncate()), + } + }); + let changed = budget.finish(); + // When the above gives us `Break`, + // gix doesn't return partial-success + // but instead `Error::Cancelled`. + // So if the listing comes out truncated, + // the "error" in `walked` is our own stop-sign given + // back at us and we ignore it on purpose. + // If the listing is complete, + // nothing ever asked to stop + // and when `walked` errors out it's actually + // from the diff itself that we should believe. + match changed.listing() { + Listing::Truncated => Ok(changed), + Listing::Complete => walked.map(|_| changed).map_err(backend), + } + } + + pub fn commit_patches( + &self, + range: PatchRange, + budget: &mut BinaryBudget, + ) -> Result, GitError> { + let (old_tree, new_tree) = self.diff_trees(range)?; + let mut sides: Vec<(PatchStatus, String, Side, Side)> = Vec::new(); + old_tree + .changes() + .map_err(backend)? + .options(|options| { + options.track_rewrites(None); + }) + .for_each_to_obtain_tree(&new_tree, |change| { + use gix::object::tree::diff::Change; + match change { + Change::Addition { + location, + id, + entry_mode, + .. + } => sides.push(( + PatchStatus::Added, + location.to_string(), + Side::Absent, + Side::Present { + oid: Oid::from(id.detach()), + kind: crate::objects::map_kind(entry_mode.kind()), + }, + )), + Change::Deletion { + location, + id, + entry_mode, + .. + } => sides.push(( + PatchStatus::Deleted, + location.to_string(), + Side::Present { + oid: Oid::from(id.detach()), + kind: crate::objects::map_kind(entry_mode.kind()), + }, + Side::Absent, + )), + Change::Modification { + location, + previous_id, + id, + previous_entry_mode, + entry_mode, + } => sides.push(( + PatchStatus::Modified, + location.to_string(), + Side::Present { + oid: Oid::from(previous_id.detach()), + kind: crate::objects::map_kind(previous_entry_mode.kind()), + }, + Side::Present { + oid: Oid::from(id.detach()), + kind: crate::objects::map_kind(entry_mode.kind()), + }, + )), + Change::Rewrite { .. } => {} + } + Ok::<_, std::convert::Infallible>(std::ops::ControlFlow::Continue(())) + }) + .map_err(backend)?; + + sides + .into_iter() + .filter(|(_, _, old, new)| { + !matches!( + (old, new), + ( + Side::Present { + kind: EntryKind::Tree, + .. + }, + _ + ) | ( + _, + Side::Present { + kind: EntryKind::Tree, + .. + } + ) + ) + }) + .map(|(status, path, old, new)| { + let path = + RepoPath::new(path).map_err(|error| GitError::Decode(error.to_string()))?; + self.file_patch(status, path, old, new, budget) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_wire_bound_covers_every_byte_the_patch_writes() { + let payload = |len: usize, fill: fn(usize) -> u8| (0..len).map(fill).collect::>(); + [0usize, 1, 3, 4, 51, 52, 53, 1000, 65_536] + .into_iter() + .flat_map(|len| { + let noise = payload(len, |index| (index as u8).wrapping_mul(37) ^ 0x5a); + [(payload(len, |_| 0), noise.clone()), (noise, Vec::new())] + }) + .for_each(|(old, new)| { + let BinaryDiff::Encoded { sizes, text } = encode_binary(&old, &new).unwrap() else { + panic!("encode_binary returns Encoded for every payload"); + }; + assert!( + text.len() as u64 <= sizes.wire_bound(), + "payload of {} bytes: expected at most {}, wrote {}", + old.len().max(new.len()), + sizes.wire_bound(), + text.len() + ); + }); + } + + #[test] + fn the_budget_reports_the_first_payload_it_refuses() { + let sizes = BinarySizes { old: 0, new: 4096 }; + let mut budget = BinaryBudget::new(sizes.wire_bound()); + assert!(budget.admit(sizes)); + assert!( + !budget.omitted(), + "omitted is false while admit returns true" + ); + assert!(!budget.admit(sizes)); + assert!(budget.omitted(), "omitted is true once admit returns false"); + + let mut omit = BinaryBudget::Omit; + assert!(!omit.admit(sizes)); + assert!( + !omit.omitted(), + "omitted is false under Omit, where admit always returns false" + ); + } +} diff --git a/knot2/crates/knot-git/src/patch_apply.rs b/knot2/crates/knot-git/src/patch_apply.rs new file mode 100644 index 0000000000000000000000000000000000000000..047d5951c658af8a0c1f3e896c7bc4c947637693 --- /dev/null +++ b/knot2/crates/knot-git/src/patch_apply.rs @@ -0,0 +1,735 @@ +use std::collections::BTreeMap; + +use knot_types::{Oid, RepoPath}; + +use crate::error::{GitError, backend}; +use crate::objects::{EntryKind, Identity, signature}; +use crate::patch::{Hunk, LineOp, MAX_DIFF_BLOB_BYTES, subproject_line}; +use crate::patch_parse::{FileIntent, ParsedFile, PatchPayload}; +use crate::repo::Repo; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConflictReason { + AlreadyExists, + DoesNotExist, + DoesNotApply, +} + +impl ConflictReason { + pub fn as_str(self) -> &'static str { + match self { + ConflictReason::AlreadyExists => "file already exists", + ConflictReason::DoesNotExist => "file doesn't exist", + ConflictReason::DoesNotApply => "patch doesn't apply", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Conflict { + pub path: String, + pub reason: ConflictReason, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StagedAction { + Put { content: Vec, kind: EntryKind }, + PutBlob { oid: Oid, kind: EntryKind }, + PutGitlink { oid: Oid }, + Remove, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StagedChange { + pub path: RepoPath, + pub action: StagedAction, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum ApplyOutcome { + Clean(Vec), + Conflicted(Vec), +} + +#[derive(Debug, thiserror::Error)] +pub enum ApplyError { + #[error(transparent)] + Git(#[from] GitError), + #[error("file touched by patch exceeds {MAX_DIFF_BLOB_BYTES}-byte limit")] + TooLarge, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NewCommit { + pub tree: Oid, + pub parents: Vec, + pub author: Identity, + pub committer: Identity, + pub message: String, + pub extra_headers: Vec<(String, Vec)>, +} + +fn patch_path(raw: &str) -> Option { + RepoPath::new(raw).ok().filter(|path| !path.names_dot_git()) +} + +fn split_lines(content: &[u8]) -> Vec<&[u8]> { + content.split_inclusive(|&byte| byte == b'\n').collect() +} + +fn images(hunk: &Hunk) -> (Vec<&[u8]>, Vec<&[u8]>) { + let pick = |keep: fn(LineOp) -> bool| { + hunk.lines + .iter() + .filter(move |line| keep(line.op)) + .map(|line| line.text.as_slice()) + .collect() + }; + ( + pick(|op| matches!(op, LineOp::Context | LineOp::Delete)), + pick(|op| matches!(op, LineOp::Context | LineOp::Add)), + ) +} + +fn find_match(lines: &[&[u8]], pre: &[&[u8]], cursor: usize, expected: usize) -> Option { + let last = lines.len().checked_sub(pre.len())?; + if last < cursor { + return None; + } + let anchor = expected.clamp(cursor, last); + let matches_at = |at: usize| { + lines[at..at + pre.len()] + .iter() + .zip(pre) + .all(|(a, b)| a == b) + }; + (0..=last - cursor) + .flat_map(|distance| [anchor.checked_add(distance), anchor.checked_sub(distance)]) + .flatten() + .filter(|&at| at >= cursor && at <= last) + .find(|&at| matches_at(at)) +} + +pub(crate) fn apply_hunks(old: &[u8], hunks: &[Hunk]) -> Option> { + let lines = split_lines(old); + let (out, cursor) = + hunks + .iter() + .try_fold((Vec::::new(), 0usize), |(mut out, cursor), hunk| { + let (pre, post) = images(hunk); + let has_context = hunk.lines.iter().any(|line| line.op == LineOp::Context); + let expected = match hunk.old_lines.get() { + 0 => hunk.old_start.get() as usize, + _ => (hunk.old_start.get() as usize).saturating_sub(1), + }; + let at = match pre.is_empty() { + true => expected.clamp(cursor, lines.len()), + false => find_match(&lines, &pre, cursor, expected)?, + }; + if !has_context && !pre.is_empty() && at + pre.len() != lines.len() { + return None; + } + lines + .get(cursor..at)? + .iter() + .for_each(|line| out.extend_from_slice(line)); + post.iter().for_each(|line| out.extend_from_slice(line)); + Some((out, at + pre.len())) + })?; + Some(lines.get(cursor..)?.iter().fold(out, |mut out, line| { + out.extend_from_slice(line); + out + })) +} + +pub(crate) fn apply_delta(base: &[u8], delta: &[u8]) -> Option> { + let mut pos = 0usize; + let declared_base = read_size(delta, &mut pos)?; + let declared_target = read_size(delta, &mut pos)?; + if declared_base != base.len() as u64 || declared_target > MAX_DIFF_BLOB_BYTES { + return None; + } + let mut out: Vec = Vec::with_capacity(declared_target as usize); + std::iter::from_fn(|| { + let opcode = *delta.get(pos)?; + pos += 1; + Some(match opcode { + 0 => None, + literal if literal & 0x80 == 0 => { + let take = literal as usize; + delta.get(pos..pos + take).map(|bytes| { + pos += take; + out.extend_from_slice(bytes); + }) + } + copy => { + let mut field = |bit: u8| -> u64 { + match copy & bit { + 0 => 0, + _ => { + let byte = delta.get(pos).copied().unwrap_or(0); + pos += 1; + u64::from(byte) + } + } + }; + let offset = field(0x01) | field(0x02) << 8 | field(0x04) << 16 | field(0x08) << 24; + let size = match field(0x10) | field(0x20) << 8 | field(0x40) << 16 { + 0 => 0x10000, + size => size, + }; + base.get(offset as usize..(offset + size) as usize) + .map(|bytes| out.extend_from_slice(bytes)) + } + }) + }) + .try_for_each(|step| step.map(|_| ()))?; + (pos == delta.len() && out.len() as u64 == declared_target).then_some(out) +} + +fn read_size(delta: &[u8], pos: &mut usize) -> Option { + let mut shift = 0u32; + let mut acc = 0u64; + std::iter::from_fn(|| { + let byte = *delta.get(*pos)?; + *pos += 1; + acc |= u64::from(byte & 0x7f) << shift; + shift += 7; + Some(byte & 0x80 != 0) + }) + .take(10) + .find(|more| !more) + .map(|_| acc) +} + +enum OverlayEntry { + Put { content: Vec, kind: EntryKind }, + Gitlink { oid: Oid }, + Removed, +} + +fn parse_subproject(content: &[u8]) -> Option { + let text = std::str::from_utf8(content).ok()?; + Oid::from_hex(text.strip_prefix("Subproject commit ")?.trim()).ok() +} + +struct FoundFile { + content: Vec, + kind: EntryKind, + oid: Option, +} + +struct StepView<'r, 'a> { + repo: &'r Repo, + base: Oid, + accumulated: &'a BTreeMap, + step: BTreeMap, +} + +impl StepView<'_, '_> { + fn overlaid(&self, path: &RepoPath) -> Option<&OverlayEntry> { + self.step.get(path).or_else(|| self.accumulated.get(path)) + } + + fn current(&self, path: &RepoPath) -> Result, ApplyError> { + match self.overlaid(path) { + Some(OverlayEntry::Removed) => Ok(None), + Some(OverlayEntry::Put { content, kind }) => Ok(Some(FoundFile { + content: content.clone(), + kind: *kind, + oid: Some(overlay_oid(self.repo, content)?), + })), + Some(OverlayEntry::Gitlink { oid }) => Ok(Some(FoundFile { + content: subproject_line(*oid), + kind: EntryKind::Commit, + oid: Some(*oid), + })), + None => match self.repo.entry_at(self.base, path)? { + Some(entry) + if matches!( + entry.kind, + EntryKind::Blob | EntryKind::BlobExecutable | EntryKind::Link + ) => + { + if self.repo.blob_size(entry.oid)? > MAX_DIFF_BLOB_BYTES { + return Err(ApplyError::TooLarge); + } + Ok(Some(FoundFile { + content: self.repo.read_blob(entry.oid)?, + kind: entry.kind, + oid: Some(entry.oid), + })) + } + Some(entry) if entry.kind == EntryKind::Commit => Ok(Some(FoundFile { + content: subproject_line(entry.oid), + kind: EntryKind::Commit, + oid: Some(entry.oid), + })), + _ => Ok(None), + }, + } + } + + fn occupied(&self, path: &RepoPath) -> Result { + match self.overlaid(path) { + Some(OverlayEntry::Removed) => Ok(false), + Some(OverlayEntry::Put { .. } | OverlayEntry::Gitlink { .. }) => Ok(true), + None => Ok(self.repo.entry_at(self.base, path)?.is_some()), + } + } + + fn prefix_is_file(&self, prefix: &RepoPath) -> Result { + match self.overlaid(prefix) { + Some(OverlayEntry::Removed) => Ok(false), + Some(OverlayEntry::Put { kind, .. }) => Ok(!matches!(kind, EntryKind::Tree)), + Some(OverlayEntry::Gitlink { .. }) => Ok(true), + None => Ok(matches!( + self.repo.entry_at(self.base, prefix)?, + Some(entry) if !matches!(entry.kind, EntryKind::Tree) + )), + } + } + + fn ancestor_is_file(&self, path: &RepoPath) -> Result { + let parts: Vec<&str> = path.as_str().split('/').collect(); + (1..parts.len()) + .map(|end| parts[..end].join("/")) + .try_fold(false, |blocked, prefix| { + let prefix = + RepoPath::new(prefix).expect("prefix of a valid repo path is well-formed"); + Ok(blocked || self.prefix_is_file(&prefix)?) + }) + } + + fn put(&mut self, path: &RepoPath, content: Vec, kind: EntryKind) { + self.step + .insert(path.clone(), OverlayEntry::Put { content, kind }); + } + + fn put_gitlink(&mut self, path: &RepoPath, oid: Oid) { + self.step + .insert(path.clone(), OverlayEntry::Gitlink { oid }); + } + + fn remove(&mut self, path: &RepoPath) { + self.step.insert(path.clone(), OverlayEntry::Removed); + } +} + +fn index_matches(actual: Option, declared: Option) -> bool { + matches!((actual, declared), (Some(actual), Some(declared)) if actual == declared) +} + +fn overlay_oid(repo: &Repo, content: &[u8]) -> Result { + gix::objs::compute_hash(repo.git().object_hash(), gix::objs::Kind::Blob, content) + .map(Oid::from) + .map_err(|error| ApplyError::Git(backend(error))) +} + +fn transform( + payload: &PatchPayload, + old: &[u8], + old_oid: Option, + declared_old: Option, +) -> Option> { + match payload { + PatchPayload::Text(hunks) => apply_hunks(old, hunks), + PatchPayload::BinaryLiteral(data) => { + index_matches(old_oid, declared_old).then(|| data.clone()) + } + PatchPayload::BinaryDelta(delta) => index_matches(old_oid, declared_old) + .then(|| apply_delta(old, delta)) + .flatten(), + PatchPayload::BinaryOpaque => None, + } +} + +fn fresh_content(payload: &PatchPayload) -> Option> { + match payload { + PatchPayload::Text(hunks) => apply_hunks(&[], hunks), + PatchPayload::BinaryLiteral(data) => Some(data.clone()), + PatchPayload::BinaryDelta(_) | PatchPayload::BinaryOpaque => None, + } +} + +fn file_kind(kind: Option, fallback: EntryKind) -> Option { + match kind.unwrap_or(fallback) { + EntryKind::Tree => None, + usable => Some(usable), + } +} + +fn stage_content( + overlay: &mut StepView<'_, '_>, + path: &RepoPath, + kind: EntryKind, + content: Vec, +) -> Option { + match kind { + EntryKind::Commit => match parse_subproject(&content) { + Some(oid) => { + overlay.put_gitlink(path, oid); + None + } + None => Some(Conflict { + path: path.to_string(), + reason: ConflictReason::DoesNotApply, + }), + }, + _ => { + overlay.put(path, content, kind); + None + } + } +} + +fn apply_file( + overlay: &mut StepView<'_, '_>, + file: &ParsedFile, +) -> Result, ApplyError> { + let conflict = |path: &str, reason: ConflictReason| { + Ok(Some(Conflict { + path: path.to_string(), + reason, + })) + }; + let Some(path) = patch_path(&file.path) else { + return conflict(&file.path, ConflictReason::DoesNotApply); + }; + match &file.intent { + FileIntent::Create => { + let Some(kind) = file_kind(file.new_kind, EntryKind::Blob) else { + return conflict(&file.path, ConflictReason::DoesNotApply); + }; + if overlay.occupied(&path)? { + return conflict(&file.path, ConflictReason::AlreadyExists); + } + if overlay.ancestor_is_file(&path)? { + return conflict(&file.path, ConflictReason::DoesNotApply); + } + match fresh_content(&file.payload) { + Some(content) => Ok(stage_content(overlay, &path, kind, content)), + None => conflict(&file.path, ConflictReason::DoesNotApply), + } + } + FileIntent::Delete => match overlay.current(&path)? { + None => conflict(&file.path, ConflictReason::DoesNotExist), + Some(found) => { + let emptied = match &file.payload { + PatchPayload::BinaryOpaque => { + index_matches(found.oid, file.old_index).then(Vec::new) + } + payload => transform(payload, &found.content, found.oid, file.old_index), + }; + match emptied { + Some(rest) if rest.is_empty() => { + overlay.remove(&path); + Ok(None) + } + _ => conflict(&file.path, ConflictReason::DoesNotApply), + } + } + }, + FileIntent::Modify => match overlay.current(&path)? { + None => conflict(&file.path, ConflictReason::DoesNotExist), + Some(found) => { + let Some(kind) = file_kind(file.new_kind, found.kind) else { + return conflict(&file.path, ConflictReason::DoesNotApply); + }; + match evolved(&file.payload, &found, file.old_index) { + Some(next) => Ok(stage_content(overlay, &path, kind, next)), + None => conflict(&file.path, ConflictReason::DoesNotApply), + } + } + }, + FileIntent::Rename { from } | FileIntent::Copy { from } => { + let Some(source) = patch_path(from) else { + return conflict(from, ConflictReason::DoesNotApply); + }; + if overlay.occupied(&path)? { + return conflict(&file.path, ConflictReason::AlreadyExists); + } + if overlay.ancestor_is_file(&path)? { + return conflict(&file.path, ConflictReason::DoesNotApply); + } + match overlay.current(&source)? { + None => conflict(from, ConflictReason::DoesNotExist), + Some(found) => { + let Some(kind) = file_kind(file.new_kind, found.kind) else { + return conflict(&file.path, ConflictReason::DoesNotApply); + }; + match evolved(&file.payload, &found, file.old_index) { + Some(next) => { + if matches!(&file.intent, FileIntent::Rename { .. }) { + overlay.remove(&source); + } + Ok(stage_content(overlay, &path, kind, next)) + } + None => conflict(&file.path, ConflictReason::DoesNotApply), + } + } + } + } + } +} + +fn evolved( + payload: &PatchPayload, + found: &FoundFile, + declared_old: Option, +) -> Option> { + match payload { + PatchPayload::Text(hunks) if hunks.is_empty() => Some(found.content.clone()), + payload => transform(payload, &found.content, found.oid, declared_old), + } +} + +pub struct PatchApplier<'r> { + repo: &'r Repo, + base: Oid, + accumulated: BTreeMap, +} + +impl<'r> PatchApplier<'r> { + pub fn new(repo: &'r Repo, base_commit: Oid) -> Self { + Self { + repo, + base: base_commit, + accumulated: BTreeMap::new(), + } + } + + pub fn step(&mut self, files: &[ParsedFile]) -> Result { + let mut view = StepView { + repo: self.repo, + base: self.base, + accumulated: &self.accumulated, + step: BTreeMap::new(), + }; + let conflicts: Vec = files + .iter() + .map(|file| apply_file(&mut view, file)) + .collect::, ApplyError>>()? + .into_iter() + .flatten() + .collect(); + if !conflicts.is_empty() { + return Ok(ApplyOutcome::Conflicted(conflicts)); + } + let step = view.step; + let staged: Vec = step + .iter() + .map(|(path, entry)| StagedChange { + path: path.clone(), + action: match entry { + OverlayEntry::Put { content, kind } => StagedAction::Put { + content: content.clone(), + kind: *kind, + }, + OverlayEntry::Gitlink { oid } => StagedAction::PutGitlink { oid: *oid }, + OverlayEntry::Removed => StagedAction::Remove, + }, + }) + .collect(); + self.accumulated.extend(step); + Ok(ApplyOutcome::Clean(staged)) + } +} + +impl Repo { + pub fn write_staged_tree( + &self, + base_tree: Oid, + staged: &[StagedChange], + ) -> Result { + let mut editor = self + .git() + .edit_tree(base_tree.object_id()) + .map_err(backend)?; + staged + .iter() + .try_for_each(|change| -> Result<(), GitError> { + match &change.action { + StagedAction::Put { content, kind } => { + let blob = self.git().write_blob(content).map_err(backend)?.detach(); + editor + .upsert(change.path.as_str(), (*kind).into(), blob) + .map_err(backend)?; + } + StagedAction::PutBlob { oid, kind } => { + editor + .upsert(change.path.as_str(), (*kind).into(), oid.object_id()) + .map_err(backend)?; + } + StagedAction::PutGitlink { oid } => { + editor + .upsert( + change.path.as_str(), + EntryKind::Commit.into(), + oid.object_id(), + ) + .map_err(backend)?; + } + StagedAction::Remove => { + editor.remove(change.path.as_str()).map_err(backend)?; + } + } + Ok(()) + })?; + Ok(Oid::from(editor.write().map_err(backend)?.detach())) + } + + pub fn write_commit(&self, new: &NewCommit) -> Result { + let message = match new.message.ends_with('\n') { + true => new.message.clone(), + false => format!("{}\n", new.message), + }; + let commit = gix::objs::Commit { + tree: new.tree.object_id(), + parents: new + .parents + .iter() + .map(|parent| parent.object_id()) + .collect(), + author: signature(&new.author), + committer: signature(&new.committer), + encoding: None, + message: message.into(), + extra_headers: new + .extra_headers + .iter() + .map(|(name, value)| { + ( + name.as_str().into(), + gix::bstr::BString::from(value.clone()), + ) + }) + .collect(), + }; + Ok(Oid::from( + self.git().write_object(commit).map_err(backend)?.detach(), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::patch::{HunkLine, LineCount, LineNumber}; + + type ApplyCase<'a> = (&'a [u8], Vec, Option<&'a [u8]>); + + fn hunk(old_start: u32, old_lines: u32, new_start: u32, new_lines: u32, spec: &str) -> Hunk { + let lines = spec + .split('\n') + .filter(|line| !line.is_empty()) + .map(|line| { + let (op, text) = match line.as_bytes()[0] { + b'-' => (LineOp::Delete, &line[1..]), + b'+' => (LineOp::Add, &line[1..]), + _ => (LineOp::Context, &line[1..]), + }; + HunkLine { + op, + text: format!("{text}\n").into_bytes(), + } + }) + .collect(); + Hunk { + old_start: LineNumber::new(old_start), + old_lines: LineCount::new(old_lines), + new_start: LineNumber::new(new_start), + new_lines: LineCount::new(new_lines), + lines, + } + } + + #[test] + fn apply_hunks_tracks_position_drift_and_boundaries() { + let cases: Vec = vec![ + ( + b"one\ntwo\nthree\n", + vec![hunk(1, 3, 1, 3, " one\n-two\n+TWO\n three\n")], + Some(b"one\nTWO\nthree\n".as_slice()), + ), + ( + b"zero\nzero\none\ntwo\nthree\n", + vec![hunk(1, 3, 1, 3, " one\n-two\n+TWO\n three\n")], + Some(b"zero\nzero\none\nTWO\nthree\n".as_slice()), + ), + ( + b"one\nTWO ALREADY\nthree\n", + vec![hunk(1, 3, 1, 3, " one\n-two\n+TWO\n three\n")], + None, + ), + ( + b"", + vec![hunk(0, 0, 1, 2, "+alpha\n+beta\n")], + Some(b"alpha\nbeta\n".as_slice()), + ), + ( + b"only\n", + vec![hunk(1, 1, 0, 0, "-only\n")], + Some(b"".as_slice()), + ), + ( + b"a\nb\nc\nd\ne\nf\ng\n", + vec![ + hunk(1, 2, 1, 3, " a\n+inserted\n b\n"), + hunk(6, 2, 7, 2, " f\n-g\n+G\n"), + ], + Some(b"a\ninserted\nb\nc\nd\ne\nf\nG\n".as_slice()), + ), + ( + b"one\ntwo\nthree\nfour\n", + vec![hunk(2, 1, 2, 1, "-two\n+TWO\n")], + None, + ), + ( + b"one\ntwo\nthree\nfour\n", + vec![hunk(2, 1, 1, 0, "-two\n")], + None, + ), + ( + b"one\ntwo\nthree\nfour\n", + vec![hunk(4, 1, 4, 1, "-four\n+FOUR\n")], + Some(b"one\ntwo\nthree\nFOUR\n".as_slice()), + ), + ]; + cases.iter().for_each(|(old, hunks, expected)| { + assert_eq!( + apply_hunks(old, hunks).as_deref(), + *expected, + "apply_hunks mismatch for {old:?}" + ); + }); + } + + #[test] + fn delta_application_round_trips_copy_and_insert() { + let base = b"hello world"; + let delta: Vec = vec![11, 9, 0x90, 5, 4, b'-', b'g', b'i', b'x']; + assert_eq!(apply_delta(base, &delta), Some(b"hello-gix".to_vec())); + assert_eq!(apply_delta(b"wrong size base", &delta), None); + assert_eq!(apply_delta(base, &delta[..5]), None); + } + + #[test] + fn unsafe_paths_are_rejected() { + assert!(patch_path("../escape").is_none()); + assert!(patch_path("/absolute").is_none()); + assert!(patch_path("nested/../escape").is_none()); + assert!(patch_path(".git/hooks/pre-receive").is_none()); + assert!(patch_path("dir/.GIT/config").is_none()); + assert!(patch_path("").is_none()); + assert!(patch_path("src/lib.rs").is_some()); + assert!(patch_path("a b/c.txt").is_some()); + } + + #[test] + fn subproject_text_round_trips_through_a_commit_oid() { + let oid = Oid::from_hex("0123456789abcdef0123456789abcdef01234567").unwrap(); + assert_eq!(parse_subproject(&subproject_line(oid)), Some(oid)); + assert_eq!(parse_subproject(b"not a subproject line\n"), None); + } +} diff --git a/knot2/crates/knot-git/src/patch_parse.rs b/knot2/crates/knot-git/src/patch_parse.rs new file mode 100644 index 0000000000000000000000000000000000000000..c8d8bc0cb413bdcd1ccbd04137e1540781f7b4f6 --- /dev/null +++ b/knot2/crates/knot-git/src/patch_parse.rs @@ -0,0 +1,1189 @@ +use std::io::Read; + +use base64::Engine; +use knot_types::{AuthorName, Email, Oid}; + +use crate::base85; +use crate::objects::{CommitChangeId, EntryKind}; +use crate::patch::{Hunk, HunkLine, LineCount, LineNumber, LineOp, MAX_DIFF_BLOB_BYTES}; + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum PatchParseError { + #[error("patch is empty")] + Empty, + #[error("patch contains no file changes")] + NoFiles, + #[error("malformed patch: {0}")] + Malformed(String), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileIntent { + Create, + Delete, + Modify, + Rename { from: String }, + Copy { from: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PatchPayload { + Text(Vec), + BinaryLiteral(Vec), + BinaryDelta(Vec), + BinaryOpaque, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedFile { + pub path: String, + pub intent: FileIntent, + pub old_kind: Option, + pub new_kind: Option, + pub old_index: Option, + pub payload: PatchPayload, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MailPatch { + pub author_name: AuthorName, + pub author_email: Email, + pub subject: String, + pub body: String, + pub change_id: Option, + pub files: Vec, +} + +impl MailPatch { + pub fn commit_message(&self) -> String { + match self.body.is_empty() { + true => self.subject.clone(), + false => format!("{}\n\n{}", self.subject, self.body), + } + } +} + +pub fn is_format_patch(patch: &str) -> bool { + let lines: Vec<&str> = patch.split('\n').collect(); + if lines.len() < 2 { + return false; + } + let first = lines[0].trim(); + if first.starts_with("From ") && first.contains(" Mon Sep 17 00:00:00 2001") { + return true; + } + lines + .iter() + .take(10) + .map(|line| line.trim()) + .filter(|line| { + line.starts_with("From: ") + || line.starts_with("Date: ") + || line.starts_with("Subject: ") + || line.starts_with("commit ") + }) + .count() + >= 2 +} + +struct Cursor<'a> { + lines: &'a [&'a str], + pos: usize, +} + +impl<'a> Cursor<'a> { + fn new(lines: &'a [&'a str]) -> Self { + Self { lines, pos: 0 } + } + + fn peek(&self) -> Option<&'a str> { + self.lines.get(self.pos).copied() + } + + fn next(&mut self) -> Option<&'a str> { + let line = self.peek()?; + self.pos += 1; + Some(line) + } + + fn take_prefix(&mut self, prefix: &str) -> Option<&'a str> { + let rest = self.peek()?.strip_prefix(prefix)?; + self.pos += 1; + Some(rest) + } +} + +fn malformed(message: impl Into) -> PatchParseError { + PatchParseError::Malformed(message.into()) +} + +const MAX_TOTAL_PATCH_BYTES: u64 = 128 * 1024 * 1024; + +struct Budget { + remaining: u64, +} + +impl Budget { + fn new(limit: u64) -> Self { + Self { remaining: limit } + } + + fn charge(&mut self, bytes: u64) -> Result<(), PatchParseError> { + self.remaining = self + .remaining + .checked_sub(bytes) + .ok_or_else(|| malformed("patch exceeds total decompressed size budget"))?; + Ok(()) + } +} + +fn unescape_c(bytes: &[u8]) -> Option> { + let mut pos = 0usize; + std::iter::from_fn(move || match bytes.get(pos..) { + None | Some([]) => None, + Some(slice) => { + let decoded: Option = match slice { + [b'\\', b'n', ..] => apply(&mut pos, 2, b'\n'), + [b'\\', b't', ..] => apply(&mut pos, 2, b'\t'), + [b'\\', b'a', ..] => apply(&mut pos, 2, 0x07), + [b'\\', b'b', ..] => apply(&mut pos, 2, 0x08), + [b'\\', b'v', ..] => apply(&mut pos, 2, 0x0b), + [b'\\', b'f', ..] => apply(&mut pos, 2, 0x0c), + [b'\\', b'r', ..] => apply(&mut pos, 2, b'\r'), + [b'\\', b'"', ..] => apply(&mut pos, 2, b'"'), + [b'\\', b'\\', ..] => apply(&mut pos, 2, b'\\'), + [b'\\', a @ b'0'..=b'3', b @ b'0'..=b'7', c @ b'0'..=b'7', ..] => { + apply(&mut pos, 4, (a - b'0') * 64 + (b - b'0') * 8 + (c - b'0')) + } + [b'\\', ..] => { + pos += 1; + None + } + [byte, ..] => apply(&mut pos, 1, *byte), + [] => None, + }; + Some(decoded) + } + }) + .collect() +} + +fn apply(pos: &mut usize, width: usize, byte: u8) -> Option { + *pos += width; + Some(byte) +} + +fn quoted_end(bytes: &[u8]) -> Option { + let mut idx = 0usize; + std::iter::from_fn(move || match bytes.get(idx) { + Some(b'"') => Some(Some(idx)), + Some(b'\\') => { + idx += 2; + Some(None) + } + Some(_) => { + idx += 1; + Some(None) + } + None => None, + }) + .flatten() + .next() +} + +fn unquote(raw: &str) -> Result { + match raw.strip_prefix('"') { + None => Ok(raw.to_string()), + Some(inner) => { + let end = + quoted_end(inner.as_bytes()).ok_or_else(|| malformed("unclosed quoted path"))?; + let unescaped = unescape_c(&inner.as_bytes()[..end]) + .ok_or_else(|| malformed("bad escape in quoted path"))?; + String::from_utf8(unescaped).map_err(|_| malformed("quoted path isn't utf-8")) + } + } +} + +#[cfg(test)] +const PRINTABLE_ASCII: std::ops::Range = 0x20..0x7f; + +#[cfg(test)] +fn needs_quoting(byte: u8) -> bool { + !PRINTABLE_ASCII.contains(&byte) || matches!(byte, b'"' | b'\\') +} + +#[cfg(test)] +fn quote_path(path: &str) -> String { + match path.bytes().any(needs_quoting) { + false => path.to_string(), + true => { + let mut quoted = path.bytes().fold(String::from("\""), |mut out, byte| { + match byte { + b'\n' => out.push_str("\\n"), + b'\t' => out.push_str("\\t"), + 0x07 => out.push_str("\\a"), + 0x08 => out.push_str("\\b"), + 0x0b => out.push_str("\\v"), + 0x0c => out.push_str("\\f"), + b'\r' => out.push_str("\\r"), + b'"' => out.push_str("\\\""), + b'\\' => out.push_str("\\\\"), + other if !PRINTABLE_ASCII.contains(&other) => { + out.push_str(&format!("\\{other:03o}")) + } + other => out.push(other as char), + } + out + }); + quoted.push('"'); + quoted + } + } +} + +fn strip_level(path: &str) -> String { + path.split_once('/') + .map(|(_, rest)| rest.to_string()) + .unwrap_or_else(|| path.to_string()) +} + +fn parse_label(raw: &str) -> Result, PatchParseError> { + let bare = match raw.starts_with('"') { + true => unquote(raw)?, + false => raw.split('\t').next().unwrap_or(raw).trim_end().to_string(), + }; + Ok(match bare.as_str() { + "/dev/null" => None, + _ => Some(strip_level(&bare)), + }) +} + +fn diff_paths(rest: &str) -> Option<(String, String)> { + match rest.contains('"') { + true => { + let (old, after) = take_path_token(rest)?; + let (new, _) = take_path_token(after.strip_prefix(' ')?)?; + Some((strip_level(&old), strip_level(&new))) + } + false => unquoted_diff_paths(rest), + } +} + +fn unquoted_diff_paths(rest: &str) -> Option<(String, String)> { + let split_at = |at: usize| { + let old = rest.get(..at)?.strip_prefix("a/")?; + let new = rest.get(at + " b/".len()..)?; + Some((old, new)) + }; + rest.match_indices(" b/") + .filter_map(|(at, _)| split_at(at)) + .find(|(old, new)| old == new) + .or_else(|| split_at(rest.rfind(" b/")?)) + .map(|(old, new)| (old.to_string(), new.to_string())) +} + +fn take_path_token(rest: &str) -> Option<(String, &str)> { + match rest.strip_prefix('"') { + Some(inner) => { + let end = quoted_end(inner.as_bytes())?; + let token = String::from_utf8(unescape_c(&inner.as_bytes()[..end])?).ok()?; + Some((token, inner.get(end + 1..)?)) + } + None => { + let end = rest.find(' ').unwrap_or(rest.len()); + Some((rest[..end].to_string(), &rest[end..])) + } + } +} + +fn full_oid(hex: &str) -> Option { + Oid::from_hex(hex).ok() +} + +fn parse_hunk_header(line: &str) -> Option<(LineNumber, LineCount, LineNumber, LineCount)> { + let rest = line.strip_prefix("@@ -")?; + let (old, rest) = rest.split_once(" +")?; + let (new, _) = rest.split_once(" @@")?; + let span = |raw: &str| -> Option<(LineNumber, LineCount)> { + match raw.split_once(',') { + Some((start, lines)) => Some(( + LineNumber::new(start.parse().ok()?), + LineCount::new(lines.parse().ok()?), + )), + None => Some((LineNumber::new(raw.parse().ok()?), LineCount::new(1))), + } + }; + let (old_start, old_lines) = span(old)?; + let (new_start, new_lines) = span(new)?; + Some((old_start, old_lines, new_start, new_lines)) +} + +fn parse_hunk(cursor: &mut Cursor<'_>, budget: &mut Budget) -> Result { + let header = cursor.next().ok_or_else(|| malformed("truncated hunk"))?; + let (old_start, old_lines, new_start, new_lines) = + parse_hunk_header(header).ok_or_else(|| malformed(format!("bad hunk header {header}")))?; + let mut lines: Vec = Vec::new(); + let mut old_left = old_lines.get() as i64; + let mut new_left = new_lines.get() as i64; + std::iter::from_fn(|| { + (old_left > 0 || new_left > 0).then(|| -> Result<(), PatchParseError> { + let line = cursor + .next() + .ok_or_else(|| malformed("hunk ends before its declared length"))?; + budget.charge(line.len() as u64 + 1)?; + let push = |lines: &mut Vec, op: LineOp| { + let mut text = line.get(1..).unwrap_or("").as_bytes().to_vec(); + text.push(b'\n'); + lines.push(HunkLine { op, text }); + }; + match line.as_bytes().first() { + Some(b' ') | None => { + old_left -= 1; + new_left -= 1; + push(&mut lines, LineOp::Context); + Ok(()) + } + Some(b'-') => { + old_left -= 1; + push(&mut lines, LineOp::Delete); + Ok(()) + } + Some(b'+') => { + new_left -= 1; + push(&mut lines, LineOp::Add); + Ok(()) + } + Some(b'\\') => { + strip_last_newline(&mut lines); + Ok(()) + } + _ => Err(malformed(format!("unexpected hunk line {line}"))), + } + }) + }) + .try_for_each(|outcome| outcome)?; + if cursor.peek().is_some_and(|line| line.starts_with('\\')) { + cursor.next(); + strip_last_newline(&mut lines); + } + Ok(Hunk { + old_start, + old_lines, + new_start, + new_lines, + lines, + }) +} + +fn strip_last_newline(lines: &mut [HunkLine]) { + if let Some(last) = lines.last_mut() + && last.text.last() == Some(&b'\n') + { + last.text.pop(); + } +} + +fn parse_hunks(cursor: &mut Cursor<'_>, budget: &mut Budget) -> Result, PatchParseError> { + std::iter::from_fn(|| { + cursor + .peek() + .is_some_and(|line| line.starts_with("@@ -")) + .then(|| parse_hunk(cursor, budget)) + }) + .collect() +} + +fn parse_binary_block( + cursor: &mut Cursor<'_>, + budget: &mut Budget, +) -> Result<(bool, Vec), PatchParseError> { + let header = cursor + .next() + .ok_or_else(|| malformed("truncated binary patch"))?; + let (kind, size) = header + .split_once(' ') + .ok_or_else(|| malformed(format!("bad binary patch header {header}")))?; + let is_delta = match kind { + "literal" => false, + "delta" => true, + _ => return Err(malformed(format!("unknown binary patch kind {kind}"))), + }; + let size: u64 = size + .trim() + .parse() + .map_err(|_| malformed("bad binary patch size"))?; + if size > MAX_DIFF_BLOB_BYTES { + return Err(malformed("binary patch exceeds size limit")); + } + budget.charge(size)?; + let mut packed: Vec = Vec::new(); + std::iter::from_fn(|| { + cursor + .peek() + .is_some_and(|line| !line.is_empty()) + .then(|| cursor.next().expect("peeked line is present")) + }) + .try_for_each(|line| { + base85::decode_line(line, &mut packed) + .map_err(|_| malformed("invalid base85 line in binary patch")) + })?; + cursor.next(); + let mut inflated: Vec = Vec::new(); + flate2::read::ZlibDecoder::new(packed.as_slice()) + .take(size + 1) + .read_to_end(&mut inflated) + .map_err(|error| malformed(format!("bad zlib stream in binary patch: {error}")))?; + if inflated.len() as u64 != size { + return Err(malformed("binary patch size doesn't match its header")); + } + Ok((is_delta, inflated)) +} + +fn parse_binary_payload( + cursor: &mut Cursor<'_>, + budget: &mut Budget, +) -> Result { + cursor.next(); + let (is_delta, data) = parse_binary_block(cursor, budget)?; + if cursor + .peek() + .is_some_and(|line| line.starts_with("literal ") || line.starts_with("delta ")) + { + parse_binary_block(cursor, budget)?; + } + Ok(match is_delta { + true => PatchPayload::BinaryDelta(data), + false => PatchPayload::BinaryLiteral(data), + }) +} + +#[derive(Default)] +struct FileHeaders { + diff_old: Option, + diff_new: Option, + old_mode: Option, + new_mode: Option, + created: bool, + deleted: bool, + rename_from: Option, + rename_to: Option, + copy_from: Option, + copy_to: Option, + old_index: Option, + label_old: Option>, + label_new: Option>, +} + +fn parse_extended_headers( + cursor: &mut Cursor<'_>, + headers: &mut FileHeaders, +) -> Result<(), PatchParseError> { + std::iter::from_fn(|| { + let line = cursor.peek()?; + let step: Option> = if let Some(rest) = + line.strip_prefix("old mode ") + { + headers.old_mode = EntryKind::from_git_mode(rest); + Some(Ok(())) + } else if let Some(rest) = line.strip_prefix("new mode ") { + headers.new_mode = EntryKind::from_git_mode(rest); + Some(Ok(())) + } else if let Some(rest) = line.strip_prefix("new file mode ") { + headers.created = true; + headers.new_mode = EntryKind::from_git_mode(rest); + Some(Ok(())) + } else if let Some(rest) = line.strip_prefix("deleted file mode ") { + headers.deleted = true; + headers.old_mode = EntryKind::from_git_mode(rest); + Some(Ok(())) + } else if let Some(rest) = line.strip_prefix("rename from ") { + Some(unquote(rest).map(|path| { + headers.rename_from = Some(path); + })) + } else if let Some(rest) = line.strip_prefix("rename to ") { + Some(unquote(rest).map(|path| { + headers.rename_to = Some(path); + })) + } else if let Some(rest) = line.strip_prefix("copy from ") { + Some(unquote(rest).map(|path| { + headers.copy_from = Some(path); + })) + } else if let Some(rest) = line.strip_prefix("copy to ") { + Some(unquote(rest).map(|path| { + headers.copy_to = Some(path); + })) + } else if line.starts_with("similarity index ") || line.starts_with("dissimilarity index ") + { + Some(Ok(())) + } else if let Some(rest) = line.strip_prefix("index ") { + let (oids, mode) = rest + .split_once(' ') + .map(|(oids, mode)| (oids, Some(mode))) + .unwrap_or((rest, None)); + if let Some((old, _)) = oids.split_once("..") { + headers.old_index = full_oid(old); + } + if let Some(kind) = mode.and_then(EntryKind::from_git_mode) { + headers.old_mode = headers.old_mode.or(Some(kind)); + headers.new_mode = headers.new_mode.or(Some(kind)); + } + Some(Ok(())) + } else { + None + }; + step.inspect(|_| { + cursor.next(); + }) + }) + .try_for_each(|outcome| outcome) +} + +fn parse_labels_and_hunks( + cursor: &mut Cursor<'_>, + headers: &mut FileHeaders, + budget: &mut Budget, +) -> Result { + let old_raw = cursor + .take_prefix("--- ") + .ok_or_else(|| malformed("expected --- label"))?; + headers.label_old = Some(parse_label(old_raw)?); + let new_raw = cursor + .take_prefix("+++ ") + .ok_or_else(|| malformed("expected +++ label"))?; + headers.label_new = Some(parse_label(new_raw)?); + Ok(PatchPayload::Text(parse_hunks(cursor, budget)?)) +} + +fn assemble(headers: FileHeaders, payload: PatchPayload) -> Result { + let FileHeaders { + diff_old, + diff_new, + old_mode, + new_mode, + created, + deleted, + rename_from, + rename_to, + copy_from, + copy_to, + old_index, + label_old, + label_new, + } = headers; + let created = created || matches!(label_old, Some(None)); + let deleted = deleted || matches!(label_new, Some(None)); + let need = |path: Option, what: &str| { + path.ok_or_else(|| malformed(format!("file section is missing its {what} path"))) + }; + let (intent, path) = match (rename_from, rename_to, copy_from, copy_to) { + (Some(from), to, _, _) => (FileIntent::Rename { from }, need(to, "rename target")?), + (_, _, Some(from), to) => (FileIntent::Copy { from }, need(to, "copy target")?), + _ if created => ( + FileIntent::Create, + need(label_new.flatten().or(diff_new), "new")?, + ), + _ if deleted => ( + FileIntent::Delete, + need(label_old.flatten().or(diff_old), "old")?, + ), + _ => ( + FileIntent::Modify, + need(label_new.flatten().or(diff_new), "target")?, + ), + }; + Ok(ParsedFile { + path, + intent, + old_kind: old_mode, + new_kind: new_mode, + old_index, + payload, + }) +} + +fn parse_git_file( + cursor: &mut Cursor<'_>, + budget: &mut Budget, +) -> Result { + let rest = cursor + .take_prefix("diff --git ") + .ok_or_else(|| malformed("expected diff --git header"))?; + let mut headers = FileHeaders::default(); + if let Some((old, new)) = diff_paths(rest) { + headers.diff_old = Some(old); + headers.diff_new = Some(new); + } + parse_extended_headers(cursor, &mut headers)?; + let payload = match cursor.peek() { + Some(line) if line.starts_with("--- ") => { + parse_labels_and_hunks(cursor, &mut headers, budget)? + } + Some("GIT binary patch") => parse_binary_payload(cursor, budget)?, + Some(line) if line.starts_with("Binary files ") => { + cursor.next(); + PatchPayload::BinaryOpaque + } + _ => PatchPayload::Text(Vec::new()), + }; + assemble(headers, payload) +} + +fn parse_traditional_file( + cursor: &mut Cursor<'_>, + budget: &mut Budget, +) -> Result { + let mut headers = FileHeaders::default(); + let payload = parse_labels_and_hunks(cursor, &mut headers, budget)?; + assemble(headers, payload) +} + +fn at_file_start(cursor: &Cursor<'_>) -> bool { + match cursor.peek() { + Some(line) if line.starts_with("diff --git ") => true, + Some(line) if line.starts_with("--- ") => cursor + .lines + .get(cursor.pos + 1) + .is_some_and(|next| next.starts_with("+++ ")), + _ => false, + } +} + +fn skip_to_file_start(cursor: &mut Cursor<'_>) -> bool { + std::iter::from_fn(|| { + (!at_file_start(cursor) && cursor.peek().is_some()).then(|| cursor.next()) + }) + .for_each(|_| ()); + cursor.peek().is_some() +} + +pub fn parse_patch(text: &str) -> Result, PatchParseError> { + parse_patch_bounded(text, MAX_TOTAL_PATCH_BYTES) +} + +pub fn parse_patch_bounded(text: &str, max_bytes: u64) -> Result, PatchParseError> { + parse_patch_budgeted(text, &mut Budget::new(max_bytes)) +} + +fn parse_patch_budgeted( + text: &str, + budget: &mut Budget, +) -> Result, PatchParseError> { + if text.trim().is_empty() { + return Err(PatchParseError::Empty); + } + let lines: Vec<&str> = text.split('\n').collect(); + let mut cursor = Cursor::new(&lines); + let files: Vec = std::iter::from_fn(|| { + skip_to_file_start(&mut cursor).then(|| match cursor.peek() { + Some(line) if line.starts_with("diff --git ") => parse_git_file(&mut cursor, budget), + _ => parse_traditional_file(&mut cursor, budget), + }) + }) + .collect::>()?; + match files.is_empty() { + true => Err(PatchParseError::NoFiles), + false => Ok(files), + } +} + +fn is_mail_divider(line: &str) -> bool { + line.strip_prefix("From ").is_some_and(|rest| { + rest.len() > 40 + && rest.as_bytes()[..40] + .iter() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)) + && rest.as_bytes()[40] == b' ' + }) +} + +fn split_mail(text: &str) -> Vec<&str> { + let starts: Vec = text + .split_inclusive('\n') + .scan(0usize, |offset, line| { + let start = *offset; + *offset += line.len(); + Some((start, line)) + }) + .filter(|(_, line)| is_mail_divider(line.trim_end_matches('\n'))) + .map(|(start, _)| start) + .collect(); + match starts.is_empty() { + true => vec![text], + false => { + let ends = starts + .iter() + .skip(1) + .copied() + .chain(std::iter::once(text.len())); + starts + .iter() + .copied() + .zip(ends) + .map(|(start, end)| &text[start..end]) + .collect() + } + } +} + +fn decode_q(bytes: &[u8]) -> Option> { + let hex = |c: u8| (c as char).to_digit(16).map(|d| d as u8); + let mut pos = 0usize; + std::iter::from_fn(move || match bytes.get(pos..) { + None | Some([]) => None, + Some(slice) => { + let decoded: Option = match slice { + [b'_', ..] => apply(&mut pos, 1, b' '), + [b'=', high, low, ..] => { + let byte = hex(*high).zip(hex(*low)).map(|(high, low)| high * 16 + low); + pos += 3; + byte + } + [b'=', ..] => { + pos += 1; + None + } + [byte, ..] => apply(&mut pos, 1, *byte), + [] => None, + }; + Some(decoded) + } + }) + .collect() +} + +fn decode_rfc2047_word(word: &str) -> Option { + let inner = word.strip_prefix("=?")?.strip_suffix("?=")?; + let (charset, rest) = inner.split_once('?')?; + let (encoding, payload) = rest.split_once('?')?; + if !charset.eq_ignore_ascii_case("utf-8") { + return None; + } + let bytes = match encoding { + "Q" | "q" => decode_q(payload.as_bytes())?, + "B" | "b" => base64::engine::general_purpose::STANDARD + .decode(payload) + .ok()?, + _ => return None, + }; + Some(String::from_utf8_lossy(&bytes).into_owned()) +} + +fn decode_rfc2047(value: &str) -> String { + value + .split(' ') + .filter(|token| !token.is_empty()) + .map(|token| match decode_rfc2047_word(token) { + Some(decoded) => (true, decoded), + None => (false, token.to_string()), + }) + .fold( + (String::new(), false), + |(mut acc, prev_encoded), (encoded, text)| { + if !(acc.is_empty() || prev_encoded && encoded) { + acc.push(' '); + } + acc.push_str(&text); + (acc, encoded) + }, + ) + .0 +} + +fn strip_subject_prefix(subject: &str) -> String { + let stripped = std::iter::successors(Some(subject.trim_start()), |current| { + current + .strip_prefix('[') + .and_then(|rest| rest.split_once(']')) + .map(|(_, tail)| tail.trim_start()) + }) + .last() + .unwrap_or(""); + match stripped.starts_with('[') { + true => stripped.to_string(), + false => stripped.trim_end().to_string(), + } +} + +fn parse_address(raw: &str) -> (AuthorName, Email) { + let decoded = decode_rfc2047(raw.trim()); + match decoded.rsplit_once('<') { + Some((name, rest)) => { + let email = rest.split('>').next().unwrap_or(rest).trim(); + let name = name.trim().trim_matches('"').trim(); + (AuthorName::new(name), Email::new(email)) + } + None => { + let bare = decoded.trim(); + (AuthorName::new(bare), Email::new(bare)) + } + } +} + +fn fold_headers(lines: &[&str]) -> Vec<(String, String)> { + lines.iter().fold(Vec::new(), |mut acc, line| { + match line.strip_prefix(' ').or_else(|| line.strip_prefix('\t')) { + Some(continuation) => { + if let Some(last) = acc.last_mut() { + last.1.push(' '); + last.1.push_str(continuation.trim()); + } + } + None => { + if let Some((name, value)) = line.split_once(':') { + acc.push((name.trim().to_string(), value.trim().to_string())); + } + } + } + acc + }) +} + +fn parse_mail(chunk: &str, budget: &mut Budget) -> Result { + let lines: Vec<&str> = chunk.split('\n').collect(); + let after_divider: &[&str] = match lines.split_first() { + Some((first, rest)) if is_mail_divider(first) => rest, + _ => &lines, + }; + let header_end = after_divider + .iter() + .position(|line| line.trim().is_empty()) + .ok_or_else(|| malformed("mail patch has no header separator"))?; + let headers = fold_headers(&after_divider[..header_end]); + let header = |name: &str| { + headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.clone()) + }; + let (author_name, author_email) = header("From") + .map(|raw| parse_address(&raw)) + .ok_or_else(|| malformed("mail patch has no From header"))?; + let rest = &after_divider[header_end + 1..]; + let body_end = rest + .iter() + .position(|line| line.trim_end() == "---" || line.starts_with("diff --git ")) + .unwrap_or(rest.len()); + let body = rest[..body_end].join("\n").trim().to_string(); + let files = parse_patch_budgeted(&rest[body_end..].join("\n"), budget)?; + Ok(MailPatch { + author_name, + author_email, + subject: strip_subject_prefix(&decode_rfc2047(&header("Subject").unwrap_or_default())), + body, + change_id: header("Change-Id").and_then(|raw| CommitChangeId::new(raw).ok()), + files, + }) +} + +pub fn parse_mailbox(text: &str) -> Result, PatchParseError> { + parse_mailbox_bounded(text, MAX_TOTAL_PATCH_BYTES) +} + +pub fn parse_mailbox_bounded( + text: &str, + max_bytes: u64, +) -> Result, PatchParseError> { + if text.trim().is_empty() { + return Err(PatchParseError::Empty); + } + let mut budget = Budget::new(max_bytes); + split_mail(text) + .into_iter() + .map(|chunk| parse_mail(chunk, &mut budget)) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn format_patch_detection_matches_the_mailbox_heuristic() { + assert!(is_format_patch( + "From 0123456789012345678901234567890123456789 Mon Sep 17 00:00:00 2001\nFrom: nel \n" + )); + assert!(is_format_patch( + "From: nel \nSubject: [PATCH] tide pool\n\n" + )); + assert!(!is_format_patch( + "diff --git a/reef.txt b/reef.txt\n--- a/reef.txt\n+++ b/reef.txt\n" + )); + assert!(!is_format_patch("")); + } + + #[test] + fn a_simple_modification_parses() { + let patch = "diff --git a/reef.txt b/reef.txt\nindex 1111111..2222222 100644\n--- a/reef.txt\n+++ b/reef.txt\n@@ -1,2 +1,2 @@\n-old line\n+new line\n context\n"; + let files = parse_patch(patch).unwrap(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].path, "reef.txt"); + assert_eq!(files[0].intent, FileIntent::Modify); + let PatchPayload::Text(hunks) = &files[0].payload else { + panic!("expected text payload"); + }; + assert_eq!(hunks.len(), 1); + assert_eq!(hunks[0].lines.len(), 3); + } + + #[test] + fn creations_deletions_and_renames_parse() { + let patch = concat!( + "diff --git a/new.txt b/new.txt\n", + "new file mode 100644\n", + "index 0000000..2222222\n", + "--- /dev/null\n", + "+++ b/new.txt\n", + "@@ -0,0 +1 @@\n", + "+hello\n", + "diff --git a/gone.txt b/gone.txt\n", + "deleted file mode 100755\n", + "index 2222222..0000000\n", + "--- a/gone.txt\n", + "+++ /dev/null\n", + "@@ -1 +0,0 @@\n", + "-bye\n", + "diff --git a/old.txt b/moved.txt\n", + "similarity index 100%\n", + "rename from old.txt\n", + "rename to moved.txt\n", + ); + let files = parse_patch(patch).unwrap(); + assert_eq!(files.len(), 3); + assert_eq!(files[0].intent, FileIntent::Create); + assert_eq!(files[0].new_kind, Some(EntryKind::Blob)); + assert_eq!(files[1].intent, FileIntent::Delete); + assert_eq!(files[1].old_kind, Some(EntryKind::BlobExecutable)); + assert_eq!( + files[2].intent, + FileIntent::Rename { + from: "old.txt".to_string() + } + ); + assert_eq!(files[2].path, "moved.txt"); + } + + #[test] + fn the_no_newline_marker_strips_the_trailing_newline() { + let patch = "diff --git a/reef.txt b/reef.txt\nindex 1111111..2222222 100644\n--- a/reef.txt\n+++ b/reef.txt\n@@ -1 +1 @@\n-old\n+new\n\\ No newline at end of file\n"; + let files = parse_patch(patch).unwrap(); + let PatchPayload::Text(hunks) = &files[0].payload else { + panic!("expected text payload"); + }; + assert_eq!(hunks[0].lines[0].text, b"old\n".to_vec()); + assert_eq!(hunks[0].lines[1].text, b"new".to_vec()); + } + + #[test] + fn quoted_paths_unescape() { + assert_eq!(unquote("\"a/sp ace.txt\"").unwrap(), "a/sp ace.txt"); + assert_eq!(unquote("\"a/tab\\there\"").unwrap(), "a/tab\there"); + assert_eq!(unquote("\"a/\\303\\251\"").unwrap(), "a/é"); + assert!(unquote("\"a/broken").is_err()); + } + + #[test] + fn a_path_git_quotes_with_letter_escapes_parses() { + let patch = concat!( + "diff --git \"a/c\\a\\b\\v\\f\\r.txt\" \"b/c\\a\\b\\v\\f\\r.txt\"\n", + "new file mode 100644\n", + "index 0000000..587be6b\n", + "--- /dev/null\n", + "+++ \"b/c\\a\\b\\v\\f\\r.txt\"\n", + "@@ -0,0 +1 @@\n", + "+x\n", + ); + let control = "c\u{7}\u{8}\u{b}\u{c}\r.txt"; + assert_eq!(parse_patch(patch).unwrap()[0].path, control); + assert_eq!(quote_path(control), "\"c\\a\\b\\v\\f\\r.txt\""); + } + + #[test] + fn a_plain_path_stays_bare_and_a_quoted_one_unquotes_back() { + [ + "a/reef.txt", + "a/~tilde", + "a/{brace}", + "a/sp ace.txt", + "a/b/ b/c", + ] + .into_iter() + .for_each(|plain| { + assert_eq!(quote_path(plain), plain, "git leaves {plain} bare too"); + }); + [ + "a/quote\".txt", + "a/back\\slash", + "a/tab\there", + "a/new\nline", + "a/é", + "a/\u{7f}del", + ] + .into_iter() + .for_each(|awkward| { + let quoted = quote_path(awkward); + assert!(quoted.starts_with('"'), "{awkward} must come out quoted"); + assert_eq!( + unquote("ed).unwrap(), + awkward, + "{awkward} must unquote back to itself" + ); + }); + } + + #[test] + fn diff_paths_splits_a_header_whose_path_holds_the_separator() { + [ + ("a/b/ b/c.bin b/b/ b/c.bin", "b/ b/c.bin", "b/ b/c.bin"), + ( + "\"a/quote\\\".bin\" \"b/quote\\\".bin\"", + "quote\".bin", + "quote\".bin", + ), + ( + "a/old name.txt b/new name.txt", + "old name.txt", + "new name.txt", + ), + ] + .into_iter() + .for_each(|(header, old, new)| { + assert_eq!( + diff_paths(header), + Some((old.to_string(), new.to_string())), + "the a-side and b-side must agree on the split: {header}" + ); + }); + } + + #[test] + fn a_binary_file_named_around_the_separator_keeps_its_path() { + let patch = concat!( + "diff --git a/b/ b/c.bin b/b/ b/c.bin\n", + "index 1111111111111111111111111111111111111111..2222222222222222222222222222222222222222 100644\n", + "GIT binary patch\n", + "literal 4\n", + "LcmZRms;UA20^k8}\n", + "\n", + "literal 5\n", + "Mcmb\n", + "Date: Tue, 5 Sep 2023 12:00:00 +0530\n", + "Subject: [PATCH 1/2] first\n", + "\n", + "body text\n", + "---\n", + " reef.txt | 1 +\n", + " 1 file changed, 1 insertion(+)\n", + "\n", + "diff --git a/reef.txt b/reef.txt\n", + "new file mode 100644\n", + "index 0000000..2222222\n", + "--- /dev/null\n", + "+++ b/reef.txt\n", + "@@ -0,0 +1 @@\n", + "+one\n", + "-- \n2.43.0\n\n", + "From 2222222222222222222222222222222222222222 Mon Sep 17 00:00:00 2001\n", + "From: =?UTF-8?q?t=C3=A9q?= \n", + "Date: Tue, 5 Sep 2023 13:00:00 +0530\n", + "Subject: [PATCH 2/2] second\n", + "Change-Id: I0123456789abcdef\n", + "\n", + "---\n", + "diff --git a/reef.txt b/reef.txt\n", + "index 2222222..3333333 100644\n", + "--- a/reef.txt\n", + "+++ b/reef.txt\n", + "@@ -1 +1 @@\n", + "-one\n", + "+two\n", + ); + let mails = parse_mailbox(mbox).unwrap(); + assert_eq!(mails.len(), 2); + assert_eq!(mails[0].author_name.as_str(), "nel"); + assert_eq!(mails[0].author_email.as_str(), "nel@oyster.cafe"); + assert_eq!(mails[0].subject, "first"); + assert_eq!(mails[0].body, "body text"); + assert_eq!(mails[0].commit_message(), "first\n\nbody text"); + assert_eq!(mails[0].files.len(), 1); + assert_eq!(mails[1].author_name.as_str(), "téq"); + assert_eq!( + mails[1].change_id, + Some(CommitChangeId::new("I0123456789abcdef").unwrap()) + ); + assert_eq!(mails[1].files[0].intent, FileIntent::Modify); + } + + #[test] + fn malformed_input_yields_typed_errors() { + assert_eq!(parse_patch(" \n "), Err(PatchParseError::Empty)); + assert_eq!(parse_patch("hello world\n"), Err(PatchParseError::NoFiles)); + + let patch = "diff --git a/r.txt b/r.txt\n--- a/r.txt\n+++ b/r.txt\n@@ -0,0 +1 @@\n+a line that is wider than four bytes\n"; + let mut tight = Budget { remaining: 4 }; + assert_eq!( + parse_patch_budgeted(patch, &mut tight), + Err(malformed("patch exceeds total decompressed size budget")), + ); + let mut roomy = Budget { remaining: 1_000 }; + assert!(parse_patch_budgeted(patch, &mut roomy).is_ok()); + } + + #[test] + fn pathological_depth_inputs_do_not_overflow_the_stack() { + let huge = "a".repeat(1_000_000); + assert_eq!(unquote(&format!("\"{huge}\"")).unwrap(), huge); + let brackets = "[x]".repeat(500_000); + assert_eq!(strip_subject_prefix(&brackets), ""); + let encoded = format!("=?utf-8?q?{}?=", "=41".repeat(400_000)); + assert_eq!(decode_rfc2047(&encoded), "A".repeat(400_000)); + } + + #[test] + fn a_bare_email_from_header_becomes_both_name_and_email() { + let mbox = concat!( + "From 1111111111111111111111111111111111111111 Mon Sep 17 00:00:00 2001\n", + "From: nel@oyster.cafe\n", + "Date: Tue, 5 Sep 2023 12:00:00 +0000\n", + "Subject: [PATCH] bare\n", + "\n", + "diff --git a/reef.txt b/reef.txt\n", + "new file mode 100644\n", + "--- /dev/null\n", + "+++ b/reef.txt\n", + "@@ -0,0 +1 @@\n", + "+hi\n", + ); + let mails = parse_mailbox(mbox).unwrap(); + assert_eq!(mails[0].author_name.as_str(), "nel@oyster.cafe"); + assert_eq!(mails[0].author_email.as_str(), "nel@oyster.cafe"); + } +} diff --git a/knot2/crates/knot-git/src/reads.rs b/knot2/crates/knot-git/src/reads.rs new file mode 100644 index 0000000000000000000000000000000000000000..50559153b9f5e9424ebb95b6b0efe2a30fec4100 --- /dev/null +++ b/knot2/crates/knot-git/src/reads.rs @@ -0,0 +1,817 @@ +use std::collections::{HashMap, HashSet}; +use std::ops::ControlFlow; +use std::time::Instant; + +use gix::bstr::ByteSlice; +use knot_types::{BranchName, Oid, OriginUrl, RepoPath, TagName, UnixSeconds}; + +use crate::error::{GitError, backend}; +use crate::objects::{Commit, CommitRange, EntryKind, Identity, identity, map_kind}; +use crate::repo::Repo; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SizedEntry { + pub name: String, + pub oid: Oid, + pub kind: EntryKind, + pub size: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PathEntry { + pub oid: Oid, + pub kind: EntryKind, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LastCommit { + pub id: Oid, + pub subject: String, + pub time: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlameHunk { + pub start: u32, + pub lines: u32, + pub commit: Oid, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BranchTip { + Commit(Box), + Opaque { + id: Oid, + message: String, + created_at: UnixSeconds, + }, +} + +impl BranchTip { + pub fn created_at(&self) -> UnixSeconds { + match self { + BranchTip::Commit(commit) => commit.committer.time, + BranchTip::Opaque { created_at, .. } => *created_at, + } + } + + pub fn id(&self) -> Oid { + match self { + BranchTip::Commit(commit) => commit.id, + BranchTip::Opaque { id, .. } => *id, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BranchInfo { + pub name: BranchName, + pub tip: BranchTip, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AnnotatedTag { + pub tagger: Option, + pub pgp_signature: Option, + pub target: Oid, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TagInfo { + pub name: TagName, + pub id: Oid, + pub created_at: UnixSeconds, + pub message: String, + pub annotated: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Submodule { + pub name: String, + pub path: RepoPath, + pub url: OriginUrl, + pub branch: Option, +} + +knot_types::scalar_newtype! { + pub struct LogSkip(usize); + pub struct LogLimit(usize); +} + +#[derive(Debug, thiserror::Error)] +#[error("blame computation deadline exceeded")] +struct BlameDeadline { + // gix's transparent boxed find error forwards only this source. + source: BlameDeadlineMarker, +} + +#[derive(Debug, thiserror::Error)] +#[error("blame deadline marker")] +struct BlameDeadlineMarker; + +struct DeadlineObjects<'a> { + git: &'a gix::Repository, + deadline: Instant, +} + +impl DeadlineObjects<'_> { + fn check(&self) -> Result<(), gix::objs::find::Error> { + if Instant::now() >= self.deadline { + return Err(Box::new(BlameDeadline { + source: BlameDeadlineMarker, + })); + } + Ok(()) + } +} + +impl gix::objs::FindHeader for DeadlineObjects<'_> { + fn try_header( + &self, + id: &gix::hash::oid, + ) -> Result, gix::objs::find::Error> { + self.check()?; + gix::objs::FindHeader::try_header(self.git, id) + } +} + +impl gix::objs::Find for DeadlineObjects<'_> { + fn try_find<'a>( + &self, + id: &gix::hash::oid, + buffer: &'a mut Vec, + ) -> Result>, gix::objs::find::Error> { + self.check()?; + gix::objs::Find::try_find(self.git, id, buffer) + } +} + +fn caused_by( + error: &(dyn std::error::Error + 'static), + predicate: fn(&T) -> bool, +) -> bool { + error.downcast_ref::().is_some_and(predicate) + || error + .source() + .is_some_and(|source| caused_by(source, predicate)) +} + +fn allocation_limited(error: &(dyn std::error::Error + 'static)) -> bool { + caused_by::(error, |error| { + matches!(error, gix_odb::loose::find::Error::OutOfMemory { .. }) + }) || caused_by::(error, |error| { + matches!(error, gix_odb::pack::data::decode::Error::OutOfMemory) + }) +} + +fn blame_error(error: impl std::error::Error + 'static, max_object_bytes: Option) -> GitError { + if caused_by::(&error, |_| true) { + GitError::BlameTimedOut + } else if let Some(limit) = max_object_bytes.filter(|_| allocation_limited(&error)) { + GitError::BlameObjectTooLarge { limit } + } else { + backend(error) + } +} + +fn blame_file( + git: &gix::Repository, + commit: Oid, + path: &RepoPath, + max_object_bytes: Option, + deadline: Option, +) -> Result, GitError> { + let file_path = path.as_str().as_bytes().as_bstr(); + let outcome = match deadline { + Some(deadline) => { + let cache = git.commit_graph_if_enabled().map_err(backend)?; + let mut resource_cache = git.diff_resource_cache_for_tree_diff().map_err(backend)?; + let options = gix::blame::Options { + diff_algorithm: git.diff_algorithm().map_err(backend)?, + rewrites: Some(gix::diff::Rewrites::default()), + ..Default::default() + }; + gix::blame::file( + DeadlineObjects { git, deadline }, + commit.object_id(), + cache, + &mut resource_cache, + file_path, + options, + ) + .map_err(|error| blame_error(error, max_object_bytes))? + } + None => { + let options = gix::repository::blame_file::Options { + rewrites: Some(gix::diff::Rewrites::default()), + ..Default::default() + }; + git.blame_file(file_path, commit.object_id(), options) + .map_err(|error| blame_error(error, max_object_bytes))? + } + }; + Ok(outcome + .entries + .into_iter() + .map(|entry| BlameHunk { + start: entry.start_in_blamed_file + 1, + lines: entry.len.get(), + commit: Oid::from(entry.commit_id), + }) + .collect()) +} + +impl Repo { + pub fn resolve_revision(&self, spec: &str) -> Option { + if spec.is_empty() || spec.contains('\0') { + return None; + } + self.git() + .rev_parse_single(spec.as_bytes()) + .ok() + .map(|id| Oid::from(id.detach())) + } + + pub fn peel_to_commit(&self, oid: Oid) -> Result { + let peeled = self + .git() + .find_object(oid.object_id()) + .map_err(backend)? + .peel_tags_to_end() + .map_err(backend)?; + match peeled.kind { + gix::object::Kind::Commit => Ok(Oid::from(peeled.id)), + _ => Err(GitError::ObjectType { + oid, + expected: "commit", + }), + } + } + + fn walk_from( + &self, + start: Oid, + hidden: Option, + ) -> Result> + '_, GitError> { + let hidden = hidden.filter(|oid| self.contains(*oid)).map(Oid::object_id); + Ok(self + .git() + .rev_walk(Some(start.object_id())) + .sorting(gix::revision::walk::Sorting::ByCommitTime( + gix::traverse::commit::simple::CommitTimeOrder::NewestFirst, + )) + .with_hidden(hidden) + .all() + .map_err(|error| GitError::RevWalk(error.to_string()))? + .map(|info| { + info.map(|info| Oid::from(info.id)) + .map_err(|error| GitError::RevWalk(error.to_string())) + })) + } + + pub fn commits_between( + &self, + range: CommitRange, + limit: LogLimit, + ) -> Result, GitError> { + self.walk_from(range.head, Some(range.base))? + .take(limit.get()) + .collect() + } + + pub fn log_window( + &self, + start: Oid, + skip: LogSkip, + limit: LogLimit, + ) -> Result<(Vec, usize), GitError> { + self.walk_from(start, None)?.enumerate().try_fold( + (Vec::new(), 0usize), + |(mut window, _), (index, oid)| { + let oid = oid?; + if index >= skip.get() && window.len() < limit.get() { + window.push(self.find_commit(oid)?); + } + Ok((window, index + 1)) + }, + ) + } + + pub fn log_window_bounded( + &self, + start: Oid, + skip: LogSkip, + limit: LogLimit, + ) -> Result, GitError> { + self.walk_from(start, None)? + .enumerate() + .filter(|(index, _)| *index >= skip.get()) + .take(limit.get()) + .map(|(_, oid)| oid.and_then(|oid| self.find_commit(oid))) + .collect() + } + + pub fn log_past_bounded( + &self, + start: Oid, + past: Oid, + limit: LogLimit, + ) -> Result>, GitError> { + let mut found = false; + let mut window = Vec::new(); + for oid in self.walk_from(start, None)? { + let oid = oid?; + if found { + if window.len() < limit.get() { + window.push(self.find_commit(oid)?); + } else { + break; + } + } else if oid == past { + found = true; + } + } + Ok(found.then_some(window)) + } + + pub fn merge_base(&self, one: Oid, two: Oid) -> Result, GitError> { + use gix::repository::merge_base::Error; + match self.git().merge_base(one.object_id(), two.object_id()) { + Ok(id) => Ok(Some(Oid::from(id.detach()))), + Err(Error::NotFound { .. }) => Ok(None), + Err(error) => Err(backend(error)), + } + } + + // For security purposes. + // Without this, anyone who knows an oid can read objects from + // a deleted branch/ unreferenced push. + // + // COBs and forky staging refs aren't counted ofc. + // + // Oh btw to that end `advertised_refs()` *isn't* what upload-pack "advertises": + // upload-pack uses + // `advertised_refs_for(AdvertScope::Upload)` which also omits + // refs matching `transfer.hideRefs`/`uploadpack.hideRefs`. + pub fn reachable_from_public(&self, target: Oid) -> Result { + let tips: Vec = self + .advertised_refs()? + .iter() + // skip any broken refs + .filter_map(|record| self.peel_to_commit(record.target).ok()) + .collect(); + if tips.contains(&target) { + return Ok(true); + } + tips.iter().try_fold(false, |found, tip| { + Ok(found + || self + // Traverse graph, but shouldn't be too hard on CPU + // because `commit_graph_if_enabled` isn't directly + // on the object db. + .merge_base(target, *tip)? + .is_some_and(|base| base == target)) + }) + } + + pub fn branch_list(&self) -> Result, GitError> { + self.branches()? + .into_iter() + .filter_map(|record| record.name.branch_name().map(|name| (name, record.target))) + .map(|(name, target)| self.branch_tip(target).map(|tip| BranchInfo { name, tip })) + .collect() + } + + fn branch_tip(&self, target: Oid) -> Result { + let object = self + .git() + .find_object(target.object_id()) + .map_err(backend)?; + match object.kind { + gix::object::Kind::Commit => self + .find_commit(target) + .map(|commit| BranchTip::Commit(Box::new(commit))), + gix::object::Kind::Tag => { + let tag = object.try_into_tag().map_err(backend)?; + let decoded = tag.decode().map_err(backend)?; + let created_at = decoded + .tagger() + .map_err(|error| GitError::Decode(error.to_string()))? + .map(identity) + .transpose()? + .map(|tagger| tagger.time) + .unwrap_or(UnixSeconds::new(0)); + Ok(BranchTip::Opaque { + id: target, + message: decoded.message.to_string(), + created_at, + }) + } + _ => Ok(BranchTip::Opaque { + id: target, + message: String::new(), + created_at: UnixSeconds::new(0), + }), + } + } + + pub fn tag_list(&self) -> Result, GitError> { + self.tags()? + .into_iter() + .filter_map(|record| record.name.tag_name().map(|name| (name, record.target))) + .map(|(name, target)| self.tag_info(name, target)) + .collect() + } + + fn tag_info(&self, name: TagName, target: Oid) -> Result { + let object = self + .git() + .find_object(target.object_id()) + .map_err(backend)?; + match object.kind { + gix::object::Kind::Tag => { + let tag = object.try_into_tag().map_err(backend)?; + let decoded = tag.decode().map_err(backend)?; + let tagger = decoded + .tagger() + .map_err(|error| GitError::Decode(error.to_string()))? + .map(identity) + .transpose()?; + let created_at = tagger + .as_ref() + .map(|tagger| tagger.time) + .unwrap_or(UnixSeconds::new(0)); + Ok(TagInfo { + name, + id: target, + created_at, + message: decoded.message.to_string(), + annotated: Some(AnnotatedTag { + tagger, + pgp_signature: decoded.pgp_signature.map(|signature| signature.to_string()), + target: Oid::from(decoded.target()), + }), + }) + } + gix::object::Kind::Commit => { + let commit = self.find_commit(target)?; + Ok(TagInfo { + name, + id: target, + created_at: commit.committer.time, + message: commit.message, + annotated: None, + }) + } + _ => Ok(TagInfo { + name, + id: target, + created_at: UnixSeconds::new(0), + message: String::new(), + annotated: None, + }), + } + } + + fn dir_tree_id( + &self, + commit: Oid, + dir: Option<&RepoPath>, + ) -> Result, GitError> { + let root = self.commit_tree(commit)?; + let Some(dir) = dir else { + return Ok(Some(root)); + }; + let tree = self.git().find_tree(root).map_err(backend)?; + match tree.lookup_entry_by_path(dir.as_str()).map_err(backend)? { + Some(entry) if entry.mode().is_tree() => Ok(Some(entry.object_id())), + _ => Ok(None), + } + } + + pub(crate) fn root_tree(&self, commit: Oid) -> Result, GitError> { + let root = self.commit_tree(commit)?; + self.git().find_tree(root).map_err(backend) + } + + pub fn blame_file(&self, commit: Oid, path: &RepoPath) -> Result, GitError> { + blame_file(self.git(), commit, path, None, None) + } + + fn blame_repo(&self, max_object_bytes: u64) -> Result { + gix::open_opts( + self.path(), + crate::repo::isolated_open_options() + .config_overrides([format!("gitoxide.objects.allocLimit={max_object_bytes}")]), + ) + .map_err(|error| GitError::Open { + path: self.path().to_path_buf(), + message: error.to_string(), + }) + } + + pub fn blame_file_bounded( + &self, + commit: Oid, + path: &RepoPath, + max_object_bytes: u64, + deadline: Instant, + ) -> Result, GitError> { + blame_file( + &self.blame_repo(max_object_bytes)?, + commit, + path, + Some(max_object_bytes), + Some(deadline), + ) + } + + pub fn entry_at(&self, commit: Oid, path: &RepoPath) -> Result, GitError> { + let tree = self.root_tree(commit)?; + Ok(tree + .lookup_entry_by_path(path.as_str()) + .map_err(backend)? + .map(|entry| PathEntry { + oid: Oid::from(entry.object_id()), + kind: map_kind(entry.mode().kind()), + })) + } + + pub fn tree_entries_at( + &self, + commit: Oid, + path: Option<&RepoPath>, + ) -> Result>, GitError> { + let Some(path) = path else { + let tree = self.root_tree(commit)?; + return self.sized_entries(&tree).map(Some); + }; + match self.entry_at(commit, path)? { + None => Ok(None), + Some(entry) if entry.kind == EntryKind::Tree => self.tree_entries(entry.oid).map(Some), + Some(entry) if entry.kind == EntryKind::Commit => Ok(None), + Some(_) => Ok(Some(Vec::new())), + } + } + + pub fn tree_entries(&self, tree: Oid) -> Result, GitError> { + let tree = self.git().find_tree(tree.object_id()).map_err(backend)?; + self.sized_entries(&tree) + } + + fn sized_entries(&self, tree: &gix::Tree<'_>) -> Result, GitError> { + let decoded = tree + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + decoded + .entries + .iter() + .map(|entry| { + let oid = Oid::from(entry.oid.to_owned()); + let kind = map_kind(entry.mode.kind()); + let size = match kind { + EntryKind::Blob | EntryKind::BlobExecutable | EntryKind::Link => self + .git() + .try_find_header(oid.object_id()) + .map_err(|error| GitError::Corrupt { + oid, + message: error.to_string(), + })? + .map(|header| header.size()) + .unwrap_or(0), + EntryKind::Tree | EntryKind::Commit => 0, + }; + Ok(SizedEntry { + name: entry.filename.to_string(), + oid, + kind, + size, + }) + }) + .collect() + } + + fn entry_oids_of_tree(&self, tree: gix::ObjectId) -> Result, GitError> { + let tree = self.git().find_tree(tree).map_err(backend)?; + let decoded = tree + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + Ok(decoded + .entries + .iter() + .map(|entry| (entry.filename.to_string(), Oid::from(entry.oid.to_owned()))) + .collect()) + } + + pub fn last_commits( + &self, + start: Oid, + dir: Option<&RepoPath>, + names: &[String], + deadline: Option, + ) -> Result, GitError> { + let mut pending: HashSet<&str> = names.iter().map(String::as_str).collect(); + let mut attributed = HashMap::new(); + let mut dir_trees: HashMap> = HashMap::new(); + let mut dir_tree_of = |commit: Oid| -> Result, GitError> { + match dir_trees.get(&commit) { + Some(known) => Ok(*known), + None => { + let id = self.dir_tree_id(commit, dir)?; + dir_trees.insert(commit, id); + Ok(id) + } + } + }; + + let mut step = |oid: Result| -> Result { + if pending.is_empty() || deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Ok(false); + } + let oid = oid?; + let commit = self.find_commit(oid)?; + if commit.parents.len() > 1 { + return Ok(true); + } + let here_tree = dir_tree_of(oid)?; + let parent_tree = commit + .parents + .first() + .copied() + .map(&mut dir_tree_of) + .transpose()? + .flatten(); + if here_tree == parent_tree || here_tree.is_none() { + return Ok(true); + } + let here = self.entry_oids_of_tree(here_tree.expect("checked above"))?; + let parent = parent_tree + .map(|tree| self.entry_oids_of_tree(tree)) + .transpose()? + .unwrap_or_default(); + let changed: Vec = pending + .iter() + .filter(|name| here.contains_key(**name) && here.get(**name) != parent.get(**name)) + .map(|name| name.to_string()) + .collect(); + changed.iter().for_each(|name| { + pending.remove(name.as_str()); + }); + changed.into_iter().for_each(|name| { + attributed.insert( + name, + LastCommit { + id: oid, + subject: subject_line(&commit.message), + time: commit.author.time, + }, + ); + }); + Ok(true) + }; + + let flow = + self.walk_from(start, None)? + .try_for_each(|oid| -> ControlFlow> { + match step(oid) { + Ok(true) => ControlFlow::Continue(()), + Ok(false) => ControlFlow::Break(None), + Err(error) => ControlFlow::Break(Some(error)), + } + }); + match flow { + ControlFlow::Break(Some(error)) => Err(error), + _ => Ok(attributed), + } + } + + pub fn submodules(&self, commit: Oid) -> Result, GitError> { + let gitmodules = RepoPath::new(".gitmodules").expect("literal path is well-formed"); + let Some(entry) = self.entry_at(commit, &gitmodules)? else { + return Ok(Vec::new()); + }; + if !entry.kind.is_file() { + return Ok(Vec::new()); + } + let raw = self.read_blob(entry.oid)?; + Ok(parse_gitmodules(raw.as_bstr().to_str_lossy().as_ref())) + } +} + +fn subject_line(message: &str) -> String { + message.lines().next().unwrap_or_default().to_string() +} + +fn strip_config_comment(line: &str) -> String { + let flow = line.chars().try_fold( + (String::new(), false, false), + |(mut out, quoted, escaped), ch| match (escaped, quoted, ch) { + (false, false, '#' | ';') => ControlFlow::Break(out), + (false, _, '"') => { + out.push(ch); + ControlFlow::Continue((out, !quoted, false)) + } + (false, _, '\\') => { + out.push(ch); + ControlFlow::Continue((out, quoted, true)) + } + _ => { + out.push(ch); + ControlFlow::Continue((out, quoted, false)) + } + }, + ); + match flow { + ControlFlow::Continue((out, _, _)) | ControlFlow::Break(out) => out, + } +} + +fn unquote_config_value(raw: &str) -> String { + raw.trim() + .chars() + .fold((String::new(), false), |(mut out, escaped), ch| { + match (escaped, ch) { + (true, 'n') => { + out.push('\n'); + (out, false) + } + (true, 't') => { + out.push('\t'); + (out, false) + } + (true, 'b') => { + out.push('\u{0008}'); + (out, false) + } + (true, other) => { + out.push(other); + (out, false) + } + (false, '\\') => (out, true), + (false, '"') => (out, false), + (false, other) => { + out.push(other); + (out, false) + } + } + }) + .0 +} + +fn parse_gitmodules(content: &str) -> Vec { + struct Partial { + name: String, + path: Option, + url: Option, + branch: Option, + } + let finish = |partial: Partial| -> Option { + Some(Submodule { + name: partial.name, + path: RepoPath::new(partial.path?).ok()?, + url: OriginUrl::new(partial.url?), + branch: partial + .branch + .and_then(|branch| BranchName::new(branch).ok()), + }) + }; + let (mut sections, last) = content.lines().map(strip_config_comment).fold( + (Vec::new(), None::), + |(mut done, current), line| { + let line = line.trim(); + if let Some(rest) = line.strip_prefix("[submodule \"") + && let Some(name) = rest.strip_suffix("\"]") + { + done.extend(current.and_then(&finish)); + return ( + done, + Some(Partial { + name: name.to_string(), + path: None, + url: None, + branch: None, + }), + ); + } + if line.starts_with('[') { + done.extend(current.and_then(&finish)); + return (done, None); + } + let current = current.map(|mut partial| { + if let Some((key, value)) = line.split_once('=') { + let value = unquote_config_value(value); + match key.trim() { + "path" => partial.path = Some(value), + "url" => partial.url = Some(value), + "branch" => partial.branch = Some(value), + _ => {} + } + } + partial + }); + (done, current) + }, + ); + sections.extend(last.and_then(&finish)); + sections +} diff --git a/knot2/crates/knot-git/src/repo.rs b/knot2/crates/knot-git/src/repo.rs new file mode 100644 index 0000000000000000000000000000000000000000..09323352bf2262b420c60fafae3b1906757af03f --- /dev/null +++ b/knot2/crates/knot-git/src/repo.rs @@ -0,0 +1,1969 @@ +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + +use gix::refs::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}; +use gix::refs::{FullName, Target}; +use knot_cache::{Cache, Moka, Weight}; +use knot_storage::{ + AdvertScope, HeadRef, PackBudget, PackHash, RefClass, RefEpoch, RefGeneration, RefRecord, + RefUpdate, ReflogUpdate, +}; +use knot_types::{BranchName, KnotId, ObjectFormat, Oid, OriginUrl, RefName, RepoDid, UnixSeconds}; + +use crate::error::GitError; +use crate::objects::{Haves, Walked, Wants}; + +const REFLOG_COMMITTER_NAME: &str = "knot"; +const REFLOG_COMMITTER_EMAIL: &str = "noreply@knot"; +const HEADS_PREFIX: &str = "refs/heads/"; +const TAGS_PREFIX: &str = "refs/tags/"; +const MAX_SYMREF_DEPTH: usize = 5; +const ADVERT_BYTES_PER_REF: u64 = 128; + +fn tuned(mut git: gix::Repository) -> gix::Repository { + git.object_cache_size_if_unset(knot_resource::object_cache_bytes().get() as usize); + pin_reflog_identity(&mut git); + git +} + +fn assembled(git: gix::Repository, path: PathBuf) -> Repo { + Repo { + git: tuned(git), + path, + commit_graph: OnceLock::new(), + } +} + +fn pin_reflog_identity(git: &mut gix::Repository) { + use gix::config::tree::{Committer, Core}; + let mut config = git.config_snapshot_mut(); + let pinned = config.set_value(&Core::LOG_ALL_REF_UPDATES, "true").is_ok() + && config + .set_value(&Committer::NAME, REFLOG_COMMITTER_NAME) + .is_ok() + && config + .set_value(&Committer::EMAIL, REFLOG_COMMITTER_EMAIL) + .is_ok(); + if pinned { + let _ = config.commit(); + } +} + +struct RefState { + lock: Mutex<()>, + generation: AtomicU64, + epoch: RefEpoch, +} + +type RefRegistry = Mutex>>; + +fn ref_registry() -> &'static RefRegistry { + static REGISTRY: OnceLock = OnceLock::new(); + REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn next_epoch() -> RefEpoch { + static EPOCH: AtomicU64 = AtomicU64::new(0); + RefEpoch::new(EPOCH.fetch_add(1, Ordering::Relaxed)) +} + +fn ref_state(git_dir: &Path) -> Arc { + let mut states = ref_registry() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + Arc::clone(states.entry(git_dir.to_path_buf()).or_insert_with(|| { + Arc::new(RefState { + lock: Mutex::new(()), + generation: AtomicU64::new(0), + epoch: next_epoch(), + }) + })) +} + +fn forget_ref_state(git_dir: &Path) { + ref_registry() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(git_dir); +} + +type AdvertCache = Moka<(RefEpoch, RefGeneration), Arc>>; + +fn advert_cache() -> &'static Arc { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| { + let cache = Arc::new(Moka::by_weight( + Weight::new(knot_resource::advert_cache_bytes().get()), + |refs: &Arc>| { + Weight::new( + (refs.len() as u64) + .max(1) + .saturating_mul(ADVERT_BYTES_PER_REF), + ) + }, + )); + knot_cache::register(&cache); + cache + }) +} + +pub fn repo_shard(did: &RepoDid) -> Result { + knot_storage::StorableRepoDid::from_did(did) + .map(|repo| shard_join(&repo)) + .map_err(|_| GitError::UnsafeRepoDid(did.as_str().to_string())) +} + +fn shard_join(repo: &knot_storage::StorableRepoDid) -> PathBuf { + let (method, shard, remainder) = repo.shard_parts(); + PathBuf::from(method).join(shard).join(remainder) +} + +fn knot_shard(knot: &KnotId) -> Result { + let did = RepoDid::new(knot.as_str()) + .map_err(|_| GitError::UnsafeRepoDid(knot.as_str().to_string()))?; + repo_shard(&did) +} + +#[derive(Debug, Clone)] +pub struct Layout { + scan_path: PathBuf, + head: RefName, + reserved: Option, + object_format: ObjectFormat, +} + +fn default_head() -> RefName { + RefName::new(format!("{HEADS_PREFIX}main")).expect("refs/heads/main is valid ref name") +} + +impl Layout { + pub fn new(scan_path: impl Into) -> Self { + Self { + scan_path: scan_path.into(), + head: default_head(), + reserved: None, + object_format: ObjectFormat::default(), + } + } + + pub fn with_default_branch(mut self, branch: BranchName) -> Self { + self.head = branch.head_ref(); + self + } + + pub fn with_object_format(mut self, object_format: ObjectFormat) -> Self { + self.object_format = object_format; + self + } + + pub fn reserving_meta(mut self, knot: &KnotId) -> Result { + let reserved = self.meta_path(knot)?; + self.reserved = Some(reserved); + Ok(self) + } + + pub fn repo_path(&self, did: &RepoDid) -> Result { + Ok(self.scan_path.join(repo_shard(did)?)) + } + + pub fn scratch_dir(&self) -> &Path { + &self.scan_path + } + + pub fn meta_path(&self, knot: &KnotId) -> Result { + Ok(self.scan_path.join(knot_shard(knot)?)) + } + + pub fn guarded_path(&self, did: &RepoDid) -> Result { + let path = self.repo_path(did)?; + match &self.reserved { + Some(reserved) if *reserved == path => { + Err(GitError::ReservedDid(did.as_str().to_string())) + } + _ => Ok(path), + } + } + + pub fn open(&self, did: &RepoDid) -> Result { + Repo::open(self.guarded_path(did)?) + } + + pub fn create(&self, did: &RepoDid) -> Result { + self.create_with_format(did, self.object_format) + } + + pub fn create_with_format( + &self, + did: &RepoDid, + format: ObjectFormat, + ) -> Result { + let repo = Repo::create_with_format(self.guarded_path(did)?, format)?; + repo.set_head(&self.head)?; + Ok(repo) + } + + pub fn remove(&self, did: &RepoDid) -> Result<(), GitError> { + let path = self.guarded_path(did)?; + if let Ok(repo) = Repo::open(&path) { + forget_ref_state(repo.git.git_dir()); + } + match std::fs::remove_dir_all(&path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(GitError::Remove { + path, + message: error.to_string(), + }), + } + } + + pub fn bootstrap_meta(&self, knot: &KnotId) -> Result { + init_bare_idempotent(self.meta_path(knot)?) + } +} + +// `worktree_stream` builds a `gix_filter::Pipeline` +// out of whatever config it has loaded, +// so a filter command defined in any of those sources +// runs against the tree being archived. +// The repository's own config is then the only source gix reads, +// and mr knot wrote that file when it created the repo under the scan path. +// We pin the trust because gix otherwise works it out from who owns the git dir, +// and it reduces the trust when someone else owns that dir, +// at which point it applies a 16MiB object allocation limit +// and treats the repository's own config sections as untrusted. +// Under isolation no safe.directory entry can restore `Full` trust, +// since gix honors that key from only system/global config. +pub(crate) fn isolated_open_options() -> gix::open::Options { + gix::open::Options::isolated().with(gix::sec::Trust::Full) +} + +pub(crate) fn init_bare_with_format( + path: &Path, + format: ObjectFormat, +) -> Result { + let object_hash = (format != ObjectFormat::SHA1).then(|| format.kind()); + gix::ThreadSafeRepository::init_opts( + path, + gix::create::Kind::Bare, + gix::create::Options { + object_hash, + ..Default::default() + }, + isolated_open_options(), + ) + .map(Into::into) + .map_err(|error| error.to_string()) +} + +pub(crate) fn nonce_dir(parent: &Path, prefix: &str) -> PathBuf { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nonce = COUNTER.fetch_add(1, Ordering::Relaxed); + parent.join(format!("{prefix}{}.{nonce}", std::process::id())) +} + +fn staging_path(parent: &Path) -> PathBuf { + nonce_dir(parent, ".knot-staging.") +} + +fn init_bare_idempotent(path: PathBuf) -> Result { + if let Ok(git) = gix::open_opts(&path, isolated_open_options()) { + return Ok(assembled(git, path)); + } + let parent = path.parent().ok_or_else(|| GitError::Create { + path: path.clone(), + message: "meta path has no parent directory".to_string(), + })?; + std::fs::create_dir_all(parent).map_err(|error| GitError::Create { + path: path.clone(), + message: error.to_string(), + })?; + let staging = staging_path(parent); + let _ = std::fs::remove_dir_all(&staging); + init_bare_with_format(&staging, ObjectFormat::SHA1).map_err(|message| GitError::Create { + path: staging.clone(), + message, + })?; + match std::fs::rename(&staging, &path) { + Ok(()) => Repo::open(path), + Err(_) => { + let _ = std::fs::remove_dir_all(&staging); + Repo::open(path) + } + } +} + +pub struct Repo { + git: gix::Repository, + path: PathBuf, + commit_graph: OnceLock>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackfileUrl(String); + +impl PackfileUrl { + pub fn new(value: impl Into) -> Option { + let value = value.into(); + let authority = value + .strip_prefix("https://") + .or_else(|| value.strip_prefix("http://")) + .filter(|rest| !rest.is_empty() && !rest.starts_with('/')); + (authority.is_some() && !value.chars().any(|c| c.is_whitespace() || c.is_control())) + .then_some(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackfileUri { + pub oid: Oid, + pub pack_hash: PackHash, + pub uri: PackfileUrl, +} + +pub fn is_branch(name: &RefName) -> bool { + name.as_str().starts_with(HEADS_PREFIX) +} + +fn advert_config_key(scope: AdvertScope) -> &'static str { + match scope { + AdvertScope::Upload => "uploadpack.hideRefs", + AdvertScope::Receive => "receive.hideRefs", + } +} + +fn ref_hidden_by(name: &RefName, patterns: &[String]) -> bool { + patterns.iter().any(|pattern| { + name.as_str() == pattern || name.as_str().starts_with(&format!("{pattern}/")) + }) +} + +pub(crate) fn fsync_if_present(path: &Path) -> Result<(), GitError> { + knot_resource::fsync_path(path).map_err(|error| GitError::Fsync { + path: error.path, + message: error.source.to_string(), + }) +} + +impl Repo { + pub fn open(path: impl Into) -> Result { + let path = path.into(); + let git = + gix::open_opts(&path, isolated_open_options()).map_err(|error| GitError::Open { + path: path.clone(), + message: error.to_string(), + })?; + Ok(assembled(git, path)) + } + + pub fn create(path: impl Into) -> Result { + Self::create_with_format(path, ObjectFormat::default()) + } + + pub fn create_with_format( + path: impl Into, + format: ObjectFormat, + ) -> Result { + let path = path.into(); + if path.exists() { + return Err(GitError::AlreadyExists(path)); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| GitError::Create { + path: path.clone(), + message: error.to_string(), + })?; + } + let git = init_bare_with_format(&path, format).map_err(|message| GitError::Create { + path: path.clone(), + message, + })?; + Ok(assembled(git, path)) + } + + pub fn git(&self) -> &gix::Repository { + &self.git + } + + pub fn object_format(&self) -> ObjectFormat { + ObjectFormat::from_kind(self.git.object_hash()) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn objects_dir(&self) -> PathBuf { + self.git.git_dir().join("objects") + } + + pub fn set_reference(&self, name: &RefName, target: Oid) -> Result<(), GitError> { + self.git + .reference( + name.as_str(), + target, + PreviousValue::Any, + "update ref to target commit", + ) + .map_err(|error| GitError::Backend(error.to_string()))?; + Ok(()) + } + + pub fn rename_references( + &self, + renames: impl IntoIterator, + ) -> Result<(), GitError> { + renames + .into_iter() + .try_for_each(|(legacy, modern, target)| { + self.set_reference(&modern, target)?; + self.update_ref(&RefUpdate::Delete { + name: legacy, + old: target, + }) + }) + } + + pub fn references(&self) -> Result, GitError> { + self.git + .references() + .map_err(|error| GitError::Backend(error.to_string()))? + .all() + .map_err(|error| GitError::Backend(error.to_string()))? + .filter_map(|reference| { + let reference = match reference { + Ok(reference) => reference, + Err(error) => return Some(Err(GitError::Backend(error.to_string()))), + }; + let raw = reference.name().as_bstr().to_string(); + let target = Oid::from(self.direct_target(&reference, MAX_SYMREF_DEPTH)?); + let name = RefName::new(raw).ok()?; + Some(Ok(RefRecord { name, target })) + }) + .collect() + } + + pub fn reflog_updates_since(&self, since_seconds: UnixSeconds) -> Vec { + let Ok(references) = self.git.references() else { + return Vec::new(); + }; + let Ok(all) = references.all() else { + return Vec::new(); + }; + all.filter_map(Result::ok) + .filter(|reference| { + let name = reference.name().as_bstr().to_string(); + name.starts_with(HEADS_PREFIX) || name.starts_with(TAGS_PREFIX) + }) + .flat_map(|reference| self.ref_reflog_since(&reference, since_seconds)) + .collect() + } + + fn ref_reflog_since( + &self, + reference: &gix::Reference<'_>, + since_seconds: UnixSeconds, + ) -> Vec { + let Ok(name) = RefName::new(reference.name().as_bstr().to_string()) else { + return Vec::new(); + }; + let mut platform = reference.log_iter(); + let Ok(Some(reverse)) = platform.rev() else { + return Vec::new(); + }; + reverse + .filter_map(Result::ok) + .take_while(|line| UnixSeconds::new(line.signature.time.seconds) >= since_seconds) + .filter_map(|line| { + (!line.new_oid.is_null()).then(|| ReflogUpdate { + name: name.clone(), + old: Some(line.previous_oid) + .filter(|previous| !previous.is_null()) + .map(Oid::from), + new: Oid::from(line.new_oid), + seconds: UnixSeconds::new(line.signature.time.seconds), + }) + }) + .collect() + } + + pub fn retained_oids( + &self, + floor: UnixSeconds, + tip_classes: impl Fn(RefClass) -> bool, + ) -> Result, GitError> { + let mut oids: HashSet = self + .references()? + .into_iter() + .filter(|record| tip_classes(RefClass::of(&record.name))) + .map(|record| record.target) + .collect(); + self.reflog_updates_since(floor) + .into_iter() + .for_each(|update| { + oids.insert(update.new); + oids.extend(update.old); + }); + oids.retain(|oid| self.contains(*oid)); + Ok(oids) + } + + pub fn find_ref(&self, name: &RefName) -> Result, GitError> { + match self + .git + .try_find_reference(name.as_str()) + .map_err(|error| GitError::Backend(error.to_string()))? + { + Some(reference) => Ok(self + .direct_target(&reference, MAX_SYMREF_DEPTH) + .map(Oid::from)), + None => Ok(None), + } + } + + fn direct_target(&self, reference: &gix::Reference<'_>, depth: usize) -> Option { + match (depth, reference.follow()) { + (_, None) => reference.try_id().map(|id| id.detach()), + (0, Some(_)) => None, + (_, Some(Ok(next))) => self.direct_target(&next, depth - 1), + (_, Some(Err(_))) => None, + } + } + + pub(crate) fn commit_graph(&self) -> Option<&gix::commitgraph::Graph> { + self.commit_graph + .get_or_init(|| self.git.commit_graph().ok()) + .as_ref() + } + + pub fn commit_graph_misses_a_tip(&self) -> Result { + let Ok(graph) = self.git.commit_graph() else { + return Ok(true); + }; + let in_graph = |oid: Oid| graph.lookup(oid.object_id()).is_some(); + Ok(self.references()?.into_iter().any(|record| { + !in_graph(record.target) + && self + .peel_to_commit(record.target) + .is_ok_and(|tip| !in_graph(tip)) + })) + } + + pub fn with_ref_lock(&self, body: impl FnOnce() -> R) -> R { + let state = ref_state(self.git.git_dir()); + let _guard = state + .lock + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + body() + } + + fn locked_value(&self, body: impl FnOnce() -> R) -> R { + self.with_ref_lock(|| { + let outcome = body(); + let state = ref_state(self.git.git_dir()); + let previous = RefGeneration::new(state.generation.fetch_add(1, Ordering::SeqCst)); + advert_cache().invalidate(&(state.epoch, previous)); + outcome + }) + } + + pub(crate) fn locked( + &self, + body: impl FnOnce() -> Result, + ) -> Result { + self.locked_value(body) + } + + pub fn with_ref_txn(&self, body: impl FnOnce(&RefTxn<'_>) -> R) -> R { + self.locked_value(|| body(&RefTxn { repo: self })) + } + + pub fn set_head(&self, target: &RefName) -> Result<(), GitError> { + self.locked(|| self.set_head_locked(target)) + } + + pub fn has_valid_head(&self) -> Result { + let Some(branch) = self.default_branch() else { + return Ok(false); + }; + Ok(self.find_ref(&branch)?.is_some()) + } + + pub fn ensure_valid_head<'a>( + &self, + updates: &'a [RefUpdate], + ) -> Result, GitError> { + if self.has_valid_head()? { + return Ok(None); + } + + let Some(branch) = updates + .iter() + .map(RefUpdate::name) + .find(|name| name.as_str().starts_with("refs/heads/")) + else { + return Ok(None); + }; + + self.set_head(branch)?; + Ok(Some(branch)) + } + + fn set_head_locked(&self, target: &RefName) -> Result<(), GitError> { + let raw = target.as_str(); + let target_name = FullName::try_from(raw).map_err(|error| GitError::Reference { + name: raw.to_string(), + message: error.to_string(), + })?; + let edit = RefEdit { + change: Change::Update { + log: LogChange { + mode: RefLog::AndReference, + force_create_reflog: false, + message: "knot set HEAD".into(), + }, + expected: PreviousValue::Any, + new: Target::Symbolic(target_name), + }, + name: FullName::try_from("HEAD").map_err(|error| GitError::Reference { + name: "HEAD".to_string(), + message: error.to_string(), + })?, + deref: false, + }; + self.git + .edit_reference(edit) + .map_err(|error| GitError::Reference { + name: "HEAD".to_string(), + message: error.to_string(), + })?; + let git_dir = self.git.git_dir(); + fsync_if_present(&git_dir.join("HEAD"))?; + fsync_if_present(git_dir) + } + + fn persist_refs<'a>( + &self, + mut names: impl Iterator, + ) -> Result<(), GitError> { + let git_dir = self.git.git_dir(); + let mut dirs = BTreeSet::from([git_dir.to_path_buf()]); + names.try_for_each(|name| -> Result<(), GitError> { + let ref_path = git_dir.join(name.as_str()); + fsync_if_present(&ref_path)?; + std::iter::successors(ref_path.parent(), |path| path.parent()) + .take_while(|path| path.starts_with(git_dir)) + .for_each(|path| { + dirs.insert(path.to_path_buf()); + }); + Ok(()) + })?; + fsync_if_present(&git_dir.join("packed-refs"))?; + dirs.iter().try_for_each(|dir| fsync_if_present(dir)) + } + + pub fn origin_url(&self) -> Option { + self.git + .config_snapshot() + .string("remote.origin.url") + .map(|value| OriginUrl::new(value.to_string())) + } + + pub fn set_origin_url(&self, url: &OriginUrl) -> Result<(), GitError> { + let path = self.git.git_dir().join("config"); + let report = |message: String| GitError::Config { + path: path.clone(), + message, + }; + let mut file = + gix::config::File::from_path_no_includes(path.clone(), gix::config::Source::Local) + .map_err(|error| report(error.to_string()))?; + file.set_raw_value_by( + "remote", + Some(gix::bstr::BStr::new("origin")), + "url", + gix::bstr::BStr::new(url.as_str()), + ) + .map_err(|error| report(error.to_string()))?; + knot_resource::atomic_write(&path, knot_resource::FileMode::Inherited, |out| { + file.write_to(out) + .map_err(|error| report(error.to_string())) + })?; + fsync_if_present(self.git.git_dir()) + } + + pub fn branches(&self) -> Result, GitError> { + self.references().map(|records| { + records + .into_iter() + .filter(|record| record.name.as_str().starts_with(HEADS_PREFIX)) + .collect() + }) + } + + pub fn tags(&self) -> Result, GitError> { + self.references().map(|records| { + records + .into_iter() + .filter(|record| record.name.as_str().starts_with(TAGS_PREFIX)) + .collect() + }) + } + + pub fn advertised_refs(&self) -> Result>, GitError> { + let state = ref_state(self.git.git_dir()); + let key = ( + state.epoch, + RefGeneration::new(state.generation.load(Ordering::SeqCst)), + ); + advert_cache() + .get_or_try_insert_with(key, || self.public_refs().map(Arc::new)) + .map_err(|error: Arc| GitError::Backend(error.to_string())) + } + + pub fn public_refs(&self) -> Result, GitError> { + self.references().map(|records| { + records + .into_iter() + .filter(|record| RefClass::of(&record.name).is_public()) + .collect() + }) + } + + pub fn advertised_refs_for(&self, scope: AdvertScope) -> Result, GitError> { + let patterns = self.hidden_ref_patterns(scope); + let base = self.advertised_refs()?; + if patterns.is_empty() { + return Ok(base.to_vec()); + } + Ok(base + .iter() + .filter(|record| !ref_hidden_by(&record.name, &patterns)) + .cloned() + .collect()) + } + + pub fn blob_packfile_uris(&self) -> Vec { + let snapshot = self.git.config_snapshot(); + snapshot + .strings("uploadpack.blobPackfileUri") + .into_iter() + .flatten() + .filter_map(|value| { + let text = value.to_string(); + let mut parts = text.split_whitespace(); + let oid = Oid::from_hex(parts.next()?).ok()?; + let pack_hash = PackHash::new(parts.next()?)?; + let uri = PackfileUrl::new(parts.next()?)?; + Some(PackfileUri { + oid, + pack_hash, + uri, + }) + }) + .collect() + } + + fn hidden_ref_patterns(&self, scope: AdvertScope) -> Vec { + let snapshot = self.git.config_snapshot(); + ["transfer.hideRefs", advert_config_key(scope)] + .into_iter() + .filter_map(|key| snapshot.strings(key)) + .flatten() + .map(|value| value.to_string()) + .collect() + } + + pub fn head(&self) -> Option { + let target = self.git.head_id().ok()?.detach(); + Some(HeadRef { + name: self.default_branch()?, + target: Oid::from(target), + }) + } + + pub fn default_branch(&self) -> Option { + let raw = self.git.head_name().ok()??.as_bstr().to_string(); + RefName::new(raw) + .ok() + .filter(|name| RefClass::of(name).is_public()) + } + + pub fn contains(&self, oid: Oid) -> bool { + self.git.has_object(oid.object_id()) + } + + pub fn is_shallow(&self) -> bool { + self.git.is_shallow() + } + + pub(crate) fn shallow_grafts(&self) -> Result, GitError> { + Ok(self + .git + .shallow_commits() + .map_err(|error| GitError::Decode(format!("shallow file: {error}")))? + .map(|commits| commits.iter().copied().collect()) + .unwrap_or_default()) + } + + pub fn rev_walk(&self, wants: Wants, haves: Haves) -> Result, GitError> { + let mut walked = Walked::new(PackBudget::unbounded()); + self.rev_walk_each(wants, haves, &mut walked) + } + + pub(crate) fn rev_walk_each( + &self, + wants: Wants<'_>, + haves: Haves<'_>, + walked: &mut Walked, + ) -> Result, GitError> { + let present: Vec = haves + .as_slice() + .iter() + .copied() + .filter(|oid| self.contains(*oid)) + .map(Oid::object_id) + .collect(); + let mut probe = *walked; + let collected = self + .git + .rev_walk(wants.as_slice().iter().copied().map(Oid::object_id)) + .with_hidden(present.iter().copied()) + .all() + .ok() + .and_then(|walk| { + walk.map(|info| { + probe.tick()?; + info.map(|info| Oid::from(info.id)) + .map_err(|error| GitError::RevWalk(error.to_string())) + }) + .collect::, _>>() + .ok() + }); + match collected { + Some(commits) => { + *walked = probe; + Ok(commits) + } + None => self.rev_walk_lenient(wants.as_slice(), &present, walked), + } + } + + fn rev_walk_lenient( + &self, + wants: &[Oid], + hidden_tips: &[gix::ObjectId], + walked: &mut Walked, + ) -> Result, GitError> { + let mut hidden: HashSet = HashSet::new(); + let mut stack = hidden_tips.to_vec(); + while let Some(oid) = stack.pop() { + if hidden.insert(oid) + && let Ok((_, parents)) = self.commit_tree_and_parents(Oid::from(oid)) + { + stack.extend(parents); + } + } + let mut visited: HashSet = HashSet::new(); + let mut commits = Vec::new(); + let mut stack: Vec = wants.iter().copied().map(Oid::object_id).collect(); + while let Some(oid) = stack.pop() { + if hidden.contains(&oid) || !visited.insert(oid) { + continue; + } + walked.tick()?; + let (_, parents) = self.commit_tree_and_parents(Oid::from(oid))?; + commits.push(Oid::from(oid)); + stack.extend(parents); + } + Ok(commits) + } + + fn ref_edit(update: &RefUpdate, via_head: bool) -> Result { + let edited = if via_head { + "HEAD" + } else { + update.name().as_str() + }; + let name = FullName::try_from(edited).map_err(|error| GitError::Reference { + name: edited.to_string(), + message: error.to_string(), + })?; + let log = LogChange { + mode: RefLog::AndReference, + force_create_reflog: true, + message: "knot ref update".into(), + }; + let change = match update { + RefUpdate::Create { new, .. } => Change::Update { + log, + expected: PreviousValue::MustNotExist, + new: Target::Object(new.object_id()), + }, + RefUpdate::Update { old, new, .. } => Change::Update { + log, + expected: PreviousValue::MustExistAndMatch(Target::Object(old.object_id())), + new: Target::Object(new.object_id()), + }, + RefUpdate::Delete { old, .. } => Change::Delete { + expected: PreviousValue::MustExistAndMatch(Target::Object(old.object_id())), + log: RefLog::AndReference, + }, + }; + Ok(RefEdit { + change, + name, + deref: via_head, + }) + } + + fn updates_head_branch(&self, update: &RefUpdate) -> bool { + !matches!(update, RefUpdate::Delete { .. }) + && self + .default_branch() + .is_some_and(|head| head.as_str() == update.name().as_str()) + } + + pub fn update_ref(&self, update: &RefUpdate) -> Result<(), GitError> { + self.locked(|| self.update_ref_locked(update)) + } + + fn update_ref_locked(&self, update: &RefUpdate) -> Result<(), GitError> { + self.reject_df_conflicts(std::slice::from_ref(update))?; + let raw = update.name().as_str().to_string(); + let via_head = self.updates_head_branch(update); + self.git + .edit_reference(Self::ref_edit(update, via_head)?) + .map_err(|error| GitError::Reference { + name: raw, + message: error.to_string(), + })?; + self.persist_refs(std::iter::once(update.name())) + } + + pub fn update_refs(&self, updates: &[RefUpdate]) -> Result<(), GitError> { + self.locked(|| self.update_refs_locked(updates)) + } + + fn name_taken(&self, raw: &str, creates: &HashSet<&str>) -> Result { + match creates.contains(raw) { + true => Ok(true), + false => match RefName::new(raw.to_string()) { + Ok(name) => self.find_ref(&name).map(|found| found.is_some()), + Err(_) => Ok(false), + }, + } + } + + fn blocking_ancestor( + &self, + name: &str, + creates: &HashSet<&str>, + deletes: &HashSet<&str>, + ) -> Result, GitError> { + name.match_indices('/') + .map(|(at, _)| &name[..at]) + .filter(|ancestor| !deletes.contains(ancestor)) + .map(|ancestor| Ok((ancestor, self.name_taken(ancestor, creates)?))) + .collect::, GitError>>() + .map(|probed| { + probed + .into_iter() + .find_map(|(ancestor, taken)| taken.then(|| ancestor.to_string())) + }) + } + + fn leaf_under( + &self, + directory: &str, + deletes: &HashSet<&str>, + ) -> Result, GitError> { + let backend = |error: String| GitError::Backend(error); + match self + .git + .references() + .map_err(|error| backend(error.to_string()))? + .prefixed(directory) + { + Ok(walk) => Ok(walk + .filter_map(Result::ok) + .map(|reference| reference.name().as_bstr().to_string()) + .find(|leaf| !deletes.contains(leaf.as_str()))), + Err(gix::reference::iter::init::Error::RelativePath(_)) => Ok(self + .references()? + .into_iter() + .map(|record| record.name) + .find(|name| { + name.as_str().starts_with(directory) && !deletes.contains(name.as_str()) + }) + .map(|name| name.as_str().to_string())), + Err(error) => Err(backend(error.to_string())), + } + } + + fn blocked_leaf( + &self, + name: &str, + creates: &HashSet<&str>, + deletes: &HashSet<&str>, + ) -> Result, GitError> { + let directory = format!("{name}/"); + match creates + .iter() + .find(|other| other.starts_with(&directory)) + .copied() + { + Some(leaf) => Ok(Some(leaf.to_string())), + None => self.leaf_under(&directory, deletes), + } + } + + fn reject_df_conflicts(&self, updates: &[RefUpdate]) -> Result<(), GitError> { + let creates: HashSet<&str> = updates + .iter() + .filter_map(|update| match update { + RefUpdate::Create { name, .. } => Some(name.as_str()), + _ => None, + }) + .collect(); + let deletes: HashSet<&str> = updates + .iter() + .filter_map(|update| match update { + RefUpdate::Delete { name, .. } => Some(name.as_str()), + _ => None, + }) + .collect(); + creates.iter().try_for_each(|name| { + match self.blocking_ancestor(name, &creates, &deletes)? { + Some(directory) => Err(GitError::AtomicRefs(format!( + "d/f conflict: {directory} blocks {name}" + ))), + None => match self.blocked_leaf(name, &creates, &deletes)? { + Some(leaf) => Err(GitError::AtomicRefs(format!( + "d/f conflict: {name} blocks {leaf}" + ))), + None => Ok(()), + }, + } + }) + } + + fn update_refs_locked(&self, updates: &[RefUpdate]) -> Result<(), GitError> { + self.reject_df_conflicts(updates)?; + let edits = updates + .iter() + .map(|update| { + let via_head = self.updates_head_branch(update); + Self::ref_edit(update, via_head) + }) + .collect::, _>>()?; + self.git + .edit_references(edits) + .map_err(|error| GitError::AtomicRefs(error.to_string()))?; + self.persist_refs(updates.iter().map(RefUpdate::name)) + } +} + +pub struct RefTxn<'a> { + repo: &'a Repo, +} + +impl RefTxn<'_> { + pub fn update_ref(&self, update: &RefUpdate) -> Result<(), GitError> { + self.repo.update_ref_locked(update) + } + + pub fn update_refs(&self, updates: &[RefUpdate]) -> Result<(), GitError> { + self.repo.update_refs_locked(updates) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const A: &str = "1111111111111111111111111111111111111111"; + const B: &str = "2222222222222222222222222222222222222222"; + + fn oid(hex: &str) -> Oid { + Oid::from_hex(hex).unwrap() + } + + fn head_ref() -> RefName { + RefName::new("refs/heads/main").unwrap() + } + + fn repo() -> (tempfile::TempDir, Layout, RepoDid) { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + (dir, layout, did) + } + + #[test] + fn every_repo_opens_against_its_own_config_and_nothing_ambient() { + let permissions = isolated_open_options().permissions; + let config = permissions.config; + assert!( + !config.system + && !config.git + && !config.user + && !config.env + && !config.includes + && !config.git_binary, + "a filter driver in ambient config would execute when worktree_stream archives a \ + pushed tree, so we read only the repository's own config: {config:?}" + ); + assert!( + !permissions.attributes.system + && !permissions.attributes.git + && !permissions.attributes.git_binary, + "only the archived tree's own .gitattributes may set filter= on a path: {:?}", + permissions.attributes + ); + let denied = |permission| matches!(permission, gix::sec::Permission::Deny); + let env = permissions.env; + assert!( + denied(env.xdg_config_home) + && denied(env.home) + && denied(env.git_prefix) + && denied(env.ssh_prefix) + && denied(env.identity) + && denied(env.objects) + && denied(env.http_transport), + "gix resolves GIT_CONFIG_KEY_n, HOME and XDG_CONFIG_HOME into config that can define a \ + filter driver: {env:?}" + ); + assert!( + permissions.is_isolated(), + "every permission must match the set gix itself calls isolated, including any field a \ + gix upgrade adds that the three checks above don't name: {permissions:?}" + ); + } + + #[cfg(unix)] + #[test] + fn the_knot_keeps_full_trust_on_a_git_dir_it_no_longer_owns() { + const NOBODY: u32 = 65534; + let (_dir, layout, did) = repo(); + layout.create(&did).unwrap(); + let path = layout.repo_path(&did).unwrap(); + let unreachable_uid = |kind| { + matches!( + kind, + std::io::ErrorKind::PermissionDenied | std::io::ErrorKind::InvalidInput + ) + }; + match std::os::unix::fs::chown(&path, Some(NOBODY), None) { + Err(error) if unreachable_uid(error.kind()) => { + eprintln!( + "skipping the foreign-owner trust check: chown to {NOBODY} needs root and a \ + uid mapping reaching that far" + ); + return; + } + outcome => outcome.unwrap(), + } + assert_eq!( + layout.open(&did).unwrap().git().git_dir_trust(), + gix::sec::Trust::Full, + "reduced trust applies a 16MiB limit to every object allocation" + ); + assert_eq!( + gix::open_opts(&path, gix::open::Options::isolated()) + .unwrap() + .git_dir_trust(), + gix::sec::Trust::Reduced, + "gix raised the trust of a git dir owned by another user with no safe.directory entry \ + in reach" + ); + } + + #[test] + fn layout_paths_shard_and_stay_within_scan() { + let layout = Layout::new("/srv/git"); + let cases: &[(&str, &str)] = &[ + ("did:plc:squid", "plc/sq/uid"), + ("did:web:oyster.cafe", "web/oy/ster.cafe"), + ("did:web:nel.pet", "web/ne/l.pet"), + ]; + cases.iter().for_each(|&(raw, suffix)| { + let path = layout.repo_path(&RepoDid::new(raw).unwrap()).unwrap(); + assert!(path.ends_with(suffix), "{path:?} missing shard {suffix}"); + assert!(path.starts_with("/srv/git"), "{path:?} escaped scan path"); + }); + } + + #[test] + fn dot_only_did_cannot_escape_scan_path() { + let layout = Layout::new("/srv/git/scan"); + ["did:plc:....", "did:web:....", "did:plc:...", "did:plc:.."] + .into_iter() + .map(|raw| RepoDid::new(raw).unwrap()) + .for_each(|did| { + assert!( + matches!(layout.repo_path(&did), Err(GitError::UnsafeRepoDid(_))), + "dot-only method-specific-id must be refused, never resolved to path" + ); + assert!(matches!(layout.open(&did), Err(GitError::UnsafeRepoDid(_)))); + assert!(matches!( + layout.create(&did), + Err(GitError::UnsafeRepoDid(_)) + )); + }); + + let real = RepoDid::new("did:web:oyster.cafe").unwrap(); + assert!( + layout.repo_path(&real).is_ok(), + "legitimate did:web with dots in its domain must still resolve" + ); + } + + #[test] + fn meta_repo_path_is_sharded_and_never_collides() { + let layout = Layout::new("/srv/git"); + let knot = KnotId::new("did:web:oyster.cafe").unwrap(); + let meta = layout.meta_path(&knot).unwrap(); + assert!(meta.ends_with("web/oy/ster.cafe")); + ["did:plc:squid", "did:web:nel.pet"] + .into_iter() + .map(|raw| RepoDid::new(raw).unwrap()) + .for_each(|did| { + assert_ne!(layout.repo_path(&did).unwrap(), meta); + }); + } + + #[test] + fn bootstrap_meta_creates_then_opens_idempotently() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let knot = KnotId::new("did:web:oyster.cafe").unwrap(); + + let created = layout.bootstrap_meta(&knot).unwrap(); + assert!(created.references().unwrap().is_empty()); + assert_eq!(created.path(), layout.meta_path(&knot).unwrap()); + + let reopened = layout.bootstrap_meta(&knot).unwrap(); + assert_eq!(reopened.path(), layout.meta_path(&knot).unwrap()); + } + + #[test] + fn concurrent_bootstrap_meta_converges_for_every_caller() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let knot = KnotId::new("did:web:oyster.cafe").unwrap(); + let meta = layout.meta_path(&knot).unwrap(); + + let paths = std::thread::scope(|scope| { + let handles: Vec<_> = (0..8) + .map(|_| { + let layout = layout.clone(); + let knot = knot.clone(); + scope.spawn(move || { + layout + .bootstrap_meta(&knot) + .map(|repo| repo.path().to_path_buf()) + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect::>() + }); + + assert!( + paths + .iter() + .all(|outcome| matches!(outcome, Ok(path) if path == &meta)), + "every racing bootstrap must converge on one meta repo, not fail: {paths:?}" + ); + let reopened = layout.bootstrap_meta(&knot).unwrap(); + assert!(reopened.references().unwrap().is_empty()); + } + + #[test] + fn reserving_meta_refuses_the_knot_did_for_open_and_create() { + let dir = tempfile::tempdir().unwrap(); + let knot = KnotId::new("did:web:oyster.cafe").unwrap(); + let layout = Layout::new(dir.path()).reserving_meta(&knot).unwrap(); + layout.bootstrap_meta(&knot).unwrap(); + + let knot_as_repo = RepoDid::new("did:web:oyster.cafe").unwrap(); + assert!(matches!( + layout.open(&knot_as_repo), + Err(GitError::ReservedDid(_)) + )); + assert!(matches!( + layout.create(&knot_as_repo), + Err(GitError::ReservedDid(_)) + )); + + let ordinary = RepoDid::new("did:plc:squid").unwrap(); + assert!(layout.create(&ordinary).is_ok()); + assert!(layout.open(&ordinary).is_ok()); + } + + #[test] + fn creating_a_bare_repo_is_sha1_and_rejects_a_second_create() { + let (_dir, layout, did) = repo(); + + let repo = layout.create(&did).unwrap(); + assert!(repo.references().unwrap().is_empty()); + assert!(repo.head().is_none()); + assert_eq!(repo.object_format(), ObjectFormat::SHA1); + + assert!(layout.open(&did).unwrap().references().unwrap().is_empty()); + assert!(matches!( + layout.create(&did), + Err(GitError::AlreadyExists(_)) + )); + } + + #[test] + fn with_ref_txn_holds_the_ref_lock_across_its_whole_body() { + use std::sync::Mutex; + use std::sync::mpsc::channel; + + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + let contender_repo = layout.open(&did).unwrap(); + + let order: Mutex> = Mutex::new(Vec::new()); + let order_ref = ℴ + let (contending, observed) = channel(); + + std::thread::scope(|scope| { + repo.with_ref_txn(|_txn| { + order_ref.lock().unwrap().push("txn-enter"); + scope.spawn(move || { + contending.send(()).unwrap(); + contender_repo.with_ref_lock(|| order_ref.lock().unwrap().push("contender")); + }); + observed.recv().unwrap(); + std::thread::yield_now(); + order_ref.lock().unwrap().push("txn-exit"); + }); + }); + + assert_eq!( + *order.lock().unwrap(), + ["txn-enter", "txn-exit", "contender"], + "object migration runs inside the transaction body, so no other ref-lock holder can observe the half-applied push" + ); + } + + #[test] + fn create_with_sha256_object_format_round_trips() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()).with_object_format(ObjectFormat::SHA256); + let did = RepoDid::new("did:plc:squid").unwrap(); + + let repo = layout.create(&did).unwrap(); + assert_eq!(repo.object_format(), ObjectFormat::SHA256); + let oid = Oid::from(repo.git().write_blob(b"hello sha256\n").unwrap().detach()); + assert_eq!( + oid.to_hex().len(), + 64, + "sha256 repo names objects with 32-byte digests" + ); + + let reopened = layout.open(&did).unwrap(); + assert_eq!( + reopened.object_format(), + ObjectFormat::SHA256, + "object format survives reopen, read from repo config" + ); + } + + #[test] + fn compare_and_swap_governs_every_ref_write() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + let main = head_ref(); + + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: oid(A), + }) + .unwrap(); + assert!( + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: oid(B), + }) + .is_err() + ); + assert!( + repo.update_ref(&RefUpdate::Update { + name: main.clone(), + old: oid(B), + new: oid(A), + }) + .is_err() + ); + repo.update_ref(&RefUpdate::Update { + name: main.clone(), + old: oid(A), + new: oid(B), + }) + .unwrap(); + let refs = repo.references().unwrap(); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].target, oid(B)); + + drop(repo); + let repo = layout.open(&did).unwrap(); + assert_eq!( + repo.find_ref(&main).unwrap(), + Some(oid(B)), + "update is visible after reopen" + ); + + repo.update_ref(&RefUpdate::Delete { + name: main.clone(), + old: oid(B), + }) + .unwrap(); + assert!(repo.references().unwrap().is_empty()); + + let x = RefName::new("refs/heads/x").unwrap(); + let y = RefName::new("refs/heads/y").unwrap(); + repo.update_refs(&[ + RefUpdate::Create { + name: x.clone(), + new: oid(A), + }, + RefUpdate::Create { + name: y.clone(), + new: oid(A), + }, + ]) + .unwrap(); + let result = repo.update_refs(&[ + RefUpdate::Update { + name: x.clone(), + old: oid(A), + new: oid(B), + }, + RefUpdate::Update { + name: y.clone(), + old: oid(B), + new: oid(A), + }, + ]); + assert!( + result.is_err(), + "batch with one stale compare-and-swap must fail as a whole" + ); + assert_eq!( + repo.find_ref(&x).unwrap(), + Some(oid(A)), + "valid edit in failed batch must roll back" + ); + assert_eq!(repo.find_ref(&y).unwrap(), Some(oid(A))); + } + + #[test] + fn ref_writes_leave_a_recoverable_reflog_under_the_knot_identity() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + + let committer = repo + .git() + .committer() + .expect("committer is always pinned so reflog writes never depend on ambient config") + .expect("pinned committer signature parses"); + assert_eq!(committer.name.to_string(), REFLOG_COMMITTER_NAME); + assert_eq!(committer.email.to_string(), REFLOG_COMMITTER_EMAIL); + + repo.update_ref(&RefUpdate::Create { + name: head_ref(), + new: oid(A), + }) + .unwrap(); + repo.update_ref(&RefUpdate::Update { + name: head_ref(), + old: oid(A), + new: oid(B), + }) + .unwrap(); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/tags/v1").unwrap(), + new: oid(A), + }) + .unwrap(); + + let logs = repo.git().git_dir().join("logs"); + let branch = std::fs::read_to_string(logs.join("refs/heads/main")) + .expect("branch update must leave reflog so clobbering push is recoverable"); + assert!( + branch.contains(A) && branch.contains(B) && branch.contains(REFLOG_COMMITTER_NAME), + "branch reflog records both tips under knot identity:\n{branch}" + ); + assert!( + logs.join("refs/tags/v1").exists(), + "force_create_reflog must log tags too, not just conventional refs/heads set" + ); + + let updates = repo.reflog_updates_since(UnixSeconds::new(0)); + let head_new: Vec = updates + .iter() + .filter(|update| update.name == head_ref()) + .map(|update| update.new) + .collect(); + assert!( + head_new.contains(&oid(A)) && head_new.contains(&oid(B)), + "both branch tips are recovered from reflog: {head_new:?}" + ); + let create = updates + .iter() + .find(|update| update.name == head_ref() && update.new == oid(A)) + .unwrap(); + assert_eq!(create.old, None, "branch creation has no previous oid"); + let update = updates + .iter() + .find(|update| update.name == head_ref() && update.new == oid(B)) + .unwrap(); + assert_eq!(update.old, Some(oid(A)), "branch update records prior oid"); + assert!( + updates + .iter() + .any(|update| update.name.as_str() == "refs/tags/v1" && update.new == oid(A)), + "tag update is recovered too" + ); + assert!( + repo.reflog_updates_since(UnixSeconds::new(i64::MAX)) + .is_empty(), + "horizon past every entry filters whole reflog out" + ); + } + + #[test] + fn advertisement_hides_reserved_refs_and_tracks_each_change() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + let main = head_ref(); + let feature = RefName::new("refs/heads/feature").unwrap(); + + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: oid(A), + }) + .unwrap(); + let first = repo.advertised_refs().unwrap(); + assert_eq!(first.len(), 1); + assert_eq!(first[0].name.as_str(), "refs/heads/main"); + assert_eq!( + repo.advertised_refs().unwrap(), + first, + "repeated advertisement with no ref change serves same answer" + ); + + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/cobs/org.tangled.graph.collaborator/limpet").unwrap(), + new: oid(B), + }) + .unwrap(); + let advertised = repo.advertised_refs().unwrap(); + assert_eq!(advertised.len(), 1, "cob ref is hidden from advertisement"); + assert_eq!(advertised[0].name.as_str(), "refs/heads/main"); + assert_eq!(repo.references().unwrap().len(), 2); + assert_eq!( + RefClass::of(&RefName::new("refs/cobs/x/y").unwrap()), + RefClass::Cob + ); + + repo.update_ref(&RefUpdate::Create { + name: feature.clone(), + new: oid(B), + }) + .unwrap(); + assert_eq!( + repo.advertised_refs().unwrap().len(), + 2, + "ref created after advertisement invalidates cached answer" + ); + repo.update_ref(&RefUpdate::Delete { + name: feature, + old: oid(B), + }) + .unwrap(); + assert_eq!( + repo.advertised_refs().unwrap().len(), + 1, + "delete after advertisement invalidates cached answer" + ); + + drop(repo); + layout.remove(&did).unwrap(); + let repo = layout.create(&did).unwrap(); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/new").unwrap(), + new: oid(B), + }) + .unwrap(); + let advertised = repo.advertised_refs().unwrap(); + assert_eq!( + advertised.len(), + 1, + "recreated repo advertises only its own ref" + ); + assert_eq!( + advertised[0].name.as_str(), + "refs/heads/new", + "deleted repo's cached advertisement mustn't survive recreation at same path" + ); + } + + #[test] + fn each_reserved_namespace_covers_its_children_however_the_ref_is_spelled() { + let cases = [ + ("refs/heads/main", RefClass::Public), + ("refs/atproto", RefClass::Atproto), + ("refs/atproto/commit", RefClass::Atproto), + ("refs/atproto-mirror/commit", RefClass::Public), + ("refs/cobs", RefClass::Cob), + ("refs/cobs/org.tangled.knot.member/beef", RefClass::Cob), + ("refs/cob-checkpoints", RefClass::Checkpoint), + ("refs/cob-checkpoints/main", RefClass::Checkpoint), + ("refs/hidden", RefClass::Hidden), + ("refs/hidden/feature/main", RefClass::Hidden), + ("refs/tngl", RefClass::Tngl), + ( + "refs/tngl/keep/did%3aplc%3asquid/org.tangled.review.patch/x/y", + RefClass::Tngl, + ), + ]; + cases.iter().for_each(|(raw, class)| { + assert_eq!( + RefClass::of(&RefName::new(*raw).unwrap()), + *class, + "{raw}: a namespace must cover its own name and its children" + ); + assert_eq!( + RefClass::of_revspec(raw), + *class, + "{raw}: an unparsed revspec must classify like a parsed RefName" + ); + assert_eq!( + RefClass::of_revspec(raw.strip_prefix("refs/").unwrap()), + *class, + "{raw}: the refs/ prefix is optional in a revspec" + ); + }); + } + + #[test] + fn every_ref_class_owns_one_namespace_and_the_classifier_maps_it_back() { + let namespaces: Vec<&str> = RefClass::ALL + .iter() + .filter_map(|class| class.namespace()) + .collect(); + assert_eq!( + namespaces.len(), + RefClass::ALL.len() - 1, + "Public is the only class without a namespace" + ); + assert_eq!( + namespaces.iter().collect::>().len(), + namespaces.len(), + "two classes on one namespace would route one class's refs through the other's rule" + ); + RefClass::ALL + .iter() + .filter_map(|class| class.namespace().map(|namespace| (class, namespace))) + .for_each(|(class, namespace)| { + assert_eq!( + RefClass::of_revspec(namespace), + *class, + "{namespace}: a class the classifier doesn't know falls through to public" + ); + assert_eq!( + RefClass::of_revspec(&format!("{namespace}/child")), + *class, + "{namespace}: a namespace must own its children" + ); + assert_eq!( + RefClass::of_revspec(&format!("{namespace}-beside/child")), + RefClass::Public, + "{namespace}: a prefix match isn't a match, so this name stays public" + ); + }); + } + + #[test] + fn knot_writes_its_atproto_chain_and_advertisement_stops_at_main() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + let chain = RefName::new("refs/atproto/commit").unwrap(); + repo.update_ref(&RefUpdate::Create { + name: head_ref(), + new: oid(A), + }) + .unwrap(); + repo.update_ref(&RefUpdate::Create { + name: chain.clone(), + new: oid(B), + }) + .unwrap(); + + assert_eq!( + repo.find_ref(&chain).unwrap(), + Some(oid(B)), + "the reservation is a wire rule, so the knot's own write still reaches the ref" + ); + assert_eq!( + repo.advertised_refs() + .unwrap() + .iter() + .map(|record| record.name.as_str().to_string()) + .collect::>(), + vec!["refs/heads/main".to_string()], + "a client can't push to the chain, so the advertisement mustn't offer it" + ); + repo.set_head(&chain).unwrap(); + assert_eq!( + repo.default_branch(), + None, + "a clone would follow HEAD to the chain, so default_branch must reject it" + ); + assert!(repo.head().is_none()); + } + + #[test] + fn ensure_valid_head_points_to_first_pushed_branch() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + + // newly created repo has "unborn" HEAD (pointing to refs/heads/main) + assert!(!repo.has_valid_head().unwrap()); + assert_eq!(repo.default_branch().unwrap().as_str(), "refs/heads/main"); + + // pushing non-branch ref (e.g. tag or atproto) must not update HEAD + let non_branch = RefName::new("refs/tags/v1.0.0").unwrap(); + let tag_updates = vec![RefUpdate::Create { + name: non_branch.clone(), + new: oid(A), + }]; + assert_eq!(repo.ensure_valid_head(&tag_updates).unwrap(), None); + assert!(repo.head().is_none()); + + // pushing first branch is used as HEAD + let master = RefName::new("refs/heads/master").unwrap(); + let master_updates = vec![RefUpdate::Create { + name: master.clone(), + new: oid(A), + }]; + repo.update_refs(&master_updates).unwrap(); + + let adopted = repo.ensure_valid_head(&master_updates).unwrap(); + assert_eq!(adopted, Some(&master)); + assert_eq!(repo.default_branch().unwrap().as_str(), "refs/heads/master"); + assert!(repo.has_valid_head().unwrap()); + assert_eq!(repo.find_ref(&master).unwrap(), Some(oid(A))); + + // a branch pushed while HEAD is already valid must not update + let dev = RefName::new("refs/heads/dev").unwrap(); + let dev_updates = vec![RefUpdate::Create { + name: dev.clone(), + new: oid(B), + }]; + repo.update_refs(&dev_updates).unwrap(); + let adopted_again = repo.ensure_valid_head(&dev_updates).unwrap(); + assert_eq!(adopted_again, None); + assert_eq!(repo.default_branch().unwrap().as_str(), "refs/heads/master"); + assert!(repo.has_valid_head().unwrap()); + } + + #[test] + fn a_ref_at_the_bare_namespace_name_blocks_its_children_on_both_write_paths() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/atproto").unwrap(), + new: oid(A), + }) + .unwrap(); + + let chain = RefUpdate::Create { + name: RefName::new("refs/atproto/commit").unwrap(), + new: oid(B), + }; + [ + repo.update_ref(&chain), + repo.update_refs(std::slice::from_ref(&chain)), + ] + .iter() + .for_each(|outcome| { + assert!( + matches!(outcome, Err(GitError::AtomicRefs(message)) if message.contains("d/f conflict")), + "gix must refuse the chain write as a d/f conflict, on whichever path the writer \ + took: {outcome:?}" + ); + }); + } + + #[test] + fn the_df_screen_stores_every_name_git_allows_and_refuses_a_relative_path() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + let awkward = ["refs/heads/a PathBuf { + nonce_dir(git_dir, INCOMING_PREFIX) +} + +pub struct Staging { + dir: PathBuf, + repo: Repo, +} + +impl Staging { + pub fn new(live: &Repo) -> Result { + let dir = incoming_path(live.path()); + let _ = std::fs::remove_dir_all(&dir); + init_bare_with_format(&dir, live.object_format()) + .map_err(|error| GitError::Staging(format!("init: {error}")))?; + let info = dir.join("objects").join("info"); + std::fs::create_dir_all(&info) + .map_err(|error| GitError::Staging(format!("objects/info: {error}")))?; + let live_objects = std::path::absolute(live.objects_dir()) + .map_err(|error| GitError::Staging(format!("alternates: {error}")))?; + std::fs::write( + info.join("alternates"), + format!("{}\n", live_objects.display()), + ) + .map_err(|error| GitError::Staging(format!("alternates: {error}")))?; + match Repo::open(&dir) { + Ok(repo) => Ok(Self { dir, repo }), + Err(error) => { + let _ = std::fs::remove_dir_all(&dir); + Err(error) + } + } + } + + pub fn repo(&self) -> &Repo { + &self.repo + } + + pub fn migrate_into(&self, live: &Repo) -> Result<(), GitError> { + migrate( + &StagingObjects(self.repo.objects_dir()), + &LiveObjects(live.objects_dir()), + ) + } +} + +impl Drop for Staging { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn is_shard(name: &str) -> bool { + name.len() == 2 && name.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn fsync_object_path(path: &Path) -> Result<(), GitError> { + let report = + |error: std::io::Error| GitError::Staging(format!("fsync {}: {error}", path.display())); + match std::fs::File::open(path) { + Ok(file) => file.sync_all().map_err(report), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(report(error)), + } +} + +struct StagingObjects(PathBuf); + +struct LiveObjects(PathBuf); + +fn migrate(from_objects: &StagingObjects, to_objects: &LiveObjects) -> Result<(), GitError> { + migrate_packs(from_objects, to_objects)?; + migrate_loose(from_objects, to_objects) +} + +fn migrate_packs(from_objects: &StagingObjects, to_objects: &LiveObjects) -> Result<(), GitError> { + let from_pack = from_objects.0.join("pack"); + let entries = match std::fs::read_dir(&from_pack) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(GitError::Staging(error.to_string())), + }; + let to_pack = to_objects.0.join("pack"); + std::fs::create_dir_all(&to_pack).map_err(|error| GitError::Staging(error.to_string()))?; + let names: Vec = entries + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_type() + .map(|kind| kind.is_file()) + .unwrap_or(false) + }) + .map(|entry| entry.file_name()) + .collect(); + let is_idx = + |name: &std::ffi::OsString| Path::new(name).extension().is_some_and(|ext| ext == "idx"); + let ordered = names + .iter() + .filter(|name| !is_idx(name)) + .chain(names.iter().filter(|name| is_idx(name))); + ordered.into_iter().try_for_each(|name| { + let dest = to_pack.join(name); + std::fs::rename(from_pack.join(name), &dest) + .map_err(|error| GitError::Staging(format!("migrate pack file: {error}")))?; + fsync_object_path(&dest) + })?; + fsync_object_path(&to_pack) +} + +fn migrate_loose(from_objects: &StagingObjects, to_objects: &LiveObjects) -> Result<(), GitError> { + let entries = + std::fs::read_dir(&from_objects.0).map_err(|error| GitError::Staging(error.to_string()))?; + entries + .filter_map(Result::ok) + .filter(|entry| { + entry.file_type().map(|kind| kind.is_dir()).unwrap_or(false) + && entry.file_name().to_str().is_some_and(is_shard) + }) + .try_for_each(|shard| { + let dest_shard = to_objects.0.join(shard.file_name()); + std::fs::create_dir_all(&dest_shard) + .map_err(|error| GitError::Staging(error.to_string()))?; + std::fs::read_dir(shard.path()) + .map_err(|error| GitError::Staging(error.to_string()))? + .filter_map(Result::ok) + .try_for_each(|object| { + let dest = dest_shard.join(object.file_name()); + std::fs::rename(object.path(), &dest).map_err(|error| { + GitError::Staging(format!("migrate loose object: {error}")) + })?; + fsync_object_path(&dest) + })?; + fsync_object_path(&dest_shard) + })?; + fsync_object_path(&to_objects.0) +} diff --git a/knot2/crates/knot-git/tests/apply.rs b/knot2/crates/knot-git/tests/apply.rs new file mode 100644 index 0000000000000000000000000000000000000000..9c3816cfa07a6e429eadd65c4ab4ea2e47c797bf --- /dev/null +++ b/knot2/crates/knot-git/tests/apply.rs @@ -0,0 +1,432 @@ +use std::path::Path; + +use knot_git::{ + ApplyOutcome, ConflictReason, Identity, Layout, NewCommit, PatchApplier, RefUpdate, Repo, + is_format_patch, parse_mailbox, parse_patch, +}; +use knot_types::{AuthorName, Email, Oid, RefName, RepoDid, UnixSeconds}; + +mod common; +use common::{git_ok as git, seeded}; + +fn stage_and_commit(work: &Path, message: &str) { + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", message]); +} + +fn push_main(work: &Path, layout: &Layout, did: &RepoDid) { + let bare = layout.repo_path(did).unwrap(); + git(work, &["push", "-q", bare.to_str().unwrap(), "main"]); + git(&bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); +} + +fn main_ref() -> RefName { + RefName::new("refs/heads/main").unwrap() +} + +fn committer() -> Identity { + Identity { + name: AuthorName::new("Tangled"), + email: Email::new("noreply@tangled.sh"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } +} + +fn apply_mailbox_natively(bare: &Repo, patch: &str) -> Vec { + let mails = parse_mailbox(patch).unwrap(); + let tip = bare.find_ref(&main_ref()).unwrap().unwrap(); + let base_tree = bare.find_commit(tip).unwrap().tree; + let mut applier = PatchApplier::new(bare, tip); + let (commits, _, new_tip) = mails.iter().fold( + (Vec::new(), base_tree, tip), + |(mut commits, tree, parent), mail| { + let staged = match applier.step(&mail.files).unwrap() { + ApplyOutcome::Clean(staged) => staged, + ApplyOutcome::Conflicted(conflicts) => { + panic!("expected clean apply, got conflicts {conflicts:?}") + } + }; + let next_tree = bare.write_staged_tree(tree, &staged).unwrap(); + let commit = bare + .write_commit(&NewCommit { + tree: next_tree, + parents: vec![parent], + author: Identity { + name: mail.author_name.clone(), + email: mail.author_email.clone(), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + }, + committer: committer(), + message: mail.commit_message(), + extra_headers: mail + .change_id + .iter() + .map(|id| ("change-id".to_string(), id.as_str().as_bytes().to_vec())) + .collect(), + }) + .unwrap(); + commits.push(commit); + (commits, next_tree, commit) + }, + ); + bare.update_ref(&RefUpdate::Update { + name: main_ref(), + old: tip, + new: new_tip, + }) + .unwrap(); + commits +} + +#[test] +fn a_format_patch_series_applies_tree_identical_to_git_am() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + + let blob: Vec = (0u32..8192).map(|i| (i * 31 % 251) as u8).collect(); + std::fs::write(work.join("a.txt"), "alpha\nbeta\ngamma\n").unwrap(); + std::fs::create_dir_all(work.join("sub")).unwrap(); + std::fs::write(work.join("sub/inner.txt"), "nested\n").unwrap(); + std::fs::write(work.join("noeol.txt"), "tail without newline").unwrap(); + std::fs::write(work.join("data.bin"), &blob).unwrap(); + std::fs::write(work.join("drop.bin"), [0u8, 1, 2, 3, 0, 9]).unwrap(); + stage_and_commit(work, "base"); + push_main(work, &layout, &did); + let base = git(work, &["rev-parse", "HEAD"]); + + std::fs::write(work.join("a.txt"), "alpha\nBETA\ngamma\n").unwrap(); + stage_and_commit(work, "first subject\n\nfirst body line"); + + git(work, &["mv", "a.txt", "moved.txt"]); + std::fs::write(work.join("moved.txt"), "alpha\nBETA\ngamma\ndelta\n").unwrap(); + std::fs::write(work.join("run.sh"), "#!/bin/sh\necho reef\n").unwrap(); + git(work, &["add", "-A"]); + git(work, &["update-index", "--chmod=+x", "run.sh"]); + git(work, &["commit", "-q", "-m", "second"]); + + std::fs::remove_file(work.join("noeol.txt")).unwrap(); + std::fs::write(work.join("sp ace.txt"), "spaced\n").unwrap(); + std::fs::write(work.join("café.txt"), "unicode\n").unwrap(); + stage_and_commit(work, "third"); + + let mutated: Vec = blob + .iter() + .copied() + .chain([0u8, 255, 254, 7]) + .map(|byte| match byte { + 42 => 24, + other => other, + }) + .collect(); + std::fs::write(work.join("data.bin"), &mutated).unwrap(); + std::fs::remove_file(work.join("drop.bin")).unwrap(); + std::fs::write(work.join("new.bin"), [9u8, 0, 8, 0, 7]).unwrap(); + stage_and_commit(work, "binary churn"); + + let patch = git( + work, + &["format-patch", "--stdout", &format!("{base}..HEAD")], + ); + assert!(is_format_patch(&patch)); + assert!(patch.contains("GIT binary patch")); + + let bare = layout.open(&did).unwrap(); + let ours = apply_mailbox_natively(&bare, &patch); + + let expected: Vec = git(work, &["rev-list", "--reverse", &format!("{base}..HEAD")]) + .lines() + .map(str::to_string) + .collect(); + assert_eq!(ours.len(), expected.len()); + ours.iter().zip(&expected).for_each(|(our_oid, real)| { + let our_commit = bare.find_commit(*our_oid).unwrap(); + let real_tree = + Oid::from_hex(&git(work, &["rev-parse", &format!("{real}^{{tree}}")])).unwrap(); + assert_eq!( + our_commit.tree, real_tree, + "natively applied tree must be byte-identical to git am's" + ); + let real_message = git(work, &["log", "-1", "--format=%B", real]); + assert_eq!(our_commit.message.trim_end(), real_message.trim_end()); + assert_eq!(our_commit.author.name.as_str(), "nel"); + assert_eq!(our_commit.author.email.as_str(), "nel@oyster.cafe"); + assert_eq!(our_commit.committer.name.as_str(), "Tangled"); + assert_eq!(our_commit.committer.email.as_str(), "noreply@tangled.sh"); + }); + assert_eq!( + bare.find_ref(&main_ref()).unwrap(), + Some(*ours.last().unwrap()) + ); + + git(work, &["checkout", "-q", "-b", "subline"]); + std::fs::write(work.join("seed.txt"), "one\n").unwrap(); + stage_and_commit(work, "seed one"); + let old_oid = git(work, &["rev-parse", "HEAD"]); + std::fs::write(work.join("seed.txt"), "two\n").unwrap(); + stage_and_commit(work, "seed two"); + let new_oid = git(work, &["rev-parse", "HEAD"]); + git( + work, + &[ + "update-index", + "--add", + "--cacheinfo", + &format!("160000,{old_oid},vendor"), + ], + ); + git(work, &["commit", "-q", "-m", "add submodule"]); + let sub_base = git(work, &["rev-parse", "HEAD"]); + let bare_path = layout.repo_path(&did).unwrap(); + git( + work, + &[ + "push", + "-q", + "-f", + bare_path.to_str().unwrap(), + &format!("{sub_base}:refs/heads/main"), + ], + ); + + git( + work, + &[ + "update-index", + "--cacheinfo", + &format!("160000,{new_oid},vendor"), + ], + ); + git(work, &["commit", "-q", "-m", "bump submodule"]); + let bump = git( + work, + &["format-patch", "--stdout", &format!("{sub_base}..HEAD")], + ); + assert!(bump.contains("Subproject commit")); + + let bumped = layout.open(&did).unwrap(); + let applied = apply_mailbox_natively(&bumped, &bump); + let real_tree = Oid::from_hex(&git(work, &["rev-parse", "HEAD^{tree}"])).unwrap(); + assert_eq!(bumped.find_commit(applied[0]).unwrap().tree, real_tree); +} + +#[test] +fn a_unified_diff_applies_tree_identical_to_git_apply() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + + std::fs::write(work.join("a.txt"), "one\ntwo\nthree\n").unwrap(); + std::fs::write(work.join("gone.txt"), "doomed\n").unwrap(); + std::fs::write(work.join("noeol.txt"), "no newline here").unwrap(); + stage_and_commit(work, "base"); + push_main(work, &layout, &did); + let base = git(work, &["rev-parse", "HEAD"]); + + std::fs::write(work.join("a.txt"), "one\nTWO\nthree\nfour\n").unwrap(); + std::fs::remove_file(work.join("gone.txt")).unwrap(); + std::fs::write(work.join("fresh.txt"), "brand new\n").unwrap(); + std::fs::write(work.join("noeol.txt"), "still no newline").unwrap(); + stage_and_commit(work, "changes"); + + let patch = git(work, &["diff", &base, "HEAD"]); + assert!(!is_format_patch(&patch)); + let files = parse_patch(&patch).unwrap(); + + let bare = layout.open(&did).unwrap(); + let tip = bare.find_ref(&main_ref()).unwrap().unwrap(); + let base_tree = bare.find_commit(tip).unwrap().tree; + let mut applier = PatchApplier::new(&bare, tip); + let staged = match applier.step(&files).unwrap() { + ApplyOutcome::Clean(staged) => staged, + ApplyOutcome::Conflicted(conflicts) => panic!("unexpected conflicts {conflicts:?}"), + }; + let our_tree = bare.write_staged_tree(base_tree, &staged).unwrap(); + + let real_tree = Oid::from_hex(&git(work, &["rev-parse", "HEAD^{tree}"])).unwrap(); + assert_eq!(our_tree, real_tree); +} + +#[test] +fn a_stale_patch_conflicts_instead_of_applying() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + + std::fs::write(work.join("a.txt"), "original\n").unwrap(); + stage_and_commit(work, "base"); + let base = git(work, &["rev-parse", "HEAD"]); + + std::fs::write(work.join("a.txt"), "patched from original\n").unwrap(); + stage_and_commit(work, "feature"); + let patch = git(work, &["diff", &base, "HEAD"]); + + git(work, &["checkout", "-q", &base]); + git(work, &["checkout", "-q", "-b", "drifted"]); + std::fs::write(work.join("a.txt"), "diverged\n").unwrap(); + stage_and_commit(work, "drift"); + git(work, &["branch", "-q", "-f", "main", "HEAD"]); + push_main(work, &layout, &did); + + let bare = layout.open(&did).unwrap(); + let tip = bare.find_ref(&main_ref()).unwrap().unwrap(); + let files = parse_patch(&patch).unwrap(); + let mut applier = PatchApplier::new(&bare, tip); + match applier.step(&files).unwrap() { + ApplyOutcome::Conflicted(conflicts) => { + assert_eq!(conflicts.len(), 1); + assert_eq!(conflicts[0].path, "a.txt"); + assert_eq!(conflicts[0].reason, ConflictReason::DoesNotApply); + } + ApplyOutcome::Clean(_) => panic!("stale patch mustn't apply cleanly"), + } +} + +fn git_apply_applies(cwd: &Path, patch: &str) -> bool { + use std::io::Write; + use std::process::Stdio; + let mut child = knot_fixtures::command(cwd) + .args(["apply", "--check"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("git is available"); + child + .stdin + .take() + .expect("stdin is piped") + .write_all(patch.as_bytes()) + .expect("write patch to git apply"); + child.wait().expect("git apply completes").success() +} + +#[test] +fn apply_verdict_matches_git_apply() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + std::fs::write(work.join("a.txt"), "one\ntwo\nthree\nfour\nfive\n").unwrap(); + stage_and_commit(work, "base"); + push_main(work, &layout, &did); + + let bare = layout.open(&did).unwrap(); + let tip = bare.find_ref(&main_ref()).unwrap().unwrap(); + + let cases: [&str; 4] = [ + "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n@@ -1,3 +1,3 @@\n one\n-two\n+TWO\n three\n", + "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n@@ -1,3 +1,3 @@\n nope\n-gone\n+HERE\n zero\n", + "diff --git a/fresh.txt b/fresh.txt\nnew file mode 100644\n--- /dev/null\n+++ b/fresh.txt\n@@ -0,0 +1 @@\n+brand new\n", + "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n@@ -2,1 +1,0 @@\n-two\n", + ]; + + cases.iter().for_each(|patch| { + let git_clean = git_apply_applies(work, patch); + let mut applier = PatchApplier::new(&bare, tip); + let ours_clean = matches!( + applier.step(&parse_patch(patch).unwrap()).unwrap(), + ApplyOutcome::Clean(_) + ); + assert_eq!( + git_clean, ours_clean, + "verdict disagrees with git apply for patch:\n{patch}" + ); + }); +} + +fn git_apply_to_worktree_fails(cwd: &Path, patch: &str) -> bool { + use std::io::Write; + use std::process::Stdio; + let mut child = knot_fixtures::command(cwd) + .args(["apply"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("git is available"); + child + .stdin + .take() + .expect("stdin is piped") + .write_all(patch.as_bytes()) + .expect("write patch to git apply"); + !child.wait().expect("git apply completes").success() +} + +#[test] +fn refused_patches_conflict_with_the_reason_git_apply_rejects() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + std::fs::write(work.join("a.txt"), "present\n").unwrap(); + std::fs::write(work.join("dir"), "i am a file\n").unwrap(); + stage_and_commit(work, "base"); + push_main(work, &layout, &did); + + let bare = layout.open(&did).unwrap(); + let tip = bare.find_ref(&main_ref()).unwrap().unwrap(); + + let create_existing = concat!( + "diff --git a/a.txt b/a.txt\n", + "new file mode 100644\n", + "--- /dev/null\n", + "+++ b/a.txt\n", + "@@ -0,0 +1 @@\n", + "+x\n", + ); + let delete_missing = concat!( + "diff --git a/ghost.txt b/ghost.txt\n", + "deleted file mode 100644\n", + "--- a/ghost.txt\n", + "+++ /dev/null\n", + "@@ -1 +0,0 @@\n", + "-x\n", + ); + let escape = concat!( + "diff --git a/../escape.txt b/../escape.txt\n", + "new file mode 100644\n", + "--- /dev/null\n", + "+++ b/../escape.txt\n", + "@@ -0,0 +1 @@\n", + "+boom\n", + ); + let under_a_file = concat!( + "diff --git a/dir/inner.txt b/dir/inner.txt\n", + "new file mode 100644\n", + "--- /dev/null\n", + "+++ b/dir/inner.txt\n", + "@@ -0,0 +1 @@\n", + "+nested\n", + ); + + let cases: &[(&str, ConflictReason, &str)] = &[ + ( + create_existing, + ConflictReason::AlreadyExists, + "file already exists", + ), + ( + delete_missing, + ConflictReason::DoesNotExist, + "file doesn't exist", + ), + (escape, ConflictReason::DoesNotApply, "patch doesn't apply"), + ( + under_a_file, + ConflictReason::DoesNotApply, + "patch doesn't apply", + ), + ]; + + cases.iter().for_each(|(patch, reason, message)| { + assert!( + git_apply_to_worktree_fails(work, patch), + "git apply must refuse:\n{patch}" + ); + let mut applier = PatchApplier::new(&bare, tip); + match applier.step(&parse_patch(patch).unwrap()).unwrap() { + ApplyOutcome::Conflicted(conflicts) => { + assert_eq!(conflicts[0].reason, *reason); + assert_eq!(conflicts[0].reason.as_str(), *message); + } + ApplyOutcome::Clean(_) => panic!("must conflict, not apply cleanly:\n{patch}"), + } + }); +} diff --git a/knot2/crates/knot-git/tests/bitmap.rs b/knot2/crates/knot-git/tests/bitmap.rs new file mode 100644 index 0000000000000000000000000000000000000000..2ede815f566d1eec531c0743376f4d30bc76edf0 --- /dev/null +++ b/knot2/crates/knot-git/tests/bitmap.rs @@ -0,0 +1,377 @@ +use std::collections::HashSet; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use knot_git::{ + Haves, Repo, Wants, count_commits, open_bitmap_index, reachable_via_bitmap, + verbatim_clone_pack, write_bitmap, write_midx_bitmap, +}; +use knot_types::Oid; + +mod common; +use common::{commit_file as commit, git, git_available, git_ok as ok}; + +fn test_bitmap(path: &Path) { + let (ok, report) = git(path, &["rev-list", "--test-bitmap", "HEAD"]); + assert!(ok, "git rejected the bitmap: {report}"); +} + +fn seed(format: &str) -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path(); + ok( + path, + &[ + "init", + "-q", + "--object-format", + format, + "--initial-branch", + "main", + ], + ); + commit(path, "a.txt", "alpha\n", "root"); + commit(path, "b.txt", "beta\n", "second"); + ok(path, &["checkout", "-q", "-b", "feature"]); + commit(path, "c.txt", "gamma\n", "feature work"); + ok(path, &["checkout", "-q", "main"]); + commit(path, "d.txt", "delta\n", "more main"); + ok(path, &["checkout", "-q", "-b", "topic"]); + commit(path, "t.txt", "topic\n", "topic work"); + ok(path, &["checkout", "-q", "main"]); + ok( + path, + &["merge", "-q", "--no-ff", "-m", "merge topic", "topic"], + ); + ok(path, &["tag", "-a", "v1", "-m", "release one"]); + ok(path, &["repack", "-adq"]); + dir +} + +fn find_pack(path: &Path, suffix: &str) -> PathBuf { + std::fs::read_dir(path.join(".git/objects/pack")) + .unwrap() + .filter_map(Result::ok) + .map(|e| e.path()) + .find(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.ends_with(suffix)) + }) + .unwrap_or_else(|| panic!("a pack file ending {suffix}")) +} + +fn idx_count(path: &Path) -> usize { + std::fs::read_dir(path.join(".git/objects/pack")) + .unwrap() + .filter_map(Result::ok) + .filter(|e| e.path().extension().is_some_and(|ext| ext == "idx")) + .count() +} + +fn rev_parse(path: &Path, name: &str) -> Oid { + Oid::from_hex(ok(path, &["rev-parse", name]).trim()).unwrap() +} + +fn rev_list(path: &Path, revs: &[&str]) -> HashSet { + let args: Vec<&str> = ["rev-list", "--objects"] + .into_iter() + .chain(revs.iter().copied()) + .collect(); + ok(path, &args) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter_map(|token| Oid::from_hex(token).ok()) + .collect() +} + +fn closure(repo: &Repo, wants: &[Oid], haves: &[Oid]) -> HashSet { + reachable_via_bitmap(repo, Wants::new(wants), Haves::new(haves)) + .unwrap() + .expect("bitmap fast path resolves") + .into_iter() + .collect() +} + +fn walked(repo: &Repo, want: Oid) -> HashSet { + repo.select_pack_objects(Wants::new(&[want]), Haves::new(&[])) + .unwrap() + .into_iter() + .collect() +} + +fn reject_corruption(repo: &Repo, bitmap: &Path, head: Oid) { + let intact = std::fs::read(bitmap).unwrap(); + let mut flipped = intact.clone(); + *flipped.last_mut().unwrap() ^= 0xff; + std::fs::write(bitmap, &flipped).unwrap(); + assert!( + reachable_via_bitmap(repo, Wants::new(&[head]), Haves::new(&[])).is_err(), + "a corrupt checksum is rejected" + ); + std::fs::write(bitmap, &intact[..intact.len() / 2]).unwrap(); + assert!( + reachable_via_bitmap(repo, Wants::new(&[head]), Haves::new(&[])).is_err(), + "a truncated bitmap is rejected without a panic" + ); +} + +fn run_lifecycle(format: &str) { + if !git_available() { + eprintln!("skipping bitmap lifecycle: git unavailable"); + return; + } + let dir = seed(format); + let path = dir.path(); + let repo = Repo::open(path).unwrap(); + let idx = find_pack(path, ".idx"); + assert!( + write_bitmap(&repo, &idx, None).unwrap(), + "a single-pack repo gets a bitmap" + ); + test_bitmap(path); + + let head = rev_parse(path, "HEAD"); + let ours = closure(&repo, &[head], &[]); + assert_eq!( + ours, + rev_list(path, &["HEAD"]), + "closure equals rev-list --objects HEAD" + ); + assert_eq!(ours, walked(&repo, head), "closure equals the walk closure"); + assert_eq!( + ours, + closure(&repo, &vec![head; 2048], &[]), + "duplicate wants fold to one closure" + ); + + let feature = rev_parse(path, "feature"); + let main = rev_parse(path, "main"); + assert_eq!( + closure(&repo, &[feature], &[main]), + rev_list(path, &["feature", "^main"]), + "feature minus main" + ); + + let tag = rev_parse(path, "v1"); + let mut whole = verbatim_clone_pack(&repo, Wants::new(&[main, feature, tag])) + .unwrap() + .expect("full clone reuses whole pack"); + let mut header = [0u8; 12]; + whole.read_exact(&mut header).unwrap(); + assert_eq!( + &header[..4], + b"PACK", + "verbatim reuse returns the on-disk pack" + ); + let count = u32::from_be_bytes([header[8], header[9], header[10], header[11]]) as usize; + assert_eq!( + count, + rev_list(path, &["--all"]).len(), + "verbatim reuse streams every object" + ); + assert!( + verbatim_clone_pack(&repo, Wants::new(&[main])) + .unwrap() + .is_none(), + "a partial closure never reuses the whole pack" + ); + + ok(path, &["tag", "-a", "v2", "-m", "release two", "HEAD"]); + let v2 = rev_parse(path, "v2"); + assert_ne!(v2, head); + assert_eq!( + closure(&repo, &[v2], &[]), + rev_list(path, &["v2"]), + "a want outside the bitmapped pack walks down to the bitmap" + ); + + ok( + path, + &["tag", "-a", "tree-tag", "-m", "a tree", "HEAD^{tree}"], + ); + let tree_tag = rev_parse(path, "tree-tag"); + assert_eq!( + closure(&repo, &[tree_tag, head], &[]), + rev_list(path, &["tree-tag", "HEAD"]), + "a tag pointing at a tree resolves through the bitmap" + ); + + commit(path, "s.txt", "stale\n", "after the bitmap"); + let fresh = rev_parse(path, "HEAD"); + assert_eq!( + closure(&repo, &[fresh], &[]), + rev_list(path, &["HEAD"]), + "a stale bitmap still answers a clone" + ); + assert_eq!( + closure(&repo, &[fresh], &[head]), + rev_list(path, &["HEAD", &format!("^{head}")]), + "a stale bitmap answers a fetch" + ); + commit(path, "u.txt", "unbitmapped\n", "a have without a bitmap"); + let newest = rev_parse(path, "HEAD"); + assert_eq!( + closure(&repo, &[newest], &[fresh]), + rev_list(path, &["HEAD", &format!("^{fresh}")]), + "a have without a bitmap is walked" + ); + + let previous = open_bitmap_index(&repo) + .unwrap() + .expect("the pack bitmap opens"); + std::fs::remove_file(idx.with_extension("bitmap")).unwrap(); + commit(path, "e.txt", "epsilon\n", "post-bitmap growth"); + commit(path, "f.txt", "zeta\n", "more growth"); + ok(path, &["repack", "-dq"]); + ok(path, &["multi-pack-index", "write"]); + assert!(idx_count(path) >= 2, "the repo now spans multiple packs"); + + let repo = Repo::open(path).unwrap(); + assert!( + write_midx_bitmap(&repo, None).unwrap(), + "a multi-pack repo gets a midx bitmap" + ); + let midx_bitmap = find_pack(path, ".bitmap"); + let rebuilt = std::fs::read(&midx_bitmap).unwrap(); + assert!(write_midx_bitmap(&repo, Some(&previous)).unwrap()); + assert_eq!( + std::fs::read(&midx_bitmap).unwrap(), + rebuilt, + "reusing the pack bitmap writes the same midx bitmap" + ); + test_bitmap(path); + let head = rev_parse(path, "HEAD"); + assert_eq!( + closure(&repo, &[head], &[]), + rev_list(path, &["HEAD"]), + "midx closure equals rev-list HEAD" + ); + + reject_corruption(&repo, &find_pack(path, ".bitmap"), head); +} + +#[test] +fn bitmap_single_pack_then_midx_round_trips_against_canonical_git_sha1() { + run_lifecycle("sha1"); +} + +#[test] +fn bitmap_single_pack_then_midx_round_trips_against_canonical_git_sha256() { + run_lifecycle("sha256"); +} + +fn bitmap_entries(bitmap: &Path) -> u32 { + let bytes = std::fs::read(bitmap).unwrap(); + u32::from_be_bytes(bytes[8..12].try_into().unwrap()) +} + +#[test] +fn bitmap_selection_samples_history_not_refs() { + use std::io::Write; + if !git_available() { + eprintln!("skipping bitmap selection: git unavailable"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path(); + ok(path, &["init", "-q", "--initial-branch", "main"]); + let refs = 300; + let mut script = String::new(); + for n in 0..refs { + let from = if n == 0 { + String::new() + } else { + format!("from :{n}\n") + }; + script += &format!( + "commit refs/heads/b{n}\nmark :{}\ncommitter a {} +0000\ndata 1\nx\n{from}M 644 inline f\ndata {}\n{n}\n\n", + n + 1, + 1_000_000 + n, + n.to_string().len() + 1, + ); + } + let mut child = std::process::Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(path) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(script.as_bytes()) + .unwrap(); + assert!(child.wait().unwrap().success()); + ok( + path, + &[ + "update-ref", + "refs/heads/main", + &format!("refs/heads/b{}", refs - 1), + ], + ); + ok(path, &["repack", "-adbq"]); + + let idx = find_pack(path, ".idx"); + let bitmap = idx.with_extension("bitmap"); + let canonical = bitmap_entries(&bitmap); + std::fs::remove_file(&bitmap).unwrap(); + let repo = Repo::open(path).unwrap(); + assert!(write_bitmap(&repo, &idx, None).unwrap()); + let ours = bitmap_entries(&bitmap); + assert_eq!(ours, canonical, "selection matches git's sampling"); + assert!(ours < refs, "not every ref tip gets a bitmap"); + test_bitmap(path); + + let unsampled = rev_parse(path, "b150"); + assert_eq!( + closure(&repo, &[unsampled], &[]), + rev_list(path, &["b150"]), + "a ref without its own bitmap walks down to one" + ); +} + +fn rev_count(path: &Path, rev: &str) -> u64 { + ok(path, &["rev-list", "--count", rev]) + .trim() + .parse() + .unwrap() +} + +#[test] +fn count_commits_matches_rev_list_with_and_without_bitmap() { + if !git_available() { + eprintln!("skipping count_commits: git unavailable"); + return; + } + let dir = seed("sha1"); + let path = dir.path(); + let later = std::time::Instant::now() + std::time::Duration::from_secs(60); + let count = |rev: &str| { + let repo = Repo::open(path).unwrap(); + count_commits(&repo, rev_parse(path, rev), later).unwrap() + }; + assert_eq!(count("HEAD"), Some(rev_count(path, "HEAD")), "plain walk"); + + let repo = Repo::open(path).unwrap(); + assert!(write_bitmap(&repo, &find_pack(path, ".idx"), None).unwrap()); + for rev in ["HEAD", "feature", "HEAD~1"] { + assert_eq!(count(rev), Some(rev_count(path, rev)), "bitmap {rev}"); + } + + commit(path, "e.txt", "epsilon\n", "after repack"); + assert_eq!( + count("HEAD"), + Some(rev_count(path, "HEAD")), + "commits outside the bitmap are walked" + ); + + let past = std::time::Instant::now(); + assert_eq!( + count_commits(&Repo::open(path).unwrap(), rev_parse(path, "HEAD"), past).unwrap(), + None, + "an elapsed deadline gives up" + ); +} diff --git a/knot2/crates/knot-git/tests/common/mod.rs b/knot2/crates/knot-git/tests/common/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..ef6a6ffd61835a672114592d7b24649c797c5556 --- /dev/null +++ b/knot2/crates/knot-git/tests/common/mod.rs @@ -0,0 +1,18 @@ +#![allow(dead_code, unused_imports)] + +use knot_git::Layout; +use knot_types::RepoDid; + +pub use knot_fixtures::{ + available as git_available, commit as commit_file, contains, must as git_ok, run as git, +}; + +pub fn seeded() -> (tempfile::TempDir, tempfile::TempDir, Layout, RepoDid) { + let scan = tempfile::tempdir().unwrap(); + let work = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + git_ok(work.path(), &["init", "-q", "-b", "main"]); + (scan, work, layout, did) +} diff --git a/knot2/crates/knot-git/tests/config_isolation.rs b/knot2/crates/knot-git/tests/config_isolation.rs new file mode 100644 index 0000000000000000000000000000000000000000..82ea69953797f67e8c229d141502fec30915595d --- /dev/null +++ b/knot2/crates/knot-git/tests/config_isolation.rs @@ -0,0 +1,142 @@ +use std::path::Path; +use std::sync::atomic::AtomicBool; + +use knot_git::{ArchiveFormat, ArchiveLimit}; +use knot_types::Oid; + +mod common; +use common::{commit_file, contains, git_ok as git, seeded}; + +fn archive_without_isolation(bare_path: &Path, head: Oid) -> Vec { + let permissive = gix::open_opts(bare_path, gix::open::Options::default()).unwrap(); + let tree = permissive + .find_object(head.object_id()) + .unwrap() + .peel_to_tree() + .unwrap(); + let (stream, _index) = permissive.worktree_stream(tree.id).unwrap(); + let mut out = std::io::Cursor::new(Vec::new()); + permissive + .worktree_archive( + stream, + &mut out, + gix::progress::Discard, + &AtomicBool::new(false), + gix_archive::Options { + format: gix_archive::Format::Tar, + tree_prefix: None, + modification_time: 0, + }, + ) + .unwrap(); + out.into_inner() +} + +#[test] +fn a_filter_driver_pulled_in_by_an_include_never_runs_for_a_served_archive() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + let bare_path = layout.repo_path(&did).unwrap(); + std::fs::write(work.join(".gitattributes"), "payload.txt filter=knotpwn\n").unwrap(); + commit_file(work, "payload.txt", "kelp\n", "seed"); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + + let ambient = tempfile::tempdir().unwrap(); + let driver = ambient.path().join("driver.cfg"); + std::fs::write( + &driver, + "[filter \"knotpwn\"]\n\tsmudge = sed s/kelp/pwned/\n\trequired = true\n", + ) + .unwrap(); + let config_path = bare_path.join("config"); + let local = std::fs::read_to_string(&config_path).unwrap(); + std::fs::write( + &config_path, + format!("{local}[include]\n\tpath = {}\n", driver.display()), + ) + .unwrap(); + + let via_git = knot_fixtures::command(&bare_path) + .args(["archive", "--format=tar", "main"]) + .output() + .unwrap(); + assert!( + via_git.status.success(), + "git archive failed:\n{}", + String::from_utf8_lossy(&via_git.stderr) + ); + assert!( + contains(&via_git.stdout, b"pwned"), + "git archived with the include in place and its output has no pwned in it, so the driver \ + never ran" + ); + assert!( + contains(&archive_without_isolation(&bare_path, head), b"pwned"), + "gix archived the same repo under default permissions and its output has no pwned in it, \ + so worktree_stream no longer applies filters" + ); + + let bare = layout.open(&did).unwrap(); + let tree = bare.peel_to_tree(head).unwrap(); + let mut out = std::io::Cursor::new(Vec::new()); + bare.write_archive( + tree, + ArchiveFormat::Tar, + None, + ArchiveLimit::new(u64::MAX), + &mut out, + ) + .unwrap(); + let served = out.into_inner(); + + assert!( + std::fs::read_to_string(&config_path) + .unwrap() + .contains("[include]"), + "opening the repo rewrote its config and removed the include, so gix never read the \ + driver definition for the archive below" + ); + assert!( + contains(&served, b"kelp"), + "the served archive contains the blob as it was pushed" + ); + assert!( + !contains(&served, b"pwned"), + "the knot ran a filter driver defined by config outside the repository" + ); +} + +#[test] +fn a_pushed_replace_ref_never_substitutes_an_object_the_knot_reads() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + let bare_path = layout.repo_path(&did).unwrap(); + + commit_file(work, "payload.txt", "kelp\n", "seed"); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let original = Oid::from_hex(&git(work, &["rev-parse", "HEAD:payload.txt"])).unwrap(); + commit_file(work, "payload.txt", "pwned\n", "second"); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let substitute = Oid::from_hex(&git(work, &["rev-parse", "HEAD:payload.txt"])).unwrap(); + + git( + &bare_path, + &[ + "update-ref", + &format!("refs/replace/{}", original.to_hex()), + &substitute.to_hex(), + ], + ); + + assert_eq!( + git(&bare_path, &["cat-file", "blob", &original.to_hex()]), + "pwned", + "git read the replaced object as itself, so this fixture never armed the substitution" + ); + assert_eq!( + layout.open(&did).unwrap().read_blob(original).unwrap(), + b"kelp\n", + "a pushed replace ref rewrote what the knot serves for an object" + ); +} diff --git a/knot2/crates/knot-git/tests/fuzz_smoke.rs b/knot2/crates/knot-git/tests/fuzz_smoke.rs new file mode 100644 index 0000000000000000000000000000000000000000..e242d405ed3122f90b2cd46c8ef4bab4f380cd81 --- /dev/null +++ b/knot2/crates/knot-git/tests/fuzz_smoke.rs @@ -0,0 +1,10 @@ +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn the_patch_parsers_never_panic(data in proptest::collection::vec(any::(), 0..4096)) { + knot_git::fuzz::patch(&data); + } +} diff --git a/knot2/crates/knot-git/tests/reads.rs b/knot2/crates/knot-git/tests/reads.rs new file mode 100644 index 0000000000000000000000000000000000000000..652b1586d29613db968bfcfd5c88408c7ca953b8 --- /dev/null +++ b/knot2/crates/knot-git/tests/reads.rs @@ -0,0 +1,851 @@ +use std::path::Path; +use std::time::{Duration, Instant}; + +use knot_git::{ + BinaryBudget, BinaryDiff, CommitRange, EntryKind, FileChange, FilePatch, GitError, Layout, + LineCount, LogLimit, LogSkip, PatchBody, Repo, +}; +use knot_types::{Listing, Oid, RefName, RepoDid, RepoPath}; + +fn rp(path: &str) -> RepoPath { + RepoPath::new(path).unwrap() +} + +fn patches_of( + bare: &Repo, + base: Option, + head: Oid, + budget: &mut BinaryBudget, +) -> Vec { + bare.commit_patches(knot_git::PatchRange { base, head }, budget) + .unwrap() +} + +fn named<'a>(patches: &'a [FilePatch], path: &str) -> &'a FilePatch { + patches + .iter() + .find(|patch| patch.path.as_str() == path) + .unwrap() +} + +fn binary_diff(patch: &FilePatch) -> &BinaryDiff { + match &patch.body { + PatchBody::Binary(diff) => diff, + PatchBody::Text(_) => panic!("expected a binary patch for {}, found text", patch.path), + } +} + +mod common; +use common::{commit_file, contains, git_ok as git}; + +#[test] +fn typed_reads_over_a_seeded_repo() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + let bare_path = layout.repo_path(&did).unwrap(); + let bare_str = bare_path.to_str().unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git(work, &["init", "-q", "-b", "main"]); + commit_file(work, "a.txt", "one\n", "first"); + git(work, &["push", "-q", bare_str, "main"]); + commit_file(work, "b.txt", "two\n", "second"); + std::fs::write(work.join("a.txt"), "one updated\n").unwrap(); + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", "third"]); + git(work, &["push", "-q", bare_str, "main"]); + git( + bare_path.as_path(), + &["symbolic-ref", "HEAD", "refs/heads/main"], + ); + + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let parent = Oid::from_hex(&git(work, &["rev-parse", "HEAD~1"])).unwrap(); + let root = Oid::from_hex(&git(work, &["rev-parse", "HEAD~2"])).unwrap(); + let tree = Oid::from_hex(&git(work, &["rev-parse", "HEAD^{tree}"])).unwrap(); + let blob_b = Oid::from_hex(&git(work, &["rev-parse", "HEAD:b.txt"])).unwrap(); + + let head_ref = bare.head().expect("HEAD resolves"); + assert_eq!(head_ref.name.as_str(), "refs/heads/main"); + assert_eq!(head_ref.target, head); + + let commit = bare.find_commit(head).unwrap(); + assert_eq!(commit.id, head); + assert_eq!(commit.tree, tree); + assert_eq!(commit.parents, vec![parent]); + assert_eq!(commit.author.name.as_str(), "nel"); + assert!(commit.message.starts_with("third")); + + let tree_entries = bare.find_tree(tree).unwrap(); + let names: Vec<&str> = tree_entries + .entries + .iter() + .map(|entry| entry.name.as_str()) + .collect(); + assert!(names.contains(&"a.txt")); + assert!(names.contains(&"b.txt")); + assert!( + tree_entries + .entries + .iter() + .all(|entry| entry.kind == EntryKind::Blob) + ); + + assert_eq!(bare.read_blob(blob_b).unwrap(), b"two\n"); + + let changes = bare.diff(CommitRange { base: root, head }).unwrap(); + assert!(changes.iter().any( + |change| matches!(change, FileChange::Added { path, .. } if path.as_str() == "b.txt") + )); + assert!(changes.iter().any( + |change| matches!(change, FileChange::Modified { path, .. } if path.as_str() == "a.txt") + )); + + let comparison = bare.compare(CommitRange { base: root, head }).unwrap(); + assert_eq!(comparison.commits.len(), 2); + assert!(comparison.commits.contains(&head)); + assert!(comparison.commits.contains(&parent)); + assert_eq!(comparison.changes, changes); +} + +fn seed_main() -> (tempfile::TempDir, Layout, RepoDid, std::path::PathBuf, Oid) { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare_path = layout.repo_path(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git(work, &["init", "-q", "-b", "main"]); + commit_file(work, "a.txt", "one\n", "first"); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + (scan, layout, did, bare_path, head) +} + +fn seed_rich() -> (tempfile::TempDir, tempfile::TempDir, Layout, RepoDid) { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare_path = layout.repo_path(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git(work, &["init", "-q", "-b", "main"]); + commit_file(work, "a.txt", "one\ntwo\nthree\n", "first"); + std::fs::create_dir_all(work.join("src")).unwrap(); + commit_file(work, "src/lib.rs", "pub fn nel() {}\n", "add lib"); + commit_file(work, "a.txt", "one\ntwo\nthree\nfour\n", "extend a"); + git(work, &["tag", "light"]); + git(work, &["tag", "-a", "v1.0.0", "-m", "release one"]); + commit_file( + work, + "src/lib.rs", + "pub fn nel() {}\npub fn teq() {}\n", + "extend lib", + ); + git( + work, + &["push", "-q", "--tags", bare_path.to_str().unwrap(), "main"], + ); + git( + bare_path.as_path(), + &["symbolic-ref", "HEAD", "refs/heads/main"], + ); + (scan, work_dir, layout, did) +} + +#[test] +fn typed_reads_over_a_rich_repo() { + let (_scan, work_dir, layout, did) = seed_rich(); + let work = work_dir.path(); + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let parent = Oid::from_hex(&git(work, &["rev-parse", "HEAD~1"])).unwrap(); + let base = Oid::from_hex(&git(work, &["rev-parse", "HEAD~2"])).unwrap(); + let root = Oid::from_hex(&git(work, &["rev-parse", "HEAD~3"])).unwrap(); + + assert_eq!(bare.resolve_revision("main"), Some(head)); + assert_eq!(bare.resolve_revision("HEAD"), Some(head)); + assert_eq!(bare.resolve_revision(&head.to_hex()), Some(head)); + assert_eq!(bare.resolve_revision("does-not-exist"), None); + let tag_object = bare.resolve_revision("v1.0.0").unwrap(); + assert_eq!( + bare.peel_to_commit(tag_object).unwrap(), + Oid::from_hex(&git(work, &["rev-parse", "v1.0.0^{commit}"])).unwrap(), + "annotated tag peels to its commit" + ); + + let expected: Vec = git(work, &["rev-list", "HEAD"]) + .lines() + .map(str::to_string) + .collect(); + let (walked, total) = bare + .log_window(head, LogSkip::new(0), LogLimit::new(100)) + .unwrap(); + let walked: Vec = walked.iter().map(|commit| commit.id.to_hex()).collect(); + assert_eq!(walked, expected, "log order matches git rev-list"); + assert_eq!(total, expected.len()); + + let (page, total) = bare + .log_window(head, LogSkip::new(1), LogLimit::new(2)) + .unwrap(); + assert_eq!(page.len(), 2); + assert_eq!(page[0].id.to_hex(), expected[1]); + assert_eq!(page[1].id.to_hex(), expected[2]); + assert_eq!(total, expected.len(), "window still reports full count"); + + let between: Vec = bare + .commits_between(CommitRange { base, head }, LogLimit::new(100)) + .unwrap() + .iter() + .map(|oid| oid.to_hex()) + .collect(); + let expected_between: Vec = + git(work, &["rev-list", &format!("{}..HEAD", base.to_hex())]) + .lines() + .map(str::to_string) + .collect(); + assert_eq!(between, expected_between); + assert_eq!( + bare.commits_between(CommitRange { base, head }, LogLimit::new(1)) + .unwrap() + .len(), + 1, + "walk stops at limit instead of collecting full range" + ); + assert_eq!(bare.merge_base(head, base).unwrap(), Some(base)); + + let commit = bare.find_commit(head).unwrap(); + assert_eq!(commit.author.name.as_str(), "nel"); + assert!(commit.pgp_signature.is_none()); + assert!(commit.extra_headers.is_empty()); + assert!(commit.change_id().is_none()); + + let branches = bare.branch_list().unwrap(); + assert_eq!(branches.len(), 1); + assert_eq!(branches[0].name.as_str(), "main"); + assert_eq!(branches[0].tip.id(), head); + assert!(matches!(branches[0].tip, knot_git::BranchTip::Commit(_))); + + let tags = bare.tag_list().unwrap(); + assert_eq!(tags.len(), 2); + let light = tags + .iter() + .find(|tag| tag.name.as_str() == "light") + .unwrap(); + assert!(light.annotated.is_none()); + assert!(light.message.starts_with("extend a")); + let annotated = tags + .iter() + .find(|tag| tag.name.as_str() == "v1.0.0") + .unwrap(); + let detail = annotated.annotated.as_ref().unwrap(); + assert_eq!(annotated.message, "release one\n"); + assert_eq!(detail.tagger.as_ref().unwrap().name.as_str(), "nel"); + assert_eq!( + detail.target, + Oid::from_hex(&git(work, &["rev-parse", "v1.0.0^{commit}"])).unwrap() + ); + assert_eq!( + annotated.id, + Oid::from_hex(&git(work, &["rev-parse", "v1.0.0"])).unwrap(), + "tag info id is tag object itself" + ); + + let tree_root = bare.tree_entries_at(head, None).unwrap().unwrap(); + let names: Vec<&str> = tree_root.iter().map(|entry| entry.name.as_str()).collect(); + assert_eq!(names, vec!["a.txt", "src"]); + let a = tree_root + .iter() + .find(|entry| entry.name == "a.txt") + .unwrap(); + assert_eq!(a.size, "one\ntwo\nthree\nfour\n".len() as u64); + assert_eq!(a.kind.mode_octal(), "0100644"); + let src = tree_root.iter().find(|entry| entry.name == "src").unwrap(); + assert_eq!(src.kind, EntryKind::Tree); + assert_eq!(src.size, 0); + + let sub = bare + .tree_entries_at(head, Some(&rp("src"))) + .unwrap() + .unwrap(); + assert_eq!(sub.len(), 1); + assert_eq!(sub[0].name, "lib.rs"); + assert_eq!( + bare.tree_entries_at(head, Some(&rp("a.txt"))) + .unwrap() + .unwrap(), + Vec::new(), + "file path lists as empty" + ); + assert!( + bare.tree_entries_at(head, Some(&rp("missing"))) + .unwrap() + .is_none() + ); + assert!(RepoPath::new("../escape").is_err()); + + let entry = bare.entry_at(head, &rp("src/lib.rs")).unwrap().unwrap(); + assert_eq!( + entry.oid, + Oid::from_hex(&git(work, &["rev-parse", "HEAD:src/lib.rs"])).unwrap() + ); + + let deadline = Some(std::time::Instant::now() + std::time::Duration::from_secs(10)); + let names: Vec = tree_root.iter().map(|entry| entry.name.clone()).collect(); + let attributed = bare.last_commits(head, None, &names, deadline).unwrap(); + assert_eq!( + attributed["a.txt"].id.to_hex(), + git(work, &["log", "-1", "--format=%H", "--", "a.txt"]) + ); + assert_eq!( + attributed["src"].id.to_hex(), + git(work, &["log", "-1", "--format=%H", "--", "src"]) + ); + assert_eq!(attributed["a.txt"].subject, "extend a"); + let nested = bare + .last_commits(head, Some(&rp("src")), &["lib.rs".to_string()], deadline) + .unwrap(); + assert_eq!( + nested["lib.rs"].id.to_hex(), + git(work, &["log", "-1", "--format=%H", "--", "src/lib.rs"]) + ); + + let patches = patches_of(&bare, Some(parent), head, &mut BinaryBudget::Omit); + assert_eq!(patches.len(), 1); + let patch = &patches[0]; + assert_eq!(patch.path.as_str(), "src/lib.rs"); + assert_eq!(patch.status, knot_git::PatchStatus::Modified); + assert!(!patch.is_binary()); + assert_eq!(patch.hunks().len(), 1); + let hunk = &patch.hunks()[0]; + assert_eq!( + ( + hunk.old_start.get(), + hunk.old_lines.get(), + hunk.new_start.get(), + hunk.new_lines.get() + ), + (1, 1, 1, 2) + ); + assert_eq!(hunk.added(), LineCount::new(1)); + assert_eq!(hunk.deleted(), LineCount::new(0)); + assert_eq!( + hunk.lines + .iter() + .map(|line| String::from_utf8_lossy(&line.text).into_owned()) + .collect::>(), + vec!["pub fn nel() {}\n", "pub fn teq() {}\n"] + ); + + let initial = patches_of(&bare, None, root, &mut BinaryBudget::Omit); + assert_eq!(initial.len(), 1); + assert_eq!(initial[0].status, knot_git::PatchStatus::Added); + assert_eq!( + initial[0].hunks()[0].old_start.get(), + 0, + "added file hunk starts at -0,0" + ); + assert_eq!(initial[0].hunks()[0].old_lines.get(), 0); + + let tag_commit = bare.peel_to_commit(tag_object).unwrap(); + assert_eq!( + bare.changed_paths(knot_git::PatchRange { + base: None, + head: tag_object, + }) + .unwrap(), + bare.changed_paths(knot_git::PatchRange { + base: None, + head: tag_commit, + }) + .unwrap(), + "an annotated tag peels to its commit before the trees are diffed" + ); + let created = bare + .changed_paths(knot_git::PatchRange { base: None, head }) + .unwrap(); + assert_eq!( + created.paths(), + [rp("a.txt"), rp("src/lib.rs")], + "a ref creation lists every blob in the tree and no directory of them" + ); + assert_eq!(created.listing(), Listing::Complete); +} + +type TopoRow = (fn(&Path, &Oid), fn(&Repo, &Oid)); + +#[test] +fn ref_topology_reads_are_total() { + let rows: &[TopoRow] = &[ + ( + |bare, _head| { + git(bare, &["update-ref", "-d", "refs/heads/main"]); + }, + |bare, _head| { + assert!(bare.head().is_none()); + assert_eq!(bare.default_branch().unwrap().as_str(), "refs/heads/main"); + assert!(bare.references().unwrap().is_empty()); + assert!(bare.branches().unwrap().is_empty()); + assert!(bare.tags().unwrap().is_empty()); + assert!(bare.advertised_refs().unwrap().is_empty()); + }, + ), + ( + |bare, head| { + git(bare, &["update-ref", "--no-deref", "HEAD", &head.to_hex()]); + }, + |bare, head| { + assert!(bare.head().is_none()); + assert!(bare.default_branch().is_none()); + let branches = bare.branches().unwrap(); + assert_eq!(branches.len(), 1); + assert_eq!(branches[0].target, *head); + assert_eq!( + bare.find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap(), + Some(*head) + ); + }, + ), + ( + |bare, _head| { + git(bare, &["symbolic-ref", "HEAD", "refs/heads/nursery"]); + }, + |bare, _head| { + assert!(bare.head().is_none()); + assert_eq!( + bare.default_branch().unwrap().as_str(), + "refs/heads/nursery" + ); + let branches = bare.branches().unwrap(); + assert_eq!(branches.len(), 1); + assert_eq!(branches[0].name.as_str(), "refs/heads/main"); + }, + ), + ( + |bare, _head| { + git( + bare, + &["symbolic-ref", "refs/heads/mirror", "refs/heads/gone"], + ); + }, + |bare, head| { + let refs = bare.references().unwrap(); + assert!( + refs.iter() + .all(|record| record.name.as_str() != "refs/heads/mirror"), + "symref to missing target must be dropped, not panic or resolve" + ); + assert!( + refs.iter() + .any(|record| record.name.as_str() == "refs/heads/main" + && record.target == *head) + ); + assert_eq!( + bare.find_ref(&RefName::new("refs/heads/mirror").unwrap()) + .unwrap(), + None + ); + }, + ), + ( + |bare, _head| { + git(bare, &["pack-refs", "--all"]); + assert!(!bare.join("refs/heads/main").exists()); + assert!(bare.join("packed-refs").exists()); + }, + |bare, head| { + let refs = bare.references().unwrap(); + assert!( + refs.iter() + .any(|record| record.name.as_str() == "refs/heads/main" + && record.target == *head) + ); + assert_eq!( + bare.find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap(), + Some(*head) + ); + }, + ), + ]; + + rows.iter().for_each(|(setup, check)| { + let (_scan, layout, did, bare_path, head) = seed_main(); + setup(bare_path.as_path(), &head); + let bare = layout.open(&did).unwrap(); + check(&bare, &head); + }); +} + +#[test] +fn reachable_from_public_excludes_cob_only_commits() { + let (_scan, work_dir, layout, did) = seed_rich(); + let work = work_dir.path(); + let bare_path = layout.repo_path(&did).unwrap(); + let bare = layout.open(&did).unwrap(); + + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let ancestor = Oid::from_hex(&git(work, &["rev-parse", "HEAD~2"])).unwrap(); + let tag_commit = Oid::from_hex(&git(work, &["rev-parse", "v1.0.0^{commit}"])).unwrap(); + assert!(bare.reachable_from_public(head).unwrap()); + assert!(bare.reachable_from_public(ancestor).unwrap()); + assert!(bare.reachable_from_public(tag_commit).unwrap()); + + commit_file(work, "hidden.txt", "secret\n", "hidden"); + let hidden = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + git( + work, + &[ + "push", + "-q", + bare_path.to_str().unwrap(), + "HEAD:refs/cobs/org.tangled.graph.collaborator/secret", + ], + ); + + let bare = layout.open(&did).unwrap(); + assert!(bare.contains(hidden), "object lives in odb"); + assert!( + !bare.reachable_from_public(hidden).unwrap(), + "commit held only by cob ref isn't reachable from any public ref" + ); +} + +#[test] +fn a_branch_tipped_by_a_tag_object_lists_opaquely() { + let (_scan, work_dir, layout, did) = seed_rich(); + let work = work_dir.path(); + let bare_path = layout.repo_path(&did).unwrap(); + let tag_object = git(work, &["rev-parse", "v1.0.0"]); + std::fs::write( + bare_path.join("refs/heads/tagtip"), + format!("{tag_object}\n"), + ) + .unwrap(); + + let bare = layout.open(&did).unwrap(); + let branches = bare.branch_list().unwrap(); + assert_eq!(branches.len(), 2); + let tagtip = branches + .iter() + .find(|branch| branch.name.as_str() == "tagtip") + .unwrap(); + match &tagtip.tip { + knot_git::BranchTip::Opaque { + id, + message, + created_at, + } => { + assert_eq!(*id, Oid::from_hex(&tag_object).unwrap()); + assert_eq!(message, "release one\n"); + assert!(created_at.get() > 0, "annotated tag records tagger time"); + } + other => panic!("expected opaque tip, got {other:?}"), + } +} + +#[test] +fn extended_history_reads() { + let (_scan, work_dir, layout, did) = seed_rich(); + let work = work_dir.path(); + let bare_path = layout.repo_path(&did).unwrap(); + + let seed_head = git(work, &["rev-parse", "HEAD"]); + git( + work, + &[ + "update-index", + "--add", + "--cacheinfo", + &format!("160000,{seed_head},vendor/dep"), + ], + ); + git(work, &["commit", "-q", "-m", "add gitlink"]); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let bare = layout.open(&did).unwrap(); + let linked = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + assert!( + bare.tree_entries_at(linked, Some(&rp("vendor/dep"))) + .unwrap() + .is_none(), + "submodule path isn't found" + ); + assert!( + bare.tree_entries_at(linked, Some(&rp("vendor"))) + .unwrap() + .is_some(), + "directory holding gitlink still lists" + ); + + commit_file( + work, + ".gitmodules", + "# top comment\n[submodule \"kelp\"]\n\tpath = libs/kelp ; trailing comment\n\turl = \"https://oyster.cafe/kelp.git\"\n\tbranch = main\n[submodule \"whelk\"]\n\tpath = libs/whelk\n\turl = https://nel.pet/whelk.git # mirror\n", + "add submodules", + ); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let bare = layout.open(&did).unwrap(); + let with_mods = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let submodules = bare.submodules(with_mods).unwrap(); + assert_eq!(submodules.len(), 2); + assert_eq!(submodules[0].name, "kelp"); + assert_eq!(submodules[0].path.as_str(), "libs/kelp"); + assert_eq!(submodules[0].url.as_str(), "https://oyster.cafe/kelp.git"); + assert_eq!( + submodules[0].branch, + Some(knot_types::BranchName::new("main").unwrap()) + ); + assert_eq!(submodules[1].branch, None); + + std::fs::write(work.join("blob.bin"), [0u8, 159, 146, 150, 0, 1]).unwrap(); + std::fs::write(work.join("noeol.txt"), "no newline at end").unwrap(); + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", "binary and noeol"]); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let parent = Oid::from_hex(&git(work, &["rev-parse", "HEAD~1"])).unwrap(); + let patches = patches_of(&bare, Some(parent), head, &mut BinaryBudget::new(1024)); + let blob = binary_diff(named(&patches, "blob.bin")); + assert!( + matches!(blob, BinaryDiff::Encoded { .. }), + "a blob under the budget is encoded" + ); + let mut thin = BinaryBudget::new(5); + let thinned = patches_of(&bare, Some(parent), head, &mut thin); + let starved = binary_diff(named(&thinned, "blob.bin")); + assert!( + matches!(starved, BinaryDiff::Omitted(_)), + "a blob past the budget is omitted" + ); + assert!(thin.omitted(), "omitted is true once a blob is left out"); + assert_eq!( + [blob.sizes(), starved.sizes()].map(|sizes| (sizes.old, sizes.new)), + [(0, 6); 2], + "a new file has an empty pre-image, encoded or omitted" + ); + let last = named(&patches, "noeol.txt").hunks()[0] + .lines + .last() + .unwrap(); + assert_eq!(last.text, b"no newline at end".to_vec()); +} + +#[test] +fn archives_round_trip_through_tar() { + let (_scan, work_dir, layout, did) = seed_rich(); + let work = work_dir.path(); + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let tree = bare.peel_to_tree(head).unwrap(); + + let mut out = std::io::Cursor::new(Vec::new()); + bare.write_archive( + tree, + knot_git::ArchiveFormat::TarGz, + Some(&knot_git::TreePrefix::new("squid-main/").unwrap()), + knot_git::ArchiveLimit::new(u64::MAX), + &mut out, + ) + .unwrap(); + let compressed = out.into_inner(); + assert_eq!( + &compressed[..2], + &[0x1f, 0x8b], + "tar.gz starts with gzip magic" + ); + + let mut decoder = flate2::read::GzDecoder::new(compressed.as_slice()); + let mut tar = Vec::new(); + std::io::Read::read_to_end(&mut decoder, &mut tar).unwrap(); + assert!( + contains(&tar, b"squid-main/src/lib.rs"), + "tar contains prefixed entries" + ); +} + +#[test] +fn an_archive_stops_at_its_limit_instead_of_spooling_the_whole_tree() { + let (_scan, work_dir, layout, did) = seed_rich(); + let work = work_dir.path(); + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let tree = bare.peel_to_tree(head).unwrap(); + + [ + knot_git::ArchiveFormat::Tar, + knot_git::ArchiveFormat::TarGz, + knot_git::ArchiveFormat::Zip, + ] + .iter() + .for_each(|format| { + let mut out = std::io::Cursor::new(Vec::new()); + let refused = bare.write_archive( + tree, + *format, + None, + knot_git::ArchiveLimit::new(512), + &mut out, + ); + assert!( + matches!(refused, Err(knot_git::GitError::ArchiveTooLarge { .. })), + "a {format:?} archive past its limit must be refused, got {refused:?}" + ); + assert!( + out.into_inner().len() <= 512, + "the {format:?} writer took bytes past the limit before the refusal" + ); + }); +} + +#[test] +fn a_compressible_tree_is_measured_before_the_compressor_ever_sees_it() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:whelk").unwrap(); + layout.create(&did).unwrap(); + let bare_path = layout.repo_path(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git(work, &["init", "-q", "-b", "main"]); + commit_file( + work, + "kelp.txt", + &"kelp\n".repeat(200_000), + "one very compressible blob", + ); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let tree = bare.peel_to_tree(head).unwrap(); + + let mut out = std::io::Cursor::new(Vec::new()); + let refused = bare.write_archive( + tree, + knot_git::ArchiveFormat::TarGz, + None, + knot_git::ArchiveLimit::new(64 * 1024), + &mut out, + ); + assert!( + matches!(refused, Err(knot_git::GitError::ArchiveTooLarge { .. })), + "a tree that gzips under the limit still costs its full size to read, so it must be refused, got {refused:?}" + ); + assert!( + out.into_inner().is_empty(), + "the knot must refuse before the compressor writes a byte" + ); +} + +#[test] +fn a_filename_with_a_backslash_is_addressable() { + let (_scan, layout, did, bare_path, _head) = seed_main(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git( + work, + &["clone", "-q", bare_path.to_str().unwrap(), "checkout"], + ); + let clone = work.join("checkout"); + commit_file(&clone, "back\\slash.txt", "escaped\n", "backslash name"); + git(&clone, &["push", "-q", "origin", "main"]); + + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(&clone, &["rev-parse", "HEAD"])).unwrap(); + let entry = bare + .entry_at(head, &rp("back\\slash.txt")) + .unwrap() + .expect("backslash in filename is legal and addressable"); + assert_eq!(bare.read_blob(entry.oid).unwrap(), b"escaped\n"); +} + +#[test] +fn an_oversized_blob_diffs_as_binary_without_loading_it() { + let (_scan, layout, did, bare_path, _head) = seed_main(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git( + work, + &["clone", "-q", bare_path.to_str().unwrap(), "checkout"], + ); + let clone = work.join("checkout"); + let oversized = vec![b'a'; (knot_git::MAX_DIFF_BLOB_BYTES + 1) as usize]; + std::fs::write(clone.join("huge.txt"), &oversized).unwrap(); + git(&clone, &["add", "-A"]); + git(&clone, &["commit", "-q", "-m", "huge text file"]); + git(&clone, &["push", "-q", "origin", "main"]); + + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(&clone, &["rev-parse", "HEAD"])).unwrap(); + let parent = Oid::from_hex(&git(&clone, &["rev-parse", "HEAD~1"])).unwrap(); + let patches = patches_of(&bare, Some(parent), head, &mut BinaryBudget::new(u64::MAX)); + let huge = named(&patches, "huge.txt"); + assert!( + huge.is_binary() && huge.hunks().is_empty(), + "blob past diff budget falls back to binary instead of being loaded" + ); + assert!( + matches!(binary_diff(huge), BinaryDiff::Omitted(_)), + "a blob nobody read has no bytes to encode, whatever the budget allows" + ); + assert_eq!( + binary_diff(huge).sizes().new, + knot_git::MAX_DIFF_BLOB_BYTES + 1, + "the size comes from the blob header the knot did read" + ); +} + +#[test] +fn blame_object_limit_applies_to_large_history_loose_and_packed() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:blamelimit").unwrap(); + let bare = layout.create(&did).unwrap(); + let bare_path = layout.repo_path(&did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git(work, &["init", "-q", "-b", "main"]); + commit_file(work, "exact.txt", &"x\n".repeat(512 * 1024), "exact"); + commit_file(work, "large.txt", &"x\n".repeat(1024 * 1024), "large"); + commit_file(work, "exact.txt", "x\nsentinel\n", "shrink exact"); + commit_file(work, "large.txt", "x\nsentinel\n", "shrink large"); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let exact = rp("exact.txt"); + let large = rp("large.txt"); + let check = |repo: &Repo| { + let far_future = Instant::now() + Duration::from_secs(3600); + assert!( + repo.blame_file_bounded(head, &exact, 1024 * 1024, far_future) + .is_ok() + ); + let result = repo.blame_file_bounded(head, &large, 1024 * 1024, far_future); + assert!( + matches!( + result, + Err(GitError::BlameObjectTooLarge { limit }) if limit == 1024 * 1024 + ), + "{result:#?}" + ); + let timed_out = repo.blame_file_bounded(head, &exact, 1024 * 1024, Instant::now()); + assert!( + matches!(timed_out, Err(GitError::BlameTimedOut)), + "{timed_out:#?}" + ); + }; + + check(&bare); + drop(bare); + git(&bare_path, &["gc", "--prune=now"]); + check(&layout.open(&did).unwrap()); +} diff --git a/knot2/crates/knot-index/Cargo.toml b/knot2/crates/knot-index/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..083bcb539c3227d73037df497fcd9247bad59194 --- /dev/null +++ b/knot2/crates/knot-index/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "knot-index" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[features] +test-support = [] + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-cob = { workspace = true } +knot-resource = { workspace = true } +knot-cobs = { workspace = true } +scc = { workspace = true } +lasso = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +knot-runtime = { workspace = true } +knot-record = { workspace = true } +tempfile = { workspace = true } +proptest = { workspace = true } +serde = { workspace = true } +serde_ipld_dagcbor = { workspace = true } diff --git a/knot2/crates/knot-index/src/coverage.rs b/knot2/crates/knot-index/src/coverage.rs new file mode 100644 index 0000000000000000000000000000000000000000..d9049f2606f3ae27790c32be68b0dbed4e9ec71b --- /dev/null +++ b/knot2/crates/knot-index/src/coverage.rs @@ -0,0 +1,70 @@ +use std::sync::atomic::{AtomicU8, Ordering}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Coverage { + Warming, + Ready, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Resolved { + Warming, + Ready(T), +} + +impl Resolved { + pub fn is_warming(&self) -> bool { + matches!(self, Resolved::Warming) + } + + pub fn map(self, f: impl FnOnce(T) -> U) -> Resolved { + match self { + Resolved::Ready(value) => Resolved::Ready(f(value)), + Resolved::Warming => Resolved::Warming, + } + } + + pub fn ready_or_default(self) -> T + where + T: Default, + { + match self { + Resolved::Ready(value) => value, + Resolved::Warming => T::default(), + } + } +} + +const WARMING: u8 = 0; +const READY: u8 = 1; + +#[derive(Debug)] +pub(crate) struct CoverageCell(AtomicU8); + +impl CoverageCell { + pub(crate) fn new(initial: Coverage) -> Self { + Self(AtomicU8::new(encode(initial))) + } + + pub(crate) fn get(&self) -> Coverage { + decode(self.0.load(Ordering::Acquire)) + } + + pub(crate) fn set(&self, coverage: Coverage) { + self.0.store(encode(coverage), Ordering::Release); + } +} + +fn encode(coverage: Coverage) -> u8 { + match coverage { + Coverage::Warming => WARMING, + Coverage::Ready => READY, + } +} + +fn decode(raw: u8) -> Coverage { + match raw { + READY => Coverage::Ready, + _ => Coverage::Warming, + } +} diff --git a/knot2/crates/knot-index/src/error.rs b/knot2/crates/knot-index/src/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..69928ea9fc7be55a9e0dbc1ea2044863adc3a6d2 --- /dev/null +++ b/knot2/crates/knot-index/src/error.rs @@ -0,0 +1,25 @@ +use knot_cob::CobError; +use knot_git::GitError; +use knot_types::{ChangeId, TypeName}; + +#[derive(Debug, thiserror::Error)] +pub enum IndexError { + #[error(transparent)] + Cob(#[from] CobError), + #[error(transparent)] + Git(#[from] GitError), + #[error("expected at most one {type_name} object, found {count}")] + Ambiguous { type_name: TypeName, count: usize }, + #[error("change {change} in {type_name} projection does not decode: {reason}")] + Decode { + change: ChangeId, + type_name: TypeName, + reason: String, + }, + #[error("change {change} is {found} change in {expected} projection")] + UnexpectedType { + change: ChangeId, + expected: TypeName, + found: TypeName, + }, +} diff --git a/knot2/crates/knot-index/src/intern.rs b/knot2/crates/knot-index/src/intern.rs new file mode 100644 index 0000000000000000000000000000000000000000..9a9e761da8763b210705e23ba5fbde252f224b8a --- /dev/null +++ b/knot2/crates/knot-index/src/intern.rs @@ -0,0 +1,102 @@ +use std::sync::Arc; + +use knot_types::{AccountDid, OwnerDid, RecordCollection, RecordRkey, RepoDid, RepoName, RepoRkey}; +use lasso::{Spur, ThreadedRodeo}; + +#[derive(Debug, Clone, Default)] +pub(crate) struct Interner(Arc); + +impl Interner { + pub(crate) fn new() -> Self { + Self(Arc::new(ThreadedRodeo::new())) + } +} + +macro_rules! interned { + (@lookup $key:ident of $value:ty { $intern:ident, $get:ident }) => { + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub(crate) struct $key(Spur); + + impl Interner { + pub(crate) fn $intern(&self, value: &$value) -> $key { + $key(self.0.get_or_intern(value.as_str())) + } + + pub(crate) fn $get(&self, value: &$value) -> Option<$key> { + self.0.get(value.as_str()).map($key) + } + } + }; + ($( + $key:ident of $value:ty { + $intern:ident, $get:ident $(, $resolve:ident, $label:literal)? + } + )+) => {$( + interned!(@lookup $key of $value { $intern, $get }); + + $( + impl Interner { + pub(crate) fn $resolve(&self, key: $key) -> $value { + <$value>::new(self.0.resolve(&key.0)) + .expect(concat!("Interned ", $label, " is valid ", $label)) + } + } + )? + )+}; +} + +interned! { + AccountKey of AccountDid { intern_account, account, resolve_account, "account DID" } + RepoKey of RepoDid { intern_repo, repo, resolve_repo, "repo DID" } + OwnerKey of OwnerDid { intern_owner, owner, resolve_owner, "owner DID" } + RkeyKey of RepoRkey { intern_rkey, rkey, resolve_rkey, "rkey" } + NameKey of RepoName { intern_name, name } + CollectionKey of RecordCollection { intern_collection, collection, resolve_collection, "record collection" } +} + +const TID_BYTES: usize = 13; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) enum RecordKey { + Tid([u8; TID_BYTES]), + Slug(Spur), +} + +fn tid_bytes(value: &RecordRkey) -> Option<[u8; TID_BYTES]> { + let tid_shaped = match value { + RecordRkey::Minted(_) => true, + RecordRkey::Composed(key) => { + matches!(RecordRkey::new(key.as_str()), Ok(RecordRkey::Minted(_))) + } + RecordRkey::Subject(_) => false, + }; + tid_shaped.then(|| { + <[u8; TID_BYTES]>::try_from(value.as_str().as_bytes()) + .expect("A TID-shaped record key is thirteen bytes") + }) +} + +impl Interner { + pub(crate) fn key_of(&self, value: &RecordRkey) -> RecordKey { + match tid_bytes(value) { + Some(bytes) => RecordKey::Tid(bytes), + None => RecordKey::Slug(self.0.get_or_intern(value.as_str())), + } + } + + pub(crate) fn known_key(&self, value: &RecordRkey) -> Option { + match tid_bytes(value) { + Some(bytes) => Some(RecordKey::Tid(bytes)), + None => self.0.get(value.as_str()).map(RecordKey::Slug), + } + } + + pub(crate) fn key_str(&self, key: RecordKey) -> String { + match key { + RecordKey::Tid(bytes) => std::str::from_utf8(&bytes) + .expect("Record key bytes are TID ASCII") + .to_owned(), + RecordKey::Slug(spur) => self.0.resolve(&spur).to_owned(), + } + } +} diff --git a/knot2/crates/knot-index/src/lib.rs b/knot2/crates/knot-index/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..ee07b2a69f47de1d087e850ecb657cf4371ae025 --- /dev/null +++ b/knot2/crates/knot-index/src/lib.rs @@ -0,0 +1,959 @@ +mod coverage; +mod error; +mod intern; +mod projections; +mod social; + +pub use coverage::{Coverage, Resolved}; +pub use error::IndexError; +pub use social::{SocialEntry, SocialEntryState, SocialGuard, SocialResidencyBytes}; + +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use knot_cob::{ChangePayload, CobStore, Evaluate}; +pub use knot_cobs::Entry; +use knot_cobs::{BlocklistCob, CollaboratorsCob, MembersCob, RepoRef, RepoRegistryCob}; +use knot_git::{Layout, Repo}; +use knot_resource::StructureBytes; +use knot_types::{ + AccountDid, ClonePath, CobId, OfferedKey, OwnerDid, RecordAddress, RecordCollection, RepoDid, + RepoPolicy, RepoRkey, SubjectNumber, TypeName, UnixSeconds, +}; +use tokio::sync::watch; + +use intern::{Interner, RepoKey}; +use projections::{CollaboratorsProjection, GrantSetProjection, KeyProjection, RegistryProjection}; + +pub struct Answer<'a, S> { + scope: S, + subject: &'a AccountDid, + granted: bool, +} + +pub type MemberStatus<'a> = Answer<'a, ()>; +pub type CollaborationConsent<'a> = Answer<'a, &'a RepoDid>; + +impl<'a, S> Answer<'a, S> { + pub const fn subject(&self) -> &'a AccountDid { + self.subject + } + + pub const fn granted(&self) -> bool { + self.granted + } + + #[cfg(feature = "test-support")] + #[doc(hidden)] + pub const fn assumed(scope: S, subject: &'a AccountDid, granted: bool) -> Self { + Self { + scope, + subject, + granted, + } + } +} + +impl<'a> MemberStatus<'a> { + pub const fn unread(subject: &'a AccountDid) -> Self { + Self { + scope: (), + subject, + granted: false, + } + } +} + +impl<'a> CollaborationConsent<'a> { + pub const fn repo(&self) -> &'a RepoDid { + self.scope + } +} + +knot_types::scalar_newtype! { + pub struct IndexGeneration(u64); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IndexCoverage { + pub members: Coverage, + pub blocklist: Coverage, + pub collaborators: Coverage, + pub registry: Coverage, + pub keys: Coverage, +} + +macro_rules! account_list { + ($($name:ident),+ $(,)?) => {$( + #[derive(Debug, Clone, PartialEq, Eq)] + pub struct $name(Vec); + + impl $name { + pub fn new(accounts: Vec) -> Self { + Self(accounts) + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn as_slice(&self) -> &[AccountDid] { + &self.0 + } + + pub fn into_vec(self) -> Vec { + self.0 + } + } + )+}; +} + +account_list!( + Pushers, + KeptAccounts, + StalePushers, + SuspectPushers, + UnreadMembers, + StaleMembers, +); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HostedCoverage { + Whole, + Partial { unread: usize }, +} + +impl HostedCoverage { + const fn over(unread: usize) -> Self { + match unread { + 0 => Self::Whole, + unread => Self::Partial { unread }, + } + } +} + +struct Granted { + subjects: Vec, + hosted: HostedCoverage, +} + +enum RepoGrants { + Ready(Vec), + Pending, + Unreadable, +} + +impl RepoGrants { + fn into_accounts(self) -> Option> { + match self { + RepoGrants::Ready(accounts) => Some(accounts), + RepoGrants::Pending | RepoGrants::Unreadable => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SweepInterval(Duration); + +impl SweepInterval { + pub const DEFAULT: Self = Self::from_secs(60); + + pub const fn from_secs(secs: u64) -> Self { + Self(Duration::from_secs(secs)) + } + + pub const fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyTtl(Duration); + +impl KeyTtl { + pub const DEFAULT: Self = Self::from_secs(3_600); + + pub const fn from_secs(secs: u64) -> Self { + Self(Duration::from_secs(secs)) + } + + pub const fn get(self) -> Duration { + self.0 + } + + pub const fn longest(self, other: Self) -> Self { + match self.0.as_secs() >= other.0.as_secs() { + true => self, + false => other, + } + } + + pub const fn lease_from(self, now: UnixSeconds) -> KeyLease { + KeyLease { + read_at: now, + expires_at: after(now, self.0), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyLease { + read_at: UnixSeconds, + expires_at: UnixSeconds, +} + +impl KeyLease { + pub(crate) const fn is_live(self, now: UnixSeconds) -> bool { + self.expires_at.get() > now.get() + } + + pub(crate) const fn renewal_due(self, now: UnixSeconds) -> bool { + let held = self.expires_at.get().saturating_sub(self.read_at.get()); + now.get() >= self.read_at.saturating_add_secs(held / 2).get() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyReprieve { + retry: Duration, + budget: Duration, +} + +impl KeyReprieve { + pub const DEFAULT: Self = Self::from_secs(300, 21_600); + + pub const fn from_secs(retry: u64, budget: u64) -> Self { + Self { + retry: Duration::from_secs(retry), + budget: Duration::from_secs(budget), + } + } + + pub const fn budgeted_for(self, ttl: KeyTtl) -> Self { + match self.budget.as_secs() >= ttl.0.as_secs() { + true => self, + false => Self { + retry: self.retry, + budget: ttl.0, + }, + } + } + + pub(crate) const fn first_failure(self, now: UnixSeconds) -> KeyLease { + KeyLease { + read_at: now, + expires_at: after(now, self.retry), + } + } + + pub(crate) const fn extend(self, lease: KeyLease, now: UnixSeconds) -> Option { + let horizon = after(lease.read_at, self.budget); + match horizon.get() > now.get() { + false => None, + true => { + let retry = after(now, self.retry); + let granted = match retry.get() < horizon.get() { + true => retry, + false => horizon, + }; + Some(KeyLease { + read_at: lease.read_at, + expires_at: match lease.expires_at.get() > granted.get() { + true => lease.expires_at, + false => granted, + }, + }) + } + } + } +} + +pub(crate) const fn whole_secs(span: Duration) -> i64 { + let secs = span.as_secs(); + match secs > i64::MAX as u64 { + true => i64::MAX, + false => secs as i64, + } +} + +pub(crate) const fn after(now: UnixSeconds, span: Duration) -> UnixSeconds { + now.saturating_add_secs(whole_secs(span)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyRecord { + Stored, + Unheld, + Saturated, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyReprieved { + Extended, + Pending, + Exhausted, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyBudget(usize); + +impl KeyBudget { + pub const DEFAULT: Self = Self::from_mib(64); + + pub const fn from_mib(mib: usize) -> Self { + Self(mib * 1024 * 1024) + } + + pub const fn from_bytes(bytes: usize) -> Self { + Self(bytes) + } + + pub const fn get(self) -> usize { + self.0 + } +} + +pub struct MemberWork { + pub unread: UnreadMembers, + pub due: StaleMembers, + pub kept: KeptAccounts, +} + +pub struct KeyWork { + pub generation: IndexGeneration, + pub tracked: usize, + pub hosted: HostedCoverage, + pub pushers: Pushers, + pub due: StalePushers, + pub suspected: SuspectPushers, + pub complete: bool, + pub members: Resolved, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Recheck { + Renewals, + Everything, +} + +fn sorted(mut subjects: Vec) -> Vec { + subjects.sort(); + subjects.dedup(); + subjects +} + +pub struct Index { + meta_path: PathBuf, + layout: Layout, + interner: Interner, + members: GrantSetProjection, + blocklist: GrantSetProjection, + collaborators: CollaboratorsProjection, + registry: RegistryProjection, + social: social::SocialProjection, + keys: KeyProjection, + unreadable: scc::HashSet, + generation: AtomicU64, + generations: watch::Sender, +} + +impl Index { + pub fn new(meta_path: impl Into, layout: Layout) -> Self { + Self::with_key_budget(meta_path, layout, KeyBudget::DEFAULT) + } + + pub fn with_key_budget( + meta_path: impl Into, + layout: Layout, + budget: KeyBudget, + ) -> Self { + Self { + meta_path: meta_path.into(), + layout, + interner: Interner::new(), + members: GrantSetProjection::new(), + blocklist: GrantSetProjection::new(), + collaborators: CollaboratorsProjection::new(), + registry: RegistryProjection::new(), + social: social::SocialProjection::new(SocialResidencyBytes::DEFAULT), + keys: KeyProjection::new(budget), + unreadable: scc::HashSet::new(), + generation: AtomicU64::new(0), + generations: watch::Sender::new(IndexGeneration::new(0)), + } + } + + pub fn with_social_residency(mut self, residency: SocialResidencyBytes) -> Self { + self.social = social::SocialProjection::new(residency); + self + } + + pub fn generation(&self) -> IndexGeneration { + IndexGeneration(self.generation.load(Ordering::Acquire)) + } + + pub fn generations(&self) -> watch::Receiver { + self.generations.subscribe() + } + + fn bump_generation(&self) { + self.generation.fetch_add(1, Ordering::Release); + self.generations.send_replace(self.generation()); + } + + pub fn rebuild(&self) -> Result<(), IndexError> { + self.refresh_members()?; + self.refresh_blocklist()?; + self.refresh_registry()?; + Ok(()) + } + + pub fn warm_collaborators(&self) -> usize { + self.hosted_repos() + .iter() + .filter(|repo| match self.ensure_collaborators(repo) { + Ok(()) => false, + Err(error) => { + tracing::warn!( + repo = repo.as_str(), + %error, + "Collaborators unread, the knot can't open the repo" + ); + true + } + }) + .count() + } + + fn unreadable_repo(&self, repo: &RepoDid) -> bool { + self.interner + .repo(repo) + .is_some_and(|repo| self.unreadable.contains_sync(&repo)) + } + + fn sole_cob(store: &CobStore) -> Result, IndexError> { + match store.list::()?.as_slice() { + [] => Ok(None), + [object] => Ok(Some(*object)), + many => Err(IndexError::Ambiguous { + type_name: E::Change::type_name(), + count: many.len(), + }), + } + } + + pub fn refresh_members(&self) -> Result<(), IndexError> { + let meta = Repo::open(&self.meta_path)?; + let store = CobStore::new(&meta); + match Self::sole_cob::(&store)? { + None => self.members.reset(), + Some(object) => self.members.refresh(&self.interner, &store, object)?, + } + self.bump_generation(); + Ok(()) + } + + pub fn refresh_blocklist(&self) -> Result<(), IndexError> { + let meta = Repo::open(&self.meta_path)?; + let store = CobStore::new(&meta); + match Self::sole_cob::(&store)? { + None => self.blocklist.reset(), + Some(object) => self.blocklist.refresh(&self.interner, &store, object)?, + } + self.bump_generation(); + Ok(()) + } + + pub fn refresh_registry(&self) -> Result<(), IndexError> { + self.registry_refresh() + .inspect_err(|_| self.registry.warm()) + } + + fn registry_refresh(&self) -> Result<(), IndexError> { + let meta = Repo::open(&self.meta_path)?; + let store = CobStore::new(&meta); + let evacuated = match Self::sole_cob::(&store)? { + None => self.registry.reset(&self.interner), + Some(object) => self.registry.refresh(&self.interner, &store, object)?, + }; + evacuated.iter().for_each(|repo| { + if let Some(key) = self.interner.repo(repo) { + self.collaborators.drop_repo(key); + self.social.drop_repo(key); + self.unreadable.remove_sync(&key); + } + }); + self.bump_generation(); + Ok(()) + } + + pub fn ensure_collaborators(&self, repo: &RepoDid) -> Result<(), IndexError> { + if self.collaborators.is_materialized(&self.interner, repo) { + return Ok(()); + } + self.refresh_collaborators(repo) + } + + pub fn refresh_collaborators(&self, repo: &RepoDid) -> Result<(), IndexError> { + let repo_key = self.interner.intern_repo(repo); + self.materialize_collaborators(repo, repo_key) + .inspect(|()| { + self.unreadable.remove_sync(&repo_key); + }) + .inspect_err(|_| { + let _ = self.unreadable.insert_sync(repo_key); + }) + } + + fn materialize_collaborators( + &self, + repo: &RepoDid, + repo_key: RepoKey, + ) -> Result<(), IndexError> { + let git = self.layout.open(repo)?; + let store = CobStore::new(&git); + match Self::sole_cob::(&store)? { + None => self.collaborators.mark_repo_empty(repo_key), + Some(object) => { + self.collaborators + .refresh_repo(&self.interner, &store, repo_key, object)? + } + } + self.bump_generation(); + Ok(()) + } + + pub fn ensure_social(&self, repo: &RepoDid) -> Result<(), IndexError> { + if self.social.is_materialized(&self.interner, repo) { + return Ok(()); + } + let git = self.layout.open(repo)?; + let store = CobStore::new(&git); + let repo_key = self.interner.intern_repo(repo); + if self.social.ensure_repo(&self.interner, &store, repo_key)? { + self.bump_generation(); + } + Ok(()) + } + + pub fn guard_social(&self, repo: &RepoDid) -> Result, IndexError> { + let guard = self.social.guard(self.interner.intern_repo(repo)); + self.ensure_social(repo).map(|()| guard) + } + + pub fn refresh_social(&self, repo: &RepoDid) -> Result<(), IndexError> { + let git = self.layout.open(repo)?; + let store = CobStore::new(&git); + let repo_key = self.interner.intern_repo(repo); + self.social + .refresh_repo(&self.interner, &store, repo_key) + .inspect(|()| self.bump_generation()) + } + + pub fn refresh_social_object( + &self, + repo: &RepoDid, + type_name: &TypeName, + object: CobId, + ) -> Result<(), IndexError> { + let git = self.layout.open(repo)?; + let store = CobStore::new(&git); + let repo_key = self.interner.intern_repo(repo); + self.social + .refresh_object(&self.interner, &store, repo_key, type_name, object) + .inspect(|()| self.bump_generation()) + } + + pub fn social_object_at( + &self, + repo: &RepoDid, + address: &RecordAddress, + ) -> Resolved> { + self.social.object_at(&self.interner, repo, address) + } + + pub fn social_address_of( + &self, + repo: &RepoDid, + object: CobId, + ) -> Resolved> { + self.social.address_of(&self.interner, repo, object) + } + + pub fn social_of_kind( + &self, + repo: &RepoDid, + collection: &RecordCollection, + ) -> Resolved> { + self.social.of_kind(&self.interner, repo, collection) + } + + pub fn social_children_of( + &self, + repo: &RepoDid, + parent: &RecordAddress, + ) -> Resolved> { + self.social.children_of(&self.interner, repo, parent) + } + + pub fn social_latest_child( + &self, + repo: &RepoDid, + parent: &RecordAddress, + collection: &RecordCollection, + ) -> Resolved> { + self.social + .latest_child(&self.interner, repo, parent, collection) + } + + pub fn social_live_reaction( + &self, + repo: &RepoDid, + subject: &RecordAddress, + actor: &AccountDid, + kind: knot_cobs::ReactionKind, + ) -> Resolved> { + self.social + .reaction_of(&self.interner, repo, subject, actor, kind) + } + + pub fn social_labels_of( + &self, + repo: &RepoDid, + subject: &RecordAddress, + ) -> Resolved> { + self.social.labels_of(&self.interner, repo, subject) + } + + pub fn social_entries(&self, repo: &RepoDid) -> Resolved> { + self.social.entries(&self.interner, repo) + } + + pub fn next_subject_number( + &self, + repo: &RepoDid, + collection: &RecordCollection, + ) -> Resolved> { + self.social + .highest_number(&self.interner, repo, collection) + .map(social::next_number) + } + + pub fn social_structure(&self, repo: &RepoDid) -> Resolved { + self.social.structure(&self.interner, repo) + } + + pub fn social_serving_bytes(&self, repo: &RepoDid) -> Resolved { + self.social.serving_bytes(&self.interner, repo) + } + + pub fn social_serving_bytes_of( + &self, + repo: &RepoDid, + object: CobId, + ) -> Resolved> { + self.social.serving_bytes_of(&self.interner, repo, object) + } + + pub fn effective_member(&self, did: &AccountDid) -> Resolved { + self.members.holds(&self.interner, did) + } + + pub fn member_status<'a>(&self, subject: &'a AccountDid) -> MemberStatus<'a> { + Answer { + scope: (), + subject, + granted: self.effective_member(subject).ready_or_default(), + } + } + + pub fn member_entries(&self) -> Resolved> { + self.members.entries(&self.interner) + } + + pub fn effective_members(&self) -> Resolved> { + self.members.subjects(&self.interner) + } + + pub fn is_blocked(&self, did: &AccountDid) -> Resolved { + self.blocklist.holds(&self.interner, did) + } + + pub fn blocklist_entries(&self) -> Resolved> { + self.blocklist.entries(&self.interner) + } + + pub fn blocked(&self) -> Resolved> { + self.blocklist.subjects(&self.interner) + } + + pub fn effective_collaborator(&self, repo: &RepoDid, did: &AccountDid) -> Resolved { + self.collaborators.holds(&self.interner, repo, did) + } + + pub fn collaboration_consent<'a>( + &self, + repo: &'a RepoDid, + subject: &'a AccountDid, + ) -> CollaborationConsent<'a> { + Answer { + scope: repo, + subject, + granted: self + .effective_collaborator(repo, subject) + .ready_or_default(), + } + } + + pub fn collaborator_entries(&self, repo: &RepoDid) -> Resolved> { + self.collaborators.entries(&self.interner, repo) + } + + pub fn effective_collaborators(&self, repo: &RepoDid) -> Resolved> { + self.collaborators.subjects(&self.interner, repo) + } + + pub fn resolve_repo(&self, owner: &OwnerDid, rkey: &RepoRkey) -> Resolved> { + self.registry.resolve(&self.interner, owner, rkey) + } + + pub fn resolve_clone_path( + &self, + owner: &OwnerDid, + path: &ClonePath, + ) -> Resolved> { + self.registry + .resolve_clone_path(&self.interner, owner, path) + } + + pub fn owner_of(&self, repo: &RepoDid) -> Resolved> { + self.registry.owner_of(&self.interner, repo) + } + pub fn is_fostered(&self, repo: &RepoDid) -> Resolved { + self.registry.fostered_of(&self.interner, repo) + } + + pub fn rkey_of(&self, repo: &RepoDid) -> Resolved> { + self.registry.rkey_of(&self.interner, repo) + } + + pub fn contribution_policy(&self, repo: &RepoDid) -> Resolved> { + self.registry + .policy_of(&self.interner, repo) + .map(|policy| policy.map(|policy| RepoPolicy::new(repo.clone(), policy))) + } + + pub fn ownership_of(&self, repo: &RepoDid) -> Resolved> { + self.registry.ownership(&self.interner, repo) + } + + pub fn hosted_repos(&self) -> Vec { + self.registry.hosted_repos(&self.interner) + } + + pub fn native_repos(&self) -> Vec { + self.registry.native_repos(&self.interner) + } + + pub fn owner_of_key(&self, key: &OfferedKey, now: UnixSeconds) -> Resolved> { + self.keys.owner(&self.interner, key, now) + } + + pub fn keys(&self) -> KeySet<'_> { + KeySet(self) + } + + fn push_grants(&self) -> Resolved { + match self.registry.coverage() { + Coverage::Warming => Resolved::Warming, + Coverage::Ready => { + let repos: Vec = self + .hosted_repos() + .iter() + .map(|repo| self.grants_of(repo)) + .collect(); + let hosted = HostedCoverage::over( + repos + .iter() + .filter(|repo| matches!(repo, RepoGrants::Pending)) + .count(), + ); + let grants = repos + .into_iter() + .filter_map(RepoGrants::into_accounts) + .fold(Vec::new(), |mut grants, accounts| { + grants.extend(accounts); + grants + }); + Resolved::Ready(Granted { + subjects: sorted(grants), + hosted, + }) + } + } + } + + fn grants_of(&self, repo: &RepoDid) -> RepoGrants { + match (self.owner_of(repo), self.effective_collaborators(repo)) { + (Resolved::Ready(owner), Resolved::Ready(granted)) => RepoGrants::Ready( + owner + .map(AccountDid::from) + .into_iter() + .chain(granted) + .collect(), + ), + _ if self.unreadable_repo(repo) => RepoGrants::Unreadable, + _ => RepoGrants::Pending, + } + } + + fn due_among( + &self, + subjects: &[AccountDid], + now: UnixSeconds, + against: Recheck, + ) -> Vec { + subjects + .iter() + .filter(|did| match against { + Recheck::Everything => true, + Recheck::Renewals => self.keys.renewal_due(&self.interner, did, now), + }) + .cloned() + .collect() + } + + pub fn coverage(&self) -> IndexCoverage { + IndexCoverage { + members: self.members.coverage(), + blocklist: self.blocklist.coverage(), + collaborators: self.collaborators.coverage(), + registry: self.registry.coverage(), + keys: self.keys.coverage(self.generation()), + } + } +} + +pub struct KeySet<'a>(&'a Index); + +impl KeySet<'_> { + pub fn coverage(&self) -> Coverage { + self.0.keys.coverage(self.0.generation()) + } + + pub fn mark_ready(&self, generation: IndexGeneration) { + self.0.keys.mark_ready(generation); + } + + pub fn mark_warming(&self) { + self.0.keys.mark_warming(); + } + + pub fn record(&self, did: &AccountDid, keys: Vec, lease: KeyLease) -> KeyRecord { + self.0.keys.record(&self.0.interner, did, keys, lease) + } + + pub fn reprieve( + &self, + did: &AccountDid, + now: UnixSeconds, + grace: KeyReprieve, + exhausted: KeyLease, + ) -> KeyReprieved { + self.0 + .keys + .reprieve(&self.0.interner, did, now, grace, exhausted) + } + + pub fn retain(&self, kept: &KeptAccounts) { + self.0.keys.retain(&self.0.interner, kept.as_slice()); + } + + pub fn is_fresh(&self, did: &AccountDid, now: UnixSeconds) -> bool { + self.0.keys.is_fresh(&self.0.interner, did, now) + } + + pub fn publisher_among( + &self, + candidates: &[AccountDid], + key: &OfferedKey, + now: UnixSeconds, + ) -> Option { + self.0 + .keys + .publisher_among(&self.0.interner, candidates, key, now) + } + + pub fn note_miss(&self) { + self.0.keys.suspect_now(); + } + + pub fn any_unheld(&self) -> bool { + self.0.keys.any_unheld() + } + + pub fn work(&self, now: UnixSeconds, interval: SweepInterval) -> Resolved { + let index = self.0; + let generation = index.generation(); + let Resolved::Ready(granted) = index.push_grants() else { + return Resolved::Warming; + }; + let pushers = granted.subjects; + let against = match index.keys.take_suspicion(now, interval) { + true => Recheck::Everything, + false => Recheck::Renewals, + }; + let members = index.effective_members().map(|members| { + let outside: Vec = members + .into_iter() + .filter(|did| pushers.binary_search(did).is_err()) + .collect(); + let (unread, due): (Vec, Vec) = index + .due_among(&outside, now, against) + .into_iter() + .partition(|did| !index.keys.on_file(&index.interner, did)); + let kept = pushers.iter().cloned().chain(outside).collect(); + MemberWork { + unread: UnreadMembers(unread), + due: StaleMembers(due), + kept: KeptAccounts(kept), + } + }); + let tracked = match &members { + Resolved::Ready(work) => work.kept.len(), + Resolved::Warming => pushers.len(), + }; + let due = index.due_among(&pushers, now, Recheck::Renewals); + let suspected = match against { + Recheck::Renewals => Vec::new(), + Recheck::Everything => pushers + .iter() + .filter(|did| !index.keys.renewal_due(&index.interner, did, now)) + .cloned() + .collect(), + }; + let complete = granted.hosted == HostedCoverage::Whole + && index.keys.all_live(&index.interner, &pushers, now); + Resolved::Ready(KeyWork { + generation, + tracked, + hosted: granted.hosted, + pushers: Pushers(pushers), + due: StalePushers(due), + suspected: SuspectPushers(suspected), + complete, + members, + }) + } + + pub fn all_live(&self, pushers: &Pushers, now: UnixSeconds) -> bool { + self.0 + .keys + .all_live(&self.0.interner, pushers.as_slice(), now) + } +} diff --git a/knot2/crates/knot-index/src/projections.rs b/knot2/crates/knot-index/src/projections.rs new file mode 100644 index 0000000000000000000000000000000000000000..73ce4302c40adad41057577fb66c8f0375576f17 --- /dev/null +++ b/knot2/crates/knot-index/src/projections.rs @@ -0,0 +1,1412 @@ +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use knot_cob::{Change, ChangePayload, Checkpoint, CobStore, Evaluate}; +use knot_cobs::{ + CollaboratorsChange, CollaboratorsCob, Effect, Entry, Grant, GrantChange, GrantList, Policy, + Registration, Registry, RegistryChange, Rename, RepoRef, RepoRegistryCob, +}; +use knot_types::{ + AccountDid, ChangeId, ClonePath, CobId, ContributionPolicy, OfferedKey, OwnerDid, RepoDid, + RepoRkey, UnixSeconds, +}; + +use crate::coverage::{Coverage, CoverageCell, Resolved}; +use crate::error::IndexError; +use crate::intern::{AccountKey, Interner, NameKey, OwnerKey, RepoKey, RkeyKey}; +use crate::{ + IndexGeneration, KeyBudget, KeyLease, KeyRecord, KeyReprieve, KeyReprieved, SweepInterval, +}; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Provenance { + added_by: AccountKey, + created_at: UnixSeconds, +} + +impl Provenance { + fn admitted(interner: &Interner, grant: &Grant) -> Self { + Self { + added_by: interner.intern_account(&grant.added_by), + created_at: grant.created_at, + } + } + + fn seeded(interner: &Interner, entry: &Entry) -> Self { + Self { + added_by: interner.intern_account(&entry.added_by), + created_at: entry.created_at, + } + } + + fn entry(self, interner: &Interner) -> Entry { + Entry { + added_by: interner.resolve_account(self.added_by), + created_at: self.created_at, + } + } +} + +fn decode_change(change: &Change) -> Result { + if change.type_name != P::type_name() { + return Err(IndexError::UnexpectedType { + change: change.id, + expected: P::type_name(), + found: change.type_name.clone(), + }); + } + P::decode(change.payload()).map_err(|error| IndexError::Decode { + change: change.id, + type_name: P::type_name(), + reason: error.to_string(), + }) +} + +fn decode_delta(changes: &[Change]) -> Result, IndexError> { + changes.iter().map(decode_change::

    ).collect() +} + +fn group_by(items: Vec, key: impl Fn(&T) -> K) -> BTreeMap> { + items.into_iter().fold(BTreeMap::new(), |mut acc, item| { + acc.entry(key(&item)).or_default().push(item); + acc + }) +} + +pub(crate) struct GrantSetProjection +where + Cob: Evaluate, + Cob::Change: ChangePayload + GrantChange, +{ + membership: scc::HashMap, + coverage: CoverageCell, + tip: Mutex>, + _cob: PhantomData Cob>, +} + +impl GrantSetProjection +where + Cob: Evaluate, + Cob::Change: ChangePayload + GrantChange, +{ + pub(crate) fn new() -> Self { + Self { + membership: scc::HashMap::new(), + coverage: CoverageCell::new(Coverage::Warming), + tip: Mutex::new(None), + _cob: PhantomData, + } + } + + pub(crate) fn coverage(&self) -> Coverage { + self.coverage.get() + } + + pub(crate) fn reset(&self) { + let mut tip = self + .tip + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + self.membership.clear_sync(); + *tip = None; + self.coverage.set(Coverage::Ready); + } + + pub(crate) fn holds(&self, interner: &Interner, did: &AccountDid) -> Resolved { + match self.coverage.get() { + Coverage::Warming => Resolved::Warming, + Coverage::Ready => Resolved::Ready( + interner + .account(did) + .is_some_and(|key| self.membership.contains_sync(&key)), + ), + } + } + + fn rows( + &self, + interner: &Interner, + row: impl Fn(AccountDid, Provenance) -> T, + ) -> Resolved> { + match self.coverage.get() { + Coverage::Warming => Resolved::Warming, + Coverage::Ready => { + let mut out = BTreeMap::new(); + self.membership.iter_sync(|&subject, slot| { + let did = interner.resolve_account(subject); + out.insert(did.clone(), row(did, *slot)); + true + }); + Resolved::Ready(out.into_values().collect()) + } + } + } + + pub(crate) fn entries(&self, interner: &Interner) -> Resolved> { + self.rows(interner, |did, slot| (did, slot.entry(interner))) + } + + pub(crate) fn subjects(&self, interner: &Interner) -> Resolved> { + self.rows(interner, |did, _| did) + } + + pub(crate) fn refresh( + &self, + interner: &Interner, + store: &CobStore, + object: CobId, + ) -> Result<(), IndexError> + where + Cob: Checkpoint + Evaluate, + { + let mut tip = self + .tip + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + match *tip { + None => { + let (list, seeded) = store + .materialize::(object) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + self.seed(interner, &list); + *tip = Some(seeded); + } + Some(prev) => { + let delta = store + .changes_since::(object, Some(prev)) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + let decoded = decode_delta::(&delta.changes) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + self.apply_delta(interner, decoded); + *tip = Some(delta.tip); + } + } + self.coverage.set(Coverage::Ready); + Ok(()) + } + + fn seed(&self, interner: &Interner, list: &GrantList) { + self.membership.clear_sync(); + list.entries().for_each(|(subject, entry)| { + let _ = self.membership.insert_sync( + interner.intern_account(subject), + Provenance::seeded(interner, entry), + ); + }); + } + + fn apply_delta(&self, interner: &Interner, changes: Vec) { + group_by(changes, |change| change.subject().clone()) + .into_iter() + .for_each(|(did, ops)| { + let current = interner + .account(&did) + .and_then(|key| self.membership.read_sync(&key, |_, slot| *slot)); + let net = ops + .iter() + .fold(current, |slot, change| match change.effect() { + Effect::Admit(grant) => { + slot.or_else(|| Some(Provenance::admitted(interner, grant))) + } + Effect::Noop(_) => slot, + Effect::Revoke(_) => None, + }); + match net { + Some(slot) => { + *self + .membership + .entry_sync(interner.intern_account(&did)) + .or_insert(slot) + .get_mut() = slot; + } + None => { + if let Some(key) = interner.account(&did) { + let _ = self.membership.remove_sync(&key); + } + } + } + }); + } +} + +struct RepoGrantList { + tip: Option, + entries: BTreeMap, +} + +const REPO_LOCK_STRIPES: usize = 256; + +pub(crate) struct CollaboratorsProjection { + grant_lists: scc::HashMap, + locks: [Mutex<()>; REPO_LOCK_STRIPES], +} + +impl CollaboratorsProjection { + pub(crate) fn new() -> Self { + Self { + grant_lists: scc::HashMap::new(), + locks: std::array::from_fn(|_| Mutex::new(())), + } + } + + fn repo_lock(&self, repo: RepoKey) -> &Mutex<()> { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + repo.hash(&mut hasher); + &self.locks[(hasher.finish() % REPO_LOCK_STRIPES as u64) as usize] + } + + pub(crate) fn coverage(&self) -> Coverage { + Coverage::Ready + } + + pub(crate) fn is_materialized(&self, interner: &Interner, repo: &RepoDid) -> bool { + interner + .repo(repo) + .is_some_and(|repo| self.grant_lists.contains_sync(&repo)) + } + + pub(crate) fn holds( + &self, + interner: &Interner, + repo: &RepoDid, + did: &AccountDid, + ) -> Resolved { + let Some(repo) = interner.repo(repo) else { + return Resolved::Warming; + }; + match self.grant_lists.read_sync(&repo, |_, list| { + interner + .account(did) + .is_some_and(|account| list.entries.contains_key(&account)) + }) { + Some(value) => Resolved::Ready(value), + None => Resolved::Warming, + } + } + + fn rows( + &self, + interner: &Interner, + repo: &RepoDid, + row: impl Fn(AccountDid, Provenance) -> T, + ) -> Resolved> { + let Some(repo) = interner.repo(repo) else { + return Resolved::Warming; + }; + match self.grant_lists.read_sync(&repo, |_, list| { + list.entries + .iter() + .map(|(&subject, slot)| { + let did = interner.resolve_account(subject); + (did.clone(), row(did, *slot)) + }) + .collect::>() + .into_values() + .collect() + }) { + Some(rows) => Resolved::Ready(rows), + None => Resolved::Warming, + } + } + + pub(crate) fn entries( + &self, + interner: &Interner, + repo: &RepoDid, + ) -> Resolved> { + self.rows(interner, repo, |did, slot| (did, slot.entry(interner))) + } + + pub(crate) fn subjects( + &self, + interner: &Interner, + repo: &RepoDid, + ) -> Resolved> { + self.rows(interner, repo, |did, _| did) + } + + pub(crate) fn mark_repo_empty(&self, repo: RepoKey) { + let lock = self.repo_lock(repo); + let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + self.install( + repo, + RepoGrantList { + tip: None, + entries: BTreeMap::new(), + }, + ); + } + + pub(crate) fn refresh_repo( + &self, + interner: &Interner, + store: &CobStore, + repo: RepoKey, + object: CobId, + ) -> Result<(), IndexError> { + let lock = self.repo_lock(repo); + let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + self.materialize_repo(interner, store, repo, object) + .inspect_err(|_| self.purge_repo(repo)) + } + + fn materialize_repo( + &self, + interner: &Interner, + store: &CobStore, + repo: RepoKey, + object: CobId, + ) -> Result<(), IndexError> { + let prev = self + .grant_lists + .read_sync(&repo, |_, list| list.tip) + .flatten(); + match prev { + None => { + let (list, tip) = store.materialize::(object)?; + let entries = list + .entries() + .map(|(subject, entry)| { + ( + interner.intern_account(subject), + Provenance::seeded(interner, entry), + ) + }) + .collect(); + self.install( + repo, + RepoGrantList { + tip: Some(tip), + entries, + }, + ); + } + Some(prev) => { + let delta = store.changes_since::(object, Some(prev))?; + let decoded = decode_delta::(&delta.changes)?; + let mut occupied = + self.grant_lists + .entry_sync(repo) + .or_insert_with(|| RepoGrantList { + tip: None, + entries: BTreeMap::new(), + }); + let list = occupied.get_mut(); + list.entries = decoded + .into_iter() + .fold(std::mem::take(&mut list.entries), |entries, change| { + apply(interner, entries, change) + }); + list.tip = Some(delta.tip); + } + } + Ok(()) + } + + pub(crate) fn drop_repo(&self, repo: RepoKey) { + let lock = self.repo_lock(repo); + let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + self.purge_repo(repo); + } + + fn purge_repo(&self, repo: RepoKey) { + let _ = self.grant_lists.remove_sync(&repo); + } + + fn install(&self, repo: RepoKey, list: RepoGrantList) { + match self.grant_lists.entry_sync(repo) { + scc::hash_map::Entry::Occupied(mut occupied) => { + let _ = occupied.insert(list); + } + scc::hash_map::Entry::Vacant(vacant) => { + vacant.insert_entry(list); + } + } + } +} + +fn apply( + interner: &Interner, + mut entries: BTreeMap, + change: CollaboratorsChange, +) -> BTreeMap { + match change.effect() { + Effect::Admit(grant) => { + entries + .entry(interner.intern_account(&grant.subject)) + .or_insert_with(|| Provenance::admitted(interner, grant)); + } + Effect::Noop(_) => {} + Effect::Revoke(subject) => { + if let Some(key) = interner.account(subject) { + entries.remove(&key); + } + } + } + entries +} + +struct RecordSlot { + owner: OwnerKey, + rkey: RkeyKey, + name: NameKey, + created_at: UnixSeconds, + policy: ContributionPolicy, + fostered: bool, +} + +pub(crate) struct RegistryProjection { + aliases: scc::HashMap<(OwnerKey, RkeyKey), RepoKey>, + names: scc::HashMap<(OwnerKey, NameKey), BTreeSet<(UnixSeconds, RepoKey)>>, + records: scc::HashMap, + coverage: CoverageCell, + tip: Mutex>, +} + +impl RegistryProjection { + pub(crate) fn new() -> Self { + Self { + aliases: scc::HashMap::new(), + names: scc::HashMap::new(), + records: scc::HashMap::new(), + coverage: CoverageCell::new(Coverage::Warming), + tip: Mutex::new(None), + } + } + + pub(crate) fn coverage(&self) -> Coverage { + self.coverage.get() + } + + pub(crate) fn warm(&self) { + self.coverage.set(Coverage::Warming); + } + + pub(crate) fn reset(&self, interner: &Interner) -> Vec { + let mut tip = self + .tip + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let evacuated = self.hosted_repos(interner); + self.coverage.set(Coverage::Warming); + self.aliases.clear_sync(); + self.names.clear_sync(); + self.records.clear_sync(); + *tip = None; + self.coverage.set(Coverage::Ready); + evacuated + } + + pub(crate) fn resolve( + &self, + interner: &Interner, + owner: &OwnerDid, + rkey: &RepoRkey, + ) -> Resolved> { + match self.coverage.get() { + Coverage::Warming => Resolved::Warming, + Coverage::Ready => Resolved::Ready( + interner + .owner(owner) + .zip(interner.rkey(rkey)) + .and_then(|key| self.aliases.read_sync(&key, |_, repo| *repo)) + .map(|repo| interner.resolve_repo(repo)), + ), + } + } + + pub(crate) fn resolve_clone_path( + &self, + interner: &Interner, + owner: &OwnerDid, + path: &ClonePath, + ) -> Resolved> { + if self.coverage.get() == Coverage::Warming { + return Resolved::Warming; + } + let Some(owner) = interner.owner(owner) else { + return Resolved::Ready(None); + }; + let by_rkey = path + .rkeys() + .filter_map(|rkey| interner.rkey(rkey)) + .find_map(|rkey| self.aliases.read_sync(&(owner, rkey), |_, repo| *repo)); + if let Some(repo) = by_rkey { + return Resolved::Ready(Some(interner.resolve_repo(repo))); + } + Resolved::Ready( + path.names() + .filter_map(|name| interner.name(name)) + .find_map(|name| self.oldest_registration_for_name(interner, owner, name)), + ) + } + + fn oldest_registration_for_name( + &self, + interner: &Interner, + owner: OwnerKey, + name: NameKey, + ) -> Option { + // Notice how set keys sort by whichever string the interner saw first, + // and a cold rebuild will see them in a different order than live replay, + // so 2 entries with the same timestamp will settle by comparing + // DIDs instead. + self.names + .read_sync(&(owner, name), |_, registered| { + let earliest = registered.first()?.0; + registered + .iter() + .take_while(|(created_at, _)| *created_at == earliest) + .map(|(_, repo)| interner.resolve_repo(*repo)) + .min() + }) + .flatten() + } + + pub(crate) fn owner_of( + &self, + interner: &Interner, + repo: &RepoDid, + ) -> Resolved> { + if self.coverage.get() == Coverage::Warming { + return Resolved::Warming; + } + let Some(target) = interner.repo(repo) else { + return Resolved::Ready(None); + }; + Resolved::Ready( + self.records + .read_sync(&target, |_, slot| slot.owner) + .map(|owner| interner.resolve_owner(owner)), + ) + } + + pub(crate) fn rkey_of( + &self, + interner: &Interner, + repo: &RepoDid, + ) -> Resolved> { + if self.coverage.get() == Coverage::Warming { + return Resolved::Warming; + } + let Some(target) = interner.repo(repo) else { + return Resolved::Ready(None); + }; + Resolved::Ready( + self.records + .read_sync(&target, |_, slot| slot.rkey) + .map(|rkey| interner.resolve_rkey(rkey)), + ) + } + + pub(crate) fn policy_of( + &self, + interner: &Interner, + repo: &RepoDid, + ) -> Resolved> { + if self.coverage.get() == Coverage::Warming { + return Resolved::Warming; + } + let Some(target) = interner.repo(repo) else { + return Resolved::Ready(None); + }; + Resolved::Ready(self.records.read_sync(&target, |_, slot| slot.policy)) + } + + pub(crate) fn ownership( + &self, + interner: &Interner, + repo: &RepoDid, + ) -> Resolved> { + if self.coverage.get() == Coverage::Warming { + return Resolved::Warming; + } + let Some(target) = interner.repo(repo) else { + return Resolved::Ready(None); + }; + Resolved::Ready( + self.records + .read_sync(&target, |_, slot| (slot.owner, slot.rkey)) + .map(|(owner, rkey)| RepoRef { + owner: interner.resolve_owner(owner), + rkey: interner.resolve_rkey(rkey), + }), + ) + } + pub(crate) fn fostered_of(&self, interner: &Interner, repo: &RepoDid) -> Resolved { + if self.coverage.get() == Coverage::Warming { + return Resolved::Warming; + } + let Some(target) = interner.repo(repo) else { + return Resolved::Ready(false); + }; + Resolved::Ready( + self.records + .read_sync(&target, |_, slot| slot.fostered) + .unwrap_or(false), + ) + } + + pub(crate) fn hosted_repos(&self, interner: &Interner) -> Vec { + let mut repos = BTreeSet::new(); + self.records.iter_sync(|repo, _| { + repos.insert(interner.resolve_repo(*repo)); + true + }); + repos.into_iter().collect() + } + + pub(crate) fn native_repos(&self, interner: &Interner) -> Vec { + let mut repos = BTreeSet::new(); + self.records.iter_sync(|repo, slot| { + if !slot.fostered { + repos.insert(interner.resolve_repo(*repo)); + } + true + }); + repos.into_iter().collect() + } + + pub(crate) fn refresh( + &self, + interner: &Interner, + store: &CobStore, + object: CobId, + ) -> Result, IndexError> { + let mut tip = self + .tip + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + match *tip { + None => { + let (registry, seeded) = store + .materialize::(object) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + self.seed(interner, ®istry); + *tip = Some(seeded); + self.coverage.set(Coverage::Ready); + Ok(Vec::new()) + } + Some(prev) => { + let delta = store + .changes_since::(object, Some(prev)) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + let decoded = decode_delta::(&delta.changes) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + let displaced = self.apply_delta(interner, decoded); + *tip = Some(delta.tip); + self.coverage.set(Coverage::Ready); + Ok(self.evacuated(interner, displaced)) + } + } + } + + fn seed(&self, interner: &Interner, registry: &Registry) { + self.coverage.set(Coverage::Warming); + self.aliases.clear_sync(); + self.names.clear_sync(); + self.records.clear_sync(); + registry.records().for_each(|(repo, record)| { + let repo = interner.intern_repo(repo); + let owner = interner.intern_owner(&record.owner); + let name = interner.intern_name(&record.name); + self.upsert_record( + repo, + owner, + interner.intern_rkey(&record.rkey), + name, + record.created_at, + record.policy, + record.fostered, + ); + self.bind_name(owner, name, record.created_at, repo); + }); + registry.aliases().for_each(|(owner, rkey, repo)| { + self.upsert_alias( + interner.intern_owner(owner), + interner.intern_rkey(rkey), + interner.intern_repo(repo), + ); + }); + } + + fn evacuated(&self, interner: &Interner, displaced: Vec) -> Vec { + if displaced.is_empty() { + return Vec::new(); + } + let live = self.live_repos(); + displaced + .into_iter() + .collect::>() + .into_iter() + .filter(|repo| !live.contains(repo)) + .map(|repo| interner.resolve_repo(repo)) + .collect() + } + + fn live_repos(&self) -> BTreeSet { + let mut live = BTreeSet::new(); + self.records.iter_sync(|repo, _| { + live.insert(*repo); + true + }); + live + } + + fn apply_delta(&self, interner: &Interner, changes: Vec) -> Vec { + changes + .into_iter() + .fold(Vec::new(), |displaced, change| match change { + RegistryChange::Register(registration) => { + self.apply_register(interner, registration, displaced) + } + RegistryChange::Rename(rename) => self.apply_rename(interner, rename, displaced), + RegistryChange::Deregister(target) => { + self.apply_deregister(interner, target, displaced) + } + RegistryChange::SetPolicy(policy) => self.apply_policy(interner, policy, displaced), + RegistryChange::Retired(_) => displaced, + }) + } + + fn apply_register( + &self, + interner: &Interner, + registration: Registration, + mut displaced: Vec, + ) -> Vec { + let repo = interner.intern_repo(®istration.repo); + let owner = interner.intern_owner(®istration.owner); + let rkey = interner.intern_rkey(®istration.rkey); + let name = interner.intern_name(®istration.name); + if self.records.contains_sync(&repo) { + self.drop_record(repo); + displaced.push(repo); + } + displaced.extend(self.steal_alias(owner, rkey, repo)); + self.upsert_record( + repo, + owner, + rkey, + name, + registration.created_at, + registration.policy, + registration.fostered, + ); + self.upsert_alias(owner, rkey, repo); + self.bind_name(owner, name, registration.created_at, repo); + displaced + } + + fn apply_rename( + &self, + interner: &Interner, + rename: Rename, + mut displaced: Vec, + ) -> Vec { + let repo = interner.intern_repo(&rename.repo); + let owner = interner.intern_owner(&rename.owner); + let rkey = interner.intern_rkey(&rename.rkey); + let name = interner.intern_name(&rename.name); + let record = self + .records + .read_sync(&repo, |_, slot| { + (slot.owner == owner).then_some(( + slot.created_at, + slot.name, + slot.policy, + slot.fostered, + )) + }) + .flatten(); + let Some((created_at, previous, policy, fostered)) = record else { + return displaced; + }; + displaced.extend(self.steal_alias(owner, rkey, repo)); + self.upsert_record(repo, owner, rkey, name, created_at, policy, fostered); + self.upsert_alias(owner, rkey, repo); + self.bind_name(owner, name, created_at, repo); + if previous != name { + self.unbind_name(owner, previous, created_at, repo); + } + displaced + } + + fn apply_policy( + &self, + interner: &Interner, + change: Policy, + displaced: Vec, + ) -> Vec { + let repo = interner.intern_repo(&change.repo); + let owner = interner.intern_owner(&change.owner); + let applied = self + .records + .update_sync(&repo, |_, slot| { + let owner_match = slot.owner == owner; + if owner_match { + slot.policy = change.policy; + } + owner_match + }) + .unwrap_or(false); + if !applied { + tracing::warn!( + repo = %change.repo, + owner = %change.owner, + policy = %change.policy, + "contribution policy change names repository registry doesn't have under that owner. Its policy stays put" + ); + } + displaced + } + + fn apply_deregister( + &self, + interner: &Interner, + target: RepoRef, + mut displaced: Vec, + ) -> Vec { + let Some(owner) = interner.owner(&target.owner) else { + return displaced; + }; + let Some(rkey) = interner.rkey(&target.rkey) else { + return displaced; + }; + let Some(repo) = self.aliases.read_sync(&(owner, rkey), |_, repo| *repo) else { + return displaced; + }; + self.drop_record(repo); + displaced.push(repo); + displaced + } + + fn steal_alias(&self, owner: OwnerKey, rkey: RkeyKey, target: RepoKey) -> Option { + let holder = self.aliases.read_sync(&(owner, rkey), |_, repo| *repo)?; + if holder == target { + return None; + } + let canonical = self + .records + .read_sync(&holder, |_, slot| slot.rkey == rkey) + .unwrap_or(false); + if canonical { + self.drop_record(holder); + Some(holder) + } else { + let _ = self.aliases.remove_sync(&(owner, rkey)); + None + } + } + + fn drop_record(&self, repo: RepoKey) { + if let Some((_, slot)) = self.records.remove_sync(&repo) { + self.unbind_name(slot.owner, slot.name, slot.created_at, repo); + } + self.aliases.retain_sync(|_, holder| *holder != repo); + } + + fn bind_name(&self, owner: OwnerKey, name: NameKey, created_at: UnixSeconds, repo: RepoKey) { + match self.names.entry_sync((owner, name)) { + scc::hash_map::Entry::Occupied(mut occupied) => { + occupied.get_mut().insert((created_at, repo)); + } + scc::hash_map::Entry::Vacant(vacant) => { + vacant.insert_entry(BTreeSet::from([(created_at, repo)])); + } + } + } + + fn unbind_name(&self, owner: OwnerKey, name: NameKey, created_at: UnixSeconds, repo: RepoKey) { + let _ = self.names.remove_if_sync(&(owner, name), |registered| { + registered.remove(&(created_at, repo)); + registered.is_empty() + }); + } + + #[allow(clippy::too_many_arguments)] + fn upsert_record( + &self, + repo: RepoKey, + owner: OwnerKey, + rkey: RkeyKey, + name: NameKey, + created_at: UnixSeconds, + policy: ContributionPolicy, + fostered: bool, + ) { + if self + .records + .update_sync(&repo, |_, slot| { + slot.owner = owner; + slot.rkey = rkey; + slot.name = name; + slot.created_at = created_at; + slot.policy = policy; + slot.fostered = fostered; + }) + .is_none() + { + let _ = self.records.insert_sync( + repo, + RecordSlot { + owner, + rkey, + name, + created_at, + policy, + fostered, + }, + ); + } + } + + fn upsert_alias(&self, owner: OwnerKey, rkey: RkeyKey, repo: RepoKey) { + let key = (owner, rkey); + if self + .aliases + .update_sync(&key, |_, slot| *slot = repo) + .is_none() + { + let _ = self.aliases.insert_sync(key, repo); + } + } +} + +enum Reading { + Published(Vec>), + Unheld, + Unread, +} + +impl Reading { + fn keys(&self) -> &[Arc] { + match self { + Reading::Published(keys) => keys, + Reading::Unheld | Reading::Unread => &[], + } + } + + fn is_published(&self) -> bool { + matches!(self, Reading::Published(_)) + } +} + +struct Held { + reading: Reading, + lease: KeyLease, +} + +const PER_KEY_OVERHEAD: usize = 192; + +const PER_ACCOUNT_OVERHEAD: usize = 128; + +impl Held { + fn bytes(&self) -> usize { + PER_ACCOUNT_OVERHEAD + + self + .reading + .keys() + .iter() + .map(|key| key.as_bytes().len() + PER_KEY_OVERHEAD) + .sum::() + } + + fn answers(&self, now: UnixSeconds) -> bool { + self.reading.is_published() && self.lease.is_live(now) + } + + fn settled(&self, now: UnixSeconds) -> bool { + !matches!(self.reading, Reading::Unread) && self.lease.is_live(now) + } + + fn renewal_due(&self, now: UnixSeconds) -> bool { + match self.reading { + Reading::Published(_) => self.lease.renewal_due(now), + Reading::Unheld | Reading::Unread => !self.lease.is_live(now), + } + } + + fn publishes(&self, key: &OfferedKey, now: UnixSeconds) -> bool { + self.answers(now) + && self + .reading + .keys() + .iter() + .any(|published| published.as_ref() == key) + } + + fn is_unheld(&self) -> bool { + matches!(self.reading, Reading::Unheld) + } +} + +#[derive(Default)] +struct Publishers(Vec); + +impl Publishers { + fn add(&mut self, account: AccountKey) { + if let Err(at) = self.0.binary_search(&account) { + self.0.insert(at, account); + } + } + + fn remove(&mut self, account: AccountKey) -> bool { + if let Ok(at) = self.0.binary_search(&account) { + self.0.remove(at); + } + self.0.is_empty() + } + + fn to_vec(&self) -> Vec { + self.0.clone() + } +} + +const NEVER: u64 = u64::MAX; + +pub(crate) struct KeyProjection { + owners: scc::HashMap, Publishers>, + held: scc::HashMap, + budget: KeyBudget, + tracked_bytes: AtomicUsize, + unheld: AtomicUsize, + suspect: AtomicBool, + swept_at: AtomicI64, + ready_at: AtomicU64, +} + +impl KeyProjection { + pub(crate) fn new(budget: KeyBudget) -> Self { + Self { + owners: scc::HashMap::new(), + held: scc::HashMap::new(), + budget, + tracked_bytes: AtomicUsize::new(0), + unheld: AtomicUsize::new(0), + suspect: AtomicBool::new(false), + swept_at: AtomicI64::new(i64::MIN), + ready_at: AtomicU64::new(NEVER), + } + } + + pub(crate) fn coverage(&self, generation: IndexGeneration) -> Coverage { + match self.ready_at.load(Ordering::Acquire) == generation.get() { + true => Coverage::Ready, + false => Coverage::Warming, + } + } + + pub(crate) fn suspect_now(&self) { + self.suspect.store(true, Ordering::Release); + } + + pub(crate) fn take_suspicion(&self, now: UnixSeconds, interval: SweepInterval) -> bool { + let due_at = |swept: i64| swept.saturating_add(crate::whole_secs(interval.get())); + match self.suspect.load(Ordering::Acquire) { + false => false, + true => { + self.swept_at + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |swept| { + (now.get() >= due_at(swept)).then_some(now.get()) + }) + .is_ok() + && self.suspect.swap(false, Ordering::AcqRel) + } + } + } + + pub(crate) fn mark_ready(&self, generation: IndexGeneration) { + self.ready_at.store(generation.get(), Ordering::Release); + } + + pub(crate) fn mark_warming(&self) { + self.ready_at.store(NEVER, Ordering::Release); + } + + pub(crate) fn owner( + &self, + interner: &Interner, + key: &OfferedKey, + now: UnixSeconds, + ) -> Resolved> { + let publishers = self + .owners + .read_sync(key, |_, publishers| publishers.to_vec()) + .unwrap_or_default(); + Resolved::Ready( + publishers + .into_iter() + .find(|account| { + self.held_by(*account, |held| held.publishes(key, now)) == Some(true) + }) + .map(|account| interner.resolve_account(account)), + ) + } + + pub(crate) fn any_unheld(&self) -> bool { + self.unheld.load(Ordering::Acquire) > 0 + } + + fn track_unheld(&self, lost: bool, gained: bool) { + match (lost, gained) { + (false, true) => { + self.unheld.fetch_add(1, Ordering::AcqRel); + } + (true, false) => { + self.unheld.fetch_sub(1, Ordering::AcqRel); + } + (true, true) | (false, false) => {} + } + } + + pub(crate) fn record( + &self, + interner: &Interner, + did: &AccountDid, + keys: Vec, + lease: KeyLease, + ) -> KeyRecord { + let account = interner.intern_account(did); + let incoming = Held { + reading: Reading::Published(keys.into_iter().map(Arc::new).collect()), + lease, + }; + let entry = self.held.entry_sync(account); + let held = match &entry { + scc::hash_map::Entry::Occupied(slot) => slot.get().bytes(), + scc::hash_map::Entry::Vacant(_) => 0, + }; + if self.reserve_bytes(incoming.bytes() as isize - held as isize) { + self.settle(entry, account, incoming); + return KeyRecord::Stored; + } + let unheld = Held { + reading: Reading::Unheld, + lease, + }; + if !self.reserve_bytes(unheld.bytes() as isize - held as isize) { + return KeyRecord::Saturated; + } + self.settle(entry, account, unheld); + KeyRecord::Unheld + } + + fn settle( + &self, + entry: scc::hash_map::Entry<'_, AccountKey, Held>, + account: AccountKey, + incoming: Held, + ) { + let published: HashSet> = incoming.reading.keys().iter().cloned().collect(); + let gained = incoming.is_unheld(); + match entry { + scc::hash_map::Entry::Occupied(mut slot) => { + let replaced = slot.insert(incoming); + self.track_unheld(replaced.is_unheld(), gained); + self.rewire(account, &published, replaced.reading.keys()); + } + scc::hash_map::Entry::Vacant(slot) => { + let _locked = slot.insert_entry(incoming); + self.track_unheld(false, gained); + self.rewire(account, &published, &[]); + } + } + } + + fn rewire( + &self, + account: AccountKey, + published: &HashSet>, + replaced: &[Arc], + ) { + published + .iter() + .for_each(|key| self.claim(Arc::clone(key), account)); + replaced + .iter() + .filter(|key| !published.contains(*key)) + .for_each(|key| self.disown(key, account)); + } + + pub(crate) fn reprieve( + &self, + interner: &Interner, + did: &AccountDid, + now: UnixSeconds, + grace: KeyReprieve, + exhausted: KeyLease, + ) -> KeyReprieved { + let account = interner.intern_account(did); + let mut slot = match self.held.entry_sync(account) { + scc::hash_map::Entry::Vacant(slot) => { + let pending = Held { + reading: Reading::Unread, + lease: grace.first_failure(now), + }; + if self.reserve_bytes(pending.bytes() as isize) { + let _locked = slot.insert_entry(pending); + } + return KeyReprieved::Pending; + } + scc::hash_map::Entry::Occupied(slot) => slot, + }; + match grace.extend(slot.get().lease, now) { + Some(lease) => { + let published = slot.get().reading.is_published(); + slot.get_mut().lease = lease; + match published { + true => KeyReprieved::Extended, + false => KeyReprieved::Pending, + } + } + None => { + let given_up = Held { + reading: Reading::Published(Vec::new()), + lease: exhausted, + }; + let _ = self.reserve_bytes(given_up.bytes() as isize - slot.get().bytes() as isize); + self.track_unheld(slot.get().is_unheld(), false); + self.disown_all(&slot, account); + *slot.get_mut() = given_up; + KeyReprieved::Exhausted + } + } + } + + pub(crate) fn on_file(&self, interner: &Interner, did: &AccountDid) -> bool { + interner + .account(did) + .is_some_and(|account| self.held.contains_sync(&account)) + } + + fn held_by(&self, account: AccountKey, ready: impl Fn(&Held) -> bool) -> Option { + self.held.read_sync(&account, |_, held| ready(held)) + } + + fn held_for( + &self, + interner: &Interner, + did: &AccountDid, + ready: impl Fn(&Held) -> bool, + ) -> Option { + interner + .account(did) + .and_then(|account| self.held_by(account, ready)) + } + + pub(crate) fn is_fresh(&self, interner: &Interner, did: &AccountDid, now: UnixSeconds) -> bool { + self.held_for(interner, did, |held| held.answers(now)) + .unwrap_or(false) + } + + pub(crate) fn renewal_due( + &self, + interner: &Interner, + did: &AccountDid, + now: UnixSeconds, + ) -> bool { + self.held_for(interner, did, |held| held.renewal_due(now)) + .unwrap_or(true) + } + + pub(crate) fn all_live( + &self, + interner: &Interner, + subjects: &[AccountDid], + now: UnixSeconds, + ) -> bool { + subjects.iter().all(|did| { + self.held_for(interner, did, |held| held.settled(now)) + .unwrap_or(false) + }) + } + + pub(crate) fn publisher_among( + &self, + interner: &Interner, + candidates: &[AccountDid], + key: &OfferedKey, + now: UnixSeconds, + ) -> Option { + candidates + .iter() + .find(|did| { + self.held_for(interner, did, |held| held.publishes(key, now)) + .unwrap_or(false) + }) + .cloned() + } + + pub(crate) fn retain(&self, interner: &Interner, kept: &[AccountDid]) { + let keep: BTreeSet = kept + .iter() + .filter_map(|did| interner.account(did)) + .collect(); + let mut released = Vec::new(); + self.held.iter_sync(|account, _| { + if !keep.contains(account) { + released.push(*account); + } + true + }); + released + .into_iter() + .for_each(|account| self.release(account)); + } + + fn release(&self, account: AccountKey) { + if let scc::hash_map::Entry::Occupied(slot) = self.held.entry_sync(account) { + self.evict(&slot, account); + let _ = slot.remove(); + } + } + + fn evict( + &self, + slot: &scc::hash_map::OccupiedEntry<'_, AccountKey, Held>, + account: AccountKey, + ) { + self.track_unheld(slot.get().is_unheld(), false); + self.disown_all(slot, account); + let _ = self.reserve_bytes(-(slot.get().bytes() as isize)); + } + + fn disown_all( + &self, + slot: &scc::hash_map::OccupiedEntry<'_, AccountKey, Held>, + account: AccountKey, + ) { + slot.get() + .reading + .keys() + .iter() + .for_each(|key| self.disown(key, account)); + } + + fn reserve_bytes(&self, growth: isize) -> bool { + self.tracked_bytes + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |tracked| { + let next = tracked.saturating_add_signed(growth); + (growth <= 0 || next <= self.budget.get()).then_some(next) + }) + .is_ok() + } + + fn claim(&self, key: Arc, account: AccountKey) { + self.owners + .entry_sync(key) + .or_default() + .get_mut() + .add(account); + } + + fn disown(&self, key: &OfferedKey, account: AccountKey) { + let _ = self + .owners + .remove_if_sync(key, |publishers| publishers.remove(account)); + } +} diff --git a/knot2/crates/knot-index/src/social.rs b/knot2/crates/knot-index/src/social.rs new file mode 100644 index 0000000000000000000000000000000000000000..23cb3947f32b8137f198a4cea805daf4b500114a --- /dev/null +++ b/knot2/crates/knot-index/src/social.rs @@ -0,0 +1,1008 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::hash::{Hash, Hasher}; +use std::ops::ControlFlow; +use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; +use std::sync::{LazyLock, Mutex}; + +use knot_cob::{Change, ChangePayload, CobStore, Delta, HistoryModel}; +use knot_resource::StructureBytes; +use knot_types::{ + AccountDid, ChangeId, CobId, RecordAddress, RecordCid, RecordCollection, RecordRkey, RepoDid, + SubjectNumber, TypeName, UnixSeconds, +}; + +use crate::coverage::Resolved; +use crate::error::IndexError; +use crate::intern::{AccountKey, CollectionKey, Interner, RecordKey, RepoKey}; + +const REPO_LOCK_STRIPES: usize = 256; + +knot_types::scalar_newtype! { + pub struct SocialResidencyBytes(u64) => ordered; + struct TouchStamp(u64) => ordered; +} + +impl SocialResidencyBytes { + pub const DEFAULT: Self = Self::new(256 * 1024 * 1024); +} +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct Slot { + collection: CollectionKey, + rkey: RecordKey, +} + +impl Slot { + fn of(interner: &Interner, address: &RecordAddress) -> Self { + Self { + collection: interner.intern_collection(address.collection()), + rkey: interner.key_of(address.rkey()), + } + } + + fn known(interner: &Interner, address: &RecordAddress) -> Option { + Some(Self { + collection: interner.collection(address.collection())?, + rkey: interner.known_key(address.rkey())?, + }) + } + + fn address(self, interner: &Interner) -> RecordAddress { + RecordAddress::new( + interner.resolve_collection(self.collection), + RecordRkey::new(interner.key_str(self.rkey)) + .expect("An interned record key is a valid rkey"), + ) + } +} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SocialEntryState { + Serving(RecordCid), + Erased, + Contested, +} + +impl SocialEntryState { + pub const fn serving(self) -> Option { + match self { + Self::Serving(version) => Some(version), + Self::Erased | Self::Contested => None, + } + } + + pub const fn erased(self) -> bool { + matches!(self, Self::Erased) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SocialEntry { + pub address: RecordAddress, + pub object: CobId, + pub state: SocialEntryState, +} + +type Labelings = Box<[knot_types::LabelKey]>; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Objected { + slot: Slot, + number: SubjectNumber, + parent: Option, + reacted: Option, + labeled: Option, + created_at: UnixSeconds, + version: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ReactionPlacement { + actor: AccountKey, + kind: knot_cobs::ReactionKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct Ordered(SubjectNumber, UnixSeconds, CobId); + +impl Ordered { + const fn of(objected: &Objected, object: CobId) -> Self { + Self(objected.number, objected.created_at, object) + } + + const fn object(self) -> CobId { + self.2 + } +} + +#[derive(Debug)] +struct RepoSocial { + touched: AtomicU64, + charged: BTreeMap, + objects: BTreeMap, + claiming: BTreeMap>, + addressed: BTreeMap, + children: BTreeMap>>, + numbered: BTreeMap>, + reacted: BTreeMap<(Slot, AccountKey, knot_cobs::ReactionKind), BTreeSet>, + structure: StructureBytes, +} + +impl RepoSocial { + fn new(touched: TouchStamp) -> Self { + Self { + touched: AtomicU64::new(touched.get()), + charged: BTreeMap::new(), + objects: BTreeMap::new(), + claiming: BTreeMap::new(), + addressed: BTreeMap::new(), + children: BTreeMap::new(), + numbered: BTreeMap::new(), + reacted: BTreeMap::new(), + structure: StructureBytes::new(0), + } + } + + fn touch(&self, at: TouchStamp) { + self.touched.store(at.get(), AtomicOrdering::Relaxed); + } + + fn progress(&self, object: CobId) -> Option { + self.charged.get(&object).copied() + } + + fn install(&mut self, object: CobId, projected: Projected) { + let left = self.retract(object); + self.structure = + StructureBytes::new(self.structure.get().saturating_add(projected.bytes.get())); + self.charged.insert( + object, + Progress { + through: projected.through, + bytes: projected.bytes, + serving: projected.serving, + }, + ); + let arrived = projected.objected.map(|objected| { + let ordered = Ordered::of(&objected, object); + self.claiming + .entry(objected.slot) + .or_default() + .insert(ordered); + self.numbered + .entry(objected.slot.collection) + .or_default() + .insert(ordered); + objected.parent.iter().for_each(|parent| { + self.children + .entry(*parent) + .or_default() + .entry(objected.slot.collection) + .or_default() + .insert(ordered); + }); + Self::reaction_key(&objected).iter().for_each(|key| { + self.reacted.entry(*key).or_default().insert(object); + }); + let slot = objected.slot; + self.objects.insert(object, objected); + slot + }); + left.into_iter() + .chain(arrived.filter(|slot| Some(*slot) != left)) + .for_each(|slot| self.settle(slot)); + } + + fn retract(&mut self, object: CobId) -> Option { + let charged = self + .charged + .remove(&object) + .map_or(0, |progress| progress.bytes.get()); + self.structure = StructureBytes::new(self.structure.get().saturating_sub(charged)); + self.objects.remove(&object).map(|objected| { + let ordered = Ordered::of(&objected, object); + self.unlist(&objected, ordered); + Self::without(&mut self.numbered, &objected.slot.collection, &ordered); + Self::without(&mut self.claiming, &objected.slot, &ordered); + Self::reaction_key(&objected) + .iter() + .for_each(|key| Self::without(&mut self.reacted, key, &object)); + objected.slot + }) + } + + fn settle(&mut self, slot: Slot) { + let Some(Ordered(.., winner)) = self + .claiming + .get(&slot) + .and_then(|claiming| claiming.first().copied()) + else { + self.addressed.remove(&slot); + return; + }; + self.addressed.insert(slot, winner); + self.claiming + .get(&slot) + .into_iter() + .flatten() + .for_each(|Ordered(.., claimant)| { + if *claimant != winner { + tracing::warn!( + object = %claimant, + "Second object under one record address never answers lookups" + ); + } + }); + } + + fn labels_of(&self, subject: &Slot) -> Vec { + self.addressed + .get(subject) + .and_then(|object| self.objects.get(object)) + .and_then(|objected| objected.labeled.as_ref()) + .map(|labels| labels.to_vec()) + .unwrap_or_default() + } + + fn unlist(&mut self, objected: &Objected, ordered: Ordered) { + objected.parent.iter().for_each(|parent| { + let emptied = self.children.get_mut(parent).is_some_and(|kinds| { + Self::without(kinds, &objected.slot.collection, &ordered); + kinds.is_empty() + }); + if emptied { + self.children.remove(parent); + } + }); + } + + fn without(index: &mut BTreeMap>, key: &K, value: &V) { + let emptied = index.get_mut(key).is_some_and(|set| { + set.remove(value); + set.is_empty() + }); + if emptied { + index.remove(key); + } + } + + fn reaction_key(objected: &Objected) -> Option<(Slot, AccountKey, knot_cobs::ReactionKind)> { + Some(( + objected.parent?, + objected.reacted?.actor, + objected.reacted?.kind, + )) + } + + fn contests(&self, object: &CobId, slot: &Slot) -> bool { + self.addressed.get(slot) != Some(object) + } + + fn answers(&self, object: &CobId) -> bool { + self.objects + .get(object) + .is_some_and(|objected| !self.contests(object, &objected.slot)) + } + + fn serving(&self, object: CobId) -> Option<&Objected> { + let objected = self.objects.get(&object)?; + self.answers(&object).then_some(objected) + } + + fn is_serving(&self, object: &CobId) -> bool { + self.objects.get(object).is_some_and(|objected| { + !self.contests(object, &objected.slot) && objected.version.is_some() + }) + } + + fn live_at(&self, slot: &Slot) -> bool { + self.addressed + .get(slot) + .is_some_and(|object| self.is_serving(object)) + } + + fn live_reaction( + &self, + subject: &Slot, + actor: AccountKey, + kind: knot_cobs::ReactionKind, + ) -> Option { + self.reacted + .get(&(*subject, actor, kind))? + .iter() + .copied() + .find(|object| self.is_serving(object)) + } + + fn listed(&self, ordered: &BTreeSet) -> Vec { + ordered + .iter() + .map(|ordered| ordered.object()) + .filter(|object| self.answers(object)) + .collect() + } + + fn state_of(&self, object: CobId, objected: &Objected) -> SocialEntryState { + self.contests(&object, &objected.slot) + .then_some(SocialEntryState::Contested) + .or_else(|| objected.version.map(SocialEntryState::Serving)) + .unwrap_or(SocialEntryState::Erased) + } + + fn serving_total(&self) -> StructureBytes { + StructureBytes::new( + self.charged + .iter() + .filter(|(object, _)| self.answers(object)) + .map(|(_, progress)| progress.serving.get()) + .fold(0, u64::saturating_add), + ) + } + + fn serving_of(&self, object: CobId) -> Option { + self.answers(&object) + .then(|| self.charged.get(&object)) + .flatten() + .map(|progress| progress.serving) + } +} + +#[derive(Debug, Clone, Copy)] +struct Progress { + through: ChangeId, + bytes: StructureBytes, + serving: StructureBytes, +} + +struct Projected { + bytes: StructureBytes, + serving: StructureBytes, + through: ChangeId, + objected: Option, +} + +impl Projected { + fn of( + base: StructureBytes, + served: StructureBytes, + delta: &Delta, + objected: Option, + ) -> Self { + let bytes = delta.changes.iter().fold(base.get(), |total, change| { + let bytes_of = change.payload().len() + change.record.bytes().map_or(0, <[u8]>::len); + total.saturating_add(bytes_of as u64) + }); + Self { + bytes: StructureBytes::new(bytes), + serving: served, + through: delta.tip, + objected, + } + } +} + +fn serving(delta: &Delta, version: Option, served: StructureBytes) -> StructureBytes { + let Some(version) = version else { + return StructureBytes::new(0); + }; + let standing = delta.changes.iter().rev().find_map(|change| { + let produces = + knot_cobs::Social::emission(&knot_cobs::decode_social(change.payload()).ok()?) + .produces()?; + (produces == version).then(|| change.record.bytes().map_or(0, <[u8]>::len) as u64) + }); + StructureBytes::new(standing.unwrap_or(served.get())) +} + +pub struct SocialGuard<'index> { + projection: &'index SocialProjection, + repo: RepoKey, +} + +impl Drop for SocialGuard<'_> { + fn drop(&mut self) { + self.projection.release(self.repo); + } +} + +pub(crate) struct SocialProjection { + repos: scc::HashMap, + guards: scc::HashMap, + locks: [Mutex<()>; REPO_LOCK_STRIPES], + residency: SocialResidencyBytes, + epoch: AtomicU64, +} + +impl SocialProjection { + pub(crate) fn new(residency: SocialResidencyBytes) -> Self { + Self { + repos: scc::HashMap::new(), + guards: scc::HashMap::new(), + locks: std::array::from_fn(|_| Mutex::new(())), + residency, + epoch: AtomicU64::new(0), + } + } + + fn repo_lock(&self, repo: RepoKey) -> &Mutex<()> { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + repo.hash(&mut hasher); + &self.locks[(hasher.finish() % REPO_LOCK_STRIPES as u64) as usize] + } + + fn advance(&self) -> TouchStamp { + TouchStamp::new(self.epoch.fetch_add(1, AtomicOrdering::Relaxed)) + } + + fn stamp(&self) -> TouchStamp { + TouchStamp::new(self.epoch.load(AtomicOrdering::Relaxed)) + } + + pub(crate) fn is_materialized(&self, interner: &Interner, repo: &RepoDid) -> bool { + interner + .repo(repo) + .is_some_and(|repo| self.repos.contains_sync(&repo)) + } + pub(crate) fn guard(&self, repo: RepoKey) -> SocialGuard<'_> { + let _guard = self + .guards + .entry_sync(repo) + .and_modify(|guards| *guards += 1) + .or_insert(1); + SocialGuard { + projection: self, + repo, + } + } + + fn release(&self, repo: RepoKey) { + self.guards + .update_sync(&repo, |_, guards| *guards = guards.saturating_sub(1)); + let _ = self.guards.remove_if_sync(&repo, |guards| *guards == 0); + } + + pub(crate) fn drop_repo(&self, repo: RepoKey) { + let lock = self.repo_lock(repo); + let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + self.repos.remove_sync(&repo); + } + + pub(crate) fn ensure_repo( + &self, + interner: &Interner, + store: &CobStore, + repo: RepoKey, + ) -> Result { + let lock = self.repo_lock(repo); + let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if self.repos.contains_sync(&repo) { + return Ok(false); + } + self.project_and_publish(interner, store, repo) + .map(|()| true) + } + + pub(crate) fn refresh_repo( + &self, + interner: &Interner, + store: &CobStore, + repo: RepoKey, + ) -> Result<(), IndexError> { + let lock = self.repo_lock(repo); + let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + self.project_and_publish(interner, store, repo) + } + + pub(crate) fn refresh_object( + &self, + interner: &Interner, + store: &CobStore, + repo: RepoKey, + type_name: &TypeName, + object: CobId, + ) -> Result<(), IndexError> { + let lock = self.repo_lock(repo); + let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if !is_social(type_name) { + return Ok(()); + } + let from = self + .repos + .read_sync(&repo, |_, social| social.progress(object)) + .flatten(); + let projected = + project_object(interner, store, type_name, object, from).inspect_err(|_| { + self.repos.remove_sync(&repo); + })?; + let stored = self + .repos + .update_sync(&repo, |_, social| { + social.install(object, projected); + social.touch(self.advance()); + }) + .is_some(); + if stored { + return Ok(()); + } + self.project_and_publish(interner, store, repo) + } + + fn project_and_publish( + &self, + interner: &Interner, + store: &CobStore, + repo: RepoKey, + ) -> Result<(), IndexError> { + let social = self.project_repo(interner, store).inspect_err(|_| { + self.repos.remove_sync(&repo); + })?; + self.publish(repo, social); + Ok(()) + } + + fn publish(&self, repo: RepoKey, social: RepoSocial) { + self.evict_past_residency(repo, social.structure); + match self.repos.entry_sync(repo) { + scc::hash_map::Entry::Occupied(mut occupied) => *occupied.get_mut() = social, + scc::hash_map::Entry::Vacant(vacant) => { + vacant.insert_entry(social); + } + } + } + + fn evict_past_residency(&self, keeping: RepoKey, arriving: StructureBytes) { + let budget = self.residency.get(); + let mut resident: Vec<(TouchStamp, RepoKey, StructureBytes)> = Vec::new(); + let mut total = arriving.get(); + self.repos.iter_sync(|repo, social| { + if *repo != keeping { + let structure = social.structure; + total = total.saturating_add(structure.get()); + resident.push(( + TouchStamp::new(social.touched.load(AtomicOrdering::Relaxed)), + *repo, + structure, + )); + } + true + }); + if total <= budget { + return; + } + resident.sort_unstable(); + let _ = resident + .into_iter() + .filter(|(_, repo, _)| !self.guards.contains_sync(repo)) + .try_fold(total, |total, (_, repo, bytes)| { + self.repos.remove_sync(&repo); + let left = total.saturating_sub(bytes.get()); + (left > budget) + .then_some(left) + .map_or(ControlFlow::Break(()), ControlFlow::Continue) + }); + } + + fn project_repo( + &self, + interner: &Interner, + store: &CobStore, + ) -> Result { + let projected = store + .cobs()? + .into_iter() + .filter(|(type_name, _)| is_social(type_name)) + .map(|(type_name, object)| { + project_object(interner, store, &type_name, object, None) + .map(|projected| (object, projected)) + }) + .collect::, IndexError>>()?; + Ok(projected.into_iter().fold( + RepoSocial::new(self.advance()), + |mut social, (object, projected)| { + social.install(object, projected); + social + }, + )) + } + + fn read( + &self, + interner: &Interner, + repo: &RepoDid, + read: impl FnOnce(&RepoSocial) -> T, + ) -> Resolved { + let Some(repo) = interner.repo(repo) else { + return Resolved::Warming; + }; + match self.repos.read_sync(&repo, |_, social| { + social.touch(self.stamp()); + read(social) + }) { + Some(value) => Resolved::Ready(value), + None => Resolved::Warming, + } + } + + pub(crate) fn object_at( + &self, + interner: &Interner, + repo: &RepoDid, + address: &RecordAddress, + ) -> Resolved> { + let slot = Slot::known(interner, address); + self.read(interner, repo, |social| { + slot.and_then(|slot| social.addressed.get(&slot).copied()) + }) + } + + pub(crate) fn labels_of( + &self, + interner: &Interner, + repo: &RepoDid, + subject: &RecordAddress, + ) -> Resolved> { + let slot = Slot::known(interner, subject); + self.read(interner, repo, |social| { + slot.filter(|slot| social.live_at(slot)) + .map_or_else(Vec::new, |slot| social.labels_of(&slot)) + }) + } + + pub(crate) fn address_of( + &self, + interner: &Interner, + repo: &RepoDid, + object: CobId, + ) -> Resolved> { + self.read(interner, repo, |social| { + social + .serving(object) + .map(|objected| objected.slot.address(interner)) + }) + } + + pub(crate) fn entries( + &self, + interner: &Interner, + repo: &RepoDid, + ) -> Resolved> { + self.read(interner, repo, |social| { + social + .objects + .iter() + .map(|(object, objected)| SocialEntry { + address: objected.slot.address(interner), + object: *object, + state: social.state_of(*object, objected), + }) + .collect() + }) + } + + pub(crate) fn of_kind( + &self, + interner: &Interner, + repo: &RepoDid, + collection: &RecordCollection, + ) -> Resolved> { + let key = interner.collection(collection); + self.read(interner, repo, |social| { + key.and_then(|key| social.numbered.get(&key)) + .map(|numbered| social.listed(numbered)) + .unwrap_or_default() + }) + } + + pub(crate) fn children_of( + &self, + interner: &Interner, + repo: &RepoDid, + parent: &RecordAddress, + ) -> Resolved> { + let slot = Slot::known(interner, parent); + self.read(interner, repo, |social| { + let Some(slot) = slot else { + return Vec::new(); + }; + if !social.live_at(&slot) { + return Vec::new(); + } + social + .children + .get(&slot) + .map(|kinds| social.listed(&kinds.values().flatten().copied().collect())) + .unwrap_or_default() + }) + } + + pub(crate) fn latest_child( + &self, + interner: &Interner, + repo: &RepoDid, + parent: &RecordAddress, + collection: &RecordCollection, + ) -> Resolved> { + let slot = Slot::known(interner, parent); + let kind = interner.collection(collection); + self.read(interner, repo, |social| { + let (Some(slot), Some(kind)) = (slot, kind) else { + return None; + }; + if !social.live_at(&slot) { + return None; + } + social + .children + .get(&slot)? + .get(&kind)? + .iter() + .rev() + .filter_map(|ordered| social.serving(ordered.object())) + .find(|objected| objected.version.is_some()) + .map(|objected| objected.slot.address(interner)) + }) + } + + pub(crate) fn reaction_of( + &self, + interner: &Interner, + repo: &RepoDid, + subject: &RecordAddress, + actor: &AccountDid, + kind: knot_cobs::ReactionKind, + ) -> Resolved> { + let slot = Slot::known(interner, subject); + let actor = interner.account(actor); + self.read(interner, repo, |social| { + let (Some(slot), Some(actor)) = (slot, actor) else { + return None; + }; + social.live_reaction(&slot, actor, kind) + }) + } + + pub(crate) fn highest_number( + &self, + interner: &Interner, + repo: &RepoDid, + collection: &RecordCollection, + ) -> Resolved> { + let key = interner.collection(collection); + self.read(interner, repo, |social| { + key.and_then(|key| social.numbered.get(&key)) + .and_then(|numbered| numbered.last().map(|Ordered(number, ..)| *number)) + }) + } + + pub(crate) fn structure( + &self, + interner: &Interner, + repo: &RepoDid, + ) -> Resolved { + self.read(interner, repo, |social| social.structure) + } + + pub(crate) fn serving_bytes( + &self, + interner: &Interner, + repo: &RepoDid, + ) -> Resolved { + self.read(interner, repo, RepoSocial::serving_total) + } + + pub(crate) fn serving_bytes_of( + &self, + interner: &Interner, + repo: &RepoDid, + object: CobId, + ) -> Resolved> { + self.read(interner, repo, |social| social.serving_of(object)) + } +} + +static PROJECTED: LazyLock<[RecordCollection; 7]> = LazyLock::new(knot_cobs::social_collections); + +fn is_social(type_name: &TypeName) -> bool { + PROJECTED + .iter() + .any(|collection| collection.as_str() == type_name.as_str()) +} + +fn with_latest_change( + store: &CobStore, + type_name: &TypeName, + object: CobId, + known: Option<&Change>, + state: &knot_cobs::SocialState, + place: impl FnOnce(&Change) -> Result, +) -> Result, IndexError> { + if state.object().is_none() { + return Ok(None); + } + let fetched = known + .is_none() + .then(|| store.changes_of(type_name, object, HistoryModel::Convergent, None)) + .transpose()?; + known + .or_else(|| fetched.as_ref().and_then(|fetched| fetched.changes.last())) + .map(place) + .transpose() +} + +fn label_placement( + store: &CobStore, + type_name: &TypeName, + object: CobId, + known: Option<&Change>, + state: &knot_cobs::SocialState, +) -> Result, IndexError> { + if type_name.as_str() != knot_cobs::TicketCob::collection().as_str() { + return Ok(None); + } + with_latest_change(store, type_name, object, known, state, |latest| { + let decoded = knot_cobs::TicketChange::decode(latest.payload()).map_err(|error| { + IndexError::Decode { + change: latest.id, + type_name: type_name.clone(), + reason: error.to_string(), + } + })?; + Ok(match decoded { + knot_cobs::SocialChange::Open { body, .. } + | knot_cobs::SocialChange::Edit { body, .. } => { + body.tangled_labels.map(Vec::into_boxed_slice) + } + knot_cobs::SocialChange::Erase { .. } => None, + }) + }) + .map(Option::flatten) +} + +fn project_object( + interner: &Interner, + store: &CobStore, + type_name: &TypeName, + object: CobId, + from: Option, +) -> Result { + match from.and_then(|progress| replay_from(interner, store, type_name, object, progress)) { + Some(projected) => Ok(projected), + None => rebuild_whole(interner, store, type_name, object), + } +} + +fn replay_from( + interner: &Interner, + store: &CobStore, + type_name: &TypeName, + object: CobId, + progress: Progress, +) -> Option { + let (state, _) = knot_cobs::social_state(store, type_name, object).ok()??; + let delta = store + .changes_of( + type_name, + object, + HistoryModel::Convergent, + Some(progress.through), + ) + .ok()?; + Some(Projected::of( + progress.bytes, + serving( + &delta, + state.object().and_then(|object| object.version()), + progress.serving, + ), + &delta, + objected( + interner, + &state, + reaction_placement(interner, store, type_name, object, None, &state).ok()?, + label_placement(store, type_name, object, None, &state).ok()?, + ), + )) +} + +fn rebuild_whole( + interner: &Interner, + store: &CobStore, + type_name: &TypeName, + object: CobId, +) -> Result { + let delta = store.changes_of(type_name, object, HistoryModel::Convergent, None)?; + let (state, served) = delta.changes.iter().try_fold( + (knot_cobs::SocialState::Unopened, StructureBytes::new(0)), + |(state, served), change| { + knot_cobs::decode_social(change.payload()) + .map(|decoded| { + let before = state.object().map(|object| object.version()); + let applied = state.apply(&decoded); + let accepted = applied + .object() + .map(|object| object.version()) + .is_some_and(|version| Some(version) != before); + let served = if accepted { + StructureBytes::new(change.record.bytes().map_or(0, <[u8]>::len) as u64) + } else { + served + }; + (applied, served) + }) + .map_err(|error| IndexError::Decode { + change: change.id, + type_name: type_name.clone(), + reason: error.to_string(), + }) + }, + )?; + Ok(Projected::of( + StructureBytes::new(0), + served, + &delta, + objected( + interner, + &state, + reaction_placement( + interner, + store, + type_name, + object, + delta.changes.last(), + &state, + )?, + label_placement(store, type_name, object, delta.changes.last(), &state)?, + ), + )) +} + +fn reaction_placement( + interner: &Interner, + store: &CobStore, + type_name: &TypeName, + object: CobId, + latest: Option<&Change>, + state: &knot_cobs::SocialState, +) -> Result, IndexError> { + if type_name.as_str() != knot_cobs::ReactionCob::collection().as_str() { + return Ok(None); + } + with_latest_change(store, type_name, object, latest, state, |latest| { + let decoded = knot_cobs::ReactionChange::decode(latest.payload()).map_err(|error| { + IndexError::Decode { + change: latest.id, + type_name: type_name.clone(), + reason: error.to_string(), + } + })?; + Ok(match decoded { + knot_cobs::SocialChange::Open { body, .. } => { + state.object().map(|live| ReactionPlacement { + actor: interner.intern_account(live.author()), + kind: body.kind, + }) + } + knot_cobs::SocialChange::Edit { .. } | knot_cobs::SocialChange::Erase { .. } => None, + }) + }) + .map(Option::flatten) +} + +fn objected( + interner: &Interner, + state: &knot_cobs::SocialState, + reacted: Option, + labeled: Option, +) -> Option { + state.object().map(|object| Objected { + slot: Slot::of(interner, object.address()), + number: object.number(), + parent: object.parent().map(|parent| Slot::of(interner, parent)), + reacted, + labeled, + created_at: object.created_at(), + version: object.version(), + }) +} + +pub(crate) fn next_number(highest: Option) -> Option { + highest.map_or(Some(SubjectNumber::FIRST), SubjectNumber::after) +} diff --git a/knot2/crates/knot-index/tests/common/mod.rs b/knot2/crates/knot-index/tests/common/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..915c2dddd32944703eca975b3932f35486ef6cf9 --- /dev/null +++ b/knot2/crates/knot-index/tests/common/mod.rs @@ -0,0 +1,714 @@ +#![allow(dead_code)] + +use std::path::PathBuf; + +use knot_cob::{ + Approved, Authored, Authorize, ChangePayload, Checkpoint, CobHome, CobStore, KnotAuthored, + Materialized, Record, WriteContext, +}; +use knot_cobs::{ + CollaboratorsChange, Creation, Editing, Erasure, Grant, MembersChange, Registration, + RegistryChange, Removal, TicketChange, TicketCob, +}; +use knot_git::{Layout, Repo}; +use knot_index::{Index, KeyBudget}; +use knot_record::comment::CommentRef; +use knot_record::pull::PatchRecord; +use knot_record::reaction::ReactionRecord; +use knot_record::ticket::TicketState; +use knot_runtime::{K256Signer, SeededEntropy}; +use knot_types::{ + AccountDid, Blocked, CobId, ContributionPermission, ContributionPolicy, KnotId, LabelKey, Oid, + OwnerDid, RecordAddress, RecordBody, RecordCid, RecordCollection, RecordRkey, RepoDid, + RepoName, RepoRkey, RepoRole, SubjectNumber, TypeName, UnixSeconds, dag_cbor_cid, +}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use tempfile::TempDir; + +pub fn acc(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{suffix}")).unwrap() +} + +pub fn own(suffix: &str) -> OwnerDid { + OwnerDid::new(format!("did:plc:{suffix}")).unwrap() +} + +pub fn repo_did(suffix: &str) -> RepoDid { + RepoDid::new(format!("did:plc:{suffix}")).unwrap() +} + +pub fn rkey(value: &str) -> RepoRkey { + RepoRkey::new(value).unwrap() +} + +pub fn tid(value: &str) -> RecordRkey { + RecordRkey::new(value).unwrap() +} + +pub fn at(seconds: i64) -> UnixSeconds { + UnixSeconds::new(seconds) +} + +fn num(number: u32) -> SubjectNumber { + SubjectNumber::new(number).expect("Subject is numbered from one") +} + +pub fn label_set(keys: &[&str]) -> Vec { + keys.iter() + .map(|key| LabelKey::new(*key).unwrap()) + .collect() +} + +pub fn meta_home() -> CobHome { + CobHome::from(&KnotId::new("did:web:knot.nel.pet").unwrap()) +} + +pub fn grant(subject: &str, added_by: &str, seconds: i64) -> Grant { + Grant { + subject: acc(subject), + added_by: acc(added_by), + created_at: at(seconds), + } +} + +pub fn registration(owner_id: &str, key: &str, repo: &RepoDid, seconds: i64) -> Registration { + Registration { + owner: own(owner_id), + rkey: rkey(key), + name: RepoName::new(key).unwrap(), + repo: repo.clone(), + created_at: at(seconds), + policy: ContributionPolicy::default(), + fostered: false, + retired_at: None, + } +} + +pub fn registration_with_name( + owner_id: &str, + key: &str, + display: &str, + repo: &RepoDid, + seconds: i64, +) -> Registration { + Registration { + owner: own(owner_id), + rkey: rkey(key), + name: RepoName::new(display).unwrap(), + repo: repo.clone(), + created_at: at(seconds), + policy: ContributionPolicy::default(), + fostered: false, + retired_at: None, + } +} + +pub struct World { + _dir: TempDir, + pub meta_path: PathBuf, + pub layout: Layout, + pub signer: K256Signer, +} + +impl World { + pub fn new() -> Self { + Self::seeded(1) + } + + pub fn seeded(seed: u64) -> Self { + let dir = tempfile::tempdir().unwrap(); + let meta_path = dir.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(dir.path().join("repos")); + let signer = K256Signer::generate(&SeededEntropy::new(seed)); + Self { + _dir: dir, + meta_path, + layout, + signer, + } + } + + pub fn index(&self) -> Index { + Index::new(&self.meta_path, self.layout.clone()) + } + + pub fn index_within(&self, budget: KeyBudget) -> Index { + Index::with_key_budget(&self.meta_path, self.layout.clone(), budget) + } + + pub fn on_repo(&self, repo: &RepoDid, object: CobId, change: &P, t: i64) { + let git = self.layout.open(repo).unwrap(); + CobStore::new(&git) + .update(&CobHome::from(repo), object, change, &self.signer, at(t)) + .unwrap(); + } + + pub fn seed_members(&self) -> CobId { + let meta = Repo::open(&self.meta_path).unwrap(); + let store = CobStore::new(&meta); + let created = store + .create( + &meta_home(), + &MembersChange::Add(grant("nel", "nel", 1)), + &self.signer, + at(1), + ) + .unwrap(); + store + .update( + &meta_home(), + created.object, + &MembersChange::Add(grant("olaren", "nel", 2)), + &self.signer, + at(2), + ) + .unwrap(); + created.object + } + + pub fn add_member(&self, object: CobId, subject: &str, seconds: i64) { + let meta = Repo::open(&self.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .update( + &meta_home(), + object, + &MembersChange::Add(grant(subject, "nel", seconds)), + &self.signer, + at(seconds), + ) + .unwrap(); + } + + pub fn seed_registry(&self, repo: &RepoDid) -> CobId { + let meta = Repo::open(&self.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .create( + &meta_home(), + &RegistryChange::Register(registration("nel", "anemone", repo, 1)), + &self.signer, + at(1), + ) + .unwrap() + .object + } + + pub fn register_extra(&self, repo: &RepoDid, key: &str, registry: CobId) { + self.register_owned(repo, key, "nel", registry); + } + + pub fn register_owned(&self, repo: &RepoDid, key: &str, owner: &str, registry: CobId) { + let meta = Repo::open(&self.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .update( + &meta_home(), + registry, + &RegistryChange::Register(registration(owner, key, repo, 2)), + &self.signer, + at(2), + ) + .unwrap(); + } + + pub fn seed_collaborator(&self, repo: &RepoDid, subject: &str) -> CobId { + let git = self.layout.create(repo).unwrap(); + let store = CobStore::new(&git); + store + .create( + &CobHome::from(repo), + &CollaboratorsChange::Add(grant(subject, "nel", 1)), + &self.signer, + at(1), + ) + .unwrap() + .object + } + + pub fn remove_collaborator(&self, repo: &RepoDid, object: CobId, subject: &str, seconds: i64) { + let git = self.layout.open(repo).unwrap(); + let store = CobStore::new(&git); + store + .update( + &CobHome::from(repo), + object, + &CollaboratorsChange::Remove(Removal { + subject: acc(subject), + }), + &self.signer, + at(seconds), + ) + .unwrap(); + } + + pub fn open_cob( + &self, + repo: &RepoDid, + change: &E::Change, + record: Record, + seconds: i64, + ) -> CobId + where + E: Authorize, + E::Change: ChangePayload + Authored + Clone, + { + let git = self + .layout + .open(repo) + .unwrap_or_else(|_| self.layout.create(repo).unwrap()); + let author = change.author().clone(); + let cx = as_author(&author, repo); + CobStore::new(&git) + .create_approved::( + &CobHome::from(repo), + &Approved::signed_by( + &cx, + Materialized { + change: change.clone(), + record, + }, + ) + .unwrap(), + &self.signer, + at(seconds), + ) + .unwrap() + .object + } + + pub fn append_cob( + &self, + repo: &RepoDid, + object: CobId, + change: &E::Change, + record: Record, + seconds: i64, + ) where + E: Authorize + Checkpoint, + E::State: Serialize + DeserializeOwned, + E::Change: ChangePayload + Authored + Clone, + { + let git = self.layout.open(repo).unwrap(); + let author = change.author().clone(); + let cx = as_author(&author, repo); + CobStore::new(&git) + .append_approved::( + &CobHome::from(repo), + object, + &Approved::signed_by( + &cx, + Materialized { + change: change.clone(), + record, + }, + ) + .unwrap(), + &self.signer, + at(seconds), + ) + .unwrap(); + } + + pub fn open_ticket( + &self, + repo: &RepoDid, + change: &TicketChange, + record: Record, + seconds: i64, + ) -> CobId { + self.open_cob::(repo, change, record, seconds) + } + + pub fn append_ticket( + &self, + repo: &RepoDid, + object: CobId, + change: &TicketChange, + record: Record, + seconds: i64, + ) { + self.append_cob::(repo, object, change, record, seconds); + } + + #[allow(clippy::too_many_arguments)] + pub fn edit_ticket( + &self, + repo: &RepoDid, + object: CobId, + author: &AccountDid, + from: RecordCid, + text: &str, + tangled_labels: Option>, + seconds: i64, + ) { + let (record, _) = note_record(text); + self.append_ticket( + repo, + object, + &ticket_editing(author, from, text, tangled_labels, seconds), + Record::version(record), + seconds, + ); + } + + pub fn ticket( + &self, + repo: &RepoDid, + key: &str, + author: &AccountDid, + number: u32, + text: &str, + seconds: i64, + ) -> CobId { + self.opened( + repo, + key, + author, + number, + None, + knot_cobs::Ticket { + state: TicketState::Open, + tangled_labels: None, + assignees: None, + }, + text, + seconds, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn labeled_ticket( + &self, + repo: &RepoDid, + key: &str, + author: &AccountDid, + number: u32, + labels: Vec, + text: &str, + seconds: i64, + ) -> CobId { + self.opened( + repo, + key, + author, + number, + None, + knot_cobs::Ticket { + state: TicketState::Open, + tangled_labels: Some(labels), + assignees: None, + }, + text, + seconds, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn opened( + &self, + repo: &RepoDid, + key: &str, + author: &AccountDid, + number: u32, + parent: Option<&RecordAddress>, + body: B, + prose: &str, + seconds: i64, + ) -> CobId { + let (change, record) = + contribution_of(key, author, number, parent, body, prose, at(seconds)); + self.open_cob::>(repo, &change, Record::version(record), seconds) + } + + pub fn erase( + &self, + repo: &RepoDid, + object: CobId, + author: &AccountDid, + from: RecordCid, + seconds: i64, + ) { + self.append_cob::>( + repo, + object, + &erased(author, from, at(seconds)), + Record::Tombstone, + seconds, + ); + } + + #[allow(clippy::too_many_arguments)] + pub fn comment( + &self, + repo: &RepoDid, + key: &str, + author: &AccountDid, + number: u32, + parent: &RecordAddress, + text: &str, + seconds: i64, + ) -> CobId { + self.opened( + repo, + key, + author, + number, + Some(parent), + knot_cobs::Comment { + subject: RecordCid::new(dag_cbor_cid(b"subject")).unwrap(), + reply: None, + }, + text, + seconds, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn reaction( + &self, + repo: &RepoDid, + key: &str, + author: &AccountDid, + number: u32, + parent: &RecordAddress, + kind: knot_record::reaction::ReactionKind, + seconds: i64, + ) -> CobId { + let subject = CommentRef::new( + parent.clone(), + RecordCid::new(dag_cbor_cid(b"subject")).unwrap(), + ); + let bytes = ReactionRecord::new( + repo.clone(), + subject.clone(), + kind, + at(seconds), + author.clone(), + ) + .unwrap() + .body() + .unwrap(); + let change = contribution( + reaction_address(&tid(key)), + author, + number, + Some(parent.clone()), + bytes.cid(), + knot_cobs::Reaction { + subject: subject.cid(), + kind, + }, + at(seconds), + ); + self.open_cob::(repo, &change, Record::version(bytes), seconds) + } + + #[allow(clippy::too_many_arguments)] + pub fn patch( + &self, + repo: &RepoDid, + key: &str, + author: &AccountDid, + number: u32, + parent: &RecordAddress, + seconds: i64, + ) -> CobId { + let record = PatchRecord::new( + parent.rkey().clone(), + knot_git::CommitRange { + base: Oid::from_hex(&"1".repeat(40)).unwrap(), + head: Oid::from_hex(&"2".repeat(40)).unwrap(), + }, + None, + at(seconds), + author.clone(), + ) + .unwrap() + .body() + .unwrap(); + let change = contribution( + patch_address(&tid(key)), + author, + number, + Some(parent.clone()), + record.cid(), + knot_cobs::Patch, + at(seconds), + ); + self.open_cob::(repo, &change, Record::version(record), seconds) + } +} + +fn as_author<'a>(author: &'a AccountDid, repo: &'a RepoDid) -> WriteContext<'a> { + WriteContext::new( + author, + repo, + ContributionPermission::new( + Blocked::No, + RepoRole::Collaborator, + ContributionPolicy::Anyone, + ), + ) +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct NoteBody { + pub text: String, +} + +knot_cobs::social_cob! { + body = NoteBody, + change = NoteChange, + cob = NoteCob, + collection = "org.tangled.test.note", + opening = knot_cobs::Opening::Contribution, + stored = note_stored, +} + +fn note_stored(_: &knot_types::RecordBody) -> Result { + Ok(NoteBody { + text: String::new(), + }) +} + +macro_rules! addressed { + ($( + $collection:ident of $source:expr, $address:ident $(, $name:ident of $change:ty)? + )+) => {$( + pub fn $collection() -> RecordCollection { + $source + } + + pub fn $address(tid: &RecordRkey) -> RecordAddress { + RecordAddress::new($collection(), tid.clone()) + } + + $( + pub fn $name() -> TypeName { + <$change>::type_name() + } + )? + )+}; +} + +pub fn ticket_at(key: &str) -> RecordAddress { + ticket_address(&tid(key)) +} + +pub fn comment_at(key: &str) -> RecordAddress { + comment_address(&tid(key)) +} + +pub fn patch_at(key: &str) -> RecordAddress { + patch_address(&tid(key)) +} + +addressed! { + note_collection of NoteCob::collection(), note_address + ticket_collection of knot_record::ticket::ticket_collection(), ticket_address, + ticket_type_name of TicketChange + comment_collection of knot_record::comment::comment_collection(), comment_address, + comment_type_name of knot_cobs::CommentChange + reaction_collection of knot_record::reaction::reaction_collection(), reaction_address + label_def_collection of knot_record::label::label_def_collection(), + label_def_address + patch_collection of knot_record::pull::patch_collection(), patch_address, + patch_type_name of knot_cobs::PatchChange +} + +pub fn note_record(text: &str) -> (RecordBody, RecordCid) { + let body = RecordBody::new( + serde_ipld_dagcbor::to_vec(&NoteBody { + text: text.to_owned(), + }) + .unwrap(), + ) + .unwrap(); + let cid = body.cid(); + (body, cid) +} + +pub fn contribution( + address: RecordAddress, + author: &AccountDid, + number: u32, + parent: Option, + produces: RecordCid, + body: B, + seconds: UnixSeconds, +) -> knot_cobs::SocialChange { + knot_cobs::SocialChange::Open { + creation: Creation { + address, + author: author.clone(), + number: num(number), + parent, + created_at: seconds, + produces, + }, + body, + } +} + +pub fn erased( + author: &AccountDid, + from: RecordCid, + seconds: UnixSeconds, +) -> knot_cobs::SocialChange { + knot_cobs::SocialChange::Erase { + erasure: Erasure { + author: author.clone(), + from, + erased_at: seconds, + }, + } +} + +pub fn contribution_of( + key: &str, + author: &AccountDid, + number: u32, + parent: Option<&RecordAddress>, + body: B, + prose: &str, + seconds: UnixSeconds, +) -> (knot_cobs::SocialChange, RecordBody) { + let (bytes, produces) = note_record(prose); + ( + contribution( + RecordAddress::new(RecordCollection::new(B::COLLECTION).unwrap(), tid(key)), + author, + number, + parent.cloned(), + produces, + body, + seconds, + ), + bytes, + ) +} + +pub fn ticket_editing( + author: &AccountDid, + from: RecordCid, + text: &str, + tangled_labels: Option>, + seconds: i64, +) -> TicketChange { + TicketChange::Edit { + editing: Editing { + author: author.clone(), + from, + produces: note_record(text).1, + edited_at: at(seconds), + }, + body: knot_cobs::Ticket { + state: TicketState::Open, + tangled_labels, + assignees: None, + }, + } +} diff --git a/knot2/crates/knot-index/tests/lifecycle.rs b/knot2/crates/knot-index/tests/lifecycle.rs new file mode 100644 index 0000000000000000000000000000000000000000..cfdbdfea51dc9f84e5b49569f29f584ecd2f601e --- /dev/null +++ b/knot2/crates/knot-index/tests/lifecycle.rs @@ -0,0 +1,448 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Accept, CollaboratorsChange, CollaboratorsCob, Invite, MembersChange}; +use knot_git::Repo; +use knot_index::{Coverage, Entry, IndexCoverage, IndexError, Resolved}; +use knot_types::{AccountDid, OfferedKey}; + +mod common; +use common::{World, acc, at, grant, meta_home, own, repo_did, rkey}; + +#[test] +fn rebuild_materializes_members_and_registry_and_collaborators_materialize_on_access() { + let world = World::new(); + let repo = repo_did("squid"); + world.seed_members(); + world.seed_registry(&repo); + world.seed_collaborator(&repo, "lyna"); + + let index = world.index(); + index.rebuild().unwrap(); + + let empty = World::new(); + let empty_index = empty.index(); + empty_index.rebuild().unwrap(); + assert_eq!( + empty_index.effective_member(&acc("nel")), + Resolved::Ready(false), + "a boot over a meta repo with no cobs reads empty, not warming" + ); + assert_eq!( + empty_index.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Ready(None) + ); + + assert_eq!(index.effective_member(&acc("nel")), Resolved::Ready(true)); + assert_eq!( + index.effective_member(&acc("olaren")), + Resolved::Ready(true) + ); + assert_eq!(index.effective_member(&acc("teq")), Resolved::Ready(false)); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Ready(Some(repo.clone())) + ); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("nautilus")), + Resolved::Ready(None) + ); + assert_eq!( + index.effective_collaborator(&repo, &acc("lyna")), + Resolved::Warming, + "rebuild doesn't materialize collaborators, so the grant list reads warming until first access" + ); + + index.ensure_collaborators(&repo).unwrap(); + assert_eq!( + index.effective_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true) + ); + assert_eq!( + index.effective_collaborator(&repo, &acc("bailey")), + Resolved::Ready(false) + ); + assert_eq!( + index.coverage(), + IndexCoverage { + members: Coverage::Ready, + blocklist: Coverage::Ready, + collaborators: Coverage::Ready, + registry: Coverage::Ready, + keys: Coverage::Warming, + } + ); +} + +#[test] +fn every_accessor_fails_closed_while_warming() { + let world = World::new(); + world.seed_members(); + world.seed_collaborator(&repo_did("squid"), "lyna"); + let index = world.index(); + + assert_eq!(index.effective_member(&acc("nel")), Resolved::Warming); + assert_eq!(index.member_entries(), Resolved::Warming); + assert_eq!( + index.effective_collaborator(&repo_did("squid"), &acc("lyna")), + Resolved::Warming + ); + assert_eq!( + index.collaborator_entries(&repo_did("squid")), + Resolved::Warming, + "A grant list that hasn't materialized fails closed" + ); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Warming + ); + assert_eq!( + index.owner_of(&repo_did("squid")), + Resolved::Warming, + "repo lookup before rebuild fails closed" + ); + assert_eq!(index.coverage().members, Coverage::Warming); + + assert_eq!( + index.owner_of_key(&OfferedKey::from_bytes(vec![1, 2, 3]), at(0)), + Resolved::Ready(None), + "a key lookup answers from the first request, even while the key set is warming" + ); +} + +fn asserted(added_by: &str, seconds: i64) -> Entry { + Entry { + added_by: acc(added_by), + created_at: at(seconds), + } +} + +#[test] +fn member_entries_record_provenance() { + let world = World::new(); + world.seed_members(); + let index = world.index(); + index.rebuild().unwrap(); + assert_eq!( + index.member_entries(), + Resolved::Ready(vec![ + (acc("nel"), asserted("nel", 1)), + (acc("olaren"), asserted("nel", 2)), + ]) + ); +} + +#[test] +fn a_re_added_member_keeps_the_first_provenance() { + let world = World::new(); + let members = world.seed_members(); + let index = world.index(); + index.rebuild().unwrap(); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .update( + &meta_home(), + members, + &MembersChange::Add(grant("olaren", "teq", 9)), + &world.signer, + at(9), + ) + .unwrap(); + index.refresh_members().unwrap(); + + assert_eq!( + index.member_entries(), + Resolved::Ready(vec![ + (acc("nel"), asserted("nel", 1)), + (acc("olaren"), asserted("nel", 2)), + ]), + "duplicate add never rewrites original provenance, matching the canonical grant list" + ); +} + +#[test] +fn collaborator_entries_match_the_canonical_grant_list() { + let world = World::new(); + let repo = repo_did("squid"); + world.seed_registry(&repo); + let object = world.seed_collaborator(&repo, "lyna"); + let git = world.layout.open(&repo).unwrap(); + let store = CobStore::new(&git); + store + .update( + &CobHome::from(&repo), + object, + &CollaboratorsChange::Add(grant("bailey", "olaren", 2)), + &world.signer, + at(2), + ) + .unwrap(); + world.remove_collaborator(&repo, object, "lyna", 3); + store + .update( + &CobHome::from(&repo), + object, + &CollaboratorsChange::Add(grant("lyna", "teq", 5)), + &world.signer, + at(5), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + index.ensure_collaborators(&repo).unwrap(); + + let canonical = store.get::(object).unwrap(); + let expected: Vec<(AccountDid, Entry)> = canonical + .state() + .entries() + .map(|(subject, entry)| (subject.clone(), entry.clone())) + .collect(); + assert_eq!( + index.collaborator_entries(&repo), + Resolved::Ready(expected), + "projected entries disagree with the canonical Evaluate replay" + ); +} + +#[test] +fn a_legacy_invite_admits_and_its_acceptance_changes_nothing() { + let world = World::new(); + let members = world.seed_members(); + let repo = repo_did("squid"); + world.seed_registry(&repo); + let collaborators = world.seed_collaborator(&repo, "lyna"); + let meta = Repo::open(&world.meta_path).unwrap(); + let meta_store = CobStore::new(&meta); + let git = world.layout.open(&repo).unwrap(); + let repo_store = CobStore::new(&git); + let home = CobHome::from(&repo); + let on_knot = |change: MembersChange, seconds: i64| { + meta_store + .update(&meta_home(), members, &change, &world.signer, at(seconds)) + .unwrap(); + }; + let on_repo = |change: CollaboratorsChange, seconds: i64| { + repo_store + .update(&home, collaborators, &change, &world.signer, at(seconds)) + .unwrap(); + }; + let invite = |added_by: &str| Invite(grant("teq", added_by, 4)); + let accepted = Accept { + subject: acc("teq"), + verified_at: at(6), + }; + + on_knot(MembersChange::Invite(invite("nel")), 4); + on_repo(CollaboratorsChange::Invite(invite("olaren")), 4); + let index = world.index(); + index.rebuild().unwrap(); + index.ensure_collaborators(&repo).unwrap(); + let acl = || { + ( + index.effective_member(&acc("teq")), + index.effective_collaborator(&repo, &acc("teq")), + index.effective_members(), + index.effective_collaborators(&repo), + ) + }; + + let admitted = ( + Resolved::Ready(true), + Resolved::Ready(true), + Resolved::Ready(vec![acc("nel"), acc("olaren"), acc("teq")]), + Resolved::Ready(vec![acc("lyna"), acc("teq")]), + ); + assert_eq!( + acl(), + admitted, + "an invite replayed out of a live chain is a membership, and it doesn't take an acceptance" + ); + + on_knot(MembersChange::Accept(accepted.clone()), 6); + on_repo(CollaboratorsChange::Accept(accepted), 6); + index.refresh_members().unwrap(); + index.refresh_collaborators(&repo).unwrap(); + assert_eq!( + acl(), + admitted, + "an acceptance from the consent model doesn't change the grant list" + ); +} + +#[test] +fn a_materialized_repo_serves_while_unaccessed_repos_stay_warming() { + let world = World::new(); + let present = repo_did("squid"); + let absent = repo_did("kelp"); + let registry = world.seed_registry(&present); + world.register_extra(&absent, "barnacle", registry); + world.seed_collaborator(&present, "lyna"); + + let index = world.index(); + index.rebuild().unwrap(); + assert_eq!( + index.coverage().collaborators, + Coverage::Ready, + "collaborators projection is operational from boot" + ); + index.ensure_collaborators(&present).unwrap(); + + assert_eq!( + index.effective_collaborator(&present, &acc("lyna")), + Resolved::Ready(true) + ); + assert_eq!( + index + .collaborator_entries(&present) + .map(|entries| entries.len()), + Resolved::Ready(1), + "an accessed repo serves its grant list" + ); + assert_eq!( + index.collaborator_entries(&absent), + Resolved::Warming, + "registered repo that was never accessed stays fail-closed until materialized" + ); + assert_eq!( + index.effective_collaborator(&repo_did("conch"), &acc("lyna")), + Resolved::Warming, + "a repo the index never read can't be checked, so access must be denied" + ); +} + +#[test] +fn an_ambiguous_meta_cob_fails_refresh_and_rebuild() { + let world = World::new(); + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .create( + &meta_home(), + &MembersChange::Add(grant("nel", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap(); + store + .create( + &meta_home(), + &MembersChange::Add(grant("olaren", "olaren", 2)), + &world.signer, + at(2), + ) + .unwrap(); + + let index = world.index(); + assert!(matches!( + index.refresh_members(), + Err(IndexError::Ambiguous { count: 2, .. }) + )); + assert!( + matches!(index.rebuild(), Err(IndexError::Ambiguous { .. })), + "broken meta COB fails whole boot instead of reporting partial one" + ); +} + +#[test] +fn a_repo_missing_on_disk_does_not_fail_the_boot_and_isolates_its_materialization() { + let world = World::new(); + let present = repo_did("squid"); + let absent = repo_did("kelp"); + world.seed_members(); + let registry = world.seed_registry(&present); + world.register_extra(&absent, "barnacle", registry); + world.seed_collaborator(&present, "lyna"); + + let index = world.index(); + index.rebuild().expect( + "boot materializes members and registry only, so a missing repo dir never fails it", + ); + + index.ensure_collaborators(&present).unwrap(); + assert_eq!( + index.effective_collaborator(&present, &acc("lyna")), + Resolved::Ready(true) + ); + assert!( + index.ensure_collaborators(&absent).is_err(), + "materializing a repo with no dir on disk fails for that repo alone" + ); + assert_eq!( + index.effective_collaborator(&absent, &acc("lyna")), + Resolved::Warming, + "repo the index couldn't materialize stays fail-closed" + ); +} + +#[test] +fn concurrent_refreshes_of_distinct_repos_all_land() { + let world = World::new(); + let repos = ["squid", "clam", "whelk", "conch"]; + repos.iter().for_each(|repo| { + world.seed_collaborator(&repo_did(repo), "lyna"); + }); + + let index = Arc::new(world.index()); + std::thread::scope(|scope| { + repos.iter().for_each(|repo| { + let index = Arc::clone(&index); + let repo = repo_did(repo); + scope.spawn(move || index.refresh_collaborators(&repo).unwrap()); + }); + }); + + repos.iter().for_each(|repo| { + assert_eq!( + index.effective_collaborator(&repo_did(repo), &acc("lyna")), + Resolved::Ready(true) + ); + }); +} + +#[test] +fn a_refresh_is_eventually_consistent_not_an_atomic_snapshot() { + let world = World::new(); + let members = world.seed_members(); + let index = Arc::new(world.index()); + index.rebuild().unwrap(); + + (0..32).for_each(|i| world.add_member(members, &format!("m{i}"), 10 + i as i64)); + + let done = Arc::new(AtomicBool::new(false)); + std::thread::scope(|scope| { + let writer = Arc::clone(&index); + let writer_done = Arc::clone(&done); + scope.spawn(move || { + writer.refresh_members().unwrap(); + writer_done.store(true, Ordering::Release); + }); + + let reader = Arc::clone(&index); + let reader_done = Arc::clone(&done); + scope.spawn(move || { + while !reader_done.load(Ordering::Acquire) { + assert_eq!( + reader.effective_member(&acc("nel")), + Resolved::Ready(true), + "stable member stays visible and read never blocks on writer" + ); + assert!( + !reader.effective_member(&acc("m0")).is_warming(), + "already-ready projection serves reads mid-refresh, it never re-warms" + ); + } + }); + }); + + (0..32).for_each(|i| { + assert_eq!( + index.effective_member(&acc(&format!("m{i}"))), + Resolved::Ready(true), + "once writer returns, whole delta has converged" + ); + }); +} diff --git a/knot2/crates/knot-index/tests/meta_repo.rs b/knot2/crates/knot-index/tests/meta_repo.rs new file mode 100644 index 0000000000000000000000000000000000000000..fe78f310e8cedcfda2f8feab402fde7ced0fcd54 --- /dev/null +++ b/knot2/crates/knot-index/tests/meta_repo.rs @@ -0,0 +1,174 @@ +use knot_cob::{ChangePayload, Checkpoint, CobHome, CobStore}; +use knot_cobs::{ + CollaboratorsChange, MembersChange, Registration, RegistryChange, RepoRegistryCob, +}; +use knot_git::{Layout, Repo}; +use knot_index::{Index, Resolved}; +use knot_runtime::{K256Signer, SeededEntropy}; +use knot_types::{KnotId, RepoDid, RepoName}; + +mod common; +use common::{acc, at, grant, own, rkey}; + +#[test] +fn the_meta_repo_round_trips_every_projection_from_git() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let knot = KnotId::new("did:web:oyster.cafe").unwrap(); + let signer = K256Signer::generate(&SeededEntropy::new(1)); + + let meta = layout.bootstrap_meta(&knot).unwrap(); + let store = CobStore::new(&meta); + let members = store + .create( + &CobHome::from(&knot), + &MembersChange::Add(grant("nel", "nel", 1)), + &signer, + at(1), + ) + .unwrap(); + store + .update( + &CobHome::from(&knot), + members.object, + &MembersChange::Add(grant("olaren", "nel", 2)), + &signer, + at(2), + ) + .unwrap(); + + let repo = RepoDid::new("did:plc:squid").unwrap(); + let shadow = RepoDid::new("did:plc:feralpearl").unwrap(); + let registry = store + .create( + &CobHome::from(&knot), + &RegistryChange::Register(Registration { + owner: own("nel"), + rkey: rkey("anemone"), + name: RepoName::new("anemone").unwrap(), + repo: repo.clone(), + created_at: at(1), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }), + &signer, + at(1), + ) + .unwrap(); + store + .update( + &CobHome::from(&knot), + registry.object, + &RegistryChange::Register(Registration { + owner: own("nel"), + rkey: rkey("pearl"), + name: RepoName::new("pearl").unwrap(), + repo: shadow.clone(), + created_at: at(2), + policy: knot_types::ContributionPolicy::default(), + fostered: true, + retired_at: None, + }), + &signer, + at(2), + ) + .unwrap(); + + let git = layout.create(&repo).unwrap(); + CobStore::new(&git) + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &signer, + at(1), + ) + .unwrap(); + + let meta_path = layout.meta_path(&knot).unwrap(); + let boot = || { + let index = Index::new(meta_path.clone(), layout.clone()); + index.rebuild().unwrap(); + index.warm_collaborators(); + index + }; + + let first = boot(); + assert_eq!(first.effective_member(&acc("nel")), Resolved::Ready(true)); + assert_eq!( + first.hosted_repos(), + vec![shadow.clone(), repo.clone()], + "registry should be able to host native and fostered repos" + ); + assert_eq!( + first.native_repos(), + vec![repo.clone()], + "PLC sweeper's candidateset shouldn't include fostered repos" + ); + assert_eq!( + first.effective_member(&acc("olaren")), + Resolved::Ready(true) + ); + assert_eq!( + first.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Ready(Some(repo.clone())) + ); + assert_eq!( + first.effective_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true) + ); + + let second = boot(); + ["nel", "olaren", "lyna", "stranger"] + .into_iter() + .for_each(|who| { + assert_eq!( + first.effective_member(&acc(who)), + second.effective_member(&acc(who)) + ); + assert_eq!( + first.effective_collaborator(&repo, &acc(who)), + second.effective_collaborator(&repo, &acc(who)), + ); + }); + assert_eq!( + first.resolve_repo(&own("nel"), &rkey("anemone")), + second.resolve_repo(&own("nel"), &rkey("anemone")), + ); + assert_eq!( + second.native_repos(), + vec![repo.clone()], + "a rebuilt index still leaves the fostered repo out of the sweeper's candidates" + ); +} + +#[test] +fn when_rebuilding_we_save_checkpoint_that_can_be_used_later() { + let world = common::World::new(); + let repo = RepoDid::new("did:plc:anemone").unwrap(); + let registry = world.seed_registry(&repo); + let total = ::SNAPSHOT_STRIDE.get(); + (1..total).for_each(|index| { + world.register_extra( + &RepoDid::new(format!("did:plc:repo{index:03}")).unwrap(), + &format!("repo{index:03}"), + registry, + ); + }); + + world.index().rebuild().unwrap(); + let meta = Repo::open(&world.meta_path).unwrap(); + let checkpoint = knot_cob::checkpoint_ref_name(&RegistryChange::type_name(), registry).unwrap(); + assert!( + meta.find_ref(&checkpoint).unwrap().is_some(), + "When rebuilding, we saved a registry checkpoint" + ); + + let rebooted = world.index(); + rebooted.rebuild().unwrap(); + assert_eq!( + rebooted.hosted_repos().len(), + total, + "same exact length of checkpoint rebuild vs non-checkpoint" + ); +} diff --git a/knot2/crates/knot-index/tests/projections.rs b/knot2/crates/knot-index/tests/projections.rs new file mode 100644 index 0000000000000000000000000000000000000000..87547a3cffc12e4bbb80355bcd896c908aa3ba7f --- /dev/null +++ b/knot2/crates/knot-index/tests/projections.rs @@ -0,0 +1,2711 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use knot_cob::{ChangePayload, CobHome, CobStore, KnotAuthored, Record}; +use knot_cobs::{CollaboratorsChange, MembersChange, RegistryChange, Removal, Rename, RepoRef}; +use knot_git::{RefUpdate, Repo}; +use knot_index::{ + Coverage, HostedCoverage, IndexError, KeptAccounts, KeyBudget, KeyRecord, KeyReprieve, + KeyReprieved, KeyTtl, Resolved, SocialEntryState, SocialResidencyBytes, SweepInterval, +}; +use knot_types::{ + ClonePath, CobId, OfferedKey, RecordAddress, RecordCid, RefName, RepoDid, RepoName, + SubjectNumber, +}; +use serde::{Deserialize, Serialize}; + +mod common; +use common::{ + NoteBody, NoteCob, World, acc, at, comment_at, comment_collection, comment_type_name, + contribution_of, erased, grant, label_def_collection, label_set, meta_home, note_address, + note_record, own, patch_at, patch_collection, patch_type_name, reaction_collection, + registration, registration_with_name, repo_did, rkey, ticket_at, ticket_collection, + ticket_type_name, tid, +}; +use knot_record::reaction::ReactionKind; + +#[derive(Serialize, Deserialize)] +#[serde(tag = "op", content = "data", rename_all = "snake_case")] +enum BadMembers { + Explode(u8), +} +impl ChangePayload for BadMembers { + const TYPE: &'static str = "org.tangled.knot.member"; +} +impl KnotAuthored for BadMembers {} + +#[test] +fn a_concurrent_reader_never_sees_a_net_absent_subject() { + let world = World::new(); + let object = world.seed_members(); + let index = Arc::new(world.index()); + index.rebuild().unwrap(); + assert_eq!(index.effective_member(&acc("teq")), Resolved::Ready(false)); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .update( + &meta_home(), + object, + &MembersChange::Add(grant("teq", "nel", 10)), + &world.signer, + at(10), + ) + .unwrap(); + (0..1000).for_each(|i| { + store + .update( + &meta_home(), + object, + &MembersChange::Add(grant(&format!("f{i}"), "nel", 11 + i)), + &world.signer, + at(11 + i), + ) + .unwrap(); + }); + store + .update( + &meta_home(), + object, + &MembersChange::Remove(Removal { + subject: acc("teq"), + }), + &world.signer, + at(20_000), + ) + .unwrap(); + + let done = Arc::new(AtomicBool::new(false)); + let leaked = Arc::new(AtomicBool::new(false)); + std::thread::scope(|scope| { + let reader = Arc::clone(&index); + let reader_done = Arc::clone(&done); + let reader_leaked = Arc::clone(&leaked); + scope.spawn(move || { + while !reader_done.load(Ordering::Acquire) { + if reader.effective_member(&acc("teq")) == Resolved::Ready(true) { + reader_leaked.store(true, Ordering::Release); + } + } + }); + index.refresh_members().unwrap(); + done.store(true, Ordering::Release); + }); + + assert!( + !leaked.load(Ordering::Acquire), + "Net-absent subject is never written, so no reader can observe it mid-delta" + ); + assert_eq!(index.effective_member(&acc("teq")), Resolved::Ready(false)); + assert_eq!(index.effective_member(&acc("f0")), Resolved::Ready(true)); + assert_eq!(index.effective_member(&acc("f999")), Resolved::Ready(true)); +} + +#[test] +fn a_concurrent_reader_never_sees_a_collaborator_grant_list_emptied_mid_refresh() { + let world = World::new(); + let repo = repo_did("squid"); + let git = world.layout.create(&repo).unwrap(); + let store = CobStore::new(&git); + let object = store + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("anchor", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap() + .object; + (0..128).for_each(|i| { + store + .update( + &CobHome::from(&repo), + object, + &CollaboratorsChange::Add(grant(&format!("c{i}"), "nel", 2 + i)), + &world.signer, + at(2 + i), + ) + .unwrap(); + }); + + let index = Arc::new(world.index()); + index.rebuild().unwrap(); + index.ensure_collaborators(&repo).unwrap(); + assert_eq!( + index.effective_collaborator(&repo, &acc("anchor")), + Resolved::Ready(true) + ); + + let done = Arc::new(AtomicBool::new(false)); + let leaked = Arc::new(AtomicBool::new(false)); + std::thread::scope(|scope| { + let reader = Arc::clone(&index); + let reader_done = Arc::clone(&done); + let reader_leaked = Arc::clone(&leaked); + let target = repo.clone(); + scope.spawn(move || { + while !reader_done.load(Ordering::Acquire) { + if reader.effective_collaborator(&target, &acc("anchor")) != Resolved::Ready(true) { + reader_leaked.store(true, Ordering::Release); + } + } + }); + (0..500).for_each(|_| index.refresh_collaborators(&repo).unwrap()); + done.store(true, Ordering::Release); + }); + + assert!( + !leaked.load(Ordering::Acquire), + "a concurrent reader never sees anchor absent or warming mid-refresh, since the in-place \ + `mem::take` runs under the per-repo lock" + ); + assert_eq!( + index.effective_collaborator(&repo, &acc("anchor")), + Resolved::Ready(true) + ); +} + +fn cob_ref(type_name: &str, object: CobId) -> RefName { + RefName::new(format!("refs/cobs/{type_name}/{}", object.oid())).unwrap() +} + +#[test] +fn a_diverged_collaborators_tip_purges_the_grant_list_instead_of_serving_it_stale() { + let world = World::new(); + let repo = repo_did("squid"); + let git = world.layout.create(&repo).unwrap(); + let store = CobStore::new(&git); + let created = store + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap(); + let tip = store + .update( + &CobHome::from(&repo), + created.object, + &CollaboratorsChange::Add(grant("bailey", "nel", 2)), + &world.signer, + at(2), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + index.ensure_collaborators(&repo).unwrap(); + assert_eq!( + index.effective_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true) + ); + + git.update_ref(&RefUpdate::Update { + name: cob_ref(CollaboratorsChange::TYPE, created.object), + old: tip.oid(), + new: created.object.oid(), + }) + .unwrap(); + + assert!( + index.refresh_collaborators(&repo).is_err(), + "Tip that doesn't descend from the materialized tip anymore is structural error" + ); + assert_eq!( + index.effective_collaborator(&repo, &acc("lyna")), + Resolved::Warming, + "diverged COB tip purges the grant list and fails closed without serving \ + pre-divergence collaborators" + ); +} + +#[test] +fn a_diverged_members_tip_fails_closed_to_warming() { + let world = World::new(); + let object = world.seed_members(); + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let tip = store + .update( + &meta_home(), + object, + &MembersChange::Add(grant("teq", "nel", 3)), + &world.signer, + at(3), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + assert_eq!(index.effective_member(&acc("nel")), Resolved::Ready(true)); + + meta.update_ref(&RefUpdate::Update { + name: cob_ref(MembersChange::TYPE, object), + old: tip.oid(), + new: object.oid(), + }) + .unwrap(); + + assert!( + index.refresh_members().is_err(), + "Tip that doesn't descend from the materialized tip anymore is structural error" + ); + assert_eq!(index.coverage().members, Coverage::Warming); + assert_eq!( + index.effective_member(&acc("nel")), + Resolved::Warming, + "diverged members COB fails projection closed instead of serving stale members" + ); +} + +#[test] +fn an_undecodable_change_fails_closed_with_no_partial_apply() { + let world = World::new(); + let object = world.seed_members(); + let index = world.index(); + index.rebuild().unwrap(); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .update( + &meta_home(), + object, + &MembersChange::Add(grant("teq", "nel", 3)), + &world.signer, + at(3), + ) + .unwrap(); + store + .update( + &meta_home(), + object, + &BadMembers::Explode(0), + &world.signer, + at(4), + ) + .unwrap(); + store + .update( + &meta_home(), + object, + &MembersChange::Remove(Removal { + subject: acc("teq"), + }), + &world.signer, + at(5), + ) + .unwrap(); + + assert!(matches!( + index.refresh_members(), + Err(IndexError::Decode { .. }) + )); + + assert_eq!(index.coverage().members, Coverage::Warming); + assert_eq!( + index.effective_member(&acc("teq")), + Resolved::Warming, + "no partial apply: teq from pre-error change was never committed" + ); + assert_eq!( + index.effective_member(&acc("nel")), + Resolved::Warming, + "structurally broken COB fails whole projection closed" + ); + + assert!(matches!( + index.refresh_members(), + Err(IndexError::Decode { .. }) + )); + assert_eq!(index.coverage().members, Coverage::Warming); +} + +#[test] +fn deregister_purges_collaborators_fail_closed() { + let world = World::new(); + let repo = repo_did("clam"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let registry = store + .create( + &meta_home(), + &RegistryChange::Register(registration("nel", "anemone", &repo, 1)), + &world.signer, + at(1), + ) + .unwrap() + .object; + + let git = world.layout.create(&repo).unwrap(); + let cstore = CobStore::new(&git); + cstore + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + index.warm_collaborators(); + assert_eq!( + index.effective_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true) + ); + + store + .update( + &meta_home(), + registry, + &RegistryChange::Deregister(RepoRef { + owner: own("nel"), + rkey: rkey("anemone"), + }), + &world.signer, + at(2), + ) + .unwrap(); + index.refresh_registry().unwrap(); + + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Ready(None) + ); + assert_eq!( + index.effective_collaborator(&repo, &acc("lyna")), + Resolved::Warming, + "Deregistered repo's collaborators are purged and fail closed, not served stale" + ); +} + +#[test] +fn a_renamed_repo_keeps_both_rkeys_and_its_collaborators() { + let world = World::new(); + let repo = repo_did("squid"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let registry = store + .create( + &meta_home(), + &RegistryChange::Register(registration("nel", "anemone", &repo, 1)), + &world.signer, + at(1), + ) + .unwrap() + .object; + + let git = world.layout.create(&repo).unwrap(); + CobStore::new(&git) + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + index.warm_collaborators(); + assert_eq!(index.rkey_of(&repo), Resolved::Ready(Some(rkey("anemone")))); + + store + .update( + &meta_home(), + registry, + &RegistryChange::Rename(Rename { + owner: own("nel"), + rkey: rkey("barnacle"), + name: RepoName::new("barnacle").unwrap(), + repo: repo.clone(), + }), + &world.signer, + at(2), + ) + .unwrap(); + index.refresh_registry().unwrap(); + + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Ready(Some(repo.clone())), + "Prior rkey keeps resolving as alias after rename is delta-applied" + ); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("barnacle")), + Resolved::Ready(Some(repo.clone())) + ); + assert_eq!( + index.rkey_of(&repo), + Resolved::Ready(Some(rkey("barnacle"))), + "New rkey is canonical" + ); + assert_eq!( + index.effective_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true), + "Rename never evacuates repo, so its collaborators survive" + ); + + store + .update( + &meta_home(), + registry, + &RegistryChange::Deregister(RepoRef { + owner: own("nel"), + rkey: rkey("anemone"), + }), + &world.signer, + at(3), + ) + .unwrap(); + index.refresh_registry().unwrap(); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("barnacle")), + Resolved::Ready(None), + "Deregistering through retained alias removes repo and every alias" + ); + assert_eq!( + index.effective_collaborator(&repo, &acc("lyna")), + Resolved::Warming, + "Deregistered repo's collaborators are evacuated" + ); +} + +#[test] +fn a_repo_moved_within_one_delta_is_not_evacuated() { + let world = World::new(); + let repo = repo_did("squid"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let registry = store + .create( + &meta_home(), + &RegistryChange::Register(registration("nel", "anemone", &repo, 1)), + &world.signer, + at(1), + ) + .unwrap() + .object; + + let git = world.layout.create(&repo).unwrap(); + CobStore::new(&git) + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + index.warm_collaborators(); + assert_eq!( + index.effective_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true) + ); + + store + .update( + &meta_home(), + registry, + &RegistryChange::Deregister(RepoRef { + owner: own("nel"), + rkey: rkey("anemone"), + }), + &world.signer, + at(2), + ) + .unwrap(); + store + .update( + &meta_home(), + registry, + &RegistryChange::Register(registration("nel", "barnacle", &repo, 3)), + &world.signer, + at(3), + ) + .unwrap(); + index.refresh_registry().unwrap(); + + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Ready(None) + ); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("barnacle")), + Resolved::Ready(Some(repo.clone())) + ); + assert_eq!( + index.effective_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true), + "deregister and re-register within single delta leaves repo hosted, so its collaborators survive" + ); +} + +#[test] +fn recording_and_retaining_track_who_still_publishes_each_key() { + let world = World::new(); + let index = world.index(); + let key = |i: u32| OfferedKey::from_bytes(i.to_le_bytes().to_vec()); + + index + .keys() + .record(&acc("nel"), vec![key(0), key(1)], hour(0)); + index + .keys() + .record(&acc("nel"), vec![key(1), key(2)], hour(0)); + assert_eq!( + index.owner_of_key(&key(0), at(0)), + Resolved::Ready(None), + "A key the account stopped publishing stops resolving to it" + ); + assert_eq!( + index.owner_of_key(&key(1), at(0)), + Resolved::Ready(Some(acc("nel"))), + "A key present in both sets survives the replacement" + ); + + index + .keys() + .record(&acc("cuttle"), vec![key(1), key(3)], hour(0)); + assert_eq!( + index + .keys() + .publisher_among(&[acc("cuttle")], &key(1), at(0)), + Some(acc("cuttle")), + "a deploy key two accounts publish must answer for the candidate that may push here, \ + whichever of them the knot happened to read first" + ); + assert_eq!( + index.keys().publisher_among(&[acc("teq")], &key(1), at(0)), + None, + "The key mustn't answer for an account that doesn't publish it" + ); + + index.keys().retain(&KeptAccounts::new(vec![acc("nel")])); + assert_eq!( + index.owner_of_key(&key(3), at(0)), + Resolved::Ready(None), + "An account outside the grant set has its keys released" + ); + assert_eq!( + index.owner_of_key(&key(1), at(0)), + Resolved::Ready(Some(acc("nel"))), + "one account leaving the grant set mustn't take a shared deploy key away from the \ + account that still publishes it" + ); + + index.keys().record(&acc("nel"), vec![key(2)], hour(0)); + assert_eq!( + index.owner_of_key(&key(1), at(0)), + Resolved::Ready(None), + "With its last publisher not offering it anymore, the key stops answering for anybody" + ); +} + +#[test] +fn full_budget_reports_unheld_readings_and_then_stops_growing_entirely() { + let world = World::new(); + let index = world.index_within(KeyBudget::from_bytes(560)); + let fat = |seed: u8| OfferedKey::from_bytes(vec![seed; 48]); + + assert_eq!( + index.keys().record(&acc("nel"), vec![fat(1)], hour(0)), + KeyRecord::Stored + ); + assert!( + !index.keys().any_unheld(), + "A set that kept every reading it took can vouch for its misses" + ); + + assert_eq!( + index.keys().record(&acc("cuttle"), vec![fat(2)], hour(0)), + KeyRecord::Unheld, + "a budget too small for another account's keys still has room to record that the \ + knot read the account" + ); + assert!( + index.keys().any_unheld(), + "a reading the budget couldn't fit means an unknown key may still belong to a \ + candidate, so the handshake must defer to the push check instead of refusing" + ); + assert!( + !index.keys().is_fresh(&acc("cuttle"), at(0)), + "the refused account stays stale, so the fill keeps reporting the set incomplete and \ + the knot keeps checking its pushes against its PDS" + ); + assert_eq!( + index.owner_of_key(&fat(2), at(0)), + Resolved::Ready(None), + "A key the budget refused mustn't answer for anybody" + ); + + assert_eq!( + index.keys().record(&acc("teq"), vec![fat(3)], hour(0)), + KeyRecord::Saturated, + "A knot whose grant set publishes more key bytes than it budgeted for must stop growing" + ); + assert_eq!( + index.owner_of_key(&fat(1), at(0)), + Resolved::Ready(Some(acc("nel"))), + "Refusing the accounts that didn't fit mustn't release the accounts already on file" + ); + + index.keys().retain(&KeptAccounts::new(vec![acc("cuttle")])); + assert!( + index.keys().any_unheld(), + "Releasing a different account mustn't clear the report while the unheld reading stays" + ); + assert_eq!( + index.keys().record(&acc("cuttle"), vec![fat(2)], hour(0)), + KeyRecord::Stored, + "The budget frees up with the accounts that leave the grant set" + ); + assert!( + !index.keys().any_unheld(), + "with every reading back on file the handshake can refuse unknown keys outright" + ); + + assert_eq!( + index.keys().record(&acc("teq"), vec![fat(3)], hour(0)), + KeyRecord::Unheld + ); + assert_eq!( + index + .keys() + .reprieve(&acc("teq"), at(21_601), grace(), hour(21_601)), + KeyReprieved::Exhausted + ); + assert!( + !index.keys().any_unheld(), + "an account the knot gave up rereading is recorded with an empty key set, so its \ + unheld reading mustn't keep the handshake open" + ); +} + +#[test] +fn a_pass_renews_at_the_lease_halfway_and_rereads_once_per_miss_outside_the_floor() { + let (_world, index) = materialized(); + index + .keys() + .record(&acc("nel"), vec![OfferedKey::from_bytes(vec![7])], hour(0)); + + assert!( + matches!(index.keys().work(at(0), sweep()), Resolved::Ready(work) + if work.due.is_empty() && work.suspected.is_empty()), + "with every account's keys read inside the ttl the fill won't fetch anything" + ); + + index.keys().note_miss(); + let Resolved::Ready(work) = index.keys().work(at(100), sweep()) else { + panic!("The registry is materialized, so the knot knows who may push"); + }; + assert_eq!( + work.suspected.len(), + 1, + "A key the accounts on file don't publish is the only evidence the knot gets that an \ + account published a new key, so the next pass must reread rather than wait out the TTL" + ); + assert!( + work.due.is_empty(), + "a reread a stranger asked for must stay separate from the reads coverage waits on, or \ + the knot spends a whole pass on it without renewing anything" + ); + assert!( + matches!(index.keys().work(at(100), sweep()), Resolved::Ready(work) if work.suspected.is_empty()), + "one miss is worth one reread, so a stranger offering keys can't set the fill's pace" + ); + + index.keys().note_miss(); + assert!( + matches!(index.keys().work(at(159), sweep()), Resolved::Ready(work) if work.suspected.is_empty()), + "a stranger can offer an unsigned key for free, so a knot that swept a moment ago \ + mustn't spend another read at every PDS in the grant set" + ); + assert!( + matches!(index.keys().work(at(160), sweep()), Resolved::Ready(work) if work.suspected.len() == 1), + "the miss the floor delayed is honored once the floor has passed, so a key published \ + between passes is still picked up" + ); + + assert!( + matches!(index.keys().work(at(1_799), sweep()), Resolved::Ready(work) if work.due.is_empty()), + "an account inside the first half of its lease is left alone" + ); + let Resolved::Ready(work) = index.keys().work(at(1_800), sweep()) else { + panic!("The registry is materialized, so the knot knows who may push"); + }; + assert_eq!( + work.due.len(), + 1, + "the fill renews at the halfway mark, so a pass no longer than the lease it is working \ + to finishes before anything expires" + ); + assert!( + work.complete, + "the lease the account still has answers for it, so a knot busy renewing keeps \ + refusing keys the accounts on file don't publish instead of opening the handshake \ + to every stranger" + ); +} + +#[test] +fn an_unmaterialized_repo_keeps_the_pass_partial_and_an_unopenable_repo_never_grants() { + let world = World::new(); + let created = repo_did("squid"); + let never_created = repo_did("limpet"); + world.layout.create(&created).unwrap(); + let registry = world.seed_registry(&created); + world.register_owned(&never_created, "limpet", "cuttle", registry); + let index = world.index(); + index.refresh_registry().unwrap(); + index.refresh_collaborators(&created).unwrap(); + + let Resolved::Ready(work) = index.keys().work(at(0), sweep()) else { + panic!( + "One repo the knot hasn't materialized mustn't stop it filling the keys it can enumerate" + ); + }; + assert_eq!( + work.hosted, + HostedCoverage::Partial { unread: 1 }, + "the pass must report it read less than the whole grant list, or it evicts the accounts it \ + couldn't enumerate" + ); + assert!( + !work.due.is_empty(), + "the owner of the repo that did materialize still needs its keys read" + ); + assert!( + !work.complete, + "a partial grant list mustn't read as complete, or the knot rejects a \ + pusher whose repo it couldn't read" + ); + + assert_eq!( + index.warm_collaborators(), + 1, + "the repo registered without ever being created is what the knot can't open" + ); + let Resolved::Ready(work) = index.keys().work(at(0), sweep()) else { + panic!("The registry is materialized, so the knot knows who may push"); + }; + assert_eq!( + work.hosted, + HostedCoverage::Whole, + "a repo the knot can't open won't serve a push, so counting it as unread would leave \ + the set warming and the handshake open to every stranger for good" + ); + assert_eq!( + work.pushers.as_slice(), + [acc("nel")], + "only the repo the knot can open grants anybody" + ); + + index + .keys() + .record(&acc("nel"), vec![OfferedKey::from_bytes(vec![7])], hour(0)); + assert!( + pushers_complete(&index, at(0)), + "with the one readable repo's owner on file the set is complete, so the knot can go \ + back to refusing keys the accounts on file don't publish" + ); +} + +#[test] +fn an_acl_write_puts_the_key_set_back_to_warming_even_mid_pass() { + let (world, index) = materialized(); + index + .keys() + .record(&acc("nel"), vec![OfferedKey::from_bytes(vec![7])], hour(0)); + let Resolved::Ready(work) = index.keys().work(at(0), sweep()) else { + panic!("The registry is materialized, so the knot knows who may push"); + }; + assert!(work.complete, "The one pusher's keys are on file"); + index.keys().mark_ready(work.generation); + assert_eq!(index.keys().coverage(), Coverage::Ready); + + let members = world.seed_members(); + index.refresh_members().unwrap(); + assert_eq!( + index.keys().coverage(), + Coverage::Warming, + "the knot mustn't check a grant it has never read keys for against a set it calls \ + complete, or a new collaborator is refused until the next fill pass" + ); + + let Resolved::Ready(rereading) = index.keys().work(at(0), sweep()) else { + panic!("The registry is materialized, so the knot knows who may push"); + }; + world.add_member(members, "teq", 5); + index.refresh_members().unwrap(); + index.keys().mark_ready(rereading.generation); + assert_eq!( + index.keys().coverage(), + Coverage::Warming, + "a pass claims the grant set it read, so a grant written while it was reading keys \ + mustn't count as covered, or whoever it grants is refused at the handshake" + ); +} + +#[test] +fn an_account_the_budget_cant_fit_is_checked_against_its_pds_without_delaying_coverage() { + let world = World::new(); + let hosted = repo_did("squid"); + let second = repo_did("limpet"); + world.layout.create(&hosted).unwrap(); + world.layout.create(&second).unwrap(); + let registry = world.seed_registry(&hosted); + world.register_owned(&second, "limpet", "cuttle", registry); + let index = world.index_within(KeyBudget::from_bytes(560)); + index.refresh_registry().unwrap(); + index.warm_collaborators(); + let fat = |seed: u8| OfferedKey::from_bytes(vec![seed; 48]); + + assert_eq!( + index.keys().record(&acc("nel"), vec![fat(1)], hour(0)), + KeyRecord::Stored + ); + assert_eq!( + index.keys().record(&acc("cuttle"), vec![fat(2)], hour(0)), + KeyRecord::Unheld + ); + assert!( + pushers_complete(&index, at(0)), + "a full budget mustn't leave the set warming for good, or the knot spends the rest of \ + its life accepting every key offered to it" + ); + let Resolved::Ready(later) = index.keys().work(at(3_601), sweep()) else { + panic!("The registry is materialized, so the knot knows who may push"); + }; + assert!( + later.due.as_slice().contains(&acc("cuttle")), + "the knot tries the account again once the lease runs out, so a budget that frees up \ + starts keeping its keys. Every pass until then costs one read rather than a reread \ + of the whole grant set" + ); +} + +#[test] +fn a_key_read_outside_its_lease_stops_answering_for_its_publisher() { + let world = World::new(); + let index = world.index(); + let key = OfferedKey::from_bytes(vec![7]); + index.keys().record(&acc("nel"), vec![key.clone()], hour(0)); + + assert_eq!( + index.owner_of_key(&key, at(3_599)), + Resolved::Ready(Some(acc("nel"))), + "inside the lease the set answers for the account that published the key" + ); + assert_eq!( + index.keys().publisher_among(&[acc("nel")], &key, at(3_599)), + Some(acc("nel")) + ); + assert_eq!( + index.owner_of_key(&key, at(3_601)), + Resolved::Ready(None), + "past the lease the knot no longer knows the account publishes the key, so a key \ + revoked while the fill was behind mustn't keep clearing the handshake" + ); + assert_eq!( + index.keys().publisher_among(&[acc("nel")], &key, at(3_601)), + None, + "a push check is bound by the same lease, or a key revoked while the fill was behind \ + keeps authorizing pushes" + ); +} + +#[test] +fn a_warming_member_grant_list_still_yields_the_accounts_that_may_push() { + let (_world, index) = materialized(); + + let Resolved::Ready(work) = index.keys().work(at(0), sweep()) else { + panic!("The registry is materialized, so the knot knows who may push"); + }; + assert_eq!(work.pushers.len(), 1, "The one hosted repo has one owner"); + assert_eq!( + work.due.len(), + 1, + "the owner of the one hosted repo may push and hasn't had its keys read yet" + ); + assert!( + work.members.is_warming(), + "the knot still reads the push keys while the member grant list is warming" + ); +} + +#[test] +fn the_sweep_retains_keys_a_push_recorded_for_a_collaborator() { + let world = World::new(); + let repo = repo_did("squid"); + world.seed_collaborator(&repo, "lyna"); + world.seed_registry(&repo); + let index = world.index(); + index.rebuild().unwrap(); + index.warm_collaborators(); + + let kept = || match index.keys().work(at(0), sweep()).map(|work| work.members) { + Resolved::Ready(Resolved::Ready(members)) => members.kept, + _ => panic!("The registry or the member grant list is still warming"), + }; + index + .keys() + .record(&acc("lyna"), vec![OfferedKey::from_bytes(vec![7])], hour(0)); + let vouching = kept(); + assert!( + vouching.as_slice().contains(&acc("lyna")), + "a collaborator may push, so the sweep keeps the keys a push already read" + ); + + index.keys().retain(&vouching); + assert_eq!( + index.owner_of_key(&OfferedKey::from_bytes(vec![7]), at(0)), + Resolved::Ready(Some(acc("lyna"))), + "retain kept the key, so owner_of_key still finds lyna between sweeps" + ); +} + +#[test] +fn a_reprieve_coasts_on_the_last_read_until_its_budget_runs_out() { + let (_world, index) = materialized(); + let key = OfferedKey::from_bytes(vec![7]); + index.keys().record(&acc("nel"), vec![key.clone()], hour(0)); + + assert_eq!( + index + .keys() + .reprieve(&acc("nel"), at(10), grace(), hour(10)), + KeyReprieved::Extended + ); + assert!( + index.keys().is_fresh(&acc("nel"), at(3_599)), + "a reread the knot attempted early and couldn't finish mustn't cut the lease it already \ + has down to one retry, or a flaky PDS multiplies what the knot reads from it" + ); + + assert_eq!( + index + .keys() + .reprieve(&acc("nel"), at(4_000), grace(), hour(4_000)), + KeyReprieved::Extended + ); + assert!( + pushers_complete(&index, at(4_000)), + "the reprieve keeps the keys the knot last read, so one unreachable PDS mustn't reopen \ + the knot to every offered key" + ); + assert_eq!( + index.owner_of_key(&key, at(4_000)), + Resolved::Ready(Some(acc("nel"))) + ); + assert!( + !index.keys().is_fresh(&acc("nel"), at(4_301)), + "a reprieve is one retry's worth, so the knot tries the account again shortly" + ); + + assert_eq!( + index + .keys() + .reprieve(&acc("nel"), at(21_500), grace(), hour(21_500)), + KeyReprieved::Extended + ); + assert!( + !index.keys().is_fresh(&acc("nel"), at(21_600)), + "the last reprieve before the budget runs out mustn't stretch past it" + ); + + assert_eq!( + index + .keys() + .reprieve(&acc("nel"), at(21_601), grace(), hour(21_601)), + KeyReprieved::Exhausted, + "past the reprieve budget the knot stops coasting on keys it hasn't reread" + ); + assert_eq!( + index.owner_of_key(&key, at(21_601)), + Resolved::Ready(None), + "a key revoked while its PDS was unreachable mustn't keep authorizing pushes forever" + ); +} + +#[test] +fn an_account_the_knot_never_reads_stops_delaying_the_whole_set() { + let (_world, index) = materialized(); + let offered = OfferedKey::from_bytes(vec![7]); + + assert_eq!( + index.keys().reprieve(&acc("nel"), at(0), grace(), hour(0)), + KeyReprieved::Pending, + "an account the knot has never managed to read is still unread after a reprieve" + ); + assert!( + !pushers_complete(&index, at(0)), + "an unread account mustn't count toward a complete set, or the knot refuses the keys \ + the account publishes" + ); + assert!( + matches!(index.keys().work(at(299), sweep()), Resolved::Ready(work) if work.due.is_empty()), + "a PDS that just refused the knot is left alone until the retry comes round" + ); + assert!( + matches!(index.keys().work(at(400), sweep()), Resolved::Ready(work) if work.due.len() == 1), + "the knot retries an unread account every reprieve interval rather than every pass" + ); + + assert_eq!( + index + .keys() + .reprieve(&acc("nel"), at(21_601), grace(), hour(21_601)), + KeyReprieved::Exhausted, + "the reprieve budget runs from the first failure, so a PDS that never answers stops \ + being a pending read" + ); + assert!( + pushers_complete(&index, at(21_601)), + "one dead account mustn't keep the set warming forever, or the knot spends its whole \ + life accepting every key offered to it" + ); + assert_eq!( + index + .keys() + .publisher_among(&[acc("nel")], &offered, at(21_601)), + None, + "giving up on an account records what the knot knows, an empty key set" + ); +} + +#[test] +fn concurrent_reads_of_one_account_never_leave_a_key_answering_for_a_set_it_left() { + let world = World::new(); + let index = Arc::new(world.index()); + let hour = KeyTtl::from_secs(3_600).lease_from(at(0)); + let key = |i: u8| OfferedKey::from_bytes(vec![i]); + + (0..2_000u32).for_each(|round| { + let first = key((round % 7) as u8); + let second = key(7 + (round % 7) as u8); + let start = std::sync::Barrier::new(6); + std::thread::scope(|scope| { + (0..6).for_each(|slot| { + let index = Arc::clone(&index); + let keys = match slot % 2 { + 0 => vec![first.clone()], + _ => vec![second.clone()], + }; + let start = &start; + scope.spawn(move || { + start.wait(); + index.keys().record(&acc("nel"), keys, hour) + }); + }); + }); + + [first, second].iter().for_each(|key| { + let reverse = index.owner_of_key(key, at(0)); + let forward = index.keys().publisher_among(&[acc("nel")], key, at(0)); + assert_eq!( + reverse, + Resolved::Ready(forward), + "a key the account no longer publishes must stop answering for it, or two reads \ + arriving at once leave a revoked key authorizing pushes for good" + ); + }); + }); +} + +fn materialized() -> (World, knot_index::Index) { + let world = World::new(); + let repo = repo_did("squid"); + world.layout.create(&repo).unwrap(); + world.seed_registry(&repo); + let index = world.index(); + index.refresh_registry().unwrap(); + index.warm_collaborators(); + (world, index) +} + +fn grace() -> KeyReprieve { + KeyReprieve::from_secs(300, 21_600) +} + +fn hour(from: i64) -> knot_index::KeyLease { + KeyTtl::from_secs(3_600).lease_from(at(from)) +} + +fn sweep() -> SweepInterval { + SweepInterval::from_secs(60) +} + +fn pushers_complete(index: &knot_index::Index, now: knot_types::UnixSeconds) -> bool { + matches!(index.keys().work(now, sweep()), Resolved::Ready(work) if work.complete) +} + +fn path(raw: &str) -> ClonePath { + ClonePath::parse(raw).unwrap() +} + +#[test] +fn a_clone_path_resolves_by_name_when_the_record_key_is_a_tid() { + let world = World::new(); + let repo = repo_did("squid"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .create( + &meta_home(), + &RegistryChange::Register(registration_with_name( + "nel", + "3mizfnpxii522", + "substratum.cloud", + &repo, + 1, + )), + &world.signer, + at(1), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("substratum.cloud")), + Resolved::Ready(Some(repo.clone())), + "a PDS-native record key leaves the display name as the only human clone path" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("3mizfnpxii522")), + Resolved::Ready(Some(repo.clone())), + "the record key still resolves" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("substratum.cloud.git")), + Resolved::Ready(Some(repo)), + "the conventional .git suffix strips before the name lookup" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("periwinkle")), + Resolved::Ready(None) + ); +} + +#[test] +fn a_record_key_outranks_another_repos_name() { + let world = World::new(); + let by_name = repo_did("limpet"); + let by_rkey = repo_did("mussel"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let object = store + .create( + &meta_home(), + &RegistryChange::Register(registration_with_name( + "nel", "limpet", "mussel", &by_name, 1, + )), + &world.signer, + at(1), + ) + .unwrap() + .object; + store + .update( + &meta_home(), + object, + &RegistryChange::Register(registration_with_name( + "nel", "mussel", "scallop", &by_rkey, 2, + )), + &world.signer, + at(2), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("mussel")), + Resolved::Ready(Some(by_rkey)), + "a record key match wins over another repo holding that string as its name" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("scallop")), + Resolved::Ready(Some(repo_did("mussel"))) + ); +} + +#[test] +fn a_name_two_repos_share_resolves_to_the_older_registration() { + let world = World::new(); + let first = repo_did("whelk"); + let second = repo_did("conch"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let object = store + .create( + &meta_home(), + &RegistryChange::Register(registration_with_name( + "nel", + "3lubrptx57d22", + "kelp", + &first, + 1, + )), + &world.signer, + at(1), + ) + .unwrap() + .object; + store + .update( + &meta_home(), + object, + &RegistryChange::Register(registration_with_name( + "nel", + "3mqydma3re27z", + "kelp", + &second, + 2, + )), + &world.signer, + at(2), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("kelp")), + Resolved::Ready(Some(first.clone())), + "a contested name resolves to whichever repo registered first, ordered by \ + created_at so a cold seed and an incremental replay agree" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("3lubrptx57d22")), + Resolved::Ready(Some(first)), + "each record key stays unambiguous" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("3mqydma3re27z")), + Resolved::Ready(Some(second)), + "the repo that lost the name is still reachable by its record key" + ); +} + +#[test] +fn a_repo_registered_later_never_takes_a_contested_name_by_renaming_onto_it() { + let world = World::new(); + let holder = repo_did("whelk"); + let latecomer = repo_did("conch"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let object = store + .create( + &meta_home(), + &RegistryChange::Register(registration_with_name( + "nel", + "3lubrptx57d22", + "kelp", + &holder, + 1, + )), + &world.signer, + at(1), + ) + .unwrap() + .object; + store + .update( + &meta_home(), + object, + &RegistryChange::Register(registration_with_name( + "nel", + "3mqydma3re27z", + "uni", + &latecomer, + 2, + )), + &world.signer, + at(2), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + + store + .update( + &meta_home(), + object, + &RegistryChange::Rename(Rename { + owner: own("nel"), + rkey: rkey("3mqydma3re27z"), + name: RepoName::new("kelp").unwrap(), + repo: latecomer.clone(), + }), + &world.signer, + at(3), + ) + .unwrap(); + index.refresh_registry().unwrap(); + + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("kelp")), + Resolved::Ready(Some(holder)), + "a rename keeps the repo's original created_at, so renaming onto a name \ + another repo registered earlier can't take it" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("uni")), + Resolved::Ready(None), + "the renamed repo's previous name stops resolving" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("3mqydma3re27z")), + Resolved::Ready(Some(latecomer)) + ); +} + +#[test] +fn deregistering_the_older_registration_moves_a_shared_name_to_the_survivor() { + let world = World::new(); + let first = repo_did("whelk"); + let survivor = repo_did("conch"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let object = store + .create( + &meta_home(), + &RegistryChange::Register(registration_with_name( + "nel", + "3lubrptx57d22", + "kelp", + &first, + 1, + )), + &world.signer, + at(1), + ) + .unwrap() + .object; + store + .update( + &meta_home(), + object, + &RegistryChange::Register(registration_with_name( + "nel", + "3mqydma3re27z", + "kelp", + &survivor, + 2, + )), + &world.signer, + at(2), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("kelp")), + Resolved::Ready(Some(first)), + "the name resolves to the older registration while both exist" + ); + + store + .update( + &meta_home(), + object, + &RegistryChange::Deregister(RepoRef { + owner: own("nel"), + rkey: rkey("3lubrptx57d22"), + }), + &world.signer, + at(3), + ) + .unwrap(); + index.refresh_registry().unwrap(); + + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("kelp")), + Resolved::Ready(Some(survivor)), + "the name resolves to the remaining repo once the older one is deregistered" + ); +} + +#[test] +fn a_rename_moves_name_resolution_off_the_old_name() { + let world = World::new(); + let repo = repo_did("squid"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let object = store + .create( + &meta_home(), + &RegistryChange::Register(registration_with_name( + "nel", + "3mizfnpxii522", + "anemone", + &repo, + 1, + )), + &world.signer, + at(1), + ) + .unwrap() + .object; + + let index = world.index(); + index.rebuild().unwrap(); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("anemone")), + Resolved::Ready(Some(repo.clone())) + ); + + store + .update( + &meta_home(), + object, + &RegistryChange::Rename(Rename { + owner: own("nel"), + rkey: rkey("3mizfnpxii522"), + name: RepoName::new("barnacle").unwrap(), + repo: repo.clone(), + }), + &world.signer, + at(2), + ) + .unwrap(); + index.refresh_registry().unwrap(); + + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("barnacle")), + Resolved::Ready(Some(repo.clone())), + "the new name resolves after a rename that keeps the record key" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("anemone")), + Resolved::Ready(None), + "the superseded name stops resolving" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("3mizfnpxii522")), + Resolved::Ready(Some(repo)) + ); +} + +#[test] +fn a_rename_that_changes_only_the_record_key_keeps_the_name_resolving() { + let world = World::new(); + let repo = repo_did("squid"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let object = store + .create( + &meta_home(), + &RegistryChange::Register(registration_with_name( + "nel", + "3lubrptx57d22", + "kelp", + &repo, + 1, + )), + &world.signer, + at(1), + ) + .unwrap() + .object; + + let index = world.index(); + index.rebuild().unwrap(); + + store + .update( + &meta_home(), + object, + &RegistryChange::Rename(Rename { + owner: own("nel"), + rkey: rkey("3mqydma3re27z"), + name: RepoName::new("kelp").unwrap(), + repo: repo.clone(), + }), + &world.signer, + at(2), + ) + .unwrap(); + index.refresh_registry().unwrap(); + + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("kelp")), + Resolved::Ready(Some(repo.clone())), + "the unchanged name survives a rename that swaps the record key" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("3mqydma3re27z")), + Resolved::Ready(Some(repo.clone())) + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("3lubrptx57d22")), + Resolved::Ready(Some(repo)), + "the superseded record key keeps resolving through its retained alias" + ); +} + +#[test] +fn seeding_and_replaying_agree_on_name_resolution() { + let world = World::new(); + let first = repo_did("whelk"); + let second = repo_did("conch"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let object = store + .create( + &meta_home(), + &RegistryChange::Register(registration_with_name( + "nel", + "3lubrptx57d22", + "kelp", + &first, + 1, + )), + &world.signer, + at(1), + ) + .unwrap() + .object; + + let replayed = world.index(); + replayed.rebuild().unwrap(); + + store + .update( + &meta_home(), + object, + &RegistryChange::Register(registration_with_name( + "nel", + "3mqydma3re27z", + "uni", + &second, + 2, + )), + &world.signer, + at(2), + ) + .unwrap(); + replayed.refresh_registry().unwrap(); + + let seeded = world.index(); + seeded.rebuild().unwrap(); + + ["kelp", "uni", "3lubrptx57d22", "3mqydma3re27z", "nautilus"] + .into_iter() + .for_each(|segment| { + assert_eq!( + replayed.resolve_clone_path(&own("nel"), &path(segment)), + seeded.resolve_clone_path(&own("nel"), &path(segment)), + "incremental replay and a cold seed disagree on {segment}" + ); + }); +} + +#[test] +fn a_name_differing_from_its_record_key_only_by_case_resolves() { + let world = World::new(); + let repo = repo_did("squid"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .create( + &meta_home(), + &RegistryChange::Register(registration_with_name( + "nel", + "runic_lang", + "Runic_lang", + &repo, + 1, + )), + &world.signer, + at(1), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("Runic_lang")), + Resolved::Ready(Some(repo.clone())), + "the appview lowercases the record key but keeps the display name's case, \ + so the mixed-case path resolves by name" + ); + assert_eq!( + index.resolve_clone_path(&own("nel"), &path("runic_lang")), + Resolved::Ready(Some(repo)), + "the lowercased record key still resolves" + ); +} + +fn kelp_ticket() -> (World, RepoDid, CobId) { + let world = World::new(); + let repo = repo_did("squid"); + let object = world.ticket(&repo, "3lubrptx57d22", &acc("nel"), 1, "kelp", 10); + (world, repo, object) +} + +fn materialized_kelp() -> (World, RepoDid, CobId, knot_index::Index) { + let (world, repo, object) = kelp_ticket(); + let index = world.index(); + index.refresh_social(&repo).unwrap(); + (world, repo, object, index) +} + +fn assert_projections_agree( + incremental: &knot_index::Index, + cold: &knot_index::Index, + repo: &RepoDid, + objects: &[CobId], + addresses: &[RecordAddress], +) { + [ + ticket_collection(), + comment_collection(), + reaction_collection(), + label_def_collection(), + patch_collection(), + ] + .into_iter() + .for_each(|collection| { + assert_eq!( + incremental.social_of_kind(repo, &collection), + cold.social_of_kind(repo, &collection), + "Incremental replay and cold rebuild mustn't disagree about which {collection} objects exist" + ); + assert_eq!( + incremental.next_subject_number(repo, &collection), + cold.next_subject_number(repo, &collection), + "nor must they disagree about the next number {collection} hands out" + ); + }); + assert_eq!( + incremental.social_structure(repo), + cold.social_structure(repo), + "bytes repository has are the same, whichever way projection got there" + ); + assert_eq!( + incremental.social_entries(repo), + cold.social_entries(repo), + "and so should every object's address and serving state" + ); + objects.iter().for_each(|object| { + assert_eq!( + incremental.social_address_of(repo, *object), + cold.social_address_of(repo, *object) + ); + }); + addresses.iter().for_each(|at| { + assert_eq!( + incremental.social_object_at(repo, at), + cold.social_object_at(repo, at), + "incremental replay, and cold replay, disagree about which object serves at {at}" + ); + assert_eq!( + incremental.social_children_of(repo, at), + cold.social_children_of(repo, at), + "Incremental replay and cold replay disagree about children of {at}" + ); + assert_eq!( + incremental.social_labels_of(repo, at), + cold.social_labels_of(repo, at), + "Incremental replay and cold replay disagree about the current labels on {at}" + ); + [ + ticket_collection(), + comment_collection(), + reaction_collection(), + patch_collection(), + ] + .into_iter() + .for_each(|collection| { + assert_eq!( + incremental.social_latest_child(repo, at, &collection), + cold.social_latest_child(repo, at, &collection), + "Incremental replay and cold rebuild mustn't disagree about the latest \ + {collection} under {at}" + ); + }); + }); +} + +#[test] +fn social_projection_answers_by_address_by_kind_and_by_parent() { + let (world, repo, issue_object) = kelp_ticket(); + let second_object = world.ticket(&repo, "3lubrptx57d33", &acc("olaren"), 2, "urchin", 20); + let comment_object = world.comment( + &repo, + "3lubrptx57d44", + &acc("nel"), + 1, + &ticket_at("3lubrptx57d22"), + "kelp reply", + 30, + ); + + let index = world.index(); + index.refresh_social(&repo).unwrap(); + + assert_eq!( + index.social_object_at(&repo, &ticket_at("3lubrptx57d22")), + Resolved::Ready(Some(issue_object)), + "Rkey resolves to object that materializes there" + ); + assert_eq!( + index.social_address_of(&repo, issue_object), + Resolved::Ready(Some(ticket_at("3lubrptx57d22"))), + "and object resolves back to its address" + ); + assert_eq!( + index.social_of_kind(&repo, &ticket_collection()), + Resolved::Ready(vec![issue_object, second_object]), + "Kind must read in write order: number first, then createdAt" + ); + assert_eq!( + index.social_of_kind(&repo, &comment_collection()), + Resolved::Ready(vec![comment_object]), + "under its own collection, never the ticket's, a comment should read" + ); + assert_eq!( + index.social_children_of(&repo, &ticket_at("3lubrptx57d22")), + Resolved::Ready(vec![comment_object]), + "child names its parent's address, and parent reads its children back" + ); + assert_eq!( + index.social_latest_child(&repo, &ticket_at("3lubrptx57d22"), &comment_collection()), + Resolved::Ready(Some(comment_at("3lubrptx57d44"))), + "Latest comment is the one replay's order puts last" + ); + assert_eq!( + index.social_latest_child(&repo, &ticket_at("3lubrptx57d22"), &ticket_collection()), + Resolved::Ready(None), + "asking under a kind without children under this parent shouldn't hand back the \ + comment either" + ); + assert_eq!( + index.social_children_of(&repo, &ticket_at("3lubrptx57d33")), + Resolved::Ready(Vec::new()), + "Object nobody answered has no children" + ); +} + +#[test] +fn backdated_comment_still_ranks_latest_by_write_order() { + let (world, repo, _issue) = kelp_ticket(); + let parent = ticket_at("3lubrptx57d22"); + world.comment(&repo, "3lubrptx57d44", &acc("nel"), 1, &parent, "first", 30); + world.comment(&repo, "3lubrptx57d55", &acc("nel"), 2, &parent, "second", 5); + let index = world.index(); + index.refresh_social(&repo).unwrap(); + assert_eq!( + index.social_latest_child(&repo, &parent, &comment_collection()), + Resolved::Ready(Some(comment_at("3lubrptx57d55"))), + "the third write must be current even though its clock read earlier than the second's" + ); +} + +#[test] +fn erased_subject_stops_answering_children_and_comment_keeps_serving() { + let (world, repo, issue_object) = kelp_ticket(); + let parent = ticket_at("3lubrptx57d22"); + let comment_object = world.comment( + &repo, + "3lubrptx57d33", + &acc("nel"), + 1, + &parent, + "kelp reply", + 20, + ); + let index = world.index(); + index.refresh_social(&repo).unwrap(); + assert_eq!( + index.social_children_of(&repo, &parent), + Resolved::Ready(vec![comment_object]), + "while the issue exists, comments on it ought to be listed under it" + ); + assert_eq!( + index.social_latest_child(&repo, &parent, &comment_collection()), + Resolved::Ready(Some(comment_at("3lubrptx57d33"))), + "the newest comment under the issue should probably come back too" + ); + let before = state_of(&index, &repo, comment_object).expect("Index should contain the comment"); + + world.erase::( + &repo, + issue_object, + &acc("nel"), + version_of(&index, &repo, issue_object).expect("Issue should probably have a version"), + 30, + ); + index.refresh_social(&repo).unwrap(); + + assert_eq!( + index.social_children_of(&repo, &parent), + Resolved::Ready(Vec::new()), + "comments on a deleted issue mustn't be listed" + ); + assert_eq!( + index.social_latest_child(&repo, &parent, &comment_collection()), + Resolved::Ready(None), + "nor should the newest comment be listed" + ); + assert_eq!( + state_of(&index, &repo, comment_object), + Some(before), + "deleting the issue shouldn't change the comment's own record" + ); + assert_eq!( + index.social_object_at(&repo, &comment_at("3lubrptx57d33")), + Resolved::Ready(Some(comment_object)), + "the comment should still be findable by address" + ); +} + +#[test] +fn reaction_index_answers_live_reactions_only() { + let (world, repo, _issue) = kelp_ticket(); + let parent = ticket_at("3lubrptx57d22"); + let nels_like = world.reaction( + &repo, + "3lubrptx57d44", + &acc("nel"), + 1, + &parent, + ReactionKind::new("👍").unwrap(), + 20, + ); + let olarens_like = world.reaction( + &repo, + "3lubrptx57d55", + &acc("olaren"), + 2, + &parent, + ReactionKind::new("👍").unwrap(), + 30, + ); + let nels_heart = world.reaction( + &repo, + "3lubrptx57d66", + &acc("nel"), + 3, + &parent, + ReactionKind::new("❤️").unwrap(), + 40, + ); + let index = world.index(); + index.refresh_social(&repo).unwrap(); + let cold = world.index(); + cold.refresh_social(&repo).unwrap(); + + for (actor, kind, expected) in [ + ( + &acc("nel"), + ReactionKind::new("👍").unwrap(), + Some(nels_like), + ), + ( + &acc("olaren"), + ReactionKind::new("👍").unwrap(), + Some(olarens_like), + ), + ( + &acc("nel"), + ReactionKind::new("❤️").unwrap(), + Some(nels_heart), + ), + (&acc("olaren"), ReactionKind::new("❤️").unwrap(), None), + (&acc("nel"), ReactionKind::new("👀").unwrap(), None), + ] { + assert_eq!( + index.social_live_reaction(&repo, &parent, actor, kind), + Resolved::Ready(expected), + "existing reactions have to be found by actor, subject, & kind" + ); + assert_eq!( + index.social_live_reaction(&repo, &parent, actor, kind), + cold.social_live_reaction(&repo, &parent, actor, kind), + "an index rebuilt from scratch must match the incrementally updated index" + ); + } + + world.erase::( + &repo, + nels_like, + &acc("nel"), + version_of(&index, &repo, nels_like).expect("Reaction should have a version"), + 50, + ); + index.refresh_social(&repo).unwrap(); + assert_eq!( + index.social_live_reaction( + &repo, + &parent, + &acc("nel"), + ReactionKind::new("👍").unwrap() + ), + Resolved::Ready(None), + "a deleted reaction should probably stop being found" + ); + assert_eq!( + index.social_live_reaction( + &repo, + &parent, + &acc("olaren"), + ReactionKind::new("👍").unwrap() + ), + Resolved::Ready(Some(olarens_like)), + "another actor's same-kind reaction should still be found" + ); +} + +#[test] +fn untouched_repo_stays_warming_until_write_reaches_answer() { + let (world, repo, object) = kelp_ticket(); + let index = world.index(); + + assert_eq!( + index.social_object_at(&repo, &ticket_at("3lubrptx57d22")), + Resolved::Warming, + "unmaterialized repository fails closed, turning away write that reads it" + ); + assert_eq!( + index.next_subject_number(&repo, &ticket_collection()), + Resolved::Warming + ); + + index.ensure_social(&repo).unwrap(); + assert_eq!( + index.next_subject_number(&repo, &ticket_collection()), + Resolved::Ready(SubjectNumber::new(2)), + "Next number is one past the highest the projection has" + ); + + world.edit_ticket( + &repo, + object, + &acc("nel"), + note_record("kelp").1, + "kelp forest", + None, + 20, + ); + index.refresh_social(&repo).unwrap(); + assert_eq!( + version_of(&index, &repo, object), + Some(note_record("kelp forest").1), + "Projection follows version latest change produced" + ); + + world.erase::(&repo, object, &acc("nel"), note_record("kelp forest").1, 30); + index.refresh_social(&repo).unwrap(); + assert_eq!( + state_of(&index, &repo, object), + Some(SocialEntryState::Erased), + "Erased object doesn't have version" + ); + assert_eq!( + index.next_subject_number(&repo, &ticket_collection()), + Resolved::Ready(SubjectNumber::new(2)), + "Deletion leaves its number spent, as every forge does" + ); +} + +fn state_of(index: &knot_index::Index, repo: &RepoDid, object: CobId) -> Option { + let Resolved::Ready(entries) = index.social_entries(repo) else { + return None; + }; + entries + .into_iter() + .find(|entry| entry.object == object) + .map(|entry| entry.state) +} + +fn version_of(index: &knot_index::Index, repo: &RepoDid, object: CobId) -> Option { + state_of(index, repo, object).and_then(SocialEntryState::serving) +} + +#[test] +fn git_rebuild_and_lone_rebuilds_agree_with_incremental_replay() { + let (world, repo, object) = kelp_ticket(); + let untouched_object = world.ticket(&repo, "3lubrptx57d33", &acc("olaren"), 2, "urchin", 15); + let parent = ticket_at("3lubrptx57d22"); + let reply_object = world.comment( + &repo, + "3lubrptx57d44", + &acc("olaren"), + 1, + &parent, + "urchin reply", + 16, + ); + + let replayed = world.index(); + replayed.refresh_social(&repo).unwrap(); + world.edit_ticket( + &repo, + object, + &acc("nel"), + note_record("kelp").1, + "kelp forest", + None, + 20, + ); + replayed + .refresh_social_object(&repo, &ticket_type_name(), object) + .unwrap(); + + let reopened_object = world.comment( + &repo, + "3lubrptx57d55", + &acc("nel"), + 2, + &parent, + "kelp reply", + 21, + ); + replayed + .refresh_social_object(&repo, &comment_type_name(), reopened_object) + .unwrap(); + assert_eq!( + replayed.social_latest_child(&repo, &parent, &comment_collection()), + Resolved::Ready(Some(comment_at("3lubrptx57d55"))), + "a comment replayed on its own must still become the ticket's latest child" + ); + + let patch_object = world.patch(&repo, "3lubrptx57d66", &acc("nel"), 1, &parent, 22); + replayed + .refresh_social_object(&repo, &patch_type_name(), patch_object) + .unwrap(); + assert_eq!( + replayed.social_latest_child(&repo, &parent, &patch_collection()), + Resolved::Ready(Some(patch_at("3lubrptx57d66"))), + "a patch replayed on its own must still become the ticket's latest patch" + ); + + let cold = world.index(); + cold.refresh_social(&repo).unwrap(); + assert_projections_agree( + &replayed, + &cold, + &repo, + &[ + object, + untouched_object, + reply_object, + reopened_object, + patch_object, + ], + &[ + ticket_at("3lubrptx57d22"), + ticket_at("3lubrptx57d33"), + comment_at("3lubrptx57d44"), + comment_at("3lubrptx57d55"), + patch_at("3lubrptx57d66"), + ], + ); + assert_eq!( + cold.social_children_of(&repo, &parent), + Resolved::Ready(vec![reply_object, patch_object, reopened_object]), + "in the child index, each reply oughtta list under the object it answered" + ); + assert_eq!( + version_of(&cold, &repo, object), + Some(note_record("kelp forest").1), + "the tree is missing the record blob, so the version ought to come off the signed payload" + ); + assert_eq!( + cold.social_entries(&repo).map(|entries| entries + .into_iter() + .filter(|entry| entry.state.serving().is_some()) + .count()), + Resolved::Ready(5) + ); +} + +#[test] +fn ticket_labels_serve_the_tangled_set_and_an_unlabeled_ticket_serves_none() { + let world = World::new(); + let repo = repo_did("squid"); + world.labeled_ticket( + &repo, + "3lubrptx57d22", + &acc("nel"), + 1, + label_set(&["wontfix", "reviewer"]), + "kelp", + 10, + ); + world.ticket(&repo, "3lubrptx57d22z", &acc("nel"), 2, "kelp", 11); + + let index = world.index(); + index.refresh_social(&repo).unwrap(); + assert_eq!( + index.social_labels_of(&repo, &ticket_at("3lubrptx57d22")), + Resolved::Ready(label_set(&["wontfix", "reviewer"])), + "the ticket's tangledLabels is the live set" + ); + assert_eq!( + index.social_labels_of(&repo, &ticket_at("3lubrptx57d22z")), + Resolved::Ready(Vec::new()), + "a ticket without tangledLabels serves an empty set" + ); +} + +#[test] +fn incremental_label_refresh_matches_a_cold_replay() { + let world = World::new(); + let repo = repo_did("squid"); + let labels = label_set(&["wontfix", "reviewer"]); + let object = world.labeled_ticket(&repo, "3lubrptx57d22", &acc("nel"), 1, labels, "kelp", 10); + let subject = ticket_at("3lubrptx57d22"); + + let warmed = world.index(); + warmed.refresh_social(&repo).unwrap(); + let cold = world.index(); + cold.refresh_social(&repo).unwrap(); + assert_eq!( + cold.social_labels_of(&repo, &subject), + warmed.social_labels_of(&repo, &subject), + "a ticket replayed incrementally onto a warm projection serves the same tangled set \ + as a cold replay from the git history" + ); + + let labels = Some(label_set(&["reviewer"])); + world.edit_ticket( + &repo, + object, + &acc("nel"), + note_record("kelp").1, + "kelp forest", + labels, + 20, + ); + warmed + .refresh_social_object(&repo, &ticket_type_name(), object) + .unwrap(); + let replayed = world.index(); + replayed.refresh_social(&repo).unwrap(); + let edited = Resolved::Ready(label_set(&["reviewer"])); + assert_eq!( + warmed.social_labels_of(&repo, &subject), + edited, + "an edit's tangledLabels replaces the opened set on a warm projection" + ); + assert_eq!( + replayed.social_labels_of(&repo, &subject), + edited, + "and a cold replay off the git history serves the edited set too" + ); +} + +#[test] +fn edit_omitting_tangled_labels_clears_existing_set() { + let world = World::new(); + let repo = repo_did("squid"); + let object = world.labeled_ticket( + &repo, + "3lubrptx57d22", + &acc("nel"), + 1, + label_set(&["wontfix", "reviewer"]), + "kelp", + 10, + ); + let subject = ticket_at("3lubrptx57d22"); + + let index = world.index(); + index.refresh_social(&repo).unwrap(); + assert_eq!( + index.social_labels_of(&repo, &subject), + Resolved::Ready(label_set(&["wontfix", "reviewer"])), + "The indexed ticket serves its labels." + ); + + world.edit_ticket( + &repo, + object, + &acc("nel"), + note_record("kelp").1, + "kelp forest", + None, + 20, + ); + + let edited = world.index(); + edited.refresh_social(&repo).unwrap(); + assert_eq!( + edited.social_labels_of(&repo, &subject), + Resolved::Ready(Vec::new()), + "An edit that omits tangledLabels clears the existing set, so a client \ + must send the full live set on every ticket put." + ); +} + +#[test] +fn deleted_subject_stops_serving_its_labels() { + let world = World::new(); + let repo = repo_did("squid"); + let labels = label_set(&["wontfix"]); + let issue = world.labeled_ticket(&repo, "3lubrptx57d22", &acc("nel"), 1, labels, "kelp", 10); + + let subject = ticket_at("3lubrptx57d22"); + let index = world.index(); + index.refresh_social(&repo).unwrap(); + assert_eq!( + index.social_labels_of(&repo, &subject), + Resolved::Ready(label_set(&["wontfix"])), + "while the ticket stands, so does the label" + ); + + let (_, live) = note_record("kelp"); + world.erase::(&repo, issue, &acc("nel"), live, 30); + index + .refresh_social_object(&repo, &ticket_type_name(), issue) + .unwrap(); + assert_eq!( + index.social_labels_of(&repo, &subject), + Resolved::Ready(Vec::new()), + "a deleted subject stops serving the label set along with the children" + ); +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "op", content = "data", rename_all = "snake_case")] +enum BadIssue { + Explode(u8), +} +impl ChangePayload for BadIssue { + const TYPE: &'static str = "org.tangled.track.ticket"; +} +impl KnotAuthored for BadIssue {} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "op", content = "data", rename_all = "snake_case")] +enum BadNote { + Explode(u8), +} +impl ChangePayload for BadNote { + const TYPE: &'static str = "org.tangled.test.note"; +} +impl KnotAuthored for BadNote {} + +fn served(index: &knot_index::Index, repo: &RepoDid) -> u64 { + match index.social_serving_bytes(repo) { + Resolved::Ready(bytes) => bytes.get(), + Resolved::Warming => panic!("Projection oughtta be built"), + } +} + +#[test] +fn serving_bytes_answer_live_versions_and_erasure_frees_them() { + let (world, repo, object) = kelp_ticket(); + let index = world.index(); + index.refresh_social(&repo).unwrap(); + + let urchin = world.ticket(&repo, "3lubrptx57d33", &acc("olaren"), 2, "urchin", 15); + index + .refresh_social_object(&repo, &ticket_type_name(), urchin) + .unwrap(); + + let kelp = note_record("kelp").0.len() as u64; + let urchin_bytes = note_record("urchin").0.len() as u64; + let forest = note_record("kelp forest").0.len() as u64; + assert_eq!( + served(&index, &repo), + kelp + urchin_bytes, + "each object answers the bytes of the version it serves, and no past version counts" + ); + + world.edit_ticket( + &repo, + object, + &acc("nel"), + note_record("kelp").1, + "kelp forest", + None, + 20, + ); + index + .refresh_social_object(&repo, &ticket_type_name(), object) + .unwrap(); + assert_eq!( + served(&index, &repo), + forest + urchin_bytes, + "an edit answers its new version, freed of the one it replaced" + ); + + world.erase::(&repo, object, &acc("nel"), note_record("kelp forest").1, 30); + index + .refresh_social_object(&repo, &ticket_type_name(), object) + .unwrap(); + assert_eq!( + served(&index, &repo), + urchin_bytes, + "erasure frees the bytes" + ); + + index.refresh_social(&repo).unwrap(); + assert_eq!( + served(&index, &repo), + urchin_bytes, + "a cold rebuild answers what the incremental replay answered" + ); +} + +#[test] +fn contested_claims_stop_counting_toward_serving_bytes() { + let world = World::new(); + let repo = repo_did("squid"); + world.ticket(&repo, "3lubrptx57d22", &acc("nel"), 1, "kelp", 10); + + world.ticket(&repo, "3lubrptx57d22", &acc("olaren"), 2, "urchin", 20); + let index = world.index(); + index.refresh_social(&repo).unwrap(); + assert_eq!( + served(&index, &repo), + note_record("kelp").0.len() as u64, + "the claim the projection settled on answers its bytes; the contested one is free" + ); +} + +#[test] +fn unreadable_social_change_answers_warming_until_repo_or_object_refresh() { + let (world, repo, object, index) = materialized_kelp(); + assert_eq!( + index.social_object_at(&repo, &ticket_at("3lubrptx57d22")), + Resolved::Ready(Some(object)) + ); + + world.on_repo(&repo, object, &BadIssue::Explode(0), 11); + assert!(matches!( + index.refresh_social(&repo), + Err(IndexError::Decode { .. }) + )); + assert_eq!( + index.social_object_at(&repo, &ticket_at("3lubrptx57d22")), + Resolved::Warming, + "object projection can't read, whole repository goes unanswered, and nobody else gets handed address it claimed" + ); + assert_eq!( + index.next_subject_number(&repo, &ticket_collection()), + Resolved::Warming + ); + assert!(matches!( + index.refresh_social_object(&repo, &ticket_type_name(), object), + Err(IndexError::Decode { .. }) + )); + assert_eq!( + index.social_object_at(&repo, &ticket_at("3lubrptx57d22")), + Resolved::Warming, + "an object refresh that can't read git shouldn't leave a stale answer behind for the next write" + ); +} + +#[test] +fn unprojected_cob_skips_without_failing_repository() { + let world = World::new(); + let repo = repo_did("squid"); + world.seed_collaborator(&repo, "olaren"); + let object = world.ticket(&repo, "3lubrptx57d22", &acc("nel"), 1, "kelp", 10); + let (note, note_body) = contribution_of::( + "3lubrptx57d33", + &acc("nel"), + 1, + None, + NoteBody { + text: "foreign".to_owned(), + }, + "foreign", + at(11), + ); + let foreign = world.open_cob::(&repo, ¬e, Record::version(note_body), 11); + world.on_repo(&repo, foreign, &BadNote::Explode(0), 12); + + let index = world.index(); + index.refresh_social(&repo).unwrap(); + assert_eq!( + index.social_of_kind(&repo, &ticket_collection()), + Resolved::Ready(vec![object]), + "without becoming social objects, a grant list and a COB of a type this crate doesn't project \ + ought to share the repository, and an unreadable one of them shouldn't take the \ + repository down" + ); + assert_eq!( + index.social_object_at(&repo, ¬e_address(&tid("3lubrptx57d33"))), + Resolved::Ready(None), + "the foreign object shouldn't claim an address here either" + ); +} + +fn resident(multiple: u64) -> (World, Vec, knot_index::Index) { + let world = World::new(); + let repos: Vec<_> = ["squid", "anemone", "barnacle"] + .into_iter() + .map(repo_did) + .collect(); + repos.iter().for_each(|repo| { + world.ticket(repo, "3lubrptx57d22", &acc("nel"), 1, "kelp", 10); + }); + let measured = world.index(); + measured.ensure_social(&repos[0]).unwrap(); + let Resolved::Ready(one) = measured.social_structure(&repos[0]) else { + panic!("Projection oughtta be built"); + }; + let index = world + .index() + .with_social_residency(SocialResidencyBytes::new(one.get() * multiple)); + (world, repos, index) +} + +#[test] +fn eviction_past_residency_forgets_coldest_and_skips_guarded_repo() { + let (_world, repos, index) = resident(2); + repos + .iter() + .for_each(|repo| index.ensure_social(repo).unwrap()); + assert_eq!( + index.social_of_kind(&repos[0], &ticket_collection()), + Resolved::Warming, + "repository rebuilt longest ago ought to make room for a newcomer" + ); + assert!(matches!( + index.social_of_kind(&repos[2], &ticket_collection()), + Resolved::Ready(_) + )); + index.ensure_social(&repos[0]).unwrap(); + assert!( + matches!( + index.social_of_kind(&repos[0], &ticket_collection()), + Resolved::Ready(_) + ), + "touch must rebuild an evicted repository" + ); + + let (_world, repos, index) = resident(1); + let guard = index.guard_social(&repos[0]).unwrap(); + index.ensure_social(&repos[1]).unwrap(); + assert!( + matches!( + index.social_of_kind(&repos[0], &ticket_collection()), + Resolved::Ready(_) + ), + "a write keeping its repository resident oughtn't lose it to somebody else's publish" + ); + assert!( + matches!( + index.social_of_kind(&repos[1], &ticket_collection()), + Resolved::Ready(_) + ), + "the publish that couldn't make room should still serve what it rebuilt" + ); + + drop(guard); + index.ensure_social(&repos[2]).unwrap(); + assert_eq!( + index.social_of_kind(&repos[0], &ticket_collection()), + Resolved::Warming + ); + assert_eq!( + index.social_of_kind(&repos[1], &ticket_collection()), + Resolved::Warming, + "once the guard drops, eviction should run until the resident bytes fit" + ); +} + +#[test] +fn two_creates_racing_on_one_repo_both_take_the_same_number() { + let (world, repo, _first_object, index) = materialized_kelp(); + + let read = std::sync::Barrier::new(2); + let write = std::sync::Barrier::new(2); + let numbers: Vec = std::thread::scope(|scope| { + let handles: Vec<_> = ["3lubrptx57d33", "3lubrptx57d44"] + .into_iter() + .map(|key| { + let (index, world, repo, read, write) = (&index, &world, &repo, &read, &write); + scope.spawn(move || { + read.wait(); + let Resolved::Ready(Some(number)) = + index.next_subject_number(repo, &ticket_collection()) + else { + panic!("Projection is materialized before either create reads it"); + }; + write.wait(); + let object = world.ticket(repo, key, &acc("nel"), number.get(), "raced", 20); + index + .refresh_social_object(repo, &ticket_type_name(), object) + .unwrap(); + number + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().expect("Create thread runs to its answer")) + .collect() + }); + + assert_eq!( + numbers, + vec![ + SubjectNumber::new(2).unwrap(), + SubjectNumber::new(2).unwrap() + ], + "both creates read projection before either refreshed it: label #2 went" + ); + assert_eq!( + index.next_subject_number(&repo, &ticket_collection()), + Resolved::Ready(SubjectNumber::new(3)), + "Projection that took both refreshes counts past the shared label once" + ); + index.refresh_social(&repo).unwrap(); + assert_eq!( + index + .social_of_kind(&repo, &ticket_collection()) + .map(|objects| objects.len()), + Resolved::Ready(3), + "both raced objects exist, each with its own address, and only label collides" + ); +} + +#[test] +fn two_objects_claiming_one_address_leave_one_reachable() { + let (world, repo, first_object, index) = materialized_kelp(); + let second_object = world.ticket(&repo, "3lubrptx57d22", &acc("olaren"), 2, "squatting", 20); + index.refresh_social(&repo).unwrap(); + + assert_eq!( + index.social_object_at(&repo, &ticket_at("3lubrptx57d22")), + Resolved::Ready(Some(first_object)), + "Object that took its address first keeps said address, whatever order git lists refs in" + ); + assert_eq!( + index.social_of_kind(&repo, &ticket_collection()), + Resolved::Ready(vec![first_object]), + "lookup by address, and one by kind, agree, loser doesn't answer listings" + ); + assert_eq!( + state_of(&index, &repo, second_object), + Some(SocialEntryState::Contested), + "the loser stays in the projection as contested" + ); + assert_eq!( + index.social_address_of(&repo, second_object), + Resolved::Ready(None), + "but it doesn't answer its address, which keeps address_of and object_at one-to-one" + ); + assert_eq!( + index.social_address_of(&repo, first_object), + Resolved::Ready(Some(ticket_at("3lubrptx57d22"))) + ); + assert_eq!( + index.next_subject_number(&repo, &ticket_collection()), + Resolved::Ready(SubjectNumber::new(3)), + "still keeping its number, the loser of an address mustn't answer any listing, or the \ + next create numbers two objects the same" + ); + + let cold = world.index(); + cold.refresh_social(&repo).unwrap(); + assert_projections_agree( + &index, + &cold, + &repo, + &[first_object, second_object], + &[ticket_at("3lubrptx57d22")], + ); +} + +#[test] +fn refreshing_one_object_leaves_other_answers_where_cold_rebuild_puts_them() { + let (world, repo, incumbent, index) = materialized_kelp(); + let untouched = world.ticket(&repo, "3lubrptx57d33", &acc("nel"), 2, "shell", 20); + index.refresh_social(&repo).unwrap(); + let before = index.social_of_kind(&repo, &ticket_collection()); + + let loser = world.ticket(&repo, "3lubrptx57d22", &acc("olaren"), 3, "squatting", 30); + index + .refresh_social_object(&repo, &ticket_type_name(), loser) + .unwrap(); + + assert_eq!( + index.social_object_at(&repo, &ticket_at("3lubrptx57d22")), + Resolved::Ready(Some(incumbent)), + "replaying the latecomer alone doesn't hand it incumbent's address" + ); + assert_eq!( + index.social_of_kind(&repo, &ticket_collection()), + before, + "contested arrival leaves listing alone, and sibling's order where it was" + ); + assert_eq!( + index.social_object_at(&repo, &ticket_at("3lubrptx57d33")), + Resolved::Ready(Some(untouched)), + "and object nobody contested answers for itself" + ); + + let cold = world.index(); + cold.refresh_social(&repo).unwrap(); + assert_projections_agree( + &index, + &cold, + &repo, + &[incumbent, untouched, loser], + &[ticket_at("3lubrptx57d22"), ticket_at("3lubrptx57d33")], + ); +} + +#[test] +fn a_change_that_opens_no_object_still_costs_the_repository_its_bytes() { + let (world, repo, object, index) = materialized_kelp(); + let Resolved::Ready(opened) = index.social_structure(&repo) else { + panic!("Projection must be built"); + }; + + world.open_ticket( + &repo, + &erased(&acc("nel"), note_record("kelp").1, at(20)), + Record::Tombstone, + 20, + ); + index.refresh_social(&repo).unwrap(); + let Resolved::Ready(both) = index.social_structure(&repo) else { + panic!("Projection must be built"); + }; + assert!( + both > opened, + "History that never opens its object still has bytes, and the budget \ + that skipped them wouldn't bound anything" + ); + assert_eq!( + index.social_of_kind(&repo, &ticket_collection()), + Resolved::Ready(vec![object]), + "it shouldn't read as an object either" + ); +} diff --git a/knot2/crates/knot-index/tests/properties.rs b/knot2/crates/knot-index/tests/properties.rs new file mode 100644 index 0000000000000000000000000000000000000000..ff2fac617f41b3417d0058f2cc180c885a2a5845 --- /dev/null +++ b/knot2/crates/knot-index/tests/properties.rs @@ -0,0 +1,262 @@ +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{ + Accept, CollaboratorsChange, CollaboratorsCob, Grant, Invite, MembersChange, MembersCob, + Registration, RegistryChange, Removal, Rename, RepoRef, RepoRegistryCob, +}; +use knot_git::Repo; +use knot_index::{Index, Resolved}; +use knot_types::{AccountDid, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds}; +use proptest::prelude::*; + +mod common; +use common::{World, meta_home}; + +fn acc(n: u8) -> AccountDid { + AccountDid::new(format!("did:plc:s{n}")).unwrap() +} + +fn owner(n: u8) -> OwnerDid { + match n { + 0 => OwnerDid::new("did:plc:nel").unwrap(), + _ => OwnerDid::new("did:plc:olaren").unwrap(), + } +} + +fn repo_rkey(n: u8) -> RepoRkey { + RepoRkey::new(format!("r{n}")).unwrap() +} + +fn repo_did(n: u8) -> RepoDid { + RepoDid::new(format!("did:plc:r{n}")).unwrap() +} + +fn grant(subject: u8, t: i64) -> Grant { + Grant { + subject: acc(subject), + added_by: AccountDid::new("did:plc:nel").unwrap(), + created_at: UnixSeconds::new(t), + } +} + +fn accept(subject: u8, t: i64) -> Accept { + Accept { + subject: acc(subject), + verified_at: UnixSeconds::new(t), + } +} + +fn member_change(op: u8, subject: u8, t: i64) -> MembersChange { + match op { + 0 => MembersChange::Add(grant(subject, t)), + 1 => MembersChange::Accept(accept(subject, t)), + 2 => MembersChange::Invite(Invite(grant(subject, t))), + _ => MembersChange::Remove(Removal { + subject: acc(subject), + }), + } +} + +fn collaborator_change(op: u8, subject: u8, t: i64) -> CollaboratorsChange { + match op { + 0 => CollaboratorsChange::Add(grant(subject, t)), + 1 => CollaboratorsChange::Accept(accept(subject, t)), + 2 => CollaboratorsChange::Invite(Invite(grant(subject, t))), + _ => CollaboratorsChange::Remove(Removal { + subject: acc(subject), + }), + } +} + +fn registry_change(op: u8, who: u8, rkey: u8, repo: u8, t: i64) -> RegistryChange { + match op { + 0 => RegistryChange::Register(Registration { + owner: owner(who), + rkey: repo_rkey(rkey), + name: RepoName::new(format!("r{rkey}")).unwrap(), + repo: repo_did(repo), + created_at: UnixSeconds::new(t), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }), + 1 => RegistryChange::Rename(Rename { + owner: owner(who), + rkey: repo_rkey(rkey), + name: RepoName::new(format!("r{rkey}")).unwrap(), + repo: repo_did(repo), + }), + _ => RegistryChange::Deregister(RepoRef { + owner: owner(who), + rkey: repo_rkey(rkey), + }), + } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 40, ..ProptestConfig::default() })] + + #[test] + fn members_fold_equals_canonical_evaluate( + ops in prop::collection::vec((0u8..4, 0u8..4), 1..14) + ) { + let world = World::seeded(7); + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + + let incremental = Index::new(&world.meta_path, world.layout.clone()); + + let (op0, subject0) = ops[0]; + let object = store + .create(&meta_home(), &member_change(op0, subject0, 1), &world.signer, UnixSeconds::new(1)) + .unwrap() + .object; + incremental.refresh_members().unwrap(); + + ops.iter().enumerate().skip(1).for_each(|(index, (op, subject))| { + let t = index as i64 + 1; + store + .update(&meta_home(), object, &member_change(*op, *subject, t), &world.signer, UnixSeconds::new(t)) + .unwrap(); + incremental.refresh_members().unwrap(); + }); + + let full = Index::new(&world.meta_path, world.layout.clone()); + full.rebuild().unwrap(); + + let canonical = store.get::(object).unwrap(); + let list = canonical.state(); + let expected: Vec> = (0u8..4) + .map(|subject| Resolved::Ready(list.contains(&acc(subject)))) + .collect(); + prop_assert_eq!( + (0u8..4).map(|s| incremental.effective_member(&acc(s))).collect::>(), + expected.clone() + ); + prop_assert_eq!( + (0u8..4).map(|s| full.effective_member(&acc(s))).collect::>(), + expected + ); + } + + #[test] + fn collaborators_fold_equals_canonical_evaluate( + ops in prop::collection::vec((0u8..4, 0u8..4), 1..14) + ) { + let world = World::seeded(8); + let repo = repo_did(0); + let git = world.layout.create(&repo).unwrap(); + let store = CobStore::new(&git); + + let incremental = Index::new(&world.meta_path, world.layout.clone()); + + let (op0, subject0) = ops[0]; + let object = store + .create(&CobHome::from(&repo), &collaborator_change(op0, subject0, 1), &world.signer, UnixSeconds::new(1)) + .unwrap() + .object; + incremental.rebuild().unwrap(); + incremental.refresh_collaborators(&repo).unwrap(); + + ops.iter().enumerate().skip(1).for_each(|(index, (op, subject))| { + let t = index as i64 + 1; + store + .update(&CobHome::from(&repo), object, &collaborator_change(*op, *subject, t), &world.signer, UnixSeconds::new(t)) + .unwrap(); + incremental.refresh_collaborators(&repo).unwrap(); + }); + + let full = Index::new(&world.meta_path, world.layout.clone()); + full.rebuild().unwrap(); + full.refresh_collaborators(&repo).unwrap(); + + let canonical = store.get::(object).unwrap(); + let list = canonical.state(); + let expected: Vec> = (0u8..4) + .map(|subject| Resolved::Ready(list.contains(&acc(subject)))) + .collect(); + prop_assert_eq!( + (0u8..4).map(|s| incremental.effective_collaborator(&repo, &acc(s))).collect::>(), + expected.clone() + ); + prop_assert_eq!( + (0u8..4).map(|s| full.effective_collaborator(&repo, &acc(s))).collect::>(), + expected + ); + } + + #[test] + fn registry_fold_equals_canonical_evaluate( + ops in prop::collection::vec((0u8..3, 0u8..2, 0u8..4, 0u8..4), 1..14) + ) { + let world = World::seeded(9); + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + + let incremental = Index::new(&world.meta_path, world.layout.clone()); + + let (op0, who0, name0, repo0) = ops[0]; + let object = store + .create(&meta_home(), ®istry_change(op0, who0, name0, repo0, 1), &world.signer, UnixSeconds::new(1)) + .unwrap() + .object; + incremental.refresh_registry().unwrap(); + + ops.iter().enumerate().skip(1).for_each(|(index, (op, who, name, repo))| { + let t = index as i64 + 1; + store + .update(&meta_home(), object, ®istry_change(*op, *who, *name, *repo, t), &world.signer, UnixSeconds::new(t)) + .unwrap(); + incremental.refresh_registry().unwrap(); + }); + + let full = Index::new(&world.meta_path, world.layout.clone()); + full.rebuild().unwrap(); + + let canonical = store.get::(object).unwrap(); + let registry = canonical.state(); + let lookups: Vec<(u8, u8)> = (0u8..2) + .flat_map(|who| (0u8..4).map(move |rkey| (who, rkey))) + .collect(); + let expected: Vec>> = lookups + .iter() + .map(|(who, rkey)| { + Resolved::Ready(registry.resolve(&owner(*who), &repo_rkey(*rkey)).cloned()) + }) + .collect(); + prop_assert_eq!( + lookups + .iter() + .map(|(who, rkey)| incremental.resolve_repo(&owner(*who), &repo_rkey(*rkey))) + .collect::>(), + expected.clone() + ); + prop_assert_eq!( + lookups + .iter() + .map(|(who, rkey)| full.resolve_repo(&owner(*who), &repo_rkey(*rkey))) + .collect::>(), + expected + ); + let expected_records: Vec<_> = (0u8..4) + .map(|n| { + let record = registry.record_of(&repo_did(n)); + ( + Resolved::Ready(record.map(|record| record.owner.clone())), + Resolved::Ready(record.map(|record| record.rkey.clone())), + ) + }) + .collect(); + prop_assert_eq!( + (0u8..4) + .map(|n| (incremental.owner_of(&repo_did(n)), incremental.rkey_of(&repo_did(n)))) + .collect::>(), + expected_records.clone() + ); + prop_assert_eq!( + (0u8..4) + .map(|n| (full.owner_of(&repo_did(n)), full.rkey_of(&repo_did(n)))) + .collect::>(), + expected_records + ); + } +} diff --git a/knot2/crates/knot-keyfill/Cargo.toml b/knot2/crates/knot-keyfill/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..ce6fc5fae86be73d320b8ce9a2f377409f64194b --- /dev/null +++ b/knot2/crates/knot-keyfill/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "knot-keyfill" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-atproto = { workspace = true } +knot-index = { workspace = true } +knot-resource = { workspace = true } +knot-runtime = { workspace = true } +knot-types = { workspace = true } +futures = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-git = { workspace = true } +bytes = { workspace = true } +http = { workspace = true } +serde_json = { workspace = true } +tempfile = { workspace = true } diff --git a/knot2/crates/knot-keyfill/src/lib.rs b/knot2/crates/knot-keyfill/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..1473f2f2d3d56a2ae1ed902f780614c0aed53c41 --- /dev/null +++ b/knot2/crates/knot-keyfill/src/lib.rs @@ -0,0 +1,620 @@ +use std::future::Future; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use futures::future::OptionFuture; +use futures::{FutureExt, StreamExt}; +use knot_atproto::{Atproto, AtprotoError}; +use knot_index::{ + Coverage, HostedCoverage, Index, IndexGeneration, KeyLease, KeyRecord, KeyReprieve, + KeyReprieved, KeyTtl, MemberWork, Pushers, Resolved, StalePushers, SuspectPushers, + SweepInterval, +}; +use knot_resource::{Burst, HostKey, HostPacer, RateLimit, RefillMicros, SlotPermit, Slots}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, OfferedKey, UnixMicros, UnixSeconds}; +use tokio::sync::watch; +use tokio_util::sync::CancellationToken; +use url::Url; + +const FILL_FANOUT: usize = 8; + +const PASS_HEADROOM: u32 = 2; + +macro_rules! span { + ($($name:ident from $unit:ident),+ $(,)?) => {$( + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct $name(Duration); + + impl $name { + pub const fn $unit(value: u64) -> Self { + Self(Duration::$unit(value)) + } + + pub const fn get(self) -> Duration { + self.0 + } + } + )+}; +} + +span!( + BusyRetry from from_millis, + SettleFloor from from_millis, + StalledBackoff from from_secs, + SettledPause from from_secs, +); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AccountBudget(usize); + +impl AccountBudget { + pub const fn new(accounts: usize) -> Self { + Self(if accounts == 0 { 1 } else { accounts }) + } + + pub const fn get(self) -> usize { + self.0 + } +} + +#[derive(Debug, Default)] +pub struct Cursor(AtomicUsize); + +impl Cursor { + fn advance(&self, by: usize, len: usize) -> usize { + match len { + 0 => 0, + len => self + .0 + .try_update(Ordering::Relaxed, Ordering::Relaxed, |seen| { + Some((seen % len).wrapping_add(by) % len) + }) + .map_or(0, |seen| seen % len), + } + } +} + +#[derive(Debug, Default)] +pub struct Cursors { + members: Cursor, + suspected: Cursor, +} + +fn portion(accounts: &[AccountDid], budget: AccountBudget, cursor: &Cursor) -> Vec { + let taken = budget.get().min(accounts.len()); + let start = cursor.advance(taken, accounts.len()); + accounts + .iter() + .cycle() + .skip(start) + .take(taken) + .cloned() + .collect() +} + +#[derive(Debug, Clone, Copy)] +pub struct Pace { + pub busy: BusyRetry, + pub floor: SettleFloor, + pub ttl: KeyTtl, + pub reprieve: KeyReprieve, + pub sweep: SweepInterval, + pub stalled: StalledBackoff, + pub settled: SettledPause, + pub members: AccountBudget, + pub suspected: AccountBudget, + pub host: RateLimit, +} + +impl Default for Pace { + fn default() -> Self { + Self { + busy: BusyRetry::from_millis(50), + floor: SettleFloor::from_millis(1_000), + ttl: KeyTtl::DEFAULT, + reprieve: KeyReprieve::DEFAULT, + sweep: SweepInterval::DEFAULT, + stalled: StalledBackoff::from_secs(30), + settled: SettledPause::from_secs(60), + members: AccountBudget::new(64), + suspected: AccountBudget::new(256), + host: RateLimit { + burst: Burst::literal(10), + refill: RefillMicros::literal(200_000), + }, + } + } +} + +impl Pace { + fn ttl_covering(self, accounts: usize) -> KeyTtl { + let micros = (accounts as u64).saturating_mul(self.host.refill.get()); + let secs = (micros / 1_000_000).saturating_mul(u64::from(PASS_HEADROOM)); + self.ttl.longest(KeyTtl::from_secs(secs)) + } +} + +pub fn spawn( + index: Arc, + atproto: Arc>, + slots: Slots, + pace: Pace, + shutdown: CancellationToken, +) -> tokio::task::JoinHandle<()> { + let stop = shutdown.clone(); + let driver = Driver { + generations: index.generations(), + pacer: HostPacer::new(pace.host), + cursors: Cursors::default(), + index, + atproto, + slots, + pace, + shutdown, + }; + tokio::spawn(async move { + futures::stream::unfold(driver, |mut driver| async move { + driver.pass().await; + Some(((), driver)) + }) + .take_until(stop.cancelled_owned()) + .for_each(|()| std::future::ready(())) + .await; + tracing::info!("key fill stopped"); + }) +} + +struct Driver { + index: Arc, + atproto: Arc>, + slots: Slots, + pacer: HostPacer, + cursors: Cursors, + pace: Pace, + shutdown: CancellationToken, + generations: watch::Receiver, +} + +impl Driver { + async fn pass(&mut self) { + self.generations.mark_unchanged(); + let filling = fill_once( + &self.index, + &self.atproto, + &self.slots, + &self.pacer, + self.pace, + &self.cursors, + ); + let pause = tokio::select! { + pause = guarded(filling, self.pace) => pause, + () = self.shutdown.cancelled() => self.pace.stalled.get(), + }; + tokio::select! { + () = self.shutdown.cancelled() => {} + () = settle(&mut self.generations, pause, self.pace.floor) => {} + } + } +} + +async fn guarded(pass: impl Future, pace: Pace) -> Duration { + std::panic::AssertUnwindSafe(pass) + .catch_unwind() + .await + .unwrap_or_else(|_| { + tracing::error!("key fill pass panicked, backing off before the next pass"); + pace.stalled.get() + }) +} + +async fn settle( + generations: &mut watch::Receiver, + pause: Duration, + floor: SettleFloor, +) { + let held = floor.get().min(pause); + tokio::time::sleep(held).await; + let _ = tokio::time::timeout(pause.saturating_sub(held), generations.changed()).await; +} + +struct Pass<'a, H, C> { + index: &'a Arc, + atproto: &'a Arc>, + slots: &'a Slots, + pacer: &'a HostPacer, + pace: Pace, + now: UnixSeconds, + lease: KeyLease, + reprieve: KeyReprieve, +} + +pub async fn fill_once( + index: &Arc, + atproto: &Arc>, + slots: &Slots, + pacer: &HostPacer, + pace: Pace, + cursors: &Cursors, +) -> Duration { + fold_hosted(index).await; + let now = atproto.now().seconds(); + let Resolved::Ready(work) = index.keys().work(now, pace.sweep) else { + index.keys().mark_warming(); + return pace.stalled.get(); + }; + if let HostedCoverage::Partial { unread } = work.hosted { + tracing::debug!( + repos = unread, + "partial grant set, a repo was registered while this pass was working out who may push" + ); + } + let members = match (work.hosted, work.members) { + (HostedCoverage::Whole, Resolved::Ready(members)) => { + index.keys().retain(&members.kept); + Some(members) + } + (HostedCoverage::Partial { .. }, Resolved::Ready(members)) => Some(members), + (_, Resolved::Warming) => None, + }; + let ttl = pace.ttl_covering(work.tracked); + if ttl != pace.ttl { + tracing::debug!( + tracked = work.tracked, + ttl_secs = ttl.get().as_secs(), + "one paced pass over the grant set outruns the key ttl, so the fill stretches it" + ); + } + let pass = Pass { + index, + atproto, + slots, + pacer, + pace, + now, + lease: ttl.lease_from(now), + reprieve: pace.reprieve.budgeted_for(ttl), + }; + let pause = fill_pushers( + &pass, + PushWork { + generation: work.generation, + hosted: work.hosted, + pushers: work.pushers, + due: work.due, + suspected: work.suspected, + complete: work.complete, + }, + &cursors.suspected, + ) + .await; + OptionFuture::from(members.map(|work| fill_members(&pass, work, &cursors.members))).await; + pause +} + +struct PushWork { + generation: IndexGeneration, + hosted: HostedCoverage, + pushers: Pushers, + due: StalePushers, + suspected: SuspectPushers, + complete: bool, +} + +async fn fold_hosted(index: &Arc) { + let index = Arc::clone(index); + match tokio::task::spawn_blocking(move || index.warm_collaborators()).await { + Ok(0) | Err(_) => {} + Ok(unreadable) => tracing::warn!( + repos = unreadable, + "the knot hosts registered repos it can't open, so it won't grant anybody through \ + them until an operator restores or deregisters each repo" + ), + } +} + +async fn fill_pushers( + pass: &Pass<'_, H, C>, + work: PushWork, + cursor: &Cursor, +) -> Duration { + let PushWork { + generation, + hosted, + pushers, + due, + suspected, + complete, + } = work; + if !complete { + pass.index.keys().mark_warming(); + } + let wanted = due.len(); + let recorded = match wanted { + 0 => 0, + _ => { + let recorded = record_each(pass, due.into_vec()).await; + tracing::debug!(wanted, recorded, "pusher key fill pass"); + recorded + } + }; + recheck_suspected(pass, &suspected, cursor).await; + match recorded < wanted { + true => pass.pace.stalled.get(), + false => claim_ready(pass, generation, hosted, &pushers), + } +} + +async fn recheck_suspected( + pass: &Pass<'_, H, C>, + suspected: &SuspectPushers, + cursor: &Cursor, +) { + if suspected.is_empty() { + return; + } + let batch = portion(suspected.as_slice(), pass.pace.suspected, cursor); + let wanted = batch.len(); + let recorded = record_each(pass, batch).await; + tracing::debug!( + wanted, + recorded, + deferred = suspected.len().saturating_sub(wanted), + "pusher key recheck pass, a client offered a key the accounts on file don't publish" + ); +} + +fn claim_ready( + pass: &Pass<'_, H, C>, + generation: IndexGeneration, + hosted: HostedCoverage, + pushers: &Pushers, +) -> Duration { + let keys = pass.index.keys(); + let settled = + hosted == HostedCoverage::Whole && keys.all_live(pushers, pass.atproto.now().seconds()); + if !settled { + return pass.pace.stalled.get(); + } + keys.mark_ready(generation); + match keys.coverage() { + Coverage::Ready => pass.pace.settled.get(), + Coverage::Warming => pass.pace.stalled.get(), + } +} + +async fn fill_members( + pass: &Pass<'_, H, C>, + work: MemberWork, + cursor: &Cursor, +) { + if !work.unread.is_empty() { + let wanted = work.unread.len(); + let recorded = record_each(pass, work.unread.into_vec()).await; + tracing::debug!(wanted, recorded, "member key first-read pass"); + } + if work.due.is_empty() { + return; + } + let batch = portion(work.due.as_slice(), pass.pace.members, cursor); + let wanted = batch.len(); + let recorded = record_each(pass, batch).await; + tracing::debug!( + wanted, + recorded, + deferred = work.due.len().saturating_sub(wanted), + "member key renewal pass" + ); +} + +async fn record_each( + pass: &Pass<'_, H, C>, + stale: Vec, +) -> usize { + futures::stream::iter(stale) + .map(|did| async move { + match published_keys(pass, &did).await { + Ok(keys) => usize::from(record(pass, &did, keys)), + Err(error) if error.is_gone() => { + tracing::debug!( + did = did.as_str(), + %error, + "key fill records an empty key set for an account whose DID document is gone" + ); + usize::from(record(pass, &did, Vec::new())) + } + Err(error) => reprieve(pass, &did, error), + } + }) + .buffer_unordered(FILL_FANOUT) + .fold(0, |total, recorded| async move { total + recorded }) + .await +} + +fn reprieve(pass: &Pass<'_, H, C>, did: &AccountDid, error: AtprotoError) -> usize { + let outcome = pass + .index + .keys() + .reprieve(did, pass.now, pass.reprieve, pass.lease); + match outcome { + KeyReprieved::Exhausted => tracing::warn!( + did = did.as_str(), + %error, + "the knot spent the whole reprieve failing to read an account, so it records an \ + empty key set for the account until a later read succeeds" + ), + KeyReprieved::Extended | KeyReprieved::Pending => tracing::debug!( + did = did.as_str(), + %error, + ?outcome, + "key fill couldn't read an account's records" + ), + } + match outcome { + KeyReprieved::Extended | KeyReprieved::Exhausted => 1, + KeyReprieved::Pending => 0, + } +} + +fn record(pass: &Pass<'_, H, C>, did: &AccountDid, keys: Vec) -> bool { + match pass.index.keys().record(did, keys, pass.lease) { + KeyRecord::Stored => true, + KeyRecord::Unheld => { + tracing::warn!( + did = did.as_str(), + "the key set is full, so the knot will check this account's pushes against its \ + PDS every time instead of against the set" + ); + true + } + KeyRecord::Saturated => { + tracing::warn!( + did = did.as_str(), + "the key budget can't record even that it read this account, so the knot keeps \ + reporting the set incomplete and defers every offered key to the push check" + ); + false + } + } +} + +async fn published_keys( + pass: &Pass<'_, H, C>, + did: &AccountDid, +) -> Result, AtprotoError> { + let atproto = pass.atproto; + if let Ok(document) = atproto.document_url(did) { + wait_for_turn(pass.pacer, &document, atproto.now()).await; + } + let identity = { + let _permit = idle_permit(pass.slots, pass.pace).await; + atproto.resolve_identity(did).await? + }; + wait_for_turn(pass.pacer, identity.pds.url(), atproto.now()).await; + let _permit = idle_permit(pass.slots, pass.pace).await; + atproto.pubkeys_at(&identity, did).await +} + +async fn wait_for_turn(pacer: &HostPacer, url: &Url, now: UnixMicros) { + if let Some(host) = url.host_str().map(HostKey::new) { + tokio::time::sleep(pacer.reserve(&host, now)).await; + } +} + +async fn idle_permit(slots: &Slots, pace: Pace) -> SlotPermit { + let attempts = futures::stream::repeat(()).filter_map(|()| async { + match slots.resolve.try_acquire() { + Some(permit) => Some(permit), + None => { + tokio::time::sleep(pace.busy.get()).await; + None + } + } + }); + futures::pin_mut!(attempts); + attempts + .next() + .await + .expect("an endless stream of attempts yields a permit") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_grant_set_the_pacer_cant_reread_within_the_ttl_stretches_it() { + let pace = Pace::default(); + assert_eq!( + pace.ttl_covering(1_000), + KeyTtl::DEFAULT, + "a set one paced pass covers well inside the ttl keeps the ttl it was configured with" + ); + assert_eq!( + pace.ttl_covering(100_000), + KeyTtl::from_secs(40_000), + "at one directory turn per 200ms a hundred thousand accounts take 20_000s to reread, \ + so the entries the pass wrote first must outlive the pass that writes the last" + ); + } + + #[test] + fn the_reprieve_budget_never_undercuts_the_ttl_the_fill_is_working_to() { + let stretched = KeyTtl::from_secs(40_000); + assert_eq!( + KeyReprieve::DEFAULT.budgeted_for(stretched), + KeyReprieve::from_secs(300, 40_000), + "an account would be released while the pass that would reread it is still running, \ + if the budget stayed under the ttl" + ); + assert_eq!( + KeyReprieve::DEFAULT.budgeted_for(KeyTtl::DEFAULT), + KeyReprieve::DEFAULT, + "a ttl the budget already covers leaves the budget alone" + ); + } + + #[test] + fn the_member_cursor_turns_over_without_running_past_its_type() { + let cursor = Cursor(AtomicUsize::new(usize::MAX)); + assert_eq!( + cursor.advance(3, 4), + usize::MAX % 4, + "a cursor at the end of its range wraps instead of overflowing" + ); + assert_eq!(cursor.advance(3, 4), (usize::MAX % 4 + 3) % 4); + assert_eq!( + cursor.advance(3, 4), + (usize::MAX % 4 + 6) % 4, + "every pass after the wrap steps by its budget, so one member can't keep the front \ + of the queue" + ); + } + + #[test] + fn a_set_larger_than_its_budget_comes_round_in_turns_that_cover_everybody() { + let accounts: Vec = ["nel", "olaren", "teq", "bailey", "cuttle"] + .iter() + .map(|name| AccountDid::new(format!("did:plc:{name}")).unwrap()) + .collect(); + let cursor = Cursor::default(); + let budget = AccountBudget::new(2); + let turns: Vec> = (0..3) + .map(|_| { + portion(&accounts, budget, &cursor) + .iter() + .map(|did| did.as_str().to_string()) + .collect() + }) + .collect(); + assert_eq!( + turns, + vec![ + vec!["did:plc:nel", "did:plc:olaren"], + vec!["did:plc:teq", "did:plc:bailey"], + vec!["did:plc:cuttle", "did:plc:nel"], + ], + "a stranger offering an unrecognized key mustn't cost the knot a read of every \ + account it grants. The accounts it defers must come round on later passes" + ); + } + + #[test] + fn a_set_the_budget_covers_is_read_whole_without_repeats() { + let accounts: Vec = ["nel", "olaren"] + .iter() + .map(|name| AccountDid::new(format!("did:plc:{name}")).unwrap()) + .collect(); + let cursor = Cursor::default(); + assert_eq!( + portion(&accounts, AccountBudget::new(256), &cursor), + accounts, + "a set that fits inside one budget must behave as though there were no budget" + ); + assert!( + portion(&[], AccountBudget::new(256), &cursor).is_empty(), + "an empty set must yield an empty portion, or the cycle turns forever" + ); + } +} diff --git a/knot2/crates/knot-keyfill/tests/fill.rs b/knot2/crates/knot-keyfill/tests/fill.rs new file mode 100644 index 0000000000000000000000000000000000000000..8def1ab20c8c3faeb281f6a8a886664fe833f607 --- /dev/null +++ b/knot2/crates/knot-keyfill/tests/fill.rs @@ -0,0 +1,445 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use futures::StreamExt; +use knot_atproto::Atproto; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Grant, MembersChange, Registration, RegistryChange}; +use knot_git::{Layout, Repo}; +use knot_index::{Coverage, Index, KeyReprieve, KeyTtl, Resolved, SweepInterval}; +use knot_keyfill::{ + AccountBudget, BusyRetry, Cursors, Pace, SettleFloor, SettledPause, StalledBackoff, fill_once, +}; +use knot_resource::{Burst, HostKey, HostPacer, RateLimit, RefillMicros, Slots}; +use knot_runtime::{ + FakeHttp, HttpRequest, HttpResponse, K256Signer, ManualClock, NetworkError, SeededEntropy, + Signer, +}; +use knot_types::{ + AccountDid, KnotId, OfferedKey, OwnerDid, RepoDid, RepoName, RepoRkey, UnixMicros, UnixSeconds, +}; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; +use url::Url; + +const KNOT_DID: &str = "did:web:nel.pet"; +const NEL: &str = "did:plc:nel"; +const OLAREN: &str = "did:plc:olaren"; +const TEQ: &str = "did:plc:teq"; +const BAILEY: &str = "did:plc:bailey"; +const NEL_PDS: &str = "https://pds.nel.pet"; +const OLAREN_PDS: &str = "https://pds.olaren.dev"; + +type Responder = Box Result + Send + Sync>; + +fn responding(status: http::StatusCode, body: bytes::Bytes) -> HttpResponse { + HttpResponse { + status, + headers: http::HeaderMap::new(), + body, + } +} + +struct Hosted { + owner: OwnerDid, + repo: RepoDid, + name: RepoName, +} + +fn account(did: &str) -> AccountDid { + AccountDid::new(did).unwrap() +} + +fn anemone() -> Hosted { + Hosted { + owner: OwnerDid::new(NEL).unwrap(), + repo: RepoDid::new("did:plc:squid").unwrap(), + name: RepoName::new("anemone").unwrap(), + } +} + +fn barnacle() -> Hosted { + Hosted { + owner: OwnerDid::new(OLAREN).unwrap(), + repo: RepoDid::new("did:plc:limpet").unwrap(), + name: RepoName::new("barnacle").unwrap(), + } +} + +fn hosted_index(scratch: &TempDir, repos: &[Hosted], members: &[AccountDid]) -> Arc { + let meta_path = scratch.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scratch.path().join("repos")); + let meta = Repo::open(&meta_path).unwrap(); + let store = CobStore::new(&meta); + let home = CobHome::from(&KnotId::new(KNOT_DID).unwrap()); + let signer = K256Signer::generate(&SeededEntropy::new(3)); + let registration = |hosted: &Hosted| { + layout.create(&hosted.repo).unwrap(); + RegistryChange::Register(Registration { + owner: hosted.owner.clone(), + rkey: RepoRkey::new(hosted.name.as_str()).unwrap(), + name: hosted.name.clone(), + repo: hosted.repo.clone(), + created_at: UnixSeconds::new(1), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }) + }; + let (first, rest) = repos.split_first().expect("a knot under test hosts a repo"); + let registry = store + .create(&home, ®istration(first), &signer, UnixSeconds::new(1)) + .unwrap() + .object; + rest.iter().for_each(|hosted| { + store + .update( + &home, + registry, + ®istration(hosted), + &signer, + UnixSeconds::new(2), + ) + .unwrap(); + }); + + let granted = |subject: &AccountDid| { + MembersChange::Add(Grant { + subject: subject.clone(), + added_by: account(NEL), + created_at: UnixSeconds::new(1), + }) + }; + if let Some((first, rest)) = members.split_first() { + let list = store + .create(&home, &granted(first), &signer, UnixSeconds::new(1)) + .unwrap() + .object; + rest.iter().for_each(|subject| { + store + .update(&home, list, &granted(subject), &signer, UnixSeconds::new(2)) + .unwrap(); + }); + } + + let index = Arc::new(Index::new(meta_path, layout)); + index.rebuild().unwrap(); + index.warm_collaborators(); + index +} + +fn atproto_with(responder: Responder) -> Arc, ManualClock>> { + Arc::new(Atproto::new( + FakeHttp::new(responder), + ManualClock::new(UnixMicros::new(1_000_000_000)), + KnotId::new(KNOT_DID).unwrap(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + )) +} + +fn atproto_answering( + status: http::StatusCode, + calls: Arc, +) -> Arc, ManualClock>> { + atproto_with(Box::new(move |_request: &HttpRequest| { + calls.fetch_add(1, Ordering::SeqCst); + Ok(responding(status, bytes::Bytes::new())) + })) +} + +fn did_document(did: &str, handle: &str, pds: &str) -> bytes::Bytes { + let signing = K256Signer::generate(&SeededEntropy::new(5)); + let multibase = knot_types::crypto::multikey(0xe7, signing.public_key().as_bytes()); + let body = serde_json::json!({ + "id": did, + "alsoKnownAs": [format!("at://{handle}")], + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multibase, + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": pds, + }] + }); + bytes::Bytes::from(serde_json::to_vec(&body).unwrap()) +} + +fn atproto_serving_two_accounts() -> Arc, ManualClock>> { + atproto_with(Box::new(move |request: &HttpRequest| { + let url = request.url.as_str(); + let body = if url.contains("listRecords") { + bytes::Bytes::from_static(br#"{"records":[]}"#) + } else if url.ends_with(NEL) { + did_document(NEL, "nel.pet", NEL_PDS) + } else if url.ends_with(OLAREN) { + did_document(OLAREN, "olaren.dev", OLAREN_PDS) + } else { + return Ok(responding(http::StatusCode::NOT_FOUND, bytes::Bytes::new())); + }; + Ok(responding(http::StatusCode::OK, body)) + })) +} + +fn atproto_recording( + seen: Arc>>, +) -> Arc, ManualClock>> { + atproto_with(Box::new(move |request: &HttpRequest| { + let url = request.url.as_str(); + if url.contains("listRecords") { + return Ok(responding( + http::StatusCode::OK, + bytes::Bytes::from_static(br#"{"records":[]}"#), + )); + } + if url.ends_with(NEL) { + return Ok(responding( + http::StatusCode::OK, + did_document(NEL, "nel.pet", NEL_PDS), + )); + } + if let Some(did) = url.rsplit('/').next() { + seen.lock().unwrap().push(did.to_string()); + } + Ok(responding( + http::StatusCode::SERVICE_UNAVAILABLE, + bytes::Bytes::new(), + )) + })) +} + +fn now() -> UnixSeconds { + UnixSeconds::new(1_000) +} + +fn brisk() -> Pace { + Pace { + busy: BusyRetry::from_millis(0), + floor: SettleFloor::from_millis(0), + ttl: KeyTtl::from_secs(3_600), + reprieve: KeyReprieve::from_secs(300, 21_600), + sweep: SweepInterval::DEFAULT, + stalled: StalledBackoff::from_secs(1), + settled: SettledPause::from_secs(1), + members: AccountBudget::new(64), + suspected: AccountBudget::new(256), + host: RateLimit { + burst: Burst::literal(1), + refill: RefillMicros::literal(1_000), + }, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_gone_document_completes_the_set_and_an_outage_leaves_it_warming() { + let outcome = |status: http::StatusCode| async move { + let scratch = tempfile::tempdir().unwrap(); + let index = hosted_index(&scratch, &[anemone()], &[]); + let calls = Arc::new(AtomicUsize::new(0)); + let atproto = atproto_answering(status, Arc::clone(&calls)); + let pacer = HostPacer::new(brisk().host); + assert_eq!(index.keys().coverage(), Coverage::Warming); + fill_once( + &index, + &atproto, + &Slots::testing(4), + &pacer, + brisk(), + &Cursors::default(), + ) + .await; + assert!( + calls.load(Ordering::SeqCst) > 0, + "the fill made an outbound request" + ); + index.keys().coverage() + }; + + assert_eq!( + outcome(http::StatusCode::NOT_FOUND).await, + Coverage::Ready, + "a permanently unresolvable owner is recorded with an empty key set, so the set is complete" + ); + assert_eq!( + outcome(http::StatusCode::SERVICE_UNAVAILABLE).await, + Coverage::Warming, + "a transient failure doesn't teach the knot anything, so it mustn't claim a complete set" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn keys_the_knot_already_read_survive_an_outage_and_a_refused_listing() { + let scratch = tempfile::tempdir().unwrap(); + let index = hosted_index(&scratch, &[anemone()], &[]); + let key = OfferedKey::from_bytes(vec![9]); + let spent = KeyTtl::from_secs(1).lease_from(UnixSeconds::new(100)); + index.keys().record(&account(NEL), vec![key.clone()], spent); + let pacer = HostPacer::new(brisk().host); + + let unreachable = atproto_answering(http::StatusCode::SERVICE_UNAVAILABLE, Arc::default()); + fill_once( + &index, + &unreachable, + &Slots::testing(4), + &pacer, + brisk(), + &Cursors::default(), + ) + .await; + assert_eq!( + index.keys().coverage(), + Coverage::Ready, + "an owner the knot has read before keeps its last keys through an outage, so one \ + unreachable PDS mustn't reopen the knot to every offered key" + ); + assert_eq!( + index.owner_of_key(&key, now()), + Resolved::Ready(Some(account(NEL))), + "the reprieve keeps the keys the knot last read" + ); + + index.keys().record(&account(NEL), vec![key.clone()], spent); + let listing = Arc::new(AtomicUsize::new(0)); + let refusing = { + let listing = Arc::clone(&listing); + atproto_with(Box::new(move |request: &HttpRequest| { + let url = request.url.as_str(); + if url.contains("listRecords") { + listing.fetch_add(1, Ordering::SeqCst); + return Ok(responding( + http::StatusCode::BAD_REQUEST, + bytes::Bytes::new(), + )); + } + Ok(responding( + http::StatusCode::OK, + did_document(NEL, "nel.pet", NEL_PDS), + )) + })) + }; + fill_once( + &index, + &refusing, + &Slots::testing(4), + &pacer, + brisk(), + &Cursors::default(), + ) + .await; + assert!( + listing.load(Ordering::SeqCst) > 0, + "the fill read from the PDS" + ); + assert_eq!( + index.owner_of_key(&key, now()), + Resolved::Ready(Some(account(NEL))), + "an account whose DID document resolves hasn't gone anywhere, so the knot mustn't read \ + a 400 from a record listing as proof the account stopped publishing keys" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_fill_takes_a_turn_at_each_host_it_reads_from_and_stops_on_shutdown() { + let scratch = tempfile::tempdir().unwrap(); + let index = hosted_index(&scratch, &[anemone(), barnacle()], &[]); + let atproto = atproto_serving_two_accounts(); + let pacer = HostPacer::new(brisk().host); + + fill_once( + &index, + &atproto, + &Slots::testing(4), + &pacer, + brisk(), + &Cursors::default(), + ) + .await; + + assert_eq!( + index.keys().coverage(), + Coverage::Ready, + "both owners resolved, so every account that may push has a record" + ); + ["plc.directory", "pds.nel.pet", "pds.olaren.dev"] + .iter() + .for_each(|host| { + assert!( + !pacer.reserve_now(&HostKey::new(host), UnixMicros::new(0)), + "the fill must take a turn at {host} before it reads from it, \ + or a knot whose members share one PDS spends its whole rate at that host" + ); + }); + assert!( + pacer.reserve_now(&HostKey::new("pds.teq.dev"), UnixMicros::new(0)), + "a host the fill never read from is due immediately. The bookings above are the fill's \ + own work" + ); + + let shutdown = CancellationToken::new(); + let task = knot_keyfill::spawn( + Arc::clone(&index), + Arc::clone(&atproto), + Slots::testing(4), + brisk(), + shutdown.clone(), + ); + shutdown.cancel(); + tokio::time::timeout(std::time::Duration::from_secs(5), task) + .await + .expect("a shutting-down knot mustn't wait out the pause between fill passes") + .expect("the fill task stops without panicking"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn members_are_read_whole_on_first_contact_then_renewed_one_budget_turn_per_pass() { + let scratch = tempfile::tempdir().unwrap(); + let members = [account(OLAREN), account(TEQ), account(BAILEY)]; + let index = hosted_index(&scratch, &[anemone()], &members); + let seen = Arc::new(Mutex::new(Vec::new())); + let atproto = atproto_recording(Arc::clone(&seen)); + let pacer = HostPacer::new(brisk().host); + let pace = Pace { + members: AccountBudget::new(1), + ..brisk() + }; + let cursor = Cursors::default(); + + fill_once(&index, &atproto, &Slots::testing(4), &pacer, pace, &cursor).await; + assert_eq!( + index.keys().coverage(), + Coverage::Ready, + "the pushers are what coverage waits on, so an unreadable member mustn't make the knot \ + doubt the keys it checks pushes against" + ); + let mut attempted = seen.lock().unwrap().clone(); + attempted.sort(); + assert_eq!( + attempted, + vec![BAILEY.to_string(), OLAREN.to_string(), TEQ.to_string()], + "the renewal budget paces rereads, so a member the knot has never read mustn't wait \ + its turn behind it and be refused at the handshake for the passes in between" + ); + + let spent = KeyTtl::from_secs(1).lease_from(UnixSeconds::new(0)); + members + .iter() + .for_each(|member| _ = index.keys().record(member, Vec::new(), spent)); + seen.lock().unwrap().clear(); + + futures::stream::iter(0..3) + .for_each(|_| async { + fill_once(&index, &atproto, &Slots::testing(4), &pacer, pace, &cursor).await; + }) + .await; + let order = seen.lock().unwrap().clone(); + assert_eq!( + order.iter().map(String::as_str).collect::>(), + vec![BAILEY, OLAREN, TEQ], + "one member per pass in turn, or members whose PDS stays down keep the front of \ + the queue and the knot never reads the rest" + ); +} diff --git a/knot2/crates/knot-langs/Cargo.toml b/knot2/crates/knot-langs/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..b19e8095c675626205607e6d591fb35b8543746e --- /dev/null +++ b/knot2/crates/knot-langs/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "knot-langs" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +gengo-language = { workspace = true } +regex = { workspace = true } diff --git a/knot2/crates/knot-langs/src/langs.rs b/knot2/crates/knot-langs/src/langs.rs new file mode 100644 index 0000000000000000000000000000000000000000..1faca7499bccdcf784f1679ac8191730c3e481fb --- /dev/null +++ b/knot2/crates/knot-langs/src/langs.rs @@ -0,0 +1,127 @@ +use std::collections::HashMap; +use std::ops::ControlFlow; +use std::time::Instant; + +use gengo_language::{Category, Language}; +use knot_git::{EntryKind, GitError, MAX_TREE_DEPTH, Repo, SizedEntry}; +use knot_types::{LanguageBytes, Oid}; + +use crate::linguist; + +const READ_LIMIT: usize = 16 * 1024; +const SIZE_LIMIT: u64 = 1024 * 1024; + +use knot_types::LanguageName; + +fn looks_binary(content: &[u8]) -> bool { + content.contains(&0) +} + +fn category_of(name: &'static str, fallback: Category) -> Category { + name.parse::() + .map(|language| language.category()) + .unwrap_or(fallback) +} + +pub fn analyze( + repo: &Repo, + commit: Oid, + deadline: Option, +) -> Result, GitError> { + let mut sizes: HashMap = HashMap::new(); + let root = repo.peel_to_tree(commit)?; + let _budget = walk(repo, root, "", 0, deadline, &mut sizes)?; + Ok(sizes) +} + +fn walk( + repo: &Repo, + tree: Oid, + dir: &str, + depth: usize, + deadline: Option, + sizes: &mut HashMap, +) -> Result, GitError> { + if depth > MAX_TREE_DEPTH || deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Ok(ControlFlow::Break(())); + } + let entries = repo.tree_entries(tree)?; + entries + .iter() + .try_fold(ControlFlow::Continue(()), |flow, entry| { + if flow.is_break() || deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Ok(ControlFlow::Break(())); + } + let path = match dir.is_empty() { + true => entry.name.clone(), + false => format!("{dir}/{}", entry.name), + }; + match entry.kind { + EntryKind::Tree => match linguist::is_vendor_dir(&path) { + true => Ok(ControlFlow::Continue(())), + false => walk(repo, entry.oid, &path, depth + 1, deadline, sizes), + }, + EntryKind::Blob | EntryKind::BlobExecutable => { + if !linguist::is_skipped_path(&path) { + tally(repo, entry, &path, sizes)?; + } + Ok(ControlFlow::Continue(())) + } + EntryKind::Link | EntryKind::Commit => Ok(ControlFlow::Continue(())), + } + }) +} + +fn tally( + repo: &Repo, + entry: &SizedEntry, + path: &str, + sizes: &mut HashMap, +) -> Result<(), GitError> { + let content = match entry.size <= SIZE_LIMIT { + true => { + let blob = repo.read_blob(entry.oid)?; + blob[..blob.len().min(READ_LIMIT)].to_vec() + } + false => Vec::new(), + }; + if looks_binary(&content) { + return Ok(()); + } + let Some(language) = Language::pick(path, &content, READ_LIMIT) else { + return Ok(()); + }; + let name = LanguageName::new(linguist::group(language.name())); + if !matches!( + category_of(name.as_str(), language.category()), + Category::Programming | Category::Markup + ) { + return Ok(()); + } + let slot = sizes.entry(name).or_default(); + *slot = slot.saturating_add_bytes(entry.size); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn language_name_round_trips_through_as_str() { + let rust = LanguageName::new("Rust"); + assert_eq!(rust.as_str(), "Rust"); + } + + #[test] + fn language_names_compare_and_hash_by_value() { + use std::collections::HashSet; + + assert_eq!(LanguageName::new("Go"), LanguageName::new("Go")); + assert_ne!(LanguageName::new("Go"), LanguageName::new("Zig")); + let set: HashSet = [LanguageName::new("Go"), LanguageName::new("Go")] + .into_iter() + .collect(); + assert_eq!(set.len(), 1); + } +} diff --git a/knot2/crates/knot-langs/src/lib.rs b/knot2/crates/knot-langs/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..62aac8700c9bc24d608aeb43c65ea4eaa2c59ab4 --- /dev/null +++ b/knot2/crates/knot-langs/src/lib.rs @@ -0,0 +1,4 @@ +mod langs; +mod linguist; + +pub use langs::analyze; diff --git a/knot2/crates/knot-langs/src/linguist.rs b/knot2/crates/knot-langs/src/linguist.rs new file mode 100644 index 0000000000000000000000000000000000000000..244cfcb4cd6f8329cb3b8c36f9a86cfe196312a2 --- /dev/null +++ b/knot2/crates/knot-langs/src/linguist.rs @@ -0,0 +1,196 @@ +use std::sync::LazyLock; + +use regex::Regex; + +static VENDOR: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:^(?:(?:[Dd]ependencies/)|(?:debian/)|(?:deps/)|(?:rebar$)))|(?:(?:^|/)(?:(?:BuddyBuildSDK\.framework/)|(?:Carthage/)|(?:Chart\.js$)|(?:Control\.FullScreen\.css)|(?:Control\.FullScreen\.js)|(?:Crashlytics\.framework/)|(?:Fabric\.framework/)|(?:Godeps/_workspace/)|(?:Jenkinsfile$)|(?:Leaflet\.Coordinates-\d+\.\d+\.\d+\.src\.js$)|(?:MathJax/)|(?:MochiKit\.js$)|(?:RealmSwift\.framework)|(?:Realm\.framework)|(?:Sparkle/)|(?:Vagrantfile$)|(?:[Bb]ourbon/.*\.(css|less|scss|styl)$)|(?:[Cc]ode[Mm]irror/(\d+\.\d+/)?(lib|mode|theme|addon|keymap|demo))|(?:[Ee]xtern(als?)?/)|(?:[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$)|(?:[Pp]ackages/.+\.\d+/)|(?:[Ss]pecs?/fixtures/)|(?:[Tt]ests?/fixtures/)|(?:[Vv]+endor/)|(?:\.[Dd][Ss]_[Ss]tore$)|(?:\.gitattributes$)|(?:\.github/)|(?:\.gitignore$)|(?:\.gitmodules$)|(?:\.gitpod\.Dockerfile$)|(?:\.google_apis/)|(?:\.indent\.pro)|(?:\.mvn/wrapper/)|(?:\.obsidian/)|(?:\.osx$)|(?:\.sublime-project)|(?:\.sublime-workspace)|(?:\.teamcity/)|(?:\.vscode/)|(?:\.yarn/plugins/)|(?:\.yarn/releases/)|(?:\.yarn/sdks/)|(?:\.yarn/unplugged/)|(?:\.yarn/versions/)|(?:_esy$)|(?:ace-builds/)|(?:aclocal\.m4)|(?:activator$)|(?:activator\.bat$)|(?:admin_media/)|(?:angular([^.]*)\.js$)|(?:animate\.(css|less|scss|styl)$)|(?:bootbox\.js)|(?:bootstrap([^/.]*)(\..*)?\.(js|css|less|scss|styl)$)|(?:bootstrap-datepicker/)|(?:bower_components/)|(?:bulma\.(css|sass|scss)$)|(?:cache/)|(?:ckeditor\.js$)|(?:config\.guess$)|(?:config\.sub$)|(?:configure$)|(?:controls\.js$)|(?:cordova([^.]*)\.js$)|(?:cordova\-\d\.\d(\.\d)?\.js$)|(?:cpplint\.py)|(?:custom\.bootstrap([^\s]*)(js|css|less|scss|styl)$)|(?:dist/)|(?:docs?/_?(build|themes?|templates?|static)/)|(?:dojo\.js$)|(?:dotnet-install\.(ps1|sh)$)|(?:dragdrop\.js$)|(?:effects\.js$)|(?:env/)|(?:erlang\.mk)|(?:extjs/.*?\.html$)|(?:extjs/.*?\.js$)|(?:extjs/.*?\.properties$)|(?:extjs/.*?\.txt$)|(?:extjs/.*?\.xml$)|(?:extjs/\.sencha/)|(?:extjs/builds/)|(?:extjs/cmd/)|(?:extjs/docs/)|(?:extjs/examples/)|(?:extjs/locale/)|(?:extjs/packages/)|(?:extjs/plugins/)|(?:extjs/resources/)|(?:extjs/src/)|(?:extjs/welcome/)|(?:fabfile\.py$)|(?:flow-typed/.*\.js$)|(?:font-?awesome/.*\.(css|less|scss|styl)$)|(?:font-?awesome\.(css|less|scss|styl)$)|(?:fontello(.*?)\.css$)|(?:foundation(\..*)?\.js$)|(?:foundation\.(css|less|scss|styl)$)|(?:fuelux\.js)|(?:gradle/wrapper/)|(?:gradlew$)|(?:gradlew\.bat$)|(?:html5shiv\.js$)|(?:inst/extdata/)|(?:jquery([^.]*)\.js$)|(?:jquery([^.]*)\.unobtrusive\-ajax\.js$)|(?:jquery([^.]*)\.validate(\.unobtrusive)?\.js$)|(?:jquery\-\d\.\d+(\.\d+)?\.js$)|(?:jquery\-ui(\-\d\.\d+(\.\d+)?)?(\.\w+)?\.(js|css)$)|(?:jquery\.(ui|effects)\.([^.]*)\.(js|css)$)|(?:jquery\.dataTables\.js)|(?:jquery\.fancybox\.(js|css))|(?:jquery\.fileupload(-\w+)?\.js$)|(?:jquery\.fn\.gantt\.js)|(?:knockout-(\d+\.){3}(debug\.)?js$)|(?:leaflet\.draw-src\.js)|(?:leaflet\.draw\.css)|(?:leaflet\.spin\.js)|(?:libtool\.m4)|(?:ltoptions\.m4)|(?:ltsugar\.m4)|(?:ltversion\.m4)|(?:lt~obsolete\.m4)|(?:materialize\.(css|less|scss|styl|js)$)|(?:modernizr\-\d\.\d+(\.\d+)?\.js$)|(?:modernizr\.custom\.\d+\.js$)|(?:mootools([^.]*)\d+\.\d+.\d+([^.]*)\.js$)|(?:mvnw$)|(?:mvnw\.cmd$)|(?:node_modules/)|(?:normalize\.(css|less|scss|styl)$)|(?:octicons\.css)|(?:pdf\.worker\.js)|(?:proguard-rules\.pro$)|(?:proguard\.pro$)|(?:prototype(.*)\.js$)|(?:puphpet/)|(?:react(-[^.]*)?\.js$)|(?:run\.n$)|(?:select2/.*\.(css|scss|js)$)|(?:shBrush([^.]*)\.js$)|(?:shCore\.js$)|(?:shLegacy\.js$)|(?:skeleton\.(css|less|scss|styl)$)|(?:slick\.\w+.js$)|(?:sprockets-octicons\.scss)|(?:testdata/)|(?:tiny_mce([^.]*)\.js$)|(?:tiny_mce/(langs|plugins|themes|utils))|(?:vendors?/)|(?:waf$)|(?:wicket-leaflet\.js)|(?:xvba_modules/)|(?:yahoo-([^.]*)\.js$)|(?:yui([^.]*)\.js$)))|(?:(.*?)\.d\.ts$)|(?:(3rd|[Tt]hird)[-_]?[Pp]arty/)|(?:([^\s]*)import\.(css|less|scss|styl)$)|(?:(\.|-)min\.(js|css)$)|(?:(^|/)d3(\.v\d+)?([^.]*)\.js$)|(?:-vsdoc\.js$)|(?:\.imageset/)|(?:\.intellisense\.js$)|(?:\.xctemplate/)").expect("enry vendor regex compiles") +}); + +static DOCUMENTATION: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:^[Dd]ocs?/)|(?:(^|/)[Dd]ocumentation/)|(?:(^|/)[Gg]roovydoc/)|(?:(^|/)[Jj]avadoc/)|(?:^[Mm]an/)|(?:^[Ee]xamples/)|(?:^[Dd]emos?/)|(?:(^|/)inst/doc/)|(?:(^|/)CITATION(\.cff|(S)?(\.(bib|md))?)$)|(?:(^|/)CHANGE(S|LOG)?(\.|$))|(?:(^|/)CONTRIBUTING(\.|$))|(?:(^|/)COPYING(\.|$))|(?:(^|/)INSTALL(\.|$))|(?:(^|/)LICEN[CS]E(\.|$))|(?:(^|/)[Ll]icen[cs]e(\.|$))|(?:(^|/)README(\.|$))|(?:(^|/)[Rr]eadme(\.|$))|(?:^[Ss]amples?/)").expect("enry documentation regex compiles") +}); + +static GENERATED_NAME: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:(?:^|/)\.idea/)|(?:(^Pods|/Pods)/)|(?:(^|/)Carthage/Build/)|(?:(?i)\.designer\.(cs|vb)$)|(?:(?i)\.feature\.cs$)|(?:vendor/([-0-9A-Za-z]+\.)+(com|edu|gov|in|me|net|org|fm|io))|(?:(^|/)(\w+\.)?esy.lock$)|(?:(^|/)\.pnp\..*$)|(?:.\.zep\.(?:c|h|php)$)|(?:(^|/)flake\.lock$)|(?:(^|/)MODULE\.bazel\.lock$)|(?:(?:^|/)\.terraform\.lock\.hcl$)|(?:(?i)_tlb\.pas$)|(?:(?:^|/)htmlcov/)|(?:(?:^|.*/)\.sqlx/query-.+\.json$)").expect("enry generated-name regex compiles") +}); + +const GENERATED_SUFFIXES: &[&str] = &[ + "Gopkg.lock", + "glide.lock", + "poetry.lock", + "pdm.lock", + "uv.lock", + "deno.lock", + "npm-shrinkwrap.json", + "package-lock.json", + "pnpm-lock.yaml", + "composer.lock", + "Cargo.lock", + "Cargo.toml.orig", + "Pipfile.lock", + "bun.lock", +]; + +const GENERATED_CONTAINS: &[&str] = &["node_modules/", "Godeps/", "__generated__/"]; + +const GENERATED_EXTENSIONS: &[&str] = &[".nib", ".xcworkspacedata", ".xcuserstate"]; + +fn is_dotfile(path: &str) -> bool { + path.rsplit('/') + .next() + .is_some_and(|base| base.starts_with('.') && base != ".") +} + +fn is_generated_name(path: &str) -> bool { + GENERATED_SUFFIXES + .iter() + .any(|suffix| path.ends_with(suffix)) + || GENERATED_CONTAINS + .iter() + .any(|needle| path.contains(needle)) + || GENERATED_EXTENSIONS + .iter() + .any(|extension| path.ends_with(extension)) + || GENERATED_NAME.is_match(path) +} + +pub(crate) fn is_skipped_path(path: &str) -> bool { + is_dotfile(path) + || VENDOR.is_match(path) + || DOCUMENTATION.is_match(path) + || is_generated_name(path) +} + +pub(crate) fn is_vendor_dir(path: &str) -> bool { + VENDOR.is_match(&format!("{path}/")) +} + +fn language_group(name: &str) -> Option<&'static str> { + match name { + "Alpine Abuild" => Some("Shell"), + "Apollo Guidance Computer" => Some("Assembly"), + "BibTeX" => Some("TeX"), + "Bison" => Some("Yacc"), + "Bluespec BH" => Some("Bluespec"), + "C2hs Haskell" => Some("Haskell"), + "Cairo" => Some("Cairo"), + "Cairo Zero" => Some("Cairo"), + "CameLIGO" => Some("LigoLANG"), + "ColdFusion CFC" => Some("ColdFusion"), + "Cylc" => Some("INI"), + "ECLiPSe" => Some("Prolog"), + "Easybuild" => Some("Python"), + "Ecere Projects" => Some("JavaScript"), + "Ecmarkup" => Some("HTML"), + "EditorConfig" => Some("INI"), + "Elvish Transcript" => Some("Elvish"), + "Filterscript" => Some("RenderScript"), + "Fortran" => Some("Fortran"), + "Fortran Free Form" => Some("Fortran"), + "Gentoo Ebuild" => Some("Shell"), + "Gentoo Eclass" => Some("Shell"), + "Git Config" => Some("INI"), + "Glimmer JS" => Some("JavaScript"), + "Glimmer TS" => Some("TypeScript"), + "Gradle Kotlin DSL" => Some("Gradle"), + "Groovy Server Pages" => Some("Groovy"), + "HTML+ECR" => Some("HTML"), + "HTML+EEX" => Some("HTML"), + "HTML+ERB" => Some("HTML"), + "HTML+PHP" => Some("HTML"), + "HTML+Razor" => Some("HTML"), + "Isabelle ROOT" => Some("Isabelle"), + "JFlex" => Some("Lex"), + "JSON with Comments" => Some("JSON"), + "Java Server Pages" => Some("Java"), + "Java Template Engine" => Some("Java"), + "JavaScript+ERB" => Some("JavaScript"), + "Jison" => Some("Yacc"), + "Jison Lex" => Some("Lex"), + "Julia REPL" => Some("Julia"), + "Lean 4" => Some("Lean"), + "LigoLANG" => Some("LigoLANG"), + "Literate Agda" => Some("Agda"), + "Literate CoffeeScript" => Some("CoffeeScript"), + "Literate Haskell" => Some("Haskell"), + "M4Sugar" => Some("M4"), + "MUF" => Some("Forth"), + "Maven POM" => Some("XML"), + "Motorola 68K Assembly" => Some("Assembly"), + "NPM Config" => Some("INI"), + "NumPy" => Some("Python"), + "OASv2-json" => Some("OpenAPI Specification v2"), + "OASv2-yaml" => Some("OpenAPI Specification v2"), + "OASv3-json" => Some("OpenAPI Specification v3"), + "OASv3-yaml" => Some("OpenAPI Specification v3"), + "OpenCL" => Some("C"), + "OpenRC runscript" => Some("Shell"), + "Parrot Assembly" => Some("Parrot"), + "Parrot Internal Representation" => Some("Parrot"), + "Pic" => Some("Roff"), + "PostCSS" => Some("CSS"), + "Python console" => Some("Python"), + "Python traceback" => Some("Python"), + "RBS" => Some("Ruby"), + "Readline Config" => Some("INI"), + "ReasonLIGO" => Some("LigoLANG"), + "Roff Manpage" => Some("Roff"), + "SSH Config" => Some("INI"), + "STON" => Some("Smalltalk"), + "Simple File Verification" => Some("Checksums"), + "Snakemake" => Some("Python"), + "TSX" => Some("TypeScript"), + "Tcsh" => Some("Shell"), + "Terraform Template" => Some("HCL"), + "Unified Parallel C" => Some("C"), + "Unix Assembly" => Some("Assembly"), + "Wget Config" => Some("INI"), + "X BitMap" => Some("C"), + "X PixMap" => Some("C"), + "XML Property List" => Some("XML"), + "cURL Config" => Some("INI"), + "fish" => Some("Shell"), + "nanorc" => Some("INI"), + _ => None, + } +} + +pub(crate) fn group(name: &'static str) -> &'static str { + language_group(name).unwrap_or(name) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn enry_matchers_compile_and_match_known_paths() { + assert!(is_skipped_path("node_modules/left-pad/index.js")); + assert!(is_skipped_path("src/jquery-3.6.0.js")); + assert!(is_skipped_path("docs/guide.md")); + assert!(is_skipped_path("README.md")); + assert!(is_skipped_path("LICENSE")); + assert!(is_skipped_path("Cargo.lock")); + assert!(is_skipped_path("package-lock.json")); + assert!(is_skipped_path("app/main.designer.cs")); + assert!(is_skipped_path(".github/workflows/ci.yml")); + assert!(is_skipped_path("third_party/zlib/zlib.c")); + assert!(is_skipped_path("web/app.min.js")); + assert!(!is_skipped_path("src/main.rs")); + assert!(!is_skipped_path("internal/server.go")); + } + + #[test] + fn vendor_dir_pruning_matches_directory_paths() { + assert!(is_vendor_dir("node_modules")); + assert!(is_vendor_dir("a/b/dist")); + assert!(!is_vendor_dir("src")); + } + + #[test] + fn grouping_folds_known_languages() { + assert_eq!(group("TSX"), "TypeScript"); + assert_eq!(group("HTML+ERB"), "HTML"); + assert_eq!(group("Rust"), "Rust"); + } +} diff --git a/knot2/crates/knot-lexicons/Cargo.toml b/knot2/crates/knot-lexicons/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..0a71f219d377a3e36a0b59be60357dd2bdbddc13 --- /dev/null +++ b/knot2/crates/knot-lexicons/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "knot-lexicons" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +jacquard-common = { workspace = true } +jacquard-derive = { workspace = true } +jacquard-lexicon = { workspace = true, default-features = false } +bytes = { workspace = true } +cid = { workspace = true } +miette = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +url = { workspace = true } + +[build-dependencies] +anyhow = { workspace = true } +jacquard-lexicon = { workspace = true, features = ["codegen"] } +serde_json = { workspace = true } +walkdir = { workspace = true } + +[features] +default = ["com_atproto", "org_tangled"] +com_atproto = [] +org_tangled = [] +streaming = [] + +[lints.clippy] +unnecessary_lazy_evaluations = "allow" diff --git a/knot2/crates/knot-lexicons/build.rs b/knot2/crates/knot-lexicons/build.rs new file mode 100644 index 0000000000000000000000000000000000000000..9d8758f9d9014a546d4013727905ed7451d6f702 --- /dev/null +++ b/knot2/crates/knot-lexicons/build.rs @@ -0,0 +1,103 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use jacquard_lexicon::codegen::{CodeGenerator, CodegenMode}; +use jacquard_lexicon::corpus::LexiconCorpus; +use walkdir::WalkDir; + +const LEXICONS_SUBDIR: &str = "lexicons"; +const STAGED_SUBDIR: &str = "lexicons-staged"; +const GENERATED_SUBDIR: &str = "_lex"; +const GENERATED_ROOT_FILE: &str = "_lex_root.rs"; +const SKIPPED_SEGMENTS: [&str; 2] = ["temp", "legacy"]; + +fn main() -> Result<()> { + let lexicons_dir = match std::env::var("TANGLED_LEXICONS_DIR") { + Ok(dir) => PathBuf::from(dir), + Err(_) => { + let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR")?); + manifest_dir + .parent() + .and_then(Path::parent) + .and_then(Path::parent) + .context("resolve workspace root from manifest dir")? + .join(LEXICONS_SUBDIR) + } + }; + + println!("cargo:rerun-if-env-changed=TANGLED_LEXICONS_DIR"); + println!("cargo:rerun-if-changed={}", lexicons_dir.display()); + println!("cargo:rerun-if-changed=build.rs"); + + let out_dir = PathBuf::from(std::env::var("OUT_DIR")?); + let staged = out_dir.join(STAGED_SUBDIR); + if staged.exists() { + std::fs::remove_dir_all(&staged).context("clean staged lexicons")?; + } + stage_lexicons(&lexicons_dir, &staged)?; + + let corpus = LexiconCorpus::load_from_dir(&staged) + .map_err(|e| anyhow::anyhow!("load lexicon corpus: {e:?}"))?; + + let generated = out_dir.join(GENERATED_SUBDIR); + if generated.exists() { + std::fs::remove_dir_all(&generated).context("clean generated dir")?; + } + std::fs::create_dir_all(&generated).context("create generated dir")?; + + let codegen = CodeGenerator::with_mode(&corpus, "crate", CodegenMode::Pretty); + codegen + .write_to_disk(&generated) + .map_err(|e| anyhow::anyhow!("write generated code: {e:?}"))?; + write_generated_root(&generated, &out_dir.join(GENERATED_ROOT_FILE))?; + + Ok(()) +} + +/// absolute #[path] so the generated `mod x;` files resolve inside OUT_DIR, not next to lib.rs +fn write_generated_root(generated: &Path, path: &Path) -> Result<()> { + let source = format!( + "#[path = {:?}]\nmod generated;\npub use generated::*;\n", + generated.join("lib.rs") + ); + std::fs::write(path, source).context("write generated root") +} + +fn stage_lexicons(src: &Path, dst: &Path) -> Result<()> { + WalkDir::new(src) + .into_iter() + .filter_map(std::result::Result::ok) + .filter(|entry| entry.file_type().is_file()) + .filter(|entry| { + entry + .path() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("json")) + }) + .filter(|entry| { + !entry.path().components().any(|component| { + SKIPPED_SEGMENTS + .iter() + .any(|skip| component.as_os_str() == *skip) + }) + }) + .try_for_each(|entry| -> Result<()> { + let staged_namespace = + std::fs::read(entry.path()).context("read lexicon for staging")?; + let staged_namespace = serde_json::from_slice::(&staged_namespace) + .context("parse lexicon for staging")?; + let staged_namespace = staged_namespace["id"].as_str().unwrap_or_default(); + if !staged_namespace.starts_with("com.atproto.") + && !staged_namespace.starts_with("org.tangled.") + { + return Ok(()); + } + let rel = entry.path().strip_prefix(src).context("strip src prefix")?; + let target = dst.join(rel); + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).context("create staged parent")?; + } + std::fs::copy(entry.path(), &target).context("copy lexicon file")?; + Ok(()) + }) +} diff --git a/knot2/crates/knot-lexicons/src/lib.rs b/knot2/crates/knot-lexicons/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..5f5cc8914b2f7b36048628c2abb821a9c4f88e3f --- /dev/null +++ b/knot2/crates/knot-lexicons/src/lib.rs @@ -0,0 +1,57 @@ +extern crate alloc; + +#[allow(non_snake_case, unused_imports, unused_extern_crates)] +#[allow(clippy::all)] +#[rustfmt::skip] +mod _lex { + include!(concat!(env!("OUT_DIR"), "/_lex_root.rs")); +} + +pub use _lex::*; + +#[cfg(test)] +mod tests { + use jacquard_common::DefaultStr; + use jacquard_common::types::{collection::Collection, string::Datetime}; + + use crate::org_tangled::{graph, knot}; + + const WHEN: &str = "2026-06-11T00:00:00Z"; + const NSIDS: [&str; 2] = [ + ::NSID, + ::NSID, + ]; + + #[test] + fn a_grant_record_is_the_bare_offer_shape() { + let member = knot::member::Member:: { + created_at: Datetime::raw_str(WHEN), + extra_data: None, + }; + let collaborator = graph::collaborator::Collaborator:: { + created_at: Datetime::raw_str(WHEN), + extra_data: None, + }; + + assert_eq!( + serde_json::json!([member, collaborator]), + serde_json::json!([ + { "$type": "org.tangled.knot.member", "createdAt": WHEN }, + { "$type": "org.tangled.graph.collaborator", "createdAt": WHEN } + ]), + "a grant record is the bare offer keyed by its subject and holds no other fact" + ); + } + + #[test] + fn every_consent_collection_owns_its_own_nsid() { + assert_eq!( + NSIDS + .into_iter() + .collect::>() + .len(), + NSIDS.len(), + "two consent collections have the same NSID, so their records are stored together" + ); + } +} diff --git a/knot2/crates/knot-lfs/Cargo.toml b/knot2/crates/knot-lfs/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..bb48fe3a69e0a39f6a79bcf7451a141ba1d2ead6 --- /dev/null +++ b/knot2/crates/knot-lfs/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "knot-lfs" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-messages = { workspace = true } +knot-git = { workspace = true } +knot-runtime = { workspace = true } +knot-resource = { workspace = true } +gix-packetline = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } +tempfile = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/knot2/crates/knot-lfs/fuzz/.gitignore b/knot2/crates/knot-lfs/fuzz/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1a45eee7760d240bfa8ac1989088e0349169f264 --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-lfs/fuzz/Cargo.lock b/knot2/crates/knot-lfs/fuzz/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..17c292d0f757a6238765e88bcec10f27bd199007 --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/Cargo.lock @@ -0,0 +1,5414 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base45" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240e56f4d3c453c36faacb695c535a4d5f8c7d23dac175014f32eb0a71012a03" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cee35f73844aa3014bb606320a6c1f010249dbdf43342fe54b5a4f6a8ed4b79" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +dependencies = [ + "serde", +] + +[[package]] +name = "bytesize" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7c8918969267b2932ffd5655509bbbea0833823058c378876953217f5fc50e" + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "clru" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "gix" +version = "0.84.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae54ae0ebd1a5a3c3f8d95dd3b5ca6e63f4fed9bfd585e13801a97d7bde8f9ce" +dependencies = [ + "gix-actor", + "gix-archive", + "gix-attributes", + "gix-blame", + "gix-command", + "gix-commitgraph", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-ignore", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-negotiate", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-pathspec", + "gix-prompt", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-shallow", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "gix-worktree", + "gix-worktree-state", + "gix-worktree-stream", + "nonempty", + "parking_lot", + "regex", + "signal-hook", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-actor" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc998b8f746dda8565450d08a63b792ced9165d8c27a1ed3f02799ec6a7820f" +dependencies = [ + "bstr", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-archive" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16909cacc78936ab96f6c3be08379d0a2e88bfa3a7527972d2ed75c7517ef31e" +dependencies = [ + "bstr", + "flate2", + "gix-date", + "gix-error", + "gix-object", + "gix-path", + "gix-worktree-stream", + "rawzip", + "tar", +] + +[[package]] +name = "gix-attributes" +version = "0.33.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39b40888d0ed415c0744a6cdc61eebf0304c9d26ab726725b718443c322e5ba4" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-bitmap" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ebef0c26ad305747649e727bbcd56a7b7910754eb7cea88f6dff6f93c51283" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-blame" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d39a0c14af94c2edaa5eefe06d5ef2cdea55316ae9a9321314288e3f55fa4c0" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-diff", + "gix-error", + "gix-hash", + "gix-object", + "gix-revwalk", + "gix-trace", + "gix-traverse", + "gix-worktree", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-chunk" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9faee47943b638e58ddd5e275a4906ad3e4b6c8584f1d41bd18ab9032ec52afb" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-command" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00706d4fef135ef4b01680d5218c6ee40cda8baf697b864296cbc887d19118f6" +dependencies = [ + "bstr", + "gix-path", + "gix-quote", + "gix-trace", + "shell-words", +] + +[[package]] +name = "gix-commitgraph" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" +dependencies = [ + "bstr", + "gix-chunk", + "gix-error", + "gix-hash", + "memmap2", + "nonempty", +] + +[[package]] +name = "gix-config" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2372d4b49ca28431e7d150cab9d25edc1890f0184bd57eb0e917c7799e63de" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-config-value" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-path", + "libc", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-credentials" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40cd22f0dd71988be12d6e78b1709de2370e1957c5f107ff31e56caeba3745d" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-date", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-trace", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-date" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d63f9e28b59ddeb1a1eb9e5cf986a9222b5d484947445edbc20473939cc7fd0" +dependencies = [ + "bstr", + "gix-error", + "itoa", + "jiff", +] + +[[package]] +name = "gix-diff" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6d9528f32d94cef2edf39a1ac01fe5a0fc44ddbb18d9e44099936047c3302b" +dependencies = [ + "bstr", + "gix-attributes", + "gix-command", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-imara-diff", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-dir" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bb2a53a6fd917ec499ed0bfb5b6887de7a15bd79197dcea7c987938749a9f1" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-discover" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77bacdd12b7879d2178a80c58c2f319995e4654e1a7a23e3181e5c8a12b824f7" +dependencies = [ + "bstr", + "dunce", + "gix-fs", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-error" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57831e199be480af90dcd7e459abed8a174c09ec9a6e2cc8f7ca6c54598b06b" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "bytes", + "bytesize", + "crc32fast", + "crossbeam-channel", + "gix-path", + "gix-trace", + "gix-utils", + "libc", + "once_cell", + "parking_lot", + "prodash", + "thiserror 2.0.18", + "walkdir", + "zlib-rs", +] + +[[package]] +name = "gix-filter" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf74b7d16f6694ce4a3049074c41be0c7987105743674f1671807bd6dce09fa" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash", + "gix-object", + "gix-packetline", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-fs" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" +dependencies = [ + "bstr", + "fastrand", + "gix-features", + "gix-path", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-glob" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-hashtable" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e261d54091f0d1c729bc83f54548c071bdec60a697de1e58e88bdfd7a99d24e" +dependencies = [ + "gix-hash", + "hashbrown 0.17.1", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-imara-diff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b305d85504de270ad3525d726a6b69cc59ee7b2269b014387651107ab9f0755b" +dependencies = [ + "bstr", + "hashbrown 0.15.5", +] + +[[package]] +name = "gix-index" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6b28cc592dc753adb58302bb14a64e412ee591a3bec77aa4df87bff74fa80d" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils", + "gix-validate", + "hashbrown 0.17.1", + "itoa", + "libc", + "memmap2", + "rustix", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-lock" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-mailmap" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195fd20808055824531be2fd0d34136d900e5fbca3ffb0a3c07e8beeefb9c828" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-negotiate" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890c936a215bae25818c076cb881cb2e54d2c66ba947ba58b8dd47cff921bf55" +dependencies = [ + "bitflags 2.13.0", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-object", + "gix-revwalk", +] + +[[package]] +name = "gix-object" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5cd857e29429c7213bdef3f5aef83f8cc124774fe8ae0d27b1607d218d6d525" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-odb" +version = "0.81.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d004c32858b1556f2d7874405edb3c97dc78fc09beaa87d57bb077ee2858a7d" +dependencies = [ + "arc-swap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "memmap2", + "parking_lot", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pack" +version = "0.71.0" +dependencies = [ + "clru", + "gix-chunk", + "gix-diff", + "gix-error", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-traverse", + "parking_lot", + "smallvec", + "thiserror 2.0.18", + "uluru", +] + +[[package]] +name = "gix-packetline" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b217dd0ee0c4021ecf169a4a519b1b4f80d15e3f3765f3dc466223dc0ac891d7" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-path" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa6ac14cd14939ea94a496ce7460daa6511c09f5b84757e9cfc6f9c8d0f93a6" +dependencies = [ + "bstr", + "gix-trace", + "gix-validate", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pathspec" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-prompt" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee604d7746080ae7e1023bf47204bcc2c5f307bfbe2306a3c90b1bfd1a2c6d8" +dependencies = [ + "gix-command", + "gix-config-value", + "parking_lot", + "rustix", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-protocol" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51dea3acb390707ab868f1f9584f18449eb95d869deffae96768e47d303595ee" +dependencies = [ + "bstr", + "gix-date", + "gix-features", + "gix-hash", + "gix-ref", + "gix-shallow", + "gix-transport", + "gix-utils", + "maybe-async", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-quote" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" +dependencies = [ + "bstr", + "gix-error", + "gix-utils", +] + +[[package]] +name = "gix-ref" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c04f64c37eb7e6feb73c7060f8dc6f381cc5de5d53249bfd450bc48a86b2e8b" +dependencies = [ + "gix-actor", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils", + "gix-validate", + "memmap2", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-refspec" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b216ae06ec74b5f24ad0142026a997fb0a935b7410eaf9c1616fc3f0e6c5a6d3" +dependencies = [ + "bstr", + "gix-error", + "gix-glob", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-revision" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b47c88884dd3c1a19a39da19d10211fcdea2809aadc86869b6e824a1774340f" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "nonempty", +] + +[[package]] +name = "gix-revwalk" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f5756abffe0917827aac683b13684ed99875bc398fa1f9b8f479b0681ef9e6" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-sec" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8519976e4c7e486270740a5400369f37940779b80bd1377d94cfa1125d01b3" +dependencies = [ + "bitflags 2.13.0", + "gix-path", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "gix-shallow" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" +dependencies = [ + "bstr", + "gix-hash", + "gix-lock", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-status" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22042e385d28a34275e029d98f4970285045be14b9073658ca897923f2ed8700" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-submodule" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3059890ef054066c22a94bfc6a3eaba0d806aedcd630a0bc9e5783fd88884781" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-tempfile" +version = "23.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ef60812443484e67bf84e444cc71b4c78ae62deb822221774a4fa0c57fdb17f" +dependencies = [ + "dashmap", + "gix-fs", + "libc", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "gix-transport" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186874f7ad1fb2f9a2f2aa9c2dabc7f9dd087bef74c1a0eee2b4a9cf0248fcb3" +dependencies = [ + "bstr", + "gix-command", + "gix-features", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-traverse" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8de590ecc86a3b2870665f2288324fa9f7f8672c7fc2d4e020fdd81cd1f7aed" +dependencies = [ + "bitflags 2.13.0", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-url" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bb01ec69d55e82ccb7a19e264501ead4e6aac38463a8cebfdd81e22bb67ab2" +dependencies = [ + "bstr", + "gix-path", + "percent-encoding", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-utils" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c50966184123caf580ffa64e28031a878597f1c7fceb8fe19566c38eb1b771" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc6fc771c4063ba7cd2f47b91fb6076251c6a823b64b7fe7b8874b0fe4afae3" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-worktree" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef414ed275e8407cd5d53d301e83be19700b0dd3f859d2434417b58f454a2d1" +dependencies = [ + "bstr", + "gix-attributes", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate", +] + +[[package]] +name = "gix-worktree-state" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bffae8b3ca258fdd50370cd51f06deb4c76a3b43db3868bc28dde45ffa77d69" +dependencies = [ + "bstr", + "gix-features", + "gix-filter", + "gix-fs", + "gix-index", + "gix-object", + "gix-path", + "gix-worktree", + "io-close", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-worktree-stream" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25e9ed30100c63f7590bc581c225e53f731a53e06aa79a245739c07f7dcc557" +dependencies = [ + "gix-attributes", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-traverse", + "parking_lot", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "human_format" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaec953f16e5bcf6b8a3cb3aa959b17e5577dbd2693e94554c462c08be22624b" + +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest 0.12.28", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jiff" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" +dependencies = [ + "defmt", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-static" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6142247df1a93c2b3587402a19710be3e6e942f1581a1702e76408f2c21d6590" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-git" +version = "0.1.0" +dependencies = [ + "base64", + "dashmap", + "flate2", + "gix", + "gix-archive", + "gix-bitmap", + "gix-hash", + "gix-pack", + "knot-resource", + "knot-types", + "moka", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "knot-lfs" +version = "0.1.0" +dependencies = [ + "gix-packetline", + "knot-git", + "knot-resource", + "knot-runtime", + "knot-types", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", + "tracing", + "url", +] + +[[package]] +name = "knot-lfs-fuzz" +version = "0.0.0" +dependencies = [ + "knot-lfs", + "libfuzzer-sys", +] + +[[package]] +name = "knot-resource" +version = "0.1.0" +dependencies = [ + "rustix", +] + +[[package]] +name = "knot-runtime" +version = "0.1.0" +dependencies = [ + "bytes", + "futures", + "getrandom 0.4.3", + "http", + "k256", + "reqwest 0.13.1", + "thiserror 2.0.18", + "tokio", + "url", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multibase" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e0e4a371cbf1dfd666b658ba137763edb23c45beb43cfe369b5593cd6b437b6" +dependencies = [ + "base-x", + "base256emoji", + "base45", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nonempty" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "bytesize", + "human_format", + "parking_lot", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.0", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "reqwest" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags 2.13.0", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uluru" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8a2469e56e6e5095c82ccd3afb98dad95f7af7929aab6d8ba8d6e0f73657da" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75726053136156d419e285b9b7eddaaea9e3fea6ce32eed44a89901f0bd98de1" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5431d5661c32445236631278f27946e444ddafe4684cac70b185272d4f9c52d5" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/knot2/crates/knot-lfs/fuzz/Cargo.toml b/knot2/crates/knot-lfs/fuzz/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..defd74581bdaaff13917cc3b77bf88f5f979a953 --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "knot-lfs-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-lfs] +path = ".." + +[[bin]] +name = "transfer" +path = "fuzz_targets/transfer.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "batch" +path = "fuzz_targets/batch.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "pointer" +path = "fuzz_targets/pointer.rs" +test = false +doc = false +bench = false + +[patch.crates-io] +gix-pack = { path = "../../../third_party/gix-pack" } diff --git a/knot2/crates/knot-lfs/fuzz/fuzz_targets/batch.rs b/knot2/crates/knot-lfs/fuzz/fuzz_targets/batch.rs new file mode 100644 index 0000000000000000000000000000000000000000..9ee5e91cd0ae5ff82a95d15be9e06aeafe1afdc2 --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/fuzz_targets/batch.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_lfs::fuzz::batch(data); +}); diff --git a/knot2/crates/knot-lfs/fuzz/fuzz_targets/pointer.rs b/knot2/crates/knot-lfs/fuzz/fuzz_targets/pointer.rs new file mode 100644 index 0000000000000000000000000000000000000000..289a8720acd9992ec03e9d882cc71838ad67c8c7 --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/fuzz_targets/pointer.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_lfs::fuzz::pointer(data); +}); diff --git a/knot2/crates/knot-lfs/fuzz/fuzz_targets/transfer.rs b/knot2/crates/knot-lfs/fuzz/fuzz_targets/transfer.rs new file mode 100644 index 0000000000000000000000000000000000000000..53b3aacf068e7b454361af71fca64e96712b74ba --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/fuzz_targets/transfer.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_lfs::fuzz::transfer(data); +}); diff --git a/knot2/crates/knot-lfs/src/admission.rs b/knot2/crates/knot-lfs/src/admission.rs new file mode 100644 index 0000000000000000000000000000000000000000..6410ecca762726ce36d2dd21361f098edcc2d89d --- /dev/null +++ b/knot2/crates/knot-lfs/src/admission.rs @@ -0,0 +1,177 @@ +use knot_resource::{BelowFloor, DiskGovernor, DiskReservation, FreeBytes}; + +use crate::{ClaimedSize, FreeSpaceFloor, LfsError, LfsSize, LfsStorePath}; + +pub trait UploadAdmission: Send + Sync { + fn admit(&self, declared: ClaimedSize) -> Result; + + fn max_object(&self) -> LfsSize; +} + +pub struct UploadPermit { + _reservation: Option, +} + +impl UploadPermit { + fn unreserved() -> Self { + Self { _reservation: None } + } +} + +pub struct StoreAdmission { + root: LfsStorePath, + max_object: LfsSize, + floor: FreeSpaceFloor, + governor: DiskGovernor, +} + +impl StoreAdmission { + pub fn new(root: LfsStorePath, max_object: LfsSize, floor: FreeSpaceFloor) -> Self { + let governor = DiskGovernor::new(knot_resource::DiskFloorBytes::new(floor.get())); + Self { + root, + max_object, + floor, + governor, + } + } +} + +impl UploadAdmission for StoreAdmission { + fn admit(&self, declared: ClaimedSize) -> Result { + if declared.get() > self.max_object.get() { + tracing::warn!( + declared = declared.get(), + limit = self.max_object.get(), + "lfs upload denied by the object size limit" + ); + return Err(LfsError::SizeLimitExceeded { + declared, + limit: self.max_object, + }); + } + if self.floor.get() == 0 { + return Ok(UploadPermit::unreserved()); + } + let free = + knot_resource::disk_free_bytes(self.root.as_path()).map_err(|source| LfsError::Io { + op: "probe free space under", + path: self.root.as_path().to_path_buf(), + source, + })?; + self.admit_against(free, declared) + } + + fn max_object(&self) -> LfsSize { + self.max_object + } +} + +impl StoreAdmission { + fn admit_against( + &self, + free: FreeBytes, + declared: ClaimedSize, + ) -> Result { + match self + .governor + .reserve_against(free, knot_resource::ReserveBytes::new(declared.get())) + { + Ok(reservation) => Ok(UploadPermit { + _reservation: Some(reservation), + }), + Err(BelowFloor { free, .. }) => { + tracing::warn!( + declared = declared.get(), + free = free.get(), + floor = self.floor.get(), + "lfs upload denied below the free-space floor" + ); + Err(LfsError::FreeSpaceDenied { + free: LfsSize::new(free.get()), + floor: self.floor, + }) + } + } + } +} + +#[cfg(test)] +pub(crate) struct Unbounded; + +#[cfg(test)] +impl UploadAdmission for Unbounded { + fn admit(&self, _declared: ClaimedSize) -> Result { + Ok(UploadPermit::unreserved()) + } + + fn max_object(&self) -> LfsSize { + LfsSize::new(u64::MAX) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_size_limit_rejects_before_touching_the_disk() { + let gate = StoreAdmission::new( + LfsStorePath::new("/definitely/not/mounted"), + LfsSize::new(8), + FreeSpaceFloor::new(0), + ); + assert!(matches!( + gate.admit(ClaimedSize::new(9)), + Err(LfsError::SizeLimitExceeded { .. }) + )); + assert!(gate.admit(ClaimedSize::new(8)).is_ok()); + } + + #[test] + fn an_absurd_floor_denies_and_a_zero_floor_opts_out() { + let dir = tempfile::tempdir().unwrap(); + let strict = StoreAdmission::new( + LfsStorePath::new(dir.path()), + LfsSize::new(u64::MAX), + FreeSpaceFloor::new(u64::MAX), + ); + assert!(matches!( + strict.admit(ClaimedSize::new(1)), + Err(LfsError::FreeSpaceDenied { .. }) + )); + let opted_out = StoreAdmission::new( + LfsStorePath::new(dir.path()), + LfsSize::new(u64::MAX), + FreeSpaceFloor::new(0), + ); + assert!(opted_out.admit(ClaimedSize::new(u64::MAX)).is_ok()); + } + + #[test] + fn a_held_permit_reserves_against_the_next_admission() { + let dir = tempfile::tempdir().unwrap(); + let free = FreeBytes::new(10_240); + let gate = StoreAdmission::new( + LfsStorePath::new(dir.path()), + LfsSize::new(u64::MAX), + FreeSpaceFloor::new(4_096), + ); + let held = gate + .admit_against(free, ClaimedSize::new(4_096)) + .expect("a free-space reading the floor leaves room in admits a lone upload"); + assert!( + matches!( + gate.admit_against(free, ClaimedSize::new(4_096)), + Err(LfsError::FreeSpaceDenied { .. }) + ), + "a second upload cannot pass the floor while the first is in flight" + ); + drop(held); + assert!( + gate.admit_against(free, ClaimedSize::new(4_096)).is_ok(), + "the same reading must admit again once the first upload finishes, or the refusal \ + above was the floor rather than the reservation still being held" + ); + } +} diff --git a/knot2/crates/knot-lfs/src/batch.rs b/knot2/crates/knot-lfs/src/batch.rs new file mode 100644 index 0000000000000000000000000000000000000000..ca5018748a8e929c617a0a33e2fbe3e1ad80955a --- /dev/null +++ b/knot2/crates/knot-lfs/src/batch.rs @@ -0,0 +1,222 @@ +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use url::Url; + +use knot_types::{HttpStatus, RefName}; + +use crate::{ClaimedSize, LfsOid}; + +pub const BATCH_MEDIA_TYPE: &str = "application/vnd.git-lfs+json"; +pub const HASH_ALGO: &str = "sha256"; +pub const BASIC_TRANSFER: &str = "basic"; +pub const MAX_BATCH_OBJECTS: usize = 1000; + +// Unknown adapters parse instead of failing the whole body, +// so that a client offering something we don't serve +// will at least get a 422 that shows a mismatch not a serde error! +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TransferAdapter { + Basic, + Other(String), +} + +impl TransferAdapter { + pub fn is_basic(&self) -> bool { + matches!(self, Self::Basic) + } + + pub fn as_str(&self) -> &str { + match self { + Self::Basic => BASIC_TRANSFER, + Self::Other(value) => value, + } + } +} + +impl Serialize for TransferAdapter { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for TransferAdapter { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + Ok(if raw == BASIC_TRANSFER { + Self::Basic + } else { + Self::Other(raw) + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HashAlgo { + Sha256, + Other(String), +} + +impl HashAlgo { + pub fn is_sha256(&self) -> bool { + matches!(self, Self::Sha256) + } + + pub fn as_str(&self) -> &str { + match self { + Self::Sha256 => HASH_ALGO, + Self::Other(value) => value, + } + } +} + +impl Serialize for HashAlgo { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for HashAlgo { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + Ok(if raw == HASH_ALGO { + Self::Sha256 + } else { + Self::Other(raw) + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum BatchOperation { + Download, + Upload, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchRef { + pub name: RefName, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BatchObject { + pub oid: LfsOid, + pub size: ClaimedSize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchRequest { + pub operation: BatchOperation, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub transfers: Vec, + #[serde(default, rename = "ref", skip_serializing_if = "Option::is_none")] + pub reference: Option, + pub objects: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hash_algo: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchAction { + pub href: Url, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchActions { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub download: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub upload: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchObjectError { + pub code: HttpStatus, + pub message: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchResponseObject { + pub oid: LfsOid, + pub size: ClaimedSize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authenticated: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub actions: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +fn basic_transfer() -> TransferAdapter { + TransferAdapter::Basic +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchResponse { + #[serde(default = "basic_transfer")] + pub transfer: TransferAdapter, + pub objects: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hash_algo: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + const OID: &str = "6c17f2007cbe934aee6e309b28b2fba3c119d98be6ea4156da3aa3173456ad16"; + + #[test] + fn a_client_batch_request_round_trips() { + let body = format!( + r#"{{"operation":"download","transfers":["basic","ssh"],"ref":{{"name":"refs/heads/main"}},"objects":[{{"oid":"{OID}","size":42}}],"hash_algo":"sha256"}}"# + ); + let request: BatchRequest = serde_json::from_str(&body).unwrap(); + assert_eq!(request.operation, BatchOperation::Download); + assert_eq!(request.objects[0].oid.as_str(), OID); + assert_eq!(request.objects[0].size, ClaimedSize::new(42)); + assert_eq!( + request.reference.as_ref().unwrap().name.as_str(), + "refs/heads/main" + ); + } + + #[test] + fn a_hostile_oid_fails_the_whole_parse() { + let body = r#"{"operation":"download","objects":[{"oid":"../../etc/passwd","size":1}]}"#; + assert!(serde_json::from_str::(body).is_err()); + } + + #[test] + fn a_response_serializes_the_lfs_shape() { + let response = BatchResponse { + transfer: TransferAdapter::Basic, + objects: vec![BatchResponseObject { + oid: LfsOid::new(OID).unwrap(), + size: ClaimedSize::new(42), + authenticated: Some(true), + actions: Some(BatchActions { + download: Some(BatchAction { + href: Url::parse(&format!( + "https://nel.pet/did:plc:squid/media/info/lfs/objects/{OID}" + )) + .unwrap(), + }), + upload: None, + }), + error: None, + }], + hash_algo: Some(HashAlgo::Sha256), + }; + let json = serde_json::to_value(&response).unwrap(); + assert_eq!(json["transfer"], "basic"); + assert_eq!(json["objects"][0]["oid"], OID); + assert_eq!(json["objects"][0]["authenticated"], true); + assert!( + json["objects"][0]["actions"]["download"]["href"] + .as_str() + .unwrap() + .ends_with(OID) + ); + assert_eq!(json["objects"][0].get("error"), None); + } +} diff --git a/knot2/crates/knot-lfs/src/error.rs b/knot2/crates/knot-lfs/src/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..ee7f40eb1d804b94cf99645d80c5dd28e408ffa4 --- /dev/null +++ b/knot2/crates/knot-lfs/src/error.rs @@ -0,0 +1,51 @@ +use std::path::PathBuf; + +use crate::{ClaimedSize, FreeSpaceFloor, LfsOid, LfsSize}; + +#[derive(Debug, thiserror::Error)] +pub enum LfsError { + #[error("invalid LFS oid {value:?}")] + InvalidOid { value: String }, + #[error("unsafe repo DID {did:?}")] + UnsafeRepoDid { did: String }, + #[error("oid mismatch, declared {declared}, computed {computed}")] + HashMismatch { declared: LfsOid, computed: LfsOid }, + #[error("size mismatch, declared {declared}, received {received}")] + SizeMismatch { + declared: ClaimedSize, + received: LfsSize, + }, + #[error("object size {declared} exceeds limit {limit}")] + SizeLimitExceeded { + declared: ClaimedSize, + limit: LfsSize, + }, + #[error("free space {free} below floor {floor}")] + FreeSpaceDenied { + free: LfsSize, + floor: FreeSpaceFloor, + }, + #[error("object {oid} not found")] + NotFound { oid: LfsOid }, + #[error("protocol framing fault: {detail}")] + Framing { detail: String }, + #[error("too many {what} in one message, limit {limit}")] + TooMany { what: &'static str, limit: usize }, + #[error("transfer channel fault")] + Channel { + #[source] + source: std::io::Error, + }, + #[error("object body read failed")] + BodyRead { + #[source] + source: std::io::Error, + }, + #[error("{op} {path} failed")] + Io { + op: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, +} diff --git a/knot2/crates/knot-lfs/src/gc.rs b/knot2/crates/knot-lfs/src/gc.rs new file mode 100644 index 0000000000000000000000000000000000000000..4bf6b099a30d551b00c4bda91ce3b78543f569d4 --- /dev/null +++ b/knot2/crates/knot-lfs/src/gc.rs @@ -0,0 +1,338 @@ +use std::collections::HashSet; +use std::time::{Duration, SystemTime}; + +use knot_git::{GitError, Haves, RefClass, Repo, Wants}; +use knot_types::{Oid, RepoDid, UnixSeconds}; + +use crate::store::{DiskStore, Reclaimed, expired}; +use crate::{LfsError, LfsOid, LfsSize, Pool, scan_pointers}; + +#[derive(Debug, thiserror::Error)] +pub enum GcError { + #[error("git: {0}")] + Git(#[from] GitError), + #[error("store: {0}")] + Store(#[from] LfsError), +} +#[derive(Debug, thiserror::Error)] +pub enum RootsDenied { + #[error("The repo wouldn't open: {0}.")] + Unopened(GitError), + #[error("This repo's history is longer than the blob index window.")] + Windowed, + #[error("This repo's history stopped traversing or a stored frame wouldn't parse.")] + Untraversable, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct GcReport { + pub scanned: usize, + pub marked: usize, + pub swept: usize, + pub bytes: LfsSize, +} + +fn unix_seconds(now: SystemTime) -> UnixSeconds { + let secs = now + .duration_since(SystemTime::UNIX_EPOCH) + .map(|delta| delta.as_secs() as i64) + .unwrap_or(0); + UnixSeconds::new(secs) +} + +fn reachable_pointers(repo: &Repo, floor: UnixSeconds) -> Result, GitError> { + let roots: Vec = repo + .retained_oids(floor, |class| match class { + RefClass::Public | RefClass::Hidden | RefClass::Tngl => true, + RefClass::Cob | RefClass::Checkpoint | RefClass::Atproto => false, + })? + .into_iter() + .collect(); + if roots.is_empty() { + return Ok(HashSet::new()); + } + Ok(scan_pointers(repo, Wants::new(&roots), Haves::new(&[]))? + .into_keys() + .collect()) +} + +pub fn collect_repo( + store: &DiskStore, + repo: &Repo, + did: &RepoDid, + grace: Duration, + now: SystemTime, +) -> Result { + let stored = store.enumerate(Pool::Lfs, did)?; + if stored.is_empty() { + return Ok(GcReport::default()); + } + let floor = unix_seconds(now).saturating_sub_secs(grace.as_secs().min(i64::MAX as u64) as i64); + let reachable = reachable_pointers(repo, floor)?; + stored + .iter() + .filter(|object| !reachable.contains(&object.oid)) + .filter(|object| expired(now, object.mtime, grace)) + .map(|object| store.collect_expired(Pool::Lfs, did, &object.oid, grace, now)) + .try_fold( + GcReport { + scanned: stored.len(), + ..GcReport::default() + }, + |mut report, outcome| { + report.marked += 1; + if let Reclaimed::Swept(size) = outcome? { + report.swept += 1; + report.bytes = report.bytes.saturating_add(size); + } + Ok::<_, GcError>(report) + }, + ) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::store::DiskStore; + use crate::{ClaimedSize, LfsOid, LfsSize, LfsStore, LfsStorePath}; + use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, + }; + use knot_types::{AuthorName, BranchName, Email, Oid, RefName, RepoDid}; + + const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + const DAY: Duration = Duration::from_secs(86_400); + const MONTHS: Duration = Duration::from_secs(60 * 86_400); + + struct Fixture { + _scan: tempfile::TempDir, + _lfs: tempfile::TempDir, + store: DiskStore, + did: RepoDid, + repo: Repo, + } + + fn fixture() -> Fixture { + let scan = tempfile::tempdir().unwrap(); + let lfs = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + let store = DiskStore::open(LfsStorePath::new(lfs.path())).unwrap(); + Fixture { + _scan: scan, + _lfs: lfs, + store, + did, + repo, + } + } + + fn who(secs: i64) -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: knot_types::UnixSeconds::new(secs), + offset_seconds: 0, + } + } + + fn put_media(f: &Fixture, bytes: &[u8]) -> (LfsOid, LfsSize) { + let oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(bytes)); + let bytes_len = bytes.len() as u64; + f.store + .put( + Pool::Lfs, + &f.did, + &oid, + ClaimedSize::new(bytes_len), + &mut &bytes[..], + ) + .unwrap(); + (oid, LfsSize::new(bytes_len)) + } + + fn empty_commit(f: &Fixture, message: &str, secs: i64) -> Oid { + f.repo + .write_commit(&NewCommit { + tree: Oid::from_hex(EMPTY_TREE).unwrap(), + parents: Vec::new(), + author: who(secs), + committer: who(secs), + message: message.to_string(), + extra_headers: Vec::new(), + }) + .unwrap() + } + + fn commit_pointer(f: &Fixture, name: &str, oid: &LfsOid, size: LfsSize) -> Oid { + let pointer = + format!("version https://git-lfs.github.com/spec/v1\noid sha256:{oid}\nsize {size}\n") + .into_bytes(); + let tree = f + .repo + .write_staged_tree( + Oid::from_hex(EMPTY_TREE).unwrap(), + &[StagedChange { + path: knot_types::RepoPath::new("clip.bin").unwrap(), + action: StagedAction::Put { + content: pointer, + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + let tip = f + .repo + .write_commit(&NewCommit { + tree, + parents: Vec::new(), + author: who(1_700_000_000), + committer: who(1_700_000_000), + message: "add media".to_string(), + extra_headers: Vec::new(), + }) + .unwrap(); + f.repo + .update_ref(&RefUpdate::Create { + name: RefName::new(name).unwrap(), + new: tip, + }) + .unwrap(); + tip + } + + fn age(f: &Fixture, oid: &LfsOid, past: Duration) { + let path = f + .store + .object_file(Pool::Lfs, &f.did, oid) + .unwrap() + .unwrap() + .1; + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_modified(SystemTime::now() - past) + .unwrap(); + } + + fn collect(f: &Fixture, grace: Duration) -> GcReport { + collect_repo(&f.store, &f.repo, &f.did, grace, SystemTime::now()).unwrap() + } + + #[test] + fn a_reachable_object_is_never_swept_regardless_of_retention_source() { + let f = fixture(); + + let (live, live_size) = put_media(&f, b"referenced media"); + commit_pointer(&f, "refs/heads/keep", &live, live_size); + age(&f, &live, MONTHS); + + let (fresh, _) = put_media(&f, b"just uploaded, pointer still in flight"); + + let (forced, forced_size) = put_media(&f, b"orphaned by a force push"); + let old = commit_pointer(&f, "refs/heads/main", &forced, forced_size); + let replacement = empty_commit(&f, "drop media", 1_700_000_100); + f.repo + .update_ref(&RefUpdate::Update { + name: RefName::new("refs/heads/main").unwrap(), + old, + new: replacement, + }) + .unwrap(); + age(&f, &forced, MONTHS); + + let report = collect(&f, Duration::from_secs(14 * 86_400)); + assert_eq!(report.marked, 0, "no reachable object is ever marked"); + assert_eq!(report.swept, 0); + [&live, &fresh, &forced].iter().for_each(|oid| { + assert!( + f.store.probe(Pool::Lfs, &f.did, oid).unwrap().is_some(), + "a live ref, the grace window, and the reflog old tip each keep their object" + ); + }); + } + + #[test] + fn an_unreferenced_expired_object_is_swept_even_after_its_branch_is_gone() { + let f = fixture(); + + let (orphan, orphan_size) = put_media(&f, b"orphaned media"); + age(&f, &orphan, MONTHS); + + let (dropped, dropped_size) = put_media(&f, b"lived on a branch that was deleted"); + let tip = commit_pointer(&f, "refs/heads/topic", &dropped, dropped_size); + f.repo + .update_ref(&RefUpdate::Delete { + name: RefName::new("refs/heads/topic").unwrap(), + old: tip, + }) + .unwrap(); + age(&f, &dropped, MONTHS); + + let report = collect(&f, DAY); + assert_eq!(report.marked, 2); + assert_eq!( + report.swept, 2, + "a plain orphan and a deleted-branch orphan are both swept once the mtime grace expires" + ); + assert_eq!(report.bytes, orphan_size.saturating_add(dropped_size)); + assert_eq!(f.store.probe(Pool::Lfs, &f.did, &orphan).unwrap(), None); + assert_eq!(f.store.probe(Pool::Lfs, &f.did, &dropped).unwrap(), None); + } + + #[test] + fn a_knot_written_pointer_is_swept_and_a_hidden_one_keeps_its_bytes() { + let f = fixture(); + + let (cob, cob_size) = put_media(&f, b"media under a collaborative object"); + commit_pointer(&f, "refs/cobs/org.tangled.knot.member/beef", &cob, cob_size); + age(&f, &cob, MONTHS); + + let (chain, chain_size) = put_media(&f, b"media under the atproto commit chain"); + commit_pointer(&f, "refs/atproto/commit", &chain, chain_size); + age(&f, &chain, MONTHS); + + let (snapshot, snapshot_size) = put_media(&f, b"media under a cob checkpoint"); + commit_pointer( + &f, + "refs/cob-checkpoints/org.tangled.knot.member/beef", + &snapshot, + snapshot_size, + ); + age(&f, &snapshot, MONTHS); + + let (staged, staged_size) = put_media(&f, b"media under a fork staging ref"); + commit_pointer(&f, "refs/hidden/feature/main", &staged, staged_size); + age(&f, &staged, MONTHS); + + let report = collect(&f, DAY); + assert_eq!(report.swept, 3); + assert_eq!( + report.bytes, + cob_size + .saturating_add(chain_size) + .saturating_add(snapshot_size) + ); + assert_eq!(f.store.probe(Pool::Lfs, &f.did, &cob).unwrap(), None); + assert_eq!(f.store.probe(Pool::Lfs, &f.did, &chain).unwrap(), None); + assert_eq!(f.store.probe(Pool::Lfs, &f.did, &snapshot).unwrap(), None); + assert!( + f.store.probe(Pool::Lfs, &f.did, &staged).unwrap().is_some(), + "fork staging roots its bytes, so the three sweeps above are the class filter \ + at work rather than every reserved name losing its objects" + ); + } + + #[test] + fn a_repo_with_no_stored_objects_never_walks_git() { + let f = fixture(); + let orphan_pointer = + LfsOid::from_digest(knot_types::Sha256Digest::hash(b"pointer without bytes")); + commit_pointer(&f, "refs/heads/main", &orphan_pointer, LfsSize::new(21)); + assert_eq!(collect(&f, DAY), GcReport::default()); + } +} diff --git a/knot2/crates/knot-lfs/src/lib.rs b/knot2/crates/knot-lfs/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..aa4ea3f7c444af58754f433985136aa7cee1e7a9 --- /dev/null +++ b/knot2/crates/knot-lfs/src/lib.rs @@ -0,0 +1,77 @@ +mod admission; +mod batch; +mod error; +mod gc; +mod pointer; +mod scan; +mod store; +mod transfer; +mod types; + +pub use admission::{StoreAdmission, UploadAdmission, UploadPermit}; +pub use batch::{ + BASIC_TRANSFER, BATCH_MEDIA_TYPE, BatchAction, BatchActions, BatchObject, BatchObjectError, + BatchOperation, BatchRef, BatchRequest, BatchResponse, BatchResponseObject, HASH_ALGO, + HashAlgo, MAX_BATCH_OBJECTS, TransferAdapter, +}; +pub use error::LfsError; +pub use gc::{GcError, GcReport, RootsDenied, collect_repo}; +pub use pointer::{LfsPointer, POINTER_MAX_BYTES, parse_pointer}; +pub use scan::scan_pointers; +pub use store::{ + DiskStore, LfsHandle, LfsStore, MemoryStore, OrphanSweep, Reclaimed, StoredObject, +}; +pub use transfer::{TransferOp, serve_transfer}; +pub use types::{ClaimedSize, FreeSpaceFloor, LfsOid, LfsSize, LfsStorePath, ObjectRelPath, Pool}; + +#[doc(hidden)] +pub mod fuzz { + use knot_types::RepoDid; + + use crate::{ + BatchRequest, FreeSpaceFloor, LfsSize, LfsStore, LfsStorePath, MAX_BATCH_OBJECTS, + MemoryStore, Pool, StoreAdmission, TransferOp, parse_pointer, serve_transfer, + }; + + pub fn transfer(data: &[u8]) { + let admission = StoreAdmission::new( + LfsStorePath::new("/"), + LfsSize::new(u64::MAX), + FreeSpaceFloor::new(0), + ); + let repo = RepoDid::new("did:plc:squid").expect("static DID is valid"); + let messages = &knot_messages::default_catalog().lfs; + [TransferOp::Upload, TransferOp::Download] + .iter() + .for_each(|op| { + let store = MemoryStore::new(); + let _ = serve_transfer( + &store, + &admission, + &repo, + *op, + messages, + data, + std::io::sink(), + ); + }); + } + + pub fn batch(data: &[u8]) { + let Ok(request) = serde_json::from_slice::(data) else { + return; + }; + if request.objects.len() > MAX_BATCH_OBJECTS { + return; + } + let repo = RepoDid::new("did:plc:squid").expect("static DID is valid"); + let store = MemoryStore::new(); + request.objects.iter().for_each(|object| { + let _ = store.probe(Pool::Lfs, &repo, &object.oid); + }); + } + + pub fn pointer(data: &[u8]) { + let _ = parse_pointer(data); + } +} diff --git a/knot2/crates/knot-lfs/src/pointer.rs b/knot2/crates/knot-lfs/src/pointer.rs new file mode 100644 index 0000000000000000000000000000000000000000..d4cdde973c4ee5a344a94c03614437fa339a8816 --- /dev/null +++ b/knot2/crates/knot-lfs/src/pointer.rs @@ -0,0 +1,88 @@ +use crate::{ClaimedSize, LfsOid}; + +pub const POINTER_MAX_BYTES: u64 = 1024; + +const SPEC_URLS: [&str; 2] = [ + "https://git-lfs.github.com/spec/v1", + "https://hawser.github.com/spec/v1", +]; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LfsPointer { + pub oid: LfsOid, + pub size: ClaimedSize, +} + +pub fn parse_pointer(blob: &[u8]) -> Option { + if blob.len() as u64 > POINTER_MAX_BYTES { + return None; + } + let text = std::str::from_utf8(blob).ok()?; + let mut lines = text.lines(); + let spec = lines.next()?.strip_prefix("version ")?; + SPEC_URLS.contains(&spec).then_some(())?; + let fields: Vec<(&str, &str)> = lines.filter_map(|line| line.split_once(' ')).collect(); + let value_of = |key: &str| { + fields + .iter() + .find_map(|(name, value)| (*name == key).then_some(*value)) + }; + let oid = LfsOid::new(value_of("oid")?.strip_prefix("sha256:")?).ok()?; + let size = ClaimedSize::new(value_of("size")?.parse().ok()?); + Some(LfsPointer { oid, size }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const OID: &str = "6c17f2007cbe934aee6e309b28b2fba3c119d98be6ea4156da3aa3173456ad16"; + + fn pointer_text() -> String { + format!("version https://git-lfs.github.com/spec/v1\noid sha256:{OID}\nsize 12345\n") + } + + #[test] + fn valid_variants_beyond_the_canonical_form_still_parse() { + let hawser = pointer_text().replace("git-lfs.github.com", "hawser.github.com"); + assert!( + parse_pointer(hawser.as_bytes()).is_some(), + "the legacy hawser spec still counts" + ); + let extra = format!( + "version https://git-lfs.github.com/spec/v1\nname media/clip.mp4\noid sha256:{OID}\nsize 7\n" + ); + assert!( + parse_pointer(extra.as_bytes()).is_some(), + "extra sorted keys are tolerated" + ); + } + + #[test] + fn non_pointers_are_rejected() { + let cases: Vec> = vec![ + b"".to_vec(), + b"plain source file\n".to_vec(), + format!("oid sha256:{OID}\nsize 7\n").into_bytes(), + format!("version https://oyster.cafe/spec/v1\noid sha256:{OID}\nsize 7\n").into_bytes(), + pointer_text().replace("sha256:", "sha512:").into_bytes(), + pointer_text() + .replace("size 12345", "size lots") + .into_bytes(), + format!("version https://git-lfs.github.com/spec/v1\noid sha256:{OID}\n").into_bytes(), + b"version https://git-lfs.github.com/spec/v1\nsize 7\n".to_vec(), + [pointer_text().into_bytes(), vec![0xff, 0xfe]].concat(), + [ + pointer_text().into_bytes(), + vec![b' '; POINTER_MAX_BYTES as usize], + ] + .concat(), + ]; + cases.iter().enumerate().for_each(|(index, blob)| { + assert!( + parse_pointer(blob).is_none(), + "case {index} parsed as a pointer" + ); + }); + } +} diff --git a/knot2/crates/knot-lfs/src/scan.rs b/knot2/crates/knot-lfs/src/scan.rs new file mode 100644 index 0000000000000000000000000000000000000000..0f865239d030f14389cda744299f40eac1a3b84c --- /dev/null +++ b/knot2/crates/knot-lfs/src/scan.rs @@ -0,0 +1,32 @@ +use std::collections::BTreeMap; + +use knot_git::{Filter, GitError, Haves, PackBudget, Repo, Wants}; + +use crate::pointer::{POINTER_MAX_BYTES, parse_pointer}; +use crate::{ClaimedSize, LfsOid}; + +pub fn scan_pointers( + repo: &Repo, + wants: Wants<'_>, + haves: Haves<'_>, +) -> Result, GitError> { + let selection = repo.select_pack_objects_filtered( + wants, + haves, + Filter::BlobLimit(POINTER_MAX_BYTES + 1), + PackBudget::unbounded(), + )?; + selection + .send + .iter() + .filter_map(|oid| match repo.blob_size(*oid) { + Ok(_) => Some(repo.read_blob(*oid)), + Err(GitError::ObjectType { .. }) => None, + Err(fault) => Some(Err(fault)), + }) + .filter_map(|blob| match blob { + Ok(bytes) => parse_pointer(&bytes).map(|pointer| Ok((pointer.oid, pointer.size))), + Err(fault) => Some(Err(fault)), + }) + .collect() +} diff --git a/knot2/crates/knot-lfs/src/store.rs b/knot2/crates/knot-lfs/src/store.rs new file mode 100644 index 0000000000000000000000000000000000000000..be3a73e4d599313f56dca791b19bee64196c3d7b --- /dev/null +++ b/knot2/crates/knot-lfs/src/store.rs @@ -0,0 +1,1151 @@ +use std::collections::{HashMap, HashSet}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::{Duration, SystemTime}; + +use knot_types::RepoDid; +use sha2::{Digest, Sha256}; + +use crate::types::RepoPrefix; +use crate::{ + ClaimedSize, FreeSpaceFloor, LfsError, LfsOid, LfsSize, LfsStorePath, ObjectRelPath, Pool, +}; + +pub trait LfsStore: Send + Sync { + fn put( + &self, + pool: Pool, + repo: &RepoDid, + oid: &LfsOid, + size: ClaimedSize, + body: &mut dyn Read, + ) -> Result<(), LfsError>; + + fn read( + &self, + pool: Pool, + repo: &RepoDid, + oid: &LfsOid, + ) -> Result, LfsError>; + + fn probe(&self, pool: Pool, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError>; + + fn touch(&self, pool: Pool, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError>; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredObject { + pub oid: LfsOid, + pub size: LfsSize, + pub mtime: SystemTime, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Reclaimed { + Swept(LfsSize), + Spared, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct OrphanSweep { + pub prefixes: usize, + pub objects: usize, + pub bytes: LfsSize, +} + +pub(crate) fn expired(now: SystemTime, mtime: SystemTime, grace: Duration) -> bool { + now.duration_since(mtime) + .map(|age| age >= grace) + .unwrap_or(false) +} + +const COPY_CHUNK: usize = 64 * 1024; + +fn io_at(op: &'static str, path: &Path) -> impl FnOnce(std::io::Error) -> LfsError { + let path = path.to_path_buf(); + move |source| LfsError::Io { op, path, source } +} + +pub(crate) fn for_each_chunk( + body: &mut dyn Read, + mut step: impl FnMut(&[u8]) -> Result<(), LfsError>, +) -> Result<(), LfsError> { + let mut buffer = vec![0u8; COPY_CHUNK]; + std::iter::from_fn(|| match body.read(&mut buffer) { + Ok(0) => None, + Ok(count) => Some(step(&buffer[..count])), + Err(source) if source.kind() == std::io::ErrorKind::Interrupted => Some(Ok(())), + Err(source) => Some(Err(LfsError::BodyRead { source })), + }) + .try_for_each(std::convert::identity) +} + +fn write_verified( + declared: &LfsOid, + size: ClaimedSize, + body: &mut dyn Read, + mut sink: impl FnMut(&[u8]) -> Result<(), LfsError>, +) -> Result<(), LfsError> { + let mut hasher = Sha256::new(); + let mut received: u64 = 0; + for_each_chunk(body, |chunk| { + received += chunk.len() as u64; + if received > size.get() { + return Err(LfsError::SizeMismatch { + declared: size, + received: LfsSize::new(received), + }); + } + hasher.update(chunk); + sink(chunk) + })?; + if received != size.get() { + return Err(LfsError::SizeMismatch { + declared: size, + received: LfsSize::new(received), + }); + } + let computed = LfsOid::from_digest(knot_types::Sha256Digest::new(hasher.finalize().into())); + match computed == *declared { + true => Ok(()), + false => Err(LfsError::HashMismatch { + declared: declared.clone(), + computed, + }), + } +} + +fn fsync_dir(path: &Path) -> Result<(), LfsError> { + std::fs::File::open(path) + .and_then(|dir| dir.sync_all()) + .map_err(io_at("sync dir", path)) +} + +fn fsync_chain(root: &Path, leaf: &Path) -> Result<(), LfsError> { + leaf.ancestors() + .take_while(|dir| dir.starts_with(root)) + .try_for_each(fsync_dir) +} + +const INCOMING_DIR: &str = ".incoming"; +const OID_LOCK_STRIPES: usize = 64; + +fn set_mtime_now(path: &Path) -> Result<(), LfsError> { + std::fs::OpenOptions::new() + .write(true) + .open(path) + .and_then(|file| file.set_modified(SystemTime::now())) + .map_err(io_at("touch mtime", path)) +} + +fn subdirs(path: &Path) -> Result, LfsError> { + match std::fs::read_dir(path) { + Ok(entries) => entries + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(io_at("read dir", path)) + }) + .filter(|entry| entry.as_ref().map(|path| path.is_dir()).unwrap_or(true)) + .collect(), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(source) => Err(io_at("read dir", path)(source)), + } +} + +fn stored_object(path: &Path) -> Result, LfsError> { + let Some(oid) = path + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| LfsOid::new(name).ok()) + else { + return Ok(None); + }; + let meta = std::fs::metadata(path).map_err(io_at("stat", path))?; + let mtime = meta.modified().map_err(io_at("read mtime", path))?; + Ok(Some(StoredObject { + oid, + size: LfsSize::new(meta.len()), + mtime, + })) +} + +fn is_object_file(path: &Path) -> bool { + path.is_file() + && path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| LfsOid::new(name).is_ok()) +} + +fn is_shard_nibble(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + name.len() == 2 + && name + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) + }) +} + +fn is_object_leaf(dir: &Path) -> bool { + is_shard_nibble(dir) && dir.parent().is_some_and(is_shard_nibble) +} + +fn discover_prefixes(dir: &Path) -> Result, LfsError> { + let entries = match std::fs::read_dir(dir) { + Ok(entries) => entries + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(io_at("read dir", dir)) + }) + .collect::, _>>()?, + Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(source) => return Err(io_at("read dir", dir)(source)), + }; + if is_object_leaf(dir) && entries.iter().any(|path| is_object_file(path)) { + return Ok(dir + .parent() + .and_then(|shard| shard.parent()) + .map(Path::to_path_buf) + .into_iter() + .collect()); + } + entries + .iter() + .filter(|path| path.is_dir()) + .map(|sub| discover_prefixes(sub)) + .collect::, _>>() + .map(|nested| nested.into_iter().flatten().collect()) +} + +fn enumerate_prefix(prefix: &Path) -> Result, LfsError> { + subdirs(prefix)? + .iter() + .map(|shard| subdirs(shard)) + .collect::, _>>()? + .into_iter() + .flatten() + .map(|nibble| match std::fs::read_dir(&nibble) { + Ok(entries) => entries + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(io_at("read dir", &nibble)) + }) + .collect::, _>>(), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(source) => Err(io_at("read dir", &nibble)(source)), + }) + .collect::, _>>()? + .into_iter() + .flatten() + .filter_map(|file| stored_object(&file).transpose()) + .collect() +} + +pub struct DiskStore { + root: LfsStorePath, + locks: Box<[Mutex<()>]>, +} + +impl DiskStore { + pub fn open(root: LfsStorePath) -> Result { + let incoming = root.as_path().join(INCOMING_DIR); + std::fs::create_dir_all(&incoming).map_err(io_at("create dir", &incoming))?; + std::fs::read_dir(&incoming) + .map_err(io_at("read dir", &incoming))? + .try_for_each(|entry| { + let path = entry.map_err(io_at("read dir", &incoming))?.path(); + match std::fs::remove_file(&path) { + Ok(()) => Ok(()), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(io_at("remove abandoned upload", &path)(source)), + } + })?; + migrate_legacy_prefixes(&root)?; + let locks = std::iter::repeat_with(|| Mutex::new(())) + .take(OID_LOCK_STRIPES) + .collect(); + Ok(Self { root, locks }) + } + + fn object_path(&self, pool: Pool, repo: &RepoDid, oid: &LfsOid) -> Result { + Ok(self.root.object_path(&ObjectRelPath::new(pool, repo, oid)?)) + } + + fn prefix_path(&self, pool: Pool, repo: &RepoDid) -> Result { + Ok(self + .root + .as_path() + .join(pool.dir()) + .join(RepoPrefix::new(repo)?.as_path())) + } + + fn oid_lock(&self, oid: &LfsOid) -> &Mutex<()> { + let stripe = u8::from_str_radix(&oid.as_str()[0..2], 16).unwrap_or(0) as usize; + &self.locks[stripe % OID_LOCK_STRIPES] + } +} + +fn migrate_legacy_prefixes(root: &LfsStorePath) -> Result<(), LfsError> { + let lfs_pool = root.as_path().join(Pool::Lfs.dir()); + let legacy = subdirs(root.as_path())? + .into_iter() + .filter(|dir| { + dir.file_name().is_some_and(|name| { + !Pool::ALL.iter().any(|pool| name == pool.dir()) + && name != INCOMING_DIR + && name.to_str().is_some_and(is_did_method) + }) + }) + .collect::>(); + if legacy.is_empty() { + return Ok(()); + } + std::fs::create_dir_all(&lfs_pool).map_err(io_at("create dir", &lfs_pool))?; + legacy.iter().try_for_each(|dir| { + let into = lfs_pool.join( + dir.file_name() + .expect("a legacy prefix always has a final component"), + ); + merge_legacy_dir(dir, &into)?; + std::fs::remove_dir_all(dir).map_err(io_at("remove migrated prefix", dir)) + }) +} + +fn is_did_method(name: &str) -> bool { + name.chars() + .next() + .is_some_and(|first| first.is_ascii_lowercase()) + && name + .chars() + .all(|rest| rest.is_ascii_lowercase() || rest.is_ascii_digit()) +} + +fn merge_legacy_dir(from: &Path, into: &Path) -> Result<(), LfsError> { + std::fs::read_dir(from) + .map_err(io_at("read legacy prefix", from))? + .try_for_each(|entry| { + let entry = entry.map_err(io_at("read legacy entry", from))?; + let target = into.join(entry.file_name()); + let is_dir = entry + .file_type() + .map_err(io_at("stat legacy entry", &target))? + .is_dir(); + if is_dir { + std::fs::create_dir_all(&target).map_err(io_at("create dir", &target))?; + merge_legacy_dir(&entry.path(), &target) + } else if target.exists() { + Ok(()) + } else { + std::fs::rename(entry.path(), &target) + .map_err(io_at("migrate object into", &target)) + } + }) +} + +impl LfsStore for DiskStore { + fn put( + &self, + pool: Pool, + repo: &RepoDid, + oid: &LfsOid, + size: ClaimedSize, + body: &mut dyn Read, + ) -> Result<(), LfsError> { + let target = self.object_path(pool, repo, oid)?; + let incoming = self.root.as_path().join(INCOMING_DIR); + let mut temp = tempfile::Builder::new() + .prefix("put-") + .tempfile_in(&incoming) + .map_err(io_at("create temp under", &incoming))?; + let temp_path = temp.path().to_path_buf(); + write_verified(oid, size, body, |chunk| { + temp.as_file_mut() + .write_all(chunk) + .map_err(io_at("write", &temp_path)) + })?; + temp.as_file() + .sync_all() + .map_err(io_at("sync", &temp_path))?; + let parent = target + .parent() + .expect("object path always has a shard parent"); + let _guard = self + .oid_lock(oid) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + std::fs::create_dir_all(parent).map_err(io_at("create dir", parent))?; + temp.persist(&target).map_err(|fault| LfsError::Io { + op: "rename into", + path: target.clone(), + source: fault.error, + })?; + fsync_chain(self.root.as_path(), parent) + } + + fn read( + &self, + pool: Pool, + repo: &RepoDid, + oid: &LfsOid, + ) -> Result, LfsError> { + let path = self.object_path(pool, repo, oid)?; + match std::fs::File::open(&path) { + Ok(file) => Ok(Box::new(file)), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + Err(LfsError::NotFound { oid: oid.clone() }) + } + Err(source) => Err(io_at("open", &path)(source)), + } + } + + fn probe(&self, pool: Pool, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError> { + Ok(self.object_file(pool, repo, oid)?.map(|(size, _)| size)) + } + + fn touch(&self, pool: Pool, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError> { + let path = self.object_path(pool, repo, oid)?; + let _guard = self + .oid_lock(oid) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match std::fs::metadata(&path) { + Ok(meta) => { + set_mtime_now(&path)?; + Ok(Some(LfsSize::new(meta.len()))) + } + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(source) => Err(io_at("stat", &path)(source)), + } + } +} + +impl DiskStore { + pub fn object_file( + &self, + pool: Pool, + repo: &RepoDid, + oid: &LfsOid, + ) -> Result, LfsError> { + let path = self.object_path(pool, repo, oid)?; + match std::fs::metadata(&path) { + Ok(meta) => Ok(Some((LfsSize::new(meta.len()), path))), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(source) => Err(io_at("stat", &path)(source)), + } + } + + pub fn probe_ready(&self) -> Result<(), LfsError> { + let incoming = self.root.as_path().join(INCOMING_DIR); + tempfile::tempfile_in(&incoming) + .map(|_| ()) + .map_err(io_at("probe writability under", &incoming)) + } + + pub fn remove_repo(&self, repo: &RepoDid) -> Result<(), LfsError> { + Pool::ALL.iter().try_for_each(|pool| { + let prefix = self.prefix_path(*pool, repo)?; + match std::fs::remove_dir_all(&prefix) { + Ok(()) => Ok(()), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(io_at("remove repo prefix", &prefix)(source)), + } + }) + } + + pub fn enumerate(&self, pool: Pool, repo: &RepoDid) -> Result, LfsError> { + enumerate_prefix(&self.prefix_path(pool, repo)?) + } + + pub fn collect_expired( + &self, + pool: Pool, + repo: &RepoDid, + oid: &LfsOid, + grace: Duration, + now: SystemTime, + ) -> Result { + let path = self.object_path(pool, repo, oid)?; + let _guard = self + .oid_lock(oid) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let meta = match std::fs::metadata(&path) { + Ok(meta) => meta, + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + return Ok(Reclaimed::Spared); + } + Err(source) => return Err(io_at("stat", &path)(source)), + }; + let mtime = meta.modified().map_err(io_at("read mtime", &path))?; + if !expired(now, mtime, grace) { + return Ok(Reclaimed::Spared); + } + match std::fs::remove_file(&path) { + Ok(()) => Ok(Reclaimed::Swept(LfsSize::new(meta.len()))), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(Reclaimed::Spared), + Err(source) => Err(io_at("remove object", &path)(source)), + } + } + + pub fn sweep_unreferenced( + &self, + repo: &RepoDid, + referenced: &HashSet, + grace: Duration, + now: SystemTime, + ) -> Result { + let doomed: Vec = self + .enumerate(Pool::Attachments, repo)? + .into_iter() + .filter(|object| !referenced.contains(&object.oid) && expired(now, object.mtime, grace)) + .collect(); + doomed + .iter() + .map(|object| self.collect_expired(Pool::Attachments, repo, &object.oid, grace, now)) + .try_fold(OrphanSweep::default(), |acc, outcome| match outcome? { + Reclaimed::Swept(bytes) => Ok(OrphanSweep { + prefixes: acc.prefixes, + objects: acc.objects + 1, + bytes: acc.bytes.saturating_add(bytes), + }), + Reclaimed::Spared => Ok(acc), + }) + } + + pub fn sweep_orphans( + &self, + hosted: &HashSet, + grace: Duration, + now: SystemTime, + ) -> Result { + let expected: HashSet = hosted + .iter() + .filter_map(|repo| RepoPrefix::new(repo).ok()) + .map(|prefix| prefix.as_path().to_path_buf()) + .collect(); + Pool::ALL + .iter() + .map(|pool| { + let pool_root = self.root.as_path().join(pool.dir()); + let orphans: HashSet = discover_prefixes(&pool_root)? + .into_iter() + .filter(|prefix| { + prefix + .strip_prefix(&pool_root) + .ok() + .filter(|rel| matches!(rel.components().count(), 2 | 3)) + .map(|rel| !expected.contains(rel)) + .unwrap_or(false) + }) + .collect(); + orphans + .iter() + .map(|prefix| self.reclaim_orphan(prefix, grace, now)) + .try_fold(OrphanSweep::default(), |acc, outcome| { + let outcome = outcome?; + Ok(OrphanSweep { + prefixes: acc.prefixes + outcome.prefixes, + objects: acc.objects + outcome.objects, + bytes: acc.bytes.saturating_add(outcome.bytes), + }) + }) + }) + .try_fold(OrphanSweep::default(), |acc, per_pool| { + let swept = per_pool?; + Ok(OrphanSweep { + prefixes: acc.prefixes + swept.prefixes, + objects: acc.objects + swept.objects, + bytes: acc.bytes.saturating_add(swept.bytes), + }) + }) + } + + fn reclaim_orphan( + &self, + prefix: &Path, + grace: Duration, + now: SystemTime, + ) -> Result { + let objects = enumerate_prefix(prefix)?; + let live = objects + .iter() + .any(|object| !expired(now, object.mtime, grace)); + if live { + return Ok(OrphanSweep::default()); + } + match std::fs::remove_dir_all(prefix) { + Ok(()) => Ok(OrphanSweep { + prefixes: 1, + objects: objects.len(), + bytes: objects + .iter() + .map(|object| object.size) + .fold(LfsSize::new(0), LfsSize::saturating_add), + }), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + Ok(OrphanSweep::default()) + } + Err(source) => Err(io_at("remove orphan prefix", prefix)(source)), + } + } +} + +#[derive(Clone)] +pub struct LfsHandle { + pub store: std::sync::Arc, + pub admission: std::sync::Arc, +} + +impl LfsHandle { + pub fn open( + root: LfsStorePath, + max_object: LfsSize, + free_space_floor: FreeSpaceFloor, + ) -> Result { + let admission = crate::StoreAdmission::new(root.clone(), max_object, free_space_floor); + Ok(Self { + store: std::sync::Arc::new(DiskStore::open(root)?), + admission: std::sync::Arc::new(admission), + }) + } +} + +#[derive(Default)] +pub struct MemoryStore { + objects: Mutex>>, +} + +impl MemoryStore { + pub fn new() -> Self { + Self::default() + } + + fn locked(&self) -> std::sync::MutexGuard<'_, HashMap>> { + self.objects.lock().expect("lfs memory store lock poisoned") + } +} + +impl LfsStore for MemoryStore { + fn put( + &self, + pool: Pool, + repo: &RepoDid, + oid: &LfsOid, + size: ClaimedSize, + body: &mut dyn Read, + ) -> Result<(), LfsError> { + let rel = ObjectRelPath::new(pool, repo, oid)?; + let mut bytes = Vec::new(); + write_verified(oid, size, body, |chunk| { + bytes.extend_from_slice(chunk); + Ok(()) + })?; + self.locked().insert(rel, bytes); + Ok(()) + } + + fn read( + &self, + pool: Pool, + repo: &RepoDid, + oid: &LfsOid, + ) -> Result, LfsError> { + let rel = ObjectRelPath::new(pool, repo, oid)?; + self.locked() + .get(&rel) + .cloned() + .map(|bytes| Box::new(std::io::Cursor::new(bytes)) as Box) + .ok_or_else(|| LfsError::NotFound { oid: oid.clone() }) + } + + fn probe(&self, pool: Pool, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError> { + let rel = ObjectRelPath::new(pool, repo, oid)?; + Ok(self + .locked() + .get(&rel) + .map(|bytes| LfsSize::new(bytes.len() as u64))) + } + + fn touch(&self, pool: Pool, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError> { + self.probe(pool, repo, oid) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const MONTH: Duration = Duration::from_secs(30 * 86_400); + const GRACE: Duration = Duration::from_secs(14 * 86_400); + + fn oid_of(bytes: &[u8]) -> LfsOid { + LfsOid::from_digest(knot_types::Sha256Digest::hash(bytes)) + } + + fn disk() -> (DiskStore, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + (store, dir) + } + + fn seed(store: &DiskStore, pool: Pool, repo: &RepoDid, body: &[u8]) -> (LfsOid, LfsSize) { + let oid = oid_of(body); + let bytes = body.len() as u64; + store + .put(pool, repo, &oid, ClaimedSize::new(bytes), &mut &body[..]) + .unwrap(); + (oid, LfsSize::new(bytes)) + } + + fn backdate(store: &DiskStore, pool: Pool, repo: &RepoDid, oid: &LfsOid, past: Duration) { + let path = store.object_file(pool, repo, oid).unwrap().unwrap().1; + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_modified(SystemTime::now() - past) + .unwrap(); + } + + fn read_back(store: &dyn LfsStore, pool: Pool, repo: &RepoDid, oid: &LfsOid) -> Vec { + let mut out = Vec::new(); + store + .read(pool, repo, oid) + .unwrap() + .read_to_end(&mut out) + .unwrap(); + out + } + + fn store_contract(store: &dyn LfsStore, pool: Pool) { + let repo = RepoDid::new("did:plc:squid").unwrap(); + let body: &[u8] = b"lfs media bytes for the round trip"; + let oid = oid_of(body); + let size = LfsSize::new(body.len() as u64); + + assert_eq!(store.probe(pool, &repo, &oid).unwrap(), None); + assert!(matches!( + store.read(pool, &repo, &oid), + Err(LfsError::NotFound { .. }) + )); + + let claim = ClaimedSize::new(size.get()); + store.put(pool, &repo, &oid, claim, &mut &body[..]).unwrap(); + store.put(pool, &repo, &oid, claim, &mut &body[..]).unwrap(); + assert_eq!( + store.probe(pool, &repo, &oid).unwrap(), + Some(size), + "a re-put of identical bytes is idempotent" + ); + + let other = RepoDid::new("did:plc:limpet").unwrap(); + assert_eq!(store.probe(pool, &other, &oid).unwrap(), None); + assert!(matches!( + store.read(pool, &other, &oid), + Err(LfsError::NotFound { .. }) + )); + + let other = match pool { + Pool::Lfs => Pool::Attachments, + Pool::Attachments => Pool::Lfs, + }; + assert_eq!( + store.probe(other, &repo, &oid).unwrap(), + None, + "Storing a digest in one namespace keeps it out of the other." + ); + } + + #[test] + fn every_store_honors_contract_in_each_pool() { + Pool::ALL.iter().for_each(|pool| { + store_contract(&MemoryStore::new(), *pool); + let (store, _dir) = disk(); + store_contract(&store, *pool); + }); + } + + #[test] + fn a_body_longer_than_its_declared_size_errors_before_the_end() { + let store = MemoryStore::new(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + let mut endless = std::io::repeat(0x5a); + assert!(matches!( + store.put( + Pool::Lfs, + &repo, + &oid_of(b"whatever"), + ClaimedSize::new(8), + &mut endless + ), + Err(LfsError::SizeMismatch { .. }) + )); + } + + #[test] + fn disk_writes_are_sharded_and_the_boot_sweep_clears_only_temp_files() { + let (store, dir) = disk(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + let (oid, size) = seed(&store, Pool::Lfs, &repo, b"sharded placement"); + let sharded = dir + .path() + .join(".lfs") + .join("plc/sq/uid") + .join(&oid.as_str()[0..2]) + .join(&oid.as_str()[2..4]) + .join(oid.as_str()); + assert_eq!(std::fs::read(&sharded).unwrap(), b"sharded placement"); + + let method = RepoDid::new("did:incoming:squid").unwrap(); + seed( + &store, + Pool::Lfs, + &method, + b"an incoming method mustn't alias the temp dir", + ); + let lfs_method_did = RepoDid::new("did:lfs:squid").unwrap(); + seed( + &store, + Pool::Lfs, + &lfs_method_did, + b"an lfs method mustn't alias the pool dir", + ); + + let tampered = oid_of(b"a different object"); + assert!(matches!( + store.put( + Pool::Lfs, + &repo, + &tampered, + ClaimedSize::new(4), + &mut &b"nope"[..] + ), + Err(LfsError::HashMismatch { .. }) + )); + assert_eq!(store.probe(Pool::Lfs, &repo, &tampered).unwrap(), None); + let incoming = dir.path().join(INCOMING_DIR); + assert!( + std::fs::read_dir(&incoming).unwrap().next().is_none(), + "a failed put leaves no files in the incoming dir" + ); + + std::fs::write(incoming.join("put-torn4321"), b"partial bytes from a crash").unwrap(); + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + assert!( + std::fs::read_dir(&incoming).unwrap().next().is_none(), + "the boot sweep clears abandoned uploads" + ); + assert_eq!(store.probe(Pool::Lfs, &repo, &oid).unwrap(), Some(size)); + assert_eq!( + read_back(&store, Pool::Lfs, &repo, &oid), + b"sharded placement" + ); + } + + #[test] + fn opening_moves_legacy_layout_into_lfs_pool_untouched() { + let dir = tempfile::tempdir().unwrap(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + let body: &[u8] = b"written before namespaces existed"; + let oid = oid_of(body); + let legacy = dir + .path() + .join("plc/sq/uid") + .join(&oid.as_str()[0..2]) + .join(&oid.as_str()[2..4]) + .join(oid.as_str()); + let stray = dir.path().join("lost+found"); + std::fs::create_dir_all(&stray).unwrap(); + std::fs::write(stray.join("salvage"), b"the filesystem's own business").unwrap(); + std::fs::create_dir_all(legacy.parent().unwrap()).unwrap(); + std::fs::write(&legacy, body).unwrap(); + + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + assert_eq!( + store.probe(Pool::Lfs, &repo, &oid).unwrap(), + Some(LfsSize::new(body.len() as u64)) + ); + assert_eq!(read_back(&store, Pool::Lfs, &repo, &oid), body); + assert!( + !legacy.exists(), + "The old path is gone rather than duplicated." + ); + assert!( + stray.join("salvage").exists(), + "Only DID-method directories are claimed as legacy prefixes." + ); + let moved = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + assert_eq!( + moved.probe(Pool::Lfs, &repo, &oid).unwrap(), + Some(LfsSize::new(body.len() as u64)), + "Second open is a no-op, so a crash mid-migration resumes cleanly." + ); + } + + #[test] + fn opening_merges_partly_stored_legacy_prefix() { + let dir = tempfile::tempdir().unwrap(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + let rolled_back: &[u8] = b"written by a pre-namespace binary"; + let shared: &[u8] = b"same digest on both sides of the upgrade"; + let rolled_oid = oid_of(rolled_back); + let shared_oid = oid_of(shared); + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + store + .put( + Pool::Lfs, + &repo, + &shared_oid, + ClaimedSize::new(shared.len() as u64), + &mut &shared[..], + ) + .unwrap(); + drop(store); + + let shard = |oid: &LfsOid| { + dir.path() + .join("plc/sq/uid") + .join(&oid.as_str()[0..2]) + .join(&oid.as_str()[2..4]) + }; + [(&rolled_oid, rolled_back), (&shared_oid, shared)] + .iter() + .for_each(|(oid, body)| { + let legacy = shard(oid).join(oid.as_str()); + std::fs::create_dir_all(legacy.parent().unwrap()).unwrap(); + std::fs::write(&legacy, body).unwrap(); + }); + + let merged = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + assert_eq!( + merged.probe(Pool::Lfs, &repo, &rolled_oid).unwrap(), + Some(LfsSize::new(rolled_back.len() as u64)), + "The rolled-back object is stored beside the pool's own, so boot survives the collision." + ); + assert_eq!( + merged.probe(Pool::Lfs, &repo, &shared_oid).unwrap(), + Some(LfsSize::new(shared.len() as u64)), + "The pool stores a shared digest exactly once." + ); + assert!( + !dir.path().join("plc").exists(), + "The legacy prefix is gone rather than left half-migrated." + ); + } + + #[test] + fn remove_repo_reclaims_prefix_in_both_namespaces() { + let (store, _dir) = disk(); + let doomed = RepoDid::new("did:plc:squid").unwrap(); + let neighbor = RepoDid::new("did:plc:squirrel").unwrap(); + let (oid, size) = seed(&store, Pool::Lfs, &doomed, b"prefix removal"); + seed(&store, Pool::Lfs, &neighbor, b"prefix removal"); + seed(&store, Pool::Attachments, &doomed, b"prefix removal"); + store.remove_repo(&doomed).unwrap(); + assert_eq!(store.probe(Pool::Lfs, &doomed, &oid).unwrap(), None); + assert_eq!( + store.probe(Pool::Attachments, &doomed, &oid).unwrap(), + None, + "The attachments prefix goes with the LFS one." + ); + assert_eq!(store.probe(Pool::Lfs, &neighbor, &oid).unwrap(), Some(size)); + store.remove_repo(&doomed).unwrap(); + } + + #[test] + fn collect_touch_and_enumerate_govern_the_sweep_per_object() { + let (store, _dir) = disk(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + assert!( + store + .enumerate(Pool::Lfs, &RepoDid::new("did:plc:limpet").unwrap()) + .unwrap() + .is_empty(), + "a missing prefix enumerates to nothing" + ); + + let (stale, stale_size) = seed(&store, Pool::Lfs, &repo, b"long unreferenced"); + let (fresh, _) = seed(&store, Pool::Lfs, &repo, b"still within grace"); + let (vouched, vouched_size) = seed( + &store, + Pool::Lfs, + &repo, + b"vouched for moments before the sweep", + ); + backdate(&store, Pool::Lfs, &repo, &stale, MONTH); + backdate(&store, Pool::Lfs, &repo, &vouched, MONTH); + let now = SystemTime::now(); + + let listed: HashSet = store + .enumerate(Pool::Lfs, &repo) + .unwrap() + .into_iter() + .map(|object| object.oid) + .collect(); + assert_eq!( + listed, + HashSet::from([stale.clone(), fresh.clone(), vouched.clone()]) + ); + + assert_eq!( + store + .collect_expired(Pool::Lfs, &repo, &fresh, GRACE, now) + .unwrap(), + Reclaimed::Spared, + "a fresh object is inside its grace window" + ); + assert_eq!( + store + .collect_expired(Pool::Lfs, &repo, &stale, GRACE, now) + .unwrap(), + Reclaimed::Swept(stale_size) + ); + assert_eq!(store.probe(Pool::Lfs, &repo, &stale).unwrap(), None); + assert_eq!( + store + .collect_expired(Pool::Lfs, &repo, &stale, GRACE, now) + .unwrap(), + Reclaimed::Spared, + "collecting an already-gone object is a no-op" + ); + + assert_eq!( + store.touch(Pool::Lfs, &repo, &vouched).unwrap(), + Some(vouched_size) + ); + assert_eq!( + store + .collect_expired(Pool::Lfs, &repo, &vouched, GRACE, SystemTime::now()) + .unwrap(), + Reclaimed::Spared, + "the touch bumped the mtime inside the grace window" + ); + assert_eq!( + store.probe(Pool::Lfs, &repo, &vouched).unwrap(), + Some(vouched_size) + ); + } + + #[test] + fn attachment_sweep_takes_only_unreferenced_objects_past_grace() { + let (store, _dir) = disk(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + let (loose, loose_size) = seed( + &store, + Pool::Attachments, + &repo, + b"uploaded and unreferenced", + ); + let (referenced, referenced_size) = seed( + &store, + Pool::Attachments, + &repo, + b"a live record references this", + ); + let (twin, twin_size) = seed(&store, Pool::Lfs, &repo, b"uploaded and unreferenced"); + backdate(&store, Pool::Attachments, &repo, &loose, MONTH); + backdate(&store, Pool::Attachments, &repo, &referenced, MONTH); + + let swept = store + .sweep_unreferenced( + &repo, + &HashSet::from([referenced.clone()]), + GRACE, + SystemTime::now(), + ) + .unwrap(); + assert_eq!((swept.objects, swept.bytes), (1, loose_size)); + assert_eq!(store.probe(Pool::Attachments, &repo, &loose).unwrap(), None); + assert_eq!( + store.probe(Pool::Attachments, &repo, &referenced).unwrap(), + Some(referenced_size), + "Referenced attachment survives any mtime age." + ); + assert_eq!( + store.probe(Pool::Lfs, &repo, &twin).unwrap(), + Some(twin_size), + "The attachment sweep leaves the LFS namespace alone." + ); + } + #[test] + fn the_orphan_sweep_reclaims_unregistered_prefixes_and_spares_every_other_class() { + let (store, dir) = disk(); + let hosted = RepoDid::new("did:plc:squid").unwrap(); + let orphan = RepoDid::new("did:plc:limpet").unwrap(); + let fresh = RepoDid::new("did:plc:cuttle").unwrap(); + let short_hosted = RepoDid::new("did:web:ab").unwrap(); + let short_orphan = RepoDid::new("did:web:cd").unwrap(); + assert_eq!( + RepoPrefix::new(&short_hosted) + .unwrap() + .as_path() + .components() + .count(), + 2, + "a short method-specific-id shards to a two-component prefix" + ); + + let (kept, kept_size) = seed(&store, Pool::Lfs, &hosted, b"belongs to a live repo"); + let (doomed, doomed_size) = seed(&store, Pool::Lfs, &orphan, b"repo was deleted"); + let (graced, graced_size) = seed(&store, Pool::Lfs, &fresh, b"deleted repo, but only just"); + let (short_kept, short_kept_size) = + seed(&store, Pool::Lfs, &short_hosted, b"live short-did object"); + let (short_doomed, short_doomed_size) = seed( + &store, + Pool::Lfs, + &short_orphan, + b"orphaned short-did media", + ); + [ + (&hosted, &kept), + (&orphan, &doomed), + (&short_hosted, &short_kept), + (&short_orphan, &short_doomed), + ] + .iter() + .for_each(|(did, oid)| backdate(&store, Pool::Lfs, did, oid, MONTH)); + + std::fs::write( + dir.path().join(".lfs").join(doomed.as_str()), + b"stray at the wrong depth", + ) + .unwrap(); + + let registry = HashSet::from([hosted.clone(), short_hosted.clone()]); + let sweep = store + .sweep_orphans(®istry, GRACE, SystemTime::now()) + .unwrap(); + + assert_eq!(sweep.prefixes, 2, "both past-grace orphans are reclaimed"); + assert_eq!(sweep.objects, 2, "one object under each reclaimed prefix"); + assert_eq!(sweep.bytes, doomed_size.saturating_add(short_doomed_size)); + assert_eq!(store.probe(Pool::Lfs, &orphan, &doomed).unwrap(), None); + assert_eq!( + store + .probe(Pool::Lfs, &short_orphan, &short_doomed) + .unwrap(), + None + ); + assert_eq!( + store.probe(Pool::Lfs, &hosted, &kept).unwrap(), + Some(kept_size), + "a hosted prefix is never an orphan" + ); + assert_eq!( + store.probe(Pool::Lfs, &short_hosted, &short_kept).unwrap(), + Some(short_kept_size), + "a hosted repo shallower than the oid shards survives too" + ); + assert_eq!( + store.probe(Pool::Lfs, &fresh, &graced).unwrap(), + Some(graced_size), + "Objects from a just-deleted repo stay for the grace window." + ); + } +} diff --git a/knot2/crates/knot-lfs/src/transfer.rs b/knot2/crates/knot-lfs/src/transfer.rs new file mode 100644 index 0000000000000000000000000000000000000000..fd05aa576b6708b39a8d81f91ecd90763541b9db --- /dev/null +++ b/knot2/crates/knot-lfs/src/transfer.rs @@ -0,0 +1,1191 @@ +use std::io::{Read, Write}; + +use gix_packetline::PacketLineRef; +use gix_packetline::blocking_io::{StreamingPeekableIter, encode}; +use knot_messages::{ + AlgorithmKey, CommandKey, DeclaredComputedKey, DeclaredLimitKey, DeclaredReceivedKey, + DetailKey, FreeFloorKey, LfsMessages, OidKey, ValueKey, VersionKey, WhatLimitKey, +}; +use knot_types::{HttpStatus, RepoDid}; + +use crate::store::for_each_chunk; +use crate::{ + BatchObject, ClaimedSize, LfsError, LfsOid, LfsSize, LfsStore, MAX_BATCH_OBJECTS, Pool, + UploadAdmission, +}; + +pub const CAPABILITY_VERSION: &str = "version=1"; +const PKT_DATA_MAX: usize = 65516; +const MAX_MESSAGE_ARGS: usize = 64; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TransferOp { + Upload, + Download, +} + +impl TransferOp { + pub fn parse(token: &str) -> Option { + match token { + "upload" => Some(Self::Upload), + "download" => Some(Self::Download), + _ => None, + } + } +} + +#[allow(clippy::too_many_arguments)] +pub fn serve_transfer( + store: &dyn LfsStore, + admission: &dyn UploadAdmission, + repo: &RepoDid, + op: TransferOp, + messages: &LfsMessages, + input: impl Read, + mut output: impl Write, +) -> Result<(), LfsError> { + write_text(&mut output, CAPABILITY_VERSION)?; + write_flush(&mut output)?; + let mut session = Session { + store, + admission, + repo, + pool: Pool::Lfs, + op, + messages, + lines: StreamingPeekableIter::new(input, &[], false), + out: output, + }; + let mut done = false; + std::iter::from_fn(|| { + (!done).then(|| { + session.step().map(|flow| match flow { + Flow::Continue => (), + Flow::Quit => done = true, + }) + }) + }) + .try_for_each(std::convert::identity) +} + +enum Flow { + Continue, + Quit, +} + +enum Pkt { + Data(Vec), + Flush, + Delim, + Eof, +} + +enum Ended { + Flush, + Delim, +} + +fn next_pkt(lines: &mut StreamingPeekableIter) -> Result { + match lines.read_line() { + None => Ok(Pkt::Eof), + Some(Err(source)) if source.kind() == std::io::ErrorKind::UnexpectedEof => Ok(Pkt::Eof), + Some(Err(source)) => Err(LfsError::Channel { source }), + Some(Ok(Err(fault))) => Err(LfsError::Framing { + detail: fault.to_string(), + }), + Some(Ok(Ok(PacketLineRef::Data(payload)))) => Ok(Pkt::Data(payload.to_vec())), + Some(Ok(Ok(PacketLineRef::Flush))) => Ok(Pkt::Flush), + Some(Ok(Ok(PacketLineRef::Delimiter))) => Ok(Pkt::Delim), + Some(Ok(Ok(PacketLineRef::ResponseEnd))) => Err(LfsError::Framing { + detail: "unexpected response-end packet".to_string(), + }), + } +} + +fn text_of(payload: Vec) -> Result { + String::from_utf8(payload) + .map(|line| line.trim_end_matches('\n').to_string()) + .map_err(|_| LfsError::Framing { + detail: "non-utf8 text packet".to_string(), + }) +} + +fn fault_text(fault: &LfsError, messages: &LfsMessages) -> String { + match fault { + LfsError::InvalidOid { value } => messages + .invalid_oid + .line(|ValueKey::Value| format!("{value:?}")), + LfsError::HashMismatch { declared, computed } => { + messages.hash_mismatch.line(|key| match key { + DeclaredComputedKey::Declared => declared.to_string(), + DeclaredComputedKey::Computed => computed.to_string(), + }) + } + LfsError::SizeMismatch { declared, received } => { + messages.size_mismatch.line(|key| match key { + DeclaredReceivedKey::Declared => declared.to_string(), + DeclaredReceivedKey::Received => received.to_string(), + }) + } + LfsError::SizeLimitExceeded { declared, limit } => { + messages.size_limit_exceeded.line(|key| match key { + DeclaredLimitKey::Declared => declared.to_string(), + DeclaredLimitKey::Limit => limit.to_string(), + }) + } + LfsError::FreeSpaceDenied { free, floor } => { + messages.free_space_denied.line(|key| match key { + FreeFloorKey::Free => free.to_string(), + FreeFloorKey::Floor => floor.to_string(), + }) + } + LfsError::NotFound { oid } => messages.not_found.line(|OidKey::Oid| oid.to_string()), + LfsError::Framing { detail } => messages.framing.line(|DetailKey::Detail| detail.clone()), + LfsError::TooMany { what, limit } => messages.too_many.line(|key| match key { + WhatLimitKey::What => what.to_string(), + WhatLimitKey::Limit => limit.to_string(), + }), + other => other.to_string(), + } +} + +fn status_of(fault: &LfsError) -> HttpStatus { + HttpStatus::new(match fault { + LfsError::NotFound { .. } => 404, + LfsError::InvalidOid { .. } + | LfsError::HashMismatch { .. } + | LfsError::SizeMismatch { .. } + | LfsError::Framing { .. } => 400, + LfsError::SizeLimitExceeded { .. } => 413, + LfsError::FreeSpaceDenied { .. } => 429, + _ => 500, + }) +} + +fn write_text(out: &mut impl Write, line: &str) -> Result<(), LfsError> { + encode::data_to_write(format!("{line}\n").as_bytes(), &mut *out) + .map(|_| ()) + .map_err(|source| LfsError::Channel { source }) +} + +fn write_flush(out: &mut impl Write) -> Result<(), LfsError> { + encode::flush_to_write(&mut *out) + .and_then(|_| out.flush().map(|()| 0)) + .map(|_| ()) + .map_err(|source| LfsError::Channel { source }) +} + +fn write_delim(out: &mut impl Write) -> Result<(), LfsError> { + encode::delim_to_write(&mut *out) + .map(|_| ()) + .map_err(|source| LfsError::Channel { source }) +} + +fn arg_value<'a>(args: &'a [String], key: &str) -> Option<&'a str> { + args.iter().find_map(|arg| { + arg.strip_prefix(key) + .and_then(|rest| rest.strip_prefix('=')) + }) +} + +struct Session<'a, R: Read, W: Write> { + store: &'a dyn LfsStore, + admission: &'a dyn UploadAdmission, + repo: &'a RepoDid, + pool: Pool, + op: TransferOp, + messages: &'a LfsMessages, + lines: StreamingPeekableIter, + out: W, +} + +impl Session<'_, R, W> { + fn step(&mut self) -> Result { + match next_pkt(&mut self.lines)? { + Pkt::Eof => Ok(Flow::Quit), + Pkt::Flush | Pkt::Delim => Err(LfsError::Framing { + detail: "expected a command packet".to_string(), + }), + Pkt::Data(payload) => self.dispatch(text_of(payload)?), + } + } + + fn dispatch(&mut self, command: String) -> Result { + let (verb, rest) = command + .split_once(' ') + .map_or((command.as_str(), ""), |(verb, rest)| (verb, rest)); + match verb { + "version" => self.handle_version(rest), + "batch" => self.handle_batch(), + "put-object" => self.handle_put(rest), + "verify-object" => self.handle_verify(rest), + "get-object" => self.handle_get(rest), + "quit" => { + self.drain_message()?; + self.respond_ok(&[])?; + Ok(Flow::Quit) + } + _ => { + self.drain_message()?; + let message = self + .messages + .unknown_command + .line(|CommandKey::Command| format!("{verb:?}")); + self.respond_error(HttpStatus::new(400), &message)?; + Ok(Flow::Continue) + } + } + } + + fn read_args(&mut self) -> Result<(Vec, Ended), LfsError> { + let mut args = Vec::new(); + std::iter::from_fn(|| Some(next_pkt(&mut self.lines))) + .find_map(|pkt| match pkt { + Err(fault) => Some(Err(fault)), + Ok(Pkt::Data(payload)) => match text_of(payload) { + Ok(_) if args.len() >= MAX_MESSAGE_ARGS => Some(Err(LfsError::TooMany { + what: "arguments", + limit: MAX_MESSAGE_ARGS, + })), + Ok(line) => { + args.push(line); + None + } + Err(fault) => Some(Err(fault)), + }, + Ok(Pkt::Flush) => Some(Ok(Ended::Flush)), + Ok(Pkt::Delim) => Some(Ok(Ended::Delim)), + Ok(Pkt::Eof) => Some(Err(LfsError::Framing { + detail: "message truncated before flush".to_string(), + })), + }) + .expect("an endless packet iterator always yields a terminator") + .map(|ended| (args, ended)) + } + + fn drain_budget(&self) -> LfsSize { + self.admission.max_object() + } + + fn drain_to_flush(&mut self, limit: LfsSize) -> Result<(), LfsError> { + let mut discarded = LfsSize::new(0); + std::iter::from_fn(|| Some(next_pkt(&mut self.lines))) + .find_map(|pkt| match pkt { + Err(fault) => Some(Err(fault)), + Ok(Pkt::Flush) => Some(Ok(())), + Ok(Pkt::Eof) => Some(Err(LfsError::Framing { + detail: "message truncated before flush".to_string(), + })), + Ok(Pkt::Data(payload)) => { + discarded = discarded.saturating_add(LfsSize::new(payload.len() as u64)); + match discarded > limit { + true => Some(Err(LfsError::Framing { + detail: "message body exceeds the drain bound".to_string(), + })), + false => None, + } + } + Ok(Pkt::Delim) => None, + }) + .expect("an endless packet iterator always yields a terminator") + } + + fn drain_message(&mut self) -> Result<(), LfsError> { + match self.read_args()?.1 { + Ended::Flush => Ok(()), + Ended::Delim => self.drain_to_flush(self.drain_budget()), + } + } + + fn respond_ok(&mut self, args: &[String]) -> Result<(), LfsError> { + write_text(&mut self.out, "status 200")?; + args.iter() + .try_for_each(|arg| write_text(&mut self.out, arg))?; + write_flush(&mut self.out) + } + + fn respond_error(&mut self, code: HttpStatus, message: &str) -> Result<(), LfsError> { + write_text(&mut self.out, &format!("status {:03}", code.get()))?; + write_delim(&mut self.out)?; + write_text(&mut self.out, &format!("error: {message}"))?; + write_flush(&mut self.out) + } + + fn respond_fault(&mut self, fault: &LfsError) -> Result<(), LfsError> { + let message = match fault { + LfsError::Io { .. } => { + tracing::warn!(repo = self.repo.as_str(), %fault, "lfs store fault"); + "internal storage fault".to_string() + } + other => fault_text(other, self.messages), + }; + self.respond_error(status_of(fault), &message) + } + + fn handle_version(&mut self, rest: &str) -> Result { + self.drain_message()?; + match rest.trim() { + "1" => self.respond_ok(&[])?, + other => { + let message = self + .messages + .unsupported_version + .line(|VersionKey::Version| format!("{other:?}")); + self.respond_error(HttpStatus::new(400), &message)?; + } + } + Ok(Flow::Continue) + } + + fn handle_batch(&mut self) -> Result { + let (args, ended) = self.read_args()?; + if let Some(algo) = arg_value(&args, "hash-algo") + && algo != crate::HASH_ALGO + { + if matches!(ended, Ended::Delim) { + self.drain_to_flush(self.drain_budget())?; + } + let message = self + .messages + .unsupported_hash + .line(|AlgorithmKey::Algorithm| format!("{algo:?}")); + self.respond_error(HttpStatus::new(400), &message)?; + return Ok(Flow::Continue); + } + let items = match ended { + Ended::Flush => Ok(Vec::new()), + Ended::Delim => self.read_batch_items(), + }; + let items = match items { + Ok(items) => items, + Err(fault @ (LfsError::InvalidOid { .. } | LfsError::Framing { .. })) => { + self.drain_to_flush(self.drain_budget())?; + self.respond_fault(&fault)?; + return Ok(Flow::Continue); + } + Err(fault) => return Err(fault), + }; + let lines: Result, LfsError> = + items.iter().map(|item| self.batch_line(item)).collect(); + match lines { + Ok(lines) => { + write_text(&mut self.out, "status 200")?; + write_text(&mut self.out, &format!("hash-algo={}", crate::HASH_ALGO))?; + write_delim(&mut self.out)?; + lines + .iter() + .try_for_each(|line| write_text(&mut self.out, line))?; + write_flush(&mut self.out)?; + } + Err(fault) => self.respond_fault(&fault)?, + } + Ok(Flow::Continue) + } + + fn read_batch_items(&mut self) -> Result, LfsError> { + let mut items = Vec::new(); + std::iter::from_fn(|| Some(next_pkt(&mut self.lines))) + .find_map(|pkt| match pkt { + Err(fault) => Some(Err(fault)), + Ok(Pkt::Data(payload)) => match text_of(payload).and_then(parse_batch_item) { + Ok(_) if items.len() >= MAX_BATCH_OBJECTS => Some(Err(LfsError::TooMany { + what: "batch items", + limit: MAX_BATCH_OBJECTS, + })), + Ok(item) => { + items.push(item); + None + } + Err(fault) => Some(Err(fault)), + }, + Ok(Pkt::Flush) => Some(Ok(())), + Ok(Pkt::Delim | Pkt::Eof) => Some(Err(LfsError::Framing { + detail: "batch items truncated before flush".to_string(), + })), + }) + .expect("an endless packet iterator always yields a terminator") + .map(|()| items) + } + + fn batch_line(&self, item: &BatchObject) -> Result { + let stored = match self.op { + TransferOp::Upload => self.store.touch(self.pool, self.repo, &item.oid)?, + TransferOp::Download => self.store.probe(self.pool, self.repo, &item.oid)?, + }; + let (size, action) = match (self.op, stored) { + (TransferOp::Upload, Some(_)) => (item.size.get(), "noop"), + (TransferOp::Upload, None) => (item.size.get(), "upload"), + (TransferOp::Download, Some(actual)) => (actual.get(), "download"), + (TransferOp::Download, None) => (item.size.get(), "download"), + }; + Ok(format!("{} {} {}", item.oid, size, action)) + } + + fn handle_put(&mut self, rest: &str) -> Result { + if self.op != TransferOp::Upload { + self.drain_message()?; + self.respond_error(HttpStatus::new(403), &self.messages.put_on_download.text())?; + return Ok(Flow::Continue); + } + let (args, ended) = self.read_args()?; + let checked = LfsOid::new(rest).and_then(|oid| { + let declared = arg_value(&args, "size") + .and_then(|value| value.parse().ok()) + .map(ClaimedSize::new) + .ok_or_else(|| LfsError::Framing { + detail: "put-object requires a size argument".to_string(), + })?; + let permit = self.admission.admit(declared)?; + Ok((oid, declared, permit)) + }); + let (oid, declared, permit) = match checked { + Ok(admitted) => admitted, + Err(fault) => { + if matches!(ended, Ended::Delim) { + self.drain_to_flush(self.drain_budget())?; + } + self.respond_fault(&fault)?; + return Ok(Flow::Continue); + } + }; + if matches!(ended, Ended::Flush) { + self.respond_error(HttpStatus::new(400), &self.messages.put_no_body.text())?; + return Ok(Flow::Continue); + } + let mut body = PktBody { + lines: &mut self.lines, + buffer: Vec::new(), + offset: 0, + done: false, + }; + let stored = self + .store + .put(self.pool, self.repo, &oid, declared, &mut body); + drop(permit); + let synced = body.done; + if !synced { + self.drain_to_flush(self.drain_budget())?; + } + match stored { + Ok(()) => { + tracing::info!( + repo = self.repo.as_str(), + oid = oid.as_str(), + size = declared.get(), + "lfs object received over ssh" + ); + self.respond_ok(&[])? + } + Err(fault) => self.respond_fault(&fault)?, + } + Ok(Flow::Continue) + } + + fn handle_verify(&mut self, rest: &str) -> Result { + if self.op != TransferOp::Upload { + self.drain_message()?; + self.respond_error( + HttpStatus::new(403), + &self.messages.verify_on_download.text(), + )?; + return Ok(Flow::Continue); + } + let (args, ended) = self.read_args()?; + if matches!(ended, Ended::Delim) { + self.drain_to_flush(self.drain_budget())?; + } + let declared = arg_value(&args, "size") + .and_then(|value| value.parse().ok()) + .map(ClaimedSize::new); + let verdict = LfsOid::new(rest).and_then(|oid| { + match (self.store.touch(self.pool, self.repo, &oid)?, declared) { + (None, _) => Err(LfsError::NotFound { oid }), + (Some(actual), Some(declared)) if !declared.matches(actual) => { + Err(LfsError::SizeMismatch { + declared, + received: actual, + }) + } + (Some(_), _) => Ok(()), + } + }); + match verdict { + Ok(()) => self.respond_ok(&[])?, + Err(fault) => self.respond_fault(&fault)?, + } + Ok(Flow::Continue) + } + + fn handle_get(&mut self, rest: &str) -> Result { + if self.op != TransferOp::Download { + self.drain_message()?; + self.respond_error(HttpStatus::new(403), &self.messages.get_on_upload.text())?; + return Ok(Flow::Continue); + } + self.drain_message()?; + let opened = LfsOid::new(rest).and_then(|oid| { + let size = self + .store + .probe(self.pool, self.repo, &oid)? + .ok_or(LfsError::NotFound { oid: oid.clone() })?; + let body = self.store.read(self.pool, self.repo, &oid)?; + Ok((size, body)) + }); + let (size, mut body) = match opened { + Ok(found) => found, + Err(fault) => { + self.respond_fault(&fault)?; + return Ok(Flow::Continue); + } + }; + write_text(&mut self.out, "status 200")?; + write_text(&mut self.out, &format!("size={size}"))?; + write_delim(&mut self.out)?; + let out = &mut self.out; + for_each_chunk(&mut body, |chunk| { + chunk.chunks(PKT_DATA_MAX).try_for_each(|piece| { + encode::data_to_write(piece, &mut *out) + .map(|_| ()) + .map_err(|source| LfsError::Channel { source }) + }) + })?; + write_flush(&mut self.out)?; + tracing::info!( + repo = self.repo.as_str(), + oid = rest, + size = size.get(), + "lfs object served over ssh" + ); + Ok(Flow::Continue) + } +} + +fn parse_batch_item(line: String) -> Result { + let mut tokens = line.split(' '); + let oid = LfsOid::new(tokens.next().unwrap_or_default())?; + let size = tokens + .next() + .and_then(|token| token.parse().ok()) + .map(ClaimedSize::new) + .ok_or_else(|| LfsError::Framing { + detail: format!("malformed batch item {line:?}"), + })?; + Ok(BatchObject { oid, size }) +} + +struct PktBody<'a, R: Read> { + lines: &'a mut StreamingPeekableIter, + buffer: Vec, + offset: usize, + done: bool, +} + +impl Read for PktBody<'_, R> { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.done { + return Ok(0); + } + if self.offset >= self.buffer.len() { + match next_pkt(self.lines).map_err(std::io::Error::other)? { + Pkt::Data(payload) => { + self.buffer = payload; + self.offset = 0; + } + Pkt::Flush => { + self.done = true; + return Ok(0); + } + Pkt::Delim => { + return Err(std::io::Error::other("unexpected delimiter in object body")); + } + Pkt::Eof => return Err(std::io::Error::other("object body truncated")), + } + } + let take = out.len().min(self.buffer.len() - self.offset); + out[..take].copy_from_slice(&self.buffer[self.offset..self.offset + take]); + self.offset += take; + Ok(take) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::admission::Unbounded; + use crate::{FreeSpaceFloor, MemoryStore}; + + fn oid_of(bytes: &[u8]) -> LfsOid { + LfsOid::from_digest(knot_types::Sha256Digest::hash(bytes)) + } + + fn repo() -> RepoDid { + RepoDid::new("did:plc:squid").unwrap() + } + + fn put_text(buf: &mut Vec, line: &str) { + encode::data_to_write(format!("{line}\n").as_bytes(), &mut *buf).unwrap(); + } + + fn msg(buf: &mut Vec, command: &str, args: &[&str]) { + put_text(buf, command); + args.iter().for_each(|arg| put_text(buf, arg)); + encode::flush_to_write(&mut *buf).unwrap(); + } + + fn msg_lines(buf: &mut Vec, command: &str, args: &[&str], lines: &[String]) { + put_text(buf, command); + args.iter().for_each(|arg| put_text(buf, arg)); + encode::delim_to_write(&mut *buf).unwrap(); + lines.iter().for_each(|line| put_text(buf, line)); + encode::flush_to_write(&mut *buf).unwrap(); + } + + fn msg_data(buf: &mut Vec, command: &str, args: &[&str], data: &[u8]) { + put_text(buf, command); + args.iter().for_each(|arg| put_text(buf, arg)); + encode::delim_to_write(&mut *buf).unwrap(); + data.chunks(PKT_DATA_MAX).for_each(|chunk| { + encode::data_to_write(chunk, &mut *buf).unwrap(); + }); + encode::flush_to_write(&mut *buf).unwrap(); + } + + #[derive(Debug, PartialEq, Eq, Clone)] + enum Out { + Line(String), + Bin(Vec), + Delim, + Flush, + } + + fn parse_out(bytes: &[u8]) -> Vec { + let mut lines = StreamingPeekableIter::new(bytes, &[], false); + std::iter::from_fn(|| { + lines.read_line().and_then(|pkt| { + if matches!(&pkt, Err(fault) if fault.kind() == std::io::ErrorKind::UnexpectedEof) { + return None; + } + Some( + match pkt.expect("readable output").expect("well-formed output") { + PacketLineRef::Data(payload) => std::str::from_utf8(payload) + .ok() + .filter(|text| text.ends_with('\n')) + .map(|text| Out::Line(text.trim_end_matches('\n').to_string())) + .unwrap_or_else(|| Out::Bin(payload.to_vec())), + PacketLineRef::Flush => Out::Flush, + PacketLineRef::Delimiter => Out::Delim, + PacketLineRef::ResponseEnd => panic!("server never writes response-end"), + }, + ) + }) + }) + .collect() + } + + fn line(text: &str) -> Out { + Out::Line(text.to_string()) + } + + fn run(op: TransferOp, store: &dyn LfsStore, script: &[u8]) -> Vec { + let mut output = Vec::new(); + serve_transfer( + store, + &Unbounded, + &repo(), + op, + &knot_messages::default_catalog().lfs, + script, + &mut output, + ) + .unwrap(); + parse_out(&output) + } + + fn responses(out: &[Out]) -> Vec> { + out.split_inclusive(|item| *item == Out::Flush) + .map(<[Out]>::to_vec) + .collect() + } + + #[test] + fn an_upload_session_lands_and_verifies_an_object() { + let store = MemoryStore::new(); + let seeded: &[u8] = b"already on the server"; + let seeded_oid = oid_of(seeded); + store + .put( + Pool::Lfs, + &repo(), + &seeded_oid, + ClaimedSize::new(seeded.len() as u64), + &mut &seeded[..], + ) + .unwrap(); + + let fresh: &[u8] = b"\xffnew binary media\x00with raw bytes"; + let fresh_oid = oid_of(fresh); + let fresh_len = fresh.len(); + + let mut script = Vec::new(); + msg(&mut script, "version 1", &[]); + msg_lines( + &mut script, + "batch", + &[ + "transfer=ssh", + "hash-algo=sha256", + "refname=refs/heads/main", + ], + &[ + format!("{seeded_oid} {} extra=ignored", seeded.len()), + format!("{fresh_oid} {fresh_len}"), + ], + ); + msg_data( + &mut script, + &format!("put-object {fresh_oid}"), + &[&format!("size={fresh_len}")], + fresh, + ); + msg( + &mut script, + &format!("verify-object {fresh_oid}"), + &[&format!("size={fresh_len}")], + ); + msg(&mut script, "quit", &[]); + + let out = run(TransferOp::Upload, &store, &script); + let turns = responses(&out); + assert_eq!(turns[0], vec![line(CAPABILITY_VERSION), Out::Flush]); + assert_eq!(turns[1], vec![line("status 200"), Out::Flush]); + assert_eq!( + turns[2], + vec![ + line("status 200"), + line("hash-algo=sha256"), + Out::Delim, + line(&format!("{seeded_oid} {} noop", seeded.len())), + line(&format!("{fresh_oid} {fresh_len} upload")), + Out::Flush, + ] + ); + assert_eq!(turns[3], vec![line("status 200"), Out::Flush]); + assert_eq!(turns[4], vec![line("status 200"), Out::Flush]); + assert_eq!(turns[5], vec![line("status 200"), Out::Flush]); + assert_eq!( + store.probe(Pool::Lfs, &repo(), &fresh_oid).unwrap(), + Some(LfsSize::new(fresh_len as u64)) + ); + } + + #[test] + fn a_download_session_streams_bytes_and_404s_the_missing() { + let store = MemoryStore::new(); + let media: &[u8] = b"\xff\x00streamable media"; + let media_oid = oid_of(media); + let absent = oid_of(b"never uploaded"); + store + .put( + Pool::Lfs, + &repo(), + &media_oid, + ClaimedSize::new(media.len() as u64), + &mut &media[..], + ) + .unwrap(); + + let mut script = Vec::new(); + msg_lines( + &mut script, + "batch", + &["transfer=ssh", "hash-algo=sha256"], + &[format!("{media_oid} 1"), format!("{absent} 9")], + ); + msg(&mut script, &format!("get-object {media_oid}"), &[]); + msg(&mut script, &format!("get-object {absent}"), &[]); + msg(&mut script, "quit", &[]); + + let out = run(TransferOp::Download, &store, &script); + let turns = responses(&out); + assert_eq!( + turns[1], + vec![ + line("status 200"), + line("hash-algo=sha256"), + Out::Delim, + line(&format!("{media_oid} {} download", media.len())), + line(&format!("{absent} 9 download")), + Out::Flush, + ] + ); + assert_eq!( + turns[2], + vec![ + line("status 200"), + line(&format!("size={}", media.len())), + Out::Delim, + Out::Bin(media.to_vec()), + Out::Flush, + ] + ); + assert_eq!(turns[3][0], line("status 404")); + assert_eq!(turns[4], vec![line("status 200"), Out::Flush]); + } + + #[test] + fn the_channel_mode_gates_every_write_and_read_verb() { + let store = MemoryStore::new(); + let oid = oid_of(b"whatever"); + + let mut download_script = Vec::new(); + msg_data( + &mut download_script, + &format!("put-object {oid}"), + &["size=3"], + b"abc", + ); + msg( + &mut download_script, + &format!("verify-object {oid}"), + &["size=3"], + ); + let out = run(TransferOp::Download, &store, &download_script); + let turns = responses(&out); + assert_eq!(turns[1][0], line("status 403")); + assert_eq!(turns[2][0], line("status 403")); + assert_eq!(store.probe(Pool::Lfs, &repo(), &oid).unwrap(), None); + + let mut upload_script = Vec::new(); + msg(&mut upload_script, &format!("get-object {oid}"), &[]); + let out = run(TransferOp::Upload, &store, &upload_script); + assert_eq!(responses(&out)[1][0], line("status 403")); + } + + #[test] + fn admission_rejections_surface_as_typed_statuses() { + struct Deny(LfsSize); + impl UploadAdmission for Deny { + fn admit(&self, declared: ClaimedSize) -> Result { + match declared.get() > self.0.get() { + true => Err(LfsError::SizeLimitExceeded { + declared, + limit: self.0, + }), + false => Err(LfsError::FreeSpaceDenied { + free: LfsSize::new(1), + floor: FreeSpaceFloor::new(2), + }), + } + } + + fn max_object(&self) -> LfsSize { + self.0 + } + } + let store = MemoryStore::new(); + let body: &[u8] = b"denied"; + let oid = oid_of(body); + let mut script = Vec::new(); + msg_data( + &mut script, + &format!("put-object {oid}"), + &["size=999"], + body, + ); + msg_data(&mut script, &format!("put-object {oid}"), &["size=6"], body); + let mut output = Vec::new(); + serve_transfer( + &store, + &Deny(LfsSize::new(10)), + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &script[..], + &mut output, + ) + .unwrap(); + let turns = responses(&parse_out(&output)); + assert_eq!(turns[1][0], line("status 413")); + assert_eq!(turns[2][0], line("status 429")); + assert_eq!(store.probe(Pool::Lfs, &repo(), &oid).unwrap(), None); + } + + #[test] + fn tampered_and_truncated_uploads_fail_closed_and_the_session_survives() { + let store = MemoryStore::new(); + let body: &[u8] = b"the true bytes"; + let liar = oid_of(b"some other bytes"); + let valid = oid_of(body); + + let mut script = Vec::new(); + msg_data( + &mut script, + &format!("put-object {liar}"), + &[&format!("size={}", body.len())], + body, + ); + msg_data( + &mut script, + &format!("put-object {valid}"), + &[&format!("size={}", body.len() + 5)], + body, + ); + msg( + &mut script, + &format!("verify-object {valid}"), + &[&format!("size={}", body.len())], + ); + msg_data( + &mut script, + &format!("put-object {valid}"), + &[&format!("size={}", body.len())], + body, + ); + msg(&mut script, "quit", &[]); + + let out = run(TransferOp::Upload, &store, &script); + let turns = responses(&out); + assert_eq!(turns[1][0], line("status 400"), "hash mismatch"); + assert_eq!(turns[2][0], line("status 400"), "size mismatch"); + assert_eq!(turns[3][0], line("status 404"), "nothing landed"); + assert_eq!( + turns[4][0], + line("status 200"), + "retry with matching bytes succeeds" + ); + assert_eq!( + store.probe(Pool::Lfs, &repo(), &valid).unwrap(), + Some(LfsSize::new(body.len() as u64)) + ); + assert_eq!(store.probe(Pool::Lfs, &repo(), &liar).unwrap(), None); + } + + #[test] + fn hostile_commands_get_clean_rejections_and_never_a_panic() { + let store = MemoryStore::new(); + let mut script = Vec::new(); + msg(&mut script, "version 9", &[]); + msg(&mut script, "steal-the-objects now", &[]); + msg_lines( + &mut script, + "batch", + &["hash-algo=sha256"], + &["../../../etc/passwd0000000000000000000000000000000000000000 5".to_string()], + ); + msg_lines( + &mut script, + "batch", + &["hash-algo=sha1"], + &[format!("{} 5", oid_of(b"x"))], + ); + msg(&mut script, "quit", &[]); + + let out = run(TransferOp::Upload, &store, &script); + let turns = responses(&out); + assert_eq!(turns[1][0], line("status 400"), "unsupported version"); + assert_eq!(turns[2][0], line("status 400"), "unknown command"); + assert_eq!(turns[3][0], line("status 400"), "traversal oid"); + assert_eq!(turns[4][0], line("status 400"), "foreign hash algo"); + assert_eq!(turns[5][0], line("status 200"), "quit still answers"); + } + + #[test] + fn floods_kill_the_session_instead_of_accumulating() { + let store = MemoryStore::new(); + let flooded_batch: Vec = (0..MAX_BATCH_OBJECTS + 1) + .map(|index| format!("{} 1", oid_of(index.to_string().as_bytes()))) + .collect(); + let mut script = Vec::new(); + msg_lines(&mut script, "batch", &["hash-algo=sha256"], &flooded_batch); + let mut output = Vec::new(); + let verdict = serve_transfer( + &store, + &Unbounded, + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &script[..], + &mut output, + ); + assert!(matches!( + verdict, + Err(LfsError::TooMany { + what: "batch items", + .. + }) + )); + + let flooded_args: Vec<&str> = std::iter::repeat_n("size=1", MAX_MESSAGE_ARGS + 1).collect(); + let mut script = Vec::new(); + msg( + &mut script, + &format!("put-object {}", oid_of(b"flooded")), + &flooded_args, + ); + let mut output = Vec::new(); + let verdict = serve_transfer( + &store, + &Unbounded, + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &script[..], + &mut output, + ); + assert!(matches!( + verdict, + Err(LfsError::TooMany { + what: "arguments", + .. + }) + )); + } + + #[test] + fn malformed_and_oversized_put_streams_end_the_session() { + let store = MemoryStore::new(); + + let mut sink = Vec::new(); + let garbage = serve_transfer( + &store, + &Unbounded, + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &b"zzzz not pkt-line at all"[..], + &mut sink, + ); + assert!( + matches!(garbage, Err(LfsError::Framing { .. })), + "raw garbage on the wire is a framing fault, not a hang" + ); + + let admission = crate::StoreAdmission::new( + crate::LfsStorePath::new("/"), + LfsSize::new(10), + FreeSpaceFloor::new(0), + ); + let oid = oid_of(b"whatever"); + let mut script = Vec::new(); + put_text(&mut script, &format!("put-object {oid}")); + put_text(&mut script, "size=1"); + encode::delim_to_write(&mut script).unwrap(); + (0..3).for_each(|_| { + encode::data_to_write(&[0u8; 100][..], &mut script).unwrap(); + }); + encode::flush_to_write(&mut script).unwrap(); + let mut output = Vec::new(); + let overshoot = serve_transfer( + &store, + &admission, + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &script[..], + &mut output, + ); + assert!( + matches!(overshoot, Err(LfsError::Framing { .. })), + "a body overshooting the object size limit ends the session instead of draining unbounded bytes" + ); + assert_eq!(store.probe(Pool::Lfs, &repo(), &oid).unwrap(), None); + } + + #[test] + fn store_faults_reach_the_client_without_the_path() { + struct BrokenStore; + impl LfsStore for BrokenStore { + fn put( + &self, + _pool: Pool, + _repo: &RepoDid, + _oid: &LfsOid, + _size: ClaimedSize, + _body: &mut dyn std::io::Read, + ) -> Result<(), LfsError> { + // for now!!!! + unreachable!("this session never puts") + } + + fn read( + &self, + _pool: Pool, + _repo: &RepoDid, + _oid: &LfsOid, + ) -> Result, LfsError> { + unreachable!("this session never reads") + } + + fn probe( + &self, + _pool: Pool, + _repo: &RepoDid, + _oid: &LfsOid, + ) -> Result, LfsError> { + Err(LfsError::Io { + op: "stat", + path: "/srv/secret-lfs-root/plc/sq/uid".into(), + source: std::io::Error::other("disk fell off"), + }) + } + + fn touch( + &self, + pool: Pool, + repo: &RepoDid, + oid: &LfsOid, + ) -> Result, LfsError> { + self.probe(pool, repo, oid) + } + } + + let mut script = Vec::new(); + msg_lines( + &mut script, + "batch", + &["hash-algo=sha256"], + &[format!("{} 5", oid_of(b"whatever"))], + ); + let out = run(TransferOp::Upload, &BrokenStore, &script); + let turns = responses(&out); + assert_eq!(turns[1][0], line("status 500")); + assert!(turns[1].contains(&line("error: internal storage fault"))); + turns[1].iter().for_each(|item| { + if let Out::Line(text) = item { + assert!(!text.contains("secret-lfs-root"), "leaked path in {text:?}"); + } + }); + } + + #[test] + fn a_large_object_round_trips_across_many_packets() { + let store = MemoryStore::new(); + let media: Vec = (0..500_000u32).map(|n| (n % 251) as u8).collect(); + let oid = oid_of(&media); + + let mut script = Vec::new(); + msg_data( + &mut script, + &format!("put-object {oid}"), + &[&format!("size={}", media.len())], + &media, + ); + let out = run(TransferOp::Upload, &store, &script); + assert_eq!(responses(&out)[1][0], line("status 200")); + + let mut fetch = Vec::new(); + msg(&mut fetch, &format!("get-object {oid}"), &[]); + let out = run(TransferOp::Download, &store, &fetch); + let body: Vec = out + .iter() + .skip_while(|item| **item != Out::Delim) + .filter_map(|item| match item { + Out::Bin(chunk) => Some(chunk.clone()), + Out::Line(text) => Some(format!("{text}\n").into_bytes()), + _ => None, + }) + .flatten() + .collect(); + assert_eq!(body, media); + } +} diff --git a/knot2/crates/knot-lfs/src/types.rs b/knot2/crates/knot-lfs/src/types.rs new file mode 100644 index 0000000000000000000000000000000000000000..91e1470c7859d63343069bf3505f88c74424d796 --- /dev/null +++ b/knot2/crates/knot-lfs/src/types.rs @@ -0,0 +1,315 @@ +use std::fmt; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use knot_types::RepoDid; + +use crate::LfsError; + +const OID_HEX_LEN: usize = 64; + +#[derive( + Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, +)] +#[serde(try_from = "String", into = "String")] +pub struct LfsOid(String); + +impl LfsOid { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let valid = value.len() == OID_HEX_LEN + && value + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')); + match valid { + true => Ok(Self(value)), + false => Err(LfsError::InvalidOid { value }), + } + } + + pub fn from_digest(digest: knot_types::Sha256Digest) -> Self { + Self(knot_types::lowercase_hex(&digest.bytes())) + } + + pub fn digest(&self) -> knot_types::Sha256Digest { + knot_types::Sha256Digest::new( + self.0 + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| { + u8::from_str_radix(std::str::from_utf8(pair).expect("an oid is hex"), 16) + .expect("an oid is hex") + }) + .collect::>() + .try_into() + .expect("an oid digest is 32 bytes"), + ) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for LfsOid { + type Err = LfsError; + + fn from_str(s: &str) -> Result { + Self::new(s) + } +} + +impl TryFrom for LfsOid { + type Error = LfsError; + + fn try_from(value: String) -> Result { + Self::new(value) + } +} + +impl From for String { + fn from(oid: LfsOid) -> Self { + oid.0 + } +} + +impl fmt::Display for LfsOid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl AsRef for LfsOid { + fn as_ref(&self) -> &str { + &self.0 + } +} + +#[derive( + Debug, + Clone, + Copy, + Default, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + serde::Serialize, + serde::Deserialize, +)] +#[serde(transparent)] +pub struct LfsSize(u64); + +impl LfsSize { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn saturating_add(self, other: LfsSize) -> LfsSize { + LfsSize(self.0.saturating_add(other.0)) + } +} + +impl fmt::Display for LfsSize { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, +)] +#[serde(transparent)] +// The mass that the client says the incoming object has. +// Admission will reserve disk space +// using this before the obj bytes arrive, +// and the count of what actually showed up in the end is `LfsSize`. +pub struct ClaimedSize(u64); + +impl ClaimedSize { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn matches(self, actual: LfsSize) -> bool { + self.0 == actual.get() + } +} + +impl fmt::Display for ClaimedSize { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct FreeSpaceFloor(u64); + +impl FreeSpaceFloor { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } +} + +impl fmt::Display for FreeSpaceFloor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Pool { + Lfs, + Attachments, +} + +impl Pool { + pub(crate) const fn dir(self) -> &'static str { + match self { + Pool::Lfs => ".lfs", + Pool::Attachments => ".attachments", + } + } + + pub const ALL: [Pool; 2] = [Pool::Lfs, Pool::Attachments]; +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RepoPrefix(PathBuf); + +impl RepoPrefix { + pub fn new(repo: &RepoDid) -> Result { + knot_git::repo_shard(repo) + .map(Self) + .map_err(|_| LfsError::UnsafeRepoDid { + did: repo.as_str().to_string(), + }) + } + + pub fn as_path(&self) -> &Path { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ObjectRelPath(PathBuf); + +impl ObjectRelPath { + pub fn new(pool: Pool, repo: &RepoDid, oid: &LfsOid) -> Result { + let prefix = RepoPrefix::new(repo)?; + let hex = oid.as_str(); + Ok(Self( + Path::new(pool.dir()) + .join(prefix.0) + .join(&hex[0..2]) + .join(&hex[2..4]) + .join(hex), + )) + } + + pub fn as_path(&self) -> &Path { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LfsStorePath(PathBuf); + +impl LfsStorePath { + pub fn new(root: impl Into) -> Self { + Self(root.into()) + } + + pub fn as_path(&self) -> &Path { + &self.0 + } + + pub fn object_path(&self, rel: &ObjectRelPath) -> PathBuf { + self.0.join(rel.as_path()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE_OID: &str = "6c17f2007cbe934aee6e309b28b2fba3c119d98be6ea4156da3aa3173456ad16"; + + #[test] + fn oid_accepts_lowercase_hex_and_round_trips_a_digest() { + let oid = LfsOid::new(SAMPLE_OID).unwrap(); + assert_eq!(oid.as_str(), SAMPLE_OID); + assert_eq!(oid.to_string(), SAMPLE_OID); + assert_eq!(SAMPLE_OID.parse::().unwrap(), oid); + + let from_digest = LfsOid::from_digest(knot_types::Sha256Digest::new([0xab; 32])); + assert_eq!(from_digest.as_str().len(), 64); + assert_eq!( + from_digest.digest(), + knot_types::Sha256Digest::new([0xab; 32]) + ); + assert_eq!(LfsOid::new(from_digest.as_str()).unwrap(), from_digest); + } + + #[test] + fn oid_rejects_everything_else() { + let hostile = [ + "", + "abc123", + &SAMPLE_OID[..63], + &format!("{SAMPLE_OID}0"), + &SAMPLE_OID.to_uppercase(), + &format!("{}g", &SAMPLE_OID[..63]), + "../../../../../../etc/passwd0000000000000000000000000000000000000", + "..%2f..%2f..%2f..%2f..%2fetc%2fpasswd0000000000000000000000000000", + &format!("{}\0", &SAMPLE_OID[..63]), + ]; + hostile.iter().for_each(|value| { + assert!( + matches!(LfsOid::new(*value), Err(LfsError::InvalidOid { .. })), + "accepted {value:?}" + ); + }); + } + + #[test] + fn object_paths_shard_by_pool_repo_then_oid_and_stay_under_root() { + let repo = RepoDid::new("did:plc:squid").unwrap(); + let oid = LfsOid::new(SAMPLE_OID).unwrap(); + let rel = ObjectRelPath::new(Pool::Lfs, &repo, &oid).unwrap(); + assert_eq!( + rel.as_path(), + Path::new(".lfs/plc/sq/uid/6c/17").join(SAMPLE_OID) + ); + let attachments = ObjectRelPath::new(Pool::Attachments, &repo, &oid).unwrap(); + assert_eq!( + attachments.as_path(), + Path::new(".attachments/plc/sq/uid/6c/17").join(SAMPLE_OID) + ); + + let root = LfsStorePath::new("/srv/lfs"); + Pool::ALL.iter().for_each(|pool| { + let path = root.object_path(&ObjectRelPath::new(*pool, &repo, &oid).unwrap()); + assert!(path.starts_with(root.as_path())); + assert!( + path.components() + .all(|part| !matches!(part, std::path::Component::ParentDir)) + ); + }); + } +} diff --git a/knot2/crates/knot-lfs/tests/chaos.rs b/knot2/crates/knot-lfs/tests/chaos.rs new file mode 100644 index 0000000000000000000000000000000000000000..cafb2e876aa9fecc76b122272beabd509ecc24f0 --- /dev/null +++ b/knot2/crates/knot-lfs/tests/chaos.rs @@ -0,0 +1,320 @@ +mod common; + +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant, SystemTime}; + +use common::{backdate, incompressible, object_path, oid_of, pointer_blob}; +use knot_git::{EntryKind, Identity, Layout, NewCommit, RefUpdate, StagedAction, StagedChange}; +use knot_lfs::{ + ClaimedSize, DiskStore, LfsOid, LfsSize, LfsStore, LfsStorePath, Pool, collect_repo, +}; +use knot_types::{AuthorName, BranchName, Email, Oid, RefName, RepoDid, UnixSeconds}; + +const DID: &str = "did:plc:squid"; +const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; +const PUT_BYTES: usize = 48 * 1024 * 1024; +const REFERENCED: usize = 24; +const UNREFERENCED: usize = 320; +const GRACE: Duration = Duration::from_secs(86_400); +const BACKDATE: Duration = Duration::from_secs(60 * 86_400); + +fn did() -> RepoDid { + RepoDid::new(DID).unwrap() +} + +fn spawn_worker(role: &str, envs: &[(&str, &Path)]) -> std::process::Child { + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", &format!("chaos_{role}_worker"), "--nocapture"]) + .env("KNOT_CHAOS_ROLE", role) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + envs.iter().for_each(|(key, value)| { + command.env(key, value); + }); + command.spawn().expect("spawn chaos worker") +} + +fn kill_after(mut child: std::process::Child, delay: Duration) { + std::thread::sleep(delay); + let _ = child.kill(); + child.wait().unwrap(); +} + +fn delays(full: Duration) -> Vec { + let fractions = [0.20, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95]; + std::iter::once(Duration::from_millis(1)) + .chain(std::iter::once(Duration::from_millis(3))) + .chain(fractions.iter().map(|fraction| full.mul_f64(*fraction))) + .chain(std::iter::once(full.mul_f64(2.0))) + .collect() +} + +#[test] +fn chaos_put_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("put") { + return; + } + let store_dir = std::env::var("KNOT_CHAOS_STORE").unwrap(); + let store = DiskStore::open(LfsStorePath::new(&store_dir)).unwrap(); + let body = incompressible(PUT_BYTES, 0x2545_f491_4f6c_dd1d); + let oid = oid_of(&body); + let _ = store.put( + Pool::Lfs, + &did(), + &oid, + ClaimedSize::new(body.len() as u64), + &mut &body[..], + ); +} + +#[test] +fn kill9_during_put_object_never_leaves_a_torn_object() { + let body = incompressible(PUT_BYTES, 0x2545_f491_4f6c_dd1d); + let oid = oid_of(&body); + let scratch = tempfile::tempdir().unwrap(); + + let warm = scratch.path().join("warm"); + std::fs::create_dir_all(&warm).unwrap(); + let started = Instant::now(); + spawn_worker("put", &[("KNOT_CHAOS_STORE", &warm)]) + .wait() + .unwrap(); + let full = started.elapsed(); + assert!( + object_path(&warm, &oid).is_file(), + "an uninterrupted put stores the object" + ); + + let landed: Vec = delays(full) + .iter() + .enumerate() + .map(|(trial, delay)| { + let store_dir = scratch.path().join(format!("store-{trial}")); + std::fs::create_dir_all(&store_dir).unwrap(); + kill_after( + spawn_worker("put", &[("KNOT_CHAOS_STORE", &store_dir)]), + *delay, + ); + + let final_path = object_path(&store_dir, &oid); + let present = final_path.is_file(); + if present { + let bytes = std::fs::read(&final_path).unwrap(); + assert_eq!( + bytes.len(), + PUT_BYTES, + "trial {trial}: a visible object is never truncated" + ); + assert_eq!( + oid_of(&bytes), + oid, + "trial {trial}: a visible object always hashes to its oid" + ); + } + + let store = DiskStore::open(LfsStorePath::new(&store_dir)).unwrap(); + let incoming: Vec<_> = std::fs::read_dir(store_dir.join(".incoming")) + .unwrap() + .collect(); + assert!( + incoming.is_empty(), + "trial {trial}: boot sweep clears abandoned uploads, found {incoming:?}" + ); + assert_eq!( + store.probe(Pool::Lfs, &did(), &oid).unwrap().is_some(), + present, + "trial {trial}: the boot sweep never deletes a stored object" + ); + present + }) + .collect(); + + assert!( + landed.iter().any(|present| !present), + "some trial must be killed before the rename, or the kill delays are all too long" + ); + assert!( + landed.iter().any(|present| *present), + "some trial must complete, or the kill delays are all too short" + ); +} + +fn identity() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } +} + +struct GcFixture { + referenced: Vec<(LfsOid, Vec)>, + unreferenced: Vec, +} + +fn build_gc_fixture(scan: &Path, store_dir: &Path) -> GcFixture { + let layout = Layout::new(scan).with_default_branch(BranchName::new("main").unwrap()); + let repo = layout.create(&did()).unwrap(); + let store = DiskStore::open(LfsStorePath::new(store_dir)).unwrap(); + + let referenced: Vec<(LfsOid, Vec)> = (0..REFERENCED) + .map(|index| { + let body = incompressible(2048, 0x9e37_79b9_7f4a_7c15 ^ index as u64); + let oid = oid_of(&body); + store + .put( + Pool::Lfs, + &did(), + &oid, + ClaimedSize::new(body.len() as u64), + &mut &body[..], + ) + .unwrap(); + backdate(&object_path(store_dir, &oid), BACKDATE); + (oid, body) + }) + .collect(); + + let unreferenced: Vec = (0..UNREFERENCED) + .map(|index| { + let body = incompressible(512, 0xdead_beef_cafe_f00d ^ index as u64); + let oid = oid_of(&body); + store + .put( + Pool::Lfs, + &did(), + &oid, + ClaimedSize::new(body.len() as u64), + &mut &body[..], + ) + .unwrap(); + backdate(&object_path(store_dir, &oid), BACKDATE); + oid + }) + .collect(); + + let changes: Vec = referenced + .iter() + .enumerate() + .map(|(index, (oid, body))| StagedChange { + path: knot_types::RepoPath::new(format!("media/clip{index}.bin")).unwrap(), + action: StagedAction::Put { + content: pointer_blob(oid, LfsSize::new(body.len() as u64)), + kind: EntryKind::Blob, + }, + }) + .collect(); + let empty = Oid::from_hex(EMPTY_TREE).unwrap(); + let tree = repo.write_staged_tree(empty, &changes).unwrap(); + let tip = repo + .write_commit(&NewCommit { + tree, + parents: Vec::new(), + author: identity(), + committer: identity(), + message: "add media".to_string(), + extra_headers: Vec::new(), + }) + .unwrap(); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: tip, + }) + .unwrap(); + + GcFixture { + referenced, + unreferenced, + } +} + +fn run_gc(scan: &Path, store_dir: &Path) { + let layout = Layout::new(scan); + let repo = layout.open(&did()).unwrap(); + let store = DiskStore::open(LfsStorePath::new(store_dir)).unwrap(); + let _ = collect_repo(&store, &repo, &did(), GRACE, SystemTime::now()); +} + +#[test] +fn chaos_gc_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("gc") { + return; + } + let scan = std::env::var("KNOT_CHAOS_SCAN").unwrap(); + let store_dir = std::env::var("KNOT_CHAOS_STORE").unwrap(); + run_gc(Path::new(&scan), Path::new(&store_dir)); +} + +#[test] +fn kill9_during_gc_never_loses_a_referenced_object() { + let scratch = tempfile::tempdir().unwrap(); + + let warm_scan = scratch.path().join("warm-scan"); + let warm_store = scratch.path().join("warm-store"); + let warm_fixture = build_gc_fixture(&warm_scan, &warm_store); + let started = Instant::now(); + spawn_worker( + "gc", + &[ + ("KNOT_CHAOS_SCAN", &warm_scan), + ("KNOT_CHAOS_STORE", &warm_store), + ], + ) + .wait() + .unwrap(); + let full = started.elapsed(); + let warm_disk = DiskStore::open(LfsStorePath::new(&warm_store)).unwrap(); + warm_fixture.referenced.iter().for_each(|(oid, _)| { + assert!( + warm_disk.probe(Pool::Lfs, &did(), oid).unwrap().is_some(), + "an uninterrupted gc keeps every referenced object" + ); + }); + warm_fixture.unreferenced.iter().for_each(|oid| { + assert_eq!( + warm_disk.probe(Pool::Lfs, &did(), oid).unwrap(), + None, + "an uninterrupted gc reclaims every expired orphan" + ); + }); + + delays(full).iter().enumerate().for_each(|(trial, delay)| { + let scan = scratch.path().join(format!("scan-{trial}")); + let store_dir = scratch.path().join(format!("store-{trial}")); + let fixture = build_gc_fixture(&scan, &store_dir); + kill_after( + spawn_worker( + "gc", + &[("KNOT_CHAOS_SCAN", &scan), ("KNOT_CHAOS_STORE", &store_dir)], + ), + *delay, + ); + + let store = DiskStore::open(LfsStorePath::new(&store_dir)).unwrap(); + fixture.referenced.iter().for_each(|(oid, body)| { + assert_eq!( + store.probe(Pool::Lfs, &did(), oid).unwrap(), + Some(LfsSize::new(body.len() as u64)), + "trial {trial}: a referenced object remains stored after a killed sweep" + ); + }); + + run_gc(&scan, &store_dir); + fixture.referenced.iter().for_each(|(oid, _)| { + assert!( + store.probe(Pool::Lfs, &did(), oid).unwrap().is_some(), + "trial {trial}: a referenced object remains stored after the self-heal pass" + ); + }); + fixture.unreferenced.iter().for_each(|oid| { + assert_eq!( + store.probe(Pool::Lfs, &did(), oid).unwrap(), + None, + "trial {trial}: the self-heal pass finishes the interrupted reclaim" + ); + }); + }); +} diff --git a/knot2/crates/knot-lfs/tests/common/mod.rs b/knot2/crates/knot-lfs/tests/common/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..5d15fc31f75b5039e589c2050d53112e335e5f88 --- /dev/null +++ b/knot2/crates/knot-lfs/tests/common/mod.rs @@ -0,0 +1,98 @@ +#![allow(dead_code)] + +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use gix_packetline::blocking_io::encode; +use knot_lfs::{LfsOid, LfsSize}; +use knot_types::RepoDid; + +pub const SQUID: &str = "did:plc:squid"; +pub const PKT_DATA_MAX: usize = 65516; +pub const PEAK_CEILING: u64 = 512 * 1024 * 1024; +pub const GROWTH_SLACK: u64 = 64 * 1024 * 1024; + +pub fn repo() -> RepoDid { + RepoDid::new(SQUID).unwrap() +} + +pub fn oid_of(bytes: &[u8]) -> LfsOid { + LfsOid::from_digest(knot_types::Sha256Digest::hash(bytes)) +} + +pub fn incompressible(len: usize, seed: u64) -> Vec { + let mut state = seed | 1; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() +} + +pub fn pointer_blob(oid: &LfsOid, size: LfsSize) -> Vec { + format!("version https://git-lfs.github.com/spec/v1\noid sha256:{oid}\nsize {size}\n") + .into_bytes() +} + +pub fn object_path(store_dir: &Path, oid: &LfsOid) -> PathBuf { + store_dir + .join(".lfs") + .join("plc/sq/uid") + .join(&oid.as_str()[0..2]) + .join(&oid.as_str()[2..4]) + .join(oid.as_str()) +} + +pub fn backdate(path: &Path, past: Duration) { + std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_modified(SystemTime::now() - past) + .unwrap(); +} + +pub fn put_text(buf: &mut Vec, line: &str) { + encode::data_to_write(format!("{line}\n").as_bytes(), &mut *buf).unwrap(); +} + +pub fn upload_script(body: &[u8]) -> (LfsOid, Vec) { + let oid = oid_of(body); + let mut script = Vec::with_capacity(body.len() + 4096); + put_text(&mut script, &format!("put-object {oid}")); + put_text(&mut script, &format!("size={}", body.len())); + encode::delim_to_write(&mut script).unwrap(); + body.chunks(PKT_DATA_MAX).for_each(|chunk| { + encode::data_to_write(chunk, &mut script).unwrap(); + }); + encode::flush_to_write(&mut script).unwrap(); + put_text(&mut script, &format!("verify-object {oid}")); + put_text(&mut script, &format!("size={}", body.len())); + encode::flush_to_write(&mut script).unwrap(); + put_text(&mut script, "quit"); + encode::flush_to_write(&mut script).unwrap(); + (oid, script) +} + +pub fn download_script(oid: &LfsOid) -> Vec { + let mut script = Vec::new(); + put_text(&mut script, &format!("get-object {oid}")); + encode::flush_to_write(&mut script).unwrap(); + put_text(&mut script, "quit"); + encode::flush_to_write(&mut script).unwrap(); + script +} + +pub fn rss_bytes() -> u64 { + const PAGE_BYTES: u64 = 4096; + let statm = std::fs::read_to_string("/proc/self/statm").expect("/proc/self/statm is readable"); + statm + .split_whitespace() + .nth(1) + .and_then(|pages| pages.parse::().ok()) + .map(|pages| pages * PAGE_BYTES) + .expect("statm lists the resident page count") +} diff --git a/knot2/crates/knot-lfs/tests/fuzz_smoke.rs b/knot2/crates/knot-lfs/tests/fuzz_smoke.rs new file mode 100644 index 0000000000000000000000000000000000000000..883880ee2c663fda2dc1aaabbe8516304a4ef042 --- /dev/null +++ b/knot2/crates/knot-lfs/tests/fuzz_smoke.rs @@ -0,0 +1,20 @@ +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn the_transfer_engine_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_lfs::fuzz::transfer(&data); + } + + #[test] + fn the_batch_json_parser_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_lfs::fuzz::batch(&data); + } + + #[test] + fn the_pointer_parser_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_lfs::fuzz::pointer(&data); + } +} diff --git a/knot2/crates/knot-lfs/tests/properties.rs b/knot2/crates/knot-lfs/tests/properties.rs new file mode 100644 index 0000000000000000000000000000000000000000..e32e40412449579d75ae7d839ed92eaaa1348a64 --- /dev/null +++ b/knot2/crates/knot-lfs/tests/properties.rs @@ -0,0 +1,124 @@ +mod common; + +use common::{oid_of, repo}; +use knot_lfs::{ + ClaimedSize, DiskStore, LfsError, LfsOid, LfsSize, LfsStore, LfsStorePath, MemoryStore, Pool, +}; +use knot_types::Sha256Digest; +use proptest::prelude::*; + +fn round_trip(store: &dyn LfsStore, body: &[u8]) -> Result<(), TestCaseError> { + let repo = repo(); + let oid = oid_of(body); + let bytes = body.len() as u64; + store + .put( + Pool::Lfs, + &repo, + &oid, + ClaimedSize::new(bytes), + &mut &body[..], + ) + .expect("put with matching size and oid succeeds"); + prop_assert_eq!( + store.probe(Pool::Lfs, &repo, &oid).expect("probe"), + Some(LfsSize::new(bytes)) + ); + let mut out = Vec::new(); + store + .read(Pool::Lfs, &repo, &oid) + .expect("stored object opens") + .read_to_end(&mut out) + .expect("stored object reads"); + prop_assert_eq!(out, body); + Ok(()) +} + +#[derive(Debug, Clone)] +enum Tamper { + Flip { at: usize, xor: u8 }, + Truncate { keep: usize }, + Extend { extra: Vec }, +} + +fn tampered(body: &[u8], tamper: &Tamper) -> Vec { + match tamper { + Tamper::Flip { at, xor } => { + let mut bytes = body.to_vec(); + bytes[at % body.len()] ^= xor; + bytes + } + Tamper::Truncate { keep } => body[..keep % body.len()].to_vec(), + Tamper::Extend { extra } => [body, extra].concat(), + } +} + +fn tamper_strategy() -> impl Strategy { + prop_oneof![ + (any::(), 1u8..).prop_map(|(at, xor)| Tamper::Flip { at, xor }), + any::().prop_map(|keep| Tamper::Truncate { keep }), + proptest::collection::vec(any::(), 1..64).prop_map(|extra| Tamper::Extend { extra }), + ] +} + +fn rejects_tampering( + store: &dyn LfsStore, + body: &[u8], + tamper: &Tamper, +) -> Result<(), TestCaseError> { + let repo = repo(); + let oid = oid_of(body); + let size = ClaimedSize::new(body.len() as u64); + let forged = tampered(body, tamper); + let outcome = store.put(Pool::Lfs, &repo, &oid, size, &mut &forged[..]); + prop_assert!( + matches!( + outcome, + Err(LfsError::HashMismatch { .. } | LfsError::SizeMismatch { .. }) + ), + "A tampered body must fail the verifier, got {outcome:?}." + ); + prop_assert_eq!(store.probe(Pool::Lfs, &repo, &oid).expect("probe"), None); + Ok(()) +} + +fn disk_store() -> (DiskStore, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let store = DiskStore::open(LfsStorePath::new(dir.path())).expect("store opens"); + (store, dir) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(64))] + + #[test] + fn any_body_round_trips_through_both_stores( + body in proptest::collection::vec(any::(), 0..4096) + ) { + round_trip(&MemoryStore::new(), &body)?; + let (store, _dir) = disk_store(); + round_trip(&store, &body)?; + } + + #[test] + fn the_verifier_rejects_any_tampered_or_truncated_body( + body in proptest::collection::vec(any::(), 1..2048), + tamper in tamper_strategy(), + ) { + prop_assume!(!matches!(&tamper, Tamper::Flip { xor: 0, .. })); + rejects_tampering(&MemoryStore::new(), &body, &tamper)?; + let (store, _dir) = disk_store(); + rejects_tampering(&store, &body, &tamper)?; + } + + #[test] + fn a_pointer_file_round_trips(digest in any::<[u8; 32]>(), size in any::()) { + let oid = LfsOid::from_digest(Sha256Digest::new(digest)); + let text = format!( + "version https://git-lfs.github.com/spec/v1\noid sha256:{oid}\nsize {size}\n" + ); + let parsed = knot_lfs::parse_pointer(text.as_bytes()).expect("a spec pointer parses"); + prop_assert_eq!(parsed.oid, oid); + prop_assert_eq!(parsed.size, ClaimedSize::new(size)); + } +} diff --git a/knot2/crates/knot-lfs/tests/race.rs b/knot2/crates/knot-lfs/tests/race.rs new file mode 100644 index 0000000000000000000000000000000000000000..6d383bc5df72c8c9a3f37b5438ff3952c0ee337a --- /dev/null +++ b/knot2/crates/knot-lfs/tests/race.rs @@ -0,0 +1,132 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, SystemTime}; + +mod common; + +use common::{oid_of, repo}; +use knot_lfs::{ClaimedSize, DiskStore, LfsOid, LfsSize, LfsStore, LfsStorePath, Pool, Reclaimed}; + +const ROUNDS: usize = 400; +const GRACE: Duration = Duration::from_secs(14 * 86_400); +const BACKDATE: Duration = Duration::from_secs(60 * 86_400); + +#[derive(Clone, Copy)] +enum Bias { + TouchFirst, + SweepFirst, + Simultaneous, +} + +impl Bias { + fn of_round(round: usize) -> Self { + match round % 3 { + 0 => Self::TouchFirst, + 1 => Self::SweepFirst, + _ => Self::Simultaneous, + } + } +} + +fn seed_expired(store: &DiskStore, round: usize) -> LfsOid { + let body = format!("past-grace media, round {round}").into_bytes(); + let oid = oid_of(&body); + store + .put( + Pool::Lfs, + &repo(), + &oid, + ClaimedSize::new(body.len() as u64), + &mut &body[..], + ) + .unwrap(); + let path = store + .object_file(Pool::Lfs, &repo(), &oid) + .unwrap() + .unwrap() + .1; + std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_modified(SystemTime::now() - BACKDATE) + .unwrap(); + oid +} + +#[test] +fn a_mention_concurrent_with_the_sweep_never_yields_a_dangling_pointer() { + let dir = tempfile::tempdir().unwrap(); + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + + let outcomes: Vec<(Option, Reclaimed)> = (0..ROUNDS) + .map(|round| { + let oid = seed_expired(&store, round); + let bias = Bias::of_round(round); + let go = AtomicBool::new(false); + let touch_started = AtomicBool::new(false); + let sweep_started = AtomicBool::new(false); + let (vouched, reclaimed) = std::thread::scope(|scope| { + let toucher = scope.spawn(|| { + while !go.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + if matches!(bias, Bias::SweepFirst) { + while !sweep_started.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + } + touch_started.store(true, Ordering::Release); + store.touch(Pool::Lfs, &repo(), &oid).unwrap() + }); + let sweeper = scope.spawn(|| { + while !go.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + if matches!(bias, Bias::TouchFirst) { + while !touch_started.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + } + sweep_started.store(true, Ordering::Release); + store + .collect_expired(Pool::Lfs, &repo(), &oid, GRACE, SystemTime::now()) + .unwrap() + }); + go.store(true, Ordering::Release); + (toucher.join().unwrap(), sweeper.join().unwrap()) + }); + + if let Some(size) = vouched { + assert!( + matches!(reclaimed, Reclaimed::Spared), + "round {round}: the sweep deleted an object the server just reported stored" + ); + assert_eq!( + store.probe(Pool::Lfs, &repo(), &oid).unwrap(), + Some(size), + "round {round}: an object reported stored must remain readable" + ); + } else { + assert!( + matches!(reclaimed, Reclaimed::Swept(_)), + "round {round}: a touch that reports missing means the sweeper unlinked first" + ); + assert_eq!( + store.probe(Pool::Lfs, &repo(), &oid).unwrap(), + None, + "round {round}: a swept object reports missing" + ); + } + (vouched, reclaimed) + }) + .collect(); + + assert!( + outcomes.iter().any(|(vouched, _)| vouched.is_some()), + "some round must complete the touch first, or the interleaving never varied" + ); + assert!( + outcomes.iter().any(|(vouched, _)| vouched.is_none()), + "some round must complete the sweep first, or the interleaving never varied" + ); +} diff --git a/knot2/crates/knot-lfs/tests/soak.rs b/knot2/crates/knot-lfs/tests/soak.rs new file mode 100644 index 0000000000000000000000000000000000000000..ce7817e2a828a3a5704436835914c36d7b026366 --- /dev/null +++ b/knot2/crates/knot-lfs/tests/soak.rs @@ -0,0 +1,161 @@ +mod common; + +use std::io::Write; + +use common::{ + GROWTH_SLACK, PEAK_CEILING, download_script, incompressible, oid_of, repo, rss_bytes, + upload_script, +}; +use knot_lfs::{ + ClaimedSize, DiskStore, FreeSpaceFloor, LfsOid, LfsSize, LfsStore, LfsStorePath, Pool, + StoreAdmission, TransferOp, serve_transfer, +}; + +const OBJECT_BYTES: usize = 8 * 1024 * 1024; +const SEEDS: usize = 4; +const WRITERS: u64 = 4; +const READERS: usize = 4; +const ROUNDS: u64 = 3; + +struct CountingSink(u64); + +impl Write for CountingSink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 += buf.len() as u64; + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[test] +fn sustained_concurrent_transfers_stay_bounded_and_leak_nothing() { + let dir = tempfile::tempdir().unwrap(); + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + let admission = StoreAdmission::new( + LfsStorePath::new(dir.path()), + LfsSize::new(u64::MAX), + FreeSpaceFloor::new(0), + ); + + let seeded: Vec = (0..SEEDS) + .map(|seed| { + let body = incompressible(OBJECT_BYTES, 0x5eed_0000 + seed as u64); + let oid = oid_of(&body); + store + .put( + Pool::Lfs, + &repo(), + &oid, + ClaimedSize::new(body.len() as u64), + &mut &body[..], + ) + .unwrap(); + oid + }) + .collect(); + + let storm = |round: u64| { + std::thread::scope(|scope| { + let writers: Vec<_> = (0..WRITERS) + .map(|writer| { + let store = &store; + let admission = &admission; + scope.spawn(move || { + let body = + incompressible(OBJECT_BYTES, 0xfeed_0000 + round * WRITERS + writer); + let (oid, script) = upload_script(&body); + let mut out = Vec::new(); + serve_transfer( + store, + admission, + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &script[..], + &mut out, + ) + .unwrap(); + let replies = String::from_utf8_lossy(&out); + assert!( + !replies.contains("status 4") && !replies.contains("status 5"), + "round {round} writer {writer}: upload session failed:\n{replies}" + ); + oid + }) + }) + .collect(); + let readers: Vec<_> = (0..READERS) + .map(|reader| { + let store = &store; + let admission = &admission; + let oid = seeded[reader % SEEDS].clone(); + scope.spawn(move || { + let script = download_script(&oid); + let mut sink = CountingSink(0); + serve_transfer( + store, + admission, + &repo(), + TransferOp::Download, + &knot_messages::default_catalog().lfs, + &script[..], + &mut sink, + ) + .unwrap(); + assert!( + sink.0 >= OBJECT_BYTES as u64, + "round {round} reader {reader}: streamed {} bytes", + sink.0 + ); + }) + }) + .collect(); + let uploaded: Vec = writers + .into_iter() + .map(|writer| writer.join().unwrap()) + .collect(); + readers.into_iter().for_each(|reader| { + reader.join().unwrap(); + }); + uploaded + }) + }; + + let mut uploaded = storm(0); + let settled = rss_bytes(); + + let peaks: Vec = (1..ROUNDS) + .map(|round| { + uploaded.extend(storm(round)); + rss_bytes() + }) + .collect(); + + let peak = peaks.iter().copied().max().unwrap_or(settled); + assert!( + peak < PEAK_CEILING, + "concurrent transfers peaked at {peak} bytes, ceiling {PEAK_CEILING}" + ); + let last = *peaks.last().unwrap_or(&settled); + assert!( + last <= settled + GROWTH_SLACK, + "rss grew from {settled} to {last} across rounds, transfers are leaking" + ); + + uploaded.iter().chain(seeded.iter()).for_each(|oid| { + assert!( + store.probe(Pool::Lfs, &repo(), oid).unwrap().is_some(), + "object {oid} must be readable after the concurrent rounds" + ); + }); + let leftover: Vec<_> = std::fs::read_dir(dir.path().join(".incoming")) + .unwrap() + .collect(); + assert!( + leftover.is_empty(), + "temporary upload files leaked: {leftover:?}" + ); +} diff --git a/knot2/crates/knot-maintenance/Cargo.toml b/knot2/crates/knot-maintenance/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..e16245cda457c436205a538b2f03b0b6928820c9 --- /dev/null +++ b/knot2/crates/knot-maintenance/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "knot-maintenance" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-resource = { workspace = true } +knot-config = { workspace = true } +knot-git = { workspace = true } +knot-lfs = { workspace = true } +knot-pack = { workspace = true } +knot-runtime = { workspace = true } +tracing = { workspace = true } +gix = { workspace = true } +gix-pack = { workspace = true } +gix-hash = { workspace = true } +walkdir = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } + +[dev-dependencies] +knot-fixtures = { workspace = true } +tempfile = { workspace = true } +sha2 = { workspace = true } diff --git a/knot2/crates/knot-maintenance/src/bitmap.rs b/knot2/crates/knot-maintenance/src/bitmap.rs new file mode 100644 index 0000000000000000000000000000000000000000..ba15f3ae206ac1b414ce5c217e54d2f5f9b0a892 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/bitmap.rs @@ -0,0 +1,85 @@ +use std::path::Path; + +use knot_git::{BitmapIndex, Repo}; + +use crate::MaintError; +use crate::fsio::{self, MIDX_SIDECAR_PREFIX, PackStem}; + +pub fn missing(objects_dir: &Path) -> bool { + let idxs = fsio::pack_idx_paths(objects_dir); + let (bitmap, rev) = match idxs.as_slice() { + [] => return false, + [only] => (only.with_extension("bitmap"), only.with_extension("rev")), + _ => { + let Some(stem) = current_midx_stem(objects_dir) else { + return true; + }; + let pack_dir = objects_dir.join("pack"); + (stem.file(&pack_dir, "bitmap"), stem.file(&pack_dir, "rev")) + } + }; + !bitmap.exists() || !rev.exists() +} + +pub fn refresh( + repo: &Repo, + objects_dir: &Path, + previous: Option<&BitmapIndex>, +) -> Result { + let idxs = fsio::pack_idx_paths(objects_dir); + match idxs.as_slice() { + [only] => { + let wrote = knot_git::write_bitmap(repo, only, previous) + .map_err(|error| MaintError::Pack(error.to_string()))?; + let stem = PackStem::of(only); + prune_sidecars(objects_dir, stem.as_ref()); + Ok(wrote) + } + [] => { + prune_sidecars(objects_dir, None); + Ok(false) + } + _ => { + let wrote = knot_git::write_midx_bitmap(repo, previous) + .map_err(|error| MaintError::Pack(error.to_string()))?; + let keep = current_midx_stem(objects_dir); + prune_sidecars(objects_dir, keep.as_ref()); + Ok(wrote) + } + } +} + +fn current_midx_stem(objects_dir: &Path) -> Option { + let path = objects_dir.join("pack").join("multi-pack-index"); + let file = + gix_pack::multi_index::File::at(path, Some(crate::midx::MIDX_ALLOC_LIMIT_BYTES)).ok()?; + Some(PackStem::midx_sidecar( + &file.checksum().to_hex().to_string(), + )) +} + +fn prune_sidecars(objects_dir: &Path, keep_stem: Option<&PackStem>) { + let pack_dir = objects_dir.join("pack"); + let Ok(entries) = std::fs::read_dir(&pack_dir) else { + return; + }; + entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| is_bitmap_sidecar(path)) + .filter(|path| PackStem::of(path).as_ref() != keep_stem) + .for_each(|path| { + let _ = std::fs::remove_file(path); + }); +} + +fn is_bitmap_sidecar(path: &Path) -> bool { + match path.extension().and_then(|ext| ext.to_str()) { + Some("bitmap") => true, + Some("rev") => path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with(MIDX_SIDECAR_PREFIX)), + _ => false, + } +} diff --git a/knot2/crates/knot-maintenance/src/commitgraph.rs b/knot2/crates/knot-maintenance/src/commitgraph.rs new file mode 100644 index 0000000000000000000000000000000000000000..377845e7308d31eebc830ba47215d4ec7abbb239 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/commitgraph.rs @@ -0,0 +1,825 @@ +use std::collections::{HashMap, HashSet}; + +use gix::ObjectId; +use gix::commitgraph::{Graph, write}; +use gix::objs::tree::EntryKind as TreeEntryKind; +use gix::objs::{CommitRef, Kind, TagRefIter, TreeRef}; +use gix::prelude::FindExt; +use knot_git::Repo; +use knot_types::UnixSeconds; + +use crate::MaintError; +use crate::fsio; + +const GRAPH_PARENT_NONE: u32 = 0x7000_0000; +const GRAPH_EXTRA_EDGES_NEEDED: u32 = 0x8000_0000; +const GRAPH_LAST_EDGE: u32 = 0x8000_0000; +const GRAPH_GENERATION_MAX: u32 = 0x3FFF_FFFF; +const MAX_PEEL_DEPTH: usize = 32; + +const CORRECTED_OFFSET_OVERFLOW: u32 = 0x8000_0000; +const CORRECTED_OFFSET_MAX: u64 = (1 << 31) - 1; + +const BLOOM_HASH_VERSION: u32 = 2; +const BLOOM_NUM_HASHES: u32 = 7; +const BLOOM_BITS_PER_ENTRY: u32 = 10; +const BLOOM_MAX_CHANGED_PATHS: usize = 512; +const BLOOM_SEED0: u32 = 0x293a_e76f; +const BLOOM_SEED1: u32 = 0x7e64_6e2c; +const MURMUR_C1: u32 = 0xcc9e_2d51; +const MURMUR_C2: u32 = 0x1b87_3593; +const MURMUR_N: u32 = 0xe654_6b64; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +struct Generation(u32); + +impl Generation { + const fn new(value: u32) -> Self { + Self(value) + } + + const fn get(self) -> u32 { + self.0 + } + + const fn succ(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +knot_types::scalar_newtype! { + struct GraphPosition(u32); +} + +#[derive(Debug, Clone, Copy)] +struct ParentField(u32); + +impl ParentField { + const NONE: Self = Self(GRAPH_PARENT_NONE); + + fn from_position(position: Option) -> Self { + position.map_or(Self::NONE, |position| Self(position.get())) + } + + fn extra_edges(index: usize) -> Self { + Self(GRAPH_EXTRA_EDGES_NEEDED | index as u32) + } + + fn to_be_bytes(self) -> [u8; 4] { + self.0.to_be_bytes() + } +} + +#[derive(Debug, Clone, Copy)] +struct ParentFields { + first: ParentField, + second: ParentField, +} + +#[derive(Debug, Clone, Copy)] +struct EdgeField(u32); + +impl EdgeField { + fn new(parent: ParentField, last: bool) -> Self { + Self(parent.0 | if last { GRAPH_LAST_EDGE } else { 0 }) + } + + fn to_be_bytes(self) -> [u8; 4] { + self.0.to_be_bytes() + } +} + +knot_types::scalar_newtype! { + struct CorrectedDate(u64); +} + +struct ChangedPathFilter(Vec); + +impl ChangedPathFilter { + fn bytes(&self) -> &[u8] { + &self.0 + } + + fn len_bytes(&self) -> usize { + self.0.len() + } +} + +struct CommitMeta { + tree: ObjectId, + parents: Vec, + seconds: UnixSeconds, +} + +pub fn graph_path(repo: &Repo) -> std::path::PathBuf { + repo.objects_dir().join("info").join("commit-graph") +} + +pub fn write(repo: &Repo) -> Result { + let kind = repo.object_format().kind(); + let Some(commits) = collect(repo, kind)? else { + return Ok(false); + }; + if commits.is_empty() { + return Ok(false); + } + let blooms = changed_path_filters(repo, &commits, kind)?; + let bytes = serialize(&commits, &blooms, kind); + let path = graph_path(repo); + let info_dir = repo.objects_dir().join("info"); + std::fs::create_dir_all(&info_dir).map_err(|error| fsio::io_error(&info_dir, error))?; + clear_chain(&info_dir)?; + knot_resource::atomic_write_bytes(&path, &bytes, knot_resource::FileMode::Inherited)?; + Ok(true) +} + +pub fn write_layer(repo: &Repo) -> Result { + let kind = repo.object_format().kind(); + let odb = repo.git().objects.clone(); + let tips: Vec = repo + .references()? + .into_iter() + .filter_map(|record| peel_to_commit(&odb, record.target.object_id(), kind, MAX_PEEL_DEPTH)) + .collect(); + let info_dir = repo.objects_dir().join("info"); + let graph = Graph::from_info_dir(&info_dir).ok(); + let options = write::Options { + object_hash: kind, + bloom_settings: write::bloom_filter_settings( + graph.as_ref(), + true, + Some(BLOOM_HASH_VERSION), + ), + max_new_filters: None, + thread_limit: knot_resource::gix_thread_limit().map(knot_resource::ThreadCount::get), + split: write::Split::Merge { + size_multiple: 2, + max_commits: None, + }, + }; + match write::write_to_info_dir(&odb, &info_dir, graph, tips, options, None) { + Ok(outcome) => Ok(outcome.is_some()), + Err(error) => { + tracing::warn!(repo = %repo.path().display(), %error, "commit-graph layer write failed"); + Ok(false) + } + } +} + +pub fn remove(repo: &Repo) -> Result<(), MaintError> { + let path = graph_path(repo); + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(fsio::io_error(&path, error)), + } + clear_chain(&repo.objects_dir().join("info")) +} + +fn clear_chain(info_dir: &std::path::Path) -> Result<(), MaintError> { + let chain_dir = info_dir.join("commit-graphs"); + match std::fs::remove_dir_all(&chain_dir) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(fsio::io_error(&chain_dir, error)), + } +} + +fn collect( + repo: &Repo, + kind: gix::hash::Kind, +) -> Result>, MaintError> { + let odb = &repo.git().objects; + let mut stack: Vec = repo + .references()? + .into_iter() + .filter_map(|record| peel_to_commit(odb, record.target.object_id(), kind, MAX_PEEL_DEPTH)) + .collect(); + let mut commits: HashMap = HashMap::new(); + while let Some(oid) = stack.pop() { + if commits.contains_key(&oid) { + continue; + } + let mut buf = Vec::new(); + let data = match odb.find(&oid, &mut buf) { + Ok(data) => data, + Err(_) => return Ok(None), + }; + if data.kind != Kind::Commit { + return Ok(None); + } + let commit = CommitRef::from_bytes(data.data, kind) + .map_err(|error| MaintError::CommitGraph(error.to_string()))?; + let tree = commit.tree(); + let parents: Vec = commit.parents().collect(); + let seconds = UnixSeconds::new(commit.committer().map(|sig| sig.seconds()).unwrap_or(0)); + parents.iter().for_each(|parent| stack.push(*parent)); + commits.insert( + oid, + CommitMeta { + tree, + parents, + seconds, + }, + ); + } + Ok(Some(commits)) +} + +fn peel_to_commit( + odb: &gix::odb::Handle, + oid: ObjectId, + kind: gix::hash::Kind, + depth: usize, +) -> Option { + if depth == 0 { + return None; + } + let mut buf = Vec::new(); + let data = odb.find(&oid, &mut buf).ok()?; + match data.kind { + Kind::Commit => Some(oid), + Kind::Tag => { + let target = TagRefIter::from_bytes(data.data, kind).target_id().ok()?; + peel_to_commit(odb, target, kind, depth - 1) + } + _ => None, + } +} + +fn serialize( + commits: &HashMap, + blooms: &HashMap, + kind: gix::hash::Kind, +) -> Vec { + let hash_len = match kind { + gix::hash::Kind::Sha256 => 32, + _ => 20, + }; + let mut oids: Vec = commits.keys().copied().collect(); + oids.sort(); + let position: HashMap = oids + .iter() + .enumerate() + .map(|(index, oid)| (*oid, GraphPosition::new(index as u32))) + .collect(); + let generations = generations(commits, &oids); + let corrected = corrected_dates(commits, &oids); + + let mut edges: Vec = Vec::new(); + let mut cdat: Vec = Vec::with_capacity(oids.len() * (hash_len + 16)); + oids.iter().for_each(|oid| { + let meta = &commits[oid]; + cdat.extend_from_slice(meta.tree.as_slice()); + let parents = parent_fields(&meta.parents, &position, &mut edges); + cdat.extend_from_slice(&parents.first.to_be_bytes()); + cdat.extend_from_slice(&parents.second.to_be_bytes()); + let generation = generations + .get(oid) + .copied() + .unwrap_or(Generation::new(0)) + .get() as u64; + let date = (meta.seconds.get().max(0) as u64) & 0x3_FFFF_FFFF; + let packed = (generation << 34) | date; + cdat.extend_from_slice(&packed.to_be_bytes()); + }); + + let (gda2, overflow) = oids.iter().fold( + (Vec::with_capacity(oids.len() * 4), Vec::::new()), + |(mut bytes, mut ovf), oid| { + let date = commits[oid].seconds.get().max(0) as u64; + let offset = corrected + .get(oid) + .copied() + .unwrap_or(CorrectedDate::new(date)) + .get() + .saturating_sub(date); + let packed = if offset > CORRECTED_OFFSET_MAX { + let index = ovf.len() as u32; + ovf.push(offset); + CORRECTED_OFFSET_OVERFLOW | index + } else { + offset as u32 + }; + bytes.extend_from_slice(&packed.to_be_bytes()); + (bytes, ovf) + }, + ); + let gdo2: Vec = overflow + .iter() + .flat_map(|value| value.to_be_bytes()) + .collect(); + + let bidx: Vec = oids + .iter() + .scan(0u32, |acc, oid| { + *acc = acc.saturating_add(filter_for(blooms, oid).len_bytes() as u32); + Some(acc.to_be_bytes()) + }) + .flatten() + .collect(); + let bdat: Vec = [BLOOM_HASH_VERSION, BLOOM_NUM_HASHES, BLOOM_BITS_PER_ENTRY] + .iter() + .flat_map(|value| value.to_be_bytes()) + .chain( + oids.iter() + .flat_map(|oid| filter_for(blooms, oid).bytes().iter().copied()), + ) + .collect(); + + let oidf = fanout(&oids); + let mut oidl: Vec = Vec::with_capacity(oids.len() * hash_len); + oids.iter() + .for_each(|oid| oidl.extend_from_slice(oid.as_slice())); + let mut edge_bytes: Vec = Vec::with_capacity(edges.len() * 4); + edges + .iter() + .for_each(|edge| edge_bytes.extend_from_slice(&edge.to_be_bytes())); + + let mut chunks: Vec<(&[u8; 4], Vec)> = vec![ + (b"OIDF", oidf), + (b"OIDL", oidl), + (b"CDAT", cdat), + (b"GDA2", gda2), + ]; + if !gdo2.is_empty() { + chunks.push((b"GDO2", gdo2)); + } + if !edge_bytes.is_empty() { + chunks.push((b"EDGE", edge_bytes)); + } + chunks.push((b"BIDX", bidx)); + chunks.push((b"BDAT", bdat)); + + assemble(chunks, kind) +} + +fn filter_for<'a>( + blooms: &'a HashMap, + oid: &ObjectId, +) -> &'a ChangedPathFilter { + static EMPTY: ChangedPathFilter = ChangedPathFilter(Vec::new()); + blooms.get(oid).unwrap_or(&EMPTY) +} + +fn parent_fields( + parents: &[ObjectId], + position: &HashMap, + edges: &mut Vec, +) -> ParentFields { + let pos = |oid: &ObjectId| ParentField::from_position(position.get(oid).copied()); + match parents { + [] => ParentFields { + first: ParentField::NONE, + second: ParentField::NONE, + }, + [first] => ParentFields { + first: pos(first), + second: ParentField::NONE, + }, + [first, second] => ParentFields { + first: pos(first), + second: pos(second), + }, + [first, rest @ ..] => { + let edge_index = edges.len(); + let last = rest.len() - 1; + rest.iter().enumerate().for_each(|(index, parent)| { + edges.push(EdgeField::new(pos(parent), index == last)); + }); + ParentFields { + first: pos(first), + second: ParentField::extra_edges(edge_index), + } + } + } +} + +fn fanout(oids: &[ObjectId]) -> Vec { + let mut buckets = [0u32; 256]; + oids.iter() + .for_each(|oid| buckets[oid.as_slice()[0] as usize] += 1); + (1..256).for_each(|index| buckets[index] += buckets[index - 1]); + buckets + .iter() + .flat_map(|count| count.to_be_bytes()) + .collect() +} + +fn resolve_topo( + commits: &HashMap, + oids: &[ObjectId], + transform: impl Fn(&[V], &CommitMeta) -> V, +) -> HashMap { + let mut value: HashMap = HashMap::new(); + oids.iter().for_each(|root| { + if value.contains_key(root) { + return; + } + let mut stack = vec![*root]; + while let Some(top) = stack.last().copied() { + if value.contains_key(&top) { + stack.pop(); + continue; + } + let parents = &commits[&top].parents; + let unresolved: Vec = parents + .iter() + .filter(|parent| commits.contains_key(*parent) && !value.contains_key(*parent)) + .copied() + .collect(); + if unresolved.is_empty() { + let resolved: Vec = parents + .iter() + .filter_map(|parent| value.get(parent)) + .copied() + .collect(); + let computed = transform(&resolved, &commits[&top]); + value.insert(top, computed); + stack.pop(); + } else { + unresolved.into_iter().for_each(|parent| stack.push(parent)); + } + } + }); + value +} + +fn generations( + commits: &HashMap, + oids: &[ObjectId], +) -> HashMap { + resolve_topo(commits, oids, |parents: &[Generation], _meta| { + parents + .iter() + .copied() + .max() + .unwrap_or(Generation::new(0)) + .succ() + .min(Generation::new(GRAPH_GENERATION_MAX)) + }) +} + +fn corrected_dates( + commits: &HashMap, + oids: &[ObjectId], +) -> HashMap { + resolve_topo(commits, oids, |parents: &[CorrectedDate], meta| { + let max_parent = parents.iter().map(|date| date.get()).max().unwrap_or(0); + let date = meta.seconds.get().max(0) as u64; + let base = if date > max_parent { + date - 1 + } else { + max_parent + }; + CorrectedDate::new(base + 1) + }) +} + +fn tuned(handle: &gix::odb::Handle) -> gix::odb::Handle { + let mut odb = handle.clone(); + odb.refresh_never(); + odb.prevent_pack_unload(); + odb +} + +fn one_filter( + odb: &gix::odb::Handle, + commits: &HashMap, + meta: &CommitMeta, + kind: gix::hash::Kind, +) -> Result { + let parent_tree = meta + .parents + .first() + .and_then(|parent| commits.get(parent)) + .map(|found| found.tree); + let changed = diff_trees(odb, parent_tree, Some(meta.tree), kind)?; + Ok(build_filter(&changed)) +} + +fn changed_path_filters( + repo: &Repo, + commits: &HashMap, + kind: gix::hash::Kind, +) -> Result, MaintError> { + let entries: Vec<(&ObjectId, &CommitMeta)> = commits.iter().collect(); + let path = repo.path().to_owned(); + let produced = knot_resource::map_chunks(&entries, |batch| { + let local = Repo::open(&path)?; + let odb = tuned(&local.git().objects); + batch + .iter() + .map(|(oid, meta)| Ok((**oid, one_filter(&odb, commits, meta, kind)?))) + .collect::, MaintError>>() + })?; + Ok(produced.into_iter().collect()) +} + +fn tree_entries( + odb: &gix::odb::Handle, + oid: Option, + kind: gix::hash::Kind, +) -> Result, (TreeEntryKind, ObjectId)>, MaintError> { + let Some(oid) = oid else { + return Ok(HashMap::new()); + }; + if oid == ObjectId::empty_tree(kind) { + return Ok(HashMap::new()); + } + let mut buf = Vec::new(); + let data = odb + .find(&oid, &mut buf) + .map_err(|error| MaintError::CommitGraph(error.to_string()))?; + if data.kind != Kind::Tree { + return Ok(HashMap::new()); + } + let tree = TreeRef::from_bytes(data.data, kind) + .map_err(|error| MaintError::CommitGraph(error.to_string()))?; + Ok(tree + .entries + .into_iter() + .map(|entry| { + ( + entry.filename.to_vec(), + (entry.mode.kind(), entry.oid.to_owned()), + ) + }) + .collect()) +} + +fn diff_trees( + odb: &gix::odb::Handle, + parent: Option, + commit: Option, + kind: gix::hash::Kind, +) -> Result>, MaintError> { + let is_tree = |kind: &TreeEntryKind| matches!(kind, TreeEntryKind::Tree); + let mut out: Vec> = Vec::new(); + let mut stack: Vec<(Option, Option, Vec)> = + vec![(parent, commit, Vec::new())]; + while let Some((parent, commit, prefix)) = stack.pop() { + let parent_entries = tree_entries(odb, parent, kind)?; + let commit_entries = tree_entries(odb, commit, kind)?; + let names: HashSet<&Vec> = parent_entries.keys().chain(commit_entries.keys()).collect(); + names.into_iter().for_each(|name| { + let full: Vec = prefix.iter().copied().chain(name.iter().copied()).collect(); + let subprefix = + || -> Vec { full.iter().copied().chain(std::iter::once(b'/')).collect() }; + match (parent_entries.get(name), commit_entries.get(name)) { + (None, Some((ck, co))) => { + if is_tree(ck) { + stack.push((None, Some(*co), subprefix())); + } else { + out.push(full); + } + } + (Some((pk, po)), None) => { + if is_tree(pk) { + stack.push((Some(*po), None, subprefix())); + } else { + out.push(full); + } + } + (Some((pk, po)), Some((ck, co))) => match (is_tree(pk), is_tree(ck)) { + (true, true) => { + if po != co { + stack.push((Some(*po), Some(*co), subprefix())); + } + } + (false, false) => { + if po != co || pk != ck { + out.push(full); + } + } + (true, false) => { + stack.push((Some(*po), None, subprefix())); + out.push(full); + } + (false, true) => { + stack.push((None, Some(*co), subprefix())); + out.push(full); + } + }, + (None, None) => {} + } + }); + } + Ok(out) +} + +fn build_filter(changed: &[Vec]) -> ChangedPathFilter { + if changed.len() > BLOOM_MAX_CHANGED_PATHS { + // `0xFF` = git's "too many changed paths" marker. + return ChangedPathFilter(vec![0xFF]); + } + let paths: HashSet> = changed.iter().flat_map(|path| prefixes(path)).collect(); + if paths.len() > BLOOM_MAX_CHANGED_PATHS { + return ChangedPathFilter(vec![0xFF]); + } + let bits = paths.len() * BLOOM_BITS_PER_ENTRY as usize; + let len_bytes = bits.div_ceil(8).max(1); + let modulus = (len_bytes * 8) as u64; + let data = paths.iter().fold(vec![0u8; len_bytes], |mut data, path| { + let hash0 = murmur3(BLOOM_SEED0, path); + let hash1 = murmur3(BLOOM_SEED1, path); + (0..BLOOM_NUM_HASHES).for_each(|index| { + let combined = hash0.wrapping_add(index.wrapping_mul(hash1)); + let position = (combined as u64) % modulus; + data[(position / 8) as usize] |= 1 << (position % 8); + }); + data + }); + ChangedPathFilter(data) +} + +fn prefixes(path: &[u8]) -> Vec> { + std::iter::once(path.to_vec()) + .chain( + path.iter() + .enumerate() + .filter(|(_, byte)| **byte == b'/') + .map(|(index, _)| path[..index].to_vec()), + ) + .collect() +} + +fn murmur3(seed: u32, data: &[u8]) -> u32 { + let body = data.len() / 4; + let mixed = (0..body).fold(seed, |seed, index| { + let base = index * 4; + let block = + u32::from_le_bytes([data[base], data[base + 1], data[base + 2], data[base + 3]]); + let block = block + .wrapping_mul(MURMUR_C1) + .rotate_left(15) + .wrapping_mul(MURMUR_C2); + (seed ^ block) + .rotate_left(13) + .wrapping_mul(5) + .wrapping_add(MURMUR_N) + }); + let tail = &data[body * 4..]; + let tail_key = tail.iter().enumerate().fold(0u32, |key, (index, byte)| { + key | ((*byte as u32) << (8 * index)) + }); + let mixed = if tail.is_empty() { + mixed + } else { + mixed + ^ tail_key + .wrapping_mul(MURMUR_C1) + .rotate_left(15) + .wrapping_mul(MURMUR_C2) + }; + let mixed = mixed ^ (data.len() as u32); + let mixed = (mixed ^ (mixed >> 16)).wrapping_mul(0x85eb_ca6b); + let mixed = (mixed ^ (mixed >> 13)).wrapping_mul(0xc2b2_ae35); + mixed ^ (mixed >> 16) +} + +fn assemble(chunks: Vec<(&[u8; 4], Vec)>, kind: gix::hash::Kind) -> Vec { + let num_chunks = chunks.len() as u8; + let table_len = (chunks.len() + 1) * 12; + let data_start = 8 + table_len; + let hash_version = match kind { + gix::hash::Kind::Sha256 => 2u8, + _ => 1u8, + }; + + let mut out: Vec = Vec::new(); + out.extend_from_slice(b"CGPH"); + out.push(1); + out.push(hash_version); + out.push(num_chunks); + out.push(0); + + let mut offset = data_start as u64; + chunks.iter().for_each(|(id, body)| { + out.extend_from_slice(*id); + out.extend_from_slice(&offset.to_be_bytes()); + offset += body.len() as u64; + }); + out.extend_from_slice(&[0, 0, 0, 0]); + out.extend_from_slice(&offset.to_be_bytes()); + + chunks + .iter() + .for_each(|(_, body)| out.extend_from_slice(body)); + + let mut hasher = gix_hash::hasher(kind); + hasher.update(&out); + let checksum = hasher + .try_finalize() + .expect("commit-graph checksum finalizes"); + out.extend_from_slice(checksum.as_slice()); + out +} + +#[cfg(test)] +mod tests { + use super::{GRAPH_GENERATION_MAX, Generation, build_filter, murmur3}; + + #[test] + fn succ_advances_by_one_and_orders_above_its_source() { + let base = Generation::new(7); + assert_eq!(base.succ(), Generation::new(8)); + assert!(base.succ() > base); + } + + #[test] + fn succ_saturates_at_the_numeric_ceiling() { + assert_eq!(Generation::new(u32::MAX).succ(), Generation::new(u32::MAX)); + } + + #[test] + fn a_child_sits_one_above_its_highest_parent() { + let parents = [Generation::new(2), Generation::new(5), Generation::new(3)]; + let child = parents.into_iter().max().unwrap().succ(); + assert_eq!(child, Generation::new(6)); + } + + #[test] + fn clamping_holds_the_value_at_the_format_maximum() { + let value = Generation::new(GRAPH_GENERATION_MAX) + .succ() + .min(Generation::new(GRAPH_GENERATION_MAX)); + assert_eq!(value, Generation::new(GRAPH_GENERATION_MAX)); + } + + #[test] + fn empty_change_set_is_a_single_zero_word() { + let filter = build_filter(&[]); + assert_eq!(filter.bytes(), &[0u8]); + } + + #[test] + fn overlarge_change_set_is_a_single_saturated_word() { + let many: Vec> = (0..600).map(|n| format!("p{n}").into_bytes()).collect(); + let filter = build_filter(&many); + assert_eq!(filter.bytes(), &[0xFFu8]); + } + + #[test] + fn murmur3_matches_known_vectors() { + assert_eq!(murmur3(0, b""), 0); + assert_eq!(murmur3(0, b"hello"), 0x248bfa47); + } + + #[test] + fn diff_trees_walks_a_deeply_nested_tree_without_overflowing_the_stack() { + use knot_git::{EntryKind, Repo, StagedAction, StagedChange}; + use knot_types::Oid; + + let depth = 4000usize; + let dir = tempfile::tempdir().unwrap(); + let git_dir = dir.path().join("deep.git"); + + let build_dir = git_dir.clone(); + let tree = std::thread::Builder::new() + .stack_size(64 * 1024 * 1024) + .spawn(move || { + let repo = Repo::create(&build_dir).unwrap(); + let kind = repo.object_format().kind(); + let base = Oid::from(gix::ObjectId::empty_tree(kind)); + let path = (0..depth) + .map(|_| "d") + .chain(std::iter::once("leaf.txt")) + .collect::>() + .join("/"); + repo.write_staged_tree( + base, + &[StagedChange { + path: knot_types::RepoPath::new(path).unwrap(), + action: StagedAction::Put { + content: b"leaf\n".to_vec(), + kind: EntryKind::Blob, + }, + }], + ) + .unwrap() + .object_id() + }) + .unwrap() + .join() + .unwrap(); + + let changed = std::thread::Builder::new() + .stack_size(256 * 1024) + .spawn(move || { + let repo = Repo::open(&git_dir).unwrap(); + let kind = repo.object_format().kind(); + super::diff_trees(&repo.git().objects, None, Some(tree), kind).unwrap() + }) + .unwrap() + .join() + .expect("diff_trees on a 256 KiB stack mustn't overflow on a 4000-deep tree"); + + assert_eq!(changed.len(), 1, "the single leaf is the only changed path"); + assert_eq!( + changed[0].iter().filter(|byte| **byte == b'/').count(), + depth, + "the changed path retains every nesting level" + ); + } +} diff --git a/knot2/crates/knot-maintenance/src/cruft.rs b/knot2/crates/knot-maintenance/src/cruft.rs new file mode 100644 index 0000000000000000000000000000000000000000..17fd16a38ff038d81d4615b62daea1e45b13e464 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/cruft.rs @@ -0,0 +1,531 @@ +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; +use std::time::{Duration, SystemTime}; + +use gix::progress::Discard; +use knot_types::{Oid, UnixSeconds}; + +use crate::fsio::{self, PackStem}; +use crate::{FileCount, MaintError, ObjectCount, PruneReport}; + +const MTIMES_MAGIC: u32 = 0x4d54_4d45; +const MTIMES_VERSION: u32 = 1; + +pub fn run( + objects_dir: &Path, + kind: gix::hash::Kind, + reachable: &HashSet, + new_reachable_stem: Option<&PackStem>, + kept_large: &[PackStem], + loose: &[(Oid, PathBuf)], + grace: Duration, +) -> Result { + let pack_dir = objects_dir.join("pack"); + let idxs = fsio::pack_idx_paths(objects_dir); + let now = SystemTime::now(); + let now_mtime = PackedMtime::from_system_time(now); + + let kept_pack = + |stem: &PackStem| -> bool { Some(stem) == new_reachable_stem || kept_large.contains(stem) }; + + let kept_pack_oids: HashSet = idxs + .iter() + .filter(|idx| PackStem::of(idx).is_some_and(|stem| kept_pack(&stem))) + .filter_map(|idx| fsio::pack_oids(idx, kind)) + .flatten() + .collect(); + + let recorded = read_recorded_mtimes(&idxs, kind); + let mtime_of = mtime_index(&idxs, &pack_dir, kind, &recorded, loose); + + let unreachable = + unreachable_candidates(&idxs, kind, loose, reachable, &kept_pack_oids, &kept_pack); + let mut keep: Vec = unreachable + .into_iter() + .filter(|oid| !is_expired(mtime_of.get(oid), now, grace)) + .collect(); + keep.sort(); + + let cruft_stem = if keep.is_empty() { + None + } else { + let stem = write_cruft_pack(objects_dir, keep.clone(), kind)?; + if let Some(stem) = &stem { + write_mtimes(&pack_dir, stem, kind, |oid| { + mtime_of + .get(&oid) + .map(PackedMtime::from_unix) + .unwrap_or(now_mtime) + })?; + } + stem + }; + knot_resource::fsync_path(&pack_dir)?; + + let kept_oids: HashSet = kept_pack_oids + .iter() + .copied() + .chain(cruft_stem.as_ref().into_iter().flat_map(|stem| { + fsio::pack_oids(&stem.file(&pack_dir, "idx"), kind).unwrap_or_default() + })) + .collect(); + + let keep_covered = keep.iter().all(|oid| kept_oids.contains(oid)); + if !closure_is_covered(reachable, &kept_oids) || !keep_covered { + if let Some(stem) = cruft_stem.as_ref() { + fsio::remove_pack_files(&stem.file(&pack_dir, "idx"), &pack_dir); + } + return Ok(PruneReport::skipped()); + } + + let removed_packs = idxs + .iter() + .filter(|idx| { + PackStem::of(idx) + .is_some_and(|stem| !kept_pack(&stem) && Some(&stem) != cruft_stem.as_ref()) + }) + .filter(|idx| fsio::remove_pack_files(idx, &pack_dir)) + .count(); + let removed_loose = loose + .iter() + .filter(|(oid, _)| !reachable.contains(oid)) + .filter(|(_, path)| std::fs::remove_file(path).is_ok()) + .count(); + knot_resource::fsync_path(&pack_dir)?; + knot_resource::fsync_path(objects_dir)?; + + Ok(PruneReport { + removed: FileCount::new(removed_loose), + removed_packs: FileCount::new(removed_packs), + crufted: ObjectCount::new(keep.len()), + ran: true, + }) +} + +fn closure_is_covered(reachable: &HashSet, kept: &HashSet) -> bool { + reachable.is_subset(kept) +} + +fn unreachable_candidates bool>( + idxs: &[PathBuf], + kind: gix::hash::Kind, + loose: &[(Oid, PathBuf)], + reachable: &HashSet, + kept_pack_oids: &HashSet, + kept_pack: &K, +) -> Vec { + idxs.iter() + .filter(|idx| PackStem::of(idx).is_some_and(|stem| !kept_pack(&stem))) + .filter_map(|idx| fsio::pack_oids(idx, kind)) + .flatten() + .chain(loose.iter().map(|(oid, _)| *oid)) + .filter(|oid| !reachable.contains(oid)) + .filter(|oid| !kept_pack_oids.contains(oid)) + .collect::>() + .into_iter() + .collect() +} + +fn mtime_index( + idxs: &[PathBuf], + pack_dir: &Path, + kind: gix::hash::Kind, + recorded: &HashMap, + loose: &[(Oid, PathBuf)], +) -> HashMap { + let from_packs = idxs.iter().flat_map(|idx| { + let is_cruft = idx.with_extension("mtimes").exists(); + let pack_secs = fsio::pack_mtime(idx, pack_dir); + fsio::pack_oids(idx, kind) + .unwrap_or_default() + .into_iter() + .filter_map(move |oid| { + let secs = if is_cruft { + recorded.get(&oid).copied() + } else { + pack_secs + }; + secs.map(|secs| (oid, secs)) + }) + }); + let from_loose = loose + .iter() + .filter_map(|(oid, path)| loose_mtime(path).map(|secs| (*oid, secs))); + newest_by_oid(from_packs.chain(from_loose)) +} + +fn newest_by_oid(pairs: impl Iterator) -> HashMap { + pairs.fold(HashMap::new(), |mut acc, (oid, secs)| { + acc.entry(oid) + .and_modify(|current| { + if secs.get() > current.get() { + *current = secs; + } + }) + .or_insert(secs); + acc + }) +} + +fn read_recorded_mtimes(idxs: &[PathBuf], kind: gix::hash::Kind) -> HashMap { + idxs.iter() + .filter(|idx| idx.with_extension("mtimes").exists()) + .filter_map(|idx| { + let bytes = std::fs::read(idx.with_extension("mtimes")).ok()?; + let oids = fsio::pack_oids(idx, kind)?; + let checksum = gix_pack::data::File::at(idx.with_extension("pack"), kind) + .ok()? + .checksum(); + let table = validated_mtimes(&bytes, oids.len(), kind, checksum.as_slice())?; + Some( + oids.into_iter() + .zip(table) + .map(|(oid, mtime)| (oid, mtime.to_unix())) + .collect::>(), + ) + }) + .flatten() + .collect() +} + +fn validated_mtimes( + bytes: &[u8], + count: usize, + kind: gix::hash::Kind, + pack_checksum: &[u8], +) -> Option> { + let hash_len = kind.len_in_bytes(); + let header = 12usize; + let total = header + count * 4 + hash_len * 2; + if bytes.len() != total + || bytes[0..4] != MTIMES_MAGIC.to_be_bytes() + || bytes[4..8] != MTIMES_VERSION.to_be_bytes() + || bytes[8..12] != hash_id(kind).to_be_bytes() + { + return None; + } + let table_end = header + count * 4; + if bytes[table_end..table_end + hash_len] != *pack_checksum { + return None; + } + let mut hasher = gix_hash::hasher(kind); + hasher.update(&bytes[..total - hash_len]); + let digest = hasher.try_finalize().ok()?; + if digest.as_slice() != &bytes[total - hash_len..] { + return None; + } + Some( + (0..count) + .map(|index| { + let offset = header + index * 4; + PackedMtime(u32::from_be_bytes( + bytes[offset..offset + 4].try_into().unwrap(), + )) + }) + .collect(), + ) +} + +fn is_expired(mtime: Option<&UnixSeconds>, now: SystemTime, grace: Duration) -> bool { + let Some(mtime) = mtime else { + return false; + }; + let when = SystemTime::UNIX_EPOCH + Duration::from_secs(mtime.get().max(0) as u64); + now.duration_since(when) + .map(|age| age >= grace) + .unwrap_or(false) +} + +fn loose_mtime(path: &Path) -> Option { + let modified = path.metadata().ok()?.modified().ok()?; + Some(PackedMtime::from_system_time(modified).to_unix()) +} + +#[derive(Debug, Clone, Copy)] +struct PackedMtime(u32); + +impl PackedMtime { + fn from_unix(secs: &UnixSeconds) -> Self { + Self(secs.get().clamp(0, u32::MAX as i64) as u32) + } + + fn from_system_time(time: SystemTime) -> Self { + Self( + time.duration_since(SystemTime::UNIX_EPOCH) + .map(|delta| delta.as_secs().min(u32::MAX as u64) as u32) + .unwrap_or(0), + ) + } + + fn to_unix(self) -> UnixSeconds { + UnixSeconds::new(self.0 as i64) + } + + fn to_be_bytes(self) -> [u8; 4] { + self.0.to_be_bytes() + } +} + +fn hash_id(kind: gix::hash::Kind) -> u32 { + match kind { + gix::hash::Kind::Sha256 => 2, + _ => 1, + } +} + +fn write_cruft_pack( + objects_dir: &Path, + oids: Vec, + kind: gix::hash::Kind, +) -> Result, MaintError> { + let pack_dir = objects_dir.join("pack"); + std::fs::create_dir_all(&pack_dir).map_err(|error| fsio::io_error(&pack_dir, error))?; + knot_resource::clear_stale(&pack_dir, ".knot-cruft."); + let staging = pack_dir.join(format!( + ".knot-cruft.{}.pack", + knot_resource::staging_nonce() + )); + let outcome = stream_pack(objects_dir, oids, kind, &staging) + .and_then(|()| install_pack(&pack_dir, &staging, kind)); + let _ = std::fs::remove_file(&staging); + outcome +} + +fn stream_pack( + objects_dir: &Path, + oids: Vec, + kind: gix::hash::Kind, + staging: &Path, +) -> Result<(), MaintError> { + let file = std::fs::File::create(staging).map_err(|error| fsio::io_error(staging, error))?; + let mut writer = std::io::BufWriter::new(file); + knot_pack::write_pack(objects_dir, oids, None, &mut writer, kind) + .map_err(|error| MaintError::Pack(error.to_string()))?; + writer + .into_inner() + .map(|_| ()) + .map_err(|error| fsio::io_error(staging, error.into_error())) +} + +fn install_pack( + pack_dir: &Path, + staging: &Path, + kind: gix::hash::Kind, +) -> Result, MaintError> { + let file = std::fs::File::open(staging).map_err(|error| fsio::io_error(staging, error))?; + let mut reader = std::io::BufReader::new(file); + let outcome = gix_pack::Bundle::write_to_directory( + &mut reader, + Some(pack_dir), + &mut Discard, + &AtomicBool::new(false), + None::, + gix_pack::bundle::write::Options { + thread_limit: Some(1), + iteration_mode: gix_pack::data::input::Mode::Verify, + index_version: gix_pack::index::Version::default(), + object_hash: kind, + }, + ) + .map_err(|error| MaintError::Pack(error.to_string()))?; + + if let Some(keep) = &outcome.keep_path { + let _ = std::fs::remove_file(keep); + } + [&outcome.data_path, &outcome.index_path] + .into_iter() + .flatten() + .try_for_each(|path| knot_resource::fsync_path(path))?; + + Ok(outcome + .data_path + .as_ref() + .and_then(|path| PackStem::of(path))) +} + +fn write_mtimes( + pack_dir: &Path, + stem: &PackStem, + kind: gix::hash::Kind, + mtime_for: impl Fn(Oid) -> PackedMtime, +) -> Result<(), MaintError> { + let idx = stem.file(pack_dir, "idx"); + let pack = stem.file(pack_dir, "pack"); + let index = gix_pack::index::File::at(&idx, kind) + .map_err(|error| MaintError::Pack(format!("open cruft index: {error}")))?; + let checksum = gix_pack::data::File::at(&pack, kind) + .map_err(|error| MaintError::Pack(format!("open cruft pack: {error}")))? + .checksum(); + + let mut out = Vec::new(); + out.extend_from_slice(&MTIMES_MAGIC.to_be_bytes()); + out.extend_from_slice(&MTIMES_VERSION.to_be_bytes()); + out.extend_from_slice(&hash_id(kind).to_be_bytes()); + index + .iter() + .for_each(|entry| out.extend_from_slice(&mtime_for(Oid::from(entry.oid)).to_be_bytes())); + out.extend_from_slice(checksum.as_slice()); + let mut hasher = gix_hash::hasher(kind); + hasher.update(&out); + let digest = hasher + .try_finalize() + .map_err(|error| MaintError::Pack(format!("cruft mtimes checksum: {error}")))?; + out.extend_from_slice(digest.as_slice()); + + knot_resource::atomic_write_bytes( + &stem.file(pack_dir, "mtimes"), + &out, + knot_resource::FileMode::Inherited, + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn oid(byte: u8) -> Oid { + Oid::from_hex(&format!("{byte:02x}").repeat(20)).unwrap() + } + + #[test] + fn closure_covered_when_every_reachable_oid_survives() { + let reachable: HashSet = [oid(1), oid(2)].into_iter().collect(); + let kept: HashSet = [oid(1), oid(2), oid(3)].into_iter().collect(); + assert!(closure_is_covered(&reachable, &kept)); + } + + #[test] + fn closure_uncovered_when_a_reachable_oid_is_missing() { + let reachable: HashSet = [oid(1), oid(2)].into_iter().collect(); + let kept: HashSet = [oid(1)].into_iter().collect(); + assert!(!closure_is_covered(&reachable, &kept)); + } + + #[test] + fn unknown_mtime_is_never_expired() { + assert!(!is_expired(None, SystemTime::now(), Duration::ZERO)); + } + + #[test] + fn future_mtime_is_never_expired() { + let future = PackedMtime::from_system_time(SystemTime::now()) + .to_unix() + .saturating_add_secs(100_000); + assert!(!is_expired( + Some(&future), + SystemTime::now(), + Duration::from_secs(1) + )); + } + + #[test] + fn newest_mtime_wins_regardless_of_iteration_order() { + let shared = oid(7); + let old = UnixSeconds::new(1_000); + let fresh = UnixSeconds::new(2_000); + let forward = newest_by_oid([(shared, old), (shared, fresh)].into_iter()); + let reverse = newest_by_oid([(shared, fresh), (shared, old)].into_iter()); + assert_eq!(forward.get(&shared), Some(&fresh)); + assert_eq!(reverse.get(&shared), Some(&fresh)); + } + + #[test] + fn old_mtime_past_grace_is_expired() { + let old = UnixSeconds::new(1_000); + assert!(is_expired( + Some(&old), + SystemTime::now(), + Duration::from_secs(60) + )); + } + + #[test] + fn corrupt_mtimes_are_rejected_rather_than_trusted() { + let count = 3usize; + let hash_len = gix::hash::Kind::Sha1.len_in_bytes(); + let total = 12 + count * 4 + hash_len * 2; + let zero_checksum = vec![0u8; hash_len]; + let unsigned = vec![0u8; total]; + assert!( + validated_mtimes(&unsigned, count, gix::hash::Kind::Sha1, &zero_checksum).is_none(), + "a correctly-sized but unsigned mtimes table isn't trusted" + ); + let truncated = vec![0u8; total - 1]; + assert!( + validated_mtimes(&truncated, count, gix::hash::Kind::Sha1, &zero_checksum).is_none() + ); + let mut wrong_pack = vec![0u8; total]; + wrong_pack[0..4].copy_from_slice(&MTIMES_MAGIC.to_be_bytes()); + wrong_pack[4..8].copy_from_slice(&MTIMES_VERSION.to_be_bytes()); + wrong_pack[8..12].copy_from_slice(&hash_id(gix::hash::Kind::Sha1).to_be_bytes()); + let mismatched = vec![0xabu8; hash_len]; + assert!( + validated_mtimes(&wrong_pack, count, gix::hash::Kind::Sha1, &mismatched).is_none(), + "an mtimes table whose pack checksum names a different pack isn't trusted" + ); + } + + #[test] + fn run_fail_closes_when_survivors_miss_the_closure() { + use knot_git::{Layout, RefUpdate}; + use knot_types::{BranchName, RefName, RepoDid}; + + use crate::test_support::{commit_on, empty_tree}; + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + let repo = layout.create(&did).unwrap(); + let tip = commit_on(&repo, empty_tree(repo.object_format()), Vec::new(), "a"); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: tip, + }) + .unwrap(); + + let options = crate::Options { + repack_max_objects: crate::ObjectCount::new(1_000_000), + geometric_factor: crate::GeometricFactor::full_repack(), + loose_threshold: crate::ObjectCount::new(0), + prune_grace: crate::PruneGrace::from_secs(0), + full_repack_interval: crate::FullRepackInterval::from_secs(432_000), + reflog_floor: crate::ReflogRetention::from_secs(i64::MAX as u64 / 4), + commit_graph: false, + multi_pack_index: false, + bitmap: false, + }; + crate::run_repo(&repo, UnixSeconds::new(1_700_000_500), &options).unwrap(); + + let objects_dir = repo.objects_dir(); + let kind = repo.object_format().kind(); + let reachable: HashSet = repo + .select_pack_objects(knot_git::Wants::new(&[tip]), knot_git::Haves::new(&[])) + .unwrap() + .into_iter() + .collect(); + assert!(!reachable.is_empty()); + + let absent = PackStem::of(Path::new( + "pack-0000000000000000000000000000000000000000.idx", + )) + .unwrap(); + let report = run( + &objects_dir, + kind, + &reachable, + Some(&absent), + &[], + &[], + Duration::ZERO, + ) + .unwrap(); + assert!(!report.ran, "an uncovered closure fail-closes the prune"); + let reopened = knot_git::Repo::open(repo.git().git_dir()).unwrap(); + assert!( + reachable.iter().all(|oid| reopened.contains(*oid)), + "no reachable object is deleted when survivors don't cover the closure" + ); + } +} diff --git a/knot2/crates/knot-maintenance/src/fsio.rs b/knot2/crates/knot-maintenance/src/fsio.rs new file mode 100644 index 0000000000000000000000000000000000000000..a4834b0e931d805955bc56fd1afa5e60ddc9a2a7 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/fsio.rs @@ -0,0 +1,206 @@ +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use knot_types::{Oid, UnixSeconds}; + +use crate::MaintError; + +pub(crate) const MIDX_SIDECAR_PREFIX: &str = "multi-pack-index-"; +const FULL_REPACK_TIMESTAMP: &str = "knot-full-repack-timestamp"; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +// Every file in a pack-set has only the one stem, +// so maintenance finds siblings by just swapping the file extension. +// Checking the stem once here is more efficient than +// checking it at each place that makes a sibling path. +pub struct PackStem(String); + +impl PackStem { + pub fn of(path: &Path) -> Option { + path.file_stem() + .and_then(|stem| stem.to_str()) + .filter(|stem| stem.starts_with("pack-") || stem.starts_with(MIDX_SIDECAR_PREFIX)) + .map(|stem| Self(stem.to_string())) + } + + pub(crate) fn midx_sidecar(checksum_hex: &str) -> Self { + Self(format!("{MIDX_SIDECAR_PREFIX}{checksum_hex}")) + } + + pub fn file(&self, pack_dir: &Path, extension: &str) -> PathBuf { + pack_dir.join(format!("{}.{extension}", self.0)) + } +} + +pub fn io_error(path: &Path, error: std::io::Error) -> MaintError { + MaintError::Io { + path: path.to_path_buf(), + message: error.to_string(), + } +} + +pub fn loose_objects(objects_dir: &Path) -> Vec<(Oid, PathBuf)> { + let Ok(shards) = std::fs::read_dir(objects_dir) else { + return Vec::new(); + }; + shards + .filter_map(Result::ok) + .filter(|shard| is_shard_name(&shard.file_name())) + .flat_map(|shard| loose_in_shard(&shard.path(), &shard.file_name())) + .collect() +} + +fn is_shard_name(name: &std::ffi::OsString) -> bool { + name.to_str() + .is_some_and(|text| text.len() == 2 && text.bytes().all(|byte| byte.is_ascii_hexdigit())) +} + +fn loose_in_shard(shard_path: &Path, shard_name: &std::ffi::OsString) -> Vec<(Oid, PathBuf)> { + let Some(prefix) = shard_name.to_str() else { + return Vec::new(); + }; + let Ok(entries) = std::fs::read_dir(shard_path) else { + return Vec::new(); + }; + entries + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_file())) + .filter_map(|entry| { + let name = entry.file_name(); + let rest = name.to_str()?; + let oid = Oid::from_hex(&format!("{prefix}{rest}")).ok()?; + Some((oid, entry.path())) + }) + .collect() +} + +pub fn has_loose_refs(git_dir: &Path) -> bool { + walkdir::WalkDir::new(git_dir.join("refs")) + .into_iter() + .filter_map(Result::ok) + .any(|entry| entry.file_type().is_file()) +} + +pub fn pack_idx_paths(objects_dir: &Path) -> Vec { + let pack_dir = objects_dir.join("pack"); + let Ok(entries) = std::fs::read_dir(&pack_dir) else { + return Vec::new(); + }; + entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "idx")) + .collect() +} + +pub fn pack_oids(idx: &Path, kind: gix::hash::Kind) -> Option> { + let index = gix_pack::index::File::at(idx, kind).ok()?; + Some(index.iter().map(|entry| Oid::from(entry.oid)).collect()) +} + +pub fn pack_object_count(idx: &Path, kind: gix::hash::Kind) -> Option { + let index = gix_pack::index::File::at(idx, kind).ok()?; + Some(index.num_objects() as usize) +} + +pub fn pack_file(idx: &Path, pack_dir: &Path) -> Option { + let stem = idx.file_stem().and_then(|stem| stem.to_str())?; + Some(pack_dir.join(format!("{stem}.pack"))) +} + +pub fn remove_pack_files(idx: &Path, pack_dir: &Path) -> bool { + let Some(stem) = idx.file_stem().and_then(|stem| stem.to_str()) else { + return false; + }; + let idx_removed = std::fs::remove_file(idx).is_ok(); + let pack_removed = std::fs::remove_file(pack_dir.join(format!("{stem}.pack"))).is_ok(); + let _ = std::fs::remove_file(pack_dir.join(format!("{stem}.rev"))); + let _ = std::fs::remove_file(pack_dir.join(format!("{stem}.bitmap"))); + let _ = std::fs::remove_file(pack_dir.join(format!("{stem}.mtimes"))); + idx_removed || pack_removed +} + +pub fn pack_mtime(idx: &Path, pack_dir: &Path) -> Option { + let pack = pack_file(idx, pack_dir)?; + let modified = pack.metadata().ok()?.modified().ok()?; + let secs = modified + .duration_since(SystemTime::UNIX_EPOCH) + .ok()? + .as_secs(); + Some(UnixSeconds::new(secs as i64)) +} + +pub fn last_full_repack(git_dir: &Path) -> Option { + let modified = git_dir + .join(FULL_REPACK_TIMESTAMP) + .metadata() + .ok()? + .modified() + .ok()?; + let secs = modified + .duration_since(SystemTime::UNIX_EPOCH) + .ok()? + .as_secs(); + Some(UnixSeconds::new(secs as i64)) +} + +pub fn mark_full_repack(git_dir: &Path, at: UnixSeconds) -> Result<(), MaintError> { + let path = git_dir.join(FULL_REPACK_TIMESTAMP); + let modified = SystemTime::UNIX_EPOCH + Duration::from_secs(at.get().max(0) as u64); + std::fs::File::create(&path) + .and_then(|file| file.set_modified(modified)) + .map_err(|error| io_error(&path, error)) +} + +pub fn older_than(path: &Path, grace: Duration) -> bool { + path.metadata() + .and_then(|meta| meta.modified()) + .map(|modified| { + SystemTime::now() + .duration_since(modified) + .unwrap_or(Duration::ZERO) + >= grace + }) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn touch(path: &Path) { + std::fs::write(path, b"x").unwrap(); + } + + #[test] + fn remove_pack_files_clears_idx_and_pack_together() { + let dir = tempfile::tempdir().unwrap(); + let pack_dir = dir.path(); + let idx = pack_dir.join("pack-scallop.idx"); + touch(&idx); + touch(&pack_dir.join("pack-scallop.pack")); + touch(&pack_dir.join("pack-scallop.mtimes")); + + assert!(remove_pack_files(&idx, pack_dir)); + assert!(!idx.exists()); + assert!(!pack_dir.join("pack-scallop.pack").exists()); + assert!(!pack_dir.join("pack-scallop.mtimes").exists()); + } + + #[test] + fn remove_pack_files_reclaims_an_orphan_idx_with_no_pack() { + let dir = tempfile::tempdir().unwrap(); + let pack_dir = dir.path(); + let idx = pack_dir.join("pack-whelk.idx"); + touch(&idx); + + assert!( + remove_pack_files(&idx, pack_dir), + "a lone idx left by a crash mid-deletion is still reclaimed" + ); + assert!( + !idx.exists(), + "the orphan idx no longer advertises phantom objects" + ); + } +} diff --git a/knot2/crates/knot-maintenance/src/lib.rs b/knot2/crates/knot-maintenance/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..a06097620018e94a7c1411c7b43d26876c5889cf --- /dev/null +++ b/knot2/crates/knot-maintenance/src/lib.rs @@ -0,0 +1,477 @@ +use std::collections::HashSet; +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +use knot_git::{PackRefsReport, RefClass, ReflogReport, Repo}; +use knot_types::{ObjectCount, Oid, UnixSeconds}; + +use policy::{Snapshot, Task}; + +mod bitmap; +mod commitgraph; +mod cruft; +mod fsio; +mod midx; +mod policy; +mod prune; +mod repack; +mod scheduler; +#[cfg(test)] +mod test_support; + +pub use midx::MidxStatus; +pub use scheduler::{ + AttachmentRoots, GrantScope, GrantSync, LfsGc, MaintenanceHandle, PushBytes, RepoSource, + Scheduler, +}; + +pub const MIN_REFLOG_RETENTION_SECS: i64 = 30 * 24 * 60 * 60; + +#[derive(Debug, thiserror::Error)] +pub enum MaintError { + #[error("git: {0}")] + Git(#[from] knot_git::GitError), + #[error("pack: {0}")] + Pack(String), + #[error("io {path}: {message}")] + Io { path: PathBuf, message: String }, + #[error("commit-graph: {0}")] + CommitGraph(String), +} + +impl From for MaintError { + fn from(error: knot_resource::FsError) -> Self { + MaintError::Io { + path: error.path, + message: error.source.to_string(), + } + } +} + +knot_types::scalar_newtype! { + pub struct FileCount(usize); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct GeometricFactor(u64); + +impl GeometricFactor { + pub const fn new(value: u64) -> Self { + Self(if value < 2 { 2 } else { value }) + } + + pub const fn full_repack() -> Self { + Self(u64::MAX) + } + + pub const fn get(self) -> u64 { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct Options { + pub repack_max_objects: ObjectCount, + pub geometric_factor: GeometricFactor, + pub loose_threshold: ObjectCount, + pub prune_grace: PruneGrace, + pub full_repack_interval: FullRepackInterval, + pub reflog_floor: ReflogRetention, + pub commit_graph: bool, + pub multi_pack_index: bool, + pub bitmap: bool, +} + +impl Options { + pub fn from_config(config: &knot_config::MaintenanceConfig) -> Self { + Self { + repack_max_objects: ObjectCount::new(config.repack_max_objects as usize), + geometric_factor: GeometricFactor::new(config.repack_geometric_factor), + loose_threshold: ObjectCount::new(config.repack_loose_threshold as usize), + prune_grace: PruneGrace::from_secs(config.prune_grace_secs), + full_repack_interval: FullRepackInterval::from_secs(config.full_repack_interval_secs), + reflog_floor: ReflogRetention::from_secs(config.reflog_expire_secs), + commit_graph: config.commit_graph, + multi_pack_index: config.multi_pack_index, + bitmap: config.bitmap, + } + } +} + +const LFS_GRACE_MIN: Duration = Duration::from_secs(86_400); + +#[derive(Debug, Clone, Copy)] +pub struct GcGrace(Duration); + +impl GcGrace { + pub const fn from_secs(secs: u64) -> Self { + Self(Duration::from_secs(secs)) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct ReflogRetention(Duration); + +impl ReflogRetention { + pub const fn from_secs(secs: u64) -> Self { + Self(Duration::from_secs(secs)) + } + + pub const fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct PruneGrace(Duration); + +impl PruneGrace { + pub const fn from_secs(secs: u64) -> Self { + Self(Duration::from_secs(secs)) + } + + pub const fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct FullRepackInterval(Duration); + +impl FullRepackInterval { + pub const fn from_secs(secs: u64) -> Self { + Self(Duration::from_secs(secs)) + } + + pub const fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LfsGrace(Duration); + +impl LfsGrace { + pub const fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct SweepInterval(Duration); + +impl SweepInterval { + pub const fn new(interval: Duration) -> Self { + Self(interval) + } + + pub const fn get(self) -> Duration { + self.0 + } +} + +pub fn lfs_grace(gc_grace: GcGrace, reflog_retention: ReflogRetention) -> LfsGrace { + let ceiling = reflog_retention + .0 + .max(Duration::from_secs(MIN_REFLOG_RETENTION_SECS as u64)) + .max(LFS_GRACE_MIN); + LfsGrace(gc_grace.0.clamp(LFS_GRACE_MIN, ceiling)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RepackStatus { + Repacked, + Clean, + SkippedTooLarge, + ClosureFailed, + NothingReachable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RepackReport { + pub status: RepackStatus, + pub packed_objects: ObjectCount, + pub removed_loose: FileCount, + pub removed_packs: FileCount, +} + +impl RepackReport { + fn skipped(status: RepackStatus) -> Self { + Self { + status, + packed_objects: ObjectCount::new(0), + removed_loose: FileCount::new(0), + removed_packs: FileCount::new(0), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PruneReport { + pub removed: FileCount, + pub removed_packs: FileCount, + pub crufted: ObjectCount, + pub ran: bool, +} + +impl PruneReport { + fn skipped() -> Self { + Self { + removed: FileCount::new(0), + removed_packs: FileCount::new(0), + crufted: ObjectCount::new(0), + ran: false, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Report { + pub packed_refs: PackRefsReport, + pub reflog: ReflogReport, + pub commit_graph: bool, + pub repack: RepackReport, + pub prune: PruneReport, + pub multi_pack_index: MidxStatus, + pub bitmap: bool, +} + +pub fn run_repo( + repo: &Repo, + now_seconds: UnixSeconds, + opts: &Options, +) -> Result { + let started = Instant::now(); + let objects_dir = repo.objects_dir(); + let kind = repo.object_format().kind(); + let snapshot = Snapshot::take(repo, opts, now_seconds); + let loose = &snapshot.loose; + tracing::info!( + repo = %repo.path().display(), + loose = loose.len(), + loose_refs = snapshot.loose_refs, + "maintenance started" + ); + // open previous bitmap before repack. rollup and cruft clear the midx and delete packs but old + // midx is needed to map the old bitmap's positions to OIDs. + // TODO(boltless): follow git's flow instead + let previous_bitmap = if opts.bitmap { + timed(repo, "open_bitmap", || knot_git::open_bitmap_index(repo)) + .ok() + .flatten() + } else { + None + }; + + if !opts.commit_graph { + commitgraph::remove(repo)?; + } + + let packed_refs = if snapshot.due(Task::PackRefs) { + timed(repo, "pack_refs", || repo.pack_refs())? + } else { + PackRefsReport { packed: 0 } + }; + let floor_secs = (opts.reflog_floor.get().as_secs() as i64).max(MIN_REFLOG_RETENTION_SECS); + let retention_floor = now_seconds.saturating_sub_secs(floor_secs); + let reflog = if snapshot.due(Task::ExpireReflog) { + timed(repo, "expire_reflogs", || { + repo.expire_reflogs(retention_floor) + })? + } else { + ReflogReport { + files: 0, + dropped: 0, + } + }; + + let full_repack = snapshot.due(Task::FullRepack); + if full_repack { + fsio::mark_full_repack(repo.git().git_dir(), now_seconds)?; + } + + let (repack, reachable, roots, new_stem, kept_large) = if full_repack { + let roots = collect_roots(repo, retention_floor)?; + let (report, reachable, new_stem, kept_large) = timed(repo, "full_repack", || { + repack::run( + repo, + &objects_dir, + kind, + roots.iter().copied().collect(), + opts.repack_max_objects, + opts.geometric_factor, + loose, + ) + })?; + (report, reachable, roots, new_stem, kept_large) + } else if snapshot.due(Task::Repack) { + ( + timed(repo, "roll_up", || { + repack::roll_up(&objects_dir, kind, opts.geometric_factor, loose) + })?, + None, + HashSet::new(), + None, + Vec::new(), + ) + } else { + ( + RepackReport::skipped(RepackStatus::Clean), + None, + HashSet::new(), + None, + Vec::new(), + ) + }; + + let prune = match &reachable { + Some(set) => timed(repo, "prune", || { + repo.with_ref_lock(|| { + let current = collect_roots(repo, retention_floor)?; + if current != roots { + return Ok(PruneReport::skipped()); + } + if repack.status == RepackStatus::Repacked { + midx::clear(&objects_dir)?; + cruft::run( + &objects_dir, + kind, + set, + new_stem.as_ref(), + &kept_large, + loose, + opts.prune_grace.get(), + ) + } else { + prune::run(&objects_dir, set, loose, opts.prune_grace.get()) + } + }) + })?, + None => PruneReport::skipped(), + }; + + let commit_graph = if !opts.commit_graph { + false + } else if full_repack { + timed(repo, "commit_graph", || commitgraph::write(repo))? + } else if snapshot.due(Task::CommitGraph) { + timed(repo, "commit_graph_layer", || { + commitgraph::write_layer(repo) + })? + } else { + false + }; + + let multi_pack_index = if !opts.multi_pack_index { + MidxStatus::Absent + } else if snapshot.due(Task::MultiPackIndex) { + timed(repo, "multi_pack_index", || midx::write(repo))? + } else { + MidxStatus::Unchanged + }; + + let bitmap = if opts.bitmap && snapshot.due(Task::Bitmap) { + timed(repo, "bitmap", || { + bitmap::refresh(repo, &objects_dir, previous_bitmap.as_ref()) + })? + } else { + false + }; + + let report = Report { + packed_refs, + reflog, + commit_graph, + repack, + prune, + multi_pack_index, + bitmap, + }; + tracing::info!( + repo = %repo.path().display(), + duration_ms = started.elapsed().as_millis() as u64, + ?report, + "maintenance finished" + ); + Ok(report) +} + +fn timed(repo: &Repo, task: &'static str, run: impl FnOnce() -> T) -> T { + let started = Instant::now(); + let out = run(); + tracing::info!( + repo = %repo.path().display(), + task, + duration_ms = started.elapsed().as_millis() as u64, + "maintenance task finished" + ); + out +} + +fn collect_roots(repo: &Repo, retention_floor: UnixSeconds) -> Result, MaintError> { + repo.retained_oids(retention_floor, |class| match class { + RefClass::Public + | RefClass::Cob + | RefClass::Checkpoint + | RefClass::Hidden + | RefClass::Atproto + | RefClass::Tngl => true, + }) + .map_err(MaintError::from) +} + +#[cfg(test)] +mod tests { + use super::{GcGrace, LFS_GRACE_MIN, MIN_REFLOG_RETENTION_SECS, ReflogRetention, lfs_grace}; + + #[test] + fn the_default_grace_is_not_clamped_by_the_coupling() { + let fourteen_days = 14 * 86_400; + let ninety_days = 90 * 86_400; + assert_eq!( + lfs_grace( + GcGrace::from_secs(fourteen_days), + ReflogRetention::from_secs(ninety_days) + ) + .get() + .as_secs(), + fourteen_days, + "the 14-day default is within the window and never clamped" + ); + } + + #[test] + fn a_small_grace_is_clamped_to_the_hard_minimum() { + assert_eq!( + lfs_grace( + GcGrace::from_secs(0), + ReflogRetention::from_secs(90 * 86_400) + ) + .get(), + LFS_GRACE_MIN + ); + assert_eq!( + lfs_grace( + GcGrace::from_secs(60), + ReflogRetention::from_secs(90 * 86_400) + ) + .get(), + LFS_GRACE_MIN + ); + } + + #[test] + fn grace_never_exceeds_the_reflog_retention() { + let short_reflog = MIN_REFLOG_RETENTION_SECS as u64; + assert_eq!( + lfs_grace( + GcGrace::from_secs(u64::MAX), + ReflogRetention::from_secs(short_reflog) + ) + .get() + .as_secs(), + short_reflog, + "a grace above the reflog retention is clamped to it" + ); + } +} diff --git a/knot2/crates/knot-maintenance/src/midx.rs b/knot2/crates/knot-maintenance/src/midx.rs new file mode 100644 index 0000000000000000000000000000000000000000..abab8bf73069f9f9d3ae92612a1a78519a503beb --- /dev/null +++ b/knot2/crates/knot-maintenance/src/midx.rs @@ -0,0 +1,279 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; + +use gix::progress::Discard; +use knot_git::Repo; + +use crate::fsio; +use crate::{FileCount, MaintError}; + +const FILE_NAME: &str = "multi-pack-index"; + +pub(crate) const MIDX_ALLOC_LIMIT_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MidxStatus { + Written(FileCount), + Removed, + Unchanged, + Absent, +} + +pub(crate) fn clear(objects_dir: &Path) -> Result<(), MaintError> { + let pack_dir = objects_dir.join("pack"); + knot_resource::clear_temps(&pack_dir, FILE_NAME); + knot_resource::fsync_path(&pack_dir).map_err(Into::into) +} + +pub fn write(repo: &Repo) -> Result { + let objects_dir = repo.objects_dir(); + let kind = repo.object_format().kind(); + let target = objects_dir.join("pack").join(FILE_NAME); + let idx_paths = fsio::pack_idx_paths(&objects_dir); + match FileCount::new(idx_paths.len()) { + count if count.get() >= 2 => { + write_atomic(idx_paths, kind, &target)?; + Ok(MidxStatus::Written(count)) + } + _ => remove_if_present(&target), + } +} + +pub fn is_stale(objects_dir: &Path) -> bool { + let mut current: Vec<_> = fsio::pack_idx_paths(objects_dir) + .iter() + .filter_map(|idx| idx.file_name().map(ToOwned::to_owned)) + .collect(); + let target = objects_dir.join("pack").join(FILE_NAME); + if current.len() < 2 { + return target.exists(); + } + let Ok(file) = gix_pack::multi_index::File::at(&target, Some(MIDX_ALLOC_LIMIT_BYTES)) else { + return true; + }; + let mut indexed: Vec<_> = file + .index_names() + .iter() + .filter_map(|name| name.file_name().map(ToOwned::to_owned)) + .collect(); + current.sort(); + indexed.sort(); + current != indexed +} + +fn write_atomic( + idx_paths: Vec, + kind: gix::hash::Kind, + target: &Path, +) -> Result<(), MaintError> { + knot_resource::atomic_write(target, knot_resource::FileMode::Inherited, |file| { + let mut writer = std::io::BufWriter::new(file); + gix_pack::multi_index::write_from_index_paths( + idx_paths, + &mut writer, + &mut Discard, + &AtomicBool::new(false), + gix_pack::multi_index::write::Options { object_hash: kind }, + ) + .map_err(|e| MaintError::Pack(e.to_string()))?; + std::io::Write::flush(&mut writer).map_err(|e| fsio::io_error(target, e)) + }) +} + +fn remove_if_present(target: &Path) -> Result { + match std::fs::remove_file(target) { + Ok(()) => { + if let Some(dir) = target.parent() { + knot_resource::fsync_path(dir)?; + } + Ok(MidxStatus::Removed) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(MidxStatus::Absent), + Err(error) => Err(fsio::io_error(target, error)), + } +} + +#[cfg(test)] +mod tests { + use knot_git::{Layout, RefUpdate}; + use knot_types::{BranchName, ObjectFormat, Oid, RefName, RepoDid}; + + use super::*; + use crate::test_support::{commit_on, empty_tree}; + + fn midx_file(repo: &knot_git::Repo) -> PathBuf { + repo.objects_dir().join("pack").join(FILE_NAME) + } + + fn pack_from(repo: &knot_git::Repo, tip: Oid, kind: gix::hash::Kind) { + let closure = repo + .select_pack_objects(knot_git::Wants::new(&[tip]), knot_git::Haves::new(&[])) + .unwrap(); + let mut bytes = Vec::new(); + knot_pack::write_pack(&repo.objects_dir(), closure, None, &mut bytes, kind).unwrap(); + knot_pack::ingest_pack( + &repo.objects_dir(), + &bytes, + &knot_pack::PackLimits::default(), + kind, + ) + .unwrap(); + } + + fn two_pack_repo(format: ObjectFormat) -> (tempfile::TempDir, knot_git::Repo, Oid, Oid) { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()) + .with_object_format(format) + .with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + let repo = layout.create(&did).unwrap(); + let kind = format.kind(); + let first = commit_on(&repo, empty_tree(format), Vec::new(), "scallop"); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: first, + }) + .unwrap(); + pack_from(&repo, first, kind); + let second = commit_on(&repo, empty_tree(format), Vec::new(), "whelk"); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/side").unwrap(), + new: second, + }) + .unwrap(); + pack_from(&repo, second, kind); + (dir, layout.open(&did).unwrap(), first, second) + } + + fn assert_two_pack_midx(format: ObjectFormat) { + let (_dir, repo, scallop, whelk) = two_pack_repo(format); + assert_eq!(fsio::pack_idx_paths(&repo.objects_dir()).len(), 2); + + let status = write(&repo).unwrap(); + let count = match status { + MidxStatus::Written(count) => count, + other => panic!("expected a written midx, got {other:?}"), + }; + assert_eq!(count, FileCount::new(2)); + assert!(midx_file(&repo).exists()); + + let parsed = + gix_pack::multi_index::File::at(midx_file(&repo), Some(MIDX_ALLOC_LIMIT_BYTES)) + .unwrap(); + assert_eq!(parsed.num_indices() as usize, 2); + assert!(parsed.num_objects() >= 4); + + assert!(repo.contains(scallop)); + assert!(repo.contains(whelk)); + assert_eq!(repo.object_format().kind(), format.kind()); + } + + #[test] + fn stale_tracks_the_pack_set() { + let format = ObjectFormat::SHA1; + let (_dir, repo, _, _) = two_pack_repo(format); + let objects_dir = repo.objects_dir(); + assert!(is_stale(&objects_dir), "two packs without a midx"); + write(&repo).unwrap(); + assert!(!is_stale(&objects_dir), "a midx over the current packs"); + + let third = commit_on(&repo, empty_tree(format), Vec::new(), "periwinkle"); + pack_from(&repo, third, format.kind()); + assert!(is_stale(&objects_dir), "a pack the midx does not name"); + write(&repo).unwrap(); + assert!(!is_stale(&objects_dir)); + + fsio::pack_idx_paths(&objects_dir) + .iter() + .skip(1) + .for_each(|idx| { + fsio::remove_pack_files(idx, &objects_dir.join("pack")); + }); + assert!(is_stale(&objects_dir), "a midx left over a single pack"); + write(&repo).unwrap(); + assert!(!is_stale(&objects_dir)); + } + + #[test] + fn writes_a_multi_pack_index_over_two_packs_sha1() { + assert_two_pack_midx(ObjectFormat::SHA1); + } + + #[test] + fn writes_a_multi_pack_index_over_two_packs_sha256() { + assert_two_pack_midx(ObjectFormat::SHA256); + } + + #[test] + fn clear_removes_the_index_and_sidecars_but_keeps_packs() { + let dir = tempfile::tempdir().unwrap(); + let objects_dir = dir.path(); + let pack_dir = objects_dir.join("pack"); + std::fs::create_dir_all(&pack_dir).unwrap(); + let make = |name: &str| std::fs::write(pack_dir.join(name), b"x").unwrap(); + make(FILE_NAME); + make("multi-pack-index-abc.bitmap"); + make("multi-pack-index-abc.rev"); + make("pack-scallop.idx"); + make("pack-scallop.pack"); + + clear(objects_dir).unwrap(); + + assert!(!pack_dir.join(FILE_NAME).exists()); + assert!(!pack_dir.join("multi-pack-index-abc.bitmap").exists()); + assert!(!pack_dir.join("multi-pack-index-abc.rev").exists()); + assert!( + pack_dir.join("pack-scallop.idx").exists(), + "real packs are left in place" + ); + assert!(pack_dir.join("pack-scallop.pack").exists()); + } + + #[test] + fn lookup_resolves_through_the_midx_once_idx_files_are_gone() { + let (_dir, repo, scallop, whelk) = two_pack_repo(ObjectFormat::SHA1); + let objects_dir = repo.objects_dir(); + assert!(matches!(write(&repo).unwrap(), MidxStatus::Written(_))); + + fsio::loose_objects(&objects_dir) + .iter() + .for_each(|(_, path)| std::fs::remove_file(path).unwrap()); + fsio::pack_idx_paths(&objects_dir) + .iter() + .for_each(|idx| std::fs::remove_file(idx).unwrap()); + assert!( + midx_file(&repo).exists(), + "the multi-pack-index is the only index left on disk" + ); + + let via_midx = knot_git::Repo::open(repo.git().git_dir()).unwrap(); + assert!( + via_midx.contains(scallop) && via_midx.contains(whelk), + "objects resolve through the multi-pack-index with no per-pack idx present" + ); + + std::fs::remove_file(midx_file(&repo)).unwrap(); + let bare = knot_git::Repo::open(repo.git().git_dir()).unwrap(); + assert!( + !bare.contains(scallop) && !bare.contains(whelk), + "with the index gone the packs are unreadable, proving the midx served the lookup" + ); + } + + #[test] + fn fewer_than_two_packs_writes_nothing_and_clears_stale() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:conch").unwrap(); + let repo = layout.create(&did).unwrap(); + + assert_eq!(write(&repo).unwrap(), MidxStatus::Absent); + assert!(!midx_file(&repo).exists()); + + let target = midx_file(&repo); + std::fs::create_dir_all(target.parent().unwrap()).unwrap(); + std::fs::write(&target, b"stale").unwrap(); + assert_eq!(write(&repo).unwrap(), MidxStatus::Removed); + assert!(!midx_file(&repo).exists()); + } +} diff --git a/knot2/crates/knot-maintenance/src/policy.rs b/knot2/crates/knot-maintenance/src/policy.rs new file mode 100644 index 0000000000000000000000000000000000000000..c6c73c2b5f710876ca74c81ab7caf6d0d61264ae --- /dev/null +++ b/knot2/crates/knot-maintenance/src/policy.rs @@ -0,0 +1,60 @@ +use std::path::PathBuf; + +use knot_git::Repo; +use knot_types::{Oid, UnixSeconds}; + +use crate::{Options, bitmap, fsio, midx, repack}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Task { + PackRefs, + ExpireReflog, + CommitGraph, + Repack, + FullRepack, + MultiPackIndex, + Bitmap, +} + +pub struct Snapshot<'a> { + repo: &'a Repo, + opts: &'a Options, + now: UnixSeconds, + pub loose: Vec<(Oid, PathBuf)>, + pub loose_refs: bool, +} + +impl<'a> Snapshot<'a> { + pub fn take(repo: &'a Repo, opts: &'a Options, now: UnixSeconds) -> Self { + Self { + repo, + opts, + now, + loose: fsio::loose_objects(&repo.objects_dir()), + loose_refs: fsio::has_loose_refs(repo.git().git_dir()), + } + } + + pub fn due(&self, task: Task) -> bool { + match task { + Task::PackRefs => self.loose_refs, + Task::Repack => { + self.loose.len() > self.opts.loose_threshold.get() + || repack::rolls_up( + &self.repo.objects_dir(), + self.repo.object_format().kind(), + self.opts.geometric_factor, + ) + } + Task::FullRepack | Task::ExpireReflog => { + fsio::last_full_repack(self.repo.git().git_dir()).is_none_or(|last| { + self.now.get().saturating_sub(last.get()) + >= self.opts.full_repack_interval.get().as_secs() as i64 + }) + } + Task::MultiPackIndex => midx::is_stale(&self.repo.objects_dir()), + Task::Bitmap => bitmap::missing(&self.repo.objects_dir()), + Task::CommitGraph => self.repo.commit_graph_misses_a_tip().unwrap_or(true), + } + } +} diff --git a/knot2/crates/knot-maintenance/src/prune.rs b/knot2/crates/knot-maintenance/src/prune.rs new file mode 100644 index 0000000000000000000000000000000000000000..9a3a60c47d2f230f0eb10955f333ea5c51991ec4 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/prune.rs @@ -0,0 +1,31 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use knot_types::Oid; + +use crate::fsio; +use crate::{FileCount, MaintError, ObjectCount, PruneReport}; + +pub fn run( + objects_dir: &Path, + reachable: &HashSet, + loose: &[(Oid, PathBuf)], + grace: Duration, +) -> Result { + let removed = loose + .iter() + .filter(|(oid, _)| !reachable.contains(oid)) + .filter(|(_, path)| fsio::older_than(path, grace)) + .filter(|(_, path)| std::fs::remove_file(path).is_ok()) + .count(); + if removed > 0 { + knot_resource::fsync_path(objects_dir)?; + } + Ok(PruneReport { + removed: FileCount::new(removed), + removed_packs: FileCount::new(0), + crufted: ObjectCount::new(0), + ran: true, + }) +} diff --git a/knot2/crates/knot-maintenance/src/repack.rs b/knot2/crates/knot-maintenance/src/repack.rs new file mode 100644 index 0000000000000000000000000000000000000000..8a21ed7b46d1bab6c36fb7a279b7dbeebfc54c15 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/repack.rs @@ -0,0 +1,324 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; + +use gix::progress::Discard; +use knot_git::{Haves, Repo, Wants}; +use knot_types::Oid; + +use crate::fsio::{self, PackStem}; +use crate::{ + FileCount, GeometricFactor, MaintError, ObjectCount, RepackReport, RepackStatus, midx, +}; + +type RepackOutcome = ( + RepackReport, + Option>, + Option, + Vec, +); + +pub fn run( + repo: &Repo, + objects_dir: &Path, + kind: gix::hash::Kind, + roots: Vec, + max_objects: ObjectCount, + factor: GeometricFactor, + loose: &[(Oid, PathBuf)], +) -> Result { + let closure = match repo.select_pack_objects(Wants::new(&roots), Haves::new(&[])) { + Ok(closure) => closure, + Err(_) => { + return Ok(( + RepackReport::skipped(RepackStatus::ClosureFailed), + None, + None, + Vec::new(), + )); + } + }; + if closure.is_empty() { + return Ok(( + RepackReport::skipped(RepackStatus::NothingReachable), + Some(HashSet::new()), + None, + Vec::new(), + )); + } + if closure.len() > max_objects.get() { + return Ok(( + RepackReport::skipped(RepackStatus::SkippedTooLarge), + None, + None, + Vec::new(), + )); + } + let reachable: HashSet = closure.iter().copied().collect(); + + let kept_large = if factor == GeometricFactor::full_repack() { + Vec::new() + } else { + kept_large_packs(objects_dir, kind, factor) + }; + let kept_large_oids: HashSet = kept_large + .iter() + .filter_map(|idx| fsio::pack_oids(idx, kind)) + .flatten() + .collect(); + let new_objects: Vec = closure + .into_iter() + .filter(|oid| !kept_large_oids.contains(oid)) + .collect(); + + let new_stem = if new_objects.is_empty() { + None + } else { + build_and_install_pack(objects_dir, new_objects, kind)? + }; + + let removed_loose = loose + .iter() + .filter(|(oid, _)| reachable.contains(oid)) + .filter(|(_, path)| std::fs::remove_file(path).is_ok()) + .count(); + + let kept_large_stems: Vec = kept_large + .iter() + .filter_map(|idx| PackStem::of(idx)) + .collect(); + + Ok(( + RepackReport { + status: RepackStatus::Repacked, + packed_objects: ObjectCount::new(reachable.len()), + removed_loose: FileCount::new(removed_loose), + removed_packs: FileCount::new(0), + }, + Some(reachable), + new_stem, + kept_large_stems, + )) +} + +pub fn roll_up( + objects_dir: &Path, + kind: gix::hash::Kind, + factor: GeometricFactor, + loose: &[(Oid, PathBuf)], +) -> Result { + let pack_dir = objects_dir.join("pack"); + let eligible = eligible_packs(objects_dir, kind); + let weights: Vec = eligible.iter().map(|(_, count)| *count).collect(); + let rolled: Vec<(PathBuf, Vec)> = eligible + .into_iter() + .take(compute_split(&weights, factor)) + .filter_map(|(idx, _)| fsio::pack_oids(&idx, kind).map(|oids| (idx, oids))) + .collect(); + let objects: HashSet = rolled + .iter() + .flat_map(|(_, oids)| oids.iter().copied()) + .chain(loose.iter().map(|(oid, _)| *oid)) + .collect(); + if objects.is_empty() { + return Ok(RepackReport::skipped(RepackStatus::Clean)); + } + let packed_objects = ObjectCount::new(objects.len()); + + let Some(new_stem) = build_and_install_pack(objects_dir, objects.into_iter().collect(), kind)? + else { + return Err(MaintError::Pack("the rollup wrote no pack".into())); + }; + midx::clear(objects_dir)?; + let removed_packs = rolled + .iter() + .filter(|(idx, _)| PackStem::of(idx).as_ref() != Some(&new_stem)) + .filter(|(idx, _)| fsio::remove_pack_files(idx, &pack_dir)) + .count(); + let removed_loose = loose + .iter() + .filter(|(_, path)| std::fs::remove_file(path).is_ok()) + .count(); + knot_resource::fsync_path(&pack_dir)?; + knot_resource::fsync_path(objects_dir)?; + + Ok(RepackReport { + status: RepackStatus::Repacked, + packed_objects, + removed_loose: FileCount::new(removed_loose), + removed_packs: FileCount::new(removed_packs), + }) +} + +pub fn rolls_up(objects_dir: &Path, kind: gix::hash::Kind, factor: GeometricFactor) -> bool { + let eligible = eligible_packs(objects_dir, kind); + let weights: Vec = eligible.iter().map(|(_, count)| *count).collect(); + compute_split(&weights, factor) >= 2 +} + +fn kept_large_packs( + objects_dir: &Path, + kind: gix::hash::Kind, + factor: GeometricFactor, +) -> Vec { + let eligible = eligible_packs(objects_dir, kind); + let weights: Vec = eligible.iter().map(|(_, count)| *count).collect(); + let split = compute_split(&weights, factor); + eligible + .into_iter() + .skip(split) + .map(|(idx, _)| idx) + .collect() +} + +fn eligible_packs(objects_dir: &Path, kind: gix::hash::Kind) -> Vec<(PathBuf, usize)> { + let pack_dir = objects_dir.join("pack"); + let mut eligible: Vec<(PathBuf, usize)> = fsio::pack_idx_paths(objects_dir) + .into_iter() + .filter(|idx| !is_excluded(idx, &pack_dir)) + .filter_map(|idx| fsio::pack_object_count(&idx, kind).map(|count| (idx, count))) + .collect(); + eligible.sort_by_key(|(_, count)| *count); + eligible +} + +fn is_excluded(idx: &Path, pack_dir: &Path) -> bool { + let Some(stem) = idx.file_stem().and_then(|stem| stem.to_str()) else { + return true; + }; + pack_dir.join(format!("{stem}.mtimes")).exists() +} + +fn compute_split(weights: &[usize], factor: GeometricFactor) -> usize { + let n = weights.len(); + if n == 0 { + return 0; + } + let geometric = + |big: usize, small: usize| (small as u64).saturating_mul(factor.get()) <= big as u64; + let split1 = (1..n) + .rev() + .find(|&i| !geometric(weights[i], weights[i - 1])) + .map(|i| i + 1) + .unwrap_or(0); + let total: u64 = weights[..split1].iter().map(|weight| *weight as u64).sum(); + let extended = (split1..n).try_fold((split1, total), |(split, total), j| { + match total.checked_mul(factor.get()) { + Some(threshold) if (weights[j] as u64) < threshold => std::ops::ControlFlow::Continue( + (split + 1, total.saturating_add(weights[j] as u64)), + ), + _ => std::ops::ControlFlow::Break((split, total)), + } + }); + match extended { + std::ops::ControlFlow::Continue((split, _)) => split, + std::ops::ControlFlow::Break((split, _)) => split, + } +} + +fn build_and_install_pack( + objects_dir: &Path, + closure: Vec, + kind: gix::hash::Kind, +) -> Result, MaintError> { + let pack_dir = objects_dir.join("pack"); + std::fs::create_dir_all(&pack_dir).map_err(|error| fsio::io_error(&pack_dir, error))?; + knot_resource::clear_stale(&pack_dir, ".knot-repack."); + let staging = pack_dir.join(format!( + ".knot-repack.{}.pack", + knot_resource::staging_nonce() + )); + let outcome = write_streaming_pack(objects_dir, closure, kind, &staging) + .and_then(|()| install_streamed_pack(&pack_dir, &staging, kind)); + let _ = std::fs::remove_file(&staging); + outcome +} + +fn write_streaming_pack( + objects_dir: &Path, + closure: Vec, + kind: gix::hash::Kind, + staging: &Path, +) -> Result<(), MaintError> { + let file = std::fs::File::create(staging).map_err(|error| fsio::io_error(staging, error))?; + let mut writer = std::io::BufWriter::new(file); + knot_pack::write_pack(objects_dir, closure, None, &mut writer, kind) + .map_err(|error| MaintError::Pack(error.to_string()))?; + writer + .into_inner() + .map(|_| ()) + .map_err(|error| fsio::io_error(staging, error.into_error())) +} + +fn install_streamed_pack( + pack_dir: &Path, + staging: &Path, + kind: gix::hash::Kind, +) -> Result, MaintError> { + let file = std::fs::File::open(staging).map_err(|error| fsio::io_error(staging, error))?; + let mut reader = std::io::BufReader::new(file); + let outcome = gix_pack::Bundle::write_to_directory( + &mut reader, + Some(pack_dir), + &mut Discard, + &AtomicBool::new(false), + None::, + gix_pack::bundle::write::Options { + thread_limit: Some(1), + iteration_mode: gix_pack::data::input::Mode::Verify, + index_version: gix_pack::index::Version::default(), + object_hash: kind, + }, + ) + .map_err(|error| MaintError::Pack(error.to_string()))?; + + if let Some(keep) = &outcome.keep_path { + let _ = std::fs::remove_file(keep); + } + [&outcome.data_path, &outcome.index_path] + .into_iter() + .flatten() + .try_for_each(|path| knot_resource::fsync_path(path))?; + knot_resource::fsync_path(pack_dir)?; + + Ok(outcome + .data_path + .as_ref() + .and_then(|path| PackStem::of(path))) +} + +#[cfg(test)] +mod tests { + use super::{GeometricFactor, compute_split}; + + #[test] + fn uniform_small_packs_all_roll_up() { + assert_eq!(compute_split(&[1, 1, 1, 1], GeometricFactor::new(2)), 4); + } + + #[test] + fn a_clean_geometric_progression_rolls_up_nothing() { + assert_eq!(compute_split(&[1, 2, 4, 8], GeometricFactor::new(2)), 0); + } + + #[test] + fn one_large_pack_with_tiny_additions_keeps_the_large_one() { + let split = compute_split(&[1, 1, 1, 1000], GeometricFactor::new(2)); + assert_eq!(split, 3); + } + + #[test] + fn a_single_pack_is_never_rolled_up() { + assert_eq!(compute_split(&[42], GeometricFactor::new(2)), 0); + } + + #[test] + fn empty_input_rolls_up_nothing() { + assert_eq!(compute_split(&[], GeometricFactor::new(2)), 0); + } + + #[test] + fn a_huge_factor_saturates_without_overflow() { + assert_eq!(compute_split(&[1, 1, 1], GeometricFactor::full_repack()), 3); + } +} diff --git a/knot2/crates/knot-maintenance/src/scheduler.rs b/knot2/crates/knot-maintenance/src/scheduler.rs new file mode 100644 index 0000000000000000000000000000000000000000..1a57596950a142c22d7878449f0986d85589bd19 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/scheduler.rs @@ -0,0 +1,789 @@ +use std::collections::HashSet; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use knot_git::Layout; +use knot_lfs::DiskStore; +use knot_runtime::Clock; +use knot_types::{RepoDid, UnixSeconds}; +use tokio::sync::{mpsc, watch}; + +use crate::{LfsGrace, Options, RepackStatus, Report, SweepInterval, run_repo}; + +pub trait RepoSource: Send + Sync + 'static { + fn repos(&self) -> Vec; + fn ready_repos(&self) -> Option>; +} + +// "Grace" means how old an unreferenced object can get before we reap it, +// and the "interval" is how often we look. +pub struct LfsGc { + store: Arc, + grace: LfsGrace, + interval: SweepInterval, + attachments: Option>, +} + +impl LfsGc { + pub fn new(store: Arc, grace: LfsGrace, interval: SweepInterval) -> Self { + Self { + store, + grace, + interval, + attachments: None, + } + } +} + +pub trait AttachmentRoots: Send + Sync { + fn referenced( + &self, + repo: &RepoDid, + ) -> Result, knot_lfs::RootsDenied>; +} +pub enum GrantScope { + Knot, + Repo(RepoDid), +} + +pub trait GrantSync: Send + Sync + 'static { + fn sweep(&self, scope: GrantScope); +} + +const MIN_ATTACHMENT_GRACE: std::time::Duration = std::time::Duration::from_secs(60); + +fn attachment_grace(configured: std::time::Duration) -> std::time::Duration { + if configured < MIN_ATTACHMENT_GRACE { + tracing::warn!( + configured = configured.as_secs(), + "The attachment sweep keeps a sixty-second grace minimum, so a write-admitted \ + blob outlives the gap between admission and commit." + ); + MIN_ATTACHMENT_GRACE + } else { + configured + } +} + +const TRIGGER_CAPACITY: usize = 1024; + +knot_types::scalar_newtype! { + pub struct PushBytes(u64) => ordered; +} + +#[derive(Clone)] +pub struct MaintenanceHandle { + trigger: Option>, + large_push: PushBytes, +} + +impl MaintenanceHandle { + pub fn disabled() -> Self { + Self { + trigger: None, + large_push: PushBytes::new(u64::MAX), + } + } + + pub fn note_push(&self, repo: &RepoDid, pack_bytes: PushBytes) { + if pack_bytes >= self.large_push + && let Some(trigger) = &self.trigger + { + let _ = trigger.try_send(repo.clone()); + } + } +} + +pub struct Scheduler { + layout: Layout, + source: Arc, + clock: C, + options: Options, + interval: Duration, + triggers: mpsc::Receiver, + lfs: LfsGc, + grants: Option>, +} + +impl Scheduler { + pub fn new( + layout: Layout, + source: Arc, + clock: C, + options: Options, + interval: Duration, + large_push: PushBytes, + lfs: LfsGc, + ) -> (Self, MaintenanceHandle) { + let (trigger, triggers) = mpsc::channel(TRIGGER_CAPACITY); + let handle = MaintenanceHandle { + trigger: Some(trigger), + large_push, + }; + let scheduler = Self { + layout, + source, + clock, + options, + interval, + triggers, + lfs, + grants: None, + }; + (scheduler, handle) + } + + pub fn with_grant_sync(mut self, grants: Arc) -> Self { + self.grants = Some(grants); + self + } + + pub fn with_attachment_roots(mut self, roots: Arc) -> Self { + self.lfs.attachments = Some(roots); + self + } + + fn now_seconds(&self) -> UnixSeconds { + UnixSeconds::new((self.clock.now_unix_micros().get() / 1_000_000) as i64) + } + + fn now(&self) -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::from_micros(self.clock.now_unix_micros().get()) + } + + async fn gc_repo(&self, repo: &RepoDid) -> knot_lfs::GcReport { + let report = self.sweep_lfs_objects(&self.lfs, repo).await; + self.sweep_unreferenced_attachments(&self.lfs, repo).await; + report + } + + async fn sweep_lfs_objects(&self, lfs: &LfsGc, repo: &RepoDid) -> knot_lfs::GcReport { + let layout = self.layout.clone(); + let store = Arc::clone(&lfs.store); + let grace = lfs.grace.get(); + let now = self.now(); + let target = repo.clone(); + let started = std::time::Instant::now(); + let outcome = tokio::task::spawn_blocking(move || { + layout + .open(&target) + .map_err(knot_lfs::GcError::from) + .and_then(|opened| knot_lfs::collect_repo(&store, &opened, &target, grace, now)) + }) + .await; + match outcome { + Ok(Ok(report)) => { + if report.swept > 0 { + tracing::info!( + repo = %repo, + scanned = report.scanned, + marked = report.marked, + swept = report.swept, + bytes = report.bytes.get(), + duration_ms = started.elapsed().as_millis() as u64, + "lfs gc reclaimed objects" + ); + } + report + } + Ok(Err(error)) => { + tracing::warn!(repo = %repo, %error, "lfs gc skipped, projection uncertain"); + knot_lfs::GcReport::default() + } + Err(join) => { + tracing::error!(repo = %repo, %join, "lfs gc task panicked"); + knot_lfs::GcReport::default() + } + } + } + + async fn sweep_unreferenced_attachments(&self, lfs: &LfsGc, repo: &RepoDid) { + let Some(roots) = &lfs.attachments else { + return; + }; + let roots = Arc::clone(roots); + let queried = repo.clone(); + let lookup = tokio::task::spawn_blocking(move || roots.referenced(&queried)).await; + let referenced = match lookup { + Ok(Ok(referenced)) => referenced, + Ok(Err(reason)) => { + tracing::warn!( + repo = %repo, + %reason, + "The attachment roots query failed, so the sweep keeps every attachment." + ); + return; + } + Err(join) => { + tracing::error!(repo = %repo, %join, "Attachment roots task panicked."); + return; + } + }; + let store = Arc::clone(&lfs.store); + let grace = attachment_grace(lfs.grace.get()); + let now = self.now(); + let target = repo.clone(); + let outcome = tokio::task::spawn_blocking(move || { + store.sweep_unreferenced(&target, &referenced, grace, now) + }) + .await; + match outcome { + Ok(Ok(sweep)) if sweep.objects > 0 => tracing::info!( + repo = %repo, + objects = sweep.objects, + bytes = sweep.bytes.get(), + "Attachment sweep reclaimed unreferenced blobs." + ), + Ok(Ok(_)) => (), + Ok(Err(error)) => { + tracing::warn!(repo = %repo, %error, "Attachment sweep skipped.") + } + Err(join) => { + tracing::error!(repo = %repo, %join, "Attachment sweep task panicked.") + } + } + } + + async fn sweep_orphans(&self) { + let lfs = &self.lfs; + let Some(hosted) = self.source.ready_repos() else { + return; + }; + let store = Arc::clone(&lfs.store); + let grace = lfs.grace.get(); + let now = self.now(); + let hosted: HashSet = hosted.into_iter().collect(); + let outcome = + tokio::task::spawn_blocking(move || store.sweep_orphans(&hosted, grace, now)).await; + match outcome { + Ok(Ok(sweep)) if sweep.prefixes > 0 => tracing::info!( + prefixes = sweep.prefixes, + objects = sweep.objects, + bytes = sweep.bytes.get(), + "lfs gc reclaimed orphan prefixes" + ), + Ok(Ok(_)) => {} + Ok(Err(error)) => tracing::warn!(%error, "lfs orphan sweep failed"), + Err(join) => tracing::error!(%join, "lfs orphan sweep task panicked"), + } + } + + pub async fn run(mut self, mut shutdown: watch::Receiver) { + let mut tick = tokio::time::interval(self.interval); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + tick.tick().await; + let mut lfs_tick = tokio::time::interval(self.lfs.interval.get()); + lfs_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + lfs_tick.tick().await; + let probe = shutdown.clone(); + loop { + tokio::select! { + biased; + _ = shutdown.changed() => break, + _ = tick.tick() => self.sweep_all(&probe).await, + _ = lfs_tick.tick() => self.lfs_sweep_all(&probe).await, + Some(repo) = self.triggers.recv() => self.maintain_batch(repo).await, + } + } + } + + async fn sweep_all(&self, shutdown: &watch::Receiver) { + self.sync_grants(GrantScope::Knot).await; + for repo in self.source.repos() { + if *shutdown.borrow() { + return; + } + self.sync_grants(GrantScope::Repo(repo.clone())).await; + self.maintain_one(repo).await; + } + } + + async fn sync_grants(&self, scope: GrantScope) { + let Some(grants) = self.grants.clone() else { + return; + }; + if let Err(join) = tokio::task::spawn_blocking(move || grants.sweep(scope)).await { + tracing::error!(%join, "a grant sync task panicked"); + } + } + + async fn lfs_sweep_all(&self, shutdown: &watch::Receiver) { + let started = std::time::Instant::now(); + let mut totals = knot_lfs::GcReport::default(); + let mut repos = 0usize; + for repo in self.source.repos() { + if *shutdown.borrow() { + return; + } + let report = self.gc_repo(&repo).await; + repos += 1; + totals.scanned += report.scanned; + totals.marked += report.marked; + totals.swept += report.swept; + totals.bytes = totals.bytes.saturating_add(report.bytes); + } + if !*shutdown.borrow() { + self.sweep_orphans().await; + } + tracing::info!( + repos, + scanned = totals.scanned, + marked = totals.marked, + swept = totals.swept, + bytes = totals.bytes.get(), + duration_ms = started.elapsed().as_millis() as u64, + "lfs gc pass finished" + ); + } + + async fn maintain_batch(&mut self, first: RepoDid) { + let pending: HashSet = std::iter::once(first) + .chain(std::iter::from_fn(|| self.triggers.try_recv().ok())) + .collect(); + for repo in pending { + self.maintain_one(repo.clone()).await; + self.gc_repo(&repo).await; + } + } + + async fn maintain_one(&self, repo: RepoDid) { + let layout = self.layout.clone(); + let options = self.options; + let now_seconds = self.now_seconds(); + let target = repo.clone(); + let outcome = tokio::task::spawn_blocking(move || { + layout + .open(&target) + .map_err(crate::MaintError::from) + .and_then(|opened| run_repo(&opened, now_seconds, &options)) + }) + .await; + match outcome { + Ok(Ok(report)) => report_skips(&repo, &report), + Ok(Err(error)) => tracing::error!(repo = %repo, %error, "maintenance run failed"), + Err(join) => tracing::error!(repo = %repo, %join, "maintenance task panicked"), + } + } +} + +fn report_skips(repo: &RepoDid, report: &Report) { + match report.repack.status { + RepackStatus::SkippedTooLarge => { + tracing::warn!( + repo = %repo, + reason = "reachable set exceeds repack_max_objects", + "skipped repack" + ) + } + RepackStatus::ClosureFailed => { + tracing::warn!( + repo = %repo, + reason = "couldn't compute reachable set", + "skipped repack and prune" + ) + } + _ => {} + } +} + +#[cfg(test)] +mod tests { + use knot_git::{Layout, RefUpdate}; + use std::collections::HashSet; + use std::sync::Arc; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + use knot_lfs::{ClaimedSize, DiskStore, LfsOid, LfsStore, LfsStorePath, Pool}; + use knot_runtime::{ManualClock, SystemClock}; + use knot_types::{BranchName, RefName, RepoDid, UnixMicros}; + + use super::{ + LfsGc, MIN_ATTACHMENT_GRACE, MaintenanceHandle, PushBytes, RepoSource, Scheduler, + attachment_grace, + }; + use crate::test_support::{commit_on, empty_tree, options}; + + struct Fixed(Vec); + impl RepoSource for Fixed { + fn repos(&self) -> Vec { + self.0.clone() + } + + fn ready_repos(&self) -> Option> { + Some(self.0.clone()) + } + } + + fn seed_repo(layout: &Layout, did: &RepoDid) { + let repo = layout.create(did).unwrap(); + let tip = commit_on(&repo, empty_tree(repo.object_format()), Vec::new(), "a"); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: tip, + }) + .unwrap(); + } + + #[test] + fn note_push_fires_only_past_the_threshold() { + let (sender, mut receiver) = tokio::sync::mpsc::channel(16); + let handle = MaintenanceHandle { + trigger: Some(sender), + large_push: PushBytes::new(1_000), + }; + let did = RepoDid::new("did:plc:squid").unwrap(); + handle.note_push(&did, PushBytes::new(999)); + assert!(receiver.try_recv().is_err(), "small push is ignored"); + handle.note_push(&did, PushBytes::new(1_000)); + assert_eq!(receiver.try_recv().unwrap(), did, "large push triggers"); + } + + #[test] + fn disabled_handle_never_triggers() { + let did = RepoDid::new("did:plc:squid").unwrap(); + MaintenanceHandle::disabled().note_push(&did, PushBytes::new(u64::MAX)); + } + + #[tokio::test] + async fn the_lfs_interval_collects_without_a_push_trigger() { + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + use knot_lfs::{DiskStore, LfsOid, LfsStore, LfsStorePath, Pool}; + use knot_runtime::ManualClock; + use knot_types::UnixMicros; + + let scan = tempfile::tempdir().unwrap(); + let lfs_dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + seed_repo(&layout, &did); + + let store = Arc::new(DiskStore::open(LfsStorePath::new(lfs_dir.path())).unwrap()); + let body: &[u8] = b"unreferenced media reclaimed on the interval alone"; + let oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(body)); + let size = knot_lfs::ClaimedSize::new(body.len() as u64); + store + .put(Pool::Lfs, &did, &oid, size, &mut &body[..]) + .unwrap(); + + let real_micros = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros() as u64; + let future = ManualClock::new(UnixMicros::new(real_micros + 5 * 86_400 * 1_000_000)); + + let (scheduler, _handle) = Scheduler::new( + layout.clone(), + Arc::new(Fixed(vec![did.clone()])), + future, + options(), + std::time::Duration::from_secs(3_600), + PushBytes::new(1_000), + LfsGc::new( + Arc::clone(&store), + crate::lfs_grace( + crate::GcGrace::from_secs(86_400), + crate::ReflogRetention::from_secs(90 * 86_400), + ), + crate::SweepInterval::new(Duration::from_millis(40)), + ), + ); + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let task = tokio::spawn(scheduler.run(shutdown_rx)); + + let mut waited = 0; + while store.probe(Pool::Lfs, &did, &oid).unwrap().is_some() && waited < 200 { + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + waited += 1; + } + assert_eq!( + store.probe(Pool::Lfs, &did, &oid).unwrap(), + None, + "the lfs interval alone reclaimed the unreferenced, past-grace object" + ); + + shutdown_tx.send(true).unwrap(); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_triggered_push_collects_an_unreferenced_lfs_object() { + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + use knot_lfs::{DiskStore, LfsOid, LfsStore, LfsStorePath, Pool}; + use knot_runtime::ManualClock; + use knot_types::UnixMicros; + + let scan = tempfile::tempdir().unwrap(); + let lfs_dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + seed_repo(&layout, &did); + + let store = Arc::new(DiskStore::open(LfsStorePath::new(lfs_dir.path())).unwrap()); + let body: &[u8] = b"unreferenced media the sweep should reclaim"; + let oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(body)); + let size = knot_lfs::ClaimedSize::new(body.len() as u64); + store + .put(Pool::Lfs, &did, &oid, size, &mut &body[..]) + .unwrap(); + + let real_micros = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros() as u64; + let future = ManualClock::new(UnixMicros::new(real_micros + 5 * 86_400 * 1_000_000)); + + let (scheduler, handle) = Scheduler::new( + layout.clone(), + Arc::new(Fixed(vec![did.clone()])), + future, + options(), + std::time::Duration::from_secs(3_600), + PushBytes::new(1_000), + LfsGc::new( + Arc::clone(&store), + crate::lfs_grace( + crate::GcGrace::from_secs(86_400), + crate::ReflogRetention::from_secs(90 * 86_400), + ), + crate::SweepInterval::new(Duration::from_secs(3_600)), + ), + ); + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let task = tokio::spawn(scheduler.run(shutdown_rx)); + + handle.note_push(&did, PushBytes::new(10_000)); + + let mut waited = 0; + while store.probe(Pool::Lfs, &did, &oid).unwrap().is_some() && waited < 200 { + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + waited += 1; + } + assert_eq!( + store.probe(Pool::Lfs, &did, &oid).unwrap(), + None, + "the triggered gc reclaimed the unreferenced, past-grace object" + ); + + shutdown_tx.send(true).unwrap(); + task.await.unwrap(); + } + + struct Referenced(Vec); + + impl crate::AttachmentRoots for Referenced { + fn referenced(&self, _repo: &RepoDid) -> Result, knot_lfs::RootsDenied> { + Ok(HashSet::from_iter(self.0.iter().cloned())) + } + } + + struct DeniedRoots; + + impl crate::AttachmentRoots for DeniedRoots { + fn referenced(&self, _repo: &RepoDid) -> Result, knot_lfs::RootsDenied> { + Err(knot_lfs::RootsDenied::Untraversable) + } + } + + struct Sweeping { + store: Arc, + did: RepoDid, + shutdown: tokio::sync::watch::Sender, + task: tokio::task::JoinHandle<()>, + _scan: tempfile::TempDir, + _lfs: tempfile::TempDir, + } + + fn future_clock() -> ManualClock { + let real_micros = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros() as u64; + ManualClock::new(UnixMicros::new(real_micros + 5 * 86_400 * 1_000_000)) + } + + async fn sweeping( + grace_secs: u64, + clock: ManualClock, + roots: Arc, + ) -> Sweeping { + let scan = tempfile::tempdir().unwrap(); + let lfs = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + seed_repo(&layout, &did); + let store = Arc::new(DiskStore::open(LfsStorePath::new(lfs.path())).unwrap()); + let (scheduler, _handle) = Scheduler::new( + layout, + Arc::new(Fixed(vec![did.clone()])), + clock, + options(), + Duration::from_secs(3_600), + PushBytes::new(1_000), + LfsGc::new( + Arc::clone(&store), + crate::lfs_grace( + crate::GcGrace::from_secs(grace_secs), + crate::ReflogRetention::from_secs(90 * 86_400), + ), + crate::SweepInterval::new(Duration::from_millis(40)), + ), + ); + let scheduler = scheduler.with_attachment_roots(roots); + let (shutdown, shutdown_rx) = tokio::sync::watch::channel(false); + let task = tokio::spawn(scheduler.run(shutdown_rx)); + Sweeping { + store, + did, + shutdown, + task, + _scan: scan, + _lfs: lfs, + } + } + + fn seed(store: &DiskStore, pool: Pool, did: &RepoDid, body: &[u8]) -> LfsOid { + let oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(body)); + store + .put( + pool, + did, + &oid, + ClaimedSize::new(body.len() as u64), + &mut &body[..], + ) + .unwrap(); + oid + } + + #[tokio::test] + async fn attachment_sweep_takes_unreferenced_and_keeps_referenced() { + let referenced: &[u8] = b"a live record references this"; + let referenced_oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(referenced)); + let swept = sweeping( + 86_400, + future_clock(), + Arc::new(Referenced(vec![referenced_oid.clone()])), + ) + .await; + let loose_body: &[u8] = b"no record references this upload"; + let loose = seed(&swept.store, Pool::Attachments, &swept.did, loose_body); + seed(&swept.store, Pool::Attachments, &swept.did, referenced); + + let mut waited = 0; + while swept + .store + .probe(Pool::Attachments, &swept.did, &loose) + .unwrap() + .is_some() + && waited < 200 + { + tokio::time::sleep(Duration::from_millis(25)).await; + waited += 1; + } + assert_eq!( + swept + .store + .probe(Pool::Attachments, &swept.did, &loose) + .unwrap(), + None, + "The interval sweep reclaimed the unreferenced, past-grace attachment." + ); + assert_eq!( + swept + .store + .probe(Pool::Attachments, &swept.did, &referenced_oid) + .unwrap(), + Some(knot_lfs::LfsSize::new(referenced.len() as u64)), + "A live record's attachment survives the grace window." + ); + + swept.shutdown.send(true).unwrap(); + swept.task.await.unwrap(); + } + + #[test] + fn attachment_grace_keeps_sixty_second_minimum_under_shorter_setting() { + assert_eq!( + attachment_grace(Duration::from_secs(0)), + MIN_ATTACHMENT_GRACE, + "The minimum keeps a write-admitted blob through the gap between admission \ + and commit." + ); + assert_eq!( + attachment_grace(Duration::from_secs(3_600)), + Duration::from_secs(3_600) + ); + } + + #[tokio::test] + async fn failed_roots_query_keeps_every_attachment() { + let swept = sweeping(86_400, future_clock(), Arc::new(DeniedRoots)).await; + let oid = seed( + &swept.store, + Pool::Attachments, + &swept.did, + b"unreferenced, but the roots won't say so", + ); + + tokio::time::sleep(Duration::from_millis(250)).await; + assert!( + swept + .store + .probe(Pool::Attachments, &swept.did, &oid) + .unwrap() + .is_some(), + "Failed roots query keeps every attachment rather than guessing." + ); + + swept.shutdown.send(true).unwrap(); + swept.task.await.unwrap(); + } + + #[tokio::test] + async fn a_triggered_repo_is_maintained() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + seed_repo(&layout, &did); + + let lfs_dir = tempfile::tempdir().unwrap(); + let store = Arc::new(DiskStore::open(LfsStorePath::new(lfs_dir.path())).unwrap()); + let (scheduler, handle) = Scheduler::new( + layout.clone(), + Arc::new(Fixed(vec![did.clone()])), + SystemClock, + options(), + std::time::Duration::from_secs(3_600), + PushBytes::new(1_000), + LfsGc::new( + Arc::clone(&store), + crate::lfs_grace( + crate::GcGrace::from_secs(86_400), + crate::ReflogRetention::from_secs(90 * 86_400), + ), + crate::SweepInterval::new(Duration::from_secs(3_600)), + ), + ); + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let task = tokio::spawn(scheduler.run(shutdown_rx)); + + handle.note_push(&did, PushBytes::new(10_000)); + + let graph = layout + .open(&did) + .unwrap() + .objects_dir() + .join("info/commit-graph"); + let mut waited = 0; + while !graph.exists() && waited < 200 { + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + waited += 1; + } + assert!(graph.exists(), "triggered repo got commit-graph"); + + shutdown_tx.send(true).unwrap(); + task.await.unwrap(); + } +} diff --git a/knot2/crates/knot-maintenance/src/test_support.rs b/knot2/crates/knot-maintenance/src/test_support.rs new file mode 100644 index 0000000000000000000000000000000000000000..ba00e260d8e13c0d9c041a936b0973ac256e0a52 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/test_support.rs @@ -0,0 +1,57 @@ +use knot_git::{EntryKind, Identity, NewCommit, Repo, StagedAction, StagedChange}; +use knot_types::{AuthorName, Email, ObjectFormat, Oid, UnixSeconds}; + +use crate::{ + FullRepackInterval, GeometricFactor, ObjectCount, Options, PruneGrace, ReflogRetention, +}; + +pub fn identity() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } +} + +pub fn empty_tree(format: ObjectFormat) -> Oid { + Oid::from(gix::ObjectId::empty_tree(format.kind())) +} + +pub fn commit_on(repo: &Repo, empty_tree: Oid, parents: Vec, marker: &str) -> Oid { + let tree = repo + .write_staged_tree( + empty_tree, + &[StagedChange { + path: knot_types::RepoPath::new(format!("{marker}.txt")).unwrap(), + action: StagedAction::Put { + content: marker.as_bytes().to_vec(), + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + repo.write_commit(&NewCommit { + tree, + parents, + author: identity(), + committer: identity(), + message: marker.to_string(), + extra_headers: Vec::new(), + }) + .unwrap() +} + +pub fn options() -> Options { + Options { + repack_max_objects: ObjectCount::new(1_000_000), + geometric_factor: GeometricFactor::full_repack(), + loose_threshold: ObjectCount::new(0), + prune_grace: PruneGrace::from_secs(0), + full_repack_interval: FullRepackInterval::from_secs(432_000), + reflog_floor: ReflogRetention::from_secs(i64::MAX as u64 / 4), + commit_graph: true, + multi_pack_index: true, + bitmap: true, + } +} diff --git a/knot2/crates/knot-maintenance/tests/chaos.rs b/knot2/crates/knot-maintenance/tests/chaos.rs new file mode 100644 index 0000000000000000000000000000000000000000..b76d0ff3edf27745d332b4c28c8d6ce1d7aaddd6 --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/chaos.rs @@ -0,0 +1,237 @@ +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, +}; +use knot_maintenance::{ + FullRepackInterval, GeometricFactor, Options, PruneGrace, ReflogRetention, run_repo, +}; +use knot_types::{AuthorName, BranchName, Email, ObjectCount, Oid, RefName, RepoDid, UnixSeconds}; + +const DID: &str = "did:plc:squid"; +const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; +const BLOB_BYTES: usize = 8 * 1024 * 1024; +const HISTORY: u32 = 40; +const NOW_SECONDS: UnixSeconds = UnixSeconds::new(1_700_000_500); + +fn incompressible(len: usize) -> Vec { + let mut state = 0x2545_f491_4f6c_dd1du64; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() +} + +fn identity() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } +} + +fn options() -> Options { + Options { + repack_max_objects: ObjectCount::new(5_000_000), + geometric_factor: GeometricFactor::full_repack(), + loose_threshold: ObjectCount::new(0), + prune_grace: PruneGrace::from_secs(0), + full_repack_interval: FullRepackInterval::from_secs(432_000), + reflog_floor: ReflogRetention::from_secs(i64::MAX as u64 / 4), + commit_graph: true, + multi_pack_index: true, + bitmap: true, + } +} + +fn build_template(scan: &Path) { + let layout = Layout::new(scan).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new(DID).unwrap(); + let repo = layout.create(&did).unwrap(); + let main = RefName::new("refs/heads/main").unwrap(); + let empty = Oid::from_hex(EMPTY_TREE).unwrap(); + + let big_tree = repo + .write_staged_tree( + empty, + &[StagedChange { + path: knot_types::RepoPath::new("big.bin").unwrap(), + action: StagedAction::Put { + content: incompressible(BLOB_BYTES), + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + let mut tip = repo + .write_commit(&NewCommit { + tree: big_tree, + parents: Vec::new(), + author: identity(), + committer: identity(), + message: "big".to_string(), + extra_headers: Vec::new(), + }) + .unwrap(); + (0..HISTORY).for_each(|index| { + let tree = repo + .write_staged_tree( + empty, + &[StagedChange { + path: knot_types::RepoPath::new(format!("file{index}.txt")).unwrap(), + action: StagedAction::Put { + content: format!("contents {index}").into_bytes(), + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + let next = repo + .write_commit(&NewCommit { + tree, + parents: vec![tip], + author: identity(), + committer: identity(), + message: format!("commit {index}"), + extra_headers: Vec::new(), + }) + .unwrap(); + let update = match index { + 0 => RefUpdate::Create { + name: main.clone(), + new: next, + }, + _ => RefUpdate::Update { + name: main.clone(), + old: tip, + new: next, + }, + }; + repo.update_ref(&update).unwrap(); + tip = next; + }); +} + +fn copy_tree(src: &Path, dst: &Path) { + walkdir::WalkDir::new(src) + .into_iter() + .filter_map(Result::ok) + .for_each(|entry| { + let relative = entry.path().strip_prefix(src).unwrap(); + let target = dst.join(relative); + if entry.file_type().is_dir() { + std::fs::create_dir_all(&target).unwrap(); + } else { + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::copy(entry.path(), &target).unwrap(); + } + }); +} + +fn spawn_worker(scan: &Path) -> std::process::Child { + Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "chaos_maintenance_worker", "--nocapture"]) + .env("KNOT_CHAOS_ROLE", "worker") + .env("KNOT_CHAOS_SCAN", scan) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn chaos worker") +} + +fn fsck_clean(bare: &Path) -> Result<(), String> { + knot_fixtures::fsck(bare) +} + +fn main_tip(scan: &Path) -> Option { + let layout = Layout::new(scan); + let repo = layout + .open(&RepoDid::new(DID).unwrap()) + .expect("repo must reopen cleanly after kill"); + repo.find_ref(&RefName::new("refs/heads/main").unwrap()) + .expect("references must be readable after kill") +} + +#[test] +fn chaos_maintenance_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("worker") { + return; + } + let scan = std::env::var("KNOT_CHAOS_SCAN").unwrap(); + let layout = Layout::new(&scan); + let repo = layout.open(&RepoDid::new(DID).unwrap()).unwrap(); + let _ = run_repo(&repo, NOW_SECONDS, &options()); +} + +#[test] +fn kill9_during_maintenance_leaves_a_consistent_repo() { + let scratch = tempfile::tempdir().unwrap(); + let template = scratch.path().join("template"); + build_template(&template); + let did = RepoDid::new(DID).unwrap(); + let tip = main_tip(&template).expect("template has a main tip"); + + let warm = scratch.path().join("warm"); + copy_tree(&template, &warm); + let started = Instant::now(); + let mut child = spawn_worker(&warm); + child.wait().unwrap(); + let full = started.elapsed(); + let warm_repo = Repo::open(Layout::new(&warm).repo_path(&did).unwrap()).unwrap(); + assert!( + warm_repo + .git() + .git_dir() + .join("objects/info/commit-graph") + .exists(), + "uninterrupted maintenance run writes commit-graph" + ); + + let fractions = [0.20, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 1.10]; + let delays: Vec = std::iter::once(Duration::from_millis(1)) + .chain(std::iter::once(Duration::from_millis(3))) + .chain(fractions.iter().map(|fraction| full.mul_f64(*fraction))) + .chain(std::iter::once(full.mul_f64(2.0))) + .collect(); + + delays.iter().enumerate().for_each(|(trial, delay)| { + let scan = scratch.path().join(format!("scan-{trial}")); + copy_tree(&template, &scan); + let mut child = spawn_worker(&scan); + std::thread::sleep(*delay); + let _ = child.kill(); + child.wait().unwrap(); + + let bare = Layout::new(&scan).repo_path(&did).unwrap(); + fsck_clean(&bare).unwrap_or_else(|errors| { + panic!("trial {trial}: killed maintenance run left corrupt repo:\n{errors}") + }); + assert_eq!( + main_tip(&scan), + Some(tip), + "trial {trial}: maintenance never changes branch value, so main must still resolve to tip" + ); + + let recovered = Repo::open(&bare).unwrap(); + run_repo(&recovered, NOW_SECONDS, &options()).unwrap_or_else(|error| { + panic!("trial {trial}: maintenance must self-heal after crash, got {error}") + }); + fsck_clean(&bare).unwrap_or_else(|errors| { + panic!("trial {trial}: self-heal pass left corrupt repo:\n{errors}") + }); + assert_eq!( + main_tip(&scan), + Some(tip), + "trial {trial}: recovered repo still resolves main to tip" + ); + }); +} diff --git a/knot2/crates/knot-maintenance/tests/commitgraph.rs b/knot2/crates/knot-maintenance/tests/commitgraph.rs new file mode 100644 index 0000000000000000000000000000000000000000..12b75676e2a247e0c2b5dcd2796e3683d8ffb401 --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/commitgraph.rs @@ -0,0 +1,325 @@ +use std::collections::HashMap; +use std::path::Path; + +use knot_git::Repo; +use knot_maintenance::run_repo; + +mod common; +use common::{git_available, now, options}; + +fn skip() -> bool { + if git_available() { + return false; + } + eprintln!("skipping commit-graph differential: git unavailable"); + true +} + +fn git_at(dir: &Path, date: i64, args: &[&str]) { + let stamp = format!("{date} +0000"); + let out = knot_fixtures::command_at(dir, &stamp) + .args(args) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); +} + +fn git(dir: &Path, args: &[&str]) { + git_at(dir, 1_700_000_000, args); +} + +fn write_file(dir: &Path, rel: &str, contents: &str) { + let path = dir.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, contents).unwrap(); +} + +fn commit(dir: &Path, rel: &str, contents: &str, message: &str) { + write_file(dir, rel, contents); + git(dir, &["add", "-A"]); + git(dir, &["commit", "-m", message]); +} + +fn seed_history(dir: &Path, format: &str) { + git(dir, &["init", "--object-format", format, "-b", "main"]); + write_file(dir, "dir1/a.txt", "a1"); + write_file(dir, "dir1/sub/b.txt", "b1"); + commit(dir, "c.txt", "c1", "root"); + commit(dir, "dir1/sub/b.txt", "b2", "edit nested"); + git(dir, &["checkout", "-b", "feature"]); + commit(dir, "c.txt", "c2", "feature edit"); + git(dir, &["checkout", "main"]); + commit(dir, "dir2/d.txt", "d1", "add dir2"); + git(dir, &["merge", "--no-ff", "-m", "merge feature", "feature"]); + git(dir, &["tag", "v1"]); +} + +fn read_chunks(bytes: &[u8]) -> HashMap<[u8; 4], Vec> { + let count = bytes[6] as usize; + let table = &bytes[8..8 + (count + 1) * 12]; + let entry = |index: usize| -> ([u8; 4], u64) { + let base = index * 12; + let id = table[base..base + 4].try_into().unwrap(); + ( + id, + u64::from_be_bytes(table[base + 4..base + 12].try_into().unwrap()), + ) + }; + (0..count) + .map(|index| { + let (id, start) = entry(index); + let (_, end) = entry(index + 1); + (id, bytes[start as usize..end as usize].to_vec()) + }) + .collect() +} + +fn graph_verify(root: &Path) -> (bool, String) { + let out = knot_fixtures::command(root) + .args(["commit-graph", "verify"]) + .output() + .expect("git commit-graph verify runs"); + ( + out.status.success(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +fn path_log(dir: &Path, path: &str, use_graph: bool) -> String { + let out = knot_fixtures::command(dir) + .args([ + "-c", + &format!("core.commitGraph={use_graph}"), + "-c", + "commitGraph.readChangedPaths=true", + "log", + "--format=%H", + "--", + path, + ]) + .output() + .expect("git log runs"); + assert!(out.status.success()); + String::from_utf8(out.stdout).unwrap() +} + +fn build_and_compare(root: &Path, format: &str, chunks: &[&[u8; 4]]) { + git( + root, + &[ + "-c", + "commitGraph.changedPathsVersion=2", + "commit-graph", + "write", + "--reachable", + "--changed-paths", + ], + ); + let graph_file = root.join(".git/objects/info/commit-graph"); + let canonical = read_chunks(&std::fs::read(&graph_file).unwrap()); + std::fs::remove_file(&graph_file).unwrap(); + + assert!( + run_repo(&Repo::open(root).unwrap(), now(), &options()) + .unwrap() + .commit_graph, + "knot wrote a graph ({format})" + ); + let ours = read_chunks(&std::fs::read(&graph_file).unwrap()); + chunks.iter().for_each(|id| { + assert_eq!( + ours.get(*id), + canonical.get(*id), + "chunk {} differs ({format})", + String::from_utf8_lossy(*id) + ); + }); + let (ok, stderr) = graph_verify(root); + assert!(ok, "git commit-graph verify failed ({format}): {stderr}"); +} + +fn check_against_git(format: &str) { + if skip() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + seed_history(root, format); + build_and_compare(root, format, &[b"OIDL", b"CDAT", b"GDA2", b"BIDX", b"BDAT"]); + + ["dir1/sub/b.txt", "dir1/sub", "dir1", "c.txt", "dir2/d.txt"] + .iter() + .for_each(|path| { + assert_eq!( + path_log(root, path, true), + path_log(root, path, false), + "changed-path bloom altered `git log -- {path}` ({format})" + ); + }); +} + +#[test] +fn matches_canonical_git_sha1() { + check_against_git("sha1"); +} + +#[test] +fn matches_canonical_git_sha256() { + check_against_git("sha256"); +} + +#[test] +fn writing_the_graph_clears_a_pre_existing_split_chain() { + if skip() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + seed_history(root, "sha1"); + + git(root, &["commit-graph", "write", "--reachable", "--split"]); + let chain = root.join(".git/objects/info/commit-graphs"); + assert!(chain.exists(), "git wrote a split commit-graph chain"); + commit(root, "c.txt", "c3", "after the chain"); + + assert!( + run_repo(&Repo::open(root).unwrap(), now(), &options()) + .unwrap() + .commit_graph, + "knot wrote a monolithic graph" + ); + assert!( + !chain.exists(), + "the stale split chain is removed so it cannot shadow the fresh graph" + ); + assert!(root.join(".git/objects/info/commit-graph").exists()); + let (ok, stderr) = graph_verify(root); + assert!( + ok, + "git verify passes after the chain is replaced: {stderr}" + ); +} + +#[test] +fn large_filter_sentinel_matches_git_when_dirs_overflow_the_limit() { + if skip() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + git(root, &["init", "--object-format", "sha1", "-b", "main"]); + (0..200).for_each(|i| write_file(root, &format!("a{i}/b{i}/c.txt"), "x")); + git(root, &["add", "-A"]); + git(root, &["commit", "-m", "wide refactor"]); + build_and_compare(root, "sha1", &[b"BIDX", b"BDAT"]); +} + +#[test] +fn corrected_date_overflow_matches_git() { + if skip() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + git(root, &["init", "--object-format", "sha1", "-b", "main"]); + write_file(root, "a.txt", "1"); + git_at(root, 4_000_000_000, &["add", "-A"]); + git_at(root, 4_000_000_000, &["commit", "-m", "far-future root"]); + write_file(root, "a.txt", "2"); + git_at(root, 1_000_000_000, &["add", "-A"]); + git_at(root, 1_000_000_000, &["commit", "-m", "past child"]); + build_and_compare(root, "sha1", &[b"CDAT", b"GDA2", b"GDO2"]); +} + +#[test] +fn changed_path_filter_matches_git_for_high_byte_names() { + if skip() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + git(root, &["init", "--object-format", "sha1", "-b", "main"]); + write_file(root, "café/résumé.txt", "1"); + commit(root, "naïve.md", "2", "non-ascii paths"); + commit(root, "café/résumé.txt", "2", "edit non-ascii"); + build_and_compare(root, "sha1", &[b"BIDX", b"BDAT"]); +} + +fn chain_layers(root: &Path) -> usize { + std::fs::read_to_string(root.join(".git/objects/info/commit-graphs/commit-graph-chain")) + .map_or(0, |chain| chain.lines().count()) +} + +fn maintain(root: &Path, at: knot_types::UnixSeconds) -> bool { + run_repo(&Repo::open(root).unwrap(), at, &options()) + .unwrap() + .commit_graph +} + +#[test] +fn later_passes_write_layers_and_the_full_repack_collapses_them() { + if skip() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let single = root.join(".git/objects/info/commit-graph"); + seed_history(root, "sha1"); + + assert!(maintain(root, now())); + assert!(single.exists()); + assert_eq!(chain_layers(root), 0); + + commit(root, "dir1/a.txt", "a2", "first layer"); + assert!(maintain(root, now())); + assert!(!single.exists(), "the single graph migrates into the chain"); + assert_eq!(chain_layers(root), 2); + let (ok, stderr) = graph_verify(root); + assert!(ok, "verify after the first layer: {stderr}"); + + commit(root, "dir2/d.txt", "d2", "second layer"); + assert!(maintain(root, now())); + assert_eq!(chain_layers(root), 2, "the small top layers merge"); + let (ok, stderr) = graph_verify(root); + assert!(ok, "verify after the second layer: {stderr}"); + ["dir1/a.txt", "dir2", "c.txt"].iter().for_each(|path| { + assert_eq!(path_log(root, path, true), path_log(root, path, false)); + }); + + let later = knot_types::UnixSeconds::new(now().get() + 432_000); + assert!(maintain(root, later)); + assert!(single.exists()); + assert!(!root.join(".git/objects/info/commit-graphs").exists()); + let (ok, stderr) = graph_verify(root); + assert!(ok, "verify after the full repack: {stderr}"); +} + +#[test] +fn gate_skips_graphed_refs_and_annotated_tags() { + if skip() { + return; + } + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + seed_history(root, "sha1"); + let misses = || { + Repo::open(root) + .unwrap() + .commit_graph_misses_a_tip() + .unwrap() + }; + assert!(misses(), "no graph yet"); + git(root, &["commit-graph", "write", "--reachable"]); + assert!(!misses(), "every tip is graphed"); + git( + root, + &["tag", "-a", "-m", "annotated", "v1-annotated", "v1"], + ); + assert!(!misses(), "an annotated tag peels to a graphed commit"); + commit(root, "c.txt", "c3", "ungraphed"); + assert!(misses(), "a new tip is missing from the graph"); +} diff --git a/knot2/crates/knot-maintenance/tests/common/mod.rs b/knot2/crates/knot-maintenance/tests/common/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..2270ef9cec35a7ef127252080a881e5ef9e45d3a --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/common/mod.rs @@ -0,0 +1,227 @@ +#![allow(dead_code, unused_imports)] + +use std::collections::BTreeSet; +use std::ops::Range; +use std::path::Path; + +use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, +}; +use knot_maintenance::{FullRepackInterval, GeometricFactor, Options, PruneGrace, ReflogRetention}; +use knot_types::{ + AuthorName, BranchName, Email, ObjectCount, ObjectFormat, Oid, RefName, RepoDid, UnixSeconds, +}; + +pub const EMPTY_TREE_SHA1: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; +pub const EMPTY_TREE_SHA256: &str = + "6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321"; + +pub fn empty_tree(format: ObjectFormat) -> &'static str { + if format == ObjectFormat::SHA256 { + EMPTY_TREE_SHA256 + } else { + EMPTY_TREE_SHA1 + } +} + +pub fn now() -> UnixSeconds { + UnixSeconds::new(1_700_000_500) +} + +pub use knot_fixtures::available as git_available; + +pub fn identity() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } +} + +pub fn options() -> Options { + Options { + repack_max_objects: ObjectCount::new(1_000_000), + geometric_factor: GeometricFactor::full_repack(), + loose_threshold: ObjectCount::new(0), + prune_grace: PruneGrace::from_secs(0), + full_repack_interval: FullRepackInterval::from_secs(432_000), + reflog_floor: ReflogRetention::from_secs(i64::MAX as u64 / 4), + commit_graph: true, + multi_pack_index: true, + bitmap: true, + } +} + +pub fn create_repo(scan: &Path, format: ObjectFormat, did: &str) -> Repo { + Layout::new(scan) + .with_object_format(format) + .with_default_branch(BranchName::new("main").unwrap()) + .create(&RepoDid::new(did).unwrap()) + .unwrap() +} + +pub fn commit_on(repo: &Repo, empty_tree: &str, body: u8, parents: Vec) -> Oid { + let tree = repo + .write_staged_tree( + Oid::from_hex(empty_tree).unwrap(), + &[StagedChange { + path: knot_types::RepoPath::new(format!("file{body}.txt")).unwrap(), + action: StagedAction::Put { + content: vec![body, body, body], + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + repo.write_commit(&NewCommit { + tree, + parents, + author: identity(), + committer: identity(), + message: format!("commit {body}"), + extra_headers: Vec::new(), + }) + .unwrap() +} + +pub fn commit(repo: &Repo, body: u8, parents: Vec) -> Oid { + commit_on(repo, EMPTY_TREE_SHA1, body, parents) +} + +pub fn chain(repo: &Repo, empty_tree: &str, bodies: Range, start: Option) -> Oid { + bodies + .fold(start, |parent, body| { + let parents = parent.map(|tip| vec![tip]).unwrap_or_default(); + Some(commit_on(repo, empty_tree, body, parents)) + }) + .expect("a non-empty body range yields a tip") +} + +pub fn set_ref(repo: &Repo, name: &str, new: Oid) { + let refname = RefName::new(name).unwrap(); + let update = match repo.find_ref(&refname).unwrap() { + Some(old) => RefUpdate::Update { + name: refname, + old, + new, + }, + None => RefUpdate::Create { name: refname, new }, + }; + repo.update_ref(&update).unwrap(); +} + +pub fn delete_ref(repo: &Repo, name: &str) { + let refname = RefName::new(name).unwrap(); + let old = repo.find_ref(&refname).unwrap().unwrap(); + repo.update_ref(&RefUpdate::Delete { name: refname, old }) + .unwrap(); +} + +pub fn set_reflog_seconds(repo: &Repo, name: &str, seconds: &[i64]) { + let path = repo.git().git_dir().join("logs").join(name); + let text = std::fs::read_to_string(&path).expect("reflog file exists"); + let lines: Vec<&str> = text.lines().collect(); + assert_eq!( + lines.len(), + seconds.len(), + "set_reflog_seconds needs one timestamp per reflog line" + ); + let rewritten = lines + .iter() + .zip(seconds) + .map(|(line, secs)| rewrite_reflog_seconds(line, *secs)) + .collect::>() + .join("\n"); + std::fs::write(&path, format!("{rewritten}\n")).expect("rewrite reflog"); +} + +fn rewrite_reflog_seconds(line: &str, secs: i64) -> String { + let (meta, message) = line + .split_once('\t') + .expect("reflog line has a message tab"); + let tokens: Vec<&str> = meta.split(' ').collect(); + let tz = tokens.last().expect("reflog line has a timezone"); + let head = tokens[..tokens.len() - 2].join(" "); + format!("{head} {secs} {tz}\t{message}") +} + +pub fn git(repo: &Repo, args: &[&str]) -> (bool, String) { + let out = knot_fixtures::command(repo.git().git_dir()) + .args(args) + .output() + .expect("git runs"); + ( + out.status.success(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +pub fn fsck(repo: &Repo) -> (bool, String) { + git(repo, &["fsck", "--no-dangling", "--no-progress"]) +} + +pub fn fsck_clean(repo: &Repo) -> bool { + fsck(repo).0 +} + +pub fn assert_fsck_clean(repo: &Repo) { + let (clean, stderr) = fsck(repo); + assert!(clean, "fsck failed: {stderr}"); +} + +pub fn reachable_objects(repo: &Repo) -> BTreeSet { + let out = knot_fixtures::command(repo.git().git_dir()) + .args(["rev-list", "--objects", "--all"]) + .output() + .expect("git rev-list runs"); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .map(str::to_string) + .collect() +} + +pub fn midx_verifies(repo: &Repo) -> Option<(bool, String)> { + let midx = repo.git().git_dir().join("objects/pack/multi-pack-index"); + midx.exists() + .then(|| git(repo, &["multi-pack-index", "verify"])) +} + +fn pack_entries(repo: &Repo) -> impl Iterator { + std::fs::read_dir(repo.git().git_dir().join("objects/pack")) + .into_iter() + .flatten() + .filter_map(Result::ok) + .map(|entry| entry.path()) +} + +pub fn idx_stems(repo: &Repo) -> BTreeSet { + pack_entries(repo) + .filter(|path| path.extension().is_some_and(|ext| ext == "idx")) + .filter_map(|path| { + path.file_stem() + .and_then(|stem| stem.to_str()) + .map(str::to_string) + }) + .collect() +} + +pub fn has_cruft_pack(repo: &Repo) -> bool { + pack_entries(repo).any(|path| path.extension().is_some_and(|ext| ext == "mtimes")) +} + +pub fn has_bitmap(repo: &Repo) -> bool { + pack_entries(repo).any(|path| path.extension().is_some_and(|ext| ext == "bitmap")) +} + +pub fn has_midx_bitmap(repo: &Repo) -> bool { + pack_entries(repo) + .filter_map(|path| { + path.file_name() + .and_then(|n| n.to_str()) + .map(str::to_string) + }) + .any(|name| name.starts_with("multi-pack-index-") && name.ends_with(".bitmap")) +} diff --git a/knot2/crates/knot-maintenance/tests/cruft.rs b/knot2/crates/knot-maintenance/tests/cruft.rs new file mode 100644 index 0000000000000000000000000000000000000000..eea9514f9bcec4fdca8f865caf2309078d631977 --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/cruft.rs @@ -0,0 +1,134 @@ +use knot_git::Repo; +use knot_maintenance::{Options, PruneGrace, ReflogRetention, run_repo}; +use knot_types::{ObjectFormat, UnixSeconds}; + +mod common; +use common::{ + EMPTY_TREE_SHA1, assert_fsck_clean, commit_on, create_repo, delete_ref, empty_tree, + has_cruft_pack, now, options, set_ref, set_reflog_seconds, +}; + +fn opts(grace: PruneGrace) -> Options { + Options { + prune_grace: grace, + commit_graph: false, + multi_pack_index: false, + bitmap: false, + ..options() + } +} + +fn retain_repeat_then_expire(format: ObjectFormat) { + let scan = tempfile::tempdir().unwrap(); + let empty = empty_tree(format); + let repo = create_repo(scan.path(), format, "did:plc:cuttle"); + + let base = commit_on(&repo, empty, 0, Vec::new()); + let doomed = commit_on(&repo, empty, 1, vec![base]); + set_ref(&repo, "refs/heads/main", base); + set_ref(&repo, "refs/heads/feature", doomed); + run_repo(&repo, now(), &opts(PruneGrace::from_secs(86_400))).unwrap(); + assert!(repo.contains(doomed), "doomed is packed while reachable"); + + delete_ref(&repo, "refs/heads/feature"); + set_ref( + &repo, + "refs/heads/main", + commit_on(&repo, empty, 2, vec![base]), + ); + + let interval_later = |intervals: i64| now().saturating_add_secs(intervals * 432_000); + let retained = run_repo( + &repo, + interval_later(1), + &opts(PruneGrace::from_secs(86_400)), + ) + .unwrap(); + assert!( + retained.prune.crufted.get() >= 1, + "young unreachable objects are crufted, not dropped" + ); + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + has_cruft_pack(&reopened) && reopened.contains(doomed), + "a cruft pack holds the young object" + ); + assert_fsck_clean(&reopened); + + assert!( + run_repo( + &reopened, + interval_later(2), + &opts(PruneGrace::from_secs(86_400)) + ) + .unwrap() + .prune + .ran + ); + let recycled = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + recycled.contains(doomed), + "the young object survives a repeat cruft cycle within grace" + ); + assert_fsck_clean(&recycled); + + assert!( + run_repo( + &recycled, + interval_later(3), + &opts(PruneGrace::from_secs(0)) + ) + .unwrap() + .prune + .ran + ); + let settled = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + !settled.contains(doomed), + "the past-grace object is finally dropped" + ); + assert!( + settled.contains(base) && !has_cruft_pack(&settled), + "base survives, no cruft lingers" + ); + assert_fsck_clean(&settled); +} + +#[test] +fn cruft_retains_survives_a_repeat_cycle_then_expires_sha1() { + retain_repeat_then_expire(ObjectFormat::SHA1); +} + +#[test] +fn cruft_retains_survives_a_repeat_cycle_then_expires_sha256() { + retain_repeat_then_expire(ObjectFormat::SHA256); +} +#[test] +fn a_rewound_away_commit_survives_on_its_reflog_old_pointer_alone() { + let scan = tempfile::tempdir().unwrap(); + let repo = create_repo(scan.path(), ObjectFormat::SHA1, "did:plc:periwinkle"); + + let lost = commit_on(&repo, EMPTY_TREE_SHA1, 7, Vec::new()); + set_ref(&repo, "refs/heads/main", lost); + let survivor = commit_on(&repo, EMPTY_TREE_SHA1, 9, Vec::new()); + set_ref(&repo, "refs/heads/main", survivor); + + set_reflog_seconds(&repo, "refs/heads/main", &[1_000_000, 2_000_000_000]); + + let opts = Options { + reflog_floor: ReflogRetention::from_secs(500_000_000), + ..opts(PruneGrace::from_secs(0)) + }; + run_repo(&repo, UnixSeconds::new(2_000_000_000), &opts).unwrap(); + + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + reopened.contains(lost), + "a force-rewound commit reachable only through a reflog old-pointer must not be pruned" + ); + assert!( + reopened.contains(survivor), + "the live tip survives the prune" + ); + assert_fsck_clean(&reopened); +} diff --git a/knot2/crates/knot-maintenance/tests/engine.rs b/knot2/crates/knot-maintenance/tests/engine.rs new file mode 100644 index 0000000000000000000000000000000000000000..03c778721678bea8553fb9133fb39617f6eef74a --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/engine.rs @@ -0,0 +1,240 @@ +use std::collections::HashSet; + +use knot_git::Repo; +use knot_maintenance::{MidxStatus, Options, PruneGrace, RepackStatus, run_repo}; +use knot_types::{ObjectFormat, Oid}; + +mod common; +use common::{ + commit, create_repo, delete_ref, git, has_bitmap, has_midx_bitmap, now, options, set_ref, +}; + +fn create(scan: &std::path::Path, did: &str) -> Repo { + create_repo(scan, ObjectFormat::SHA1, did) +} + +fn loose_count(repo: &Repo) -> usize { + walkdir::WalkDir::new(repo.git().git_dir().join("objects")) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .filter(|e| { + e.path() + .parent() + .and_then(|p| p.file_name()) + .and_then(|n| n.to_str()) + .is_some_and(|n| n.len() == 2) + }) + .count() +} + +#[test] +fn a_single_pack_maintenance_pass_packs_graphs_bitmaps_prunes_then_settles() { + let scan = tempfile::tempdir().unwrap(); + let repo = create(scan.path(), "did:plc:squid"); + + let base = commit(&repo, 0, Vec::new()); + let main_tip = commit(&repo, 1, vec![base]); + set_ref(&repo, "refs/heads/main", main_tip); + set_ref(&repo, "refs/heads/feature", commit(&repo, 2, vec![base])); + let orphan = commit(&repo, 9, vec![main_tip]); + + let info = repo.git().git_dir().join("objects/info"); + std::fs::create_dir_all(&info).unwrap(); + let leaked = info.join("commit-graph.knot-tmp.999999"); + std::fs::write(&leaked, b"a graph write that a kill -9 interrupted").unwrap(); + let long_ago = std::time::SystemTime::now() - std::time::Duration::from_secs(7 * 3600); + std::fs::File::options() + .write(true) + .open(&leaked) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(long_ago)) + .unwrap(); + assert!(loose_count(&repo) >= 4 && repo.contains(orphan)); + + let report = run_repo(&repo, now(), &options()).unwrap(); + assert_eq!(report.repack.status, RepackStatus::Repacked); + assert!( + report.repack.removed_loose.get() >= 4, + "loose objects folded into the pack" + ); + assert!(report.commit_graph && info.join("commit-graph").exists()); + assert!( + !leaked.exists(), + "maintenance sweeps a crashed graph write's temp once it is too old to still have a writer" + ); + assert!(report.bitmap && has_bitmap(&repo)); + let (ok, stderr) = git(&repo, &["rev-list", "--test-bitmap", "main"]); + assert!(ok, "canonical git accepts our bitmap: {stderr}"); + assert!(report.packed_refs.packed >= 1); + assert!(report.reflog.files >= 1, "the first pass expires reflogs"); + assert!( + report.prune.ran && report.prune.removed.get() >= 1, + "the orphan is pruned" + ); + + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + assert!(!reopened.contains(orphan) && reopened.contains(main_tip)); + let (clean, stderr) = git(&reopened, &["fsck", "--no-progress"]); + assert!(clean, "fsck-clean after the pass: {stderr}"); + + let settled = run_repo(&reopened, now(), &options()).unwrap(); + assert_eq!(settled.repack.status, RepackStatus::Clean); + assert!( + !settled.prune.ran && !settled.commit_graph && !settled.bitmap, + "a settled repo is a no-op" + ); + assert_eq!(settled.packed_refs.packed, 0); + assert_eq!( + settled.reflog.files, 0, + "reflogs wait for the next full repack" + ); + + set_ref( + &reopened, + "refs/heads/main", + commit(&reopened, 10, vec![main_tip]), + ); + let moved = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + run_repo(&moved, now(), &options()).unwrap().commit_graph, + "a tip outside the graph rewrites it" + ); +} + +#[test] +fn the_written_graph_verifies_and_accelerates_selection_through_an_octopus_merge() { + let scan = tempfile::tempdir().unwrap(); + let repo = create(scan.path(), "did:plc:cuttle"); + + let a = commit(&repo, 1, Vec::new()); + let b = commit(&repo, 2, Vec::new()); + let c = commit(&repo, 3, Vec::new()); + let pair = commit(&repo, 4, vec![a, b]); + let octopus = commit(&repo, 5, vec![a, b, c]); + let tip = commit(&repo, 6, vec![pair, octopus]); + set_ref(&repo, "refs/heads/main", tip); + + let info = repo.git().git_dir().join("objects/info"); + let has_graph = || info.join("commit-graph").exists() || info.join("commit-graphs").exists(); + let closure = |r: &Repo| -> HashSet { + r.select_pack_objects(knot_git::Wants::new(&[tip]), knot_git::Haves::new(&[])) + .unwrap() + .into_iter() + .collect() + }; + let decode_closure = closure(&repo); + assert!(!has_graph()); + + assert!(run_repo(&repo, now(), &options()).unwrap().commit_graph && has_graph()); + let (ok, stderr) = git(&repo, &["commit-graph", "verify"]); + assert!( + ok, + "git accepts the hand-written graph with an octopus: {stderr}" + ); + + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + assert_eq!( + closure(&reopened), + decode_closure, + "graph-accelerated selection equals the decode walk" + ); +} + +#[test] +fn commit_graph_knob_lifecycle_skips_backfills_and_sweeps() { + let scan = tempfile::tempdir().unwrap(); + let repo = create(scan.path(), "did:plc:limpet"); + + let base = commit(&repo, 0, Vec::new()); + set_ref(&repo, "refs/heads/main", commit(&repo, 1, vec![base])); + + let off = Options { + commit_graph: false, + ..options() + }; + let info = repo.git().git_dir().join("objects/info"); + let has_graph = || info.join("commit-graph").exists() || info.join("commit-graphs").exists(); + + let first = run_repo(&repo, now(), &off).unwrap(); + assert_eq!(first.repack.status, RepackStatus::Repacked); + assert!( + !first.commit_graph && !has_graph(), + "no graph is written when the knob is off" + ); + + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + let backfill = run_repo(&reopened, now(), &options()).unwrap(); + assert!( + backfill.commit_graph && has_graph(), + "a packed repo with no graph backfills it" + ); + assert_eq!(backfill.repack.status, RepackStatus::Clean); + + let settled = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + !run_repo(&settled, now(), &options()).unwrap().commit_graph, + "a present graph settles to no-op" + ); + + let swept = Repo::open(repo.git().git_dir()).unwrap(); + assert!(!run_repo(&swept, now(), &off).unwrap().commit_graph); + assert!( + !has_graph(), + "flipping the knob off sweeps the orphan graph" + ); +} + +#[test] +fn cruft_second_pack_gets_a_midx_bitmap_canonical_git_accepts() { + let scan = tempfile::tempdir().unwrap(); + let repo = create(scan.path(), "did:plc:mussel"); + + let base = commit(&repo, 0, Vec::new()); + let doomed = commit(&repo, 1, vec![base]); + set_ref(&repo, "refs/heads/main", base); + set_ref(&repo, "refs/heads/feature", doomed); + + let opts = Options { + prune_grace: PruneGrace::from_secs(86_400), + ..options() + }; + run_repo(&repo, now(), &opts).unwrap(); + + delete_ref(&repo, "refs/heads/feature"); + set_ref(&repo, "refs/heads/main", commit(&repo, 2, vec![base])); + + let later = now().saturating_add_secs(432_000); + let report = run_repo(&repo, later, &opts).unwrap(); + assert!( + report.prune.crufted.get() >= 1, + "young unreachable object is crufted" + ); + assert!( + report.bitmap && has_midx_bitmap(&repo), + "the multi-pack repo gets a midx bitmap" + ); + let (ok, stderr) = git(&repo, &["rev-list", "--test-bitmap", "main"]); + assert!(ok, "canonical git accepts our midx bitmap: {stderr}"); + + let unchanged = run_repo(&repo, later, &opts).unwrap(); + assert_eq!(unchanged.multi_pack_index, MidxStatus::Unchanged); + assert!(!unchanged.bitmap, "an unchanged pack set keeps its bitmap"); + let pack_dir = repo.objects_dir().join("pack"); + std::fs::read_dir(&pack_dir) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| { + path.extension().is_some_and(|ext| ext == "rev") + && path + .file_name() + .unwrap() + .to_string_lossy() + .starts_with("multi-pack-index-") + }) + .for_each(|path| std::fs::remove_file(path).unwrap()); + assert!( + run_repo(&repo, later, &opts).unwrap().bitmap, + "a bitmap without its .rev is rebuilt" + ); +} diff --git a/knot2/crates/knot-maintenance/tests/geometric.rs b/knot2/crates/knot-maintenance/tests/geometric.rs new file mode 100644 index 0000000000000000000000000000000000000000..c6c21a1be078ebc73484436a6e57f703f52c8bac --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/geometric.rs @@ -0,0 +1,216 @@ +use std::collections::BTreeSet; + +use knot_git::Repo; +use knot_maintenance::{GeometricFactor, Options, PruneGrace, run_repo}; +use knot_types::{ObjectFormat, Oid, RefName}; + +mod common; +use common::{ + EMPTY_TREE_SHA1, chain, create_repo, delete_ref, empty_tree, fsck_clean, git_available, + has_cruft_pack, idx_stems, midx_verifies, now, options, reachable_objects, set_ref, +}; + +fn opts(factor: u64, grace: PruneGrace) -> Options { + Options { + geometric_factor: GeometricFactor::new(factor), + prune_grace: grace, + commit_graph: false, + bitmap: false, + ..options() + } +} + +fn main_ref(repo: &Repo) -> Option { + repo.find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap() +} + +fn no_reachable_loss(format: ObjectFormat) { + if !git_available() { + eprintln!("skipping geometric differential: git unavailable"); + return; + } + let scan = tempfile::tempdir().unwrap(); + let empty = empty_tree(format); + let repo = create_repo(scan.path(), format, "did:plc:scallop"); + let opts = opts(2, PruneGrace::from_secs(86_400)); + + (0..6).fold((None, BTreeSet::new()), |(tip, seen), round| { + let next = chain(&repo, empty, round * 2..round * 2 + 2, tip); + set_ref(&repo, "refs/heads/main", next); + run_repo(&repo, now(), &opts).unwrap(); + + let r = Repo::open(repo.git().git_dir()).unwrap(); + assert!(fsck_clean(&r), "{format:?} r{round}: not fsck-clean"); + if let Some((ok, stderr)) = midx_verifies(&r) { + assert!(ok, "{format:?} r{round} midx: {stderr}"); + } + assert_eq!( + main_ref(&r), + Some(next), + "{format:?} r{round}: main lost its tip" + ); + let present = reachable_objects(&r); + assert!( + seen.is_subset(&present), + "{format:?} r{round}: a reachable object went missing" + ); + (Some(next), present) + }); +} + +#[test] +fn geometric_no_reachable_loss_sha1() { + no_reachable_loss(ObjectFormat::SHA1); +} + +#[test] +fn geometric_no_reachable_loss_sha256() { + no_reachable_loss(ObjectFormat::SHA256); +} + +#[test] +fn geometric_keeps_the_large_pack_while_rolling_up_then_crufting_small_packs() { + if !git_available() { + eprintln!("skipping geometric behavior test: git unavailable"); + return; + } + let scan = tempfile::tempdir().unwrap(); + let repo = create_repo(scan.path(), ObjectFormat::SHA1, "did:plc:conch"); + let opts = opts(2, PruneGrace::from_secs(86_400)); + + let big = chain(&repo, EMPTY_TREE_SHA1, 0..10, None); + set_ref(&repo, "refs/heads/main", big); + run_repo(&repo, now(), &opts).unwrap(); + let large = idx_stems(&repo); + assert_eq!( + large.len(), + 1, + "the initial repack settles to one large pack" + ); + + let advanced = chain(&repo, EMPTY_TREE_SHA1, 100..102, Some(big)); + set_ref(&repo, "refs/heads/main", advanced); + run_repo(&repo, now(), &opts).unwrap(); + assert!( + large.is_subset(&idx_stems(&repo)), + "the large pack is kept verbatim" + ); + assert_eq!( + idx_stems(&repo).len(), + 2, + "small additions roll into a second pack" + ); + + let feature = chain(&repo, EMPTY_TREE_SHA1, 200..202, Some(advanced)); + set_ref(&repo, "refs/heads/feature", feature); + run_repo(&repo, now(), &opts).unwrap(); + delete_ref(&repo, "refs/heads/feature"); + + let report = run_repo(&repo, now().saturating_add_secs(432_000), &opts).unwrap(); + assert!( + report.prune.crufted.get() >= 1, + "young rolled-up garbage is crufted" + ); + assert!( + large.is_subset(&idx_stems(&repo)) && has_cruft_pack(&repo), + "large pack untouched, cruft written" + ); + + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + fsck_clean(&reopened), + "not fsck-clean after roll-up and cruft" + ); + assert!( + reopened.contains(advanced) && reopened.contains(feature), + "reachable tip and young orphan both survive" + ); +} + +#[test] +fn a_rollup_copies_in_pack_deltas_and_unreachable_objects_without_walking() { + if !git_available() { + eprintln!("skipping rollup delta test: git unavailable"); + return; + } + let scan = tempfile::tempdir().unwrap(); + let repo = create_repo(scan.path(), ObjectFormat::SHA1, "did:plc:nautilus"); + set_ref( + &repo, + "refs/heads/main", + chain(&repo, EMPTY_TREE_SHA1, 0..2, None), + ); + run_repo(&repo, now(), &options()).unwrap(); + + let git_dir = repo.git().git_dir(); + let run_git = |args: &[&str], stdin: &str| { + let mut child = knot_fixtures::command(git_dir) + .args(args) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + std::io::Write::write_all(&mut child.stdin.take().unwrap(), stdin.as_bytes()).unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git {args:?}"); + String::from_utf8(out.stdout).unwrap() + }; + let pack_dir = git_dir.join("objects/pack"); + let pack_prefix = pack_dir.join("pack"); + let pack_similar_blobs = |seed: usize| -> Vec { + let oids: Vec = (0..4) + .map(|variant| { + let lines: String = (0..200) + .map(|line| format!("line {line} of a large blob, variant {variant}\n")) + .collect(); + run_git( + &["hash-object", "-w", "--stdin"], + &format!("{seed}\n{lines}"), + ) + .trim() + .to_string() + }) + .collect(); + run_git( + &[ + "pack-objects", + "-q", + "--delta-base-offset", + pack_prefix.to_str().unwrap(), + ], + &oids.join("\n"), + ); + run_git(&["prune-packed", "-q"], ""); + oids.iter().map(|oid| Oid::from_hex(oid).unwrap()).collect() + }; + let unreachable: Vec = pack_similar_blobs(1) + .into_iter() + .chain(pack_similar_blobs(2)) + .collect(); + let reachable = reachable_objects(&repo); + assert_eq!(idx_stems(&repo).len(), 3); + + let report = run_repo(&repo, now(), &options()).unwrap(); + assert!( + !report.prune.ran, + "a rollup between full repacks never prunes" + ); + let stems = idx_stems(&repo); + assert_eq!(stems.len(), 1, "every pack rolls into one"); + + let idx = pack_dir.join(format!("{}.idx", stems.first().unwrap())); + let listing = run_git(&["verify-pack", "-v", idx.to_str().unwrap()], ""); + let deltas = listing + .lines() + .filter(|line| line.split_whitespace().count() == 7) + .count(); + assert!(deltas >= 4, "in-pack deltas are copied, not inflated"); + + let reopened = Repo::open(git_dir).unwrap(); + assert!( + unreachable.iter().all(|oid| reopened.contains(*oid)), + "unreachable objects ride along until the next full repack" + ); + assert!(reachable.is_subset(&reachable_objects(&reopened))); +} diff --git a/knot2/crates/knot-messages/Cargo.toml b/knot2/crates/knot-messages/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..5748e00e5b69cfafbddc2f6d224fb8b2afdc8a18 --- /dev/null +++ b/knot2/crates/knot-messages/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "knot-messages" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +confique = { workspace = true } +thiserror = { workspace = true } diff --git a/knot2/crates/knot-messages/src/lib.rs b/knot2/crates/knot-messages/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..0a815bbe1d196311cbda952be8132ee18b456bd9 --- /dev/null +++ b/knot2/crates/knot-messages/src/lib.rs @@ -0,0 +1,455 @@ +mod template; + +use confique::Config; + +pub use template::{Key, Line, Lines, NoKeys, Segment, Shape, Template, TemplateError}; + +#[derive(Debug)] +pub struct MessageLines(Vec); + +impl std::ops::Deref for MessageLines { + type Target = [String]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'de> confique::serde::Deserialize<'de> for MessageLines { + fn deserialize(deserializer: D) -> Result + where + D: confique::serde::Deserializer<'de>, + { + struct Shaped; + + impl<'de> confique::serde::de::Visitor<'de> for Shaped { + type Value = MessageLines; + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("a list of lines, or one newline-separated string") + } + + fn visit_str( + self, + value: &str, + ) -> Result { + Ok(MessageLines(value.lines().map(str::to_string).collect())) + } + + fn visit_seq>( + self, + mut seq: A, + ) -> Result { + std::iter::from_fn(|| seq.next_element().transpose()) + .collect::, _>>() + .map(MessageLines) + } + } + + deserializer.deserialize_any(Shaped) + } +} + +// Each msg field defines the enum of placeholders it accepts, +// so a typo'ed `{handel}` in a given config gets caught at startup, +// rather than printed at some poor pusher mid-push. +macro_rules! keys { + ( $( $name:ident { $( $variant:ident = $placeholder:literal ),+ $(,)? } )+ ) => { + $( + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum $name { + $( $variant, )+ + } + + impl Key for $name { + const PLACEHOLDERS: &'static [(&'static str, Self)] = + &[ $( ($placeholder, Self::$variant), )+ ]; + } + )+ + }; +} + +macro_rules! config_type { + (Lines) => { + MessageLines + }; + (Line) => { + String + }; +} + +macro_rules! parse_field { + (Lines, $field:expr, $value:expr) => { + Template::parse_lines($field, $value) + }; + (Line, $field:expr, $value:expr) => { + Template::parse($field, $value) + }; +} + +macro_rules! message_group { + ( + $config:ident => $catalog:ident @ $prefix:literal { + $( $field:ident : $shape:ident<$keys:ty> @ $env:literal = $default:tt ),+ $(,)? + } + ) => { + #[derive(Debug, ::confique::Config)] + pub struct $config { + $( + #[config(env = $env, default = $default)] + pub $field: config_type!($shape), + )+ + } + + #[derive(Debug)] + pub struct $catalog { + $( pub $field: Template<$keys, $shape>, )+ + } + + impl $catalog { + pub fn parse(config: &$config) -> Result { + Ok(Self { + $( + $field: parse_field!( + $shape, + concat!($prefix, ".", stringify!($field)), + &config.$field + )?, + )+ + }) + } + } + }; +} + +keys! { + KnotKey { Knot = "knot" } + PushAckKey { Knot = "knot", Refs = "refs" } + UrlKey { Url = "url" } + GreetingKey { User = "user", Knot = "knot" } + AuthorizedKey { Authorized = "authorized" } + CountKey { Count = "count" } + RefKey { Ref = "ref" } + ErrorKey { Error = "error" } + CommandKey { Command = "command" } + VersionKey { Version = "version" } + AlgorithmKey { Algorithm = "algorithm" } + ValueKey { Value = "value" } + OidKey { Oid = "oid" } + DetailKey { Detail = "detail" } + DeclaredComputedKey { Declared = "declared", Computed = "computed" } + DeclaredReceivedKey { Declared = "declared", Received = "received" } + DeclaredLimitKey { Declared = "declared", Limit = "limit" } + FreeFloorKey { Free = "free", Floor = "floor" } + WhatLimitKey { What = "what", Limit = "limit" } +} + +message_group! { + PushConfig => PushMessages @ "messages.push" { + ack: Lines @ "KNOT_MESSAGES_PUSH_ACK" = ["{knot} received {refs}."], + pull_request: Lines @ "KNOT_MESSAGES_PUSH_PULL_REQUEST" = [ + "", + "-> Open pull request for this branch:", + " {url}", + "" + ], + pipeline_clean: Lines @ "KNOT_MESSAGES_PUSH_PIPELINE_CLEAN" = ["pipeline compiled with no diagnostics"], + pipeline_none: Lines @ "KNOT_MESSAGES_PUSH_PIPELINE_NONE" = ["no pipelines to compile"], + } +} + +message_group! { + FetchConfig => FetchMessages @ "messages.fetch" { + motd: Lines @ "KNOT_MESSAGES_FETCH_MOTD" = ["Thanks for using {knot}!"], + enumerating: Lines @ "KNOT_MESSAGES_FETCH_ENUMERATING" = ["Enumerating objects: {count}, done."], + total: Lines @ "KNOT_MESSAGES_FETCH_TOTAL" = ["Total {count}, done."], + fatal: Line @ "KNOT_MESSAGES_FETCH_FATAL" = "knot: {error}", + } +} + +message_group! { + RejectConfig => RejectMessages @ "messages.reject" { + reserved_refs: Line @ "KNOT_MESSAGES_REJECT_RESERVED_REFS" = "refs/cobs/*, refs/cob-checkpoints/*, refs/hidden/*, refs/atproto/* and refs/tngl/* are reserved and can't be pushed", + cob_create_only: Line @ "KNOT_MESSAGES_REJECT_COB_CREATE_ONLY" = "existing refs/cobs/* object cannot be modified or deleted over the wire", + cob_delete: Line @ "KNOT_MESSAGES_REJECT_COB_DELETE" = "refs/cobs/* stores append-only collaborative objects and cannot be deleted", + hidden_reserved: Line @ "KNOT_MESSAGES_REJECT_HIDDEN_RESERVED" = "refs/hidden/* is reserved and can't be pushed", + atproto_reserved: Line @ "KNOT_MESSAGES_REJECT_ATPROTO_RESERVED" = "refs/atproto/* is reserved for the knot's atproto commit chain and can't be pushed", + tngl_reserved: Line @ "KNOT_MESSAGES_REJECT_TNGL_RESERVED" = "refs/tngl/* is reserved for knot-written keep refs and can't be pushed", + checkpoint_reserved: Line @ "KNOT_MESSAGES_REJECT_CHECKPOINT_RESERVED" = "refs/cob-checkpoints/* is reserved for knot-written collaborative-object snapshots and can't be pushed", + cob_verification: Line @ "KNOT_MESSAGES_REJECT_COB_VERIFICATION" = "collaborative-object verification failed: {error}", + ref_exists: Line @ "KNOT_MESSAGES_REJECT_REF_EXISTS" = "reference already exists", + stale_old_value: Line @ "KNOT_MESSAGES_REJECT_STALE_OLD_VALUE" = "stale info: old value doesn't match", + missing_objects: Line @ "KNOT_MESSAGES_REJECT_MISSING_OBJECTS" = "missing necessary objects", + missing_objects_for: Line @ "KNOT_MESSAGES_REJECT_MISSING_OBJECTS_FOR" = "missing necessary objects for {ref}", + atomic_failed: Line @ "KNOT_MESSAGES_REJECT_ATOMIC_FAILED" = "atomic transaction failed", + atomic_aborted: Line @ "KNOT_MESSAGES_REJECT_ATOMIC_ABORTED" = "atomic push aborted", + authorization_unavailable: Line @ "KNOT_MESSAGES_REJECT_AUTHORIZATION_UNAVAILABLE" = "authorization unavailable", + unpacker_error: Line @ "KNOT_MESSAGES_REJECT_UNPACKER_ERROR" = "unpacker error", + ref_snapshot_unavailable: Line @ "KNOT_MESSAGES_REJECT_REF_SNAPSHOT_UNAVAILABLE" = "ref snapshot unavailable", + object_migration_failed: Line @ "KNOT_MESSAGES_REJECT_OBJECT_MIGRATION_FAILED" = "object migration failed", + } +} + +message_group! { + SshConfig => SshMessages @ "messages.ssh" { + greeting: Lines @ "KNOT_MESSAGES_SSH_GREETING" = [ + "Hi {user}! You're authenticated to {knot} knot.", + "This knot serves git over ssh, so there's no shell here. :P", + "Clone repo with: git clone {knot}:" + ], + greeting_unknown: Lines @ "KNOT_MESSAGES_SSH_GREETING_UNKNOWN" = [ + "Hi there! This is the {knot} knot.", + "This knot serves git over ssh, so there's no shell here. :P", + "Clone repo with: git clone {knot}:", + "Publish your ssh key to your atproto account so this knot can identify your pushes.", + "Put your handle in the url, as in yourhandle@{knot}:, so your ssh client can find your registered key on its own." + ], + unsupported_command: Line @ "KNOT_MESSAGES_SSH_UNSUPPORTED_COMMAND" = "knot: unsupported command", + too_many_operations: Line @ "KNOT_MESSAGES_SSH_TOO_MANY_OPERATIONS" = "knot: too many concurrent operations from your address, try again shortly", + repo_not_found: Line @ "KNOT_MESSAGES_SSH_REPO_NOT_FOUND" = "knot: repository not found", + index_warming: Line @ "KNOT_MESSAGES_SSH_INDEX_WARMING" = "knot: repository index is warming, retry shortly", + key_not_registered: Line @ "KNOT_MESSAGES_SSH_KEY_NOT_REGISTERED" = "knot: this ssh key doesn't match any key published by the accounts that may push here. Authorized: {authorized}. If your agent offers several keys, add -o IdentitiesOnly=yes so it offers your registered key.", + identity_unavailable: Line @ "KNOT_MESSAGES_SSH_IDENTITY_UNAVAILABLE" = "knot: couldn't read the account records needed to check your ssh key, retry shortly", + push_denied: Line @ "KNOT_MESSAGES_SSH_PUSH_DENIED" = "knot: you aren't authorized to push to this repository.", + shutting_down: Line @ "KNOT_MESSAGES_SSH_SHUTTING_DOWN" = "knot: server is shutting down", + archive_malformed: Line @ "KNOT_MESSAGES_SSH_ARCHIVE_MALFORMED" = "knot: malformed upload-archive request", + archive_timeout: Line @ "KNOT_MESSAGES_SSH_ARCHIVE_TIMEOUT" = "knot: upload-archive request timed out", + archive_failed: Line @ "KNOT_MESSAGES_SSH_ARCHIVE_FAILED" = "knot: upload-archive failed", + advertise_failed: Line @ "KNOT_MESSAGES_SSH_ADVERTISE_FAILED" = "knot: cannot advertise refs", + push_too_large: Line @ "KNOT_MESSAGES_SSH_PUSH_TOO_LARGE" = "knot: push exceeds configured size limit", + receive_deadline: Line @ "KNOT_MESSAGES_SSH_RECEIVE_DEADLINE" = "knot: receive exceeded its time budget", + malformed_pack: Line @ "KNOT_MESSAGES_SSH_MALFORMED_PACK" = "knot: malformed pack stream", + receive_read_error: Line @ "KNOT_MESSAGES_SSH_RECEIVE_READ_ERROR" = "knot: receive read error", + receive_ended_early: Line @ "KNOT_MESSAGES_SSH_RECEIVE_ENDED_EARLY" = "knot: receive stream ended early", + receive_failed: Line @ "KNOT_MESSAGES_SSH_RECEIVE_FAILED" = "knot: receive-pack failed", + } +} + +message_group! { + HttpConfig => HttpMessages @ "messages.http" { + push_denied: Line @ "KNOT_MESSAGES_HTTP_PUSH_DENIED" = "you aren't authorized to push to this repository", + repo_not_found: Line @ "KNOT_MESSAGES_HTTP_REPO_NOT_FOUND" = "repository not found", + push_too_large: Line @ "KNOT_MESSAGES_HTTP_PUSH_TOO_LARGE" = "push exceeds the configured size limit", + malformed_pack: Line @ "KNOT_MESSAGES_HTTP_MALFORMED_PACK" = "malformed pack stream: {error}", + receive_ended_early: Line @ "KNOT_MESSAGES_HTTP_RECEIVE_ENDED_EARLY" = "receive stream ended early", + } +} + +message_group! { + LfsConfig => LfsMessages @ "messages.lfs" { + invalid_oid: Line @ "KNOT_MESSAGES_LFS_INVALID_OID" = "invalid LFS oid {value}", + hash_mismatch: Line @ "KNOT_MESSAGES_LFS_HASH_MISMATCH" = "oid mismatch, declared {declared}, computed {computed}", + size_mismatch: Line @ "KNOT_MESSAGES_LFS_SIZE_MISMATCH" = "size mismatch, declared {declared}, received {received}", + size_limit_exceeded: Line @ "KNOT_MESSAGES_LFS_SIZE_LIMIT_EXCEEDED" = "object size {declared} exceeds limit {limit}", + free_space_denied: Line @ "KNOT_MESSAGES_LFS_FREE_SPACE_DENIED" = "free space {free} below floor {floor}", + not_found: Line @ "KNOT_MESSAGES_LFS_NOT_FOUND" = "object {oid} not found", + framing: Line @ "KNOT_MESSAGES_LFS_FRAMING" = "protocol framing fault: {detail}", + too_many: Line @ "KNOT_MESSAGES_LFS_TOO_MANY" = "too many {what} in one message, limit {limit}", + unknown_command: Line @ "KNOT_MESSAGES_LFS_UNKNOWN_COMMAND" = "unknown command {command}", + unsupported_version: Line @ "KNOT_MESSAGES_LFS_UNSUPPORTED_VERSION" = "unsupported version {version}", + unsupported_hash: Line @ "KNOT_MESSAGES_LFS_UNSUPPORTED_HASH" = "unsupported hash algorithm {algorithm}", + put_on_download: Line @ "KNOT_MESSAGES_LFS_PUT_ON_DOWNLOAD" = "put-object isn't allowed on a download channel", + verify_on_download: Line @ "KNOT_MESSAGES_LFS_VERIFY_ON_DOWNLOAD" = "verify-object isn't allowed on a download channel", + get_on_upload: Line @ "KNOT_MESSAGES_LFS_GET_ON_UPLOAD" = "get-object isn't allowed on an upload channel", + put_no_body: Line @ "KNOT_MESSAGES_LFS_PUT_NO_BODY" = "put-object is missing its object body", + } +} + +#[derive(Debug, Config)] +pub struct MessagesConfig { + #[config(nested)] + pub push: PushConfig, + #[config(nested)] + pub fetch: FetchConfig, + #[config(nested)] + pub reject: RejectConfig, + #[config(nested)] + pub ssh: SshConfig, + #[config(nested)] + pub http: HttpConfig, + #[config(nested)] + pub lfs: LfsConfig, +} + +impl MessagesConfig { + pub fn defaults() -> Self { + Self::builder() + .load() + .expect("message defaults satisfy every field") + } +} + +#[derive(Debug)] +pub struct Catalog { + pub push: PushMessages, + pub fetch: FetchMessages, + pub reject: RejectMessages, + pub ssh: SshMessages, + pub http: HttpMessages, + pub lfs: LfsMessages, +} + +impl Catalog { + pub fn parse(config: &MessagesConfig) -> Result { + Ok(Self { + push: PushMessages::parse(&config.push)?, + fetch: FetchMessages::parse(&config.fetch)?, + reject: RejectMessages::parse(&config.reject)?, + ssh: SshMessages::parse(&config.ssh)?, + http: HttpMessages::parse(&config.http)?, + lfs: LfsMessages::parse(&config.lfs)?, + }) + } + + pub fn defaults() -> Self { + Self::parse(&MessagesConfig::defaults()).expect("built-in message templates parse") + } +} + +pub fn default_catalog() -> &'static Catalog { + static DEFAULTS: std::sync::LazyLock = std::sync::LazyLock::new(Catalog::defaults); + &DEFAULTS +} + +pub fn count_refs(applied: usize) -> String { + match applied { + 1 => "1 ref".to_string(), + n => format!("{n} refs"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_defaults_parse_into_a_full_catalog() { + let catalog = Catalog::defaults(); + assert_eq!(catalog.reject.ref_exists.text(), "reference already exists"); + assert_eq!( + catalog.ssh.repo_not_found.text(), + "knot: repository not found" + ); + } + + #[test] + fn the_pull_request_block_matches_the_shipped_shape() { + let catalog = Catalog::defaults(); + let url = "https://oyster.cafe/nel.pet/anemone/pulls/new"; + let block = catalog + .push + .pull_request + .lines(|UrlKey::Url| url.to_string()); + assert_eq!( + block, + vec![ + "\u{200b}".to_string(), + "-> Open pull request for this branch:".to_string(), + format!(" {url}"), + "\u{200b}".to_string(), + ] + ); + } + + #[test] + fn the_greeting_names_the_user_and_the_knot() { + let catalog = Catalog::defaults(); + let lines = catalog.ssh.greeting.lines(|key| match key { + GreetingKey::User => "@nel.pet".to_string(), + GreetingKey::Knot => "oyster.cafe".to_string(), + }); + assert!(lines[0].contains("@nel.pet")); + assert!(lines.iter().any(|line| line.contains("oyster.cafe"))); + } + + #[test] + fn an_unidentified_visitor_is_greeted_and_shown_what_a_push_needs() { + let catalog = Catalog::defaults(); + let lines = catalog + .ssh + .greeting_unknown + .lines(|KnotKey::Knot| "oyster.cafe".to_string()); + assert!(lines[0].contains("oyster.cafe")); + assert!( + lines.iter().any(|line| line.contains("ssh key")), + "a visitor the knot can't identify learns what a push needs: {lines:?}" + ); + + let denial = catalog + .ssh + .key_not_registered + .line(|AuthorizedKey::Authorized| "@nel.pet".to_string()); + assert!( + denial.contains("@nel.pet"), + "the denial lists who may push instead: {denial}" + ); + } + + #[test] + fn an_empty_lines_template_mutes_the_message() { + let template: Template = + Template::parse_lines("messages.test", &[]).unwrap(); + assert!(template.text_lines().is_empty()); + } + + #[test] + fn an_unknown_placeholder_is_a_parse_error() { + let error = Template::::parse_lines( + "messages.fetch.motd", + &["hi {handle}".to_string()], + ) + .unwrap_err(); + assert_eq!( + error, + TemplateError::UnknownPlaceholder { + field: "messages.fetch.motd", + name: "handle".to_string(), + } + ); + } + + #[test] + fn doubled_braces_render_as_literal_braces() { + let template: Template = + Template::parse("messages.test", "a {{literal}} brace").unwrap(); + assert_eq!(template.text(), "a {literal} brace"); + } + + #[test] + fn line_templates_reject_empty_and_multiline_text() { + assert_eq!( + Template::::parse("messages.test", "").unwrap_err(), + TemplateError::Empty { + field: "messages.test" + } + ); + assert_eq!( + Template::::parse("messages.test", "a\nb").unwrap_err(), + TemplateError::Multiline { + field: "messages.test" + } + ); + } + + #[test] + fn unbalanced_braces_are_parse_errors() { + assert_eq!( + Template::::parse("messages.test", "open {").unwrap_err(), + TemplateError::UnclosedBrace { + field: "messages.test" + } + ); + assert_eq!( + Template::::parse("messages.test", "close }").unwrap_err(), + TemplateError::StrayBrace { + field: "messages.test" + } + ); + } + + #[test] + fn ref_counts_pluralize() { + assert_eq!(count_refs(1), "1 ref"); + assert_eq!(count_refs(3), "3 refs"); + } +} diff --git a/knot2/crates/knot-messages/src/template.rs b/knot2/crates/knot-messages/src/template.rs new file mode 100644 index 0000000000000000000000000000000000000000..f2c2095b305efa0d489fe9c48e8ccbd59d32f06a --- /dev/null +++ b/knot2/crates/knot-messages/src/template.rs @@ -0,0 +1,188 @@ +use std::fmt; + +const SPACER: &str = "\u{200b}"; + +pub trait Key: Copy + 'static { + const PLACEHOLDERS: &'static [(&'static str, Self)]; +} + +#[derive(Debug, Clone, Copy)] +pub enum NoKeys {} + +impl Key for NoKeys { + const PLACEHOLDERS: &'static [(&'static str, Self)] = &[]; +} + +pub trait Shape { + type Repr; +} + +#[derive(Debug)] +pub struct Lines; + +impl Shape for Lines { + type Repr = Vec>>; +} + +#[derive(Debug)] +pub struct Line; + +impl Shape for Line { + type Repr = Vec>; +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum TemplateError { + #[error("{field}: unknown placeholder {{{name}}}")] + UnknownPlaceholder { field: &'static str, name: String }, + #[error("{field}: unclosed {{ in template")] + UnclosedBrace { field: &'static str }, + #[error("{field}: stray }} in template")] + StrayBrace { field: &'static str }, + #[error("{field}: template mustn't be empty")] + Empty { field: &'static str }, + #[error("{field}: template must be a single line")] + Multiline { field: &'static str }, +} + +#[derive(Debug)] +pub enum Segment { + Literal(String), + Placeholder(K), +} + +pub struct Template { + repr: S::Repr, +} + +impl fmt::Debug for Template +where + S::Repr: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Template") + .field("repr", &self.repr) + .finish() + } +} + +impl Template { + pub fn parse_lines(field: &'static str, texts: &[String]) -> Result { + Ok(Self { + repr: texts + .iter() + .map(|text| line_segments(field, text)) + .collect::>()?, + }) + } + + pub fn lines(&self, resolve: impl Fn(K) -> String) -> Vec { + self.repr + .iter() + .map(|segments| { + let rendered = render_segments(segments, &resolve); + match rendered.is_empty() { + true => SPACER.to_string(), + false => rendered, + } + }) + .collect() + } +} + +impl Template { + // A reject reason is a Line - `receive-pack` gives it as + // "ng \n" inside one `pkt-line`. + // `line_segments` won't accept newlines + // so the reason stays on that one line. + pub fn parse(field: &'static str, text: &str) -> Result { + match text.is_empty() { + true => Err(TemplateError::Empty { field }), + false => Ok(Self { + repr: line_segments(field, text)?, + }), + } + } + + pub fn line(&self, resolve: impl Fn(K) -> String) -> String { + render_segments(&self.repr, &resolve) + } +} + +impl Template { + pub fn text_lines(&self) -> Vec { + self.lines(|key| match key {}) + } +} + +impl Template { + pub fn text(&self) -> String { + self.line(|key| match key {}) + } +} + +fn line_segments( + field: &'static str, + text: &str, +) -> Result>, TemplateError> { + match text.contains('\n') { + true => Err(TemplateError::Multiline { field }), + false => segments(field, text), + } +} + +fn render_segments(segments: &[Segment], resolve: &dyn Fn(K) -> String) -> String { + segments + .iter() + .map(|segment| match segment { + Segment::Literal(text) => text.clone(), + Segment::Placeholder(key) => resolve(*key), + }) + .collect() +} + +fn segments(field: &'static str, text: &str) -> Result>, TemplateError> { + let Some(at) = text.find(['{', '}']) else { + return Ok(literal(text)); + }; + let (before, rest) = text.split_at(at); + let (parsed, remainder) = brace(field, rest)?; + Ok(literal(before) + .into_iter() + .chain(parsed) + .chain(segments(field, remainder)?) + .collect()) +} + +fn literal(text: &str) -> Vec> { + match text.is_empty() { + true => Vec::new(), + false => vec![Segment::Literal(text.to_string())], + } +} + +type Braced<'a, K> = (Option>, &'a str); + +fn brace<'a, K: Key>(field: &'static str, rest: &'a str) -> Result, TemplateError> { + if let Some(after) = rest.strip_prefix("{{") { + return Ok((Some(Segment::Literal("{".to_string())), after)); + } + if let Some(after) = rest.strip_prefix("}}") { + return Ok((Some(Segment::Literal("}".to_string())), after)); + } + if rest.starts_with('}') { + return Err(TemplateError::StrayBrace { field }); + } + let close = rest + .find('}') + .ok_or(TemplateError::UnclosedBrace { field })?; + let name = &rest[1..close]; + K::PLACEHOLDERS + .iter() + .find(|(candidate, _)| *candidate == name) + .map(|(_, key)| (Some(Segment::Placeholder(*key)), &rest[close + 1..])) + .ok_or_else(|| TemplateError::UnknownPlaceholder { + field, + name: name.to_string(), + }) +} diff --git a/knot2/crates/knot-migrate/Cargo.toml b/knot2/crates/knot-migrate/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..f190197c231a3e7d9288df17c719d8f558ec6018 --- /dev/null +++ b/knot2/crates/knot-migrate/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "knot-migrate" +version = "1.20.0" +edition = { workspace = true } +rust-version = { workspace = true } +license = { workspace = true } +publish = false + +[[bin]] +name = "knot-migrate" +path = "src/main.rs" + +[dependencies] +knot-types = { workspace = true } +knot-runtime = { workspace = true } +knot-git = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-config = { workspace = true } +knot-secrets = { workspace = true } +rusqlite = { version = "0.38", features = ["bundled"] } +rustix = { workspace = true, features = ["fs"] } +ssh-key = { version = "0.7.0-rc.10", default-features = false, features = ["std", "ed25519", "ecdsa"] } +chrono = { workspace = true } +walkdir = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +url = { workspace = true } +base64 = { workspace = true } +thiserror = { workspace = true } +zeroize = { workspace = true } + +[dev-dependencies] +knot-index = { workspace = true } +tempfile = { workspace = true } diff --git a/knot2/crates/knot-migrate/src/adopt.rs b/knot2/crates/knot-migrate/src/adopt.rs new file mode 100644 index 0000000000000000000000000000000000000000..8dd8ae74513e367b1928fb2955402767c2070f73 --- /dev/null +++ b/knot2/crates/knot-migrate/src/adopt.rs @@ -0,0 +1,378 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use knot_git::{GitError, Layout, Repo}; +use knot_types::{ObjectFormat, RepoDid}; + +use crate::mapping::AdoptRepo; +use crate::source::SourceRepoDid; + +#[derive(Debug, thiserror::Error)] +pub enum AdoptError { + #[error("layout path for {repo}: {source}")] + Layout { repo: RepoDid, source: GitError }, + #[error("repo {repo} resolves to the reserved knot meta-repo path")] + ReservesMeta { repo: RepoDid }, + #[error("place {path}: {source}")] + Place { + path: PathBuf, + source: std::io::Error, + }, + #[error("sync {path}: {source}")] + Sync { + path: PathBuf, + source: std::io::Error, + }, + #[error("adopted repo {repo} doesn't open as a git repository: {source}")] + Unopenable { repo: RepoDid, source: GitError }, + #[error("source directory for {repo} vanished between mapping and adoption")] + Vanished { repo: RepoDid }, + #[error("consuming the source needs {scan_path} and {target} on one filesystem")] + CrossDeviceConsume { scan_path: PathBuf, target: PathBuf }, + #[error( + "consuming the source will move the repos out of {scan_path}, which this process can't write: {source}" + )] + UnwritableSource { + scan_path: PathBuf, + source: std::io::Error, + }, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct AdoptOutcome { + pub transfer: Transfer, + pub adopted: u64, + pub already_present: u64, + pub sha1: u64, + pub sha256: u64, +} + +impl AdoptOutcome { + fn empty(transfer: Transfer) -> Self { + Self { + transfer, + adopted: 0, + already_present: 0, + sha1: 0, + sha256: 0, + } + } + + fn merge(self, other: Self) -> Self { + Self { + transfer: self.transfer, + adopted: self.adopted + other.adopted, + already_present: self.already_present + other.already_present, + sha1: self.sha1 + other.sha1, + sha256: self.sha256 + other.sha256, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourcePolicy { + Preserve, + Consume, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Transfer { + Rename, + Copy, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceProbe { + Repo, + Absent, + Unreadable(std::io::ErrorKind), +} + +impl std::fmt::Display for Transfer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Rename => f.write_str("rename"), + Self::Copy => f.write_str("copy"), + } + } +} + +enum Placement { + AlreadyPresent, + Staged(PathBuf), + Moved, +} + +struct Staged<'repo> { + did: &'repo RepoDid, + destination: PathBuf, + placement: Placement, +} + +pub fn source_dir(source_root: &Path, repo_did: &SourceRepoDid) -> PathBuf { + source_root.join(repo_did.as_str()) +} + +pub fn reads_as_absent(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) +} + +pub fn probe_source(source_root: &Path, repo_did: &SourceRepoDid) -> SourceProbe { + match std::fs::metadata(source_dir(source_root, repo_did).join("HEAD")) { + Ok(head) if head.is_file() => SourceProbe::Repo, + Ok(_) => SourceProbe::Absent, + Err(error) if reads_as_absent(&error) => SourceProbe::Absent, + Err(error) => SourceProbe::Unreadable(error.kind()), + } +} + +pub fn adopt_all( + layout: &Layout, + source_root: &Path, + repos: &[AdoptRepo], + policy: SourcePolicy, +) -> Result { + let root = layout.scratch_dir(); + std::fs::create_dir_all(root).map_err(|source| AdoptError::Place { + path: root.to_path_buf(), + source, + })?; + let transfer = transfer_mode(source_root, root, policy)?; + let staged = in_lanes(repos, |repo| stage_one(layout, source_root, repo, transfer))?; + sync_filesystem(root)?; + staged.iter().try_for_each(commit_one)?; + sync_filesystem(root)?; + in_lanes(&staged, |staged| count_one(staged, transfer)).map(|counted| { + counted + .into_iter() + .fold(AdoptOutcome::empty(transfer), AdoptOutcome::merge) + }) +} + +pub fn transfer_mode( + source_root: &Path, + target_root: &Path, + policy: SourcePolicy, +) -> Result { + use std::os::unix::fs::MetadataExt; + let device = |path: &Path| { + std::fs::metadata(path) + .map(|meta| meta.dev()) + .map_err(|source| AdoptError::Place { + path: path.to_path_buf(), + source, + }) + }; + match policy { + SourcePolicy::Preserve => Ok(Transfer::Copy), + SourcePolicy::Consume => match device(source_root)? == device(target_root)? { + true => writable(source_root) + .map(|()| Transfer::Rename) + .map_err(|source| AdoptError::UnwritableSource { + scan_path: source_root.to_path_buf(), + source: source.into(), + }), + false => Err(AdoptError::CrossDeviceConsume { + scan_path: source_root.to_path_buf(), + target: target_root.to_path_buf(), + }), + }, + } +} + +pub fn writable(path: &Path) -> Result<(), rustix::io::Errno> { + rustix::fs::accessat( + rustix::fs::CWD, + path, + rustix::fs::Access::WRITE_OK | rustix::fs::Access::EXEC_OK, + rustix::fs::AtFlags::EACCESS, + ) +} + +fn in_lanes<'items, T, R, F>(items: &'items [T], work: F) -> Result, AdoptError> +where + T: Sync, + R: Send, + F: Fn(&'items T) -> Result + Sync, +{ + let cursor = AtomicUsize::new(0); + std::thread::scope(|scope| { + (0..lane_count(items.len())) + .map(|_| { + scope.spawn(|| { + std::iter::from_fn(|| items.get(cursor.fetch_add(1, Ordering::Relaxed))) + .map(&work) + .collect::, AdoptError>>() + }) + }) + .collect::>() + .into_iter() + .map(|lane| lane.join().expect("adoption lane doesn't panic")) + .collect::>, AdoptError>>() + .map(|lanes| lanes.into_iter().flatten().collect()) + }) +} + +fn lane_count(items: usize) -> usize { + std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1) + .min(items.max(1)) +} + +fn stage_one<'repo>( + layout: &Layout, + source_root: &Path, + repo: &'repo AdoptRepo, + transfer: Transfer, +) -> Result, AdoptError> { + let destination = layout + .guarded_path(&repo.did) + .map_err(|source| match source { + GitError::ReservedDid(_) => AdoptError::ReservesMeta { + repo: repo.did.clone(), + }, + source => AdoptError::Layout { + repo: repo.did.clone(), + source, + }, + })?; + match destination.exists() { + true => Ok(Staged { + did: &repo.did, + destination, + placement: Placement::AlreadyPresent, + }), + false => { + let source = source_dir(source_root, &repo.source_did); + match source.is_dir() { + false => Err(AdoptError::Vanished { + repo: repo.did.clone(), + }), + true => place(&source, &destination, transfer).map(|placement| Staged { + did: &repo.did, + destination, + placement, + }), + } + } + } +} + +fn place(source: &Path, destination: &Path, transfer: Transfer) -> Result { + match transfer { + Transfer::Rename => { + make_parent(destination)?; + std::fs::rename(source, destination) + .map(|()| Placement::Moved) + .map_err(|error| AdoptError::Place { + path: destination.to_path_buf(), + source: error, + }) + } + Transfer::Copy => stage_tree(source, destination).map(Placement::Staged), + } +} + +fn commit_one(staged: &Staged<'_>) -> Result<(), AdoptError> { + match &staged.placement { + Placement::AlreadyPresent | Placement::Moved => Ok(()), + Placement::Staged(staging) => { + std::fs::rename(staging, &staged.destination).map_err(|source| AdoptError::Place { + path: staged.destination.clone(), + source, + }) + } + } +} + +fn count_one(staged: &Staged<'_>, transfer: Transfer) -> Result { + let opened = Repo::open(&staged.destination).map_err(|source| AdoptError::Unopenable { + repo: staged.did.clone(), + source, + })?; + let fresh = u64::from(!matches!(staged.placement, Placement::AlreadyPresent)); + let counted = AdoptOutcome { + adopted: fresh, + already_present: 1 - fresh, + ..AdoptOutcome::empty(transfer) + }; + Ok(match opened.object_format() { + ObjectFormat::SHA1 => AdoptOutcome { sha1: 1, ..counted }, + _ => AdoptOutcome { + sha256: 1, + ..counted + }, + }) +} + +fn make_parent(destination: &Path) -> Result<&Path, AdoptError> { + let parent = destination + .parent() + .expect("layout repo paths always have a parent"); + std::fs::create_dir_all(parent) + .map(|()| parent) + .map_err(|source| AdoptError::Place { + path: parent.to_path_buf(), + source, + }) +} + +fn stage_tree(source: &Path, destination: &Path) -> Result { + let io = |path: &Path| { + let path = path.to_path_buf(); + move |source: std::io::Error| AdoptError::Place { path, source } + }; + let parent = make_parent(destination)?; + let staging = parent.join(format!( + ".migrate-staging.{}", + destination + .file_name() + .expect("layout repo paths always have a file name") + .to_string_lossy() + )); + if staging.exists() { + std::fs::remove_dir_all(&staging).map_err(io(&staging))?; + } + place_tree(source, &staging).map(|()| staging) +} + +fn place_tree(source: &Path, destination: &Path) -> Result<(), AdoptError> { + walkdir::WalkDir::new(source) + .into_iter() + .try_for_each(|entry| { + let entry = entry.map_err(|error| AdoptError::Place { + path: source.to_path_buf(), + source: error.into(), + })?; + let relative = entry + .path() + .strip_prefix(source) + .expect("walkdir yields paths under its root"); + let target = destination.join(relative); + let io = |source: std::io::Error| AdoptError::Place { + path: entry.path().to_path_buf(), + source, + }; + match entry.file_type() { + kind if kind.is_dir() => std::fs::create_dir_all(&target).map_err(io), + #[cfg(unix)] + kind if kind.is_symlink() => std::fs::read_link(entry.path()) + .and_then(|link| std::os::unix::fs::symlink(link, &target)) + .map_err(io), + _ => std::fs::copy(entry.path(), &target).map(|_| ()).map_err(io), + } + }) +} + +fn sync_filesystem(root: &Path) -> Result<(), AdoptError> { + std::fs::File::open(root) + .and_then(|anchor| rustix::fs::syncfs(&anchor).map_err(std::io::Error::from)) + .map_err(|source| AdoptError::Sync { + path: root.to_path_buf(), + source, + }) +} diff --git a/knot2/crates/knot-migrate/src/casbin.rs b/knot2/crates/knot-migrate/src/casbin.rs new file mode 100644 index 0000000000000000000000000000000000000000..0a1e22285122114db0b0e785fcd15bd6cbd6b6ce --- /dev/null +++ b/knot2/crates/knot-migrate/src/casbin.rs @@ -0,0 +1,301 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::source::{AclRow, SourceDid, SourceRepoDid, SourceRepoName, SourceRepoObject}; + +const DOMAIN: &str = "thisserver"; +const SKIPPED_ROLES: [&str; 6] = [ + "repo:push", + "repo:settings", + "repo:invite", + "repo:delete", + "repo:create", + "server:invite", +]; + +#[derive(Debug, thiserror::Error)] +pub enum CasbinError { + #[error("unrecognized acl row: {p_type},{v0},{v1},{v2},{v3}")] + UnrecognizedRow { + p_type: String, + v0: String, + v1: String, + v2: String, + v3: String, + }, + #[error("acl names two server owners: {first} and {second}")] + TwoServerOwners { first: SourceDid, second: SourceDid }, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct AclRoster { + pub server_owner: Option, + pub members: BTreeSet, + pub owner_markers: BTreeMap>, + pub collaborators: BTreeMap>, + pub slash_collaborators: BTreeMap>, + pub slash_owner_markers: u64, + pub slash_collab_rows: u64, + pub unresolved_slash_forms: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SlashTarget { + Unique(SourceRepoDid), + Ambiguous, +} + +pub type SlashResolver = BTreeMap<(SourceDid, SourceRepoName), SlashTarget>; + +pub fn resolver( + pairs: impl Iterator, +) -> SlashResolver { + pairs.fold(BTreeMap::new(), |mut map, (owner, name, repo)| { + map.entry((owner, name)) + .and_modify(|target| *target = SlashTarget::Ambiguous) + .or_insert(SlashTarget::Unique(repo)); + map + }) +} + +pub fn decode(rows: &[AclRow], resolve: &SlashResolver) -> Result { + rows.iter().try_fold(AclRoster::default(), |roster, row| { + step(roster, row, resolve) + }) +} + +fn step( + mut roster: AclRoster, + row: &AclRow, + resolve: &SlashResolver, +) -> Result { + match ( + row.p_type.as_str(), + row.v0.as_str(), + row.v1.as_str(), + row.v2.as_str(), + row.v3.as_str(), + ) { + // loadbearing ordering because of how old knots used to work + ("g", "server:owner", "server:member", DOMAIN, _) => Ok(roster), + ("g", did, "server:member", DOMAIN, _) => { + roster.members.insert(SourceDid::from_column(did)); + Ok(roster) + } + ("g", did, "server:owner", DOMAIN, _) => match roster.server_owner.take() { + Some(first) if first.as_str() != did => Err(CasbinError::TwoServerOwners { + first, + second: SourceDid::from_column(did), + }), + _ => { + roster.server_owner = Some(SourceDid::from_column(did)); + Ok(roster) + } + }, + ("p", _, DOMAIN, _, role) if SKIPPED_ROLES.contains(&role) => Ok(roster), + ("p", did, DOMAIN, object, "repo:owner") => Ok(mark( + roster, + Marker::Owner, + &SourceDid::from_column(did), + &SourceRepoObject::from_column(object), + resolve, + )), + ("p", did, DOMAIN, object, "repo:collaborator") => Ok(mark( + roster, + Marker::Collaborator, + &SourceDid::from_column(did), + &SourceRepoObject::from_column(object), + resolve, + )), + _ => Err(CasbinError::UnrecognizedRow { + p_type: row.p_type.clone(), + v0: row.v0.clone(), + v1: row.v1.clone(), + v2: row.v2.clone(), + v3: row.v3.clone(), + }), + } +} + +#[derive(Clone, Copy)] +enum Marker { + Owner, + Collaborator, +} + +fn mark( + mut roster: AclRoster, + marker: Marker, + did: &SourceDid, + object: &SourceRepoObject, + resolve: &SlashResolver, +) -> AclRoster { + match (object.as_str().split_once('/'), marker) { + (None, Marker::Owner) => { + roster + .owner_markers + .entry(SourceRepoDid::from_column(object.as_str())) + .or_default() + .insert(did.clone()); + } + (None, Marker::Collaborator) => { + roster + .collaborators + .entry(SourceRepoDid::from_column(object.as_str())) + .or_default() + .insert(did.clone()); + } + (Some((owner, name)), marker) => { + match marker { + Marker::Owner => roster.slash_owner_markers += 1, + Marker::Collaborator => roster.slash_collab_rows += 1, + } + let resolved = resolve.get(&( + SourceDid::from_column(owner), + SourceRepoName::from_column(name), + )); + match (resolved, marker) { + (Some(SlashTarget::Unique(repo)), Marker::Collaborator) => { + roster + .slash_collaborators + .entry(repo.clone()) + .or_default() + .insert(did.clone()); + } + (Some(SlashTarget::Unique(_)), Marker::Owner) => {} + (Some(SlashTarget::Ambiguous), _) | (None, _) => { + roster.unresolved_slash_forms.push(object.to_string()) + } + } + } + } + roster +} + +#[cfg(test)] +mod tests { + use super::*; + + fn g(did: &str, role: &str) -> AclRow { + AclRow { + p_type: "g".into(), + v0: did.into(), + v1: role.into(), + v2: DOMAIN.into(), + v3: String::new(), + } + } + + fn p(did: &str, object: &str, role: &str) -> AclRow { + AclRow { + p_type: "p".into(), + v0: did.into(), + v1: DOMAIN.into(), + v2: object.into(), + v3: role.into(), + } + } + + #[test] + fn decodes_members_owner_and_repo_markers() { + let rows = [ + g("did:plc:nel", "server:owner"), + g("server:owner", "server:member"), + g("did:plc:olaren", "server:member"), + g("did:plc:teq", "server:member"), + p("did:plc:nel", "did:plc:squid", "repo:owner"), + p("did:plc:nel", "did:plc:squid", "repo:push"), + p("did:plc:nel", "did:plc:squid", "repo:settings"), + p("did:plc:nel", "did:plc:squid", "repo:invite"), + p("did:plc:nel", "did:plc:squid", "repo:delete"), + p("did:plc:teq", "did:plc:squid", "repo:collaborator"), + p("did:plc:nel", "", "repo:create"), + p("did:plc:nel", "", "server:invite"), + ]; + let roster = decode(&rows, &SlashResolver::new()).unwrap(); + assert_eq!( + roster.server_owner.as_ref().map(SourceDid::as_str), + Some("did:plc:nel") + ); + assert_eq!(roster.members.len(), 2); + assert_eq!( + roster.owner_markers["did:plc:squid"], + BTreeSet::from([SourceDid::from_column("did:plc:nel")]) + ); + assert_eq!( + roster.collaborators["did:plc:squid"], + BTreeSet::from([SourceDid::from_column("did:plc:teq")]) + ); + } + + #[test] + fn slash_forms_resolve_through_repo_keys_unless_ambiguous() { + let resolve = resolver( + [ + ("did:plc:nel", "anemone", "did:plc:limpet"), + ("did:plc:isabel", "mussel", "did:plc:whelk"), + ("did:plc:isabel", "mussel", "did:plc:conch"), + ] + .into_iter() + .map(|(owner, name, repo)| { + ( + SourceDid::from_column(owner), + SourceRepoName::from_column(name), + SourceRepoDid::from_column(repo), + ) + }), + ); + assert_eq!( + resolve[&( + SourceDid::from_column("did:plc:isabel"), + SourceRepoName::from_column("mussel") + )], + SlashTarget::Ambiguous + ); + + let roster = decode( + &[ + p("did:plc:nel", "did:plc:nel/anemone", "repo:owner"), + p("did:plc:isabel", "did:plc:nel/anemone", "repo:collaborator"), + p("did:plc:nel", "did:plc:nel/vanished", "repo:owner"), + p("did:plc:teq", "did:plc:isabel/mussel", "repo:collaborator"), + ], + &resolve, + ) + .unwrap(); + assert_eq!(roster.slash_owner_markers, 2); + assert_eq!(roster.slash_collab_rows, 2); + assert!(roster.owner_markers.is_empty()); + assert!(roster.collaborators.is_empty()); + assert!(roster.slash_collaborators["did:plc:limpet"].contains("did:plc:isabel")); + assert_eq!( + roster.slash_collaborators.len(), + 1, + "an ambiguous slash form grants nobody" + ); + assert_eq!( + roster.unresolved_slash_forms, + ["did:plc:nel/vanished", "did:plc:isabel/mussel"] + ); + } + + #[test] + fn decode_rejects_unknown_roles_and_two_server_owners() { + assert!(matches!( + decode( + &[p("did:plc:nel", "did:plc:squid", "repo:mystery")], + &SlashResolver::new() + ), + Err(CasbinError::UnrecognizedRow { .. }) + )); + assert!(matches!( + decode( + &[ + g("did:plc:nel", "server:owner"), + g("did:plc:bailey", "server:owner"), + ], + &SlashResolver::new() + ), + Err(CasbinError::TwoServerOwners { .. }) + )); + } +} diff --git a/knot2/crates/knot-migrate/src/emit.rs b/knot2/crates/knot-migrate/src/emit.rs new file mode 100644 index 0000000000000000000000000000000000000000..e29bc5c9c11e56d145877a43829cd6f33079a1d6 --- /dev/null +++ b/knot2/crates/knot-migrate/src/emit.rs @@ -0,0 +1,774 @@ +use std::path::{Path, PathBuf}; + +use knot_cob::{Checkpoint, CobError, CobHome, CobStore, Evaluate, KnotAuthored}; +use knot_cobs::{ + CollaboratorsChange, CollaboratorsCob, Grant, MembersChange, MembersCob, Registration, + RegistrationField, RegistryChange, RegistryError, RepoRegistryCob, +}; +use knot_git::{GitError, Layout}; +use knot_runtime::Signer; +use knot_secrets::{MasterKey, SecretsError}; +use knot_types::{AccountDid, ActorId, KnotHostname, KnotId, ObjectFormat, RepoDid, RepoName}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use url::Url; + +use crate::adopt; +use crate::mapping::{AdoptRepo, MappedGrant, Mapping}; + +#[derive(Debug, thiserror::Error)] +pub enum EmitError { + #[error("Meta-repo bootstrap: {0}")] + Meta(#[from] GitError), + #[error("Open adopted repo {repo}: {source}")] + OpenRepo { repo: RepoDid, source: GitError }, + #[error("Registry record for {repo} has name {existing:?} and the mapping has {name:?}")] + RegistryNameChanged { + repo: RepoDid, + existing: RepoName, + name: RepoName, + }, + #[error("{cob} write failed: {source}")] + Cob { + cob: &'static str, + source: Box, + }, + #[error("Registry write failed: {0}")] + Registry(Box), + #[error("{cob} is split across {count} objects")] + SplitObject { cob: &'static str, count: usize }, + #[error("Write {path}: {source}")] + Io { + path: PathBuf, + source: std::io::Error, + }, + #[error("Read host key {path}: {source}")] + ReadHostKey { + path: PathBuf, + source: std::io::Error, + }, + #[error( + "Host key {path} is the public half of a key pair. Point --host-key at the private key, usually the same path without the .pub." + )] + PublicHostKey { path: PathBuf }, + #[error("Host key {path} doesn't parse as an OpenSSH private key: {source}")] + HostKey { + path: PathBuf, + source: ssh_key::Error, + }, + #[error("Knot can't load the passphrase-protected host key {path} unattended")] + EncryptedHostKey { path: PathBuf }, + #[error("Config template has no line for {section}.{key}")] + TemplateDrift { + section: &'static str, + key: &'static str, + }, + #[error("{cob} is missing {missing} entries after append")] + Incomplete { cob: &'static str, missing: usize }, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct GrantSetOutcome { + pub appended: u64, + pub already_present: u64, +} + +pub struct CobSummary { + pub members: GrantSetOutcome, + pub registrations: GrantSetOutcome, + pub collaborators: GrantSetOutcome, +} + +pub fn write_cobs( + layout: &Layout, + knot: &KnotId, + mapping: &Mapping, + signer: &dyn Signer, +) -> Result { + let meta = layout.bootstrap_meta(knot)?; + let home = CobHome::from(knot); + let store = CobStore::new(&meta); + + let members = write_grant_set::( + &store, + &home, + "members", + &mapping.members, + signer, + MembersChange::Add, + )?; + let registrations = write_registry(&store, &home, &mapping.repos, signer)?; + let collaborators = + mapping + .repos + .iter() + .try_fold(GrantSetOutcome::default(), |outcome, repo| { + let sum = write_repo_collaborators(layout, repo, signer)?; + Ok::<_, EmitError>(GrantSetOutcome { + appended: outcome.appended + sum.appended, + already_present: outcome.already_present + sum.already_present, + }) + })?; + + Ok(CobSummary { + members, + registrations, + collaborators, + }) +} + +fn write_repo_collaborators( + layout: &Layout, + repo: &AdoptRepo, + signer: &dyn Signer, +) -> Result { + if repo.collaborators.is_empty() { + return Ok(GrantSetOutcome::default()); + } + let git = layout + .open(&repo.did) + .map_err(|source| EmitError::OpenRepo { + repo: repo.did.clone(), + source, + })?; + let home = CobHome::from(&repo.did); + let store = CobStore::new(&git); + write_grant_set::( + &store, + &home, + "collaborators", + &repo.collaborators, + signer, + CollaboratorsChange::Add, + ) +} + +fn write_grant_set( + store: &CobStore, + home: &CobHome, + cob: &'static str, + grants: &[MappedGrant], + signer: &dyn Signer, + make: impl Fn(Grant) -> E::Change, +) -> Result +where + E: Checkpoint + Evaluate, + E::State: Serialize + DeserializeOwned, + E::Change: KnotAuthored + Clone, +{ + write_batch::( + store, + home, + cob, + grants, + signer, + |grant| make(to_grant(grant)), + |grant| grant.created_at, + |list, grant| list.contains(&grant.subject), + |_| Ok(()), + ) +} + +fn write_registry( + store: &CobStore, + home: &CobHome, + repos: &[AdoptRepo], + signer: &dyn Signer, +) -> Result { + write_batch::( + store, + home, + "registry", + repos, + signer, + |repo| RegistryChange::Register(registration(repo)), + |repo| repo.created_at, + |registry, repo| { + registry.record_of(&repo.did).is_some_and(|record| { + record.owner == repo.owner && record.rkey == repo.rkey && record.name == repo.name + }) + }, + |registry| { + repos.iter().try_for_each(|repo| { + match ( + registry.record_of(&repo.did), + registry.resolve(&repo.owner, &repo.rkey), + ) { + (Some(record), _) if record.owner != repo.owner => Err(EmitError::Registry( + Box::new(RegistryError::AlreadyRegistered { + repo: repo.did.clone(), + owner: record.owner.clone(), + rkey: record.rkey.clone(), + field: RegistrationField::Owner, + }), + )), + (Some(record), _) if record.rkey != repo.rkey => Err(EmitError::Registry( + Box::new(RegistryError::AlreadyRegistered { + repo: repo.did.clone(), + owner: record.owner.clone(), + rkey: record.rkey.clone(), + field: RegistrationField::Rkey, + }), + )), + (Some(record), _) if record.name != repo.name => { + Err(EmitError::RegistryNameChanged { + repo: repo.did.clone(), + existing: record.name.clone(), + name: repo.name.clone(), + }) + } + (None, Some(holder)) if holder != &repo.did => { + Err(EmitError::Registry(Box::new(RegistryError::RkeyTaken { + owner: repo.owner.clone(), + rkey: repo.rkey.clone(), + existing: holder.clone(), + }))) + } + _ => Ok(()), + } + }) + }, + ) +} + +#[allow(clippy::too_many_arguments)] +fn write_batch( + store: &CobStore, + home: &CobHome, + cob: &'static str, + items: &[T], + signer: &dyn Signer, + make: impl Fn(&T) -> E::Change, + stamp: impl Fn(&T) -> knot_types::UnixSeconds, + present: impl Fn(&E::State, &T) -> bool, + precheck: impl Fn(&E::State) -> Result<(), EmitError>, +) -> Result +where + E: Checkpoint, + E::State: Serialize + DeserializeOwned, + E::Change: KnotAuthored + Clone, +{ + let fail = |source: CobError| EmitError::Cob { + cob, + source: Box::new(source), + }; + let objects = store.list::().map_err(fail)?; + let (object, state, created) = match (objects.as_slice(), items) { + (_, []) => return Ok(GrantSetOutcome::default()), + ([], [first, ..]) => { + let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); + let change = make(first); + let state = E::apply(E::initial(), change.clone(), &author); + precheck(&state)?; + let created = store + .create(home, &change, signer, stamp(first)) + .map_err(fail)?; + (created.object, state, 1) + } + ([object], _) => { + let (state, _) = store.materialize::(*object).map_err(fail)?; + precheck(&state)?; + (*object, state, 0) + } + (many, _) => { + return Err(EmitError::SplitObject { + cob, + count: many.len(), + }); + } + }; + + let missing: Vec<&T> = items.iter().filter(|item| !present(&state, item)).collect(); + missing.split_last().map_or(Ok(()), |(last, head)| { + let changes: Vec = head.iter().map(|item| make(item)).collect(); + store + .extend( + home, + object, + changes.iter().zip(head.iter().map(|item| stamp(item))), + signer, + ) + .map_err(fail)?; + store + .update_with_checkpointed::(home, object, signer, stamp(last), |_| { + Ok(make(last)) + }) + .map(|_| ()) + .map_err(fail) + })?; + + let (list, _) = store.materialize::(object).map_err(fail)?; + let absent = items.iter().filter(|item| !present(&list, item)).count(); + match absent { + 0 => Ok(GrantSetOutcome { + appended: created + missing.len() as u64, + already_present: (items.len() as u64) + .saturating_sub(missing.len() as u64) + .saturating_sub(created), + }), + count => Err(EmitError::Incomplete { + cob, + missing: count, + }), + } +} + +fn registration(repo: &AdoptRepo) -> Registration { + Registration { + owner: repo.owner.clone(), + rkey: repo.rkey.clone(), + name: repo.name.clone(), + repo: repo.did.clone(), + created_at: repo.created_at, + policy: knot_types::ContributionPolicy::default(), + fostered: false, + + retired_at: None, + } +} + +fn to_grant(grant: &MappedGrant) -> Grant { + Grant { + subject: grant.subject.clone(), + added_by: grant.added_by.clone(), + created_at: grant.created_at, + } +} + +#[derive(Serialize)] +struct ArchivedKey<'a> { + repo_did: &'a RepoDid, + key_type: &'a str, + #[serde(serialize_with = "secret_str")] + secret_key_hex: zeroize::Zeroizing, +} + +fn secret_str( + value: &zeroize::Zeroizing, + serializer: S, +) -> Result { + serializer.serialize_str(value) +} + +// do not be alarmed, for there is a plan for this +pub fn write_key_archive(path: &Path, repos: &[AdoptRepo]) -> Result<(), EmitError> { + let keys: Vec> = repos + .iter() + .map(|repo| ArchivedKey { + repo_did: &repo.did, + key_type: "k256", + secret_key_hex: repo.signing_key.to_hex(), + }) + .collect(); + let body = zeroize::Zeroizing::new( + serde_json::to_string_pretty(&keys).expect("Key archive serializes"), + ); + write_private(path, body.as_bytes()) +} + +pub struct HostKey { + bytes: zeroize::Zeroizing>, + pub algorithm: ssh_key::Algorithm, + pub fingerprint: ssh_key::Fingerprint, +} + +impl HostKey { + pub fn write_to(&self, destination: &Path) -> Result<(), EmitError> { + write_private(destination, &self.bytes) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HostKeyPolicy { + Keep, + Replace, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HostKeyPlacement { + Fresh, + Unchanged, + Replacing, +} + +#[derive(Debug)] +pub struct Fingerprints { + pub found: ssh_key::Fingerprint, + pub importing: ssh_key::Fingerprint, +} + +#[derive(Debug, thiserror::Error)] +pub enum HostKeyConflict { + #[error( + "The migration will replace the different host key at {path}, whose fingerprint {} your users already trust, with {}", + .fingerprints.found, + .fingerprints.importing + )] + Different { + path: PathBuf, + fingerprints: Box, + }, + #[error("The migration won't write over the key already at the target: {source}")] + Unparsable { source: EmitError }, + #[error( + "Read {path}, which this process can't examine and the old knot might still be serving: {source}" + )] + Unreadable { + path: PathBuf, + source: std::io::Error, + }, + #[error("{path} isn't a regular file, so the migration won't write the host key over it")] + NotAFile { path: PathBuf }, + #[error("Examine {path}: {source}")] + Unexaminable { + path: PathBuf, + source: std::io::Error, + }, + #[error( + "The migration will write the host key over {path}, which this process can't write: {source}" + )] + Unwritable { + path: PathBuf, + source: rustix::io::Errno, + }, + #[error( + "the migration will write the host key to {path} under {blocked}, which this process can't write: {source}" + )] + Uncreatable { + path: PathBuf, + blocked: PathBuf, + source: rustix::io::Errno, + }, +} + +enum Occupant { + Absent, + NotAFile, + Unexaminable(std::io::Error), + Unreadable(std::io::Error), + Unparsable(EmitError), + SameKey, + DifferentKey(ssh_key::Fingerprint), +} + +fn occupant(destination: &Path, key: &HostKey) -> Occupant { + match std::fs::symlink_metadata(destination) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Occupant::Absent, + Err(error) => Occupant::Unexaminable(error), + Ok(metadata) if !metadata.is_file() => Occupant::NotAFile, + Ok(_) => match load_host_key(destination) { + Err(EmitError::ReadHostKey { source, .. }) => Occupant::Unreadable(source), + Err(error) => Occupant::Unparsable(error), + Ok(found) => match found.fingerprint == key.fingerprint { + true => Occupant::SameKey, + false => Occupant::DifferentKey(found.fingerprint), + }, + }, + } +} + +pub fn plan_host_key( + destination: &Path, + key: &HostKey, + policy: HostKeyPolicy, +) -> Result { + let path = destination.to_path_buf(); + let placement = match (occupant(destination, key), policy) { + (Occupant::Absent, _) => Ok(HostKeyPlacement::Fresh), + (Occupant::SameKey, _) => Ok(HostKeyPlacement::Unchanged), + ( + Occupant::DifferentKey(_) | Occupant::Unparsable(_) | Occupant::Unreadable(_), + HostKeyPolicy::Replace, + ) => Ok(HostKeyPlacement::Replacing), + (Occupant::DifferentKey(found), HostKeyPolicy::Keep) => Err(HostKeyConflict::Different { + path, + fingerprints: Box::new(Fingerprints { + found, + importing: key.fingerprint, + }), + }), + (Occupant::Unparsable(source), HostKeyPolicy::Keep) => { + Err(HostKeyConflict::Unparsable { source }) + } + (Occupant::Unreadable(source), HostKeyPolicy::Keep) => { + Err(HostKeyConflict::Unreadable { path, source }) + } + (Occupant::NotAFile, _) => Err(HostKeyConflict::NotAFile { path }), + (Occupant::Unexaminable(source), _) => Err(HostKeyConflict::Unexaminable { path, source }), + }?; + writable_target(destination, placement).map(|()| placement) +} + +fn writable_target(destination: &Path, placement: HostKeyPlacement) -> Result<(), HostKeyConflict> { + match placement { + HostKeyPlacement::Fresh => { + let blocked = match destination.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + }; + match adopt::writable(blocked) { + Ok(()) => Ok(()), + Err(source) if source == rustix::io::Errno::NOENT => Ok(()), + Err(source) => Err(HostKeyConflict::Uncreatable { + path: destination.to_path_buf(), + blocked: blocked.to_path_buf(), + source, + }), + } + } + HostKeyPlacement::Unchanged | HostKeyPlacement::Replacing => rustix::fs::accessat( + rustix::fs::CWD, + destination, + rustix::fs::Access::WRITE_OK, + rustix::fs::AtFlags::EACCESS, + ) + .map_err(|source| HostKeyConflict::Unwritable { + path: destination.to_path_buf(), + source, + }), + } +} + +pub fn load_host_key(source: &Path) -> Result { + let bytes = + zeroize::Zeroizing::new( + std::fs::read(source).map_err(|error| EmitError::ReadHostKey { + path: source.to_path_buf(), + source: error, + })?, + ); + let key = ssh_key::PrivateKey::from_openssh(bytes.as_slice()).map_err(|error| { + match std::str::from_utf8(bytes.as_slice()) + .ok() + .is_some_and(|text| ssh_key::PublicKey::from_openssh(text).is_ok()) + { + true => EmitError::PublicHostKey { + path: source.to_path_buf(), + }, + false => EmitError::HostKey { + path: source.to_path_buf(), + source: error, + }, + } + })?; + if key.is_encrypted() { + return Err(EmitError::EncryptedHostKey { + path: source.to_path_buf(), + }); + } + Ok(HostKey { + algorithm: key.algorithm(), + fingerprint: key.fingerprint(ssh_key::HashAlg::Sha256), + bytes, + }) +} + +fn write_private(path: &Path, bytes: &[u8]) -> Result<(), EmitError> { + use std::io::Write; + let io = |error: std::io::Error| EmitError::Io { + path: path.to_path_buf(), + source: error, + }; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + options.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32); + } + let mut file = options.open(path).map_err(io)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode(0o600)) + .map_err(io)?; + } + file.write_all(bytes).map_err(io)?; + file.sync_all().map_err(io)?; + Ok(()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +// This name goes into the generated config as `secrets.master_key_env`, +// where knot-config runs `is_env_var_name`. +// So same rule here such the migration fails *now* instead of at the end. +pub struct MasterKeyEnv(String); + +impl MasterKeyEnv { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let valid = !value.is_empty() + && !value.starts_with(|c: char| c.is_ascii_digit()) + && value + .chars() + .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_'); + match valid { + true => Ok(Self(value)), + false => Err(value), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn read(&self) -> Result { + let value = zeroize::Zeroizing::new( + std::env::var_os(&self.0) + .ok_or_else(|| MasterKeyError::Unset(self.clone()))? + .into_string() + .map_err(|_| MasterKeyError::NotText(self.clone()))?, + ); + self.decode(&value) + } + + pub fn decode(&self, value: &str) -> Result { + use base64::Engine; + let bytes = zeroize::Zeroizing::new( + base64::engine::general_purpose::STANDARD + .decode(value.trim()) + .map_err(|_| MasterKeyError::NotBase64(self.clone()))?, + ); + MasterKey::new(bytes.as_slice()).map_err(|source| MasterKeyError::Weak { + env: self.clone(), + source, + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum MasterKeyError { + #[error("Master key env var {0} isn't set")] + Unset(MasterKeyEnv), + #[error("Master key env var {0} is set to bytes that aren't text")] + NotText(MasterKeyEnv), + #[error("Master key env var {0} isn't base64")] + NotBase64(MasterKeyEnv), + #[error("{source}, decoded from master key env var {env}")] + Weak { + env: MasterKeyEnv, + source: SecretsError, + }, +} + +impl std::fmt::Display for MasterKeyEnv { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +pub struct ConfigValues { + pub hostname: KnotHostname, + pub admins: Vec, + pub contact_email: String, + pub scan_path: PathBuf, + pub lfs_store_path: PathBuf, + pub ssh_host_key_file: PathBuf, + pub sealed_key_file: PathBuf, + pub master_key_env: MasterKeyEnv, + pub object_format: ObjectFormat, + pub plc_directory: Url, +} + +pub fn render_config(values: &ConfigValues) -> Result { + let fills: Vec<((&'static str, &'static str), String)> = [ + (("server", "hostname"), quote(values.hostname.as_str())), + ( + ("server", "admins"), + format!( + "[{}]", + values + .admins + .iter() + .map(|admin| quote(admin.as_str())) + .collect::>() + .join(", ") + ), + ), + ( + ("server", "ssh_host_key_file"), + quote_path(&values.ssh_host_key_file), + ), + (("acl", "admission"), quote("closed")), + (("repo", "scan_path"), quote_path(&values.scan_path)), + (("lfs", "store_path"), quote_path(&values.lfs_store_path)), + ( + ("git", "object_format"), + quote(values.object_format.capability()), + ), + ( + ("secrets", "sealed_key_file"), + quote_path(&values.sealed_key_file), + ), + ( + ("secrets", "master_key_env"), + quote(values.master_key_env.as_str()), + ), + ( + ("atproto", "plc_directory"), + quote(values.plc_directory.as_str()), + ), + (("contact", "email"), quote(&values.contact_email)), + ] + .into_iter() + .collect(); + + let template = knot_config::template(); + let (lines, pending, _) = template.lines().fold( + (Vec::new(), fills, ""), + |(mut lines, pending, section), line| { + let section = line + .trim() + .strip_prefix('[') + .and_then(|rest| rest.strip_suffix(']')) + .unwrap_or(section); + let matched = pending.iter().position(|((expected, key), _)| { + *expected == section && line.trim().starts_with(&format!("#{key} =")) + }); + let remaining = match matched { + Some(index) => { + let ((_, key), value) = &pending[index]; + lines.push(format!("{key} = {value}")); + pending + .into_iter() + .enumerate() + .filter(|(position, _)| *position != index) + .map(|(_, fill)| fill) + .collect() + } + None => { + lines.push(line.to_string()); + pending + } + }; + (lines, remaining, section) + }, + ); + pending.first().map_or(Ok(()), |((section, key), _)| { + Err(EmitError::TemplateDrift { section, key }) + })?; + Ok(lines.join("\n") + "\n") +} + +fn quote(value: &str) -> String { + let escaped: String = value + .chars() + .map(|c| match c { + '"' => "\\\"".to_string(), + '\\' => "\\\\".to_string(), + '\u{8}' => "\\b".to_string(), + '\t' => "\\t".to_string(), + '\n' => "\\n".to_string(), + '\u{c}' => "\\f".to_string(), + '\r' => "\\r".to_string(), + c if c.is_control() => format!("\\u{:04X}", u32::from(c)), + c => c.to_string(), + }) + .collect(); + format!("\"{escaped}\"") +} + +fn quote_path(path: &Path) -> String { + quote(&path.to_string_lossy()) +} diff --git a/knot2/crates/knot-migrate/src/envfile.rs b/knot2/crates/knot-migrate/src/envfile.rs new file mode 100644 index 0000000000000000000000000000000000000000..e7ac825e8d51844246a75b4a79bfff409629ad83 --- /dev/null +++ b/knot2/crates/knot-migrate/src/envfile.rs @@ -0,0 +1,114 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +#[derive(Debug, thiserror::Error)] +pub enum EnvFileError { + #[error("read env file {path}: {source}")] + Io { + path: PathBuf, + source: std::io::Error, + }, +} + +#[derive(Debug, Default)] +pub struct EnvFile { + values: BTreeMap, +} + +impl EnvFile { + pub fn read(path: &Path) -> Result { + let body = std::fs::read_to_string(path).map_err(|source| EnvFileError::Io { + path: path.to_path_buf(), + source, + })?; + Ok(Self::parse(&body)) + } + + pub fn parse(body: &str) -> Self { + let values = body + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(|line| line.strip_prefix("export ").unwrap_or(line)) + .filter_map(|line| line.split_once('=')) + .filter_map(|(key, value)| unquote(value).map(|value| (key.trim().to_string(), value))) + .collect(); + Self { values } + } + + pub fn get(&self, key: &str) -> Option<&str> { + self.values.get(key).map(String::as_str) + } +} + +fn unquote(raw: &str) -> Option { + let trimmed = raw.trim(); + ['"', '\''] + .into_iter() + .find_map(|quote| trimmed.strip_prefix(quote).map(|rest| (quote, rest))) + .map_or_else( + || Some(strip_comment(trimmed).to_string()), + |(quote, rest)| rest.split_once(quote).map(|(inner, _)| inner.to_string()), + ) +} + +fn strip_comment(value: &str) -> &str { + value + .match_indices('#') + .find(|(index, _)| { + value[..*index] + .chars() + .next_back() + .is_some_and(char::is_whitespace) + }) + .map_or(value, |(index, _)| &value[..index]) + .trim_end() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parsing_keeps_assignments_and_drops_every_other_line() { + let file = EnvFile::parse( + r#"# knot1 +KNOT_SERVER_HOSTNAME=oyster.cafe +export KNOT_SERVER_OWNER="did:plc:nel" + +KNOT_SERVER_PLC_URL='https://plc.oyster.cafe' +broken line +KNOT_REPO_SCAN_PATH=/data/repos # tangled default +KNOT_QUOTED_HOSTNAME="nel.pet" # quoted +KNOT_SERVER_SECRET='a # b' +KNOT_DEV_FLAGS=#literal +KNOT_APPVIEW_URL=https://tangled.test/#frag +KNOT_UNTERMINATED="oyster.cafe +KNOT_ALSO_UNTERMINATED='did:plc:nel +KNOT_LAST=kept +"#, + ); + assert_eq!(file.get("KNOT_SERVER_HOSTNAME"), Some("oyster.cafe")); + assert_eq!(file.get("KNOT_SERVER_OWNER"), Some("did:plc:nel")); + assert_eq!( + file.get("KNOT_SERVER_PLC_URL"), + Some("https://plc.oyster.cafe") + ); + assert_eq!(file.get("broken"), None); + assert_eq!(file.get("KNOT_REPO_SCAN_PATH"), Some("/data/repos")); + assert_eq!(file.get("KNOT_QUOTED_HOSTNAME"), Some("nel.pet")); + assert_eq!(file.get("KNOT_SERVER_SECRET"), Some("a # b")); + assert_eq!(file.get("KNOT_DEV_FLAGS"), Some("#literal")); + assert_eq!( + file.get("KNOT_APPVIEW_URL"), + Some("https://tangled.test/#frag") + ); + assert_eq!(file.get("KNOT_UNTERMINATED"), None); + assert_eq!(file.get("KNOT_ALSO_UNTERMINATED"), None); + assert_eq!( + file.get("KNOT_LAST"), + Some("kept"), + "an unterminated quote drops its own line and nothing after it" + ); + } +} diff --git a/knot2/crates/knot-migrate/src/lib.rs b/knot2/crates/knot-migrate/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..c89d3ba793a2c3485e110fe5a3fbeb0a44553c9e --- /dev/null +++ b/knot2/crates/knot-migrate/src/lib.rs @@ -0,0 +1,8 @@ +pub mod adopt; +pub mod casbin; +pub mod emit; +pub mod envfile; +pub mod mapping; +pub mod rehearse; +pub mod report; +pub mod source; diff --git a/knot2/crates/knot-migrate/src/main.rs b/knot2/crates/knot-migrate/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..fbb63e57b65c7b5ce148d718a26bda634ba67e79 --- /dev/null +++ b/knot2/crates/knot-migrate/src/main.rs @@ -0,0 +1,700 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use knot_migrate::adopt::{self, AdoptError, SourcePolicy}; +use knot_migrate::casbin::{self, CasbinError}; +use knot_migrate::emit::{ + self, ConfigValues, EmitError, HostKeyConflict, HostKeyPlacement, HostKeyPolicy, MasterKeyEnv, + MasterKeyError, +}; +use knot_migrate::envfile::{EnvFile, EnvFileError}; +use knot_migrate::mapping::{self, Mapping, MappingError, RepoList, SkipReason}; +use knot_migrate::rehearse::{self, Rehearsal}; +use knot_migrate::report::{Phase, Report}; +use knot_migrate::source::{SourceDb, SourceError, SourceRepoDid, SourceRkey, SourceSchema}; +use knot_runtime::OsEntropy; +use knot_secrets::{SealedStore, SecretsError}; +use knot_types::{AccountDid, KnotHostname, ObjectFormat, RepoDid}; +use url::Url; + +// TODO: I wanted to see how well I could work without clap. I shoulda just used clap. +const USAGE: &str = "\ +knot-migrate: offline conversion of a tangled-knot deployment into a knot deployment + +usage: + knot-migrate --source-db --host-key --target

    [options] + +options: + --source-db tangled-knot SQLite database, opened read-only + --source-repos tangled-knot scan path holding directories + defaults to KNOT_REPO_SCAN_PATH from the env file + --env-file tangled-knot environment file + --host-key system sshd host key to import + --target knot data directory to create + --hostname knot hostname, defaults to KNOT_SERVER_HOSTNAME + --plc-url PLC directory, defaults to KNOT_SERVER_PLC_URL + --contact-email knot contact email, defaults to KNOT_CONTACT_EMAIL + --object-format sha1 or sha256 for repos knot creates, default sha1 + --master-key-env env var holding the base64 master key, default KNOT_MASTER_KEY + --consume-source move the source repos into place instead of copying them, + which empties the source tree and needs one filesystem + --skip-unreadable migrate the rest when knot-migrate can't read a source path, + and leave those repos on the old knot without failing the exit + --force-host-key replace the host key already at the target, + whose fingerprint your users already trust + --dry-run print the mapping and reconciliation report, leave the target + alone, and exit non-zero while an input is still missing +"; + +#[derive(Debug, thiserror::Error)] +enum MigrateError { + #[error(transparent)] + Source(#[from] SourceError), + #[error(transparent)] + Casbin(#[from] CasbinError), + #[error(transparent)] + Mapping(#[from] MappingError), + #[error(transparent)] + Adopt(#[from] AdoptError), + #[error(transparent)] + Emit(#[from] EmitError), + #[error(transparent)] + HostKeyConflict(#[from] HostKeyConflict), + #[error(transparent)] + EnvFile(#[from] EnvFileError), + #[error(transparent)] + Git(#[from] knot_git::GitError), + #[error(transparent)] + Secrets(#[from] SecretsError), + #[error("{0}")] + Usage(String), + #[error("env file specifies knot owner {env} while the acl specifies {acl}")] + OwnerMismatch { env: String, acl: String }, + #[error("--hostname {flag} doesn't match the env file's KNOT_SERVER_HOSTNAME {env}")] + HostnameMismatch { flag: String, env: String }, + #[error(transparent)] + MasterKey(#[from] MasterKeyError), + #[error("{0}")] + Refused(Refusals), + #[error("the rehearsal lists what the migration still needs")] + RehearsalIncomplete, + #[error("the migration skipped {count} repos, each with its reason in the skip list above")] + SkippedRepos { count: usize }, + #[error("{context}: {source}")] + Io { + context: String, + source: std::io::Error, + }, +} + +#[derive(Debug, thiserror::Error)] +enum Refusal { + #[error( + "knot-migrate can't read the source path of {repos}. Each repo is in the skip list above with the error behind it. Re-run as root, or as a user in the group that owns those trees, when that error is permission denied. Pass --skip-unreadable to migrate everything else and leave those repos on the old knot." + )] + UnreadableSources { repos: RepoList }, + #[error( + "the acl and repo_keys are recorded with different owners for {repos}. Both DIDs per repo are in the drift section above. Settle each repo in the old knot's database, by deleting the acl rows for the wrong owner or by correcting repo_keys.owner_did, since knot-migrate won't pick a winner for you." + )] + ConflictingOwners { repos: RepoList }, +} + +#[derive(Debug)] +struct Refusals(Vec); + +impl Refusals { + fn gather(mapping: &Mapping, skip_unreadable: bool) -> Self { + let conflicting = mapping.conflicting_owners(); + let unreadable = mapping.unreadable_sources(); + Self( + [ + (!conflicting.is_empty()) + .then_some(Refusal::ConflictingOwners { repos: conflicting }), + (!unreadable.is_empty() && !skip_unreadable) + .then_some(Refusal::UnreadableSources { repos: unreadable }), + ] + .into_iter() + .flatten() + .collect(), + ) + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl std::fmt::Display for Refusals { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0 + .iter() + .enumerate() + .try_for_each(|(position, refusal)| match position { + 0 => write!(f, "{refusal}"), + _ => write!(f, "\n{refusal}"), + }) + } +} + +struct Args { + source_db: PathBuf, + source_repos: Option, + env_file: Option, + host_key: Option, + target: PathBuf, + hostname: Option, + plc_url: Option, + contact_email: Option, + object_format: ObjectFormat, + master_key_env: MasterKeyEnv, + source_policy: SourcePolicy, + host_key_policy: HostKeyPolicy, + skip_unreadable: bool, + dry_run: bool, +} + +#[derive(Debug, Default, Clone, Copy)] +struct Switches { + dry_run: bool, + consume_source: bool, + skip_unreadable: bool, + force_host_key: bool, +} + +const KNOWN_FLAGS: [&str; 10] = [ + "source-db", + "source-repos", + "env-file", + "host-key", + "target", + "hostname", + "plc-url", + "contact-email", + "object-format", + "master-key-env", +]; + +fn parse_args(args: &[String]) -> Result { + let (mut flags, switches, pending) = args.iter().try_fold( + ( + BTreeMap::::new(), + Switches::default(), + None::, + ), + |(mut flags, switches, pending), arg| match (pending, arg.as_str()) { + (Some(key), value) if value.starts_with("--") => { + Err(MigrateError::Usage(format!("--{key} needs a value"))) + } + (Some(key), value) => match flags.insert(key.clone(), value.to_string()) { + None => Ok((flags, switches, None)), + Some(_) => Err(MigrateError::Usage(format!("--{key} given twice"))), + }, + (None, "--dry-run") => Ok(( + flags, + Switches { + dry_run: true, + ..switches + }, + None, + )), + (None, "--consume-source") => Ok(( + flags, + Switches { + consume_source: true, + ..switches + }, + None, + )), + (None, "--skip-unreadable") => Ok(( + flags, + Switches { + skip_unreadable: true, + ..switches + }, + None, + )), + (None, "--force-host-key") => Ok(( + flags, + Switches { + force_host_key: true, + ..switches + }, + None, + )), + (None, flag) => match flag.strip_prefix("--").map(|rest| { + rest.split_once('=') + .map_or((rest, None), |(key, value)| (key, Some(value))) + }) { + Some((key, None)) if KNOWN_FLAGS.contains(&key) => { + Ok((flags, switches, Some(key.to_string()))) + } + Some((key, Some(value))) if KNOWN_FLAGS.contains(&key) => { + match flags.insert(key.to_string(), value.to_string()) { + None => Ok((flags, switches, None)), + Some(_) => Err(MigrateError::Usage(format!("--{key} given twice"))), + } + } + _ => Err(MigrateError::Usage(format!("unexpected argument {flag}"))), + }, + }, + )?; + pending.map_or(Ok(()), |key| { + Err(MigrateError::Usage(format!("--{key} needs a value"))) + })?; + let mut take = |key: &str| flags.remove(key); + let required = |key: &str, value: Option| { + value.ok_or_else(|| MigrateError::Usage(format!("--{key} is required"))) + }; + let object_format = take("object-format").map_or(Ok(ObjectFormat::SHA1), |value| { + ObjectFormat::from_capability(&value).ok_or_else(|| { + MigrateError::Usage(format!( + "--object-format must be sha1 or sha256, not {value}" + )) + }) + })?; + Ok(Args { + source_db: required("source-db", take("source-db"))?.into(), + source_repos: take("source-repos").map(PathBuf::from), + env_file: take("env-file").map(PathBuf::from), + host_key: take("host-key").map(PathBuf::from), + target: required("target", take("target"))?.into(), + hostname: take("hostname"), + plc_url: take("plc-url"), + contact_email: take("contact-email"), + object_format, + master_key_env: take("master-key-env") + .map_or_else(|| MasterKeyEnv::new("KNOT_MASTER_KEY"), MasterKeyEnv::new) + .map_err(|value| { + MigrateError::Usage(format!( + "--master-key-env must be an uppercase env var name, not {value}" + )) + })?, + source_policy: match switches.consume_source { + true => SourcePolicy::Consume, + false => SourcePolicy::Preserve, + }, + host_key_policy: match switches.force_host_key { + true => HostKeyPolicy::Replace, + false => HostKeyPolicy::Keep, + }, + skip_unreadable: switches.skip_unreadable, + dry_run: switches.dry_run, + }) +} + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + if args.is_empty() || args.iter().any(|arg| arg == "--help" || arg == "-h") { + print!("{USAGE}"); + return ExitCode::SUCCESS; + } + match run(&args) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("error: {error}"); + ExitCode::FAILURE + } + } +} + +fn run(args: &[String]) -> Result<(), MigrateError> { + let args = parse_args(args)?; + let env = args + .env_file + .as_deref() + .map(EnvFile::read) + .transpose()? + .unwrap_or_default(); + + args.hostname + .as_deref() + .zip(env.get("KNOT_SERVER_HOSTNAME")) + .filter(|(flag, env_value)| flag != env_value) + .map_or(Ok(()), |(flag, env_value)| { + Err(MigrateError::HostnameMismatch { + flag: flag.to_string(), + env: env_value.to_string(), + }) + })?; + let hostname = args + .hostname + .clone() + .or_else(|| env.get("KNOT_SERVER_HOSTNAME").map(str::to_string)) + .ok_or_else(|| { + MigrateError::Usage( + "--hostname is required when the env file specifies none".to_string(), + ) + })?; + let hostname = KnotHostname::new(hostname.as_str()).map_err(|error| { + MigrateError::Usage(format!("hostname {hostname} isn't valid: {error}")) + })?; + let plc_url = args + .plc_url + .clone() + .or_else(|| env.get("KNOT_SERVER_PLC_URL").map(str::to_string)) + .ok_or_else(|| { + MigrateError::Usage( + "--plc-url is required when the env file specifies none".to_string(), + ) + })?; + let plc_directory = Url::parse(&plc_url) + .ok() + .filter(|url| url.scheme() == "https" && url.host().is_some()) + .ok_or_else(|| { + MigrateError::Usage(format!("PLC directory {plc_url} isn't an https URL")) + })?; + let contact_email = args + .contact_email + .clone() + .or_else(|| env.get("KNOT_CONTACT_EMAIL").map(str::to_string)) + .ok_or_else(|| { + MigrateError::Usage( + "--contact-email is required when the env file specifies none".to_string(), + ) + })?; + if !knot_config::is_contact_email(&contact_email) { + return Err(MigrateError::Usage(format!( + "--contact-email must be a contact email, not {contact_email}" + ))); + } + let source_repos = args + .source_repos + .clone() + .or_else(|| env.get("KNOT_REPO_SCAN_PATH").map(PathBuf::from)) + .ok_or_else(|| { + MigrateError::Usage( + "--source-repos is required when the env file specifies no scan path".to_string(), + ) + })?; + match source_repos.is_dir() { + true => Ok(()), + false => Err(MigrateError::Usage(format!( + "source repos path {} isn't a directory", + source_repos.display() + ))), + }?; + + let db = SourceDb::open(&args.source_db)?; + let schema = db.schema()?; + let repos = db.repos()?; + let rkeys = repos + .iter() + .filter_map(|repo| { + db.current_rkey(&repo.repo_did) + .map(|rkey| rkey.map(|rkey| (repo.repo_did.clone(), rkey))) + .transpose() + }) + .collect::, SourceError>>()?; + let resolver = casbin::resolver(repos.iter().map(|repo| { + ( + repo.owner_did.clone(), + repo.repo_name.clone(), + repo.repo_did.clone(), + ) + })); + let acl = casbin::decode(&db.acl()?, &resolver)?; + let probe = |repo_did: &SourceRepoDid| adopt::probe_source(&source_repos, repo_did); + let mapping = match schema { + SourceSchema::Tables => mapping::map_tables( + &repos, + &rkeys, + &db.members()?, + &db.collaborators()?, + &acl, + probe, + )?, + SourceSchema::PreFlip => mapping::map_preflip(&repos, &rkeys, &db.members()?, &acl, probe)?, + }; + env.get("KNOT_SERVER_OWNER") + .filter(|owner| AccountDid::new(*owner).ok().as_ref() != Some(&mapping.knot_owner)) + .map_or(Ok(()), |owner| { + Err(MigrateError::OwnerMismatch { + env: owner.to_string(), + acl: mapping.knot_owner.to_string(), + }) + })?; + let orphan_alias_count = db.orphan_alias_count()?; + + let refusals = Refusals::gather(&mapping, args.skip_unreadable); + match (args.dry_run, refusals.is_empty()) { + (true, ready) => { + let rehearsal = timed("rehearsal", || { + Rehearsal::run(rehearse::Inputs { + source_repos: &source_repos, + adopted: &mapping.repos, + scan_path: &repos_dir(&args.target), + policy: args.source_policy, + host_key: args.host_key.as_deref(), + host_key_target: &host_key_file(&args.target), + host_key_policy: args.host_key_policy, + master_key: &args.master_key_env, + }) + }); + report(&mapping, orphan_alias_count, Phase::Rehearsed(&rehearsal)); + match ready { + false => Err(MigrateError::Refused(refusals)), + true => rehearsal + .ready() + .then(|| { + println!(); + println!("we left the target alone. Re-run without --dry-run to migrate."); + }) + .ok_or(MigrateError::RehearsalIncomplete), + } + } + (false, false) => { + report(&mapping, orphan_alias_count, Phase::Refused); + Err(MigrateError::Refused(refusals)) + } + (false, true) => { + let written = materialize( + &args, + &hostname, + &plc_directory, + &contact_email, + &source_repos, + &mapping, + )?; + report( + &mapping, + orphan_alias_count, + Phase::Written { + adoption: &written.adoption, + cobs: &written.cobs, + }, + ); + println!(); + println!("knot key identity: {}", written.knot_did); + println!("host key algorithm: {}", written.host_key_algorithm); + println!("host key fingerprint: {}", written.host_key_fingerprint); + match written.host_key_placement { + HostKeyPlacement::Fresh => (), + HostKeyPlacement::Unchanged => { + println!("host key: this key was already at the target") + } + HostKeyPlacement::Replacing => { + println!("host key: the migration replaced the different key at the target") + } + } + println!("config: {}", written.config_file.display()); + println!("key archive: {}", written.archive_file.display()); + let unopted = mapping + .skipped + .iter() + .filter(|skip| !matches!(skip.reason, SkipReason::UnreadableSource { .. })) + .count(); + if unopted == 0 { + Ok(()) + } else { + Err(MigrateError::SkippedRepos { count: unopted }) + } + } + } +} + +fn repos_dir(target: &Path) -> PathBuf { + target.join("repos") +} + +fn host_key_file(target: &Path) -> PathBuf { + target.join("ssh_host_key") +} + +fn report<'a>(mapping: &'a Mapping, orphan_alias_count: u64, phase: Phase<'a>) { + print!( + "{}", + Report { + mapping, + orphan_alias_count, + phase, + } + ); +} + +fn timed(phase: &str, work: impl FnOnce() -> T) -> T { + let started = std::time::Instant::now(); + let outcome = work(); + eprintln!("{phase}: {:.1}s", started.elapsed().as_secs_f64()); + outcome +} + +struct Written { + adoption: adopt::AdoptOutcome, + cobs: emit::CobSummary, + knot_did: knot_types::KnotId, + host_key_algorithm: ssh_key::Algorithm, + host_key_fingerprint: ssh_key::Fingerprint, + host_key_placement: HostKeyPlacement, + config_file: PathBuf, + archive_file: PathBuf, +} + +fn materialize( + args: &Args, + hostname: &KnotHostname, + plc_directory: &Url, + contact_email: &str, + source_repos: &Path, + mapping: &Mapping, +) -> Result { + let host_key_source = args.host_key.as_deref().ok_or_else(|| { + MigrateError::Usage("--host-key is required for the migration".to_string()) + })?; + let host_key = emit::load_host_key(host_key_source)?; + let host_key_placement = emit::plan_host_key( + &host_key_file(&args.target), + &host_key, + args.host_key_policy, + )?; + adopt::transfer_mode( + source_repos, + rehearse::nearest_existing(&repos_dir(&args.target)), + args.source_policy, + )?; + std::fs::create_dir_all(&args.target).map_err(|source| MigrateError::Io { + context: format!("create {}", args.target.display()), + source, + })?; + let target = args + .target + .canonicalize() + .map_err(|source| MigrateError::Io { + context: format!("canonicalize {}", args.target.display()), + source, + })?; + let scan_path = repos_dir(&target); + let lfs_store_path = target.join("lfs"); + let sealed_key_file = target.join("sealed-keys"); + let host_key_destination = host_key_file(&target); + let archive_file = target.join("repo-signing-keys.json"); + let config_file = target.join("config.toml"); + + let knot_did = hostname.knot_did(); + + let master_key = args.master_key_env.read()?; + let secrets = SealedStore::open(sealed_key_file.clone(), &master_key, Box::new(OsEntropy))?; + secrets.ensure(&knot_did)?; + let signer = secrets.signer(&knot_did)?; + + let layout = knot_git::Layout::new(&scan_path) + .with_object_format(args.object_format) + .reserving_meta(&knot_did)?; + let adoption = timed("adoption", || { + adopt::adopt_all(&layout, source_repos, &mapping.repos, args.source_policy) + })?; + let cobs = timed("cobs", || { + emit::write_cobs(&layout, &knot_did, mapping, &signer) + })?; + emit::write_key_archive(&archive_file, &mapping.repos)?; + host_key.write_to(&host_key_destination)?; + std::fs::create_dir_all(&lfs_store_path).map_err(|source| MigrateError::Io { + context: format!("create {}", lfs_store_path.display()), + source, + })?; + + let config = emit::render_config(&ConfigValues { + hostname: hostname.clone(), + admins: vec![mapping.knot_owner.clone()], + contact_email: contact_email.to_string(), + scan_path, + lfs_store_path, + sealed_key_file, + ssh_host_key_file: host_key_destination, + master_key_env: args.master_key_env.clone(), + object_format: args.object_format, + plc_directory: plc_directory.clone(), + })?; + std::fs::write(&config_file, config).map_err(|source| MigrateError::Io { + context: format!("write {}", config_file.display()), + source, + })?; + + Ok(Written { + adoption, + cobs, + knot_did, + host_key_algorithm: host_key.algorithm, + host_key_fingerprint: host_key.fingerprint, + host_key_placement, + config_file, + archive_file, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(list: &[&str]) -> Result { + let owned: Vec = list.iter().map(|arg| arg.to_string()).collect(); + parse_args(&owned) + } + + #[test] + fn accepts_space_and_equals_forms() { + let args = parse(&[ + "--source-db=/data/knotserver.db", + "--target", + "/data/knot", + "--object-format=sha256", + "--dry-run", + ]) + .unwrap(); + assert_eq!(args.source_db, PathBuf::from("/data/knotserver.db")); + assert_eq!(args.target, PathBuf::from("/data/knot")); + assert_eq!(args.object_format, ObjectFormat::SHA256); + assert!(args.dry_run); + } + + #[test] + fn every_switch_is_off_until_it_is_named() { + let off = parse(&["--source-db=/db", "--target=/t"]).unwrap(); + assert!(!off.dry_run); + assert_eq!(off.contact_email, None); + assert!(!off.skip_unreadable); + assert_eq!(off.source_policy, SourcePolicy::Preserve); + assert_eq!( + off.host_key_policy, + HostKeyPolicy::Keep, + "a fingerprint your users already trust survives a migration nobody asked to force" + ); + let on = parse(&[ + "--source-db=/db", + "--target=/t", + "--dry-run", + "--consume-source", + "--skip-unreadable", + "--force-host-key", + ]) + .unwrap(); + assert!(on.dry_run); + assert!(on.skip_unreadable); + assert_eq!(on.source_policy, SourcePolicy::Consume); + assert_eq!(on.host_key_policy, HostKeyPolicy::Replace); + } + + #[test] + fn rejects_duplicates_missing_values_and_unknown_flags() { + [ + &[ + "--source-db=/data/knotserver.db", + "--target=/data/knot", + "--target", + "/data/other", + ][..], + &["--source-db"], + &["--source-db", "--target"], + &["--mystery=1", "--source-db=/data/knotserver.db"], + &["--object-format=blake3", "--source-db=/db", "--target=/t"], + &["--force-host-key=yes", "--source-db=/db", "--target=/t"], + ] + .into_iter() + .for_each(|args| { + assert!( + matches!(parse(args), Err(MigrateError::Usage(_))), + "{args:?} mustn't parse" + ); + }); + } +} diff --git a/knot2/crates/knot-migrate/src/mapping.rs b/knot2/crates/knot-migrate/src/mapping.rs new file mode 100644 index 0000000000000000000000000000000000000000..3816f7ea4233c62692a5536b01d17a97eae0a5f4 --- /dev/null +++ b/knot2/crates/knot-migrate/src/mapping.rs @@ -0,0 +1,743 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; + +use knot_types::{AccountDid, OwnerDid, ParseError, RepoDid, RepoName, RepoRkey, UnixSeconds}; +use zeroize::{Zeroize, ZeroizeOnDrop}; + +use crate::adopt::SourceProbe; +use crate::casbin::AclRoster; +use crate::source::{ + CollabRow, MemberRow, RepoRow, SourceDid, SourceKeyType, SourceRepoDid, SourceRepoName, + SourceRkey, SourceTimestamp, +}; + +#[derive(Debug, thiserror::Error)] +pub enum MappingError { + #[error("repo DID {value} doesn't parse: {source}")] + BadRepoDid { + value: SourceRepoDid, + source: ParseError, + }, + #[error("owner {value} on repo {repo} doesn't parse: {source}")] + BadOwnerDid { + repo: SourceRepoDid, + value: SourceDid, + source: ParseError, + }, + #[error("member DID {value} doesn't parse: {source}")] + BadMemberDid { + value: SourceDid, + source: ParseError, + }, + #[error("collaborator DID {value} on repo {repo} doesn't parse: {source}")] + BadCollaboratorDid { + repo: SourceRepoDid, + value: SourceDid, + source: ParseError, + }, + #[error("{context} timestamp {value} isn't RFC 3339")] + BadTimestamp { + context: &'static str, + value: SourceTimestamp, + }, + #[error("repo {repo} has a {key_type} signing key of {bytes} bytes instead of 32-byte k256")] + BadSigningKey { + repo: SourceRepoDid, + key_type: SourceKeyType, + bytes: usize, + }, + #[error("acl names no server owner")] + MissingServerOwner, + #[error("record key {owner}/{rkey} has no single alias-backed holder")] + AmbiguousRkey { owner: OwnerDid, rkey: RepoRkey }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MappedGrant { + pub subject: AccountDid, + pub added_by: AccountDid, + pub created_at: UnixSeconds, + pub unioned: bool, +} + +#[derive(Clone, PartialEq, Eq, Zeroize, ZeroizeOnDrop)] +pub struct SigningKey([u8; 32]); + +impl fmt::Debug for SigningKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("SigningKey").finish_non_exhaustive() + } +} + +impl SigningKey { + pub fn to_hex(&self) -> zeroize::Zeroizing { + zeroize::Zeroizing::new(knot_types::lowercase_hex(&self.0)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AdoptRepo { + pub source_did: SourceRepoDid, + pub did: RepoDid, + pub owner: OwnerDid, + pub name: RepoName, + pub rkey: RepoRkey, + pub created_at: UnixSeconds, + pub signing_key: SigningKey, + pub collaborators: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SkipReason { + Name { value: SourceRepoName }, + Rkey { value: SourceRkey }, + RkeyCollision { rkey: RepoRkey, winner: RepoDid }, + NoSourceRepo, + UnreadableSource { kind: std::io::ErrorKind }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SkippedRepo { + pub repo_did: RepoDid, + pub reason: SkipReason, + pub lost_collaborators: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OwnerConflict { + pub repo: SourceRepoDid, + pub acl_owner: SourceDid, + pub key_owner: SourceDid, +} + +impl fmt::Display for OwnerConflict { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{} acl {}, repo_keys {}", + self.repo, self.acl_owner, self.key_owner + ) + } +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Drift { + pub acl_only_collaborators: Vec<(SourceRepoDid, SourceDid)>, + pub table_only_collaborators: Vec<(SourceRepoDid, SourceDid)>, + pub slash_resolved_collaborators: Vec<(SourceRepoDid, SourceDid)>, + pub orphan_collaborator_pairs: Vec<(SourceRepoDid, SourceDid)>, + pub markerless_owner_repos: Vec, + pub orphan_owner_markers: Vec, + pub conflicting_owner_markers: Vec, + pub extra_owner_markers: Vec<(SourceRepoDid, SourceDid)>, + pub acl_only_members: Vec, + pub table_only_members: Vec, + pub slash_owner_markers: u64, + pub slash_collab_rows: u64, + pub unresolved_slash_forms: Vec, +} + +impl Drift { + pub fn is_clean(&self) -> bool { + self == &Self::default() + } +} + +#[derive(Debug)] +pub struct Mapping { + pub knot_owner: AccountDid, + pub members: Vec, + pub repos: Vec, + pub skipped: Vec, + pub drift: Drift, +} + +impl Mapping { + pub fn unreadable_sources(&self) -> RepoList { + RepoList( + self.skipped + .iter() + .filter(|skip| matches!(skip.reason, SkipReason::UnreadableSource { .. })) + .map(|skip| skip.repo_did.clone()) + .collect(), + ) + } + + pub fn conflicting_owners(&self) -> RepoList { + RepoList( + self.drift + .conflicting_owner_markers + .iter() + .map(|conflict| conflict.repo.clone()) + .collect::>() + .into_iter() + .collect(), + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepoList(Vec); + +impl RepoList { + const LISTED: usize = 5; + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl fmt::Display for RepoList { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.iter().take(Self::LISTED).enumerate().try_for_each( + |(position, did)| match position { + 0 => write!(f, "{did}"), + _ => write!(f, ", {did}"), + }, + )?; + match self.0.len().saturating_sub(Self::LISTED) { + 0 => Ok(()), + rest => write!(f, ", and {rest} more"), + } + } +} + +pub fn map_tables( + repos: &[RepoRow], + rkeys: &BTreeMap, + members: &[MemberRow], + collabs: &[CollabRow], + acl: &AclRoster, + probe: impl Fn(&SourceRepoDid) -> SourceProbe, +) -> Result { + let knot_owner = server_owner(acl)?; + let repo_index: BTreeMap<&SourceRepoDid, &RepoRow> = + repos.iter().map(|repo| (&repo.repo_did, repo)).collect(); + + let table_members: Vec = members + .iter() + .filter(|row| row.subject.as_str() != knot_owner.as_str()) + .map(|row| { + Ok(MappedGrant { + subject: account(&row.subject).map_err(|source| MappingError::BadMemberDid { + value: row.subject.clone(), + source, + })?, + added_by: account(&row.did).map_err(|source| MappingError::BadMemberDid { + value: row.did.clone(), + source, + })?, + created_at: unix("knot_members.created", &row.created)?, + unioned: false, + }) + }) + .collect::>()?; + + let table_member_dids: BTreeSet<&SourceDid> = members.iter().map(|row| &row.subject).collect(); + let acl_only_members: Vec = acl + .members + .iter() + .filter(|did| !table_member_dids.contains(*did) && did.as_str() != knot_owner.as_str()) + .cloned() + .collect(); + let table_only_members: Vec = members + .iter() + .filter(|row| !acl.members.contains(&row.subject)) + .map(|row| row.subject.clone()) + .collect(); + let unioned_members: Vec = acl_only_members + .iter() + .map(|did| { + Ok(MappedGrant { + subject: account(did).map_err(|source| MappingError::BadMemberDid { + value: did.clone(), + source, + })?, + added_by: knot_owner.clone(), + // every casbin-only owner is at epoch + created_at: UnixSeconds::new(0), + unioned: true, + }) + }) + .collect::>()?; + + let (live_collabs, orphan_collabs): (Vec<&CollabRow>, Vec<&CollabRow>) = collabs + .iter() + .partition(|row| repo_index.contains_key(&row.repo_did)); + let table_pairs: BTreeSet<(&SourceRepoDid, &SourceDid)> = collabs + .iter() + .map(|row| (&row.repo_did, &row.subject_did)) + .collect(); + let acl_extra: Vec<(&SourceRepoDid, &SourceDid)> = acl + .collaborators + .iter() + .flat_map(|(repo, dids)| dids.iter().map(move |did| (repo, did))) + .filter(|(repo, did)| !table_pairs.contains(&(*repo, *did))) + .collect(); + let (acl_only, acl_orphans): (Vec<_>, Vec<_>) = acl_extra + .into_iter() + .partition(|(repo, _)| repo_index.contains_key(*repo)); + let orphan_pairs: BTreeSet<(SourceRepoDid, SourceDid)> = acl_orphans + .into_iter() + .map(|(repo, did)| (repo.clone(), did.clone())) + .chain( + orphan_collabs + .iter() + .map(|row| (row.repo_did.clone(), row.subject_did.clone())), + ) + .collect(); + let table_only_collaborators: Vec<(SourceRepoDid, SourceDid)> = live_collabs + .iter() + .filter(|row| { + [&acl.collaborators, &acl.slash_collaborators] + .into_iter() + .all(|grants| { + grants + .get(&row.repo_did) + .is_none_or(|dids| !dids.contains(&row.subject_did)) + }) + }) + .map(|row| (row.repo_did.clone(), row.subject_did.clone())) + .collect(); + + let table_grants = live_collabs.iter().copied().try_fold( + BTreeMap::>::new(), + |mut grants, row| { + let grant = MappedGrant { + subject: account(&row.subject_did).map_err(|source| { + MappingError::BadCollaboratorDid { + repo: row.repo_did.clone(), + value: row.subject_did.clone(), + source, + } + })?, + added_by: account(&row.added_by_did).map_err(|source| { + MappingError::BadCollaboratorDid { + repo: row.repo_did.clone(), + value: row.added_by_did.clone(), + source, + } + })?, + created_at: unix("collaborators.created", &row.created)?, + unioned: false, + }; + let entry = grants.entry(row.repo_did.clone()).or_default(); + if !entry.iter().any(|held| held.subject == grant.subject) { + entry.push(grant); + } + Ok::<_, MappingError>(grants) + }, + )?; + let collab_grants = owner_attributed_grants(&acl_only, &repo_index, true, table_grants)?; + + let owners = owner_drift(repos, &repo_index, acl); + let (adopted, skipped) = classify(repos, rkeys, &collab_grants, probe)?; + + Ok(Mapping { + knot_owner, + members: table_members.into_iter().chain(unioned_members).collect(), + repos: adopted, + skipped, + drift: Drift { + acl_only_collaborators: acl_only + .into_iter() + .map(|(repo, did)| (repo.clone(), did.clone())) + .collect(), + table_only_collaborators, + slash_resolved_collaborators: acl + .slash_collaborators + .iter() + .flat_map(|(repo, dids)| dids.iter().map(move |did| (repo, did))) + .filter(|(repo, did)| !table_pairs.contains(&(*repo, *did))) + .map(|(repo, did)| (repo.clone(), did.clone())) + .collect(), + orphan_collaborator_pairs: orphan_pairs.into_iter().collect(), + markerless_owner_repos: owners.markerless, + orphan_owner_markers: owners.orphans, + conflicting_owner_markers: owners.conflicts, + extra_owner_markers: owners.extras, + acl_only_members, + table_only_members, + slash_owner_markers: acl.slash_owner_markers, + slash_collab_rows: acl.slash_collab_rows, + unresolved_slash_forms: acl.unresolved_slash_forms.clone(), + }, + }) +} + +pub fn map_preflip( + repos: &[RepoRow], + rkeys: &BTreeMap, + members: &[MemberRow], + acl: &AclRoster, + probe: impl Fn(&SourceRepoDid) -> SourceProbe, +) -> Result { + let knot_owner = server_owner(acl)?; + let repo_index: BTreeMap<&SourceRepoDid, &RepoRow> = + repos.iter().map(|repo| (&repo.repo_did, repo)).collect(); + let enrich: BTreeMap<&SourceDid, &MemberRow> = + members.iter().map(|row| (&row.subject, row)).collect(); + let table_only_members: Vec = members + .iter() + .filter(|row| !acl.members.contains(&row.subject)) + .map(|row| row.subject.clone()) + .collect(); + + let mapped_members: Vec = acl + .members + .iter() + .filter(|did| did.as_str() != knot_owner.as_str()) + .map(|did| { + let subject = account(did).map_err(|source| MappingError::BadMemberDid { + value: did.clone(), + source, + })?; + match enrich.get(did) { + Some(row) => Ok(MappedGrant { + subject, + added_by: account(&row.did).map_err(|source| MappingError::BadMemberDid { + value: row.did.clone(), + source, + })?, + created_at: unix("knot_members.created", &row.created)?, + unioned: false, + }), + None => Ok(MappedGrant { + subject, + added_by: knot_owner.clone(), + created_at: UnixSeconds::new(0), + unioned: false, + }), + } + }) + .collect::>()?; + + let combined: BTreeMap<&SourceRepoDid, BTreeSet<&SourceDid>> = acl + .collaborators + .iter() + .chain(acl.slash_collaborators.iter()) + .flat_map(|(repo, dids)| dids.iter().map(move |did| (repo, did))) + .fold(BTreeMap::new(), |mut pairs, (repo, did)| { + pairs.entry(repo).or_default().insert(did); + pairs + }); + let live = combined + .iter() + .flat_map(|(repo, dids)| dids.iter().map(move |did| (*repo, *did))); + let (resolvable, orphan_pairs): (Vec<_>, Vec<_>) = + live.partition(|(repo, _)| repo_index.contains_key(*repo)); + let collab_grants = owner_attributed_grants(&resolvable, &repo_index, false, BTreeMap::new())?; + + let owners = owner_drift(repos, &repo_index, acl); + let (adopted, skipped) = classify(repos, rkeys, &collab_grants, probe)?; + + Ok(Mapping { + knot_owner, + members: mapped_members, + repos: adopted, + skipped, + drift: Drift { + orphan_collaborator_pairs: orphan_pairs + .into_iter() + .map(|(repo, did)| (repo.clone(), did.clone())) + .collect(), + markerless_owner_repos: owners.markerless, + orphan_owner_markers: owners.orphans, + conflicting_owner_markers: owners.conflicts, + extra_owner_markers: owners.extras, + table_only_members, + slash_owner_markers: acl.slash_owner_markers, + slash_collab_rows: acl.slash_collab_rows, + unresolved_slash_forms: acl.unresolved_slash_forms.clone(), + ..Drift::default() + }, + }) +} + +fn owner_attributed_grants( + pairs: &[(&SourceRepoDid, &SourceDid)], + repo_index: &BTreeMap<&SourceRepoDid, &RepoRow>, + unioned: bool, + base: BTreeMap>, +) -> Result>, MappingError> { + pairs + .iter() + .copied() + .try_fold(base, |mut grants, (repo, did)| { + let row = repo_index[repo]; + let grant = MappedGrant { + subject: account(did).map_err(|source| MappingError::BadCollaboratorDid { + repo: repo.clone(), + value: did.clone(), + source, + })?, + added_by: account(&row.owner_did).map_err(|source| MappingError::BadOwnerDid { + repo: repo.clone(), + value: row.owner_did.clone(), + source, + })?, + created_at: unix("repo_keys.created_at", &row.created_at)?, + unioned, + }; + grants.entry(repo.clone()).or_default().push(grant); + Ok(grants) + }) +} + +fn server_owner(acl: &AclRoster) -> Result { + let did = acl + .server_owner + .as_ref() + .ok_or(MappingError::MissingServerOwner)?; + account(did).map_err(|source| MappingError::BadMemberDid { + value: did.clone(), + source, + }) +} + +struct OwnerDrift { + markerless: Vec, + orphans: Vec, + conflicts: Vec, + extras: Vec<(SourceRepoDid, SourceDid)>, +} + +fn owner_drift( + repos: &[RepoRow], + repo_index: &BTreeMap<&SourceRepoDid, &RepoRow>, + acl: &AclRoster, +) -> OwnerDrift { + let (conflicting, agreeing): (Vec<_>, Vec<_>) = repos + .iter() + .filter_map(|repo| { + acl.owner_markers + .get(&repo.repo_did) + .map(|markers| (repo, markers)) + }) + .partition(|(repo, markers)| !markers.contains(&repo.owner_did)); + OwnerDrift { + markerless: repos + .iter() + .filter(|repo| !acl.owner_markers.contains_key(&repo.repo_did)) + .map(|repo| repo.repo_did.clone()) + .collect(), + orphans: acl + .owner_markers + .keys() + .filter(|repo| !repo_index.contains_key(*repo)) + .cloned() + .collect(), + conflicts: conflicting + .into_iter() + .flat_map(|(repo, markers)| { + markers.iter().map(move |marker| OwnerConflict { + repo: repo.repo_did.clone(), + acl_owner: marker.clone(), + key_owner: repo.owner_did.clone(), + }) + }) + .collect(), + extras: agreeing + .into_iter() + .flat_map(|(repo, markers)| { + markers + .iter() + .filter(move |marker| *marker != &repo.owner_did) + .map(move |marker| (repo.repo_did.clone(), marker.clone())) + }) + .collect(), + } +} + +fn classify( + repos: &[RepoRow], + rkeys: &BTreeMap, + collab_grants: &BTreeMap>, + probe: impl Fn(&SourceRepoDid) -> SourceProbe, +) -> Result<(Vec, Vec), MappingError> { + let (adopted, skipped) = repos.iter().try_fold( + (Vec::new(), Vec::new()), + |(mut adopted, mut skipped), row| { + match classify_one(row, rkeys, collab_grants, &probe)? { + Ok(repo) => adopted.push(repo), + Err(skip) => skipped.push(skip), + } + Ok::<_, MappingError>((adopted, skipped)) + }, + )?; + let alias_backed: BTreeSet = rkeys + .keys() + .filter_map(|did| RepoDid::new(did.as_str()).ok()) + .collect(); + resolve_rkey_collisions(adopted, skipped, &alias_backed) +} + +fn resolve_rkey_collisions( + adopted: Vec, + skipped: Vec, + alias_backed: &BTreeSet, +) -> Result<(Vec, Vec), MappingError> { + let groups: BTreeMap<(&OwnerDid, &RepoRkey), Vec> = + adopted + .iter() + .enumerate() + .fold(BTreeMap::new(), |mut groups, (index, repo)| { + groups + .entry((&repo.owner, &repo.rkey)) + .or_default() + .push(index); + groups + }); + let losers: BTreeMap = groups + .into_iter() + .filter(|(_, indices)| indices.len() > 1) + .map(|((owner, rkey), indices)| { + let backed: Vec = indices + .iter() + .copied() + .filter(|index| alias_backed.contains(&adopted[*index].did)) + .collect(); + match backed.as_slice() { + [winner] => Ok(indices + .into_iter() + .filter(|index| index != &*winner) + .map(|index| { + ( + index, + SkippedRepo { + repo_did: adopted[index].did.clone(), + reason: SkipReason::RkeyCollision { + rkey: rkey.clone(), + winner: adopted[*winner].did.clone(), + }, + lost_collaborators: adopted[index] + .collaborators + .iter() + .map(|grant| grant.subject.clone()) + .collect(), + }, + ) + }) + .collect::>()), + _ => Err(MappingError::AmbiguousRkey { + owner: owner.clone(), + rkey: rkey.clone(), + }), + } + }) + .collect::, MappingError>>()? + .into_iter() + .flatten() + .collect(); + let (kept, demoted) = adopted.into_iter().enumerate().fold( + (Vec::new(), losers), + |(mut kept, demoted), (index, repo)| { + if !demoted.contains_key(&index) { + kept.push(repo); + } + (kept, demoted) + }, + ); + Ok(( + kept, + skipped.into_iter().chain(demoted.into_values()).collect(), + )) +} + +fn classify_one( + row: &RepoRow, + rkeys: &BTreeMap, + collab_grants: &BTreeMap>, + probe: &impl Fn(&SourceRepoDid) -> SourceProbe, +) -> Result, MappingError> { + let did = RepoDid::new(row.repo_did.as_str()).map_err(|source| MappingError::BadRepoDid { + value: row.repo_did.clone(), + source, + })?; + let owner = + OwnerDid::new(row.owner_did.as_str()).map_err(|source| MappingError::BadOwnerDid { + repo: row.repo_did.clone(), + value: row.owner_did.clone(), + source, + })?; + let created_at = unix("repo_keys.created_at", &row.created_at)?; + let key_bytes: [u8; 32] = + row.signing_key + .as_bytes() + .try_into() + .map_err(|_| MappingError::BadSigningKey { + repo: row.repo_did.clone(), + key_type: row.key_type.clone(), + bytes: row.signing_key.as_bytes().len(), + })?; + if !row.key_type.is_k256() { + return Err(MappingError::BadSigningKey { + repo: row.repo_did.clone(), + key_type: row.key_type.clone(), + bytes: row.signing_key.as_bytes().len(), + }); + } + + let skip = |reason: SkipReason| SkippedRepo { + repo_did: did.clone(), + reason, + lost_collaborators: collab_grants + .get(&row.repo_did) + .map(|grants| grants.iter().map(|grant| grant.subject.clone()).collect()) + .unwrap_or_default(), + }; + let name = match RepoName::new(row.repo_name.as_str()) { + Ok(name) => name, + Err(_) => { + return Ok(Err(skip(SkipReason::Name { + value: row.repo_name.clone(), + }))); + } + }; + let raw_rkey = rkeys + .get(&row.repo_did) + .map(SourceRkey::as_str) + .unwrap_or(row.repo_name.as_str()); + let rkey = match RepoRkey::new(raw_rkey) { + Ok(rkey) => rkey, + Err(_) => { + return Ok(Err(skip(SkipReason::Rkey { + value: SourceRkey::from_column(raw_rkey), + }))); + } + }; + match probe(&row.repo_did) { + SourceProbe::Absent => return Ok(Err(skip(SkipReason::NoSourceRepo))), + SourceProbe::Unreadable(kind) => { + return Ok(Err(skip(SkipReason::UnreadableSource { kind }))); + } + SourceProbe::Repo => {} + } + + Ok(Ok(AdoptRepo { + source_did: row.repo_did.clone(), + did, + owner, + name, + rkey, + created_at, + signing_key: SigningKey(key_bytes), + collaborators: collab_grants + .get(&row.repo_did) + .cloned() + .unwrap_or_default(), + })) +} + +fn account(value: &SourceDid) -> Result { + AccountDid::new(value.as_str()) +} + +fn unix(context: &'static str, value: &SourceTimestamp) -> Result { + chrono::DateTime::parse_from_rfc3339(value.as_str()) + .map(|parsed| UnixSeconds::new(parsed.timestamp())) + .map_err(|_| MappingError::BadTimestamp { + context, + value: value.clone(), + }) +} diff --git a/knot2/crates/knot-migrate/src/rehearse.rs b/knot2/crates/knot-migrate/src/rehearse.rs new file mode 100644 index 0000000000000000000000000000000000000000..3bce2ac93b261f73b89d2ca420fc52d058b3c3be --- /dev/null +++ b/knot2/crates/knot-migrate/src/rehearse.rs @@ -0,0 +1,237 @@ +use std::path::{Path, PathBuf}; + +use knot_types::scalar_newtype; + +use crate::adopt::{self, AdoptError, SourcePolicy, Transfer}; +use crate::emit::{ + self, EmitError, HostKeyConflict, HostKeyPlacement, HostKeyPolicy, MasterKeyEnv, MasterKeyError, +}; +use crate::mapping::AdoptRepo; + +#[derive(Debug, thiserror::Error)] +pub enum HostKeyError { + #[error("--host-key is required for the migration")] + Unset, + #[error(transparent)] + Unusable(#[from] EmitError), +} + +#[derive(Debug, thiserror::Error)] +pub enum ScanPathError { + #[error( + "the migration will create {path} under {blocked}, which this process can't write: {source}" + )] + Uncreatable { + path: PathBuf, + blocked: PathBuf, + source: rustix::io::Errno, + }, + #[error("the migration will write repos into {path}, which this process can't write: {source}")] + Unwritable { + path: PathBuf, + source: rustix::io::Errno, + }, + #[error("the migration can't create {path}, which is a symlink to a missing target")] + Dangling { path: PathBuf }, + #[error("read {path}: {source}")] + Unreadable { + path: PathBuf, + source: std::io::Error, + }, +} + +#[derive(Debug, thiserror::Error)] +pub enum RoomError { + #[error("measure {path}: {source}")] + Source { + path: PathBuf, + source: std::io::Error, + }, + #[error("read the free space on {path}: {source}")] + Free { + path: PathBuf, + source: rustix::io::Errno, + }, +} + +scalar_newtype! { + pub struct Bytes(u64) => ordered; +} + +impl std::fmt::Display for Bytes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + const UNITS: [(&str, u64); 3] = [("GiB", 1 << 30), ("MiB", 1 << 20), ("KiB", 1 << 10)]; + match UNITS.iter().find(|(_, size)| self.get() >= *size) { + None => write!(f, "{}B", self.get()), + Some((unit, size)) => write!(f, "{:.1}{unit}", self.get() as f64 / *size as f64), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Occupancy { + Fresh, + Occupied, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Fit { + Short, + Clear, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Room { + pub source: Bytes, + pub free: Bytes, +} + +impl Room { + pub fn fit(self) -> Fit { + match self.free >= self.source { + true => Fit::Clear, + false => Fit::Short, + } + } +} + +pub struct Inputs<'a> { + pub source_repos: &'a Path, + pub adopted: &'a [AdoptRepo], + pub scan_path: &'a Path, + pub policy: SourcePolicy, + pub host_key: Option<&'a Path>, + pub host_key_target: &'a Path, + pub host_key_policy: HostKeyPolicy, + pub master_key: &'a MasterKeyEnv, +} + +pub struct Rehearsal { + pub fallback: Option, + pub transfer: Result, + pub scan_path: Result, + pub room: Option>, + pub host_key: Result, + pub host_key_target: Option>, + pub master_key: Result, +} + +impl Rehearsal { + pub fn run(inputs: Inputs<'_>) -> Self { + let probed = Probed::nearest(inputs.scan_path); + let transfer = adopt::transfer_mode(inputs.source_repos, probed.existing, inputs.policy); + let source_key = inputs + .host_key + .ok_or(HostKeyError::Unset) + .and_then(|path| emit::load_host_key(path).map_err(HostKeyError::Unusable)); + Self { + fallback: probed.fallback().map(Path::to_path_buf), + room: match transfer { + Ok(Transfer::Copy) => Some(room(inputs.source_repos, inputs.adopted, probed)), + Ok(Transfer::Rename) | Err(_) => None, + }, + transfer, + scan_path: occupancy(probed), + host_key_target: source_key.as_ref().ok().map(|key| { + emit::plan_host_key(inputs.host_key_target, key, inputs.host_key_policy) + }), + host_key: source_key.map(|key| key.algorithm), + master_key: inputs.master_key.read().map(|_| inputs.master_key.clone()), + } + } + + pub fn ready(&self) -> bool { + self.transfer.is_ok() + && self.scan_path.is_ok() + && self.host_key.is_ok() + && self.host_key_target.as_ref().is_none_or(Result::is_ok) + && self.master_key.is_ok() + && self.room.as_ref().is_none_or(|room| { + room.as_ref() + .is_ok_and(|measured| matches!(measured.fit(), Fit::Clear)) + }) + } +} + +#[derive(Debug, Clone, Copy)] +struct Probed<'a> { + scan_path: &'a Path, + existing: &'a Path, +} + +pub fn nearest_existing(path: &Path) -> &Path { + path.ancestors() + .find(|candidate| match std::fs::symlink_metadata(candidate) { + Ok(_) => true, + Err(error) => !adopt::reads_as_absent(&error), + }) + .unwrap_or_else(|| Path::new(".")) +} + +impl<'a> Probed<'a> { + fn nearest(scan_path: &'a Path) -> Self { + Self { + scan_path, + existing: nearest_existing(scan_path), + } + } + + fn fallback(self) -> Option<&'a Path> { + (self.existing != self.scan_path).then_some(self.existing) + } +} + +fn occupancy(probed: Probed<'_>) -> Result { + match (adopt::writable(probed.existing), probed.fallback()) { + (Err(source), None) if source == rustix::io::Errno::NOENT => Err(ScanPathError::Dangling { + path: probed.scan_path.to_path_buf(), + }), + (Err(source), None) => Err(ScanPathError::Unwritable { + path: probed.scan_path.to_path_buf(), + source, + }), + (Err(source), Some(blocked)) => Err(ScanPathError::Uncreatable { + path: probed.scan_path.to_path_buf(), + blocked: blocked.to_path_buf(), + source, + }), + (Ok(()), Some(_)) => Ok(Occupancy::Fresh), + (Ok(()), None) => std::fs::read_dir(probed.scan_path) + .and_then(|mut entries| entries.next().transpose()) + .map(|entry| match entry { + Some(_) => Occupancy::Occupied, + None => Occupancy::Fresh, + }) + .map_err(|source| ScanPathError::Unreadable { + path: probed.scan_path.to_path_buf(), + source, + }), + } +} + +fn room(source_repos: &Path, adopted: &[AdoptRepo], probed: Probed<'_>) -> Result { + use std::os::unix::fs::MetadataExt; + let source = adopted.iter().try_fold(0_u64, |total, repo| { + let path = adopt::source_dir(source_repos, &repo.source_did); + walkdir::WalkDir::new(&path) + .into_iter() + .try_fold(total, |total, entry| { + entry + .and_then(|entry| entry.metadata()) + .map(|meta| total + meta.blocks() * 512) + }) + .map_err(|source| RoomError::Source { + path, + source: source.into(), + }) + })?; + rustix::fs::statvfs(probed.existing) + .map(|stat| Room { + source: Bytes::new(source), + free: Bytes::new(stat.f_bavail * stat.f_frsize), + }) + .map_err(|source| RoomError::Free { + path: probed.existing.to_path_buf(), + source, + }) +} diff --git a/knot2/crates/knot-migrate/src/report.rs b/knot2/crates/knot-migrate/src/report.rs new file mode 100644 index 0000000000000000000000000000000000000000..8895d380440e129aba15441ebabe6d4bc499f012 --- /dev/null +++ b/knot2/crates/knot-migrate/src/report.rs @@ -0,0 +1,254 @@ +use std::fmt::{self, Display, Formatter}; + +use crate::adopt::AdoptOutcome; +use crate::emit::CobSummary; +use crate::emit::HostKeyPlacement; +use crate::mapping::{Mapping, SkipReason}; +use crate::rehearse::{Fit, Occupancy, Rehearsal}; +use crate::source::{SourceDid, SourceRepoDid}; + +pub struct Report<'a> { + pub mapping: &'a Mapping, + pub orphan_alias_count: u64, + pub phase: Phase<'a>, +} + +pub enum Phase<'a> { + Refused, + Rehearsed(&'a Rehearsal), + Written { + adoption: &'a AdoptOutcome, + cobs: &'a CobSummary, + }, +} + +impl Display for Report<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let mapping = self.mapping; + let drift = &mapping.drift; + writeln!(f, "knot owner: {}", mapping.knot_owner)?; + writeln!(f, "members to grant: {}", mapping.members.len())?; + writeln!(f, "repos to adopt: {}", mapping.repos.len())?; + writeln!( + f, + "collaborator grants: {}", + mapping + .repos + .iter() + .map(|repo| repo.collaborators.len()) + .sum::() + )?; + writeln!(f)?; + match drift.is_clean() && self.orphan_alias_count == 0 { + true => writeln!(f, "casbin cross-check: the acl and the tables agree")?, + false => { + writeln!(f, "casbin cross-check drift:")?; + listed( + f, + "acl-only collaborator grants unioned in", + drift.acl_only_collaborators.iter().map(GrantRow::from), + )?; + listed( + f, + "table-only collaborator grants missing from acl", + drift.table_only_collaborators.iter().map(GrantRow::from), + )?; + listed( + f, + "repos with no acl owner marker where the owner regains push", + drift.markerless_owner_repos.iter(), + )?; + counted( + f, + "orphan owner markers on unknown repos", + drift.orphan_owner_markers.len() as u64, + )?; + listed( + f, + "repos recorded with different owners in the acl and repo_keys", + drift.conflicting_owner_markers.iter(), + )?; + listed( + f, + "extra acl owner markers dropped", + drift.extra_owner_markers.iter().map(GrantRow::from), + )?; + counted( + f, + "orphan collaborator pairs on unknown repos", + drift.orphan_collaborator_pairs.len() as u64, + )?; + listed( + f, + "acl-only members unioned in", + drift.acl_only_members.iter(), + )?; + counted( + f, + "table-only members missing from acl", + drift.table_only_members.len() as u64, + )?; + counted(f, "slash-form owner markers", drift.slash_owner_markers)?; + counted(f, "slash-form collaborator rows", drift.slash_collab_rows)?; + listed( + f, + "slash-resolved collaborator grants left out of the union", + drift + .slash_resolved_collaborators + .iter() + .map(GrantRow::from), + )?; + listed( + f, + "unresolved slash forms", + drift.unresolved_slash_forms.iter(), + )?; + counted(f, "orphan aliases", self.orphan_alias_count)?; + } + } + writeln!(f)?; + writeln!(f, "skipped repos: {}", mapping.skipped.len())?; + mapping.skipped.iter().try_for_each(|skip| { + writeln!(f, "{} {}", skip.repo_did, describe(&skip.reason))?; + skip.lost_collaborators + .iter() + .try_for_each(|did| writeln!(f, "drops collaborator grant for {did}")) + })?; + match self.phase { + Phase::Refused => Ok(()), + Phase::Rehearsed(rehearsal) => { + writeln!(f)?; + match &rehearsal.transfer { + Err(error) => writeln!(f, "transfer mode: {error}"), + Ok(transfer) => writeln!(f, "transfer mode: {transfer}"), + }?; + rehearsal.fallback.as_ref().map_or(Ok(()), |fallback| { + writeln!( + f, + "the filesystem checks used {}, since the scan path doesn't exist yet", + fallback.display() + ) + })?; + match &rehearsal.scan_path { + Ok(Occupancy::Fresh) => writeln!(f, "scan path: writable"), + Ok(Occupancy::Occupied) => writeln!( + f, + "scan path: writable, with repos already in it that the migration will keep" + ), + Err(error) => writeln!(f, "scan path: {error}"), + }?; + rehearsal.room.as_ref().map_or(Ok(()), |room| match room { + Ok(room) => match room.fit() { + Fit::Clear => writeln!( + f, + "room to copy: {} free is enough for the {} that adoption will copy", + room.free, room.source + ), + Fit::Short => writeln!( + f, + "room to copy: {} free isn't enough for the {} that adoption will copy", + room.free, room.source + ), + }, + Err(error) => writeln!(f, "room to copy: {error}"), + })?; + match &rehearsal.host_key { + Ok(algorithm) => writeln!(f, "host key algorithm: {algorithm}"), + Err(error) => writeln!(f, "host key: {error}"), + }?; + match &rehearsal.host_key_target { + None | Some(Ok(HostKeyPlacement::Fresh)) => Ok(()), + Some(Ok(HostKeyPlacement::Unchanged)) => { + writeln!(f, "host key: the target already has the imported key") + } + Some(Ok(HostKeyPlacement::Replacing)) => { + writeln!( + f, + "host key: the migration will replace the different key at the target" + ) + } + Some(Err(error)) => writeln!(f, "host key: {error}"), + }?; + match &rehearsal.master_key { + Ok(env) => writeln!(f, "master key: {env} decodes to a usable key"), + Err(error) => writeln!(f, "{error}"), + } + } + Phase::Written { adoption, cobs } => { + writeln!(f)?; + writeln!( + f, + "adopted by {}: {} new, {} already present, {} sha1, {} sha256", + adoption.transfer, + adoption.adopted, + adoption.already_present, + adoption.sha1, + adoption.sha256 + )?; + writeln!(f)?; + writeln!( + f, + "member grants: {} appended, {} already present", + cobs.members.appended, cobs.members.already_present + )?; + writeln!( + f, + "registrations: {} appended, {} already present", + cobs.registrations.appended, cobs.registrations.already_present + )?; + writeln!( + f, + "collaborator grants: {} appended, {} already present", + cobs.collaborators.appended, cobs.collaborators.already_present + ) + } + } + } +} + +struct GrantRow<'a>(&'a SourceRepoDid, &'a SourceDid); + +impl<'a> From<&'a (SourceRepoDid, SourceDid)> for GrantRow<'a> { + fn from((repo, did): &'a (SourceRepoDid, SourceDid)) -> Self { + Self(repo, did) + } +} + +impl Display for GrantRow<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{} <- {}", self.0, self.1) + } +} + +fn counted(f: &mut Formatter<'_>, label: &str, count: u64) -> fmt::Result { + match count { + 0 => Ok(()), + count => writeln!(f, "{label}: {count}"), + } +} + +fn listed( + f: &mut Formatter<'_>, + label: &str, + mut rows: impl ExactSizeIterator, +) -> fmt::Result { + counted(f, label, rows.len() as u64)?; + rows.try_for_each(|row| writeln!(f, "{row}")) +} + +fn describe(reason: &SkipReason) -> String { + match reason { + SkipReason::Name { value } => format!("unrepresentable name {:?}", value.as_str()), + SkipReason::Rkey { value } => format!("unrepresentable rkey {:?}", value.as_str()), + SkipReason::RkeyCollision { rkey, winner } => { + format!( + "record key {:?} belongs to the alias-backed {winner}", + rkey.as_str() + ) + } + SkipReason::NoSourceRepo => "no git repository at the source path".to_string(), + SkipReason::UnreadableSource { kind } => { + format!("a source path that this process can't read: {kind}") + } + } +} diff --git a/knot2/crates/knot-migrate/src/source.rs b/knot2/crates/knot-migrate/src/source.rs new file mode 100644 index 0000000000000000000000000000000000000000..d5e18de441da79f78d8418022a58c11bcb0403b7 --- /dev/null +++ b/knot2/crates/knot-migrate/src/source.rs @@ -0,0 +1,317 @@ +use std::collections::BTreeSet; +use std::path::Path; + +use rusqlite::{Connection, OpenFlags, OptionalExtension, Row}; + +#[derive(Debug, thiserror::Error)] +pub enum SourceError { + #[error("source database query failed: {0}")] + Sqlite(#[from] rusqlite::Error), + #[error( + "source database predates DID-keyed repos. Upgrade tangled-knot to its latest release, let it finish its own migrations, then run knot-migrate again." + )] + PreDid, + #[error("source database has no acl table for the casbin cross-check")] + MissingAcl, + #[error( + "source database has a collaborators table but no knot_members table. Upgrade tangled-knot to its latest release, let it finish its own migrations, then run knot-migrate again." + )] + CollaboratorsWithoutMembers, + #[error( + "source table `{table}` is missing expected columns {}. This tangled-knot predates the schema knot-migrate reads. Upgrade tangled-knot to its latest release, let it finish its own migrations, then run knot-migrate again.", + .missing.join(", ") + )] + SchemaMismatch { table: String, missing: Vec }, +} + +knot_types::text_newtype! { + pub struct SourceRepoDid(String) => verbatim as from_column; + pub struct SourceDid(String) => verbatim as from_column; + pub struct SourceRkey(String) => verbatim as from_column; + pub struct SourceRepoName(String) => verbatim as from_column; + pub struct SourceRepoObject(String) => verbatim as from_column; + pub struct SourceTimestamp(String) => verbatim as from_column; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SourceKeyType { + K256, + Other(String), +} + +impl SourceKeyType { + pub fn from_column(value: impl Into) -> Self { + let value = value.into(); + match value.as_str() { + "k256" => Self::K256, + _ => Self::Other(value), + } + } + + pub fn is_k256(&self) -> bool { + matches!(self, Self::K256) + } +} + +impl ::std::fmt::Display for SourceKeyType { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + match self { + Self::K256 => f.write_str("k256"), + Self::Other(value) => f.write_str(value), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceSchema { + Tables, + PreFlip, +} + +#[derive(Clone, PartialEq, Eq, zeroize::Zeroize, zeroize::ZeroizeOnDrop)] +// Every repo's private key, straight outta the old knot's db. +// `Debug` doesn't prints any bytes on purpose, +// as to not compromise a migration. +pub struct SourceSigningKey(Vec); + +impl SourceSigningKey { + pub fn from_column(bytes: Vec) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +impl std::fmt::Debug for SourceSigningKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("SourceSigningKey").finish_non_exhaustive() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepoRow { + pub repo_did: SourceRepoDid, + pub owner_did: SourceDid, + pub repo_name: SourceRepoName, + pub signing_key: SourceSigningKey, + pub key_type: SourceKeyType, + pub created_at: SourceTimestamp, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MemberRow { + pub did: SourceDid, + pub subject: SourceDid, + pub created: SourceTimestamp, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CollabRow { + pub repo_did: SourceRepoDid, + pub subject_did: SourceDid, + pub added_by_did: SourceDid, + pub created: SourceTimestamp, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AclRow { + pub p_type: String, + pub v0: String, + pub v1: String, + pub v2: String, + pub v3: String, +} + +pub struct SourceDb { + conn: Connection, +} + +impl SourceDb { + pub fn open(path: &Path) -> Result { + let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + Ok(Self { conn }) + } + + pub fn schema(&self) -> Result { + let variant = match ( + self.has_table("repo_keys")?, + self.has_table("repo_aliases")?, + self.has_table("knot_members")?, + self.has_table("collaborators")?, + ) { + (true, true, true, true) => SourceSchema::Tables, + (true, true, false, true) => return Err(SourceError::CollaboratorsWithoutMembers), + (true, true, _, false) => SourceSchema::PreFlip, + _ => return Err(SourceError::PreDid), + }; + let checks: Vec<(&str, &[&str])> = [ + Some(( + "repo_keys", + &[ + "repo_did", + "owner_did", + "repo_name", + "signing_key", + "key_type", + "created_at", + ][..], + )), + Some(("repo_aliases", &["rkey", "repo_did", "rev"][..])), + self.has_table("acl")? + .then_some(("acl", &["p_type", "v0", "v1", "v2", "v3"][..])), + self.has_table("knot_members")? + .then_some(("knot_members", &["id", "did", "subject", "created"][..])), + (variant == SourceSchema::Tables).then_some(( + "collaborators", + &["id", "repo_did", "subject_did", "added_by_did", "created"][..], + )), + ] + .into_iter() + .flatten() + .collect(); + checks + .into_iter() + .try_for_each(|(table, columns)| self.require_columns(table, columns))?; + Ok(variant) + } + + fn has_table(&self, name: &str) -> Result { + let count: i64 = self.conn.query_row( + "select count(*) from sqlite_master where type = 'table' and name = ?1", + [name], + |row| row.get(0), + )?; + Ok(count > 0) + } + + fn require_columns(&self, table: &str, required: &[&str]) -> Result<(), SourceError> { + let present: BTreeSet = self + .conn + .prepare("select name from pragma_table_info(?1)")? + .query_map([table], |row| row.get::<_, String>(0))? + .collect::>()?; + let missing: Vec = required + .iter() + .filter(|column| !present.contains(**column)) + .map(|column| (*column).to_string()) + .collect(); + missing + .is_empty() + .then_some(()) + .ok_or(SourceError::SchemaMismatch { + table: table.to_string(), + missing, + }) + } + + pub fn repos(&self) -> Result, SourceError> { + self.collect( + "select repo_did, owner_did, repo_name, signing_key, key_type, created_at + from repo_keys order by created_at, repo_did", + |row| { + Ok(RepoRow { + repo_did: SourceRepoDid::from_column(row.get::<_, String>(0)?), + owner_did: SourceDid::from_column(row.get::<_, String>(1)?), + repo_name: SourceRepoName::from_column(row.get::<_, String>(2)?), + signing_key: SourceSigningKey::from_column(row.get(3)?), + key_type: SourceKeyType::from_column(row.get::<_, String>(4)?), + created_at: SourceTimestamp::from_column(row.get::<_, String>(5)?), + }) + }, + ) + } + + pub fn members(&self) -> Result, SourceError> { + if !self.has_table("knot_members")? { + return Ok(Vec::new()); + } + self.collect( + "select did, subject, created from knot_members + where id in (select min(id) from knot_members group by subject) + order by id", + |row| { + Ok(MemberRow { + did: SourceDid::from_column(row.get::<_, String>(0)?), + subject: SourceDid::from_column(row.get::<_, String>(1)?), + created: SourceTimestamp::from_column(row.get::<_, String>(2)?), + }) + }, + ) + } + + pub fn collaborators(&self) -> Result, SourceError> { + if !self.has_table("collaborators")? { + return Ok(Vec::new()); + } + self.collect( + "select repo_did, subject_did, added_by_did, created from collaborators order by id", + |row| { + Ok(CollabRow { + repo_did: SourceRepoDid::from_column(row.get::<_, String>(0)?), + subject_did: SourceDid::from_column(row.get::<_, String>(1)?), + added_by_did: SourceDid::from_column(row.get::<_, String>(2)?), + created: SourceTimestamp::from_column(row.get::<_, String>(3)?), + }) + }, + ) + } + + pub fn acl(&self) -> Result, SourceError> { + if !self.has_table("acl")? { + return Err(SourceError::MissingAcl); + } + self.collect( + "select p_type, v0, v1, v2, v3 from acl order by rowid", + |row| { + Ok(AclRow { + p_type: row.get(0)?, + v0: row.get(1)?, + v1: row.get(2)?, + v2: row.get(3)?, + v3: row.get(4)?, + }) + }, + ) + } + + pub fn current_rkey( + &self, + repo_did: &SourceRepoDid, + ) -> Result, SourceError> { + self.conn + .query_row( + "select rkey from repo_aliases + where repo_did = ? + order by rev desc + limit 1", + [repo_did.as_str()], + |row| row.get::<_, String>(0).map(SourceRkey::from_column), + ) + .optional() + .map_err(Into::into) + } + + pub fn orphan_alias_count(&self) -> Result { + let count: i64 = self.conn.query_row( + "select count(*) from repo_aliases ra + where not exists (select 1 from repo_keys rk where rk.repo_did = ra.repo_did)", + [], + |row| row.get(0), + )?; + Ok(count as u64) + } + + fn collect( + &self, + sql: &str, + map: impl Fn(&Row<'_>) -> rusqlite::Result, + ) -> Result, SourceError> { + let mut statement = self.conn.prepare(sql)?; + let rows = statement + .query_map([], map)? + .collect::>>()?; + Ok(rows) + } +} diff --git a/knot2/crates/knot-migrate/tests/migrate.rs b/knot2/crates/knot-migrate/tests/migrate.rs new file mode 100644 index 0000000000000000000000000000000000000000..37846d3731361917fa1d3ece8ff4f9435c0a043a --- /dev/null +++ b/knot2/crates/knot-migrate/tests/migrate.rs @@ -0,0 +1,1907 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use knot_index::{Index, Resolved}; +use knot_migrate::adopt; +use knot_migrate::casbin; +use knot_migrate::emit::MasterKeyEnv; +use knot_migrate::emit::{self, ConfigValues}; +use knot_migrate::mapping::{self, SkipReason}; +use knot_migrate::rehearse::{self, Rehearsal}; +use knot_migrate::report; +use knot_migrate::source::{ + SourceDb, SourceDid, SourceError, SourceRepoDid, SourceRkey, SourceSchema, +}; +use knot_runtime::{K256Signer, SeededEntropy}; +use knot_types::{AccountDid, KnotHostname, KnotId, ObjectFormat, RepoDid}; +use url::Url; + +const SCHEMA: &str = " +create table repo_keys ( + repo_did text primary key, + signing_key blob, + created_at text not null, + owner_did text, + repo_name text, + key_type text not null default 'k256' +); +create table repo_aliases ( + owner_did text not null, + rkey text not null, + repo_did text not null, + rev text not null, + primary key (owner_did, rkey) +); +create table knot_members ( + id integer primary key autoincrement, + did text not null, + rkey text, + subject text not null, + created text not null +); +create table collaborators ( + id integer primary key autoincrement, + repo_did text not null, + subject_did text not null, + added_by_did text not null, + created text not null +); +create table acl ( + p_type varchar(32) default '' not null, + v0 varchar(255) default '' not null, + v1 varchar(255) default '' not null, + v2 varchar(255) default '' not null, + v3 varchar(255) default '' not null, + v4 varchar(255) default '' not null, + v5 varchar(255) default '' not null +); +"; + +fn fixture_db(path: &Path, with_collaborators_table: bool) { + let conn = rusqlite::Connection::open(path).unwrap(); + conn.execute_batch(SCHEMA).unwrap(); + conn.execute_batch( + " + insert into repo_keys (repo_did, signing_key, created_at, owner_did, repo_name) values + ('did:plc:squid', x'0101010101010101010101010101010101010101010101010101010101010101', '2026-01-05T10:00:00Z', 'did:plc:nel', 'anemone'), + ('did:plc:limpet', x'0202020202020202020202020202020202020202020202020202020202020202', '2026-02-01T09:30:00Z', 'did:plc:nel', 'barnacle'), + ('did:plc:conch', x'0303030303030303030303030303030303030303030303030303030303030303', '2026-03-10T14:00:00Z', 'did:plc:isabel', 'Test knot'), + ('did:plc:whelk', x'0404040404040404040404040404040404040404040404040404040404040404', '2026-04-20T08:15:00Z', 'did:plc:isabel', 'mussel'), + ('did:plc:nautilus', x'0505050505050505050505050505050505050505050505050505050505050505', '2026-05-01T10:00:00Z', 'did:plc:isabel', 'coralline'), + ('did:plc:scallop', x'0606060606060606060606060606060606060606060606060606060606060606', '2026-05-02T10:00:00Z', 'did:plc:isabel', 'seagrass'), + ('did:plc:clam', x'0707070707070707070707070707070707070707070707070707070707070707', '2026-05-03T10:00:00Z', 'did:plc:isabel', '|'), + ('did:web:nel.pet', x'0808080808080808080808080808080808080808080808080808080808080808', '2026-06-01T10:00:00Z', 'did:plc:nel', 'seashell'); + insert into repo_aliases (owner_did, rkey, repo_did, rev) values + ('did:plc:nel', 'anemone-old', 'did:plc:squid', '1_2026-01-05T10:00:00Z'), + ('did:plc:nel', 'anemone', 'did:plc:squid', '3mq2bmuwq7v2t'), + ('did:plc:isabel', 'Test knot', 'did:plc:conch', '3mniy6vtxn22y'), + ('did:plc:isabel', 'mussel', 'did:plc:whelk', '3moo4vihsva2t'), + ('did:plc:isabel', 'seagrass', 'did:plc:nautilus', '3mpwduty3pw2z'), + ('did:plc:isabel', '|', 'did:plc:clam', '3mq3bmuwq7v2t'), + ('did:plc:nel', 'vanished', 'did:plc:kelp', '3mq4bmuwq7v2t'); + insert into knot_members (did, subject, created) values + ('did:plc:bailey', 'did:plc:nel', '2026-01-02T00:00:00Z'), + ('did:plc:nel', 'did:plc:teq', '2026-01-03T00:00:00Z'), + ('did:plc:bailey', 'did:plc:teq', '2026-01-04T00:00:00Z'), + ('did:plc:nel', 'did:plc:olaren', '2026-01-05T00:00:00Z'), + ('did:plc:teq', 'did:plc:bailey', '2026-01-06T00:00:00Z'); + insert into acl (p_type, v0, v1, v2, v3) values + ('g', 'did:plc:bailey', 'server:owner', 'thisserver', ''), + ('g', 'server:owner', 'server:member', 'thisserver', ''), + ('g', 'did:plc:bailey', 'server:member', 'thisserver', ''), + ('g', 'did:plc:nel', 'server:member', 'thisserver', ''), + ('g', 'did:plc:teq', 'server:member', 'thisserver', ''), + ('g', 'did:plc:uni', 'server:member', 'thisserver', ''), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:squid', 'repo:owner'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:squid', 'repo:push'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:squid', 'repo:settings'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:squid', 'repo:invite'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:squid', 'repo:delete'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:limpet', 'repo:owner'), + ('p', 'did:plc:bailey', 'thisserver', 'did:plc:limpet', 'repo:owner'), + ('p', 'did:plc:isabel', 'thisserver', 'did:plc:whelk', 'repo:owner'), + ('p', 'did:plc:isabel', 'thisserver', 'did:plc:nautilus', 'repo:owner'), + ('p', 'did:plc:isabel', 'thisserver', 'did:plc:scallop', 'repo:owner'), + ('p', 'did:plc:isabel', 'thisserver', 'did:plc:clam', 'repo:owner'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:kelp', 'repo:owner'), + ('p', 'did:plc:isabel', 'thisserver', 'did:plc:squid', 'repo:collaborator'), + ('p', 'did:plc:teq', 'thisserver', 'did:plc:limpet', 'repo:collaborator'), + ('p', 'did:plc:uni', 'thisserver', 'did:plc:whelk', 'repo:collaborator'), + ('p', 'did:plc:cuttle', 'thisserver', 'did:plc:kelp', 'repo:collaborator'), + ('p', 'did:plc:periwinkle', 'thisserver', 'did:plc:nel/anemone', 'repo:collaborator'), + ('p', 'did:plc:teq', 'thisserver', 'did:plc:nel/anemone', 'repo:collaborator'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:nel/vanished', 'repo:owner'), + ('p', 'did:plc:nel', 'thisserver', 'did:web:nel.pet', 'repo:owner'), + ('p', 'did:plc:nel', 'thisserver', '', 'repo:create'), + ('p', 'did:plc:nel', 'thisserver', '', 'server:invite'); + ", + ) + .unwrap(); + if with_collaborators_table { + conn.execute_batch( + " + insert into collaborators (repo_did, subject_did, added_by_did, created) values + ('did:plc:squid', 'did:plc:isabel', 'did:plc:nel', '2026-01-06T11:00:00Z'), + ('did:plc:squid', 'did:plc:olaren', 'did:plc:nel', '2026-01-07T12:00:00Z'), + ('did:plc:kelp', 'did:plc:teq', 'did:plc:nel', '2026-01-08T13:00:00Z'), + ('did:plc:squid', 'did:plc:isabel', 'did:plc:bailey', '2026-01-09T14:00:00Z'), + ('did:plc:whelk', 'did:plc:uni', 'did:plc:isabel', '2026-01-10T15:00:00Z'), + ('did:plc:squid', 'did:plc:teq', 'did:plc:nel', '2026-01-11T16:00:00Z'); + ", + ) + .unwrap(); + } else { + conn.execute_batch("drop table collaborators;").unwrap(); + } +} + +struct Fixture { + _dir: tempfile::TempDir, + db_path: PathBuf, + source_repos: PathBuf, + target: PathBuf, +} + +fn fixture(with_collaborators_table: bool) -> Fixture { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("knotserver.db"); + fixture_db(&db_path, with_collaborators_table); + let source_repos = dir.path().join("source-repos"); + [ + "did:plc:squid", + "did:plc:limpet", + "did:plc:conch", + "did:plc:nautilus", + "did:plc:scallop", + "did:plc:clam", + "did:web:nel.pet", + ] + .iter() + .for_each(|did| { + knot_git::Repo::create_with_format(source_repos.join(did), ObjectFormat::SHA1).unwrap(); + }); + std::fs::create_dir_all(source_repos.join("did:plc:whelk")).unwrap(); + Fixture { + db_path, + source_repos, + target: dir.path().join("target"), + _dir: dir, + } +} + +fn map(fx: &Fixture) -> mapping::Mapping { + map_probing(fx, |did| adopt::probe_source(&fx.source_repos, did)) +} + +fn map_probing( + fx: &Fixture, + probe: impl Fn(&SourceRepoDid) -> adopt::SourceProbe, +) -> mapping::Mapping { + let db = SourceDb::open(&fx.db_path).unwrap(); + let repos = db.repos().unwrap(); + let rkeys: BTreeMap = repos + .iter() + .filter_map(|repo| { + db.current_rkey(&repo.repo_did) + .unwrap() + .map(|rkey| (repo.repo_did.clone(), rkey)) + }) + .collect(); + let resolver = casbin::resolver(repos.iter().map(|repo| { + ( + repo.owner_did.clone(), + repo.repo_name.clone(), + repo.repo_did.clone(), + ) + })); + let acl = casbin::decode(&db.acl().unwrap(), &resolver).unwrap(); + match db.schema().unwrap() { + SourceSchema::Tables => mapping::map_tables( + &repos, + &rkeys, + &db.members().unwrap(), + &db.collaborators().unwrap(), + &acl, + probe, + ) + .unwrap(), + SourceSchema::PreFlip => { + mapping::map_preflip(&repos, &rkeys, &db.members().unwrap(), &acl, probe).unwrap() + } + } +} + +fn acc(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{suffix}")).unwrap() +} + +fn srepo(value: &str) -> SourceRepoDid { + SourceRepoDid::from_column(value) +} + +fn sdid(value: &str) -> SourceDid { + SourceDid::from_column(value) +} + +fn subjects(grants: &[mapping::MappedGrant]) -> Vec<&str> { + grants.iter().map(|grant| grant.subject.as_str()).collect() +} + +#[test] +fn table_mapping_reproduces_roster_and_drift() { + let fx = fixture(true); + let db = SourceDb::open(&fx.db_path).unwrap(); + assert_eq!(db.schema().unwrap(), SourceSchema::Tables); + assert_eq!(db.orphan_alias_count().unwrap(), 1); + + let mapping = map(&fx); + assert_eq!(mapping.knot_owner, acc("bailey")); + + assert_eq!( + subjects(&mapping.members), + [ + "did:plc:nel", + "did:plc:teq", + "did:plc:olaren", + "did:plc:uni" + ] + ); + let teq = &mapping.members[1]; + assert_eq!(teq.added_by, acc("nel")); + let uni = &mapping.members[3]; + assert!(uni.unioned); + assert_eq!(uni.added_by, acc("bailey")); + + let dids: Vec<&str> = mapping.repos.iter().map(|repo| repo.did.as_str()).collect(); + assert_eq!( + dids, + [ + "did:plc:squid", + "did:plc:limpet", + "did:plc:nautilus", + "did:web:nel.pet" + ] + ); + let web = &mapping.repos[3]; + assert_eq!(web.rkey.as_str(), "seashell"); + assert_eq!(web.name.as_str(), "seashell"); + assert!(web.collaborators.is_empty()); + let nautilus = &mapping.repos[2]; + assert_eq!(nautilus.rkey.as_str(), "seagrass"); + assert_eq!(nautilus.name.as_str(), "coralline"); + let squid = &mapping.repos[0]; + assert_eq!(squid.rkey.as_str(), "anemone"); + assert_eq!( + subjects(&squid.collaborators), + ["did:plc:isabel", "did:plc:olaren", "did:plc:teq"] + ); + assert_eq!(squid.collaborators[0].added_by, acc("nel")); + let limpet = &mapping.repos[1]; + assert_eq!(limpet.rkey.as_str(), "barnacle"); + assert_eq!(subjects(&limpet.collaborators), ["did:plc:teq"]); + assert!(limpet.collaborators[0].unioned); + assert_eq!(limpet.collaborators[0].added_by, acc("nel")); + + let reasons: Vec<(&str, &SkipReason)> = mapping + .skipped + .iter() + .map(|skip| (skip.repo_did.as_str(), &skip.reason)) + .collect(); + assert_eq!(reasons.len(), 4); + assert!(matches!( + reasons[0], + ("did:plc:conch", SkipReason::Name { .. }) + )); + assert!( + matches!(reasons[1], ("did:plc:whelk", SkipReason::NoSourceRepo)), + "a source directory that exists but holds no git repository skips the repo" + ); + match reasons[2] { + ("did:plc:clam", SkipReason::Rkey { value }) => { + assert_eq!( + value.as_str(), + "|", + "an rkey RepoRkey rejects skips the repo even where RepoName accepts the same text" + ); + } + other => panic!("unexpected third skip {other:?}"), + } + match reasons[3] { + ("did:plc:scallop", SkipReason::RkeyCollision { rkey, winner }) => { + assert_eq!(rkey.as_str(), "seagrass"); + assert_eq!(winner.as_str(), "did:plc:nautilus"); + } + other => panic!("unexpected fourth skip {other:?}"), + } + let lost: Vec> = mapping + .skipped + .iter() + .map(|skip| { + skip.lost_collaborators + .iter() + .map(AccountDid::as_str) + .collect() + }) + .collect(); + assert_eq!(lost, [vec![], vec!["did:plc:uni"], vec![], vec![]]); + + let drift = &mapping.drift; + assert_eq!( + drift.acl_only_collaborators, + [(srepo("did:plc:limpet"), sdid("did:plc:teq"))] + ); + assert_eq!( + drift.table_only_collaborators, + [(srepo("did:plc:squid"), sdid("did:plc:olaren"))] + ); + assert_eq!( + drift.slash_resolved_collaborators, + [(srepo("did:plc:squid"), sdid("did:plc:periwinkle"))] + ); + assert_eq!( + drift.orphan_collaborator_pairs, + [ + (srepo("did:plc:kelp"), sdid("did:plc:cuttle")), + (srepo("did:plc:kelp"), sdid("did:plc:teq")) + ] + ); + assert_eq!(drift.markerless_owner_repos, [srepo("did:plc:conch")]); + assert_eq!(drift.orphan_owner_markers, [srepo("did:plc:kelp")]); + assert_eq!( + drift.extra_owner_markers, + [(srepo("did:plc:limpet"), sdid("did:plc:bailey"))] + ); + assert_eq!(drift.acl_only_members, [sdid("did:plc:uni")]); + assert_eq!(drift.table_only_members, [sdid("did:plc:olaren")]); + assert_eq!(drift.slash_owner_markers, 1); + assert_eq!(drift.slash_collab_rows, 2); + assert_eq!(drift.unresolved_slash_forms, ["did:plc:nel/vanished"]); +} + +#[test] +fn collaborators_without_knot_members_is_rejected() { + let fx = fixture(true); + rusqlite::Connection::open(&fx.db_path) + .unwrap() + .execute_batch("drop table knot_members;") + .unwrap(); + let db = SourceDb::open(&fx.db_path).unwrap(); + assert!(matches!( + db.schema(), + Err(SourceError::CollaboratorsWithoutMembers) + )); +} + +#[test] +fn a_source_table_missing_expected_columns_is_rejected_early() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("knotserver.db"); + rusqlite::Connection::open(&db_path) + .unwrap() + .execute_batch( + " + create table repo_keys ( + repo_did text primary key, + signing_key blob, + created_at text not null + ); + create table repo_aliases ( + owner_did text not null, + rkey text not null, + repo_did text not null, + rev text not null + ); + ", + ) + .unwrap(); + let db = SourceDb::open(&db_path).unwrap(); + match db.schema() { + Err(SourceError::SchemaMismatch { table, missing }) => { + assert_eq!(table, "repo_keys"); + assert_eq!(missing, ["owner_did", "repo_name", "key_type"]); + } + other => panic!("a repo_keys older than knot's schema must be rejected, got {other:?}"), + } +} + +#[test] +fn preflip_mapping_reads_casbin() { + let fx = fixture(false); + let db = SourceDb::open(&fx.db_path).unwrap(); + assert_eq!(db.schema().unwrap(), SourceSchema::PreFlip); + + let mapping = map(&fx); + assert_eq!(mapping.knot_owner, acc("bailey")); + assert_eq!( + subjects(&mapping.members), + ["did:plc:nel", "did:plc:teq", "did:plc:uni"] + ); + let nel = &mapping.members[0]; + assert_eq!(nel.added_by, acc("bailey")); + assert_eq!(nel.created_at, knot_types::UnixSeconds::new(1767312000)); + let uni = &mapping.members[2]; + assert_eq!(uni.added_by, acc("bailey")); + assert_eq!(uni.created_at, knot_types::UnixSeconds::new(0)); + assert_eq!(mapping.drift.table_only_members, [sdid("did:plc:olaren")]); + + let squid = &mapping.repos[0]; + assert_eq!( + subjects(&squid.collaborators), + ["did:plc:isabel", "did:plc:periwinkle", "did:plc:teq"] + ); + let limpet = &mapping.repos[1]; + assert_eq!(subjects(&limpet.collaborators), ["did:plc:teq"]); + assert_eq!( + mapping.drift.orphan_collaborator_pairs, + [(srepo("did:plc:kelp"), sdid("did:plc:cuttle"))] + ); + assert_eq!( + mapping.drift.extra_owner_markers, + [(srepo("did:plc:limpet"), sdid("did:plc:bailey"))] + ); + let whelk = mapping + .skipped + .iter() + .find(|skip| skip.repo_did.as_str() == "did:plc:whelk") + .unwrap(); + assert_eq!(whelk.lost_collaborators, [acc("uni")]); +} + +#[test] +fn adoption_and_cobs_boot_a_working_index() { + let fx = fixture(true); + let mapping = map(&fx); + let knot = KnotId::new("did:web:knot.oyster.cafe").unwrap(); + let scan_path = fx.target.join("repos"); + std::fs::create_dir_all(&scan_path).unwrap(); + let layout = knot_git::Layout::new(&scan_path) + .with_object_format(ObjectFormat::SHA1) + .reserving_meta(&knot) + .unwrap(); + let signer = K256Signer::generate(&SeededEntropy::new(7)); + std::os::unix::fs::symlink( + "config", + fx.source_repos.join("did:plc:squid").join("config-link"), + ) + .unwrap(); + + let adoption = adopt::adopt_all( + &layout, + &fx.source_repos, + &mapping.repos, + adopt::SourcePolicy::Preserve, + ) + .unwrap(); + assert_eq!(adoption.adopted, 4); + assert_eq!(adoption.transfer, adopt::Transfer::Copy); + assert_eq!(adoption.already_present, 0); + assert_eq!(adoption.sha1, 4); + let adopted_link = layout + .repo_path(&RepoDid::new("did:plc:squid").unwrap()) + .unwrap() + .join("config-link"); + assert!( + std::fs::symlink_metadata(&adopted_link) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read_link(&adopted_link).unwrap(), + Path::new("config") + ); + + let cobs = emit::write_cobs(&layout, &knot, &mapping, &signer).unwrap(); + assert_eq!(cobs.members.appended, 4); + assert_eq!(cobs.registrations.appended, 4); + assert_eq!(cobs.collaborators.appended, 4); + + let index = Index::new(layout.meta_path(&knot).unwrap(), layout.clone()); + index.rebuild().unwrap(); + assert_eq!(index.hosted_repos().len(), 4); + let web = RepoDid::new("did:web:nel.pet").unwrap(); + assert_eq!( + index.owner_of(&web), + Resolved::Ready(Some(knot_types::OwnerDid::new("did:plc:nel").unwrap())) + ); + let squid = RepoDid::new("did:plc:squid").unwrap(); + index.ensure_collaborators(&squid).unwrap(); + assert_eq!( + index.effective_collaborator(&squid, &acc("isabel")), + Resolved::Ready(true) + ); + assert_eq!( + index.effective_collaborator(&squid, &acc("olaren")), + Resolved::Ready(true) + ); + assert_eq!( + index.effective_collaborator(&squid, &acc("teq")), + Resolved::Ready(true) + ); + assert_eq!( + index.effective_collaborator(&squid, &acc("periwinkle")), + Resolved::Ready(false) + ); + assert_eq!( + index.owner_of(&squid), + Resolved::Ready(Some(knot_types::OwnerDid::new("did:plc:nel").unwrap())) + ); + + let again = adopt::adopt_all( + &layout, + &fx.source_repos, + &mapping.repos, + adopt::SourcePolicy::Preserve, + ) + .unwrap(); + assert_eq!(again.adopted, 0); + assert_eq!(again.already_present, 4); + let recobs = emit::write_cobs(&layout, &knot, &mapping, &signer).unwrap(); + assert_eq!(recobs.members.appended, 0); + assert_eq!(recobs.members.already_present, 4); + assert_eq!(recobs.registrations.appended, 0); + assert_eq!(recobs.registrations.already_present, 4); + assert_eq!(recobs.collaborators.appended, 0); + assert_eq!(recobs.collaborators.already_present, 4); +} + +#[test] +fn consuming_the_source_moves_each_adopted_repo_out_of_the_scan_path() { + let fx = fixture(true); + let mapping = map(&fx); + let knot = KnotId::new("did:web:knot.oyster.cafe").unwrap(); + let scan_path = fx.target.join("repos"); + std::fs::create_dir_all(&scan_path).unwrap(); + let layout = knot_git::Layout::new(&scan_path) + .with_object_format(ObjectFormat::SHA1) + .reserving_meta(&knot) + .unwrap(); + + let adoption = adopt::adopt_all( + &layout, + &fx.source_repos, + &mapping.repos, + adopt::SourcePolicy::Consume, + ) + .unwrap(); + assert_eq!(adoption.transfer, adopt::Transfer::Rename); + assert_eq!(adoption.adopted, 4); + assert_eq!(adoption.already_present, 0); + assert!( + mapping + .repos + .iter() + .all(|repo| !adopt::source_dir(&fx.source_repos, &repo.source_did).exists()), + "every adopted repo leaves the source tree" + ); + assert!( + fx.source_repos.join("did:plc:conch").is_dir(), + "a skipped repo stays where it was" + ); + + let again = adopt::adopt_all( + &layout, + &fx.source_repos, + &mapping.repos, + adopt::SourcePolicy::Consume, + ) + .unwrap(); + assert_eq!(again.adopted, 0); + assert_eq!(again.already_present, 4); +} + +#[test] +fn adopting_a_repo_that_resolves_to_the_knot_meta_path_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("knotserver.db"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch(SCHEMA).unwrap(); + conn.execute_batch( + " + insert into repo_keys (repo_did, signing_key, created_at, owner_did, repo_name) values + ('did:web:knot.oyster.cafe', x'0909090909090909090909090909090909090909090909090909090909090909', '2026-06-01T10:00:00Z', 'did:plc:nel', 'seashell'); + insert into acl (p_type, v0, v1, v2, v3) values + ('g', 'did:plc:bailey', 'server:owner', 'thisserver', ''); + ", + ) + .unwrap(); + + let source_repos = dir.path().join("source-repos"); + knot_git::Repo::create_with_format( + source_repos.join("did:web:knot.oyster.cafe"), + ObjectFormat::SHA1, + ) + .unwrap(); + + let fx = Fixture { + db_path, + source_repos, + target: dir.path().join("target"), + _dir: dir, + }; + let mapping = map(&fx); + assert_eq!(mapping.repos.len(), 1); + + let knot = KnotId::new("did:web:knot.oyster.cafe").unwrap(); + let scan_path = fx.target.join("repos"); + std::fs::create_dir_all(&scan_path).unwrap(); + let layout = knot_git::Layout::new(&scan_path) + .with_object_format(ObjectFormat::SHA1) + .reserving_meta(&knot) + .unwrap(); + + let error = adopt::adopt_all( + &layout, + &fx.source_repos, + &mapping.repos, + adopt::SourcePolicy::Preserve, + ) + .unwrap_err(); + assert!(matches!( + error, + adopt::AdoptError::ReservesMeta { repo } if repo.as_str() == "did:web:knot.oyster.cafe" + )); + assert!(!layout.meta_path(&knot).unwrap().exists()); +} + +#[test] +fn rendered_config_loads() { + let dir = tempfile::tempdir().unwrap(); + let scan_path = dir.path().join("re\"pos\u{7f}"); + std::fs::create_dir_all(&scan_path).unwrap(); + let config = emit::render_config(&ConfigValues { + hostname: KnotHostname::new("knot.oyster.cafe").unwrap(), + admins: vec![acc("bailey")], + contact_email: "ops@oyster.cafe".to_string(), + scan_path: scan_path.clone(), + lfs_store_path: dir.path().join("lfs"), + ssh_host_key_file: dir.path().join("ssh_host_key"), + sealed_key_file: dir.path().join("sealed-keys"), + master_key_env: MasterKeyEnv::new("KNOT_MASTER_KEY").unwrap(), + object_format: ObjectFormat::SHA1, + plc_directory: Url::parse("https://plc.directory").unwrap(), + }) + .unwrap(); + assert!(config.contains("hostname = \"knot.oyster.cafe\"")); + assert!(config.contains("admins = [\"did:plc:bailey\"]")); + assert!(config.contains("admission = \"closed\"")); + assert!(config.contains("object_format = \"sha1\"")); + assert!(config.contains("email = \"ops@oyster.cafe\"")); + assert!(config.contains("store_path")); + let path = dir.path().join("config.toml"); + std::fs::write(&path, &config).unwrap(); + knot_config::load(Some(&path)).unwrap(); +} + +const HOST_KEY: &str = "-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACAuLv0N4MHTuclN/afhoL60chkky1gCLCFCA2T1qOKGhwAAAJgv0qFlL9Kh +ZQAAAAtzc2gtZWQyNTUxOQAAACAuLv0N4MHTuclN/afhoL60chkky1gCLCFCA2T1qOKGhw +AAAEAnnapXprdwlEwD6xIxSqm3szQrvfQdhRp6UfONp85Uky4u/Q3gwdO5yU39p+GgvrRy +GSTLWAIsIUIDZPWo4oaHAAAAEWtub3QtbWlncmF0ZS10ZXN0AQIDBA== +-----END OPENSSH PRIVATE KEY----- +"; + +const ECDSA_HOST_KEY: &str = "-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAaAAAABNlY2RzYS +1zaGEyLW5pc3RwMjU2AAAACG5pc3RwMjU2AAAAQQS6SLu5jEz+0ScKcByJBs53LlSkz8dT +ELlhV5QrNPQvk+h5UduwxR7ShN3IL9AhjiVugVN3I9vHHB1BwcNXE6exAAAAsD5jy3k+Y8 +t5AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLpIu7mMTP7RJwpw +HIkGzncuVKTPx1MQuWFXlCs09C+T6HlR27DFHtKE3cgv0CGOJW6BU3cj28ccHUHBw1cTp7 +EAAAAhAI2ARTG/6mM9qJfmdg8rASQudcrZ5KFLkH6FjB0V6EUgAAAAEWtub3QtbWlncmF0 +ZS10ZXN0AQIDBAUG +-----END OPENSSH PRIVATE KEY----- +"; + +#[test] +fn host_key_import_preserves_every_algorithm() { + let dir = tempfile::tempdir().unwrap(); + [ + ( + "ssh_host_ed25519_key", + HOST_KEY, + ssh_key::Algorithm::Ed25519, + ), + ( + "ssh_host_ecdsa_key", + ECDSA_HOST_KEY, + ssh_key::Algorithm::Ecdsa { + curve: ssh_key::EcdsaCurve::NistP256, + }, + ), + ] + .into_iter() + .for_each(|(name, pem, algorithm)| { + let source = dir.path().join(name); + std::fs::write(&source, pem).unwrap(); + let destination = dir.path().join(format!("{name}.imported")); + let host_key = emit::load_host_key(&source).unwrap(); + assert_eq!(host_key.algorithm, algorithm); + host_key.write_to(&destination).unwrap(); + assert_eq!( + std::fs::read(&source).unwrap(), + std::fs::read(&destination).unwrap(), + "{name} must round-trip byte-for-byte so the pinned fingerprint survives" + ); + }); +} + +#[test] +fn the_public_half_of_a_host_key_is_refused_with_its_own_error() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("ssh_host_ed25519_key.pub"); + let public = ssh_key::PrivateKey::from_openssh(HOST_KEY) + .unwrap() + .public_key() + .to_openssh() + .unwrap(); + std::fs::write(&source, format!("{public}\n")).unwrap(); + match emit::load_host_key(&source) { + Err(error @ emit::EmitError::PublicHostKey { .. }) => assert!( + error.to_string().contains("without the .pub"), + "the refusal has to point at the private half: {error}" + ), + other => panic!( + "a public key mustn't read as a corrupt private key: {:?}", + other.err() + ), + } + std::fs::write(&source, "not a key of any kind\n").unwrap(); + match emit::load_host_key(&source) { + Err(emit::EmitError::HostKey { .. }) => {} + other => panic!( + "a file that isn't a key of either kind is still a parse failure: {:?}", + other.err() + ), + } +} + +fn honors_permission_bits(dir: &Path) -> bool { + use std::os::unix::fs::PermissionsExt; + let probe = dir.join("permission-probe"); + std::fs::create_dir(&probe).unwrap(); + std::fs::set_permissions(&probe, std::fs::Permissions::from_mode(0o000)).unwrap(); + let denied = std::fs::read_dir(&probe).is_err(); + std::fs::set_permissions(&probe, std::fs::Permissions::from_mode(0o755)).unwrap(); + std::fs::remove_dir(&probe).unwrap(); + if !denied { + eprintln!( + "skipping the permission case: this process reads a 0000 directory, so it's running \ + as root" + ); + } + denied +} + +#[test] +fn probing_a_source_separates_a_repo_from_an_absence_and_from_an_unreadable_path() { + let fx = fixture(true); + std::fs::write(fx.source_repos.join("did:plc:mussel"), "").unwrap(); + let looped = fx.source_repos.join("did:plc:cuttle"); + std::os::unix::fs::symlink(&looped, &looped).unwrap(); + let probe = |did: &str| adopt::probe_source(&fx.source_repos, &srepo(did)); + if honors_permission_bits(&fx.source_repos) { + use std::os::unix::fs::PermissionsExt; + let squid = fx.source_repos.join("did:plc:squid"); + std::fs::set_permissions(&squid, std::fs::Permissions::from_mode(0o000)).unwrap(); + assert_eq!( + probe("did:plc:squid"), + adopt::SourceProbe::Unreadable(std::io::ErrorKind::PermissionDenied), + "secure mode leaves repo trees at a per-owner uid, and that's how a wrong-user \ + migration sees them" + ); + std::fs::set_permissions(&squid, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + assert_eq!(probe("did:plc:squid"), adopt::SourceProbe::Repo); + assert_eq!( + probe("did:plc:kelp"), + adopt::SourceProbe::Absent, + "a repo whose directory an operator deleted mustn't refuse every future migration" + ); + assert_eq!( + probe("did:plc:whelk"), + adopt::SourceProbe::Absent, + "did:plc:whelk has a readable directory without a HEAD, which the mapping skips on its own" + ); + assert_eq!( + probe("did:plc:mussel"), + adopt::SourceProbe::Absent, + "a regular file where a repo directory belongs isn't a repository either" + ); + assert!( + matches!(probe("did:plc:cuttle"), adopt::SourceProbe::Unreadable(_)), + "root can't step over a symlink loop, so this case covers the unreadable path under any uid" + ); +} + +#[test] +fn an_unreadable_source_is_skipped_separately_from_an_absent_source() { + let fx = fixture(true); + let mapping = map_probing(&fx, |did| match did.as_str() { + "did:plc:squid" => adopt::SourceProbe::Unreadable(std::io::ErrorKind::PermissionDenied), + "did:plc:limpet" => { + adopt::SourceProbe::Unreadable(std::io::ErrorKind::StaleNetworkFileHandle) + } + _ => adopt::SourceProbe::Repo, + }); + let reason = |wanted: &str| { + mapping + .skipped + .iter() + .find(|skip| skip.repo_did.as_str() == wanted) + .map(|skip| skip.reason.clone()) + }; + assert_eq!( + reason("did:plc:squid"), + Some(SkipReason::UnreadableSource { + kind: std::io::ErrorKind::PermissionDenied + }), + "the report mustn't call an unreadable repo missing" + ); + assert_eq!( + reason("did:plc:limpet"), + Some(SkipReason::UnreadableSource { + kind: std::io::ErrorKind::StaleNetworkFileHandle + }), + "an error that isn't a permission error mustn't lose the repo either" + ); + assert_eq!( + reason("did:plc:kelp"), + None, + "did:plc:kelp has an alias and an acl grant without a repo_keys row" + ); + assert_eq!( + mapping.unreadable_sources().to_string(), + "did:plc:squid, did:plc:limpet" + ); + let readable = map(&fx); + assert!(readable.unreadable_sources().is_empty()); + assert!( + readable + .skipped + .iter() + .any(|skip| skip.reason == SkipReason::NoSourceRepo), + "did:plc:whelk has a directory without a HEAD in it, which stays a plain absence" + ); +} + +#[test] +fn naming_unreadable_repos_stops_at_five_and_counts_the_rest() { + let fx = fixture(true); + let mapping = map_probing(&fx, |_| { + adopt::SourceProbe::Unreadable(std::io::ErrorKind::PermissionDenied) + }); + let listed = mapping.unreadable_sources().to_string(); + assert!( + listed.starts_with( + "did:plc:squid, did:plc:limpet, did:plc:whelk, did:plc:nautilus, did:plc:scallop" + ), + "{listed}" + ); + assert!( + listed.ends_with(", and 1 more"), + "did:plc:conch and did:plc:clam are skipped before the probe, so 6 of the 8 rows are \ + unreadable: {listed}" + ); +} + +fn set_acl_owner(fx: &Fixture, repo: &str, acl_owner: &str) { + rusqlite::Connection::open(&fx.db_path) + .unwrap() + .execute( + "update acl set v0 = ?1 where v2 = ?2 and v3 = 'repo:owner'", + rusqlite::params![acl_owner, repo], + ) + .unwrap(); +} + +#[test] +fn two_owners_for_one_repo_are_drift_that_the_report_can_render() { + let fx = fixture(true); + set_acl_owner(&fx, "did:plc:scallop", "did:plc:teq"); + let mapping = map(&fx); + assert_eq!( + mapping.drift.conflicting_owner_markers, + vec![mapping::OwnerConflict { + repo: SourceRepoDid::from_column("did:plc:scallop"), + acl_owner: SourceDid::from_column("did:plc:teq"), + key_owner: SourceDid::from_column("did:plc:isabel"), + }] + ); + assert_eq!(mapping.conflicting_owners().to_string(), "did:plc:scallop"); + assert!( + !mapping + .drift + .extra_owner_markers + .iter() + .any(|(repo, _)| repo.as_str() == "did:plc:scallop"), + "a repo recorded with a different owner mustn't also appear as an extra marker: {:?}", + mapping.drift.extra_owner_markers + ); + let rendered = report::Report { + mapping: &mapping, + orphan_alias_count: 0, + phase: report::Phase::Refused, + } + .to_string(); + assert!( + rendered.contains("repos recorded with different owners in the acl and repo_keys: 1"), + "{rendered}" + ); + assert!( + rendered.contains("did:plc:scallop acl did:plc:teq, repo_keys did:plc:isabel"), + "{rendered}" + ); +} + +#[test] +fn a_report_names_drift_only_where_there_is_drift() { + let render = |drift, orphan_alias_count| { + let mapping = mapping::Mapping { + knot_owner: AccountDid::new("did:plc:akshay").unwrap(), + members: Vec::new(), + repos: Vec::new(), + skipped: Vec::new(), + drift, + }; + report::Report { + mapping: &mapping, + orphan_alias_count, + phase: report::Phase::Refused, + } + .to_string() + }; + + let agreed = render(mapping::Drift::default(), 0); + assert!( + agreed.contains("casbin cross-check: the acl and the tables agree"), + "{agreed}" + ); + assert!( + !agreed.contains("drift"), + "thirteen zeroes are what an operator has to read past to find the one line that matters: \ + {agreed}" + ); + + let drifted = render( + mapping::Drift { + slash_owner_markers: 2, + ..mapping::Drift::default() + }, + 0, + ); + assert!(drifted.contains("casbin cross-check drift:"), "{drifted}"); + assert!(drifted.contains("slash-form owner markers: 2"), "{drifted}"); + assert!( + !drifted.contains("acl-only members unioned in"), + "{drifted}" + ); + + let orphaned = render(mapping::Drift::default(), 3); + assert!(orphaned.contains("orphan aliases: 3"), "{orphaned}"); + assert!( + !orphaned.contains("the acl and the tables agree"), + "orphan aliases are a reconciliation the operator still owes: {orphaned}" + ); +} + +fn rehearse_scan_path(source: &Path, scan_path: &Path, policy: adopt::SourcePolicy) -> Rehearsal { + rehearse_adopting(source, &[], scan_path, policy) +} + +fn rehearse_adopting( + source: &Path, + adopted: &[mapping::AdoptRepo], + scan_path: &Path, + policy: adopt::SourcePolicy, +) -> Rehearsal { + Rehearsal::run(rehearse::Inputs { + source_repos: source, + adopted, + scan_path, + policy, + host_key: None, + host_key_target: &scan_path.with_file_name("ssh_host_key"), + host_key_policy: emit::HostKeyPolicy::Keep, + master_key: &master_key_env(), + }) +} + +#[test] +fn a_rehearsal_plans_its_transfer_and_probes_the_path_that_the_real_run_will_create() { + let fx = fixture(true); + let scan_path = fx.target.join("repos"); + let missing = rehearse_scan_path(&fx.source_repos, &scan_path, adopt::SourcePolicy::Consume); + assert_eq!(missing.transfer.unwrap(), adopt::Transfer::Rename); + assert_eq!( + missing.fallback.as_deref(), + fx.target.parent(), + "the migration will create the scan path, so the filesystem checks use the deepest path \ + that exists now" + ); + + std::fs::create_dir_all(&scan_path).unwrap(); + let fresh = rehearse_scan_path(&fx.source_repos, &scan_path, adopt::SourcePolicy::Consume); + assert_eq!(fresh.transfer.unwrap(), adopt::Transfer::Rename); + assert_eq!(fresh.fallback, None); + assert_eq!(fresh.scan_path.unwrap(), rehearse::Occupancy::Fresh); + assert!( + fresh.room.is_none(), + "a rename doesn't need a second copy of anything" + ); + + std::fs::create_dir(scan_path.join("did:plc:squid")).unwrap(); + let occupied = rehearse_scan_path(&fx.source_repos, &scan_path, adopt::SourcePolicy::Preserve); + assert_eq!( + occupied.transfer.unwrap(), + adopt::Transfer::Copy, + "the default policy copies, so it will never compare the two filesystems" + ); + assert_eq!(occupied.scan_path.unwrap(), rehearse::Occupancy::Occupied); + + let relative = rehearse_scan_path( + &fx.source_repos, + Path::new("knot-migrate-nowhere/repos"), + adopt::SourcePolicy::Preserve, + ); + assert_eq!( + relative.fallback.as_deref(), + Some(Path::new(".")), + "probing / instead would answer for a filesystem that the migration never touches" + ); +} + +fn on_another_filesystem(reference: &Path) -> Option { + use std::os::unix::fs::MetadataExt; + let device = |path: &Path| std::fs::metadata(path).ok().map(|meta| meta.dev()); + tempfile::TempDir::new_in("/dev/shm") + .ok() + .filter(|elsewhere| device(elsewhere.path()) != device(reference)) +} + +#[test] +fn rehearsing_a_cross_filesystem_consume_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let Some(elsewhere) = on_another_filesystem(dir.path()) else { + eprintln!("skipping the cross-filesystem case: /dev/shm is on this tempdir's filesystem"); + return; + }; + assert!(matches!( + rehearse_scan_path(dir.path(), elsewhere.path(), adopt::SourcePolicy::Consume).transfer, + Err(adopt::AdoptError::CrossDeviceConsume { .. }) + )); + assert_eq!( + rehearse_scan_path(dir.path(), elsewhere.path(), adopt::SourcePolicy::Preserve) + .transfer + .unwrap(), + adopt::Transfer::Copy, + "the default policy copies, so two filesystems suit it fine" + ); +} + +#[test] +fn a_rehearsal_measures_the_room_that_adoption_will_copy() { + let fx = fixture(true); + let mapping = map(&fx); + let measure = || { + rehearse_adopting( + &fx.source_repos, + &mapping.repos, + &fx.target.join("repos"), + adopt::SourcePolicy::Preserve, + ) + .room + .unwrap() + .unwrap() + }; + let measured = measure(); + assert!(measured.source > rehearse::Bytes::new(0), "{measured:?}"); + assert_eq!(measured.fit(), rehearse::Fit::Clear, "{measured:?}"); + std::fs::write( + fx.source_repos.join("did:plc:whelk/stray.pack"), + vec![0_u8; 1 << 20], + ) + .unwrap(); + assert_eq!( + measure().source, + measured.source, + "did:plc:whelk doesn't have a HEAD and stays out of the mapping, so adoption will never \ + read a byte of it" + ); + let fit = |source: u64, free: u64| { + rehearse::Room { + source: rehearse::Bytes::new(source), + free: rehearse::Bytes::new(free), + } + .fit() + }; + assert_eq!(fit(1_000, 999), rehearse::Fit::Short); + assert_eq!(fit(1_000, 1_000), rehearse::Fit::Clear); + assert_eq!( + fit(0, 0), + rehearse::Fit::Clear, + "a rehearsal that won't adopt a repo doesn't need room" + ); +} + +fn scan_path_refusal(build: impl FnOnce(&Path) -> PathBuf) -> Rehearsal { + let dir = tempfile::tempdir().unwrap(); + let scan_path = build(dir.path()); + rehearse_scan_path(dir.path(), &scan_path, adopt::SourcePolicy::Preserve) +} + +#[test] +fn a_scan_path_that_the_real_run_cannot_reach_is_refused_whichever_user_runs_it() { + let under_a_file = scan_path_refusal(|dir| { + let blocker = dir.join("not-a-directory"); + std::fs::write(&blocker, "").unwrap(); + blocker.join("knot/repos") + }); + assert!( + matches!( + under_a_file.scan_path, + Err(rehearse::ScanPathError::Uncreatable { .. }) + ), + "root can't traverse a regular file either, so this case covers the refusal under any uid: \ + {:?}", + under_a_file.scan_path + ); + let looped = scan_path_refusal(|dir| { + let loop_path = dir.join("loop"); + std::os::unix::fs::symlink(&loop_path, &loop_path).unwrap(); + loop_path.join("repos") + }); + assert!( + matches!( + looped.scan_path, + Err(rehearse::ScanPathError::Unwritable { .. }) + ), + "a path that this process can't examine mustn't read as a path that the migration will \ + create: {:?}", + looped.scan_path + ); + let dangling = scan_path_refusal(|dir| { + let scan_path = dir.join("repos"); + std::os::unix::fs::symlink(dir.join("nowhere"), &scan_path).unwrap(); + scan_path + }); + assert!( + matches!( + dangling.scan_path, + Err(rehearse::ScanPathError::Dangling { .. }) + ), + "std::fs::create_dir_all refuses a symlink to a missing target with AlreadyExists, so the \ + rehearsal mustn't read it as a path that the migration will create: {:?}", + dangling.scan_path + ); + assert_eq!( + dangling.fallback, None, + "the symlink itself is what the migration fails on, so the checks stay on it and don't step \ + up to its parent" + ); + [under_a_file, looped, dangling] + .iter() + .for_each(|rehearsal| assert!(!rehearsal.ready())); +} + +#[test] +fn a_scan_path_that_this_process_cannot_write_is_refused() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + if !honors_permission_bits(dir.path()) { + return; + } + let closed = |name: &str| { + let path = dir.path().join(name); + std::fs::create_dir(&path).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o555)).unwrap(); + path + }; + let parent = closed("parent"); + let existing = closed("repos"); + let uncreatable = rehearse_scan_path( + dir.path(), + &parent.join("knot/repos"), + adopt::SourcePolicy::Preserve, + ); + let unwritable = rehearse_scan_path(dir.path(), &existing, adopt::SourcePolicy::Preserve); + [&parent, &existing].iter().for_each(|path| { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + }); + assert!( + matches!( + uncreatable.scan_path, + Err(rehearse::ScanPathError::Uncreatable { .. }) + ), + "the migration will create the scan path, so an unwritable ancestor stops it: {:?}", + uncreatable.scan_path + ); + assert!( + matches!( + unwritable.scan_path, + Err(rehearse::ScanPathError::Unwritable { .. }) + ), + "a scan path that already exists doesn't need creating, so the refusal mustn't blame its \ + parent: {:?}", + unwritable.scan_path + ); + assert!(!uncreatable.ready() && !unwritable.ready()); +} + +fn master_key_env() -> MasterKeyEnv { + MasterKeyEnv::new("KNOT_MASTER_KEY").unwrap() +} + +fn unset_master_key_env() -> MasterKeyEnv { + MasterKeyEnv::new("KNOT_MASTER_KEY_THAT_NOBODY_SETS").unwrap() +} + +fn ready_rehearsal() -> Rehearsal { + Rehearsal { + fallback: None, + transfer: Ok(adopt::Transfer::Copy), + scan_path: Ok(rehearse::Occupancy::Fresh), + room: Some(Ok(rehearse::Room { + source: rehearse::Bytes::new(1), + free: rehearse::Bytes::new(2), + })), + host_key: Ok(ssh_key::Algorithm::Ed25519), + host_key_target: Some(Ok(emit::HostKeyPlacement::Fresh)), + master_key: Ok(master_key_env()), + } +} + +#[test] +fn a_rehearsal_is_ready_only_once_the_copy_has_room() { + assert!(ready_rehearsal().ready()); + let cramped = Rehearsal { + room: Some(Ok(rehearse::Room { + source: rehearse::Bytes::new(2), + free: rehearse::Bytes::new(1), + })), + ..ready_rehearsal() + }; + assert!(!cramped.ready()); + let unmeasured = Rehearsal { + room: None, + ..ready_rehearsal() + }; + assert!( + unmeasured.ready(), + "a rename doesn't measure room at all, which mustn't read as a copy that won't fit" + ); +} + +#[test] +fn a_rehearsal_reads_a_usable_host_key_and_reports_an_unusable_file() { + let dir = tempfile::tempdir().unwrap(); + let target_key = dir.path().join("knot/ssh_host_key"); + let rehearse_host_key = |path: PathBuf| { + Rehearsal::run(rehearse::Inputs { + source_repos: dir.path(), + adopted: &[], + scan_path: &dir.path().join("knot/repos"), + policy: adopt::SourcePolicy::Preserve, + host_key: Some(&path), + host_key_target: &target_key, + host_key_policy: emit::HostKeyPolicy::Keep, + master_key: &master_key_env(), + }) + }; + assert!(matches!( + rehearse_host_key(host_key_file(dir.path())).host_key, + Ok(ssh_key::Algorithm::Ed25519) + )); + let missing = rehearse_host_key(dir.path().join("no-such-key")); + assert!(matches!( + missing.host_key, + Err(rehearse::HostKeyError::Unusable(_)) + )); + assert!( + missing.transfer.is_ok() && missing.scan_path.is_ok(), + "one unreadable input mustn't take the other checks down with it" + ); + assert!(!missing.ready()); +} + +#[test] +fn a_host_key_already_at_the_target_is_kept_until_the_switchover_is_forced() { + let dir = tempfile::tempdir().unwrap(); + let source = host_key_file(dir.path()); + let target = dir.path().join("target"); + std::fs::create_dir_all(&target).unwrap(); + let destination = target.join("ssh_host_key"); + let rehearse = |policy| { + Rehearsal::run(rehearse::Inputs { + source_repos: dir.path(), + adopted: &[], + scan_path: &target.join("repos"), + policy: adopt::SourcePolicy::Preserve, + host_key: Some(&source), + host_key_target: &destination, + host_key_policy: policy, + master_key: &master_key_env(), + }) + }; + + assert!(matches!( + rehearse(emit::HostKeyPolicy::Keep).host_key_target, + Some(Ok(emit::HostKeyPlacement::Fresh)) + )); + + emit::load_host_key(&source) + .unwrap() + .write_to(&destination) + .unwrap(); + assert!(matches!( + rehearse(emit::HostKeyPolicy::Keep).host_key_target, + Some(Ok(emit::HostKeyPlacement::Unchanged)), + )); + + std::fs::write(&destination, ECDSA_HOST_KEY).unwrap(); + let clash = rehearse(emit::HostKeyPolicy::Keep); + assert!( + matches!( + clash.host_key_target, + Some(Err(emit::HostKeyConflict::Different { .. })) + ), + "a knot that started before the switchover leaves a key of its own" + ); + assert!( + clash.transfer.is_ok() && clash.scan_path.is_ok(), + "the conflict mustn't take the other checks down with it" + ); + assert!(matches!( + rehearse(emit::HostKeyPolicy::Replace).host_key_target, + Some(Ok(emit::HostKeyPlacement::Replacing)) + )); + + std::fs::write(&destination, b"whatever this file is, it isn't a key").unwrap(); + assert!( + matches!( + rehearse(emit::HostKeyPolicy::Keep).host_key_target, + Some(Err(emit::HostKeyConflict::Unparsable { .. })) + ), + "a file that doesn't parse here might still be the key the old knot is serving" + ); + assert!( + matches!( + rehearse(emit::HostKeyPolicy::Replace).host_key_target, + Some(Ok(emit::HostKeyPlacement::Replacing)) + ), + "the file the migration can't read is the one an operator most wants gone" + ); + + std::fs::remove_file(&destination).unwrap(); + std::os::unix::fs::symlink(&source, &destination).unwrap(); + let dangling = target.join("dangling"); + std::os::unix::fs::symlink(target.join("gone"), &dangling).unwrap(); + [ + (emit::HostKeyPolicy::Keep, &destination), + (emit::HostKeyPolicy::Replace, &destination), + (emit::HostKeyPolicy::Replace, &dangling), + ] + .into_iter() + .for_each(|(policy, path)| { + assert!( + matches!( + Rehearsal::run(rehearse::Inputs { + source_repos: dir.path(), + adopted: &[], + scan_path: &target.join("repos"), + policy: adopt::SourcePolicy::Preserve, + host_key: Some(&source), + host_key_target: path, + host_key_policy: policy, + master_key: &master_key_env(), + }) + .host_key_target, + Some(Err(emit::HostKeyConflict::NotAFile { .. })) + ), + "OpenOptions::open follows a symlink, so the key would go somewhere the operator \ + never named: {policy:?} {path:?}" + ); + }); + + std::fs::remove_file(&destination).unwrap(); + std::fs::create_dir(&destination).unwrap(); + assert!( + matches!( + rehearse(emit::HostKeyPolicy::Replace).host_key_target, + Some(Err(emit::HostKeyConflict::NotAFile { .. })) + ), + "a directory would fail at write time, long after adoption has moved every repo" + ); + std::fs::remove_dir(&destination).unwrap(); + + assert!( + !Rehearsal { + host_key_target: Some(Err(emit::HostKeyConflict::Different { + path: destination, + fingerprints: Box::new(emit::Fingerprints { + found: emit::load_host_key(&source).unwrap().fingerprint, + importing: emit::load_host_key(&source).unwrap().fingerprint, + }), + })), + ..ready_rehearsal() + } + .ready() + ); + assert!( + Rehearsal { + host_key_target: Some(Ok(emit::HostKeyPlacement::Replacing)), + ..ready_rehearsal() + } + .ready() + ); +} + +#[test] +fn a_host_key_target_this_process_cannot_write_is_refused_before_adoption() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + if !honors_permission_bits(dir.path()) { + return; + } + let key = emit::load_host_key(&host_key_file(dir.path())).unwrap(); + let target = dir.path().join("target"); + std::fs::create_dir(&target).unwrap(); + let destination = target.join("ssh_host_key"); + std::fs::write(&destination, ECDSA_HOST_KEY).unwrap(); + let chmod = |path: &Path, mode| { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap() + }; + + chmod(&destination, 0o000); + assert!( + matches!( + emit::plan_host_key(&destination, &key, emit::HostKeyPolicy::Keep), + Err(emit::HostKeyConflict::Unreadable { .. }) + ), + "a key this process can't open is not a key it has read and found unparsable" + ); + assert!( + matches!( + emit::plan_host_key(&destination, &key, emit::HostKeyPolicy::Replace), + Err(emit::HostKeyConflict::Unwritable { .. }) + ), + "forcing this would otherwise fail at write time, long after adoption has moved every repo" + ); + + chmod(&destination, 0o200); + assert!( + matches!( + emit::plan_host_key(&destination, &key, emit::HostKeyPolicy::Replace), + Ok(emit::HostKeyPlacement::Replacing) + ), + "a file this process can write is one that --force-host-key can still land on" + ); + chmod(&destination, 0o600); + + let sealed = dir.path().join("sealed"); + std::fs::create_dir(&sealed).unwrap(); + chmod(&sealed, 0o500); + assert!( + matches!( + emit::plan_host_key( + &sealed.join("ssh_host_key"), + &key, + emit::HostKeyPolicy::Keep + ), + Err(emit::HostKeyConflict::Uncreatable { .. }) + ), + "an absent key under a directory nobody can write is a write that fails after adoption" + ); + chmod(&sealed, 0o700); + + assert!( + matches!( + emit::plan_host_key( + &dir.path().join("nothing/here/ssh_host_key"), + &key, + emit::HostKeyPolicy::Keep + ), + Ok(emit::HostKeyPlacement::Fresh) + ), + "a target directory that doesn't exist yet belongs to create_dir_all, which runs seconds \ + later and long before adoption" + ); +} + +#[test] +fn a_symlink_where_a_private_file_goes_is_never_followed() { + let dir = tempfile::tempdir().unwrap(); + let key = emit::load_host_key(&host_key_file(dir.path())).unwrap(); + let decoy = dir.path().join("decoy"); + let archive = dir.path().join("repo-signing-keys.json"); + let planted = dir.path().join("ssh_host_key"); + std::fs::write(&decoy, "untouched").unwrap(); + std::os::unix::fs::symlink(&decoy, &archive).unwrap(); + std::os::unix::fs::symlink(&decoy, &planted).unwrap(); + + assert!( + emit::write_key_archive(&archive, &[]).is_err(), + "every repo's signing key would land wherever the symlink points, and chmod 0600 would \ + dress up the wrong file" + ); + assert!( + key.write_to(&planted).is_err(), + "plan_host_key refuses a symlink long before the write, which is check-then-use unless the \ + open refuses it too" + ); + assert_eq!(std::fs::read_to_string(&decoy).unwrap(), "untouched"); +} + +#[test] +fn a_rehearsal_report_names_what_the_key_at_the_target_costs() { + let mapping = mapping::Mapping { + knot_owner: AccountDid::new("did:plc:akshay").unwrap(), + members: Vec::new(), + repos: Vec::new(), + skipped: Vec::new(), + drift: mapping::Drift::default(), + }; + let render = + |host_key_target: Option>| { + let rehearsal = Rehearsal { + host_key_target, + ..ready_rehearsal() + }; + report::Report { + mapping: &mapping, + orphan_alias_count: 0, + phase: report::Phase::Rehearsed(&rehearsal), + } + .to_string() + }; + + [None, Some(Ok(emit::HostKeyPlacement::Fresh))] + .into_iter() + .for_each(|quiet| { + let rendered = render(quiet); + assert!( + rendered.contains("host key algorithm: ") && !rendered.contains("host key: "), + "a target with no key on it costs the operator nothing to read: {rendered}" + ); + }); + + let unchanged = render(Some(Ok(emit::HostKeyPlacement::Unchanged))); + assert!( + unchanged.contains("host key: the target already has the imported key"), + "{unchanged}" + ); + + let replacing = render(Some(Ok(emit::HostKeyPlacement::Replacing))); + assert!( + replacing.contains("host key: the migration will replace the different key at the target"), + "{replacing}" + ); + + let unwritable = render(Some(Err(emit::HostKeyConflict::Unwritable { + path: PathBuf::from("/srv/knot/ssh_host_key"), + source: rustix::io::Errno::ACCESS, + }))); + assert!( + unwritable.contains( + "host key: The migration will write the host key over \ + /srv/knot/ssh_host_key, which this process can't write" + ), + "{unwritable}" + ); + + let not_a_file = render(Some(Err(emit::HostKeyConflict::NotAFile { + path: PathBuf::from("/srv/knot/ssh_host_key"), + }))); + assert!( + not_a_file.contains("host key: /srv/knot/ssh_host_key isn't a regular file"), + "{not_a_file}" + ); +} + +#[test] +fn a_rehearsal_states_the_inputs_that_the_real_run_still_needs() { + let fx = fixture(true); + let mapping = map(&fx); + let rehearsal = Rehearsal::run(rehearse::Inputs { + source_repos: &fx.source_repos, + adopted: &mapping.repos, + scan_path: &fx.target.join("repos"), + policy: adopt::SourcePolicy::Preserve, + host_key: None, + host_key_target: &fx.target.join("ssh_host_key"), + host_key_policy: emit::HostKeyPolicy::Keep, + master_key: &unset_master_key_env(), + }); + assert!(!rehearsal.ready()); + let rendered = report::Report { + mapping: &mapping, + orphan_alias_count: 0, + phase: report::Phase::Rehearsed(&rehearsal), + } + .to_string(); + assert!(rendered.contains("transfer mode: copy"), "{rendered}"); + assert!(rendered.contains("scan path: writable"), "{rendered}"); + assert!(rendered.contains("room to copy: "), "{rendered}"); + assert!( + rendered.contains(", since the scan path doesn't exist yet"), + "{rendered}" + ); + assert!( + rendered.contains("host key: --host-key is required for the migration"), + "{rendered}" + ); + assert!( + rendered.contains("Master key env var KNOT_MASTER_KEY_THAT_NOBODY_SETS isn't set"), + "{rendered}" + ); +} + +#[test] +fn a_master_key_reads_from_its_env_var_and_refuses_a_weak_or_malformed_value() { + assert!(matches!( + unset_master_key_env().read(), + Err(emit::MasterKeyError::Unset(_)) + )); + let padded = format!(" {}\n", base64_standard(&[7_u8; 32])); + assert!(master_key_env().decode(&padded).is_ok()); + let short = master_key_env().decode(&base64_standard(&[7_u8; 31])); + assert!( + matches!(short, Err(emit::MasterKeyError::Weak { .. })), + "{short:?}" + ); + assert!(matches!( + master_key_env().decode("not base64 at all!"), + Err(emit::MasterKeyError::NotBase64(_)) + )); +} + +fn base64_standard(bytes: &[u8]) -> String { + use base64::Engine; + base64::engine::general_purpose::STANDARD.encode(bytes) +} + +fn host_key_file(dir: &Path) -> PathBuf { + let path = dir.join("ssh_host_ed25519_key"); + std::fs::write(&path, HOST_KEY).unwrap(); + path +} + +fn run_migrate(fx: &Fixture, host_key: &Path, extra: &[&str]) -> std::process::Output { + std::process::Command::new(env!("CARGO_BIN_EXE_knot-migrate")) + .args([ + "--source-db", + fx.db_path.to_str().unwrap(), + "--source-repos", + fx.source_repos.to_str().unwrap(), + "--host-key", + host_key.to_str().unwrap(), + "--hostname", + "knot.oyster.cafe", + "--plc-url", + "https://plc.directory", + "--contact-email", + "ops@oyster.cafe", + "--target", + fx.target.to_str().unwrap(), + ]) + .args(extra) + .env("KNOT_MASTER_KEY", base64_standard(&[7_u8; 32])) + .output() + .unwrap() +} + +fn with_unreadable_repo(fx: &Fixture, work: impl FnOnce() -> T) -> Option { + use std::os::unix::fs::PermissionsExt; + if !honors_permission_bits(&fx.source_repos) { + return None; + } + let limpet = fx.source_repos.join("did:plc:limpet"); + std::fs::set_permissions(&limpet, std::fs::Permissions::from_mode(0o000)).unwrap(); + let outcome = work(); + std::fs::set_permissions(&limpet, std::fs::Permissions::from_mode(0o755)).unwrap(); + Some(outcome) +} + +#[test] +fn a_real_run_refuses_the_key_at_the_target_before_it_touches_a_repo() { + let fx = fixture(true); + let dir = tempfile::tempdir().unwrap(); + let host_key = host_key_file(dir.path()); + std::fs::create_dir_all(&fx.target).unwrap(); + let destination = fx.target.join("ssh_host_key"); + std::fs::write(&destination, ECDSA_HOST_KEY).unwrap(); + + let importing = emit::load_host_key(&host_key).unwrap().fingerprint; + let found = emit::load_host_key(&destination).unwrap().fingerprint; + + let refused = run_migrate(&fx, &host_key, &[]); + let refusal = String::from_utf8_lossy(&refused.stderr).to_string(); + assert!(!refused.status.success(), "{refusal}"); + assert!(refusal.contains("your users already trust"), "{refusal}"); + assert!( + refusal.contains(&format!( + "whose fingerprint {found} your users already trust" + )) && refusal.contains(&importing.to_string()), + "an operator deciding whether to force needs both fingerprints, not the word: {refusal}" + ); + assert!( + !fx.target.join("repos").exists() && !fx.target.join("sealed-keys").exists(), + "the refusal has to come before adoption, which moves every repo" + ); + assert_eq!( + std::fs::read_to_string(&destination).unwrap(), + ECDSA_HOST_KEY, + "a refused run mustn't touch the key it refused" + ); + + let forced = run_migrate(&fx, &host_key, &["--force-host-key"]); + let stdout = String::from_utf8_lossy(&forced.stdout).to_string(); + let stderr = String::from_utf8_lossy(&forced.stderr).to_string(); + assert!(!forced.status.success(), "{stdout}{stderr}"); + assert!( + stderr.contains("the migration skipped 4 repos"), + "the fixture skips 4, and a scripted cutover needs the skips in the exit: {stderr}" + ); + assert_eq!( + std::fs::read(&destination).unwrap(), + std::fs::read(&host_key).unwrap(), + "--force-host-key has to leave the imported key at the target" + ); + assert!( + stdout.contains("host key: the migration replaced the different key at the target"), + "an operator who forced the switchover has to read what it cost: {stdout}" + ); + assert!( + stdout.contains(&format!("host key fingerprint: {importing}")), + "the fingerprint everybody has to trust now is the one worth printing: {stdout}" + ); +} + +#[test] +fn a_real_run_refuses_an_unreadable_source_until_it_is_told_to_skip_it() { + let fx = fixture(true); + let dir = tempfile::tempdir().unwrap(); + let host_key = host_key_file(dir.path()); + let Some((refused, target_after_refusal, outcome)) = with_unreadable_repo(&fx, || { + let refused = run_migrate(&fx, &host_key, &[]); + let target_after_refusal = fx.target.exists(); + ( + refused, + target_after_refusal, + run_migrate(&fx, &host_key, &["--skip-unreadable"]), + ) + }) else { + return; + }; + let refusal = String::from_utf8_lossy(&refused.stderr).to_string(); + assert!(!refused.status.success(), "{refusal}"); + assert!(refusal.contains("did:plc:limpet"), "{refusal}"); + assert!( + !target_after_refusal, + "a refused run mustn't leave a half-migrated target behind" + ); + let stdout = String::from_utf8_lossy(&outcome.stdout).to_string(); + let stderr = String::from_utf8_lossy(&outcome.stderr).to_string(); + assert!(!outcome.status.success(), "{stdout}{stderr}"); + assert!( + stderr.contains("the migration skipped 4 repos"), + "the unreadable repo opted out of the exit, so the four the operator never chose still reach it: {stderr}" + ); + let adopted = map_probing(&fx, |did| match did.as_str() { + "did:plc:limpet" => adopt::SourceProbe::Unreadable(std::io::ErrorKind::PermissionDenied), + _ => adopt::probe_source(&fx.source_repos, did), + }) + .repos + .len(); + assert!( + stdout.contains(&format!("adopted by copy: {adopted} new")), + "{stdout}" + ); + assert!( + stdout.contains("a source path that this process can't read: permission denied"), + "{stdout}" + ); + assert!(fx.target.join("config.toml").is_file(), "{stdout}"); +} + +#[test] +fn a_run_refuses_two_owners_for_one_repo_after_it_reports_them() { + let fx = fixture(true); + set_acl_owner(&fx, "did:plc:scallop", "did:plc:teq"); + let dir = tempfile::tempdir().unwrap(); + let host_key = host_key_file(dir.path()); + [vec![], vec!["--dry-run"], vec!["--skip-unreadable"]] + .into_iter() + .for_each(|extra| { + let outcome = run_migrate(&fx, &host_key, &extra); + let stdout = String::from_utf8_lossy(&outcome.stdout).to_string(); + let stderr = String::from_utf8_lossy(&outcome.stderr).to_string(); + assert!(!outcome.status.success(), "{extra:?}: {stdout}{stderr}"); + assert!( + stdout.contains("did:plc:scallop acl did:plc:teq, repo_keys did:plc:isabel"), + "the report has to be on stdout before the refusal, {extra:?}: {stdout}" + ); + assert!( + stderr.contains( + "the acl and repo_keys are recorded with different owners for did:plc:scallop" + ), + "{extra:?}: {stderr}" + ); + assert!( + stderr.contains("repo_keys.owner_did"), + "the refusal has to point at the column to fix, {extra:?}: {stderr}" + ); + assert!( + !fx.target.exists(), + "a refused run mustn't leave a half-migrated target behind, {extra:?}" + ); + }); +} + +#[test] +fn a_rehearsal_reports_an_unreadable_source_and_a_second_owner_together() { + use std::os::unix::fs::PermissionsExt; + let fx = fixture(true); + if !honors_permission_bits(&fx.source_repos) { + return; + } + set_acl_owner(&fx, "did:plc:scallop", "did:plc:teq"); + let dir = tempfile::tempdir().unwrap(); + let host_key = host_key_file(dir.path()); + let limpet = fx.source_repos.join("did:plc:limpet"); + std::fs::set_permissions(&limpet, std::fs::Permissions::from_mode(0o000)).unwrap(); + let outcome = run_migrate(&fx, &host_key, &["--dry-run"]); + std::fs::set_permissions(&limpet, std::fs::Permissions::from_mode(0o755)).unwrap(); + let stderr = String::from_utf8_lossy(&outcome.stderr).to_string(); + assert!(!outcome.status.success(), "{stderr}"); + assert!( + stderr.contains("different owners for did:plc:scallop"), + "{stderr}" + ); + assert!( + stderr.contains("can't read the source path of did:plc:limpet"), + "both refusals have to appear together: {stderr}" + ); +} + +#[test] +fn consuming_a_source_that_this_process_cannot_write_is_refused() { + use std::os::unix::fs::PermissionsExt; + let fx = fixture(true); + if !honors_permission_bits(&fx.source_repos) { + return; + } + let dir = tempfile::tempdir().unwrap(); + let host_key = host_key_file(dir.path()); + std::fs::set_permissions(&fx.source_repos, std::fs::Permissions::from_mode(0o555)).unwrap(); + let rehearsal = Rehearsal::run(rehearse::Inputs { + source_repos: &fx.source_repos, + adopted: &[], + scan_path: &fx.target.join("repos"), + policy: adopt::SourcePolicy::Consume, + host_key: Some(&host_key), + host_key_target: &fx.target.join("ssh_host_key"), + host_key_policy: emit::HostKeyPolicy::Keep, + master_key: &master_key_env(), + }); + let preserving = rehearse_scan_path( + &fx.source_repos, + &fx.target.join("repos"), + adopt::SourcePolicy::Preserve, + ); + std::fs::set_permissions(&fx.source_repos, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!( + matches!( + rehearsal.transfer, + Err(adopt::AdoptError::UnwritableSource { .. }) + ), + "a rename will move every repo out of the source, so the source has to be writable: {:?}", + rehearsal.transfer + ); + assert!(!rehearsal.ready()); + assert_eq!( + preserving.transfer.unwrap(), + adopt::Transfer::Copy, + "a copy will read the source and write elsewhere, so it doesn't need write permission there" + ); +} diff --git a/knot2/crates/knot-pack/Cargo.toml b/knot2/crates/knot-pack/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..d249e4234f69d274446adff71138f31833a7b424 --- /dev/null +++ b/knot2/crates/knot-pack/Cargo.toml @@ -0,0 +1,57 @@ +[package] +name = "knot-pack" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[features] +instrument = ["knot-git/instrument"] + +[dependencies] +knot-types = { workspace = true } +knot-storage = { workspace = true } +knot-git = { workspace = true } +knot-index = { workspace = true } +knot-runtime = { workspace = true } +knot-resource = { workspace = true } +knot-cache = { workspace = true } +knot-edge = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-messages = { workspace = true } +url = { workspace = true } +gix = { workspace = true } +gix-pack = { workspace = true } +gix-hash = { workspace = true } +scc = { workspace = true } +gix-packetline = { workspace = true } +flate2 = { workspace = true } +axum = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +thiserror = { workspace = true } +walkdir = { workspace = true } +tempfile = { workspace = true } + +[dev-dependencies] +knot-fixtures = { workspace = true } +tempfile = { workspace = true } +flate2 = { workspace = true } +gix-hash = { workspace = true } +tracing-subscriber = { workspace = true } +tikv-jemallocator = { workspace = true } +knot-resource = { workspace = true } +proptest = { workspace = true } +tower = { version = "0.5", features = ["util"] } +http-body-util = "0.1" +knot-bench = { workspace = true } +knot-maintenance = { workspace = true } +quinn = { workspace = true } +h3 = { workspace = true } +h3-quinn = { workspace = true } +rustls = { workspace = true } +rcgen = { workspace = true } +tokio-util = { workspace = true } +http = { workspace = true } +bytes = { workspace = true } diff --git a/knot2/crates/knot-pack/examples/ingest_mem.rs b/knot2/crates/knot-pack/examples/ingest_mem.rs new file mode 100644 index 0000000000000000000000000000000000000000..0fd0e53a2f1cda25704510108301f71e5cbfa2e7 --- /dev/null +++ b/knot2/crates/knot-pack/examples/ingest_mem.rs @@ -0,0 +1,86 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Instant; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +const PAGE: u64 = 4096; + +fn rss_bytes() -> u64 { + let statm = std::fs::read_to_string("/proc/self/statm").unwrap(); + statm + .split_whitespace() + .nth(1) + .and_then(|pages| pages.parse::().ok()) + .map(|pages| pages * PAGE) + .unwrap() +} + +fn vm_hwm_bytes() -> u64 { + let status = std::fs::read_to_string("/proc/self/status").unwrap(); + status + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|kb| kb.parse::().ok()) + .map(|kb| kb * 1024) + .unwrap() +} + +fn mib(bytes: u64) -> u64 { + bytes / (1024 * 1024) +} + +fn main() { + let pack_path = std::path::PathBuf::from( + std::env::args() + .nth(1) + .expect("usage: ingest_mem "), + ); + + let pack_size = std::fs::metadata(&pack_path).map(|m| m.len()).unwrap_or(0); + println!( + "pack: {} MiB on disk, streamed not resident", + mib(pack_size) + ); + println!( + "governor: {} threads, {}", + knot_resource::threads().get(), + knot_resource::available_bytes() + .map(|bytes| format!("{} MiB available in cgroup", mib(bytes.get()))) + .unwrap_or_else(|| "unconstrained".to_string()) + ); + + let dir = tempfile::tempdir().expect("tempdir"); + let objects_dir = dir.path().to_path_buf(); + + let peak = Arc::new(AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let sampler = { + let peak = Arc::clone(&peak); + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + let now = rss_bytes(); + peak.fetch_max(now, Ordering::Relaxed); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + }) + }; + + let before = rss_bytes(); + let start = Instant::now(); + let folded = knot_pack::bench_ingest_fresh(&objects_dir, &pack_path, gix::hash::Kind::Sha1) + .expect("ingest"); + let elapsed = start.elapsed(); + stop.store(true, Ordering::Relaxed); + sampler.join().ok(); + + let sampled_peak = peak.load(Ordering::Relaxed); + println!("fold engaged: {folded}"); + println!("took: {:.1}s", elapsed.as_secs_f64()); + println!("rss before ingest: {} MiB", mib(before)); + println!("rss sampled peak: {} MiB", mib(sampled_peak)); + println!("VmHWM, kernel peak: {} MiB", mib(vm_hwm_bytes())); +} diff --git a/knot2/crates/knot-pack/examples/knot_http_backend.rs b/knot2/crates/knot-pack/examples/knot_http_backend.rs new file mode 100644 index 0000000000000000000000000000000000000000..6f0197cb0eb65dd9e48651b2b01b7d56dac34d13 --- /dev/null +++ b/knot2/crates/knot-pack/examples/knot_http_backend.rs @@ -0,0 +1,205 @@ +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use knot_git::Repo; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +const SUFFIXES: [&str; 3] = ["/info/refs", "/git-upload-pack", "/git-receive-pack"]; + +fn env(key: &str) -> Option { + std::env::var(key).ok() +} + +fn resolve() -> Option<(PathBuf, &'static str)> { + let path_info = env("PATH_INFO")?; + let (repo_sub, suffix) = SUFFIXES + .iter() + .find_map(|suffix| path_info.strip_suffix(suffix).map(|rest| (rest, *suffix)))?; + let root = env("GIT_PROJECT_ROOT") + .filter(|root| !root.is_empty()) + .or_else(|| { + let translated = env("PATH_TRANSLATED")?; + translated + .strip_suffix(path_info.as_str()) + .map(str::to_string) + })?; + let repo_sub = repo_sub.trim_start_matches('/'); + Some((Path::new(&root).join(repo_sub), suffix)) +} + +fn wants_v2() -> bool { + env("HTTP_GIT_PROTOCOL") + .or_else(|| env("GIT_PROTOCOL")) + .is_some_and(|value| value.split(':').any(|token| token.trim() == "version=2")) +} + +fn dev_push_allowed() -> bool { + env("KNOT_DEV_ALLOW_HTTP_PUSH").is_some_and(|value| { + let value = value.trim(); + value == "1" || value.eq_ignore_ascii_case("true") + }) +} + +fn dev_pack_limits() -> knot_pack::PackLimits { + let mut limits = knot_pack::PackLimits::default(); + if let Some(value) = env("KNOT_DEV_MAX_OBJECTS").and_then(|value| value.trim().parse().ok()) { + limits.max_objects = knot_types::ObjectCount::new(value); + } + if let Some(value) = env("KNOT_DEV_MAX_TOTAL_BYTES").and_then(|value| value.trim().parse().ok()) + { + limits.max_total_bytes = knot_pack::MaxTotalBytes::new(value); + } + limits +} + +fn apply_dev_selection_limits() { + let max_objects = env("KNOT_DEV_SELECTION_MAX_OBJECTS") + .and_then(|value| value.trim().parse().ok()) + .map(knot_types::ObjectCount::new); + let secs = + env("KNOT_DEV_SELECTION_TIME_BUDGET_SECS").and_then(|value| value.trim().parse().ok()); + if max_objects.is_none() && secs.is_none() { + return; + } + let base = knot_pack::SelectionLimits::default(); + knot_pack::init_selection_limits(knot_pack::SelectionLimits { + max_objects: max_objects.unwrap_or(base.max_objects), + time_budget: secs + .map(std::time::Duration::from_secs) + .unwrap_or(base.time_budget), + }); +} + +fn apply_dev_resources() { + let max_threads = env("KNOT_DEV_MAX_THREADS") + .and_then(|value| value.trim().parse::().ok()) + .filter(|threads| *threads != 0) + .map(knot_resource::ThreadCount::new); + knot_resource::init(knot_resource::Ceilings { + max_threads, + ..knot_resource::Ceilings::default() + }); +} + +fn read_body() -> Vec { + let mut raw = Vec::new(); + std::io::stdin().read_to_end(&mut raw).ok(); + let gzipped = env("HTTP_CONTENT_ENCODING").is_some_and(|value| { + value + .split(',') + .any(|token| token.trim().eq_ignore_ascii_case("gzip")) + }); + if !gzipped { + return raw; + } + let mut decoded = Vec::new(); + flate2::read::GzDecoder::new(raw.as_slice()) + .read_to_end(&mut decoded) + .ok(); + decoded +} + +fn emit(out: &mut dyn Write, content_type: &str, body: &[u8]) { + let _ = write!( + out, + "Expires: Fri, 01 Jan 1980 00:00:00 GMT\r\nPragma: no-cache\r\nCache-Control: no-cache, max-age=0, must-revalidate\r\nContent-Type: {content_type}\r\n\r\n" + ); + let _ = out.write_all(body); +} + +fn fail(out: &mut dyn Write, status: &str, message: &str) { + let _ = write!( + out, + "Status: {status}\r\nContent-Type: text/plain\r\n\r\n{message}\n" + ); +} + +const PUSH_REFUSED: &str = "knot accepts pushes over SSH, not HTTP"; + +fn serve_receive_advert(out: &mut dyn Write, repo: &Repo) { + if !dev_push_allowed() { + return fail(out, "403 Forbidden", PUSH_REFUSED); + } + match knot_pack::advertise_receive(repo) { + Ok(body) => emit(out, "application/x-git-receive-pack-advertisement", &body), + Err(error) => fail(out, "500 Internal Server Error", &error.to_string()), + } +} + +fn serve_receive_pack(out: &mut dyn Write, repo: &Repo) { + if !dev_push_allowed() { + return fail(out, "403 Forbidden", PUSH_REFUSED); + } + match knot_pack::receive_pack_with_limits(repo, &read_body(), &dev_pack_limits()) { + Ok(result) => emit(out, "application/x-git-receive-pack-result", &result), + Err(error) => fail(out, "500 Internal Server Error", &error.to_string()), + } +} + +fn main() { + apply_dev_selection_limits(); + apply_dev_resources(); + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + + let (repo_dir, suffix) = match resolve() { + Some(parts) => parts, + None => { + return fail( + &mut out, + "400 Bad Request", + "unrecognized git smart-http path", + ); + } + }; + let repo = match Repo::open(&repo_dir) { + Ok(repo) => repo, + Err(_) => return fail(&mut out, "404 Not Found", "no such repository"), + }; + let method = env("REQUEST_METHOD").unwrap_or_default(); + + match (method.as_str(), suffix) { + ("GET", "/info/refs") => { + let service = env("QUERY_STRING") + .and_then(|query| { + query + .split('&') + .find_map(|pair| pair.strip_prefix("service=").map(str::to_string)) + }) + .unwrap_or_default(); + match service.as_str() { + "git-upload-pack" => { + let body = if wants_v2() { + knot_pack::advertise_upload(&repo) + } else { + knot_pack::advertise_upload_v0(&repo) + }; + match body { + Ok(body) => emit( + &mut out, + "application/x-git-upload-pack-advertisement", + &body, + ), + Err(error) => { + fail(&mut out, "500 Internal Server Error", &error.to_string()) + } + } + } + "git-receive-pack" => serve_receive_advert(&mut out, &repo), + _ => fail(&mut out, "403 Forbidden", "unsupported service"), + } + } + ("POST", "/git-upload-pack") => match knot_pack::upload_pack(&repo, &read_body()) { + Ok(result) => emit(&mut out, "application/x-git-upload-pack-result", &result), + Err(error) => fail(&mut out, "500 Internal Server Error", &error.to_string()), + }, + ("POST", "/git-receive-pack") => serve_receive_pack(&mut out, &repo), + _ => fail( + &mut out, + "400 Bad Request", + "unsupported git smart-http request", + ), + } +} diff --git a/knot2/crates/knot-pack/examples/receive_mem.rs b/knot2/crates/knot-pack/examples/receive_mem.rs new file mode 100644 index 0000000000000000000000000000000000000000..c2f22d5c89bd621c29b5a90a2f2efdccc3a53692 --- /dev/null +++ b/knot2/crates/knot-pack/examples/receive_mem.rs @@ -0,0 +1,129 @@ +use std::io::Read; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Instant; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +const PAGE: u64 = 4096; + +fn rss_bytes() -> u64 { + let statm = std::fs::read_to_string("/proc/self/statm").unwrap(); + statm + .split_whitespace() + .nth(1) + .and_then(|pages| pages.parse::().ok()) + .map(|pages| pages * PAGE) + .unwrap() +} + +fn vm_hwm_bytes() -> u64 { + let status = std::fs::read_to_string("/proc/self/status").unwrap(); + status + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|kb| kb.parse::().ok()) + .map(|kb| kb * 1024) + .unwrap() +} + +fn mib(bytes: u64) -> u64 { + bytes / (1024 * 1024) +} + +fn pktline(data: &[u8]) -> Vec { + let mut out = format!("{:04x}", 4 + data.len()).into_bytes(); + out.extend_from_slice(data); + out +} + +fn main() { + let max_threads = std::env::var("KNOT_MAX_THREADS") + .ok() + .and_then(|value| value.parse::().ok()); + knot_resource::init(knot_resource::Ceilings { + max_threads: max_threads.map(knot_resource::ThreadCount::new), + max_memory: None, + }); + + let pack_path = std::path::PathBuf::from( + std::env::args() + .nth(1) + .expect("usage: receive_mem "), + ); + let pack_size = std::fs::metadata(&pack_path).map(|m| m.len()).unwrap_or(0); + println!("pack: {} MiB on disk", mib(pack_size)); + + let mut preamble = pktline( + b"0000000000000000000000000000000000000000 \ + 1111111111111111111111111111111111111111 refs/heads/main\0report-status\n", + ); + preamble.extend_from_slice(b"0000"); + + let dir = tempfile::tempdir().expect("tempdir"); + let limit = knot_pack::MaxWireBytes::new(16 * 1024 * 1024 * 1024); + let mut receiver = knot_pack::PackReceiver::new( + dir.path(), + limit, + knot_pack::PackLimits::default(), + gix::hash::Kind::Sha1, + ) + .expect("receiver"); + + let peak = Arc::new(AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let sampler = { + let peak = Arc::clone(&peak); + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + peak.fetch_max(rss_bytes(), Ordering::Relaxed); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + }) + }; + + let start = Instant::now(); + receiver.write(&preamble).expect("preamble"); + let mut file = std::fs::File::open(&pack_path).expect("open pack"); + let mut buf = vec![0u8; 1 << 20]; + loop { + let read = file.read(&mut buf).expect("read pack"); + if read == 0 { + break; + } + receiver.write(&buf[..read]).expect("receive"); + } + let received = receiver.finish().expect("finish"); + let receipt_elapsed = start.elapsed(); + let receipt_peak = peak.load(Ordering::Relaxed); + println!( + "receipt: {:.1}s, peak {} MiB (pack streamed to a temp file, never resident)", + receipt_elapsed.as_secs_f64(), + mib(receipt_peak) + ); + + let staged = received.open_pack().expect("open pack").expect("has pack"); + let objects = tempfile::tempdir().expect("objects tempdir"); + let result = + knot_pack::bench_admit_and_ingest(objects.path(), staged.path(), gix::hash::Kind::Sha1); + let total_elapsed = start.elapsed(); + let total_peak = peak.load(Ordering::Relaxed); + stop.store(true, Ordering::Relaxed); + sampler.join().ok(); + + match result { + Ok(folded) => { + println!("ADMITTED, fold engaged: {folded}"); + println!("receive + ingest: {:.1}s", total_elapsed.as_secs_f64()); + println!("receive + ingest peak: {} MiB", mib(total_peak)); + println!("VmHWM,kernel peak: {} MiB", mib(vm_hwm_bytes())); + } + Err(error) => { + println!("REFUSED cleanly before the fold: {error:?}"); + println!("peak at refusal: {} MiB", mib(total_peak)); + } + } +} diff --git a/knot2/crates/knot-pack/fuzz/.gitignore b/knot2/crates/knot-pack/fuzz/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1a45eee7760d240bfa8ac1989088e0349169f264 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-pack/fuzz/Cargo.lock b/knot2/crates/knot-pack/fuzz/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..8a7c9dc767f1d28460ab840f09497807d9cc436d --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/Cargo.lock @@ -0,0 +1,6275 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive 0.5.1", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive 0.6.0", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-http-codec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "096146020b08dbc4587685b0730a7ba905625af13c65f8028035cdfd69573c91" +dependencies = [ + "anyhow", + "futures", + "http", + "httparse", + "log", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-net" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" +dependencies = [ + "async-io", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-web-client" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8caf502b44d6d4be6154ac33af012cbb5fef11e6066edcfb42834217fbaf501b" +dependencies = [ + "async-http-codec", + "async-net", + "futures", + "futures-rustls", + "http", + "lazy_static", + "log", + "rustls-pki-types", + "serde", + "thiserror 1.0.69", + "webpki-roots 0.26.11", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "bon" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] + +[[package]] +name = "bytesize" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "clru" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs 0.6.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs 0.7.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "forwarded-header-value" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" +dependencies = [ + "nonempty 0.7.0", + "thiserror 1.0.69", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls", + "rustls-pki-types", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "gix" +version = "0.84.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae54ae0ebd1a5a3c3f8d95dd3b5ca6e63f4fed9bfd585e13801a97d7bde8f9ce" +dependencies = [ + "gix-actor", + "gix-archive", + "gix-attributes", + "gix-blame", + "gix-command", + "gix-commitgraph", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-ignore", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-negotiate", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-pathspec", + "gix-prompt", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-shallow", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "gix-worktree", + "gix-worktree-state", + "gix-worktree-stream", + "nonempty 0.12.0", + "parking_lot", + "regex", + "signal-hook", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-actor" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc998b8f746dda8565450d08a63b792ced9165d8c27a1ed3f02799ec6a7820f" +dependencies = [ + "bstr", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-archive" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16909cacc78936ab96f6c3be08379d0a2e88bfa3a7527972d2ed75c7517ef31e" +dependencies = [ + "bstr", + "flate2", + "gix-date", + "gix-error", + "gix-object", + "gix-path", + "gix-worktree-stream", + "rawzip", + "tar", +] + +[[package]] +name = "gix-attributes" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d43f12e246d3bf7ec624c8fc15ac4a4b62b7c4c6f586cb82be6c90bf84c9d02" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-bitmap" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ebef0c26ad305747649e727bbcd56a7b7910754eb7cea88f6dff6f93c51283" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-blame" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d39a0c14af94c2edaa5eefe06d5ef2cdea55316ae9a9321314288e3f55fa4c0" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-diff", + "gix-error", + "gix-hash", + "gix-object", + "gix-revwalk", + "gix-trace", + "gix-traverse", + "gix-worktree", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-chunk" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9faee47943b638e58ddd5e275a4906ad3e4b6c8584f1d41bd18ab9032ec52afb" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-command" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00706d4fef135ef4b01680d5218c6ee40cda8baf697b864296cbc887d19118f6" +dependencies = [ + "bstr", + "gix-path", + "gix-quote", + "gix-trace", + "shell-words", +] + +[[package]] +name = "gix-commitgraph" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" +dependencies = [ + "bstr", + "gix-chunk", + "gix-error", + "gix-hash", + "memmap2", + "nonempty 0.12.0", +] + +[[package]] +name = "gix-config" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2372d4b49ca28431e7d150cab9d25edc1890f0184bd57eb0e917c7799e63de" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-config-value" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" +dependencies = [ + "bitflags", + "bstr", + "gix-path", + "libc", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-credentials" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40cd22f0dd71988be12d6e78b1709de2370e1957c5f107ff31e56caeba3745d" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-date", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-trace", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-date" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ecab64a98bbac9f8e02990a9ea5e3c974a7d49b95f2bd70ad94ad22fa6b48c" +dependencies = [ + "bstr", + "gix-error", + "itoa", + "jiff", +] + +[[package]] +name = "gix-diff" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6d9528f32d94cef2edf39a1ac01fe5a0fc44ddbb18d9e44099936047c3302b" +dependencies = [ + "bstr", + "gix-attributes", + "gix-command", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-imara-diff", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-dir" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bb2a53a6fd917ec499ed0bfb5b6887de7a15bd79197dcea7c987938749a9f1" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-discover" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77bacdd12b7879d2178a80c58c2f319995e4654e1a7a23e3181e5c8a12b824f7" +dependencies = [ + "bstr", + "dunce", + "gix-fs", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-error" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57831e199be480af90dcd7e459abed8a174c09ec9a6e2cc8f7ca6c54598b06b" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "bytes", + "bytesize", + "crc32fast", + "crossbeam-channel", + "gix-path", + "gix-trace", + "gix-utils", + "libc", + "once_cell", + "parking_lot", + "prodash", + "thiserror 2.0.18", + "walkdir", + "zlib-rs", +] + +[[package]] +name = "gix-filter" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf74b7d16f6694ce4a3049074c41be0c7987105743674f1671807bd6dce09fa" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash", + "gix-object", + "gix-packetline", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-fs" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" +dependencies = [ + "bstr", + "fastrand", + "gix-features", + "gix-path", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-glob" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" +dependencies = [ + "bitflags", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-hashtable" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0e30b93eea8718baf7d8153fcb938e2926175bbf18097c09f1c01b6f0be0563" +dependencies = [ + "gix-hash", + "hashbrown 0.17.1", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-imara-diff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19753d40da53d0ec41604750eeb969097a90fb2d7f7992730d904541c04e2c19" +dependencies = [ + "bstr", + "hashbrown 0.17.1", +] + +[[package]] +name = "gix-index" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6b28cc592dc753adb58302bb14a64e412ee591a3bec77aa4df87bff74fa80d" +dependencies = [ + "bitflags", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils", + "gix-validate", + "hashbrown 0.17.1", + "itoa", + "libc", + "memmap2", + "rustix", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-lock" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-mailmap" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195fd20808055824531be2fd0d34136d900e5fbca3ffb0a3c07e8beeefb9c828" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-negotiate" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890c936a215bae25818c076cb881cb2e54d2c66ba947ba58b8dd47cff921bf55" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-object", + "gix-revwalk", +] + +[[package]] +name = "gix-object" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5cd857e29429c7213bdef3f5aef83f8cc124774fe8ae0d27b1607d218d6d525" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-odb" +version = "0.81.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d004c32858b1556f2d7874405edb3c97dc78fc09beaa87d57bb077ee2858a7d" +dependencies = [ + "arc-swap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "memmap2", + "parking_lot", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pack" +version = "0.71.0" +dependencies = [ + "clru", + "gix-chunk", + "gix-diff", + "gix-error", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-traverse", + "parking_lot", + "smallvec", + "thiserror 2.0.18", + "uluru", +] + +[[package]] +name = "gix-packetline" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb18337ba2830bb43367d1af43819c8c78f31337f079fc76d0f1f1750a173126" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-path" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa6ac14cd14939ea94a496ce7460daa6511c09f5b84757e9cfc6f9c8d0f93a6" +dependencies = [ + "bstr", + "gix-trace", + "gix-validate", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pathspec" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" +dependencies = [ + "bitflags", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-prompt" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee604d7746080ae7e1023bf47204bcc2c5f307bfbe2306a3c90b1bfd1a2c6d8" +dependencies = [ + "gix-command", + "gix-config-value", + "parking_lot", + "rustix", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-protocol" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51dea3acb390707ab868f1f9584f18449eb95d869deffae96768e47d303595ee" +dependencies = [ + "bstr", + "gix-date", + "gix-features", + "gix-hash", + "gix-ref", + "gix-shallow", + "gix-transport", + "gix-utils", + "maybe-async", + "nonempty 0.12.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-quote" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" +dependencies = [ + "bstr", + "gix-error", + "gix-utils", +] + +[[package]] +name = "gix-ref" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c04f64c37eb7e6feb73c7060f8dc6f381cc5de5d53249bfd450bc48a86b2e8b" +dependencies = [ + "gix-actor", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils", + "gix-validate", + "memmap2", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-refspec" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b216ae06ec74b5f24ad0142026a997fb0a935b7410eaf9c1616fc3f0e6c5a6d3" +dependencies = [ + "bstr", + "gix-error", + "gix-glob", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-revision" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b47c88884dd3c1a19a39da19d10211fcdea2809aadc86869b6e824a1774340f" +dependencies = [ + "bitflags", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "nonempty 0.12.0", +] + +[[package]] +name = "gix-revwalk" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f5756abffe0917827aac683b13684ed99875bc398fa1f9b8f479b0681ef9e6" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-sec" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8519976e4c7e486270740a5400369f37940779b80bd1377d94cfa1125d01b3" +dependencies = [ + "bitflags", + "gix-path", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "gix-shallow" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" +dependencies = [ + "bstr", + "gix-hash", + "gix-lock", + "nonempty 0.12.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-status" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22042e385d28a34275e029d98f4970285045be14b9073658ca897923f2ed8700" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-submodule" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3059890ef054066c22a94bfc6a3eaba0d806aedcd630a0bc9e5783fd88884781" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-tempfile" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27850097e1ff9515f46a0dad0f5f9c9d020e972727772dabab9450690c4adb22" +dependencies = [ + "dashmap", + "gix-fs", + "libc", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "gix-transport" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd0e34995b1aab0fa8dff2af8db726a0bfad3e119c89302604463264046e7ff" +dependencies = [ + "bstr", + "gix-command", + "gix-features", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-traverse" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8de590ecc86a3b2870665f2288324fa9f7f8672c7fc2d4e020fdd81cd1f7aed" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-url" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bb01ec69d55e82ccb7a19e264501ead4e6aac38463a8cebfdd81e22bb67ab2" +dependencies = [ + "bstr", + "gix-path", + "percent-encoding", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-utils" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c50966184123caf580ffa64e28031a878597f1c7fceb8fe19566c38eb1b771" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc6fc771c4063ba7cd2f47b91fb6076251c6a823b64b7fe7b8874b0fe4afae3" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-worktree" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef414ed275e8407cd5d53d301e83be19700b0dd3f859d2434417b58f454a2d1" +dependencies = [ + "bstr", + "gix-attributes", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate", +] + +[[package]] +name = "gix-worktree-state" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bffae8b3ca258fdd50370cd51f06deb4c76a3b43db3868bc28dde45ffa77d69" +dependencies = [ + "bstr", + "gix-features", + "gix-filter", + "gix-fs", + "gix-index", + "gix-object", + "gix-path", + "gix-worktree", + "io-close", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-worktree-stream" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25e9ed30100c63f7590bc581c225e53f731a53e06aa79a245739c07f7dcc557" +dependencies = [ + "gix-attributes", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-traverse", + "parking_lot", +] + +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "dashmap", + "futures-sink", + "futures-timer", + "futures-util", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand 0.9.4", + "smallvec", + "spinning_top", + "web-time", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h3" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10872b55cfb02a821b69dc7cf8dc6a71d6af25eb9a79662bec4a9d016056b3be" +dependencies = [ + "bytes", + "fastrand", + "futures-util", + "http", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "h3-quinn" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2e732c8d91a74731663ac8479ab505042fbf547b9a207213ab7fbcbfc4f8b4" +dependencies = [ + "bytes", + "futures", + "h3", + "quinn", + "tokio", + "tokio-util", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "human_format" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaec953f16e5bcf6b8a3cb3aa959b17e5577dbd2693e94554c462c08be22624b" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.7", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest 0.12.28", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jiff" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-static" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-edge" +version = "0.1.0" +dependencies = [ + "arc-swap", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "governor", + "h3", + "h3-quinn", + "http", + "http-body", + "hyper", + "hyper-util", + "knot-types", + "quinn", + "rustls", + "rustls-acme", + "rustls-pemfile", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.7.0", + "tower_governor", + "tracing", + "x509-parser 0.18.1", +] + +[[package]] +name = "knot-git" +version = "0.1.0" +dependencies = [ + "base64", + "dashmap", + "flate2", + "gix", + "gix-archive", + "gix-bitmap", + "gix-hash", + "gix-pack", + "knot-resource", + "knot-types", + "moka", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "knot-pack" +version = "0.1.0" +dependencies = [ + "axum", + "dashmap", + "flate2", + "gix", + "gix-hash", + "gix-pack", + "gix-packetline", + "knot-edge", + "knot-git", + "knot-resource", + "knot-runtime", + "knot-types", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "url", + "walkdir", +] + +[[package]] +name = "knot-pack-fuzz" +version = "0.0.0" +dependencies = [ + "knot-pack", + "libfuzzer-sys", +] + +[[package]] +name = "knot-resource" +version = "0.1.0" +dependencies = [ + "rustix", +] + +[[package]] +name = "knot-runtime" +version = "0.1.0" +dependencies = [ + "bytes", + "futures", + "getrandom 0.4.2", + "http", + "k256", + "reqwest 0.13.1", + "thiserror 2.0.18", + "tokio", + "url", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multibase" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" +dependencies = [ + "base-x", + "base256emoji", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nonempty" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" + +[[package]] +name = "nonempty" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs 0.6.2", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs 0.7.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "bytesize", + "human_format", + "parking_lot", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "futures-io", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.7", +] + +[[package]] +name = "reqwest" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.7", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-acme" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c70a17ecb067d5067565a16a2e0f26a4a2ea0924f49739d558c45186facc75" +dependencies = [ + "async-io", + "async-trait", + "async-web-client", + "aws-lc-rs", + "base64", + "blocking", + "chrono", + "futures", + "futures-rustls", + "http", + "log", + "pem", + "rcgen", + "serde", + "serde_json", + "thiserror 2.0.18", + "webpki-roots 1.0.7", + "x509-parser 0.16.0", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tower_governor" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44de9b94d849d3c46e06a883d72d408c2de6403367b39df2b1c9d9e7b6736fe6" +dependencies = [ + "axum", + "forwarded-header-value", + "governor", + "http", + "pin-project", + "thiserror 2.0.18", + "tower", + "tracing", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uluru" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8a2469e56e6e5095c82ccd3afb98dad95f7af7929aab6d8ba8d6e0f73657da" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.7", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs 0.6.2", + "data-encoding", + "der-parser 9.0.0", + "lazy_static", + "nom", + "oid-registry 0.7.1", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs 0.7.2", + "data-encoding", + "der-parser 10.0.0", + "lazy_static", + "nom", + "oid-registry 0.8.1", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/knot2/crates/knot-pack/fuzz/Cargo.toml b/knot2/crates/knot-pack/fuzz/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..f96b85540c8db2d78dcd7ce55eebde543e9fbfa7 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "knot-pack-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-pack] +path = ".." + +[[bin]] +name = "pkt" +path = "fuzz_targets/pkt.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "pack" +path = "fuzz_targets/pack.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "receive_commands" +path = "fuzz_targets/receive_commands.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "upload_args" +path = "fuzz_targets/upload_args.rs" +test = false +doc = false +bench = false + +[patch.crates-io] +gix-pack = { path = "../../../third_party/gix-pack" } diff --git a/knot2/crates/knot-pack/fuzz/fuzz_targets/pack.rs b/knot2/crates/knot-pack/fuzz/fuzz_targets/pack.rs new file mode 100644 index 0000000000000000000000000000000000000000..f13e5214b7b47482f25aec0763d16cc3d14937b5 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/fuzz_targets/pack.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_pack::fuzz::pack(data); +}); diff --git a/knot2/crates/knot-pack/fuzz/fuzz_targets/pkt.rs b/knot2/crates/knot-pack/fuzz/fuzz_targets/pkt.rs new file mode 100644 index 0000000000000000000000000000000000000000..6d58e00ab460812e80bf580cad46d37db0605ae1 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/fuzz_targets/pkt.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_pack::fuzz::pkt(data); +}); diff --git a/knot2/crates/knot-pack/fuzz/fuzz_targets/receive_commands.rs b/knot2/crates/knot-pack/fuzz/fuzz_targets/receive_commands.rs new file mode 100644 index 0000000000000000000000000000000000000000..719af6d09328c5f819db1a2921adc1fd55c5cb13 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/fuzz_targets/receive_commands.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_pack::fuzz::receive_commands(data); +}); diff --git a/knot2/crates/knot-pack/fuzz/fuzz_targets/upload_args.rs b/knot2/crates/knot-pack/fuzz/fuzz_targets/upload_args.rs new file mode 100644 index 0000000000000000000000000000000000000000..2b5e4d3f75879c27976d3bddba619950759e7cf0 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/fuzz_targets/upload_args.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_pack::fuzz::upload_args(data); +}); diff --git a/knot2/crates/knot-pack/src/archive.rs b/knot2/crates/knot-pack/src/archive.rs new file mode 100644 index 0000000000000000000000000000000000000000..8a2971026e92b89867dcc67c969749ff5fadeb3a --- /dev/null +++ b/knot2/crates/knot-pack/src/archive.rs @@ -0,0 +1,149 @@ +use std::io::{self, Read, Seek, SeekFrom}; + +use knot_git::{ArchiveFormat, ArchiveLimit, Repo, TreePrefix}; +use knot_types::Oid; + +use crate::error::PackError; +use crate::pkt; + +struct Request { + treeish: String, + format: ArchiveFormat, + prefix: Option, +} + +pub fn stream( + repo: &Repo, + request: &[u8], + limit: ArchiveLimit, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let args = parse_arguments(request)?; + match build(repo, &args, limit) { + Ok(mut spool) => { + let mut head = Vec::new(); + pkt::write_data(&mut head, b"ACK\n")?; + pkt::write_flush(&mut head)?; + emit(sink, &head)?; + std::iter::from_fn(|| { + let mut chunk = vec![0u8; pkt::MAX_BAND]; + match spool.read(&mut chunk) { + Ok(0) => None, + Ok(read) => { + chunk.truncate(read); + Some(Ok(chunk)) + } + Err(error) => Some(Err(error)), + } + }) + .try_for_each(|chunk| -> Result<(), PackError> { + let chunk = chunk.map_err(|error| PackError::Pack(error.to_string()))?; + let mut framed = Vec::new(); + pkt::write_band(&mut framed, &chunk)?; + emit(sink, &framed) + })?; + let mut tail = Vec::new(); + pkt::write_flush(&mut tail)?; + emit(sink, &tail) + } + Err(error) => { + let mut buf = Vec::new(); + pkt::write_data( + &mut buf, + format!("NACK {}\n", error.to_string().replace('\n', " ")).as_bytes(), + )?; + pkt::write_flush(&mut buf)?; + emit(sink, &buf) + } + } +} + +fn emit(sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, bytes: &[u8]) -> Result<(), PackError> { + sink(bytes).map_err(|error| PackError::Pack(error.to_string())) +} + +fn parse_arguments(request: &[u8]) -> Result, PackError> { + Ok(pkt::data_payloads(request)? + .iter() + .filter_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end_matches('\n') + .strip_prefix("argument ") + .map(str::to_string) + }) + .collect()) +} + +fn interpret(args: &[String]) -> Result { + let format = args + .iter() + .find_map(|arg| arg.strip_prefix("--format=")) + .map(format_from) + .unwrap_or(ArchiveFormat::Tar); + let prefix = args + .iter() + .find_map(|arg| arg.strip_prefix("--prefix=")) + .map(|raw| { + TreePrefix::new(raw).map_err(|_| { + PackError::Protocol(format!( + "archive prefix mustn't escape archive root or exceed {} bytes", + TreePrefix::MAX_BYTES + )) + }) + }) + .transpose()?; + let treeish = args + .iter() + .find(|arg| !arg.starts_with('-')) + .cloned() + .ok_or_else(|| PackError::Protocol("archive request has no tree-ish".to_string()))?; + Ok(Request { + treeish, + format, + prefix, + }) +} + +fn format_from(value: &str) -> ArchiveFormat { + match value { + "zip" => ArchiveFormat::Zip, + "tar.gz" | "tgz" => ArchiveFormat::TarGz, + _ => ArchiveFormat::Tar, + } +} + +fn build(repo: &Repo, args: &[String], limit: ArchiveLimit) -> Result { + let request = interpret(args)?; + let id = repo + .resolve_revision(&request.treeish) + .ok_or_else(|| PackError::Protocol(format!("cannot resolve {}", request.treeish)))?; + let commit = ensure_public_commit(repo, id)?; + let tree = repo + .peel_to_tree(commit) + .map_err(|error| PackError::Pack(error.to_string()))?; + let mut spool = tempfile::tempfile().map_err(|error| PackError::Pack(error.to_string()))?; + repo.write_archive( + tree, + request.format, + request.prefix.as_ref(), + limit, + &mut spool, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + spool + .seek(SeekFrom::Start(0)) + .map_err(|error| PackError::Pack(error.to_string()))?; + Ok(spool) +} + +fn ensure_public_commit(repo: &Repo, id: Oid) -> Result { + let unreachable = + || PackError::Protocol("tree-ish is not reachable from public ref".to_string()); + let commit = repo.peel_to_commit(id).map_err(|_| unreachable())?; + match repo.reachable_from_public(commit) { + Ok(true) => Ok(commit), + Ok(false) => Err(unreachable()), + Err(error) => Err(PackError::Pack(error.to_string())), + } +} diff --git a/knot2/crates/knot-pack/src/cache.rs b/knot2/crates/knot-pack/src/cache.rs new file mode 100644 index 0000000000000000000000000000000000000000..bae23d6cb86d259fc4dc0ed049f1b6b39faa0134 --- /dev/null +++ b/knot2/crates/knot-pack/src/cache.rs @@ -0,0 +1,512 @@ +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::body::Bytes; +use knot_cache::{Cache, EntryCount, Lru, Reclaimable, Weight}; +use knot_runtime::Clock; +use tokio::sync::watch; + +const MAX_ENTRIES: usize = 1024; + +knot_types::scalar_newtype! { + pub struct MaxEntryBytes(usize); + pub struct MaxCacheBytes(usize); +} + +#[derive(Debug, Clone, Copy)] +pub struct CacheConfig { + pub enabled: bool, + pub ttl: Duration, + pub max_entry_bytes: MaxEntryBytes, + pub max_total_bytes: MaxCacheBytes, +} + +impl Default for CacheConfig { + fn default() -> Self { + Self { + enabled: true, + ttl: Duration::from_secs(60), + max_entry_bytes: MaxEntryBytes::new(64 * 1024 * 1024), + max_total_bytes: MaxCacheBytes::new(512 * 1024 * 1024), + } + } +} + +#[derive(Clone, PartialEq, Eq, Hash)] +pub(crate) struct RequestKey { + objects_dir: PathBuf, + digest: [u8; 32], +} + +impl RequestKey { + pub(crate) fn new(objects_dir: &Path, ref_token: &crate::ids::RefsDigest, body: &[u8]) -> Self { + let mut hasher = gix::hash::hasher(gix::hash::Kind::Sha256); + hasher.update(ref_token.as_bytes()); + hasher.update(body); + let id = hasher.try_finalize().expect("sha256 digest finalizes"); + let mut digest = [0u8; 32]; + digest.copy_from_slice(id.as_slice()); + Self { + objects_dir: objects_dir.to_path_buf(), + digest, + } + } +} + +#[derive(Clone)] +pub(crate) enum Signal { + Pending, + Ready(Bytes), + Retry, + Regenerate, +} + +#[derive(Clone)] +enum Slot { + Ready(Bytes), + TooLarge, +} + +fn weigh(slot: &Slot) -> Weight { + match slot { + Slot::Ready(bytes) => Weight::new(bytes.len() as u64), + Slot::TooLarge => Weight::new(0), + } +} + +enum Settlement { + Ready(Bytes), + TooLarge, + Regenerate, + Retry, +} + +pub(crate) struct PackCache { + config: CacheConfig, + store: Lru>, + inflight: Mutex>>, +} + +pub(crate) enum Decision { + Serve(Bytes), + Await(watch::Receiver), + Lead(Lease), + Stream, + Off, +} + +impl PackCache { + pub(crate) fn new(mut config: CacheConfig, clock: Arc) -> Arc { + config.max_entry_bytes = MaxEntryBytes::new( + config + .max_entry_bytes + .get() + .min(config.max_total_bytes.get()), + ); + let store = Lru::by_weight_with_ttl( + Weight::new(config.max_total_bytes.get() as u64), + config.ttl, + clock, + weigh, + ) + .with_entry_cap(EntryCount::new(MAX_ENTRIES as u64)); + let cache = Arc::new(Self { + config, + store, + inflight: Mutex::new(HashMap::new()), + }); + knot_cache::register(&cache); + cache + } + + pub(crate) fn decide(self: &Arc, key: RequestKey) -> Decision { + if !self.config.enabled { + return Decision::Off; + } + if let Some(decision) = self.serve_cached(&key) { + return decision; + } + let mut inflight = self.inflight_lock(); + if let Some(sender) = inflight.get(&key) { + return Decision::Await(sender.subscribe()); + } + // `store` & `inflight` are separate locks, + // such that a leader can finish settling in the gap between + // a fast-path miss and this lock acquisition, + // by which point it has inserted the pack and taken its sender away. + // Reading the store a second time will cover that gap, + // since `settle` always inserts before it removes. + // + // Without it, + // the race-losing caller would elect itself and rebuild a pack + // the store does in fact already have. + if let Some(decision) = self.serve_cached(&key) { + return decision; + } + let (sender, _) = watch::channel(Signal::Pending); + inflight.insert(key.clone(), sender); + Decision::Lead(Lease { + key, + cache: Arc::clone(self), + max_entry_bytes: self.config.max_entry_bytes, + settled: false, + }) + } + + fn serve_cached(&self, key: &RequestKey) -> Option { + match self.store.get(key) { + Some(Slot::Ready(bytes)) => Some(Decision::Serve(bytes)), + Some(Slot::TooLarge) => Some(Decision::Stream), + None => None, + } + } + + fn settle(&self, key: &RequestKey, settlement: Settlement) { + let signal = match settlement { + Settlement::Ready(bytes) => { + self.store.insert(key.clone(), Slot::Ready(bytes.clone())); + Signal::Ready(bytes) + } + Settlement::TooLarge => { + self.store.insert(key.clone(), Slot::TooLarge); + Signal::Regenerate + } + Settlement::Regenerate => Signal::Regenerate, + Settlement::Retry => Signal::Retry, + }; + if let Some(sender) = self.inflight_lock().remove(key) { + let _ = sender.send(signal); + } + } + + fn inflight_lock( + &self, + ) -> std::sync::MutexGuard<'_, HashMap>> { + self.inflight + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + #[cfg(test)] + fn entry_count(&self) -> u64 { + self.store.entry_count().get() + } +} + +impl Reclaimable for PackCache { + fn footprint(&self) -> Weight { + self.store.footprint() + } + + fn reclaim(&self) { + self.store.reclaim(); + } +} + +pub(crate) struct Lease { + key: RequestKey, + cache: Arc, + max_entry_bytes: MaxEntryBytes, + settled: bool, +} + +impl Lease { + pub(crate) fn max_entry_bytes(&self) -> MaxEntryBytes { + self.max_entry_bytes + } + + pub(crate) fn ready(mut self, bytes: Bytes) { + self.cache.settle(&self.key, Settlement::Ready(bytes)); + self.settled = true; + } + + pub(crate) fn too_large(mut self) { + self.cache.settle(&self.key, Settlement::TooLarge); + self.settled = true; + } + + pub(crate) fn regenerate(mut self) { + self.cache.settle(&self.key, Settlement::Regenerate); + self.settled = true; + } + + pub(crate) fn retry(mut self) { + self.cache.settle(&self.key, Settlement::Retry); + self.settled = true; + } +} + +impl Drop for Lease { + fn drop(&mut self) { + if !self.settled { + self.cache.settle(&self.key, Settlement::Retry); + } + } +} + +pub(crate) enum Resolved { + Bytes(Bytes), + Retry, + Regenerate, +} + +pub(crate) async fn wait(mut receiver: watch::Receiver) -> Resolved { + let resolved = classify(&receiver.borrow_and_update()); + match resolved { + Some(resolved) => resolved, + None => match receiver.changed().await { + Err(_) => Resolved::Regenerate, + Ok(()) => Box::pin(wait(receiver)).await, + }, + } +} + +fn classify(signal: &Signal) -> Option { + match signal { + Signal::Pending => None, + Signal::Ready(bytes) => Some(Resolved::Bytes(bytes.clone())), + Signal::Retry => Some(Resolved::Retry), + Signal::Regenerate => Some(Resolved::Regenerate), + } +} + +pub(crate) enum Capture { + Buffering { buffer: Vec, limit: usize }, + Overflow, + Off, +} + +impl Capture { + pub(crate) fn new(limit: Option) -> Self { + match limit { + Some(limit) => Capture::Buffering { + buffer: Vec::new(), + limit: limit.get(), + }, + None => Capture::Off, + } + } + + pub(crate) fn record(&mut self, chunk: &[u8]) { + match self { + Capture::Buffering { buffer, limit } if buffer.len() + chunk.len() <= *limit => { + buffer.extend_from_slice(chunk) + } + Capture::Buffering { .. } => *self = Capture::Overflow, + _ => {} + } + } + + pub(crate) fn into_bytes(self) -> Option> { + match self { + Capture::Buffering { buffer, .. } => Some(buffer), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use knot_runtime::{ManualClock, SystemClock}; + use knot_types::UnixMicros; + + use super::*; + + fn built(config: CacheConfig) -> Arc { + PackCache::new(config, Arc::new(SystemClock)) + } + + fn key(body: &[u8]) -> RequestKey { + let mut token = [0u8; 32]; + token[..4].copy_from_slice(b"refs"); + RequestKey::new( + Path::new("/scan/did:plc:squid/objects"), + &crate::ids::RefsDigest::new(token), + body, + ) + } + + fn lead(cache: &Arc, body: &[u8]) -> Lease { + match cache.decide(key(body)) { + Decision::Lead(lease) => lease, + _ => panic!("the first request for a key leads"), + } + } + + #[tokio::test] + async fn a_second_identical_request_serves_the_cached_pack() { + let cache = built(CacheConfig::default()); + lead(&cache, b"want").ready(Bytes::from_static(b"PACK-bytes")); + match cache.decide(key(b"want")) { + Decision::Serve(bytes) => assert_eq!(bytes.as_ref(), b"PACK-bytes"), + _ => panic!("second identical request hits the cache"), + } + } + + #[tokio::test] + async fn a_concurrent_request_awaits_the_leader_then_shares_its_bytes() { + let cache = built(CacheConfig::default()); + let leader = lead(&cache, b"clone"); + let receiver = match cache.decide(key(b"clone")) { + Decision::Await(receiver) => receiver, + _ => panic!("concurrent request awaits the inflight leader"), + }; + leader.ready(Bytes::from_static(b"shared")); + match wait(receiver).await { + Resolved::Bytes(bytes) => assert_eq!(bytes.as_ref(), b"shared"), + _ => panic!("the follower shares the leader's bytes"), + } + } + + #[tokio::test] + async fn a_regenerating_leader_wakes_followers_to_regenerate() { + let cache = built(CacheConfig::default()); + let leader = lead(&cache, b"err"); + let receiver = match cache.decide(key(b"err")) { + Decision::Await(receiver) => receiver, + _ => panic!("follower awaits"), + }; + leader.regenerate(); + assert!( + matches!(wait(receiver).await, Resolved::Regenerate), + "an errored leader tells the follower to regenerate in parallel" + ); + match cache.decide(key(b"err")) { + Decision::Lead(_) => {} + _ => panic!("a regenerated key isn't remembered, so the next request leads afresh"), + } + } + + #[tokio::test] + async fn a_retrying_leader_tells_followers_to_re_elect() { + let cache = built(CacheConfig::default()); + let leader = lead(&cache, b"vanish"); + let receiver = match cache.decide(key(b"vanish")) { + Decision::Await(receiver) => receiver, + _ => panic!("follower awaits"), + }; + leader.retry(); + assert!( + matches!(wait(receiver).await, Resolved::Retry), + "a vanished leader tells the follower to re-elect a fresh leader" + ); + match cache.decide(key(b"vanish")) { + Decision::Lead(_) => {} + _ => panic!("a retried key isn't remembered, so the next request leads afresh"), + } + } + + #[tokio::test] + async fn a_dropped_lease_re_elects_rather_than_stranding_the_follower() { + let cache = built(CacheConfig::default()); + let leader = lead(&cache, b"dropped"); + let receiver = match cache.decide(key(b"dropped")) { + Decision::Await(receiver) => receiver, + _ => panic!("follower awaits"), + }; + drop(leader); + assert!( + matches!(wait(receiver).await, Resolved::Retry), + "an unsettled lease that drops re-elects instead of stranding the follower" + ); + } + + #[tokio::test] + async fn an_oversized_leader_marks_the_key_for_direct_streaming() { + let cache = built(CacheConfig::default()); + let leader = lead(&cache, b"huge"); + let receiver = match cache.decide(key(b"huge")) { + Decision::Await(receiver) => receiver, + _ => panic!("follower awaits"), + }; + leader.too_large(); + assert!( + matches!(wait(receiver).await, Resolved::Regenerate), + "an oversized leader sends its followers to stream directly" + ); + match cache.decide(key(b"huge")) { + Decision::Stream => {} + _ => panic!("an oversized key streams directly without re-buffering"), + } + } + + #[tokio::test] + async fn an_expired_entry_is_regenerated() { + let clock = Arc::new(ManualClock::new(UnixMicros::new(0))); + let cache = PackCache::new( + CacheConfig { + ttl: Duration::from_secs(60), + ..CacheConfig::default() + }, + Arc::clone(&clock) as Arc, + ); + lead(&cache, b"stale").ready(Bytes::from_static(b"old")); + clock.advance(Duration::from_secs(61)); + match cache.decide(key(b"stale")) { + Decision::Lead(_) => {} + _ => panic!("an expired entry forces a fresh generation"), + } + } + + #[tokio::test] + async fn the_total_byte_limit_evicts_the_oldest_entry() { + let cache = built(CacheConfig { + max_total_bytes: MaxCacheBytes::new(8), + ..CacheConfig::default() + }); + lead(&cache, b"a").ready(Bytes::from(vec![0u8; 5])); + lead(&cache, b"b").ready(Bytes::from(vec![0u8; 5])); + match cache.decide(key(b"a")) { + Decision::Lead(_) => {} + _ => panic!("the oldest entry is evicted once the total limit is exceeded"), + } + match cache.decide(key(b"b")) { + Decision::Serve(_) => {} + _ => panic!("the newest entry survives eviction"), + } + } + + #[tokio::test] + async fn the_entry_limit_never_exceeds_the_total_cache_size() { + let cache = built(CacheConfig { + max_entry_bytes: MaxEntryBytes::new(64), + max_total_bytes: MaxCacheBytes::new(16), + ..CacheConfig::default() + }); + let lease = lead(&cache, b"probe"); + assert_eq!( + lease.max_entry_bytes().get(), + 16, + "a per-entry limit above the whole-cache size is clamped so a full entry can be retained" + ); + } + + #[tokio::test] + async fn oversized_entries_cannot_grow_the_cache_without_bound() { + let cache = built(CacheConfig::default()); + (0..(MAX_ENTRIES + 64)).for_each(|nonce| { + lead(&cache, format!("oversized-{nonce}").as_bytes()).too_large(); + }); + assert!( + cache.entry_count() <= MAX_ENTRIES as u64, + "a flood of distinct oversized requests stays within the entry bound" + ); + } + + #[test] + fn capture_stops_buffering_once_the_limit_is_passed() { + let mut capture = Capture::new(Some(MaxEntryBytes::new(4))); + capture.record(b"abc"); + capture.record(b"de"); + assert!(capture.into_bytes().is_none(), "overflow drops the buffer"); + } + + #[test] + fn capture_keeps_bytes_under_the_limit() { + let mut capture = Capture::new(Some(MaxEntryBytes::new(8))); + capture.record(b"abcd"); + assert_eq!(capture.into_bytes().unwrap(), b"abcd"); + } +} diff --git a/knot2/crates/knot-pack/src/error.rs b/knot2/crates/knot-pack/src/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..d4cb3d5b44402be05f64de92f8daaf271a1de9cf --- /dev/null +++ b/knot2/crates/knot-pack/src/error.rs @@ -0,0 +1,71 @@ +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; + +pub use knot_storage::PackLimit; + +#[derive(Debug, thiserror::Error)] +pub enum PackError { + #[error("repository not found")] + NotFound, + #[error("repository index is warming, retry shortly")] + Unavailable, + #[error("invalid request path: {0}")] + BadPath(String), + #[error("unsupported service")] + UnsupportedService, + #[error("push is served over SSH, not HTTP")] + PushOverSsh, + #[error("protocol: {0}")] + Protocol(String), + #[error("unsupported request content-encoding: {0}")] + UnsupportedEncoding(String), + #[error("pkt-line: {0}")] + PktLine(#[from] std::io::Error), + #[error("pack: {0}")] + Pack(String), + #[error("pack exceeds {0} limit")] + LimitExceeded(PackLimit), + #[error("upload-pack selection exceeded its object-set limit")] + SelectionTooLarge, + #[error("upload-pack selection exceeded its time budget")] + SelectionTimeout, + #[error("insufficient memory to ingest this push, retry when the server is less busy")] + InsufficientMemory, + #[error(transparent)] + Git(knot_git::GitError), +} + +impl From for PackError { + fn from(error: knot_git::GitError) -> Self { + use knot_git::SelectionLimit; + match error { + knot_git::GitError::Selection(SelectionLimit::Objects) => PackError::SelectionTooLarge, + knot_git::GitError::Selection(SelectionLimit::Time) => PackError::SelectionTimeout, + other => PackError::Git(other), + } + } +} + +impl PackError { + pub fn http_status(&self) -> StatusCode { + match self { + PackError::NotFound => StatusCode::NOT_FOUND, + PackError::Unavailable => StatusCode::SERVICE_UNAVAILABLE, + PackError::PushOverSsh => StatusCode::FORBIDDEN, + PackError::BadPath(_) | PackError::UnsupportedService => StatusCode::BAD_REQUEST, + PackError::Protocol(_) | PackError::PktLine(_) => StatusCode::BAD_REQUEST, + PackError::UnsupportedEncoding(_) => StatusCode::UNSUPPORTED_MEDIA_TYPE, + PackError::LimitExceeded(_) => StatusCode::PAYLOAD_TOO_LARGE, + PackError::SelectionTooLarge + | PackError::SelectionTimeout + | PackError::InsufficientMemory => StatusCode::SERVICE_UNAVAILABLE, + PackError::Pack(_) | PackError::Git(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl IntoResponse for PackError { + fn into_response(self) -> Response { + (self.http_status(), self.to_string()).into_response() + } +} diff --git a/knot2/crates/knot-pack/src/fetch.rs b/knot2/crates/knot-pack/src/fetch.rs new file mode 100644 index 0000000000000000000000000000000000000000..d19a6434cc16f7c382dbb895f442ef437938488c --- /dev/null +++ b/knot2/crates/knot-pack/src/fetch.rs @@ -0,0 +1,763 @@ +use std::io::Write; + +use axum::http::{HeaderMap, HeaderValue, Method, header}; +use knot_git::{Filter, Haves, RefRecord, Repo, Wants}; +use knot_runtime::{HttpRequest, HttpResponse, HttpTransport, NetworkError}; +use knot_types::{HttpStatus, ObjectFormat, Oid, RefName}; +use tokio_stream::StreamExt; +use url::Url; + +use crate::error::PackError; +use crate::ids::MaxWireBytes; +use crate::pkt; +use crate::{HaveOids, WantOids}; + +#[derive(Debug, thiserror::Error)] +pub enum FetchError { + #[error("upstream url: {0}")] + Url(String), + #[error("upstream network: {0}")] + Network(#[from] NetworkError), + #[error("upstream returned http status {0}")] + Status(HttpStatus), + #[error("upstream protocol: {0}")] + Protocol(String), + #[error("upstream reported: {0}")] + Remote(String), + #[error("fetched pack exceeds {} bytes", limit.get())] + PackTooLarge { limit: MaxWireBytes }, + #[error(transparent)] + Pack(#[from] PackError), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpstreamRefs { + pub object_format: ObjectFormat, + pub head_symref: Option, + pub head_target: Option, + pub refs: Vec, +} + +impl UpstreamRefs { + pub fn tips(&self) -> Vec { + let mut tips: Vec = self.refs.iter().map(|record| record.target).collect(); + tips.sort_unstable(); + tips.dedup(); + tips + } + + pub fn find(&self, name: &RefName) -> Option { + self.refs + .iter() + .find(|record| record.name == *name) + .map(|record| record.target) + } +} + +fn protocol(message: impl Into) -> FetchError { + FetchError::Protocol(message.into()) +} + +fn endpoint(base: &Url, suffix: &str) -> Result { + let trimmed = base.as_str().trim_end_matches('/'); + Url::parse(&format!("{trimmed}/{suffix}")).map_err(|error| FetchError::Url(error.to_string())) +} + +fn headers_v2(content_type: Option<&'static str>) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert("git-protocol", HeaderValue::from_static("version=2")); + if let Some(content_type) = content_type { + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type)); + } + headers +} + +async fn execute( + http: &dyn HttpTransport, + request: HttpRequest, +) -> Result { + let response = http.execute(request).await?; + if !response.status.is_success() { + return Err(FetchError::Status(HttpStatus::new( + response.status.as_u16(), + ))); + } + Ok(response) +} + +pub fn parse_advertisement(body: &[u8]) -> Result { + let lines = pkt::data_payloads_all(body).map_err(|error| protocol(error.to_string()))?; + let lines: Vec<&str> = lines + .iter() + .map(|line| std::str::from_utf8(line).unwrap_or_default().trim_end()) + .collect(); + let has = |name: &str| { + lines + .iter() + .any(|line| *line == name || line.starts_with(&format!("{name}="))) + }; + if !has("version 2") { + return Err(protocol("upstream doesn't speak git protocol v2")); + } + if !has("ls-refs") || !has("fetch") { + return Err(protocol("upstream is missing ls-refs or fetch v2 command")); + } + lines + .iter() + .find_map(|line| line.strip_prefix("object-format=")) + .map_or(Ok(ObjectFormat::SHA1), |token| { + ObjectFormat::from_capability(token).ok_or_else(|| { + protocol(format!("upstream advertises unknown object format {token}")) + }) + }) +} + +// git refuses non-sha1 unless we echo the format back +fn write_object_format(buf: &mut Vec, object_format: ObjectFormat) -> Result<(), PackError> { + if object_format != ObjectFormat::SHA1 { + pkt::write_data( + buf, + format!("object-format={}\n", object_format.capability()).as_bytes(), + )?; + } + Ok(()) +} + +pub fn ls_refs_request( + prefixes: &[&str], + object_format: ObjectFormat, +) -> Result, PackError> { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"command=ls-refs\n")?; + pkt::write_data(&mut buf, b"agent=knot/0\n")?; + write_object_format(&mut buf, object_format)?; + pkt::write_delim(&mut buf)?; + pkt::write_data(&mut buf, b"symrefs\n")?; + prefixes.iter().try_for_each(|prefix| { + pkt::write_data(&mut buf, format!("ref-prefix {prefix}\n").as_bytes()) + })?; + pkt::write_flush(&mut buf)?; + Ok(buf) +} + +pub fn parse_ls_refs(body: &[u8], object_format: ObjectFormat) -> Result { + let lines = pkt::data_payloads(body).map_err(|error| protocol(error.to_string()))?; + lines.iter().try_fold( + UpstreamRefs { + object_format, + head_symref: None, + head_target: None, + refs: Vec::new(), + }, + |mut refs, line| { + let text = std::str::from_utf8(line) + .map_err(|_| protocol("ref line isn't utf-8"))? + .trim_end(); + if let Some(message) = text.strip_prefix("ERR ") { + return Err(FetchError::Remote(message.to_string())); + } + let (oid, rest) = text + .split_once(' ') + .ok_or_else(|| protocol(format!("malformed ref line: {text}")))?; + let target = + Oid::from_hex(oid).map_err(|_| protocol(format!("malformed ref oid: {oid}")))?; + if target.object_id().kind() != object_format.kind() { + return Err(protocol(format!( + "ref oid {oid} doesn't match advertised object format {}", + object_format.capability() + ))); + } + let mut attributes = rest.split(' '); + match attributes.next() { + Some("HEAD") => { + refs.head_target = Some(target); + refs.head_symref = attributes + .find_map(|attribute| attribute.strip_prefix("symref-target:")) + .and_then(|symref| RefName::new(symref).ok()); + } + Some(name) => { + if let Ok(name) = RefName::new(name) { + refs.refs.push(RefRecord { name, target }); + } + } + None => return Err(protocol(format!("malformed ref line: {text}"))), + } + Ok(refs) + }, + ) +} + +pub fn fetch_request( + wants: &WantOids, + haves: &HaveOids, + object_format: ObjectFormat, +) -> Result, PackError> { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"command=fetch\n")?; + pkt::write_data(&mut buf, b"agent=knot/0\n")?; + write_object_format(&mut buf, object_format)?; + pkt::write_delim(&mut buf)?; + pkt::write_data(&mut buf, b"no-progress\n")?; + pkt::write_data(&mut buf, b"ofs-delta\n")?; + wants + .iter() + .try_for_each(|want| pkt::write_data(&mut buf, format!("want {want}\n").as_bytes()))?; + haves + .iter() + .try_for_each(|have| pkt::write_data(&mut buf, format!("have {have}\n").as_bytes()))?; + pkt::write_data(&mut buf, b"done\n")?; + pkt::write_flush(&mut buf)?; + Ok(buf) +} + +pub struct PackDemux { + pending: Vec, + in_packfile: bool, + written: MaxWireBytes, + limit: MaxWireBytes, +} + +impl PackDemux { + pub fn new(max_pack_bytes: MaxWireBytes) -> Self { + Self { + pending: Vec::new(), + in_packfile: false, + written: MaxWireBytes::new(0), + limit: max_pack_bytes, + } + } + + pub fn feed(&mut self, chunk: &[u8], out: &mut dyn Write) -> Result<(), FetchError> { + self.pending.extend_from_slice(chunk); + let mut done = 0usize; + // one sideband payload held outside `self` so the sink can run while + // the pending buffer is still being drained + let mut scratch = Vec::new(); + loop { + let rest = &self.pending[done..]; + if rest.len() < 4 { + break; + } + let line = pkt::frame_len(&rest[..4]).map_err(|error| protocol(error.to_string()))?; + if line < 4 { + done += 4; + continue; + } + if rest.len() < line { + break; + } + let payload = &rest[4..line]; + if self.in_packfile { + match payload.split_first() { + Some((1, data)) => { + scratch.clear(); + scratch.extend_from_slice(data); + self.admit(&scratch, out)?; + } + Some((2, _)) => {} + Some((3, message)) => { + return Err(FetchError::Remote( + String::from_utf8_lossy(message).trim_end().to_string(), + )); + } + _ => return Err(protocol("empty sideband frame in packfile section")), + } + } else if let Some(message) = payload.strip_prefix(b"ERR ".as_slice()) { + return Err(FetchError::Remote( + String::from_utf8_lossy(message).trim_end().to_string(), + )); + } else { + let line = payload.strip_suffix(b"\n".as_slice()).unwrap_or(payload); + self.in_packfile |= line == b"packfile"; + } + done += line; + } + self.pending.drain(..done); + Ok(()) + } + + fn admit(&mut self, data: &[u8], out: &mut dyn Write) -> Result<(), FetchError> { + let next = self.written.get() + data.len(); + if next > self.limit.get() { + return Err(FetchError::PackTooLarge { limit: self.limit }); + } + out.write_all(data) + .map_err(|error| FetchError::Pack(error.into()))?; + self.written = MaxWireBytes::new(next); + Ok(()) + } + + pub fn finish(&mut self) -> Result<(), FetchError> { + if !self.pending.is_empty() { + return Err(protocol("truncated pkt-line in upstream pack response")); + } + if !self.in_packfile { + return Err(protocol("upstream response has no packfile section")); + } + Ok(()) + } +} + +pub async fn remote_refs( + http: &dyn HttpTransport, + base: &Url, + prefixes: &[&str], +) -> Result { + let advertise = endpoint(base, "info/refs?service=git-upload-pack")?; + let response = execute( + http, + HttpRequest { + method: Method::GET, + url: advertise, + headers: headers_v2(None), + body: None, + }, + ) + .await?; + let object_format = parse_advertisement(&response.body)?; + + let upload = endpoint(base, "git-upload-pack")?; + let response = execute( + http, + HttpRequest { + method: Method::POST, + url: upload, + headers: headers_v2(Some("application/x-git-upload-pack-request")), + body: Some(ls_refs_request(prefixes, object_format)?.into()), + }, + ) + .await?; + parse_ls_refs(&response.body, object_format) +} + +pub async fn remote_pack( + http: &dyn HttpTransport, + base: &Url, + wants: &WantOids, + haves: &HaveOids, + object_format: ObjectFormat, + max_pack_bytes: MaxWireBytes, +) -> Result, FetchError> { + let mut pack = Vec::new(); + remote_pack_streamed( + http, + base, + wants, + haves, + object_format, + max_pack_bytes, + &mut pack, + ) + .await?; + Ok(pack) +} + +#[allow(clippy::too_many_arguments)] +pub async fn remote_pack_streamed( + http: &dyn HttpTransport, + base: &Url, + wants: &WantOids, + haves: &HaveOids, + object_format: ObjectFormat, + max_pack_bytes: MaxWireBytes, + out: &mut W, +) -> Result<(), FetchError> { + if wants.is_empty() { + return Ok(()); + } + let upload = endpoint(base, "git-upload-pack")?; + let response = http + .execute_streamed(HttpRequest { + method: Method::POST, + url: upload, + headers: headers_v2(Some("application/x-git-upload-pack-request")), + body: Some(fetch_request(wants, haves, object_format)?.into()), + }) + .await?; + if !response.status.is_success() { + return Err(FetchError::Status(HttpStatus::new( + response.status.as_u16(), + ))); + } + let mut body = response.body; + let mut demux = PackDemux::new(max_pack_bytes); + while let Some(chunk) = body.try_next().await? { + demux.feed(&chunk, out)?; + } + demux.finish() +} + +pub fn local_refs(source: &Repo, prefixes: &[&str]) -> Result { + let refs = source + .advertised_refs() + .map_err(PackError::from)? + .iter() + .filter(|record| crate::upload::matches_prefix(record.name.as_str(), prefixes)) + .cloned() + .collect(); + let head = source.head(); + Ok(UpstreamRefs { + object_format: source.object_format(), + head_symref: head.as_ref().map(|head| head.name.clone()), + head_target: head.map(|head| head.target), + refs, + }) +} + +struct BoundedPack<'a> { + out: &'a mut dyn Write, + limit: u64, + written: u64, + overflowed: bool, +} + +impl Write for BoundedPack<'_> { + fn write(&mut self, data: &[u8]) -> std::io::Result { + if self.written + data.len() as u64 > self.limit { + self.overflowed = true; + return Err(std::io::Error::other("pack byte limit exceeded")); + } + self.out.write_all(data)?; + self.written += data.len() as u64; + Ok(data.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.out.flush() + } +} + +pub fn local_pack_to( + source: &Repo, + wants: &WantOids, + haves: &HaveOids, + max_pack_bytes: MaxWireBytes, + out: &mut W, +) -> Result<(), FetchError> { + if wants.is_empty() { + return Ok(()); + } + let oids = source + .select_pack_objects_filtered( + Wants::new(wants.as_slice()), + Haves::new(haves.as_slice()), + Filter::None, + crate::upload::selection_budget(), + ) + .map_err(PackError::from)? + .send; + let mut out = BoundedPack { + out, + limit: max_pack_bytes.get() as u64, + written: 0, + overflowed: false, + }; + match crate::objects::write_pack( + &source.objects_dir(), + oids, + None, + &mut out, + source.object_format().kind(), + ) { + Ok(()) => Ok(()), + Err(_) if out.overflowed => Err(FetchError::PackTooLarge { + limit: max_pack_bytes, + }), + Err(error) => Err(FetchError::Pack(error)), + } +} + +pub fn local_pack( + source: &Repo, + wants: &WantOids, + haves: &HaveOids, + max_pack_bytes: MaxWireBytes, +) -> Result, FetchError> { + let mut buf = Vec::new(); + local_pack_to(source, wants, haves, max_pack_bytes, &mut buf)?; + Ok(buf) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn data(buf: &mut Vec, line: &[u8]) { + pkt::write_data(buf, line).unwrap(); + } + + #[test] + fn the_v2_advertisement_parses_to_its_object_format_and_v0_is_refused() { + let scan = tempfile::tempdir().unwrap(); + let repo = knot_git::Layout::new(scan.path()) + .create(&knot_types::RepoDid::new("did:plc:squid").unwrap()) + .unwrap(); + let v2 = crate::upload::advertise(&repo).unwrap(); + assert_eq!(parse_advertisement(&v2).unwrap(), ObjectFormat::SHA1); + + let advertise = |extra: Option<&str>| { + let mut buf = Vec::new(); + data(&mut buf, b"version 2\n"); + data(&mut buf, b"ls-refs\n"); + data(&mut buf, b"fetch\n"); + if let Some(extra) = extra { + data(&mut buf, format!("{extra}\n").as_bytes()); + } + pkt::write_flush(&mut buf).unwrap(); + parse_advertisement(&buf) + }; + assert_eq!( + advertise(Some("object-format=sha256")).unwrap(), + ObjectFormat::SHA256 + ); + assert_eq!( + advertise(None).unwrap(), + ObjectFormat::SHA1, + "protocol v2 treats an absent object-format as sha1" + ); + assert!(matches!( + advertise(Some("object-format=sha3")), + Err(FetchError::Protocol(_)) + )); + + let mut v0 = Vec::new(); + data(&mut v0, b"# service=git-upload-pack\n"); + pkt::write_flush(&mut v0).unwrap(); + data( + &mut v0, + b"95d09f2b10159347eece71399a7e2e907ea3df4f HEAD\0side-band-64k\n", + ); + pkt::write_flush(&mut v0).unwrap(); + assert!(matches!( + parse_advertisement(&v0), + Err(FetchError::Protocol(_)) + )); + } + + #[test] + fn ls_refs_lines_parse_with_symref_skip_head_and_enforce_the_object_format() { + let mut body = Vec::new(); + data( + &mut body, + b"95d09f2b10159347eece71399a7e2e907ea3df4f HEAD symref-target:refs/heads/main\n", + ); + data( + &mut body, + b"95d09f2b10159347eece71399a7e2e907ea3df4f refs/heads/main\n", + ); + pkt::write_flush(&mut body).unwrap(); + let refs = parse_ls_refs(&body, ObjectFormat::SHA1).unwrap(); + assert_eq!( + refs.head_symref.as_ref().map(RefName::as_str), + Some("refs/heads/main") + ); + assert_eq!( + refs.head_target.map(|oid| oid.to_string()), + Some("95d09f2b10159347eece71399a7e2e907ea3df4f".to_string()) + ); + assert_eq!(refs.refs.len(), 1); + assert_eq!(refs.refs[0].name.as_str(), "refs/heads/main"); + assert_eq!(refs.tips().len(), 1); + + let single = |oid: &str| { + let mut buf = Vec::new(); + data(&mut buf, format!("{oid} refs/heads/main\n").as_bytes()); + pkt::write_flush(&mut buf).unwrap(); + buf + }; + let sha1 = "95d09f2b10159347eece71399a7e2e907ea3df4f"; + let sha256 = "6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321"; + assert!(parse_ls_refs(&single(sha256), ObjectFormat::SHA256).is_ok()); + assert!(matches!( + parse_ls_refs(&single(sha256), ObjectFormat::SHA1), + Err(FetchError::Protocol(_)) + )); + assert!(matches!( + parse_ls_refs(&single(sha1), ObjectFormat::SHA256), + Err(FetchError::Protocol(_)) + )); + } + + #[test] + fn head_target_survives_when_the_server_omits_its_symref() { + let mut body = Vec::new(); + data( + &mut body, + b"95d09f2b10159347eece71399a7e2e907ea3df4f HEAD\n", + ); + data( + &mut body, + b"95d09f2b10159347eece71399a7e2e907ea3df4f refs/heads/master\n", + ); + pkt::write_flush(&mut body).unwrap(); + + let refs = parse_ls_refs(&body, ObjectFormat::SHA1).unwrap(); + assert_eq!(refs.head_symref, None); + assert_eq!( + refs.head_target, + refs.refs.first().map(|record| record.target) + ); + } + + #[test] + fn a_malformed_ref_oid_is_a_protocol_error() { + let mut body = Vec::new(); + data(&mut body, b"zzzz refs/heads/main\n"); + pkt::write_flush(&mut body).unwrap(); + assert!(matches!( + parse_ls_refs(&body, ObjectFormat::SHA1), + Err(FetchError::Protocol(_)) + )); + } + + #[test] + fn an_err_line_is_surfaced_as_remote() { + let mut body = Vec::new(); + data(&mut body, b"ERR access denied\n"); + pkt::write_flush(&mut body).unwrap(); + assert!(matches!( + parse_ls_refs(&body, ObjectFormat::SHA1), + Err(FetchError::Remote(message)) if message == "access denied" + )); + } + + fn fed(body: &[u8], split: usize) -> Result, FetchError> { + let mut demux = PackDemux::new(MaxWireBytes::new(1024)); + let mut pack = Vec::new(); + let (head, tail) = body.split_at(split.min(body.len())); + demux.feed(head, &mut pack)?; + demux.feed(tail, &mut pack)?; + demux.finish()?; + Ok(pack) + } + + #[test] + fn the_packfile_section_demuxes_data_drops_progress_and_survives_split_chunks() { + let mut body = Vec::new(); + data(&mut body, b"packfile\n"); + data(&mut body, b"\x01PACKDATA"); + data(&mut body, b"\x02counting objects\n"); + data(&mut body, b"\x01MORE"); + pkt::write_flush(&mut body).unwrap(); + for split in [0, 1, 3, 9, body.len()] { + let pack = fed(&body, split).unwrap(); + assert_eq!(pack, b"PACKDATAMORE", "split at {split}"); + } + } + + #[test] + fn a_sideband_error_band_is_remote_and_the_limit_holds() { + let mut body = Vec::new(); + data(&mut body, b"packfile\n"); + data(&mut body, b"\x03out of disk\n"); + pkt::write_flush(&mut body).unwrap(); + assert!(matches!( + fed(&body, 0), + Err(FetchError::Remote(message)) if message == "out of disk" + )); + + let mut big = Vec::new(); + data(&mut big, b"packfile\n"); + data(&mut big, b"\x01PACKDATA"); + pkt::write_flush(&mut big).unwrap(); + let mut demux = PackDemux::new(MaxWireBytes::new(4)); + let mut pack = Vec::new(); + assert!(matches!( + demux.feed(&big, &mut pack), + Err(FetchError::PackTooLarge { limit }) if limit == MaxWireBytes::new(4) + )); + } + + #[test] + fn a_response_without_a_packfile_section_is_refused() { + let mut body = Vec::new(); + data(&mut body, b"acknowledgments\n"); + data(&mut body, b"NAK\n"); + pkt::write_flush(&mut body).unwrap(); + assert!(matches!(fed(&body, 0), Err(FetchError::Protocol(_)))); + } + + #[test] + fn a_truncated_pack_response_is_refused_instead_of_silently_accepted() { + // the trailing pkt-line is cut in half, finish must refuse instead of + // reporting success over a response it never read to the end + let mut body = Vec::new(); + data(&mut body, b"packfile\n"); + data(&mut body, b"\x01PACKDATA"); + let mut demux = PackDemux::new(MaxWireBytes::new(1024)); + let mut pack = Vec::new(); + demux.feed(&body[..body.len() - 2], &mut pack).unwrap(); + assert!(matches!(demux.finish(), Err(FetchError::Protocol(_)))); + } + + #[test] + fn an_err_line_before_the_packfile_section_is_remote() { + let mut body = Vec::new(); + data(&mut body, b"ERR no such object\n"); + assert!( + matches!(fed(&body, 0), Err(FetchError::Remote(message)) if message == "no such object") + ); + } + + #[test] + fn the_fetch_request_includes_wants_haves_and_done() { + let want = Oid::from_hex("95d09f2b10159347eece71399a7e2e907ea3df4f").unwrap(); + let have = Oid::from_hex("2222222222222222222222222222222222222222").unwrap(); + let body = fetch_request( + &WantOids::new(vec![want]), + &HaveOids::new(vec![have]), + ObjectFormat::SHA1, + ) + .unwrap(); + let lines = pkt::data_payloads_all(&body).unwrap(); + let text: Vec<&str> = lines + .iter() + .map(|line| std::str::from_utf8(line).unwrap().trim_end()) + .collect(); + assert!(text.contains(&"command=fetch")); + assert!(text.contains(&"want 95d09f2b10159347eece71399a7e2e907ea3df4f")); + assert!(text.contains(&"have 2222222222222222222222222222222222222222")); + assert!(text.contains(&"done")); + assert!(text.contains(&"no-progress")); + assert!(!text.iter().any(|line| line.starts_with("object-format="))); + } + + #[test] + fn the_requests_repeat_a_non_default_object_format() { + let want = + Oid::from_hex("95d09f2b10159347eece71399a7e2e907ea3df4f0123456789abcdef01234567") + .unwrap(); + let text = |body: &[u8]| -> Vec { + pkt::data_payloads_all(body) + .unwrap() + .iter() + .map(|line| std::str::from_utf8(line).unwrap().trim_end().to_string()) + .collect() + }; + + let fetch = fetch_request( + &WantOids::new(vec![want]), + &HaveOids::default(), + ObjectFormat::SHA256, + ) + .unwrap(); + let lines = text(&fetch); + assert!(lines.contains(&"object-format=sha256".to_string())); + let capability = lines.iter().position(|line| line == "object-format=sha256"); + let delimiter = lines.iter().position(|line| line.as_str() == "0001"); + assert!( + capability < delimiter || delimiter.is_none(), + "the capability belongs in the request header, before the argument section" + ); + + let ls_refs = ls_refs_request(&["refs/heads/"], ObjectFormat::SHA256).unwrap(); + assert!(text(&ls_refs).contains(&"object-format=sha256".to_string())); + + let sha1 = ls_refs_request(&["refs/heads/"], ObjectFormat::SHA1).unwrap(); + assert!( + !text(&sha1) + .iter() + .any(|line| line.starts_with("object-format=")) + ); + } +} diff --git a/knot2/crates/knot-pack/src/frame.rs b/knot2/crates/knot-pack/src/frame.rs new file mode 100644 index 0000000000000000000000000000000000000000..3216c4dcf87ff87ce4eb65c8f3dcdc335ba556c3 --- /dev/null +++ b/knot2/crates/knot-pack/src/frame.rs @@ -0,0 +1,457 @@ +use std::collections::HashMap; + +use flate2::{Decompress, FlushDecompress, Status}; +use gix_pack::data::{Entry, entry::Header}; + +use knot_types::ObjectCount; + +use crate::error::{PackError, PackLimit}; +use crate::ids::PackOffset; +use knot_storage::PackLimits; + +use crate::meter::{check_depth, malformed, pack_object_count}; +use crate::pkt::{self, Frame}; + +const HEADER_SLACK: usize = 64; + +fn pack_start(buf: &[u8]) -> Option { + let caps = pkt::first_command(buf) + .map(pkt::parse_caps) + .unwrap_or_default(); + let boundary = if caps.push_options { 2 } else { 1 }; + pkt::frames(buf, Some(boundary)) + .filter_map(|item| match item { + Ok((Frame::Flush, at)) => Some(at), + _ => None, + }) + .nth(boundary - 1) +} + +fn new_oid_field(line: &[u8]) -> Option<&[u8]> { + let line = line.split(|byte| *byte == 0).next().unwrap_or(line); + line.split(|byte| *byte == b' ').nth(1) +} + +fn no_pack_needed(buf: &[u8]) -> bool { + pkt::frames(buf, Some(1)) + .filter_map(|item| match item { + Ok((Frame::Data(payload), _)) => Some(payload), + _ => None, + }) + .all(|line| { + new_oid_field(line) + .map(|oid| oid.iter().all(|byte| *byte == b'0')) + .unwrap_or(false) + }) +} + +const PREAMBLE_SCAN_LIMIT: usize = 16 * 1024 * 1024; + +trait PackSource { + fn len(&self) -> u64; + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result; +} + +impl PackSource for [u8] { + fn len(&self) -> u64 { + <[u8]>::len(self) as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result { + let start = usize::try_from(offset) + .unwrap_or(usize::MAX) + .min(<[u8]>::len(self)); + let read = (<[u8]>::len(self) - start).min(buf.len()); + buf[..read].copy_from_slice(&self[start..start + read]); + Ok(read) + } +} + +struct FileSource<'a> { + file: &'a std::fs::File, + len: u64, +} + +impl PackSource for FileSource<'_> { + fn len(&self) -> u64 { + self.len + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result { + use std::os::unix::fs::FileExt; + let read = self.len.saturating_sub(offset).min(buf.len() as u64) as usize; + self.file.read_exact_at(&mut buf[..read], offset)?; + Ok(read) + } +} + +fn read_head(source: &S, upto: u64) -> Result, PackError> { + let limit = upto.min(PREAMBLE_SCAN_LIMIT as u64) as usize; + let mut head = vec![0u8; limit]; + let read = source + .read_at(0, &mut head) + .map_err(|error| PackError::Pack(format!("pack read: {error}")))?; + head.truncate(read); + Ok(head) +} + +struct EntryInflate { + data_offset: PackOffset, + decompressed_size: u64, + decompress: Decompress, + produced: u64, +} + +impl EntryInflate { + fn feed( + &mut self, + pack: &S, + scratch: &mut [u8], + chunk: &mut [u8], + ) -> Result, PackError> { + loop { + let consumed = self.decompress.total_in(); + let out_before = self.decompress.total_out(); + let read = pack + .read_at(self.data_offset.get() + consumed, chunk) + .map_err(|error| PackError::Pack(format!("pack read: {error}")))?; + let status = self + .decompress + .decompress(&chunk[..read], scratch, FlushDecompress::None) + .map_err(|error| PackError::Pack(format!("inflate: {error}")))?; + self.produced += self.decompress.total_out() - out_before; + if self.produced > self.decompressed_size { + return Err(malformed("object inflates beyond its declared size")); + } + match status { + Status::StreamEnd => { + return if self.produced == self.decompressed_size { + self.data_offset + .get() + .checked_add(self.decompress.total_in()) + .map(|offset| Some(PackOffset::new(offset))) + .ok_or_else(|| malformed("pack offset overflow")) + } else { + Err(malformed("object decompressed size mismatch")) + }; + } + Status::Ok | Status::BufError => { + if self.decompress.total_in() == consumed + && self.decompress.total_out() == out_before + { + return Ok(None); + } + } + } + } + } +} + +#[derive(Default)] +struct PackProgress { + num_objects: ObjectCount, + objects_done: ObjectCount, + next_offset: PackOffset, + total_decompressed: u64, + base_of: HashMap, + current: Option, + depth_checked: bool, +} + +impl PackProgress { + fn scan( + &mut self, + pack: &S, + limits: &PackLimits, + kind: gix::hash::Kind, + ) -> Result, PackError> { + let hash_len = kind.len_in_bytes(); + let len = pack.len(); + let mut scratch = [0u8; 8192]; + let mut chunk = [0u8; 8192]; + let mut header = [0u8; HEADER_SLACK]; + loop { + if self.objects_done == self.num_objects { + if !self.depth_checked { + check_depth(&self.base_of, limits.max_delta_depth)?; + self.depth_checked = true; + } + let total_len = (self.next_offset.get() as usize) + .checked_add(hash_len) + .ok_or_else(|| malformed("pack length overflow"))?; + return Ok((len as usize >= total_len).then_some(total_len)); + } + match self.current.as_mut() { + Some(entry) => match entry.feed(pack, &mut scratch, &mut chunk)? { + Some(next_offset) => { + self.next_offset = next_offset; + self.objects_done = self.objects_done.succ(); + self.current = None; + } + None => return Ok(None), + }, + None => { + let start = self.next_offset; + if start.get() >= len { + return Ok(None); + } + let read = pack + .read_at(start.get(), &mut header) + .map_err(|error| PackError::Pack(format!("pack read: {error}")))?; + let mut reader: &[u8] = &header[..read]; + let entry = match Entry::from_read(&mut reader, start.get(), hash_len) { + Ok(entry) => entry, + Err(error) => { + return if len.saturating_sub(start.get()) < HEADER_SLACK as u64 { + Ok(None) + } else { + Err(PackError::Pack(error.to_string())) + }; + } + }; + if limits.max_object_bytes.exceeded_by(entry.decompressed_size) { + return Err(PackError::LimitExceeded(PackLimit::ObjectBytes)); + } + self.total_decompressed = self + .total_decompressed + .checked_add(entry.decompressed_size) + .ok_or_else(|| malformed("decompressed size overflow"))?; + if limits.max_total_bytes.exceeded_by(self.total_decompressed) { + return Err(PackError::LimitExceeded(PackLimit::TotalBytes)); + } + if let Header::OfsDelta { base_distance } = entry.header { + let base = entry + .checked_base_pack_offset(base_distance) + .ok_or_else(|| malformed("ofs-delta base out of range"))?; + self.base_of.insert(self.next_offset, PackOffset::new(base)); + } + self.current = Some(EntryInflate { + data_offset: PackOffset::new(entry.data_offset), + decompressed_size: entry.decompressed_size, + decompress: Decompress::new(true), + produced: 0, + }); + } + } + } + } +} + +pub struct ReceiveFramer { + limits: PackLimits, + kind: gix::hash::Kind, + pack_start: Option, + pack: Option, +} + +impl ReceiveFramer { + pub fn new(limits: PackLimits, kind: gix::hash::Kind) -> Self { + Self { + limits, + kind, + pack_start: None, + pack: None, + } + } + + pub fn pack_start(&self) -> Option { + self.pack_start + } + + pub fn advance_bytes(&mut self, buf: &[u8]) -> Result, PackError> { + self.advance(buf) + } + + pub fn advance_file( + &mut self, + file: &std::fs::File, + len: u64, + ) -> Result, PackError> { + self.advance(&FileSource { file, len }) + } + + fn advance(&mut self, source: &S) -> Result, PackError> { + let len = source.len(); + let pack_start = match self.pack_start { + Some(start) => start, + None => { + let head = read_head(source, len)?; + match pack_start(&head) { + Some(start) => { + self.pack_start = Some(start); + start + } + None => return Ok(None), + } + } + }; + if self.pack.is_none() { + let pack_len = len.saturating_sub(pack_start as u64); + if pack_len == 0 { + let head = read_head(source, pack_start as u64)?; + return Ok(no_pack_needed(&head).then_some(pack_start)); + } + if pack_len < 12 { + return Ok(None); + } + let mut header = [0u8; 12]; + source + .read_at(pack_start as u64, &mut header) + .map_err(|error| PackError::Pack(format!("pack read: {error}")))?; + if &header[..4] != b"PACK" { + return Err(malformed("packfile is missing its PACK signature")); + } + let num_objects = pack_object_count(&header)?; + if num_objects > self.limits.max_objects { + return Err(PackError::LimitExceeded(PackLimit::Objects)); + } + self.pack = Some(PackProgress { + num_objects, + next_offset: PackOffset::new(pack_start as u64 + 12), + ..PackProgress::default() + }); + } + let kind = self.kind; + self.pack + .as_mut() + .expect("pack progress initialized") + .scan(source, &self.limits, kind) + } +} + +pub fn receive_request_complete( + buf: &[u8], + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result, PackError> { + ReceiveFramer::new(*limits, kind).advance_bytes(buf) +} + +pub fn archive_request_complete(buf: &[u8]) -> Option { + pkt::frames(buf, Some(1)).find_map(|item| match item { + Ok((Frame::Flush, end)) => Some(end), + _ => None, + }) +} + +#[derive(Default)] +pub struct UploadFramer { + scanned: usize, + v2: bool, + flushes: usize, + complete: Option, +} + +impl UploadFramer { + pub fn new() -> Self { + Self::default() + } + + pub fn advance(&mut self, buf: &[u8]) -> Option { + if self.complete.is_some() { + return self.complete; + } + let base = self.scanned; + for item in pkt::frames(&buf[base..], None) { + let Ok((frame, at)) = item else { break }; + let boundary = base + at; + match frame { + Frame::Data(payload) => { + if payload.starts_with(b"command=") { + self.v2 = true; + } + let trimmed = payload + .iter() + .rposition(|byte| !byte.is_ascii_whitespace()) + .map(|end| &payload[..=end]) + .unwrap_or(payload); + if !self.v2 && trimmed == b"done" { + self.complete = Some(boundary); + return self.complete; + } + } + Frame::Flush => { + self.flushes += 1; + if self.v2 { + self.complete = Some(boundary); + return self.complete; + } + } + _ => {} + } + self.scanned = boundary; + } + None + } + + pub fn unanswered_flushes(&self) -> usize { + self.flushes.saturating_sub(1) + } +} + +pub fn upload_v0_nak() -> Vec { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"NAK\n").expect("write to in-memory buffer never fails"); + buf +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v2_fetch_request() -> Vec { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"command=fetch\n").unwrap(); + pkt::write_delim(&mut buf).unwrap(); + pkt::write_data(&mut buf, b"want 1111111111111111111111111111111111111111\n").unwrap(); + pkt::write_data(&mut buf, b"want 2222222222222222222222222222222222222222\n").unwrap(); + pkt::write_data(&mut buf, b"done\n").unwrap(); + pkt::write_flush(&mut buf).unwrap(); + buf + } + + #[test] + fn upload_framer_completes_a_v2_request_at_the_terminating_flush() { + let request = v2_fetch_request(); + assert_eq!(UploadFramer::new().advance(&request), Some(request.len())); + } + + #[test] + fn upload_framer_fed_one_byte_at_a_time_never_overruns_the_buffer() { + let request = v2_fetch_request(); + let mut framer = UploadFramer::new(); + let mut buf = Vec::new(); + let mut completed = None; + for byte in &request { + buf.push(*byte); + if let Some(len) = framer.advance(&buf) { + assert!( + len <= buf.len(), + "advance returned {len} past buffer of {}", + buf.len() + ); + completed = Some(len); + break; + } + } + assert_eq!( + completed, + Some(request.len()), + "incrementally fed request completes exactly once the whole buffer has arrived" + ); + } + + #[test] + fn upload_framer_counts_v0_have_batch_flushes_without_completing() { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"want 1111111111111111111111111111111111111111\n").unwrap(); + pkt::write_flush(&mut buf).unwrap(); + pkt::write_data(&mut buf, b"have 2222222222222222222222222222222222222222\n").unwrap(); + pkt::write_flush(&mut buf).unwrap(); + let mut framer = UploadFramer::new(); + assert_eq!(framer.advance(&buf), None, "v0 request is open until done"); + assert_eq!( + framer.unanswered_flushes(), + 1, + "two flushes seen, one have-batch awaits NAK" + ); + } +} diff --git a/knot2/crates/knot-pack/src/guard.rs b/knot2/crates/knot-pack/src/guard.rs new file mode 100644 index 0000000000000000000000000000000000000000..578dfb12e4e1d92e672a358f3d3ea8f81f6b0496 --- /dev/null +++ b/knot2/crates/knot-pack/src/guard.rs @@ -0,0 +1,62 @@ +use std::sync::Arc; + +use knot_cob::{CobHome, CobStore}; +use knot_git::{RefClass, Repo}; +use knot_messages::{Catalog, ErrorKey}; +use knot_types::{ActorId, RefName}; + +use crate::{ReceiveCommand, ReceiveGuard, RefDecision}; + +pub struct PushGuard { + pub cob_authority: ActorId, + pub home: CobHome, + pub messages: Arc, +} + +impl ReceiveGuard for PushGuard { + fn authorize(&self, staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands + .iter() + .map(|command| self.decide(staged, command)) + .collect() + } +} + +impl PushGuard { + fn decide(&self, staged: &Repo, command: &ReceiveCommand) -> RefDecision { + let Some(name) = command.name() else { + return RefDecision::Reject( + self.messages + .reject + .cob_verification + .line(|ErrorKey::Error| crate::receive::invalid_refname(command.refname())), + ); + }; + match RefClass::of(name) { + RefClass::Public => RefDecision::Allow, + RefClass::Cob if command.is_delete() => { + RefDecision::Reject(self.messages.reject.cob_delete.text()) + } + RefClass::Cob => self.verify_cob(staged, name), + RefClass::Checkpoint => { + RefDecision::Reject(self.messages.reject.checkpoint_reserved.text()) + } + RefClass::Hidden => RefDecision::Reject(self.messages.reject.hidden_reserved.text()), + RefClass::Atproto => RefDecision::Reject(self.messages.reject.atproto_reserved.text()), + RefClass::Tngl => RefDecision::Reject(self.messages.reject.tngl_reserved.text()), + } + } + + fn verify_cob(&self, staged: &Repo, name: &RefName) -> RefDecision { + let store = CobStore::new(staged); + match knot_cobs::verify_cob_ref(&store, &self.home, name, &self.cob_authority) { + Ok(_) => RefDecision::Allow, + Err(error) => RefDecision::Reject( + self.messages + .reject + .cob_verification + .line(|ErrorKey::Error| error.to_string()), + ), + } + } +} diff --git a/knot2/crates/knot-pack/src/ids.rs b/knot2/crates/knot-pack/src/ids.rs new file mode 100644 index 0000000000000000000000000000000000000000..f1c14b6dc5cb563afdde7557369493b4991d2113 --- /dev/null +++ b/knot2/crates/knot-pack/src/ids.rs @@ -0,0 +1,51 @@ +knot_types::scalar_newtype! { + pub(crate) struct Crc32(u32); + pub struct MaxWireBytes(usize); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +pub(crate) struct PackOffset(u64); + +impl PackOffset { + pub(crate) fn new(value: u64) -> Self { + Self(value) + } + + pub(crate) fn get(self) -> u64 { + self.0 + } + + // Hostile packs will ask to go back past the start of a file, + // so just making sure. + pub(crate) fn checked_sub_distance(self, distance: u64) -> Option { + self.0.checked_sub(distance).map(Self) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Rounds(usize); + +impl Rounds { + pub(crate) fn new(rounds: usize) -> Self { + Self(rounds) + } + + // Ensuring chains deeper than the limit return `None` instead of + // wrapping and doing like 18 quintillion more passes. + pub(crate) fn next(self) -> Option { + self.0.checked_sub(1).map(Self) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct RefsDigest([u8; 32]); + +impl RefsDigest { + pub(crate) fn new(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + pub(crate) fn as_bytes(&self) -> &[u8] { + &self.0 + } +} diff --git a/knot2/crates/knot-pack/src/idxwrite.rs b/knot2/crates/knot-pack/src/idxwrite.rs new file mode 100644 index 0000000000000000000000000000000000000000..e4157b6ee4f65ab2e3bb84acf87400127b356980 --- /dev/null +++ b/knot2/crates/knot-pack/src/idxwrite.rs @@ -0,0 +1,218 @@ +use std::io::{self, BufWriter, Write}; +use std::os::unix::fs::FileExt; +use std::sync::Mutex; + +use gix::ObjectId; + +use crate::error::PackError; +use crate::ids::{Crc32, PackOffset}; + +const V2_SIGNATURE: &[u8] = &[0xff, 0x74, 0x4f, 0x63]; +const V2_VERSION: u32 = 2; +const HIGH_BIT: u32 = 0x8000_0000; +const LARGE_OFFSET_THRESHOLD: u64 = 0x7fff_ffff; +const BUCKETS: usize = 256; +const BUCKET_BUF: usize = 64 * 1024; +const CRC_LEN: usize = 4; +const OFFSET_LEN: usize = 8; + +struct Record { + id: ObjectId, + crc32: Crc32, + offset: PackOffset, +} + +type Bucket = Mutex>>; + +pub(crate) struct Spool { + buckets: Vec, + record_len: usize, + hash_len: usize, +} + +impl Spool { + pub(crate) fn new(kind: gix::hash::Kind) -> Self { + let hash_len = kind.len_in_bytes(); + Self { + buckets: (0..BUCKETS).map(|_| Mutex::new(None)).collect(), + record_len: hash_len + CRC_LEN + OFFSET_LEN, + hash_len, + } + } + + pub(crate) fn push(&self, id: ObjectId, crc32: Crc32, offset: PackOffset) -> io::Result<()> { + let mut guard = self.buckets[id.first_byte() as usize] + .lock() + .expect("spool bucket poisoned"); + let writer = match guard.as_mut() { + Some(writer) => writer, + None => guard.insert(BufWriter::with_capacity(BUCKET_BUF, tempfile::tempfile()?)), + }; + writer.write_all(id.as_slice())?; + writer.write_all(&crc32.get().to_be_bytes())?; + writer.write_all(&offset.get().to_be_bytes()) + } + + fn cumulative_fanout(&self) -> Result<[u32; 256], PackError> { + let mut fanout = [0u32; 256]; + self.buckets.iter().enumerate().try_for_each( + |(bucket, cell)| -> Result<(), PackError> { + let mut guard = cell.lock().expect("spool bucket poisoned"); + fanout[bucket] = match guard.as_mut() { + None => 0, + Some(writer) => { + writer.flush()?; + (writer.get_ref().metadata()?.len() as usize / self.record_len) as u32 + } + }; + Ok(()) + }, + )?; + fanout.iter_mut().fold(0u32, |acc, count| { + *count += acc; + *count + }); + Ok(fanout) + } + + fn visit_sorted( + &self, + mut visit: impl FnMut(&Record) -> Result<(), PackError>, + ) -> Result<(), PackError> { + self.buckets.iter().try_for_each(|cell| { + let mut records = self.read_bucket(cell)?; + records.sort_unstable_by_key(|record| record.id); + records.iter().try_for_each(&mut visit) + }) + } + + fn read_bucket(&self, cell: &Bucket) -> Result, PackError> { + let mut guard = cell.lock().expect("spool bucket poisoned"); + let bytes = match guard.as_mut() { + None => Vec::new(), + Some(writer) => { + writer.flush()?; + let file = writer.get_ref(); + let len = file.metadata()?.len() as usize; + let mut bytes = vec![0u8; len]; + file.read_exact_at(&mut bytes, 0)?; + bytes + } + }; + drop(guard); + bytes + .chunks_exact(self.record_len) + .map(|chunk| { + let (id, rest) = chunk.split_at(self.hash_len); + Ok(Record { + id: ObjectId::try_from(id) + .map_err(|error| PackError::Pack(format!("spool record oid: {error}")))?, + crc32: Crc32::new(u32::from_be_bytes( + rest[..CRC_LEN].try_into().expect("crc slice"), + )), + offset: PackOffset::new(u64::from_be_bytes( + rest[CRC_LEN..].try_into().expect("offset slice"), + )), + }) + }) + .collect() + } +} + +fn feed(out: &mut dyn Write, hasher: &mut gix_hash::Hasher, buf: &[u8]) -> io::Result<()> { + hasher.update(buf); + out.write_all(buf) +} + +pub(crate) fn write_v2_index( + out: &mut dyn Write, + records: &Spool, + pack_hash: &ObjectId, + kind: gix::hash::Kind, +) -> Result { + let mut hasher = gix_hash::hasher(kind); + feed(out, &mut hasher, V2_SIGNATURE)?; + feed(out, &mut hasher, &V2_VERSION.to_be_bytes())?; + + records + .cumulative_fanout()? + .iter() + .try_for_each(|count| feed(out, &mut hasher, &count.to_be_bytes()))?; + records.visit_sorted(|record| Ok(feed(out, &mut hasher, record.id.as_slice())?))?; + records + .visit_sorted(|record| Ok(feed(out, &mut hasher, &record.crc32.get().to_be_bytes())?))?; + + let mut large_offsets = Vec::::new(); + records.visit_sorted(|record| { + let encoded = if record.offset.get() > LARGE_OFFSET_THRESHOLD { + let position = large_offsets.len() as u32; + large_offsets.push(record.offset.get()); + position | HIGH_BIT + } else { + record.offset.get() as u32 + }; + Ok(feed(out, &mut hasher, &encoded.to_be_bytes())?) + })?; + large_offsets + .iter() + .try_for_each(|offset| feed(out, &mut hasher, &offset.to_be_bytes()))?; + + feed(out, &mut hasher, pack_hash.as_slice())?; + + let index_hash = hasher + .try_finalize() + .map_err(|error| PackError::Pack(format!("finalize index hash: {error}")))?; + out.write_all(index_hash.as_slice())?; + out.flush()?; + Ok(index_hash) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn oid(seed: u8) -> ObjectId { + let mut raw = [0u8; 20]; + raw[0] = seed; + raw[19] = seed; + ObjectId::try_from(raw.as_slice()).unwrap() + } + + #[test] + fn large_offsets_round_trip_through_the_index_reader() { + let records = [ + (oid(0x02), 0x1111_1111u32, 12u64), + (oid(0x40), 0x2222_2222, LARGE_OFFSET_THRESHOLD), + (oid(0x80), 0x3333_3333, 0x1_2345_6789), + (oid(0xc0), 0x4444_4444, LARGE_OFFSET_THRESHOLD + 1), + ]; + let spool = Spool::new(gix::hash::Kind::Sha1); + records.iter().for_each(|(id, crc32, offset)| { + spool + .push(*id, Crc32::new(*crc32), PackOffset::new(*offset)) + .unwrap() + }); + let pack_hash = oid(0xaa); + + let mut buf = Vec::new(); + let index_hash = + write_v2_index(&mut buf, &spool, &pack_hash, gix::hash::Kind::Sha1).unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pack-under-test.idx"); + std::fs::write(&path, &buf).unwrap(); + let index = gix_pack::index::File::at(&path, gix::hash::Kind::Sha1).unwrap(); + + assert_eq!(index.num_objects(), records.len() as u32); + assert_eq!(index.index_checksum(), index_hash); + assert_eq!(index.pack_checksum(), pack_hash); + records.iter().for_each(|(id, crc32, offset)| { + let at = index + .lookup(*id) + .expect("written oid resolves in the index"); + assert_eq!(index.oid_at_index(at), id.as_ref()); + assert_eq!(index.pack_offset_at_index(at), *offset); + assert_eq!(index.crc32_at_index(at), Some(*crc32)); + }); + } +} diff --git a/knot2/crates/knot-pack/src/lib.rs b/knot2/crates/knot-pack/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..11f6bfbaa33215d918856bc1f64537e3ff77e9e5 --- /dev/null +++ b/knot2/crates/knot-pack/src/lib.rs @@ -0,0 +1,920 @@ +mod archive; +mod cache; +mod error; +mod fetch; +mod frame; +mod guard; +mod ids; +mod idxwrite; +mod meter; +mod objects; +mod pkt; +mod quarantine; +mod receive; +mod receiver; +mod resolve; +mod upload; + +use std::collections::HashMap; +use std::io::{self, Read}; + +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::{DefaultBodyLimit, Path, Query, State}; +use axum::http::{HeaderMap, header}; +use axum::response::Response; +use axum::routing::post; +use knot_git::{Layout, Repo}; +use knot_messages::{Catalog, ErrorKey, FetchMessages}; +use knot_resource::{PackSlots, SlotPermit}; +use knot_runtime::Clock; +use knot_types::{ + AccountDid, ClonePath, Handle, KnotHostname, OwnerDid, OwnerRef, ParseError, RepoDid, +}; +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; + +pub use cache::{CacheConfig, MaxCacheBytes, MaxEntryBytes}; +pub use error::{PackError, PackLimit}; +pub use fetch::{ + FetchError, PackDemux, UpstreamRefs, local_pack, local_pack_to, local_refs, remote_pack, + remote_pack_streamed, remote_refs, +}; +pub use frame::{ + ReceiveFramer, UploadFramer, archive_request_complete, receive_request_complete, upload_v0_nak, +}; +pub use guard::PushGuard; +pub use ids::MaxWireBytes; +pub use knot_storage::{DeltaDepth, HaveOids, MaxObjectBytes, MaxTotalBytes, PackLimits, WantOids}; +pub use objects::{ExpandedPack, count_expanded, write_expanded, write_pack}; +pub use pkt::frame_report; +pub use quarantine::sweep_incoming; +pub use receive::{Preflight, ReceiveCommand, ReceiveGuard, ReceiveOutcome, RefDecision}; +pub use receiver::{PackReceiver, ReceiveReadError, ReceivedPack}; +pub use upload::{SelectionLimits, init_selection_limits, selection_budget}; + +use upload::UploadOutcome; + +pub use knot_messages::default_catalog; + +pub fn default_hostname() -> &'static KnotHostname { + static HOSTNAME: std::sync::LazyLock = std::sync::LazyLock::new(|| { + KnotHostname::new("knot.invalid").expect("static hostname parses") + }); + &HOSTNAME +} + +pub fn advertise_upload(repo: &Repo) -> Result, PackError> { + upload::advertise(repo) +} + +pub fn advertise_upload_v0(repo: &Repo) -> Result, PackError> { + upload::advertise_v0(repo) +} + +pub fn upload_pack(repo: &Repo, request: &[u8]) -> Result, PackError> { + upload::buffered(repo, request, &default_catalog().fetch, default_hostname()) +} + +pub fn upload_pack_streamed( + repo: &Repo, + request: &[u8], + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + upload::streamed(repo, request, messages, knot, sink) +} + +pub fn advertise_receive(repo: &Repo) -> Result, PackError> { + receive::advertise(repo) +} + +pub fn advertise_upload_ssh(repo: &Repo) -> Result, PackError> { + upload::advertise_ssh(repo) +} + +pub fn advertise_upload_v0_ssh(repo: &Repo) -> Result, PackError> { + upload::advertise_v0_ssh(repo) +} + +pub fn advertise_receive_ssh(repo: &Repo) -> Result, PackError> { + receive::advertise_ssh(repo) +} + +#[doc(hidden)] +pub fn receive_pack(repo: &Repo, request: &[u8]) -> Result, PackError> { + receive::handle_bytes(repo, request, &PackLimits::default()) +} + +#[doc(hidden)] +pub fn receive_pack_with_limits( + repo: &Repo, + request: &[u8], + limits: &PackLimits, +) -> Result, PackError> { + receive::handle_bytes(repo, request, limits) +} + +#[doc(hidden)] +pub fn bench_ingest_fresh( + objects_dir: &std::path::Path, + pack_path: &std::path::Path, + kind: gix::hash::Kind, +) -> Result { + let pack = gix_pack::data::File::at(pack_path, kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + objects::ingest_and_close( + objects_dir, + &pack, + &PackLimits::default(), + kind, + knot_resource::ingest_base_budget().map(|budget| budget.get() as usize), + false, + ) + .map(|closure| closure.is_some()) +} + +#[doc(hidden)] +pub fn bench_ingest_external( + objects_dir: &std::path::Path, + pack_path: &std::path::Path, + kind: gix::hash::Kind, +) -> Result, PackError> { + let pack = gix_pack::data::File::at(pack_path, kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + objects::admit_ingest(&pack, kind)?; + objects::ingest_and_close( + objects_dir, + &pack, + &PackLimits::default(), + kind, + knot_resource::ingest_base_budget().map(|budget| budget.get() as usize), + true, + ) + .map(|closure| closure.map(|closure| closure.self_contained)) +} + +#[doc(hidden)] +pub fn bench_admit_and_ingest( + objects_dir: &std::path::Path, + pack_path: &std::path::Path, + kind: gix::hash::Kind, +) -> Result { + bench_ingest_with_base_budget( + objects_dir, + pack_path, + kind, + knot_resource::ingest_base_budget().map(|budget| budget.get() as usize), + ) +} + +#[doc(hidden)] +pub fn bench_ingest_with_base_budget( + objects_dir: &std::path::Path, + pack_path: &std::path::Path, + kind: gix::hash::Kind, + base_budget: Option, +) -> Result { + let pack = gix_pack::data::File::at(pack_path, kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + objects::admit_ingest(&pack, kind)?; + objects::ingest_and_close( + objects_dir, + &pack, + &PackLimits::default(), + kind, + base_budget, + false, + ) + .map(|closure| closure.is_some()) +} + +pub fn receive_pack_guarded( + repo: &Repo, + request: &[u8], + limits: &PackLimits, + guard: &dyn ReceiveGuard, + seal: &dyn Fn(&[knot_git::RefUpdate]), + messages: &knot_messages::RejectMessages, +) -> Result { + receive::handle_guarded_bytes(repo, request, limits, guard, seal, messages) +} + +pub fn receive_pack_guarded_streamed( + repo: &Repo, + received: &ReceivedPack, + limits: &PackLimits, + guard: &dyn ReceiveGuard, + seal: &dyn Fn(&[knot_git::RefUpdate]), + messages: &knot_messages::RejectMessages, +) -> Result { + receive::handle_guarded_streamed(repo, received, limits, guard, seal, messages) +} + +pub fn receive_preflight(request: &[u8]) -> Preflight { + receive::preflight(request) +} + +pub fn upload_archive_streamed( + repo: &Repo, + request: &[u8], + limit: knot_git::ArchiveLimit, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + archive::stream(repo, request, limit, sink) +} + +pub fn upload_archive( + repo: &Repo, + request: &[u8], + limit: knot_git::ArchiveLimit, +) -> Result, PackError> { + let mut buf = Vec::new(); + upload_archive_streamed(repo, request, limit, &mut |chunk| { + buf.extend_from_slice(chunk); + Ok(()) + })?; + Ok(buf) +} + +pub fn meter_pack( + pack: &[u8], + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + meter::meter(pack, limits, kind) +} + +pub fn ingest_pack( + objects_dir: &std::path::Path, + pack: &[u8], + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + objects::index_pack(objects_dir, pack, limits, kind) +} + +pub fn ingest_pack_file( + objects_dir: &std::path::Path, + pack_path: &std::path::Path, + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + objects::ingest_pack_file(objects_dir, pack_path, limits, kind) +} + +#[doc(hidden)] +pub mod fuzz { + pub fn pkt(data: &[u8]) { + let _ = crate::pkt::data_payloads(data); + let _ = crate::pkt::data_payloads_all(data); + let _ = crate::pkt::split_receive(data); + } + + pub fn pack(data: &[u8]) { + let _ = crate::meter::meter(data, &crate::PackLimits::default(), gix::hash::Kind::Sha1); + } + + pub fn receive_commands(data: &[u8]) { + crate::receive::fuzz(data); + } + + pub fn upload_args(data: &[u8]) { + crate::upload::fuzz(data); + } +} + +const MAX_REQUEST_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RepoTarget { + Did(RepoDid), + OwnerPath(OwnerDid, ClonePath), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RepoLookup { + Hosted(RepoDid), + Unhosted, + Unavailable, +} + +impl RepoLookup { + pub fn from_resolved( + resolved: knot_index::Resolved>, + found: impl FnOnce(T) -> RepoDid, + ) -> RepoLookup { + match resolved { + knot_index::Resolved::Ready(Some(value)) => RepoLookup::Hosted(found(value)), + knot_index::Resolved::Ready(None) => RepoLookup::Unhosted, + knot_index::Resolved::Warming => RepoLookup::Unavailable, + } + } +} + +pub trait RepoResolver: Send + Sync + 'static { + fn resolve(&self, target: &RepoTarget) -> RepoLookup; +} + +pub trait HandleResolver: Send + Sync + 'static { + fn resolve( + &self, + handle: Handle, + ) -> std::pin::Pin> + Send + '_>>; +} + +pub use knot_edge::SocketPeer; + +pub trait ReceiveAdvertiser: Send + Sync + 'static { + fn advertise( + &self, + repo: RepoDid, + peer: knot_edge::SocketPeer, + headers: HeaderMap, + ) -> std::pin::Pin + Send + '_>>; +} + +impl RepoResolver for F +where + F: Fn(&RepoTarget) -> RepoLookup + Send + Sync + 'static, +{ + fn resolve(&self, target: &RepoTarget) -> RepoLookup { + self(target) + } +} + +#[derive(Clone)] +struct PackState { + layout: Layout, + resolver: Arc, + receive: Option>, + handle_resolver: Option>, + pack_slots: PackSlots, + cache: Arc, + catalog: Arc, + hostname: KnotHostname, + archive_limit: knot_git::ArchiveLimit, +} + +pub struct EdgeConfig { + pub layout: Layout, + pub resolver: Arc, + pub receive: Option>, + pub handle_resolver: Option>, + pub pack_slots: PackSlots, + pub cache: CacheConfig, + pub catalog: Arc, + pub hostname: KnotHostname, + pub clock: Arc, + pub archive_limit: knot_git::ArchiveLimit, +} + +impl EdgeConfig { + pub fn serving(layout: Layout, resolver: Arc, clock: Arc) -> Self { + Self { + layout, + resolver, + receive: None, + handle_resolver: None, + pack_slots: PackSlots::new(knot_resource::threads().get()), + cache: CacheConfig::default(), + catalog: Arc::new(Catalog::defaults()), + hostname: default_hostname().clone(), + clock, + archive_limit: knot_git::ArchiveLimit::default(), + } + } + + pub fn with_pack_slots(self, pack_slots: PackSlots) -> Self { + Self { pack_slots, ..self } + } +} + +pub fn router(layout: Layout, resolver: Arc, clock: Arc) -> Router { + serving_router(EdgeConfig::serving(layout, resolver, clock)) +} + +pub fn router_with_pack_slots( + layout: Layout, + resolver: Arc, + pack_slots: PackSlots, + clock: Arc, +) -> Router { + serving_router(EdgeConfig::serving(layout, resolver, clock).with_pack_slots(pack_slots)) +} + +fn serving_router(config: EdgeConfig) -> Router { + let state = pack_state(config); + write_routes(state.clone()).merge(advertisement_routes(state).into_router()) +} + +pub fn edge_routes(config: EdgeConfig) -> (Router, knot_edge::ZeroRttRoutes) { + let state = pack_state(config); + (write_routes(state.clone()), advertisement_routes(state)) +} + +fn pack_state(config: EdgeConfig) -> PackState { + let EdgeConfig { + layout, + resolver, + receive, + handle_resolver, + pack_slots, + cache, + catalog, + hostname, + clock, + archive_limit, + } = config; + PackState { + layout, + resolver, + receive, + handle_resolver, + pack_slots, + cache: cache::PackCache::new(cache, clock), + catalog, + hostname, + archive_limit, + } +} + +fn write_routes(state: PackState) -> Router { + Router::new() + .route("/{did}/{name}/git-upload-pack", post(upload_named)) + .route( + "/{did}/{name}/git-upload-archive", + post(upload_archive_named), + ) + .route("/{did}/git-upload-pack", post(upload_did)) + .route("/{did}/git-upload-archive", post(upload_archive_did)) + .layer(DefaultBodyLimit::max(MAX_REQUEST_BYTES)) + .with_state(state) +} + +fn advertisement_routes(state: PackState) -> knot_edge::ZeroRttRoutes { + let named_state = state.clone(); + let did_state = state; + knot_edge::ZeroRttRoutes::new() + .get( + "/{did}/{name}/info/refs", + knot_edge::ZeroRttSafe::new( + move |Path((did, name)): Path<(String, String)>, + Query(query): Query>, + peer: knot_edge::SocketPeer, + headers: HeaderMap| { + let state = named_state.clone(); + async move { + let owner = resolve_owner(&state, &did).await?; + let path = ClonePath::parse(&name).ok_or(PackError::NotFound)?; + let repo_did = resolve_named_did(&state, &owner, &path)?; + read_advertisement( + &state, + repo_did, + query.get("service").map(String::as_str), + peer, + &headers, + ) + .await + } + }, + ), + ) + .get( + "/{did}/info/refs", + knot_edge::ZeroRttSafe::new( + move |Path(did): Path, + Query(query): Query>, + peer: knot_edge::SocketPeer, + headers: HeaderMap| { + let state = did_state.clone(); + async move { + let did = RepoDid::new(did.as_str()).map_err(bad_path)?; + let repo_did = resolve_did_did(&state, &did)?; + read_advertisement( + &state, + repo_did, + query.get("service").map(String::as_str), + peer, + &headers, + ) + .await + } + }, + ), + ) +} + +fn lookup_did(lookup: RepoLookup) -> Result { + match lookup { + RepoLookup::Hosted(did) => Ok(did), + RepoLookup::Unhosted => Err(PackError::NotFound), + RepoLookup::Unavailable => Err(PackError::Unavailable), + } +} + +fn bad_path(error: ParseError) -> PackError { + PackError::BadPath(error.to_string()) +} + +async fn resolve_owner(state: &PackState, owner: &str) -> Result { + match OwnerRef::parse(owner).ok_or(PackError::NotFound)? { + OwnerRef::Did(did) => Ok(did), + OwnerRef::Handle(handle) => { + let resolver = state.handle_resolver.as_ref().ok_or(PackError::NotFound)?; + let did = resolver.resolve(handle).await.ok_or(PackError::NotFound)?; + Ok(did.into()) + } + } +} + +fn resolve_named_did( + state: &PackState, + owner: &OwnerDid, + path: &ClonePath, +) -> Result { + lookup_did( + state + .resolver + .resolve(&RepoTarget::OwnerPath(owner.clone(), path.clone())), + ) +} + +fn resolve_did_did(state: &PackState, did: &RepoDid) -> Result { + lookup_did(state.resolver.resolve(&RepoTarget::Did(did.clone()))) +} + +fn open_named(state: &PackState, owner: &OwnerDid, path: &ClonePath) -> Result { + let did = resolve_named_did(state, owner, path)?; + state.layout.open(&did).map_err(|_| PackError::NotFound) +} + +fn open_did(state: &PackState, did: &RepoDid) -> Result { + let did = resolve_did_did(state, did)?; + state.layout.open(&did).map_err(|_| PackError::NotFound) +} + +async fn read_advertisement( + state: &PackState, + repo_did: RepoDid, + service: Option<&str>, + peer: knot_edge::SocketPeer, + headers: &HeaderMap, +) -> Result { + match service { + Some("git-upload-pack") => { + let repo = state + .layout + .open(&repo_did) + .map_err(|_| PackError::NotFound)?; + let body = if wants_v2(headers) { + advertise_upload(&repo)? + } else { + upload::advertise_v0(&repo)? + }; + Ok(git_response( + "application/x-git-upload-pack-advertisement", + body, + )) + } + Some("git-receive-pack") => match &state.receive { + Some(advertiser) => Ok(advertiser.advertise(repo_did, peer, headers.clone()).await), + None => Err(PackError::PushOverSsh), + }, + _ => Err(PackError::UnsupportedService), + } +} + +fn decode_request(headers: &HeaderMap, body: Bytes) -> Result, PackError> { + let codings: Vec<&str> = headers + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .map(|value| { + value + .split(',') + .map(str::trim) + .filter(|token| !token.is_empty() && !token.eq_ignore_ascii_case("identity")) + .collect() + }) + .unwrap_or_default(); + match codings.as_slice() { + [] => Ok(body.to_vec()), + [token] if token.eq_ignore_ascii_case("gzip") => { + let mut out = Vec::new(); + flate2::read::GzDecoder::new(body.as_ref()) + .take(MAX_REQUEST_BYTES as u64 + 1) + .read_to_end(&mut out) + .map_err(|error| PackError::Protocol(format!("gzip request body: {error}")))?; + match out.len() > MAX_REQUEST_BYTES { + true => Err(PackError::LimitExceeded(PackLimit::TotalBytes)), + false => Ok(out), + } + } + unsupported => Err(PackError::UnsupportedEncoding(unsupported.join(", "))), + } +} + +fn wants_v2(headers: &HeaderMap) -> bool { + headers + .get("git-protocol") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.split(':').any(|token| token.trim() == "version=2")) +} + +fn refs_digest(repo: &Repo) -> Option { + let refs = repo + .advertised_refs_for(knot_git::AdvertScope::Upload) + .ok()?; + let mut hasher = gix::hash::hasher(gix::hash::Kind::Sha256); + refs.iter().for_each(|record| { + hasher.update(record.name.as_str().as_bytes()); + hasher.update(b"\0"); + hasher.update(record.target.to_string().as_bytes()); + hasher.update(b"\n"); + }); + let id = hasher.try_finalize().ok()?; + let mut token = [0u8; 32]; + token.copy_from_slice(id.as_slice()); + Some(ids::RefsDigest::new(token)) +} + +const CACHE_RETRY_BUDGET: usize = 4; + +async fn upload_dispatch( + state: &PackState, + repo: Repo, + body: &[u8], +) -> Result { + upload_dispatch_within(state, repo, body, CACHE_RETRY_BUDGET).await +} + +async fn upload_dispatch_within( + state: &PackState, + repo: Repo, + body: &[u8], + retries: usize, +) -> Result { + let Some(token) = refs_digest(&repo) else { + return dispatch_plan(state, repo, body, None).await; + }; + let key = cache::RequestKey::new(&repo.objects_dir(), &token, body); + match state.cache.decide(key) { + cache::Decision::Serve(bytes) => Ok(cached_response(bytes)), + cache::Decision::Await(receiver) => match cache::wait(receiver).await { + cache::Resolved::Bytes(bytes) => Ok(cached_response(bytes)), + cache::Resolved::Retry => match retries { + 0 => dispatch_plan(state, repo, body, None).await, + _ => Box::pin(upload_dispatch_within(state, repo, body, retries - 1)).await, + }, + cache::Resolved::Regenerate => dispatch_plan(state, repo, body, None).await, + }, + cache::Decision::Lead(lease) => dispatch_plan(state, repo, body, Some(lease)).await, + cache::Decision::Stream | cache::Decision::Off => { + dispatch_plan(state, repo, body, None).await + } + } +} + +async fn dispatch_plan( + state: &PackState, + repo: Repo, + body: &[u8], + lease: Option, +) -> Result { + let permit = state.pack_slots.acquire().await; + let owned_body = body.to_vec(); + let planned = tokio::task::spawn_blocking(move || { + let outcome = upload::plan(&repo, &owned_body); + (repo, outcome) + }) + .await; + let (repo, plan) = match planned { + Ok((repo, Ok(plan))) => (repo, plan), + Ok((_repo, Err(error))) => { + if let Some(lease) = lease { + lease.regenerate(); + } + return Err(error); + } + Err(_join) => { + if let Some(lease) = lease { + lease.regenerate(); + } + return Err(PackError::Pack("upload planning task panicked".to_string())); + } + }; + match plan { + UploadOutcome::Buffered(bytes) => { + drop(permit); + if let Some(lease) = lease { + lease.regenerate(); + } + Ok(git_response("application/x-git-upload-pack-result", bytes)) + } + UploadOutcome::Streaming { + preamble, + wants, + haves, + opts, + } => Ok(stream_response( + repo, + preamble, + wants, + haves, + opts, + permit, + lease, + Arc::clone(&state.catalog), + state.hostname.clone(), + )), + } +} + +fn cached_response(bytes: Bytes) -> Response { + nocache( + Response::builder().header(header::CONTENT_TYPE, "application/x-git-upload-pack-result"), + ) + .body(Body::from(bytes)) + .expect("valid response") +} + +#[allow(clippy::too_many_arguments)] +fn stream_response( + repo: Repo, + preamble: Vec, + wants: WantOids, + haves: HaveOids, + opts: upload::StreamOpts, + permit: SlotPermit, + lease: Option, + catalog: Arc, + hostname: KnotHostname, +) -> Response { + let side_band = opts.side_band; + let limit = lease.as_ref().map(cache::Lease::max_entry_bytes); + let (tx, rx) = mpsc::channel::>(16); + tokio::task::spawn_blocking(move || { + let _permit = permit; + let capture = std::cell::RefCell::new(cache::Capture::new(limit)); + let mut emit = |chunk: &[u8]| -> io::Result<()> { + capture.borrow_mut().record(chunk); + tx.blocking_send(Ok(Bytes::copy_from_slice(chunk))) + .map_err(|_| io::Error::other("client disconnected")) + }; + if emit(&preamble).is_err() { + if let Some(lease) = lease { + lease.retry(); + } + return; + } + let result = upload::stream_pack( + &repo, + &wants, + &haves, + &opts, + &catalog.fetch, + &hostname, + &mut emit, + ); + match result { + Ok(()) => { + if side_band { + let mut flush = Vec::new(); + if pkt::write_flush(&mut flush).is_ok() { + let _ = emit(&flush); + } + } + if let Some(lease) = lease { + match capture.into_inner().into_bytes() { + Some(bytes) => lease.ready(Bytes::from(bytes)), + None => lease.too_large(), + } + } + } + Err(error) if side_band => { + let mut tail = Vec::new(); + let line = catalog + .fetch + .fatal + .line(|ErrorKey::Error| error.to_string().replace('\n', " ")); + let message = format!("{line}\n"); + if pkt::write_band_error(&mut tail, message.as_bytes()).is_ok() { + let _ = pkt::write_flush(&mut tail); + let _ = emit(&tail); + } + if let Some(lease) = lease { + lease.regenerate(); + } + } + Err(error) => { + let _ = tx.blocking_send(Err(io::Error::other(error.to_string()))); + if let Some(lease) = lease { + lease.regenerate(); + } + } + } + }); + nocache( + Response::builder().header(header::CONTENT_TYPE, "application/x-git-upload-pack-result"), + ) + .body(Body::from_stream(ReceiverStream::new(rx))) + .expect("valid response") +} + +async fn upload_named( + State(state): State, + Path((did, name)): Path<(String, String)>, + headers: HeaderMap, + body: Bytes, +) -> Result { + let owner = resolve_owner(&state, &did).await?; + let path = ClonePath::parse(&name).ok_or(PackError::NotFound)?; + let repo = open_named(&state, &owner, &path)?; + upload_dispatch(&state, repo, &decode_request(&headers, body)?).await +} + +async fn upload_did( + State(state): State, + Path(did): Path, + headers: HeaderMap, + body: Bytes, +) -> Result { + let did = RepoDid::new(did.as_str()).map_err(bad_path)?; + let repo = open_did(&state, &did)?; + upload_dispatch(&state, repo, &decode_request(&headers, body)?).await +} + +async fn archive_dispatch( + state: &PackState, + repo: Repo, + body: Vec, +) -> Result { + let permit = state.pack_slots.acquire().await; + Ok(archive_response(repo, body, state.archive_limit, permit)) +} + +fn archive_response( + repo: Repo, + body: Vec, + limit: knot_git::ArchiveLimit, + permit: SlotPermit, +) -> Response { + let (tx, rx) = mpsc::channel::>(16); + tokio::task::spawn_blocking(move || { + let _permit = permit; + let mut sink = |chunk: &[u8]| -> io::Result<()> { + tx.blocking_send(Ok(Bytes::copy_from_slice(chunk))) + .map_err(|_| io::Error::other("client disconnected")) + }; + if let Err(error) = upload_archive_streamed(&repo, &body, limit, &mut sink) { + let _ = tx.blocking_send(Err(io::Error::other(error.to_string()))); + } + }); + nocache(Response::builder().header( + header::CONTENT_TYPE, + "application/x-git-upload-archive-result", + )) + .body(Body::from_stream(ReceiverStream::new(rx))) + .expect("valid response") +} + +async fn upload_archive_named( + State(state): State, + Path((did, name)): Path<(String, String)>, + headers: HeaderMap, + body: Bytes, +) -> Result { + let owner = resolve_owner(&state, &did).await?; + let path = ClonePath::parse(&name).ok_or(PackError::NotFound)?; + let repo = open_named(&state, &owner, &path)?; + archive_dispatch(&state, repo, decode_request(&headers, body)?).await +} + +async fn upload_archive_did( + State(state): State, + Path(did): Path, + headers: HeaderMap, + body: Bytes, +) -> Result { + let did = RepoDid::new(did.as_str()).map_err(bad_path)?; + let repo = open_did(&state, &did)?; + archive_dispatch(&state, repo, decode_request(&headers, body)?).await +} + +fn nocache(builder: axum::http::response::Builder) -> axum::http::response::Builder { + builder + .header(header::EXPIRES, "Fri, 01 Jan 1980 00:00:00 GMT") + .header(header::PRAGMA, "no-cache") + .header( + header::CACHE_CONTROL, + "no-cache, max-age=0, must-revalidate", + ) +} + +fn git_response(content_type: &'static str, body: Vec) -> Response { + nocache(Response::builder().header(header::CONTENT_TYPE, content_type)) + .body(Body::from(body)) + .expect("valid response") +} diff --git a/knot2/crates/knot-pack/src/meter.rs b/knot2/crates/knot-pack/src/meter.rs new file mode 100644 index 0000000000000000000000000000000000000000..2cd15dbc58bcad03b7fbd486020648ae778afe21 --- /dev/null +++ b/knot2/crates/knot-pack/src/meter.rs @@ -0,0 +1,234 @@ +use std::collections::HashMap; +use std::io::{self, Write}; + +use flate2::{Decompress, FlushDecompress, Status}; +use gix_pack::data::input; +use gix_pack::data::{entry::Header, header}; + +use knot_storage::{DeltaDepth, MaxObjectBytes, PackLimits}; +use knot_types::ObjectCount; + +use crate::error::{PackError, PackLimit}; +use crate::ids::PackOffset; + +pub(crate) fn malformed(message: &str) -> PackError { + PackError::Pack(message.to_string()) +} + +pub(crate) fn pack_object_count(pack: &[u8]) -> Result { + let head: [u8; 12] = pack + .get(..12) + .and_then(|slice| slice.try_into().ok()) + .ok_or_else(|| malformed("packfile header is truncated"))?; + header::decode(&head) + .map(|(_version, num_objects)| ObjectCount::from(num_objects)) + .map_err(|error| PackError::Pack(error.to_string())) +} + +pub fn meter(pack: &[u8], limits: &PackLimits, kind: gix::hash::Kind) -> Result<(), PackError> { + if pack.len() < 12 + kind.len_in_bytes() { + return Err(malformed("packfile is truncated")); + } + meter_entries( + io::Cursor::new(pack), + pack_object_count(pack)?, + limits, + kind, + ) + .map(|_| ()) +} + +pub(crate) fn meter_file( + pack: &gix_pack::data::File, + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result { + let reader = io::BufReader::new(std::fs::File::open(pack.path())?); + meter_entries(reader, ObjectCount::from(pack.num_objects()), limits, kind) +} + +fn meter_entries( + reader: R, + num_objects: ObjectCount, + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result { + if num_objects > limits.max_objects { + return Err(PackError::LimitExceeded(PackLimit::Objects)); + } + let mut entries = input::BytesToEntriesIter::new_from_header( + reader, + input::Mode::Verify, + input::EntryDataMode::Keep, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + + let mut total = 0u64; + let mut thin = false; + let mut base_of: HashMap = HashMap::new(); + entries.try_for_each(|entry| -> Result<(), PackError> { + let entry = entry.map_err(|error| PackError::Pack(error.to_string()))?; + if limits.max_object_bytes.exceeded_by(entry.decompressed_size) { + return Err(PackError::LimitExceeded(PackLimit::ObjectBytes)); + } + total = total + .checked_add(entry.decompressed_size) + .ok_or_else(|| malformed("decompressed size overflow"))?; + if limits.max_total_bytes.exceeded_by(total) { + return Err(PackError::LimitExceeded(PackLimit::TotalBytes)); + } + match entry.header { + Header::OfsDelta { base_distance } => { + let pack_offset = PackOffset::new(entry.pack_offset); + let base = pack_offset + .checked_sub_distance(base_distance) + .ok_or_else(|| malformed("ofs-delta base out of range"))?; + base_of.insert(pack_offset, base); + check_delta_result(&entry, limits.max_object_bytes)?; + } + Header::RefDelta { .. } => { + thin = true; + check_delta_result(&entry, limits.max_object_bytes)?; + } + _ => {} + } + Ok(()) + })?; + + check_depth(&base_of, limits.max_delta_depth)?; + Ok(thin) +} + +fn check_delta_result( + entry: &input::Entry, + max_object_bytes: MaxObjectBytes, +) -> Result<(), PackError> { + let compressed = entry + .compressed + .as_deref() + .ok_or_else(|| malformed("delta entry missing compressed data"))?; + let mut peek = HeaderPeek::new(); + inflate_into(compressed, entry.decompressed_size, &mut peek)?; + if max_object_bytes.exceeded_by(delta_result_size(peek.filled())?) { + return Err(PackError::LimitExceeded(PackLimit::ObjectBytes)); + } + Ok(()) +} + +pub(crate) fn inflate_into( + input: &[u8], + expected: u64, + out: &mut dyn Write, +) -> Result { + let mut decompress = Decompress::new(true); + let mut scratch = [0u8; 8192]; + let mut produced = 0u64; + loop { + let consumed = decompress.total_in() as usize; + let out_before = decompress.total_out(); + let status = decompress + .decompress( + input.get(consumed..).unwrap_or_default(), + &mut scratch, + FlushDecompress::None, + ) + .map_err(|error| PackError::Pack(format!("inflate: {error}")))?; + let written = (decompress.total_out() - out_before) as usize; + produced += written as u64; + if produced > expected { + return Err(malformed("object inflates beyond its declared size")); + } + out.write_all(&scratch[..written]) + .map_err(|error| PackError::Pack(format!("inflate sink: {error}")))?; + match status { + Status::StreamEnd => break, + Status::Ok | Status::BufError => { + if decompress.total_in() as usize == consumed + && decompress.total_out() == out_before + { + return Err(malformed("inflate stalled or pack truncated")); + } + } + } + } + if produced != expected { + return Err(malformed("object decompressed size mismatch")); + } + Ok(decompress.total_in()) +} + +struct HeaderPeek { + bytes: [u8; 32], + len: usize, +} + +impl HeaderPeek { + fn new() -> Self { + Self { + bytes: [0u8; 32], + len: 0, + } + } + + fn filled(&self) -> &[u8] { + &self.bytes[..self.len] + } +} + +impl Write for HeaderPeek { + fn write(&mut self, data: &[u8]) -> io::Result { + let take = (self.bytes.len() - self.len).min(data.len()); + self.bytes[self.len..self.len + take].copy_from_slice(&data[..take]); + self.len += take; + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn read_delta_varint( + data: &[u8], + pos: usize, + shift: u32, + acc: u64, +) -> Result<(u64, usize), PackError> { + if shift >= u64::BITS { + return Err(malformed("delta size header overflows")); + } + let byte = *data + .get(pos) + .ok_or_else(|| malformed("delta size header truncated"))?; + let acc = acc | (u64::from(byte & 0x7f) << shift); + if byte & 0x80 == 0 { + Ok((acc, pos + 1)) + } else { + read_delta_varint(data, pos + 1, shift + 7, acc) + } +} + +fn delta_result_size(header: &[u8]) -> Result { + let (_base_size, after_base) = read_delta_varint(header, 0, 0, 0)?; + let (result_size, _) = read_delta_varint(header, after_base, 0, 0)?; + Ok(result_size) +} + +pub(crate) fn check_depth( + base_of: &HashMap, + max: DeltaDepth, +) -> Result<(), PackError> { + base_of.keys().try_for_each(|start| { + let mut depth = DeltaDepth::ZERO; + let mut cursor = *start; + while let Some(&base) = base_of.get(&cursor) { + depth = depth.deeper(); + if depth.exceeds(max) { + return Err(PackError::LimitExceeded(PackLimit::DeltaDepth)); + } + cursor = base; + } + Ok(()) + }) +} diff --git a/knot2/crates/knot-pack/src/objects.rs b/knot2/crates/knot-pack/src/objects.rs new file mode 100644 index 0000000000000000000000000000000000000000..b2f348a0a5ef76c1a6f42a84d2a6944f91e87063 --- /dev/null +++ b/knot2/crates/knot-pack/src/objects.rs @@ -0,0 +1,1032 @@ +use std::collections::{HashMap, HashSet}; +use std::error::Error; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use gix::ObjectId; +use gix::prelude::FindExt; +use gix::progress::Discard; +use gix_pack::data::{Version, output}; +use knot_types::{ObjectCount, Oid}; + +use crate::error::{PackError, PackLimit}; +use crate::ids::{Crc32, PackOffset}; +use knot_storage::{MaxObjectBytes, PackLimits}; + +use crate::meter::{self, check_depth}; +use crate::resolve; + +type OidStream = Box>> + Send>; + +fn odb_at(objects_dir: &Path, kind: gix::hash::Kind) -> Result { + gix::odb::at_opts( + objects_dir, + std::iter::empty(), + gix::odb::store::init::Options { + object_hash: kind, + ..Default::default() + }, + ) + .map_err(|error| PackError::Pack(error.to_string())) +} + +pub fn write_pack( + objects_dir: &Path, + oids: Vec, + thin_bases: Option<&HashSet>, + out: &mut dyn Write, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + let mut odb = odb_at(objects_dir, kind)?; + odb.prevent_pack_unload(); + odb.refresh_never(); + + let interrupt = AtomicBool::new(false); + let permitted_bases: Option> = + thin_bases.map(|bases| bases.iter().map(|oid| oid.object_id()).collect()); + let oids: OidStream = Box::new(oids.into_iter().map(|oid| Ok(oid.object_id()))); + + let (counts, _) = output::count::objects( + odb.clone(), + oids, + &Discard, + &interrupt, + output::count::objects::Options { + thread_limit: knot_resource::gix_thread_limit().map(knot_resource::ThreadCount::get), + chunk_size: 50, + input_object_expansion: output::count::objects::ObjectExpansion::AsIs, + }, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + + write_counts(counts, odb, permitted_bases, out, kind) +} + +pub struct ExpandedPack { + counts: Vec, + odb: gix::odb::Handle, + kind: gix::hash::Kind, +} + +impl ExpandedPack { + pub fn len(&self) -> usize { + self.counts.len() + } + + pub fn is_empty(&self) -> bool { + self.counts.is_empty() + } +} + +pub fn count_expanded( + objects_dir: &Path, + roots: Vec, + max_objects: ObjectCount, + stall: Duration, + kind: gix::hash::Kind, +) -> Result { + let mut odb = odb_at(objects_dir, kind)?; + odb.prevent_pack_unload(); + odb.refresh_never(); + + let interrupt = Arc::new(AtomicBool::new(false)); + let counter = Arc::new(AtomicUsize::new(0)); + let progress = SharedCount(Arc::clone(&counter)); + let oids: OidStream = Box::new(roots.into_iter().map(|oid| Ok(oid.object_id()))); + let result = { + let _watchdog = Watchdog::arm(interrupt.clone(), Arc::clone(&counter), max_objects, stall); + output::count::objects( + Interruptible { + inner: odb.clone(), + flag: Arc::clone(&interrupt), + }, + oids, + &progress, + &interrupt, + output::count::objects::Options { + thread_limit: knot_resource::gix_thread_limit() + .map(knot_resource::ThreadCount::get), + chunk_size: 50, + input_object_expansion: output::count::objects::ObjectExpansion::TreeContents, + }, + ) + }; + let over_limit = counter.load(Ordering::Relaxed) > max_objects.get(); + let timed_out = interrupt.load(Ordering::Relaxed); + let counts = match result { + Ok((counts, _)) => counts, + Err(_) if over_limit => return Err(PackError::SelectionTooLarge), + Err(_) if timed_out => return Err(PackError::SelectionTimeout), + Err(error) => return Err(PackError::Pack(error.to_string())), + }; + if counts.len() > max_objects.get() { + return Err(PackError::SelectionTooLarge); + } + if timed_out { + return Err(PackError::SelectionTimeout); + } + Ok(ExpandedPack { counts, odb, kind }) +} + +pub fn write_expanded(pack: ExpandedPack, out: &mut dyn Write) -> Result<(), PackError> { + write_counts(pack.counts, pack.odb, None, out, pack.kind) +} + +fn write_counts( + counts: Vec, + odb: gix::odb::Handle, + permitted_bases: Option>, + out: &mut dyn Write, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + let num_entries = counts.len() as u32; + let allow_thin_pack = permitted_bases.is_some(); + let counted = output::entry::iter_from_counts( + counts, + odb.clone(), + Box::new(Discard), + output::entry::iter_from_counts::Options { + thread_limit: knot_resource::gix_thread_limit().map(knot_resource::ThreadCount::get), + mode: output::entry::iter_from_counts::Mode::PackCopyAndBaseObjects, + allow_thin_pack, + chunk_size: 50, + version: Version::V2, + }, + ); + + let entries = gix::parallel::InOrderIter::from(counted).map( + move |chunk| -> Result, PackError> { + let entries = chunk.map_err(|error| PackError::Pack(error.to_string()))?; + entries + .into_iter() + .map(|entry| restrict_thin_base(&odb, permitted_bases.as_ref(), entry)) + .collect() + }, + ); + + let mut writer = + output::bytes::FromEntriesIter::new(entries, out, num_entries, Version::V2, kind); + writer + .try_fold((), |(), written| written.map(|_| ())) + .map_err(|error| PackError::Pack(error.to_string()))?; + Ok(()) +} + +struct SharedCount(Arc); + +impl gix::progress::Count for SharedCount { + fn set(&self, step: usize) { + self.0.store(step, Ordering::Relaxed); + } + + fn step(&self) -> usize { + self.0.load(Ordering::Relaxed) + } + + fn inc_by(&self, step: usize) { + self.0.fetch_add(step, Ordering::Relaxed); + } + + fn counter(&self) -> Arc { + Arc::clone(&self.0) + } +} + +#[derive(Debug)] +struct Halted; + +impl std::fmt::Display for Halted { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("object enumeration halted by selection budget") + } +} + +impl Error for Halted {} + +#[derive(Clone)] +struct Interruptible { + inner: gix::odb::Handle, + flag: Arc, +} + +impl gix_pack::Find for Interruptible { + fn contains(&self, id: &gix::hash::oid) -> bool { + self.inner.contains(id) + } + + fn try_find_cached<'a>( + &self, + id: &gix::hash::oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn gix_pack::cache::DecodeEntry, + ) -> Result< + Option<(gix::objs::Data<'a>, Option)>, + gix::objs::find::Error, + > { + if self.flag.load(Ordering::Relaxed) { + return Err(Box::new(Halted)); + } + self.inner.try_find_cached(id, buffer, pack_cache) + } + + fn location_by_oid( + &self, + id: &gix::hash::oid, + buf: &mut Vec, + ) -> Option { + self.inner.location_by_oid(id, buf) + } + + fn pack_offsets_and_oid( + &self, + pack_id: u32, + ) -> Option> { + self.inner.pack_offsets_and_oid(pack_id) + } + + fn entry_by_location( + &self, + location: &gix_pack::data::entry::Location, + ) -> Option { + self.inner.entry_by_location(location) + } +} + +struct Watchdog { + idle: Option>, + watch: Option>, +} + +impl Watchdog { + fn arm( + flag: Arc, + counter: Arc, + max_objects: ObjectCount, + stall: Duration, + ) -> Self { + let (idle, wake) = mpsc::channel::<()>(); + let watch = std::thread::spawn(move || { + let poll = Duration::from_millis(25).min(stall); + let mut last = counter.load(Ordering::Relaxed); + let mut since = Instant::now(); + loop { + let now = counter.load(Ordering::Relaxed); + if now != last { + last = now; + since = Instant::now(); + } + if since.elapsed() >= stall || now > max_objects.get() { + flag.store(true, Ordering::Relaxed); + return; + } + if !matches!( + wake.recv_timeout(poll), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + return; + } + } + }); + Self { + idle: Some(idle), + watch: Some(watch), + } + } +} + +impl Drop for Watchdog { + fn drop(&mut self) { + self.idle.take(); + if let Some(watch) = self.watch.take() { + let _ = watch.join(); + } + } +} + +fn restrict_thin_base( + odb: &gix::odb::Handle, + permitted: Option<&HashSet>, + entry: output::Entry, +) -> Result { + let base = match &entry.kind { + output::entry::Kind::DeltaOid { id } => *id, + _ => return Ok(entry), + }; + if permitted.is_some_and(|bases| bases.contains(&base)) { + return Ok(entry); + } + let mut buf = Vec::new(); + let object = odb + .find(&entry.id, &mut buf) + .map_err(|error| PackError::Pack(error.to_string()))?; + let count = output::Count::from_data(entry.id, None); + output::Entry::from_data(&count, &object).map_err(|error| PackError::Pack(error.to_string())) +} + +pub(crate) fn open_pack_file( + path: &Path, + kind: gix::hash::Kind, +) -> Result { + let mut signature = [0u8; 4]; + let signature_len = std::fs::File::open(path)?.read(&mut signature).unwrap_or(0); + match (signature_len, signature.as_slice()) { + (0, _) => Err(PackError::Pack("packfile is empty".to_string())), + (_, b"PACK") => { + gix_pack::data::File::at(path, kind).map_err(|error| PackError::Pack(error.to_string())) + } + _ => Err(PackError::Pack( + "packfile is missing its PACK signature".to_string(), + )), + } +} + +pub fn index_pack( + objects_dir: &Path, + pack: &[u8], + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + if pack.is_empty() { + return Ok(()); + } + std::fs::create_dir_all(objects_dir)?; + let mut tmp = tempfile::NamedTempFile::new_in(objects_dir)?; + tmp.write_all(pack)?; + tmp.flush()?; + let file = open_pack_file(tmp.path(), kind)?; + index_pack_bounded(objects_dir, &file, limits, kind) +} + +pub fn ingest_pack_file( + objects_dir: &Path, + pack_path: &Path, + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + let file = open_pack_file(pack_path, kind)?; + index_pack_bounded(objects_dir, &file, limits, kind) +} + +pub(crate) fn index_pack_bounded( + objects_dir: &Path, + pack: &gix_pack::data::File, + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + if pack.data_len() < 12 + kind.len_in_bytes() { + return Ok(()); + } + let thin = meter::meter_file(pack, limits, kind)?; + + let pack_dir = objects_dir.join("pack"); + std::fs::create_dir_all(&pack_dir)?; + + inline_and_index(objects_dir, pack, &pack_dir, limits, kind, thin).or_else(|_| { + let bytes = std::fs::read(pack.path())?; + resolve::resolve(objects_dir, &bytes, limits, kind) + }) +} + +fn inline_and_index( + objects_dir: &Path, + pack: &gix_pack::data::File, + pack_dir: &Path, + limits: &PackLimits, + kind: gix::hash::Kind, + thin: bool, +) -> Result<(), PackError> { + let (pack_path, pack_hash) = if thin { + let pack_hash = inline_thin_bases(objects_dir, pack, pack_dir, kind)?; + ( + pack_dir.join(format!("pack-{}.pack", pack_hash.to_hex())), + pack_hash, + ) + } else { + let pack_hash = pack.checksum(); + let pack_path = pack_dir.join(format!("pack-{}.pack", pack_hash.to_hex())); + let source = pack.path().to_owned(); + persist_atomic(&pack_path, |writer| { + std::io::copy(&mut std::fs::File::open(&source)?, writer)?; + Ok(()) + })?; + (pack_path, pack_hash) + }; + let pack_cleanup = RemoveOnDrop::arm(&pack_path); + + let nodes = scan_offsets(&pack_path, kind)?; + let spool = spool_pack( + &pack_path, + nodes, + kind, + knot_resource::ingest_base_budget().map(|budget| budget.get() as usize), + limits.max_object_bytes, + None, + )?; + let (_present, idx_cleanup) = persist_index(pack_dir, &pack_hash, spool, kind)?; + pack_cleanup.disarm(); + idx_cleanup.disarm(); + Ok(()) +} + +fn inline_thin_bases( + objects_dir: &Path, + pack: &gix_pack::data::File, + pack_dir: &Path, + kind: gix::hash::Kind, +) -> Result { + let odb = odb_at(objects_dir, kind)?; + let staged = tempfile::NamedTempFile::new_in(pack_dir)?; + let writer = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(staged.path())?; + let reader = std::io::BufReader::new(std::fs::File::open(pack.path())?); + let entries = gix_pack::data::input::BytesToEntriesIter::new_from_header( + reader, + gix_pack::data::input::Mode::Verify, + gix_pack::data::input::EntryDataMode::KeepAndCrc32, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let version = entries.version(); + let lookup = gix_pack::data::input::LookupRefDeltaObjectsIter::new(entries, odb); + let mut sink = gix_pack::data::input::EntriesToBytesIter::new(lookup, writer, version, kind); + sink.try_for_each(|entry| { + entry + .map(|_| ()) + .map_err(|error| PackError::Pack(error.to_string())) + })?; + let pack_hash = sink + .digest() + .ok_or_else(|| PackError::Pack("resolved pack has no trailer".to_string()))?; + drop(sink); + + let pack_path = pack_dir.join(format!("pack-{}.pack", pack_hash.to_hex())); + staged + .persist(&pack_path) + .map_err(|error| PackError::Pack(error.to_string()))?; + Ok(pack_hash) +} + +fn scan_offsets(pack_path: &Path, kind: gix::hash::Kind) -> Result, PackError> { + let reader = std::io::BufReader::new(std::fs::File::open(pack_path)?); + let mut entries = gix_pack::data::input::BytesToEntriesIter::new_from_header( + reader, + gix_pack::data::input::Mode::Verify, + gix_pack::data::input::EntryDataMode::Crc32, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let mut nodes: Vec = Vec::new(); + entries.try_for_each(|entry| -> Result<(), PackError> { + let entry = entry.map_err(|error| PackError::Pack(error.to_string()))?; + nodes.push(Node { + offset: PackOffset::new(entry.pack_offset), + crc32: Crc32::new( + entry + .crc32 + .ok_or_else(|| PackError::Pack("entry crc32 not computed".to_string()))?, + ), + }); + Ok(()) + })?; + Ok(nodes) +} + +pub(crate) struct PresentSet(gix_pack::index::File); + +impl PresentSet { + fn open(idx_path: &Path, kind: gix::hash::Kind) -> Result { + gix_pack::index::File::at(idx_path, kind) + .map(Self) + .map_err(|error| PackError::Pack(error.to_string())) + } + + pub(crate) fn contains(&self, oid: &Oid) -> bool { + self.0.lookup(oid.object_id()).is_some() + } + + fn sorted_offsets(&self) -> Vec { + self.0 + .sorted_offsets() + .into_iter() + .map(PackOffset::new) + .collect() + } +} + +pub(crate) struct FreshClosure { + pub self_contained: bool, + pub present: PresentSet, +} + +#[derive(Clone)] +struct Node { + offset: PackOffset, + crc32: Crc32, +} + +const PRESENT: u8 = 1; +const REFERENCED: u8 = 2; + +struct Connectivity { + objects: scc::HashMap, + empty_tree: Oid, +} + +impl Connectivity { + fn mark_present(&self, oid: Oid) { + self.objects + .entry_sync(oid) + .and_modify(|flags| *flags |= PRESENT) + .or_insert(PRESENT); + } + + fn check(&self, reference: Oid) { + if reference != self.empty_tree { + self.objects + .entry_sync(reference) + .and_modify(|flags| *flags |= REFERENCED) + .or_insert(REFERENCED); + } + } + + fn self_contained(&self) -> bool { + self.objects + .any_sync(|_, flags| *flags & REFERENCED != 0 && *flags & PRESENT == 0) + .is_none() + } +} + +enum Scan { + Thin, + Limit(PackLimit), + Pack(String), +} + +const INGEST_BYTES_PER_OBJECT: u64 = 128; + +fn fits_in_memory(num_objects: ObjectCount) -> bool { + knot_resource::ingest_admits(knot_resource::PayloadBytes::new( + (num_objects.get() as u64).saturating_mul(INGEST_BYTES_PER_OBJECT), + )) +} + +pub(crate) fn admit_ingest( + pack: &gix_pack::data::File, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + if pack.data_len() < 12 + kind.len_in_bytes() { + return Ok(()); + } + if fits_in_memory(ObjectCount::from(pack.num_objects())) { + Ok(()) + } else { + Err(PackError::InsufficientMemory) + } +} + +struct RemoveOnDrop(Option); + +impl RemoveOnDrop { + fn arm(path: &Path) -> Self { + Self(Some(path.to_owned())) + } + + fn disarm(mut self) { + self.0 = None; + } +} + +impl Drop for RemoveOnDrop { + fn drop(&mut self) { + if let Some(path) = self.0.take() { + let _ = std::fs::remove_file(path); + } + } +} + +pub(crate) fn ingest_and_close( + objects_dir: &Path, + pack: &gix_pack::data::File, + limits: &PackLimits, + kind: gix::hash::Kind, + base_budget: Option, + force_external: bool, +) -> Result, PackError> { + let hash_len = kind.len_in_bytes(); + if pack.data_len() < 12 + hash_len { + return Ok(None); + } + if ObjectCount::from(pack.num_objects()) > limits.max_objects { + return Err(PackError::LimitExceeded(PackLimit::Objects)); + } + + let reader = std::io::BufReader::new(std::fs::File::open(pack.path())?); + let mut entries = gix_pack::data::input::BytesToEntriesIter::new_from_header( + reader, + gix_pack::data::input::Mode::Verify, + gix_pack::data::input::EntryDataMode::Crc32, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let mut nodes: Vec = Vec::with_capacity(pack.num_objects() as usize); + let mut base_of: HashMap = HashMap::new(); + let mut total_decompressed: u64 = 0; + let mut max_object: u64 = 0; + let scan = entries.try_for_each(|entry| -> Result<(), Scan> { + let entry = entry.map_err(|error| Scan::Pack(error.to_string()))?; + match entry.header { + gix_pack::data::entry::Header::RefDelta { .. } => return Err(Scan::Thin), + gix_pack::data::entry::Header::OfsDelta { base_distance } => { + let pack_offset = PackOffset::new(entry.pack_offset); + let base = pack_offset + .checked_sub_distance(base_distance) + .ok_or_else(|| Scan::Pack("ofs-delta base out of range".to_string()))?; + base_of.insert(pack_offset, base); + } + _ => {} + } + if limits.max_object_bytes.exceeded_by(entry.decompressed_size) { + return Err(Scan::Limit(PackLimit::ObjectBytes)); + } + max_object = max_object.max(entry.decompressed_size); + total_decompressed = total_decompressed + .checked_add(entry.decompressed_size) + .ok_or_else(|| Scan::Pack("decompressed size overflow".to_string()))?; + if limits.max_total_bytes.exceeded_by(total_decompressed) { + return Err(Scan::Limit(PackLimit::TotalBytes)); + } + nodes.push(Node { + offset: PackOffset::new(entry.pack_offset), + crc32: Crc32::new( + entry + .crc32 + .ok_or_else(|| Scan::Pack("entry crc32 not computed".to_string()))?, + ), + }); + Ok(()) + }); + match scan { + Ok(()) => {} + Err(Scan::Thin) => return Ok(None), + Err(Scan::Limit(limit)) => return Err(PackError::LimitExceeded(limit)), + Err(Scan::Pack(message)) => return Err(PackError::Pack(message)), + } + check_depth(&base_of, limits.max_delta_depth)?; + drop(base_of); + let entry_count = nodes.len(); + + let base_cache = knot_resource::ingest_base_budget() + .unwrap_or(knot_resource::CacheBytes::new(0)) + .get(); + let working_set = (entry_count as u64) + .saturating_mul(INGEST_BYTES_PER_OBJECT) + .saturating_add(base_cache) + .saturating_add(max_object); + if !knot_resource::ingest_admits_churn( + knot_resource::WorkingSetBytes::new(working_set), + knot_resource::ChurnBytes::new(total_decompressed), + ) { + return Err(PackError::InsufficientMemory); + } + + let pack_hash = pack.checksum(); + let pack_dir = objects_dir.join("pack"); + std::fs::create_dir_all(&pack_dir)?; + let stem = format!("pack-{}", pack_hash.to_hex()); + let pack_path = pack_dir.join(format!("{stem}.pack")); + let source = pack.path().to_owned(); + persist_atomic(&pack_path, |writer| { + let mut reader = std::fs::File::open(&source)?; + std::io::copy(&mut reader, writer)?; + Ok(()) + })?; + let pack_cleanup = RemoveOnDrop::arm(&pack_path); + + let externalize = force_external + || knot_resource::externalize_connectivity(knot_resource::ConnectivityObjects::new( + entry_count as u64, + )); + let connectivity = (!externalize).then(|| Connectivity { + objects: scc::HashMap::with_capacity(entry_count), + empty_tree: Oid::from(ObjectId::empty_tree(kind)), + }); + let spool = spool_pack( + &pack_path, + nodes, + kind, + base_budget, + limits.max_object_bytes, + connectivity.as_ref(), + )?; + let self_contained_in_ram = connectivity.as_ref().map(Connectivity::self_contained); + drop(connectivity); + let (present, idx_cleanup) = persist_index(&pack_dir, &pack_hash, spool, kind)?; + + let self_contained = match self_contained_in_ram { + Some(value) => value, + None => verify_connectivity_retraverse( + &pack_path, + &present, + kind, + base_budget, + limits.max_object_bytes, + )?, + }; + + pack_cleanup.disarm(); + idx_cleanup.disarm(); + Ok(Some(FreshClosure { + self_contained, + present, + })) +} + +fn spool_pack( + pack_path: &Path, + nodes: Vec, + kind: gix::hash::Kind, + base_budget: Option, + max_object_bytes: MaxObjectBytes, + connectivity: Option<&Connectivity>, +) -> Result { + let interrupt = AtomicBool::new(false); + let tree = gix_pack::cache::delta::Tree::from_offsets_in_pack( + pack_path, + nodes.into_iter(), + &|node: &Node| node.offset.get(), + &|_id| None, + &mut Discard, + &interrupt, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let stored = gix_pack::data::File::at(pack_path, kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + let spool = crate::idxwrite::Spool::new(kind); + run_ingest_traverse( + tree, + &stored, + &interrupt, + base_budget, + max_object_bytes, + kind, + |node: &mut Node, _progress, context| harvest(node, context, kind, connectivity, &spool), + )?; + Ok(spool) +} + +fn persist_index( + pack_dir: &Path, + pack_hash: &ObjectId, + spool: crate::idxwrite::Spool, + kind: gix::hash::Kind, +) -> Result<(PresentSet, RemoveOnDrop), PackError> { + let idx_path = pack_dir.join(format!("pack-{}.idx", pack_hash.to_hex())); + persist_atomic(&idx_path, |writer| { + crate::idxwrite::write_v2_index(writer, &spool, pack_hash, kind).map(|_| ()) + })?; + let idx_cleanup = RemoveOnDrop::arm(&idx_path); + drop(spool); + let present = PresentSet::open(&idx_path, kind)?; + Ok((present, idx_cleanup)) +} + +// he wishes he was on the farm already +fn harvest( + node: &Node, + context: gix_pack::cache::delta::traverse::Context<'_>, + kind: gix::hash::Kind, + connectivity: Option<&Connectivity>, + spool: &crate::idxwrite::Spool, +) -> Result<(), PackError> { + let id = gix::objs::compute_hash(kind, context.object_kind, context.decompressed) + .map_err(|error| PackError::Pack(error.to_string()))?; + spool.push(id, node.crc32, node.offset)?; + if let Some(connectivity) = connectivity { + connectivity.mark_present(Oid::from(id)); + parse_references( + context.object_kind, + context.decompressed, + kind, + &mut |reference| connectivity.check(reference), + )?; + } + Ok(()) +} + +fn parse_references( + object_kind: gix::object::Kind, + bytes: &[u8], + kind: gix::hash::Kind, + check: &mut dyn FnMut(Oid), +) -> Result<(), PackError> { + match object_kind { + gix::object::Kind::Blob => Ok(()), + gix::object::Kind::Tree => harvest_tree(bytes, kind, check), + gix::object::Kind::Commit => harvest_commit(bytes, kind, check), + gix::object::Kind::Tag => harvest_tag(bytes, kind, check), + } +} + +fn harvest_tree( + bytes: &[u8], + kind: gix::hash::Kind, + check: &mut dyn FnMut(Oid), +) -> Result<(), PackError> { + gix::objs::TreeRefIter::from_bytes(bytes, kind).try_for_each(|entry| { + let entry = entry.map_err(|error| PackError::Pack(error.to_string()))?; + if !entry.mode.is_commit() { + check(Oid::from(entry.oid.to_owned())); + } + Ok(()) + }) +} + +fn harvest_commit( + bytes: &[u8], + kind: gix::hash::Kind, + check: &mut dyn FnMut(Oid), +) -> Result<(), PackError> { + let mut iter = gix::objs::CommitRefIter::from_bytes(bytes, kind); + let tree = iter + .tree_id() + .map_err(|error| PackError::Pack(error.to_string()))?; + std::iter::once(tree) + .chain(iter.parent_ids()) + .for_each(|oid| check(Oid::from(oid))); + Ok(()) +} + +fn harvest_tag( + bytes: &[u8], + kind: gix::hash::Kind, + check: &mut dyn FnMut(Oid), +) -> Result<(), PackError> { + let target = gix::objs::TagRefIter::from_bytes(bytes, kind) + .target_id() + .map_err(|error| PackError::Pack(error.to_string()))?; + check(Oid::from(target)); + Ok(()) +} + +fn verify_connectivity_retraverse( + pack_path: &Path, + present: &PresentSet, + kind: gix::hash::Kind, + base_budget: Option, + max_object_bytes: MaxObjectBytes, +) -> Result { + let interrupt = AtomicBool::new(false); + let tree = gix_pack::cache::delta::Tree::from_offsets_in_pack( + pack_path, + present.sorted_offsets().into_iter(), + &|offset: &PackOffset| offset.get(), + &|_id| None, + &mut Discard, + &interrupt, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let stored = gix_pack::data::File::at(pack_path, kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + let empty_tree = Oid::from(ObjectId::empty_tree(kind)); + let missing = AtomicBool::new(false); + run_ingest_traverse( + tree, + &stored, + &interrupt, + base_budget, + max_object_bytes, + kind, + |_offset: &mut PackOffset, _progress, context| { + parse_references( + context.object_kind, + context.decompressed, + kind, + &mut |reference| { + if reference != empty_tree && !present.contains(&reference) { + missing.store(true, Ordering::Relaxed); + } + }, + ) + }, + )?; + Ok(!missing.load(Ordering::Relaxed)) +} + +fn run_ingest_traverse( + tree: gix_pack::cache::delta::Tree, + stored: &gix_pack::data::File, + interrupt: &AtomicBool, + base_budget: Option, + max_object_bytes: MaxObjectBytes, + kind: gix::hash::Kind, + harvest: H, +) -> Result<(), PackError> +where + T: Send + Sync + Clone, + H: FnMut( + &mut T, + &dyn gix::progress::Progress, + gix_pack::cache::delta::traverse::Context<'_>, + ) -> Result<(), PackError> + + Send + + Clone, +{ + let base_spill = match base_budget { + Some(budget) => Some(Arc::new(gix_pack::cache::delta::traverse::BaseSpill::new( + tempfile::tempfile()?, + budget, + ))), + None => None, + }; + tree.traverse( + |range: gix_pack::data::EntryRange, source: &gix_pack::data::File, buf: &mut Vec| { + source.read_into(range, buf) + }, + stored, + stored.pack_end() as u64, + harvest, + gix_pack::cache::delta::traverse::Options { + object_progress: Box::new(Discard), + size_progress: &mut Discard, + thread_limit: Some(knot_resource::ingest_thread_limit().get()), + should_interrupt: interrupt, + object_hash: kind, + base_spill, + collect_items: false, + max_object_bytes: Some(max_object_bytes.get()), + }, + ) + .map(|_| ()) + .map_err(|error| PackError::Pack(error.to_string())) +} + +fn persist_atomic( + path: &Path, + write: impl FnOnce(&mut dyn Write) -> Result<(), PackError>, +) -> Result<(), PackError> { + let dir = path + .parent() + .ok_or_else(|| PackError::Pack("object path has no parent".to_string()))?; + let mut tmp = tempfile::NamedTempFile::new_in(dir)?; + let mut writer = std::io::BufWriter::new(tmp.as_file_mut()); + write(&mut writer)?; + writer.flush()?; + drop(writer); + tmp.as_file().sync_all()?; + tmp.persist(path) + .map_err(|error| PackError::Pack(error.to_string()))?; + std::fs::File::open(dir)?.sync_all()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use gix_pack::Find; + + use super::*; + + #[test] + fn an_empty_pack_file_is_not_a_successful_ingest() { + let dir = tempfile::tempdir().unwrap(); + let pack = tempfile::NamedTempFile::new().unwrap(); + let error = ingest_pack_file( + dir.path(), + pack.path(), + &PackLimits::default(), + gix::hash::Kind::Sha1, + ) + .unwrap_err(); + assert!(matches!(error, PackError::Pack(message) if message == "packfile is empty")); + } + + #[test] + fn interruptible_find_errors_once_the_flag_is_set() { + let dir = tempfile::tempdir().unwrap(); + let flag = Arc::new(AtomicBool::new(false)); + let find = Interruptible { + inner: gix::odb::at(dir.path()).unwrap(), + flag: Arc::clone(&flag), + }; + let absent = Oid::null().object_id(); + let mut buf = Vec::new(); + + assert!( + find.try_find(&absent, &mut buf).unwrap().is_none(), + "before budget trips, lookup of an absent object is a plain miss" + ); + + flag.store(true, Ordering::Relaxed); + assert!( + find.try_find(&absent, &mut buf).is_err(), + "once tripped, every decode errors, which is what aborts a breadthfirst tree walk \ + mid-closure instead of waiting for the next commit-root boundary" + ); + } +} diff --git a/knot2/crates/knot-pack/src/pkt.rs b/knot2/crates/knot-pack/src/pkt.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb6d6022de654fb7ea2ff2761a3f07897f4b5837 --- /dev/null +++ b/knot2/crates/knot-pack/src/pkt.rs @@ -0,0 +1,317 @@ +use std::io; + +use gix_packetline::{Channel, blocking_io::encode}; + +pub const MAX_BAND: usize = 65515; + +pub enum Frame<'a> { + Data(&'a [u8]), + Flush, + Delim, + ResponseEnd, +} + +fn hex4(prefix: &[u8]) -> io::Result { + std::str::from_utf8(prefix) + .ok() + .and_then(|text| u16::from_str_radix(text, 16).ok()) + .ok_or_else(|| io::Error::other("invalid pkt-line length prefix")) +} + +pub(crate) fn frame_len(prefix: &[u8]) -> io::Result { + match hex4(prefix)? { + 3 => Err(io::Error::other("invalid pkt-line length 3")), + n => Ok(usize::from(n)), + } +} + +pub fn frames( + input: &[u8], + stop_after_flushes: Option, +) -> impl Iterator, usize)>> + '_ { + let mut pos = 0usize; + let mut flushes = 0usize; + let mut stopped = false; + std::iter::from_fn(move || { + (!stopped && pos + 4 <= input.len()).then(|| { + let frame = hex4(&input[pos..pos + 4]).and_then(|len| { + pos += 4; + match len { + 0 => { + flushes += 1; + stopped = stop_after_flushes == Some(flushes); + Ok((Frame::Flush, pos)) + } + 1 => Ok((Frame::Delim, pos)), + 2 => Ok((Frame::ResponseEnd, pos)), + 3 => Err(io::Error::other("invalid pkt-line length 3")), + n => { + let end = pos - 4 + usize::from(n); + (end <= input.len()) + .then(|| { + let payload = &input[pos..end]; + pos = end; + (Frame::Data(payload), end) + }) + .ok_or_else(|| io::Error::other("truncated pkt-line")) + } + } + }); + stopped |= frame.is_err(); + frame + }) + }) +} + +pub fn data_payloads(input: &[u8]) -> io::Result> { + collect_data(input, Some(1)) +} + +pub fn data_payloads_all(input: &[u8]) -> io::Result> { + collect_data(input, None) +} + +fn collect_data(input: &[u8], stop_after_flushes: Option) -> io::Result> { + frames(input, stop_after_flushes) + .filter_map(|item| match item { + Ok((Frame::Data(payload), _)) => Some(Ok(payload)), + Ok(_) => None, + Err(err) => Some(Err(err)), + }) + .collect() +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Caps { + pub atomic: bool, + pub side_band_64k: bool, + pub push_options: bool, +} + +pub(crate) fn first_command(input: &[u8]) -> Option<&[u8]> { + frames(input, Some(1)).find_map(|item| match item { + Ok((Frame::Data(payload), _)) if !is_preamble(payload) => Some(payload), + _ => None, + }) +} + +fn is_preamble(line: &[u8]) -> bool { + line.starts_with(b"shallow ") +} + +pub(crate) fn parse_caps(first_command: &[u8]) -> Caps { + let caps = first_command + .split(|byte| *byte == 0) + .nth(1) + .and_then(|caps| std::str::from_utf8(caps).ok()) + .unwrap_or_default(); + let has = |needle: &str| caps.split_whitespace().any(|cap| cap == needle); + Caps { + atomic: has("atomic"), + side_band_64k: has("side-band-64k"), + push_options: has("push-options"), + } +} + +pub struct Receive<'a> { + pub commands: Vec<&'a [u8]>, + pub options: Vec<&'a [u8]>, + pub pack: &'a [u8], + pub caps: Caps, +} + +pub fn split_receive(input: &[u8]) -> io::Result> { + let caps = first_command(input).map(parse_caps).unwrap_or_default(); + let boundary = if caps.push_options { 2 } else { 1 }; + frames(input, Some(boundary)) + .try_fold( + (Vec::new(), Vec::new(), 0usize, None), + |(mut commands, mut options, flushes, end), item| { + item.map(|(frame, at)| match frame { + Frame::Data(payload) if flushes == 0 && !is_preamble(payload) => { + commands.push(payload); + (commands, options, flushes, end) + } + Frame::Data(payload) if flushes == 1 && caps.push_options => { + options.push(payload); + (commands, options, flushes, end) + } + Frame::Data(_) => (commands, options, flushes, end), + Frame::Flush => (commands, options, flushes + 1, Some(at)), + _ => (commands, options, flushes, end), + }) + }, + ) + .map(|(commands, options, _flushes, end)| Receive { + commands, + options, + caps, + pack: &input[end.unwrap_or(input.len())..], + }) +} + +pub fn write_data(buf: &mut Vec, payload: &[u8]) -> io::Result<()> { + encode::data_to_write(payload, buf).map(|_| ()) +} + +pub fn write_flush(buf: &mut Vec) -> io::Result<()> { + encode::flush_to_write(buf).map(|_| ()) +} + +pub fn write_delim(buf: &mut Vec) -> io::Result<()> { + encode::delim_to_write(buf).map(|_| ()) +} + +pub fn write_band(buf: &mut Vec, chunk: &[u8]) -> io::Result<()> { + encode::band_to_write(Channel::Data, chunk, buf).map(|_| ()) +} + +pub fn write_band_progress(buf: &mut Vec, message: &[u8]) -> io::Result<()> { + encode::band_to_write(Channel::Progress, message, buf).map(|_| ()) +} + +pub fn write_band_error(buf: &mut Vec, message: &[u8]) -> io::Result<()> { + encode::band_to_write(Channel::Error, message, buf).map(|_| ()) +} + +pub fn frame_report(report: &[u8], messages: &[String], side_band: bool) -> Vec { + if !side_band { + return report.to_vec(); + } + let mut buf = Vec::new(); + report + .chunks(MAX_BAND) + .for_each(|chunk| write_band(&mut buf, chunk).expect("band write to vec never fails")); + messages.iter().for_each(|message| { + format!("{message}\n") + .into_bytes() + .chunks(MAX_BAND) + .for_each(|chunk| { + write_band_progress(&mut buf, chunk).expect("band write to vec never fails") + }); + }); + write_flush(&mut buf).expect("flush write to vec never fails"); + buf +} + +#[cfg(test)] +mod tests { + use super::*; + + fn command_line(caps: &str) -> Vec { + let mut line = b"\ + 0000000000000000000000000000000000000000 \ + 1111111111111111111111111111111111111111 refs/heads/main" + .to_vec(); + line.push(0); + line.extend_from_slice(caps.as_bytes()); + line.push(b'\n'); + line + } + + #[test] + fn a_malformed_length_prefix_terminates_instead_of_spinning() { + let garbage = b"zzzz this is not a pkt-line stream at all"; + assert_eq!( + frames(garbage, None).count(), + 1, + "bad length prefix yields one error frame then the stream ends" + ); + assert!( + first_command(garbage).is_none(), + "no command is parsed out of garbage, and scan does not loop" + ); + assert!( + split_receive(garbage).is_err(), + "malformed prefix is a parse error, never an infinite loop" + ); + } + + #[test] + fn split_receive_skips_the_push_options_section_before_the_pack() { + let mut body = Vec::new(); + write_data( + &mut body, + &command_line("report-status side-band-64k push-options"), + ) + .unwrap(); + write_flush(&mut body).unwrap(); + write_data(&mut body, b"ci-skip").unwrap(); + write_data(&mut body, b"verbose-ci").unwrap(); + write_flush(&mut body).unwrap(); + body.extend_from_slice(b"PACKreal-pack-bytes"); + + let parsed = split_receive(&body).unwrap(); + assert!(parsed.caps.push_options); + assert!(parsed.caps.side_band_64k); + assert_eq!(parsed.commands.len(), 1); + assert_eq!(parsed.options, vec![&b"ci-skip"[..], &b"verbose-ci"[..]]); + assert_eq!(parsed.pack, b"PACKreal-pack-bytes"); + } + + #[test] + fn split_receive_reads_caps_past_a_shallow_preamble_line() { + let mut body = Vec::new(); + write_data( + &mut body, + b"shallow 1111111111111111111111111111111111111111\n", + ) + .unwrap(); + write_data(&mut body, &command_line("report-status side-band-64k")).unwrap(); + write_flush(&mut body).unwrap(); + body.extend_from_slice(b"PACKbytes"); + + let parsed = split_receive(&body).unwrap(); + assert!( + parsed.caps.side_band_64k, + "capabilities come from the command line, not the shallow preamble" + ); + assert_eq!( + parsed.commands.len(), + 1, + "the shallow line is not a command" + ); + assert_eq!(parsed.pack, b"PACKbytes"); + } + + #[test] + fn split_receive_without_push_options_starts_the_pack_after_the_command_flush() { + let mut body = Vec::new(); + write_data(&mut body, &command_line("report-status side-band-64k")).unwrap(); + write_flush(&mut body).unwrap(); + body.extend_from_slice(b"PACKbytes"); + + let parsed = split_receive(&body).unwrap(); + assert!(!parsed.caps.push_options); + assert!(parsed.options.is_empty()); + assert_eq!(parsed.pack, b"PACKbytes"); + } + + #[test] + fn frame_report_muxes_the_report_on_band_one_and_messages_on_band_two() { + let report = b"unpack ok\n"; + let messages = vec!["hello there".to_string()]; + let framed = frame_report(report, &messages, true); + + let bands: Vec<(u8, Vec)> = frames(&framed, None) + .filter_map(|item| match item { + Ok((Frame::Data(payload), _)) => Some((payload[0], payload[1..].to_vec())), + _ => None, + }) + .collect(); + assert_eq!(bands[0].0, 1, "report rides band 1"); + assert_eq!(bands[0].1, report); + assert_eq!(bands[1].0, 2, "message rides band 2"); + assert_eq!(bands[1].1, b"hello there\n"); + assert!(framed.ends_with(b"0000"), "outer flush closes the stream"); + } + + #[test] + fn frame_report_passes_through_raw_without_side_band() { + let report = b"unpack ok\n0000"; + assert_eq!( + frame_report(report, &["dropped".to_string()], false), + report + ); + } +} diff --git a/knot2/crates/knot-pack/src/quarantine.rs b/knot2/crates/knot-pack/src/quarantine.rs new file mode 100644 index 0000000000000000000000000000000000000000..09b7b6be8bd81ca53921b9ff7f8353765d605f05 --- /dev/null +++ b/knot2/crates/knot-pack/src/quarantine.rs @@ -0,0 +1,72 @@ +//! The same way one would do this for uploading images onto a server, +//! we stage pushes such that objects unpack into a separate bare repo +//! under the incoming prefix, because we treat a push as real only once +//! every object it referenced actually made it over. +//! +//! Hence aborting is as simple as removal of the directory that +//! represents the push in flight. +//! +//! "Why not use `GIT_QUARANTINE_PATH`?" - because we never forked `receive-pack`. +//! +//! If you were wondering, this `sweep_incoming` is for a crash between a stage +//! and migration that would otherwise leak a staging dir. + +use std::path::Path; + +use knot_git::{INCOMING_PREFIX, Repo, Staging}; + +use crate::error::PackError; +use crate::objects; +use knot_storage::PackLimits; + +pub(crate) struct Quarantine { + staging: Staging, +} + +impl Quarantine { + pub(crate) fn stage( + live: &Repo, + pack: Option<&gix_pack::data::File>, + limits: &PackLimits, + kind: gix::hash::Kind, + live_empty: bool, + ) -> Result<(Self, Option), PackError> { + let staging = Staging::new(live)?; + let (unpack, closure) = crate::receive::ingest( + &staging.repo().objects_dir(), + pack, + limits, + kind, + live_empty, + ); + unpack?; + Ok((Self { staging }, closure)) + } + + pub(crate) fn repo(&self) -> &Repo { + self.staging.repo() + } + + pub(crate) fn migrate_into(&self, live: &Repo) -> Result<(), PackError> { + self.staging.migrate_into(live).map_err(PackError::from) + } +} + +pub fn sweep_incoming(scan_path: &Path) -> usize { + walkdir::WalkDir::new(scan_path) + .into_iter() + .filter_entry(|entry| entry.file_name().to_str() != Some("objects")) + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_dir()) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(INCOMING_PREFIX)) + }) + .map(|entry| entry.into_path()) + .collect::>() + .into_iter() + .filter(|path| std::fs::remove_dir_all(path).is_ok()) + .count() +} diff --git a/knot2/crates/knot-pack/src/receive.rs b/knot2/crates/knot-pack/src/receive.rs new file mode 100644 index 0000000000000000000000000000000000000000..d8c84ffee7a6da717ff08cbd1cad2c4b12752cc8 --- /dev/null +++ b/knot2/crates/knot-pack/src/receive.rs @@ -0,0 +1,858 @@ +use std::collections::HashMap; +use std::io::Write; +use std::path::Path; + +use knot_git::{Filter, Haves, RefClass, RefTxn, RefUpdate, Repo, Wants}; +use knot_messages::{RefKey, RejectMessages}; +use knot_types::{ObjectFormat, Oid, PushOption, PushOptions, RefName}; + +use crate::error::PackError; +use crate::objects; +use crate::pkt; +use crate::quarantine::Quarantine; +use crate::receiver::ReceivedPack; +use crate::{HaveOids, WantOids}; +use knot_storage::PackLimits; + +fn stage_pack_bytes( + dir: &Path, + pack: &[u8], + kind: gix::hash::Kind, +) -> Result, PackError> { + if pack.is_empty() { + return Ok(None); + } + let mut tmp = tempfile::NamedTempFile::new_in(dir)?; + tmp.write_all(pack)?; + tmp.flush()?; + let file = objects::open_pack_file(tmp.path(), kind)?; + Ok(Some((tmp, file))) +} + +pub fn handle_bytes(repo: &Repo, body: &[u8], limits: &PackLimits) -> Result, PackError> { + let kind = repo.object_format().kind(); + let messages = &crate::default_catalog().reject; + match stage_pack_bytes(&repo.objects_dir(), pkt::split_receive(body)?.pack, kind) { + Ok(staged) => handle( + repo, + body, + Ok(staged.as_ref().map(|(_, file)| file)), + limits, + messages, + ), + Err(error) => handle(repo, body, Err(error), limits, messages), + } +} + +pub fn handle_guarded_bytes( + live: &Repo, + body: &[u8], + limits: &PackLimits, + guard: &dyn ReceiveGuard, + seal: &dyn Fn(&[RefUpdate]), + messages: &RejectMessages, +) -> Result { + let kind = live.object_format().kind(); + match stage_pack_bytes(&live.objects_dir(), pkt::split_receive(body)?.pack, kind) { + Ok(staged) => handle_guarded( + live, + body, + Ok(staged.as_ref().map(|(_, file)| file)), + limits, + guard, + seal, + messages, + ), + Err(error) => handle_guarded(live, body, Err(error), limits, guard, seal, messages), + } +} + +pub fn handle_guarded_streamed( + live: &Repo, + received: &ReceivedPack, + limits: &PackLimits, + guard: &dyn ReceiveGuard, + seal: &dyn Fn(&[RefUpdate]), + messages: &RejectMessages, +) -> Result { + match received.open_pack() { + Ok(pack) => handle_guarded( + live, + received.preamble(), + Ok(pack.as_ref()), + limits, + guard, + seal, + messages, + ), + Err(error) => handle_guarded( + live, + received.preamble(), + Err(error), + limits, + guard, + seal, + messages, + ), + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RefDecision { + Allow, + Reject(String), +} + +pub struct ReceiveOutcome { + pub report: Vec, + pub side_band: bool, + pub push_options: PushOptions, +} + +pub trait ReceiveGuard { + fn authorize(&self, staged: &Repo, commands: &[ReceiveCommand]) -> Vec; +} + +const CAPS_BASE: &str = + "report-status delete-refs atomic ofs-delta side-band-64k push-options agent=knot/0"; + +fn caps(format: ObjectFormat) -> String { + format!("{CAPS_BASE} object-format={}", format.capability()) +} + +pub fn advertise(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"# service=git-receive-pack\n")?; + pkt::write_flush(&mut buf)?; + write_advert(&mut buf, repo)?; + Ok(buf) +} + +pub fn advertise_ssh(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + write_advert(&mut buf, repo)?; + Ok(buf) +} + +fn write_advert(buf: &mut Vec, repo: &Repo) -> Result<(), PackError> { + let format = repo.object_format(); + let caps = caps(format); + let refs = repo.advertised_refs_for(knot_git::AdvertScope::Receive)?; + match refs.split_first() { + Some((first, rest)) => { + let mut line = format!("{} {}", first.target, first.name).into_bytes(); + line.push(0); + line.extend_from_slice(caps.as_bytes()); + line.push(b'\n'); + pkt::write_data(buf, &line)?; + rest.iter().try_fold(&mut *buf, |buf, record| { + pkt::write_data( + buf, + format!("{} {}\n", record.target, record.name).as_bytes(), + )?; + Ok::<_, PackError>(buf) + })?; + } + None => { + let mut line = format!("{} capabilities^{{}}", format.null_oid()).into_bytes(); + line.push(0); + line.extend_from_slice(caps.as_bytes()); + line.push(b'\n'); + pkt::write_data(buf, &line)?; + } + } + pkt::write_flush(buf)?; + Ok(()) +} + +enum CommandRef { + Named(RefName), + Unparsed(String), +} + +pub struct ReceiveCommand { + old: Oid, + new: Oid, + name: CommandRef, +} + +impl ReceiveCommand { + pub fn refname(&self) -> &str { + match &self.name { + CommandRef::Named(name) => name.as_str(), + CommandRef::Unparsed(raw) => raw, + } + } + + pub fn name(&self) -> Option<&RefName> { + match &self.name { + CommandRef::Named(name) => Some(name), + CommandRef::Unparsed(_) => None, + } + } + + pub fn is_delete(&self) -> bool { + self.new.is_null() + } + + pub fn is_create(&self) -> bool { + self.old.is_null() + } + + fn parse(line: &[u8], first: bool) -> Option { + let line = if first { + line.split(|byte| *byte == 0).next().unwrap_or(line) + } else { + line + }; + let text = std::str::from_utf8(line).ok()?; + let mut parts = text.trim_end().split(' '); + let old = Oid::from_hex(parts.next()?).ok()?; + let new = Oid::from_hex(parts.next()?).ok()?; + let raw = parts.next()?.to_string(); + let name = match RefName::new(raw.as_str()) { + Ok(name) => CommandRef::Named(name), + Err(_) => CommandRef::Unparsed(raw), + }; + Some(ReceiveCommand { old, new, name }) + } + + fn to_update(&self) -> Result { + let name = self + .name() + .cloned() + .ok_or_else(|| PackError::Protocol(invalid_refname(self.refname())))?; + Ok(match (self.old.is_null(), self.new.is_null()) { + (_, true) => RefUpdate::Delete { + name, + old: self.old, + }, + (true, false) => RefUpdate::Create { + name, + new: self.new, + }, + (false, false) => RefUpdate::Update { + name, + old: self.old, + new: self.new, + }, + }) + } +} + +pub(crate) fn invalid_refname(raw: &str) -> String { + format!("invalid ref name {raw:?}") +} + +fn forbidden_ref(command: &ReceiveCommand, messages: &RejectMessages) -> Option { + match command.name() { + Some(name) => (!RefClass::of(name).is_public()).then(|| messages.reserved_refs.text()), + None => Some(invalid_refname(command.refname())), + } +} + +fn reserved_create_only(command: &ReceiveCommand, messages: &RejectMessages) -> Option { + (command + .name() + .is_some_and(|name| RefClass::of(name) == RefClass::Cob) + && !command.is_create()) + .then(|| messages.cob_create_only.text()) +} + +struct RefSnapshot { + by_name: HashMap, +} + +impl RefSnapshot { + fn capture(repo: &Repo) -> Result { + let by_name = repo + .references()? + .into_iter() + .map(|record| (record.name, record.target)) + .collect(); + Ok(RefSnapshot { by_name }) + } + + fn tips(&self) -> HaveOids { + self.by_name.values().copied().collect() + } + + fn conflict(&self, command: &ReceiveCommand, messages: &RejectMessages) -> Option { + match ( + command.old.is_null(), + command + .name() + .and_then(|name| self.by_name.get(name).copied()), + ) { + (true, Some(_)) => Some(messages.ref_exists.text()), + (false, found) if found != Some(command.old) => Some(messages.stale_old_value.text()), + _ => None, + } + } +} + +struct RefResult { + refname: String, + failure: Option, +} + +impl RefResult { + fn of(command: &ReceiveCommand, failure: Option) -> RefResult { + RefResult { + refname: command.refname().to_string(), + failure, + } + } +} + +struct Conflict { + refname: String, + reason: String, +} + +fn first_conflict( + snapshot: &RefSnapshot, + commands: &[ReceiveCommand], + messages: &RejectMessages, +) -> Option { + commands.iter().find_map(|command| { + snapshot.conflict(command, messages).map(|reason| Conflict { + refname: command.refname().to_string(), + reason, + }) + }) +} + +fn objects_present( + repo: &Repo, + wants: &WantOids, + haves: &HaveOids, + closure: Option<&objects::FreshClosure>, +) -> bool { + if let Some(closure) = closure { + return closure.self_contained + && wants + .as_slice() + .iter() + .all(|want| closure.present.contains(want) || repo.contains(*want)); + } + matches!( + repo.select_pack_objects_filtered( + Wants::new(wants.as_slice()), + Haves::new(haves.as_slice()), + Filter::None, + crate::upload::selection_budget(), + ), + Ok(selection) if selection.send.iter().all(|oid| repo.contains(*oid)) + ) +} + +fn connectivity_reasons( + repo: &Repo, + commands: &[ReceiveCommand], + haves: &HaveOids, + closure: Option<&objects::FreshClosure>, + messages: &RejectMessages, +) -> Vec> { + let news: WantOids = commands + .iter() + .map(|command| command.new) + .filter(|new| !new.is_null()) + .collect(); + let batched_ok = news.is_empty() || objects_present(repo, &news, haves, closure); + commands + .iter() + .map(|command| { + if command.new.is_null() + || batched_ok + || objects_present(repo, &WantOids::new(vec![command.new]), haves, closure) + { + None + } else { + Some(messages.missing_objects.text()) + } + }) + .collect() +} + +pub(crate) fn fuzz(body: &[u8]) { + if let Ok(parsed) = pkt::split_receive(body) { + parsed + .commands + .iter() + .enumerate() + .for_each(|(index, line)| { + if let Some(command) = ReceiveCommand::parse(line, index == 0) { + let _ = command.to_update(); + } + }); + } +} + +pub(crate) fn is_empty(repo: &Repo) -> bool { + repo.references() + .map(|refs| refs.is_empty()) + .unwrap_or(false) +} + +pub(crate) fn ingest( + objects_dir: &Path, + pack: Option<&gix_pack::data::File>, + limits: &PackLimits, + kind: gix::hash::Kind, + live_empty: bool, +) -> (Result<(), PackError>, Option) { + let Some(pack) = pack else { + return (Ok(()), None); + }; + if let Err(error) = objects::admit_ingest(pack, kind) { + return (Err(error), None); + } + if live_empty { + match objects::ingest_and_close( + objects_dir, + pack, + limits, + kind, + knot_resource::ingest_base_budget().map(|budget| budget.get() as usize), + false, + ) { + Ok(Some(closure)) => return (Ok(()), Some(closure)), + Ok(None) => {} + Err(error) => return (Err(error), None), + } + } + ( + objects::index_pack_bounded(objects_dir, pack, limits, kind), + None, + ) +} + +pub fn handle( + repo: &Repo, + body: &[u8], + pack: Result, PackError>, + limits: &PackLimits, + messages: &RejectMessages, +) -> Result, PackError> { + let parsed = pkt::split_receive(body)?; + let atomic = parsed.caps.atomic; + let commands = parse_commands(&parsed); + let kind = repo.object_format().kind(); + + let (unpack, closure) = match pack { + Ok(pack) => ingest(&repo.objects_dir(), pack, limits, kind, is_empty(repo)), + Err(error) => (Err(error), None), + }; + let results: Vec = match &unpack { + Err(_) => all_failed(&commands, &messages.unpacker_error.text()), + Ok(()) => match RefSnapshot::capture(repo) { + Err(_) => all_failed(&commands, &messages.ref_snapshot_unavailable.text()), + Ok(snapshot) => { + let haves = snapshot.tips(); + if atomic { + apply_atomic( + repo, + &snapshot, + &commands, + &haves, + closure.as_ref(), + messages, + ) + } else { + commands + .iter() + .zip(connectivity_reasons( + repo, + &commands, + &haves, + closure.as_ref(), + messages, + )) + .map(|(command, connectivity)| { + RefResult::of(command, apply_one(repo, command, connectivity, messages)) + }) + .collect() + } + } + }, + }; + report(&unpack, &results) + .map(|report| pkt::frame_report(&report, &[], parsed.caps.side_band_64k)) +} + +fn apply_one( + repo: &Repo, + command: &ReceiveCommand, + connectivity: Option, + messages: &RejectMessages, +) -> Option { + if let Some(reason) = forbidden_ref(command, messages) { + return Some(reason); + } + if let Some(reason) = connectivity { + return Some(reason); + } + command + .to_update() + .and_then(|update| repo.update_ref(&update).map_err(PackError::from)) + .err() + .map(|error| error.to_string().replace('\n', " ")) +} + +fn atomic_failure( + snapshot: &RefSnapshot, + commands: &[ReceiveCommand], + messages: &RejectMessages, +) -> Vec { + let conflict = first_conflict(snapshot, commands, messages); + commands + .iter() + .map(|command| match &conflict { + Some(conflict) if conflict.refname == command.refname() => { + RefResult::of(command, Some(conflict.reason.clone())) + } + _ => RefResult::of(command, Some(messages.atomic_failed.text())), + }) + .collect() +} + +fn apply_atomic( + repo: &Repo, + snapshot: &RefSnapshot, + commands: &[ReceiveCommand], + haves: &HaveOids, + closure: Option<&objects::FreshClosure>, + messages: &RejectMessages, +) -> Vec { + let fail = |reason: String| all_failed(commands, &reason); + if commands + .iter() + .any(|command| forbidden_ref(command, messages).is_some()) + { + return commands + .iter() + .map(|command| { + let reason = forbidden_ref(command, messages) + .unwrap_or_else(|| messages.atomic_aborted.text()); + RefResult::of(command, Some(reason)) + }) + .collect(); + } + if let Some(command) = commands + .iter() + .zip(connectivity_reasons( + repo, commands, haves, closure, messages, + )) + .find_map(|(command, reason)| reason.map(|_| command)) + { + return fail( + messages + .missing_objects_for + .line(|RefKey::Ref| command.refname().to_string()), + ); + } + let updates = match commands + .iter() + .map(ReceiveCommand::to_update) + .collect::, _>>() + { + Ok(updates) => updates, + Err(error) => return fail(error.to_string().replace('\n', " ")), + }; + match repo.update_refs(&updates) { + Ok(()) => commands + .iter() + .map(|command| RefResult::of(command, None)) + .collect(), + Err(_) => atomic_failure(snapshot, commands, messages), + } +} + +fn report(unpack: &Result<(), PackError>, results: &[RefResult]) -> Result, PackError> { + let mut buf = Vec::new(); + match unpack { + Ok(()) => pkt::write_data(&mut buf, b"unpack ok\n")?, + Err(error) => pkt::write_data( + &mut buf, + format!("unpack {}\n", error.to_string().replace('\n', " ")).as_bytes(), + )?, + } + results.iter().try_fold(&mut buf, |buf, result| { + let line = match &result.failure { + None => format!("ok {}\n", result.refname), + Some(reason) => format!("ng {} {reason}\n", result.refname), + }; + pkt::write_data(buf, line.as_bytes())?; + Ok::<_, PackError>(buf) + })?; + pkt::write_flush(&mut buf)?; + Ok(buf) +} + +fn parse_commands(parsed: &pkt::Receive) -> Vec { + parsed + .commands + .iter() + .enumerate() + .filter_map(|(index, line)| ReceiveCommand::parse(line, index == 0)) + .collect() +} + +pub struct Preflight { + pub creates_branch: bool, +} + +pub(crate) fn preflight(body: &[u8]) -> Preflight { + pkt::split_receive(body) + .map(|parsed| { + let commands = parse_commands(&parsed); + Preflight { + creates_branch: commands.iter().any(|command| { + command.is_create() && command.name().is_some_and(knot_git::is_branch) + }), + } + }) + .unwrap_or(Preflight { + creates_branch: false, + }) +} + +fn all_failed(commands: &[ReceiveCommand], reason: &str) -> Vec { + commands + .iter() + .map(|command| RefResult::of(command, Some(reason.to_string()))) + .collect() +} + +fn stage_to_quarantine(staged: &Repo, commands: &[ReceiveCommand]) { + commands + .iter() + .filter(|command| !command.is_delete()) + .for_each(|command| { + if let Some(name) = command.name() { + let _ = staged.update_ref(&RefUpdate::Create { + name: name.clone(), + new: command.new, + }); + } + }); +} + +fn apply_guarded_atomic( + txn: &RefTxn<'_>, + snapshot: &RefSnapshot, + commands: &[ReceiveCommand], + seal: &dyn Fn(&[RefUpdate]), + messages: &RejectMessages, +) -> Vec { + let updates = match commands + .iter() + .map(ReceiveCommand::to_update) + .collect::, _>>() + { + Ok(updates) => updates, + Err(error) => return all_failed(commands, &error.to_string().replace('\n', " ")), + }; + match txn.update_refs(&updates) { + Ok(()) => { + seal(&updates); + commands + .iter() + .map(|command| RefResult::of(command, None)) + .collect() + } + Err(_) => atomic_failure(snapshot, commands, messages), + } +} + +fn apply_guarded_update( + txn: &RefTxn<'_>, + command: &ReceiveCommand, + seal: &dyn Fn(&[RefUpdate]), +) -> Option { + let update = match command.to_update() { + Ok(update) => update, + Err(error) => return Some(error.to_string().replace('\n', " ")), + }; + match txn.update_ref(&update) { + Ok(()) => { + seal(std::slice::from_ref(&update)); + None + } + Err(error) => Some(PackError::from(error).to_string().replace('\n', " ")), + } +} + +fn parse_push_options(parsed: &pkt::Receive) -> PushOptions { + PushOptions::new(parsed.options.iter().filter_map(|option| { + PushOption::new( + String::from_utf8_lossy(option).trim_matches(|byte: char| byte == '\n' || byte == '\r'), + ) + .ok() + })) +} + +#[allow(clippy::too_many_arguments)] +pub fn handle_guarded( + live: &Repo, + body: &[u8], + pack: Result, PackError>, + limits: &PackLimits, + guard: &dyn ReceiveGuard, + seal: &dyn Fn(&[RefUpdate]), + messages: &RejectMessages, +) -> Result { + let parsed = pkt::split_receive(body)?; + let atomic = parsed.caps.atomic; + let side_band = parsed.caps.side_band_64k; + let push_options = parse_push_options(&parsed); + let build = |report: Vec| ReceiveOutcome { + report, + side_band, + push_options: push_options.clone(), + }; + let commands = parse_commands(&parsed); + if commands.is_empty() { + return report(&Ok(()), &[]).map(build); + } + + let pack = match pack { + Ok(pack) => pack, + Err(error) => { + let results = all_failed(&commands, &messages.unpacker_error.text()); + return report(&Err(error), &results).map(build); + } + }; + let (quarantine, closure) = match Quarantine::stage( + live, + pack, + limits, + live.object_format().kind(), + is_empty(live), + ) { + Ok(staged) => staged, + Err(error) => { + let results = all_failed(&commands, &messages.unpacker_error.text()); + return report(&Err(error), &results).map(build); + } + }; + let staged = quarantine.repo(); + stage_to_quarantine(staged, &commands); + let snapshot = match RefSnapshot::capture(live) { + Ok(snapshot) => snapshot, + Err(_) => { + let results = all_failed(&commands, &messages.ref_snapshot_unavailable.text()); + return report(&Ok(()), &results).map(build); + } + }; + let haves = snapshot.tips(); + + let verdicts = guard.authorize(staged, &commands); + let reasons: Vec> = if verdicts.len() == commands.len() { + commands + .iter() + .zip(verdicts) + .zip(connectivity_reasons( + staged, + &commands, + &haves, + closure.as_ref(), + messages, + )) + .map(|((command, verdict), connectivity)| match verdict { + RefDecision::Reject(reason) => Some(reason), + RefDecision::Allow => reserved_create_only(command, messages) + .or(connectivity) + .or_else(|| snapshot.conflict(command, messages)), + }) + .collect() + } else { + commands + .iter() + .map(|_| Some(messages.authorization_unavailable.text())) + .collect() + }; + + let any_reject = reasons.iter().any(Option::is_some); + if atomic && any_reject { + let results = commands + .iter() + .zip(reasons) + .map(|(command, reason)| { + RefResult::of( + command, + Some(reason.unwrap_or_else(|| messages.atomic_aborted.text())), + ) + }) + .collect::>(); + return report(&Ok(()), &results).map(build); + } + + let applied = live.with_ref_txn(|txn| { + if reasons.iter().any(Option::is_none) { + quarantine.migrate_into(live)?; + } + let results = if atomic { + apply_guarded_atomic(txn, &snapshot, &commands, seal, messages) + } else { + commands + .iter() + .zip(reasons) + .map(|(command, reason)| match reason { + Some(reason) => RefResult::of(command, Some(reason)), + None => RefResult::of(command, apply_guarded_update(txn, command, seal)), + }) + .collect() + }; + Ok::<_, PackError>(results) + }); + match applied { + Ok(results) => report(&Ok(()), &results).map(build), + Err(error) => { + let results = all_failed(&commands, &messages.object_migration_failed.text()); + report(&Err(error), &results).map(build) + } + } +} + +#[cfg(test)] +mod tests { + use super::parse_push_options; + use crate::pkt; + use knot_types::{PushOption, PushOptions}; + + fn options(raw: &[&[u8]]) -> PushOptions { + parse_push_options(&pkt::Receive { + commands: Vec::new(), + options: raw.to_vec(), + pack: &[], + caps: pkt::Caps::default(), + }) + } + + #[test] + fn a_push_option_the_lexicon_rejects_never_reaches_the_event() { + let long = vec![b'x'; 1025]; + let parsed = options(&[b"verbose-ci\n", b"", &long, b"has\nnewline", b"ci-skip\r"]); + assert_eq!( + parsed + .as_slice() + .iter() + .map(PushOption::as_str) + .collect::>(), + vec!["verbose-ci", "ci-skip"], + "parsing trims trailing end-of-line and rejects empty, oversized, and multi-line options" + ); + + let raw: Vec> = (0..PushOptions::MAX + 10) + .map(|index| format!("option-{index}").into_bytes()) + .collect(); + let borrowed: Vec<&[u8]> = raw.iter().map(Vec::as_slice).collect(); + assert_eq!( + options(&borrowed).as_slice().len(), + PushOptions::MAX, + "directives parse from the same truncated list the event reports" + ); + } +} diff --git a/knot2/crates/knot-pack/src/receiver.rs b/knot2/crates/knot-pack/src/receiver.rs new file mode 100644 index 0000000000000000000000000000000000000000..1135ac997addf63f78cd945e6fd84145105ef75e --- /dev/null +++ b/knot2/crates/knot-pack/src/receiver.rs @@ -0,0 +1,162 @@ +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use tempfile::NamedTempFile; + +use crate::error::PackError; +use crate::frame::ReceiveFramer; +use crate::ids::MaxWireBytes; +use knot_storage::PackLimits; + +#[derive(Debug)] +pub enum ReceiveReadError { + Io(std::io::Error), + Pack(PackError), + TooLarge, + Truncated, +} + +pub struct ReceivedPack { + preamble: Vec, + pack: Option, + kind: gix::hash::Kind, + total_len: usize, +} + +impl ReceivedPack { + pub fn preamble(&self) -> &[u8] { + &self.preamble + } + + pub fn len(&self) -> usize { + self.total_len + } + + pub fn is_empty(&self) -> bool { + self.total_len == 0 + } + + pub fn open_pack(&self) -> Result, PackError> { + match &self.pack { + Some(tmp) => gix_pack::data::File::at(tmp.path(), self.kind) + .map(Some) + .map_err(|error| PackError::Pack(error.to_string())), + None => Ok(None), + } + } +} + +pub struct PackReceiver { + dir: PathBuf, + file: NamedTempFile, + framer: ReceiveFramer, + written: u64, + limit: MaxWireBytes, + complete: Option, + kind: gix::hash::Kind, +} + +impl PackReceiver { + pub fn new( + dir: &Path, + limit: MaxWireBytes, + limits: PackLimits, + kind: gix::hash::Kind, + ) -> std::io::Result { + Ok(Self { + dir: dir.to_path_buf(), + file: NamedTempFile::new_in(dir)?, + framer: ReceiveFramer::new(limits, kind), + written: 0, + limit, + complete: None, + kind, + }) + } + + pub fn write(&mut self, chunk: &[u8]) -> Result { + if self.complete.is_some() { + return Ok(true); + } + if self.written as usize + chunk.len() > self.limit.get() { + return Err(ReceiveReadError::TooLarge); + } + self.file.write_all(chunk).map_err(ReceiveReadError::Io)?; + self.written += chunk.len() as u64; + self.scan() + } + + fn scan(&mut self) -> Result { + if self.written == 0 { + return Ok(false); + } + self.file.flush().map_err(ReceiveReadError::Io)?; + match self + .framer + .advance_file(self.file.as_file(), self.written) + .map_err(ReceiveReadError::Pack)? + { + Some(total) => { + self.complete = Some(total); + Ok(true) + } + None => Ok(false), + } + } + + pub fn finish(mut self) -> Result { + let total = match self.complete { + Some(total) => total, + None => { + if self.written == 0 { + 0 + } else if self.scan()? { + self.complete.expect("scan recorded completion") + } else { + return Err(ReceiveReadError::Truncated); + } + } + }; + let pack_start = self.framer.pack_start().unwrap_or(total); + let preamble = + read_range(self.file.as_file(), 0..pack_start).map_err(ReceiveReadError::Io)?; + let pack = match total > pack_start { + true => { + let mut tmp = NamedTempFile::new_in(&self.dir).map_err(ReceiveReadError::Io)?; + copy_range( + self.file.as_file(), + pack_start as u64..total as u64, + tmp.as_file_mut(), + ) + .map_err(ReceiveReadError::Io)?; + tmp.flush().map_err(ReceiveReadError::Io)?; + Some(tmp) + } + false => None, + }; + Ok(ReceivedPack { + preamble, + pack, + kind: self.kind, + total_len: total, + }) + } +} + +fn read_range(file: &std::fs::File, range: std::ops::Range) -> std::io::Result> { + use std::os::unix::fs::FileExt; + let mut buf = vec![0u8; range.end.saturating_sub(range.start)]; + file.read_exact_at(&mut buf, range.start as u64)?; + Ok(buf) +} + +fn copy_range( + src: &std::fs::File, + range: std::ops::Range, + dst: &mut std::fs::File, +) -> std::io::Result<()> { + let mut reader = src.try_clone()?; + reader.seek(SeekFrom::Start(range.start))?; + std::io::copy(&mut reader.take(range.end.saturating_sub(range.start)), dst)?; + Ok(()) +} diff --git a/knot2/crates/knot-pack/src/resolve.rs b/knot2/crates/knot-pack/src/resolve.rs new file mode 100644 index 0000000000000000000000000000000000000000..95fdec1cd0435a786b3ac3b1fec9c6cc8639dd3f --- /dev/null +++ b/knot2/crates/knot-pack/src/resolve.rs @@ -0,0 +1,338 @@ +use std::collections::HashMap; +use std::path::Path; + +use gix::ObjectId; +use gix::object::Kind; +use gix::objs::{Find, Write}; +use gix_pack::data::{Entry, entry::Header}; + +use crate::error::{PackError, PackLimit}; +use crate::ids::{PackOffset, Rounds}; +use knot_storage::{DeltaDepth, PackLimits}; + +use crate::meter::{inflate_into, pack_object_count}; + +struct Raw { + offset: PackOffset, + header: Header, + data: Vec, +} + +#[derive(Clone, Copy)] +struct Resolved { + oid: ObjectId, + depth: DeltaDepth, +} + +fn malformed(message: &str) -> PackError { + PackError::Pack(message.to_string()) +} + +pub fn resolve( + objects_dir: &Path, + pack: &[u8], + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + let mut raws = parse_entries(pack, kind)?; + let odb = gix::odb::at_opts( + objects_dir, + std::iter::empty(), + gix::odb::store::init::Options { + object_hash: kind, + ..Default::default() + }, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let mut done: HashMap = HashMap::new(); + let mut by_oid: HashMap = HashMap::new(); + resolve_rounds( + &mut raws, + &odb, + &mut done, + &mut by_oid, + limits.max_delta_depth, + Rounds::new(limits.max_delta_depth.get() + 2), + ) +} + +fn parse_entries(pack: &[u8], kind: gix::hash::Kind) -> Result, PackError> { + let hash_len = kind.len_in_bytes(); + let trailer = pack + .len() + .checked_sub(hash_len) + .ok_or_else(|| malformed("packfile is truncated"))?; + let num_objects = pack_object_count(pack)?; + + (0..num_objects.get()) + .try_fold((Vec::new(), PackOffset::new(12)), |(mut acc, offset), _| { + let mut reader: &[u8] = pack + .get(offset.get() as usize..trailer) + .ok_or_else(|| malformed("entry offset past pack end"))?; + let entry = Entry::from_read(&mut reader, offset.get(), hash_len) + .map_err(|error| PackError::Pack(error.to_string()))?; + let data_start = entry.data_offset as usize; + let mut data = Vec::with_capacity(entry.decompressed_size as usize); + let consumed = inflate_into( + pack.get(data_start..) + .ok_or_else(|| malformed("entry body past pack end"))?, + entry.decompressed_size, + &mut data, + )?; + acc.push(Raw { + offset, + header: entry.header, + data, + }); + Ok::<_, PackError>((acc, PackOffset::new(entry.data_offset + consumed))) + }) + .map(|(acc, _)| acc) +} + +fn resolve_rounds( + raws: &mut [Raw], + odb: &gix::odb::Handle, + done: &mut HashMap, + by_oid: &mut HashMap, + max_depth: DeltaDepth, + rounds_left: Rounds, +) -> Result<(), PackError> { + let pending: Vec = (0..raws.len()) + .filter(|index| !done.contains_key(&raws[*index].offset)) + .collect(); + if pending.is_empty() { + return Ok(()); + } + let Some(remaining) = rounds_left.next() else { + return Err(PackError::LimitExceeded(PackLimit::DeltaDepth)); + }; + let progressed = pending.iter().try_fold(false, |progressed, &index| { + match resolve_one(&raws[index], odb, done, by_oid, max_depth)? { + Some((kind, data, depth)) => { + let oid = odb + .write_buf(kind, &data) + .map_err(|error| PackError::Pack(error.to_string()))?; + by_oid.insert(oid, depth); + done.insert(raws[index].offset, Resolved { oid, depth }); + raws[index].data = Vec::new(); + Ok::<_, PackError>(true) + } + None => Ok(progressed), + } + })?; + if !progressed { + return Err(malformed("pack contains unresolvable delta")); + } + resolve_rounds(raws, odb, done, by_oid, max_depth, remaining) +} + +fn apply_delta_checked( + base_kind: Kind, + base_data: &[u8], + base_depth: DeltaDepth, + delta: &[u8], + max_depth: DeltaDepth, +) -> Result<(Kind, Vec, DeltaDepth), PackError> { + let depth = base_depth.deeper(); + if depth.exceeds(max_depth) { + return Err(PackError::LimitExceeded(PackLimit::DeltaDepth)); + } + Ok((base_kind, apply_delta(base_data, delta)?, depth)) +} + +fn resolve_one( + raw: &Raw, + odb: &gix::odb::Handle, + done: &HashMap, + by_oid: &HashMap, + max_depth: DeltaDepth, +) -> Result, DeltaDepth)>, PackError> { + match &raw.header { + Header::Commit | Header::Tree | Header::Blob | Header::Tag => { + let kind = raw + .header + .as_kind() + .ok_or_else(|| malformed("base entry has no object kind"))?; + Ok(Some((kind, raw.data.clone(), DeltaDepth::ZERO))) + } + Header::OfsDelta { base_distance } => { + let base_offset = raw + .offset + .checked_sub_distance(*base_distance) + .filter(|_| *base_distance != 0) + .ok_or_else(|| malformed("ofs-delta base out of range"))?; + match done.get(&base_offset) { + Some(base) => { + let mut buf = Vec::new(); + let found = odb + .try_find(&base.oid, &mut buf) + .map_err(|error| PackError::Pack(error.to_string()))? + .ok_or_else(|| malformed("ofs-delta base missing from odb"))?; + apply_delta_checked(found.kind, found.data, base.depth, &raw.data, max_depth) + .map(Some) + } + None => Ok(None), + } + } + Header::RefDelta { base_id } => { + let base_depth = by_oid.get(base_id).copied(); + let mut buf = Vec::new(); + match odb + .try_find(base_id, &mut buf) + .map_err(|error| PackError::Pack(error.to_string()))? + { + Some(base) => apply_delta_checked( + base.kind, + base.data, + base_depth.unwrap_or(DeltaDepth::ZERO), + &raw.data, + max_depth, + ) + .map(Some), + None => Ok(None), + } + } + } +} + +enum Op<'a> { + Copy { start: usize, len: usize }, + Insert(&'a [u8]), +} + +fn read_varint(delta: &[u8], pos: &mut usize) -> Result { + let mut shift = 0u32; + let mut value = 0u64; + loop { + let byte = *delta + .get(*pos) + .ok_or_else(|| malformed("delta size header truncated"))?; + *pos += 1; + value |= u64::from(byte & 0x7f) << shift; + shift += 7; + if byte & 0x80 == 0 { + return Ok(value); + } + if shift >= u64::BITS { + return Err(malformed("delta size header overflows")); + } + } +} + +fn assemble( + cmd: u8, + bit_base: u32, + count: u32, + delta: &[u8], + pos: &mut usize, +) -> Result { + (0..count).try_fold(0u64, |acc, index| { + if cmd & (1 << (bit_base + index)) == 0 { + return Ok(acc); + } + let byte = *delta + .get(*pos) + .ok_or_else(|| malformed("delta copy operand truncated"))?; + *pos += 1; + Ok(acc | (u64::from(byte) << (8 * index))) + }) +} + +fn ops<'a>(delta: &'a [u8]) -> impl Iterator, PackError>> { + let mut pos = 0usize; + std::iter::from_fn(move || { + (pos < delta.len()).then(|| { + let cmd = delta[pos]; + pos += 1; + if cmd & 0x80 != 0 { + let offset = assemble(cmd, 0, 4, delta, &mut pos)?; + let raw_size = assemble(cmd, 4, 3, delta, &mut pos)?; + let size = if raw_size == 0 { 0x10000 } else { raw_size }; + Ok(Op::Copy { + start: offset as usize, + len: size as usize, + }) + } else if cmd != 0 { + let len = cmd as usize; + let bytes = delta + .get(pos..pos + len) + .ok_or_else(|| malformed("delta insert truncated"))?; + pos += len; + Ok(Op::Insert(bytes)) + } else { + Err(malformed("delta uses reserved opcode 0")) + } + }) + }) +} + +fn apply_delta(base: &[u8], delta: &[u8]) -> Result, PackError> { + let mut pos = 0usize; + let base_size = read_varint(delta, &mut pos)?; + if base_size as usize != base.len() { + return Err(malformed("delta base size doesn't match its base object")); + } + let target_size = read_varint(delta, &mut pos)?; + let out = + ops(&delta[pos..]).try_fold(Vec::with_capacity(target_size as usize), |mut out, op| { + match op? { + Op::Copy { start, len } => { + let end = start + .checked_add(len) + .ok_or_else(|| malformed("delta copy range overflows"))?; + out.extend_from_slice( + base.get(start..end) + .ok_or_else(|| malformed("delta copy reads past base"))?, + ); + } + Op::Insert(bytes) => out.extend_from_slice(bytes), + } + Ok::<_, PackError>(out) + })?; + if out.len() as u64 != target_size { + return Err(malformed("delta produced wrong target size")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn apply_delta_reconstructs_copy_and_insert() { + let base = b"hello world"; + let delta = [0x0b, 0x06, 0x90, 0x05, 0x01, b'!']; + assert_eq!(apply_delta(base, &delta).unwrap(), b"hello!"); + } + + #[test] + fn apply_delta_rejects_a_base_size_mismatch() { + let delta = [0x05, 0x00]; + assert!(apply_delta(b"hi", &delta).is_err()); + } + + #[test] + fn apply_delta_rejects_a_copy_past_the_base() { + let delta = [0x02, 0x10, 0x90, 0xff]; + assert!(apply_delta(b"hi", &delta).is_err()); + } + + #[test] + fn apply_delta_rejects_a_truncated_insert() { + let delta = [0x02, 0x05, 0x05, b'a', b'b']; + assert!(apply_delta(b"hi", &delta).is_err()); + } + + #[test] + fn apply_delta_rejects_the_reserved_opcode() { + let delta = [0x02, 0x00, 0x00]; + assert!(apply_delta(b"hi", &delta).is_err()); + } + + #[test] + fn apply_delta_rejects_a_truncated_size_header() { + let delta = [0x80]; + assert!(apply_delta(b"", &delta).is_err()); + } +} diff --git a/knot2/crates/knot-pack/src/upload.rs b/knot2/crates/knot-pack/src/upload.rs new file mode 100644 index 0000000000000000000000000000000000000000..43ba804e81307647558bc4a719b2eb937b48906a --- /dev/null +++ b/knot2/crates/knot-pack/src/upload.rs @@ -0,0 +1,1131 @@ +use std::collections::HashSet; +use std::io::{self, Read, Write}; +use std::sync::OnceLock; +use std::time::Duration; + +#[cfg(feature = "instrument")] +use knot_git::instrument::{add_odb_reads as credit, measure as counted}; +use knot_git::{ + CommitDepth, Deepen, Filter, Haves, PackBudget, PackfileUri, RefClass, Repo, ShallowCommits, + Wants, +}; +use knot_messages::{CountKey, FetchMessages, KnotKey}; +use knot_types::{KnotHostname, ObjectCount, ObjectFormat, Oid, UnixSeconds}; + +use crate::error::PackError; +use crate::objects; +use crate::pkt; +use crate::{HaveOids, WantOids}; + +const AGENT: &[u8] = b"agent=knot/0\n"; +const SELECTION_MAX_OBJECTS: ObjectCount = ObjectCount::new(16_000_000); +const SELECTION_TIME_BUDGET: Duration = Duration::from_secs(600); +const KEEPALIVE: Duration = Duration::from_secs(5); + +#[derive(Debug, Clone, Copy)] +pub struct SelectionLimits { + pub max_objects: ObjectCount, + pub time_budget: Duration, +} + +impl Default for SelectionLimits { + fn default() -> Self { + Self { + max_objects: SELECTION_MAX_OBJECTS, + time_budget: SELECTION_TIME_BUDGET, + } + } +} + +static SELECTION_LIMITS: OnceLock = OnceLock::new(); + +pub fn init_selection_limits(limits: SelectionLimits) { + if SELECTION_LIMITS.set(limits).is_err() { + debug_assert!(false, "selection limits initialized more than once"); + } +} + +fn selection_limits() -> SelectionLimits { + SELECTION_LIMITS.get().copied().unwrap_or_default() +} +const V0_CAPS_BASE: &str = "multi_ack_detailed no-done side-band-64k ofs-delta shallow deepen-since deepen-not filter agent=knot/0"; + +fn v0_caps(format: ObjectFormat) -> String { + format!("{V0_CAPS_BASE} object-format={}", format.capability()) +} + +pub struct StreamOpts { + pub side_band: bool, + pub sideband_all: bool, + pub no_progress: bool, + pub thin: bool, + pub filter: Filter, + pub shallow_commits: Option>, + pub packfile_uris: Vec, + pub emit_packfile_header: bool, +} + +pub enum UploadOutcome { + Buffered(Vec), + Streaming { + preamble: Vec, + wants: WantOids, + haves: HaveOids, + opts: StreamOpts, + }, +} + +fn write_v2_caps(buf: &mut Vec, format: ObjectFormat) -> Result<(), PackError> { + pkt::write_data(buf, b"version 2\n")?; + pkt::write_data(buf, AGENT)?; + pkt::write_data(buf, b"ls-refs\n")?; + pkt::write_data( + buf, + b"fetch=shallow filter wait-for-done packfile-uris sideband-all\n", + )?; + pkt::write_data(buf, b"server-option\n")?; + pkt::write_data( + buf, + format!("object-format={}\n", format.capability()).as_bytes(), + )?; + pkt::write_flush(buf)?; + Ok(()) +} + +pub fn advertise(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"# service=git-upload-pack\n")?; + pkt::write_flush(&mut buf)?; + write_v2_caps(&mut buf, repo.object_format())?; + Ok(buf) +} + +pub fn advertise_ssh(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + write_v2_caps(&mut buf, repo.object_format())?; + Ok(buf) +} + +pub fn advertise_v0(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"# service=git-upload-pack\n")?; + pkt::write_flush(&mut buf)?; + write_v0_advert(&mut buf, repo)?; + Ok(buf) +} + +pub fn advertise_v0_ssh(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + write_v0_advert(&mut buf, repo)?; + Ok(buf) +} + +fn write_v0_advert(buf: &mut Vec, repo: &Repo) -> Result<(), PackError> { + let format = repo.object_format(); + let caps = v0_caps(format); + let refs = repo.advertised_refs_for(knot_git::AdvertScope::Upload)?; + match repo.head() { + Some(head) => { + pkt::write_data( + buf, + format!("{} HEAD\0{caps} symref=HEAD:{}\n", head.target, head.name).as_bytes(), + )?; + write_plain_refs(buf, &refs)?; + } + None => match refs.split_first() { + Some((first, rest)) => { + pkt::write_data( + buf, + format!("{} {}\0{caps}\n", first.target, first.name).as_bytes(), + )?; + write_plain_refs(buf, rest)?; + } + None => { + pkt::write_data( + buf, + format!("{} capabilities^{{}}\0{caps}\n", format.null_oid()).as_bytes(), + )?; + } + }, + } + pkt::write_flush(buf)?; + Ok(()) +} + +fn write_plain_refs(buf: &mut Vec, refs: &[knot_git::RefRecord]) -> Result<(), PackError> { + refs.iter().try_for_each(|record| { + pkt::write_data( + buf, + format!("{} {}\n", record.target, record.name).as_bytes(), + ) + .map_err(PackError::from) + }) +} + +pub(crate) fn fuzz(body: &[u8]) { + if let Ok(lines) = pkt::data_payloads_all(body) { + let _ = parse_wants(&lines); + let _ = parse_oids(&lines, b"want "); + let _ = parse_oids(&lines, b"have "); + let _ = first_caps(&lines); + let _ = parse_ls_refs_args(&lines); + } +} + +pub fn plan(repo: &Repo, body: &[u8]) -> Result { + let peek = pkt::data_payloads(body)?; + match peek.first() { + Some(line) if line.starts_with(b"command=") => plan_v2(repo, &peek), + _ => plan_v0(repo, body), + } +} + +pub fn buffered( + repo: &Repo, + body: &[u8], + messages: &FetchMessages, + knot: &KnotHostname, +) -> Result, PackError> { + let mut out = Vec::new(); + streamed(repo, body, messages, knot, &mut |chunk| { + out.extend_from_slice(chunk); + Ok(()) + })?; + Ok(out) +} + +pub fn streamed( + repo: &Repo, + body: &[u8], + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + match plan(repo, body)? { + UploadOutcome::Buffered(bytes) => sink(&bytes).map_err(PackError::from), + UploadOutcome::Streaming { + preamble, + wants, + haves, + opts, + } => { + sink(&preamble)?; + stream_pack(repo, &wants, &haves, &opts, messages, knot, sink)?; + if opts.side_band { + let mut flush = Vec::new(); + pkt::write_flush(&mut flush)?; + sink(&flush)?; + } + Ok(()) + } + } +} + +fn plan_v2(repo: &Repo, lines: &[&[u8]]) -> Result { + match lines.first().copied().unwrap_or_default() { + command if command.starts_with(b"command=ls-refs") => { + Ok(UploadOutcome::Buffered(ls_refs(repo, lines)?)) + } + command if command.starts_with(b"command=fetch") => plan_v2_fetch(repo, lines), + _ => Err(PackError::Protocol( + "unsupported protocol v2 command".to_string(), + )), + } +} + +struct LsRefsArgs { + symrefs: bool, + peel: bool, + prefixes: Vec, +} + +fn parse_ls_refs_args(lines: &[&[u8]]) -> LsRefsArgs { + LsRefsArgs { + symrefs: lines.iter().any(|line| line.starts_with(b"symrefs")), + peel: lines.iter().any(|line| line.starts_with(b"peel")), + prefixes: lines + .iter() + .filter_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end() + .strip_prefix("ref-prefix ") + .map(str::to_string) + }) + .collect(), + } +} + +pub(crate) fn matches_prefix>(name: &str, prefixes: &[S]) -> bool { + prefixes.is_empty() + || prefixes + .iter() + .any(|prefix| name.starts_with(prefix.as_ref())) +} +fn class_refs(repo: &Repo, class: RefClass) -> Result, PackError> { + Ok(repo + .references()? + .into_iter() + .filter(|record| RefClass::of(&record.name) == class) + .collect()) +} + +fn ls_refs(repo: &Repo, lines: &[&[u8]]) -> Result, PackError> { + let args = parse_ls_refs_args(lines); + let mut buf = Vec::new(); + if let Some(head) = repo.head() + && matches_prefix("HEAD", &args.prefixes) + { + let mut line = format!("{} HEAD", head.target); + if args.symrefs { + line.push_str(&format!(" symref-target:{}", head.name)); + } + line.push('\n'); + pkt::write_data(&mut buf, line.as_bytes())?; + } + let mut refs = repo.advertised_refs_for(knot_git::AdvertScope::Upload)?; + for class in [RefClass::Cob, RefClass::Tngl] { + let namespace = class.namespace().expect("hidden classes have a namespace"); + if args + .prefixes + .iter() + .any(|prefix| prefix.starts_with(namespace)) + { + refs.extend(class_refs(repo, class)?); + } + } + refs.iter() + .filter(|record| matches_prefix(record.name.as_str(), &args.prefixes)) + .try_fold(&mut buf, |buf, record| { + let mut line = format!("{} {}", record.target, record.name); + if args.peel + && let Some(peeled) = repo.peeled_target(record.target)? + { + line.push_str(&format!(" peeled:{peeled}")); + } + line.push('\n'); + pkt::write_data(buf, line.as_bytes())?; + Ok::<_, PackError>(buf) + })?; + pkt::write_flush(&mut buf)?; + Ok(buf) +} + +fn plan_v2_fetch(repo: &Repo, lines: &[&[u8]]) -> Result { + let wants = parse_wants(lines)?; + ensure_wanted(repo, &wants)?; + let haves = HaveOids::new(parse_oids(lines, b"have ")); + let done = lines.iter().any(|line| line.starts_with(b"done")); + let wait_for_done = lines.iter().any(|line| line.starts_with(b"wait-for-done")); + let sideband_all = lines.iter().any(|line| line.starts_with(b"sideband-all")); + let no_progress = lines.iter().any(|line| line.starts_with(b"no-progress")); + let thin = lines.iter().any(|line| line.starts_with(b"thin-pack")); + let filter = parse_filter(lines)?; + let deepen = parse_deepen(repo, lines)?; + let client_shallow = parse_oids(lines, b"shallow "); + let common: HaveOids = haves + .iter() + .copied() + .filter(|oid| repo.contains(*oid)) + .collect(); + + let mut preamble = Vec::new(); + if !haves.is_empty() && !done { + seg(&mut preamble, sideband_all, b"acknowledgments\n")?; + if common.is_empty() { + seg(&mut preamble, sideband_all, b"NAK\n")?; + pkt::write_flush(&mut preamble)?; + return Ok(UploadOutcome::Buffered(preamble)); + } + common.iter().try_for_each(|oid| { + seg( + &mut preamble, + sideband_all, + format!("ACK {oid}\n").as_bytes(), + ) + })?; + if wait_for_done { + pkt::write_flush(&mut preamble)?; + return Ok(UploadOutcome::Buffered(preamble)); + } + seg(&mut preamble, sideband_all, b"ready\n")?; + pkt::write_delim(&mut preamble)?; + } + + let shallow_commits = if deepen.is_shallow_request() || repo.is_shallow() { + let plan = repo.shallow_walk( + Wants::new(wants.as_slice()), + &deepen, + ShallowCommits::new(&client_shallow), + )?; + seg(&mut preamble, sideband_all, b"shallow-info\n")?; + plan.shallow.iter().try_for_each(|oid| { + seg( + &mut preamble, + sideband_all, + format!("shallow {oid}\n").as_bytes(), + ) + })?; + plan.unshallow.iter().try_for_each(|oid| { + seg( + &mut preamble, + sideband_all, + format!("unshallow {oid}\n").as_bytes(), + ) + })?; + pkt::write_delim(&mut preamble)?; + Some(plan.commits) + } else { + None + }; + + Ok(UploadOutcome::Streaming { + preamble, + wants, + haves: common, + opts: StreamOpts { + side_band: true, + sideband_all, + no_progress, + thin, + filter, + shallow_commits, + // packfile-uris require sideband-all so keepalives can precede them + packfile_uris: match sideband_all { + true => packfile_uri_candidates(repo, lines), + false => Vec::new(), + }, + emit_packfile_header: true, + }, + }) +} + +fn seg(buf: &mut Vec, sideband_all: bool, content: &[u8]) -> io::Result<()> { + if sideband_all { + pkt::write_band(buf, content) + } else { + pkt::write_data(buf, content) + } +} + +fn packfile_uri_candidates(repo: &Repo, lines: &[&[u8]]) -> Vec { + let Some(protocols) = lines.iter().find_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end() + .strip_prefix("packfile-uris ") + .map(str::to_string) + }) else { + return Vec::new(); + }; + let allowed: Vec<&str> = protocols.split(',').map(str::trim).collect(); + repo.blob_packfile_uris() + .into_iter() + .filter(|candidate| { + candidate + .uri + .as_str() + .split_once("://") + .is_some_and(|(scheme, _)| allowed.contains(&scheme)) + }) + .collect() +} + +fn parse_filter(lines: &[&[u8]]) -> Result { + let spec = lines.iter().find_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end() + .strip_prefix("filter ") + }); + match spec { + None => Ok(Filter::None), + Some("blob:none") => Ok(Filter::BlobNone), + Some(rest) if rest.starts_with("blob:limit=") => parse_size(&rest["blob:limit=".len()..]) + .map(Filter::BlobLimit) + .ok_or_else(|| PackError::Protocol(format!("bad blob:limit filter: {rest}"))), + Some(rest) if rest.starts_with("tree:") => rest["tree:".len()..] + .parse::() + .map(|depth| Filter::TreeDepth(knot_git::TreeDepth::new(depth))) + .map_err(|_| PackError::Protocol(format!("bad tree filter: {rest}"))), + Some(other) => Err(PackError::Protocol(format!("unsupported filter: {other}"))), + } +} + +fn parse_size(text: &str) -> Option { + let (digits, scale) = match text.chars().last() { + Some('k') | Some('K') => (&text[..text.len() - 1], 1024), + Some('m') | Some('M') => (&text[..text.len() - 1], 1024 * 1024), + Some('g') | Some('G') => (&text[..text.len() - 1], 1024 * 1024 * 1024), + _ => (text, 1), + }; + digits + .parse::() + .ok() + .and_then(|value| value.checked_mul(scale)) +} + +fn parse_deepen(repo: &Repo, lines: &[&[u8]]) -> Result { + let value = |prefix: &str| -> Option<&str> { + lines.iter().find_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end() + .strip_prefix(prefix) + }) + }; + let depth = value("deepen ") + .and_then(|text| text.parse::().ok()) + .map(CommitDepth::new); + let since = value("deepen-since ") + .and_then(|text| text.parse::().ok()) + .map(UnixSeconds::new); + let relative = lines + .iter() + .any(|line| line.starts_with(b"deepen-relative")); + let not = lines + .iter() + .filter_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end() + .strip_prefix("deepen-not ") + }) + .map(|spec| resolve_commitish(repo, spec)) + .collect::, _>>()?; + Ok(Deepen { + depth, + since, + not, + relative, + }) +} + +fn resolve_commitish(repo: &Repo, spec: &str) -> Result { + if let Ok(oid) = Oid::from_hex(spec) + && repo.contains(oid) + { + return Ok(oid); + } + let candidates = [ + spec.to_string(), + format!("refs/{spec}"), + format!("refs/tags/{spec}"), + format!("refs/heads/{spec}"), + ]; + repo.advertised_refs_for(knot_git::AdvertScope::Upload)? + .iter() + .find(|record| candidates.iter().any(|name| record.name.as_str() == name)) + .map(|record| record.target) + .ok_or_else(|| PackError::Protocol(format!("deepen-not {spec}: unknown ref"))) +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct V0Caps { + side_band: bool, + no_progress: bool, + thin_pack: bool, + multi_ack_detailed: bool, + no_done: bool, +} + +fn parse_v0_caps(caps: Option<&str>) -> V0Caps { + let caps = caps.unwrap_or_default(); + let has = |needle: &str| caps.split(' ').any(|cap| cap == needle); + V0Caps { + side_band: has("side-band-64k") || has("side-band"), + no_progress: has("no-progress"), + thin_pack: has("thin-pack"), + multi_ack_detailed: has("multi_ack_detailed"), + no_done: has("no-done"), + } +} + +fn plan_v0(repo: &Repo, body: &[u8]) -> Result { + let lines = pkt::data_payloads_all(body)?; + let wants = parse_wants(&lines)?; + ensure_wanted(repo, &wants)?; + let haves = HaveOids::new(parse_oids(&lines, b"have ")); + let done = lines.iter().any(|line| line.starts_with(b"done")); + let caps = parse_v0_caps(first_caps(&lines)); + let filter = parse_filter(&lines)?; + let deepen = parse_deepen(repo, &lines)?; + let client_shallow = parse_oids(&lines, b"shallow "); + let common: HaveOids = haves + .iter() + .copied() + .filter(|oid| repo.contains(*oid)) + .collect(); + + let mut preamble = Vec::new(); + let shallow_commits = if deepen.is_shallow_request() || repo.is_shallow() { + let plan = repo.shallow_walk( + Wants::new(wants.as_slice()), + &deepen, + ShallowCommits::new(&client_shallow), + )?; + plan.shallow.iter().try_for_each(|oid| { + pkt::write_data(&mut preamble, format!("shallow {oid}\n").as_bytes()) + })?; + plan.unshallow.iter().try_for_each(|oid| { + pkt::write_data(&mut preamble, format!("unshallow {oid}\n").as_bytes()) + })?; + pkt::write_flush(&mut preamble)?; + Some(plan.commits) + } else { + None + }; + + if haves.is_empty() && !done { + if !(deepen.is_shallow_request() || repo.is_shallow()) { + pkt::write_data(&mut preamble, b"NAK\n")?; + } + return Ok(UploadOutcome::Buffered(preamble)); + } + + let ready = caps.multi_ack_detailed + && !done + && !common.is_empty() + && common.len() == haves.len() + && repo.wants_satisfied_by(Wants::new(wants.as_slice()), Haves::new(common.as_slice()))?; + + let stream = move |preamble: Vec, common: HaveOids| UploadOutcome::Streaming { + preamble, + wants, + haves: common, + opts: StreamOpts { + side_band: caps.side_band, + sideband_all: false, + no_progress: caps.no_progress, + thin: caps.thin_pack, + filter, + shallow_commits, + packfile_uris: Vec::new(), + emit_packfile_header: false, + }, + }; + + if caps.multi_ack_detailed { + common.iter().try_for_each(|oid| { + pkt::write_data(&mut preamble, format!("ACK {oid} common\n").as_bytes()) + })?; + let last = common.as_slice().last().copied(); + if done { + match last { + Some(oid) => { + pkt::write_data(&mut preamble, format!("ACK {oid}\n").as_bytes())?; + } + None => pkt::write_data(&mut preamble, b"NAK\n")?, + } + return Ok(stream(preamble, common)); + } + match (ready, last) { + (true, Some(oid)) => { + pkt::write_data(&mut preamble, format!("ACK {oid} ready\n").as_bytes())?; + pkt::write_data(&mut preamble, b"NAK\n")?; + if caps.no_done { + pkt::write_data(&mut preamble, format!("ACK {oid}\n").as_bytes())?; + return Ok(stream(preamble, common)); + } + } + _ => pkt::write_data(&mut preamble, b"NAK\n")?, + } + return Ok(UploadOutcome::Buffered(preamble)); + } + + match common.as_slice().first() { + Some(oid) => pkt::write_data(&mut preamble, format!("ACK {oid}\n").as_bytes())?, + None => pkt::write_data(&mut preamble, b"NAK\n")?, + } + match done { + true => Ok(stream(preamble, common)), + false => Ok(UploadOutcome::Buffered(preamble)), + } +} + +pub fn stream_pack( + repo: &Repo, + wants: &WantOids, + haves: &HaveOids, + opts: &StreamOpts, + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let progress = opts.side_band && !opts.no_progress; + // the `packfile` section opens before keepalives start, unless `packfile-uris` must precede it + if opts.packfile_uris.is_empty() { + write_packfile_header(opts, sink)?; + } + with_keepalive(repo, opts.side_band, KEEPALIVE, sink, |repo, sink| { + if opts.shallow_commits.is_none() + && haves.is_empty() + && opts.filter == Filter::None + && opts.packfile_uris.is_empty() + { + if let Ok(Some(pack)) = + knot_git::verbatim_clone_pack(repo, Wants::new(wants.as_slice())) + { + return stream_verbatim_pack(pack, opts, progress, messages, knot, sink); + } + if let Ok(Some(oids)) = + knot_git::reachable_via_bitmap(repo, Wants::new(wants.as_slice()), Haves::new(&[])) + { + return stream_object_set(repo, oids, opts, progress, messages, knot, sink); + } + return stream_full_clone(repo, wants.as_slice(), opts, progress, messages, knot, sink); + } + let budget = selection_budget(); + let mut selection = match &opts.shallow_commits { + Some(commits) => repo.select_shallow_objects( + Wants::new(wants.as_slice()), + ShallowCommits::new(commits), + Haves::new(haves.as_slice()), + opts.filter, + budget, + )?, + None => repo.select_pack_objects_filtered( + Wants::new(wants.as_slice()), + Haves::new(haves.as_slice()), + opts.filter, + budget, + )?, + }; + if !opts.packfile_uris.is_empty() { + offload_packfile_uris( + &mut selection.send, + &opts.packfile_uris, + opts.sideband_all, + sink, + )?; + write_packfile_header(opts, sink)?; + } + let count = ObjectCount::new(selection.send.len()); + emit_preamble(progress, messages, knot, count, sink)?; + { + let mut pack_sink = PackSink { + side_band: opts.side_band, + sink: &mut *sink, + }; + let thin_bases = opts.thin.then_some(&selection.client_has); + objects::write_pack( + &repo.objects_dir(), + selection.send, + thin_bases, + &mut pack_sink, + repo.object_format().kind(), + )?; + } + emit_total(progress, messages, count, sink) + }) +} + +fn emit_progress( + progress: bool, + lines: impl FnOnce() -> Vec, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + if !progress { + return Ok(()); + } + let rendered = lines(); + if rendered.is_empty() { + return Ok(()); + } + let mut buf = Vec::new(); + rendered.iter().try_for_each(|line| { + format!("{line}\n") + .into_bytes() + .chunks(pkt::MAX_BAND) + .try_for_each(|chunk| pkt::write_band_progress(&mut buf, chunk)) + })?; + sink(&buf).map_err(PackError::from) +} + +fn emit_preamble( + progress: bool, + messages: &FetchMessages, + knot: &KnotHostname, + count: ObjectCount, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + emit_progress( + progress, + || { + messages + .motd + .lines(|KnotKey::Knot| knot.as_str().to_string()) + .into_iter() + .chain( + messages + .enumerating + .lines(|CountKey::Count| count.to_string()), + ) + .collect() + }, + sink, + ) +} + +fn emit_total( + progress: bool, + messages: &FetchMessages, + count: ObjectCount, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + emit_progress( + progress, + || messages.total.lines(|CountKey::Count| count.to_string()), + sink, + ) +} + +fn write_packfile_header( + opts: &StreamOpts, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + if !opts.emit_packfile_header { + return Ok(()); + } + let mut buf = Vec::new(); + seg(&mut buf, opts.sideband_all, b"packfile\n")?; + sink(&buf)?; + Ok(()) +} + +fn offload_packfile_uris( + send: &mut Vec, + candidates: &[PackfileUri], + sideband_all: bool, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let present: std::collections::HashSet = send.iter().copied().collect(); + let kept: Vec<&PackfileUri> = candidates + .iter() + .filter(|candidate| present.contains(&candidate.oid)) + .collect(); + if kept.is_empty() { + return Ok(()); + } + let excluded: std::collections::HashSet = + kept.iter().map(|candidate| candidate.oid).collect(); + send.retain(|oid| !excluded.contains(oid)); + let mut buf = Vec::new(); + seg(&mut buf, sideband_all, b"packfile-uris\n")?; + kept.iter().try_for_each(|candidate| { + seg( + &mut buf, + sideband_all, + format!( + "{} {}\n", + candidate.pack_hash.to_hex(), + candidate.uri.as_str() + ) + .as_bytes(), + ) + })?; + pkt::write_delim(&mut buf)?; + sink(&buf)?; + Ok(()) +} + +fn stream_verbatim_pack( + mut file: std::fs::File, + opts: &StreamOpts, + progress: bool, + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let mut header = [0u8; 12]; + file.read_exact(&mut header)?; + if &header[..4] != b"PACK" { + return Err(PackError::Pack( + "reused pack is missing its PACK signature".to_string(), + )); + } + let count = ObjectCount::from(u32::from_be_bytes([ + header[8], header[9], header[10], header[11], + ])); + stream_pack_body(opts, progress, messages, knot, count, sink, |pack_sink| { + pack_sink.write_all(&header)?; + io::copy(&mut file, pack_sink)?; + Ok(()) + }) +} + +fn stream_object_set( + repo: &Repo, + oids: Vec, + opts: &StreamOpts, + progress: bool, + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let count = ObjectCount::new(oids.len()); + stream_pack_body(opts, progress, messages, knot, count, sink, |pack_sink| { + objects::write_pack( + &repo.objects_dir(), + oids, + None, + pack_sink, + repo.object_format().kind(), + ) + }) +} + +fn stream_full_clone( + repo: &Repo, + wants: &[Oid], + opts: &StreamOpts, + progress: bool, + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let limits = selection_limits(); + let stall = limits.time_budget; + let roots = repo.clone_roots(wants, PackBudget::new(limits.max_objects, stall))?; + let pack = objects::count_expanded( + &repo.objects_dir(), + roots, + limits.max_objects, + stall, + repo.object_format().kind(), + )?; + let count = ObjectCount::new(pack.len()); + stream_pack_body(opts, progress, messages, knot, count, sink, |pack_sink| { + objects::write_expanded(pack, pack_sink) + }) +} + +fn stream_pack_body( + opts: &StreamOpts, + progress: bool, + messages: &FetchMessages, + knot: &KnotHostname, + count: ObjectCount, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, + body: impl FnOnce(&mut PackSink<'_>) -> Result<(), PackError>, +) -> Result<(), PackError> { + emit_preamble(progress, messages, knot, count, sink)?; + { + let mut pack_sink = PackSink { + side_band: opts.side_band, + sink: &mut *sink, + }; + body(&mut pack_sink)?; + } + emit_total(progress, messages, count, sink) +} + +/// Runs `work` on a worker thread, forwarding its output to `sink` and sending an empty data +/// band whenever it stays silent for `interval`. +fn with_keepalive( + repo: &Repo, + keepalive: bool, + interval: Duration, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, + work: impl FnOnce(&Repo, &mut dyn FnMut(&[u8]) -> io::Result<()>) -> Result + Send, +) -> Result { + if !keepalive { + return work(repo, sink); + } + let path = repo.path().to_owned(); + std::thread::scope(|scope| { + let (tx, rx) = std::sync::mpsc::sync_channel::>(16); + let worker = scope.spawn(move || { + let mut send = |data: &[u8]| { + tx.send(data.to_vec()) + .map_err(|_| io::Error::from(io::ErrorKind::BrokenPipe)) + }; + // knot_git::Repo is not thread-safe. + counted(|| work(&Repo::open(path)?, &mut send)) + }); + let forwarded = loop { + let sent = match rx.recv_timeout(interval) { + Ok(data) => sink(&data), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => sink(b"0005\x01"), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break Ok(()), + }; + if let Err(error) = sent { + break Err(error); + } + }; + // unblocks the worker if the client went away + drop(rx); + let (out, reads) = worker + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)); + // the read counter is per thread + credit(reads); + forwarded?; + out + }) +} + +#[cfg(not(feature = "instrument"))] +fn counted(work: impl FnOnce() -> T) -> (T, ()) { + (work(), ()) +} + +#[cfg(not(feature = "instrument"))] +fn credit((): ()) {} + +struct PackSink<'a> { + side_band: bool, + sink: &'a mut dyn FnMut(&[u8]) -> io::Result<()>, +} + +impl Write for PackSink<'_> { + fn write(&mut self, data: &[u8]) -> io::Result { + if self.side_band { + data.chunks(pkt::MAX_BAND).try_for_each(|chunk| { + let mut framed = Vec::with_capacity(chunk.len() + 5); + pkt::write_band(&mut framed, chunk)?; + (self.sink)(&framed) + })?; + } else { + (self.sink)(data)?; + } + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub fn selection_budget() -> PackBudget { + let limits = selection_limits(); + PackBudget::new(limits.max_objects, limits.time_budget) +} + +fn ensure_wanted(repo: &Repo, wants: &WantOids) -> Result<(), PackError> { + let mut tips: Vec = repo + .advertised_refs_for(knot_git::AdvertScope::Upload)? + .iter() + .map(|record| record.target) + .collect(); + let advertised: HashSet = tips.iter().copied().collect(); + if wants.iter().all(|want| advertised.contains(want)) { + return Ok(()); + } + // commit wants pass on a commit-only walk, cheaper than unioning bitmaps + let commit_closure = repo.reachable_commits(&tips, selection_budget())?; + let unresolved: Vec = wants + .iter() + .copied() + .filter(|want| !advertised.contains(want) && !commit_closure.contains(want)) + .collect(); + if unresolved.is_empty() { + return Ok(()); + } + tips.extend( + repo.references()? + .into_iter() + .filter(|record| matches!(RefClass::of(&record.name), RefClass::Cob | RefClass::Tngl)) + .map(|record| record.target), + ); + // trees and blobs: with a bitmap, not a walk of every object + let unreachable = match knot_git::unreachable_via_bitmap(repo, &tips, &unresolved)? { + Some(unreachable) => unreachable, + None => { + let reachable: HashSet = repo + .select_pack_objects_filtered( + Wants::new(&tips), + Haves::new(&[]), + Filter::None, + selection_budget(), + )? + .send + .into_iter() + .collect(); + unresolved + .into_iter() + .filter(|want| !reachable.contains(want)) + .collect() + } + }; + match unreachable.first() { + Some(hidden) => Err(PackError::Protocol(format!( + "want {hidden} isn't reachable from any public, COB or refs/tngl ref" + ))), + None => Ok(()), + } +} + +fn first_caps<'a>(lines: &[&'a [u8]]) -> Option<&'a str> { + let line = lines.iter().find(|line| line.starts_with(b"want "))?; + let text = std::str::from_utf8(line).ok()?.trim_end(); + text.strip_prefix("want ")? + .split_once(' ') + .map(|(_oid, caps)| caps) +} + +fn parse_wants(lines: &[&[u8]]) -> Result { + lines + .iter() + .filter_map(|line| line.strip_prefix(b"want ")) + .map(|rest| { + let hex = rest + .split(|byte| *byte == b' ' || *byte == b'\n') + .next() + .unwrap_or_default(); + std::str::from_utf8(hex) + .ok() + .and_then(|text| Oid::from_hex(text).ok()) + .ok_or_else(|| PackError::Protocol("malformed want line".to_string())) + }) + .collect::, _>>() + .map(WantOids::new) +} + +fn parse_oids(lines: &[&[u8]], prefix: &[u8]) -> Vec { + lines + .iter() + .filter_map(|line| line.strip_prefix(prefix)) + .filter_map(|rest| { + let hex = rest.split(|byte| *byte == b' ' || *byte == b'\n').next()?; + let hex = std::str::from_utf8(hex).ok()?; + Oid::from_hex(hex).ok() + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn slow_work_is_kept_alive_with_empty_data_bands() { + let dir = tempfile::tempdir().unwrap(); + gix::init_bare(dir.path()).unwrap(); + let repo = Repo::open(dir.path()).unwrap(); + let interval = Duration::from_millis(20); + let mut out = Vec::new(); + let value = with_keepalive( + &repo, + true, + interval, + &mut |chunk| { + out.extend_from_slice(chunk); + Ok(()) + }, + |_, sink| { + sink(b"first")?; + std::thread::sleep(interval * 5); + sink(b"last")?; + Ok(7) + }, + ) + .unwrap(); + assert_eq!(value, 7); + let middle = out + .strip_prefix(b"first") + .and_then(|rest| rest.strip_suffix(b"last")) + .unwrap(); + assert!(middle.len() >= 5 && middle.chunks(5).all(|packet| packet == b"0005\x01")); + } +} diff --git a/knot2/crates/knot-pack/tests/chaos.rs b/knot2/crates/knot-pack/tests/chaos.rs new file mode 100644 index 0000000000000000000000000000000000000000..9abd50e2dd49889ded6f0f73f95f53d4e11c7101 --- /dev/null +++ b/knot2/crates/knot-pack/tests/chaos.rs @@ -0,0 +1,174 @@ +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use knot_git::Layout; +use knot_types::{Oid, RefName, RepoDid}; + +mod common; +use common::{must, pack_objects, receive_request}; + +const DID: &str = "did:plc:squid"; +const BLOB_BYTES: usize = 32 * 1024 * 1024; + +fn incompressible(len: usize) -> Vec { + let mut state = 0x2545_f491_4f6c_dd1du64; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() +} + +struct Seed { + c1: String, + c2: String, + request: std::path::PathBuf, +} + +fn build_seed(scratch: &Path) -> Seed { + let work = scratch.join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + std::fs::write(work.join("base.txt"), "baseline\n").unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", "c1"]); + let c1 = must(&work, &["rev-parse", "HEAD"]); + std::fs::write(work.join("big.bin"), incompressible(BLOB_BYTES)).unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", "c2"]); + let c2 = must(&work, &["rev-parse", "HEAD"]); + + let oids: Vec = must(&work, &["rev-list", "--objects", &c2, "--not", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(&work, &oids); + let request = scratch.join("c2.request"); + std::fs::write( + &request, + receive_request("refs/heads/main", &c1, &c2, &pack), + ) + .unwrap(); + Seed { c1, c2, request } +} + +fn fresh_repo(scratch: &Path, trial: usize, seed: &Seed) -> std::path::PathBuf { + let scan = scratch.join(format!("scan-{trial}")); + let layout = Layout::new(&scan); + let did = RepoDid::new(DID).unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + must( + scratch.join("work").as_path(), + &[ + "push", + "-q", + bare.to_str().unwrap(), + &format!("{}:refs/heads/main", seed.c1), + ], + ); + scan +} + +fn spawn_worker(scan: &Path, request: &Path) -> std::process::Child { + Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "chaos_receive_worker", "--nocapture"]) + .env("KNOT_CHAOS_ROLE", "worker") + .env("KNOT_CHAOS_SCAN", scan) + .env("KNOT_CHAOS_REQUEST", request) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn chaos worker") +} + +fn fsck_clean(bare: &Path) -> Result<(), String> { + knot_fixtures::fsck(bare) +} + +fn main_tip(scan: &Path) -> Option { + let layout = Layout::new(scan); + let repo = layout + .open(&RepoDid::new(DID).unwrap()) + .expect("repo must reopen cleanly after a kill"); + repo.find_ref(&RefName::new("refs/heads/main").unwrap()) + .expect("references must be readable after a kill") +} + +#[test] +fn chaos_receive_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("worker") { + return; + } + let scan = std::env::var("KNOT_CHAOS_SCAN").unwrap(); + let request = std::env::var("KNOT_CHAOS_REQUEST").unwrap(); + let layout = Layout::new(&scan); + let repo = layout.open(&RepoDid::new(DID).unwrap()).unwrap(); + let body = std::fs::read(&request).unwrap(); + let _ = knot_pack::receive_pack(&repo, &body); +} + +#[test] +fn kill9_during_receive_pack_leaves_a_consistent_repo() { + let scratch = tempfile::tempdir().unwrap(); + let seed = build_seed(scratch.path()); + let c1 = Oid::from_hex(&seed.c1).unwrap(); + let c2 = Oid::from_hex(&seed.c2).unwrap(); + let did = RepoDid::new(DID).unwrap(); + + let warm_scan = fresh_repo(scratch.path(), 9000, &seed); + let started = Instant::now(); + let mut warm = spawn_worker(&warm_scan, &seed.request); + warm.wait().unwrap(); + let full = started.elapsed(); + assert_eq!( + main_tip(&warm_scan), + Some(c2), + "uninterrupted receive must fast-forward main to new tip" + ); + + let fractions = [0.30, 0.45, 0.55, 0.62, 0.70, 0.78, 0.85, 0.92, 1.05, 1.25]; + let delays: Vec = std::iter::once(Duration::from_millis(2)) + .chain(std::iter::once(Duration::from_millis(5))) + .chain(fractions.iter().map(|fraction| full.mul_f64(*fraction))) + .chain(std::iter::once(full.mul_f64(2.0))) + .collect(); + + let outcomes: Vec = delays + .iter() + .enumerate() + .map(|(trial, delay)| { + let scan = fresh_repo(scratch.path(), trial, &seed); + let mut child = spawn_worker(&scan, &seed.request); + std::thread::sleep(*delay); + let _ = child.kill(); + child.wait().unwrap(); + + let bare = Layout::new(&scan).repo_path(&did).unwrap(); + fsck_clean(&bare).unwrap_or_else(|errors| { + panic!("trial {trial}: killed receive left a corrupt repo:\n{errors}") + }); + let tip = main_tip(&scan).unwrap_or_else(|| { + panic!("trial {trial}: main vanished after a kill, acknowledged ref was lost") + }); + assert!( + tip == c1 || tip == c2, + "trial {trial}: main must hold either acknowledged baseline or completed tip, never a torn value, got {tip}" + ); + tip + }) + .collect(); + + assert!( + outcomes.contains(&c1), + "no trial was interrupted before ref update; chaos window never opened" + ); + assert!( + outcomes.contains(&c2), + "no trial ran to completion; receive never finished under chosen delays" + ); +} diff --git a/knot2/crates/knot-pack/tests/common/mod.rs b/knot2/crates/knot-pack/tests/common/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..9707818e5168dfceab4f5d78dc8fea5607e8aa29 --- /dev/null +++ b/knot2/crates/knot-pack/tests/common/mod.rs @@ -0,0 +1,332 @@ +#![allow(dead_code, unused_imports)] + +use std::collections::BTreeSet; +use std::io::Write; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::Arc; + +use axum::Router; +use knot_git::{Layout, Repo}; +use knot_pack::{RepoLookup, RepoResolver, RepoTarget}; +use knot_types::{ObjectFormat, RepoDid}; + +pub use knot_fixtures::{commit, contains, must, run as git}; + +pub fn pkt(payload: &[u8]) -> Vec { + let mut out = format!("{:04x}", payload.len() + 4).into_bytes(); + out.extend_from_slice(payload); + out +} + +pub fn pack_objects(cwd: &Path, oids: &[String]) -> Vec { + let mut child = knot_fixtures::command(cwd) + .args(["pack-objects", "--stdout", "-q"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn pack-objects"); + child + .stdin + .take() + .unwrap() + .write_all(oids.join("\n").as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "pack-objects failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr) + ); + out.stdout +} + +pub fn pack_objects_tuned(cwd: &Path, oids: &[String], ofs: bool) -> Vec { + let args: &[&str] = if ofs { + &[ + "pack-objects", + "--stdout", + "-q", + "--delta-base-offset", + "--depth=50", + "--window=250", + ] + } else { + &[ + "-c", + "pack.useDeltaBaseOffset=false", + "pack-objects", + "--stdout", + "-q", + "--depth=50", + "--window=250", + ] + }; + let mut child = knot_fixtures::command(cwd) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn pack-objects"); + child + .stdin + .take() + .unwrap() + .write_all(oids.join("\n").as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "pack-objects failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr) + ); + out.stdout +} + +pub fn index_into_bare(extra: &[&str], pack: &[u8]) -> bool { + let bare = tempfile::tempdir().unwrap(); + knot_fixtures::must( + bare.path(), + &["init", "--bare", "-q", bare.path().to_str().unwrap()], + ); + let args: Vec<&str> = std::iter::once("index-pack") + .chain(extra.iter().copied()) + .chain(std::iter::once("--stdin")) + .collect(); + knot_fixtures::feed(bare.path(), &args, pack).0 +} + +fn zlib(data: &[u8]) -> Vec { + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() +} + +fn base128(value: u64) -> Vec { + let low = (value & 0x7f) as u8; + let rest = value >> 7; + if rest == 0 { + vec![low] + } else { + std::iter::once(low | 0x80).chain(base128(rest)).collect() + } +} + +fn obj_header(obj_type: u8, size: usize) -> Vec { + fn tail(size: usize) -> Vec { + if size == 0 { + Vec::new() + } else { + let byte = (size & 0x7f) as u8; + let rest = size >> 7; + let cont = if rest > 0 { 0x80 } else { 0 }; + std::iter::once(byte | cont).chain(tail(rest)).collect() + } + } + let rest = size >> 4; + let cont = if rest > 0 { 0x80 } else { 0 }; + std::iter::once((obj_type << 4) | (size & 0x0f) as u8 | cont) + .chain(tail(rest)) + .collect() +} + +fn ofs_distance(distance: u64) -> Vec { + fn prefix(value: u64) -> Vec { + if value == 0 { + Vec::new() + } else { + let reduced = value - 1; + prefix(reduced >> 7) + .into_iter() + .chain(std::iter::once(0x80 | (reduced & 0x7f) as u8)) + .collect() + } + } + prefix(distance >> 7) + .into_iter() + .chain(std::iter::once((distance & 0x7f) as u8)) + .collect() +} + +pub fn delta_bomb_pack(declared_result_bytes: u64) -> Vec { + let base = b"hi"; + let mut entry0 = obj_header(3, base.len()); + entry0.extend(zlib(base)); + + let delta_stream: Vec = base128(base.len() as u64) + .into_iter() + .chain(base128(declared_result_bytes)) + .chain([0x90, 0x02]) + .collect(); + let mut entry1 = obj_header(6, delta_stream.len()); + entry1.extend(ofs_distance(entry0.len() as u64)); + entry1.extend(zlib(&delta_stream)); + + let mut pack = b"PACK".to_vec(); + pack.extend_from_slice(&2u32.to_be_bytes()); + pack.extend_from_slice(&2u32.to_be_bytes()); + pack.extend_from_slice(&entry0); + pack.extend_from_slice(&entry1); + + let mut hasher = gix_hash::hasher(gix_hash::Kind::Sha1); + hasher.update(&pack); + let checksum = hasher.try_finalize().unwrap(); + pack.extend_from_slice(checksum.as_bytes()); + pack +} + +pub fn receive_request(refname: &str, old: &str, new: &str, pack: &[u8]) -> Vec { + let mut first = format!("{old} {new} {refname}").into_bytes(); + first.push(0); + first.extend_from_slice(b"report-status\n"); + let mut req = pkt(&first); + req.extend_from_slice(b"0000"); + req.extend_from_slice(pack); + req +} + +pub fn unsideband(resp: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut pos = 0usize; + while pos + 4 <= resp.len() { + let len = std::str::from_utf8(&resp[pos..pos + 4]) + .ok() + .and_then(|hex| usize::from_str_radix(hex, 16).ok()) + .unwrap_or(0); + pos += 4; + if len < 4 { + continue; + } + let end = (pos + len - 4).min(resp.len()); + let payload = &resp[pos..end]; + pos = end; + if payload.first() == Some(&1) { + out.extend_from_slice(&payload[1..]); + } + } + out +} + +pub fn object_set(dir: &Path) -> BTreeSet { + must( + dir, + &[ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname)", + ], + ) + .lines() + .map(|line| line.trim().to_string()) + .filter(|line| !line.is_empty()) + .collect() +} + +pub fn serve_dids() -> Arc { + Arc::new(|target: &RepoTarget| match target { + RepoTarget::Did(did) => RepoLookup::Hosted(did.clone()), + RepoTarget::OwnerPath(_, _) => RepoLookup::Unhosted, + }) +} + +pub async fn spawn(router: Router, bind: &str) -> SocketAddr { + let listener = tokio::net::TcpListener::bind(bind).await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + addr +} + +pub fn advance_via_receive(bare: &Path, work: &Path, old: &str, new: &str) { + let oids: Vec = must(work, &["rev-list", "--objects", new, "--not", old]) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .map(str::to_string) + .collect(); + let request = receive_request("refs/heads/main", old, new, &pack_objects(work, &oids)); + let repo = Repo::open(bare).expect("open knot bare"); + let report = knot_pack::receive_pack(&repo, &request).expect("knot receive"); + assert!( + String::from_utf8_lossy(&report).contains("ok refs/heads/main"), + "knot must accept a receive that advances main" + ); +} + +pub fn seed_branches_and_tag(work: &Path, bares: [&Path; 2], format: ObjectFormat) { + let fmt = format!("--object-format={}", format.capability()); + std::fs::create_dir_all(work).unwrap(); + must(work, &["init", &fmt, "-q", "-b", "main"]); + std::fs::write(work.join("README.md"), "seed\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c1"]); + let c1 = must(work, &["rev-parse", "HEAD"]); + std::fs::write(work.join("src.txt"), "more\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c2"]); + must(work, &["checkout", "-q", "-b", "dev", &c1]); + std::fs::write(work.join("dev.txt"), "branch\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c3"]); + must(work, &["checkout", "-q", "main"]); + must(work, &["tag", "-a", "v1", "-m", "release"]); + bares.into_iter().for_each(|bare| { + must( + work, + &["push", "-q", bare.to_str().unwrap(), "main", "dev", "v1"], + ); + must(bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); + }); +} + +pub fn seeded(layout: &Layout, did: &RepoDid) -> (Repo, tempfile::TempDir, String, Vec) { + let bare = layout.create(did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "x\n", "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + let oids: Vec = must(work, &["rev-list", "--objects", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(work, &oids); + (bare, work_dir, c1, pack) +} + +pub struct Stand { + pub scan: tempfile::TempDir, + pub scratch: tempfile::TempDir, + pub layout: Layout, + pub addr: SocketAddr, + pub bare: PathBuf, +} + +pub async fn stand(did: &RepoDid) -> Stand { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + layout.create(did).unwrap(); + let bare = layout.repo_path(did).unwrap(); + let addr = spawn( + knot_pack::router( + layout.clone(), + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + let scratch = tempfile::tempdir().unwrap(); + Stand { + scan, + scratch, + layout, + addr, + bare, + } +} diff --git a/knot2/crates/knot-pack/tests/differential.rs b/knot2/crates/knot-pack/tests/differential.rs new file mode 100644 index 0000000000000000000000000000000000000000..659c94b6c9ffebeedef2112775f8e3319798aa03 --- /dev/null +++ b/knot2/crates/knot-pack/tests/differential.rs @@ -0,0 +1,692 @@ +use std::collections::{BTreeSet, HashMap}; +use std::path::Path; + +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::{DefaultBodyLimit, Path as UrlPath, Query, State}; +use axum::http::header; +use axum::response::Response; +use axum::routing::{get, post}; +use knot_git::Layout; +use knot_pack::PackError; +use knot_types::{ObjectFormat, RepoDid, RepoRkey}; + +mod common; +use common::{ + advance_via_receive, git, must, object_set, seed_branches_and_tag, serve_dids, spawn, +}; + +#[derive(Clone)] +struct Receive { + layout: Layout, +} + +fn open(layout: &Layout, did: &str, name: &str) -> Result { + let did = RepoDid::new(did).map_err(|e| PackError::BadPath(e.to_string()))?; + RepoRkey::new(name).map_err(|e| PackError::BadPath(e.to_string()))?; + Ok(layout.open(&did)?) +} + +fn raw_response(content_type: &'static str, body: Vec) -> Response { + Response::builder() + .header(header::CONTENT_TYPE, content_type) + .body(Body::from(body)) + .unwrap() +} + +async fn receive_info_refs( + State(state): State, + UrlPath((did, name)): UrlPath<(String, String)>, + Query(query): Query>, +) -> Result { + let repo = open(&state.layout, &did, &name)?; + match query.get("service").map(String::as_str) { + Some("git-upload-pack") => Ok(raw_response( + "application/x-git-upload-pack-advertisement", + knot_pack::advertise_upload(&repo)?, + )), + Some("git-receive-pack") => Ok(raw_response( + "application/x-git-receive-pack-advertisement", + knot_pack::advertise_receive(&repo)?, + )), + _ => Err(PackError::UnsupportedService), + } +} + +async fn receive_upload( + State(state): State, + UrlPath((did, name)): UrlPath<(String, String)>, + body: Bytes, +) -> Result { + let repo = open(&state.layout, &did, &name)?; + let result = knot_pack::upload_pack(&repo, &body)?; + Ok(raw_response("application/x-git-upload-pack-result", result)) +} + +async fn receive_receive( + State(state): State, + UrlPath((did, name)): UrlPath<(String, String)>, + body: Bytes, +) -> Result { + let repo = open(&state.layout, &did, &name)?; + let result = knot_pack::receive_pack(&repo, &body)?; + Ok(raw_response( + "application/x-git-receive-pack-result", + result, + )) +} + +fn knot_receive_router(layout: Layout) -> Router { + Router::new() + .route("/{did}/{name}/info/refs", get(receive_info_refs)) + .route("/{did}/{name}/git-upload-pack", post(receive_upload)) + .route("/{did}/{name}/git-receive-pack", post(receive_receive)) + .layer(DefaultBodyLimit::disable()) + .with_state(Receive { layout }) +} + +fn clone_to(scratch: &Path, url: &str, dest: &Path) { + must(scratch, &["clone", "-q", url, dest.to_str().unwrap()]); +} + +fn publish(work: &Path, bares: [&Path; 2], refs: &[&str], head: &str) { + bares.into_iter().for_each(|bare| { + let push = [&["push", "-q", bare.to_str().unwrap()], refs].concat(); + must(work, &push); + must(bare, &["symbolic-ref", "HEAD", head]); + }); +} + +fn seed_cross_branch_deltas(work: &Path, knot_bare: &Path, canon_bare: &Path) { + std::fs::create_dir_all(work).unwrap(); + must(work, &["init", "-q", "-b", "base"]); + std::fs::write(work.join("README.md"), "thin\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "base"]); + let base = must(work, &["rev-parse", "HEAD"]); + + let bulk = |stem: usize, extra: &str| { + let body: String = (0..200) + .map(|line| format!("file {stem} line {line} with some shared payload\n")) + .collect(); + format!("{body}{extra}") + }; + let branch = |name: &str, extra: &str| { + must(work, &["checkout", "-q", "-b", name, &base]); + (0..8).for_each(|stem| { + std::fs::write(work.join(format!("bulk-{stem}.txt")), bulk(stem, extra)).unwrap(); + }); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", name]); + }; + branch("side", ""); + branch("other", "one divergent trailing line\n"); + must(work, &["checkout", "-q", "base"]); + + publish( + work, + [knot_bare, canon_bare], + &["base", "side", "other"], + "refs/heads/base", + ); + [knot_bare, canon_bare].into_iter().for_each(|bare| { + must(bare, &["repack", "-adf", "--window=50", "--depth=50"]); + }); +} + +fn seed_nested_bares(work: &Path, knot_bare: &Path, canon_bare: &Path, format: ObjectFormat) { + let fmt = format!("--object-format={}", format.capability()); + std::fs::create_dir_all(work).unwrap(); + must(work, &["init", &fmt, "-q", "-b", "main"]); + std::fs::create_dir_all(work.join("a/c")).unwrap(); + std::fs::create_dir_all(work.join("b")).unwrap(); + std::fs::create_dir_all(work.join("deep/mid/bottom")).unwrap(); + std::fs::write(work.join("root.txt"), b"01234567").unwrap(); + std::fs::write(work.join("a/c/leaf"), b"hi\n").unwrap(); + std::fs::write(work.join("b/leaf"), b"hi\n").unwrap(); + std::fs::write(work.join("deep/mid/bottom/far.txt"), "x".repeat(20)).unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c1"]); + std::fs::write(work.join("deep/mid/bottom/far.txt"), "y".repeat(40)).unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c2"]); + publish(work, [knot_bare, canon_bare], &["main"], "refs/heads/main"); +} + +#[derive(Clone, Copy, PartialEq)] +enum Mode { + Serve, + Receive, +} + +struct Servers { + canon_bare: std::path::PathBuf, + knot_bare: std::path::PathBuf, + canon_url: String, + knot_url: String, +} + +async fn stand_up( + scan: &Path, + canon_root: &Path, + did: &RepoDid, + name: &RepoRkey, + format: ObjectFormat, + mode: Mode, +) -> Servers { + let layout = Layout::new(scan).with_object_format(format); + layout.create(did).unwrap(); + let knot_bare = layout.repo_path(did).unwrap(); + + let canon_bare = canon_root.join(format!("{}.git", name.as_str())); + let fmt = format!("--object-format={}", format.capability()); + must( + Path::new("/tmp"), + &["init", "--bare", &fmt, "-q", canon_bare.to_str().unwrap()], + ); + + let knot = match mode { + Mode::Serve => { + spawn( + knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await + } + Mode::Receive => spawn(knot_receive_router(layout), "[::1]:0").await, + }; + let knot_url = match mode { + Mode::Serve => format!("http://{knot}/{}", did.as_str()), + Mode::Receive => format!("http://{knot}/{}/{}", did.as_str(), name.as_str()), + }; + Servers { + canon_url: format!("file://{}", canon_bare.to_str().unwrap()), + canon_bare, + knot_bare, + knot_url, + } +} + +fn single_pack_idx(bare: &Path) -> std::path::PathBuf { + std::fs::read_dir(bare.join("objects/pack")) + .unwrap() + .filter_map(Result::ok) + .map(|entry| entry.path()) + .find(|path| path.extension().is_some_and(|ext| ext == "idx")) + .expect("a single pack index after repack") +} + +async fn clone_lifecycle(format: ObjectFormat, did: &str, name: &str) { + let scan = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + let did = RepoDid::new(did).unwrap(); + let name = RepoRkey::new(name).unwrap(); + let s = stand_up( + scan.path(), + canon_root.path(), + &did, + &name, + format, + Mode::Serve, + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_branches_and_tag(&work, [&s.knot_bare, &s.canon_bare], format); + + let canon = scratch.path().join("clone-canon"); + let knot = scratch.path().join("clone-knot"); + clone_to(scratch.path(), &s.canon_url, &canon); + clone_to(scratch.path(), &s.knot_url, &knot); + let parse = |dir: &Path, rev: &str| must(dir, &["rev-parse", rev]); + assert_eq!( + parse(&knot, "--show-object-format"), + format.capability(), + "{format:?} clone inherits object format" + ); + assert_eq!( + parse(&canon, "HEAD^{tree}"), + parse(&knot, "HEAD^{tree}"), + "{format:?} same checked-out tree" + ); + assert_eq!( + must(&canon, &["ls-files", "-s"]), + must(&knot, &["ls-files", "-s"]), + "{format:?} identical working tree" + ); + assert_eq!( + object_set(&canon), + object_set(&knot), + "{format:?} clone transfers canonical object set" + ); + assert_eq!( + must(&canon, &["branch", "-r"]), + must(&knot, &["branch", "-r"]), + "{format:?} same remote branches" + ); + + must( + &s.knot_bare, + &["-c", "repack.writeBitmaps=false", "repack", "-adq"], + ); + let repo = knot_git::Repo::open(&s.knot_bare).unwrap(); + assert!( + knot_git::write_bitmap(&repo, &single_pack_idx(&s.knot_bare), None).unwrap(), + "{format:?} single-pack repo gets a bitmap" + ); + must(&s.knot_bare, &["rev-list", "--test-bitmap", "HEAD"]); + + let bm_canon = scratch.path().join("reclone-canon"); + let bm_knot = scratch.path().join("reclone-knot"); + clone_to(scratch.path(), &s.canon_url, &bm_canon); + clone_to(scratch.path(), &s.knot_url, &bm_knot); + assert_eq!( + object_set(&bm_canon), + object_set(&bm_knot), + "{format:?} bitmap fast path serves canonical object set" + ); + must(&bm_knot, &["fsck", "--strict"]); + + std::fs::write(work.join("incremental.txt"), "fetch me\n").unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", "c4"]); + let tip = must(&work, &["rev-parse", "HEAD"]); + let old = must(&work, &["rev-parse", "HEAD~1"]); + advance_via_receive(&s.knot_bare, &work, &old, &tip); + must( + &work, + &["push", "-q", s.canon_bare.to_str().unwrap(), "main"], + ); + must(&canon, &["fetch", "-q", "origin"]); + must(&knot, &["fetch", "-q", "origin"]); + assert_eq!( + must(&knot, &["rev-parse", "origin/main"]), + tip, + "{format:?} fetch advances origin/main" + ); + assert_eq!( + object_set(&canon), + object_set(&knot), + "{format:?} incremental fetch transfers canonical object set" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn differential_clone_fetch_and_bitmap_serving_match_canonical() { + clone_lifecycle(ObjectFormat::SHA1, "did:plc:squid", "scallop").await; + clone_lifecycle(ObjectFormat::SHA256, "did:plc:nautilus", "whelk").await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_shallow_server_repo_is_served_and_the_clone_becomes_shallow() { + let scan = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + let did = RepoDid::new("did:plc:periwinkle").unwrap(); + let name = RepoRkey::new("conch").unwrap(); + let s = stand_up( + scan.path(), + canon_root.path(), + &did, + &name, + ObjectFormat::SHA1, + Mode::Serve, + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + (1..=5).for_each(|n| { + std::fs::write(work.join("history.txt"), format!("revision {n}\n")).unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", &format!("c{n}")]); + }); + let work_url = format!("file://{}", work.to_str().unwrap()); + [&s.knot_bare, &s.canon_bare].into_iter().for_each(|bare| { + must( + bare, + &[ + "fetch", + "-q", + "--depth=2", + &work_url, + "main:refs/heads/main", + ], + ); + must(bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); + assert!(bare.join("shallow").exists(), "server repo is shallow"); + }); + + let canon = scratch.path().join("clone-canon"); + let knot = scratch.path().join("clone-knot"); + clone_to(scratch.path(), &s.canon_url, &canon); + clone_to(scratch.path(), &s.knot_url, &knot); + assert!( + knot.join(".git/shallow").exists(), + "clone of a shallow server is itself shallow" + ); + must(&knot, &["fsck"]); + assert_eq!( + must(&canon, &["rev-list", "--count", "HEAD"]), + must(&knot, &["rev-list", "--count", "HEAD"]), + "both clones see the same clamped depth" + ); + assert_eq!( + object_set(&canon), + object_set(&knot), + "shallow clone transfers canonical object set" + ); +} + +fn fetch_branch(scratch: &Path, url: &str, label: &str, branch: &str) -> BTreeSet { + let clone = scratch.join(format!("clone-{label}-{branch}")); + must( + scratch, + &[ + "clone", + "-q", + "--single-branch", + "--branch", + "base", + url, + clone.to_str().unwrap(), + ], + ); + must(&clone, &["fetch", "-q", "origin", branch]); + must(&clone, &["fsck", "--strict"]); + object_set(&clone) +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_thin_fetch_never_deltas_against_objects_the_client_lacks() { + let scan = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + let did = RepoDid::new("did:plc:limpet").unwrap(); + let name = RepoRkey::new("whelk").unwrap(); + let s = stand_up( + scan.path(), + canon_root.path(), + &did, + &name, + ObjectFormat::SHA1, + Mode::Serve, + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + seed_cross_branch_deltas(&scratch.path().join("work"), &s.knot_bare, &s.canon_bare); + + ["side", "other"].into_iter().for_each(|branch| { + assert_eq!( + fetch_branch(scratch.path(), &s.canon_url, "canon", branch), + fetch_branch(scratch.path(), &s.knot_url, "knot", branch), + "thin fetch of {branch} onto a base-only clone transfers canonical object set" + ); + }); +} + +fn push_sequence(remote: &str, scratch: &Path, label: &str) -> Vec<(String, bool)> { + let clone = scratch.join(format!("push-{label}")); + must(scratch, &["clone", "-q", remote, clone.to_str().unwrap()]); + let commit = |file: &str, body: &str, msg: &str| { + std::fs::write(clone.join(file), body).unwrap(); + must(&clone, &["add", "-A"]); + must(&clone, &["commit", "-q", "-m", msg]); + }; + let push = |args: &[&str]| git(&clone, &[&["push", "-q", "origin"], args].concat()).0; + + commit("ff.txt", "fast forward\n", "fast forward"); + let ff = push(&["main"]); + must(&clone, &["checkout", "-q", "-b", "feature"]); + commit("feature.txt", "new branch\n", "feature"); + let new_branch = push(&["feature"]); + let delete = push(&["--delete", "feature"]); + must(&clone, &["checkout", "-q", "main"]); + must(&clone, &["reset", "-q", "--hard", "HEAD~1"]); + commit("diverge.txt", "non fast forward\n", "diverge"); + let non_ff = push(&["main"]); + + vec![ + ("fast-forward".to_string(), ff), + ("new-branch".to_string(), new_branch), + ("delete-branch".to_string(), delete), + ("non-fast-forward".to_string(), non_ff), + ] +} + +async fn push_diff(format: ObjectFormat, did: &str, name: &str) { + let scan = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + let did = RepoDid::new(did).unwrap(); + let name = RepoRkey::new(name).unwrap(); + let s = stand_up( + scan.path(), + canon_root.path(), + &did, + &name, + format, + Mode::Receive, + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + seed_branches_and_tag( + &scratch.path().join("work"), + [&s.knot_bare, &s.canon_bare], + format, + ); + + let advertised = |bare: &Path| { + must( + bare, + &[ + "for-each-ref", + "--format=%(refname) %(objectname)", + "refs/heads/", + "refs/tags/", + ], + ) + }; + assert_eq!( + push_sequence(&s.canon_url, scratch.path(), "canon"), + push_sequence(&s.knot_url, scratch.path(), "knot"), + "{format:?} knot accepts and rejects the same pushes as canonical git" + ); + assert_eq!( + advertised(&s.canon_bare), + advertised(&s.knot_bare), + "{format:?} both servers hold the same refs after an identical push sequence" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn differential_push_verdicts_match_canonical() { + push_diff(ObjectFormat::SHA1, "did:plc:squid", "barnacle").await; + push_diff(ObjectFormat::SHA256, "did:plc:cuttle", "scallop").await; +} + +fn enable_filter(bare: &Path) { + must(bare, &["config", "uploadpack.allowFilter", "true"]); + must(bare, &["config", "uploadpack.allowAnySHA1InWant", "true"]); +} + +fn filtered_clone(scratch: &Path, url: &str, label: &str, filter: &str) -> BTreeSet { + let clone = scratch.join(format!("clone-{label}")); + must( + scratch, + &[ + "clone", + "-q", + "--no-checkout", + &format!("--filter={filter}"), + url, + clone.to_str().unwrap(), + ], + ); + object_set(&clone) +} + +fn fetched_set( + scratch: &Path, + url: &str, + label: &str, + oid: &str, + filter: &str, + format: ObjectFormat, +) -> BTreeSet { + let dest = scratch.join(format!("fetch-{label}")); + let fmt = format!("--object-format={}", format.capability()); + must(scratch, &["init", &fmt, "-q", dest.to_str().unwrap()]); + must( + &dest, + &["fetch", "-q", &format!("--filter={filter}"), url, oid], + ); + object_set(&dest) +} + +fn blobless_checkout(scratch: &Path, url: &str, label: &str) -> BTreeSet { + let clone = scratch.join(format!("clone-{label}")); + must( + scratch, + &[ + "clone", + "-q", + "--filter=blob:none", + url, + clone.to_str().unwrap(), + ], + ); + must(&clone, &["fsck", "--strict"]); + object_set(&clone) +} + +async fn partial_clone(format: ObjectFormat, did: &str, name: &str) { + let scan = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + let did = RepoDid::new(did).unwrap(); + let name = RepoRkey::new(name).unwrap(); + let s = stand_up( + scan.path(), + canon_root.path(), + &did, + &name, + format, + Mode::Serve, + ) + .await; + enable_filter(&s.canon_bare); + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_nested_bares(&work, &s.knot_bare, &s.canon_bare, format); + + [ + "tree:0", + "tree:1", + "tree:2", + "tree:3", + "blob:none", + "blob:limit=3", + "blob:limit=4", + "blob:limit=20", + ] + .into_iter() + .for_each(|filter| { + assert_eq!( + filtered_clone( + scratch.path(), + &s.canon_url, + &format!("canon-{filter}"), + filter + ), + filtered_clone( + scratch.path(), + &s.knot_url, + &format!("knot-{filter}"), + filter + ), + "{format:?} a --filter={filter} clone transfers canonical object set" + ); + }); + + let deep = must(&work, &["rev-parse", "HEAD:deep"]); + let far = must(&work, &["rev-parse", "HEAD:deep/mid/bottom/far.txt"]); + [ + (&deep, "tree:0"), + (&deep, "tree:1"), + (&deep, "tree:2"), + (&deep, "tree:3"), + (&deep, "blob:none"), + (&far, "blob:none"), + ] + .into_iter() + .enumerate() + .for_each(|(case, (oid, filter))| { + assert_eq!( + fetched_set( + scratch.path(), + &s.canon_url, + &format!("canon-{case}"), + oid, + filter, + format + ), + fetched_set( + scratch.path(), + &s.knot_url, + &format!("knot-{case}"), + oid, + filter, + format + ), + "{format:?} explicit want {oid} --filter={filter} transfers canonical object set" + ); + }); + + assert_eq!( + blobless_checkout(scratch.path(), &s.canon_url, "canon"), + blobless_checkout(scratch.path(), &s.knot_url, "knot"), + "{format:?} a blobless clone faults its checkout blobs in like canonical git" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn differential_partial_clone_matches_canonical() { + partial_clone(ObjectFormat::SHA1, "did:plc:anemone", "barnacle").await; + partial_clone(ObjectFormat::SHA256, "did:plc:cuttle", "uni").await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn an_empty_sha256_repo_clones_to_sha256_over_v2_and_v0() { + let scan = tempfile::tempdir().unwrap(); + let did = RepoDid::new("did:plc:limpet").unwrap(); + let layout = Layout::new(scan.path()).with_object_format(ObjectFormat::SHA256); + layout.create(&did).unwrap(); + let knot = spawn( + knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + let url = format!("http://{knot}/{}", did.as_str()); + + let scratch = tempfile::tempdir().unwrap(); + [("v2", "2"), ("v0", "0")].into_iter().for_each(|(label, version)| { + let dst = scratch.path().join(format!("clone-{label}")); + must(scratch.path(), &["-c", &format!("protocol.version={version}"), "clone", "-q", &url, dst.to_str().unwrap()]); + assert_eq!( + must(&dst, &["rev-parse", "--show-object-format"]), + "sha256", + "{label}: an empty knot sha256 repo clones to sha256 from its advertised capabilities" + ); + }); +} diff --git a/knot2/crates/knot-pack/tests/fetch.rs b/knot2/crates/knot-pack/tests/fetch.rs new file mode 100644 index 0000000000000000000000000000000000000000..7ad7c8e65a52eb3f00a677501b38932c8a6b8e94 --- /dev/null +++ b/knot2/crates/knot-pack/tests/fetch.rs @@ -0,0 +1,425 @@ +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::Arc; + +use axum::http; +use knot_git::{Layout, Repo, Staging}; +use knot_pack::{ + FetchError, HaveOids, MaxWireBytes, PackLimits, WantOids, ingest_pack, local_pack, local_refs, +}; +use knot_runtime::{FakeHttp, HttpRequest, HttpResponse, HttpTransport}; +use knot_types::{Oid, RefName, RepoDid}; +use url::Url; + +mod common; +use common::{commit, must, pack_objects}; + +fn seed_source(dir: &Path) -> PathBuf { + let work = dir.join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + commit(&work, "reef.txt", "kelp forest\n", "first"); + commit(&work, "tide.txt", "rock pool\n", "second"); + must(&work, &["tag", "v1"]); + must(&work, &["branch", "anemone"]); + let bare = dir.join("source.git"); + must( + dir, + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + ); + bare +} + +fn stock_git_server(repo: PathBuf) -> Arc { + Arc::new(FakeHttp::new(move |request: &HttpRequest| { + let response = |body: Vec| { + Ok(HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: body.into(), + }) + }; + if request.url.path().ends_with("/info/refs") { + let out = knot_fixtures::command(&repo) + .args([ + "upload-pack", + "--stateless-rpc", + "--http-backend-info-refs", + ".", + ]) + .env("GIT_PROTOCOL", "version=2") + .output() + .expect("git upload-pack advertises"); + assert!(out.status.success()); + let mut body = b"001e# service=git-upload-pack\n0000".to_vec(); + body.extend_from_slice(&out.stdout); + return response(body); + } + let mut child = knot_fixtures::command(&repo) + .args(["upload-pack", "--stateless-rpc", "."]) + .env("GIT_PROTOCOL", "version=2") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("git upload-pack serves"); + child + .stdin + .take() + .unwrap() + .write_all(request.body.as_deref().unwrap_or_default()) + .unwrap(); + let out = child.wait_with_output().expect("git upload-pack finishes"); + assert!( + out.status.success(), + "upload-pack: {}", + String::from_utf8_lossy(&out.stderr) + ); + response(out.stdout) + })) +} + +fn knot_server(repo_path: PathBuf) -> Arc { + Arc::new(FakeHttp::new(move |request: &HttpRequest| { + let repo = Repo::open(&repo_path).unwrap(); + let body = if request.url.path().ends_with("/info/refs") { + knot_pack::advertise_upload(&repo).unwrap() + } else { + knot_pack::upload_pack(&repo, request.body.as_deref().unwrap_or_default()).unwrap() + }; + Ok(HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: body.into(), + }) + })) +} + +struct ChunkedHttp { + inner: Arc, + chunk_bytes: usize, +} + +impl HttpTransport for ChunkedHttp { + fn execute(&self, request: HttpRequest) -> knot_runtime::HttpFuture { + self.inner.execute(request) + } + + fn execute_streamed(&self, request: HttpRequest) -> knot_runtime::StreamFuture { + let inner = Arc::clone(&self.inner); + let chunk_bytes = self.chunk_bytes; + Box::pin(async move { + let response = inner.execute(request).await?; + let body = response.body; + let chunks: Vec> = body + .chunks(chunk_bytes) + .map(|chunk| Ok(axum::body::Bytes::copy_from_slice(chunk))) + .collect(); + Ok(knot_runtime::StreamedResponse { + status: response.status, + headers: response.headers, + body: Box::pin(tokio_stream::iter(chunks)), + }) + }) + } +} + +fn base_url() -> Url { + Url::parse("https://kelp.oyster.cafe/did:plc:squid/uni").unwrap() +} + +const CAP: MaxWireBytes = MaxWireBytes::new(64 * 1024 * 1024); + +fn refnames(records: &[knot_git::RefRecord]) -> Vec<&str> { + records.iter().map(|record| record.name.as_str()).collect() +} + +async fn clone_through(http: &dyn HttpTransport, source: &Repo, target: &Repo) { + let refs = knot_pack::remote_refs(http, &base_url(), &["HEAD", "refs/heads/", "refs/tags/"]) + .await + .unwrap(); + assert_eq!( + refs.head_symref.as_ref().map(RefName::as_str), + Some("refs/heads/main") + ); + let mut expected = source.advertised_refs().unwrap().to_vec(); + expected.sort_by(|a, b| a.name.as_str().cmp(b.name.as_str())); + let mut got = refs.refs.clone(); + got.sort_by(|a, b| a.name.as_str().cmp(b.name.as_str())); + assert_eq!(refnames(&got), refnames(&expected)); + assert_eq!(got, expected); + + let pack = knot_pack::remote_pack( + http, + &base_url(), + &WantOids::new(refs.tips()), + &HaveOids::default(), + refs.object_format, + CAP, + ) + .await + .unwrap(); + ingest_pack( + &target.objects_dir(), + &pack, + &PackLimits::default(), + target.object_format().kind(), + ) + .unwrap(); + let closure = target + .select_pack_objects( + knot_git::Wants::new(&refs.tips()), + knot_git::Haves::new(&[]), + ) + .unwrap(); + assert!(closure.iter().all(|oid| target.contains(*oid))); +} + +#[tokio::test] +async fn the_client_clones_from_both_stock_git_and_native_knot_servers() { + let dir = tempfile::tempdir().unwrap(); + let source_path = seed_source(dir.path()); + let source = Repo::open(&source_path).unwrap(); + + let stock = Repo::create(dir.path().join("fork-stock.git")).unwrap(); + clone_through( + stock_git_server(source_path.clone()).as_ref(), + &source, + &stock, + ) + .await; + + let knot = Repo::create(dir.path().join("fork-knot.git")).unwrap(); + clone_through(knot_server(source_path.clone()).as_ref(), &source, &knot).await; +} + +#[tokio::test] +async fn an_incremental_pull_completes_through_staging() { + let dir = tempfile::tempdir().unwrap(); + let source_path = seed_source(dir.path()); + let source = Repo::open(&source_path).unwrap(); + let clone = Repo::create(dir.path().join("fork.git")).unwrap(); + let http = knot_server(source_path.clone()); + clone_through(http.as_ref(), &source, &clone).await; + let main = RefName::new("refs/heads/main").unwrap(); + let old_tip = source.find_ref(&main).unwrap().unwrap(); + clone + .update_ref(&knot_git::RefUpdate::Create { + name: main.clone(), + new: old_tip, + }) + .unwrap(); + + let work = dir.path().join("work"); + commit(&work, "spray.txt", "salt\n", "third"); + let new_hex = must(&work, &["rev-parse", "HEAD"]); + let new_tip = Oid::from_hex(&new_hex).unwrap(); + let oids: Vec = must( + &work, + &[ + "rev-list", + "--objects", + &new_hex, + "--not", + &old_tip.to_hex(), + ], + ) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .map(str::to_string) + .collect(); + let pack = pack_objects(&work, &oids); + ingest_pack( + &source.objects_dir(), + &pack, + &PackLimits::default(), + source.object_format().kind(), + ) + .unwrap(); + source + .update_ref(&knot_git::RefUpdate::Update { + name: main.clone(), + old: old_tip, + new: new_tip, + }) + .unwrap(); + assert_ne!(old_tip, new_tip); + + let pack = knot_pack::remote_pack( + http.as_ref(), + &base_url(), + &WantOids::new(vec![new_tip]), + &HaveOids::new(vec![old_tip]), + clone.object_format(), + CAP, + ) + .await + .unwrap(); + let staging = Staging::new(&clone).unwrap(); + ingest_pack( + &staging.repo().objects_dir(), + &pack, + &PackLimits::default(), + clone.object_format().kind(), + ) + .unwrap(); + let closure = staging + .repo() + .select_pack_objects( + knot_git::Wants::new(&[new_tip]), + knot_git::Haves::new(&[old_tip]), + ) + .unwrap(); + assert!(closure.iter().all(|oid| staging.repo().contains(*oid))); + staging.migrate_into(&clone).unwrap(); + assert!(clone.contains(new_tip)); +} + +#[tokio::test] +async fn a_tiny_pack_limit_refuses_both_remote_and_local_transfers() { + let dir = tempfile::tempdir().unwrap(); + let source_path = seed_source(dir.path()); + let source = Repo::open(&source_path).unwrap(); + let tips: Vec = source + .advertised_refs() + .unwrap() + .iter() + .map(|record| record.target) + .collect(); + + let http = knot_server(source_path); + let remote = knot_pack::remote_pack( + http.as_ref(), + &base_url(), + &WantOids::new(tips.clone()), + &HaveOids::default(), + source.object_format(), + MaxWireBytes::new(16), + ) + .await; + assert!(matches!( + remote, + Err(FetchError::PackTooLarge { limit }) if limit == MaxWireBytes::new(16) + )); + + let local = local_pack( + &source, + &WantOids::new(tips), + &HaveOids::default(), + MaxWireBytes::new(16), + ); + assert!(matches!( + local, + Err(FetchError::PackTooLarge { limit }) if limit == MaxWireBytes::new(16) + )); +} + +#[test] +fn local_refs_and_pack_mirror_a_same_knot_source() { + let dir = tempfile::tempdir().unwrap(); + let source_path = seed_source(dir.path()); + let source = Repo::open(&source_path).unwrap(); + let hidden = RefName::new("refs/hidden/feature/main").unwrap(); + let main = RefName::new("refs/heads/main").unwrap(); + let tip = source.find_ref(&main).unwrap().unwrap(); + source + .update_ref(&knot_git::RefUpdate::Create { + name: hidden, + new: tip, + }) + .unwrap(); + + let refs = local_refs(&source, &["HEAD", "refs/heads/", "refs/tags/"]).unwrap(); + assert_eq!( + refs.head_symref.as_ref().map(RefName::as_str), + Some("refs/heads/main") + ); + assert!( + refs.refs + .iter() + .all(|record| !record.name.as_str().starts_with("refs/hidden/")), + "hidden ref must never leave source repo through a fork" + ); + + let layout = Layout::new(dir.path().join("scan")); + let fork = layout + .create(&RepoDid::new("did:plc:limpet").unwrap()) + .unwrap(); + let pack = local_pack( + &source, + &WantOids::new(refs.tips()), + &HaveOids::default(), + CAP, + ) + .unwrap(); + ingest_pack( + &fork.objects_dir(), + &pack, + &PackLimits::default(), + fork.object_format().kind(), + ) + .unwrap(); + let closure = fork + .select_pack_objects( + knot_git::Wants::new(&refs.tips()), + knot_git::Haves::new(&[]), + ) + .unwrap(); + assert!(closure.iter().all(|oid| fork.contains(*oid))); +} + +#[tokio::test] +async fn a_streamed_fetch_lands_the_pack_on_disk_and_indexes_it_from_there() { + let dir = tempfile::tempdir().unwrap(); + let source_path = seed_source(dir.path()); + let target = Repo::create(dir.path().join("fork.git")).unwrap(); + let http = ChunkedHttp { + inner: stock_git_server(source_path.clone()), + chunk_bytes: 7, + }; + + let refs = knot_pack::remote_refs(&http, &base_url(), &["HEAD", "refs/heads/", "refs/tags/"]) + .await + .unwrap(); + let pack_path = dir.path().join("fetched.pack"); + { + let mut file = std::fs::File::create(&pack_path).unwrap(); + knot_pack::remote_pack_streamed( + &http, + &base_url(), + &WantOids::new(refs.tips()), + &HaveOids::default(), + refs.object_format, + CAP, + &mut file, + ) + .await + .unwrap(); + } + let on_disk = std::fs::read(&pack_path).unwrap(); + assert!(!on_disk.is_empty(), "the streamed fetch wrote a pack"); + assert!(on_disk.starts_with(b"PACK")); + + knot_pack::ingest_pack_file( + &target.objects_dir(), + &pack_path, + &PackLimits::default(), + target.object_format().kind(), + ) + .unwrap(); + let closure = target + .select_pack_objects( + knot_git::Wants::new(&refs.tips()), + knot_git::Haves::new(&[]), + ) + .unwrap(); + assert!(!closure.is_empty()); + assert!(closure.iter().all(|oid| target.contains(*oid))); +} diff --git a/knot2/crates/knot-pack/tests/fuzz_smoke.rs b/knot2/crates/knot-pack/tests/fuzz_smoke.rs new file mode 100644 index 0000000000000000000000000000000000000000..3428d86753c395694895c7baf32ceb0dea7e53d1 --- /dev/null +++ b/knot2/crates/knot-pack/tests/fuzz_smoke.rs @@ -0,0 +1,76 @@ +use knot_git::Layout; +use knot_types::RepoDid; +use proptest::prelude::*; +use proptest::test_runner::TestRunner; + +fn empty_repo() -> (knot_git::Repo, tempfile::TempDir) { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + (repo, scan) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn parsers_never_panic(data in proptest::collection::vec(any::(), 0..4096)) { + knot_pack::fuzz::pkt(&data); + knot_pack::fuzz::pack(&data); + knot_pack::fuzz::receive_commands(&data); + knot_pack::fuzz::upload_args(&data); + } +} + +#[test] +fn a_version3_pack_header_is_a_typed_rejection_not_a_panic() { + let header = b"PACK\x00\x00\x00\x03\x00\x00\x00\x00"; + assert!( + knot_pack::meter_pack( + header, + &knot_pack::PackLimits::default(), + gix_hash::Kind::Sha1 + ) + .is_err(), + "a v3 pack header must be declined with a typed error" + ); + knot_pack::fuzz::pack(header); +} + +#[test] +fn a_lying_decompressed_size_never_preallocates_the_declared_amount() { + let bomb = [ + 0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x20, 0xff, 0xa0, 0xa8, + 0xa8, 0xed, 0xff, 0xff, 0x54, 0x41, 0x43, 0xff, 0xf4, 0x38, 0x06, 0x3e, 0xff, 0xff, 0xff, + 0xc7, 0x00, 0xc7, 0xff, 0xff, 0xff, 0xff, 0x3e, 0xff, 0x3e, + ]; + assert!( + knot_pack::meter_pack( + &bomb, + &knot_pack::PackLimits::default(), + gix_hash::Kind::Sha1 + ) + .is_err(), + "an entry declaring a petabyte object must be a typed error, never an allocation" + ); + knot_pack::fuzz::pack(&bomb); +} + +#[test] +fn repo_entry_points_never_panic() { + let (repo, _scan) = empty_repo(); + let mut runner = TestRunner::default(); + runner + .run(&proptest::collection::vec(any::(), 0..8192), |data| { + let _ = knot_pack::upload_pack(&repo, &data); + let _ = knot_pack::receive_pack(&repo, &data); + let _ = knot_pack::meter_pack( + &data, + &knot_pack::PackLimits::default(), + repo.object_format().kind(), + ); + Ok(()) + }) + .unwrap(); +} diff --git a/knot2/crates/knot-pack/tests/git_client.rs b/knot2/crates/knot-pack/tests/git_client.rs new file mode 100644 index 0000000000000000000000000000000000000000..eee5a63aaa67e24111c68af180de4ba020be3551 --- /dev/null +++ b/knot2/crates/knot-pack/tests/git_client.rs @@ -0,0 +1,990 @@ +use std::net::SocketAddr; +use std::path::Path; +use std::sync::Arc; + +use axum::body::Body; +use axum::http::header; +use knot_git::{Layout, RefUpdate}; +use knot_pack::{RepoLookup, RepoResolver, RepoTarget}; +use knot_types::{OwnerDid, RefName, RepoDid, RepoRkey}; + +mod common; +use common::{commit, contains, git, must, pkt, serve_dids, spawn, unsideband}; + +fn seed_repo(work: &Path, bare: &str, file: &str, contents: &str) { + std::fs::create_dir_all(work).unwrap(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, file, contents, "initial"); + must(work, &["push", "-q", bare, "main"]); + must( + Path::new(bare), + &["symbolic-ref", "HEAD", "refs/heads/main"], + ); +} + +fn url(addr: SocketAddr, did: &RepoDid, _name: &RepoRkey) -> String { + format!("http://{addr}/{}", did.as_str()) +} + +fn pack_object_oids(pack: &[u8]) -> std::collections::BTreeSet { + use std::io::Write; + use std::process::Stdio; + + let bare = tempfile::tempdir().unwrap(); + let path = bare.path().to_str().unwrap(); + assert!( + knot_fixtures::command(bare.path()) + .args(["init", "--bare", "-q", path]) + .output() + .unwrap() + .status + .success() + ); + let mut child = knot_fixtures::command(bare.path()) + .args(["index-pack", "--stdin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(pack).unwrap(); + let indexed = child.wait_with_output().unwrap(); + assert!( + indexed.status.success(), + "index-pack of our pack failed:\n{}", + String::from_utf8_lossy(&indexed.stderr) + ); + let listed = knot_fixtures::command(bare.path()) + .args([ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname)", + ]) + .output() + .unwrap(); + String::from_utf8_lossy(&listed.stdout) + .lines() + .map(|line| line.trim().to_string()) + .filter(|line| !line.is_empty()) + .collect() +} + +fn canonical_pack(work: &Path, revs: &[String]) -> Vec { + use std::io::Write; + use std::process::Stdio; + + let mut child = knot_fixtures::command(work) + .args(["pack-objects", "--revs", "--stdout", "-q"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(revs.join("\n").as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "canonical pack-objects failed"); + out.stdout +} + +fn v2_fetch_body(wants: &[String], haves: &[String]) -> Vec { + let mut body = pkt(b"command=fetch\n"); + body.extend_from_slice(b"0001"); + wants + .iter() + .for_each(|want| body.extend(pkt(format!("want {want}\n").as_bytes()))); + haves + .iter() + .for_each(|have| body.extend(pkt(format!("have {have}\n").as_bytes()))); + body.extend(pkt(b"done\n")); + body.extend_from_slice(b"0000"); + body +} + +#[tokio::test(flavor = "multi_thread")] +async fn http_routing_resolves_owner_rkey_and_dot_git_and_404s_the_unhosted() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let owner = OwnerDid::new("did:plc:nel").unwrap(); + let plain_did = RepoDid::new("did:plc:squid").unwrap(); + let literal_did = RepoDid::new("did:plc:whelk").unwrap(); + layout.create(&plain_did).unwrap(); + layout.create(&literal_did).unwrap(); + + let resolver: Arc = { + let owner = owner.clone(); + let plain_did = plain_did.clone(); + let literal_did = literal_did.clone(); + Arc::new(move |target: &RepoTarget| match target { + RepoTarget::OwnerPath(o, p) + if *o == owner && p.rkeys().any(|rkey| rkey.as_str() == "barnacle.git") => + { + RepoLookup::Hosted(literal_did.clone()) + } + RepoTarget::OwnerPath(o, p) + if *o == owner && p.rkeys().any(|rkey| rkey.as_str() == "anemone") => + { + RepoLookup::Hosted(plain_did.clone()) + } + _ => RepoLookup::Unhosted, + }) + }; + let addr = spawn( + knot_pack::router( + layout.clone(), + resolver, + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + seed_repo( + &scratch.path().join("work-plain"), + layout.repo_path(&plain_did).unwrap().to_str().unwrap(), + "README.md", + "plain\n", + ); + seed_repo( + &scratch.path().join("work-literal"), + layout.repo_path(&literal_did).unwrap().to_str().unwrap(), + "README.md", + "literal\n", + ); + + let clone_ok = |name: &str, label: &str, expect: &str| { + let dest = scratch.path().join(label); + let remote = format!("http://{addr}/{}/{name}", owner.as_str()); + let (ok, out) = git( + scratch.path(), + &["clone", "-q", &remote, dest.to_str().unwrap()], + ); + assert!( + ok, + "clone of {name} must resolve through the registry:\n{out}" + ); + assert_eq!( + std::fs::read_to_string(dest.join("README.md")).unwrap(), + expect + ); + }; + clone_ok("anemone", "clone-plain", "plain\n"); + clone_ok("anemone.git", "clone-suffixed", "plain\n"); + clone_ok("barnacle.git", "clone-literal", "literal\n"); + + let clone_404 = |remote: String, why: &str| { + let dest = scratch.path().join("clone-404"); + let (ok, _out) = git( + scratch.path(), + &["clone", "-q", &remote, dest.to_str().unwrap()], + ); + assert!(!ok, "{why}"); + let _ = std::fs::remove_dir_all(&dest); + }; + clone_404( + format!("http://{addr}/{}/conch", owner.as_str()), + "rkey with no registry entry must 404, not route to the wrong repo", + ); + clone_404( + format!("http://{addr}/{}", plain_did.as_str()), + "a direct-DID path the resolver doesn't host must 404, even though the repo exists on disk", + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn http_clone_while_the_index_is_warming_is_unavailable_not_404() { + use tower::ServiceExt; + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let resolver: Arc = Arc::new(|_target: &RepoTarget| RepoLookup::Unavailable); + + let by_name = axum::http::Request::builder() + .uri("/did:plc:nel/anemone/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + let response = knot_pack::router( + layout.clone(), + Arc::clone(&resolver), + std::sync::Arc::new(knot_runtime::SystemClock), + ) + .oneshot(by_name) + .await + .unwrap(); + assert_eq!( + response.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "warming registry is a retryable 503 on owner/rkey route, never a 404" + ); + + let by_did = axum::http::Request::builder() + .uri("/did:plc:squid/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + let response = knot_pack::router( + layout, + resolver, + std::sync::Arc::new(knot_runtime::SystemClock), + ) + .oneshot(by_did) + .await + .unwrap(); + assert_eq!( + response.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "warming registry is a retryable 503 on direct-DID route, never a 404" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn shallow_clone_over_protocol_v0() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let name = RepoRkey::new("scallop").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + let addr = spawn( + knot_pack::router( + layout.clone(), + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_three_commits(&work, bare.to_str().unwrap()); + commit(&work, "d.txt", "c4\n", "c4"); + must(&work, &["push", "-q", bare.to_str().unwrap(), "main"]); + let remote = url(addr, &did, &name); + + let clone = scratch.path().join("clone"); + let (ok, out) = git( + scratch.path(), + &[ + "-c", + "protocol.version=0", + "clone", + "--depth=1", + "-q", + &remote, + clone.to_str().unwrap(), + ], + ); + assert!(ok, "v0 shallow clone failed:\n{out}"); + assert!( + clone.join(".git/shallow").exists(), + "depth-limited v0 clone must be marked shallow" + ); + assert_eq!( + must(&clone, &["rev-list", "--count", "HEAD"]).trim(), + "1", + "depth=1 over v0 must yield exactly one commit" + ); + + let (ok, out) = git( + &clone, + &[ + "-c", + "protocol.version=0", + "fetch", + "--depth=2", + "-q", + "origin", + ], + ); + assert!(ok, "v0 deepening fetch failed:\n{out}"); + assert_eq!( + must(&clone, &["rev-list", "--count", "origin/main"]).trim(), + "2", + "deepen to depth=2 over v0 must reveal second commit" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn shallow_fetch_of_an_annotated_tag() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let name = RepoRkey::new("whelk").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + let addr = spawn( + knot_pack::router( + layout.clone(), + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_repo(&work, bare.to_str().unwrap(), "a.txt", "c1\n"); + commit(&work, "b.txt", "c2\n", "c2"); + must(&work, &["tag", "-a", "release", "-m", "release"]); + must( + &work, + &["push", "-q", bare.to_str().unwrap(), "main", "release"], + ); + let remote = url(addr, &did, &name); + + ["2", "0"].iter().enumerate().for_each(|(index, version)| { + let dest = scratch.path().join(format!("tagfetch-{index}")); + std::fs::create_dir_all(&dest).unwrap(); + must(&dest, &["init", "-q"]); + let (ok, out) = git( + &dest, + &[ + "-c", + &format!("protocol.version={version}"), + "fetch", + "--depth=1", + "-q", + &remote, + "refs/tags/release:refs/tags/release", + ], + ); + assert!(ok, "shallow tag fetch over v{version} failed:\n{out}"); + assert_eq!( + must(&dest, &["cat-file", "-t", "release"]).trim(), + "tag", + "annotated tag object itself must be transferred over v{version}" + ); + assert_eq!( + must(&dest, &["rev-list", "--count", "release^{commit}"]).trim(), + "1", + "depth-1 tag fetch over v{version} must contain exactly the tagged commit" + ); + }); +} + +#[tokio::test(flavor = "multi_thread")] +async fn pack_slot_limit_serializes_concurrent_clones_without_breaking_them() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let name = RepoRkey::new("cuttle").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_repo(&work, bare.to_str().unwrap(), "README.md", "kelp\n"); + let tip = must(&work, &["rev-parse", "HEAD"]).trim().to_string(); + + let addr = spawn( + knot_pack::router_with_pack_slots( + layout, + serve_dids(), + knot_resource::PackSlots::new(1), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + let remote = url(addr, &did, &name); + + let children: Vec<(usize, std::path::PathBuf, std::process::Child)> = (0..6) + .map(|index| { + let dest = scratch.path().join(format!("clone-{index}")); + let child = knot_fixtures::command(scratch.path()) + .args(["clone", "-q", &remote, dest.to_str().unwrap()]) + .spawn() + .expect("git clone spawns"); + (index, dest, child) + }) + .collect(); + + children.into_iter().for_each(|(index, dest, mut child)| { + assert!( + child.wait().unwrap().success(), + "single pack slot must still let concurrent clone {index} complete" + ); + assert_eq!( + must(&dest, &["rev-parse", "HEAD"]).trim(), + tip, + "clone served under a one-slot limit must still check out the right tip" + ); + }); +} + +#[tokio::test(flavor = "multi_thread")] +async fn http_upload_archive_serves_a_framed_tar_and_guards_refuse_cob_raw_oids_and_traversal() { + use tower::ServiceExt as _; + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_repo(&work, bare.to_str().unwrap(), "README.md", "archive me\n"); + + let mut framed = Vec::new(); + framed.extend(pkt(b"argument --format=tar\n")); + framed.extend(pkt(b"argument HEAD\n")); + framed.extend_from_slice(b"0000"); + let response = knot_pack::router( + layout.clone(), + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ) + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(format!("/{}/git-upload-archive", did.as_str())) + .header( + header::CONTENT_TYPE, + "application/x-git-upload-archive-request", + ) + .body(Body::from(framed)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), axum::http::StatusCode::OK); + assert_eq!( + response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("application/x-git-upload-archive-result"), + ); + let body = http_body_util::BodyExt::collect(response.into_body()) + .await + .unwrap() + .to_bytes(); + assert!( + body.starts_with(b"0008ACK\n"), + "archive response opens with the ACK pkt-line" + ); + assert!( + contains(&body, b"README.md"), + "framed archive contains README.md entry" + ); + + let repo = layout.open(&did).unwrap(); + let head = repo.head().expect("seeded head").target; + let tree = repo.find_commit(head).unwrap().tree; + let archive = |args: &[&str]| { + let mut request = Vec::new(); + args.iter() + .for_each(|arg| request.extend(pkt(arg.as_bytes()))); + request.extend_from_slice(b"0000"); + knot_pack::upload_archive(&repo, &request, knot_git::ArchiveLimit::default()).unwrap() + }; + + let raw_arg = format!("argument {}\n", tree.to_hex()); + let raw_oid = archive(&["argument --format=tar\n", raw_arg.as_str()]); + assert!( + String::from_utf8_lossy(&raw_oid).contains("NACK"), + "raw tree oid must be declined like uploadArchive.allowUnreachable=false" + ); + + let traversal = archive(&[ + "argument --format=tar\n", + "argument --prefix=../evil/\n", + "argument HEAD\n", + ]); + assert!( + String::from_utf8_lossy(&traversal).contains("NACK"), + "traversal prefix must be declined" + ); + + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/cobs/org.tangled.graph.collaborator/secret").unwrap(), + new: head, + }) + .unwrap(); + repo.update_ref(&RefUpdate::Delete { + name: RefName::new("refs/heads/main").unwrap(), + old: head, + }) + .unwrap(); + let cob = archive(&[ + "argument --format=tar\n", + "argument refs/cobs/org.tangled.graph.collaborator/secret\n", + ]); + assert!( + String::from_utf8_lossy(&cob).contains("NACK"), + "archiving cob-only tree must be refused" + ); + assert!( + !contains(&cob, b"README.md"), + "refused archive mustn't leak the hidden tree's contents" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn http_upload_archive_honors_the_configured_archive_limit() { + use tower::ServiceExt as _; + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_repo(&work, bare.to_str().unwrap(), "README.md", "archive me\n"); + + let (write_routes, _advertisement) = knot_pack::edge_routes(knot_pack::EdgeConfig { + pack_slots: knot_resource::PackSlots::new(1), + archive_limit: knot_git::ArchiveLimit::new(512), + ..knot_pack::EdgeConfig::serving( + layout.clone(), + serve_dids(), + Arc::new(knot_runtime::SystemClock), + ) + }); + + let mut framed = Vec::new(); + framed.extend(pkt(b"argument --format=tar\n")); + framed.extend(pkt(b"argument HEAD\n")); + framed.extend_from_slice(b"0000"); + let response = write_routes + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(format!("/{}/git-upload-archive", did.as_str())) + .header( + header::CONTENT_TYPE, + "application/x-git-upload-archive-request", + ) + .body(Body::from(framed)) + .unwrap(), + ) + .await + .unwrap(); + let body = http_body_util::BodyExt::collect(response.into_body()) + .await + .unwrap() + .to_bytes(); + let text = String::from_utf8_lossy(&body); + assert!( + text.contains("NACK") && text.contains("archive exceeds the 512 byte limit"), + "an archive past the state's limit must be declined, got {text:?}" + ); + assert!( + !contains(&body, b"README.md"), + "the declined archive mustn't leak the tree it refused to serve" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn push_over_http_is_refused() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let name = RepoRkey::new("conch").unwrap(); + layout.create(&did).unwrap(); + let addr = spawn( + knot_pack::router( + layout.clone(), + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + commit(&work, "a.txt", "one\n", "one"); + let (ok, out) = git(&work, &["push", &url(addr, &did, &name), "main"]); + assert!(!ok, "push over HTTP must be refused, got success:\n{out}"); +} + +fn seed_three_commits(work: &Path, bare: &str) { + seed_repo(work, bare, "a.txt", "c1\n"); + commit(work, "b.txt", "c2\n", "c2"); + must(work, &["push", "-q", bare, "main"]); + commit(work, "c.txt", "c3\n", "c3"); + must(work, &["push", "-q", bare, "main"]); +} + +fn commit_dated(work: &Path, file: &str, contents: &str, message: &str, iso_date: &str) { + std::fs::write(work.join(file), contents).unwrap(); + must(work, &["add", "-A"]); + let out = knot_fixtures::command_at(work, iso_date) + .args(["commit", "-q", "-m", message]) + .output() + .unwrap(); + assert!(out.status.success(), "dated commit failed"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn shallow_clone_depth_exclude_since() { + let did = RepoDid::new("did:plc:squid").unwrap(); + let s = common::stand(&did).await; + let bare = &s.bare; + let scratch = s.scratch.path(); + + let work = scratch.join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + commit_dated(&work, "a.txt", "c1\n", "c1", "2020-01-01T00:00:00 +0000"); + must(&work, &["tag", "base"]); + commit_dated(&work, "b.txt", "c2\n", "c2", "2021-01-01T00:00:00 +0000"); + commit_dated(&work, "c.txt", "c3\n", "c3", "2022-01-01T00:00:00 +0000"); + commit_dated(&work, "d.txt", "c4\n", "c4", "2024-01-01T00:00:00 +0000"); + must( + &work, + &["push", "-q", bare.to_str().unwrap(), "main", "base"], + ); + must(bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); + let remote = format!("http://{}/{}", s.addr, did.as_str()); + + let depth = scratch.join("clone-depth"); + must( + scratch, + &["clone", "--depth=1", "-q", &remote, depth.to_str().unwrap()], + ); + assert!( + depth.join(".git/shallow").exists(), + "depth-limited clone must be marked shallow" + ); + assert_eq!( + must(&depth, &["rev-list", "--count", "HEAD"]).trim(), + "1", + "depth=1 must yield exactly one commit" + ); + let (ok, out) = git(&depth, &["fetch", "--depth=2", "-q", "origin"]); + assert!(ok, "deepening fetch failed:\n{out}"); + assert_eq!( + must(&depth, &["rev-list", "--count", "origin/main"]).trim(), + "2", + "deepen to depth=2 must reveal second commit" + ); + let (ok, out) = git(&depth, &["fetch", "--deepen=1", "-q", "origin"]); + assert!(ok, "relative deepen fetch failed:\n{out}"); + assert_eq!( + must(&depth, &["rev-list", "--count", "origin/main"]).trim(), + "3", + "--deepen=1 from depth 2 must reveal third commit" + ); + let (ok, out) = git(&depth, &["fetch", "--unshallow", "-q", "origin"]); + assert!(ok, "unshallow fetch failed:\n{out}"); + assert!( + !depth.join(".git/shallow").exists(), + "unshallow fetch must drop the shallow marker" + ); + assert_eq!( + must(&depth, &["rev-list", "--count", "origin/main"]).trim(), + "4", + "unshallow must restore full history" + ); + + let exclude = scratch.join("clone-exclude"); + let (ok, out) = git( + scratch, + &[ + "clone", + "--shallow-exclude=base", + "-q", + &remote, + exclude.to_str().unwrap(), + ], + ); + assert!(ok, "shallow-exclude clone failed:\n{out}"); + assert_eq!( + must(&exclude, &["rev-list", "--count", "HEAD"]).trim(), + "3", + "shallow-exclude=base must drop excluded commit and its ancestors" + ); + + let since = scratch.join("clone-since"); + let (ok, out) = git( + scratch, + &[ + "clone", + "--shallow-since=2023-01-01", + "-q", + &remote, + since.to_str().unwrap(), + ], + ); + assert!(ok, "shallow-since clone failed:\n{out}"); + assert_eq!( + must(&since, &["rev-list", "--count", "HEAD"]).trim(), + "1", + "shallow-since must keep only commits at or after cutoff" + ); +} + +#[test] +fn upload_pack_object_set_matches_canonical_git() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + commit(&work, "a.txt", "one\n", "c1"); + let c1 = must(&work, &["rev-parse", "HEAD"]).trim().to_string(); + commit(&work, "a.txt", "two\n", "c2"); + let c2 = must(&work, &["rev-parse", "HEAD"]).trim().to_string(); + must(&work, &["checkout", "-q", "-b", "dev", &c1]); + commit(&work, "b.txt", "three\n", "c3"); + must(&work, &["checkout", "-q", "main"]); + must(&work, &["tag", "-a", "v1", "-m", "release", &c2]); + must( + &work, + &["push", "-q", bare.to_str().unwrap(), "main", "dev", "v1"], + ); + + let repo = layout.open(&did).unwrap(); + let tips: Vec = repo + .advertised_refs() + .unwrap() + .iter() + .map(|record| record.target.to_hex().to_string()) + .collect(); + + let clone = unsideband(&knot_pack::upload_pack(&repo, &v2_fetch_body(&tips, &[])).unwrap()); + assert_eq!( + pack_object_oids(&clone), + pack_object_oids(&canonical_pack(&work, &tips)), + "full clone must transfer exactly the object set canonical git packs" + ); + + let incremental = unsideband( + &knot_pack::upload_pack( + &repo, + &v2_fetch_body(std::slice::from_ref(&c2), std::slice::from_ref(&c1)), + ) + .unwrap(), + ); + assert_eq!( + pack_object_oids(&incremental), + pack_object_oids(&canonical_pack(&work, &[c2, format!("^{c1}")])), + "incremental fetch must transfer only the objects missing from the client" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn http_boundary_encoding_and_streaming() { + use flate2::Compression; + use flate2::write::GzEncoder; + use http_body_util::BodyExt; + use std::io::Write; + use tower::ServiceExt; + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_repo(&work, bare.to_str().unwrap(), "README.md", "boundary\n"); + let tip = must(&work, &["rev-parse", "HEAD"]).trim().to_string(); + + let router = knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ); + let upload_uri = format!("/{}/git-upload-pack", did.as_str()); + + let oversized = router + .clone() + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(upload_uri.clone()) + .body(Body::from(vec![0u8; 17 * 1024 * 1024])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + oversized.status(), + axum::http::StatusCode::PAYLOAD_TOO_LARGE, + "oversized request body must be refused at the HTTP boundary before buffering" + ); + + let unsupported = router + .clone() + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(upload_uri.clone()) + .header("content-encoding", "br") + .body(Body::from(vec![0u8; 16])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + unsupported.status(), + axum::http::StatusCode::UNSUPPORTED_MEDIA_TYPE, + "a body the knot cannot decode is rejected before parsing, never mis-read as identity" + ); + + let mut plain = pkt(b"command=ls-refs\n"); + plain.extend_from_slice(b"0001"); + plain.extend(pkt(b"ref-prefix refs/heads/\n")); + plain.extend_from_slice(b"0000"); + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&plain).unwrap(); + let gzipped = encoder.finish().unwrap(); + let decoded = router + .clone() + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(upload_uri.clone()) + .header("content-encoding", "gzip") + .body(Body::from(gzipped)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(decoded.status(), axum::http::StatusCode::OK); + let body = decoded.into_body().collect().await.unwrap().to_bytes(); + assert!( + String::from_utf8_lossy(&body).contains("refs/heads/main"), + "gzip-encoded ls-refs request must be transparently decoded and answered" + ); + + let mut fetch = pkt(b"command=fetch\n"); + fetch.extend_from_slice(b"0001"); + fetch.extend(pkt(format!("want {tip}\n").as_bytes())); + fetch.extend(pkt(b"done\n")); + fetch.extend_from_slice(b"0000"); + let streamed = router + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(upload_uri) + .body(Body::from(fetch)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(streamed.status(), axum::http::StatusCode::OK); + assert!( + streamed.headers().get(header::CONTENT_LENGTH).is_none(), + "streamed pack response mustn't be buffered into a length-delimited body" + ); + let collected = streamed.into_body().collect().await.unwrap().to_bytes(); + assert!( + contains(&collected, b"PACK"), + "streamed response must contain a real PACK" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_knot_meta_repo_is_never_served_over_http() { + use tower::ServiceExt; + + let scan = tempfile::tempdir().unwrap(); + let knot = knot_types::KnotId::new("did:web:oyster.cafe").unwrap(); + let layout = Layout::new(scan.path()).reserving_meta(&knot).unwrap(); + layout.bootstrap_meta(&knot).unwrap(); + assert!( + layout.meta_path(&knot).unwrap().exists(), + "meta-repo must exist on disk so this tests the guard, not mere absence" + ); + + let visible = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&visible).unwrap(); + + let router = knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ); + + let status = |method: &'static str, uri: &'static str| { + let router = router.clone(); + async move { + router + .oneshot( + axum::http::Request::builder() + .method(method) + .uri(uri) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap() + .status() + } + }; + + let not_found = axum::http::StatusCode::NOT_FOUND; + assert_eq!( + status( + "GET", + "/did:web:oyster.cafe/info/refs?service=git-upload-pack" + ) + .await, + not_found, + "knot DID is refused on GET info/refs" + ); + assert_eq!( + status("POST", "/did:web:oyster.cafe/git-upload-pack").await, + not_found, + "knot DID is refused on POST upload-pack" + ); + assert_eq!( + status( + "GET", + "/did:web:oyster.cafe/anemone/info/refs?service=git-upload-pack" + ) + .await, + not_found, + "knot DID is refused on named info/refs route" + ); + assert_eq!( + status("POST", "/did:web:oyster.cafe/anemone/git-upload-pack").await, + not_found, + "knot DID is refused on named upload-pack route" + ); + assert_eq!( + status("POST", "/did:web:oyster.cafe/git-upload-archive").await, + not_found, + "knot DID is refused on upload-archive route" + ); + assert_eq!( + status("POST", "/did:web:oyster.cafe/anemone/git-upload-archive").await, + not_found, + "knot DID is refused on named upload-archive route" + ); + assert_eq!( + status( + "GET", + "/did:web:OYSTER.cafe/info/refs?service=git-upload-pack" + ) + .await, + not_found, + "case-variant of the knot DID canonicalizes to the same reserved repo" + ); + + assert_eq!( + status("GET", "/did:plc:squid/info/refs?service=git-upload-pack").await, + axum::http::StatusCode::OK, + "ordinary repo path still serves its advertisement" + ); +} diff --git a/knot2/crates/knot-pack/tests/h3_conformance.rs b/knot2/crates/knot-pack/tests/h3_conformance.rs new file mode 100644 index 0000000000000000000000000000000000000000..32396dbd4cebdb57067b9a88445b824827b4e8b4 --- /dev/null +++ b/knot2/crates/knot-pack/tests/h3_conformance.rs @@ -0,0 +1,574 @@ +use std::collections::BTreeSet; +use std::net::SocketAddr; +use std::num::{NonZeroU32, NonZeroU64}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::{Buf, Bytes}; +use http::{HeaderMap, Method, Uri}; +use knot_edge::{ + BodyInactivityTimeout, BurstSize, CertSource, EdgeConfig, EdgeGuards, HeaderTimeout, + IdleTimeout, ListenLimits, MaxInflightRequests, RequestTimeout, RequestsPerSecond, + RequiresFullHandshake, StaticCertPaths, TlsSetup, WriteRequestTimeout, +}; +use knot_git::Layout; +use knot_pack::{RepoLookup, RepoResolver, RepoTarget}; +use knot_types::{ObjectFormat, RepoDid}; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::crypto::aws_lc_rs; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{DigitallySignedStruct, SignatureScheme}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +mod common; +use common::{contains, must, pack_objects, receive_request}; + +type Captured = (Method, Uri, HeaderMap, Bytes); + +fn nz32(value: u32) -> NonZeroU32 { + NonZeroU32::new(value).unwrap() +} + +fn nz64(value: u64) -> NonZeroU64 { + NonZeroU64::new(value).unwrap() +} + +fn object_set(dir: &Path) -> BTreeSet { + must( + dir, + &[ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname)", + ], + ) + .lines() + .map(|line| line.trim().to_string()) + .filter(|line| !line.is_empty()) + .collect() +} + +fn serve_dids() -> Arc { + Arc::new(|target: &RepoTarget| match target { + RepoTarget::Did(did) => RepoLookup::Hosted(did.clone()), + RepoTarget::OwnerPath(_, _) => RepoLookup::Unhosted, + }) +} + +fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +fn write_self_signed(dir: &Path) -> (PathBuf, PathBuf, Vec) { + let generated = + rcgen::generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()]) + .unwrap(); + let cert_path = dir.join("cert.pem"); + let key_path = dir.join("key.pem"); + std::fs::write(&cert_path, generated.cert.pem()).unwrap(); + std::fs::write(&key_path, generated.signing_key.serialize_pem()).unwrap(); + let der = generated.cert.der().as_ref().to_vec(); + (cert_path, key_path, der) +} + +fn edge_config(addr: SocketAddr, cert: PathBuf, key: PathBuf) -> EdgeConfig { + EdgeConfig { + http_addr: knot_edge::PublicBind::new(addr), + limits: ListenLimits::new( + HeaderTimeout::from_millis(nz64(30_000)), + IdleTimeout::from_millis(nz64(120_000)), + nz32(1024), + ), + guards: EdgeGuards::new( + RequestsPerSecond::new(nz32(1_000_000)), + BurstSize::new(nz32(1_000_000)), + MaxInflightRequests::new(nz32(10_000)), + RequestTimeout::from_millis(nz64(120_000)), + BodyInactivityTimeout::from_millis(nz64(120_000)), + WriteRequestTimeout::from_millis(nz64(1_800_000)), + knot_types::ProxyTrust::default(), + ), + tls: Some(TlsSetup { + source: CertSource::Static(StaticCertPaths { + cert_path: knot_edge::CertChainPath::new(cert), + key_path: knot_edge::PrivateKeyPath::new(key), + }), + http3: true, + internal: None, + }), + } +} + +struct Edge { + addr: SocketAddr, + shutdown: CancellationToken, + log: Arc>>, + task: JoinHandle>, + client: quinn::Endpoint, +} + +async fn probe_identity( + endpoint: &quinn::Endpoint, + addr: SocketAddr, + expected_cert: &[u8], +) -> Option { + let connecting = endpoint.connect(addr, "localhost").ok()?; + let connection = tokio::time::timeout(Duration::from_millis(250), connecting) + .await + .ok()? + .ok()?; + let ours = connection + .peer_identity() + .and_then(|identity| identity.downcast::>>().ok()) + .map(|certs| { + certs + .first() + .is_some_and(|cert| cert.as_ref() == expected_cert) + }) + .unwrap_or(false); + connection.close(0u32.into(), b"probe done"); + Some(ours) +} + +async fn await_ready( + endpoint: &quinn::Endpoint, + addr: SocketAddr, + task: &mut JoinHandle>, + expected_cert: &[u8], +) -> bool { + for _ in 0..200 { + if task.is_finished() { + return false; + } + if let Some(ours) = probe_identity(endpoint, addr, expected_cert).await { + return ours; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + false +} + +async fn stand_up(layout: Layout, certdir: &Path) -> Edge { + static TRACE: std::sync::Once = std::sync::Once::new(); + TRACE.call_once(|| { + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init(); + }); + for _ in 0..8 { + let addr: SocketAddr = format!("127.0.0.1:{}", free_port()).parse().unwrap(); + let (cert, key, cert_der) = write_self_signed(certdir); + let (write_routes, advertisement) = knot_pack::edge_routes(knot_pack::EdgeConfig { + pack_slots: knot_resource::PackSlots::new(4), + ..knot_pack::EdgeConfig::serving( + layout.clone(), + serve_dids(), + Arc::new(knot_runtime::SystemClock), + ) + }); + let log: Arc>> = Arc::new(Mutex::new(Vec::new())); + let sink = log.clone(); + let recorded = write_routes.layer(axum::middleware::from_fn( + move |request: axum::extract::Request, next: axum::middleware::Next| { + let sink = sink.clone(); + async move { + let (parts, body) = request.into_parts(); + let bytes = axum::body::to_bytes(body, usize::MAX) + .await + .unwrap_or_default(); + sink.lock().unwrap().push(( + parts.method.clone(), + parts.uri.clone(), + parts.headers.clone(), + bytes.clone(), + )); + next.run(axum::extract::Request::from_parts( + parts, + axum::body::Body::from(bytes), + )) + .await + } + }, + )); + let app = RequiresFullHandshake::new(recorded); + let shutdown = CancellationToken::new(); + let mut task = tokio::spawn(knot_edge::serve( + edge_config(addr, cert, key), + app, + advertisement, + shutdown.clone(), + )); + let client = h3_client(); + if await_ready(&client, addr, &mut task, &cert_der).await { + return Edge { + addr, + shutdown, + log, + task, + client, + }; + } + client.close(0u32.into(), b"stand up retry"); + shutdown.cancel(); + let _ = task.await; + } + panic!("couldn't bind a free TCP+UDP port for the edge after several attempts"); +} + +#[derive(Debug)] +struct AcceptAnyServerCert; + +impl ServerCertVerifier for AcceptAnyServerCert { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + aws_lc_rs::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} + +fn h3_client() -> quinn::Endpoint { + let mut crypto = + rustls::ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCert)) + .with_no_client_auth(); + crypto.alpn_protocols = vec![b"h3".to_vec()]; + let quic = quinn::crypto::rustls::QuicClientConfig::try_from(crypto).unwrap(); + let mut endpoint = quinn::Endpoint::client("127.0.0.1:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(quic))); + endpoint +} + +async fn drain( + stream: &mut h3::client::RequestStream, Bytes>, +) -> Vec { + let mut out = Vec::new(); + while let Some(mut chunk) = stream.recv_data().await.unwrap() { + out.extend_from_slice(&chunk.copy_to_bytes(chunk.remaining())); + } + out +} + +async fn finish_request( + stream: &mut h3::client::RequestStream, Bytes>, +) { + match stream.finish().await { + Ok(()) => (), + Err(h3::error::StreamError::RemoteTerminate { code, .. }) + if code == h3::error::Code::H3_NO_ERROR => {} + Err(error) => panic!("finishing the request stream failed: {error}"), + } +} + +async fn replay_over_h3(edge: &Edge, did: &str, request: &Captured) -> Vec { + let (_, uri, headers, body) = request; + let connection = edge + .client + .connect(edge.addr, "localhost") + .unwrap() + .await + .unwrap(); + let quic = connection.clone(); + let (mut driver, mut sender) = h3::client::new(h3_quinn::Connection::new(connection)) + .await + .unwrap(); + let drive = tokio::spawn(async move { + let _ = std::future::poll_fn(|cx| driver.poll_close(cx)).await; + }); + + let warmup = http::Request::get(format!( + "https://localhost/{did}/info/refs?service=git-upload-pack" + )) + .header("git-protocol", "version=2") + .body(()) + .unwrap(); + let mut warm = sender.send_request(warmup).await.unwrap(); + finish_request(&mut warm).await; + assert!( + warm.recv_response().await.unwrap().status().is_success(), + "h3 info/refs advertisement must serve over QUIC" + ); + drain(&mut warm).await; + + let mut builder = http::Request::builder() + .method(Method::POST) + .uri(format!("https://localhost{}", uri.path())); + for name in ["content-type", "content-encoding", "git-protocol", "accept"] { + if let Some(value) = headers.get(name) { + builder = builder.header(name, value); + } + } + let mut stream = sender + .send_request(builder.body(()).unwrap()) + .await + .unwrap(); + stream.send_data(body.clone()).await.unwrap(); + finish_request(&mut stream).await; + let response = stream.recv_response().await.unwrap(); + assert!( + response.status().is_success(), + "h3 upload-pack returned {}", + response.status() + ); + let out = drain(&mut stream).await; + quic.close(0u32.into(), b"done"); + drive.abort(); + out +} + +fn fetch_request(log: &Arc>>) -> Captured { + let captured = log.lock().unwrap(); + captured + .iter() + .find(|(method, uri, _, body)| { + method == Method::POST + && uri.path().ends_with("/git-upload-pack") + && contains(body, b"command=fetch") + }) + .cloned() + .unwrap_or_else(|| { + let summary: Vec = captured + .iter() + .map(|(method, uri, headers, body)| { + format!( + "{method} {uri} git-protocol={:?} body[..64]={:?}", + headers.get("git-protocol"), + String::from_utf8_lossy(&body[..body.len().min(64)]) + ) + }) + .collect(); + panic!("git issued no protocol-v2 fetch over the TLS edge, captured: {summary:#?}") + }) +} + +fn extract_pack(response: &[u8]) -> Vec { + let mut channel = Vec::new(); + let mut pos = 0usize; + while pos + 4 <= response.len() { + let len = std::str::from_utf8(&response[pos..pos + 4]) + .ok() + .and_then(|hex| usize::from_str_radix(hex, 16).ok()) + .unwrap_or(0); + pos += 4; + if len < 4 { + continue; + } + let end = (pos + len - 4).min(response.len()); + let payload = &response[pos..end]; + pos = end; + if payload.first() == Some(&1) { + channel.extend_from_slice(&payload[1..]); + } + } + match channel.windows(4).position(|window| window == b"PACK") { + Some(start) => channel.split_off(start), + None => channel, + } +} + +fn index_pack(repo: &Path, pack: &[u8]) { + let (indexed, report) = + knot_fixtures::feed(repo, &["index-pack", "--stdin", "--fix-thin"], pack); + assert!(indexed, "index-pack failed: {report}"); +} + +fn advance_knot(bare: &Path, work: &Path, old: &str, new: &str) { + let oids: Vec = must(work, &["rev-list", "--objects", new, "--not", old]) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .map(str::to_string) + .collect(); + let request = receive_request("refs/heads/main", old, new, &pack_objects(work, &oids)); + let repo = knot_git::Repo::open(bare).expect("open knot bare"); + let report = knot_pack::receive_pack(&repo, &request).expect("knot receive"); + assert!( + String::from_utf8_lossy(&report).contains("ok refs/heads/main"), + "knot must accept a receive that advances main" + ); +} + +fn init(dir: &Path, format: ObjectFormat) { + std::fs::create_dir_all(dir).unwrap(); + let fmt = format!("--object-format={}", format.capability()); + must(dir, &["init", &fmt, "-q", dir.to_str().unwrap()]); +} + +fn seed(work: &Path, bares: [&Path; 2], format: ObjectFormat) { + let fmt = format!("--object-format={}", format.capability()); + std::fs::create_dir_all(work).unwrap(); + must(work, &["init", &fmt, "-q", "-b", "main"]); + std::fs::write(work.join("README.md"), "h3 conformance\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c1"]); + let c1 = must(work, &["rev-parse", "HEAD"]); + std::fs::write(work.join("src.txt"), "more\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c2"]); + must(work, &["checkout", "-q", "-b", "dev", &c1]); + std::fs::write(work.join("dev.txt"), "branch\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c3"]); + must(work, &["checkout", "-q", "main"]); + must(work, &["tag", "-a", "v1", "-m", "release"]); + bares.into_iter().for_each(|bare| { + must( + work, + &["push", "-q", bare.to_str().unwrap(), "main", "dev", "v1"], + ); + must(bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); + }); +} + +async fn h3_serves_the_canonical_object_set(format: ObjectFormat, did_str: &str) { + let scan = tempfile::tempdir().unwrap(); + let certdir = tempfile::tempdir().unwrap(); + let scratch = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + + let did = RepoDid::new(did_str).unwrap(); + let layout = Layout::new(scan.path()).with_object_format(format); + layout.create(&did).unwrap(); + let knot_bare = layout.repo_path(&did).unwrap(); + let canon_bare = canon_root.path().join("canon.git"); + let fmt = format!("--object-format={}", format.capability()); + must( + canon_root.path(), + &["init", "--bare", &fmt, "-q", canon_bare.to_str().unwrap()], + ); + + let work = scratch.path().join("work"); + seed(&work, [&knot_bare, &canon_bare], format); + + let canon_url = format!("file://{}", canon_bare.to_str().unwrap()); + let canon_clone = scratch.path().join("canon-clone"); + must( + scratch.path(), + &["clone", "-q", &canon_url, canon_clone.to_str().unwrap()], + ); + let canonical = object_set(&canon_clone); + + let edge = stand_up(layout, certdir.path()).await; + let url = format!("https://{}/{}", edge.addr, did.as_str()); + + let h1_clone = scratch.path().join("h1-clone"); + must( + scratch.path(), + &[ + "-c", + "http.sslVerify=false", + "clone", + "-q", + &url, + h1_clone.to_str().unwrap(), + ], + ); + must(&h1_clone, &["config", "http.sslVerify", "false"]); + assert_eq!( + must(&canon_clone, &["rev-parse", "HEAD^{tree}"]), + must(&h1_clone, &["rev-parse", "HEAD^{tree}"]), + "{format:?} h1/h2 TLS clone checks out the canonical tree" + ); + assert_eq!( + canonical, + object_set(&h1_clone), + "{format:?} h1/h2 TLS clone transfers the canonical object set" + ); + + let h3_clone = scratch.path().join("h3-clone"); + init(&h3_clone, format); + let pack = extract_pack(&replay_over_h3(&edge, did.as_str(), &fetch_request(&edge.log)).await); + index_pack(&h3_clone, &pack); + assert_eq!( + canonical, + object_set(&h3_clone), + "{format:?} h3 clone over QUIC transfers the canonical object set" + ); + + let old = must(&work, &["rev-parse", "HEAD"]); + std::fs::write(work.join("incremental.txt"), "fetch me\n").unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", "c4"]); + let new = must(&work, &["rev-parse", "HEAD"]); + advance_knot(&knot_bare, &work, &old, &new); + must(&work, &["push", "-q", canon_bare.to_str().unwrap(), "main"]); + + let canon_after = scratch.path().join("canon-after"); + must( + scratch.path(), + &["clone", "-q", &canon_url, canon_after.to_str().unwrap()], + ); + let canonical_after = object_set(&canon_after); + + edge.log.lock().unwrap().clear(); + must(&h1_clone, &["fetch", "-q", "origin"]); + assert_eq!( + canonical_after, + object_set(&h1_clone), + "{format:?} h1/h2 TLS fetch advances to the canonical object set" + ); + + let fetch_pack = + extract_pack(&replay_over_h3(&edge, did.as_str(), &fetch_request(&edge.log)).await); + index_pack(&h3_clone, &fetch_pack); + assert_eq!( + canonical_after, + object_set(&h3_clone), + "{format:?} h3 incremental fetch over QUIC advances to the canonical object set" + ); + + edge.shutdown.cancel(); + edge.task.abort(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_git_superset_guarantee_holds_over_h3_in_both_object_formats() { + h3_serves_the_canonical_object_set(ObjectFormat::SHA1, "did:plc:squid").await; + h3_serves_the_canonical_object_set(ObjectFormat::SHA256, "did:plc:cuttle").await; +} diff --git a/knot2/crates/knot-pack/tests/handle_owner.rs b/knot2/crates/knot-pack/tests/handle_owner.rs new file mode 100644 index 0000000000000000000000000000000000000000..a9c8b4a28196950590c77532511065e2da78ee8c --- /dev/null +++ b/knot2/crates/knot-pack/tests/handle_owner.rs @@ -0,0 +1,122 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use http_body_util::BodyExt; +use knot_git::Layout; +use knot_pack::{HandleResolver, RepoLookup, RepoResolver, RepoTarget}; +use knot_types::{AccountDid, Handle, OwnerDid, RepoDid}; +use tower::ServiceExt; + +struct FakeHandles; + +impl HandleResolver for FakeHandles { + fn resolve( + &self, + handle: Handle, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + (handle.as_str() == "nel.pet").then(|| AccountDid::new("did:plc:nel").unwrap()) + }) + } +} + +fn repo_resolver() -> Arc { + let owner = OwnerDid::new("did:plc:nel").unwrap(); + let repo = RepoDid::new("did:plc:whelk").unwrap(); + Arc::new(move |target: &RepoTarget| match target { + RepoTarget::OwnerPath(o, p) + if *o == owner && p.rkeys().any(|rkey| rkey.as_str() == "squid") => + { + RepoLookup::Hosted(repo.clone()) + } + RepoTarget::Did(d) if *d == repo => RepoLookup::Hosted(d.clone()), + _ => RepoLookup::Unhosted, + }) +} + +fn build(layout: &Layout, handle_resolver: Option>) -> axum::Router { + let (_write, advertisement) = knot_pack::edge_routes(knot_pack::EdgeConfig { + handle_resolver, + pack_slots: knot_resource::PackSlots::new(4), + ..knot_pack::EdgeConfig::serving( + layout.clone(), + repo_resolver(), + Arc::new(knot_runtime::SystemClock), + ) + }); + advertisement.into_router() +} + +async fn get(router: axum::Router, uri: &str) -> (StatusCode, Vec) { + let response = router + .oneshot(Request::get(uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + let status = response.status(); + let body = response + .into_body() + .collect() + .await + .unwrap() + .to_bytes() + .to_vec(); + (status, body) +} + +fn hosted_layout() -> (tempfile::TempDir, Layout) { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + layout + .create(&RepoDid::new("did:plc:whelk").unwrap()) + .unwrap(); + (scan, layout) +} + +#[tokio::test] +async fn a_handle_owner_serves_the_same_repo_as_its_did() { + let (_scan, layout) = hosted_layout(); + let (handle_status, handle_body) = get( + build(&layout, Some(Arc::new(FakeHandles))), + "/nel.pet/squid/info/refs?service=git-upload-pack", + ) + .await; + let (did_status, did_body) = get( + build(&layout, Some(Arc::new(FakeHandles))), + "/did:plc:nel/squid/info/refs?service=git-upload-pack", + ) + .await; + assert_eq!(handle_status, StatusCode::OK); + assert_eq!(did_status, StatusCode::OK); + assert_eq!( + handle_body, did_body, + "handle owner and DID owner must serve the same repository" + ); +} + +#[tokio::test] +async fn a_handle_owner_is_not_found_when_unknown_or_unresolvable() { + let (_scan, layout) = hosted_layout(); + let (unknown, _) = get( + build(&layout, Some(Arc::new(FakeHandles))), + "/olaren.dev/squid/info/refs?service=git-upload-pack", + ) + .await; + assert_eq!( + unknown, + StatusCode::NOT_FOUND, + "a handle the resolver rejects isn't found" + ); + let (no_resolver, _) = get( + build(&layout, None), + "/nel.pet/squid/info/refs?service=git-upload-pack", + ) + .await; + assert_eq!( + no_resolver, + StatusCode::NOT_FOUND, + "a handle owner with no resolver configured isn't found" + ); +} diff --git a/knot2/crates/knot-pack/tests/hardening.rs b/knot2/crates/knot-pack/tests/hardening.rs new file mode 100644 index 0000000000000000000000000000000000000000..36564fb188d49ab950570765999cc016f8bb5563 --- /dev/null +++ b/knot2/crates/knot-pack/tests/hardening.rs @@ -0,0 +1,1478 @@ +use std::io::Write; +use std::process::Stdio; + +use knot_git::{Layout, RefUpdate}; +use knot_pack::{DeltaDepth, PackLimits}; +use knot_types::{ObjectCount, ObjectFormat, Oid, RefName, RepoDid}; + +mod common; +use common::{ + commit, contains, delta_bomb_pack, index_into_bare, must, pack_objects, pack_objects_tuned, + pkt, receive_request, seeded, unsideband, +}; + +fn generous() -> PackLimits { + PackLimits { + max_objects: ObjectCount::new(1_000_000), + max_object_bytes: knot_pack::MaxObjectBytes::new(1 << 30), + max_total_bytes: knot_pack::MaxTotalBytes::new(1 << 31), + max_delta_depth: DeltaDepth::new(50), + } +} + +fn v2_fetch(wants: &[&str], haves: &[&str], done: bool) -> Vec { + let mut req = pkt(b"command=fetch\n"); + req.extend_from_slice(b"0001"); + wants + .iter() + .for_each(|want| req.extend(pkt(format!("want {want}\n").as_bytes()))); + haves + .iter() + .for_each(|have| req.extend(pkt(format!("have {have}\n").as_bytes()))); + if done { + req.extend(pkt(b"done\n")); + } + req.extend_from_slice(b"0000"); + req +} + +fn thin_resolves_against_base(base_pack: &[u8], thin: &[u8]) -> bool { + let bare = tempfile::tempdir().unwrap(); + knot_fixtures::must( + bare.path(), + &["init", "--bare", "-q", bare.path().to_str().unwrap()], + ); + let feed = |args: &[&str], pack: &[u8]| knot_fixtures::feed(bare.path(), args, pack).0; + feed(&["index-pack", "--stdin"], base_pack) + && feed(&["index-pack", "--stdin", "--fix-thin"], thin) +} + +fn v2_fetch_thin(want: &str, have: &str) -> Vec { + let mut req = pkt(b"command=fetch\n"); + req.extend_from_slice(b"0001"); + req.extend(pkt(b"thin-pack\n")); + req.extend(pkt(format!("want {want}\n").as_bytes())); + req.extend(pkt(format!("have {have}\n").as_bytes())); + req.extend(pkt(b"done\n")); + req.extend_from_slice(b"0000"); + req +} + +fn has_band(resp: &[u8], band: u8) -> bool { + let mut pos = 0usize; + while pos + 4 <= resp.len() { + let len = std::str::from_utf8(&resp[pos..pos + 4]) + .ok() + .and_then(|hex| usize::from_str_radix(hex, 16).ok()) + .unwrap_or(0); + pos += 4; + if len < 4 { + continue; + } + let end = (pos + len - 4).min(resp.len()); + if resp[pos..end].first() == Some(&band) { + return true; + } + pos = end; + } + false +} + +fn created_refs(bare: &knot_git::Repo) -> Vec { + bare.references() + .unwrap() + .into_iter() + .map(|record| record.name.as_str().to_string()) + .collect() +} + +#[test] +fn pack_with_missing_parent_is_now_rejected() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "old.txt", "old\n", "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + must(work, &["rm", "-q", "old.txt"]); + commit(work, "new.txt", "new\n", "c2"); + let c2 = must(work, &["rev-parse", "HEAD"]); + + let oids: Vec = must(work, &["rev-list", "--objects", &c2, "--not", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(work, &oids); + + let mut first = format!("{} {} refs/heads/dangle", Oid::null().to_hex(), c2).into_bytes(); + first.push(0); + first.extend_from_slice(b"report-status\n"); + let mut req = pkt(&first); + req.extend_from_slice(b"0000"); + req.extend_from_slice(&pack); + + let report = knot_pack::receive_pack(&bare, &req).unwrap(); + let text = String::from_utf8_lossy(&report).replace('\0', ""); + + let created = layout + .open(&did) + .unwrap() + .references() + .unwrap() + .into_iter() + .any(|r| r.name.as_str() == "refs/heads/dangle"); + + assert!( + text.contains("ng refs/heads/dangle missing necessary objects"), + "pack whose tip has a missing parent must be rejected:\n{text}" + ); + assert!(!created, "dangling ref mustn't have been created"); +} + +#[test] +fn ls_refs_prefix_and_cob_hiding() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, _work, c1, pack) = seeded(&layout, &did); + knot_pack::receive_pack( + &bare, + &receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack), + ) + .unwrap(); + bare.update_ref(&RefUpdate::Create { + name: RefName::new("refs/cobs/org.tangled.graph.collaborator/secret").unwrap(), + new: Oid::from_hex(&c1).unwrap(), + }) + .unwrap(); + let keep = "refs/tngl/keep/did%3aplc%3asquid/org.tangled.review.patch/x"; + bare.update_ref(&RefUpdate::Create { + name: RefName::new(keep).unwrap(), + new: Oid::from_hex(&c1).unwrap(), + }) + .unwrap(); + + let ls = |prefix: Option<&str>| { + let mut req = pkt(b"command=ls-refs\n"); + if let Some(prefix) = prefix { + req.extend_from_slice(b"0001"); + req.extend(pkt(format!("ref-prefix {prefix}\n").as_bytes())); + } + req.extend_from_slice(b"0000"); + String::from_utf8_lossy(&knot_pack::upload_pack(&bare, &req).unwrap()).into_owned() + }; + + let tags = ls(Some("refs/tags/")); + assert!( + !tags.contains("refs/heads/main"), + "tags-only ref-prefix must exclude heads:\n{tags}" + ); + + let heads = ls(Some("refs/heads/")); + assert!( + heads.contains("refs/heads/main"), + "heads ref-prefix must keep heads:\n{heads}" + ); + assert!( + !heads.contains("refs/cobs"), + "heads ref-prefix must exclude COBs:\n{heads}" + ); + + let cobs = ls(Some("refs/cobs/")); + assert!( + cobs.contains(&format!( + "{c1} refs/cobs/org.tangled.graph.collaborator/secret" + )), + "explicitly sending ref-prefix `refs/cobs/` must hand out COB ref:\n{cobs}" + ); + + assert!( + !heads.contains("refs/tngl"), + "heads ref-prefix must exclude keep refs:\n{heads}" + ); + let keeps = ls(Some("refs/tngl/keep/")); + assert!( + keeps.contains(&format!("{c1} {keep}")), + "explicitly sending ref-prefix `refs/tngl/keep/` must hand out keep ref:\n{keeps}" + ); + + let broad = ls(Some("refs/")); + assert!( + !broad.contains("refs/cobs"), + "a normal `refs/` ref-prefix mustn't show COBs:\n{broad}" + ); + assert!( + !broad.contains("refs/tngl"), + "a normal `refs/` ref-prefix mustn't show keep refs:\n{broad}" + ); + + let all = ls(None); + assert!( + all.contains("refs/heads/main"), + "unfiltered ls-refs must still advertise heads:\n{all}" + ); + assert!( + !all.contains("refs/cobs"), + "unfiltered ls-refs must never advertise cob refs:\n{all}" + ); +} + +#[test] +fn push_namespace_gating_accepts_only_unreserved_refs() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, _work, c1, pack) = seeded(&layout, &did); + + let cases: [(&str, &str, bool); 8] = [ + ( + "refs/cobs/org.tangled.graph.collaborator/evil", + "ng refs/cobs/org.tangled.graph.collaborator/evil", + false, + ), + ( + "refs/hidden/feature/main", + "ng refs/hidden/feature/main", + false, + ), + ("refs/atproto/commit", "ng refs/atproto/commit", false), + ("refs/atproto", "ng refs/atproto", false), + ( + "refs/cob-checkpoints/org.tangled.knot.member/beef", + "ng refs/cob-checkpoints/org.tangled.knot.member/beef", + false, + ), + ("refs/cob-checkpoints", "ng refs/cob-checkpoints", false), + ( + "refs/tngl/keep/did%3aplc%3asquid/org.tangled.review.patch/x/y", + "ng refs/tngl/keep/did%3aplc%3asquid/org.tangled.review.patch/x/y", + false, + ), + ("refs/notes/commits", "ok refs/notes/commits", true), + ]; + cases.iter().for_each(|(refname, verdict, lands)| { + let req = receive_request(refname, &Oid::null().to_hex(), &c1, &pack); + let report = String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()) + .replace('\0', ""); + assert!(report.contains(*verdict), "{verdict}:\n{report}"); + assert_eq!( + created_refs(&bare) + .iter() + .any(|name| name.as_str() == *refname), + *lands, + "ref landing mismatch for {refname}", + ); + }); +} + +#[test] +fn pack_missing_a_blob_is_rejected() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "secret\n", "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + let tree = must(work, &["rev-parse", "HEAD^{tree}"]); + let pack = pack_objects(work, &[c1.clone(), tree]); + + let req = receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()).replace('\0', ""); + assert!( + report.contains("ng refs/heads/main missing necessary objects"), + "pack whose tree references a missing blob must be rejected:\n{report}" + ); + assert!( + created_refs(&bare).is_empty(), + "ref to an object-incomplete commit mustn't be created" + ); +} + +#[test] +fn pack_with_a_submodule_gitlink_is_accepted() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + std::fs::write(work.join("a.txt"), "alpha\n").unwrap(); + must(work, &["add", "a.txt"]); + let absent = "0123456789abcdef0123456789abcdef01234567"; + let cacheinfo = format!("160000,{absent},vendor"); + must( + work, + &["update-index", "--add", "--cacheinfo", cacheinfo.as_str()], + ); + let tree = must(work, &["write-tree"]); + let head = must(work, &["commit-tree", &tree, "-m", "adds a submodule"]); + let oids: Vec = must(work, &["rev-list", "--objects", &head]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(work, &oids); + + let req = receive_request("refs/heads/main", &Oid::null().to_hex(), &head, &pack); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()).replace('\0', ""); + assert!( + report.contains("ok refs/heads/main"), + "a gitlink names a submodule commit the host needn't hold, so the push must land:\n{report}" + ); + assert!( + created_refs(&bare) + .iter() + .any(|name| name.as_str() == "refs/heads/main"), + "the submodule-bearing ref must be created" + ); +} + +#[test] +fn empty_root_commit_over_the_virtual_tree_is_accepted() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + must(work, &["commit", "-q", "--allow-empty", "-m", "empty root"]); + let head = must(work, &["rev-parse", "HEAD"]); + let pack = pack_objects(work, std::slice::from_ref(&head)); + + let req = receive_request("refs/heads/main", &Oid::null().to_hex(), &head, &pack); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()).replace('\0', ""); + assert!( + report.contains("ok refs/heads/main"), + "an empty root commit points at git's virtual empty tree, so the push must land:\n{report}" + ); + assert!( + created_refs(&bare) + .iter() + .any(|name| name.as_str() == "refs/heads/main"), + "the empty root ref must be created" + ); +} + +fn sole_idx(pack_dir: std::path::PathBuf) -> Vec { + let idx = std::fs::read_dir(&pack_dir) + .unwrap() + .filter_map(Result::ok) + .map(|entry| entry.path()) + .find(|path| path.extension().is_some_and(|ext| ext == "idx")) + .expect("exactly one idx file"); + std::fs::read(idx).unwrap() +} + +fn folded_index_matches_canonical_git(format: ObjectFormat) { + let fmt = format!("--object-format={}", format.capability()); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", &fmt, "-q", "-b", "main"]); + std::fs::create_dir_all(work.join("dir/nested")).unwrap(); + let grow = |lines: usize| { + (0..lines) + .map(|n| format!("line {n}\n")) + .collect::() + }; + (0..40).for_each(|revision| { + std::fs::write(work.join("dir/nested/a.txt"), grow(revision * 50 + 10)).unwrap(); + std::fs::write(work.join("b.txt"), grow(revision * 30 + 5)).unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", &format!("rev {revision}")]); + }); + must(work, &["tag", "-a", "v1", "-m", "release one"]); + let head = must(work, &["rev-parse", "HEAD"]); + let oids: Vec = must(work, &["rev-list", "--objects", "--all"]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects_tuned(work, &oids, true); + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_object_format(format); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + let report = String::from_utf8_lossy( + &knot_pack::receive_pack( + &bare, + &receive_request("refs/heads/main", &format.null_oid().to_hex(), &head, &pack), + ) + .unwrap(), + ) + .replace('\0', ""); + assert!( + report.contains("ok refs/heads/main"), + "fold push must land:\n{report}" + ); + let mine = sole_idx(bare.objects_dir().join("pack")); + + let scratch = tempfile::tempdir().unwrap(); + must(scratch.path(), &["init", "--bare", &fmt, "-q"]); + let mut child = knot_fixtures::command(scratch.path()) + .args(["index-pack", "--stdin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(&pack).unwrap(); + assert!( + child.wait().unwrap().success(), + "canonical index-pack failed" + ); + let canonical = sole_idx(scratch.path().join("objects/pack")); + + assert_eq!( + mine, + canonical, + "folded ingest index must be byte-identical to canonical git index-pack for {}", + format.capability() + ); +} + +fn bushy_delta_repo(work: &std::path::Path, fmt: &str) -> String { + must(work, &["init", fmt, "-q", "-b", "main"]); + let body: String = (0..400).map(|n| format!("shared line {n}\n")).collect(); + (0..10).for_each(|revision| { + (0..150).for_each(|file| { + let contents = format!( + "{body}unique {file} rev {revision}\ntail {}\n", + file * 7 + revision + ); + std::fs::write(work.join(format!("f{file:03}.txt")), contents).unwrap(); + }); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", &format!("rev {revision}")]); + }); + must(work, &["rev-parse", "HEAD"]) +} + +#[test] +fn folded_bushy_push_matches_git_through_the_parallel_delta_path() { + let format = ObjectFormat::SHA1; + let fmt = format!("--object-format={}", format.capability()); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + let head = bushy_delta_repo(work, &fmt); + let oids: Vec = must(work, &["rev-list", "--objects", "--all"]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects_tuned(work, &oids, true); + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_object_format(format); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + let report = String::from_utf8_lossy( + &knot_pack::receive_pack( + &bare, + &receive_request("refs/heads/main", &format.null_oid().to_hex(), &head, &pack), + ) + .unwrap(), + ) + .replace('\0', ""); + assert!( + report.contains("ok refs/heads/main"), + "bushy push must land:\n{report}" + ); + let mine = sole_idx(bare.objects_dir().join("pack")); + + let scratch = tempfile::tempdir().unwrap(); + must(scratch.path(), &["init", "--bare", &fmt, "-q"]); + let mut child = knot_fixtures::command(scratch.path()) + .args(["index-pack", "--stdin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(&pack).unwrap(); + assert!( + child.wait().unwrap().success(), + "canonical index-pack failed" + ); + let canonical = sole_idx(scratch.path().join("objects/pack")); + assert_eq!( + mine, canonical, + "a bushy delta pack drives the work-stealing traversal; its folded index must match canonical git" + ); +} + +#[test] +fn a_forced_base_spill_folds_the_bushy_pack_to_the_canonical_index() { + let format = ObjectFormat::SHA1; + let fmt = format!("--object-format={}", format.capability()); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + let _head = bushy_delta_repo(work, &fmt); + let oids: Vec = must(work, &["rev-list", "--objects", "--all"]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects_tuned(work, &oids, true); + + let staged = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(staged.path(), &pack).unwrap(); + let objects = tempfile::tempdir().unwrap(); + let folded = knot_pack::bench_ingest_with_base_budget( + objects.path(), + staged.path(), + format.kind(), + Some(4096), + ) + .unwrap(); + assert!( + folded, + "the bushy pack is self-contained, so a forced spill must still fold it" + ); + let mine = sole_idx(objects.path().join("pack")); + + let scratch = tempfile::tempdir().unwrap(); + must(scratch.path(), &["init", "--bare", &fmt, "-q"]); + let mut child = knot_fixtures::command(scratch.path()) + .args(["index-pack", "--stdin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(&pack).unwrap(); + assert!( + child.wait().unwrap().success(), + "canonical index-pack failed" + ); + let canonical = sole_idx(scratch.path().join("objects/pack")); + assert_eq!( + mine, canonical, + "paging the delta-base working set to disk mustn't change the folded index" + ); +} + +#[test] +fn the_streaming_connectivity_verify_agrees_with_the_in_ram_map() { + let kind = ObjectFormat::SHA1.kind(); + let stage = |pack: &[u8]| -> Option { + let staged = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(staged.path(), pack).unwrap(); + let objects = tempfile::tempdir().unwrap(); + knot_pack::bench_ingest_external(objects.path(), staged.path(), kind).unwrap() + }; + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "hello\n", "c1"); + let all: Vec = must(work, &["rev-list", "--objects", "--all"]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + assert_eq!( + stage(&pack_objects(work, &all)), + Some(true), + "the streaming probe must accept a self-contained pack" + ); + + let c1 = must(work, &["rev-parse", "HEAD"]); + let tree = must(work, &["rev-parse", "HEAD^{tree}"]); + assert_eq!( + stage(&pack_objects(work, &[c1, tree])), + Some(false), + "the streaming probe must reject a pack whose tree references a missing blob" + ); + + let sub_dir = tempfile::tempdir().unwrap(); + let sub = sub_dir.path(); + must(sub, &["init", "-q", "-b", "main"]); + std::fs::write(sub.join("x.txt"), "sub\n").unwrap(); + must(sub, &["add", "x.txt"]); + let cacheinfo = "160000,0123456789abcdef0123456789abcdef01234567,vendor"; + must(sub, &["update-index", "--add", "--cacheinfo", cacheinfo]); + let subtree = must(sub, &["write-tree"]); + let subhead = must(sub, &["commit-tree", &subtree, "-m", "with gitlink"]); + let sub_oids: Vec = must(sub, &["rev-list", "--objects", &subhead]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + assert_eq!( + stage(&pack_objects(sub, &sub_oids)), + Some(true), + "a gitlink names a submodule commit the host needn't hold, so absence is fine" + ); +} + +#[test] +fn folded_first_push_index_is_byte_identical_to_canonical_git_in_both_formats() { + folded_index_matches_canonical_git(ObjectFormat::SHA1); + folded_index_matches_canonical_git(ObjectFormat::SHA256); +} + +#[test] +fn garbage_pack_reports_unpack_error() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let fake = "1111111111111111111111111111111111111111"; + let req = receive_request( + "refs/heads/main", + &Oid::null().to_hex(), + fake, + b"not a packfile", + ); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()).replace('\0', ""); + assert!( + !report.contains("unpack ok"), + "non-PACK body mustn't report unpack ok:\n{report}" + ); + assert!( + report.contains("unpack pack:") && report.contains("PACK signature"), + "malformed pack must surface an unpack error:\n{report}" + ); + assert!( + created_refs(&bare).is_empty(), + "no ref may be created when pack is malformed" + ); +} + +#[test] +fn delete_only_push_has_no_pack() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, _work, c1, pack) = seeded(&layout, &did); + + let create = receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack); + let created = String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &create).unwrap()) + .replace('\0', ""); + assert!( + created.contains("unpack ok") && created.contains("ok refs/heads/main"), + "setup push failed:\n{created}" + ); + + let delete = receive_request("refs/heads/main", &c1, &Oid::null().to_hex(), b""); + let report = String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &delete).unwrap()) + .replace('\0', ""); + assert!( + report.contains("unpack ok"), + "delete-only push has no pack and must still report unpack ok:\n{report}" + ); + assert!( + report.contains("ok refs/heads/main"), + "deleting head must succeed:\n{report}" + ); + assert!( + created_refs(&bare).is_empty(), + "ref must be gone after a delete" + ); +} + +#[test] +fn v2_fetch_negotiation_acks_readies_waits_and_ignores_unknowns() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, _path, old, tip, _blob) = pushed_history(&layout, &did); + + let fake_have = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + let nak = String::from_utf8_lossy( + &knot_pack::upload_pack(&bare, &v2_fetch(&[&tip], &[fake_have], false)).unwrap(), + ) + .into_owned(); + assert!( + nak.contains("acknowledgments"), + "must open acknowledgments:\n{nak}" + ); + assert!(nak.contains("NAK"), "no common commit -> NAK:\n{nak}"); + assert!( + !nak.contains("packfile"), + "with no common commit server mustn't send a pack this round:\n{nak}" + ); + + let bytes = knot_pack::upload_pack(&bare, &v2_fetch(&[&tip], &[&old], false)).unwrap(); + let ready = String::from_utf8_lossy(&bytes).into_owned(); + assert!( + ready.contains(&format!("ACK {old}")), + "must ACK common commit:\n{ready}" + ); + assert!( + ready.contains("ready"), + "must declare ready once a common commit is found" + ); + assert!( + ready.contains("packfile"), + "must open packfile section after ready" + ); + assert!( + contains(&bytes, b"PACK"), + "side-band payload must contain a real PACK" + ); + + let waiting = String::from_utf8_lossy( + &knot_pack::upload_pack( + &bare, + &v2_fetch_with(&[&tip], &[&old], false, &["wait-for-done"]), + ) + .unwrap(), + ) + .into_owned(); + assert!( + waiting.contains(&format!("ACK {old}")), + "wait-for-done still acknowledges common commit:\n{waiting}" + ); + assert!( + !waiting.contains("ready"), + "wait-for-done mustn't declare ready; it waits for the client's done:\n{waiting}" + ); + assert!( + !waiting.contains("packfile"), + "wait-for-done mustn't open pack before done arrives:\n{waiting}" + ); + let finished = knot_pack::upload_pack( + &bare, + &v2_fetch_with(&[&tip], &[&old], true, &["wait-for-done"]), + ) + .unwrap(); + let finished_text = String::from_utf8_lossy(&finished).into_owned(); + assert!( + finished_text.contains("packfile"), + "once done arrives server opens the pack:\n{finished_text}" + ); + assert!( + contains(&finished, b"PACK"), + "follow-up round must contain a real PACK" + ); + + let mut req = pkt(b"command=fetch\n"); + req.extend_from_slice(b"0001"); + [ + "thin-pack\n", + "ofs-delta\n", + "include-tag\n", + "no-progress\n", + "some-future-capability-knot-does-not-know\n", + ] + .iter() + .for_each(|arg| req.extend(pkt(arg.as_bytes()))); + req.extend(pkt(format!("want {tip}\n").as_bytes())); + req.extend(pkt(b"done\n")); + req.extend_from_slice(b"0000"); + let resp = knot_pack::upload_pack(&bare, &req).unwrap(); + let text = String::from_utf8_lossy(&resp); + assert!( + text.contains("packfile"), + "unknown fetch arguments must be ignored and pack still produced:\n{text}" + ); + assert!( + resp.windows(4).any(|window| window == b"PACK"), + "response must still contain a real PACK despite unknown arguments" + ); + + let progress = knot_pack::upload_pack(&bare, &v2_fetch(&[&tip], &[], true)).unwrap(); + assert!( + has_band(&progress, 2), + "fetch must emit sideband band-2 progress" + ); + let suppressed = + knot_pack::upload_pack(&bare, &v2_fetch_with(&[&tip], &[], true, &["no-progress"])) + .unwrap(); + assert!( + !has_band(&suppressed, 2), + "no-progress must suppress band-2 output" + ); + assert!( + suppressed.windows(4).any(|window| window == b"PACK"), + "pack itself must still be sent when progress is suppressed" + ); +} + +fn v2_fetch_with(wants: &[&str], haves: &[&str], done: bool, extra: &[&str]) -> Vec { + let mut req = pkt(b"command=fetch\n"); + req.extend_from_slice(b"0001"); + extra + .iter() + .for_each(|line| req.extend(pkt(format!("{line}\n").as_bytes()))); + wants + .iter() + .for_each(|want| req.extend(pkt(format!("want {want}\n").as_bytes()))); + haves + .iter() + .for_each(|have| req.extend(pkt(format!("have {have}\n").as_bytes()))); + if done { + req.extend(pkt(b"done\n")); + } + req.extend_from_slice(b"0000"); + req +} + +fn pkt_payloads(resp: &[u8]) -> Vec> { + let mut out = Vec::new(); + let mut pos = 0usize; + while pos + 4 <= resp.len() { + let len = std::str::from_utf8(&resp[pos..pos + 4]) + .ok() + .and_then(|hex| usize::from_str_radix(hex, 16).ok()) + .unwrap_or(0); + pos += 4; + if len < 4 { + continue; + } + let end = (pos + len - 4).min(resp.len()); + out.push(resp[pos..end].to_vec()); + pos = end; + } + out +} + +fn banded(payloads: &[Vec], section: &[u8]) -> bool { + payloads + .iter() + .any(|payload| payload.first() == Some(&1) && payload[1..].starts_with(section)) +} + +fn plain(payloads: &[Vec], section: &[u8]) -> bool { + payloads.iter().any(|payload| payload == section) +} + +fn pushed_history( + layout: &Layout, + did: &RepoDid, +) -> (knot_git::Repo, std::path::PathBuf, String, String, String) { + let bare = layout.create(did).unwrap(); + let bare_path = layout.repo_path(did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "1\n", "c1"); + let old = must(work, &["rev-parse", "HEAD"]); + let blob = must(work, &["rev-parse", "HEAD:a.txt"]); + commit(work, "b.txt", "2\n", "c2"); + let tip = must(work, &["rev-parse", "HEAD"]); + must(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + (bare, bare_path, old, tip, blob) +} + +#[test] +fn v2_sideband_all_band_frames_negotiation_and_packfile_uris() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, bare_path, old, tip, blob) = pushed_history(&layout, &did); + + let plain_resp = + knot_pack::upload_pack(&bare, &v2_fetch_with(&[&tip], &[&old], false, &[])).unwrap(); + let plain_payloads = pkt_payloads(&plain_resp); + assert!( + plain(&plain_payloads, b"acknowledgments\n"), + "without sideband-all acknowledgments header is a plain pkt-line" + ); + assert!( + !banded(&plain_payloads, b"acknowledgments\n"), + "without sideband-all negotiation mustn't be band-framed" + ); + + let banded_resp = knot_pack::upload_pack( + &bare, + &v2_fetch_with(&[&tip], &[&old], false, &["sideband-all"]), + ) + .unwrap(); + let banded_payloads = pkt_payloads(&banded_resp); + assert!( + banded(&banded_payloads, b"acknowledgments\n"), + "sideband-all moves acknowledgments header into band 1" + ); + assert!( + banded(&banded_payloads, &format!("ACK {old}\n").into_bytes()), + "sideband-all wraps ACK lines too" + ); + assert!( + !plain(&banded_payloads, b"acknowledgments\n"), + "under sideband-all nothing in negotiation is sent as a plain pkt-line" + ); + + let packhash = "0123456789abcdef0123456789abcdef01234567"; + let uri = format!("https://cdn.nel.pet/{packhash}.pack"); + must( + &bare_path, + &[ + "config", + "uploadpack.blobPackfileUri", + &format!("{blob} {packhash} {uri}"), + ], + ); + let bare = layout.open(&did).unwrap(); + + let plain_uris = knot_pack::upload_pack( + &bare, + &v2_fetch_with(&[&tip], &[], false, &["packfile-uris https"]), + ) + .unwrap(); + let plain_uri_payloads = pkt_payloads(&plain_uris); + assert!( + !plain(&plain_uri_payloads, b"packfile-uris\n") + && !banded(&plain_uri_payloads, b"packfile-uris\n"), + "packfile-uris are ignored without sideband-all" + ); + + let banded_uris = knot_pack::upload_pack( + &bare, + &v2_fetch_with( + &[&tip], + &[], + false, + &["packfile-uris https", "sideband-all"], + ), + ) + .unwrap(); + let banded_uri_payloads = pkt_payloads(&banded_uris); + assert!( + banded(&banded_uri_payloads, b"packfile-uris\n"), + "sideband-all moves packfile-uris header into band 1" + ); + assert!( + !plain(&banded_uri_payloads, b"packfile-uris\n"), + "under sideband-all packfile-uris header is never a plain pkt-line" + ); +} + +fn big_blob_repo( + layout: &Layout, + did: &RepoDid, + size: usize, +) -> (knot_git::Repo, String, Vec, usize) { + let bare = layout.create(did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + let filler: String = std::iter::repeat_n('a', size).collect(); + commit(work, "big.txt", &filler, "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + let oids: Vec = must(work, &["rev-list", "--objects", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let count = oids.len(); + let pack = pack_objects(work, &oids); + (bare, c1, pack, count) +} + +#[test] +fn pack_limits_reject_oversized_and_overdeep_packs() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + + let (size_bare, size_tip, size_pack, count) = + big_blob_repo(&layout, &RepoDid::new("did:plc:squid").unwrap(), 256 * 1024); + assert!(count >= 2, "commit packs at least a commit and a tree"); + + let ofs_bare = layout + .create(&RepoDid::new("did:plc:whelk").unwrap()) + .unwrap(); + let ofs_work = tempfile::tempdir().unwrap(); + let ow = ofs_work.path(); + must(ow, &["init", "-q", "-b", "main"]); + let base: String = std::iter::repeat_n('a', 128 * 1024).collect(); + commit(ow, "big.txt", &base, "c1"); + let oc1 = must(ow, &["rev-parse", "HEAD"]); + let mut tweaked = base.clone(); + tweaked.push('b'); + commit(ow, "big.txt", &tweaked, "c2"); + let oc2 = must(ow, &["rev-parse", "HEAD"]); + let ofs_oids: Vec = must(ow, &["rev-list", "--objects", &oc1, &oc2]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let ofs_pack = pack_objects_tuned(ow, &ofs_oids, true); + + let (ref_bare, ref_tip, ref_pack) = + ref_delta_chain_repo(&layout, &RepoDid::new("did:plc:limpet").unwrap(), 4); + + let cases: [(&knot_git::Repo, &[u8], &str, PackLimits, &str); 5] = [ + ( + &size_bare, + size_pack.as_slice(), + size_tip.as_str(), + PackLimits { + max_object_bytes: knot_pack::MaxObjectBytes::new(4096), + ..generous() + }, + "unpack pack exceeds per-object size limit", + ), + ( + &size_bare, + size_pack.as_slice(), + size_tip.as_str(), + PackLimits { + max_total_bytes: knot_pack::MaxTotalBytes::new(4096), + ..generous() + }, + "unpack pack exceeds total decompressed size limit", + ), + ( + &size_bare, + size_pack.as_slice(), + size_tip.as_str(), + PackLimits { + max_objects: ObjectCount::new(count - 1), + ..generous() + }, + "unpack pack exceeds object count limit", + ), + ( + &ofs_bare, + ofs_pack.as_slice(), + oc2.as_str(), + PackLimits { + max_delta_depth: DeltaDepth::new(0), + ..generous() + }, + "unpack pack exceeds delta chain depth limit", + ), + ( + &ref_bare, + ref_pack.as_slice(), + ref_tip.as_str(), + PackLimits { + max_delta_depth: DeltaDepth::new(0), + ..generous() + }, + "unpack pack exceeds delta chain depth limit", + ), + ]; + cases + .into_iter() + .for_each(|(bare, pack, tip, limits, message)| { + let req = receive_request("refs/heads/main", &Oid::null().to_hex(), tip, pack); + let report = String::from_utf8_lossy( + &knot_pack::receive_pack_with_limits(bare, &req, &limits).unwrap(), + ) + .replace('\0', ""); + assert!(report.contains(message), "{message}:\n{report}"); + assert!( + created_refs(bare).is_empty(), + "refused pack must create no ref" + ); + }); +} + +#[test] +fn a_delta_declaring_an_oversized_result_is_rejected_on_both_ingest_paths() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let limits = generous(); + let bomb = delta_bomb_pack(1 << 40); + let absent_tip = "f".repeat(40); + + let empty = layout + .create(&RepoDid::new("did:plc:cuttle").unwrap()) + .unwrap(); + let fresh_req = receive_request("refs/heads/main", &Oid::null().to_hex(), &absent_tip, &bomb); + let fresh_report = String::from_utf8_lossy( + &knot_pack::receive_pack_with_limits(&empty, &fresh_req, &limits).unwrap(), + ) + .replace('\0', ""); + assert!( + fresh_report.contains("reconstructed size"), + "the fold traversal must refuse a delta declaring an oversized result:\n{fresh_report}" + ); + assert!( + created_refs(&empty).is_empty(), + "refused bomb must create no ref" + ); + + let (seeded_bare, tip, seed_pack, _) = + big_blob_repo(&layout, &RepoDid::new("did:plc:scallop").unwrap(), 1024); + let seed_req = receive_request("refs/heads/main", &Oid::null().to_hex(), &tip, &seed_pack); + let seed_report = String::from_utf8_lossy( + &knot_pack::receive_pack_with_limits(&seeded_bare, &seed_req, &limits).unwrap(), + ) + .replace('\0', ""); + assert!( + seed_report.contains("unpack ok"), + "seeding the repo must succeed:\n{seed_report}" + ); + + let bomb_req = receive_request("refs/heads/bomb", &Oid::null().to_hex(), &absent_tip, &bomb); + let meter_report = String::from_utf8_lossy( + &knot_pack::receive_pack_with_limits(&seeded_bare, &bomb_req, &limits).unwrap(), + ) + .replace('\0', ""); + assert!( + meter_report.contains("per-object size"), + "the meter gate must refuse the bomb on the buffered path:\n{meter_report}" + ); +} + +fn ref_delta_chain_repo( + layout: &Layout, + did: &RepoDid, + revisions: usize, +) -> (knot_git::Repo, String, Vec) { + let bare = layout.create(did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + let tips: Vec = (0..revisions) + .map(|step| { + let body: String = std::iter::repeat_n('a', 64 * 1024).collect(); + commit( + work, + "big.txt", + &format!("{body}{step}\n"), + &format!("c{step}"), + ); + must(work, &["rev-parse", "HEAD"]) + }) + .collect(); + let tip = tips.last().unwrap().clone(); + let oids: Vec = must(work, &["rev-list", "--objects", "HEAD"]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects_tuned(work, &oids, false); + (bare, tip, pack) +} + +#[test] +fn ref_delta_with_in_pack_base_is_resolved() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, tip, pack) = ref_delta_chain_repo(&layout, &did, 4); + + let req = receive_request("refs/heads/main", &Oid::null().to_hex(), &tip, &pack); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()).replace('\0', ""); + assert!( + report.contains("unpack ok") && report.contains("ok refs/heads/main"), + "self-contained ref-delta pack, which gix alone cannot index, must be resolved natively:\n{report}" + ); + assert_eq!( + bare.find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap(), + Some(Oid::from_hex(&tip).unwrap()), + "ref must point at the pushed tip" + ); + let reopened = layout.open(&did).unwrap(); + assert!( + reopened.find_commit(Oid::from_hex(&tip).unwrap()).is_ok(), + "every resolved object must be readable from the odb after push" + ); +} + +#[test] +fn crafted_stale_old_oid_is_rejected_by_server_cas() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "one\n", "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + commit(work, "a.txt", "two\n", "c2"); + let c2 = must(work, &["rev-parse", "HEAD"]); + + let oids1: Vec = must(work, &["rev-list", "--objects", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let create = receive_request( + "refs/heads/main", + &Oid::null().to_hex(), + &c1, + &pack_objects(work, &oids1), + ); + let created = String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &create).unwrap()) + .replace('\0', ""); + assert!( + created.contains("ok refs/heads/main"), + "setup push failed:\n{created}" + ); + + let wrong = "1234567812345678123456781234567812345678"; + let oids2: Vec = must(work, &["rev-list", "--objects", &c2, "--not", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let stale = receive_request("refs/heads/main", wrong, &c2, &pack_objects(work, &oids2)); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &stale).unwrap()).replace('\0', ""); + assert!( + report.contains("unpack ok"), + "pack itself is valid and must unpack:\n{report}" + ); + assert!( + report.contains("ng refs/heads/main"), + "crafted stale old-oid must be refused by the server-side compare-and-swap:\n{report}" + ); + assert_eq!( + bare.find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap(), + Some(Oid::from_hex(&c1).unwrap()), + "ref must stay at its original tip after a rejected stale update" + ); +} + +#[test] +fn fetch_emits_a_thin_pack_against_client_haves() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + let big: String = (0..20_000).map(|line| format!("line {line}\n")).collect(); + let small: String = (0..5_000).map(|line| format!("line {line}\n")).collect(); + commit(work, "f.txt", &big, "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + commit(work, "f.txt", &small, "c2"); + let c2 = must(work, &["rev-parse", "HEAD"]); + + let oids: Vec = must(work, &["rev-list", "--objects", &c2]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects_tuned(work, &oids, true); + let report = String::from_utf8_lossy( + &knot_pack::receive_pack( + &bare, + &receive_request("refs/heads/main", &Oid::null().to_hex(), &c2, &pack), + ) + .unwrap(), + ) + .replace('\0', ""); + assert!( + report.contains("ok refs/heads/main"), + "setup push failed:\n{report}" + ); + + let thin = unsideband(&knot_pack::upload_pack(&bare, &v2_fetch_thin(&c2, &c1)).unwrap()); + assert!( + !index_into_bare(&[], &thin), + "thin-pack fetch must delta against the client's have and omit it, so it cannot resolve standalone" + ); + + let base_oids: Vec = must(work, &["rev-list", "--objects", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + assert!( + thin_resolves_against_base(&pack_objects(work, &base_oids), &thin), + "client that already has the base must resolve the thin pack w/ --fix-thin" + ); + + let fat = unsideband(&knot_pack::upload_pack(&bare, &v2_fetch(&[&c2], &[&c1], true)).unwrap()); + assert!( + index_into_bare(&[], &fat), + "without thin-pack same fetch must be self-contained" + ); +} + +#[test] +fn selection_and_full_clone_expansion_abort_past_object_limit_and_deadline() { + use std::time::Duration; + + use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, build_history}; + use knot_git::{Filter, PackBudget, SelectionLimit}; + + let history = build_history(HistorySpec { + commits: CommitCount::new(8), + paths: PathCount::new(16), + churn: ChurnCount::new(2), + }); + let repo = history.repo(); + let tips = history.tips(); + let stall = Duration::from_secs(60); + + let selected = repo + .select_pack_objects_filtered( + knot_git::Wants::new(&tips), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send + .len(); + assert!( + selected > 4, + "fixture must contain more objects than the limit under test, has {selected}" + ); + assert!( + matches!( + repo.select_pack_objects_filtered( + knot_git::Wants::new(&tips), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::new(ObjectCount::new(4), stall) + ), + Err(knot_git::GitError::Selection(SelectionLimit::Objects)) + ), + "selection past the object-set limit must abort, freeing its core within the budget" + ); + assert!( + matches!( + repo.select_pack_objects_filtered( + knot_git::Wants::new(&tips), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::new(ObjectCount::new(usize::MAX), Duration::ZERO) + ), + Err(knot_git::GitError::Selection(SelectionLimit::Time)) + ), + "selection that makes no progress within its stall window must abort with a time limit, not a partial pack" + ); + + let dir = repo.objects_dir(); + let fmt = repo.object_format().kind(); + let roots = repo.clone_roots(&tips, PackBudget::unbounded()).unwrap(); + let expanded = knot_pack::count_expanded( + &dir, + roots.clone(), + ObjectCount::new(usize::MAX), + stall, + fmt, + ) + .unwrap() + .len(); + assert!( + expanded > 4, + "fixture must contain more objects than the limit under test, has {expanded}" + ); + assert!( + matches!( + knot_pack::count_expanded(&dir, roots.clone(), ObjectCount::new(4), stall, fmt), + Err(knot_pack::PackError::SelectionTooLarge) + ), + "full-clone expansion past the object limit must abort before the entry stream opens" + ); + assert!( + matches!( + knot_pack::count_expanded( + &dir, + roots, + ObjectCount::new(usize::MAX), + Duration::ZERO, + fmt + ), + Err(knot_pack::PackError::SelectionTimeout) + ), + "full-clone expansion that makes no progress within its stall window must abort with a time limit, \ + not a partial pack" + ); +} + +#[test] +fn parallel_selection_matches_the_expansion_count() { + use std::time::Duration; + + use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, build_history}; + use knot_git::{Filter, PackBudget}; + + let history = build_history(HistorySpec { + commits: CommitCount::new(5000), + paths: PathCount::new(2), + churn: ChurnCount::new(1), + }); + let repo = history.repo(); + let tips = history.tips(); + let stall = Duration::from_secs(60); + + let selected = repo + .select_pack_objects_filtered( + knot_git::Wants::new(&tips), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send; + + let dir = repo.objects_dir(); + let fmt = repo.object_format().kind(); + let roots = repo.clone_roots(&tips, PackBudget::unbounded()).unwrap(); + let expanded = knot_pack::count_expanded(&dir, roots, ObjectCount::new(usize::MAX), stall, fmt) + .unwrap() + .len(); + assert_eq!( + selected.len(), + expanded, + "the parallel selection walk must reach the same object set as the expansion oracle" + ); +} + +#[test] +fn full_clone_roots_expand_a_directly_wanted_tree() { + use std::time::Duration; + + use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, build_history}; + use knot_git::{Filter, PackBudget}; + use knot_types::Oid; + + let history = build_history(HistorySpec { + commits: CommitCount::new(4), + paths: PathCount::new(16), + churn: ChurnCount::new(2), + }); + let repo = history.repo(); + let tip = history.tip(); + let tree = Oid::from( + repo.git() + .rev_parse_single(format!("{}^{{tree}}", tip.to_hex()).as_bytes()) + .unwrap() + .detach(), + ); + + let slow = repo + .select_pack_objects_filtered( + knot_git::Wants::new(&[tree]), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send + .len(); + assert!( + slow > 1, + "directly-wanted tree must include its blobs and subtrees, the selection walk found {slow}" + ); + + let stall = Duration::from_secs(60); + let roots = repo.clone_roots(&[tree], PackBudget::unbounded()).unwrap(); + let fast = knot_pack::count_expanded( + &repo.objects_dir(), + roots, + ObjectCount::new(usize::MAX), + stall, + repo.object_format().kind(), + ) + .unwrap() + .len(); + assert_eq!( + fast, slow, + "full-clone fast path must enumerate the same object set as the selection walk \ + for a directly-wanted tree" + ); +} diff --git a/knot2/crates/knot-pack/tests/quarantine.rs b/knot2/crates/knot-pack/tests/quarantine.rs new file mode 100644 index 0000000000000000000000000000000000000000..daff968e073103c7398afeb40d7981ea50f0f721 --- /dev/null +++ b/knot2/crates/knot-pack/tests/quarantine.rs @@ -0,0 +1,651 @@ +use std::path::Path; +use std::sync::Arc; +use std::time::Instant; + +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Grant, MembersChange}; +use knot_git::{Layout, Repo}; +use knot_messages::Catalog; +use knot_pack::{ + MaxWireBytes, PackLimits, PackReceiver, PushGuard, ReceiveCommand, ReceiveFramer, ReceiveGuard, + RefDecision, receive_pack_guarded, receive_request_complete, sweep_incoming, +}; +use knot_runtime::{K256Signer, SeededEntropy, Signer}; +use knot_types::{AccountDid, ActorId, ObjectFormat, Oid, RepoDid, UnixSeconds}; + +const SHA1: ObjectFormat = ObjectFormat::SHA1; + +mod common; +use common::{commit, must, pack_objects, receive_request}; + +fn limits() -> PackLimits { + PackLimits::default() +} + +fn seeded_pack(layout: &Layout, did: &RepoDid) -> (Repo, String, Vec) { + let bare = layout.create(did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "x\n", "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + let oids: Vec = must(work, &["rev-list", "--objects", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(work, &oids); + (bare, c1, pack) +} + +fn report_text(report: &[u8]) -> String { + String::from_utf8_lossy(report).replace('\0', "") +} + +fn dir_count(path: &Path) -> usize { + std::fs::read_dir(path) + .map(|entries| entries.filter_map(Result::ok).count()) + .unwrap_or(0) +} + +fn live_object_count(repo: &Repo) -> usize { + let objects = repo.objects_dir(); + let loose: usize = std::fs::read_dir(&objects) + .map(|entries| { + entries + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.len() == 2) + && entry.path().is_dir() + }) + .map(|shard| dir_count(&shard.path())) + .sum() + }) + .unwrap_or(0); + let packs = std::fs::read_dir(objects.join("pack")) + .map(|entries| { + entries + .filter_map(Result::ok) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "pack")) + .count() + }) + .unwrap_or(0); + loose + packs +} + +struct AllowPublic; + +impl ReceiveGuard for AllowPublic { + fn authorize(&self, _staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands + .iter() + .map(|command| { + if command + .name() + .is_some_and(|name| knot_git::RefClass::of(name).is_public()) + { + RefDecision::Allow + } else { + RefDecision::Reject("not public ref".to_string()) + } + }) + .collect() + } +} + +struct DenyAll; + +impl ReceiveGuard for DenyAll { + fn authorize(&self, _staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands + .iter() + .map(|_| RefDecision::Reject("unauthorized".to_string())) + .collect() + } +} + +struct AllowAll; + +impl ReceiveGuard for AllowAll { + fn authorize(&self, _staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands.iter().map(|_| RefDecision::Allow).collect() + } +} + +struct CobVerify { + owner: ActorId, + home: CobHome, +} + +impl ReceiveGuard for CobVerify { + fn authorize(&self, staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands + .iter() + .map(|command| { + let store = CobStore::new(staged); + let Ok(name) = knot_types::RefName::new(command.refname()) else { + return RefDecision::Reject("invalid ref name".to_string()); + }; + match knot_cobs::verify_cob_ref(&store, &self.home, &name, &self.owner) { + Ok(_) => RefDecision::Allow, + Err(error) => RefDecision::Reject(error.to_string()), + } + }) + .collect() + } +} + +#[test] +fn a_denied_push_leaves_no_objects_in_the_live_odb() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, c1, pack) = seeded_pack(&layout, &did); + + assert_eq!( + live_object_count(&bare), + 0, + "fresh bare repo has no objects" + ); + let request = receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack); + let report = receive_pack_guarded( + &bare, + &request, + &limits(), + &DenyAll, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + let report = report_text(&report); + assert!( + report.contains("ng refs/heads/main unauthorized"), + "{report}" + ); + + assert!( + bare.references().unwrap().is_empty(), + "denied push mustn't create the ref" + ); + assert_eq!( + live_object_count(&bare), + 0, + "denied push must leave no objects in the live odb" + ); + assert!(!bare.contains(Oid::from_hex(&c1).unwrap())); +} + +#[test] +fn an_authorized_push_migrates_objects_then_a_stale_push_is_rejected() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, c1, pack) = seeded_pack(&layout, &did); + + let report = receive_pack_guarded( + &bare, + &receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack), + &limits(), + &AllowPublic, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + let report = report_text(&report); + assert!(report.contains("unpack ok"), "{report}"); + assert!(report.contains("ok refs/heads/main"), "{report}"); + let refs = bare.references().unwrap(); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].name.as_str(), "refs/heads/main"); + assert_eq!(refs[0].target, Oid::from_hex(&c1).unwrap()); + assert!(bare.contains(Oid::from_hex(&c1).unwrap())); + let after_first = live_object_count(&bare); + + let wrong_old = "1".repeat(40); + let fresh = "2".repeat(40); + let report = receive_pack_guarded( + &bare, + &receive_request("refs/heads/main", &wrong_old, &fresh, b""), + &limits(), + &AllowPublic, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + let report = report_text(&report); + assert!(report.contains("ng refs/heads/main"), "{report}"); + assert_eq!( + bare.find_ref(&knot_types::RefName::new("refs/heads/main").unwrap()) + .unwrap(), + Some(Oid::from_hex(&c1).unwrap()), + "stale push mustn't move the ref" + ); + assert_eq!( + live_object_count(&bare), + after_first, + "stale push mustn't add objects to the live odb" + ); +} + +#[test] +fn a_cob_ref_verifies_against_the_owner_key_and_is_refused_for_a_stranger() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let signer = K256Signer::generate(&SeededEntropy::new(7)); + + let source_did = RepoDid::new("did:plc:source").unwrap(); + let home = CobHome::from(&source_did); + let source = layout.create(&source_did).unwrap(); + let store = CobStore::new(&source); + let grant = Grant { + subject: AccountDid::new("did:plc:nel").unwrap(), + added_by: AccountDid::new("did:plc:nel").unwrap(), + created_at: UnixSeconds::new(1), + }; + let created = store + .create( + &home, + &MembersChange::Add(grant), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + let tip = created.tip.oid().to_hex(); + let refname = format!( + "refs/cobs/org.tangled.knot.member/{}", + created.object.oid().to_hex() + ); + + let reachable: Vec = must(source.path(), &["rev-list", "--objects", &tip]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(source.path(), &reachable); + let owner = ActorId::from_secp256k1(signer.public_key().as_bytes()); + + let dest = layout + .create(&RepoDid::new("did:plc:dest").unwrap()) + .unwrap(); + let report = receive_pack_guarded( + &dest, + &receive_request(&refname, &Oid::null().to_hex(), &tip, &pack), + &limits(), + &CobVerify { + owner: owner.clone(), + home: home.clone(), + }, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + assert!( + report_text(&report).contains(&format!("ok {refname}")), + "owner-signed COB ref must be accepted at the receive boundary: {}", + report_text(&report) + ); + + let stranger_key = K256Signer::generate(&SeededEntropy::new(9)); + let stranger = ActorId::from_secp256k1(stranger_key.public_key().as_bytes()); + let dest2 = layout + .create(&RepoDid::new("did:plc:dest2").unwrap()) + .unwrap(); + let report = receive_pack_guarded( + &dest2, + &receive_request(&refname, &Oid::null().to_hex(), &tip, &pack), + &limits(), + &CobVerify { + owner: stranger, + home: home.clone(), + }, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + assert!( + report_text(&report).contains(&format!("ng {refname}")), + "COB ref not signed by the resolved owner key must be refused: {}", + report_text(&report) + ); + assert_eq!( + live_object_count(&dest2), + 0, + "refused COB ref push leaves no objects behind" + ); +} + +#[test] +fn a_reserved_ref_update_is_refused_even_when_the_guard_allows_it() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let cob_ref = format!("refs/cobs/org.tangled.knot.member/{}", "a".repeat(40)); + let old = "1".repeat(40); + let new = "2".repeat(40); + let report = receive_pack_guarded( + &bare, + &receive_request(&cob_ref, &old, &new, b""), + &limits(), + &AllowAll, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + let report = report_text(&report); + + assert!( + report.contains(&format!("ng {cob_ref}")), + "non-create update to a reserved ref must be refused even under an allow-all guard: {report}" + ); + assert!( + report.contains("cannot be modified"), + "refusal must name the create-only rule, not connectivity or compare-and-swap: {report}" + ); + assert!( + bare.references().unwrap().is_empty(), + "refused reserved-ref update must land nothing" + ); + assert_eq!( + live_object_count(&bare), + 0, + "refused reserved-ref update must migrate no objects" + ); +} + +#[test] +fn the_push_guard_allows_a_branch_and_names_the_namespace_it_refuses() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, c1, pack) = seeded_pack(&layout, &did); + let signer = K256Signer::generate(&SeededEntropy::new(11)); + let guard = PushGuard { + cob_authority: ActorId::from_secp256k1(signer.public_key().as_bytes()), + home: CobHome::from(&did), + messages: Arc::new(Catalog::defaults()), + }; + let push = |refname: &str| { + report_text( + &receive_pack_guarded( + &bare, + &receive_request(refname, &Oid::null().to_hex(), &c1, &pack), + &limits(), + &guard, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report, + ) + }; + + [ + ("refs/atproto/commit", "atproto commit chain"), + ( + "refs/cob-checkpoints/org.tangled.knot.member/beef", + "knot-written collaborative-object snapshots", + ), + ("refs/hidden/feature/main", "refs/hidden/* is reserved"), + ( + "refs/tngl/keep/did%3aplc%3asquid/org.tangled.review.patch/x/y", + "knot-written keep refs", + ), + ] + .iter() + .for_each(|(refname, names_itself)| { + let refused = push(refname); + assert!( + refused.contains(&format!("ng {refname}")), + "the same push is fine on a branch, so {refname} must be refused: {refused}" + ); + assert!( + refused.contains(names_itself), + "the refusal must name the namespace that stopped this push, {refname}: {refused}" + ); + assert!( + bare.references().unwrap().is_empty(), + "a refused push to {refname} must leave the ref store empty" + ); + }); + + let branch = push("refs/heads/topic"); + assert!( + branch.contains("ok refs/heads/topic"), + "the same guard still accepts an ordinary branch: {branch}" + ); +} + +fn pseudo_random(seed: u64, len: usize) -> Vec { + let mut state = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(1); + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state >> 24) as u8 + }) + .collect() +} + +fn incompressible_pack(megabytes: usize) -> Vec { + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + (0..megabytes).for_each(|i| { + let blob = pseudo_random(i as u64 + 1, 1024 * 1024); + std::fs::write(work.join(format!("blob-{i:04}.bin")), blob).unwrap(); + }); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "bulk"]); + let head = must(work, &["rev-parse", "HEAD"]); + let oids: Vec = must(work, &["rev-list", "--objects", &head]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + pack_objects(work, &oids) +} + +#[test] +fn the_receive_framer_scans_incrementally_in_linear_time() { + const READ_CHUNK: usize = 64 * 1024; + let pack = incompressible_pack(24); + let body = receive_request( + "refs/heads/main", + &Oid::null().to_hex(), + &"1".repeat(40), + &pack, + ); + assert!( + pack.len() > 8 * 1024 * 1024, + "pack must be large enough to expose any quadratic scaling: {} bytes", + pack.len() + ); + + let single_start = Instant::now(); + let complete = ReceiveFramer::new(limits(), SHA1.kind()) + .advance_bytes(&body) + .unwrap(); + let single = single_start.elapsed(); + assert_eq!( + complete, + Some(body.len()), + "one advance over the full body frames whole request" + ); + + let chunks = body.len().div_ceil(READ_CHUNK); + let chunked_start = Instant::now(); + let mut framer = ReceiveFramer::new(limits(), SHA1.kind()); + let mut framed_at = None; + (1..=chunks).for_each(|n| { + let end = (n * READ_CHUNK).min(body.len()); + if framed_at.is_none() + && let Some(total) = framer.advance_bytes(&body[..end]).unwrap() + { + framed_at = Some(total); + } + }); + let chunked = chunked_start.elapsed(); + assert_eq!( + framed_at, + Some(body.len()), + "feeding the same body in 64KB chunks frames it at identical length" + ); + + let ratio = chunked.as_secs_f64() / single.as_secs_f64().max(1e-6); + assert!( + ratio < 5.0, + "resumable framer never re-inflates settled objects, so the {chunks}-chunk feed \ + must stay within a small constant of one pass; got {ratio:.2}x" + ); +} + +#[test] +fn receive_request_completion_framing_and_the_per_object_limit() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (_bare, c1, pack) = seeded_pack(&layout, &did); + let request = receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack); + + assert_eq!( + receive_request_complete(&request[..request.len() / 2], &limits(), SHA1.kind()).unwrap(), + None, + "half-delivered request isn't yet complete" + ); + assert_eq!( + receive_request_complete(&request, &limits(), SHA1.kind()).unwrap(), + Some(request.len()), + "whole request reports its exact length" + ); + let mut trailing = request.clone(); + trailing.extend_from_slice(b"junk-after-the-pack"); + assert_eq!( + receive_request_complete(&trailing, &limits(), SHA1.kind()).unwrap(), + Some(request.len()), + "framer stops at the pack trailer, ignoring trailing bytes" + ); + + let tight = PackLimits { + max_object_bytes: knot_pack::MaxObjectBytes::new(1), + ..PackLimits::default() + }; + assert!( + receive_request_complete(&request, &tight, SHA1.kind()).is_err(), + "object beyond the per-object limit is refused by the framer" + ); +} + +#[test] +fn sweep_incoming_removes_abandoned_staging_directories() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let staging = bare.path().join(".knot-incoming.4242.0"); + std::fs::create_dir_all(staging.join("objects")).unwrap(); + std::fs::write(staging.join("objects").join("leftover"), b"x").unwrap(); + assert!(staging.exists()); + + let swept = sweep_incoming(scan.path()); + assert_eq!(swept, 1, "exactly one staging directory is swept"); + assert!(!staging.exists(), "abandoned staging directory is gone"); + assert!( + bare.path().join("objects").exists(), + "live repository is untouched" + ); +} + +#[test] +fn streamed_receipt_reassembles_a_request_byte_for_byte() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (_bare, c1, pack) = seeded_pack(&layout, &did); + let request = receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack); + + let dir = tempfile::tempdir().unwrap(); + let mut receiver = PackReceiver::new( + dir.path(), + MaxWireBytes::new(1 << 30), + limits(), + SHA1.kind(), + ) + .unwrap(); + let completed = request + .chunks(7) + .try_fold(false, |_, chunk| receiver.write(chunk)) + .unwrap(); + assert!( + completed, + "the framer must detect completion within the streamed request" + ); + + let body = receiver.finish().unwrap(); + let mut reconstructed = body.preamble().to_vec(); + if let Some(pack) = body.open_pack().unwrap() { + reconstructed.extend_from_slice(&std::fs::read(pack.path()).unwrap()); + } + assert_eq!( + &reconstructed[..], + &request[..], + "the streamed body must be byte-identical to the buffered request" + ); + assert_eq!( + receive_request_complete(&request, &limits(), SHA1.kind()).unwrap(), + Some(request.len()), + "streamed total must agree with the buffered framer" + ); +} + +#[test] +fn streamed_receipt_handles_empty_and_delete_only_requests() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (_bare, c1, _pack) = seeded_pack(&layout, &did); + let dir = tempfile::tempdir().unwrap(); + + let empty = PackReceiver::new( + dir.path(), + MaxWireBytes::new(1 << 30), + limits(), + SHA1.kind(), + ) + .unwrap() + .finish() + .unwrap(); + assert!( + empty.is_empty(), + "a stream with no bytes yields an empty body" + ); + + let delete = receive_request("refs/heads/main", &c1, &Oid::null().to_hex(), &[]); + let mut receiver = PackReceiver::new( + dir.path(), + MaxWireBytes::new(1 << 30), + limits(), + SHA1.kind(), + ) + .unwrap(); + let completed = delete + .chunks(5) + .try_fold(false, |_, chunk| receiver.write(chunk)) + .unwrap(); + assert!(completed, "a delete-only push completes without a pack"); + let body = receiver.finish().unwrap(); + assert_eq!(body.preamble(), &delete[..]); + assert!( + body.open_pack().unwrap().is_none(), + "a delete-only push has no pack" + ); +} diff --git a/knot2/crates/knot-pack/tests/serving.rs b/knot2/crates/knot-pack/tests/serving.rs new file mode 100644 index 0000000000000000000000000000000000000000..04d04deea93bde124c41f93456f9b35f4cbb95f7 --- /dev/null +++ b/knot2/crates/knot-pack/tests/serving.rs @@ -0,0 +1,473 @@ +use std::sync::Arc; + +use axum::Router; +use axum::body::Body; +use axum::http::{Request, StatusCode, header}; +use http_body_util::BodyExt; +use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, +}; +use knot_pack::{PackError, PackLimits, RepoLookup, RepoResolver, RepoTarget, ingest_pack}; +use knot_types::{ + AuthorName, BranchName, Email, ObjectCount, ObjectFormat, Oid, RefName, RepoDid, UnixSeconds, +}; +use tempfile::TempDir; +use tower::ServiceExt; + +mod common; +use common::pkt; + +fn serve_dids() -> Arc { + Arc::new(|target: &RepoTarget| match target { + RepoTarget::Did(did) => RepoLookup::Hosted(did.clone()), + RepoTarget::OwnerPath(_, _) => RepoLookup::Unhosted, + }) +} + +fn commit_blob(repo: &Repo, path: &str, content: &[u8], parents: Vec) -> Oid { + let base = Oid::from(gix::ObjectId::empty_tree(repo.object_format().kind())); + let id = Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + }; + let tree = repo + .write_staged_tree( + base, + &[StagedChange { + path: knot_types::RepoPath::new(path).unwrap(), + action: StagedAction::Put { + content: content.to_vec(), + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + repo.write_commit(&NewCommit { + tree, + parents, + author: id.clone(), + committer: id, + message: "c".to_string(), + extra_headers: Vec::new(), + }) + .unwrap() +} + +fn create_ref(repo: &Repo, name: &str, new: Oid) { + repo.update_ref(&RefUpdate::Create { + name: RefName::new(name).unwrap(), + new, + }) + .unwrap(); +} + +fn seed(format: ObjectFormat) -> (TempDir, Layout, RepoDid, Oid) { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path().join("scan")) + .with_object_format(format) + .with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + let tip = commit_blob(&repo, "reef.txt", b"kelp forest\n", Vec::new()); + create_ref(&repo, "refs/heads/main", tip); + (dir, layout, did, tip) +} + +fn v2_fetch_body(tip: Oid, server_option: bool) -> Vec { + let mut body = pkt(b"command=fetch\n"); + body.extend_from_slice(b"0001"); + body.extend(pkt(format!("want {tip}\n").as_bytes())); + if server_option { + body.extend(pkt(b"server-option=ci-skip\n")); + } + body.extend(pkt(b"done\n")); + body.extend_from_slice(b"0000"); + body +} + +async fn post(router: &Router, did: &str, body: Vec) -> axum::http::Response { + let request = Request::builder() + .method("POST") + .uri(format!("/{did}/git-upload-pack")) + .header("git-protocol", "version=2") + .header( + header::CONTENT_TYPE, + "application/x-git-upload-pack-request", + ) + .body(Body::from(body)) + .unwrap(); + router.clone().oneshot(request).await.unwrap() +} + +async fn post_upload(router: &Router, did: &str, body: Vec) -> Vec { + let response = post(router, did, body).await; + assert_eq!(response.status(), StatusCode::OK); + response + .into_body() + .collect() + .await + .unwrap() + .to_bytes() + .to_vec() +} + +fn hide_secret_ref(repo: &Repo) { + let path = repo.git().git_dir().join("config"); + let mut config = std::fs::read_to_string(&path).unwrap(); + config.push_str("\n[uploadpack]\n\thideRefs = refs/heads/secret\n"); + std::fs::write(&path, config).unwrap(); +} + +fn refused(result: Result, PackError>) -> bool { + matches!(result, Err(PackError::Protocol(_))) +} + +#[test] +fn the_v2_advertisement_offers_server_option_and_a_fetch_using_it_is_served() { + [ObjectFormat::SHA1, ObjectFormat::SHA256] + .into_iter() + .for_each(|format| { + let (_dir, layout, did, tip) = seed(format); + let repo = layout.open(&did).unwrap(); + let advert = knot_pack::advertise_upload(&repo).unwrap(); + assert!( + String::from_utf8_lossy(&advert).contains("server-option"), + "{format:?} advert" + ); + let served = knot_pack::upload_pack(&repo, &v2_fetch_body(tip, true)).unwrap(); + assert!( + String::from_utf8_lossy(&served).contains("packfile"), + "{format:?} server-option fetch" + ); + }); +} + +#[test] +fn upload_pack_serves_cob_tips_and_doesnt_allow_malformed_unreachable_hidden_or_checkpoint_wants() { + let (dir, layout, did, tip) = seed(ObjectFormat::SHA1); + let repo = layout.open(&did).unwrap(); + + let dangling = commit_blob(&repo, "dangle.txt", b"dangling\n", Vec::new()); + assert!( + refused(knot_pack::upload_pack( + &repo, + &v2_fetch_body(dangling, false) + )), + "unreachable want" + ); + + let mut malformed = pkt(b"command=fetch\n"); + malformed.extend_from_slice(b"0001"); + malformed.extend(pkt(b"want not-a-valid-object-id\n")); + malformed.extend(pkt(b"done\n")); + malformed.extend_from_slice(b"0000"); + assert!( + refused(knot_pack::upload_pack(&repo, &malformed)), + "malformed want line" + ); + + let writer = layout.open(&did).unwrap(); + let secret = commit_blob(&writer, "secret.txt", b"hidden\n", vec![tip]); + create_ref(&writer, "refs/heads/secret", secret); + hide_secret_ref(&writer); + + let repo = layout.open(&did).unwrap(); + let named = |scope| { + repo.advertised_refs_for(scope) + .unwrap() + .iter() + .any(|record| record.name.as_str() == "refs/heads/secret") + }; + assert!( + named(knot_git::AdvertScope::Receive), + "hidden ref still public on receive advert" + ); + assert!( + !named(knot_git::AdvertScope::Upload), + "hideRefs strips it from upload advert" + ); + assert!( + refused(knot_pack::upload_pack(&repo, &v2_fetch_body(secret, false))), + "upload-hidden ref by oid" + ); + + let writer = layout.open(&did).unwrap(); + let cob_tip = commit_blob(&writer, "cob.txt", b"grumble\n", vec![tip]); + create_ref( + &writer, + "refs/cobs/org.tangled.graph.collaborator/x", + cob_tip, + ); + let checkpoint = commit_blob(&writer, "checkpoint.txt", b"checkpoint\n", vec![tip]); + create_ref( + &writer, + "refs/cob-checkpoints/org.tangled.knot.member/x", + checkpoint, + ); + + let repo = layout.open(&did).unwrap(); + let served = knot_pack::upload_pack(&repo, &v2_fetch_body(cob_tip, false)).unwrap(); + + let fork = Repo::create(dir.path().join("fork.git")).unwrap(); + ingest_pack( + &fork.objects_dir(), + &common::unsideband(&served), + &PackLimits::default(), + fork.object_format().kind(), + ) + .unwrap(); + assert!(fork.contains(cob_tip), "pack contains the COB tip"); + assert!(fork.contains(tip), "pack contains the COB tip's mama"); + + assert!( + refused(knot_pack::upload_pack( + &repo, + &v2_fetch_body(checkpoint, false) + )), + "checkpoint want" + ); +} + +#[test] +fn upload_pack_v0_serves_a_known_cob_tip() { + let (dir, layout, did, tip) = seed(ObjectFormat::SHA1); + let writer = layout.open(&did).unwrap(); + let cob_tip = commit_blob(&writer, "cob.txt", b"grumble\n", vec![tip]); + create_ref( + &writer, + "refs/cobs/org.tangled.graph.collaborator/x", + cob_tip, + ); + let repo = layout.open(&did).unwrap(); + + let mut body = pkt(format!("want {cob_tip} side-band-64k\n").as_bytes()); + body.extend(pkt(b"done\n")); + body.extend_from_slice(b"0000"); + let served = knot_pack::upload_pack(&repo, &body).unwrap(); + + let fork = Repo::create(dir.path().join("fork.git")).unwrap(); + ingest_pack( + &fork.objects_dir(), + &common::unsideband(&served), + &PackLimits::default(), + fork.object_format().kind(), + ) + .unwrap(); + assert!(fork.contains(cob_tip), "v0 pack contains the COB tip"); +} + +#[test] +fn upload_pack_serves_a_tngl_keep_tip() { + let (dir, layout, did, tip) = seed(ObjectFormat::SHA1); + let writer = layout.open(&did).unwrap(); + let kept = commit_blob(&writer, "kept.txt", b"kept\n", vec![tip]); + create_ref( + &writer, + "refs/tngl/keep/did%3aplc%3asquid/org.tangled.review.patch/x/y", + kept, + ); + let repo = layout.open(&did).unwrap(); + let served = knot_pack::upload_pack(&repo, &v2_fetch_body(kept, false)).unwrap(); + + let fork = Repo::create(dir.path().join("fork.git")).unwrap(); + ingest_pack( + &fork.objects_dir(), + &common::unsideband(&served), + &PackLimits::default(), + fork.object_format().kind(), + ) + .unwrap(); + assert!(fork.contains(kept), "pack contains the refs/tngl tip"); +} + +#[tokio::test] +async fn the_pack_cache_replays_then_invalidates_when_a_ref_is_hidden() { + let (dir, layout, did, tip) = seed(ObjectFormat::SHA1); + let writer = layout.open(&did).unwrap(); + let secret = commit_blob(&writer, "secret.txt", b"hidden\n", vec![tip]); + create_ref(&writer, "refs/heads/secret", secret); + + let router = knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ); + + let body = v2_fetch_body(tip, false); + let first = post_upload(&router, did.as_str(), body.clone()).await; + let second = post_upload(&router, did.as_str(), body).await; + assert_eq!( + first, second, + "a cache hit replays the leader's bytes exactly" + ); + let fork = Repo::create(dir.path().join("fork.git")).unwrap(); + ingest_pack( + &fork.objects_dir(), + &common::unsideband(&first), + &PackLimits::default(), + fork.object_format().kind(), + ) + .unwrap(); + assert!( + fork.contains(tip), + "the cached pack contains the wanted tip" + ); + + let secret_body = v2_fetch_body(secret, false); + let warm = post_upload(&router, did.as_str(), secret_body.clone()).await; + assert!( + !common::unsideband(&warm).is_empty(), + "the visible secret want is served and cached" + ); + + hide_secret_ref(&writer); + let after = post(&router, did.as_str(), secret_body).await.status(); + assert_eq!( + after, + StatusCode::BAD_REQUEST, + "once hidden the cached pack isn't replayed" + ); +} + +fn maint_opts() -> knot_maintenance::Options { + knot_maintenance::Options { + repack_max_objects: ObjectCount::new(1_000_000), + geometric_factor: knot_maintenance::GeometricFactor::full_repack(), + loose_threshold: ObjectCount::new(0), + prune_grace: knot_maintenance::PruneGrace::from_secs(0), + full_repack_interval: knot_maintenance::FullRepackInterval::from_secs(432_000), + reflog_floor: knot_maintenance::ReflogRetention::from_secs(i64::MAX as u64 / 4), + commit_graph: false, + multi_pack_index: false, + bitmap: true, + } +} + +fn pack_object_count(pack: &[u8]) -> u32 { + assert_eq!( + &pack[..4], + b"PACK", + "a served body begins with the pack signature" + ); + u32::from_be_bytes([pack[8], pack[9], pack[10], pack[11]]) +} + +#[test] +fn the_bitmap_fast_path_serves_the_same_pack_as_the_object_walk() { + [ObjectFormat::SHA1, ObjectFormat::SHA256] + .into_iter() + .for_each(|format| { + let (dir, layout, did, tip) = seed(format); + let repo = layout.open(&did).unwrap(); + let body = v2_fetch_body(tip, false); + let walk = common::unsideband(&knot_pack::upload_pack(&repo, &body).unwrap()); + + let now = UnixSeconds::new(1_700_000_500); + assert!( + knot_maintenance::run_repo(&repo, now, &maint_opts()) + .unwrap() + .bitmap, + "{format:?} seed packs a bitmap" + ); + + let fast = common::unsideband(&knot_pack::upload_pack(&repo, &body).unwrap()); + assert_eq!( + pack_object_count(&fast), + pack_object_count(&walk), + "{format:?} reuse vs walk count" + ); + + let did = RepoDid::new("did:plc:clam").unwrap(); + let fork = Layout::new(dir.path().join("fork")) + .with_object_format(format) + .create(&did) + .unwrap(); + ingest_pack( + &fork.objects_dir(), + &fast, + &PackLimits::default(), + fork.object_format().kind(), + ) + .unwrap(); + let closure: std::collections::HashSet = fork + .select_pack_objects(knot_git::Wants::new(&[tip]), knot_git::Haves::new(&[])) + .unwrap() + .into_iter() + .collect(); + assert!( + fork.contains(tip) && closure.len() as u32 == pack_object_count(&fast), + "{format:?} fast-path ingests as tip closure" + ); + }); +} + +fn tree_and_blob(repo: &Repo, commit: Oid) -> (Oid, Oid) { + let tree = repo + .git() + .find_commit(commit.object_id()) + .unwrap() + .tree() + .unwrap(); + let blob = tree.iter().next().unwrap().unwrap().object_id(); + (Oid::from(tree.id), Oid::from(blob)) +} + +#[test] +fn bitmapped_tree_and_blob_wants_keep_the_walks_visibility() { + [ObjectFormat::SHA1, ObjectFormat::SHA256] + .into_iter() + .for_each(|format| { + let (_dir, layout, did, tip) = seed(format); + let writer = layout.open(&did).unwrap(); + let secret = commit_blob(&writer, "secret.txt", b"hidden\n", vec![tip]); + create_ref(&writer, "refs/heads/secret", secret); + hide_secret_ref(&writer); + let cob = commit_blob(&writer, "cob.txt", b"grumble\n", vec![tip]); + create_ref(&writer, "refs/cobs/org.tangled.graph.collaborator/x", cob); + + let repo = layout.open(&did).unwrap(); + let now = UnixSeconds::new(1_700_000_500); + assert!( + knot_maintenance::run_repo(&repo, now, &maint_opts()) + .unwrap() + .bitmap, + "{format:?} seed packs a bitmap" + ); + // landed after the repack, so only the walk past the bitmap sees it + let late = commit_blob(&writer, "late.txt", b"late\n", vec![tip]); + create_ref(&writer, "refs/heads/late", late); + // after the repack too, which prunes it otherwise + let dangling = commit_blob(&writer, "dangle.txt", b"dangling\n", Vec::new()); + + let repo = layout.open(&did).unwrap(); + assert!( + knot_git::unreachable_via_bitmap(&repo, &[tip], &[]) + .unwrap() + .is_some(), + "{format:?} wants are checked against the bitmap" + ); + [ + (tip, true), + (cob, true), + (late, true), + (secret, false), + (dangling, false), + ] + .into_iter() + .for_each(|(commit, served)| { + let (tree, blob) = tree_and_blob(&repo, commit); + [("tree", tree), ("blob", blob)] + .into_iter() + .for_each(|(kind, want)| { + let refusal = + refused(knot_pack::upload_pack(&repo, &v2_fetch_body(want, false))); + assert_eq!( + !refusal, served, + "{format:?} {kind} of {commit} served={served}" + ); + }); + }); + }); +} diff --git a/knot2/crates/knot-pack/tests/soak.rs b/knot2/crates/knot-pack/tests/soak.rs new file mode 100644 index 0000000000000000000000000000000000000000..9bc84214bf4441cd80ed54c9e68a7bbb0b31634d --- /dev/null +++ b/knot2/crates/knot-pack/tests/soak.rs @@ -0,0 +1,273 @@ +use std::net::SocketAddr; +use std::path::Path; +use std::process::{Child, Stdio}; +use std::time::Duration; + +use axum::Router; +use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, write_history}; +use knot_git::Layout; +use knot_pack::{RepoLookup, RepoResolver, RepoTarget}; +use knot_types::RepoDid; + +mod common; +use common::must; + +fn serve_dids() -> std::sync::Arc { + std::sync::Arc::new(|target: &RepoTarget| match target { + RepoTarget::Did(did) => RepoLookup::Hosted(did.clone()), + RepoTarget::OwnerPath(_, _) => RepoLookup::Unhosted, + }) +} + +const BLOB_BYTES: usize = 16 * 1024 * 1024; +const CONCURRENCY: usize = 10; +const ROUNDS: usize = 5; +const PAGE_BYTES: u64 = 4096; +const OOM_CEILING: u64 = 1024 * 1024 * 1024; +const GROWTH_SLACK: u64 = 64 * 1024 * 1024; +const CURVE_LEVELS: [usize; 5] = [1, 2, 4, 8, 16]; +const PER_CONNECTION_CEILING: u64 = 96 * 1024 * 1024; +const SUBLINEAR_SLACK: u64 = 32 * 1024 * 1024; + +static RSS_GATE: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn incompressible(len: usize) -> Vec { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() +} + +async fn spawn(router: Router) -> SocketAddr { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + addr +} + +fn rss_bytes() -> u64 { + let statm = std::fs::read_to_string("/proc/self/statm").expect("/proc/self/statm is readable"); + statm + .split_whitespace() + .nth(1) + .and_then(|pages| pages.parse::().ok()) + .map(|pages| pages * PAGE_BYTES) + .expect("statm lists the resident page count") +} + +fn clone_child(remote: &str, dest: &Path) -> Child { + knot_fixtures::command(dest.parent().unwrap_or(dest)) + .args(["clone", "--bare", "--quiet", remote, dest.to_str().unwrap()]) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .expect("git clone spawns") +} + +struct Soak { + _scan: tempfile::TempDir, + scratch: tempfile::TempDir, + remote: String, + tip: String, +} + +async fn serve_large_repo() -> Soak { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + std::fs::write(work.join("big.bin"), incompressible(BLOB_BYTES)).unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", "large"]); + must(&work, &["push", "-q", bare.to_str().unwrap(), "main"]); + must(bare.as_path(), &["symbolic-ref", "HEAD", "refs/heads/main"]); + let tip = must(&work, &["rev-parse", "HEAD"]); + + let addr = spawn(knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + )) + .await; + let remote = format!("http://{addr}/{}", did.as_str()); + + Soak { + _scan: scan, + scratch, + remote, + tip, + } +} + +async fn serve_wide_history() -> Soak { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + let tip = write_history( + &repo, + HistorySpec { + commits: CommitCount::new(256), + paths: PathCount::new(4096), + churn: ChurnCount::new(16), + }, + ) + .to_hex(); + + let scratch = tempfile::tempdir().unwrap(); + let addr = spawn(knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + )) + .await; + let remote = format!("http://{addr}/{}", did.as_str()); + + Soak { + _scan: scan, + scratch, + remote, + tip, + } +} + +fn drain_storm(remote: &str, scratch: &Path, round: usize, concurrency: usize, tip: &str) -> u64 { + let dests: Vec = (0..concurrency) + .map(|index| scratch.join(format!("clone-{round}-{index}"))) + .collect(); + let mut children: Vec = dests.iter().map(|dest| clone_child(remote, dest)).collect(); + + let mut peak = rss_bytes(); + let mut pending = true; + while pending { + peak = peak.max(rss_bytes()); + std::thread::sleep(Duration::from_millis(3)); + pending = children + .iter_mut() + .any(|child| matches!(child.try_wait(), Ok(None))); + } + + children.into_iter().enumerate().for_each(|(index, child)| { + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "round {round} clone {index} failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + }); + + dests.iter().for_each(|dest| { + assert_eq!( + must(dest, &["rev-parse", "HEAD"]), + tip, + "soak clone must reproduce repo tip" + ); + must(dest, &["fsck", "--connectivity-only", "--no-progress"]); + std::fs::remove_dir_all(dest).unwrap(); + }); + + peak.max(rss_bytes()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn concurrent_clone_memory_cost_curve() { + let _rss_gate = RSS_GATE.lock().await; + let soak = serve_wide_history().await; + + let baseline = rss_bytes(); + let mut peak = baseline; + let curve: Vec<(usize, u64)> = CURVE_LEVELS + .into_iter() + .map(|concurrency| { + peak = peak.max(drain_storm( + &soak.remote, + soak.scratch.path(), + concurrency, + concurrency, + &soak.tip, + )); + (concurrency, peak.saturating_sub(baseline)) + }) + .collect(); + + curve.iter().for_each(|(concurrency, delta)| { + let per_connection = delta / *concurrency as u64; + println!( + "{concurrency} concurrent clones: cumulative +{} MiB, ~{} MiB per connection", + delta / (1024 * 1024), + per_connection / (1024 * 1024) + ); + assert!( + per_connection <= PER_CONNECTION_CEILING, + "per-connection high-water for {concurrency} clones is {} MiB, past {} MiB ceiling", + per_connection / (1024 * 1024), + PER_CONNECTION_CEILING / (1024 * 1024) + ); + }); + + let (_, single) = curve[0]; + let (top, top_delta) = *curve.last().unwrap(); + let top_per_connection = top_delta / top as u64; + assert!( + top_per_connection <= single + SUBLINEAR_SLACK, + "memory grows super-linearly with concurrency. {top} clones cost {} MiB per connection \ + against {} MiB for single clone, past the {} MiB slack. Pack is shared, so the \ + per-connection high-water must stay flat as connections rise", + top_per_connection / (1024 * 1024), + single / (1024 * 1024), + SUBLINEAR_SLACK / (1024 * 1024) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn concurrent_clones_of_a_large_repo_stay_bounded() { + let _rss_gate = RSS_GATE.lock().await; + let soak = serve_large_repo().await; + let scratch = &soak.scratch; + let remote = soak.remote.clone(); + let tip = soak.tip.clone(); + + let baseline = rss_bytes(); + let mut peak = baseline; + let after_round: Vec = (0..ROUNDS) + .map(|round| { + peak = peak.max(drain_storm( + &remote, + scratch.path(), + round, + CONCURRENCY, + &tip, + )); + rss_bytes() + }) + .collect(); + + assert!( + peak < OOM_CEILING, + "serving {CONCURRENCY} concurrent clones mustn't balloon resident memory: peak {} MiB exceeds {} MiB ceiling", + peak / (1024 * 1024), + OOM_CEILING / (1024 * 1024) + ); + + let settled = after_round[..ROUNDS - 1].iter().copied().max().unwrap(); + let last = after_round[ROUNDS - 1]; + assert!( + last <= settled + GROWTH_SLACK, + "resident memory is still climbing at final round, a leak: settled at {} MiB, round {ROUNDS} left {} MiB", + settled / (1024 * 1024), + last / (1024 * 1024) + ); +} diff --git a/knot2/crates/knot-postreceive/Cargo.toml b/knot2/crates/knot-postreceive/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..067d9c09dbb1159998c9c0a4726cf0e79e0ceceb --- /dev/null +++ b/knot2/crates/knot-postreceive/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "knot-postreceive" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-workflow = { workspace = true } +knot-messages = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +knot-runtime = { workspace = true } +knot-fixtures = { workspace = true } +tempfile = { workspace = true } +serde_json = { workspace = true } diff --git a/knot2/crates/knot-postreceive/src/lib.rs b/knot2/crates/knot-postreceive/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..c3b3278538bdd016d3e8471f1ec5fd1ae05e2e1c --- /dev/null +++ b/knot2/crates/knot-postreceive/src/lib.rs @@ -0,0 +1,317 @@ +use knot_git::{EntryKind, RefUpdate, Repo}; +use knot_messages::{PushMessages, UrlKey}; +use knot_types::{ + AccountDid, AppviewEndpoint, BranchName, ChangedFiles, Handle, Listing, Oid, OriginUrl, + OwnerDid, RefName, RefTransition, RepoDid, RepoPath, RepoRkey, +}; +use knot_workflow::{Compiled, RawWorkflow, Trigger, WorkflowName}; +use url::Url; + +const WORKFLOW_DIR: &str = ".tangled/workflows"; + +pub struct Actor { + pub committer: AccountDid, + pub repo: RepoDid, +} + +pub enum Ci { + Skip, + Compile { verbose: bool }, +} + +pub enum OwnerLabel { + Handle(Handle), + Did(OwnerDid), +} + +impl OwnerLabel { + pub fn as_str(&self) -> &str { + match self { + OwnerLabel::Handle(handle) => handle.as_str(), + OwnerLabel::Did(did) => did.as_str(), + } + } +} + +pub struct PullLink { + pub appview: AppviewEndpoint, + pub owner: OwnerLabel, + pub rkey: RepoRkey, +} + +struct SourceBranch(BranchName); +struct TargetBranch(BranchName); + +struct PushContext<'a> { + actor: &'a Actor, + ci: &'a Ci, + pull: Option<&'a PullLink>, + messages: &'a PushMessages, +} + +pub fn post_receive( + repo: &Repo, + actor: &Actor, + applied: &[RefUpdate], + ci: &Ci, + pull: Option<&PullLink>, + messages: &PushMessages, +) -> Vec { + let context = PushContext { + actor, + ci, + pull, + messages, + }; + applied + .iter() + .flat_map(|update| publish_one(repo, update, &context)) + .collect() +} + +fn publish_one(repo: &Repo, update: &RefUpdate, context: &PushContext) -> Vec { + let name = update.name(); + let transition = update.transition(); + let changed = match transition.new_oid() { + Some(new) => changed_paths(repo, name, transition.old_oid(), new), + None => ChangedFiles::none(), + }; + let pipeline = ci_messages(repo, name, transition, &changed, context); + + let pull_link = match (transition, context.pull) { + (RefTransition::Create { .. }, Some(link)) => { + pull_request_message(repo, link, name, context.messages, &context.actor.repo) + .unwrap_or_default() + } + _ => Vec::new(), + }; + pull_link.into_iter().chain(pipeline).collect() +} + +fn changed_paths(repo: &Repo, name: &RefName, old: Option, new: Oid) -> ChangedFiles { + let range = knot_git::PatchRange { + base: old, + head: new, + }; + match repo.changed_paths(range) { + Ok(changed) => { + if changed.listing() == Listing::Truncated { + tracing::warn!( + ref_name = name.as_str(), + path = %repo.path().display(), + files = changed.paths().len(), + "changed-file listing truncated at the record budget, leaving every paths constraint assumed matched" + ); + } + changed + } + Err(error) => { + tracing::warn!( + ref_name = name.as_str(), + path = %repo.path().display(), + %error, + "changed-file listing failed, leaving every paths constraint assumed matched" + ); + ChangedFiles::unknown() + } + } +} + +fn pull_request_message( + repo: &Repo, + link: &PullLink, + name: &RefName, + messages: &PushMessages, + repo_did: &RepoDid, +) -> Option> { + let branch = branch_short(name)?; + let default_ref = repo.default_branch()?; + let default = branch_short(&default_ref)?; + if branch == default { + return None; + } + repo.find_ref(&default_ref).ok().flatten()?; + + let url = match repo.origin_url() { + Some(remote) => fork_pull_url( + &link.appview, + &SourceBranch(branch), + &TargetBranch(default), + remote, + repo_did, + )?, + None => branch_pull_url( + &link.appview, + &link.owner, + &link.rkey, + &SourceBranch(branch), + &TargetBranch(default), + )?, + }; + + Some(messages.pull_request.lines(|UrlKey::Url| url.to_string())) +} + +fn branch_pull_url( + appview: &AppviewEndpoint, + owner: &OwnerLabel, + repo_rkey: &RepoRkey, + source: &SourceBranch, + target: &TargetBranch, +) -> Option { + let mut url = Url::parse(appview.as_str()).ok()?; + + url.path_segments_mut().ok()?.pop_if_empty().extend([ + owner.as_str(), + repo_rkey.as_str(), + "pulls", + "new", + ]); + + url.query_pairs_mut() + .append_pair("source", "branch") + .append_pair("sourceBranch", source.0.as_str()) + .append_pair("targetBranch", target.0.as_str()); + + Some(url) +} + +fn fork_pull_url( + appview: &AppviewEndpoint, + source: &SourceBranch, + target: &TargetBranch, + remote: OriginUrl, + repo_did: &RepoDid, +) -> Option { + let remote_url = Url::parse(remote.as_str()).ok()?; + + // TODO: We need to handle file schemes. For now though if the remote is a + // file scheme a fork PR link won't be created. + match remote_url.scheme() { + "http" | "https" => (), + _ => return None, + } + + let paths: Vec<&str> = remote_url + .path_segments() + .map(|segments| segments.collect()) + .unwrap_or_default(); + + let mut url = Url::parse(appview.as_str()).ok()?; + + url.path_segments_mut() + .ok()? + .pop_if_empty() + .extend(paths) + .extend(["pulls", "new"]); + + url.query_pairs_mut() + .append_pair("source", "fork") + .append_pair("sourceBranch", source.0.as_str()) + .append_pair("targetBranch", target.0.as_str()) + .append_pair("fork", repo_did.as_str()); + + Some(url) +} + +fn branch_short(name: &RefName) -> Option { + name.as_str() + .strip_prefix("refs/heads/") + .and_then(|short| BranchName::new(short).ok()) +} + +fn ci_messages( + repo: &Repo, + name: &RefName, + transition: RefTransition, + changed: &ChangedFiles, + context: &PushContext, +) -> Vec { + let Ci::Compile { verbose } = context.ci else { + return Vec::new(); + }; + let Some(new) = transition.new_oid() else { + return Vec::new(); + }; + let templates = context.messages; + let raws = read_workflows(repo, new); + let Compiled { + workflows, + diagnostics, + } = knot_workflow::compile( + &raws, + &Trigger::Push { + ref_name: name.clone(), + }, + changed, + ); + let mut messages = diagnostics.errors; + if *verbose { + let clean = messages.is_empty() && diagnostics.warnings.is_empty(); + messages.extend(diagnostics.warnings); + match (workflows.is_empty(), clean) { + (true, _) => messages.extend(templates.pipeline_none.text_lines()), + (false, true) => messages.extend(templates.pipeline_clean.text_lines()), + (false, false) => {} + } + } + messages +} + +fn read_workflows(repo: &Repo, new: Oid) -> Vec { + let commit = match repo.peel_to_commit(new) { + Ok(commit) => commit, + Err(error) => { + tracing::warn!( + commit = %new.to_hex(), + path = %repo.path().display(), + %error, + "workflow read failed peeling commit" + ); + return Vec::new(); + } + }; + let workflow_dir = RepoPath::new(WORKFLOW_DIR).expect("literal workflow dir is well-formed"); + let entries = match repo.tree_entries_at(commit, Some(&workflow_dir)) { + Ok(entries) => entries.unwrap_or_default(), + Err(error) => { + tracing::warn!( + path = %repo.path().display(), + commit = %new.to_hex(), + %error, + "workflow directory read failed" + ); + return Vec::new(); + } + }; + entries + .into_iter() + .filter(|entry| matches!(entry.kind, EntryKind::Blob | EntryKind::BlobExecutable)) + .filter_map(|entry| { + let name = match WorkflowName::new(entry.name.as_str()) { + Ok(name) => name, + Err(error) => { + tracing::warn!( + workflow = %entry.name, + path = %repo.path().display(), + %error, + "workflow name rejected" + ); + return None; + } + }; + match repo.read_blob(entry.oid) { + Ok(contents) => Some(RawWorkflow { name, contents }), + Err(error) => { + tracing::warn!( + workflow = %entry.name, + path = %repo.path().display(), + %error, + "workflow unreadable" + ); + None + } + } + }) + .collect() +} diff --git a/knot2/crates/knot-postreceive/tests/post_receive.rs b/knot2/crates/knot-postreceive/tests/post_receive.rs new file mode 100644 index 0000000000000000000000000000000000000000..a100738eeaf1e7a91c83f4058e1d32a1a58283ce --- /dev/null +++ b/knot2/crates/knot-postreceive/tests/post_receive.rs @@ -0,0 +1,318 @@ +use std::path::Path; + +use knot_git::{Layout, RefUpdate, Repo}; +use knot_postreceive::{Actor, Ci, OwnerLabel, PullLink, post_receive}; +use knot_types::{ + AccountDid, AppviewEndpoint, BranchName, Handle, Oid, OriginUrl, RefName, RepoDid, RepoRkey, +}; + +const DID: &str = "did:plc:limpet"; +const COMMITTER: &str = "did:plc:nel"; + +fn git(cwd: &Path, args: &[&str]) -> String { + let output = knot_fixtures::command(cwd) + .args(args) + .output() + .expect("git is available"); + assert!( + output.status.success(), + "git {args:?} failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().trim().to_string() +} + +fn commit_file(work: &Path, file: &str, contents: &str, message: &str) { + let path = work.join(file); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", message]); +} + +struct World { + _scan: tempfile::TempDir, + _work: tempfile::TempDir, + repo: Repo, + work: std::path::PathBuf, + bare: std::path::PathBuf, +} + +fn world() -> World { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new(DID).unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path().to_path_buf(); + git(&work, &["init", "-q", "-b", "main"]); + let repo = layout.open(&did).unwrap(); + World { + _scan: scan, + _work: work_dir, + repo, + work, + bare, + } +} + +fn push(world: &World, branch: &str) { + git( + &world.work, + &["push", "-q", world.bare.to_str().unwrap(), branch], + ); +} + +fn oid(world: &World, rev: &str) -> Oid { + Oid::from_hex(&git(&world.work, &["rev-parse", rev])).unwrap() +} + +fn actor() -> Actor { + Actor { + committer: AccountDid::new(COMMITTER).unwrap(), + repo: RepoDid::new(DID).unwrap(), + } +} + +fn refname(name: &str) -> RefName { + RefName::new(name).unwrap() +} + +fn pull() -> PullLink { + PullLink { + appview: AppviewEndpoint::new("https://tangled.test").unwrap(), + owner: OwnerLabel::Handle(Handle::new_owned("nel.pet").unwrap()), + rkey: RepoRkey::new("anemone").unwrap(), + } +} + +fn compile(verbose: bool) -> Ci { + Ci::Compile { verbose } +} + +fn run(world: &World, applied: &[RefUpdate], ci: &Ci, pull: Option<&PullLink>) -> Vec { + let repo = Repo::open(&world.bare).unwrap(); + post_receive( + &repo, + &actor(), + applied, + ci, + pull, + &knot_messages::default_catalog().push, + ) +} + +fn created(branch: &str, head: Oid) -> [RefUpdate; 1] { + [RefUpdate::Create { + name: refname(&format!("refs/heads/{branch}")), + new: head, + }] +} + +fn create(world: &World, branch: &str) -> [RefUpdate; 1] { + commit_file(&world.work, "a.txt", "one\n", "first"); + push(world, branch); + created(branch, oid(world, "HEAD")) +} + +fn create_feature(world: &World) -> Oid { + commit_file(&world.work, "a.txt", "one\n", "first"); + push(world, "main"); + git(&world.work, &["checkout", "-q", "-b", "feature"]); + commit_file(&world.work, "c.txt", "three\n", "feature work"); + push(world, "feature"); + oid(world, "HEAD") +} + +#[test] +fn a_push_without_workflows_says_so_when_verbose() { + let world = world(); + let applied = create(&world, "main"); + let messages = run(&world, &applied, &compile(true), None); + assert!( + messages + .iter() + .any(|line| line == "no pipelines to compile"), + "{messages:?}" + ); + assert!( + messages + .iter() + .all(|line| line != "pipeline compiled with no diagnostics"), + "{messages:?}" + ); +} + +#[test] +fn verbose_keeps_the_warning_that_explains_why_nothing_compiled() { + let world = world(); + commit_file( + &world.work, + ".tangled/workflows/ci.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: [release]\n", + "add ci", + ); + push(&world, "main"); + let head = oid(&world, "HEAD"); + + let messages = run(&world, &created("main", head), &compile(true), None); + + assert!( + messages + .iter() + .any(|line| line.contains("workflow skipped")), + "a workflow that misses the trigger still reports why: {messages:?}" + ); + assert!( + messages + .iter() + .any(|line| line == "no pipelines to compile"), + "{messages:?}" + ); +} + +#[test] +fn a_new_non_default_branch_yields_a_pull_request_link() { + let world = world(); + let head = create_feature(&world); + + let applied = created("feature", head); + let messages = run(&world, &applied, &Ci::Skip, Some(&pull())); + + let link = messages + .iter() + .find(|line| line.contains("/pulls/new")) + .expect("pull-request link is offered for new non-default branch"); + assert!( + link.contains("https://tangled.test/nel.pet/anemone/pulls/new"), + "{link}" + ); + assert!(link.contains("sourceBranch=feature"), "{link}"); + assert!(link.contains("targetBranch=main"), "{link}"); + assert!(link.contains("source=branch"), "{link}"); +} + +#[test] +fn a_new_branch_with_an_origin_remote_yields_a_pull_request_link() { + let world = world(); + world + .repo + .set_origin_url(&OriginUrl::new("https://oyster.cafe/did:plc:squid/anemone")) + .unwrap(); + let head = create_feature(&world); + + let applied = created("feature", head); + let messages = run(&world, &applied, &Ci::Skip, Some(&pull())); + + let link = messages + .iter() + .find(|line| line.contains("/pulls/new")) + .expect("pull-request link is offered for a new branch on a fork with origin remote"); + + assert!( + link.contains("https://tangled.test/did:plc:squid/anemone/pulls/new"), + "{link}" + ); + assert!(link.contains("sourceBranch=feature"), "{link}"); + assert!(link.contains("targetBranch=main"), "{link}"); + assert!(link.contains("source=fork"), "{link}"); + assert!(link.contains("fork=did%3Aplc%3Alimpet"), "{link}"); +} + +#[test] +fn no_pull_request_link_for_default_existing_forked_or_rootless_branches() { + type Case = (&'static str, fn(&World) -> [RefUpdate; 1]); + let cases: &[Case] = &[ + ("push to the default branch", |w| create(w, "main")), + ("update to an existing branch", |w| { + commit_file(&w.work, "a.txt", "one\n", "first"); + push(w, "main"); + git(&w.work, &["checkout", "-q", "-b", "feature"]); + commit_file(&w.work, "c.txt", "three\n", "feature"); + push(w, "feature"); + let old = oid(w, "HEAD"); + commit_file(&w.work, "d.txt", "four\n", "more feature"); + push(w, "feature"); + [RefUpdate::Update { + name: refname("refs/heads/feature"), + old, + new: oid(w, "HEAD"), + }] + }), + ("new branch with a file origin remote", |w| { + let head = create_feature(w); + w.repo + .set_origin_url(&OriginUrl::new( + "file://git/repos/oyster.cafe/did:plc:squid/anemone", + )) + .unwrap(); + created("feature", head) + }), + ("new branch while the default branch is absent", |w| { + commit_file(&w.work, "a.txt", "one\n", "first"); + git(&w.work, &["checkout", "-q", "-b", "feature"]); + commit_file(&w.work, "c.txt", "three\n", "feature work"); + push(w, "feature"); + created("feature", oid(w, "HEAD")) + }), + ]; + cases.iter().for_each(|(label, build)| { + let world = world(); + let applied = build(&world); + let messages = run(&world, &applied, &Ci::Skip, Some(&pull())); + assert!( + messages.iter().all(|line| !line.contains("/pulls/new")), + "{label} must offer no pull-request link: {messages:?}" + ); + }); +} + +#[test] +fn verbose_ci_reports_a_clean_pipeline_and_quiet_ci_stays_silent() { + let world = world(); + commit_file( + &world.work, + ".tangled/workflows/ci.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n", + "add ci", + ); + push(&world, "main"); + let head = oid(&world, "HEAD"); + let applied = created("main", head); + + let verbose = run(&world, &applied, &compile(true), None); + assert!( + verbose.iter().any(|line| line.contains("no diagnostics")), + "verbose ci announces clean compile: {verbose:?}" + ); + + let quiet = run(&world, &applied, &compile(false), None); + assert!( + quiet.iter().all(|line| !line.contains("no diagnostics")), + "quiet push says nothing about clean compile: {quiet:?}" + ); +} + +#[test] +fn a_pipeline_compile_error_reaches_the_pusher_even_when_quiet() { + let world = world(); + commit_file( + &world.work, + ".tangled/workflows/broken.yml", + "engine: : not valid : yaml :\n - [\n", + "add broken ci", + ); + push(&world, "main"); + let head = oid(&world, "HEAD"); + let applied = created("main", head); + + let messages = run(&world, &applied, &compile(false), None); + assert!( + messages.iter().any(|line| line.starts_with("error:")), + "malformed workflow surfaces compile error to pusher: {messages:?}" + ); +} diff --git a/knot2/crates/knot-receive/Cargo.toml b/knot2/crates/knot-receive/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..a49cb63e51c22199ac92e09bdfd27e344ed68373 --- /dev/null +++ b/knot2/crates/knot-receive/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "knot-receive" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-pack = { workspace = true } +knot-resource = { workspace = true } +knot-cob = { workspace = true } +knot-index = { workspace = true } +knot-atproto = { workspace = true } +knot-maintenance = { workspace = true } +knot-postreceive = { workspace = true } +knot-messages = { workspace = true } +knot-runtime = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } diff --git a/knot2/crates/knot-receive/src/lib.rs b/knot2/crates/knot-receive/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..cecfd9c2b5eb6fd083a3f468ee9c3186c3efd8bf --- /dev/null +++ b/knot2/crates/knot-receive/src/lib.rs @@ -0,0 +1,380 @@ +//! # How we go about receiving a push! +//! +//! `land` will run the pack ingest and the pull-link lookup at the same time, +//! then will merge both into a single post-receive pass. Whatever the repo's +//! spindle says about the push comes last, and we only wait +//! `SPINDLE_REPLY_WAIT` for it. +//! The pack receiver seals each applied ref-update while the ref lock is held, +//! so post-receive sees exactly the updates that moved a ref, in ref order. +//! +//! ```text +//! request task blocking pool +//! --------------------------------- ------------------------------ +//! read the push preamble +//! | +//! +---- spawn -------------------> open repo +//! | | +//! creates a branch? no -> no link receive pack +//! | yes | +//! look up pull link seal: collect applied updates +//! | repo-DID -> owner, rkey, handle | +//! v v +//! join <---------------------------- updates handed back sealed +//! | +//! any refs applied? no -> no messages +//! | yes +//! owner & rkey for this repo-DID +//! | +//! ci flags from push options +//! | +//! +---- spawn -------------------> ack lines +//! | | +//! | post_receive runs per update, +//! v then the ref projection +//! messages <------------------------ ..and returns its messages +//! | and the spindle's reply slots +//! wait for the spindle's replies +//! | +//! note this push for maintenance +//! | +//! frame report for the client +//! ``` +//! +//! *Figure 1: the tasks of a push.* + +use std::cell::RefCell; +use std::sync::Arc; +use std::time::Duration; + +use knot_atproto::Atproto; +use knot_cob::CobHome; +use knot_git::{Layout, RefUpdate, Repo}; +use knot_index::{Index, Resolved}; +use knot_maintenance::{MaintenanceHandle, PushBytes}; +use knot_messages::{Catalog, PushAckKey, count_refs}; +use knot_pack::{ + PackError, PackLimits, PushGuard, ReceiveOutcome, ReceivedPack, frame_report, + receive_pack_guarded_streamed, receive_preflight, +}; +use knot_postreceive::{Actor, Ci, OwnerLabel, PullLink, post_receive}; +use knot_resource::ResolveSlots; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{ + AccountDid, ActorId, AppviewEndpoint, Handle, KnotHostname, OwnerDid, PushOptions, RepoDid, +}; +use tokio::sync::oneshot; + +type Applied = Vec; + +pub type SpindleReply = oneshot::Receiver>; + +const SPINDLE_REPLY_WAIT: Duration = Duration::from_secs(2); +const MAX_SPINDLE_LINES: usize = 10; +const MAX_SPINDLE_CHARS: usize = 1000; + +pub trait RefSurface: Send + Sync { + fn project( + &self, + repo: &Repo, + repo_did: &RepoDid, + applied: &[RefUpdate], + pusher: &AccountDid, + options: &PushOptions, + ) -> Vec; +} + +pub struct Push<'a, H: HttpTransport, C: Clock> { + pub layout: &'a Layout, + pub repo_did: &'a RepoDid, + pub received: ReceivedPack, + pub limits: PackLimits, + pub knot_actor: ActorId, + pub committer: AccountDid, + pub index: &'a Index, + pub atproto: &'a Atproto, + pub resolve_slots: &'a ResolveSlots, + pub appview: &'a AppviewEndpoint, + pub maintenance: &'a MaintenanceHandle, + pub hostname: &'a KnotHostname, + pub catalog: Arc, + pub ref_surface: Option>, +} + +pub async fn land(push: Push<'_, H, C>) -> Result, PackError> { + let Push { + layout, + repo_did, + received, + limits, + knot_actor, + committer, + index, + atproto, + resolve_slots, + appview, + maintenance, + hostname, + catalog, + ref_surface, + } = push; + + let preflight = receive_preflight(received.preamble()); + let body_len = received.len(); + let home = CobHome::from(repo_did); + + let receive = { + let layout = layout.clone(); + let did = repo_did.clone(); + let catalog = Arc::clone(&catalog); + tokio::task::spawn_blocking( + move || -> Result<(Repo, ReceiveOutcome, Applied), PackError> { + let repo = layout.open(&did)?; + let guard = PushGuard { + cob_authority: knot_actor, + home, + messages: Arc::clone(&catalog), + }; + let stash: RefCell = RefCell::new(Vec::new()); + let seal = |updates: &[RefUpdate]| { + updates + .iter() + .for_each(|update| stash.borrow_mut().push(update.clone())); + }; + let outcome = receive_pack_guarded_streamed( + &repo, + &received, + &limits, + &guard, + &seal, + &catalog.reject, + )?; + Ok((repo, outcome, stash.into_inner())) + }, + ) + }; + let pull = async { + match preflight.creates_branch { + true => resolve_pull_link(index, atproto, resolve_slots, appview, repo_did).await, + false => None, + } + }; + let (received, pull) = tokio::join!(receive, pull); + let (repo, outcome, applied) = match received { + Ok(Ok(triple)) => triple, + Ok(Err(error)) => return Err(error), + Err(_) => return Err(PackError::Pack("receive-pack task panicked".to_string())), + }; + + let (mut messages, replies) = match applied.is_empty() { + true => (Vec::new(), Vec::new()), + false => { + let ci = ci_from_push_options(&outcome.push_options); + let push_options = outcome.push_options.clone(); + let did = repo_did.clone(); + let catalog = Arc::clone(&catalog); + let knot = hostname.clone(); + tokio::task::spawn_blocking(move || { + if let Err(error) = repo.ensure_valid_head(&applied) { + tracing::warn!( + repo = did.as_str(), + %error, + "failed to ensure valid HEAD", + ); + } + let actor = Actor { + committer: committer.clone(), + repo: did.clone(), + }; + let ack = catalog.push.ack.lines(|key| match key { + PushAckKey::Knot => knot.as_str().to_string(), + PushAckKey::Refs => count_refs(applied.len()), + }); + let messages = ack.into_iter().chain(post_receive( + &repo, + &actor, + &applied, + &ci, + pull.as_ref(), + &catalog.push, + )); + let replies = match ref_surface.as_deref() { + Some(project) => { + project.project(&repo, &did, &applied, &committer, &push_options) + } + None => Vec::new(), + }; + (messages.collect::>(), replies) + }) + } + .await + .unwrap_or_else(|error| { + tracing::error!(repo = repo_did.as_str(), %error, "post-receive task panicked"); + (Vec::new(), Vec::new()) + }), + }; + messages.extend(spindle_messages(replies).await); + maintenance.note_push(repo_did, PushBytes::new(body_len as u64)); + Ok(frame_report(&outcome.report, &messages, outcome.side_band)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PushDirective { + SkipCi, + VerboseCi, +} + +impl PushDirective { + fn parse(option: &str) -> Option { + match option { + "skip-ci" | "ci-skip" => Some(Self::SkipCi), + "verbose-ci" | "ci-verbose" => Some(Self::VerboseCi), + _ => None, + } + } +} + +pub fn ci_from_push_options(options: &PushOptions) -> Ci { + let directives: Vec = options + .as_slice() + .iter() + .filter_map(|option| PushDirective::parse(option.as_str())) + .collect(); + match directives.contains(&PushDirective::SkipCi) { + true => Ci::Skip, + false => Ci::Compile { + verbose: directives.contains(&PushDirective::VerboseCi), + }, + } +} + +/// a spindle can send anything, keep only plain printable lines before it reaches a terminal +async fn spindle_messages(replies: Vec) -> Vec { + let deadline = tokio::time::Instant::now() + SPINDLE_REPLY_WAIT; + let mut said = Vec::new(); + for reply in replies { + if let Ok(Ok(Some(text))) = tokio::time::timeout_at(deadline, reply).await + && !said.contains(&text) + { + said.push(text); + } + } + said.iter().flat_map(|text| terminal_lines(text)).collect() +} + +fn terminal_lines(text: &str) -> Vec { + let mut plain = String::new(); + let mut chars = text.chars().take(MAX_SPINDLE_CHARS).peekable(); + while let Some(c) = chars.next() { + match c { + '\u{1b}' => skip_escape(&mut chars), + '\n' => plain.push(c), + c if c.is_control() => {} + c => plain.push(c), + } + } + plain + .lines() + .take(MAX_SPINDLE_LINES) + .map(str::to_string) + .collect() +} + +// csi runs to a final byte in @..~, osc to bel or esc \, anything else is one char +fn skip_escape(chars: &mut std::iter::Peekable>) { + match chars.next() { + Some('[') => { + chars.find(|c| ('@'..='~').contains(c)); + } + Some(']') if chars.find(|c| matches!(c, '\u{7}' | '\u{1b}')) == Some('\u{1b}') => { + chars.next_if_eq(&'\\'); + } + _ => {} + } +} + +async fn resolve_pull_link( + index: &Index, + atproto: &Atproto, + resolve_slots: &ResolveSlots, + appview: &AppviewEndpoint, + repo_did: &RepoDid, +) -> Option { + let owner = match index.owner_of(repo_did) { + Resolved::Ready(Some(owner)) => owner, + _ => return None, + }; + let rkey = match index.rkey_of(repo_did) { + Resolved::Ready(Some(rkey)) => rkey, + _ => return None, + }; + Some(PullLink { + appview: appview.clone(), + owner: resolve_owner_label(atproto, resolve_slots, &owner).await, + rkey, + }) +} + +async fn resolve_owner_label( + atproto: &Atproto, + resolve_slots: &ResolveSlots, + owner: &OwnerDid, +) -> OwnerLabel { + let did = AccountDid::from(owner.clone()); + match resolve_handle(atproto, resolve_slots, &did).await { + Some(handle) => OwnerLabel::Handle(handle), + None => OwnerLabel::Did(owner.clone()), + } +} + +pub async fn resolve_handle( + atproto: &Atproto, + resolve_slots: &ResolveSlots, + did: &AccountDid, +) -> Option { + let _permit = resolve_slots.try_acquire()?; + let identity = atproto.resolve_identity(did).await.ok()?; + identity.primary_handle().cloned() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_spindles_text_loses_escape_sequences_and_control_characters_but_keeps_its_lines() { + let text = "-> \u{1b}[31;1mBrowse\u{1b}[0m CI logs:\r\n ssh -t \u{1b}]8;;https://evil.test\u{7}here\u{1b}]8;;\u{1b}\\\u{7}\u{8}\u{9b}2J"; + assert_eq!( + terminal_lines(text), + ["-> Browse CI logs:", " ssh -t here2J"] + ); + } + + #[test] + fn a_spindles_text_is_capped_in_lines_and_length() { + let many = (0..50).map(|n| format!("line {n}\n")).collect::(); + assert_eq!(terminal_lines(&many).len(), MAX_SPINDLE_LINES); + let long = "x".repeat(5 * MAX_SPINDLE_CHARS); + assert_eq!(terminal_lines(&long), ["x".repeat(MAX_SPINDLE_CHARS)]); + } + + #[tokio::test] + async fn each_distinct_spindle_message_is_printed_once_and_a_silent_spindle_is_not_waited_on() { + let reply = |text: Option<&str>| { + let (pusher, reply) = oneshot::channel(); + pusher.send(text.map(str::to_string)).unwrap(); + reply + }; + let (_silent, hung) = oneshot::channel(); + let started = tokio::time::Instant::now(); + let lines = spindle_messages(vec![ + reply(Some("follow it")), + reply(None), + reply(Some("follow it")), + hung, + reply(Some("and this")), + ]) + .await; + assert_eq!(lines, ["follow it", "and this"]); + assert!(started.elapsed() < SPINDLE_REPLY_WAIT + Duration::from_secs(1)); + } +} diff --git a/knot2/crates/knot-record/Cargo.toml b/knot2/crates/knot-record/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..7a8f18e943aa021185f3716cd612fd1e7e8fd0dd --- /dev/null +++ b/knot2/crates/knot-record/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "knot-record" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-git = { workspace = true } +knot-runtime = { workspace = true } +knot-types = { workspace = true } +jacquard-common = { workspace = true } +jacquard-repo = { workspace = true } +gix = { workspace = true } +bytes = { workspace = true } +cid = { workspace = true } +chrono = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +serde_bytes = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +unicode-segmentation = { workspace = true } +iroh-car = { workspace = true } +url = { workspace = true } +[dev-dependencies] +proptest = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true } diff --git a/knot2/crates/knot-record/src/artifact.rs b/knot2/crates/knot-record/src/artifact.rs new file mode 100644 index 0000000000000000000000000000000000000000..f2385db7961b5f8885db9fbeb561d3aad946bb55 --- /dev/null +++ b/knot2/crates/knot-record/src/artifact.rs @@ -0,0 +1,159 @@ +use knot_types::{AccountDid, BlobCid, Oid, RecordBody, RecordBodyError, UnixSeconds}; +use serde::{Deserialize, Serialize, Serializer}; + +use crate::attachment::{AttachmentMime, BlobInput, MimeRefused}; +use crate::{CreatedAt, RecordError, encoded}; + +pub const ARTIFACT_COLLECTION: &str = "org.tangled.repo.artifact"; + +collection!( + artifact_collection, + ARTIFACT_COLLECTION, + "Artifact collection is valid NSID" +); + +pub const MAX_ARTIFACT_BYTES: u64 = 52_428_800; + +pub const GIT_OID_TYPE: &str = "org.tangled.git.oid"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArtifactBlob { + cid: BlobCid, + mime: AttachmentMime, + size: u64, +} + +impl ArtifactBlob { + pub const fn cid(&self) -> BlobCid { + self.cid + } + + pub fn mime(&self) -> &str { + self.mime.as_str() + } + + pub const fn size(&self) -> u64 { + self.size + } +} + +impl<'de> Deserialize<'de> for ArtifactBlob { + fn deserialize>(deserializer: D) -> Result { + let blob = BlobInput::deserialize(deserializer)?; + if blob.size > MAX_ARTIFACT_BYTES { + return Err(serde::de::Error::custom(format!( + "An artifact of {} bytes is over the {MAX_ARTIFACT_BYTES}-byte limit.", + blob.size + ))); + } + let mime = match AttachmentMime::new(&blob.mime) { + Ok(mime) => mime, + Err(MimeRefused::Empty) => { + return Err(serde::de::Error::custom( + "An artifact's `mimeType` is missing.", + )); + } + Err(MimeRefused::Html(mime)) => { + return Err(serde::de::Error::custom(format!( + "An artifact's `mimeType` is never html, and this one is {mime}. Please use \ + `application/octet-stream` instead." + ))); + } + }; + Ok(Self { + cid: blob.r#ref, + mime, + size: blob.size, + }) + } +} + +#[derive(Serialize)] +struct BlobEncoding<'a> { + #[serde(rename = "$type")] + kind: &'static str, + r#ref: BlobCid, + #[serde(rename = "mimeType")] + mime: &'a str, + size: u64, +} + +impl Serialize for ArtifactBlob { + fn serialize(&self, serializer: S) -> Result { + BlobEncoding { + kind: "blob", + r#ref: self.cid, + mime: self.mime.as_str(), + size: self.size, + } + .serialize(serializer) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArtifactRecord { + oid: Oid, + name: String, + artifact: ArtifactBlob, + created_at: CreatedAt, + editor: AccountDid, +} + +impl ArtifactRecord { + pub fn new( + oid: Oid, + name: String, + artifact: ArtifactBlob, + created_at: UnixSeconds, + editor: AccountDid, + ) -> Result { + Ok(Self { + oid, + name, + artifact, + created_at: CreatedAt::at(created_at)?, + editor, + }) + } + + pub fn body(&self) -> Result { + encoded(self) + } +} + +#[derive(Serialize)] +struct OidEncoding<'a> { + #[serde(rename = "$type")] + kind: &'static str, + oid: &'a Oid, +} + +#[derive(Serialize)] +struct ArtifactEncoding<'a> { + #[serde(rename = "$type")] + record_type: knot_types::RecordCollection, + subject: OidEncoding<'a>, + name: &'a str, + artifact: &'a ArtifactBlob, + #[serde(rename = "createdAt")] + created_at: &'a CreatedAt, + #[serde(rename = "x-tngl-editor")] + editor: &'a AccountDid, +} + +impl Serialize for ArtifactRecord { + fn serialize(&self, serializer: S) -> Result { + ArtifactEncoding { + record_type: artifact_collection(), + subject: OidEncoding { + kind: GIT_OID_TYPE, + oid: &self.oid, + }, + name: &self.name, + artifact: &self.artifact, + created_at: &self.created_at, + editor: &self.editor, + } + .serialize(serializer) + } +} diff --git a/knot2/crates/knot-record/src/attachment.rs b/knot2/crates/knot-record/src/attachment.rs new file mode 100644 index 0000000000000000000000000000000000000000..36862d373824b06dfb6aafca1c181117704e95d7 --- /dev/null +++ b/knot2/crates/knot-record/src/attachment.rs @@ -0,0 +1,561 @@ +use knot_types::BlobCid; +use serde::ser::SerializeMap; +use serde::{Deserialize, Serialize, Serializer}; + +pub const IMAGE_EMBED: &str = "org.tangled.markup.embed.image"; + +pub const VIDEO_EMBED: &str = "org.tangled.markup.embed.video"; + +pub const MAX_IMAGE_BYTES: u64 = 1_000_000; + +pub const MAX_VIDEO_BYTES: u64 = 5_000_000; + +pub const MAX_UPLOAD_BYTES: u64 = 200_000_000; + +pub const SNIFF_BYTES: usize = 512; + +pub const OCTET_STREAM: &str = "application/octet-stream"; + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +#[error("An attachment of {sent} bytes is over the {max_bytes}-byte limit.")] +pub struct AttachmentTooBig { + pub sent: u64, + pub max_bytes: u64, +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum MimeRefused { + #[error("An attachment's `mimeType` is missing.")] + Empty, + #[error( + "An attachment's `mimeType` is never html, and this one is {0}. Please use `application/octet-stream` instead." + )] + Html(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct AttachmentMime(String); + +impl AttachmentMime { + pub fn new(mime: &str) -> Result { + match mime { + "" => Err(MimeRefused::Empty), + mime if html(mime) => Err(MimeRefused::Html(mime.to_owned())), + other => Ok(Self(other.to_owned())), + } + } + + pub fn detected(bytes: &[u8], hint: &str) -> Self { + Self( + match sniffed(bytes) { + Some(mime) => mime, + None => match hint { + "" | "*/*" => OCTET_STREAM, + hint if html(hint) => OCTET_STREAM, + other => other, + }, + } + .to_owned(), + ) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for AttachmentMime { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.0) + } +} + +fn html(mime: &str) -> bool { + mime.starts_with("text/html") || mime.starts_with("application/xhtml") +} + +pub fn sniffed(bytes: &[u8]) -> Option<&'static str> { + let prefix = &bytes[..bytes.len().min(SNIFF_BYTES)]; + match prefix { + [0x89, b'P', b'N', b'G', ..] => Some("image/png"), + [0xFF, 0xD8, 0xFF, ..] => Some("image/jpeg"), + [b'G', b'I', b'F', b'8', ..] => Some("image/gif"), + [ + b'R', + b'I', + b'F', + b'F', + _, + _, + _, + _, + b'W', + b'E', + b'B', + b'P', + .., + ] => Some("image/webp"), + [b'B', b'M', size @ ..] if size.len() >= 4 && size[..4] != [0, 0, 0, 0] => { + Some("image/bmp") + } + [b'%', b'P', b'D', b'F', ..] => Some("application/pdf"), + [_, _, _, _, b'f', b't', b'y', b'p', ..] => Some("video/mp4"), + text if svg(text) => Some("image/svg+xml"), + _ => None, + } +} + +fn svg(prefix: &[u8]) -> bool { + let bare = prefix + .strip_prefix(b"\xEF\xBB\xBF".as_slice()) + .unwrap_or(prefix); + let bare = bare.trim_ascii_start(); + bare.starts_with(b" Result { + if size <= MAX_UPLOAD_BYTES { + Ok(Self(size)) + } else { + Err(AttachmentTooBig { + sent: size, + max_bytes: MAX_UPLOAD_BYTES, + }) + } + } + + pub const fn get(self) -> u64 { + self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Attachment { + cid: BlobCid, + mime: AttachmentMime, + size: AttachmentSize, +} + +impl Attachment { + pub const fn new(cid: BlobCid, mime: AttachmentMime, size: AttachmentSize) -> Self { + Self { cid, mime, size } + } + + pub const fn cid(&self) -> BlobCid { + self.cid + } + + pub fn mime(&self) -> &AttachmentMime { + &self.mime + } + + pub const fn size(&self) -> AttachmentSize { + self.size + } +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum EmbedRefused { + #[error("A markup embed's `mimeType` is image/* or video/mp4, and this one is {0}.")] + Mime(String), + #[error( + "A {kind} markup embed takes {takes} attachments, and this one's `mimeType` is {mime}." + )] + Wrapped { + kind: &'static str, + takes: &'static str, + mime: String, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EmbedKind { + Image, + Video, +} + +#[derive(Clone, Copy)] +struct EmbedSpec { + kind: EmbedKind, + r#type: &'static str, + field: &'static str, + takes: &'static str, + max_bytes: u64, +} + +const EMBED_SPECS: [EmbedSpec; 2] = [ + EmbedSpec { + kind: EmbedKind::Image, + r#type: IMAGE_EMBED, + field: "image", + takes: "image/*", + max_bytes: MAX_IMAGE_BYTES, + }, + EmbedSpec { + kind: EmbedKind::Video, + r#type: VIDEO_EMBED, + field: "video", + takes: "video/mp4", + max_bytes: MAX_VIDEO_BYTES, + }, +]; + +impl EmbedKind { + const fn spec(self) -> EmbedSpec { + EMBED_SPECS[self as usize] + } + + pub const fn r#type(self) -> &'static str { + self.spec().r#type + } + + const fn field(self) -> &'static str { + self.spec().field + } + + pub const fn max_bytes(self) -> u64 { + self.spec().max_bytes + } + + pub fn takes(self) -> &'static str { + self.spec().takes + } + + pub fn accepts(self, mime: &str) -> bool { + let takes = self.spec().takes; + match takes.strip_suffix('*') { + Some(prefix) => mime.starts_with(prefix), + None => mime == takes, + } + } + + fn of(mime: &str) -> Result { + EMBED_SPECS + .iter() + .find(|spec| spec.kind.accepts(mime)) + .map(|spec| spec.kind) + .ok_or_else(|| EmbedRefused::Mime(mime.to_owned())) + } +} + +#[derive(Deserialize)] +struct EmbedInput { + #[serde(rename = "$type")] + kind: String, + image: Option, + video: Option, +} + +#[derive(Deserialize)] +pub(crate) struct BlobInput { + pub(crate) r#ref: BlobCid, + #[serde(rename = "mimeType")] + pub(crate) mime: String, + pub(crate) size: u64, +} + +#[derive(Serialize)] +struct AttachmentEncoding { + #[serde(rename = "$type")] + kind: &'static str, + r#ref: BlobCid, + #[serde(rename = "mimeType")] + mime: String, + size: AttachmentSize, +} + +impl Serialize for Attachment { + fn serialize(&self, serializer: S) -> Result { + AttachmentEncoding { + kind: "blob", + r#ref: self.cid, + mime: self.mime.as_str().to_owned(), + size: self.size, + } + .serialize(serializer) + } +} + +pub struct EmbedRef<'a>(pub &'a Attachment); + +impl Serialize for EmbedRef<'_> { + fn serialize(&self, serializer: S) -> Result { + let kind = EmbedKind::of(self.0.mime.as_str()) + .map_err(|refused| serde::ser::Error::custom(refused.to_string()))?; + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("$type", kind.r#type())?; + map.serialize_entry( + kind.field(), + &AttachmentEncoding { + kind: "blob", + r#ref: self.0.cid, + mime: self.0.mime.as_str().to_owned(), + size: self.0.size, + }, + )?; + map.end() + } +} + +pub fn embeds( + list: &Option>, + serializer: S, +) -> Result { + serializer.collect_seq(list.iter().flatten().map(EmbedRef)) +} + +impl<'de> Deserialize<'de> for Attachment { + fn deserialize>(deserializer: D) -> Result { + let sent = EmbedInput::deserialize(deserializer)?; + let (kind, blob) = match (sent.kind.as_str(), sent.image, sent.video) { + (IMAGE_EMBED, Some(blob), None) => Ok((EmbedKind::Image, blob)), + (VIDEO_EMBED, None, Some(blob)) => Ok((EmbedKind::Video, blob)), + (other, _, _) => Err(serde::de::Error::custom(format!( + "A markup embed's `$type` is {IMAGE_EMBED} or {VIDEO_EMBED}, and this one is \ + `{other}`." + ))), + }?; + let mime = AttachmentMime::new(&blob.mime) + .map_err(|refused| serde::de::Error::custom(refused.to_string()))?; + if !kind.accepts(mime.as_str()) { + return Err(serde::de::Error::custom( + EmbedRefused::Wrapped { + kind: kind.r#type(), + takes: kind.takes(), + mime: mime.as_str().to_owned(), + } + .to_string(), + )); + } + if blob.size > kind.max_bytes() { + return Err(serde::de::Error::custom( + AttachmentTooBig { + sent: blob.size, + max_bytes: kind.max_bytes(), + } + .to_string(), + )); + } + let size = AttachmentSize::new(blob.size).map_err(serde::de::Error::custom)?; + Ok(Self::new(blob.r#ref, mime, size)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE_CID: &str = "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"; + + fn sample() -> Attachment { + let cid = BlobCid::new(cid::Cid::try_from(SAMPLE_CID).expect("the sample blob CID parses")) + .expect("the sample blob CID is raw sha2-256"); + Attachment::new( + cid, + AttachmentMime::new("image/png").unwrap(), + AttachmentSize::new(1_000_000).unwrap(), + ) + } + + #[test] + fn embeds_round_trip_through_json_stock_style() { + let json = serde_json::to_value(EmbedRef(&sample())).unwrap(); + assert_eq!(json["$type"], IMAGE_EMBED); + assert_eq!(json["image"]["mimeType"], "image/png"); + assert_eq!(json["image"]["size"], 1_000_000); + assert_eq!(json["image"]["ref"]["$link"], SAMPLE_CID); + let back: Attachment = serde_json::from_value(json).unwrap(); + assert_eq!(back, sample()); + assert_eq!( + serde_json::to_value(&back).unwrap()["$type"], + "blob", + "An attachment on its own stays the flat upload receipt." + ); + + let video = serde_json::from_value::(serde_json::json!({ + "$type": VIDEO_EMBED, + "video": { + "ref": { "$link": SAMPLE_CID }, + "mimeType": "video/mp4", + "size": MAX_VIDEO_BYTES + } + })) + .unwrap(); + assert_eq!(video.mime().as_str(), "video/mp4"); + assert_eq!( + serde_json::to_value(EmbedRef(&video)).unwrap()["$type"], + VIDEO_EMBED + ); + + let bad_embeds: [(serde_json::Value, &str); 5] = [ + ( + serde_json::json!({ + "$type": IMAGE_EMBED, + "image": { + "ref": { "$link": SAMPLE_CID }, + "mimeType": "application/pdf", + "size": 9 + } + }), + "takes image/* attachments", + ), + ( + serde_json::json!({ + "$type": IMAGE_EMBED, + "image": { + "ref": { "$link": SAMPLE_CID }, + "mimeType": "image/png", + "size": MAX_IMAGE_BYTES + 1 + } + }), + "over the 1000000-byte limit", + ), + ( + serde_json::json!({ + "$type": VIDEO_EMBED, + "video": { + "ref": { "$link": SAMPLE_CID }, + "mimeType": "video/mp4", + "size": MAX_VIDEO_BYTES + 1 + } + }), + "over the 5000000-byte limit", + ), + ( + serde_json::json!({ + "$type": "blob", + "ref": { "$link": SAMPLE_CID }, + "mimeType": "image/png", + "size": 9 + }), + "A markup embed's `$type` is", + ), + ( + serde_json::json!({ + "$type": IMAGE_EMBED, + "image": { + "ref": { "$link": SAMPLE_CID }, + "mimeType": "text/html", + "size": 9 + } + }), + "An attachment's `mimeType` is never html", + ), + ]; + bad_embeds.into_iter().for_each(|(sent, message)| { + let refused = serde_json::from_value::(sent) + .expect_err("Each of these embeds is refused"); + assert!(refused.to_string().contains(message), "{refused}"); + }); + } + + #[test] + fn embed_round_trips_through_dag_cbor() { + let bytes = crate::dag_cbor(&EmbedRef(&sample())); + let back: Attachment = serde_ipld_dagcbor::from_slice(&bytes).unwrap(); + assert_eq!(back, sample()); + } + + #[test] + fn detection_trusts_the_bytes_then_sanitizes_the_hint() { + let paper = b"%PDF-1.7\n%an academic paper\n"; + assert_eq!( + AttachmentMime::detected(paper, "text/html").as_str(), + "application/pdf", + "Sniffed bytes win over any hint." + ); + [ + ("", OCTET_STREAM), + ("*/*", OCTET_STREAM), + ("text/html; charset=utf-8", OCTET_STREAM), + ("application/xhtml+xml", OCTET_STREAM), + ("application/x-whelk-report", "application/x-whelk-report"), + ] + .iter() + .for_each(|(hint, expected)| { + assert_eq!( + AttachmentMime::detected(b"mystery bytes", hint).as_str(), + *expected, + "Unsniffable bytes with hint {hint:?} become {expected}." + ); + }); + } + + #[test] + fn sniffing_matches_each_signature_and_rejects_every_truncation() { + let signatures: [(&[u8], &str, usize); 8] = [ + ( + &[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a], + "image/png", + 4, + ), + (&[0xFF, 0xD8, 0xFF, 0xE0], "image/jpeg", 3), + (&b"GIF87a"[..], "image/gif", 4), + ( + &[b'R', b'I', b'F', b'F', 1, 2, 3, 4, b'W', b'E', b'B', b'P'], + "image/webp", + 12, + ), + (&[b'B', b'M', 1, 2, 3, 4], "image/bmp", 6), + (b"\n Option<(String, String)> { + let text = cid.to_string(); + Some((text.get(7..8)?.to_owned(), text.get(8..9)?.to_owned())) +} + +fn malformed(error: impl std::fmt::Display) -> ChainError { + ChainError::Malformed(error.to_string()) +} + +fn tree_entries( + git: &gix::Repository, + oid: gix::ObjectId, +) -> Result, ChainError> { + let object = git.find_object(oid).map_err(malformed)?; + let tree = object.try_into_tree().map_err(malformed)?; + let entries: Vec<(String, gix::ObjectId)> = tree + .iter() + .map(|entry| { + entry + .map_err(malformed) + .map(|entry| (entry.filename().to_string(), entry.id().detach())) + }) + .collect::, _>>()?; + Ok(entries.into_iter().collect()) +} + +fn write_tree( + git: &gix::Repository, + entries: BTreeMap, + kind: EntryKind, +) -> Result { + let tree = gix::objs::Tree { + entries: entries + .into_iter() + .map(|(name, oid)| gix::objs::tree::Entry { + mode: kind.into(), + filename: name.into(), + oid, + }) + .collect(), + }; + Ok(git.write_object(tree).map_err(malformed)?.detach()) +} + +type Shard = (String, String); + +type ShardedBlocks<'a> = BTreeMap>; + +fn shards_of(blocks: &BTreeMap) -> Result, ChainError> { + let mut shards: ShardedBlocks<'_> = BTreeMap::new(); + blocks.iter().try_for_each(|(cid, bytes)| { + shards + .entry(shard(cid).ok_or_else(|| malformed("a block cid too short to shard"))?) + .or_default() + .push((cid, bytes)); + Ok::<_, ChainError>(()) + })?; + Ok(shards) +} + +pub fn merge_tree( + repo: &Repo, + base: Option, + blocks: &BTreeMap, +) -> Result { + let git = repo.git(); + let mut firsts = base + .map(|base| tree_entries(git, base.object_id())) + .transpose()? + .unwrap_or_default(); + shards_of(blocks)? + .into_iter() + .try_for_each(|((first, second), written)| { + let seconds = firsts + .get(&first) + .map(|tree| tree_entries(git, *tree)) + .transpose()? + .unwrap_or_default(); + let mut cids = seconds + .get(&second) + .map(|tree| tree_entries(git, *tree)) + .transpose()? + .unwrap_or_default(); + written.into_iter().try_for_each(|(cid, bytes)| { + let blob = git.write_blob(bytes).map_err(malformed)?.detach(); + cids.insert(cid.to_string(), blob); + Ok::<_, ChainError>(()) + })?; + let mut seconds = seconds; + seconds.insert(second, write_tree(git, cids, EntryKind::Blob)?); + firsts.insert(first, write_tree(git, seconds, EntryKind::Tree)?); + Ok::<_, ChainError>(()) + })?; + Ok(Oid::from(write_tree(git, firsts, EntryKind::Tree)?)) +} + +pub fn blocks_tree_of(repo: &Repo, commit: Oid) -> Result, ChainError> { + let object = repo + .git() + .find_object(commit.object_id()) + .map_err(malformed)?; + let tree = object + .try_into_commit() + .map_err(malformed)? + .tree() + .map_err(malformed)?; + Ok(tree + .find_entry(BLOCKS_TREE) + .map(|entry| Oid::from(entry.id().detach()))) +} + +fn subtree_at( + git: &gix::Repository, + tree: gix::ObjectId, + name: &str, +) -> Result, RepoError> { + let object = git + .find_object(tree) + .map_err(|error| RepoError::storage(error).with_context("reading a block shard"))?; + let tree = object.try_into_tree().map_err(RepoError::storage)?; + Ok(tree.find_entry(name).map(|entry| entry.id().detach())) +} + +fn poisoned() -> RepoError { + RepoError::storage(std::io::Error::other("block store lock poisoned")) +} + +pub struct GitBlocks { + git: Mutex, + root: gix::ObjectId, +} + +impl Clone for GitBlocks { + fn clone(&self) -> Self { + Self { + git: Mutex::new( + self.git + .lock() + .map_err(|_| poisoned()) + .expect("the holding thread never panics while holding it") + .clone(), + ), + root: self.root, + } + } +} + +impl GitBlocks { + pub fn at(repo: &Repo, root: Oid) -> Self { + Self { + git: Mutex::new(repo.git().clone()), + root: root.object_id(), + } + } + + fn blob(&self, cid: &Cid) -> Result, RepoError> { + self.in_git(cid) + } + + fn in_git(&self, cid: &Cid) -> Result, RepoError> { + let Some((first, second)) = shard(cid) else { + return Ok(None); + }; + let git = self.git.lock().map_err(|_| poisoned())?; + let Some(first_tree) = subtree_at(&git, self.root, &first)? else { + return Ok(None); + }; + let Some(second_tree) = subtree_at(&git, first_tree, &second)? else { + return Ok(None); + }; + let Some(blob) = subtree_at(&git, second_tree, &cid.to_string())? else { + return Ok(None); + }; + let object = git + .find_object(blob) + .map_err(|error| RepoError::storage(error).with_context("reading a block blob"))?; + match object.try_into_blob() { + Ok(blob) => Ok(Some(Bytes::from(blob.data.clone()))), + Err(_) => Ok(None), + } + } +} + +impl BlockStore for GitBlocks { + async fn get(&self, cid: &Cid) -> jacquard_repo::error::Result> { + self.blob(cid) + } + + async fn put(&self, _data: &[u8]) -> jacquard_repo::error::Result { + Err(writes_refused()) + } + + async fn has(&self, cid: &Cid) -> jacquard_repo::error::Result { + Ok(self.blob(cid)?.is_some()) + } + + async fn put_many( + &self, + _blocks: impl IntoIterator + Send, + ) -> jacquard_repo::error::Result<()> { + Err(writes_refused()) + } + + async fn get_many(&self, cids: &[Cid]) -> jacquard_repo::error::Result>> { + cids.iter().map(|cid| self.blob(cid)).collect() + } + + async fn apply_commit( + &self, + _commit: jacquard_repo::repo::CommitData, + ) -> jacquard_repo::error::Result<()> { + Err(writes_refused()) + } +} + +fn writes_refused() -> RepoError { + RepoError::invalid("the git block tree is written with the chain, never directly") +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use jacquard_repo::storage::BlockStore; + + use crate::cid_of; + + use super::*; + + fn repo() -> (tempfile::TempDir, Repo) { + let dir = tempfile::tempdir().unwrap(); + let repo = Repo::create(dir.path().join("repo")).unwrap(); + (dir, repo) + } + #[tokio::test] + async fn blocks_round_trip_and_merge_through_a_sharded_tree() { + let (_dir, repo) = repo(); + let blocks: BTreeMap = ["wakame", "kombu", "nori"] + .into_iter() + .map(|name| (cid_of(name.as_bytes()), Bytes::from(name))) + .collect(); + let held = *blocks.keys().next().unwrap(); + let text = held.to_string(); + let (first, second) = shard(&held).unwrap(); + assert_eq!( + (first.as_str(), second.as_str()), + (&text[7..8], &text[8..9]), + "the shard directories sit where a dag-cbor cid first varies: every one begins bafyrei", + ); + let tree = merge_tree(&repo, None, &blocks).unwrap(); + let grown = merge_tree( + &repo, + Some(tree), + &BTreeMap::from([(cid_of(b"late"), Bytes::from_static(b"late"))]), + ) + .unwrap(); + let store = GitBlocks::at(&repo, grown); + for (cid, bytes) in &blocks { + assert_eq!( + store.get(cid).await.unwrap().as_deref(), + Some(bytes.as_ref()) + ); + } + assert_eq!( + store.get(&cid_of(b"late")).await.unwrap().as_deref(), + Some(&b"late"[..]), + "a merge keeps the blocks an earlier tree holds and adds the new ones" + ); + assert!(store.has(&held).await.unwrap()); + assert_eq!( + store.get(&cid_of(b"stranger")).await.unwrap(), + None, + "a cid the tree never held answers as absent" + ); + } + + #[tokio::test] + async fn a_delta_merge_leaves_untouched_shard_trees_alone() { + let (_dir, repo) = repo(); + let first = BTreeMap::from([(cid_of(b"first"), Bytes::from_static(b"first"))]); + let tree = merge_tree(&repo, None, &first).unwrap(); + let held = *first.keys().next().unwrap(); + let second: BTreeMap = (0..64u32) + .map(|n| format!("stranger-{n}")) + .map(|text| (cid_of(text.as_bytes()), Bytes::from(text))) + .find(|(cid, _)| shard(cid) != shard(&held)) + .into_iter() + .collect(); + assert_eq!( + second.len(), + 1, + "the fixture must land in two different shards or the test pins nothing" + ); + let grown = merge_tree(&repo, Some(tree), &second).unwrap(); + let git = repo.git(); + let untouched = tree_entries(git, tree.object_id()).unwrap(); + let merged = tree_entries(git, grown.object_id()).unwrap(); + let (held_first, held_second) = shard(&held).expect("a stored cid is long enough to shard"); + assert_eq!( + tree_entries(git, *untouched.get(&held_first).unwrap()) + .unwrap() + .get(&held_second), + tree_entries(git, *merged.get(&held_first).unwrap()) + .unwrap() + .get(&held_second), + "the untouched shard tree comes through the merge byte for byte" + ); + assert_ne!(tree, grown, "the root still moves with the new blocks"); + } + + #[test] + fn leaf_blobs_carry_blob_mode_so_git_reads_the_tree_clean() { + let (_dir, repo) = repo(); + let blocks: BTreeMap = ["wakame", "kombu"] + .into_iter() + .map(|name| (cid_of(name.as_bytes()), Bytes::from(name))) + .collect(); + let root = merge_tree(&repo, None, &blocks).unwrap(); + let git = repo.git(); + let entry_at = |oid: gix::ObjectId, name: &str| { + let object = git.find_object(oid).unwrap(); + let tree = object.try_into_tree().unwrap(); + let entry = tree + .find_entry(name) + .unwrap_or_else(|| panic!("the shard tree holds {name}")); + (entry.id().detach(), entry.mode()) + }; + for cid in blocks.keys() { + let (first, second) = shard(cid).expect("a stored cid is long enough to shard"); + let (first_tree, first_mode) = entry_at(root.object_id(), &first); + assert!(first_mode.is_tree(), "the first level names directories"); + let (second_tree, second_mode) = entry_at(first_tree, &second); + assert!(second_mode.is_tree(), "the second level names directories"); + let (_, leaf_mode) = entry_at(second_tree, &cid.to_string()); + assert!( + leaf_mode.is_blob(), + "a stored block reads as a blob, not as a subtree" + ); + } + } + + #[test] + fn a_cid_too_short_to_shard_reads_as_absent_and_writes_as_an_error() { + let empty = Cid::from_str("bafkqaaa").unwrap(); + assert_eq!(empty.to_string().len(), 8); + assert_eq!( + shard(&empty), + None, + "the empty raw cidv1 stops at eight characters" + ); + let (_dir, repo) = repo(); + let stored = GitBlocks::at(&repo, merge_tree(&repo, None, &BTreeMap::new()).unwrap()); + assert_eq!( + futures::executor::block_on(stored.get(&empty)).unwrap(), + None, + "an unshardable cid answers as absent, not as a panic" + ); + assert!(merge_tree(&repo, None, &BTreeMap::from([(empty, Bytes::new())])).is_err()); + } +} diff --git a/knot2/crates/knot-record/src/chain.rs b/knot2/crates/knot-record/src/chain.rs new file mode 100644 index 0000000000000000000000000000000000000000..3f72f64a9d5f6e8791c8f822f712af64749b057b --- /dev/null +++ b/knot2/crates/knot-record/src/chain.rs @@ -0,0 +1,2533 @@ +use std::cmp::Ordering; +use std::collections::{BTreeMap, BTreeSet}; +use std::str::FromStr; +use std::sync::Arc; + +use crate::blocks::{self, GitBlocks}; +use crate::mst; +use bytes::Bytes; +use cid::Cid; +use futures::{StreamExt, TryStreamExt}; +use jacquard_common::types::did::Did; +use jacquard_common::types::tid::{Ticker, Tid}; +use jacquard_repo::Mst; +use jacquard_repo::commit::{Commit, UnsignedCommit}; +use jacquard_repo::mst::VerifiedWriteOp; +use jacquard_repo::storage::BlockStore; +use knot_git::{Identity, NewCommit, RefUpdate, Repo}; +use knot_runtime::Signer; +use knot_types::{ + AtUri, AuthorName, Email, MstKey, Oid, RecordAddress, RecordBody, RecordCid, RecordCollection, + RefName, RepoDid, UnixMicros, UnixSeconds, +}; + +pub const CHAIN_REF: &str = "refs/atproto/commit"; +const COMMIT_BLOB: &str = "commit"; +pub const FRAME_BLOB: &str = "frame"; + +pub const MAX_OPS_PER_COMMIT: usize = 200; +pub const MAX_OPS_PER_APPLY: usize = 2_000; +pub const MAX_COMMIT_BLOCKS_BYTES: usize = 2_000_000; + +pub trait Emit { + fn rev(&mut self, prev: Option<&Tid>) -> Tid; + fn frame(&self, chunk: &CommitChunk) -> Option; + fn committed(&mut self, frame: Option); +} + +impl Emit for Ticker { + fn rev(&mut self, prev: Option<&Tid>) -> Tid { + self.next(prev.cloned()) + } + + fn frame(&self, _chunk: &CommitChunk) -> Option { + None + } + + fn committed(&mut self, _frame: Option) {} +} + +#[derive(Debug, thiserror::Error)] +pub enum ChainError { + #[error("record layer error: {0}")] + Record(#[from] crate::RecordError), + #[error("atproto block error: {0}")] + Block(#[from] jacquard_repo::RepoError), + #[error("a single record still exceeds the {MAX_COMMIT_BLOCKS_BYTES} byte blocks limit")] + ChunkTooLarge, + #[error("the chain at {CHAIN_REF} is already materialized")] + AlreadyMaterialized, + #[error("the chain at {CHAIN_REF} isn't materialized")] + NotEnabled, + #[error("{uri} already has record at {current}, please update it instead of creating")] + RecordExists { + uri: AtUri, + current: RecordCid, + }, + #[error("{uri} has no record to write over")] + RecordAbsent { uri: AtUri }, + #[error( + "{uri} is at {current}, and the write was made from {claimed}; please rebase from {current}" + )] + RecordMoved { + uri: AtUri, + current: RecordCid, + claimed: RecordCid, + }, + #[error("{address} takes one op per chain write, and this one repeats op")] + RecordContested { address: RecordAddress }, + #[error("Chain write takes {limit} ops at a time and this one has {ops}")] + TooManyOps { ops: usize, limit: usize }, + #[error("{address} has {found}, which isn't dag-cbor record CID")] + LeafForeign { + address: RecordAddress, + found: Box, + }, + #[error( + "{written} of {planned} commits were written, before this chain write stopped: {reason}" + )] + PartlyApplied { + written: usize, + planned: usize, + reason: Box, + }, + #[error("git error: {0}")] + Git(#[from] knot_git::GitError), + #[error("malformed atproto chain: {0}")] + Malformed(String), +} + +#[derive(Debug, Clone)] +pub struct ChainTip { + pub git: Oid, + pub data: Cid, + pub rev: Tid, + pub blocks: Oid, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Op { + Create { key: MstKey, cid: Cid }, + Update { key: MstKey, cid: Cid, prev: Cid }, + Delete { key: MstKey, prev: Cid }, +} + +impl Op { + pub fn key(&self) -> &MstKey { + match self { + Op::Create { key, .. } | Op::Update { key, .. } | Op::Delete { key, .. } => key, + } + } + + pub fn cid(&self) -> Option { + match self { + Op::Create { cid, .. } | Op::Update { cid, .. } => Some(*cid), + Op::Delete { .. } => None, + } + } + + pub fn prev(&self) -> Option { + match self { + Op::Create { .. } => None, + Op::Update { prev, .. } | Op::Delete { prev, .. } => Some(*prev), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommitChunk { + pub commit: Vec, + pub rev: Tid, + pub prev_rev: Option, + pub data: Cid, + pub prev_data: Option, + pub ops: Vec, + pub blocks: BTreeMap, +} + +pub enum Due { + Keep(RecordCid), + Write(RecordBody), +} + +pub trait Records { + fn due(&self) -> &BTreeMap; + fn extra_blocks(&self) -> BTreeMap { + BTreeMap::new() + } +} + +pub struct InTree { + due: BTreeMap, +} + +pub fn in_tree(bodies: BTreeMap) -> impl Records { + InTree { + due: bodies + .into_iter() + .map(|(key, body)| (key, Due::Write(body))) + .collect(), + } +} + +impl Records for InTree { + fn due(&self) -> &BTreeMap { + &self.due + } +} + +pub fn blocks_bytes(blocks: &BTreeMap) -> usize { + blocks.values().map(Bytes::len).sum() +} + +pub fn tip(repo: &Repo) -> Result, ChainError> { + let name = RefName::new(CHAIN_REF).expect("chain ref name parses"); + let Some(git_oid) = repo.find_ref(&name)? else { + return Ok(None); + }; + let commit = commit_blob(repo, git_oid)?; + let blocks = blocks::blocks_tree_of(repo, git_oid)? + .ok_or_else(|| ChainError::Malformed("chain commit has no blocks tree".into()))?; + Ok(Some(ChainTip { + git: git_oid, + data: *commit.data(), + rev: commit.rev().clone(), + blocks, + })) +} + +pub fn commit_bytes(repo: &Repo, tip: &ChainTip) -> Result, ChainError> { + raw_commit_bytes(repo, tip.git) +} + +pub fn commit_cid(bytes: &[u8]) -> Result { + Ok(Commit::::from_cbor(bytes)?.to_cid()?) +} + +pub fn load_tree(repo: &Repo, tip: &ChainTip) -> Mst { + Mst::load(Arc::new(GitBlocks::at(repo, tip.blocks)), tip.data, None) +} + +pub async fn leaves(repo: &Repo) -> Result, ChainError> { + match tip(repo)? { + Some(tip) => keyed_leaves(&load_tree(repo, &tip)).await, + None => Ok(BTreeMap::new()), + } +} + +async fn keyed_leaves( + tree: &Mst, +) -> Result, ChainError> { + tree.leaves() + .await? + .into_iter() + .map(|(key, cid)| MstKey::new(key.as_str().to_owned()).map(|key| (key, cid))) + .collect::, _>>() + .map_err(|error| ChainError::Malformed(error.to_string())) +} + +pub struct ChainStep { + pub next: Option, + pub rev: Tid, + pub framed: bool, + pub frame: Option, +} + +fn step(repo: &Repo, oid: Oid, load_frame: bool) -> Result { + let git = repo.git(); + let commit = git + .find_object(oid.object_id()) + .map_err(|error| ChainError::Malformed(error.to_string()))? + .try_into_commit() + .map_err(|error| ChainError::Malformed(error.to_string()))?; + let next = commit + .parent_ids() + .next() + .map(|parent| Oid::from(parent.detach())); + let tree = commit + .tree() + .map_err(|error| ChainError::Malformed(error.to_string()))?; + let commit_bytes = blob_in(git, &tree, COMMIT_BLOB)? + .ok_or_else(|| ChainError::Malformed("chain commit is missing its commit blob".into()))?; + let rev = Commit::::from_cbor(&commit_bytes) + .map_err(ChainError::from)? + .rev() + .clone(); + let framed = tree.find_entry(FRAME_BLOB).is_some(); + let frame = if load_frame { + blob_in(git, &tree, FRAME_BLOB)? + } else { + None + }; + Ok(ChainStep { + next, + rev, + framed, + frame, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TraversalEnd { + Exhausted, + Limited, + Broken, +} + +impl TraversalEnd { + fn degraded(self) -> Self { + match self { + TraversalEnd::Exhausted => TraversalEnd::Broken, + settled => settled, + } + } +} + +pub struct FramedRevs { + pub found: Vec<(UnixMicros, Oid)>, + pub end: TraversalEnd, +} + +pub fn framed_revs(repo: &Repo, start: Oid, after: UnixMicros, limit: usize) -> FramedRevs { + let mut found = Vec::new(); + let mut end = TraversalEnd::Exhausted; + let mut next = Some(start); + while let Some(oid) = next { + match step(repo, oid, false) { + Ok(step) => { + if UnixMicros::new(step.rev.timestamp()) <= after { + break; + } + if step.framed { + if found.len() >= limit { + end = TraversalEnd::Limited; + break; + } + found.push((UnixMicros::new(step.rev.timestamp()), oid)); + } + next = step.next; + } + Err(error) => { + tracing::warn!(%error, "a chain wouldn't walk, so its older commits stay unreplayed"); + end = TraversalEnd::Broken; + break; + } + } + } + FramedRevs { found, end } +} + +pub struct FramesAfter { + pub found: Vec<(Tid, Bytes)>, + pub end: TraversalEnd, +} + +pub fn frames_after(repo: &Repo, start: Oid, after: UnixMicros, limit: usize) -> FramesAfter { + let traversal = framed_revs(repo, start, after, limit); + let mut end = traversal.end; + let mut found = Vec::with_capacity(traversal.found.len()); + traversal + .found + .into_iter() + .for_each(|(_, oid)| match step(repo, oid, true) { + Ok(stepped) => match stepped.frame { + Some(frame) => found.push((stepped.rev, frame)), + None => end = end.degraded(), + }, + Err(error) => { + tracing::warn!(%error, "A chain wouldn't yield a frame, so the blocks stay unserved."); + end = end.degraded(); + } + }); + FramesAfter { found, end } +} + +pub fn frame_of(repo: &Repo, oid: Oid) -> Result, ChainError> { + let stepped = step(repo, oid, true)?; + match stepped.frame { + Some(frame) => Ok(Some((stepped.rev, frame))), + None => Ok(None), + } +} + +pub struct FramesTo { + pub found: Vec<(Tid, Bytes)>, + pub reached: bool, +} + +pub fn frames_to(repo: &Repo, start: Oid, until: Oid, limit: usize) -> FramesTo { + let mut found = Vec::new(); + let mut reached = false; + let mut next = Some(start); + while let Some(oid) = next { + if oid == until { + reached = true; + break; + } + if found.len() >= limit { + break; + } + match step(repo, oid, true) { + Ok(stepped) => { + if let Some(frame) = stepped.frame { + found.push((stepped.rev, frame)); + } + next = stepped.next; + } + Err(error) => { + tracing::warn!( + %error, + "A chain wouldn't traverse to the requested commit, so the frames stay unserved." + ); + break; + } + } + } + FramesTo { found, reached } +} + +pub struct UnframedTip { + pub rev: Tid, + pub commit: Vec, +} + +pub fn unframed_tip(repo: &Repo) -> Result, ChainError> { + let Some(tip) = tip(repo)? else { + return Ok(None); + }; + let stepped = step(repo, tip.git, false)?; + if stepped.framed { + return Ok(None); + } + Ok(Some(UnframedTip { + rev: stepped.rev, + commit: raw_commit_bytes(repo, tip.git)?, + })) +} + +pub fn frame_blob(repo: &Repo, oid: Oid) -> Result, ChainError> { + blob_in(repo.git(), &tree_of(repo, oid)?, FRAME_BLOB) +} + +fn tree_of(repo: &Repo, git_oid: Oid) -> Result, ChainError> { + repo.git() + .find_object(git_oid.object_id()) + .map_err(|error| ChainError::Malformed(error.to_string()))? + .try_into_commit() + .map_err(|error| ChainError::Malformed(error.to_string()))? + .tree() + .map_err(|error| ChainError::Malformed(error.to_string())) +} + +fn blob_in( + git: &gix::Repository, + tree: &gix::Tree<'_>, + name: &str, +) -> Result, ChainError> { + tree.find_entry(name) + .map(|entry| { + git.find_object(entry.id()) + .map(|object| Bytes::from(object.detach().data)) + }) + .transpose() + .map_err(|error| ChainError::Malformed(error.to_string())) +} + +fn commit_blob(repo: &Repo, git_oid: Oid) -> Result, ChainError> { + let bytes = raw_commit_bytes(repo, git_oid)?; + Commit::::from_cbor(&bytes).map_err(ChainError::from) +} + +fn raw_commit_bytes(repo: &Repo, git_oid: Oid) -> Result, ChainError> { + Ok(Vec::from( + blob_in(repo.git(), &tree_of(repo, git_oid)?, COMMIT_BLOB)? + .ok_or_else(|| ChainError::Malformed("chain commit is missing its commit blob".into()))? + .as_ref(), + )) +} + +fn did_of(did: &RepoDid) -> Result, ChainError> { + Did::::from_str(did.as_str()).map_err(|error| ChainError::Malformed(error.to_string())) +} + +pub async fn initialize( + repo: &Repo, + did: &RepoDid, + signer: &dyn Signer, + emit: &mut dyn Emit, + now: UnixSeconds, +) -> Result { + if tip(repo)?.is_some() { + return Err(ChainError::AlreadyMaterialized); + } + let rev = emit.rev(None); + let empty = mst::empty(); + let (data, blocks) = empty.collect_blocks().await?; + let signed = UnsignedCommit::new_unsigned(did_of(did)?, data, rev.clone(), None) + .sign(&crate::CommitSigner::new(signer))?; + let chunk = CommitChunk { + commit: signed.to_cbor()?, + rev, + prev_rev: None, + data, + prev_data: None, + ops: Vec::new(), + blocks, + }; + let frame = emit.frame(&chunk); + let blocks_tree = blocks::merge_tree(repo, None, &chunk.blocks)?; + write_chain_commit( + repo, + None, + &chunk.commit, + now, + Some(blocks_tree), + frame.clone(), + )?; + emit.committed(frame); + Ok(chunk) +} + +pub async fn materialize( + repo: &Repo, + did: &RepoDid, + records: &dyn Records, + signer: &dyn Signer, + emit: &mut dyn Emit, + now: UnixSeconds, +) -> Result, ChainError> { + if tip(repo)?.is_some() { + return Err(ChainError::AlreadyMaterialized); + } + let ops: Vec = records + .due() + .iter() + .map(|(key, due)| match due { + Due::Write(record) => Ok(VerifiedWriteOp::Create { + key: key.as_str().into(), + cid: record.cid().cid(), + }), + Due::Keep(_) => Err(ChainError::Malformed(format!( + "a kept record can't seed an empty chain: {key}" + ))), + }) + .collect::, ChainError>>()?; + let mut plan = Plan { + repo, + did: did_of(did)?, + signer, + emit, + prev_rev: None, + prev_data: None, + tree: mst::empty(), + parent: None, + blocks_tree: None, + extra_blocks: records.extra_blocks(), + now, + }; + plan_chunks(&mut plan, records, ops).await +} + +pub async fn sync( + repo: &Repo, + did: &RepoDid, + collections: &[RecordCollection], + records: &dyn Records, + signer: &dyn Signer, + emit: &mut dyn Emit, + now: UnixSeconds, +) -> Result, ChainError> { + let prefixes: Vec = collections + .iter() + .map(|collection| format!("{}/", collection.as_str())) + .collect(); + let synced = |key: &MstKey| { + prefixes + .iter() + .any(|prefix| key.as_str().starts_with(prefix)) + }; + if let Some(key) = records.due().keys().find(|key| !synced(key)) { + return Err(ChainError::Malformed(format!( + "the record key {key} isn't in the collections this chain syncs" + ))); + } + let Some(tip) = tip(repo)? else { + return Err(ChainError::NotEnabled); + }; + let before = load_tree(repo, &tip); + let leaves = keyed_leaves(&before).await?; + let mut ops: Vec = records + .due() + .iter() + .filter_map(|(key, due)| match (due, leaves.get(key)) { + (Due::Write(record), None) => Some(Ok(VerifiedWriteOp::Create { + key: key.as_str().into(), + cid: record.cid().cid(), + })), + (Due::Write(record), Some(prev)) if *prev != record.cid().cid() => { + Some(Ok(VerifiedWriteOp::Update { + key: key.as_str().into(), + cid: record.cid().cid(), + prev: *prev, + })) + } + (Due::Keep(kept), Some(prev)) if *prev == kept.cid() => None, + (Due::Keep(_), _) => Some(Err(ChainError::Malformed(format!( + "a kept record isn't on the chain: {key}" + )))), + (Due::Write(_), Some(_)) => None, + }) + .collect::, ChainError>>()?; + ops.extend( + leaves + .iter() + .filter(|(key, _)| synced(key) && !records.due().contains_key(*key)) + .map(|(key, prev)| VerifiedWriteOp::Delete { + key: key.as_str().into(), + prev: *prev, + }), + ); + let extra_blocks = records.extra_blocks(); + if ops.is_empty() && extra_blocks.is_empty() { + return Ok(Vec::new()); + } + let mut plan = Plan { + repo, + did: did_of(did)?, + signer, + emit, + prev_rev: Some(tip.rev.clone()), + prev_data: Some(tip.data), + tree: before, + parent: Some(tip.git), + blocks_tree: Some(tip.blocks), + extra_blocks, + now, + }; + plan_chunks(&mut plan, records, ops).await +} + +pub async fn settle( + repo: &Repo, + did: &RepoDid, + collections: &[RecordCollection], + records: &dyn Records, + signer: &dyn Signer, + emit: &mut dyn Emit, + now: UnixSeconds, +) -> Result, ChainError> { + match tip(repo)? { + Some(_) => sync(repo, did, collections, records, signer, emit, now).await, + None => materialize(repo, did, records, signer, emit, now).await, + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RecordOp { + Create { record: RecordBody }, + Update { record: RecordBody, from: RecordCid }, + Delete { from: RecordCid }, +} + +impl RecordOp { + fn record(&self) -> Option<&RecordBody> { + match self { + Self::Create { record } | Self::Update { record, .. } => Some(record), + Self::Delete { .. } => None, + } + } +} + +fn adopt(address: &RecordAddress, current: Cid) -> Result { + RecordCid::new(current).ok_or_else(|| ChainError::LeafForeign { + address: address.clone(), + found: Box::new(current), + }) +} + +fn write_op( + repo: &RepoDid, + address: &RecordAddress, + key: &MstKey, + op: &RecordOp, + current: Option, +) -> Result, ChainError> { + match (op, current) { + (RecordOp::Create { record }, None) => Ok(Some(VerifiedWriteOp::Create { + key: key.as_str().into(), + cid: record.cid().cid(), + })), + (RecordOp::Create { .. }, Some(current)) => Err(ChainError::RecordExists { + uri: address.at_uri(repo), + current: adopt(address, current)?, + }), + (RecordOp::Update { .. } | RecordOp::Delete { .. }, None) => { + Err(ChainError::RecordAbsent { + uri: address.at_uri(repo), + }) + } + (RecordOp::Update { from, .. } | RecordOp::Delete { from }, Some(current)) + if from.cid() != current => + { + Err(ChainError::RecordMoved { + uri: address.at_uri(repo), + current: adopt(address, current)?, + claimed: *from, + }) + } + (RecordOp::Update { record, .. }, Some(current)) if record.cid().cid() == current => { + Ok(None) + } + (RecordOp::Update { record, .. }, Some(current)) => Ok(Some(VerifiedWriteOp::Update { + key: key.as_str().into(), + cid: record.cid().cid(), + prev: current, + })), + (RecordOp::Delete { .. }, Some(current)) => Ok(Some(VerifiedWriteOp::Delete { + key: key.as_str().into(), + prev: current, + })), + } +} + +pub async fn apply( + repo: &Repo, + did: &RepoDid, + ops: &[(RecordAddress, RecordOp)], + signer: &dyn Signer, + emit: &mut dyn Emit, + now: UnixSeconds, +) -> Result, ChainError> { + if ops.len() > MAX_OPS_PER_APPLY { + return Err(ChainError::TooManyOps { + ops: ops.len(), + limit: MAX_OPS_PER_APPLY, + }); + } + let keyed: Vec<(MstKey, &RecordAddress, &RecordOp)> = ops + .iter() + .map(|(address, op)| (address.mst_key(), address, op)) + .collect(); + keyed + .iter() + .try_fold(BTreeSet::new(), |mut seen, (key, address, _)| { + seen.insert(key) + .then_some(seen) + .ok_or_else(|| ChainError::RecordContested { + address: (*address).clone(), + }) + })?; + let Some(tip) = tip(repo)? else { + return Err(ChainError::NotEnabled); + }; + let before = load_tree(repo, &tip); + let verified = futures::stream::iter(&keyed) + .then(|(key, address, op)| { + let before = &before; + async move { + let current = before.get(key.as_str()).await?; + write_op(did, address, key, op, current) + } + }) + .try_collect::>>() + .await? + .into_iter() + .flatten() + .collect::>(); + let (mut writes, mut deletes): (Vec, Vec) = verified + .into_iter() + .partition(|op| !matches!(op, VerifiedWriteOp::Delete { .. })); + writes.sort_by_key(|op| op_key(op).to_owned()); + deletes.sort_by_key(|op| op_key(op).to_owned()); + writes.append(&mut deletes); + let verified = writes; + if verified.is_empty() { + return Ok(Vec::new()); + } + let records = in_tree( + keyed + .into_iter() + .filter_map(|(key, _, op)| op.record().map(|record| (key, record.clone()))) + .collect(), + ); + let mut plan = Plan { + repo, + did: did_of(did)?, + signer, + emit, + prev_rev: Some(tip.rev.clone()), + prev_data: Some(tip.data), + tree: before, + parent: Some(tip.git), + blocks_tree: Some(tip.blocks), + extra_blocks: records.extra_blocks(), + now, + }; + plan_chunks(&mut plan, &records, verified).await +} + +fn op_key(op: &VerifiedWriteOp) -> &str { + match op { + VerifiedWriteOp::Create { key, .. } + | VerifiedWriteOp::Update { key, .. } + | VerifiedWriteOp::Delete { key, .. } => key.as_str(), + } +} + +fn ops_of(ops: &[VerifiedWriteOp]) -> Vec { + ops.iter() + .map(|op| { + let key = MstKey::new(op_key(op).to_owned()).expect("verified op keys fit the mst"); + match op { + VerifiedWriteOp::Create { cid, .. } => Op::Create { key, cid: *cid }, + VerifiedWriteOp::Update { cid, prev, .. } => Op::Update { + key, + cid: *cid, + prev: *prev, + }, + VerifiedWriteOp::Delete { prev, .. } => Op::Delete { key, prev: *prev }, + } + }) + .collect() +} + +struct Plan<'a, S> { + repo: &'a Repo, + did: Did, + signer: &'a dyn Signer, + emit: &'a mut dyn Emit, + prev_rev: Option, + prev_data: Option, + tree: Mst, + parent: Option, + blocks_tree: Option, + extra_blocks: BTreeMap, + now: UnixSeconds, +} + +struct ChunkDraft { + tree: Mst, + data: Cid, + ops: Vec, + blocks: BTreeMap, +} + +impl ChunkDraft { + fn over_limits(&self) -> bool { + self.blocks.values().map(Bytes::len).sum::() > MAX_COMMIT_BLOCKS_BYTES + } +} + +async fn plan_chunks( + plan: &mut Plan<'_, S>, + records: &dyn Records, + ops: Vec, +) -> Result, ChainError> { + let intended = ops.len().div_ceil(MAX_OPS_PER_COMMIT); + let mut out = Vec::new(); + let planned = futures::stream::iter(ops) + .chunks(MAX_OPS_PER_COMMIT) + .map(Vec::from) + .fold( + Ok::<(&mut Plan<'_, S>, &mut Vec), ChainError>((plan, &mut out)), + |acc, slice| async move { + let (plan, out) = acc?; + emit_chunk(plan, records, slice, out).await?; + Ok((plan, out)) + }, + ) + .await + .map(|_| ()); + match (planned, out.is_empty()) { + (Ok(()), _) => {} + (Err(error), true) => return Err(error), + (Err(error), false) => { + return Err(ChainError::PartlyApplied { + written: out.len(), + planned: intended, + reason: Box::new(error), + }); + } + } + drain_extras(plan, records, &mut out).await?; + Ok(out) +} + +async fn drain_extras( + plan: &mut Plan<'_, S>, + records: &dyn Records, + out: &mut Vec, +) -> Result<(), ChainError> { + if plan.extra_blocks.is_empty() { + return Ok(()); + } + let before = plan.extra_blocks.len(); + emit_chunk(plan, records, Vec::new(), out).await?; + if plan.extra_blocks.len() == before { + return Err(ChainError::ChunkTooLarge); + } + Box::pin(drain_extras(plan, records, out)).await +} + +fn take_extra_blocks(plan: &mut Plan<'_, S>, draft: &mut ChunkDraft) { + let mut budget = MAX_COMMIT_BLOCKS_BYTES + .saturating_sub(draft.blocks.values().map(Bytes::len).sum::()); + let taken: Vec = plan + .extra_blocks + .iter() + .take_while(|(_, bytes)| { + let fits = bytes.len() <= budget; + budget = budget.saturating_sub(bytes.len().min(budget)); + fits + }) + .map(|(cid, _)| *cid) + .collect(); + draft.blocks.extend( + taken + .into_iter() + .filter_map(|cid| plan.extra_blocks.remove(&cid).map(|bytes| (cid, bytes))), + ); +} + +async fn emit_chunk( + plan: &mut Plan<'_, S>, + records: &dyn Records, + slice: Vec, + out: &mut Vec, +) -> Result<(), ChainError> { + let mut draft = draft_chunk(plan, records, &slice).await?; + take_extra_blocks(plan, &mut draft); + if !draft.over_limits() { + let chunk = sign_chunk(plan, draft).await?; + write_chunk(plan, &chunk)?; + out.push(chunk); + return Ok(()); + } + if slice.len() == 1 { + return Err(ChainError::ChunkTooLarge); + } + let (head, tail) = slice.split_at(slice.len() / 2); + Box::pin(emit_chunk(plan, records, head.to_vec(), out)).await?; + Box::pin(emit_chunk(plan, records, tail.to_vec(), out)).await +} + +fn write_chunk(plan: &mut Plan<'_, S>, chunk: &CommitChunk) -> Result<(), ChainError> { + let frame = plan.emit.frame(chunk); + let grown = blocks::merge_tree(plan.repo, plan.blocks_tree, &chunk.blocks)?; + let written = write_chain_commit( + plan.repo, + plan.parent, + &chunk.commit, + plan.now, + Some(grown), + frame.clone(), + )?; + plan.parent = Some(written); + plan.blocks_tree = Some(grown); + plan.emit.committed(frame); + Ok(()) +} + +async fn draft_chunk( + plan: &Plan<'_, S>, + records: &dyn Records, + ops: &[VerifiedWriteOp], +) -> Result, ChainError> { + let applied = mst::apply(&plan.tree, ops).await?; + let mut blocks = + crate::proof::covering_blocks(&plan.tree, &applied.tree, &applied.diff).await?; + for op in ops { + let (key, cid) = match op { + VerifiedWriteOp::Create { key, cid } | VerifiedWriteOp::Update { key, cid, .. } => { + (key, cid) + } + VerifiedWriteOp::Delete { .. } => continue, + }; + let Some(Due::Write(body)) = records.due().get(key.as_str()) else { + continue; + }; + blocks.insert(*cid, Bytes::copy_from_slice(body.as_bytes())); + } + Ok(ChunkDraft { + data: mst::root(&applied.tree).await?, + ops: ops_of(ops), + blocks, + tree: applied.tree, + }) +} +async fn sign_chunk( + plan: &mut Plan<'_, S>, + draft: ChunkDraft, +) -> Result { + let rev = plan.emit.rev(plan.prev_rev.as_ref()); + let prev_rev = plan.prev_rev.clone(); + plan.prev_rev = Some(rev.clone()); + let signed = UnsignedCommit::new_unsigned(plan.did.clone(), draft.data, rev.clone(), None) + .sign(&crate::CommitSigner::new(plan.signer))?; + let prev_data = plan.prev_data; + plan.tree = draft.tree; + plan.prev_data = Some(draft.data); + Ok(CommitChunk { + commit: signed.to_cbor()?, + rev, + prev_rev, + data: draft.data, + prev_data, + ops: draft.ops, + blocks: draft.blocks, + }) +} + +fn write_chain_commit( + repo: &Repo, + parent: Option, + commit_bytes: &[u8], + now: UnixSeconds, + blocks_tree: Option, + frame: Option, +) -> Result { + let git = repo.git(); + let blob = git + .write_blob(commit_bytes) + .map_err(|error| ChainError::Malformed(error.to_string()))? + .detach(); + let mut entries: Vec = blocks_tree + .into_iter() + .map(|tree| gix::objs::tree::Entry { + mode: gix::objs::tree::EntryKind::Tree.into(), + filename: blocks::BLOCKS_TREE.into(), + oid: tree.object_id(), + }) + .collect(); + if let Some(frame) = frame { + let oid = git + .write_blob(frame.as_ref()) + .map_err(|error| ChainError::Malformed(error.to_string()))? + .detach(); + entries.push(gix::objs::tree::Entry { + mode: gix::objs::tree::EntryKind::Blob.into(), + filename: FRAME_BLOB.into(), + oid, + }); + } + entries.push(gix::objs::tree::Entry { + mode: gix::objs::tree::EntryKind::Blob.into(), + filename: COMMIT_BLOB.into(), + oid: blob, + }); + + // The essence of trying to fit the cylinderical atproto into the square git hole. + // Every commit on atproto has a given tree w/ a blocks subtree in it, a commit blob, + // and a frame blob in which there's a firehose frame we must store. + // To make it fit, especially given that gix just orders exactly however we give it, + // we just have to be really careful to not mess up the ordering or else atproto-wise + // we'll get a wrong hash out the top. Hence the appended '/' to trick git a bit into thinking + // `blocks/` is above '.' but below '0'. + // There's a test at a_tree_entry_sorts_as_git_sorts_it_with_a_subtree_slash. + entries.sort_by(git_entry_order); + let tree = git + .write_object(gix::objs::Tree { entries }) + .map_err(|error| ChainError::Malformed(error.to_string()))? + .detach(); + let identity = Identity { + name: AuthorName::new("knot"), + email: Email::new("noreply@knot"), + time: now, + offset_seconds: 0, + }; + let id = repo.write_commit(&NewCommit { + tree: Oid::from(tree), + parents: parent.iter().copied().collect(), + author: identity.clone(), + committer: identity, + message: String::new(), + extra_headers: Vec::new(), + })?; + let name = RefName::new(CHAIN_REF).expect("chain ref name parses"); + let update = match parent { + Some(old) => RefUpdate::Update { name, old, new: id }, + None => RefUpdate::Create { name, new: id }, + }; + repo.update_ref(&update)?; + Ok(id) +} + +fn git_entry_order(left: &gix::objs::tree::Entry, right: &gix::objs::tree::Entry) -> Ordering { + let (a, b) = (left.filename.as_slice(), right.filename.as_slice()); + let shared = a.len().min(b.len()); + a[..shared].cmp(&b[..shared]).then_with(|| { + past_shared(a, shared, left.mode.is_tree()).cmp(&past_shared( + b, + shared, + right.mode.is_tree(), + )) + }) +} + +fn past_shared(name: &[u8], shared: usize, tree: bool) -> Option { + match (name.get(shared), tree) { + (Some(byte), _) => Some(*byte), + (None, true) => Some(b'/'), + (None, false) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use jacquard_repo::BlockStore; + use knot_runtime::K256Signer; + use proptest::strategy::Strategy; + use std::collections::BTreeSet; + + const DID: &str = "did:web:knot.example"; + + fn repo_did() -> RepoDid { + RepoDid::new(DID).unwrap() + } + + fn repo() -> (tempfile::TempDir, Repo) { + let dir = tempfile::tempdir().unwrap(); + let layout = knot_git::Layout::new(dir.path()); + let did = knot_types::RepoDid::new("did:plc:squid").unwrap(); + (dir, layout.create(&did).unwrap()) + } + + const COLLECTION: &str = "org.tangled.test.record"; + + fn collection() -> knot_types::RecordCollection { + knot_types::RecordCollection::new(COLLECTION).unwrap() + } + + #[derive(serde::Serialize)] + struct Fixture { + #[serde(rename = "$type")] + record_type: &'static str, + #[serde(rename = "createdAt")] + created_at: i64, + } + + fn body(at: i64) -> RecordBody { + crate::encoded(&Fixture { + record_type: COLLECTION, + created_at: at, + }) + .unwrap() + } + + fn records(count: usize) -> BTreeMap { + (0..count) + .map(|n| { + ( + MstKey::new(format!("{COLLECTION}/record{n}")).unwrap(), + body(n as i64), + ) + }) + .collect() + } + + async fn enable( + repo: &Repo, + records: &BTreeMap, + signer: &K256Signer, + ) -> Result, ChainError> { + materialize( + repo, + &repo_did(), + &in_tree(records.clone()), + signer, + &mut Ticker::new(), + UnixSeconds::new(1), + ) + .await + } + + fn key() -> K256Signer { + K256Signer::from_slice(&[9u8; 32]).unwrap() + } + + fn verified(ops: &[Op]) -> Vec { + ops.iter() + .map(|op| match op { + Op::Create { key, cid } => VerifiedWriteOp::Create { + key: key.as_str().into(), + cid: *cid, + }, + Op::Update { key, cid, prev } => VerifiedWriteOp::Update { + key: key.as_str().into(), + cid: *cid, + prev: *prev, + }, + Op::Delete { key, prev } => VerifiedWriteOp::Delete { + key: key.as_str().into(), + prev: *prev, + }, + }) + .collect() + } + + #[tokio::test] + async fn five_hundred_records_chunk_into_limited_commits() { + let (_dir, repo) = repo(); + let chunks = enable(&repo, &records(500), &key()).await.unwrap(); + assert_eq!( + chunks + .iter() + .map(|chunk| chunk.ops.len()) + .collect::>(), + vec![200, 200, 100], + "anything over 200 operations splits into several commits" + ); + chunks.windows(2).for_each(|pair| { + assert!( + pair[1].rev.newer_than(&pair[0].rev), + "each chunk carries its own strictly increasing rev" + ); + assert_eq!(pair[1].prev_data, Some(pair[0].data)); + }); + assert_eq!(chunks[0].prev_data, None); + chunks.iter().for_each(|chunk| { + assert!(blocks_bytes(&chunk.blocks) <= MAX_COMMIT_BLOCKS_BYTES); + assert!(chunk.blocks.len() >= chunk.ops.len()); + }); + assert_eq!( + tip(&repo).unwrap().map(|tip| tip.data), + chunks.last().map(|chunk| chunk.data) + ); + } + + #[tokio::test] + async fn a_second_enablement_is_refused_and_the_tip_stands() { + let (_dir, repo) = repo(); + assert!( + enable(&repo, &BTreeMap::new(), &key()) + .await + .unwrap() + .is_empty(), + "an empty grant list doesn't materialize any commit" + ); + assert!(tip(&repo).unwrap().is_none()); + assert!( + leaves_of(&repo).await.is_empty(), + "a repo without a chain serves an empty leaf list instead of an error" + ); + let first = enable(&repo, &records(1), &key()).await.unwrap(); + assert!(matches!( + enable(&repo, &records(1), &key()).await, + Err(ChainError::AlreadyMaterialized) + )); + assert_eq!( + tip(&repo).unwrap().map(|tip| tip.data), + first.last().map(|chunk| chunk.data), + "the refusal leaves the first chain as it was" + ); + } + + struct WithExtras { + due: BTreeMap, + extras: BTreeMap, + } + + impl WithExtras { + fn writing(records: &BTreeMap, extras: BTreeMap) -> Self { + Self { + due: records + .iter() + .map(|(key, body)| (key.clone(), Due::Write(body.clone()))) + .collect(), + extras, + } + } + + fn keeping(leaves: &BTreeMap, extras: BTreeMap) -> Self { + Self { + due: leaves + .iter() + .filter_map(|(key, cid)| Some((key.clone(), Due::Keep(RecordCid::new(*cid)?)))) + .collect(), + extras, + } + } + } + + impl Records for WithExtras { + fn due(&self) -> &BTreeMap { + &self.due + } + + fn extra_blocks(&self) -> BTreeMap { + self.extras.clone() + } + } + + #[tokio::test] + async fn extra_blocks_merge_into_the_chain_and_stay_servable() { + let (_dir, repo) = repo(); + let historical: BTreeMap = [body(50), body(51)] + .into_iter() + .map(|body| (body.cid().cid(), Bytes::from(body.as_bytes().to_vec()))) + .collect(); + let chunks = materialize( + &repo, + &repo_did(), + &WithExtras::writing(&records(2), historical), + &key(), + &mut Ticker::new(), + UnixSeconds::new(1), + ) + .await + .unwrap(); + let served: BTreeSet = chunks + .iter() + .flat_map(|chunk| chunk.blocks.keys().copied()) + .collect(); + let blocks_at_tip = tip(&repo).unwrap().unwrap().blocks; + futures::stream::iter([ + ( + body(50).cid().cid(), + Bytes::from(body(50).as_bytes().to_vec()), + ), + ( + body(51).cid().cid(), + Bytes::from(body(51).as_bytes().to_vec()), + ), + ]) + .for_each(|(cid, bytes)| { + let repo = &repo; + let served = &served; + async move { + assert!(served.contains(&cid), "the chunk blocks name {cid}"); + assert_eq!( + blocks::GitBlocks::at(repo, blocks_at_tip) + .get(&cid) + .await + .unwrap(), + Some(bytes), + "the block tree holds the historical body by its own cid" + ); + } + }) + .await; + } + + #[tokio::test] + async fn extras_without_ops_are_committed_on_their_own() { + let (_dir, repo) = repo(); + enable(&repo, &records(1), &key()).await.unwrap(); + let extra = body(99); + let cid = extra.cid().cid(); + let chunks = sync( + &repo, + &repo_did(), + &[collection()], + &WithExtras::keeping( + &leaves_of(&repo).await, + BTreeMap::from([(cid, Bytes::from(extra.as_bytes().to_vec()))]), + ), + &key(), + &mut Ticker::new(), + UnixSeconds::new(2), + ) + .await + .unwrap(); + assert_eq!( + chunks.len(), + 1, + "one commit is written for the extra blocks" + ); + assert!( + chunks[0].ops.is_empty(), + "the commit has no record ops, only the extra blocks" + ); + let tip = tip(&repo).unwrap().unwrap(); + assert_eq!( + blocks::GitBlocks::at(&repo, tip.blocks) + .get(&cid) + .await + .unwrap(), + Some(Bytes::from(body(99).as_bytes().to_vec())), + "a sync without record ops still writes the historical body" + ); + } + + #[tokio::test] + async fn commits_verify_against_the_signing_key_and_serialize_prev_as_an_explicit_null() { + let (_dir, repo) = repo(); + let signer = key(); + let chunks = enable(&repo, &records(3), &signer).await.unwrap(); + let explicit_null = [0x64, b'p', b'r', b'e', b'v', 0xf6]; + chunks.iter().for_each(|chunk| { + let commit = Commit::::from_cbor(&chunk.commit).unwrap(); + let unsigned = serde_ipld_dagcbor::to_vec(&UnsignedCommit { + did: commit.did.clone(), + version: commit.version, + data: *commit.data(), + rev: commit.rev().clone(), + prev: commit.prev().copied(), + }) + .unwrap(); + assert!(knot_runtime::verify( + &signer.public_key(), + &unsigned, + &knot_runtime::Signature::from_bytes(commit.sig().to_vec()), + )); + assert!( + chunk + .commit + .windows(explicit_null.len()) + .any(|window| window == explicit_null), + "omitting prev is an interop bug: it sits inside the signed bytes" + ); + }); + } + + #[tokio::test] + async fn over_wide_chunks_split_in_half_until_they_fit() { + let (_dir, repo) = repo(); + let records: BTreeMap = (0..3) + .map(|n| { + ( + MstKey::new(format!("{COLLECTION}/wide{n}")).unwrap(), + RecordBody::new( + serde_ipld_dagcbor::to_vec(&format!("{n}{}", "x".repeat(900_000))).unwrap(), + ) + .unwrap(), + ) + }) + .collect(); + let chunks = enable(&repo, &records, &key()).await.unwrap(); + assert_eq!( + chunks + .iter() + .map(|chunk| chunk.ops.len()) + .collect::>(), + vec![1, 2], + "a chunk over the blocks limit splits in half before it is signed" + ); + chunks + .iter() + .for_each(|chunk| assert!(blocks_bytes(&chunk.blocks) <= MAX_COMMIT_BLOCKS_BYTES)); + let served: BTreeSet = chunks + .iter() + .flat_map(|chunk| chunk.blocks.keys()) + .copied() + .collect(); + records + .values() + .for_each(|body| assert!(served.contains(&crate::cid_of(body.as_bytes())))); + } + + async fn leaves_of(repo: &Repo) -> BTreeMap { + leaves(repo).await.unwrap() + } + + #[tokio::test] + async fn sync_appends_create_update_and_delete_onto_the_enabled_chain() { + let (_dir, repo) = repo(); + let signer = key(); + let mut records = records(3); + enable(&repo, &records, &signer).await.unwrap(); + let enabled = tip(&repo).unwrap().unwrap(); + + records.remove(format!("{COLLECTION}/record0").as_str()); + records.insert( + MstKey::new(format!("{COLLECTION}/record3")).unwrap(), + body(99), + ); + let chunks = sync( + &repo, + &repo_did(), + std::slice::from_ref(&collection()), + &in_tree(records.clone()), + &signer, + &mut Ticker::new(), + UnixSeconds::new(2), + ) + .await + .unwrap(); + assert_eq!(chunks.len(), 1, "a changed record set appends one commit"); + let chunk = &chunks[0]; + assert_eq!( + chunk + .ops + .iter() + .map(|op| op.key().as_str().to_owned()) + .collect::>(), + BTreeSet::from([ + format!("{COLLECTION}/record0"), + format!("{COLLECTION}/record3"), + ]), + "the append carries the delete beside the create" + ); + assert!(chunk.rev.newer_than(&enabled.rev)); + assert_eq!(chunk.prev_data, Some(enabled.data)); + assert_eq!( + Some( + crate::proof::cold_invert(&chunk.blocks, chunk.data, &verified(&chunk.ops)) + .await + .unwrap(), + ), + chunk.prev_data, + "the chunk's blocks alone carry a covering proof back to the previous root" + ); + let leaves = leaves_of(&repo).await; + assert_eq!(leaves.len(), 3); + records.iter().for_each(|(key, body)| { + assert_eq!(leaves.get(key), Some(&crate::cid_of(body.as_bytes()))); + }); + } + + #[tokio::test] + async fn a_bulk_sync_splits_into_limited_commits_that_invert_cold() { + let (_dir, repo) = repo(); + let signer = key(); + enable(&repo, &records(1), &signer).await.unwrap(); + let enabled = tip(&repo).unwrap().unwrap(); + let records = records(450); + let chunks = sync( + &repo, + &repo_did(), + std::slice::from_ref(&collection()), + &in_tree(records), + &signer, + &mut Ticker::new(), + UnixSeconds::new(2), + ) + .await + .unwrap(); + assert_eq!( + chunks + .iter() + .map(|chunk| chunk.ops.len()) + .collect::>(), + vec![200, 200, 49], + "an append splits at the same operation ceiling enablement does" + ); + chunks.iter().for_each(|chunk| { + assert!(chunk.ops.len() <= MAX_OPS_PER_COMMIT); + assert!(blocks_bytes(&chunk.blocks) <= MAX_COMMIT_BLOCKS_BYTES); + }); + chunks.windows(2).for_each(|pair| { + assert!(pair[1].rev.newer_than(&pair[0].rev)); + assert_eq!(pair[1].prev_data, Some(pair[0].data)); + }); + let recovered = futures::stream::iter(chunks.iter().rev()) + .fold( + Ok::(chunks.last().unwrap().data), + |root, chunk| async move { + let root = root?; + let inverted = + crate::proof::cold_invert(&chunk.blocks, root, &verified(&chunk.ops)) + .await + .unwrap(); + assert_eq!(chunk.prev_data, Some(inverted)); + Ok(inverted) + }, + ) + .await + .unwrap(); + assert_eq!(recovered, enabled.data); + assert_eq!( + leaves_of(&repo).await.len(), + 450, + "the new tip holds every synced record" + ); + } + + #[tokio::test] + async fn sync_before_enablement_is_refused_and_unchanging_sync_leaves_tip_alone() { + let (_dir, repo) = repo(); + assert!(matches!( + sync( + &repo, + &repo_did(), + std::slice::from_ref(&collection()), + &in_tree(records(1)), + &key(), + &mut Ticker::new(), + UnixSeconds::new(1) + ) + .await, + Err(ChainError::NotEnabled) + )); + let signer = key(); + let records = records(2); + enable(&repo, &records, &signer).await.unwrap(); + let before = tip(&repo).unwrap().unwrap(); + assert!( + sync( + &repo, + &repo_did(), + std::slice::from_ref(&collection()), + &in_tree(records), + &signer, + &mut Ticker::new(), + UnixSeconds::new(2) + ) + .await + .unwrap() + .is_empty() + ); + assert_eq!(tip(&repo).unwrap().unwrap().git, before.git); + } + + #[tokio::test] + async fn sync_leaves_records_outside_its_collection_alone() { + let (_dir, repo) = repo(); + let signer = key(); + let mut mixed = records(1); + let foreign = "other.collection/record9"; + mixed.insert( + MstKey::new(foreign.to_owned()).unwrap(), + records(2).values().next().unwrap().clone(), + ); + enable(&repo, &mixed, &signer).await.unwrap(); + + let chunks = sync( + &repo, + &repo_did(), + std::slice::from_ref(&collection()), + &in_tree(records(2)), + &signer, + &mut Ticker::new(), + UnixSeconds::new(2), + ) + .await + .unwrap(); + let leaves = leaves_of(&repo).await; + assert_eq!(leaves.len(), 3, "the append adds one member invite"); + assert!( + leaves.contains_key(foreign), + "a collection's sync never deletes another collection's records" + ); + assert!( + !chunks + .iter() + .any(|chunk| chunk.ops.iter().any(|op| op.key().as_str() == foreign)), + "the foreign record rides no op" + ); + + let mut outside = records(1); + outside.insert( + MstKey::new(foreign.to_owned()).unwrap(), + records(2).values().next().unwrap().clone(), + ); + assert!(matches!( + sync( + &repo, + &repo_did(), + std::slice::from_ref(&collection()), + &in_tree(outside), + &signer, + &mut Ticker::new(), + UnixSeconds::new(3) + ) + .await, + Err(ChainError::Malformed(_)) + )); + } + + #[tokio::test] + async fn every_block_the_chain_writes_is_readable_off_the_tip_alone() { + let (_dir, repo) = repo(); + let signer = key(); + let mut records = records(2); + let mut chunks = enable(&repo, &records, &signer).await.unwrap(); + records.insert(MstKey::new(format!("{COLLECTION}/late")).unwrap(), body(7)); + chunks.extend( + sync( + &repo, + &repo_did(), + std::slice::from_ref(&collection()), + &in_tree(records.clone()), + &signer, + &mut Ticker::new(), + UnixSeconds::new(2), + ) + .await + .unwrap(), + ); + let tip = tip(&repo).unwrap().unwrap(); + let stored = GitBlocks::at(&repo, tip.blocks); + let served: Vec<(Cid, Bytes)> = chunks + .iter() + .flat_map(|chunk| chunk.blocks.clone()) + .collect(); + for (cid, bytes) in &served { + assert_eq!( + stored.get(cid).await.unwrap().as_deref(), + Some(bytes.as_ref()), + "the tip's blocks tree holds every block the chain ever wrote, {cid}" + ); + } + assert_eq!( + leaves_of(&repo).await.len(), + records.len(), + "enabled and appended records all sit in the one tree" + ); + assert_eq!( + commit_cid(&chunks.last().unwrap().commit).unwrap(), + commit_cid(&commit_bytes(&repo, &tip).unwrap()).unwrap() + ); + } + + #[tokio::test] + async fn a_fresh_account_initializes_with_an_empty_commit_and_one_more_tip() { + let (_dir, repo) = repo(); + let chunk = initialize( + &repo, + &repo_did(), + &key(), + &mut Ticker::new(), + UnixSeconds::new(1), + ) + .await + .unwrap(); + assert!(chunk.ops.is_empty()); + assert_eq!(chunk.prev_rev, None); + assert_eq!(chunk.prev_data, None); + let tip = tip(&repo).unwrap().unwrap(); + assert_eq!(tip.rev, chunk.rev); + assert_eq!(tip.data, chunk.data); + assert!( + leaves_of(&repo).await.is_empty(), + "Genesis commit has no records" + ); + assert!(matches!( + initialize( + &repo, + &repo_did(), + &key(), + &mut Ticker::new(), + UnixSeconds::new(2) + ) + .await, + Err(ChainError::AlreadyMaterialized) + )); + assert_eq!( + sync( + &repo, + &repo_did(), + std::slice::from_ref(&collection()), + &in_tree(records(1)), + &key(), + &mut Ticker::new(), + UnixSeconds::new(2), + ) + .await + .unwrap() + .len(), + 1, + "an invite after genesis appends onto the empty commit" + ); + } + + #[derive(Default)] + struct Collecting { + frames: Vec>, + } + + impl Emit for Collecting { + fn rev(&mut self, _prev: Option<&Tid>) -> Tid { + Tid::from_time(1_700_000_000_000_000 + self.frames.len() as u64, 0) + } + + fn frame(&self, chunk: &CommitChunk) -> Option { + Some(Bytes::from(chunk.rev.to_string())) + } + + fn committed(&mut self, frame: Option) { + self.frames.push(frame); + } + } + + #[tokio::test] + async fn the_replay_walk_stops_at_its_limit() { + let (_dir, repo) = repo(); + let mut emit = Collecting::default(); + let enabled = materialize( + &repo, + &repo_did(), + &in_tree(records(2)), + &key(), + &mut emit, + UnixSeconds::new(1), + ) + .await + .unwrap(); + let appended = sync( + &repo, + &repo_did(), + std::slice::from_ref(&collection()), + &in_tree(records(4)), + &key(), + &mut emit, + UnixSeconds::new(2), + ) + .await + .unwrap(); + let start = tip(&repo).unwrap().unwrap().git; + let mut read = framed_revs(&repo, start, UnixMicros::new(0), usize::MAX) + .found + .into_iter() + .map(|(_micros, oid)| frame_blob(&repo, oid).unwrap().unwrap()) + .collect::>(); + read.reverse(); + assert_eq!( + read, + enabled + .iter() + .chain(appended.iter()) + .map(|chunk| Bytes::from(chunk.rev.to_string())) + .collect::>(), + "the walk hands back every stored frame, oldest first" + ); + let one_short = framed_revs(&repo, start, UnixMicros::new(0), 1); + assert_eq!( + one_short.found, + vec![( + UnixMicros::new(tip(&repo).unwrap().unwrap().rev.timestamp()), + start, + )], + "a walk one frame short keeps only the newest commit" + ); + assert_eq!( + one_short.end, + TraversalEnd::Limited, + "the walk should report the history it left behind" + ); + assert_eq!( + framed_revs(&repo, start, UnixMicros::new(0), 2).end, + TraversalEnd::Exhausted, + "a walk shouldn't hit the limit when it reaches the bottom of the chain" + ); + } + + #[tokio::test] + async fn stalled_chain_marks_frames_incomplete() { + let (_dir, repo) = repo(); + let mut emit = Collecting::default(); + materialize( + &repo, + &repo_did(), + &in_tree(records(2)), + &key(), + &mut emit, + UnixSeconds::new(1), + ) + .await + .unwrap(); + sync( + &repo, + &repo_did(), + std::slice::from_ref(&collection()), + &in_tree(records(4)), + &key(), + &mut emit, + UnixSeconds::new(2), + ) + .await + .unwrap(); + let start = tip(&repo).unwrap().unwrap().git; + let older = step(&repo, start, false).unwrap().next.unwrap(); + let hex = older.to_string(); + let object = repo + .git() + .git_dir() + .join("objects") + .join(&hex[..2]) + .join(&hex[2..]); + std::fs::remove_file(&object).unwrap(); + std::fs::write(&object, b"not a commit anymore").unwrap(); + + let fresh = knot_git::Layout::new(_dir.path()) + .open(&RepoDid::new("did:plc:squid").unwrap()) + .unwrap(); + let start = tip(&fresh).unwrap().unwrap().git; + let traversed = frames_after(&fresh, start, UnixMicros::new(0), usize::MAX); + assert_eq!(traversed.found.len(), 1, "the tip's own frame still yields"); + assert_eq!( + traversed.end, + TraversalEnd::Broken, + "An unreadable older commit ends the traversal as broken, distinct from a \ + complete history and from the window." + ); + } + + #[tokio::test] + async fn frames_to_stops_at_requested_commit_excluding_it() { + let (_foreign_dir, right) = repo(); + let (_dir, repo) = repo(); + let mut emit = Collecting::default(); + materialize( + &repo, + &repo_did(), + &in_tree(records(2)), + &key(), + &mut emit, + UnixSeconds::new(1), + ) + .await + .unwrap(); + sync( + &repo, + &repo_did(), + std::slice::from_ref(&collection()), + &in_tree(records(4)), + &key(), + &mut emit, + UnixSeconds::new(2), + ) + .await + .unwrap(); + let start = tip(&repo).unwrap().unwrap().git; + let older = step(&repo, start, false).unwrap().next.unwrap(); + let traversed = frames_to(&repo, start, older, usize::MAX); + assert_eq!( + traversed.found.len(), + 1, + "Traversal to an ancestor returns only the commits after it." + ); + assert!(traversed.reached); + assert_eq!(traversed.found[0].0, tip(&repo).unwrap().unwrap().rev); + + let empty = frames_to(&repo, start, start, usize::MAX); + assert!(empty.reached); + assert!( + empty.found.is_empty(), + "The target's own frame belongs to the caller's base rather than the delta." + ); + + let cut = frames_to(&repo, start, older, 0); + assert!( + !cut.reached && cut.found.is_empty(), + "A zero limit is unreached, not an empty delta." + ); + + materialize( + &right, + &repo_did(), + &in_tree(records(2)), + &key(), + &mut Collecting::default(), + UnixSeconds::new(3), + ) + .await + .unwrap(); + let foreign = tip(&right).unwrap().unwrap().git; + let traversed = frames_to(&repo, start, foreign, usize::MAX); + assert!( + !traversed.reached, + "Commit from another chain is never an ancestor, so the delta must say so." + ); + assert_eq!( + traversed.found.len(), + 2, + "The traversal still returns every frame on the way down." + ); + } + + fn chain_ticket_collection() -> RecordCollection { + RecordCollection::new("org.tangled.track.ticket").unwrap() + } + + fn ticket_address(n: u64) -> RecordAddress { + RecordAddress::new( + knot_types::RecordCollection::new("org.tangled.track.ticket").unwrap(), + knot_types::RecordRkey::minted(Tid::from_time(1_700_000_000_000_000 + n, 0)), + ) + } + + fn ticket_body(revision: u64) -> RecordBody { + RecordBody::new(ticket_record_bytes(revision)).expect("Issue record is dag-cbor") + } + + fn ticket_record_bytes(revision: u64) -> Vec { + serde_ipld_dagcbor::to_vec(&BTreeMap::from([ + ("$type", "org.tangled.track.ticket".to_owned()), + ("title", format!("kelp {revision}")), + ])) + .unwrap() + } + + fn ticket_records(count: u64) -> BTreeMap { + (0..count) + .map(|n| (ticket_address(n).mst_key(), ticket_body(n))) + .collect() + } + + fn cid_at(records: &BTreeMap, address: &RecordAddress) -> RecordCid { + records[address.mst_key().as_str()].cid() + } + + struct Framing; + + impl Emit for Framing { + fn rev(&mut self, prev: Option<&Tid>) -> Tid { + Ticker::new().next(prev.cloned()) + } + + fn frame(&self, chunk: &CommitChunk) -> Option { + Some(Bytes::from( + chunk + .blocks + .keys() + .map(Cid::to_string) + .collect::>() + .join("\n"), + )) + } + + fn committed(&mut self, _frame: Option) {} + } + + #[tokio::test] + async fn planned_chunk_keeps_leaf_in_git_and_frame_lists_it() { + let (_dir, repo) = repo(); + enable(&repo, &ticket_records(1), &key()).await.unwrap(); + let body = ticket_body(7); + let chunks = apply( + &repo, + &repo_did(), + &[( + ticket_address(7), + RecordOp::Create { + record: body.clone(), + }, + )], + &key(), + &mut Framing, + UnixSeconds::new(2), + ) + .await + .unwrap(); + let [chunk] = chunks.as_slice() else { + panic!("one op must be one commit, found {}", chunks.len()); + }; + let leaf = body.cid().cid(); + assert!( + chunk.blocks.contains_key(&leaf), + "the planned chunk must include the record's bytes" + ); + let tip = tip(&repo).unwrap().unwrap(); + let stored = frame_blob(&repo, tip.git).unwrap().unwrap(); + assert!( + std::str::from_utf8(&stored) + .unwrap() + .contains(&leaf.to_string()), + "the stored frame oughtta list every block it wrote, including the leaf" + ); + assert!( + std::str::from_utf8(&stored) + .unwrap() + .contains(&tip.data.to_string()), + "the stored frame ought to keep the MST proof" + ); + let git_only = GitBlocks::at(&repo, tip.blocks); + assert_eq!( + git_only.get(&leaf).await.unwrap().as_deref(), + Some(body.as_bytes()), + "the git block tree must answer the leaf directly" + ); + assert_eq!( + leaves_of(&repo).await.get(&ticket_address(7).mst_key()), + Some(&leaf), + "the MST must point at the leaf whichever store answers it" + ); + } + + #[tokio::test] + async fn chunks_split_at_blocks_byte_limit() { + let (_dir, repo) = repo(); + enable(&repo, &ticket_records(1), &key()).await.unwrap(); + let heft = |n: u64| { + RecordBody::new( + serde_ipld_dagcbor::to_vec(&BTreeMap::from([ + ("$type", "org.tangled.track.ticket".to_owned()), + ("title", format!("kelp {n}")), + ("body", "k".repeat(900_000)), + ])) + .unwrap(), + ) + .unwrap() + }; + let ops: Vec<(RecordAddress, RecordOp)> = (0..3) + .map(|n| { + ( + ticket_address(100 + n), + RecordOp::Create { record: heft(n) }, + ) + }) + .collect(); + let chunks = apply( + &repo, + &repo_did(), + &ops, + &key(), + &mut Framing, + UnixSeconds::new(3), + ) + .await + .unwrap(); + assert_eq!( + chunks + .iter() + .map(|chunk| chunk.ops.len()) + .collect::>(), + vec![1, 2], + "three records past the blocks byte limit oughtta split, so no commit draft keeps \ + unbounded record bytes in memory" + ); + chunks.iter().for_each(|chunk| { + assert!( + blocks_bytes(&chunk.blocks) <= MAX_COMMIT_BLOCKS_BYTES, + "each commit's blocks should stay under the {MAX_COMMIT_BLOCKS_BYTES} limit" + ); + }); + } + #[tokio::test] + async fn apply_stamps_the_current_cid_when_the_record_moved() { + let (_dir, repo) = repo(); + let base = ticket_records(2); + enable(&repo, &base, &key()).await.unwrap(); + let chain_tip = tip(&repo).unwrap().unwrap().git; + let live = cid_at(&base, &ticket_address(0)); + let stale = RecordCid::new(crate::cid_of(&ticket_record_bytes(77))) + .expect("Fixture record is dag-cbor"); + + let moved = apply( + &repo, + &repo_did(), + &[( + ticket_address(0), + RecordOp::Update { + record: ticket_body(1), + from: stale, + }, + )], + &key(), + &mut Ticker::new(), + UnixSeconds::new(2), + ) + .await; + let Err(ChainError::RecordMoved { + uri, + current, + claimed, + }) = &moved + else { + panic!("Edit from version that moved on should be rejected, got {moved:?}") + }; + assert_eq!( + current, &live, + "Moved record ought to answer with the current version" + ); + assert_eq!(claimed, &stale); + assert_eq!( + uri, + &ticket_address(0).at_uri(&repo_did()), + "and should include the at-uri to rebase onto" + ); + + let existing = apply( + &repo, + &repo_did(), + &[( + ticket_address(0), + RecordOp::Create { + record: ticket_body(3), + }, + )], + &key(), + &mut Ticker::new(), + UnixSeconds::new(2), + ) + .await; + assert!( + matches!(existing, Err(ChainError::RecordExists { .. })), + "Create never writes over a live record, got {existing:?}" + ); + + let absent = apply( + &repo, + &repo_did(), + &[(ticket_address(5), RecordOp::Delete { from: live })], + &key(), + &mut Ticker::new(), + UnixSeconds::new(2), + ) + .await; + assert!( + matches!(absent, Err(ChainError::RecordAbsent { .. })), + "there's no record at that address to delete, got {absent:?}" + ); + + let twice = apply( + &repo, + &repo_did(), + &[ + ( + ticket_address(0), + RecordOp::Update { + record: ticket_body(4), + from: live, + }, + ), + (ticket_address(0), RecordOp::Delete { from: live }), + ], + &key(), + &mut Ticker::new(), + UnixSeconds::new(2), + ) + .await; + assert!( + matches!(twice, Err(ChainError::RecordContested { .. })), + "two ops on one address in one write have no defined order, got {twice:?}" + ); + + assert_eq!( + tip(&repo).unwrap().unwrap().git, + chain_tip, + "every write above was turned away and left chain alone" + ); + } + + type OpMap = BTreeMap, Option)>; + + fn op_map(chunks: &[CommitChunk]) -> OpMap { + chunks + .iter() + .flat_map(|chunk| chunk.ops.iter()) + .map(|op| (op.key().as_str().to_owned(), (op.cid(), op.prev()))) + .collect() + } + + fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("Current-thread runtime builds") + } + + const BASE_ISSUES: u64 = 20; + const REACH: u64 = 30; + const OVER_LIMIT_ISSUES: u64 = 240; + const ENABLED_AT: u64 = 1_700_000_000_000_000; + const WRITTEN_AT: u64 = 1_800_000_000_000_000; + + struct Stamped(u64); + + impl Emit for Stamped { + fn rev(&mut self, _prev: Option<&Tid>) -> Tid { + self.0 += 1; + Tid::from_time(self.0, 0) + } + + fn frame(&self, _chunk: &CommitChunk) -> Option { + None + } + + fn committed(&mut self, _frame: Option) {} + } + + fn faces(chunks: &[CommitChunk]) -> Vec<(CommitChunk, OpMap)> { + chunks + .iter() + .map(|chunk| { + ( + CommitChunk { + ops: Vec::new(), + ..chunk.clone() + }, + op_map(std::slice::from_ref(chunk)), + ) + }) + .collect() + } + + fn op_of(existing: Option, action: u8, n: u64) -> Option { + match (existing, action) { + (Some(body), 1) => Some(RecordOp::Update { + from: body.cid(), + record: body, + }), + (Some(body), 2) => Some(RecordOp::Update { + from: body.cid(), + record: ticket_body(1_000 + n), + }), + (Some(body), 3) => Some(RecordOp::Delete { from: body.cid() }), + (Some(_), _) | (None, 0) => None, + (None, _) => Some(RecordOp::Create { + record: ticket_body(n), + }), + } + } + + fn wanted(base: &BTreeMap, actions: &[u8]) -> BTreeMap { + actions + .iter() + .enumerate() + .filter_map(|(index, action)| { + let n = index as u64; + let key = ticket_address(n).mst_key(); + let existing = base.get(&key).cloned(); + match op_of(existing.clone(), *action, n) { + Some(op) => op.record().cloned().map(|record| (key, record)), + None => existing.map(|body| (key, body)), + } + }) + .collect() + } + + fn ops_in_order( + base: &BTreeMap, + actions: &[u8], + order: &[u64], + ) -> Vec<(RecordAddress, RecordOp)> { + order + .iter() + .filter_map(|n| { + let address = ticket_address(*n); + let existing = base.get(address.mst_key().as_str()).cloned(); + op_of(existing, actions[*n as usize], *n).map(|op| (address, op)) + }) + .collect() + } + + async fn synced_and_applied( + base: &BTreeMap, + desired: BTreeMap, + ops: &[(RecordAddress, RecordOp)], + ) -> (Vec, Cid, Vec, Cid) { + let (_sync_dir, sync_repo) = repo(); + let (_apply_dir, apply_repo) = repo(); + let enabled = async |repo: &Repo| { + materialize( + repo, + &repo_did(), + &in_tree(base.clone()), + &key(), + &mut Stamped(ENABLED_AT), + UnixSeconds::new(1), + ) + .await + .unwrap() + }; + let one_way = enabled(&sync_repo).await; + let other_way = enabled(&apply_repo).await; + assert_eq!( + faces(&one_way), + faces(&other_way), + "two chains over one record set must start from one chain" + ); + let synced = sync( + &sync_repo, + &repo_did(), + std::slice::from_ref(&chain_ticket_collection()), + &in_tree(desired), + &key(), + &mut Stamped(WRITTEN_AT), + UnixSeconds::new(2), + ) + .await + .unwrap(); + let applied = apply( + &apply_repo, + &repo_did(), + ops, + &key(), + &mut Stamped(WRITTEN_AT), + UnixSeconds::new(2), + ) + .await + .unwrap(); + ( + synced, + tip(&sync_repo).unwrap().unwrap().data, + applied, + tip(&apply_repo).unwrap().unwrap().data, + ) + } + + async fn inverts_cold(chunks: &[CommitChunk]) -> bool { + futures::stream::iter(chunks) + .then(|chunk| async move { + crate::proof::cold_invert(&chunk.blocks, chunk.data, &verified(&chunk.ops)) + .await + .ok() + == chunk.prev_data + }) + .all(|inverted| async move { inverted }) + .await + } + + proptest::proptest! { + #![proptest_config(proptest::prelude::ProptestConfig { cases: 16, ..proptest::prelude::ProptestConfig::default() })] + + #[test] + fn apply_in_any_order_emits_chunk_that_sync_emits( + drawn in proptest::collection::vec(0u8..5, REACH as usize), + order in Strategy::prop_shuffle(proptest::prelude::Just((0..=REACH).collect::>())) + ) { + let actions: Vec = drawn.iter().copied().chain([2]).collect(); + let base = ticket_records(BASE_ISSUES); + let ops = ops_in_order(&base, &actions, &order); + let runtime = runtime(); + let (synced, sync_root, applied, apply_root) = runtime.block_on( + synced_and_applied(&base, wanted(&base, &actions), &ops), + ); + + proptest::prop_assert_eq!( + faces(&applied), + faces(&synced), + "whatever order its ops arrive in, apply must sign the commit the \ + whole-collection diff signs and include the same proof blocks" + ); + proptest::prop_assert_eq!( + apply_root, + sync_root, + "MST, a pure function of the record set, gives both paths one root" + ); + proptest::prop_assert!( + runtime.block_on(inverts_cold(&applied)), + "every chunk apply emits inverts cold out of its own blocks" + ); + } + } + + #[tokio::test] + async fn one_op_apply_covers_neighbors_whole_collection_sync_covers() { + let base = ticket_records(BASE_ISSUES); + let base = &base; + futures::stream::iter([ + (11u64, 2u8, "an update"), + (7, 3, "a delete"), + (BASE_ISSUES + 2, 2, "a create"), + ]) + .for_each(|(n, action, shape)| async move { + let actions: Vec = (0..BASE_ISSUES + 3) + .map(|index| if index == n { action } else { 0 }) + .collect(); + let ops = ops_in_order(base, &actions, &[n]); + let (synced, sync_root, applied, apply_root) = + synced_and_applied(base, wanted(base, &actions), &ops).await; + assert_eq!( + faces(&applied), + faces(&synced), + "{shape} written through apply must cover the MST neighbors the \ + whole-collection diff covers for the very same op" + ); + assert_eq!( + apply_root, sync_root, + "{shape} oughtta reach one root either way" + ); + assert!( + inverts_cold(&applied).await, + "{shape} ought to invert cold out of the blocks its own frame includes" + ); + }) + .await; + } + + #[tokio::test] + async fn run_over_chunk_limit_emits_commits_that_sync_emits() { + let actions: Vec = (0..320u64) + .map(|n| match n { + 0..70 => 3, + 70..140 => 2, + 140..150 => 1, + 150..OVER_LIMIT_ISSUES => 0, + _ => 2, + }) + .collect(); + let order: Vec = (70..150).chain(240..320).chain(0..70).collect(); + let base = ticket_records(OVER_LIMIT_ISSUES); + let ops = ops_in_order(&base, &actions, &order); + let (synced, sync_root, applied, apply_root) = + synced_and_applied(&base, wanted(&base, &actions), &ops).await; + + assert_eq!( + applied + .iter() + .map(|chunk| chunk.ops.len()) + .collect::>(), + vec![200, 20], + "the ten byte-identical updates among them produce no op, and 220 ops in one apply \ + ought to cross the 200-op limit into two commits" + ); + assert_eq!( + faces(&applied), + faces(&synced), + "both paths oughtta sign the same commits past the limit and cover them with \ + the same blocks" + ); + assert_eq!(apply_root, sync_root); + } + + #[tokio::test] + async fn update_to_unchanged_bytes_skips_commit() { + let (_dir, repo) = repo(); + let base = ticket_records(3); + enable(&repo, &base, &key()).await.unwrap(); + let before = tip(&repo).unwrap().unwrap(); + let chunks = apply( + &repo, + &repo_did(), + &[( + ticket_address(1), + RecordOp::Update { + record: ticket_body(1), + from: cid_at(&base, &ticket_address(1)), + }, + )], + &key(), + &mut Ticker::new(), + UnixSeconds::new(2), + ) + .await + .unwrap(); + assert!( + chunks.is_empty(), + "a write of the bytes already at the address mustn't be a write, and mustn't create \ + a signed commit" + ); + assert_eq!( + tip(&repo).unwrap().unwrap().git, + before.git, + "the ref shouldn't move for it either" + ); + } + + #[test] + fn tree_entry_sorts_where_git_sorts_it_with_subtree_slash() { + let entry = |name: &str, kind: gix::objs::tree::EntryKind| gix::objs::tree::Entry { + mode: kind.into(), + filename: name.into(), + oid: gix::ObjectId::empty_blob(gix::hash::Kind::Sha1), + }; + let blob = |name: &str| entry(name, gix::objs::tree::EntryKind::Blob); + let tree = |name: &str| entry(name, gix::objs::tree::EntryKind::Tree); + let mut entries = [ + blob("blocks."), + tree("blocks"), + blob("commit"), + blob("blocks"), + blob("frame"), + ]; + entries.sort_by(git_entry_order); + let sorted: Vec = entries + .iter() + .map(|entry| { + let slash = if entry.mode.is_tree() { "/" } else { "" }; + format!("{}{slash}", entry.filename) + }) + .collect(); + assert_eq!( + sorted, + vec!["blocks", "blocks.", "blocks/", "commit", "frame"], + "git compares subtree as though its name ended in slash. It rejects \ + trees whose entries arrive in any other order" + ); + } +} diff --git a/knot2/crates/knot-record/src/comment.rs b/knot2/crates/knot-record/src/comment.rs new file mode 100644 index 0000000000000000000000000000000000000000..317d3c32135b33567ef045caf7d20ee440a9a3d4 --- /dev/null +++ b/knot2/crates/knot-record/src/comment.rs @@ -0,0 +1,308 @@ +use knot_types::{ + AccountDid, AtUri, Oid, RecordAddress, RecordBody, RecordBodyError, RecordCid, RepoDid, + UnixSeconds, +}; +use serde::{Deserialize, Serialize, Serializer}; + +use crate::ticket::MarkdownBody; +use crate::{CidString, CreatedAt, RecordError, encoded}; + +pub const COMMENT_COLLECTION: &str = "org.tangled.feed.comment"; + +pub const MARKDOWN_COLLECTION: &str = "org.tangled.markup.markdown"; + +pub const COMMIT_EMBED_TYPE: &str = "org.tangled.embed.commit"; + +collection!( + comment_collection, + COMMENT_COLLECTION, + "Comment collection is valid nsid" +); + +collection!( + markup_collection, + MARKDOWN_COLLECTION, + "Markup collection is valid nsid" +); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CommentRef { + address: RecordAddress, + cid: RecordCid, +} + +impl CommentRef { + pub fn new(address: RecordAddress, cid: RecordCid) -> Self { + Self { address, cid } + } + + pub fn address(&self) -> &RecordAddress { + &self.address + } + + pub fn cid(&self) -> RecordCid { + self.cid + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommentMarkup { + text: MarkdownBody, + attachments: Option>, +} + +impl CommentMarkup { + pub fn new( + text: MarkdownBody, + attachments: Option>, + ) -> Self { + Self { text, attachments } + } + + pub fn attachments(&self) -> &[crate::attachment::Attachment] { + self.attachments.as_deref().unwrap_or(&[]) + } + + pub fn text(&self) -> &MarkdownBody { + &self.text + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommentRecord { + repo: RepoDid, + subject: CommentRef, + body: CommentMarkup, + reply_to: Option, + embed: Option, + created_at: CreatedAt, + editor: AccountDid, +} + +impl CommentRecord { + pub fn new( + repo: RepoDid, + subject: CommentRef, + body: CommentMarkup, + reply_to: Option, + embed: Option, + created_at: UnixSeconds, + editor: AccountDid, + ) -> Result { + Ok(Self { + repo, + subject, + body, + reply_to, + embed, + created_at: CreatedAt::at(created_at)?, + editor, + }) + } + + pub fn body(&self) -> Result { + encoded(self) + } +} + +#[derive(Serialize)] +pub(crate) struct CommentRefEncoding { + uri: AtUri, + cid: CidString, +} + +pub(crate) fn ref_encoding(reference: &CommentRef, repo: &RepoDid) -> CommentRefEncoding { + CommentRefEncoding { + uri: reference.address.at_uri(repo), + cid: reference.cid.into(), + } +} + +#[derive(Serialize)] +pub(crate) struct MarkdownEncoding<'a> { + #[serde(rename = "$type")] + markup_type: &'static str, + text: &'a MarkdownBody, + #[serde( + skip_serializing_if = "Option::is_none", + serialize_with = "crate::attachment::embeds" + )] + blobs: &'a Option>, +} + +pub(crate) fn markdown_encoding<'a>( + text: &'a MarkdownBody, + blobs: &'a Option>, +) -> MarkdownEncoding<'a> { + MarkdownEncoding { + markup_type: MARKDOWN_COLLECTION, + text, + blobs, + } +} + +#[derive(Serialize)] +struct OidEncoding<'a> { + oid: &'a Oid, +} + +// the commit is always in the comment's repo, so only its oid is kept +#[derive(Serialize)] +struct CommitEmbedEncoding<'a> { + #[serde(rename = "$type")] + embed_type: &'static str, + repo: &'a RepoDid, + commit: OidEncoding<'a>, +} + +#[derive(Serialize)] +struct CommentEncoding<'a> { + #[serde(rename = "$type")] + record_type: knot_types::RecordCollection, + subject: CommentRefEncoding, + body: MarkdownEncoding<'a>, + #[serde(rename = "createdAt")] + created_at: &'a CreatedAt, + #[serde(skip_serializing_if = "Option::is_none", rename = "replyTo")] + reply_to: Option, + #[serde(skip_serializing_if = "Option::is_none")] + embed: Option>, + #[serde(rename = "x-tngl-editor")] + editor: &'a AccountDid, +} + +impl Serialize for CommentRecord { + fn serialize(&self, serializer: S) -> Result { + CommentEncoding { + record_type: comment_collection(), + subject: ref_encoding(&self.subject, &self.repo), + body: markdown_encoding(&self.body.text, &self.body.attachments), + created_at: &self.created_at, + reply_to: self + .reply_to + .as_ref() + .map(|reply| ref_encoding(reply, &self.repo)), + embed: self.embed.as_ref().map(|oid| CommitEmbedEncoding { + embed_type: COMMIT_EMBED_TYPE, + repo: &self.repo, + commit: OidEncoding { oid }, + }), + editor: &self.editor, + } + .serialize(serializer) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::frozen; + use knot_types::RecordRkey; + + const COMMENT: &str = "a564626f6479a26474657874646b656c70652474797065781b6f72672e74616e676c65642e6d61726b75702e6d61726b646f776e65247479706578186f72672e74616e676c65642e666565642e636f6d6d656e74677375626a656374a263636964783b62616679726569666a6a6570757967377877646837786c6f6c756c6e793661756f6a6d3773717a366c6c687137686966346466757068726973696963757269783961743a2f2f6469643a706c633a73717569642f6f72672e74616e676c65642e747261636b2e7469636b65742f336c75627270747835376432326963726561746564417474313937302d30312d30315430303a30303a30315a6d782d746e676c2d656469746f7278206469643a706c633a3366776563646e76746373636a6e72783270346e37616c7a"; + + use crate::test_support::{editor, squid, subject}; + + #[test] + fn comment_record_bytes_frozen() { + let record = CommentRecord::new( + squid(), + subject(), + CommentMarkup::new(MarkdownBody::new("kelp").unwrap(), None), + None, + None, + UnixSeconds::new(1), + editor(), + ) + .unwrap(); + let body = record.body().unwrap(); + assert_eq!( + body.as_bytes(), + frozen(COMMENT).as_slice(), + "the first emitted comment must freeze these bytes: absent optionals omitted, seconds precision, utc z" + ); + } + + #[test] + fn comment_change_round_trips_with_its_refs() { + #[derive(serde::Deserialize)] + struct Decoded { + #[serde(rename = "$type")] + kind: String, + subject: DecodedRef, + body: DecodedBody, + #[serde(rename = "replyTo")] + reply_to: Option, + } + #[derive(serde::Deserialize)] + struct DecodedRef { + uri: String, + cid: String, + } + #[derive(serde::Deserialize)] + struct DecodedBody { + #[serde(rename = "$type")] + kind: String, + text: String, + } + + let reply = CommentRef::new( + RecordAddress::new( + comment_collection(), + RecordRkey::new("3lubrptx57d44").unwrap(), + ), + knot_types::RecordCid::new(knot_types::dag_cbor_cid(b"reply")).unwrap(), + ); + let record = CommentRecord::new( + squid(), + subject(), + CommentMarkup::new(MarkdownBody::new("kelp").unwrap(), None), + Some(reply.clone()), + None, + UnixSeconds::new(1), + editor(), + ) + .unwrap(); + let decoded: Decoded = + serde_ipld_dagcbor::from_slice(record.body().unwrap().as_bytes()).unwrap(); + assert_eq!(decoded.kind, COMMENT_COLLECTION); + assert_eq!(decoded.body.kind, MARKDOWN_COLLECTION); + assert_eq!(decoded.body.text, "kelp"); + assert_eq!( + decoded.subject.uri, + "at://did:plc:squid/org.tangled.track.ticket/3lubrptx57d22" + ); + assert_eq!(decoded.subject.cid, subject().cid().to_string()); + let reply_to = decoded.reply_to.unwrap(); + assert_eq!( + reply_to.uri, + "at://did:plc:squid/org.tangled.feed.comment/3lubrptx57d44" + ); + assert_eq!(reply_to.cid, reply.cid().to_string()); + } + + #[test] + fn comment_embed_names_a_commit_in_its_repo() { + let oid = Oid::from_hex(&"a".repeat(40)).unwrap(); + let record = CommentRecord::new( + squid(), + subject(), + CommentMarkup::new(MarkdownBody::new("kelp").unwrap(), None), + None, + Some(oid), + UnixSeconds::new(1), + editor(), + ) + .unwrap(); + let decoded: serde_json::Value = + serde_ipld_dagcbor::from_slice(record.body().unwrap().as_bytes()).unwrap(); + assert_eq!( + decoded["embed"], + serde_json::json!({ + "$type": COMMIT_EMBED_TYPE, + "repo": squid().to_string(), + "commit": { "oid": "a".repeat(40) }, + }) + ); + } +} diff --git a/knot2/crates/knot-record/src/git_refs.rs b/knot2/crates/knot-record/src/git_refs.rs new file mode 100644 index 0000000000000000000000000000000000000000..87c04525f0decde958fc68554503d0f2ace87c88 --- /dev/null +++ b/knot2/crates/knot-record/src/git_refs.rs @@ -0,0 +1,82 @@ +use std::collections::BTreeMap; + +use crate::chain::ChainError; +use knot_git::Repo; +use knot_types::{BranchName, MstKey, RecordBody, RecordBodyError}; +use serde::Serialize; + +pub const GIT_REFS_COLLECTION: &str = "org.tangled.git.refs"; + +pub const SELF_RKEY: &str = "self"; + +const DEFAULT_DEFAULT_BRANCH: &str = "main"; + +collection!( + git_refs_collection, + GIT_REFS_COLLECTION, + "Git refs collection is valid nsid" +); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct RefsRecord { + #[serde(rename = "$type")] + record_type: &'static str, + #[serde(rename = "defaultBranch")] + default_branch: BranchName, + #[serde(with = "serde_bytes")] + refs: Vec, +} + +impl RefsRecord { + pub fn new(default_branch: BranchName, refs: Vec) -> Self { + Self { + record_type: GIT_REFS_COLLECTION, + default_branch, + refs, + } + } + + pub fn body(&self) -> Result { + RecordBody::from_cbor(self) + } +} + +pub fn desired_record(repo: &Repo) -> Result, ChainError> { + let default_branch = repo + .default_branch() + .and_then(|name| { + name.as_str() + .strip_prefix("refs/heads/") + .and_then(|short| BranchName::new(short).ok()) + }) + .unwrap_or_else(|| { + BranchName::new(DEFAULT_DEFAULT_BRANCH) + .expect("the default default branch is a valid branch name") + }); + let mut served = repo.public_refs()?; + served.sort_by(|left, right| left.name.as_str().cmp(right.name.as_str())); + let refs = served + .iter() + .fold(String::new(), |mut out, record| { + out.push_str(record.target.to_hex().as_str()); + out.push(' '); + out.push_str(record.name.as_str()); + out.push('\n'); + out + }) + .into_bytes(); + let body = match RefsRecord::new(default_branch, refs).body() { + Ok(body) => body, + Err(RecordBodyError::Oversized(len)) => { + tracing::error!( + bytes = len, + "the repository's refs are over the record size limit, so it gets no refs record at all" + ); + return Ok(BTreeMap::new()); + } + Err(error) => return Err(ChainError::Malformed(error.to_string())), + }; + let key = MstKey::new(format!("{GIT_REFS_COLLECTION}/{SELF_RKEY}")) + .expect("the refs self key fits mst"); + Ok(BTreeMap::from([(key, body)])) +} diff --git a/knot2/crates/knot-record/src/grant.rs b/knot2/crates/knot-record/src/grant.rs new file mode 100644 index 0000000000000000000000000000000000000000..f23044ab72db0909fbda2e8d1c1fb7e38334ce45 --- /dev/null +++ b/knot2/crates/knot-record/src/grant.rs @@ -0,0 +1,169 @@ +use std::collections::BTreeMap; + +use knot_types::{AccountDid, DidRkey, MstKey, UnixSeconds}; +use serde::Serialize; + +use crate::{CreatedAt, RecordBody, encoded}; + +pub const MEMBER_COLLECTION: &str = "org.tangled.knot.member"; + +pub const COLLABORATOR_COLLECTION: &str = "org.tangled.graph.collaborator"; + +pub const BLOCK_COLLECTION: &str = "org.tangled.knot.block"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GrantCollection { + Member, + Collaborator, + Block, +} + +impl GrantCollection { + pub fn nsid(&self) -> &'static str { + match self { + Self::Member => MEMBER_COLLECTION, + Self::Collaborator => COLLABORATOR_COLLECTION, + Self::Block => BLOCK_COLLECTION, + } + } + + pub fn records(&self) -> knot_types::RecordCollection { + knot_types::RecordCollection::new(self.nsid()).expect("A grant collection is a valid nsid") + } + + pub fn surface(&self) -> &'static str { + match self { + Self::Member => "members", + Self::Collaborator => "collaborators", + Self::Block => "blocklist", + } + } +} + +#[derive(Serialize)] +struct GrantEncoding<'a> { + #[serde(rename = "$type")] + record_type: &'a str, + #[serde(rename = "createdAt")] + created_at: &'a CreatedAt, +} + +pub fn record_key(subject: &AccountDid, collection: GrantCollection) -> Result { + DidRkey::new(subject.as_str()) + .map_err(|error| error.to_string()) + .and_then(|rkey| { + MstKey::new(format!("{}/{}", collection.nsid(), rkey.as_str())) + .map_err(|error| error.to_string()) + }) +} + +pub fn bodies<'a>( + entries: impl IntoIterator, + collection: GrantCollection, +) -> BTreeMap { + entries + .into_iter() + .filter_map(|(subject, offered_at)| { + let key = record_key(subject, collection); + let Ok(key) = key else { + tracing::warn!( + subject = subject.as_str(), + "a subject that isn't valid as a record key is skipped, and its access \ + stays on the grant list" + ); + return None; + }; + let stamped = CreatedAt::at(offered_at) + .map_err(|error| error.to_string()) + .and_then(|created_at| { + encoded(&GrantEncoding { + record_type: collection.nsid(), + created_at: &created_at, + }) + .map_err(|error| error.to_string()) + }); + stamped + .map(|body| (key, body)) + .inspect_err(|error| { + tracing::warn!( + subject = subject.as_str(), + %error, + "an offer that won't render is skipped, and its access stays on the grant list" + ) + }) + .ok() + }) + .collect() +} + +#[cfg(test)] +mod tests { + + use super::*; + + fn subject(did: &str) -> AccountDid { + AccountDid::new(did).unwrap() + } + + fn stamped(did: &AccountDid, secs: i64) -> (&AccountDid, UnixSeconds) { + (did, UnixSeconds::new(secs)) + } + + #[test] + fn a_grant_list_entry_becomes_one_record_keyed_by_its_subject() { + let whelk = subject("did:plc:whelk"); + let rendered = bodies( + [stamped(&whelk, 1_700_000_000)], + GrantCollection::Collaborator, + ); + let (key, body) = rendered.iter().next().unwrap(); + assert_eq!(key.as_str(), "org.tangled.graph.collaborator/did:plc:whelk"); + let decoded: BTreeMap = + serde_ipld_dagcbor::from_slice(body.as_bytes()).unwrap(); + assert_eq!( + decoded, + BTreeMap::from([ + ("$type".into(), COLLABORATOR_COLLECTION.into()), + ("createdAt".into(), "2023-11-14T22:13:20Z".into()), + ]) + ); + assert_eq!( + bodies([stamped(&whelk, 1_700_000_000)], GrantCollection::Member), + bodies([stamped(&whelk, 1_700_000_000)], GrantCollection::Member), + "the same grant list entry renders the same bytes, so a re-sync is a no-op" + ); + assert_ne!( + bodies([stamped(&whelk, 1_800_000_000)], GrantCollection::Member), + bodies([stamped(&whelk, 1_700_000_000)], GrantCollection::Member), + "a re-offer over a new stamp renders new bytes" + ); + } + + #[test] + fn an_entry_that_cant_become_a_record_is_skipped_alone() { + let whelk = subject("did:plc:whelk"); + let encoded = bodies( + [ + stamped( + &subject(&format!("did:web:{}.oyster.cafe", "a".repeat(300))), + 1, + ), + stamped(&subject("did:web:knot.nel.pet:8080"), 2), + stamped(&whelk, i64::MIN), + stamped(&whelk, 1_700_000_000), + ], + GrantCollection::Member, + ); + assert_eq!( + encoded.len(), + 1, + "only the entry that can't become a record is skipped, and the rest of the grant \ + list still gets its records" + ); + assert!( + encoded + .keys() + .all(|key| key.as_str().ends_with(whelk.as_str())) + ); + } +} diff --git a/knot2/crates/knot-record/src/label.rs b/knot2/crates/knot-record/src/label.rs new file mode 100644 index 0000000000000000000000000000000000000000..d65c0882b1271013ed546974d334601ed57eece6 --- /dev/null +++ b/knot2/crates/knot-record/src/label.rs @@ -0,0 +1,200 @@ +use std::fmt; +use std::str::FromStr; + +use crate::{CreatedAt, RecordError, encoded}; +use jacquard_common::deps::codegen::unicode_segmentation::UnicodeSegmentation; +use knot_types::{AccountDid, LabelKey, RecordBody, RecordBodyError, UnixSeconds}; +use serde::{Deserialize, Serialize, Serializer}; + +pub const LABEL_DEF_COLLECTION: &str = "org.tangled.label.definition"; + +collection!( + label_def_collection, + LABEL_DEF_COLLECTION, + "Label def collection is valid NSID" +); + +#[derive(Debug, thiserror::Error)] +pub enum LabelError { + #[error("Label name is empty")] + EmptyName, + #[error("Label name of {0} graphemes is over the {MAX_NAME_GRAPHEMES} grapheme limit")] + NameTooLong(usize), + #[error("Label name of {0} bytes is over the {MAX_NAME_BYTES} byte limit")] + NameTooWide(usize), + #[error("Color must read as a hex value like #79ffe1, of three, four, six or eight digits")] + Color, +} + +pub const MAX_NAME_GRAPHEMES: usize = 64; + +pub const MAX_NAME_BYTES: usize = 640; + +fn parse_name(name: impl Into) -> Result { + let name = name.into(); + match ( + UnicodeSegmentation::graphemes(name.as_str(), true).count(), + name.len(), + ) { + (0, _) => Err(LabelError::EmptyName), + (_, wide) if wide > MAX_NAME_BYTES => Err(LabelError::NameTooWide(wide)), + (many, _) if many > MAX_NAME_GRAPHEMES => Err(LabelError::NameTooLong(many)), + _ => Ok(LabelName(name)), + } +} + +knot_types::string_id!(LabelName, try parse_name, err LabelError); + +fn parse_color(color: impl Into) -> Result { + let color = color.into(); + let hex = matches!(color.len(), 4 | 5 | 7 | 9) + && color.starts_with('#') + && color[1..].bytes().all(|byte| byte.is_ascii_hexdigit()); + match hex { + true => Ok(LabelColor(color.to_ascii_uppercase())), + false => Err(LabelError::Color), + } +} + +knot_types::string_id!(LabelColor, try parse_color, err LabelError); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LabelDef { + name: LabelName, + color: Option, +} + +impl LabelDef { + pub fn new(name: LabelName, color: Option) -> Self { + Self { name, color } + } + + pub fn name(&self) -> &LabelName { + &self.name + } + + pub fn color(&self) -> Option<&LabelColor> { + self.color.as_ref() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LabelDefRecord { + def: LabelDef, + created_at: CreatedAt, + editor: AccountDid, +} + +impl LabelDefRecord { + pub fn new( + def: LabelDef, + created_at: UnixSeconds, + editor: AccountDid, + ) -> Result { + Ok(Self { + def, + created_at: CreatedAt::at(created_at)?, + editor, + }) + } + + pub fn def(&self) -> &LabelDef { + &self.def + } + + pub fn body(&self) -> Result { + encoded(self) + } +} + +#[derive(Serialize)] +struct LabelDefEncoding<'a> { + name: &'a LabelName, + #[serde(skip_serializing_if = "Option::is_none")] + color: &'a Option, + #[serde(rename = "$type")] + record_type: knot_types::RecordCollection, + #[serde(rename = "createdAt")] + created_at: &'a CreatedAt, + #[serde(rename = "x-tngl-editor")] + editor: &'a AccountDid, +} + +impl Serialize for LabelDefRecord { + fn serialize(&self, serializer: S) -> Result { + LabelDefEncoding { + name: &self.def.name, + color: &self.def.color, + record_type: label_def_collection(), + created_at: &self.created_at, + editor: &self.editor, + } + .serialize(serializer) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LabelToken { + key: LabelKey, +} + +impl LabelToken { + pub fn new(key: LabelKey) -> Self { + Self { key } + } + + pub fn key(&self) -> &LabelKey { + &self.key + } +} + +#[cfg(test)] +mod tests { + use super::*; + use knot_types::UnixSeconds; + + fn editor() -> AccountDid { + AccountDid::new("did:plc:3fwecdnvtcscjnrx2p4n7alz").unwrap() + } + + #[test] + fn label_def_record_embeds_name_color_and_editor() { + let record = LabelDefRecord::new( + LabelDef::new( + LabelName::new("wontfix").unwrap(), + Some(LabelColor::new("#79ffe1").unwrap()), + ), + UnixSeconds::new(1_755_090_096), + editor(), + ) + .unwrap(); + let value: serde_json::Value = + serde_ipld_dagcbor::from_slice(record.body().unwrap().as_bytes()).unwrap(); + assert_eq!(value["$type"], LABEL_DEF_COLLECTION); + assert_eq!(value["name"], "wontfix"); + assert_eq!(value["color"], "#79FFE1"); + assert_eq!(value["x-tngl-editor"], editor().as_str()); + } + + #[test] + fn label_token_serializes_as_a_key() { + let token = LabelToken::new(LabelKey::new("wont-fix.2").unwrap()); + assert_eq!( + serde_json::to_value(token).unwrap(), + serde_json::json!({ "key": "wont-fix.2" }) + ); + } + + #[test] + fn label_name_must_say_something_and_not_too_much() { + assert!(matches!(LabelName::new(""), Err(LabelError::EmptyName))); + assert!(matches!( + LabelName::new("k".repeat(MAX_NAME_GRAPHEMES + 1)), + Err(LabelError::NameTooLong(_)) + )); + assert!(matches!( + LabelName::new("👨‍👩‍👧‍👦".repeat(MAX_NAME_BYTES / 25 + 1)), + Err(LabelError::NameTooWide(_)) + )); + } +} diff --git a/knot2/crates/knot-record/src/lib.rs b/knot2/crates/knot-record/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..ff86712a56bcc23219b1b76563db4be5c3040eed --- /dev/null +++ b/knot2/crates/knot-record/src/lib.rs @@ -0,0 +1,294 @@ +macro_rules! collection { + ($accessor:ident, $name:ident, $valid:literal) => { + pub fn $accessor() -> knot_types::RecordCollection { + knot_types::RecordCollection::new($name).expect($valid) + } + }; +} + +pub mod artifact; +pub mod attachment; +pub mod blocks; +pub mod chain; +pub mod comment; +pub mod git_refs; +pub mod grant; +pub mod label; +pub mod manifest; +pub mod mst; +pub mod proof; +pub mod pull; +pub mod reaction; +pub mod ticket; + +use crate::ticket::{MarkdownBody, TicketTitle}; +use bytes::Bytes; +use cid::Cid; +use knot_types::{RecordBody, RecordBodyError, RecordCid, UnixSeconds}; +use serde::{Deserialize, Serialize, Serializer}; +#[derive(Debug, thiserror::Error)] +pub enum RecordError { + #[error("atproto block error: {0}")] + Block(#[from] jacquard_repo::RepoError), + #[error("covering proof failed to invert the commit cold")] + ColdInversion, + #[error("malformed atproto record: {0}")] + Malformed(String), + #[error("createdAt {0} is outside the RFC 3339 range")] + CreatedAtUnrenderable(i64), +} + +pub type Result = std::result::Result; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct CreatedAt(String); + +impl CreatedAt { + pub fn at(time: UnixSeconds) -> Result { + chrono::DateTime::from_timestamp(time.get(), 0) + .map(|at| Self(at.to_rfc3339_opts(chrono::SecondsFormat::Secs, true))) + .ok_or(RecordError::CreatedAtUnrenderable(time.get())) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct CidString(RecordCid); + +impl From for CidString { + fn from(cid: RecordCid) -> Self { + Self(cid) + } +} + +impl Serialize for CidString { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(&self.0) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Prose { + pub(crate) title: TicketTitle, + pub(crate) body: Option, +} + +impl Prose { + pub fn new(title: TicketTitle, body: Option) -> Self { + Self { title, body } + } +} + +#[cfg(test)] +pub(crate) fn dag_cbor(value: &T) -> Vec { + serde_ipld_dagcbor::to_vec(value).expect("Record text fields and lists encode as dag-cbor") +} + +pub(crate) fn encoded(value: &T) -> Result { + RecordBody::from_cbor(value) +} + +#[cfg(test)] +pub(crate) fn frozen(hex: &str) -> Vec { + hex.as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect() +} + +#[cfg(test)] +pub(crate) mod test_support { + use crate::comment::CommentRef; + use knot_types::{AccountDid, RecordAddress, RecordCid, RecordRkey, RepoDid}; + + pub(crate) fn editor() -> AccountDid { + AccountDid::new("did:plc:3fwecdnvtcscjnrx2p4n7alz").unwrap() + } + + pub(crate) fn squid() -> RepoDid { + RepoDid::new("did:plc:squid").unwrap() + } + + pub(crate) fn subject() -> CommentRef { + CommentRef::new( + RecordAddress::new( + crate::ticket::ticket_collection(), + RecordRkey::new("3lubrptx57d22").unwrap(), + ), + RecordCid::new(knot_types::dag_cbor_cid(b"subject")).unwrap(), + ) + } +} + +pub fn cid_of(block: &[u8]) -> Cid { + knot_types::dag_cbor_cid(block) +} + +pub fn car_bytes(root: Cid, blocks: &std::collections::BTreeMap) -> Vec { + let header = iroh_car::CarHeader::new_v1(vec![root]); + let header_bytes = header.encode().expect("A CAR header encodes"); + let mut out = Vec::new(); + varint(&mut out, header_bytes.len()); + out.extend_from_slice(&header_bytes); + blocks.iter().for_each(|(cid, data)| { + let cid_bytes = cid.to_bytes(); + varint(&mut out, cid_bytes.len() + data.len()); + out.extend_from_slice(&cid_bytes); + out.extend_from_slice(data); + }); + out +} + +fn varint(out: &mut Vec, value: usize) { + let chunks = std::iter::successors(Some(value), |value| (*value >= 0x80).then(|| value >> 7)); + out.extend(chunks.map(|value| ((value & 0x7f) as u8) | u8::from(value >= 0x80) << 7)); +} + +#[derive(Debug, thiserror::Error)] +pub enum CarError { + #[error("CAR is cut short at byte {0}")] + Truncated(usize), + #[error("CAR varint at byte {0} doesn't fit a length")] + Varint(usize), + #[error("CAR header doesn't decode: {0}")] + Header(String), + #[error("CAR section at byte {at} is missing its CID: {reason}")] + Cid { at: usize, reason: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Car { + pub roots: Vec, + pub blocks: std::collections::BTreeMap, +} + +struct Varint { + value: usize, + after: usize, +} + +fn read_varint(bytes: &[u8], at: usize) -> Result { + let tail = bytes.get(at..).ok_or(CarError::Truncated(at))?; + let last = tail + .iter() + .take(10) + .position(|byte| byte & 0x80 == 0) + .ok_or(CarError::Truncated(at))?; + let value = tail[..=last] + .iter() + .enumerate() + .try_fold(0usize, |value, (index, byte)| { + let part = usize::from(byte & 0x7f); + let shift = (7 * index) as u32; + part.checked_shl(shift) + .filter(|shifted| shifted >> shift == part) + .map(|shifted| value | shifted) + }) + .ok_or(CarError::Varint(at))?; + Ok(Varint { + value, + after: at + last + 1, + }) +} + +pub fn car_blocks(bytes: &[u8]) -> Result { + let framed = read_varint(bytes, 0)?; + let header = bytes + .get(framed.after..framed.after + framed.value) + .ok_or(CarError::Truncated(framed.after))?; + let roots = iroh_car::CarHeader::decode(header) + .map_err(|error| CarError::Header(error.to_string()))? + .roots() + .to_vec(); + let mut at = framed.after + framed.value; + let blocks = std::iter::from_fn(|| { + (at < bytes.len()).then(|| { + let framed = read_varint(bytes, at)?; + let section = bytes + .get(framed.after..framed.after + framed.value) + .ok_or(CarError::Truncated(framed.after))?; + let mut cursor = std::io::Cursor::new(§ion); + let cid = Cid::read_bytes(&mut cursor).map_err(|error| CarError::Cid { + at: framed.after, + reason: error.to_string(), + })?; + at = framed.after + framed.value; + Ok(( + cid, + Bytes::copy_from_slice(§ion[cursor.position() as usize..]), + )) + }) + }) + .collect::, CarError>>()?; + Ok(Car { roots, blocks }) +} + +pub struct CommitSigner<'a>(&'a dyn knot_runtime::Signer); + +impl<'a> CommitSigner<'a> { + pub fn new(signer: &'a dyn knot_runtime::Signer) -> Self { + Self(signer) + } +} + +impl jacquard_repo::commit::SigningKey for CommitSigner<'_> { + fn sign_bytes(&self, data: &[u8]) -> jacquard_repo::Result { + Ok(Bytes::from(self.0.sign(data).as_bytes().to_vec())) + } + + fn public_key(&self) -> Vec { + self.0.public_key().as_bytes().to_vec() + } +} +#[cfg(test)] +mod car_tests { + use super::*; + use std::collections::BTreeMap; + + #[tokio::test] + async fn the_sync_car_writer_matches_jacquards_byte_for_byte() { + let one = cid_of(b"a record body"); + let two = cid_of(b"another record body"); + let root = cid_of(b"a commit"); + let blocks = BTreeMap::from([ + (one, Bytes::from_static(b"a record body")), + (two, Bytes::from_static(b"another record body")), + (root, Bytes::from_static(b"a commit")), + ]); + assert_eq!( + car_bytes(root, &blocks), + jacquard_repo::car::write_car_bytes(root, blocks.clone()) + .await + .unwrap() + ); + } + + #[test] + fn car_reader_inverts_car_writer() { + let one = cid_of(b"a record body"); + let root = cid_of(b"a commit"); + let blocks = BTreeMap::from([ + (one, Bytes::from_static(b"a record body")), + (root, Bytes::from_static(b"a commit")), + ]); + let written = car_bytes(root, &blocks); + assert_eq!( + car_blocks(&written).unwrap(), + Car { + roots: vec![root], + blocks: blocks.clone() + } + ); + assert_eq!( + car_bytes(root, &car_blocks(&written).unwrap().blocks), + written, + "A read then written CAR is the same bytes" + ); + assert!(matches!( + car_blocks(&written[..written.len() - 3]), + Err(CarError::Truncated(_)) + )); + } +} diff --git a/knot2/crates/knot-record/src/manifest.rs b/knot2/crates/knot-record/src/manifest.rs new file mode 100644 index 0000000000000000000000000000000000000000..549c8d675ee66d8cc7c234cccfc746fd1a08daf1 --- /dev/null +++ b/knot2/crates/knot-record/src/manifest.rs @@ -0,0 +1,270 @@ +use std::fmt; +use std::str::FromStr; + +use jacquard_common::deps::codegen::unicode_segmentation::UnicodeSegmentation; +use knot_types::{ + AccountDid, ContributionPolicy, OwnerDid, RecordBody, RecordBodyError, RepoDid, RepoName, + ServiceDid, UnixSeconds, +}; +use serde::{Deserialize, Serialize, Serializer}; +use url::Url; + +use crate::{CreatedAt, RecordError, encoded}; + +pub const MANIFEST_COLLECTION: &str = "org.tangled.repo.manifest"; + +pub const MANIFEST_RKEY: &str = "self"; + +pub const SOURCE_TANGLED: &str = "org.tangled.repo.defs#sourceTangled"; + +pub const SOURCE_EXTERNAL: &str = "org.tangled.repo.defs#sourceExternal"; + +collection!( + manifest_collection, + MANIFEST_COLLECTION, + "Manifest collection is valid NSID" +); + +pub const MAX_DESCRIPTION_GRAPHEMES: usize = 140; + +pub const MAX_TOPIC_BYTES: usize = 50; + +pub const MAX_TOPICS: usize = 50; + +#[derive(Debug, thiserror::Error)] +pub enum ManifestError { + #[error("Description is empty")] + EmptyDescription, + #[error("Description of {0} graphemes is over the {MAX_DESCRIPTION_GRAPHEMES} grapheme limit")] + DescriptionTooLong(usize), + #[error("Topic of {0} bytes is over the {MAX_TOPIC_BYTES} byte limit")] + TopicTooLong(usize), + #[error("{0} topics are over the {MAX_TOPICS} limit")] + TooManyTopics(usize), +} + +fn parse_description(description: impl Into) -> Result { + let description = description.into(); + match UnicodeSegmentation::graphemes(description.as_str(), true).count() { + 0 => Err(ManifestError::EmptyDescription), + many if many > MAX_DESCRIPTION_GRAPHEMES => Err(ManifestError::DescriptionTooLong(many)), + _ => Ok(RepoDescription(description)), + } +} + +knot_types::string_id!(RepoDescription, try parse_description, err ManifestError); + +fn parse_topic(topic: impl Into) -> Result { + let topic = topic.into(); + match topic.len() { + wide if wide > MAX_TOPIC_BYTES => Err(ManifestError::TopicTooLong(wide)), + _ => Ok(RepoTopic(topic)), + } +} + +knot_types::string_id!(RepoTopic, try parse_topic, err ManifestError); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(transparent)] +pub struct RepoTopics(Vec); + +impl RepoTopics { + pub fn new(topics: Vec) -> Result { + match topics.len() { + many if many > MAX_TOPICS => Err(ManifestError::TooManyTopics(many)), + _ => Ok(Self(topics)), + } + } + + pub fn as_slice(&self) -> &[RepoTopic] { + &self.0 + } +} + +impl<'de> Deserialize<'de> for RepoTopics { + fn deserialize>(deserializer: D) -> Result { + Self::new(Vec::deserialize(deserializer)?).map_err(serde::de::Error::custom) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "$type")] +pub enum ManifestSource { + #[serde(rename = "org.tangled.repo.defs#sourceTangled")] + Tangled { did: RepoDid }, + #[serde(rename = "org.tangled.repo.defs#sourceExternal")] + External { uri: Url }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ManifestMeta { + pub spindle: Option, + pub description: Option, + pub website: Option, + pub topics: Option, + pub source: Option, + pub contribution_policy: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManifestRecord { + owner: OwnerDid, + name: RepoName, + meta: ManifestMeta, + created_at: CreatedAt, + editor: AccountDid, +} + +impl ManifestRecord { + pub fn new( + owner: OwnerDid, + name: RepoName, + meta: ManifestMeta, + created_at: UnixSeconds, + editor: AccountDid, + ) -> Result { + Ok(Self { + owner, + name, + meta, + created_at: CreatedAt::at(created_at)?, + editor, + }) + } + + pub fn body(&self) -> Result { + encoded(self) + } +} + +#[derive(Serialize)] +struct DeclarationEncoding<'a> { + owner: &'a OwnerDid, + name: &'a RepoName, +} + +#[derive(Serialize)] +struct ManifestEncoding<'a> { + #[serde(rename = "$type")] + record_type: knot_types::RecordCollection, + declaration: DeclarationEncoding<'a>, + #[serde(skip_serializing_if = "Option::is_none")] + spindle: &'a Option, + #[serde(skip_serializing_if = "Option::is_none")] + description: &'a Option, + #[serde(skip_serializing_if = "Option::is_none")] + website: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + topics: &'a Option, + #[serde(skip_serializing_if = "Option::is_none")] + source: &'a Option, + #[serde(rename = "contributionPolicy", skip_serializing_if = "Option::is_none")] + contribution_policy: Option<&'a ContributionPolicy>, + #[serde(rename = "createdAt")] + created_at: &'a CreatedAt, + #[serde(rename = "x-tngl-editor")] + editor: &'a AccountDid, +} + +impl Serialize for ManifestRecord { + fn serialize(&self, serializer: S) -> Result { + ManifestEncoding { + record_type: manifest_collection(), + declaration: DeclarationEncoding { + owner: &self.owner, + name: &self.name, + }, + spindle: &self.meta.spindle, + description: &self.meta.description, + website: self.meta.website.as_ref().map(Url::as_str), + topics: &self.meta.topics, + source: &self.meta.source, + contribution_policy: self.meta.contribution_policy.as_ref(), + created_at: &self.created_at, + editor: &self.editor, + } + .serialize(serializer) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn owner() -> AccountDid { + AccountDid::new("did:plc:3fwecdnvtcscjnrx2p4n7alz").unwrap() + } + + fn decoded(record: &ManifestRecord) -> serde_json::Value { + serde_ipld_dagcbor::from_slice(record.body().unwrap().as_bytes()).unwrap() + } + + #[test] + fn manifest_record_embeds_declaration_meta_and_editor() { + let record = ManifestRecord::new( + OwnerDid::new(owner().as_str()).unwrap(), + RepoName::new("Anemone").unwrap(), + ManifestMeta { + description: Some(RepoDescription::new("tidepool").unwrap()), + website: Some(Url::parse("https://tangled.org").unwrap()), + topics: Some(RepoTopics::new(vec![RepoTopic::new("git").unwrap()]).unwrap()), + source: Some(ManifestSource::Tangled { + did: RepoDid::new("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa").unwrap(), + }), + ..ManifestMeta::default() + }, + UnixSeconds::new(1_755_090_096), + owner(), + ) + .unwrap(); + let value = decoded(&record); + assert_eq!(value["$type"], MANIFEST_COLLECTION); + assert_eq!(value["declaration"]["owner"], owner().as_str()); + assert_eq!(value["declaration"]["name"], "Anemone"); + assert_eq!(value["description"], "tidepool"); + assert_eq!(value["website"], "https://tangled.org/"); + assert_eq!(value["topics"], serde_json::json!(["git"])); + assert_eq!(value["source"]["$type"], SOURCE_TANGLED); + assert_eq!(value["source"]["did"], "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"); + assert_eq!(value["createdAt"], "2025-08-13T13:01:36Z"); + assert_eq!(value["x-tngl-editor"], owner().as_str()); + assert!(value.get("spindle").is_none()); + } + + #[test] + fn manifest_meta_holds_to_lexicon_limits() { + assert!(matches!( + RepoDescription::new(""), + Err(ManifestError::EmptyDescription) + )); + assert!(RepoDescription::new("e\u{301}".repeat(MAX_DESCRIPTION_GRAPHEMES)).is_ok()); + assert!(matches!( + RepoDescription::new("e\u{301}".repeat(MAX_DESCRIPTION_GRAPHEMES + 1)), + Err(ManifestError::DescriptionTooLong(_)) + )); + assert!(RepoTopic::new("").is_ok()); + assert!(matches!( + RepoTopic::new("é".repeat(MAX_TOPIC_BYTES / 2 + 1)), + Err(ManifestError::TopicTooLong(_)) + )); + assert!( + serde_json::from_value::(serde_json::json!(vec!["git"; MAX_TOPICS + 1])) + .is_err() + ); + } + + #[test] + fn external_source_round_trips_by_type() { + let source: ManifestSource = serde_json::from_value(serde_json::json!({ + "$type": SOURCE_EXTERNAL, + "uri": "https://github.com/tangled/core", + })) + .unwrap(); + assert_eq!( + source, + ManifestSource::External { + uri: Url::parse("https://github.com/tangled/core").unwrap() + } + ); + } +} diff --git a/knot2/crates/knot-record/src/mst.rs b/knot2/crates/knot-record/src/mst.rs new file mode 100644 index 0000000000000000000000000000000000000000..023e8387495935766789fe1d46f2f9e522721e2b --- /dev/null +++ b/knot2/crates/knot-record/src/mst.rs @@ -0,0 +1,119 @@ +use std::collections::BTreeMap; +use std::sync::Arc; + +use bytes::Bytes; +use cid::Cid; +use futures::StreamExt; +use jacquard_repo::Mst; +use jacquard_repo::storage::MemoryBlockStore; + +use crate::{RecordError, Result}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Tree { + pub root: Cid, + pub nodes: BTreeMap, +} + +pub fn empty() -> Mst { + Mst::new(Arc::new(MemoryBlockStore::new())) +} + +async fn grow( + tree: Mst, + pairs: &BTreeMap, +) -> Result> { + futures::stream::iter(pairs) + .fold(Ok(tree), |acc, (key, cid)| async move { + acc?.add(key, *cid).await.map_err(RecordError::Block) + }) + .await +} + +pub async fn build(pairs: &BTreeMap) -> Result { + let grown = grow(empty(), pairs).await?; + let (root, nodes) = grown.collect_blocks().await.map_err(RecordError::Block)?; + Ok(Tree { root, nodes }) +} + +pub async fn tree_of(pairs: &BTreeMap) -> Result> { + grow(empty(), pairs).await +} + +pub struct Applied { + pub tree: Mst, + pub diff: jacquard_repo::MstDiff, +} + +pub async fn apply( + before: &Mst, + ops: &[jacquard_repo::mst::VerifiedWriteOp], +) -> Result> { + let after = before.batch(ops).await?; + let diff = before.diff(&after).await?; + Ok(Applied { tree: after, diff }) +} + +pub async fn root( + tree: &Mst, +) -> Result { + tree.root().await.map_err(RecordError::Block) +} + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_repo::mst::node::NodeData; + + fn keyed(entries: &[&str]) -> BTreeMap { + entries + .iter() + .map(|key| ((*key).to_owned(), crate::cid_of(b"leaf"))) + .collect() + } + + fn key_at_depth(collection: &str, depth: usize, from: u64) -> String { + (from..) + .map(|n| format!("{collection}/{n}")) + .find(|key| jacquard_repo::mst::util::layer_for_key(key) == depth) + .expect("a sha-256 depth class is reachable by enumeration") + } + + fn pass_through(tree: &Tree) -> usize { + tree.nodes + .values() + .filter(|bytes| { + let node = serde_ipld_dagcbor::from_slice::(bytes).unwrap(); + node.left.is_some() && node.entries.is_empty() + }) + .count() + } + + #[tokio::test] + async fn the_builder_reproduces_the_root_from_the_pairs_alone() { + let keys = keyed(&["alpha/1", "beta/2", "gamma/3", "delta/4"]); + assert_eq!(build(&keys).await.unwrap(), build(&keys).await.unwrap()); + } + + #[tokio::test] + async fn the_builder_emits_pass_through_nodes_at_skipped_layers() { + let shallow = keyed(&[ + &key_at_depth("a", 0, 0), + &key_at_depth("k", 4, 0), + &key_at_depth("z", 0, 0), + ]); + assert!( + pass_through(&build(&shallow).await.unwrap()) >= 1, + "a builder that skips empty layers computes the wrong root" + ); + let deep = keyed(&[ + &key_at_depth("aa", 0, 0), + &key_at_depth("d8", 8, 0), + &key_at_depth("zz", 0, 0), + ]); + assert!( + pass_through(&build(&deep).await.unwrap()) >= 7, + "a depth-eight key hangs below seven skipped layers" + ); + } +} diff --git a/knot2/crates/knot-record/src/proof.rs b/knot2/crates/knot-record/src/proof.rs new file mode 100644 index 0000000000000000000000000000000000000000..05515d0b95448d58799340895426038e0bcb53d2 --- /dev/null +++ b/knot2/crates/knot-record/src/proof.rs @@ -0,0 +1,171 @@ +use bytes::Bytes; +use cid::Cid; +use futures::{StreamExt, TryStreamExt}; +use jacquard_repo::mst::{NodeEntry, VerifiedWriteOp}; +use jacquard_repo::storage::{BlockStore, MemoryBlockStore}; +use jacquard_repo::{Mst, MstDiff}; +use std::collections::{BTreeMap, BTreeSet}; +use std::pin::Pin; + +use crate::{RecordError, Result}; +use knot_types::MstKey; + +pub async fn covering_blocks( + before: &Mst, + after: &Mst, + diff: &MstDiff, +) -> Result> { + let covered = futures::stream::iter(mutated_keys(diff)) + .then(|key| async move { + let (below_before, above_before) = around(before.clone(), &key, None, None).await?; + let (below_after, above_after) = around(after.clone(), &key, None, None).await?; + Ok::<[Option; 3], RecordError>([ + below_before.max(below_after), + nearest(above_before, above_after), + Some(key), + ]) + }) + .try_collect::>() + .await? + .into_iter() + .flatten() + .flatten() + .collect::>(); + futures::stream::iter(covered.into_iter().map(Ok::<_, RecordError>)) + .try_fold(diff.new_mst_blocks.clone(), |mut blocks, key| async move { + after + .blocks_for_path(key.as_str(), &mut blocks) + .await + .map_err(RecordError::Block)?; + Ok(blocks) + }) + .await +} + +pub async fn cold_invert( + blocks: &BTreeMap, + after_root: Cid, + ops: &[VerifiedWriteOp], +) -> Result { + let storage = std::sync::Arc::new(MemoryBlockStore::new_from_blocks(blocks.clone())); + let mut tree = Mst::load(storage, after_root, None); + futures::stream::iter(ops.iter().rev()) + .fold(Ok(&mut tree), |acc, op| async move { + let acc = acc?; + acc.invert_op(op.clone()) + .await + .map_err(RecordError::Block) + .and_then(|inverted| inverted.then_some(acc).ok_or(RecordError::ColdInversion)) + }) + .await?; + tree.root().await.map_err(RecordError::Block) +} + +fn mutated_keys(diff: &MstDiff) -> BTreeSet { + diff.creates + .iter() + .map(|(key, _)| key.as_str()) + .chain(diff.updates.iter().map(|(key, _, _)| key.as_str())) + .chain(diff.deletes.iter().map(|(key, _)| key.as_str())) + .flat_map(|key| MstKey::new(key.to_owned())) + .collect() +} + +fn nearest(left: Option, right: Option) -> Option { + match (left, right) { + (Some(left), Some(right)) => Some(left.min(right)), + (left, right) => left.or(right), + } +} + +fn checked(value: Option<&str>) -> Result> { + value + .map(|key| MstKey::new(key.to_owned())) + .transpose() + .map_err(|error| RecordError::Malformed(error.to_string())) +} + +type Picker = for<'e> fn(&'e [NodeEntry]) -> Option<&'e NodeEntry>; + +fn edge( + tree: Mst, + pick: Picker, +) -> Pin>> + Send>> { + Box::pin(async move { + let entries = entries_of(&tree).await?; + match pick(&entries) { + Some(NodeEntry::Tree(under)) => edge(under.clone(), pick).await, + Some(entry) => checked(entry.leaf_key()), + None => Ok(None), + } + }) +} + +async fn toward( + entry: Option<&NodeEntry>, + inherited: Option, + pick: Picker, +) -> Result> { + match entry { + Some(NodeEntry::Tree(under)) => edge(under.clone(), pick).await, + Some(entry) => checked(entry.leaf_key()), + None => Ok(inherited), + } +} + +type NearestKeys<'a> = + Pin, Option)>> + Send + 'a>>; + +fn around<'a, S: BlockStore + Sync + 'static>( + tree: Mst, + key: &'a MstKey, + below: Option, + above: Option, +) -> NearestKeys<'a> { + Box::pin(async move { + let entries = entries_of(&tree).await?; + let at = entries + .iter() + .position(|entry| entry.leaf_key().is_some_and(|leaf| leaf >= key.as_str())) + .unwrap_or(entries.len()); + let here = entries.get(at).and_then(NodeEntry::leaf_key); + let left = at.checked_sub(1).and_then(|index| entries.get(index)); + let higher = match here.filter(|leaf| *leaf != key.as_str()) { + Some(leaf) => checked(Some(leaf))?.or(above), + None => above, + }; + if here == Some(key.as_str()) { + return Ok(( + toward(left, below, <[NodeEntry]>::last).await?, + toward(entries.get(at + 1), higher, <[NodeEntry]>::first).await?, + )); + } + match left { + Some(NodeEntry::Tree(under)) => { + let lower = checked( + at.checked_sub(2) + .and_then(|index| entries.get(index)) + .and_then(NodeEntry::leaf_key), + )? + .or(below); + around(under.clone(), key, lower, higher).await + } + left => Ok(( + checked(left.and_then(NodeEntry::leaf_key))?.or(below), + higher, + )), + } + }) +} + +async fn entries_of(tree: &Mst) -> Result>> { + futures::stream::unfold(0isize, |index| async move { + tree.at_index(index) + .await + .map_err(RecordError::Block) + .transpose() + .map(|entry| (entry, index + 1)) + }) + .try_collect() + .await +} diff --git a/knot2/crates/knot-record/src/pull.rs b/knot2/crates/knot-record/src/pull.rs new file mode 100644 index 0000000000000000000000000000000000000000..e06f9bfda0ca0874c9ddc6a9e6dbc7ff79c5c3d0 --- /dev/null +++ b/knot2/crates/knot-record/src/pull.rs @@ -0,0 +1,173 @@ +use knot_git::CommitRange; +use knot_types::{ + AccountDid, BranchName, Oid, RecordBody, RecordBodyError, RecordRkey, RepoDid, UnixSeconds, +}; +use serde::{Deserialize, Serialize, Serializer}; + +use crate::{CreatedAt, RecordError, encoded}; + +pub const PATCH_COLLECTION: &str = "org.tangled.review.patch"; + +collection!( + patch_collection, + PATCH_COLLECTION, + "Patch collection is valid nsid" +); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PatchSource { + pub repo: RepoDid, + pub branch: BranchName, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PatchRecord { + ticket: RecordRkey, + range: CommitRange, + source: Option, + created_at: CreatedAt, + editor: AccountDid, +} + +impl PatchRecord { + pub fn new( + ticket: RecordRkey, + range: CommitRange, + source: Option, + created_at: UnixSeconds, + editor: AccountDid, + ) -> Result { + Ok(Self { + ticket, + range, + source, + created_at: CreatedAt::at(created_at)?, + editor, + }) + } + + pub fn body(&self) -> Result { + encoded(self) + } +} + +#[derive(Serialize)] +struct OidEncoding<'a> { + oid: &'a Oid, +} + +#[derive(Serialize)] +struct PatchEncoding<'a> { + #[serde(rename = "$type")] + record_type: knot_types::RecordCollection, + ticket: &'a RecordRkey, + base: OidEncoding<'a>, + head: OidEncoding<'a>, + #[serde(skip_serializing_if = "Option::is_none")] + source: Option<&'a PatchSource>, + #[serde(rename = "createdAt")] + created_at: &'a CreatedAt, + #[serde(rename = "x-tngl-editor")] + editor: &'a AccountDid, +} + +impl Serialize for PatchRecord { + fn serialize(&self, serializer: S) -> Result { + PatchEncoding { + record_type: patch_collection(), + ticket: &self.ticket, + base: OidEncoding { + oid: &self.range.base, + }, + head: OidEncoding { + oid: &self.range.head, + }, + source: self.source.as_ref(), + created_at: &self.created_at, + editor: &self.editor, + } + .serialize(serializer) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::frozen; + + const PATCH: &str = "a66462617365a1636f69647828616161616161616161616161616161616161616161616161616161616161616161616161616161616468656164a1636f696478286262626262626262626262626262626262626262626262626262626262626262626262626262626265247479706578186f72672e74616e676c65642e7265766965772e7061746368667469636b65746d336c75627270747835376432326963726561746564417474313937302d30312d30315430303a30303a30315a6d782d746e676c2d656469746f7278206469643a706c633a3366776563646e76746373636a6e72783270346e37616c7a"; + + fn editor() -> AccountDid { + AccountDid::new("did:plc:3fwecdnvtcscjnrx2p4n7alz").unwrap() + } + + fn oid(byte: u8) -> Oid { + Oid::from_hex(&(byte as char).to_string().repeat(40)).unwrap() + } + + fn range() -> CommitRange { + CommitRange { + base: oid(b'a'), + head: oid(b'b'), + } + } + + #[test] + fn patch_record_bytes_frozen() { + let record = PatchRecord::new( + RecordRkey::new("3lubrptx57d22").unwrap(), + range(), + None, + UnixSeconds::new(1), + editor(), + ) + .unwrap(); + let body = record.body().unwrap(); + assert_eq!( + body.as_bytes(), + frozen(PATCH).as_slice(), + "the first emitted patch must freeze these bytes: whole seconds, utc z" + ); + } + + #[test] + fn patch_embeds_ticket_and_oid_pair() { + let record = PatchRecord::new( + RecordRkey::new("3lubrptx57d22").unwrap(), + range(), + None, + UnixSeconds::new(1), + editor(), + ) + .unwrap(); + let value: serde_json::Value = + serde_ipld_dagcbor::from_slice(record.body().unwrap().as_bytes()).unwrap(); + assert_eq!(value["$type"], PATCH_COLLECTION); + assert_eq!(value["ticket"], "3lubrptx57d22"); + assert_eq!(value["base"]["oid"], "a".repeat(40)); + assert_eq!(value["head"]["oid"], "b".repeat(40)); + assert_eq!(value["x-tngl-editor"], editor().as_str()); + assert!(value.get("source").is_none()); + } + + #[test] + fn patch_embeds_source() { + let source = PatchSource { + repo: RepoDid::new("did:plc:3fwecdnvtcscjnrx2p4n7alz").unwrap(), + branch: BranchName::new("feat/kelp").unwrap(), + }; + let record = PatchRecord::new( + RecordRkey::new("3lubrptx57d22").unwrap(), + range(), + Some(source), + UnixSeconds::new(1), + editor(), + ) + .unwrap(); + let value: serde_json::Value = + serde_ipld_dagcbor::from_slice(record.body().unwrap().as_bytes()).unwrap(); + assert_eq!(value["source"]["repo"], "did:plc:3fwecdnvtcscjnrx2p4n7alz"); + assert_eq!(value["source"]["branch"], "feat/kelp"); + } +} diff --git a/knot2/crates/knot-record/src/reaction.rs b/knot2/crates/knot-record/src/reaction.rs new file mode 100644 index 0000000000000000000000000000000000000000..6786b35448db62d61fff52a8e07027221347077b --- /dev/null +++ b/knot2/crates/knot-record/src/reaction.rs @@ -0,0 +1,175 @@ +use knot_types::{AccountDid, RecordBody, RecordBodyError, RepoDid, UnixSeconds}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use unicode_segmentation::UnicodeSegmentation; + +use crate::comment::{CommentRef, CommentRefEncoding, ref_encoding}; +use crate::{CreatedAt, RecordError, encoded}; + +pub const REACTION_COLLECTION: &str = "org.tangled.feed.reaction"; + +collection!( + reaction_collection, + REACTION_COLLECTION, + "Reaction collection is valid nsid" +); + +pub const MAX_REACTION_BYTES: usize = 64; + +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ReactionKind { + bytes: [u8; MAX_REACTION_BYTES], + len: u8, +} + +impl ReactionKind { + pub fn new(value: &str) -> Result { + if value.len() > MAX_REACTION_BYTES { + return Err(format!( + "reaction {value:?} is longer than {MAX_REACTION_BYTES} bytes" + )); + } + if value.graphemes(true).count() > 1 { + return Err(format!("reaction {value:?} is longer than one grapheme")); + } + let mut bytes = [0; MAX_REACTION_BYTES]; + bytes[..value.len()].copy_from_slice(value.as_bytes()); + Ok(Self { + bytes, + len: value.len() as u8, + }) + } + + pub fn as_str(&self) -> &str { + std::str::from_utf8(&self.bytes[..usize::from(self.len)]) + .expect("A reaction is built from a str") + } +} + +impl std::fmt::Debug for ReactionKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.as_str().fmt(f) + } +} + +impl Serialize for ReactionKind { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for ReactionKind { + fn deserialize>(deserializer: D) -> Result { + Self::new(&String::deserialize(deserializer)?).map_err(serde::de::Error::custom) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReactionRecord { + repo: RepoDid, + subject: CommentRef, + kind: ReactionKind, + created_at: CreatedAt, + editor: AccountDid, +} + +impl ReactionRecord { + pub fn new( + repo: RepoDid, + subject: CommentRef, + kind: ReactionKind, + created_at: UnixSeconds, + editor: AccountDid, + ) -> Result { + Ok(Self { + repo, + subject, + kind, + created_at: CreatedAt::at(created_at)?, + editor, + }) + } + + pub fn body(&self) -> Result { + encoded(self) + } +} + +#[derive(Serialize)] +struct ReactionEncoding<'a> { + #[serde(rename = "$type")] + record_type: knot_types::RecordCollection, + subject: CommentRefEncoding, + reaction: &'a ReactionKind, + #[serde(rename = "createdAt")] + created_at: &'a CreatedAt, + #[serde(rename = "x-tngl-editor")] + editor: &'a AccountDid, +} + +impl Serialize for ReactionRecord { + fn serialize(&self, serializer: S) -> Result { + ReactionEncoding { + record_type: reaction_collection(), + subject: ref_encoding(&self.subject, &self.repo), + reaction: &self.kind, + created_at: &self.created_at, + editor: &self.editor, + } + .serialize(serializer) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::frozen; + use crate::test_support::{editor, squid, subject}; + + const REACTION: &str = "a565247479706578196f72672e74616e676c65642e666565642e7265616374696f6e677375626a656374a263636964783b62616679726569666a6a6570757967377877646837786c6f6c756c6e793661756f6a6d3773717a366c6c687137686966346466757068726973696963757269783961743a2f2f6469643a706c633a73717569642f6f72672e74616e676c65642e747261636b2e7469636b65742f336c7562727074783537643232687265616374696f6e66e29da4efb88f6963726561746564417474313937302d30312d30315430303a30303a30315a6d782d746e676c2d656469746f7278206469643a706c633a3366776563646e76746373636a6e72783270346e37616c7a"; + + #[test] + fn reaction_record_bytes_frozen() { + let record = ReactionRecord::new( + squid(), + subject(), + ReactionKind::new("❤️").unwrap(), + UnixSeconds::new(1), + editor(), + ) + .unwrap(); + let body = record.body().unwrap(); + assert_eq!( + body.as_bytes(), + frozen(REACTION).as_slice(), + "first reaction ever written should produce exactly these bytes: whole seconds, & utc z" + ); + } + + #[test] + fn every_reaction_kind_serializes_its_lexicon_token() { + let pairs = [ + (ReactionKind::new("👍").unwrap(), "👍"), + (ReactionKind::new("👎").unwrap(), "👎"), + (ReactionKind::new("😆").unwrap(), "😆"), + (ReactionKind::new("🎉").unwrap(), "🎉"), + (ReactionKind::new("🫤").unwrap(), "🫤"), + (ReactionKind::new("❤️").unwrap(), "❤️"), + (ReactionKind::new("🚀").unwrap(), "🚀"), + (ReactionKind::new("👀").unwrap(), "👀"), + ]; + for (kind, token) in pairs { + assert_eq!( + serde_json::to_value(kind).unwrap(), + serde_json::json!(token) + ); + } + } + + #[test] + fn reaction_is_at_most_one_grapheme() { + let decoded: Result = serde_json::from_value(serde_json::json!("🔥🔥")); + assert!(decoded.is_err()); + let decoded: ReactionKind = serde_json::from_value(serde_json::json!("🤍")).unwrap(); + assert_eq!(decoded.as_str(), "🤍"); + } +} diff --git a/knot2/crates/knot-record/src/ticket.rs b/knot2/crates/knot-record/src/ticket.rs new file mode 100644 index 0000000000000000000000000000000000000000..62e57a47b29b78a33ccc76f4db033c6b892eebb6 --- /dev/null +++ b/knot2/crates/knot-record/src/ticket.rs @@ -0,0 +1,343 @@ +use std::fmt; +use std::str::FromStr; + +use jacquard_common::deps::codegen::unicode_segmentation::UnicodeSegmentation; +use knot_types::{AccountDid, BranchName, RecordBody, RecordBodyError, RefName, UnixSeconds}; +use serde::{Deserialize, Serialize, Serializer}; + +use crate::comment::{MarkdownEncoding, markdown_encoding}; +use crate::{CreatedAt, Prose, RecordError, encoded}; + +pub const TICKET_COLLECTION: &str = "org.tangled.track.ticket"; + +pub const INLINE_MARKUP: &str = "org.tangled.markup.markdown#inline"; + +collection!( + ticket_collection, + TICKET_COLLECTION, + "Ticket collection is valid nsid" +); + +#[derive(Debug, thiserror::Error)] +pub enum ProseError { + #[error("title is empty")] + EmptyTitle, + #[error("title of {0} bytes is over the {MAX_TITLE_BYTES} byte limit")] + TitleTooLong(usize), + #[error("title of {0} graphemes is over the {MAX_TITLE_GRAPHEMES} grapheme limit")] + TitleTooDense(usize), + #[error("body of {0} bytes is over the {MAX_BODY_BYTES} byte limit")] + BodyTooLong(usize), + #[error("body of {0} graphemes is over the {MAX_BODY_GRAPHEMES} grapheme limit")] + BodyTooDense(usize), +} + +pub const MAX_TITLE_BYTES: usize = 1_024; + +pub const MAX_TITLE_GRAPHEMES: usize = 256; + +pub const MAX_BODY_BYTES: usize = 262_144; + +pub const MAX_BODY_GRAPHEMES: usize = 65_536; + +pub const MAX_ASSIGNEES: usize = 10; + +macro_rules! prose_text { + ( + text = $name:ident, + limit = $limit:expr, + graphemes = $graphemes:expr, + $(empty = $empty:ident,)? + long = $long:ident, + dense = $dense:ident + ) => { + #[derive(Debug, Clone, PartialEq, Eq, Serialize)] + #[serde(transparent)] + pub struct $name(String); + + impl $name { + pub fn new(text: impl Into) -> Result { + let text = text.into(); + $(if text.trim().is_empty() { + return Err(ProseError::$empty); + })? + match text.len() { + len if len > $limit => Err(ProseError::$long(len)), + _ => match UnicodeSegmentation::graphemes(text.as_str(), true).count() { + counted if counted > $graphemes => Err(ProseError::$dense(counted)), + _ => Ok(Self(text)), + }, + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + Self::new(text).map_err(serde::de::Error::custom) + } + } + }; +} + +prose_text!( + text = TicketTitle, + limit = MAX_TITLE_BYTES, + graphemes = MAX_TITLE_GRAPHEMES, + empty = EmptyTitle, + long = TitleTooLong, + dense = TitleTooDense +); + +prose_text!( + text = MarkdownBody, + limit = MAX_BODY_BYTES, + graphemes = MAX_BODY_GRAPHEMES, + long = BodyTooLong, + dense = BodyTooDense +); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum TicketState { + #[serde(rename = "open")] + Open, + #[serde(rename = "closed")] + Closed, + #[serde(rename = "completed")] + Completed, +} + +impl fmt::Display for TicketState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(match self { + Self::Open => "open", + Self::Closed => "closed", + Self::Completed => "completed", + }) + } +} + +pub const MAX_TARGET_BRANCH_BYTES: usize = 244; + +#[derive(Debug, thiserror::Error)] +pub enum TargetBranchError { + #[error("target branch isn't a valid branch name: {0}")] + Branch(knot_types::ParseError), + #[error("target branch is the bare branch name, without the refs/ prefix")] + Prefixed, + #[error("target branch of {0} bytes is over the {MAX_TARGET_BRANCH_BYTES} byte limit")] + TooLong(usize), +} + +#[derive(Clone)] +pub struct TargetBranch(BranchName); + +impl TargetBranch { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let branch = BranchName::new(&value).map_err(TargetBranchError::Branch)?; + if value.starts_with("refs/") { + return Err(TargetBranchError::Prefixed); + } + if value.len() > MAX_TARGET_BRANCH_BYTES { + return Err(TargetBranchError::TooLong(value.len())); + } + Ok(Self(branch)) + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } + + pub fn head_ref(&self) -> RefName { + self.0.head_ref() + } +} + +knot_types::string_id!(@traits_for TargetBranch, TargetBranchError); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TicketRecord { + body: Option, + title: TicketTitle, + state: TicketState, + tangled_labels: Option>, + assignees: Option>, + target_branch: Option, + created_at: CreatedAt, + attachments: Option>, + editor: AccountDid, +} + +impl TicketRecord { + #[allow(clippy::too_many_arguments)] + pub fn new( + prose: Prose, + state: TicketState, + tangled_labels: Option>, + assignees: Option>, + target_branch: Option, + attachments: Option>, + created_at: UnixSeconds, + editor: AccountDid, + ) -> Result { + Ok(Self { + body: prose.body, + title: prose.title, + state, + tangled_labels, + assignees, + target_branch, + created_at: CreatedAt::at(created_at)?, + attachments, + editor, + }) + } + + pub fn body(&self) -> Result { + encoded(self) + } +} + +#[derive(Serialize)] +struct InlineEncoding<'a> { + #[serde(rename = "$type")] + markup_type: &'static str, + text: &'a TicketTitle, +} + +#[derive(Serialize)] +struct TicketEncoding<'a> { + #[serde(skip_serializing_if = "Option::is_none")] + body: Option>, + #[serde(rename = "$type")] + record_type: knot_types::RecordCollection, + title: InlineEncoding<'a>, + state: &'a TicketState, + #[serde(skip_serializing_if = "Option::is_none", rename = "tangledLabels")] + tangled_labels: &'a Option>, + #[serde(skip_serializing_if = "Option::is_none")] + assignees: &'a Option>, + #[serde(skip_serializing_if = "Option::is_none", rename = "targetBranch")] + target_branch: &'a Option, + #[serde(rename = "createdAt")] + created_at: &'a CreatedAt, + #[serde(rename = "x-tngl-editor")] + editor: &'a AccountDid, +} + +impl Serialize for TicketRecord { + fn serialize(&self, serializer: S) -> Result { + TicketEncoding { + body: self + .body + .as_ref() + .map(|body| markdown_encoding(body, &self.attachments)), + record_type: ticket_collection(), + title: InlineEncoding { + markup_type: INLINE_MARKUP, + text: &self.title, + }, + state: &self.state, + tangled_labels: &self.tangled_labels, + assignees: &self.assignees, + target_branch: &self.target_branch, + created_at: &self.created_at, + editor: &self.editor, + } + .serialize(serializer) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::frozen; + + const TICKET: &str = "a664626f6479a2647465787466666f72657374652474797065781b6f72672e74616e676c65642e6d61726b75702e6d61726b646f776e65247479706578186f72672e74616e676c65642e747261636b2e7469636b6574657374617465646f70656e657469746c65a26474657874646b656c7065247479706578226f72672e74616e676c65642e6d61726b75702e6d61726b646f776e23696e6c696e656963726561746564417474323032352d30382d31335431333a30313a33365a6d782d746e676c2d656469746f7278206469643a706c633a3366776563646e76746373636a6e72783270346e37616c7a"; + + use crate::test_support::editor; + + fn prose(body: Option<&str>) -> Prose { + Prose::new( + TicketTitle::new("kelp").unwrap(), + body.map(|text| MarkdownBody::new(text).unwrap()), + ) + } + + #[test] + fn ticket_record_bytes_frozen() { + let record = TicketRecord::new( + prose(Some("forest")), + TicketState::Open, + None, + None, + None, + None, + UnixSeconds::new(1_755_090_096), + editor(), + ) + .unwrap(); + let body = record.body().unwrap(); + assert_eq!( + body.as_bytes(), + frozen(TICKET).as_slice(), + "the first emitted ticket must freeze these bytes: absent optionals omitted, seconds precision, utc z" + ); + } + + #[test] + fn absent_body_is_omitted() { + let sent = prose(None); + let record = TicketRecord::new( + sent, + TicketState::Open, + None, + None, + None, + None, + UnixSeconds::new(1_755_090_096), + editor(), + ) + .unwrap(); + let bytes = record.body().unwrap(); + let value: serde_json::Value = serde_ipld_dagcbor::from_slice(bytes.as_bytes()).unwrap(); + assert_eq!(value.get("body"), None); + } + + #[test] + fn ticket_state_spells_the_three_tokens() { + assert_eq!(TicketState::Open.to_string(), "open"); + assert_eq!(TicketState::Closed.to_string(), "closed"); + assert_eq!(TicketState::Completed.to_string(), "completed"); + } + + #[test] + fn ticket_prose_must_say_something_and_not_too_much() { + assert!(matches!( + TicketTitle::new(" "), + Err(ProseError::EmptyTitle) + )); + assert!(MarkdownBody::new("\n").is_ok()); + assert!(matches!( + TicketTitle::new("k".repeat(MAX_TITLE_BYTES + 1)), + Err(ProseError::TitleTooLong(_)) + )); + assert!(matches!( + MarkdownBody::new("k".repeat(MAX_BODY_BYTES + 1)), + Err(ProseError::BodyTooLong(_)) + )); + assert!(matches!( + TicketTitle::new("e\u{301}".repeat(MAX_TITLE_GRAPHEMES + 1)), + Err(ProseError::TitleTooDense(_)) + )); + assert!(matches!( + MarkdownBody::new("e\u{301}".repeat(MAX_BODY_GRAPHEMES + 1)), + Err(ProseError::BodyTooDense(_)) + )); + } +} diff --git a/knot2/crates/knot-record/tests/car_exports.rs b/knot2/crates/knot-record/tests/car_exports.rs new file mode 100644 index 0000000000000000000000000000000000000000..8d81ad2d838a99f5309459a6774020a976e33236 --- /dev/null +++ b/knot2/crates/knot-record/tests/car_exports.rs @@ -0,0 +1,132 @@ +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::Arc; + +use bytes::Bytes; +use cid::Cid; +use futures::StreamExt; +use jacquard_repo::Mst; +use jacquard_repo::car::reader::ParsedCar; +use jacquard_repo::commit::Commit; +use jacquard_repo::mst::node::NodeData; +use jacquard_repo::storage::MemoryBlockStore; + +use knot_record::mst; + +fn export_dir() -> Option { + std::env::var("KNOT2_ATPROTO_CAR_EXPORTS") + .map(PathBuf::from) + .ok() +} + +fn cars(dir: &PathBuf) -> Vec { + let mut cars: Vec = std::fs::read_dir(dir) + .expect("read the export directory") + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "car")) + .collect(); + cars.sort(); + cars +} + +fn node_blocks(blocks: &BTreeMap, cid: Cid, nodes: &mut BTreeMap) { + let bytes = &blocks[&cid]; + let node: NodeData = serde_ipld_dagcbor::from_slice(bytes).expect("mst blocks decode as nodes"); + nodes.insert(cid, bytes.clone()); + node.left.into_iter().for_each(|left| { + node_blocks(blocks, left, nodes); + }); + node.entries.into_iter().for_each(|entry| { + entry.tree.into_iter().for_each(|sub| { + node_blocks(blocks, sub, nodes); + }); + }); +} + +#[tokio::test] +async fn the_builder_reproduces_every_exported_tree_node_for_node() { + let Some(dir) = export_dir() else { + eprintln!("skipping: KNOT2_ATPROTO_CAR_EXPORTS isn't set"); + return; + }; + let exports = cars(&dir); + assert!( + !exports.is_empty(), + "no .car exports under {}", + dir.display() + ); + let holding_float_records = futures::stream::iter(exports.iter()) + .then(|path| async move { + let parsed = parse(&std::fs::read(path).expect("read the export")).await; + let commit: Commit = + serde_ipld_dagcbor::from_slice(&parsed.blocks[&parsed.root]) + .expect("the root block is a signed commit"); + let root = *commit.data(); + let leaves: BTreeMap = Mst::load( + Arc::new(MemoryBlockStore::new_from_blocks(parsed.blocks.clone())), + root, + None, + ) + .leaves() + .await + .expect("walk the exported tree for its records") + .into_iter() + .map(|(key, cid)| (key.to_string(), cid)) + .collect(); + let mut export_nodes = BTreeMap::new(); + node_blocks(&parsed.blocks, root, &mut export_nodes); + let rebuilt = mst::build(&leaves) + .await + .expect("rebuild the tree from pairs alone"); + assert_eq!( + rebuilt.root, + root, + "{}: the rebuilt root must equal the signed commit's data cid", + path.display() + ); + assert_eq!( + rebuilt.nodes, + export_nodes, + "{}: every rebuilt node must be byte-identical", + path.display() + ); + eprintln!( + "{}: {} records, {} mst nodes reproduced", + path.display(), + leaves.len(), + export_nodes.len() + ); + parsed.blocks.iter().for_each(|(cid, bytes)| { + if let Ok(node) = serde_ipld_dagcbor::from_slice::(bytes) { + assert_eq!( + &serde_ipld_dagcbor::to_vec(&node).unwrap(), + bytes, + "{}: {cid} re-encodes byte-identically, canonical key order included", + path.display() + ); + } + }); + parsed.blocks.values().any(|bytes| { + serde_ipld_dagcbor::from_slice::(bytes).is_err() && bytes.contains(&0xfb) + }) + }) + .collect::>() + .await + .into_iter() + .any(|held| held); + assert!( + exports.len() >= 4, + "the four real exports are the ground truth" + ); + assert!( + holding_float_records, + "real exports carry float-holding records, which is why record bytes are stored verbatim" + ); +} + +async fn parse(raw: &[u8]) -> ParsedCar { + jacquard_repo::car::reader::parse_car_bytes(raw) + .await + .expect("parse the export") +} diff --git a/knot2/crates/knot-record/tests/proof_fixtures.rs b/knot2/crates/knot-record/tests/proof_fixtures.rs new file mode 100644 index 0000000000000000000000000000000000000000..b1737b47ffdae694c42643ea53a3842738f5af5b --- /dev/null +++ b/knot2/crates/knot-record/tests/proof_fixtures.rs @@ -0,0 +1,158 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; + +use cid::Cid; +use futures::StreamExt; +use jacquard_repo::mst::VerifiedWriteOp; +use serde_json::Value; + +use knot_record::mst; +use knot_record::proof; + +const MAX_OVER_EMISSION_FACTOR: usize = 2; + +fn interop_dir() -> PathBuf { + std::env::var("KNOT_ATPROTO_INTEROP_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../third_party/atproto-interop-tests") + }) +} + +fn fixtures_available() -> bool { + std::env::var_os("KNOT_ATPROTO_INTEROP_DIR").is_some() + || interop_dir() + .join("firehose/commit-proof-fixtures.json") + .is_file() +} + +fn fixtures() -> Vec { + let path = interop_dir().join("firehose/commit-proof-fixtures.json"); + let raw = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); + serde_json::from_str(&raw).expect("commit proof fixtures parse as json") +} + +fn strings(vector: &Value, field: &str) -> Vec { + vector[field] + .as_array() + .expect("field holds an array") + .iter() + .map(|value| value.as_str().expect("entries are strings").to_owned()) + .collect() +} + +fn cid(text: &str) -> Cid { + text.parse().expect("fixture cids parse") +} + +fn pairs(keys: &[String], leaf: Cid) -> BTreeMap { + keys.iter().map(|key| (key.clone(), leaf)).collect() +} + +fn ops(vector: &Value, leaf: Cid) -> Vec { + strings(vector, "adds") + .into_iter() + .map(|key| VerifiedWriteOp::Create { + key: key.into(), + cid: leaf, + }) + .chain( + strings(vector, "dels") + .into_iter() + .map(|key| VerifiedWriteOp::Delete { + key: key.into(), + prev: leaf, + }), + ) + .collect() +} + +#[tokio::test] +async fn the_covering_set_is_a_superset_that_inverts_cold_with_a_bounded_size() { + if !fixtures_available() { + eprintln!("skipping covering-set fixture proof: atproto interop fixtures unavailable"); + return; + } + futures::stream::iter(fixtures().iter()) + .for_each(|vector| async { + let comment = vector["comment"].as_str().unwrap(); + let leaf = cid(vector["leafValue"].as_str().unwrap()); + let keys = strings(vector, "keys"); + assert_eq!( + mst::build(&pairs(&keys, leaf)) + .await + .unwrap() + .root + .to_string(), + vector["rootBeforeCommit"].as_str().unwrap(), + "{comment}: the builder must reproduce rootBeforeCommit from the pairs alone" + ); + let before = mst::tree_of(&pairs(&keys, leaf)).await.unwrap(); + let after = mst::apply(&before, &ops(vector, leaf)).await.unwrap(); + let root = mst::root(&after.tree).await.unwrap(); + assert_eq!( + root.to_string(), + vector["rootAfterCommit"].as_str().unwrap(), + "{comment}: applying the vector's ops must reach rootAfterCommit" + ); + let covering = proof::covering_blocks(&before, &after.tree, &after.diff) + .await + .unwrap(); + let expected: BTreeSet = strings(vector, "blocksInProof") + .iter() + .map(|text| cid(text)) + .collect(); + expected.iter().for_each(|cid| { + assert!( + covering.contains_key(cid), + "{comment}: the covering set must contain {cid}" + ); + }); + assert!( + covering.len() <= expected.len() * MAX_OVER_EMISSION_FACTOR, + "{comment}: emitted {} blocks over the expected {}, and over-emission can't grow \ + unwatched", + covering.len(), + expected.len(), + ); + assert_eq!( + proof::cold_invert(&covering, root, &ops(vector, leaf)) + .await + .unwrap() + .to_string(), + vector["rootBeforeCommit"].as_str().unwrap(), + "{comment}: a consumer given only the emitted blocks recovers prevData" + ); + }) + .await; +} + +#[tokio::test] +async fn mst_diff_alone_under_emits_on_neighbor_two_layers_down() { + if !fixtures_available() { + eprintln!("skipping mst diff under-emission proof: atproto interop fixtures unavailable"); + return; + } + let all = fixtures(); + let vector = all + .iter() + .find(|vector| { + vector["comment"].as_str() == Some("add on edge with neighbor two layers down") + }) + .expect("the named fixture is vendored"); + let leaf = cid(vector["leafValue"].as_str().unwrap()); + let after = mst::apply( + &mst::tree_of(&pairs(&strings(vector, "keys"), leaf)) + .await + .unwrap(), + &ops(vector, leaf), + ) + .await + .unwrap(); + assert!( + after.diff.new_mst_blocks.len() < strings(vector, "blocksInProof").len(), + "mst diff blocks alone under-emit on this vector, which is why the union exists" + ); +} diff --git a/knot2/crates/knot-resource/Cargo.toml b/knot2/crates/knot-resource/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..204ac5d44d6b48c3968eed62813a0ebb38fa5b13 --- /dev/null +++ b/knot2/crates/knot-resource/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "knot-resource" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +rustix = { workspace = true, features = ["fs"] } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt", "macros"] } +tempfile = { workspace = true } diff --git a/knot2/crates/knot-resource/src/admission.rs b/knot2/crates/knot-resource/src/admission.rs new file mode 100644 index 0000000000000000000000000000000000000000..de825c42121f26e57898c239f6c1ddc5759a0902 --- /dev/null +++ b/knot2/crates/knot-resource/src/admission.rs @@ -0,0 +1,1121 @@ +use std::collections::HashMap; +use std::hash::Hash; +use std::net::IpAddr; +use std::num::{NonZeroU32, NonZeroU64}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use knot_types::{AccountDid, UnixMicros}; + +const MAX_TRACKED_PEERS: usize = 100_000; + +const MAX_PACED_KEYS: usize = 4_096; + +const SWEEP_INTERVAL_MICROS: u64 = 1_000_000; + +const MAX_METERED_SUBJECTS: usize = 16_384; + +knot_types::scalar_newtype! { + pub struct PerPeerInflight(usize); + pub struct GlobalInflight(usize); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Burst(NonZeroU32); + +impl Burst { + pub fn of(value: u32) -> Option { + NonZeroU32::new(value).map(Self) + } + + pub const fn get(self) -> u32 { + self.0.get() + } + + pub const fn literal(value: u32) -> Self { + match NonZeroU32::new(value) { + Some(value) => Self(value), + None => panic!("Burst spelled out in source allows at least one call"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct RefillMicros(NonZeroU64); + +impl RefillMicros { + pub fn of(value: u64) -> Option { + NonZeroU64::new(value).map(Self) + } + + pub const fn get(self) -> u64 { + self.0.get() + } + + pub const fn literal(value: u64) -> Self { + match NonZeroU64::new(value) { + Some(value) => Self(value), + None => panic!("Refill interval spelled out in source is at least tick"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RateLimit { + pub burst: Burst, + pub refill: RefillMicros, +} + +#[derive(Debug, Clone, Copy)] +pub struct LimitConfig { + pub rate: Option, + pub per_peer_inflight: Option, + pub global_inflight: Option, +} + +impl Default for LimitConfig { + fn default() -> Self { + Self { + rate: Some(RateLimit { + burst: Burst::literal(20), + refill: RefillMicros::literal(100_000), + }), + per_peer_inflight: Some(PerPeerInflight::new(8)), + global_inflight: Some(GlobalInflight::new(64)), + } + } +} + +impl LimitConfig { + pub const fn per_peer_only(per_peer_inflight: PerPeerInflight) -> Self { + Self { + rate: None, + per_peer_inflight: Some(per_peer_inflight), + global_inflight: None, + } + } + + pub const fn unmetered() -> Self { + Self { + rate: None, + per_peer_inflight: None, + global_inflight: None, + } + } +} + +struct Tokens { + tokens: u32, + last_refill: UnixMicros, +} + +impl Tokens { + fn new(rate: RateLimit, now: UnixMicros) -> Self { + Self { + tokens: rate.burst.get(), + last_refill: now, + } + } + + fn intervals(&self, rate: RateLimit, now: UnixMicros) -> u64 { + now.get().saturating_sub(self.last_refill.get()) / rate.refill.get() + } + + fn tokens_at(&self, rate: RateLimit, now: UnixMicros) -> u32 { + let gained = self.intervals(rate, now).min(u64::from(rate.burst.get())) as u32; + self.tokens.saturating_add(gained).min(rate.burst.get()) + } + + fn replenish(&mut self, rate: RateLimit, now: UnixMicros) -> bool { + if now.get() < self.last_refill.get() { + self.last_refill = now; + return self.tokens > 0; + } + let intervals = self.intervals(rate, now); + if intervals > 0 { + self.tokens = self.tokens_at(rate, now); + self.last_refill = UnixMicros::new( + self.last_refill + .get() + .saturating_add(intervals.saturating_mul(rate.refill.get())), + ); + } + self.tokens > 0 + } + + fn full(&self, rate: RateLimit, now: UnixMicros) -> bool { + self.tokens_at(rate, now) >= rate.burst.get() + } +} + +struct PeerState { + bucket: Option, + inflight: usize, +} + +impl PeerState { + fn forgettable(&self) -> bool { + self.inflight == 0 && self.bucket.is_none() + } + + fn worth_tracking(&self, rate: Option, now: UnixMicros) -> bool { + match (self.inflight, &self.bucket, rate) { + (0, Some(bucket), Some(rate)) => !bucket.full(rate, now), + (0, _, _) => false, + _ => true, + } + } +} + +const CONCENTRATED_REFUSALS: u32 = 1_024; + +#[derive(Default)] +struct RejectionMajority { + peer: Option, + votes: u32, + reported: bool, +} + +impl RejectionMajority { + fn observe(&mut self, peer: IpAddr) -> Option { + match (self.peer == Some(peer), self.votes) { + (true, _) => self.votes = self.votes.saturating_add(1), + (false, 0) => { + self.peer = Some(peer); + self.votes = 1; + } + (false, _) => self.votes -= 1, + } + let crossed = self.votes >= CONCENTRATED_REFUSALS && !self.reported; + self.reported |= crossed; + crossed.then_some(peer) + } +} + +struct Inner { + peers: HashMap, PeerState>, + global_inflight: usize, + last_sweep: UnixMicros, + rejection_majority: RejectionMajority, +} + +impl Inner { + fn has_room_for( + &mut self, + peer: Option, + rate: Option, + now: UnixMicros, + ) -> bool { + if rate.is_none() || self.peers.contains_key(&peer) || self.peers.len() < MAX_TRACKED_PEERS + { + return true; + } + if now.get().saturating_sub(self.last_sweep.get()) >= SWEEP_INTERVAL_MICROS { + self.last_sweep = now; + self.peers + .retain(|_, state| state.worth_tracking(rate, now)); + } + self.peers.len() < MAX_TRACKED_PEERS + } +} + +pub struct PreAuthLimiter { + config: LimitConfig, + inner: Mutex, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Rejection { + RateLimited, + Saturated, +} + +impl Default for PreAuthLimiter { + fn default() -> Self { + Self::with_config(LimitConfig::default()) + } +} + +impl PreAuthLimiter { + pub fn with_config(config: LimitConfig) -> Self { + Self { + config, + inner: Mutex::new(Inner { + peers: HashMap::new(), + global_inflight: 0, + last_sweep: UnixMicros::new(0), + rejection_majority: RejectionMajority::default(), + }), + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub fn admit( + self: &Arc, + peer: Option, + now: UnixMicros, + ) -> Result { + let mut inner = self.lock(); + let rate = self.config.rate; + let over_global = self + .config + .global_inflight + .is_some_and(|limit| inner.global_inflight >= limit.get()); + + let per_peer = self.config.per_peer_inflight; + let decision = match inner.has_room_for(peer, rate, now) { + false => Err(Rejection::Saturated), + true => { + let state = inner.peers.entry(peer).or_insert_with(|| PeerState { + bucket: rate.map(|rate| Tokens::new(rate, now)), + inflight: 0, + }); + let ready = match (rate, state.bucket.as_mut()) { + (Some(rate), Some(bucket)) => bucket.replenish(rate, now), + _ => true, + }; + let over_peer = per_peer.is_some_and(|limit| state.inflight >= limit.get()); + match (ready, over_peer || over_global) { + (false, _) => Err(Rejection::RateLimited), + (_, true) => Err(Rejection::Saturated), + (true, false) => { + if let Some(bucket) = state.bucket.as_mut() { + bucket.tokens -= 1; + } + state.inflight += 1; + Ok(()) + } + } + } + }; + match decision { + Err(refusal) => { + if inner.peers.get(&peer).is_some_and(PeerState::forgettable) { + inner.peers.remove(&peer); + } + let concentrated = peer.and_then(|peer| inner.rejection_majority.observe(peer)); + drop(inner); + if let Some(peer) = concentrated { + tracing::warn!( + %peer, + "one address has taken {CONCENTRATED_REFUSALS} more of the pre-authentication limiter's refusals than every other address combined. If this address is a proxy, set xrpc.trusted_proxy_header to the header it forwards the client address in and add the address to xrpc.trusted_proxies, since every client behind a proxy will share its one rate-limit bucket. This warning reports the first such address only." + ); + } + Err(refusal) + } + Ok(()) => { + inner.global_inflight += 1; + Ok(AdmitGuard { + limiter: Arc::clone(self), + peer, + }) + } + } + } + + fn leave(&self, peer: Option) { + let mut inner = self.lock(); + inner.global_inflight = inner.global_inflight.saturating_sub(1); + let Some(state) = inner.peers.get_mut(&peer) else { + return; + }; + state.inflight = state.inflight.saturating_sub(1); + if state.forgettable() { + inner.peers.remove(&peer); + } + } + + fn repay(&self, peer: Option) { + let Some(rate) = self.config.rate else { + return; + }; + let mut inner = self.lock(); + if let Some(bucket) = inner + .peers + .get_mut(&peer) + .and_then(|state| state.bucket.as_mut()) + { + bucket.tokens = bucket.tokens.saturating_add(1).min(rate.burst.get()); + } + } +} + +pub struct AdmitGuard { + limiter: Arc, + peer: Option, +} + +impl AdmitGuard { + pub fn refund(self) { + self.limiter.repay(self.peer); + } +} + +impl Drop for AdmitGuard { + fn drop(&mut self) { + self.limiter.leave(self.peer); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct HostKey(String); + +impl HostKey { + pub fn new(host: &str) -> Self { + Self(host.to_ascii_lowercase()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct SubjectKey(AccountDid); + +impl SubjectKey { + pub fn new(subject: &AccountDid) -> Self { + Self(subject.clone()) + } +} + +struct Booked { + turns: HashMap, + last_sweep: UnixMicros, +} + +impl Booked { + fn has_room_for(&mut self, key: &K, now: UnixMicros) -> bool { + if self.turns.contains_key(key) || self.turns.len() < MAX_PACED_KEYS { + return true; + } + if now.get().saturating_sub(self.last_sweep.get()) >= SWEEP_INTERVAL_MICROS { + self.last_sweep = now; + self.turns.retain(|_, until| until.get() > now.get()); + } + self.turns.len() < MAX_PACED_KEYS + } +} + +pub struct Pacer { + rate: RateLimit, + inner: Mutex>, +} + +pub type HostPacer = Pacer; +pub type SubjectPacer = Pacer; +pub type PeerPacer = Pacer; + +impl Pacer { + pub fn new(rate: RateLimit) -> Self { + Self { + rate, + inner: Mutex::new(Booked { + turns: HashMap::new(), + last_sweep: UnixMicros::new(0), + }), + } + } + + pub fn reserve(&self, key: &K, now: UnixMicros) -> Duration { + let mut booked = self.lock(); + match booked.has_room_for(key, now) { + false => Duration::from_micros(self.rate.refill.get()), + true => { + let wait = self.wait_for(&booked, key, now); + self.claim_turn(&mut booked, key, now); + wait + } + } + } + + pub fn reserve_now(&self, key: &K, now: UnixMicros) -> bool { + let mut booked = self.lock(); + match booked.has_room_for(key, now) { + false => false, + true => match self.wait_for(&booked, key, now).is_zero() { + false => false, + true => { + self.claim_turn(&mut booked, key, now); + true + } + }, + } + } + + fn wait_for(&self, booked: &Booked, key: &K, now: UnixMicros) -> Duration { + let tolerance = self + .rate + .refill + .get() + .saturating_mul(u64::from(self.rate.burst.get().saturating_sub(1))); + Duration::from_micros( + self.turn(booked, key, now) + .get() + .saturating_sub(tolerance) + .saturating_sub(now.get()), + ) + } + + fn claim_turn(&self, booked: &mut Booked, key: &K, now: UnixMicros) { + let until = self + .turn(booked, key, now) + .get() + .saturating_add(self.rate.refill.get()); + booked.turns.insert(key.clone(), UnixMicros::new(until)); + } + + fn turn(&self, booked: &Booked, key: &K, now: UnixMicros) -> UnixMicros { + booked + .turns + .get(key) + .map_or(now, |until| UnixMicros::new(until.get().max(now.get()))) + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Booked> { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Bucket { + Peer, + ActorWrites, + ActorReservations, + KnotReservations, + RepoStorage, + RepoSocial, +} + +impl Bucket { + pub const fn name(self) -> &'static str { + match self { + Self::Peer => "peer", + Self::ActorWrites => "actor-writes", + Self::ActorReservations => "actor-reservations", + Self::KnotReservations => "knot-reservations", + Self::RepoStorage => "repo-storage", + Self::RepoSocial => "repo-social", + } + } +} + +struct Metering { + tokens: Tokens, + last_seen: UnixMicros, +} + +impl Metering { + fn new(rate: RateLimit, now: UnixMicros) -> Self { + Self { + tokens: Tokens::new(rate, now), + last_seen: now, + } + } + + fn spend(&mut self, rate: RateLimit, now: UnixMicros) -> Result<(), Rejection> { + self.last_seen = now; + if self.tokens.replenish(rate, now) { + self.tokens.tokens -= 1; + Ok(()) + } else { + Err(Rejection::RateLimited) + } + } +} + +struct Metered { + buckets: HashMap, + last_sweep: UnixMicros, +} + +impl Metered { + fn make_room(&mut self, rate: RateLimit, now: UnixMicros) -> bool { + if self.buckets.len() < MAX_METERED_SUBJECTS { + return true; + } + if now.get().saturating_sub(self.last_sweep.get()) >= SWEEP_INTERVAL_MICROS { + self.last_sweep = now; + self.buckets + .retain(|_, metering| !metering.tokens.full(rate, now)); + } + if self.buckets.len() < MAX_METERED_SUBJECTS { + return true; + } + let coldest = self + .buckets + .iter() + .min_by_key(|(_, metering)| metering.last_seen.get()) + .map(|(subject, metering)| (subject.clone(), metering.last_seen)); + + // Freezing cold buckets are provably evictable without consequence. + match coldest { + Some((subject, seen)) if now.get().saturating_sub(seen.get()) >= rate.refill.get() => { + self.buckets.remove(&subject); + true + } + _ => false, + } + } +} + +pub struct SubjectLimiter { + rate: RateLimit, + inner: Mutex, +} + +impl SubjectLimiter { + pub fn new(rate: RateLimit) -> Self { + Self { + rate, + inner: Mutex::new(Metered { + buckets: HashMap::new(), + last_sweep: UnixMicros::new(0), + }), + } + } + + pub fn admit(&self, subject: &SubjectKey, now: UnixMicros) -> Result<(), Rejection> { + let rate = self.rate; + let mut metered = self + .inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(metering) = metered.buckets.get_mut(subject) { + return metering.spend(rate, now); + } + if !metered.make_room(rate, now) { + return Err(Rejection::Saturated); + } + metered + .buckets + .entry(subject.clone()) + .or_insert_with(|| Metering::new(rate, now)) + .spend(rate, now) + } + + pub fn tracked(&self) -> usize { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .buckets + .len() + } +} + +knot_types::scalar_newtype! { + pub struct SocialBudgetBytes(u64); + pub struct StructureBytes(u64) => ordered; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SocialCharge { + pub current: StructureBytes, + pub adding: StructureBytes, +} + +impl SocialBudgetBytes { + pub const fn allows(self, charge: SocialCharge) -> bool { + charge.current.get().saturating_add(charge.adding.get()) <= self.get() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::Ipv4Addr; + + fn limiter(config: LimitConfig) -> Arc { + Arc::new(PreAuthLimiter::with_config(config)) + } + + fn limit(burst: u32, refill_micros: u64) -> RateLimit { + RateLimit { + burst: Burst::of(burst).expect("Test burst allows at least one call"), + refill: RefillMicros::of(refill_micros).expect("Test refill is real interval"), + } + } + + fn rate(burst: u32, refill_micros: u64) -> Option { + Some(limit(burst, refill_micros)) + } + + fn peer(last: u8) -> Option { + Some(ip(last)) + } + + fn rotating(index: u64) -> Option { + let octets = (index as u32).to_be_bytes(); + Some(IpAddr::V4(Ipv4Addr::new( + 1, octets[1], octets[2], octets[3], + ))) + } + + fn at(micros: u64) -> UnixMicros { + UnixMicros::new(micros) + } + + fn tracked(limiter: &Arc) -> usize { + limiter.lock().peers.len() + } + + fn ip(last: u8) -> IpAddr { + IpAddr::V4(Ipv4Addr::new(127, 0, 0, last)) + } + + #[test] + fn only_an_address_taking_most_of_the_refusals_is_reported_and_only_once() { + let reported = |rounds, peer: fn(u32) -> u8| { + let mut majority = RejectionMajority::default(); + (0..rounds) + .filter_map(|round| majority.observe(ip(peer(round)))) + .collect::>() + }; + [ + (CONCENTRATED_REFUSALS * 3, (|round| (round % 4 == 0) as u8) as fn(u32) -> u8, vec![ip(0)], + "the warning reports that address once, since three refusals in every four come from it"), + (CONCENTRATED_REFUSALS * 8, |round| (round % 251) as u8, vec![], + "a knot under scattered load doesn't have a proxy to point the operator at"), + (CONCENTRATED_REFUSALS - 1, |_| 1, vec![], + "a burst under the threshold is ordinary rate limiting and doesn't point at a proxy"), + (CONCENTRATED_REFUSALS, |_| 1, vec![ip(1)], + "the threshold itself is where an address earns the warning"), + ] + .into_iter() + .for_each(|(rounds, peer, expected, why)| { + assert_eq!(reported(rounds, peer), expected, "{why}"); + }); + } + + #[test] + fn a_rate_budget_refuses_a_flood_refills_over_time_and_stays_per_peer() { + let limiter = limiter(LimitConfig { + rate: rate(2, 1_000), + per_peer_inflight: Some(PerPeerInflight::new(100)), + global_inflight: Some(GlobalInflight::new(100)), + }); + assert!(limiter.admit(peer(1), at(0)).is_ok()); + assert!(limiter.admit(peer(1), at(0)).is_ok()); + assert_eq!( + limiter.admit(peer(1), at(0)).err(), + Some(Rejection::RateLimited), + "the limiter refuses a third request inside the same instant even when nothing is in flight" + ); + assert!( + limiter.admit(peer(2), at(0)).is_ok(), + "a second peer keeps its own rate budget" + ); + assert_eq!( + limiter.admit(peer(1), at(600)).err(), + Some(Rejection::RateLimited) + ); + assert!( + limiter.admit(peer(1), at(1_000)).is_ok(), + "a refusal partway through the interval mustn't reset the clock the refill measures from" + ); + } + + #[test] + fn inflight_shedding_frees_on_drop_and_spends_no_rate_budget() { + let limiter = limiter(LimitConfig { + rate: rate(4, 1_000_000), + per_peer_inflight: Some(PerPeerInflight::new(1)), + global_inflight: Some(GlobalInflight::new(2)), + }); + let guard = limiter + .admit(peer(1), at(0)) + .expect("Limiter allows first request"); + let other = limiter + .admit(peer(2), at(0)) + .expect("a second peer fills the global budget"); + assert_eq!( + limiter.admit(peer(1), at(0)).err(), + Some(Rejection::Saturated), + "the limiter sheds a second concurrent request from one peer" + ); + assert_eq!( + limiter.admit(peer(3), at(0)).err(), + Some(Rejection::Saturated), + "the limiter sheds a third peer once the global in-flight limit is reached" + ); + drop(other); + drop( + limiter + .admit(peer(3), at(0)) + .expect("freeing a global slot readmits the peer that was shed"), + ); + drop(guard); + (0..50).for_each(|_| { + limiter + .admit(peer(1), at(0)) + .expect("a refunded admission must leave the full budget available") + .refund(); + }); + (0..3).for_each(|_| { + limiter + .admit(peer(1), at(0)) + .expect("a shed request mustn't spend the rate budget it never used"); + }); + assert_eq!( + limiter.admit(peer(1), at(0)).err(), + Some(Rejection::RateLimited), + "a dropped guard without a refund keeps its token spent" + ); + } + + #[test] + fn a_budget_without_a_rate_never_rate_limits_and_keeps_no_idle_state() { + let overflowing = MAX_TRACKED_PEERS as u64 + 1_000; + + let per_peer = limiter(LimitConfig::per_peer_only(PerPeerInflight::new(2))); + (0..1_000).for_each(|_| { + per_peer + .admit(peer(1), at(0)) + .expect("a budget with no rate has nothing for a sequential flood to exhaust"); + }); + let concurrent: Vec = (0..2) + .map(|_| { + per_peer + .admit(peer(1), at(0)) + .expect("Limiter allows both concurrent operations from one peer") + }) + .collect(); + assert_eq!( + per_peer.admit(peer(1), at(0)).err(), + Some(Rejection::Saturated), + "only per-peer count turns requests away in this budget" + ); + drop(concurrent); + let guards: Vec = (0..overflowing) + .map(|index| { + per_peer + .admit(rotating(index), at(0)) + .expect("a budget that keeps no idle state has no table to overflow") + }) + .collect(); + assert_eq!(tracked(&per_peer), guards.len()); + drop(guards); + assert_eq!( + tracked(&per_peer), + 0, + "with no tokens to remember, an idle peer leaves no entry, \ + so address rotation mustn't fill the table and start shedding newcomers" + ); + + let global = limiter(LimitConfig { + rate: None, + per_peer_inflight: None, + global_inflight: Some(GlobalInflight::new(1)), + }); + let _saturating = global + .admit(peer(1), at(0)) + .expect("Limiter allows first peer"); + (0..overflowing).for_each(|index| { + assert_eq!( + global.admit(rotating(index), at(0)).err(), + Some(Rejection::Saturated) + ); + }); + assert_eq!( + tracked(&global), + 1, + "rejection doesn't release its guard, entry it left behind would never be freed, and sweep that bounds table only reclaims idle rate state" + ); + + let unmetered = limiter(LimitConfig::unmetered()); + let guards: Vec = (0..512) + .map(|_| { + unmetered + .admit(peer(1), at(0)) + .expect("Unmetered budget allows every request from every peer") + }) + .collect(); + drop(guards); + assert_eq!(tracked(&unmetered), 0); + } + + #[test] + fn a_rate_budget_bounds_its_peer_map_and_sweeps_at_most_once_per_interval() { + let limiter = limiter(LimitConfig { + rate: rate(1, 1_000), + per_peer_inflight: Some(PerPeerInflight::new(8)), + global_inflight: Some(GlobalInflight::new(64)), + }); + (0..MAX_TRACKED_PEERS as u64).for_each(|index| { + let _ = limiter.admit(rotating(index), at(0)); + }); + assert_eq!( + limiter + .admit(peer(201), at(SWEEP_INTERVAL_MICROS - 1)) + .err(), + Some(Rejection::Saturated), + "inside the sweep interval a full map sheds unseen peers without rescanning" + ); + assert!( + limiter.admit(peer(202), at(SWEEP_INTERVAL_MICROS)).is_ok(), + "once the interval elapses, sweep evicts replenished entries and readmits the newcomer" + ); + (0..(MAX_TRACKED_PEERS as u64 + 50_000)).for_each(|index| { + let _ = limiter.admit( + rotating(MAX_TRACKED_PEERS as u64 + index), + at(SWEEP_INTERVAL_MICROS + index), + ); + }); + let tracked = tracked(&limiter); + assert!( + tracked <= MAX_TRACKED_PEERS, + "a flood of distinct source addresses mustn't grow the peer map past its limit, saw {tracked}" + ); + } + + #[test] + fn a_pacer_spends_a_hosts_burst_at_once_then_spaces_it_and_leaves_every_other_host_alone() { + let pacer = HostPacer::new(limit(3, 100)); + let plc = HostKey::new("plc.directory"); + let pds = HostKey::new("PDS.Nel.Pet"); + let waits: Vec = (0..5) + .map(|_| pacer.reserve(&plc, at(0)).as_micros()) + .collect(); + assert_eq!( + waits, + vec![0, 0, 0, 100, 200], + "a cold host takes its whole burst without waiting. Every visit after that is one \ + refill interval further out" + ); + assert_eq!( + pacer.reserve(&pds, at(0)), + Duration::ZERO, + "a knot whose accounts spread over many PDSes must fill at the sum of their rates, \ + so one busy host mustn't delay another" + ); + assert_eq!( + HostKey::new("PDS.Nel.Pet"), + HostKey::new("pds.nel.pet"), + "a PDS endpoint written in mixed case is the same host and must share its schedule" + ); + assert_eq!( + pacer.reserve(&plc, at(10_000)), + Duration::ZERO, + "a host the caller hasn't visited since its last turn is due immediately" + ); + } + + #[test] + fn a_turn_that_isnt_due_is_refused_without_pushing_the_schedule_further_out() { + let pacer = SubjectPacer::new(limit(1, 1_000)); + let nel = did("nel"); + assert!( + pacer.reserve_now(&nel, at(0)), + "a subject the caller hasn't read takes its turn straight away" + ); + assert!( + !pacer.reserve_now(&nel, at(500)), + "Caller that mustn't wait is turned away inside the interval" + ); + assert!( + !pacer.reserve_now(&nel, at(999)), + "rejections must leave booking alone, or whoever keeps trying pushes their turn \ + further out every time and the knot never reads the subject again" + ); + assert!(pacer.reserve_now(&nel, at(1_000))); + } + + #[test] + fn a_flood_of_distinct_hosts_doesnt_grow_the_schedule_past_its_limit_or_delay_a_newcomer() { + let pacer = HostPacer::new(limit(1, 1_000_000)); + (0..MAX_PACED_KEYS as u64 + 5_000).for_each(|index| { + let _ = pacer.reserve(&HostKey::new(&format!("{index}.nel.pet")), at(0)); + }); + let tracked = pacer.lock().turns.len(); + assert!( + tracked <= MAX_PACED_KEYS, + "a grant set spread over more did:web hosts than the schedule can track mustn't \ + grow it past its limit, saw {tracked}" + ); + assert_eq!( + pacer.reserve(&HostKey::new("plc.directory"), at(0)), + Duration::from_micros(1_000_000), + "a host the full schedule can't track waits one refill interval, so a caller that \ + outgrows the schedule slows itself down" + ); + let settled = at(SWEEP_INTERVAL_MICROS + 2_000_000); + assert_eq!( + pacer.reserve(&HostKey::new("plc.directory"), settled), + Duration::ZERO + ); + assert_eq!( + pacer.lock().turns.len(), + 1, + "the sweep reclaims the schedule and tracks the newcomer once every booked turn \ + has passed" + ); + } + + #[test] + fn the_majority_counter_sees_a_rejection_from_a_full_peer_table() { + let limiter = limiter(LimitConfig { + rate: rate(1, 1_000), + per_peer_inflight: Some(PerPeerInflight::new(8)), + global_inflight: Some(GlobalInflight::new(64)), + }); + (0..MAX_TRACKED_PEERS as u64).for_each(|index| { + let _ = limiter.admit(rotating(index), at(0)); + }); + assert_eq!( + limiter + .admit(peer(201), at(SWEEP_INTERVAL_MICROS - 1)) + .err(), + Some(Rejection::Saturated) + ); + let inner = limiter.lock(); + assert_eq!( + ( + inner.rejection_majority.peer, + inner.rejection_majority.votes + ), + (peer(201), 1), + "Peer shed by full table is what this warning exists for, and its rejection counts like any other" + ); + } + + fn writes(burst: u32, refill_micros: u64) -> SubjectLimiter { + SubjectLimiter::new(limit(burst, refill_micros)) + } + + fn did(suffix: &str) -> SubjectKey { + SubjectKey::new(&AccountDid::new(format!("did:plc:{suffix}")).unwrap()) + } + + #[test] + fn two_accounts_behind_one_address_spend_their_own_write_budgets() { + let limiter = writes(2, 1_000); + let (nel, olaren) = (did("nel"), did("olaren")); + assert!(limiter.admit(&nel, at(0)).is_ok()); + assert!(limiter.admit(&nel, at(0)).is_ok()); + assert_eq!( + limiter.admit(&nel, at(0)), + Err(Rejection::RateLimited), + "Third write inside one refill window is over burst" + ); + assert!( + limiter.admit(&olaren, at(0)).is_ok(), + "Shared NAT is one address and two writers, and both still write" + ); + assert!( + limiter.admit(&nel, at(1_000)).is_ok(), + "one refill interval later, first writer has its token again" + ); + } + + #[test] + fn one_account_keeps_one_bucket_and_two_dids_never_share_one() { + let limiter = writes(1, 10_000); + let nel = did("nel"); + assert!(limiter.admit(&nel, at(0)).is_ok()); + assert_eq!( + limiter.admit(&nel, at(0)), + Err(Rejection::RateLimited), + "Key is account, not connection account arrived on" + ); + let upper = SubjectKey::new(&AccountDid::new("did:web:knot.test:user:Nel").unwrap()); + let lower = SubjectKey::new(&AccountDid::new("did:web:knot.test:user:nel").unwrap()); + assert!( + limiter.admit(&upper, at(0)).is_ok(), + "two spellings are two accounts, did:web path is case-sensitive" + ); + assert!( + limiter.admit(&lower, at(0)).is_ok(), + "and second spelling has its own budget to spend" + ); + } + + #[test] + fn a_clock_stepped_backwards_doesnt_mint_a_token_or_strand_a_bucket() { + let limiter = writes(2, 1_000); + let nel = did("nel"); + assert!(limiter.admit(&nel, at(5_000)).is_ok()); + assert!( + limiter.admit(&nel, at(4_000)).is_ok(), + "second token of burst is still there, when clock steps back" + ); + assert_eq!( + limiter.admit(&nel, at(4_000)), + Err(Rejection::RateLimited), + "and step back doesn't mint one" + ); + assert!( + limiter.admit(&nel, at(5_000)).is_ok(), + "measuring refill from stepped-back clock keeps bucket from waiting for old future to come around again" + ); + } + + #[test] + fn a_flood_of_accounts_never_grows_the_write_map_without_bound() { + let limiter = writes(1, 10_000); + (0..MAX_METERED_SUBJECTS + 64).for_each(|index| { + let _ = limiter.admit(&did(&format!("flood{index}")), at(0)); + }); + assert!( + limiter.tracked() <= MAX_METERED_SUBJECTS, + "stranger, with a fresh DID per request, would otherwise leak memory" + ); + } + + #[test] + fn a_full_table_sheds_a_newcomer_until_its_coldest_writer_has_been_quiet_for_a_refill() { + let limiter = writes(1, 1_000_000); + let squatters: Vec = (0..MAX_METERED_SUBJECTS) + .map(|index| did(&format!("squatter{index}"))) + .collect(); + squatters.iter().enumerate().for_each(|(index, squatter)| { + let _ = limiter.admit(squatter, at(index as u64)); + }); + assert_eq!(limiter.tracked(), MAX_METERED_SUBJECTS); + + assert_eq!( + limiter.admit(&did("newcomer"), at(500_000)), + Err(Rejection::Saturated), + "every bucket in full table was spent inside its last refill. forgetting one would hand its account fresh burst, and knot sheds instead" + ); + assert_eq!( + limiter.admit(&squatters[MAX_METERED_SUBJECTS - 1], at(500_000)), + Err(Rejection::RateLimited), + "An account the table already meters is answered by its own bucket, not shed" + ); + assert!( + limiter.admit(&did("newcomer"), at(1_000_000)).is_ok(), + "once coldest writer has been quiet for refill, its bucket makes room" + ); + assert_eq!( + limiter.tracked(), + MAX_METERED_SUBJECTS, + "Newcomer took evicted slot rather than growing table" + ); + } + + #[test] + fn a_social_budget_allows_a_write_that_fits_and_turns_away_the_one_past_it() { + let budget = SocialBudgetBytes::new(1_000); + let cases: &[(u64, u64, bool)] = &[ + (0, 1_000, true), + (900, 100, true), + (900, 101, false), + (1_000, 0, true), + (1_000, 1, false), + (u64::MAX, 1, false), + ]; + cases.iter().for_each(|(current, adding, allowed)| { + assert_eq!( + budget.allows(SocialCharge { + current: StructureBytes::new(*current), + adding: StructureBytes::new(*adding), + }), + *allowed, + "Repository with {current} bytes and taking {adding} more" + ); + }); + } +} diff --git a/knot2/crates/knot-resource/src/cpu.rs b/knot2/crates/knot-resource/src/cpu.rs new file mode 100644 index 0000000000000000000000000000000000000000..5d5f19750b185fef9ae1079a509b8481eef2147d --- /dev/null +++ b/knot2/crates/knot-resource/src/cpu.rs @@ -0,0 +1,258 @@ +use std::sync::OnceLock; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct ThreadCount(usize); + +impl ThreadCount { + pub const fn new(count: usize) -> Self { + Self(if count == 0 { 1 } else { count }) + } + + pub const fn get(self) -> usize { + self.0 + } +} + +knot_types::scalar_newtype! { + struct WorkUnits(usize); +} + +struct Budget { + ceiling: usize, + available: AtomicUsize, +} + +static BUDGET: OnceLock = OnceLock::new(); + +fn detected_threads() -> ThreadCount { + ThreadCount::new( + std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1), + ) +} + +pub(crate) fn install(ceiling: ThreadCount) -> ThreadCount { + let budget = BUDGET.get_or_init(|| Budget { + ceiling: ceiling.get(), + available: AtomicUsize::new(ceiling.get().saturating_sub(1)), + }); + ThreadCount::new(budget.ceiling) +} + +fn budget() -> &'static Budget { + BUDGET.get_or_init(|| { + let ceiling = detected_threads().get(); + Budget { + ceiling, + available: AtomicUsize::new(ceiling.saturating_sub(1)), + } + }) +} + +pub fn threads() -> ThreadCount { + ThreadCount::new(budget().ceiling) +} + +pub fn gix_thread_limit() -> Option { + let budget = budget(); + (budget.ceiling < detected_threads().get()).then_some(ThreadCount::new(budget.ceiling)) +} + +pub(crate) fn ceiling() -> usize { + budget().ceiling +} + +static SATURATE: AtomicUsize = AtomicUsize::new(0); + +pub struct Saturate(()); + +impl Drop for Saturate { + fn drop(&mut self) { + SATURATE.fetch_sub(1, Ordering::Relaxed); + } +} + +// the eat my machine button +pub fn saturate() -> Saturate { + SATURATE.fetch_add(1, Ordering::Relaxed); + Saturate(()) +} + +struct Lease { + extra: usize, + saturated: bool, +} + +impl Lease { + fn none() -> Self { + Self { + extra: 0, + saturated: false, + } + } + + fn lanes(&self) -> usize { + self.extra + 1 + } +} + +impl Drop for Lease { + fn drop(&mut self) { + if !self.saturated && self.extra > 0 { + budget().available.fetch_add(self.extra, Ordering::AcqRel); + } + } +} + +fn lease(units: WorkUnits) -> Lease { + let budget = budget(); + let want = units + .get() + .saturating_sub(1) + .min(budget.ceiling.saturating_sub(1)); + if want == 0 { + return Lease::none(); + } + if SATURATE.load(Ordering::Relaxed) > 0 { + return Lease { + extra: want, + saturated: true, + }; + } + let mut available = budget.available.load(Ordering::Relaxed); + loop { + let grant = want.min(available); + if grant == 0 { + return Lease::none(); + } + match budget.available.compare_exchange_weak( + available, + available - grant, + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => { + return Lease { + extra: grant, + saturated: false, + }; + } + Err(observed) => available = observed, + } + } +} + +pub fn map_spans(len: usize, f: F) -> Result, E> +where + R: Send, + E: Send, + F: Fn(usize, usize) -> Result, E> + Sync, +{ + if len == 0 { + return Ok(Vec::new()); + } + let f = &f; + let lease = lease(WorkUnits::new(len)); + let lanes = lease.lanes().min(len); + if lanes <= 1 { + return f(0, len); + } + let chunk = len.div_ceil(lanes); + let spans: Vec<(usize, usize)> = (0..lanes) + .map(|lane| (lane * chunk, ((lane + 1) * chunk).min(len))) + .filter(|(start, end)| start < end) + .collect(); + let (head, tail) = spans + .split_first() + .expect("a positive lane count yields at least one span"); + let ordered: Vec, E>> = std::thread::scope(|scope| { + let handles: Vec<_> = tail + .iter() + .map(|&(start, end)| scope.spawn(move || f(start, end))) + .collect(); + let head = f(head.0, head.1); + std::iter::once(head) + .chain( + handles + .into_iter() + .map(|handle| handle.join().expect("resource worker panicked")), + ) + .collect() + }); + ordered.into_iter().try_fold(Vec::new(), |mut acc, part| { + acc.extend(part?); + Ok(acc) + }) +} + +pub fn map_chunks(items: &[T], f: F) -> Result, E> +where + T: Sync, + R: Send, + E: Send, + F: Fn(&[T]) -> Result, E> + Sync, +{ + map_spans(items.len(), |start, end| f(&items[start..end])) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_positive_span_count_covers_every_index_in_order() { + let doubled = map_spans::(1000, |start, end| { + Ok((start..end).map(|value| value * 2).collect()) + }) + .unwrap(); + assert_eq!(doubled.len(), 1000); + assert!( + doubled + .iter() + .enumerate() + .all(|(index, value)| *value == index * 2) + ); + } + + #[test] + fn an_empty_span_produces_nothing() { + assert_eq!( + map_spans::(0, |_, _| Ok(vec![1])).unwrap(), + Vec::::new() + ); + } + + #[test] + fn a_worker_error_propagates() { + let outcome = + map_spans::( + 500, + |start, _| { + if start == 0 { Err("boom") } else { Ok(vec![]) } + }, + ); + assert_eq!(outcome, Err("boom")); + } + + #[test] + fn chunks_preserve_element_order() { + let items: Vec = (0..777).collect(); + let echoed = map_chunks::(&items, |batch| Ok(batch.to_vec())).unwrap(); + assert_eq!(echoed, items); + } + + #[test] + fn a_lease_never_reserves_more_than_the_ceiling() { + let lease = lease(WorkUnits::new(usize::MAX)); + assert!(lease.lanes() <= threads().get()); + } + + #[test] + fn a_saturated_lease_still_respects_the_ceiling() { + let _boost = saturate(); + let lease = lease(WorkUnits::new(usize::MAX)); + assert!(lease.lanes() <= threads().get()); + } +} diff --git a/knot2/crates/knot-resource/src/disk.rs b/knot2/crates/knot-resource/src/disk.rs new file mode 100644 index 0000000000000000000000000000000000000000..5cfed683299099763442588c974c2dcc661d812c --- /dev/null +++ b/knot2/crates/knot-resource/src/disk.rs @@ -0,0 +1,143 @@ +use std::io; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +// 3 kinds of u64 that denote "bytes" in their own way +// & look identical at a callsite. +// `reserve(path, floor)` *used to* compile perfectly happily. +knot_types::scalar_newtype! { + pub struct DiskFloorBytes(u64); + pub struct ReserveBytes(u64); + pub struct FreeBytes(u64); +} + +pub fn free_bytes(path: &Path) -> io::Result { + rustix::fs::statvfs(path) + .map(|stat| FreeBytes::new(stat.f_bavail.saturating_mul(stat.f_frsize))) + .map_err(io::Error::from) +} + +#[derive(Debug)] +pub struct BelowFloor { + pub free: FreeBytes, + pub floor: DiskFloorBytes, +} + +struct Ledger { + floor: DiskFloorBytes, + reserved: AtomicU64, +} + +#[derive(Clone)] +pub struct DiskGovernor(Arc); + +impl DiskGovernor { + pub fn new(floor: DiskFloorBytes) -> Self { + Self(Arc::new(Ledger { + floor, + reserved: AtomicU64::new(0), + })) + } + + pub fn reserve_against( + &self, + free: FreeBytes, + bytes: ReserveBytes, + ) -> Result { + let amount = bytes.get(); + let projected = self.0.reserved.fetch_add(amount, Ordering::SeqCst) + amount; + if free.get() < self.0.floor.get().saturating_add(projected) { + self.0.reserved.fetch_sub(amount, Ordering::SeqCst); + return Err(BelowFloor { + free, + floor: self.0.floor, + }); + } + Ok(DiskReservation { + ledger: Arc::clone(&self.0), + bytes: amount, + }) + } +} + +pub struct DiskReservation { + ledger: Arc, + bytes: u64, +} + +impl Drop for DiskReservation { + fn drop(&mut self) { + self.ledger.reserved.fetch_sub(self.bytes, Ordering::SeqCst); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_real_filesystem_reports_headroom() { + let dir = std::env::temp_dir(); + assert!(free_bytes(&dir).unwrap().get() > 0); + } + + #[test] + fn a_missing_path_reports_the_fault() { + assert!(free_bytes(Path::new("/definitely/not/a/mounted/path")).is_err()); + } + + #[test] + fn a_reservation_holds_bytes_until_it_drops() { + let free = FreeBytes::new(1 << 20); + let governor = DiskGovernor::new(DiskFloorBytes::new(0)); + assert_eq!(governor.0.reserved.load(Ordering::SeqCst), 0); + { + let _held = governor + .reserve_against(free, ReserveBytes::new(4_096)) + .unwrap(); + assert_eq!(governor.0.reserved.load(Ordering::SeqCst), 4_096); + let _also = governor + .reserve_against(free, ReserveBytes::new(1_024)) + .unwrap(); + assert_eq!(governor.0.reserved.load(Ordering::SeqCst), 5_120); + } + assert_eq!(governor.0.reserved.load(Ordering::SeqCst), 0); + } + + #[test] + fn concurrent_reservations_cannot_jointly_punch_through_the_floor() { + let free = FreeBytes::new(10_240); + let governor = DiskGovernor::new(DiskFloorBytes::new(4_096)); + let first = governor + .reserve_against(free, ReserveBytes::new(4_096)) + .unwrap(); + let denied = governor.reserve_against(free, ReserveBytes::new(4_096)); + assert!( + denied.is_err(), + "the second reservation must see the first still held" + ); + assert_eq!(governor.0.reserved.load(Ordering::SeqCst), 4_096); + drop(first); + assert_eq!(governor.0.reserved.load(Ordering::SeqCst), 0); + assert!( + governor + .reserve_against(free, ReserveBytes::new(4_096)) + .is_ok(), + "a free-space reading the floor leaves room in must admit a lone reservation, or \ + the refusal above was the floor rather than the reservation still being held" + ); + } + + #[test] + fn a_refused_reservation_leaves_the_ledger_untouched() { + let governor = DiskGovernor::new(DiskFloorBytes::new(4_096)); + let refused = governor.reserve_against(FreeBytes::new(4_096), ReserveBytes::new(1)); + assert!(refused.is_err()); + assert_eq!( + governor.0.reserved.load(Ordering::SeqCst), + 0, + "a refusal that left its bytes on the ledger would deny every later upload too" + ); + } +} diff --git a/knot2/crates/knot-resource/src/fsio.rs b/knot2/crates/knot-resource/src/fsio.rs new file mode 100644 index 0000000000000000000000000000000000000000..270ceaf05e3090d274abb612cea3665deab92a87 --- /dev/null +++ b/knot2/crates/knot-resource/src/fsio.rs @@ -0,0 +1,347 @@ +use std::fs::File; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, SystemTime}; + +const STAGING_INFIX: &str = ".knot-tmp."; + +// A staging file belongs to whoever is filling it, and another process filling +// the same target is normal for a knot sharing a volume with a migrate +// or maintenance run. Only reclaim one old enough that no live writer could +// still own it. +const STAGING_REAP_AFTER: Duration = Duration::from_secs(3600); + +#[derive(Debug)] +pub struct FsError { + pub path: PathBuf, + pub source: io::Error, +} + +impl FsError { + fn at(path: &Path, source: io::Error) -> Self { + Self { + path: path.to_path_buf(), + source, + } + } +} + +impl std::fmt::Display for FsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {}", self.path.display(), self.source) + } +} + +impl std::error::Error for FsError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileMode { + Inherited, + Private, +} + +pub fn staging_nonce() -> String { + static COUNTER: AtomicU64 = AtomicU64::new(0); + format!( + "{}.{}", + std::process::id(), + COUNTER.fetch_add(1, Ordering::Relaxed) + ) +} + +fn staging_prefix(name: &str) -> String { + format!(".{name}{STAGING_INFIX}") +} + +fn pre_hidden_staging_prefix(name: &str) -> String { + format!("{name}{STAGING_INFIX}") +} + +fn remove_matching( + dir: &Path, + prefixes: &[&str], + reclaimable: impl Fn(&std::fs::DirEntry) -> bool, +) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + entries + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| prefixes.iter().any(|prefix| name.starts_with(prefix))) + }) + .filter(|entry| reclaimable(entry)) + .for_each(|entry| { + let _ = std::fs::remove_file(entry.path()); + }); +} + +pub fn clear_temps(dir: &Path, prefix: &str) { + remove_matching(dir, &[prefix], |_| true); +} + +pub fn clear_stale(dir: &Path, prefix: &str) { + let now = SystemTime::now(); + remove_matching(dir, &[prefix], |entry| abandoned(entry, now)); +} + +fn abandoned(entry: &std::fs::DirEntry, now: SystemTime) -> bool { + entry + .metadata() + .and_then(|meta| meta.modified()) + .ok() + .and_then(|modified| now.duration_since(modified).ok()) + .is_some_and(|age| age >= STAGING_REAP_AFTER) +} + +pub fn clear_staging(path: &Path) { + let (Some(parent), Some(name)) = ( + path.parent(), + path.file_name().and_then(|name| name.to_str()), + ) else { + return; + }; + let (hidden, pre_hidden) = (staging_prefix(name), pre_hidden_staging_prefix(name)); + let now = SystemTime::now(); + remove_matching(parent, &[&hidden, &pre_hidden], |entry| { + abandoned(entry, now) + }); +} + +pub fn fsync_path(path: &Path) -> Result<(), FsError> { + match File::open(path) { + Ok(file) => file.sync_all().map_err(|error| FsError::at(path, error)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(FsError::at(path, error)), + } +} + +fn create(path: &Path, mode: FileMode) -> io::Result { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + if mode == FileMode::Private { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + options.open(path) +} + +pub fn atomic_write(path: &Path, mode: FileMode, fill: F) -> Result<(), E> +where + F: FnOnce(&mut File) -> Result<(), E>, + E: From, +{ + let (Some(parent), Some(name)) = ( + path.parent(), + path.file_name().and_then(|name| name.to_str()), + ) else { + return Err(FsError::at(path, io::Error::from(io::ErrorKind::InvalidInput)).into()); + }; + clear_staging(path); + let staging = parent.join(format!("{}{}", staging_prefix(name), staging_nonce())); + + let outcome = create(&staging, mode) + .map_err(|error| E::from(FsError::at(path, error))) + .and_then(|mut file| { + fill(&mut file)?; + file.sync_all() + .map_err(|error| E::from(FsError::at(path, error))) + }) + .and_then(|()| { + std::fs::rename(&staging, path).map_err(|error| E::from(FsError::at(path, error))) + }); + + match outcome { + Ok(()) => fsync_path(parent).map_err(Into::into), + Err(error) => { + let _ = std::fs::remove_file(&staging); + Err(error) + } + } +} + +pub fn atomic_write_bytes(path: &Path, contents: &[u8], mode: FileMode) -> Result<(), FsError> { + let target = path.to_path_buf(); + atomic_write(path, mode, move |file| { + file.write_all(contents) + .map_err(|error| FsError::at(&target, error)) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn names_in(dir: &Path) -> Vec { + std::fs::read_dir(dir) + .unwrap() + .filter_map(Result::ok) + .filter_map(|entry| entry.file_name().to_str().map(str::to_string)) + .collect() + } + + fn age(path: &Path, by: Duration) { + let when = SystemTime::now() - by; + File::options() + .write(true) + .open(path) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(when)) + .unwrap(); + } + + #[test] + fn a_write_stages_under_a_hidden_name_and_leaves_only_the_target_behind() { + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("keys.sealed"); + let sibling = dir.path().join("keys.sealed.v2"); + let staging = std::sync::Mutex::new(Vec::new()); + atomic_write::(&target, FileMode::Private, |file| { + *staging.lock().unwrap() = names_in(dir.path()); + file.write_all(b"first") + .map_err(|error| FsError::at(&target, error)) + }) + .unwrap(); + atomic_write_bytes(&target, b"second", FileMode::Private).unwrap(); + atomic_write_bytes(&sibling, b"two", FileMode::Inherited).unwrap(); + + let staging = staging.into_inner().unwrap(); + assert_eq!( + staging + .iter() + .filter(|name| name.starts_with("keys.sealed")) + .count(), + 0, + "a tool matching on the target's own prefix mustn't find the half-written staging file" + ); + assert!( + staging + .iter() + .any(|name| name.starts_with(".keys.sealed.knot-tmp.")), + "saw {staging:?}" + ); + assert_eq!(std::fs::read(&target).unwrap(), b"second"); + assert_eq!( + std::fs::read(&sibling).unwrap(), + b"two", + "a name that extends another mustn't share its staging path" + ); + let left = names_in(dir.path()); + assert_eq!(left.len(), 2, "left staging files behind: {left:?}"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&target).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600, "sealed material stays owner-only"); + } + assert!(fsync_path(&dir.path().join("never-written")).is_ok()); + } + + #[test] + fn a_write_that_fails_or_overlaps_another_keeps_what_is_already_stored() { + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("packed-refs"); + atomic_write_bytes(&target, b"kept", FileMode::Inherited).unwrap(); + let failed: Result<(), FsError> = atomic_write(&target, FileMode::Inherited, |_| { + Err(FsError::at(&target, io::Error::other("fill failed"))) + }); + assert!(failed.is_err()); + assert_eq!( + std::fs::read(&target).unwrap(), + b"kept", + "a failed write mustn't destroy what was already stored" + ); + assert_eq!( + names_in(dir.path()).len(), + 1, + "a failed write removes its staging file" + ); + + let overlapping: Result<(), FsError> = atomic_write(&target, FileMode::Inherited, |file| { + atomic_write_bytes( + &target, + b"the second writer's contents", + FileMode::Inherited, + )?; + file.write_all(b"first writer finishes after") + .map_err(|error| FsError::at(&target, error)) + }); + assert!( + overlapping.is_ok(), + "an overlapping write mustn't delete the staging file this one is filling: \ + {overlapping:?}" + ); + assert_eq!( + std::fs::read(&target).unwrap(), + b"first writer finishes after" + ); + } + + #[test] + fn a_write_reclaims_aged_staging_files_of_either_naming_and_spares_a_fresh_one() { + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("packed-refs"); + let crashed = dir.path().join(".packed-refs.knot-tmp.4242.7"); + let pre_hidden = dir.path().join("packed-refs.knot-tmp.4242.8"); + let in_flight = dir.path().join(".packed-refs.knot-tmp.4243.0"); + std::fs::write(&crashed, b"left by a crashed run").unwrap(); + std::fs::write(&pre_hidden, b"left beside the target by an older build").unwrap(); + std::fs::write(&in_flight, b"another process is mid-write").unwrap(); + let aged = STAGING_REAP_AFTER + Duration::from_secs(60); + age(&crashed, aged); + age(&pre_hidden, aged); + + atomic_write_bytes(&target, b"fresh", FileMode::Inherited).unwrap(); + + assert!( + !crashed.exists(), + "the write reclaims a crashed run's staging file" + ); + assert!( + !pre_hidden.exists(), + "moving staging under a dot mustn't strand the temps the previous naming left" + ); + assert!( + in_flight.exists(), + "a second knot on the same volume is a supported deployment \ + whose live staging file this write mustn't sweep out from under its rename" + ); + } + + #[test] + fn a_prefix_sweep_spares_a_live_file_only_where_another_writer_could_own_it() { + let dir = tempfile::tempdir().unwrap(); + let crashed = dir.path().join(".knot-repack.4242.7.pack"); + let in_flight = dir.path().join(".knot-repack.4243.0.pack"); + let installed = dir.path().join("multi-pack-index"); + let bitmap = dir.path().join("multi-pack-index-abc.bitmap"); + std::fs::write(&crashed, b"left by a crashed run").unwrap(); + std::fs::write(&in_flight, b"another process is streaming into this").unwrap(); + std::fs::write(&installed, b"index").unwrap(); + std::fs::write(&bitmap, b"bitmap").unwrap(); + age(&crashed, STAGING_REAP_AFTER + Duration::from_secs(60)); + + clear_stale(dir.path(), ".knot-repack."); + clear_temps(dir.path(), "multi-pack-index"); + + assert!(!crashed.exists()); + assert!( + in_flight.exists(), + "deleting the staging pack another repack is streaming into fails that run's rename" + ); + assert!(!installed.exists()); + assert!( + !bitmap.exists(), + "removing the index without its bitmap would leave a bitmap describing an index that is gone" + ); + } +} diff --git a/knot2/crates/knot-resource/src/lib.rs b/knot2/crates/knot-resource/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..1425aead9893ab772fdcff13b0c61c34204b94cf --- /dev/null +++ b/knot2/crates/knot-resource/src/lib.rs @@ -0,0 +1,65 @@ +mod admission; +mod cpu; +mod disk; +mod fsio; +mod mem; +mod slots; + +pub use admission::{ + AdmitGuard, Bucket, Burst, GlobalInflight, HostKey, HostPacer, LimitConfig, PeerPacer, + PerPeerInflight, PreAuthLimiter, RateLimit, RefillMicros, Rejection, SocialBudgetBytes, + SocialCharge, StructureBytes, SubjectKey, SubjectLimiter, SubjectPacer, +}; +pub use cpu::{Saturate, ThreadCount, gix_thread_limit, map_chunks, map_spans, saturate, threads}; +pub use disk::{ + BelowFloor, DiskFloorBytes, DiskGovernor, DiskReservation, FreeBytes, ReserveBytes, + free_bytes as disk_free_bytes, +}; +pub use fsio::{ + FileMode, FsError, atomic_write, atomic_write_bytes, clear_staging, clear_stale, clear_temps, + fsync_path, staging_nonce, +}; +pub use mem::{ + AvailableBytes, BudgetSource, CacheBytes, ChurnBytes, ConnectivityObjects, DecayMs, + MemoryBudget, MemoryHighBytes, PayloadBytes, WorkingSetBytes, advert_cache_bytes, + available_bytes, cache_shed_warranted, decay_warrants_apply, externalize_connectivity, + ingest_admits, ingest_admits_churn, ingest_base_budget, ingest_thread_limit, + object_cache_bytes, pack_cache_bytes, target_decay, +}; +pub use slots::{PackSlots, ReceiveSlots, ResolveSlots, SlotPermit, Slots}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct Ceilings { + pub max_threads: Option, + pub max_memory: Option, +} + +#[derive(Clone, Copy, Debug)] +pub struct Report { + pub threads: ThreadCount, + pub memory: Option, + pub memory_source: BudgetSource, + pub memory_high_bytes: Option, +} + +pub fn init(ceilings: Ceilings) -> Report { + let threads = cpu::install( + ceilings + .max_threads + .unwrap_or_else(|| ThreadCount::new(default_threads())), + ); + let (memory, memory_source) = mem::install(ceilings.max_memory); + let memory_high_bytes = mem::try_set_memory_high(); + Report { + threads, + memory, + memory_source, + memory_high_bytes, + } +} + +fn default_threads() -> usize { + std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1) +} diff --git a/knot2/crates/knot-resource/src/mem.rs b/knot2/crates/knot-resource/src/mem.rs new file mode 100644 index 0000000000000000000000000000000000000000..bd404bfef4a5267ced03710c486b30a901f4e282 --- /dev/null +++ b/knot2/crates/knot-resource/src/mem.rs @@ -0,0 +1,833 @@ +//! # Memory sensing & sizing. +//! +//! Everything is derived from: +//! - `MemoryBudget`: how much mem this process may use in total, +//! which we detect once then cache. +//! - `AvailableBytes`: how much is free right now, +//! which we read afresh every call. +//! +//! Sizing decisions that have to stay fixed for the lifetime of +//! a given process will refer to the mem budget, +//! while decisions that react to load will take a live reading +//! on the fly like a pulse. +//! +//! ```text +//! MemoryBudget, which is detected then cached +//! +//! configured cgroup MemTotal the budget source +//! ---------- ------ -------- ---------------- -------------------- +//! set any any configured Configured +//! - reads reads the smaller whichever won +//! - reads - the cgroup limit CgroupV2 or CgroupV1 +//! - - reads MemTotal ProcMeminfo +//! - - - None Unconstrained +//! +//! AvailableBytes, read every call +//! +//! cgroup v2 max -> anon, else v1 limit -> usage, else MemAvailable, else None +//! +//! MemoryBudget ---> object_cache_bytes, pack_cache_bytes, +//! \ ingest_base_budget +//! +--------> target_decay +//! / +//! AvailableBytes -> ingest_thread_limit, ingest_admits, +//! ingest_admits_churn, externalize_connectivity +//! ``` +//! +//! *Figure 1: where `MemoryBudget` & `AvailableBytes` originate, along with receivers.* +//! +//! In Figure 1, +//! a `-` means that source is absent or unreadable, +//! and `any` means we don't consult it at all. +//! The `cgroup` column is cgroup v2 `memory.max`, +//! or cgroup v1 `memory.limit_in_bytes` when the v2 file is +//! missing or just reads `max`. +//! +//! Every cgroup v2 reading reads this cgroup +//! up to root and takes the smallest `memory.max` it finds up the chain. +//! A `None` budget means unconstrained, +//! so every clamp returns its caller's +//! ceiling and `target_decay` reports a healthy interval. +//! +//! A `None` live-reading means the sensor is unavailable, +//! so every check takes the most permissive variant: +//! admission returns true, the thread limit stays at the CPU ceiling, +//! and `externalize_connectivity` returns false such that the +//! connectivity-map stays in RAM. +//! +//! Long story short, if the knot can't detect any limits it'll assume +//! it's allowed everything it can handle. +//! +//! In contrast, +//! `memory_high_target` uses neither of the above, +//! since `try_set_memory_high` passes it the cgroup v2 max directly, +//! which means a configured budget, +//! a `MemTotal` budget, and a cgroup v1 limit all don't affect it. +//! +//! `clamp_to_budget` will choose its cache size from a percentage of the budget, +//! between a floor and a ceiling. +//! Each of the following steps wins in some situation, +//! which Table 1 traces with the `object_cache_bytes` constants of +//! ceiling 64M, percent 2, floor 8M: +//! +//! ```text +//! ceiling.min(max(budget / 100 * percent, floor)).min(budget) +//! +//! budget budget*pct max(.,floor) min(ceiling,.) min(.,budget) winner +//! ------ ---------- ------------ -------------- ------------- ------- +//! 4M 0.08M 8M 8M 4M budget +//! 256M 5.1M 8M 8M 8M floor +//! 1G 20.5M 20.5M 20.5M 20.5M percent +//! 8G 163.8M 163.8M 64M 64M ceiling +//! ``` +//! +//! *Table 1: a budget per row, and which step decided it.* +//! +//! The 4M row is the only one where trailing `min` actually does anything, +//! since it covers a host whose entire budget is below the floor. +//! +//! `decay_for_headroom` maps headroom, meaning available over-budget, +//! onto the jemalloc dirty-page decay interval. +//! +//! // TODO: research if I can do this with mimalloc. +//! +//! When there's a lot of headroom, pages will stay cached for ten seconds, +//! but while under pressure the decay drops to zero such that pages go +//! back to the OS immediately. +//! Between those thresholds it interpolates like: +//! +//! ```text +//! decay ms +//! 10000 | ------------------ +//! | ,-' +//! | ,-' +//! | ,-' +//! | ,-' +//! | ,-' +//! 0 +===+-------------+-----------------+ +//! 0% 10% 50% 100% +//! headroom = available / budget +//! ``` +//! +//! *Figure 2: headroom mapped onto dirty-page decay interval.* +//! +//! That `=` run below 10% in Figure 2 is the curve itself, +//! I meant flat at zero, not the axis. :P +//! Between the thresholds the ramp climbs 250ms per point of headroom. +//! +//! `decay_warrants_apply` judges writes against the above curve. +//! Any move smaller than one second will be ignored, +//! so the reading has to shift like 4 points of +//! headroom before we rewrite the setting. +//! A target of 0 is exempt and always applies, +//! unless it happens to be the applied value already. + +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +use crate::cpu::ThreadCount; + +const CGROUP_V2_ROOT: &str = "/sys/fs/cgroup"; +const PROC_SELF_CGROUP: &str = "/proc/self/cgroup"; +const MEMORY_V1_LIMIT_PATH: &str = "/sys/fs/cgroup/memory/memory.limit_in_bytes"; +const MEMORY_V1_USAGE_PATH: &str = "/sys/fs/cgroup/memory/memory.usage_in_bytes"; +const MEMINFO_PATH: &str = "/proc/meminfo"; + +const CGROUP_V1_UNLIMITED: u64 = 0x7FFF_FFFF_FFFF_F000; + +const HIGH_HEADROOM_PERCENT: u64 = 10; +const HIGH_HEADROOM_LIMIT: u64 = 1024 * 1024 * 1024; + +const OBJECT_CACHE_CEILING: u64 = 64 * 1024 * 1024; +const OBJECT_CACHE_PERCENT: Percent = Percent::new(2); +const OBJECT_CACHE_FLOOR: u64 = 8 * 1024 * 1024; + +const PACK_CACHE_PERCENT: Percent = Percent::new(25); +const PACK_CACHE_FLOOR: u64 = 32 * 1024 * 1024; + +const ADVERT_CACHE_CEILING: u64 = 128 * 1024 * 1024; +const ADVERT_CACHE_PERCENT: Percent = Percent::new(5); +const ADVERT_CACHE_FLOOR: u64 = 8 * 1024 * 1024; + +const CACHE_SHED_PERCENT: u64 = 10; + +const INGEST_BASE_PERCENT: u64 = 40; + +const DECAY_HEALTHY_MS: isize = 10_000; +const DECAY_PRESSURE_MS: isize = 0; +const DECAY_HYSTERESIS_MS: isize = 1_000; +const HEADROOM_RELAXED_PERCENT: u64 = 50; +const HEADROOM_TIGHT_PERCENT: u64 = 10; + +const CONNECTIVITY_BYTES_PER_OBJECT: u64 = 96; +const INGEST_FIXED_BYTES: u64 = 16 * 1024 * 1024; +const INGEST_THREAD_WORKING_BYTES: u64 = 12 * 1024 * 1024; +const INGEST_CONCURRENCY_BYTES: u64 = 256 * 1024 * 1024; +const INGEST_CHURN_MULTIPLE: u64 = 4; + +knot_types::scalar_newtype! { + pub struct MemoryBudget(u64); +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Percent(u64); + +impl Percent { + pub const fn new(percent: u64) -> Self { + // Const so a `250` would fail build for example. + // `clamp_to_budget` would otherwise have major problems + // at runtime. + assert!(percent <= 100, "percent exceeds 100"); + Self(percent) + } + + pub const fn get(self) -> u64 { + self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BudgetSource { + Configured, + CgroupV2, + CgroupV1, + ProcMeminfo, + Unconstrained, +} + +static BUDGET: OnceLock<(Option, BudgetSource)> = OnceLock::new(); + +pub(crate) fn install(configured: Option) -> (Option, BudgetSource) { + let computed = match configured { + Some(budget) => (Some(budget), BudgetSource::Configured), + None => detect_memory_budget(), + }; + *BUDGET.get_or_init(|| computed) +} + +fn resolved() -> Option { + BUDGET.get_or_init(detect_memory_budget).0 +} + +fn detect_cgroup_limit() -> Option<(u64, BudgetSource)> { + read_cgroup_max() + .map(|limit| (limit, BudgetSource::CgroupV2)) + .or_else(|| read_cgroup_v1_max().map(|limit| (limit, BudgetSource::CgroupV1))) +} + +fn detect_memory_budget() -> (Option, BudgetSource) { + match (detect_cgroup_limit(), read_meminfo_total()) { + (Some((cgroup, source)), Some(total)) => { + if cgroup <= total { + (Some(MemoryBudget::new(cgroup)), source) + } else { + (Some(MemoryBudget::new(total)), BudgetSource::ProcMeminfo) + } + } + (Some((cgroup, source)), None) => (Some(MemoryBudget::new(cgroup)), source), + (None, Some(total)) => (Some(MemoryBudget::new(total)), BudgetSource::ProcMeminfo), + (None, None) => (None, BudgetSource::Unconstrained), + } +} + +fn cgroup_v2_dir() -> Option { + let content = std::fs::read_to_string(PROC_SELF_CGROUP).ok()?; + let relative = content + .lines() + .find_map(|line| line.strip_prefix("0::"))? + .trim(); + Some(Path::new(CGROUP_V2_ROOT).join(relative.trim_start_matches('/'))) +} + +fn read_cgroup_max() -> Option { + let root = Path::new(CGROUP_V2_ROOT); + let mut dir = cgroup_v2_dir()?; + let mut effective: Option = None; + loop { + if let Some(limit) = std::fs::read_to_string(dir.join("memory.max")) + .ok() + .and_then(|raw| parse_cgroup_max(&raw)) + { + effective = Some(effective.map_or(limit, |current| current.min(limit))); + } + if dir == root { + break; + } + match dir.parent() { + Some(parent) if parent.starts_with(root) => dir = parent.to_path_buf(), + _ => break, + } + } + effective +} + +fn parse_cgroup_max(raw: &str) -> Option { + match raw.trim() { + "max" => None, + bytes => bytes.parse::().ok(), + } +} + +fn read_meminfo_total() -> Option { + parse_meminfo_field(&std::fs::read_to_string(MEMINFO_PATH).ok()?, "MemTotal:") +} + +fn parse_meminfo_field(raw: &str, key: &str) -> Option { + raw.lines() + .find_map(|line| line.strip_prefix(key)) + .and_then(|rest| rest.trim().strip_suffix("kB")) + .and_then(|kb| kb.trim().parse::().ok()) + .map(|kb| kb.saturating_mul(1024)) +} + +fn read_u64_file(path: &str) -> Option { + std::fs::read_to_string(path) + .ok()? + .trim() + .parse::() + .ok() +} + +fn read_memory_stat_field(stat: &str, key: &str) -> Option { + stat.lines().find_map(|line| { + let mut parts = line.split_whitespace(); + match (parts.next(), parts.next()) { + (Some(name), Some(value)) if name == key => value.parse::().ok(), + _ => None, + } + }) +} + +fn cgroup_v2_available() -> Option { + let dir = cgroup_v2_dir()?; + let stat = std::fs::read_to_string(dir.join("memory.stat")).ok()?; + let anon = read_memory_stat_field(&stat, "anon")?; + Some(read_cgroup_max()?.saturating_sub(anon)) +} + +fn read_cgroup_v1_max() -> Option { + read_u64_file(MEMORY_V1_LIMIT_PATH).filter(|&limit| limit < CGROUP_V1_UNLIMITED) +} + +fn cgroup_v1_available() -> Option { + let limit = read_cgroup_v1_max()?; + Some(limit.saturating_sub(read_u64_file(MEMORY_V1_USAGE_PATH)?)) +} + +fn meminfo_available() -> Option { + parse_meminfo_field( + &std::fs::read_to_string(MEMINFO_PATH).ok()?, + "MemAvailable:", + ) +} + +pub fn available_bytes() -> Option { + cgroup_v2_available() + .or_else(cgroup_v1_available) + .or_else(meminfo_available) + .map(AvailableBytes::new) +} + +fn clamp_to_budget( + budget: Option, + ceiling: u64, + percent: Percent, + floor: u64, +) -> u64 { + match budget { + None => ceiling, + Some(budget) => ceiling + .min((budget.get() / 100 * percent.get()).max(floor)) + .min(budget.get()), + } +} + +pub fn object_cache_bytes() -> CacheBytes { + CacheBytes::new(clamp_to_budget( + resolved(), + OBJECT_CACHE_CEILING, + OBJECT_CACHE_PERCENT, + OBJECT_CACHE_FLOOR, + )) +} + +pub fn pack_cache_bytes(configured: u64) -> CacheBytes { + CacheBytes::new(clamp_to_budget( + resolved(), + configured, + PACK_CACHE_PERCENT, + PACK_CACHE_FLOOR, + )) +} + +pub fn advert_cache_bytes() -> CacheBytes { + CacheBytes::new(clamp_to_budget( + resolved(), + ADVERT_CACHE_CEILING, + ADVERT_CACHE_PERCENT, + ADVERT_CACHE_FLOOR, + )) +} + +pub fn cache_shed_warranted() -> bool { + shed_warranted_at(available_bytes(), resolved()) +} + +fn shed_warranted_at(available: Option, budget: Option) -> bool { + match (available, budget) { + (Some(available), Some(budget)) if budget.get() > 0 => { + available.get().saturating_mul(100) / budget.get() < CACHE_SHED_PERCENT + } + _ => false, + } +} + +fn ingest_base_budget_for(budget: MemoryBudget) -> CacheBytes { + CacheBytes::new(budget.get() / 100 * INGEST_BASE_PERCENT) +} + +pub fn ingest_base_budget() -> Option { + resolved().map(ingest_base_budget_for) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DecayMs(isize); + +impl DecayMs { + pub const fn ms(self) -> isize { + self.0 + } +} + +fn decay_for_headroom(available: Option, budget: Option) -> DecayMs { + let (available, budget) = match (available, budget) { + (Some(available), Some(budget)) if budget.get() > 0 => (available.get(), budget.get()), + _ => return DecayMs(DECAY_HEALTHY_MS), + }; + let headroom_percent = available.saturating_mul(100) / budget; + let decay = if headroom_percent >= HEADROOM_RELAXED_PERCENT { + DECAY_HEALTHY_MS + } else if headroom_percent <= HEADROOM_TIGHT_PERCENT { + DECAY_PRESSURE_MS + } else { + let span = (HEADROOM_RELAXED_PERCENT - HEADROOM_TIGHT_PERCENT) as isize; + let above = (headroom_percent - HEADROOM_TIGHT_PERCENT) as isize; + DECAY_HEALTHY_MS * above / span + }; + DecayMs(decay) +} + +pub fn target_decay() -> DecayMs { + decay_for_headroom(available_bytes(), resolved()) +} + +pub fn decay_warrants_apply(applied: DecayMs, target: DecayMs) -> bool { + if applied == target { + false + } else if target.ms() == DECAY_PRESSURE_MS { + true + } else { + (target.ms() - applied.ms()).abs() >= DECAY_HYSTERESIS_MS + } +} + +fn connectivity_fits(count: ConnectivityObjects, available: Option) -> bool { + match available { + Some(available) => { + count.get().saturating_mul(CONNECTIVITY_BYTES_PER_OBJECT) <= available.get() / 2 + } + None => true, + } +} + +pub fn externalize_connectivity(count: ConnectivityObjects) -> bool { + !connectivity_fits(count, available_bytes()) +} + +fn ingest_threads_for(ceiling: usize, available: Option) -> usize { + match available { + Some(available) => { + let funded = + available.get().saturating_sub(INGEST_FIXED_BYTES) / INGEST_CONCURRENCY_BYTES; + ceiling + .min(usize::try_from(funded).unwrap_or(ceiling)) + .max(1) + } + None => ceiling, + } +} + +fn ingest_floor_for(ceiling: usize, available: AvailableBytes) -> u64 { + INGEST_FIXED_BYTES + + ingest_threads_for(ceiling, Some(available)) as u64 * INGEST_THREAD_WORKING_BYTES +} + +fn ingest_admits_for( + ceiling: usize, + available: Option, + payload_bytes: PayloadBytes, +) -> bool { + match available { + Some(available) => { + ingest_floor_for(ceiling, available).saturating_add(payload_bytes.get()) + <= available.get() + } + None => true, + } +} + +pub fn ingest_thread_limit() -> ThreadCount { + ThreadCount::new(ingest_threads_for(crate::cpu::ceiling(), available_bytes())) +} + +pub fn ingest_admits(payload: PayloadBytes) -> bool { + ingest_admits_for(crate::cpu::ceiling(), available_bytes(), payload) +} + +knot_types::scalar_newtype! { + pub struct WorkingSetBytes(u64); + pub struct ChurnBytes(u64); + pub struct AvailableBytes(u64); + pub struct PayloadBytes(u64); + pub struct ConnectivityObjects(u64); + pub struct MemoryHighBytes(u64); + pub struct CacheBytes(u64); +} + +fn ingest_admits_churn_for( + ceiling: usize, + available: Option, + working_set: WorkingSetBytes, + churn: ChurnBytes, +) -> bool { + match available { + Some(available) => { + ingest_admits_for(ceiling, Some(available), PayloadBytes::new(working_set.0)) + && churn.0 <= available.get().saturating_mul(INGEST_CHURN_MULTIPLE) + } + None => true, + } +} + +pub fn ingest_admits_churn(working_set: WorkingSetBytes, churn: ChurnBytes) -> bool { + ingest_admits_churn_for(crate::cpu::ceiling(), available_bytes(), working_set, churn) +} + +fn memory_high_target(max: u64) -> u64 { + let headroom = (max / 100 * HIGH_HEADROOM_PERCENT).min(HIGH_HEADROOM_LIMIT); + max.saturating_sub(headroom) +} + +pub(crate) fn try_set_memory_high() -> Option { + let dir = cgroup_v2_dir()?; + let high = memory_high_target(read_cgroup_max()?); + std::fs::write(dir.join("memory.high"), high.to_string()) + .ok() + .map(|()| MemoryHighBytes::new(high)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const GIB: u64 = 1024 * 1024 * 1024; + const MIB: u64 = 1024 * 1024; + + #[test] + fn an_unlimited_cgroup_reads_as_no_budget() { + assert_eq!(parse_cgroup_max("max\n"), None); + assert_eq!(parse_cgroup_max("104857600\n"), Some(104_857_600)); + assert_eq!(parse_cgroup_max("garbage"), None); + } + + #[test] + fn meminfo_fields_parse_kilobytes_into_bytes() { + let sample = "MemTotal: 16384 kB\nMemFree: 100 kB\nMemAvailable: 8192 kB\n"; + assert_eq!(parse_meminfo_field(sample, "MemTotal:"), Some(16384 * 1024)); + assert_eq!( + parse_meminfo_field(sample, "MemAvailable:"), + Some(8192 * 1024) + ); + assert_eq!(parse_meminfo_field(sample, "Nothing:"), None); + } + + #[test] + fn memory_stat_matches_the_whole_key_not_a_prefix() { + let sample = "anon 2097152\nfile 8388608\nanon_thp 0\nkernel 65536\n"; + assert_eq!(read_memory_stat_field(sample, "anon"), Some(2_097_152)); + assert_eq!(read_memory_stat_field(sample, "file"), Some(8_388_608)); + assert_eq!(read_memory_stat_field(sample, "anon_thp"), Some(0)); + assert_eq!(read_memory_stat_field(sample, "missing"), None); + } + + #[test] + fn the_sensor_reads_live_memory_on_this_host() { + let available = available_bytes() + .expect("a Linux host must expose live memory availability") + .get(); + assert!( + available > 0, + "available memory must be positive, got {available}" + ); + } + + #[test] + fn an_unconstrained_host_keeps_the_ceiling() { + assert_eq!( + clamp_to_budget(None, 500_000_000, Percent::new(25), 32), + 500_000_000 + ); + } + + #[test] + fn a_constrained_host_clamps_to_the_fraction() { + let budget = Some(MemoryBudget::new(400 * MIB)); + assert_eq!( + clamp_to_budget(budget, 4 * GIB, PACK_CACHE_PERCENT, PACK_CACHE_FLOOR), + 100 * MIB + ); + } + + #[test] + fn a_tiny_host_holds_the_floor_but_never_exceeds_the_budget() { + let budget = Some(MemoryBudget::new(16 * MIB)); + assert_eq!( + clamp_to_budget(budget, 4 * GIB, PACK_CACHE_PERCENT, PACK_CACHE_FLOOR), + 16 * MIB + ); + } + + #[test] + fn a_small_host_reserves_the_headroom_percent() { + let max = 4 * GIB; + let headroom = max / 100 * HIGH_HEADROOM_PERCENT; + assert!(headroom < HIGH_HEADROOM_LIMIT); + assert_eq!(memory_high_target(max), max - headroom); + } + + #[test] + fn a_large_host_bounds_the_reclaim_headroom() { + let max = 128 * GIB; + assert!(max / 100 * HIGH_HEADROOM_PERCENT > HIGH_HEADROOM_LIMIT); + assert_eq!(memory_high_target(max), max - HIGH_HEADROOM_LIMIT); + } + + #[test] + fn a_healthy_host_keeps_the_allocator_lazy_and_a_squeezed_one_reclaims() { + let budget = Some(MemoryBudget::new(4 * GIB)); + assert_eq!( + decay_for_headroom(Some(AvailableBytes::new(3 * GIB)), budget).ms(), + DECAY_HEALTHY_MS, + "ample headroom stays fast" + ); + assert_eq!( + decay_for_headroom(Some(AvailableBytes::new(GIB / 4)), budget).ms(), + DECAY_PRESSURE_MS, + "near-exhaustion reclaims at once" + ); + assert_eq!( + decay_for_headroom(Some(AvailableBytes::new(2 * GIB)), budget).ms(), + DECAY_HEALTHY_MS, + "half-free sits at the relaxed threshold" + ); + } + + #[test] + fn cache_shedding_triggers_only_under_tight_headroom() { + let budget = Some(MemoryBudget::new(4 * GIB)); + assert!( + !shed_warranted_at(Some(AvailableBytes::new(2 * GIB)), budget), + "ample headroom keeps caches" + ); + assert!( + shed_warranted_at(Some(AvailableBytes::new(GIB / 4)), budget), + "tight headroom sheds caches" + ); + assert!( + !shed_warranted_at(Some(AvailableBytes::new(GIB)), None), + "an unmeasured budget never sheds" + ); + } + + #[test] + fn the_decay_interpolates_across_the_pressure_band() { + let budget = Some(MemoryBudget::new(100 * MIB)); + assert_eq!( + decay_for_headroom(Some(AvailableBytes::new(30 * MIB)), budget).ms(), + DECAY_HEALTHY_MS * 20 / 40, + "30% headroom is halfway through the 10..50 band" + ); + } + + #[test] + fn ingest_parallelism_backs_off_as_memory_tightens() { + assert_eq!( + ingest_threads_for(8, Some(AvailableBytes::new(4 * GIB))), + 8, + "a roomy host keeps the full cpu ceiling" + ); + assert_eq!( + ingest_threads_for(8, Some(AvailableBytes::new(GIB))), + 3, + "a 1GB limit funds only ~3 ingest threads, far below a many-core ceiling, so \ + decompression churn cannot outrun the munmap-on-free page return and grow unbounded" + ); + assert_eq!( + ingest_threads_for(8, Some(AvailableBytes::new(176 * MIB))), + 1, + "a squeezed host drops to a single ingest thread, shrinking the working set" + ); + assert_eq!( + ingest_threads_for(8, None), + 8, + "an unmeasurable host keeps the ceiling" + ); + } + + #[test] + fn the_base_spill_budget_stays_a_fraction_so_it_can_bound_a_small_host() { + assert_eq!( + ingest_base_budget_for(MemoryBudget::new(64 * GIB)), + CacheBytes::new(64 * GIB / 100 * INGEST_BASE_PERCENT), + "a roomy host spills only after the working set passes 40% of its RAM" + ); + assert!( + ingest_base_budget_for(MemoryBudget::new(300 * MIB)).get() < 300 * MIB, + "a squeezed host keeps the spill threshold under its total, or it OOMs before paging" + ); + } + + #[test] + fn ingest_admission_scales_its_floor_with_the_threads_it_will_actually_use() { + assert!( + ingest_admits_for( + 8, + Some(AvailableBytes::new(32 * MIB)), + PayloadBytes::new(MIB) + ), + "a small push fits a 32MB host by running a single ~28MB-floor ingest thread" + ); + assert!( + !ingest_admits_for( + 8, + Some(AvailableBytes::new(20 * MIB)), + PayloadBytes::new(MIB) + ), + "below the one-thread floor the push is shed, never OOM-ed part way through" + ); + assert!( + !ingest_admits_for( + 8, + Some(AvailableBytes::new(64 * MIB)), + PayloadBytes::new(200 * MIB) + ), + "a payload that dwarfs free memory is declined" + ); + assert!( + ingest_admits_for(8, None, PayloadBytes::new(u64::MAX)), + "an unmeasurable host proceeds optimistically" + ); + } + + #[test] + fn ingest_churn_sheds_a_pack_whose_decompressed_volume_dwarfs_free_memory() { + assert!( + ingest_admits_churn_for( + 8, + Some(AvailableBytes::new(GIB)), + WorkingSetBytes(MIB), + ChurnBytes(3 * GIB) + ), + "churn within a few multiples of free memory rides on the working-set floor" + ); + assert!( + !ingest_admits_churn_for( + 8, + Some(AvailableBytes::new(GIB)), + WorkingSetBytes(MIB), + ChurnBytes(5 * GIB) + ), + "decompression volume past the multiple of free memory is shed, never OOM-ed" + ); + assert!( + !ingest_admits_churn_for( + 8, + Some(AvailableBytes::new(20 * MIB)), + WorkingSetBytes(MIB), + ChurnBytes(MIB) + ), + "below the one-thread working floor the pack is shed even with trivial churn" + ); + assert!( + ingest_admits_churn_for(8, None, WorkingSetBytes(u64::MAX), ChurnBytes(u64::MAX)), + "an unmeasurable host proceeds optimistically on both gates" + ); + } + + #[test] + fn connectivity_externalizes_only_when_the_in_ram_map_would_crowd_the_host() { + assert!( + connectivity_fits( + ConnectivityObjects::new(1_000_000), + Some(AvailableBytes::new(4 * GIB)) + ), + "a small closure fits with headroom to spare" + ); + assert!( + !connectivity_fits( + ConnectivityObjects::new(7_700_000), + Some(AvailableBytes::new(512 * MIB)) + ), + "nixpkgs cannot hold its connectivity map on a 512MB box" + ); + assert!( + connectivity_fits(ConnectivityObjects::new(u64::MAX), None), + "an unmeasurable host stays on the fast in-ram path" + ); + } + + #[test] + fn an_unmeasurable_host_stays_on_the_fast_default() { + assert_eq!(decay_for_headroom(None, None).ms(), DECAY_HEALTHY_MS); + assert_eq!( + decay_for_headroom(Some(AvailableBytes::new(GIB)), None).ms(), + DECAY_HEALTHY_MS, + "no budget means no pressure signal, so don't throttle" + ); + } + + #[test] + fn a_big_host_scales_up_to_the_ceiling_only() { + let budget = Some(MemoryBudget::new(256 * GIB)); + assert_eq!( + clamp_to_budget( + budget, + OBJECT_CACHE_CEILING, + OBJECT_CACHE_PERCENT, + OBJECT_CACHE_FLOOR + ), + OBJECT_CACHE_CEILING + ); + } + + #[test] + fn decay_hysteresis_absorbs_small_wobble_but_honors_the_pressure_floor() { + let healthy = DecayMs(DECAY_HEALTHY_MS); + assert!( + !decay_warrants_apply(healthy, healthy), + "an unchanged target never rewrites the arenas" + ); + assert!( + !decay_warrants_apply(DecayMs(5_000), DecayMs(5_200)), + "a sub-band change is ignored so a percent of headroom wobble doesn't churn" + ); + assert!( + decay_warrants_apply(DecayMs(5_000), DecayMs(7_000)), + "a change past the hysteresis band is applied" + ); + assert!( + decay_warrants_apply(DecayMs(200), DecayMs(DECAY_PRESSURE_MS)), + "a move to the pressure floor is always honored so RSS reclaim is never delayed" + ); + } +} diff --git a/knot2/crates/knot-resource/src/slots.rs b/knot2/crates/knot-resource/src/slots.rs new file mode 100644 index 0000000000000000000000000000000000000000..7ff91d3a8fe140a175e02446f7e83b04105ac114 --- /dev/null +++ b/knot2/crates/knot-resource/src/slots.rs @@ -0,0 +1,160 @@ +use std::sync::{Arc, OnceLock}; + +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +use crate::cpu::threads; + +const RESOLVE_SLOTS_PER_TRANSPORT: usize = 16; + +const SHARING_TRANSPORTS: usize = 2; + +const RESOLVE_SLOTS: usize = RESOLVE_SLOTS_PER_TRANSPORT * SHARING_TRANSPORTS; + +const THREADS_PER_BLAME_SLOT: usize = 2; +const MAX_BLAME_SLOTS: usize = 4; + +static PROCESS: OnceLock = OnceLock::new(); + +pub struct SlotPermit(#[allow(dead_code)] OwnedSemaphorePermit); + +macro_rules! slot_kind { + ($($name:ident),+ $(,)?) => {$( + #[derive(Clone)] + pub struct $name(Arc); + + impl $name { + pub fn new(permits: usize) -> Self { + Self(Arc::new(Semaphore::new(permits.max(1)))) + } + + pub async fn acquire(&self) -> SlotPermit { + SlotPermit( + Arc::clone(&self.0) + .acquire_owned() + .await + .expect("a slot budget closes only with the process that owns it"), + ) + } + + pub fn available(&self) -> usize { + self.0.available_permits() + } + } + )+}; +} + +slot_kind!(ResolveSlots, ReceiveSlots, PackSlots, BlameSlots); + +impl ResolveSlots { + pub fn try_acquire(&self) -> Option { + Arc::clone(&self.0).try_acquire_owned().ok().map(SlotPermit) + } +} + +#[derive(Clone)] +pub struct Slots { + pub resolve: ResolveSlots, + pub receive: ReceiveSlots, + pub pack: PackSlots, + pub blame: BlameSlots, +} + +fn blame_slots() -> usize { + threads() + .get() + .div_ceil(THREADS_PER_BLAME_SLOT) + .min(MAX_BLAME_SLOTS) +} + +impl Slots { + pub fn for_machine() -> Self { + PROCESS + .get_or_init(|| Self { + resolve: ResolveSlots::new(RESOLVE_SLOTS), + receive: ReceiveSlots::new(threads().get()), + pack: PackSlots::new(threads().get()), + blame: BlameSlots::new(blame_slots()), + }) + .clone() + } + + pub fn testing(permits: usize) -> Self { + Self { + resolve: ResolveSlots::new(permits), + receive: ReceiveSlots::new(permits), + pack: PackSlots::new(permits), + blame: BlameSlots::new(permits), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn a_clone_shares_one_pool_per_kind_of_work() { + let slots = Slots::testing(1); + let other = slots.clone(); + let held = slots.receive.acquire().await; + assert_eq!( + other.receive.available(), + 0, + "a cloned budget mustn't grant a second permit for the one slot" + ); + assert_eq!( + other.pack.available(), + 1, + "spending a receive slot mustn't spend the pack budget" + ); + assert_eq!( + other.blame.available(), + 1, + "spending a receive slot mustn't spend the blame budget" + ); + let _blaming = slots.blame.acquire().await; + assert_eq!( + other.blame.available(), + 0, + "a cloned blame budget mustn't grant more CPU work" + ); + let _resolving = slots.resolve.acquire().await; + assert!( + other.resolve.try_acquire().is_none(), + "a cosmetic lookup mustn't wait, \ + or a push queues on it while its receive and pack slots stay spent" + ); + drop(held); + assert_eq!(other.receive.available(), 1); + } + + #[tokio::test] + async fn every_caller_of_for_machine_shares_one_budget_that_no_test_budget_touches() { + let ssh = Slots::for_machine(); + let http = Slots::for_machine(); + let isolated = Slots::testing(1); + let before = http.receive.available(); + assert_eq!(http.blame.available(), blame_slots()); + let _held = ssh.receive.acquire().await; + assert_eq!( + http.receive.available(), + before - 1, + "two transports asking the machine for a budget must get the same one, \ + or the process grants twice the concurrency it was configured for" + ); + let resolving: Vec = (0..RESOLVE_SLOTS_PER_TRANSPORT) + .filter_map(|_| ssh.resolve.try_acquire()) + .collect(); + assert_eq!(resolving.len(), RESOLVE_SLOTS_PER_TRANSPORT); + assert!( + http.resolve.try_acquire().is_some(), + "collapsing a per-transport pool into a process-wide one mustn't give a deployment \ + running both transports less outbound resolution than either had on its own" + ); + assert_eq!( + isolated.receive.available(), + 1, + "whatever the process budget is doing mustn't spend a test budget" + ); + } +} diff --git a/knot2/crates/knot-runtime/Cargo.toml b/knot2/crates/knot-runtime/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..a43bc3e43aa025232305fd1657bb481135c699ac --- /dev/null +++ b/knot2/crates/knot-runtime/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "knot-runtime" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +serde = { workspace = true } +reqwest = { workspace = true } +http = { workspace = true } +url = { workspace = true } +bytes = { workspace = true } +getrandom = { workspace = true } +k256 = { workspace = true } +thiserror = { workspace = true } +futures = { workspace = true } +tokio = { workspace = true } +hickory-resolver = { workspace = true } + +[dev-dependencies] +futures = { workspace = true } +tokio = { workspace = true } diff --git a/knot2/crates/knot-runtime/src/clock.rs b/knot2/crates/knot-runtime/src/clock.rs new file mode 100644 index 0000000000000000000000000000000000000000..d3d18a3fbafbf0ef07afa426bfbdd1caf946885e --- /dev/null +++ b/knot2/crates/knot-runtime/src/clock.rs @@ -0,0 +1,84 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +pub use knot_types::UnixMicros; + +pub trait Clock: Send + Sync + 'static { + fn now_unix_micros(&self) -> UnixMicros; +} + +impl Clock for Arc { + fn now_unix_micros(&self) -> UnixMicros { + (**self).now_unix_micros() + } +} + +pub struct SystemClock; + +impl Clock for SystemClock { + fn now_unix_micros(&self) -> UnixMicros { + let micros = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_micros() as u64) + .unwrap_or(0); + UnixMicros::new(micros) + } +} + +pub struct ManualClock { + micros: AtomicU64, +} + +impl ManualClock { + pub fn new(start: UnixMicros) -> Self { + Self { + micros: AtomicU64::new(start.get()), + } + } + + pub fn advance(&self, delta: Duration) { + self.micros + .fetch_add(delta.as_micros() as u64, Ordering::SeqCst); + } +} + +impl Clock for ManualClock { + fn now_unix_micros(&self) -> UnixMicros { + UnixMicros::new(self.micros.load(Ordering::SeqCst)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manual_clock_advances() { + let clock = ManualClock::new(UnixMicros::new(1_000)); + assert_eq!(clock.now_unix_micros().get(), 1_000); + clock.advance(Duration::from_micros(500)); + assert_eq!(clock.now_unix_micros().get(), 1_500); + } + + #[test] + fn manual_clock_same_start_same_sequence() { + let one = ManualClock::new(UnixMicros::new(1_000)); + let two = ManualClock::new(UnixMicros::new(1_000)); + let advances = [10, 250, 7, 1_000]; + advances.iter().for_each(|&step| { + one.advance(Duration::from_micros(step)); + two.advance(Duration::from_micros(step)); + assert_eq!(one.now_unix_micros(), two.now_unix_micros()); + }); + } + + #[test] + fn a_shared_clock_advances_through_the_trait_object() { + let shared = Arc::new(ManualClock::new(UnixMicros::new(1_000))); + let view: Arc = Arc::clone(&shared) as Arc; + assert_eq!(view.now_unix_micros().get(), 1_000); + shared.advance(Duration::from_micros(250)); + assert_eq!(view.now_unix_micros().get(), 1_250); + } +} diff --git a/knot2/crates/knot-runtime/src/dns.rs b/knot2/crates/knot-runtime/src/dns.rs new file mode 100644 index 0000000000000000000000000000000000000000..17e6ce90cfcc9e24776bb1c0d9317f29273810ae --- /dev/null +++ b/knot2/crates/knot-runtime/src/dns.rs @@ -0,0 +1,86 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use tokio::sync::OnceCell; + +use crate::http::NetworkError; + +pub type DnsFuture = Pin, NetworkError>> + Send>>; + +pub trait DnsTxtResolver: Send + Sync + 'static { + fn lookup_txt(&self, name: String) -> DnsFuture; +} + +pub struct SystemDns { + resolver: Arc>, +} + +impl SystemDns { + pub fn new() -> Self { + Self { + resolver: Arc::new(OnceCell::new()), + } + } +} + +impl Default for SystemDns { + fn default() -> Self { + Self::new() + } +} + +impl DnsTxtResolver for SystemDns { + fn lookup_txt(&self, name: String) -> DnsFuture { + let cell = self.resolver.clone(); + Box::pin(async move { + let resolver = cell + .get_or_try_init(|| async { + hickory_resolver::TokioAsyncResolver::tokio_from_system_conf() + .map_err(|error| NetworkError::Build(error.to_string())) + }) + .await?; + match resolver.txt_lookup(name).await { + Ok(lookup) => Ok(lookup.iter().map(render_txt).collect()), + Err(error) => match error.kind() { + hickory_resolver::error::ResolveErrorKind::NoRecordsFound { .. } => { + Ok(Vec::new()) + } + _ => Err(NetworkError::Request(error.to_string())), + }, + } + }) + } +} + +fn render_txt(record: &hickory_resolver::proto::rr::rdata::TXT) -> String { + let bytes: Vec = record + .txt_data() + .iter() + .flat_map(|chunk| chunk.iter().copied()) + .collect(); + String::from_utf8_lossy(&bytes).into_owned() +} + +pub struct FakeDns { + responder: F, +} + +impl FakeDns +where + F: Fn(&str) -> Result, NetworkError> + Send + Sync + 'static, +{ + pub fn new(responder: F) -> Self { + Self { responder } + } +} + +impl DnsTxtResolver for FakeDns +where + F: Fn(&str) -> Result, NetworkError> + Send + Sync + 'static, +{ + fn lookup_txt(&self, name: String) -> DnsFuture { + let result = (self.responder)(&name); + Box::pin(async move { result }) + } +} diff --git a/knot2/crates/knot-runtime/src/entropy.rs b/knot2/crates/knot-runtime/src/entropy.rs new file mode 100644 index 0000000000000000000000000000000000000000..0a72ccdb32413c57e47d09267cdf59d1f2575273 --- /dev/null +++ b/knot2/crates/knot-runtime/src/entropy.rs @@ -0,0 +1,146 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +// Golden gamma gem alert +const GOLDEN_GAMMA: u64 = 0x9E37_79B9_7F4A_7C15; + +fn splitmix64(z: u64) -> u64 { + let z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + let z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +pub trait Entropy: Send + Sync + 'static { + fn next_u64(&self) -> u64; + fn fill(&self, buffer: &mut [u8]); + fn derive(&self, label: u64) -> Box; +} + +pub struct OsEntropy; + +impl Entropy for OsEntropy { + fn next_u64(&self) -> u64 { + let mut bytes = [0u8; 8]; + getrandom::fill(&mut bytes).expect("OS entropy unavailable"); + u64::from_le_bytes(bytes) + } + + fn fill(&self, buffer: &mut [u8]) { + getrandom::fill(buffer).expect("OS entropy unavailable"); + } + + fn derive(&self, _label: u64) -> Box { + Box::new(OsEntropy) + } +} + +pub struct SeededEntropy { + seed: u64, + state: AtomicU64, +} + +impl SeededEntropy { + pub fn new(seed: u64) -> Self { + Self { + seed, + state: AtomicU64::new(seed), + } + } + + pub fn derive(&self, label: u64) -> SeededEntropy { + SeededEntropy::new(splitmix64(self.seed ^ splitmix64(label))) + } +} + +impl Entropy for SeededEntropy { + fn next_u64(&self) -> u64 { + let z = self + .state + .fetch_add(GOLDEN_GAMMA, Ordering::SeqCst) + .wrapping_add(GOLDEN_GAMMA); + splitmix64(z) + } + + fn fill(&self, buffer: &mut [u8]) { + buffer.chunks_mut(8).for_each(|chunk| { + let value = self.next_u64().to_le_bytes(); + chunk.copy_from_slice(&value[..chunk.len()]); + }); + } + + fn derive(&self, label: u64) -> Box { + Box::new(SeededEntropy::derive(self, label)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn stream(entropy: &SeededEntropy, count: usize) -> Vec { + std::iter::repeat_with(|| entropy.next_u64()) + .take(count) + .collect() + } + + #[test] + fn seeded_entropy_is_deterministic() { + let one = SeededEntropy::new(42); + let two = SeededEntropy::new(42); + assert_eq!(stream(&one, 256), stream(&two, 256)); + } + + #[test] + fn distinct_seeds_diverge() { + assert_ne!( + stream(&SeededEntropy::new(1), 64), + stream(&SeededEntropy::new(2), 64) + ); + } + + #[test] + fn derive_is_independent_of_parent_draw_timing() { + let early = SeededEntropy::new(42); + let child_before = early.derive(7); + + let late = SeededEntropy::new(42); + let _ = stream(&late, 100); + let child_after = late.derive(7); + + assert_eq!(stream(&child_before, 128), stream(&child_after, 128)); + } + + #[test] + fn derived_streams_differ_by_label() { + let parent = SeededEntropy::new(42); + assert_ne!(stream(&parent.derive(1), 64), stream(&parent.derive(2), 64)); + } + + fn first_fill(entropy: &dyn Entropy, label: u64) -> [u8; 16] { + let mut buffer = [0u8; 16]; + entropy.derive(label).fill(&mut buffer); + buffer + } + + #[test] + fn trait_object_derive_is_independent_of_draw_order() { + let parent: &dyn Entropy = &SeededEntropy::new(77); + let in_order = [first_fill(parent, 10), first_fill(parent, 20)]; + + let parent: &dyn Entropy = &SeededEntropy::new(77); + let reversed = [first_fill(parent, 20), first_fill(parent, 10)]; + + assert_eq!(in_order[0], reversed[1]); + assert_eq!(in_order[1], reversed[0]); + assert_ne!(in_order[0], in_order[1]); + } + + #[test] + fn seeded_fill_matches_stream() { + let stream = SeededEntropy::new(7); + let expected = stream.next_u64().to_le_bytes(); + let bytes = SeededEntropy::new(7); + let mut buffer = [0u8; 8]; + bytes.fill(&mut buffer); + assert_eq!(buffer, expected); + } +} diff --git a/knot2/crates/knot-runtime/src/http.rs b/knot2/crates/knot-runtime/src/http.rs new file mode 100644 index 0000000000000000000000000000000000000000..bce4627b83b4277013a20607fb088ba628d4bdaa --- /dev/null +++ b/knot2/crates/knot-runtime/src/http.rs @@ -0,0 +1,561 @@ +use std::future::Future; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use futures::{StreamExt, TryStreamExt}; +use http::{HeaderMap, Method, StatusCode}; +use url::{Host, Url}; + +#[derive(Debug, Clone, thiserror::Error)] +pub enum NetworkError { + #[error("build: {0}")] + Build(String), + #[error("connect: {0}")] + Connect(String), + #[error("timeout: {0}")] + Timeout(String), + #[error("request: {0}")] + Request(String), + #[error("body: {0}")] + Body(String), + #[error("response exceeds {limit} bytes")] + TooLarge { limit: u64 }, + #[error("refusing to reach non-public address {host}")] + Blocked { host: String }, +} + +pub fn is_blocked_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => is_blocked_v4(v4), + IpAddr::V6(v6) => match embedded_ipv4(v6) { + Some(embedded) => is_blocked_v4(embedded), + None => { + v6.is_loopback() + || v6.is_unspecified() + || v6.is_multicast() + || (v6.segments()[0] & 0xfe00) == 0xfc00 + || (v6.segments()[0] & 0xffc0) == 0xfe80 + } + }, + } +} + +fn is_blocked_v4(v4: Ipv4Addr) -> bool { + v4.is_loopback() + || v4.is_private() + || v4.is_link_local() + || v4.is_unspecified() + || v4.is_broadcast() + || v4.is_documentation() + || v4.is_multicast() + || v4.octets()[0] == 0 + || v4.octets()[0] >= 240 + || matches!(v4.octets(), [100, second, ..] if (64..=127).contains(&second)) + || matches!(v4.octets(), [198, second, ..] if (18..=19).contains(&second)) +} + +fn embedded_ipv4(v6: Ipv6Addr) -> Option { + if let Some(mapped) = v6.to_ipv4() { + return Some(mapped); + } + let segments = v6.segments(); + if segments[0] == 0x2002 { + return Some(Ipv4Addr::new( + (segments[1] >> 8) as u8, + segments[1] as u8, + (segments[2] >> 8) as u8, + segments[2] as u8, + )); + } + if segments[0] == 0x0064 && segments[1] == 0xff9b && segments[2..6] == [0, 0, 0, 0] { + return Some(Ipv4Addr::new( + (segments[6] >> 8) as u8, + segments[6] as u8, + (segments[7] >> 8) as u8, + segments[7] as u8, + )); + } + None +} + +pub struct GuardedResolver; + +impl reqwest::dns::Resolve for GuardedResolver { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + Box::pin(async move { + let host = name.as_str().to_owned(); + let resolved = tokio::net::lookup_host((host.as_str(), 0)).await?; + let allowed: Vec = + resolved.filter(|addr| !is_blocked_ip(addr.ip())).collect(); + if allowed.is_empty() { + return Err(Box::::from(format!( + "{host} resolves only to non-public addresses" + ))); + } + Ok(Box::new(allowed.into_iter()) as reqwest::dns::Addrs) + }) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct HttpLimits { + pub connect_timeout: Duration, + pub read_timeout: Duration, + pub request_timeout: Duration, + pub max_response_bytes: u64, + pub block_private_addresses: bool, +} + +impl Default for HttpLimits { + fn default() -> Self { + Self { + connect_timeout: Duration::from_secs(5), + read_timeout: Duration::from_secs(30), + request_timeout: Duration::from_secs(60), + max_response_bytes: 16 * 1024 * 1024, + block_private_addresses: true, + } + } +} + +pub struct HttpRequest { + pub method: Method, + pub url: Url, + pub headers: HeaderMap, + pub body: Option, +} + +impl HttpRequest { + pub fn get(url: Url) -> Self { + Self { + method: Method::GET, + url, + headers: HeaderMap::new(), + body: None, + } + } + + pub fn post(url: Url, body: Bytes) -> Self { + Self { + method: Method::POST, + url, + headers: HeaderMap::new(), + body: Some(body), + } + } +} + +#[derive(Debug)] +pub struct HttpResponse { + pub status: StatusCode, + pub headers: HeaderMap, + pub body: Bytes, +} + +pub type HttpFuture = Pin> + Send>>; + +pub type ByteStream = Pin> + Send>>; + +pub struct StreamedResponse { + pub status: StatusCode, + pub headers: HeaderMap, + pub body: ByteStream, +} + +pub type StreamFuture = + Pin> + Send>>; + +pub trait HttpTransport: Send + Sync + 'static { + fn execute(&self, request: HttpRequest) -> HttpFuture; + + fn execute_streamed(&self, request: HttpRequest) -> StreamFuture { + let response = self.execute(request); + Box::pin(async move { + let response = response.await?; + Ok(StreamedResponse { + status: response.status, + headers: response.headers, + body: Box::pin(futures::stream::once(std::future::ready(Ok(response.body)))), + }) + }) + } +} + +#[derive(Clone, Debug)] +pub struct ReqwestHttp { + client: reqwest::Client, + max_response_bytes: u64, + block_private_addresses: bool, +} + +impl ReqwestHttp { + pub fn new(limits: HttpLimits) -> Result { + let mut builder = reqwest::Client::builder() + .connect_timeout(limits.connect_timeout) + .read_timeout(limits.read_timeout) + .timeout(limits.request_timeout) + .redirect(reqwest::redirect::Policy::none()); + if limits.block_private_addresses { + builder = builder.dns_resolver(Arc::new(GuardedResolver)); + } + if let Some(path) = std::env::var_os("KNOT_EXTRA_CA_FILE") { + let pem = + std::fs::read(&path).map_err(|error| NetworkError::Build(error.to_string()))?; + builder = reqwest::Certificate::from_pem_bundle(&pem) + .map_err(|error| NetworkError::Build(error.to_string()))? + .into_iter() + .fold(builder, reqwest::ClientBuilder::add_root_certificate); + } + let client = builder + .build() + .map_err(|error| NetworkError::Build(error.to_string()))?; + Ok(Self { + client, + max_response_bytes: limits.max_response_bytes, + block_private_addresses: limits.block_private_addresses, + }) + } +} + +impl HttpTransport for ReqwestHttp { + fn execute(&self, request: HttpRequest) -> HttpFuture { + let client = self.client.clone(); + let limit = self.max_response_bytes; + let guard = self.block_private_addresses; + Box::pin(async move { + if let Some(host) = guard.then(|| blocked_literal(&request.url)).flatten() { + return Err(NetworkError::Blocked { host }); + } + let mut builder = client + .request(request.method, request.url) + .headers(request.headers); + if let Some(body) = request.body { + builder = builder.body(body); + } + let response = builder.send().await.map_err(map_reqwest)?; + let status = response.status(); + let headers = response.headers().clone(); + if response.content_length().is_some_and(|len| len > limit) { + return Err(NetworkError::TooLarge { limit }); + } + let body = bounded_body(response, limit).await?; + Ok(HttpResponse { + status, + headers, + body, + }) + }) + } + + fn execute_streamed(&self, request: HttpRequest) -> StreamFuture { + let client = self.client.clone(); + let guard = self.block_private_addresses; + let limit = self.max_response_bytes; + Box::pin(async move { + if let Some(host) = guard.then(|| blocked_literal(&request.url)).flatten() { + return Err(NetworkError::Blocked { host }); + } + let mut builder = client + .request(request.method, request.url) + .headers(request.headers); + if let Some(body) = request.body { + builder = builder.body(body); + } + let response = builder.send().await.map_err(map_reqwest)?; + let status = response.status(); + let headers = response.headers().clone(); + if response.content_length().is_some_and(|len| len > limit) { + return Err(NetworkError::TooLarge { limit }); + } + let body: ByteStream = Box::pin( + response + .bytes_stream() + .map_err(|error| { + if error.is_timeout() { + NetworkError::Timeout(error.to_string()) + } else { + NetworkError::Body(error.to_string()) + } + }) + .scan((0u64, false), move |state, item| { + let next = if state.1 { + None + } else { + Some(match item { + Ok(chunk) + if state.0.saturating_add(chunk.len() as u64) <= limit => + { + state.0 += chunk.len() as u64; + Ok(chunk) + } + Ok(_) => { + state.1 = true; + Err(NetworkError::TooLarge { limit }) + } + Err(error) => { + state.1 = true; + Err(error) + } + }) + }; + std::future::ready(next) + }), + ); + Ok(StreamedResponse { + status, + headers, + body, + }) + }) + } +} + +async fn bounded_body(response: reqwest::Response, limit: u64) -> Result { + response + .bytes_stream() + .map_err(|error| { + if error.is_timeout() { + NetworkError::Timeout(error.to_string()) + } else { + NetworkError::Body(error.to_string()) + } + }) + .try_fold(Vec::new(), |mut buffer, chunk| async move { + if buffer.len() as u64 + chunk.len() as u64 > limit { + return Err(NetworkError::TooLarge { limit }); + } + buffer.extend_from_slice(&chunk); + Ok(buffer) + }) + .await + .map(Bytes::from) +} + +fn blocked_literal(url: &Url) -> Option { + match url.host()? { + Host::Ipv4(ip) if is_blocked_ip(IpAddr::V4(ip)) => Some(ip.to_string()), + Host::Ipv6(ip) if is_blocked_ip(IpAddr::V6(ip)) => Some(ip.to_string()), + _ => None, + } +} + +fn map_reqwest(error: reqwest::Error) -> NetworkError { + if error.is_timeout() { + NetworkError::Timeout(error.to_string()) + } else if error.is_connect() { + NetworkError::Connect(error.to_string()) + } else { + NetworkError::Request(error.to_string()) + } +} + +pub struct FakeHttp { + responder: F, +} + +impl FakeHttp +where + F: Fn(&HttpRequest) -> Result + Send + Sync + 'static, +{ + pub fn new(responder: F) -> Self { + Self { responder } + } +} + +impl HttpTransport for FakeHttp +where + F: Fn(&HttpRequest) -> Result + Send + Sync + 'static, +{ + fn execute(&self, request: HttpRequest) -> HttpFuture { + let result = (self.responder)(&request); + Box::pin(async move { result }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::StreamExt; + use std::io::{Read, Write}; + use std::net::{SocketAddr, TcpListener}; + + #[test] + fn blocked_addresses_cover_the_internal_ranges() { + let blocked = [ + "127.0.0.1", + "10.0.0.5", + "192.168.1.1", + "172.16.0.1", + "169.254.169.254", + "100.64.0.1", + "198.18.0.1", + "240.0.0.1", + "0.0.0.0", + "::1", + "::ffff:127.0.0.1", + "fd00::1", + "fe80::1", + "2002:7f00:1::", + "64:ff9b::7f00:1", + ]; + for raw in blocked { + assert!( + is_blocked_ip(raw.parse().unwrap()), + "{raw} should be blocked" + ); + } + let allowed = [ + "1.1.1.1", + "8.8.8.8", + "93.184.216.34", + "2606:4700:4700::1111", + "2002:808:808::", + "64:ff9b::808:808", + ]; + for raw in allowed { + assert!( + !is_blocked_ip(raw.parse().unwrap()), + "{raw} should be allowed" + ); + } + } + + #[tokio::test] + async fn resolver_rejects_hosts_that_only_resolve_privately() { + use reqwest::dns::Resolve; + let name = "localhost" + .parse::() + .expect("localhost parses"); + let resolved = GuardedResolver.resolve(name).await; + assert!(resolved.is_err(), "localhost resolves to loopback"); + } + + fn tiny_limits(max_response_bytes: u64, request_timeout: Duration) -> HttpLimits { + HttpLimits { + connect_timeout: Duration::from_millis(200), + read_timeout: Duration::from_millis(200), + request_timeout, + max_response_bytes, + block_private_addresses: false, + } + } + + fn serve_body(body: Vec, with_content_length: bool) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("addr"); + std::thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let _ = stream.read(&mut [0u8; 1024]); + let header = if with_content_length { + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()) + } else { + "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n".to_string() + }; + let _ = stream.write_all(header.as_bytes()); + let _ = stream.write_all(&body); + } + }); + addr + } + + fn serve_hang() -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("addr"); + std::thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let _ = stream.read(&mut [0u8; 1024]); + std::thread::sleep(Duration::from_secs(5)); + drop(stream); + } + }); + addr + } + + async fn fetch(addr: SocketAddr, limits: HttpLimits) -> Result { + let transport = ReqwestHttp::new(limits).expect("client builds"); + let url = Url::parse(&format!("http://{addr}/")).expect("url"); + transport.execute(HttpRequest::get(url)).await + } + + async fn fetch_streamed(addr: SocketAddr, limits: HttpLimits) -> Result, NetworkError> { + let transport = ReqwestHttp::new(limits).expect("client builds"); + let url = Url::parse(&format!("http://{addr}/")).expect("url"); + transport + .execute_streamed(HttpRequest::get(url)) + .await? + .body + .try_fold(Vec::new(), |mut body, chunk| async move { + body.extend_from_slice(&chunk); + Ok(body) + }) + .await + } + + #[tokio::test] + async fn oversized_response_is_rejected_whether_declared_or_streamed() { + futures::stream::iter([true, false]) + .for_each(|with_content_length| async move { + let addr = serve_body(vec![0u8; 4096], with_content_length); + let result = fetch(addr, tiny_limits(64, Duration::from_secs(2))).await; + assert!(matches!(result, Err(NetworkError::TooLarge { limit: 64 }))); + }) + .await; + } + + #[tokio::test] + async fn streamed_response_keeps_the_same_total_byte_limit() { + for with_content_length in [true, false] { + let addr = serve_body(vec![0u8; 4096], with_content_length); + let result = fetch_streamed(addr, tiny_limits(64, Duration::from_secs(2))).await; + assert!(matches!(result, Err(NetworkError::TooLarge { limit: 64 }))); + } + } + + #[tokio::test] + async fn small_response_within_limit_succeeds() { + let addr = serve_body(b"pong".to_vec(), true); + let response = fetch(addr, tiny_limits(64, Duration::from_secs(2))) + .await + .expect("response within limit"); + assert_eq!(response.body.as_ref(), b"pong"); + } + + #[tokio::test] + async fn unresponsive_server_times_out() { + let addr = serve_hang(); + let result = fetch(addr, tiny_limits(1024, Duration::from_millis(150))).await; + assert!(matches!(result, Err(NetworkError::Timeout(_)))); + } + + #[tokio::test] + async fn connect_failure_surfaces_typed_error() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("addr"); + drop(listener); + let result = fetch(addr, tiny_limits(1024, Duration::from_secs(2))).await; + assert!(matches!( + result, + Err(NetworkError::Connect(_) | NetworkError::Request(_) | NetworkError::Timeout(_)) + )); + } + + #[test] + fn fake_http_returns_canned_response() { + let transport = FakeHttp::new(|request: &HttpRequest| { + assert_eq!(request.method, Method::GET); + Ok(HttpResponse { + status: StatusCode::OK, + headers: HeaderMap::new(), + body: Bytes::from_static(b"pong"), + }) + }); + let request = HttpRequest::get(Url::parse("https://oyster.cafe/ping").unwrap()); + let response = + futures::executor::block_on(transport.execute(request)).expect("fake response"); + assert_eq!(response.status, StatusCode::OK); + assert_eq!(response.body.as_ref(), b"pong"); + } +} diff --git a/knot2/crates/knot-runtime/src/lib.rs b/knot2/crates/knot-runtime/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..9c322427d3bf334411b95e68153223cfeca7ffe7 --- /dev/null +++ b/knot2/crates/knot-runtime/src/lib.rs @@ -0,0 +1,92 @@ +mod clock; +mod dns; +mod entropy; +mod http; +mod signer; + +pub use clock::{Clock, ManualClock, SystemClock, UnixMicros}; +pub use dns::{DnsFuture, DnsTxtResolver, FakeDns, SystemDns}; +pub use entropy::{Entropy, OsEntropy, SeededEntropy}; +pub use http::{ + ByteStream, FakeHttp, GuardedResolver, HttpFuture, HttpLimits, HttpRequest, HttpResponse, + HttpTransport, NetworkError, ReqwestHttp, StreamFuture, StreamedResponse, is_blocked_ip, +}; +pub use signer::{ + K256Signer, MAX_SCALAR_ATTEMPTS, PublicKeyBytes, Signature, SignatureScheme, Signer, + SignerError, verify, +}; + +#[cfg(test)] +mod contract { + use super::*; + use knot_types::UnixMicros; + + fn clock_contract(clock: &dyn Clock) { + let first = clock.now_unix_micros(); + let second = clock.now_unix_micros(); + assert!(second >= first); + } + + #[test] + fn every_clock_is_non_decreasing() { + clock_contract(&SystemClock); + clock_contract(&ManualClock::new(UnixMicros::new(1_000))); + } + + fn entropy_contract(entropy: &dyn Entropy) { + let mut buffer = [0u8; 16]; + entropy.fill(&mut buffer); + assert!(buffer.iter().any(|byte| *byte != 0)); + entropy.fill(&mut []); + let _ = entropy.next_u64(); + } + + #[test] + fn every_entropy_produces_output() { + entropy_contract(&OsEntropy); + entropy_contract(&SeededEntropy::new(1)); + } + + fn signer_contract(signer: &dyn Signer) { + let signature = signer.sign(b"the message"); + let parsed = k256::ecdsa::Signature::from_slice(signature.as_bytes()) + .expect("signatures are compact"); + assert!( + parsed.normalize_s().is_none(), + "signatures are canonical low-S" + ); + assert!(verify(&signer.public_key(), b"the message", &signature)); + assert!(!verify( + &signer.public_key(), + b"another message", + &signature + )); + } + + #[test] + fn the_seeded_signer_is_the_test_double() { + signer_contract(&K256Signer::generate(&SeededEntropy::new(3))); + } + + fn ok_response() -> HttpResponse { + HttpResponse { + status: ::http::StatusCode::OK, + headers: ::http::HeaderMap::new(), + body: bytes::Bytes::from_static(b"ok"), + } + } + + #[test] + fn http_transport_surfaces_ok_and_typed_error() { + let url = url::Url::parse("https://oyster.cafe/").unwrap(); + let okay = FakeHttp::new(|_| Ok(ok_response())); + let response = + futures::executor::block_on(okay.execute(HttpRequest::get(url.clone()))).unwrap(); + assert_eq!(response.body.as_ref(), b"ok"); + + let failing = FakeHttp::new(|_| Err(NetworkError::Timeout("slow".to_string()))); + let error = + futures::executor::block_on(failing.execute(HttpRequest::get(url))).unwrap_err(); + assert!(matches!(error, NetworkError::Timeout(_))); + } +} diff --git a/knot2/crates/knot-runtime/src/signer.rs b/knot2/crates/knot-runtime/src/signer.rs new file mode 100644 index 0000000000000000000000000000000000000000..34e7c563b41e05ff496dc7f70000ef269e9c32d6 --- /dev/null +++ b/knot2/crates/knot-runtime/src/signer.rs @@ -0,0 +1,141 @@ +use k256::ecdsa::signature::{Signer as _, Verifier as _}; +use k256::ecdsa::{Signature as K256Signature, SigningKey, VerifyingKey}; + +use crate::Entropy; + +pub const MAX_SCALAR_ATTEMPTS: usize = 64; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Signature(Vec); + +impl Signature { + pub fn from_bytes(bytes: Vec) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublicKeyBytes(Vec); + +impl PublicKeyBytes { + pub fn from_bytes(bytes: Vec) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +#[derive(Debug, thiserror::Error)] +pub enum SignerError { + #[error("invalid signing key bytes")] + InvalidKey, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SignatureScheme { + Secp256k1, + P256, +} + +pub trait Signer: Send + Sync + 'static { + fn sign(&self, message: &[u8]) -> Signature; + fn public_key(&self) -> PublicKeyBytes; + fn scheme(&self) -> SignatureScheme; +} + +pub struct K256Signer { + key: SigningKey, +} + +impl K256Signer { + pub fn from_slice(bytes: &[u8]) -> Result { + SigningKey::from_slice(bytes) + .map(|key| Self { key }) + .map_err(|_| SignerError::InvalidKey) + } + + // peak dice rolling + pub fn generate(entropy: &dyn Entropy) -> Self { + std::iter::repeat_with(|| { + let mut bytes = [0u8; 32]; + entropy.fill(&mut bytes); + SigningKey::from_slice(&bytes).ok() + }) + .take(MAX_SCALAR_ATTEMPTS) + .flatten() + .next() + .map(|key| Self { key }) + .unwrap_or_else(|| { + panic!( + "entropy failed to yield valid secp256k1 scalar in {MAX_SCALAR_ATTEMPTS} attempts" + ) + }) + } +} + +impl Signer for K256Signer { + fn sign(&self, message: &[u8]) -> Signature { + let signature: K256Signature = self.key.sign(message); + Signature(signature.to_bytes().to_vec()) + } + + fn public_key(&self) -> PublicKeyBytes { + let point = self.key.verifying_key().to_encoded_point(true); + PublicKeyBytes(point.as_bytes().to_vec()) + } + + fn scheme(&self) -> SignatureScheme { + SignatureScheme::Secp256k1 + } +} + +pub fn verify(public_key: &PublicKeyBytes, message: &[u8], signature: &Signature) -> bool { + let Ok(verifying_key) = VerifyingKey::from_sec1_bytes(public_key.as_bytes()) else { + return false; + }; + let Ok(parsed) = K256Signature::from_slice(signature.as_bytes()) else { + return false; + }; + verifying_key.verify(message, &parsed).is_ok() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::SeededEntropy; + + #[test] + fn generate_is_deterministic_from_seed() { + let one = K256Signer::generate(&SeededEntropy::new(99)); + let two = K256Signer::generate(&SeededEntropy::new(99)); + assert_eq!(one.public_key(), two.public_key()); + } + + struct BrokenEntropy; + + impl crate::Entropy for BrokenEntropy { + fn next_u64(&self) -> u64 { + 0 + } + + fn fill(&self, buffer: &mut [u8]) { + buffer.fill(0); + } + + fn derive(&self, _label: u64) -> Box { + Box::new(BrokenEntropy) + } + } + + #[test] + #[should_panic(expected = "entropy failed to yield valid secp256k1 scalar")] + fn broken_entropy_fails_stop_instead_of_spinning() { + let _ = K256Signer::generate(&BrokenEntropy); + } +} diff --git a/knot2/crates/knot-secrets/Cargo.toml b/knot2/crates/knot-secrets/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..4ab3c2b6935d7e97deb6008b19bd11b82d0a27dd --- /dev/null +++ b/knot2/crates/knot-secrets/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "knot-secrets" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-resource = { workspace = true } +knot-types = { workspace = true } +knot-runtime = { workspace = true } +k256 = { workspace = true } +aes-gcm = { workspace = true } +hkdf = { workspace = true } +sha2 = { workspace = true } +base64 = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +zeroize = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/knot2/crates/knot-secrets/src/lib.rs b/knot2/crates/knot-secrets/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..9eb64568b76de85b8a4516b66f72cefc42aa127f --- /dev/null +++ b/knot2/crates/knot-secrets/src/lib.rs @@ -0,0 +1,790 @@ +use std::collections::BTreeMap; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, RwLock}; + +use aes_gcm::aead::Aead; +use aes_gcm::{Aes256Gcm, KeyInit}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use hkdf::Hkdf; +use knot_runtime::{Entropy, K256Signer, MAX_SCALAR_ATTEMPTS, PublicKeyBytes, Signer}; +use knot_types::KnotId; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; + +// why oh why didn't I just call this knot.sealed-key-store.v1. the +// wind changed and now we're stuck with this +const HKDF_INFO: &[u8] = b"knot2.sealed-key-store.v1"; +const NONCE_LEN: usize = 12; +const SCALAR_LEN: usize = 32; +const VAULT_VERSION: u32 = 1; +const MIN_MASTER_KEY_LEN: usize = 32; + +#[derive(Debug, thiserror::Error)] +pub enum SecretsError { + #[error("sealed key store {path}: {source}")] + Io { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error( + "sealed key store couldn't be decrypted. Master key may be wrong or file may be corrupt." + )] + Decrypt, + #[error("sealed key store is malformed: {0}")] + Malformed(String), + #[error("no signing key sealed for {0}")] + Missing(String), + #[error("{0} doesn't seal a valid secp256k1 scalar")] + InvalidScalar(String), + #[error("signing key is already sealed for {0}")] + Occupied(String), + #[error("{len}-byte master key is shorter than the {MIN_MASTER_KEY_LEN}-byte minimum")] + WeakMasterKey { len: usize }, +} + +#[derive(Clone, Zeroize, ZeroizeOnDrop)] +pub struct MasterKey(Vec); + +impl MasterKey { + pub fn new(bytes: impl Into>) -> Result { + let bytes = bytes.into(); + if bytes.len() < MIN_MASTER_KEY_LEN { + return Err(SecretsError::WeakMasterKey { len: bytes.len() }); + } + Ok(Self(bytes)) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +impl fmt::Debug for MasterKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("MasterKey()") + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(transparent)] +pub struct SealedKeyId(String); + +impl SealedKeyId { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl From<&KnotId> for SealedKeyId { + fn from(did: &KnotId) -> Self { + Self(did.as_str().to_string()) + } +} + +impl<'de> serde::Deserialize<'de> for SealedKeyId { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + KnotId::new(raw) + .map(|did| Self::from(&did)) + .map_err(serde::de::Error::custom) + } +} + +#[derive(Clone, Zeroize, ZeroizeOnDrop)] +struct SecretScalar([u8; SCALAR_LEN]); + +impl SecretScalar { + fn generate(entropy: &dyn Entropy) -> Self { + std::iter::repeat_with(|| { + let mut bytes = [0u8; SCALAR_LEN]; + entropy.fill(&mut bytes); + k256::ecdsa::SigningKey::from_slice(&bytes) + .is_ok() + .then_some(bytes) + }) + .take(MAX_SCALAR_ATTEMPTS) + .flatten() + .next() + .map(Self) + .unwrap_or_else(|| { + panic!( + "entropy failed to yield valid secp256k1 scalar in {MAX_SCALAR_ATTEMPTS} attempts" + ) + }) + } + + fn signer(&self) -> K256Signer { + K256Signer::from_slice(&self.0).expect("stored scalar is always a valid signing key") + } +} + +#[derive(Zeroize, ZeroizeOnDrop)] +struct VaultKey([u8; 32]); + +impl VaultKey { + fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} + +pub struct KeyMaterial { + scalar: SecretScalar, +} + +impl KeyMaterial { + pub fn signer(&self) -> K256Signer { + self.scalar.signer() + } + + pub fn public_key(&self) -> PublicKeyBytes { + self.scalar.signer().public_key() + } +} + +#[derive(Zeroize, ZeroizeOnDrop, Serialize, Deserialize)] +#[serde(transparent)] +struct EncodedSecret(String); + +#[derive(Serialize, Deserialize)] +struct VaultFile { + version: u32, + entries: BTreeMap, +} + +pub struct SealedStore { + path: PathBuf, + enc_key: VaultKey, + entropy: Box, + entries: RwLock>, + persist_lock: Mutex<()>, +} + +impl SealedStore { + pub fn open( + path: impl Into, + master_key: &MasterKey, + entropy: Box, + ) -> Result { + let path = path.into(); + knot_resource::clear_staging(&path); + sweep_pre_knot_tmp_vaults(&path); + let enc_key = derive_enc_key(master_key); + let entries = match std::fs::read(&path) { + Ok(sealed) => decode_vault(&unseal(&enc_key, &sealed)?)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeMap::new(), + Err(source) => { + return Err(SecretsError::Io { + path: path.clone(), + source, + }); + } + }; + Ok(Self { + path, + enc_key, + entropy, + entries: RwLock::new(entries), + persist_lock: Mutex::new(()), + }) + } + + pub fn signer(&self, id: impl Into) -> Result { + let id = id.into(); + self.entries + .read() + .expect("sealed store lock") + .get(&id) + .map(SecretScalar::signer) + .ok_or(SecretsError::Missing(id.0)) + } + + pub fn public_key(&self, id: impl Into) -> Result { + let id = id.into(); + self.entries + .read() + .expect("sealed store lock") + .get(&id) + .map(|scalar| scalar.signer().public_key()) + .ok_or(SecretsError::Missing(id.0)) + } + + pub fn generate(&self) -> KeyMaterial { + KeyMaterial { + scalar: SecretScalar::generate(&*self.entropy), + } + } + + pub fn store( + &self, + id: impl Into, + material: &KeyMaterial, + ) -> Result<(), SecretsError> { + let id = id.into(); + let guard = self.persist_guard(); + let mut staged = self.staged(); + if staged.contains_key(&id) { + return Err(SecretsError::Occupied(id.0)); + } + staged.insert(id, material.scalar.clone()); + self.commit_locked(&guard, staged) + } + + pub fn ingest(&self, provided: I) -> Result + where + I: IntoIterator, + { + let guard = self.persist_guard(); + let mut staged = self.staged(); + let imported = provided + .into_iter() + .try_fold(0usize, |count, (id, bytes)| { + if staged.contains_key(&id) { + return Ok::(count); + } + K256Signer::from_slice(&bytes) + .map_err(|_| SecretsError::InvalidScalar(id.0.clone()))?; + staged.insert(id, SecretScalar(bytes)); + Ok(count + 1) + })?; + match imported { + 0 => Ok(0), + _ => self.commit_locked(&guard, staged).map(|()| imported), + } + } + + pub fn ensure(&self, id: impl Into) -> Result { + let id = id.into(); + let guard = self.persist_guard(); + if let Some(public) = self + .entries + .read() + .expect("sealed store lock") + .get(&id) + .map(|scalar| scalar.signer().public_key()) + { + return Ok(public); + } + let material = self.generate(); + let public = material.public_key(); + let mut staged = self.staged(); + staged.insert(id, material.scalar.clone()); + self.commit_locked(&guard, staged)?; + Ok(public) + } + + pub fn len(&self) -> usize { + self.entries.read().expect("sealed store lock").len() + } + + pub fn is_empty(&self) -> bool { + self.entries.read().expect("sealed store lock").is_empty() + } + + pub fn remove(&self, id: impl Into) -> Result { + let id = id.into(); + let guard = self.persist_guard(); + let mut staged = self.staged(); + if staged.remove(&id).is_none() { + return Ok(false); + } + self.commit_locked(&guard, staged)?; + Ok(true) + } + + fn persist_guard(&self) -> MutexGuard<'_, ()> { + self.persist_lock + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn staged(&self) -> BTreeMap { + self.entries.read().expect("sealed store lock").clone() + } + + fn commit_locked( + &self, + _guard: &MutexGuard<'_, ()>, + staged: BTreeMap, + ) -> Result<(), SecretsError> { + let plaintext = encode_vault(&staged); + let sealed = seal(&self.enc_key, &plaintext, &*self.entropy); + atomic_write(&self.path, &sealed)?; + *self.entries.write().expect("sealed store lock") = staged; + Ok(()) + } +} + +fn derive_enc_key(master_key: &MasterKey) -> VaultKey { + let hk = Hkdf::::new(None, master_key.as_bytes()); + let mut okm = VaultKey([0u8; 32]); + hk.expand(HKDF_INFO, &mut okm.0) + .expect("32 bytes is valid HKDF-SHA256 output length"); + okm +} + +fn seal(enc_key: &VaultKey, plaintext: &[u8], entropy: &dyn Entropy) -> Vec { + let cipher = Aes256Gcm::new_from_slice(enc_key.as_bytes()).expect("32-byte AES-256-GCM key"); + let mut nonce_bytes = [0u8; NONCE_LEN]; + entropy.fill(&mut nonce_bytes); + let ciphertext = cipher + .encrypt(&nonce_bytes.into(), plaintext) + .expect("AES-256-GCM encryption doesn't fail on valid inputs"); + [&nonce_bytes[..], &ciphertext[..]].concat() +} + +fn unseal(enc_key: &VaultKey, sealed: &[u8]) -> Result>, SecretsError> { + if sealed.len() < NONCE_LEN { + return Err(SecretsError::Decrypt); + } + let (nonce_bytes, ciphertext) = sealed.split_at(NONCE_LEN); + let nonce: [u8; NONCE_LEN] = nonce_bytes + .try_into() + .expect("split_at(NONCE_LEN) yields exactly NONCE_LEN bytes"); + let cipher = Aes256Gcm::new_from_slice(enc_key.as_bytes()).expect("32-byte AES-256-GCM key"); + cipher + .decrypt(&nonce.into(), ciphertext) + .map(Zeroizing::new) + .map_err(|_| SecretsError::Decrypt) +} + +fn encode_vault(entries: &BTreeMap) -> Zeroizing> { + let file = VaultFile { + version: VAULT_VERSION, + entries: entries + .iter() + .map(|(id, scalar)| { + ( + id.clone(), + EncodedSecret(STANDARD.encode(scalar.0.as_slice())), + ) + }) + .collect(), + }; + Zeroizing::new(serde_json::to_vec(&file).expect("vault always serializes")) +} + +fn decode_vault(plaintext: &[u8]) -> Result, SecretsError> { + let VaultFile { version, entries } = serde_json::from_slice(plaintext) + .map_err(|error| SecretsError::Malformed(error.to_string()))?; + if version != VAULT_VERSION { + return Err(SecretsError::Malformed(format!( + "unsupported vault version {version}" + ))); + } + entries + .into_iter() + .map(|(id, mut encoded)| { + let encoded = Zeroizing::new(std::mem::take(&mut encoded.0)); + let bytes = Zeroizing::new( + STANDARD + .decode(encoded.as_bytes()) + .map_err(|error| SecretsError::Malformed(error.to_string()))?, + ); + let scalar: [u8; SCALAR_LEN] = bytes.as_slice().try_into().map_err(|_| { + SecretsError::Malformed(format!("scalar for {} isn't 32 bytes", id.as_str())) + })?; + k256::ecdsa::SigningKey::from_slice(&scalar).map_err(|_| { + SecretsError::Malformed(format!("scalar for {} isn't a valid key", id.as_str())) + })?; + Ok((id, SecretScalar(scalar))) + }) + .collect() +} + +fn effective_parent(path: &Path) -> &Path { + path.parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")) +} + +fn sweep_pre_knot_tmp_vaults(path: &Path) { + let Some(stem) = path.file_name().and_then(|name| name.to_str()) else { + return; + }; + let Ok(listing) = std::fs::read_dir(effective_parent(path)) else { + return; + }; + let prefix = format!(".{stem}."); + listing + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with(&prefix) && name.ends_with(".tmp")) + }) + .for_each(|path| { + let _ = std::fs::remove_file(path); + }); +} + +fn atomic_write(path: &Path, bytes: &[u8]) -> Result<(), SecretsError> { + let parent = effective_parent(path); + std::fs::create_dir_all(parent).map_err(|source| SecretsError::Io { + path: path.to_path_buf(), + source, + })?; + knot_resource::atomic_write_bytes(path, bytes, knot_resource::FileMode::Private) + .map_err(Into::into) +} + +impl From for SecretsError { + fn from(error: knot_resource::FsError) -> Self { + SecretsError::Io { + path: error.path, + source: error.source, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use knot_runtime::{SeededEntropy, Signer, verify}; + + fn entropy(seed: u64) -> Box { + Box::new(SeededEntropy::new(seed)) + } + + fn master() -> MasterKey { + MasterKey::new([7u8; 32]).unwrap() + } + + fn kid(value: &str) -> KnotId { + KnotId::new(value).unwrap() + } + + fn store_at(path: &Path, seed: u64) -> SealedStore { + SealedStore::open(path, &master(), entropy(seed)).unwrap() + } + + fn store(seed: u64) -> (tempfile::TempDir, SealedStore) { + let dir = tempfile::tempdir().unwrap(); + let store = store_at(&dir.path().join("keys.sealed"), seed); + (dir, store) + } + + #[test] + fn an_absent_file_opens_as_an_empty_store() { + let (_dir, store) = store(1); + assert!(matches!( + store.signer(&kid("did:web:nel.pet")), + Err(SecretsError::Missing(_)) + )); + } + + #[test] + fn a_sealed_key_survives_a_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("keys.sealed"); + let public = store_at(&path, 1).ensure(&kid("did:web:nel.pet")).unwrap(); + let reopened = store_at(&path, 2); + assert_eq!( + reopened.public_key(&kid("did:web:nel.pet")).unwrap(), + public + ); + } + + #[test] + fn ensure_is_idempotent() { + let (_dir, store) = store(1); + let first = store.ensure(&kid("did:plc:squid")).unwrap(); + let second = store.ensure(&kid("did:plc:squid")).unwrap(); + assert_eq!(first, second); + } + + #[test] + fn a_generated_key_signs_verifiably_and_round_trips_through_storage() { + let (_dir, store) = store(5); + let material = store.generate(); + let public = material.public_key(); + store.store(&kid("did:plc:limpet"), &material).unwrap(); + + let signer = store.signer(&kid("did:plc:limpet")).unwrap(); + let signature = signer.sign(b"a meta-repo cob change"); + assert!(verify(&public, b"a meta-repo cob change", &signature)); + assert_eq!(signer.public_key(), public); + } + + #[test] + fn storing_over_a_sealed_key_is_refused() { + let (_dir, store) = store(1); + let original = store.ensure(&kid("did:plc:limpet")).unwrap(); + let intruder = store.generate(); + assert!(matches!( + store.store(&kid("did:plc:limpet"), &intruder), + Err(SecretsError::Occupied(_)) + )); + assert_eq!( + store.public_key(&kid("did:plc:limpet")).unwrap(), + original, + "refused overwrite must leave the sealed key untouched" + ); + } + + #[test] + fn master_key_length_is_enforced_at_construction() { + [(31usize, false), (32usize, true)] + .iter() + .for_each(|&(len, accepted)| { + let result = MasterKey::new(vec![7u8; len]); + assert_eq!(result.is_ok(), accepted, "{len}-byte master key acceptance"); + assert!( + accepted + || matches!(result, Err(SecretsError::WeakMasterKey { len: reported }) if reported == len), + "a refused master key reports its short length" + ); + }); + } + + #[test] + fn the_master_key_debug_redacts_its_bytes() { + let rendered = format!("{:?}", MasterKey::new([7u8; 32]).unwrap()); + assert_eq!(rendered, "MasterKey()"); + assert!(!rendered.contains('7')); + } + + #[test] + fn a_wrong_master_key_fails_to_decrypt() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("k"); + store_at(&path, 1).ensure(&kid("did:web:nel.pet")).unwrap(); + assert!(matches!( + SealedStore::open(&path, &MasterKey::new([9u8; 32]).unwrap(), entropy(1)), + Err(SecretsError::Decrypt) + )); + } + + #[test] + fn a_removed_key_is_gone_after_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("k"); + let store = store_at(&path, 1); + store.ensure(&kid("did:plc:squid")).unwrap(); + store.ensure(&kid("did:plc:clam")).unwrap(); + assert!(store.remove(&kid("did:plc:squid")).unwrap()); + assert!(!store.remove(&kid("did:plc:squid")).unwrap()); + + let reopened = store_at(&path, 1); + assert!(reopened.signer(&kid("did:plc:squid")).is_err()); + assert!(reopened.signer(&kid("did:plc:clam")).is_ok()); + } + + #[test] + fn concurrent_writers_never_lose_a_sealed_key_or_error() { + use std::sync::Arc; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("keys.sealed"); + let store = Arc::new( + SealedStore::open(&path, &master(), Box::new(knot_runtime::OsEntropy)).unwrap(), + ); + let dids: Vec = (0..64).map(|i| kid(&format!("did:plc:race{i}"))).collect(); + + std::thread::scope(|scope| { + dids.chunks(8).for_each(|chunk| { + let store = Arc::clone(&store); + let chunk = chunk.to_vec(); + scope.spawn(move || { + chunk.iter().for_each(|did| { + store.ensure(did).expect("concurrent seal mustn't error"); + }); + }); + }); + }); + + let reopened = SealedStore::open(&path, &master(), entropy(99)).unwrap(); + let missing: Vec<&KnotId> = dids + .iter() + .filter(|did| reopened.signer(*did).is_err()) + .collect(); + assert!( + missing.is_empty(), + "keys acknowledged in memory were lost from sealed file on disk: {missing:?}" + ); + } + + #[test] + fn concurrent_ensures_of_one_did_agree_on_a_single_key() { + use std::sync::Arc; + + let dir = tempfile::tempdir().unwrap(); + let store = Arc::new( + SealedStore::open( + dir.path().join("keys.sealed"), + &master(), + Box::new(knot_runtime::OsEntropy), + ) + .unwrap(), + ); + let contested = kid("did:plc:whelk"); + + let publics: Vec = std::thread::scope(|scope| { + let handles: Vec<_> = (0..8) + .map(|_| { + let store = Arc::clone(&store); + let did = contested.clone(); + scope.spawn(move || store.ensure(&did).unwrap()) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect() + }); + + assert!( + publics.windows(2).all(|pair| pair[0] == pair[1]), + "every racing ensure must acknowledge same sealed key" + ); + assert_eq!(store.public_key(&contested).unwrap(), publics[0]); + } + + #[test] + fn the_sealed_file_does_not_contain_raw_scalars() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("k"); + let store = store_at(&path, 1); + let material = store.generate(); + let scalar = material.scalar.0; + store.store(&kid("did:plc:squid"), &material).unwrap(); + let sealed = std::fs::read(&path).unwrap(); + assert!( + !sealed.windows(SCALAR_LEN).any(|window| window == scalar), + "plaintext scalar must never appear in the sealed file" + ); + } + + #[cfg(unix)] + fn set_mode(path: &Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + } + + #[cfg(unix)] + fn make_read_only(dir: &Path) -> bool { + set_mode(dir, 0o555); + let enforced = std::fs::write(dir.join("probe"), b"probe").is_err(); + if !enforced { + set_mode(dir, 0o755); + eprintln!("skipping permission fault injection, this user bypasses read-only modes"); + } + enforced + } + + #[cfg(unix)] + #[test] + fn a_failed_persist_acknowledges_no_key() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("keys.sealed"); + let store = store_at(&path, 1); + store.ensure(&kid("did:plc:squid")).unwrap(); + + if !make_read_only(dir.path()) { + return; + } + let material = store.generate(); + assert!(matches!( + store.store(&kid("did:plc:clam"), &material), + Err(SecretsError::Io { .. }) + )); + assert!( + matches!( + store.signer(&kid("did:plc:clam")), + Err(SecretsError::Missing(_)) + ), + "key whose persist failed mustn't be served from memory" + ); + assert!(matches!( + store.ensure(&kid("did:plc:clam")), + Err(SecretsError::Io { .. }) + )); + set_mode(dir.path(), 0o755); + + let public = store.ensure(&kid("did:plc:clam")).unwrap(); + let reopened = store_at(&path, 2); + assert_eq!(reopened.public_key(&kid("did:plc:clam")).unwrap(), public); + assert!(reopened.signer(&kid("did:plc:squid")).is_ok()); + } + + #[cfg(unix)] + #[test] + fn a_failed_removal_keeps_the_key_served_and_sealed() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("keys.sealed"); + let store = store_at(&path, 1); + store.ensure(&kid("did:plc:squid")).unwrap(); + + if !make_read_only(dir.path()) { + return; + } + assert!(matches!( + store.remove(&kid("did:plc:squid")), + Err(SecretsError::Io { .. }) + )); + assert!( + store.signer(&kid("did:plc:squid")).is_ok(), + "removal that failed to persist must leave the key in service" + ); + set_mode(dir.path(), 0o755); + + let reopened = store_at(&path, 2); + assert!(reopened.signer(&kid("did:plc:squid")).is_ok()); + } + + struct BrokenEntropy; + + impl Entropy for BrokenEntropy { + fn next_u64(&self) -> u64 { + 0 + } + + fn fill(&self, buffer: &mut [u8]) { + buffer.fill(0); + } + + fn derive(&self, _label: u64) -> Box { + Box::new(BrokenEntropy) + } + } + + #[test] + #[should_panic(expected = "entropy failed to yield valid secp256k1 scalar")] + fn broken_entropy_fails_stop_instead_of_spinning() { + let _ = SecretScalar::generate(&BrokenEntropy); + } + + #[test] + fn stale_temp_files_are_swept_on_open() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("keys.sealed"); + let stale = dir.path().join(".keys.sealed.knot-tmp.4242.7"); + let in_flight = dir.path().join(".keys.sealed.knot-tmp.4243.0"); + let pre_rename = dir.path().join(".keys.sealed.4242.7.tmp"); + std::fs::write(&stale, b"abandoned by a crashed run").unwrap(); + std::fs::write(&in_flight, b"another process is sealing right now").unwrap(); + std::fs::write(&pre_rename, b"abandoned before the staging rename").unwrap(); + let long_ago = std::time::SystemTime::now() - std::time::Duration::from_secs(7 * 3600); + std::fs::File::options() + .write(true) + .open(&stale) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(long_ago)) + .unwrap(); + + store_at(&path, 1); + assert!(!stale.exists(), "abandoned temp file must be swept at open"); + assert!( + in_flight.exists(), + "the sweep at open must keep the staging file a second process is filling" + ); + assert!( + !pre_rename.exists(), + "a vault sealed by an older build still leaves temps this build has to reclaim" + ); + } +} diff --git a/knot2/crates/knot-server/Cargo.toml b/knot2/crates/knot-server/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..2b83f21546753c61f83cee93162b0e2960684bbd --- /dev/null +++ b/knot2/crates/knot-server/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "knot-server" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-config = { workspace = true } +knot-record = { workspace = true } +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-lfs = { workspace = true } +knot-pack = { workspace = true } +knot-resource = { workspace = true } +knot-cache = { workspace = true } +knot-index = { workspace = true } +knot-keyfill = { workspace = true } +knot-atproto = { workspace = true } +knot-runtime = { workspace = true } +knot-ssh = { workspace = true } +knot-edge = { workspace = true } +knot-secrets = { workspace = true } +knot-xrpc = { workspace = true } +tikv-jemallocator = { workspace = true } +tikv-jemalloc-ctl = { workspace = true } +knot-events = { workspace = true } +knot-maintenance = { workspace = true } +knot-postreceive = { workspace = true } +knot-messages = { workspace = true } +axum = { workspace = true } +clap = { workspace = true } +reqwest = { workspace = true } +serde_json = { workspace = true } +tower-http = { workspace = true, features = ["fs", "cors"] } +tokio = { workspace = true } +tokio-util = { workspace = true } +anyhow = { workspace = true } +base64 = { workspace = true } +zeroize = { workspace = true } +rustix = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } + +[dev-dependencies] +confique = { workspace = true } +http = { workspace = true } +walkdir = { workspace = true } +tempfile = { workspace = true } diff --git a/knot2/crates/knot-server/src/admin.rs b/knot2/crates/knot-server/src/admin.rs new file mode 100644 index 0000000000000000000000000000000000000000..c111fa65bdc486726ec4bd0f55ec7c72d720386a --- /dev/null +++ b/knot2/crates/knot-server/src/admin.rs @@ -0,0 +1,127 @@ +use anyhow::Context; +use clap::Subcommand; +use knot_record::grant::GrantCollection; +use knot_types::{AccountDid, RepoDid}; +use knot_xrpc::RepoMethod; +use knot_xrpc::legacy_admin; +use serde_json::json; + +#[derive(Subcommand)] +pub enum AdminCommand { + /// Mint an invite code + CreateInviteCode, + /// Manage knot members + Member { + #[command(subcommand)] + command: MemberCommand, + }, + /// Manage anonymous users on the knot + User { + #[command(subcommand)] + command: UserCommand, + }, + /// Convert and drop the seeds a settled handover left behind. Without a repo, every repo + /// is swept in the background and the knot logs each outcome. + RepairHandover { repo: Option }, +} + +#[derive(Subcommand)] +pub enum MemberCommand { + /// Add an account as a member. + Add { subject: AccountDid }, + /// Remove a member. + Remove { subject: AccountDid }, +} + +#[derive(Subcommand)] +pub enum UserCommand { + /// Ban an account. + Ban { subject: AccountDid }, + /// Unban an account. + Unban { subject: AccountDid }, +} + +pub async fn run(config: &knot_config::Validated, command: AdminCommand) -> anyhow::Result<()> { + let response = match command { + AdminCommand::CreateInviteCode => { + post(config, legacy_admin::CREATE_INVITE_CODE_ROUTE, None).await? + } + AdminCommand::Member { command } => { + let (method, subject) = match command { + MemberCommand::Add { subject } => (RepoMethod::Create, subject), + MemberCommand::Remove { subject } => (RepoMethod::Delete, subject), + }; + grant(config, method, GrantCollection::Member, &subject).await? + } + AdminCommand::User { command } => { + let (method, subject) = match command { + UserCommand::Ban { subject } => (RepoMethod::Create, subject), + UserCommand::Unban { subject } => (RepoMethod::Delete, subject), + }; + grant(config, method, GrantCollection::Block, &subject).await? + } + AdminCommand::RepairHandover { repo } => { + let body = repo.map(|repo| json!({ "repo": repo })); + post(config, legacy_admin::REPAIR_HANDOVER_ROUTE, body).await? + } + }; + println!("{}", serde_json::to_string_pretty(&response)?); + Ok(()) +} + +async fn grant( + config: &knot_config::Validated, + method: RepoMethod, + collection: GrantCollection, + subject: &AccountDid, +) -> anyhow::Result { + let repo = config.knot_hostname().knot_did(); + let body = match method { + RepoMethod::Create => json!({ + "repo": repo.as_str(), + "collection": collection.nsid(), + "rkey": subject.as_str(), + "record": { "$type": collection.nsid() }, + }), + RepoMethod::Delete => json!({ + "repo": repo.as_str(), + "collection": collection.nsid(), + "rkey": subject.as_str(), + }), + }; + post(config, method.route(), Some(body)).await +} + +async fn post( + config: &knot_config::Validated, + route: &str, + body: Option, +) -> anyhow::Result { + let name = config + .acl + .legacy_admin_secret_env + .as_deref() + .context("acl.legacy_admin_secret_env isn't configured")?; + let secret = zeroize::Zeroizing::new(std::env::var(name).unwrap_or_default()); + let secret = secret.trim(); + anyhow::ensure!( + !secret.is_empty(), + "the environment variable {name} is unset or empty" + ); + let url = format!("https://{}{route}", config.knot_hostname().as_str()); + let request = reqwest::Client::new() + .post(&url) + .basic_auth("admin", Some(secret)); + let request = match body { + Some(body) => request.json(&body), + None => request, + }; + let response = request + .send() + .await + .with_context(|| format!("request {url}"))?; + let status = response.status(); + let body = response.text().await.context("read response body")?; + anyhow::ensure!(status.is_success(), "{url} responded {status}: {body}"); + serde_json::from_str(&body).context("decode response") +} diff --git a/knot2/crates/knot-server/src/allocator.rs b/knot2/crates/knot-server/src/allocator.rs new file mode 100644 index 0000000000000000000000000000000000000000..d87cac059247c7a8d611d93f2b363316c7dff180 --- /dev/null +++ b/knot2/crates/knot-server/src/allocator.rs @@ -0,0 +1,148 @@ +use std::time::Duration; + +use knot_resource::DecayMs; +use tokio_util::sync::CancellationToken; + +const SAMPLE_INTERVAL: Duration = Duration::from_secs(2); +const BACKGROUND_THREAD: &[u8] = b"background_thread\0"; +const NARENAS: &[u8] = b"arenas.narenas\0"; +const DIRTY_DECAY_NEW_ARENAS: &[u8] = b"arenas.dirty_decay_ms\0"; + +#[derive(Clone, Copy)] +struct ArenaIndex(u32); + +impl ArenaIndex { + fn write_decay(self, decay: DecayMs) -> bool { + let key = format!("arena.{}.dirty_decay_ms\0", self.0); + unsafe { tikv_jemalloc_ctl::raw::write(key.as_bytes(), decay.ms()) }.is_ok() + } +} + +#[derive(Clone, Copy)] +struct DecayCoverage { + reached: usize, + skipped: usize, +} + +impl DecayCoverage { + const EMPTY: Self = Self { + reached: 0, + skipped: 0, + }; + + fn record(self, reached: bool) -> Self { + match reached { + true => Self { + reached: self.reached + 1, + ..self + }, + false => Self { + skipped: self.skipped + 1, + ..self + }, + } + } +} + +fn write_or_warn(name: &[u8], value: T, control: &str) { + if let Err(error) = unsafe { tikv_jemalloc_ctl::raw::write(name, value) } { + tracing::warn!(%error, control, "jemalloc control unavailable"); + } +} + +fn apply_decay(decay: DecayMs) -> tikv_jemalloc_ctl::Result { + unsafe { tikv_jemalloc_ctl::raw::write(DIRTY_DECAY_NEW_ARENAS, decay.ms())? }; + let narenas: u32 = unsafe { tikv_jemalloc_ctl::raw::read(NARENAS)? }; + Ok((0..narenas) + .map(ArenaIndex) + .map(|arena| arena.write_decay(decay)) + .fold(DecayCoverage::EMPTY, DecayCoverage::record)) +} + +pub fn install() { + match apply_decay(knot_resource::target_decay()) { + Ok(coverage) => tracing::info!( + arenas_reached = coverage.reached, + arenas_skipped = coverage.skipped, + "jemalloc dirty_decay applied to reachable arenas" + ), + Err(error) => tracing::warn!(%error, "jemalloc dirty_decay governor unavailable"), + } + write_or_warn(BACKGROUND_THREAD, true, "background_thread"); + let background: bool = + unsafe { tikv_jemalloc_ctl::raw::read(b"opt.background_thread\0") }.unwrap_or(false); + let retain: bool = unsafe { tikv_jemalloc_ctl::raw::read(b"opt.retain\0") }.unwrap_or(true); + let dirty_decay_ms: isize = + unsafe { tikv_jemalloc_ctl::raw::read(b"arena.0.dirty_decay_ms\0") }.unwrap_or(-1); + tracing::info!( + background_thread = background, + retain, + dirty_decay_ms, + "jemalloc page-return configured" + ); +} + +pub async fn govern_decay(shutdown: CancellationToken) { + let mut ticker = tokio::time::interval(SAMPLE_INTERVAL); + let mut applied = knot_resource::target_decay(); + loop { + tokio::select! { + () = shutdown.cancelled() => return, + _ = ticker.tick() => { + let target = knot_resource::target_decay(); + if knot_resource::decay_warrants_apply(applied, target) { + match apply_decay(target) { + Ok(coverage) => tracing::info!( + target_ms = target.ms(), + arenas_reached = coverage.reached, + arenas_skipped = coverage.skipped, + "jemalloc dirty_decay retuned" + ), + Err(error) => tracing::warn!(%error, "jemalloc dirty_decay retune failed"), + } + applied = target; + } + if knot_resource::cache_shed_warranted() + && let Some(freed) = knot_cache::reclaim_largest() + { + tracing::warn!( + freed_bytes = freed.get(), + "shedding largest cache under memory pressure" + ); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn apply_decay_reaches_reachable_arenas_and_persists() { + let narenas: u32 = + unsafe { tikv_jemalloc_ctl::raw::read(NARENAS) }.expect("arenas.narenas is readable"); + assert!(narenas >= 1, "a live jemalloc has at least one arena"); + + let coverage = apply_decay(knot_resource::target_decay()) + .expect("new-arena default write and narenas read succeed"); + assert!( + coverage.reached >= 1, + "arena 0 is always initialized, so the sweep reaches at least one arena" + ); + assert_eq!( + coverage.reached + coverage.skipped, + narenas as usize, + "every arena index is accounted as reached or skipped" + ); + + unsafe { + tikv_jemalloc_ctl::raw::write(b"arena.0.dirty_decay_ms\0", 5_000isize) + .expect("per-arena dirty_decay_ms is writable"); + let back: isize = tikv_jemalloc_ctl::raw::read(b"arena.0.dirty_decay_ms\0") + .expect("per-arena dirty_decay_ms is readable"); + assert_eq!(back, 5_000, "a per-arena write reads back"); + } + } +} diff --git a/knot2/crates/knot-server/src/doctor.rs b/knot2/crates/knot-server/src/doctor.rs new file mode 100644 index 0000000000000000000000000000000000000000..3f4f0c7fb5c5f044518a84179e0b45ee6109081e --- /dev/null +++ b/knot2/crates/knot-server/src/doctor.rs @@ -0,0 +1,59 @@ +use std::time::Duration; + +pub async fn run(config: &knot_config::Validated) -> anyhow::Result<()> { + let mut problems = 0; + + // Does knot have at least one crawler? + if config.crawlers().is_empty() { + println!("[WARN] no crawlers configured (atproto.crawlers); knot won't request crawls"); + } + + // Is Knot active in configured crawlers? + let client = reqwest::Client::new(); + let hostname = config.knot_hostname().as_str(); + for crawler in config.crawlers() { + let url = format!( + "{}/xrpc/com.atproto.sync.getHostStatus?hostname={hostname}", + crawler.as_str() + ); + let response = client + .get(&url) + .timeout(Duration::from_secs(30)) + .send() + .await; + let (status, body) = match response { + Ok(response) => (response.status(), response.text().await.unwrap_or_default()), + Err(error) => { + println!("[FAIL] {}: {error}", crawler.as_str()); + problems += 1; + continue; + } + }; + let json: serde_json::Value = serde_json::from_str(&body).unwrap_or_default(); + if !status.is_success() { + let error = json["error"].as_str().unwrap_or(status.as_str()); + let message = json["message"].as_str().unwrap_or(&body); + println!("[FAIL] {}: {error}: {message}", crawler.as_str()); + problems += 1; + continue; + } + let host_status = json["status"].as_str().unwrap_or("unknown"); + let tag = if host_status == "active" { + "[ OK ]" + } else { + "[FAIL]" + }; + if host_status != "active" { + problems += 1; + } + println!( + "{tag} {}: status={host_status} seq={} repoCount={}", + crawler.as_str(), + json["seq"], + json["accountCount"], + ); + } + + anyhow::ensure!(problems == 0, "doctor found {problems} problem(s)"); + Ok(()) +} diff --git a/knot2/crates/knot-server/src/homepage.html b/knot2/crates/knot-server/src/homepage.html new file mode 100644 index 0000000000000000000000000000000000000000..84b6bbf3499cffd70ac14de5141ce62e0b151ba1 --- /dev/null +++ b/knot2/crates/knot-server/src/homepage.html @@ -0,0 +1,21 @@ + + + + + + knot + + + +

    Baby's first Knot

    +

    This is a Knot server: a git host on a network of federated servers that make up Tangled.

    +

    + Operator! Please customize this page! It's your time to shine! Something unique? Something creative? + We made it easy using the homepage.path in the knot config. +

    + + diff --git a/knot2/crates/knot-server/src/main.rs b/knot2/crates/knot-server/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..852e25760b6fa907e2f4641cc0063fec9aff9c70 --- /dev/null +++ b/knot2/crates/knot-server/src/main.rs @@ -0,0 +1,1063 @@ +mod admin; +mod allocator; +mod doctor; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[allow(non_upper_case_globals)] +#[unsafe(export_name = "_rjem_malloc_conf")] +pub static malloc_conf: &[u8] = + b"background_thread:true,retain:false,dirty_decay_ms:0,muzzy_decay_ms:0\0"; + +use std::collections::BTreeSet; +use std::num::{NonZeroU32, NonZeroU64}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use tokio_util::sync::CancellationToken; + +use anyhow::Context; +use axum::Json; +use axum::response::Html; +use axum::routing::get; +use base64::Engine; +use clap::{Parser, Subcommand}; +use knot_atproto::Atproto; +use knot_config::HomepageSource; +use knot_index::Index; +use knot_runtime::{Clock, HttpTransport, OsEntropy, ReqwestHttp, SystemClock}; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{ActorId, ObjectCount}; +use knot_xrpc::XrpcState; +use tower_http::services::ServeFile; + +const MAINTENANCE_SHUTDOWN_DRAIN: Duration = Duration::from_secs(30); +const KEYFILL_SHUTDOWN_DRAIN: Duration = Duration::from_secs(10); +const EDGE_SHUTDOWN_DRAIN: Duration = Duration::from_secs(40); +const SEQ_PERSIST_INTERVAL_SECS: u64 = 10; + +const DEFAULT_HOMEPAGE: &str = include_str!("homepage.html"); + +struct IndexRepos(Arc); + +impl knot_maintenance::RepoSource for IndexRepos { + fn repos(&self) -> Vec { + self.0.hosted_repos() + } + + fn ready_repos(&self) -> Option> { + match self.0.coverage().registry { + knot_index::Coverage::Ready => Some(self.0.hosted_repos()), + knot_index::Coverage::Warming => None, + } + } +} + +struct AtprotoHandleResolver { + atproto: Arc>, +} + +impl knot_pack::HandleResolver for AtprotoHandleResolver { + fn resolve( + &self, + handle: knot_types::Handle, + ) -> std::pin::Pin< + Box> + Send + '_>, + > { + Box::pin(async move { self.atproto.resolve_handle_to_did(&handle).await.ok() }) + } +} + +struct GrantSweeper(Arc>); + +impl knot_maintenance::GrantSync for GrantSweeper { + fn sweep(&self, scope: knot_maintenance::GrantScope) { + match scope { + knot_maintenance::GrantScope::Knot => knot_xrpc::grants::sweep_members(&self.0), + knot_maintenance::GrantScope::Repo(repo) => { + knot_xrpc::grants::sweep_collaborators(&self.0, &repo) + } + } + } +} + +fn init_tracing() { + let filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); + tracing_subscriber::fmt() + .with_env_filter(filter) + .with_writer(std::io::stderr) + .init(); +} + +fn validate(config: &knot_config::Validated, config_only: bool) -> anyhow::Result<()> { + if !config_only { + config + .verify_environment() + .context("verify runtime environment")?; + } + println!("configuration is valid"); + Ok(()) +} + +fn keyfill_pace(config: &knot_config::KeyfillConfig) -> knot_keyfill::Pace { + knot_keyfill::Pace { + ttl: knot_index::KeyTtl::from_secs(config.ttl_secs), + reprieve: knot_index::KeyReprieve::from_secs( + config.reprieve_retry_secs, + config.reprieve_budget_secs, + ), + ..knot_keyfill::Pace::default() + } +} + +#[derive(Parser)] +#[command(name = "knot")] +struct Cli { + /// Path to a TOML config file. Environment variables override file values + /// for any `KNOT_*` setting; `/etc/knot/config.toml` is consulted as a + /// final fallback so distro packaging can drop a default in place. + #[arg(short, long, global = true, value_name = "FILE", env = "KNOT_CONFIG")] + config: Option, + + #[command(subcommand)] + command: Option, +} + +// Commands that run without a configuration. +#[derive(Subcommand)] +enum Command { + /// Print a fully-commented TOML template to stdout. Use this to seed + /// `config.toml` for a fresh deploy. + ConfigTemplate, + #[command(flatten)] + Configured(ConfiguredCommand), +} + +#[derive(Subcommand)] +enum ConfiguredCommand { + /// Load and validate the configuration without starting the server. + Validate { + /// Skip runtime environment checks (paths, permissions) and only + /// validate the configuration itself. + #[arg(long)] + config_only: bool, + }, + /// Administer a running knot. + Admin { + #[command(subcommand)] + command: admin::AdminCommand, + }, + /// Diagnose the knot. + Doctor, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + #[cfg(target_os = "linux")] + rustix::process::set_dumpable_behavior(rustix::process::DumpableBehavior::NotDumpable) + .context("disable core dumps and ptrace attachment")?; + + let cli = Cli::parse(); + + let command = match cli.command { + Some(Command::ConfigTemplate) => { + print!("{}", knot_config::template()); + return Ok(()); + } + Some(Command::Configured(command)) => Some(command), + None => None, + }; + + let config = knot_config::load(cli.config.as_deref()).context("load configuration")?; + + match command { + None => serve(config).await, + Some(ConfiguredCommand::Validate { config_only }) => validate(&config, config_only), + Some(ConfiguredCommand::Admin { command }) => admin::run(&config, command).await, + Some(ConfiguredCommand::Doctor) => doctor::run(&config).await, + } +} + +async fn serve(config: knot_config::Validated) -> anyhow::Result<()> { + config + .verify_environment() + .context("verify runtime environment")?; + init_tracing(); + run(config).await +} + +async fn run(config: knot_config::Validated) -> anyhow::Result<()> { + tracing::info!("!"); + tracing::info!("!"); + tracing::info!("!"); + tracing::info!("> If knot1 was so good then why isn't there a... ( ˶°ㅁ°)"); + tracing::info!("..."); + tracing::info!("Welcome to knot2!"); + tracing::info!("This code was made with love."); + tracing::info!("Hachapuri is sho tasty, definitely worth a try. Better than pizza tbh."); + tracing::info!("!"); + tracing::info!("!"); + tracing::info!("!"); + + let resources = knot_resource::init(knot_resource::Ceilings { + max_threads: match config.resources.max_threads { + 0 => None, + n => Some(knot_resource::ThreadCount::new(n as usize)), + }, + max_memory: match config.resources.max_memory_bytes { + 0 => None, + n => Some(knot_resource::MemoryBudget::new(n)), + }, + }); + tracing::info!( + threads = resources.threads.get(), + memory_bytes = resources.memory.map(knot_resource::MemoryBudget::get), + memory_source = ?resources.memory_source, + memory_high_bytes = resources + .memory_high_bytes + .map(knot_resource::MemoryHighBytes::get), + "resource governor initialized" + ); + allocator::install(); + + let hostname = config.knot_hostname().clone(); + let knot_did = hostname.knot_did(); + let object_format = config.git.object_format; + let default_branch = config.repo.default_branch.clone(); + let layout = knot_git::Layout::new(&config.repo.scan_path) + .with_default_branch(default_branch) + .with_object_format(object_format) + .reserving_meta(&knot_did) + .context("reserve meta-repo path")?; + + let swept = knot_pack::sweep_incoming(&config.repo.scan_path); + if swept > 0 { + tracing::info!(swept, "swept abandoned receive staging directories"); + } + + layout + .bootstrap_meta(&knot_did) + .context("bootstrap meta-repo")?; + let meta_path = layout + .meta_path(&knot_did) + .context("resolve meta-repo path")?; + + let meta_repo = + knot_git::Repo::open(&meta_path).context("open the meta-repo for the legacy ref sweep")?; + let migrated = knot_xrpc::migrate::legacy_meta_refs(&meta_repo) + .context("migrate legacy sh.tangled cob refs in the meta-repo")?; + if migrated > 0 { + tracing::info!( + migrated, + "meta-repo cob refs renamed to org.tangled on boot" + ); + } + + let knot_repo = knot_types::DidRkey::new(knot_did.as_str()) + .with_context(|| { + format!( + "{} is a did:web with a port or a path, a development form that isn't valid \ + as a record key, so the knot account can't exist", + knot_did.as_str() + ) + }) + .map(|rkey| knot_types::RepoDid::from(&rkey))?; + + let key_pace = keyfill_pace(&config.keyfill); + let budget = knot_index::KeyBudget::from_mib(config.keyfill.key_budget_mib as usize); + let index = Arc::new(Index::with_key_budget( + meta_path.clone(), + layout.clone(), + budget, + )); + index.rebuild().context("rebuild index from meta-repo")?; + tracing::info!(coverage = ?index.coverage(), "index ready"); + + let warm = Arc::clone(&index); + tokio::task::spawn_blocking(move || warm.warm_collaborators()); + + let http = ReqwestHttp::new(config.http_limits()).context("build outbound HTTP client")?; + let git_http: Arc = Arc::new( + ReqwestHttp::new(config.fork_http_limits()).context("build outbound git fetch client")?, + ); + let atproto = Arc::new(Atproto::new( + http, + SystemClock, + knot_did.clone(), + knot_atproto::PlcDirectory::new(config.atproto.plc_directory.clone()) + .context("atproto.plc_directory isn't a valid PLC base URL")?, + )); + let admins: BTreeSet<_> = config.server.admins.iter().cloned().collect(); + let admission = config.acl.admission; + let contribution_policy = config.repo.contribution_policy; + let write_limits = knot_resource::RateLimit { + burst: knot_xrpc::Burst::of(config.repo.write_burst) + .context("repo.write_burst must be at least 1")?, + refill: knot_xrpc::RefillMicros::of(config.repo.write_refill_ms.saturating_mul(1_000)) + .context("repo.write_refill_ms must be at least 1")?, + }; + let social_budget = knot_resource::SocialBudgetBytes::new(config.social.repo_budget_bytes); + let foster = foster_pull(&config)?; + let liar_hydrant = liar_pull(&config)?; + let boot_width = config.social.foster_boot_width as usize; + let sweep_width = config.atproto.plc_sweep_width as usize; + let service_owner = config + .server + .admins + .first() + .cloned() + .context("at least one admin is configured")?; + + let master_key = MasterKey::new( + base64::engine::general_purpose::STANDARD + .decode( + std::env::var(&config.secrets.master_key_env) + .context("read master key from environment")? + .trim(), + ) + .context("decode master key as base64")?, + ) + .context("master key from environment")?; + if !config.secrets.sealed_key_file.exists() { + tracing::info!( + path = %config.secrets.sealed_key_file.display(), + "the knot will seal a new signing identity, since it can't find a key store at this path" + ); + } + let secrets = Arc::new( + SealedStore::open( + &config.secrets.sealed_key_file, + &master_key, + Box::new(OsEntropy), + ) + .context("open sealed key store")?, + ); + if let Some(archive) = config.repo_signing_keys() { + knot_xrpc::plc::ingest_legacy_keys(&secrets, &archive) + .map_err(|error| anyhow::anyhow!(error.to_string()).context("sealing the migration per-repo signing keys")) + .map(|count| { + tracing::info!(count, path = %archive.display(), "sealed migrated per-repo signing keys") + })?; + } + let knot_signing_key = secrets + .ensure(&knot_did) + .context("seal knot's own signing key")?; + let knot_actor = ActorId::from_secp256k1(knot_signing_key.as_bytes()); + let appview_endpoint = config.server.appview_endpoint.clone(); + let knot_service_url = hostname.service_url(); + let did_document = + knot_atproto::knot_did_document(&knot_did, &knot_signing_key, &knot_service_url); + + let http_addr = config.server.listen_addr; + let listen_limits = knot_edge::ListenLimits::new( + knot_edge::HeaderTimeout::from_millis( + NonZeroU64::new(config.server.listen_header_timeout_ms) + .context("server.listen_header_timeout_ms must be greater than zero")?, + ), + knot_edge::IdleTimeout::from_millis( + NonZeroU64::new(config.server.listen_idle_timeout_ms) + .context("server.listen_idle_timeout_ms must be greater than zero")?, + ), + NonZeroU32::new(config.server.listen_max_connections) + .context("server.listen_max_connections must be greater than zero")?, + ); + // A header name that doesn't parse will never match, + // `ProxyTrust::client_peer` falls back to socket, + // and every request in the world shares + // the proxy's address + its one ratelimit bucket. + // So... better to refuse to start. + let trusted_proxy_header = config + .xrpc + .trusted_proxy_header + .as_deref() + .map(|header| axum::http::HeaderName::from_bytes(header.as_bytes())) + .transpose() + .context("xrpc.trusted_proxy_header isn't a valid HTTP header name")?; + let proxy_trust = knot_types::ProxyTrust::new(trusted_proxy_header, config.trusted_proxies()); + if proxy_trust.trusts_any_peer() && !http_addr.ip().is_loopback() { + tracing::warn!( + bind = %http_addr, + "xrpc.trusted_proxy_header is set without xrpc.trusted_proxies while the HTTP surface takes connections from off-host, so a client that reaches this knot without passing the proxy can forge the header and pick its own rate-limit bucket. List the proxy's address in xrpc.trusted_proxies." + ); + } + let edge_guards = knot_edge::EdgeGuards::new( + knot_edge::RequestsPerSecond::new( + NonZeroU32::new(config.server.listen_rate_limit_per_second) + .context("server.listen_rate_limit_per_second must be greater than zero")?, + ), + knot_edge::BurstSize::new( + NonZeroU32::new(config.server.listen_rate_limit_burst) + .context("server.listen_rate_limit_burst must be greater than zero")?, + ), + knot_edge::MaxInflightRequests::new( + NonZeroU32::new(config.server.listen_max_inflight_requests) + .context("server.listen_max_inflight_requests must be greater than zero")?, + ), + knot_edge::RequestTimeout::from_millis( + NonZeroU64::new(config.server.listen_request_timeout_ms) + .context("server.listen_request_timeout_ms must be greater than zero")?, + ), + knot_edge::BodyInactivityTimeout::from_millis( + NonZeroU64::new(config.server.listen_body_timeout_ms) + .context("server.listen_body_timeout_ms must be greater than zero")?, + ), + knot_edge::WriteRequestTimeout::from_millis( + NonZeroU64::new(config.server.listen_write_request_timeout_ms) + .context("server.listen_write_request_timeout_ms must be greater than zero")?, + ), + proxy_trust.clone(), + ); + let tls_setup = build_tls_setup(&config, &hostname).context("assemble TLS configuration")?; + if config.tls.http3 && tls_setup.is_none() { + tracing::warn!( + "tls.http3 is set without any TLS certificate, so HTTP/3 won't start. Configure a static cert or ACME to serve h3." + ); + } + if tls_setup.is_none() && config.xrpc.trusted_proxy_header.is_none() { + tracing::warn!( + "running plaintext behind a reverse proxy without xrpc.trusted_proxy_header. Per-IP rate limiting will key on the proxy socket address, throttling all clients as one. Set xrpc.trusted_proxy_header to the header your proxy appends, and xrpc.trusted_proxies to the address it connects from." + ); + } + if config.tls.acme_enabled && http_addr.port() != 443 { + tracing::warn!( + listen_port = http_addr.port(), + "ACME validation over TLS-ALPN-01 needs the certificate authority to reach this host on TCP 443. Map 443 to the listen port if it differs." + ); + } + if config.tls.acme_enabled && config.tls.acme_staging { + tracing::warn!( + "ACME is using the Let's Encrypt staging directory. Its certificates aren't browser-trusted. Unset tls.acme_staging for real certificates." + ); + } + let ssh_addr = config.server.ssh_listen_addr; + let ssh_max_pack_bytes = config.server.ssh_max_pack_bytes as usize; + let pack_limits = knot_pack::PackLimits { + max_objects: ObjectCount::from(config.pack.max_objects), + max_total_bytes: knot_pack::MaxTotalBytes::new(config.pack.max_total_bytes), + ..knot_pack::PackLimits::default() + }; + knot_pack::init_selection_limits(knot_pack::SelectionLimits { + max_objects: ObjectCount::from(config.pack.selection_max_objects), + time_budget: Duration::from_secs(config.pack.selection_time_budget_secs), + }); + if !config.server.ssh_host_key_file.exists() { + tracing::info!( + path = %config.server.ssh_host_key_file.display(), + "the knot will generate an ssh host key, since it can't find a key at this path. \ + Anyone who already has this knot's fingerprint cached will see it change" + ); + } + let host_key = knot_ssh::load_or_create_host_key(&config.server.ssh_host_key_file) + .context("load or create SSH host key")?; + + let xrpc_limits = knot_xrpc::LimitConfig { + rate: Some(knot_xrpc::RateLimit { + burst: knot_xrpc::Burst::of(config.xrpc.preauth_burst) + .context("xrpc.preauth_burst must be at least 1")?, + refill: knot_xrpc::RefillMicros::of( + config.xrpc.preauth_refill_ms.saturating_mul(1_000), + ) + .context("xrpc.preauth_refill_ms must be at least 1")?, + }), + per_peer_inflight: Some(knot_xrpc::PerPeerInflight::new( + config.xrpc.per_peer_inflight as usize, + )), + global_inflight: Some(knot_xrpc::GlobalInflight::new( + config.xrpc.global_inflight as usize, + )), + }; + let byte_limits = knot_xrpc::ByteLimits { + body: knot_xrpc::BodyLimit::new(config.xrpc.max_body_bytes as usize), + patch: knot_xrpc::PatchLimit::new(config.xrpc.max_patch_bytes as usize), + patch_decompressed: knot_xrpc::PatchDecompressedLimit::new( + config.xrpc.max_patch_decompressed_bytes, + ), + response: knot_xrpc::ResponseLimit::new(config.xrpc.max_response_bytes as usize), + archive: knot_xrpc::ArchiveLimit::new(config.xrpc.max_archive_bytes), + fork_pack: knot_xrpc::ForkPackLimit::new(config.xrpc.fork_max_pack_bytes), + import_pack: knot_xrpc::ImportPackLimit::new(config.xrpc.import_max_pack_bytes), + pack: knot_xrpc::MaxWireBytes::new(ssh_max_pack_bytes), + }; + let budgets = knot_xrpc::Budgets { + tree_last_commit: knot_xrpc::TreeReadBudget::new(knot_xrpc::ReadBudget::Within( + Duration::from_millis(config.xrpc.tree_last_commit_budget_ms), + )), + blob_last_commit: knot_xrpc::BlobReadBudget::new(knot_xrpc::ReadBudget::Within( + Duration::from_millis(config.xrpc.blob_last_commit_budget_ms), + )), + languages: knot_xrpc::LanguagesReadBudget::new(knot_xrpc::ReadBudget::Within( + Duration::from_millis(config.xrpc.languages_budget_ms), + )), + }; + let reservations = Arc::new(knot_xrpc::Reservations::new( + knot_xrpc::ReservationTtl::new(config.xrpc.reservation_ttl_secs as i64), + knot_xrpc::PerActorQuota::new(config.xrpc.per_actor_reservations as usize), + knot_xrpc::GlobalQuota::new(config.xrpc.max_pending_reservations as usize), + )); + let replay_bounds = knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(config.xrpc.events_replay_buffer as usize) + .context("xrpc.events_replay_buffer must be greater than zero")?, + knot_events::ReplayBytes::new(config.xrpc.events_replay_bytes as usize) + .context("xrpc.events_replay_bytes must be greater than zero")?, + ); + let firehose_floor = knot_git::Repo::open(&meta_path) + .map(|meta| knot_xrpc::firehose_floor(&meta, &layout, &index.hosted_repos())) + .context("open the meta repo to seed the firehose seq floor")?; + let subscriber_gate = Arc::new(knot_events::SubscriberGate::new( + knot_events::GlobalSubscriberLimit::new(config.xrpc.events_max_subscribers as usize), + knot_events::PerPeerSubscriberLimit::new(config.xrpc.events_max_per_peer as usize), + )); + + let maintenance_enabled = config.maintenance.enabled; + let maintenance_options = knot_maintenance::Options::from_config(&config.maintenance); + let maintenance_interval = Duration::from_secs(config.maintenance.interval_secs); + let maintenance_large_push = + knot_maintenance::PushBytes::new(config.maintenance.large_push_bytes); + + let lfs_handle = knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(&config.lfs.store_path), + knot_lfs::LfsSize::new(config.lfs.max_object_bytes), + knot_lfs::FreeSpaceFloor::new(config.lfs.free_space_floor_bytes), + ) + .inspect(|_| { + tracing::info!(store = %config.lfs.store_path.display(), "git-lfs store opened"); + }) + .context("open LFS object store")?; + let lfs_max_ssh_transfers = config.lfs.max_ssh_transfers as usize; + let lfs_max_http_downloads = config.lfs.max_http_downloads as usize; + let lfs_gc_grace = knot_maintenance::lfs_grace( + knot_maintenance::GcGrace::from_secs(config.lfs.gc_grace_secs), + knot_maintenance::ReflogRetention::from_secs(config.maintenance.reflog_expire_secs), + ); + let lfs_gc_interval = + knot_maintenance::SweepInterval::new(Duration::from_secs(config.lfs.gc_interval_secs)); + if !maintenance_enabled { + tracing::warn!( + "maintenance is disabled, so unreferenced LFS objects and attachments will accumulate with no garbage collection or orphan sweep." + ); + } + + let pack_cache_config = knot_pack::CacheConfig { + enabled: config.pack_cache.enabled, + ttl: Duration::from_secs(config.pack_cache.ttl_secs), + max_entry_bytes: knot_pack::MaxEntryBytes::new(config.pack_cache.max_entry_bytes as usize), + max_total_bytes: knot_pack::MaxCacheBytes::new( + knot_resource::pack_cache_bytes(config.pack_cache.max_total_bytes).get() as usize, + ), + }; + + let homepage = config.homepage.source(); + let catalog = config.catalog(); + let legacy_admin = legacy_admin_secret(&config); + let description = knot_xrpc::KnotDescription { + links: knot_xrpc::Links { + privacy_policy: config.links.privacy_policy.clone(), + terms_of_service: config.links.terms_of_service.clone(), + }, + contact: knot_xrpc::Contact { + email: config.contact.email.clone(), + }, + }; + let ssh_proxy_protocol_from = config.ssh_proxy_protocol_from(); + let foster_ack_skip_knot_service = config.social.foster_ack_skip_knot_service; + let foster_ack_skip_origin_checks = config.social.foster_ack_skip_origin_checks; + + let outbox = knot_xrpc::Outbox::open(&meta_path, Default::default()); + let blob_views = Arc::new(knot_xrpc::BlobViews::default()); + let (maintenance_scheduler, maintenance_handle) = if maintenance_enabled { + let (scheduler, handle) = knot_maintenance::Scheduler::new( + layout.clone(), + Arc::new(IndexRepos(Arc::clone(&index))), + SystemClock, + maintenance_options, + maintenance_interval, + maintenance_large_push, + knot_maintenance::LfsGc::new( + Arc::clone(&lfs_handle.store), + lfs_gc_grace, + lfs_gc_interval, + ), + ); + let scheduler = scheduler.with_attachment_roots(Arc::new(knot_xrpc::RepoAttachments::new( + layout.clone(), + Arc::clone(&blob_views), + ))); + (Some(scheduler), handle) + } else { + (None, knot_maintenance::MaintenanceHandle::disabled()) + }; + + let slots = knot_resource::Slots::for_machine(); + let (imports, import_jobs) = knot_xrpc::imports::ImportQueue::channel(); + + let ssh_base = knot_ssh::SshState::new(knot_ssh::SshConfig { + layout: layout.clone(), + index: Arc::clone(&index), + atproto: Arc::clone(&atproto), + knot_actor, + hostname: hostname.clone(), + appview: appview_endpoint.clone(), + admins: admins.clone(), + admission, + max_pack_bytes: byte_limits.pack, + archive_limit: byte_limits.archive, + lfs: lfs_handle.clone(), + lfs_max_transfers: lfs_max_ssh_transfers, + max_inflight_per_peer: config.server.ssh_max_inflight_per_peer as usize, + }) + .with_maintenance(maintenance_handle.clone()) + .with_limits(pack_limits) + .with_slots(slots.clone()) + .with_key_ttl(key_pace.ttl) + .with_proxy_protocol_from(ssh_proxy_protocol_from) + .with_catalog(Arc::clone(&catalog)); + let xrpc_state = Arc::new(XrpcState { + layout: layout.clone(), + index: Arc::clone(&index), + atproto: Arc::clone(&atproto), + secrets, + entropy: Arc::new(OsEntropy), + invite_codes: Default::default(), + admins, + admission, + contribution_policy, + knot_did, + knot_repo, + knot_hostname: hostname, + meta_path, + knot_service_url, + writes: Arc::new(knot_resource::SubjectLimiter::new(write_limits)), + social_budget, + limiter: Arc::new(knot_xrpc::PreAuthLimiter::with_config(xrpc_limits)), + cob_locks: Arc::new(knot_xrpc::CobLocks::default()), + reservations, + proxy_trust, + byte_limits, + budgets, + git_http, + pack_limits, + service_owner, + legacy_admin_secret: legacy_admin.clone(), + description, + subscriber_gate, + maintenance: maintenance_handle, + appview: appview_endpoint, + slots: slots.clone(), + imports, + handover_sessions: Arc::new(knot_xrpc::handover::HandoverSessions::default()), + firehose: Arc::new( + knot_events::EventLog::new(SystemClock, replay_bounds).with_boot_floor(firehose_floor), + ), + lfs: knot_xrpc::LfsWeb::new(lfs_handle.clone(), lfs_max_http_downloads), + blobs: Arc::clone(&blob_views), + catalog: Arc::clone(&catalog), + rkeys: Default::default(), + outbox, + }); + let ssh_state = Arc::new( + ssh_base.with_ref_surface(Arc::new(knot_xrpc::RefProjection { + state: Arc::clone(&xrpc_state), + })), + ); + let (maintenance_shutdown, maintenance_task) = maintenance_scheduler + .map(|scheduler| { + let scheduler = + scheduler.with_grant_sync(Arc::new(GrantSweeper(Arc::clone(&xrpc_state)))); + let (shutdown, rx) = tokio::sync::watch::channel(false); + let task = tokio::spawn(scheduler.run(rx)); + tracing::info!( + interval_secs = maintenance_interval.as_secs(), + "maintenance scheduler running" + ); + (shutdown, task) + }) + .unzip(); + { + let state = Arc::clone(&xrpc_state); + let swept = tokio::task::spawn_blocking(move || knot_xrpc::migrate::legacy_state(&state)) + .await + .context("join the legacy sweep")? + .context("sweep legacy sh.tangled refs and records")?; + if swept > 0 { + tracing::info!(swept, "legacy sh.tangled refs and records swept on boot"); + } + } + { + let state = Arc::clone(&xrpc_state); + tokio::task::spawn_blocking(move || knot_xrpc::materialize::enable_surfaces(&state)); + } + let seq_state = Arc::clone(&xrpc_state); + let seq_meta_path = seq_state.meta_path.clone(); + let resolver: Arc = { + let index = Arc::clone(&index); + Arc::new(move |target: &knot_pack::RepoTarget| match target { + knot_pack::RepoTarget::Did(did) => { + knot_pack::RepoLookup::from_resolved(index.owner_of(did), |_| did.clone()) + } + knot_pack::RepoTarget::OwnerPath(owner, path) => knot_pack::RepoLookup::from_resolved( + index.resolve_clone_path(owner, path), + |found| found, + ), + }) + }; + let receive_advertiser = knot_xrpc::receive_advertiser(Arc::clone(&xrpc_state)); + let handle_resolver: Arc = Arc::new(AtprotoHandleResolver { + atproto: Arc::clone(&atproto), + }); + let (write_routes, early_data_safe) = knot_pack::edge_routes(knot_pack::EdgeConfig { + receive: Some(receive_advertiser), + handle_resolver: Some(handle_resolver), + pack_slots: slots.pack.clone(), + cache: pack_cache_config, + catalog: Arc::clone(&catalog), + hostname: xrpc_state.knot_hostname.clone(), + archive_limit: byte_limits.archive, + ..knot_pack::EdgeConfig::serving(layout, resolver, Arc::new(SystemClock)) + }); + let legacy_admin_routes = legacy_admin.as_ref().map(|_| { + tracing::warn!( + route = knot_xrpc::legacy_admin::CREATE_INVITE_CODE_ROUTE, + "serving the legacy basic-auth admin route" + ); + knot_xrpc::legacy_admin::router(Arc::clone(&xrpc_state)) + }); + let foster_routes = legacy_admin.as_ref().map(|_| { + tracing::warn!( + route = knot_xrpc::foster::SEED_ROUTE, + "serving the foster basic-auth seed route" + ); + if liar_hydrant.is_none() { + tracing::warn!( + "social.foster_liar_hydrant_url is empty, so every shadow this foster retires keeps the liar hydrant resolving these repos here" + ); + } + knot_xrpc::foster::router( + Arc::clone(&xrpc_state), + foster_ack_skip_knot_service, + foster_ack_skip_origin_checks, + liar_hydrant.clone(), + ) + }); + let base_router = write_routes + .merge(knot_xrpc::router(Arc::clone(&xrpc_state))) + .route( + "/.well-known/did.json", + get(move || { + let document = did_document.clone(); + async move { Json(document) } + }), + ); + let base_router = match legacy_admin_routes { + Some(routes) => base_router.merge(routes), + None => base_router, + }; + let base_router = match foster_routes { + Some(routes) => base_router.merge(routes), + None => base_router, + }; + let base_router = match homepage { + HomepageSource::Disabled => base_router, + HomepageSource::Default => base_router.route("/", get(|| async { Html(DEFAULT_HOMEPAGE) })), + HomepageSource::File(path) => base_router.route_service("/", ServeFile::new(path)), + }; + let app = knot_edge::RequiresFullHandshake::new(base_router); + let scheme = if tls_setup.is_some() { "https" } else { "http" }; + let edge_config = knot_edge::EdgeConfig { + http_addr: knot_edge::PublicBind::new(http_addr), + limits: listen_limits, + guards: edge_guards, + tls: tls_setup, + }; + tracing::info!("listening on {scheme}://{http_addr} and ssh://{ssh_addr}"); + + let shutdown = CancellationToken::new(); + { + let (import_stop, import_stop_rx) = tokio::sync::watch::channel(false); + let import_shutdown = shutdown.clone(); + tokio::spawn(async move { + import_shutdown.cancelled().await; + let _ = import_stop.send(true); + }); + let import_state = Arc::clone(&xrpc_state); + tokio::spawn(knot_xrpc::imports::run( + import_state, + import_jobs, + import_stop_rx, + )); + tracing::info!("import worker running"); + } + { + let state = Arc::clone(&xrpc_state); + let shutdown = shutdown.clone(); + let foster = foster.clone(); + tokio::spawn(async move { + knot_xrpc::handover::run_boot(Arc::clone(&state), shutdown.clone(), foster, boot_width) + .await; + knot_xrpc::keep_sweep::run(state, shutdown, boot_width).await; + }); + } + tokio::spawn(allocator::govern_decay(shutdown.clone())); + let keyfill_task = knot_keyfill::spawn( + Arc::clone(&index), + Arc::clone(&atproto), + slots.clone(), + key_pace, + shutdown.clone(), + ); + { + let firehose = Arc::clone(&seq_state.firehose); + let seq_meta = seq_meta_path.clone(); + let seq_shutdown = shutdown.clone(); + tokio::spawn(async move { + let mut tick = tokio::time::interval(Duration::from_secs(SEQ_PERSIST_INTERVAL_SECS)); + loop { + tokio::select! { + _ = tick.tick() => {} + () = seq_shutdown.cancelled() => break, + } + let (firehose, seq_meta) = (Arc::clone(&firehose), seq_meta.clone()); + let _ = tokio::task::spawn_blocking(move || { + knot_xrpc::persist_seq(&firehose, &seq_meta) + }) + .await; + } + }); + } + tokio::spawn(knot_xrpc::outbox::run( + Arc::clone(&xrpc_state), + shutdown.clone(), + )); + tokio::spawn(knot_xrpc::plc::run_sweeper( + Arc::clone(&xrpc_state), + shutdown.clone(), + sweep_width, + )); + tokio::spawn(knot_xrpc::foster::run_retire_gc( + Arc::clone(&xrpc_state), + shutdown.clone(), + liar_hydrant, + )); + for crawler in config.crawlers() { + let url = format!("{}/xrpc/com.atproto.sync.requestCrawl", crawler.as_str()); + let body = serde_json::json!({ "hostname": config.knot_hostname().as_str() }); + tokio::spawn(async move { + let response = reqwest::Client::new() + .post(&url) + .timeout(Duration::from_secs(30)) + .json(&body) + .send() + .await + .and_then(reqwest::Response::error_for_status); + match response { + Ok(_) => tracing::info!(crawler = %url, "requested crawl"), + Err(error) => tracing::warn!(crawler = %url, %error, "requestCrawl failed"), + } + }); + } + let mut edge_task = tokio::spawn(knot_edge::serve( + edge_config, + app, + early_data_safe, + shutdown.clone(), + )); + let mut ssh_task = { + let shutdown = shutdown.clone(); + tokio::spawn(async move { knot_ssh::serve(ssh_addr, host_key, ssh_state, shutdown).await }) + }; + + let exit = tokio::select! { + result = &mut edge_task => FirstExit::Edge(result), + result = &mut ssh_task => FirstExit::Ssh(result), + () = shutdown_signal() => { + tracing::info!("shutdown signal received"); + FirstExit::Signal + } + }; + shutdown.cancel(); + let drain = async { + match &exit { + FirstExit::Edge(_) => { + let _ = (&mut ssh_task).await; + } + FirstExit::Ssh(_) => { + let _ = (&mut edge_task).await; + } + FirstExit::Signal => { + let _ = (&mut edge_task).await; + let _ = (&mut ssh_task).await; + } + } + }; + if tokio::time::timeout(EDGE_SHUTDOWN_DRAIN, drain) + .await + .is_err() + { + tracing::warn!( + timeout_secs = EDGE_SHUTDOWN_DRAIN.as_secs(), + "aborting edge drain after timeout" + ); + } + if tokio::time::timeout(KEYFILL_SHUTDOWN_DRAIN, keyfill_task) + .await + .is_err() + { + tracing::warn!( + timeout_secs = KEYFILL_SHUTDOWN_DRAIN.as_secs(), + "aborting key fill drain after timeout" + ); + } + { + let firehose = Arc::clone(&seq_state.firehose); + let seq_meta = seq_meta_path.clone(); + let _ = + tokio::task::spawn_blocking(move || knot_xrpc::persist_seq(&firehose, &seq_meta)).await; + } + if let Some(shutdown) = maintenance_shutdown { + let _ = shutdown.send(true); + } + if let Some(task) = maintenance_task + && tokio::time::timeout(MAINTENANCE_SHUTDOWN_DRAIN, task) + .await + .is_err() + { + tracing::warn!( + timeout_secs = MAINTENANCE_SHUTDOWN_DRAIN.as_secs(), + "aborting maintenance drain after timeout :3" + ); + } + match exit { + FirstExit::Edge(result) => result + .context("edge server task panicked")? + .context("serve edge")?, + FirstExit::Ssh(result) => result + .context("ssh server task panicked")? + .context("serve ssh")?, + FirstExit::Signal => {} + } + Ok(()) +} + +enum FirstExit { + Edge(Result, tokio::task::JoinError>), + Ssh(Result, tokio::task::JoinError>), + Signal, +} + +fn legacy_admin_secret( + config: &knot_config::Validated, +) -> Option { + let name = config.acl.legacy_admin_secret_env.as_deref()?; + let value = zeroize::Zeroizing::new(std::env::var(name).unwrap_or_default()); + match knot_xrpc::legacy_admin::LegacyAdminSecret::new(&value) { + Ok(secret) => Some(secret), + Err(_) => { + tracing::warn!( + secret_env = name, + "the environment variable in acl.legacy_admin_secret_env is unset or empty, so we won't serve the admin route" + ); + None + } + } +} + +fn foster_pull( + config: &knot_config::Validated, +) -> anyhow::Result> { + let raw = config.social.foster_url.trim(); + (!raw.is_empty()) + .then(|| knot_xrpc::handover::FosterKnot::new(raw)) + .transpose() + .map_err(|error| anyhow::anyhow!(error).context("social.foster_url")) +} + +fn liar_pull( + config: &knot_config::Validated, +) -> anyhow::Result> { + let raw = config.social.foster_liar_hydrant_url.trim(); + (!raw.is_empty()) + .then(|| knot_xrpc::foster::LiarHydrant::new(raw)) + .transpose() + .map_err(|error| anyhow::anyhow!(error).context("social.foster_liar_hydrant_url")) +} + +fn build_tls_setup( + config: &knot_config::KnotConfig, + hostname: &knot_types::KnotHostname, +) -> anyhow::Result> { + let tls = &config.tls; + let source = if tls.acme_enabled { + knot_edge::CertSource::Acme(knot_edge::AcmeParams { + domains: vec![hostname.clone()], + contact: knot_edge::AcmeContact::new( + tls.acme_contact + .clone() + .context("tls.acme_contact is required when ACME is enabled")?, + )?, + cache_dir: knot_edge::AcmeCacheDir::new( + tls.acme_cache_dir + .clone() + .context("tls.acme_cache_dir is required when ACME is enabled")?, + ), + production: !tls.acme_staging, + }) + } else { + match (&tls.cert_path, &tls.key_path) { + (Some(cert_path), Some(key_path)) => { + knot_edge::CertSource::Static(knot_edge::StaticCertPaths { + cert_path: knot_edge::CertChainPath::new(cert_path.clone()), + key_path: knot_edge::PrivateKeyPath::new(key_path.clone()), + }) + } + _ => return Ok(None), + } + }; + + let internal = match tls.mtls_enabled { + true => Some(knot_edge::InternalTls { + addr: knot_edge::InternalBind::new(config.server.internal_listen_addr), + client_ca_path: knot_edge::ClientCaPath::new( + tls.mtls_client_ca_path + .clone() + .context("tls.mtls_client_ca_path is required when mTLS is enabled")?, + ), + spki_pin: knot_edge::SpkiPin::from_base64( + tls.mtls_admin_spki_pin + .as_deref() + .context("tls.mtls_admin_spki_pin is required when mTLS is enabled")?, + ) + .context("parse tls.mtls_admin_spki_pin")?, + }), + false => None, + }; + + Ok(Some(knot_edge::TlsSetup { + source, + http3: tls.http3, + internal, + })) +} + +async fn shutdown_signal() { + let interrupt = async { + let _ = tokio::signal::ctrl_c().await; + }; + #[cfg(unix)] + let terminate = async { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(mut stream) => { + stream.recv().await; + } + Err(_) => std::future::pending::<()>().await, + } + }; + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + () = interrupt => {} + () = terminate => {} + } +} diff --git a/knot2/crates/knot-server/tests/invariants.rs b/knot2/crates/knot-server/tests/invariants.rs new file mode 100644 index 0000000000000000000000000000000000000000..19bcc43223f195d4a826ff1b309d38144a78ea56 --- /dev/null +++ b/knot2/crates/knot-server/tests/invariants.rs @@ -0,0 +1,246 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use walkdir::WalkDir; + +fn knot_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .expect("knot root is two levels above server crate") + .to_path_buf() +} + +fn workspace_root() -> PathBuf { + knot_root() + .parent() + .expect("workspace root is the parent of the knot root") + .to_path_buf() +} + +fn crate_src_files() -> impl Iterator { + WalkDir::new(knot_root().join("crates")) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .map(|entry| entry.into_path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "rs")) + .filter(|path| { + path.components() + .any(|component| component.as_os_str() == "src") + }) +} + +#[test] +fn no_subprocess_spawning_in_src() { + let offenders: Vec = crate_src_files() + .filter(|path| { + std::fs::read_to_string(path) + .map(|text| text.contains("process::Command")) + .unwrap_or(false) + }) + .map(|path| path.display().to_string()) + .collect(); + assert!( + offenders.is_empty(), + "design pillar: no subprocesses for git or anything else, but these src files spawn one: {offenders:?}" + ); +} + +fn lock_field<'a>(block: &'a str, key: &str) -> Option<&'a str> { + block.lines().find_map(|line| { + line.strip_prefix(key)? + .strip_prefix(" = \"")? + .strip_suffix('"') + }) +} + +type PackageKey<'a> = (&'a str, &'a str); +type LockGraph<'a> = BTreeMap, Vec>>; + +fn lock_graph(lock: &str) -> LockGraph<'_> { + lock.split("[[package]]") + .skip(1) + .filter_map(|block| { + let name = lock_field(block, "name")?; + let version = lock_field(block, "version")?; + Some(((name, version), lock_dependencies(block))) + }) + .collect() +} + +fn lock_dependencies(block: &str) -> Vec> { + block + .split_once("dependencies = [") + .and_then(|(_, rest)| rest.split(']').next()) + .into_iter() + .flat_map(str::lines) + .filter_map(|line| line.trim().strip_prefix('"')) + .filter_map(|entry| entry.split('"').next()) + .map(|entry| { + entry + .split_once(' ') + .map_or(DepRef::Name(entry), |(name, rest)| { + DepRef::Exact(name, rest.split(' ').next().unwrap_or(rest)) + }) + }) + .collect() +} + +#[derive(Debug, Clone, Copy)] +enum DepRef<'a> { + Name(&'a str), + Exact(&'a str, &'a str), +} + +impl<'a> DepRef<'a> { + fn name(&self) -> &'a str { + match self { + DepRef::Name(name) | DepRef::Exact(name, _) => name, + } + } +} + +fn reachable_from<'a>( + graph: &LockGraph<'a>, + by_name: &BTreeMap<&'a str, Vec>>, + package: PackageKey<'a>, + seen: &mut BTreeSet>, +) { + if seen.insert(package) { + graph.get(&package).into_iter().flatten().for_each(|dep| { + let (name, version) = match dep { + DepRef::Name(name) => (*name, None), + DepRef::Exact(name, version) => (*name, Some(*version)), + }; + by_name + .get(name) + .into_iter() + .flatten() + .filter(|(_, held)| version.is_none_or(|version| *held == version)) + .for_each(|key| reachable_from(graph, by_name, *key, seen)); + }); + } +} + +#[test] +fn no_durable_state_or_native_git_crates() { + let lock = std::fs::read_to_string(workspace_root().join("Cargo.lock")) + .expect("workspace Cargo.lock is readable"); + let graph = lock_graph(&lock); + let by_name: BTreeMap<&str, Vec>> = + graph.keys().fold(BTreeMap::new(), |mut names, key| { + names.entry(key.0).or_default().push(*key); + names + }); + let server = by_name + .get("knot-server") + .and_then(|keys| keys.first()) + .copied() + .expect("the lockfile parse must find knot-server"); + assert!( + graph.get(&server).is_some_and(|deps| !deps.is_empty()), + "the lockfile parse must find knot-server's dependency list" + ); + + let mut reachable = BTreeSet::new(); + reachable_from(&graph, &by_name, server, &mut reachable); + assert!( + reachable.iter().any(|(name, _)| *name == "gix"), + "the reachability walk must reach the git engine, so an empty walk is a broken parse" + ); + + let banned = [ + "rusqlite", + "sqlx", + "sled", + "fjall", + "redb", + "git2", + "libgit2-sys", + ]; + let present: Vec<&str> = banned + .into_iter() + .filter(|name| reachable.iter().any(|(held, _)| held == name)) + .collect(); + assert!( + present.is_empty(), + "design pillar: no durable state but git and all git work through gix, but the server's dependency graph includes: {present:?}" + ); +} + +fn dependents_of<'a>(graph: &LockGraph<'a>, package: &str) -> Vec<&'a str> { + graph + .iter() + .filter(|((name, _), _)| *name != package) + .filter(|(_, deps)| deps.iter().any(|dep| dep.name() == package)) + .map(|((name, _), _)| *name) + .collect() +} + +#[test] +fn nothing_depends_on_the_offline_migration_tool() { + let lock = std::fs::read_to_string(workspace_root().join("Cargo.lock")) + .expect("workspace Cargo.lock is readable"); + let graph = lock_graph(&lock); + assert!( + !dependents_of(&graph, "knot-types").is_empty(), + "the shared newtypes have dependents, so an empty answer here is a broken parse" + ); + + let dependents = dependents_of(&graph, "knot-migrate"); + assert!( + dependents.is_empty(), + "knot-migrate is an offline one-shot tool whose rusqlite dependency must never reach the server, but it is depended on by: {dependents:?}" + ); +} + +#[test] +fn the_shared_limit_defaults_match_the_config_defaults() { + use confique::{Config, Layer}; + use knot_xrpc::{Budgets, ByteLimits, ReadBudget}; + + fn ms(budget: ReadBudget) -> u64 { + match budget { + ReadBudget::Within(within) => within.as_millis() as u64, + ReadBudget::Unbounded => u64::MAX, + } + } + + let xrpc = ::Layer::default_values(); + let server = ::Layer::default_values(); + let bytes = ByteLimits::default(); + let budgets = Budgets::default(); + + let configured = [ + ("body", xrpc.max_body_bytes), + ("patch", xrpc.max_patch_bytes), + ("patch_decompressed", xrpc.max_patch_decompressed_bytes), + ("response", xrpc.max_response_bytes), + ("archive", xrpc.max_archive_bytes), + ("fork_pack", xrpc.fork_max_pack_bytes), + ("import_pack", xrpc.import_max_pack_bytes), + ("pack", server.ssh_max_pack_bytes), + ("tree_last_commit", xrpc.tree_last_commit_budget_ms), + ("blob_last_commit", xrpc.blob_last_commit_budget_ms), + ("languages", xrpc.languages_budget_ms), + ] + .map(|(name, value)| (name, value.expect("every limit has a config default"))); + let shared = [ + ("body", bytes.body.get() as u64), + ("patch", bytes.patch.get() as u64), + ("patch_decompressed", bytes.patch_decompressed.get()), + ("response", bytes.response.get() as u64), + ("archive", bytes.archive.get()), + ("fork_pack", bytes.fork_pack.get()), + ("import_pack", bytes.import_pack.get()), + ("pack", bytes.pack.get() as u64), + ("tree_last_commit", ms(budgets.tree_last_commit.get())), + ("blob_last_commit", ms(budgets.blob_last_commit.get())), + ("languages", ms(budgets.languages.get())), + ]; + assert_eq!( + configured, shared, + "the config defaults and the in-code defaults must match. update ByteLimits::default and Budgets::default alongside the config defaults" + ); +} diff --git a/knot2/crates/knot-sim/Cargo.toml b/knot2/crates/knot-sim/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..fdc57d2c37bde7590bdaa37421c39f33dfbd455b --- /dev/null +++ b/knot2/crates/knot-sim/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "knot-sim" +version = "1.20.0" +edition = { workspace = true } +rust-version = { workspace = true } +license = { workspace = true } +publish = false + +[[bin]] +name = "knot-sim" +path = "src/main.rs" + +[dependencies] +knot-types = { workspace = true } +knot-config = { workspace = true } +knot-runtime = { workspace = true } +knot-git = { workspace = true } +knot-index = { workspace = true } +knot-pack = { workspace = true } +knot-resource = { workspace = true } +knot-atproto = { workspace = true } +knot-secrets = { workspace = true } +knot-xrpc = { workspace = true } +knot-events = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-maintenance = { workspace = true } +knot-messages = { workspace = true } +axum = { workspace = true } +tower = { workspace = true } +http = { workspace = true } +bytes = { workspace = true } +tokio = { workspace = true } +url = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +base64 = { workspace = true } +futures = { workspace = true } +tempfile = { workspace = true } +knot-lfs = { workspace = true } + +[dev-dependencies] +knot-fixtures = { workspace = true } +knot-ssh = { workspace = true } +knot-edge = { workspace = true } +knot-record = { workspace = true } +jacquard-common = { workspace = true } +sha2 = { workspace = true } +quinn = { workspace = true } +h3 = { workspace = true } +h3-quinn = { workspace = true } +rustls = { workspace = true } +rcgen = { workspace = true } +tokio-util = { workspace = true } diff --git a/knot2/crates/knot-sim/src/harness.rs b/knot2/crates/knot-sim/src/harness.rs new file mode 100644 index 0000000000000000000000000000000000000000..d5d1bd4b3088c15e95343678a10a58540d1f6409 --- /dev/null +++ b/knot2/crates/knot-sim/src/harness.rs @@ -0,0 +1,974 @@ +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use axum::{Json, Router, routing::get}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use bytes::Bytes; +use serde_json::json; +use tempfile::TempDir; +use url::Url; + +use knot_atproto::{Atproto, knot_did_document}; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Registration, RegistryChange}; +use knot_events::{EventLog, GlobalSubscriberLimit, PerPeerSubscriberLimit, SubscriberGate}; +use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, +}; +use knot_index::{Index, Resolved}; +use knot_runtime::{ + Clock, Entropy, FakeHttp, HttpRequest, HttpResponse, K256Signer, ManualClock, NetworkError, + PublicKeyBytes, SeededEntropy, Signer, UnixMicros, +}; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{ + AccountDid, AdmissionPolicy, AuthorName, Email, KnotHostname, KnotId, KnotServiceUrl, Oid, + OwnerDid, RecordRkey, RefName, RepoDid, RepoName, RepoRkey, UnixSeconds, +}; +use knot_xrpc::{ + BlobReadBudget, BodyLimit, Budgets, ByteLimits, CobLocks, GlobalInflight, GlobalQuota, + LanguagesReadBudget, LimitConfig, MaxWireBytes, PerActorQuota, PerPeerInflight, PreAuthLimiter, + ReadBudget, ReservationTtl, Reservations, ResponseLimit, TreeReadBudget, XrpcState, +}; + +use crate::realdata::{ + RealActors, SAMPLE_TAG, SAMPLED_REPOS, STRANGER_POOL, SUBJECT_POOL, admin_signer, owner_signer, +}; +use crate::trace::{RepoCollaborators, RoundNumber, Snapshot}; +use crate::workload::Rng; + +const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; +const KNOT_HOST: &str = "knot.nel.pet"; +const ADMIN_HOST: &str = "admin.nel.pet"; +const STRANGER_SEED_BASE: u64 = 1_000; +const PDS_ENDPOINT: &str = "https://pds.nel.pet"; +const START_MICROS: u64 = 1_000_000_000; +const NO_REFLOG_EXPIRY_FLOOR_SECS: i64 = i64::MAX / 4; +const NO_PRUNE_EXPIRY_GRACE_SECS: u64 = (i64::MAX / 4) as u64; +const MAINTAIN_ATTEMPTS: usize = 4; + +const SUBJECT_DIDS: [&str; 8] = [ + "did:plc:limpet", + "did:plc:whelk", + "did:plc:mussel", + "did:plc:conch", + "did:plc:scallop", + "did:plc:cuttle", + "did:plc:periwinkle", + "did:plc:nautilus", +]; + +fn knot_host() -> KnotHostname { + KnotHostname::new(KNOT_HOST).unwrap() +} + +pub(crate) fn subject_dids() -> [AccountDid; 8] { + SUBJECT_DIDS.map(|did| AccountDid::new(did).expect("subject did")) +} + +pub(crate) type Responder = + Box Result + Send + Sync>; + +pub(crate) struct Actor { + pub host: KnotHostname, + pub did: AccountDid, + pub signer: K256Signer, +} + +#[derive(Default)] +pub(crate) struct Faults { + dropped: Mutex>, +} + +impl Faults { + fn is_dropped(&self, host: &KnotHostname) -> bool { + self.dropped.lock().expect("faults lock").contains(host) + } + + pub(crate) fn drop_host(&self, host: &KnotHostname) { + self.dropped + .lock() + .expect("faults lock") + .insert(host.clone()); + } + + pub(crate) fn clear_host(&self, host: &KnotHostname) { + self.dropped.lock().expect("faults lock").remove(host); + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum SocialStage { + Ticket, + Patch, +} + +#[derive(Clone)] +pub(crate) struct SocialProgress { + pub(crate) stage: SocialStage, + pub(crate) base: Oid, + pub(crate) head: Oid, + pub(crate) ticket: Option, +} + +impl SocialProgress { + pub(crate) fn new(base: Oid, head: Oid) -> Self { + Self { + stage: SocialStage::Ticket, + base, + head, + ticket: None, + } + } + + pub(crate) fn record(&mut self, stage: SocialStage, rkey: Option) -> bool { + match stage { + SocialStage::Ticket => { + self.ticket = rkey; + self.stage = SocialStage::Patch; + false + } + SocialStage::Patch => { + self.ticket = None; + self.stage = SocialStage::Ticket; + true + } + } + } +} + +pub(crate) struct Harness { + _dir: Option, + pub clock: Arc, + pub faults: Arc, + pub layout: Layout, + pub index: Arc, + pub knot_aud: KnotId, + router: Router, + boot_state: Arc, Arc>>, + pub admin: Actor, + pub strangers: Vec, + pub subjects: Vec, + pub seed_repo: RepoDid, + pub social: Mutex>, + options: knot_maintenance::Options, + maintain_lock: Mutex<()>, +} + +impl Harness { + pub(crate) fn build(seed: u64, stranger_pool: usize) -> Self { + let dir = tempfile::tempdir().expect("sim tempdir"); + let knot = knot_host().knot_did(); + let layout = Layout::new(dir.path().join("repos")) + .reserving_meta(&knot) + .expect("reserve meta"); + layout.bootstrap_meta(&knot).expect("bootstrap meta"); + let meta_path = layout.meta_path(&knot).expect("meta path"); + + let entropy = Arc::new(SeededEntropy::new(seed ^ 0x5eed_0050)); + let secrets = Arc::new( + SealedStore::open( + dir.path().join("keys.sealed"), + &MasterKey::new([7u8; 32]).unwrap(), + Box::new(SeededEntropy::new(seed ^ 0x5ec0)), + ) + .expect("sealed store"), + ); + let knot_pubkey = secrets.ensure(&knot).expect("seal knot key"); + let knot_service_url = knot_host().service_url(); + + let admin = actor(ADMIN_HOST, 1); + let strangers: Vec = (0..stranger_pool) + .map(|index| { + let host = format!("stranger{index}.nel.pet"); + actor(&host, STRANGER_SEED_BASE + index as u64) + }) + .collect(); + let subjects: Vec = subject_dids().into(); + + let seed_repo = RepoDid::new("did:plc:squid").expect("seed repo did"); + seed_on_disk(&layout, &seed_repo); + register_seed_repo(&meta_path, &knot, &secrets, &admin.did, &seed_repo); + + let index = Arc::new(Index::new(meta_path.clone(), layout.clone())); + index.rebuild().expect("rebuild index"); + index.warm_collaborators(); + + let lfs = { + let lfs_dir = dir.path().join("lfs"); + std::fs::create_dir_all(&lfs_dir).expect("sim lfs dir"); + knot_xrpc::LfsWeb::new( + knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(&lfs_dir), + knot_lfs::LfsSize::new(64 * 1024 * 1024), + knot_lfs::FreeSpaceFloor::new(0), + ) + .expect("sim lfs store"), + 8, + ) + }; + let clock = Arc::new(ManualClock::new(UnixMicros::new(START_MICROS))); + let pubkeys = pubkey_map(&admin, &strangers); + let faults = Arc::new(Faults::default()); + let plc = Url::parse("https://plc.directory/").expect("plc url"); + let responder = build_responder( + Arc::clone(&faults), + pubkeys, + HashMap::new(), + plc.clone(), + plc_signer().public_key(), + ); + let knot_aud = knot.clone(); + let did_document = knot_did_document(&knot, &knot_pubkey, &knot_service_url); + let (router, boot_state) = assemble_router(StateParts { + layout: layout.clone(), + index: Arc::clone(&index), + responder, + secrets, + entropy: entropy as Arc, + admins: BTreeSet::from([admin.did.clone()]), + admission: AdmissionPolicy::Closed, + knot: knot.clone(), + knot_hostname: knot_host(), + meta_path, + knot_service_url, + service_owner: admin.did.clone(), + clock: Arc::clone(&clock), + lfs, + plc, + did_document, + }); + let options = maintenance_options(); + + Self { + _dir: Some(dir), + clock, + faults, + layout, + index, + knot_aud, + router, + boot_state, + admin, + strangers, + subjects, + seed_repo, + social: Mutex::new(HashMap::new()), + options, + maintain_lock: Mutex::new(()), + } + } + + pub fn open(target: &Path, seed: u64) -> Result<(Self, RealActors), OpenError> { + let config = + knot_config::load(Some(&target.join("config.toml"))).map_err(OpenError::Config)?; + let hostname = config.knot_hostname().clone(); + let knot = hostname.knot_did(); + let object_format = config.git.object_format; + let default_branch = config.repo.default_branch.clone(); + let admin_did = config + .server + .admins + .first() + .cloned() + .ok_or(OpenError::NoAdmin)?; + let admission = config.acl.admission; + let plc = config.atproto.plc_directory.clone(); + + let master_key_env = config.secrets.master_key_env.clone(); + let master_key = MasterKey::new( + STANDARD + .decode( + std::env::var(&master_key_env) + .map_err(|_| OpenError::MasterKeyEnv(master_key_env.clone()))? + .trim(), + ) + .map_err(|_| OpenError::MasterKeyDecode)?, + )?; + + let root = target.to_path_buf(); + let layout = Layout::new(root.join("repos")) + .with_default_branch(default_branch) + .with_object_format(object_format) + .reserving_meta(&knot)?; + let meta_path = layout.meta_path(&knot)?; + let secrets = Arc::new(SealedStore::open( + root.join("sealed-keys"), + &master_key, + Box::new(SeededEntropy::new(seed ^ 0x5ec0)), + )?); + let knot_pubkey = secrets.ensure(&knot)?; + let knot_service_url = hostname.service_url(); + + let index = Arc::new(Index::new(meta_path.clone(), layout.clone())); + index.rebuild()?; + index.warm_collaborators(); + + let mut hosted = index.hosted_repos(); + hosted.sort(); + let rng = Rng::new(seed ^ SAMPLE_TAG); + let owner_repos: Vec<(RepoDid, OwnerDid)> = SAMPLED_REPOS + .sample(&hosted, &rng) + .into_iter() + .filter_map(|repo| match index.owner_of(&repo) { + Resolved::Ready(Some(owner)) => Some((repo, owner)), + _ => None, + }) + .collect(); + let subjects = SUBJECT_POOL.sample(&distinct_members(&index), &rng); + + let admin = Actor { + host: hostname.clone(), + did: admin_did.clone(), + signer: admin_signer(seed), + }; + let strangers: Vec = (0..STRANGER_POOL.count()) + .map(|index| { + actor( + &format!("stranger{index}.nel.pet"), + STRANGER_SEED_BASE + index as u64, + ) + }) + .collect(); + + let mut did_overrides: HashMap = HashMap::new(); + did_overrides.insert(admin_did.clone(), admin.signer.public_key()); + owner_repos.iter().for_each(|(_, owner)| { + did_overrides + .entry(AccountDid::from(owner.clone())) + .or_insert_with(|| owner_signer(seed, owner).public_key()); + }); + + let seed_repo = owner_repos + .first() + .map(|(repo, _)| repo.clone()) + .or_else(|| hosted.first().cloned()) + .ok_or(OpenError::NoRepos)?; + + let clock = Arc::new(ManualClock::new(UnixMicros::new(START_MICROS))); + let faults = Arc::new(Faults::default()); + let responder = build_responder( + Arc::clone(&faults), + pubkey_map(&admin, &strangers), + did_overrides, + plc.clone(), + plc_signer().public_key(), + ); + let entropy = Arc::new(SeededEntropy::new(seed ^ 0x5eed_0050)); + let lfs = knot_xrpc::LfsWeb::new( + knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(root.join("lfs")), + knot_lfs::LfsSize::new(64 * 1024 * 1024), + knot_lfs::FreeSpaceFloor::new(0), + )?, + 8, + ); + let knot_aud = knot.clone(); + let did_document = knot_did_document(&knot, &knot_pubkey, &knot_service_url); + let (router, boot_state) = assemble_router(StateParts { + layout: layout.clone(), + index: Arc::clone(&index), + responder, + secrets, + entropy: entropy as Arc, + admins: BTreeSet::from([admin_did.clone()]), + admission, + knot: knot.clone(), + knot_hostname: hostname, + meta_path, + knot_service_url, + service_owner: admin_did.clone(), + clock: Arc::clone(&clock), + lfs, + plc, + did_document, + }); + + let social: Mutex> = Mutex::new( + owner_repos + .iter() + .filter_map(|(repo, _)| { + let opened = layout.open(repo).ok()?; + let main = RefName::new("refs/heads/main").expect("main ref"); + let pull = RefName::new("refs/heads/pull").expect("pull ref"); + let base = opened.find_ref(&main).ok()??; + let head = opened.find_ref(&pull).ok()??; + Some((repo.clone(), SocialProgress::new(base, head))) + }) + .collect(), + ); + + let harness = Self { + _dir: None, + clock, + faults, + layout, + index, + knot_aud, + router, + boot_state, + admin, + strangers, + subjects, + seed_repo, + social, + options: maintenance_options(), + maintain_lock: Mutex::new(()), + }; + Ok((harness, RealActors { owner_repos, seed })) + } + + pub(crate) async fn boot_surfaces(&self) { + let boot = Arc::clone(&self.boot_state); + let failed = + tokio::task::spawn_blocking(move || knot_xrpc::materialize::enable_surfaces(&boot)) + .await + .expect("enable_surfaces must finish"); + assert_eq!(failed, 0, "enable_surfaces must report zero failures"); + } + + pub(crate) fn router(&self) -> Router { + self.router.clone() + } + + pub(crate) fn now_seconds(&self) -> UnixSeconds { + UnixSeconds::new((self.clock.now_unix_micros().get() / 1_000_000) as i64) + } + + pub(crate) fn knot_repo(&self) -> &RepoDid { + &self.boot_state.knot_repo + } + + pub(crate) fn advance(&self, delta: std::time::Duration) { + self.clock.advance(delta); + } + + pub(crate) fn maintain(&self, repo: &RepoDid) -> Result<(), knot_maintenance::MaintError> { + let _serialized = self.maintain_lock.lock().expect("maintenance lock"); + let now = self.now_seconds(); + let attempt = || -> Result<(), knot_maintenance::MaintError> { + let opened = self + .layout + .open(repo) + .map_err(knot_maintenance::MaintError::from)?; + knot_maintenance::run_repo(&opened, now, &self.options).map(|_| ()) + }; + (1..MAINTAIN_ATTEMPTS).fold(attempt(), |result, _| result.or_else(|_| attempt())) + } + + pub(crate) fn populate(&self, repo: &RepoDid) { + let opened = self.layout.open(repo).expect("open created repo"); + write_history(&opened, repo); + } + + pub(crate) fn seed_social(&self, repo: &RepoDid) { + let opened = self.layout.open(repo).expect("open created repo"); + let (base, head) = write_pull(&opened, repo); + self.social + .lock() + .expect("social lock") + .insert(repo.clone(), SocialProgress::new(base, head)); + } + + pub(crate) fn snapshot(&self, round: RoundNumber, repos: &[RepoDid]) -> Snapshot { + let collaborators = repos + .iter() + .map(|repo| { + let _ = self.index.ensure_collaborators(repo); + RepoCollaborators { + repo: repo.clone(), + subjects: subjects(self.index.effective_collaborators(repo)), + } + }) + .collect(); + let repo_list = { + let mut repos: Vec = self.index.hosted_repos().to_vec(); + repos.sort(); + repos + }; + Snapshot { + round, + clock_micros: self.clock.now_unix_micros(), + members: subjects(self.index.effective_members()), + blocked: subjects(self.index.blocked()), + repos: repo_list, + collaborators, + } + } +} + +fn actor(host: &str, seed: u64) -> Actor { + Actor { + host: KnotHostname::new(host).expect("actor hostname"), + did: AccountDid::new(format!("did:web:{host}")).expect("actor did"), + signer: K256Signer::generate(&SeededEntropy::new(seed)), + } +} + +fn pubkey_map(admin: &Actor, strangers: &[Actor]) -> HashMap { + std::iter::once((admin.host.clone(), admin.signer.public_key())) + .chain( + strangers + .iter() + .map(|actor| (actor.host.clone(), actor.signer.public_key())), + ) + .collect() +} + +pub(crate) fn plc_signer() -> K256Signer { + K256Signer::generate(&SeededEntropy::new(7)) +} + +fn build_responder( + faults: Arc, + pubkeys: HashMap, + did_overrides: HashMap, + plc: Url, + plc_pubkey: PublicKeyBytes, +) -> Responder { + Box::new(move |request: &HttpRequest| { + let not_found = || HttpResponse { + status: http::StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: Bytes::new(), + }; + let Ok(host) = KnotHostname::new(request.url.host_str().unwrap_or_default()) else { + return Ok(not_found()); + }; + if faults.is_dropped(&host) { + return Err(NetworkError::Timeout( + "identity resolution dropped by simulation".to_string(), + )); + } + if request.method == http::Method::POST { + return Ok(ok_body(Bytes::new())); + } + let is_plc = request.url.host() == plc.host(); + let requested_did = if is_plc { + request + .url + .path_segments() + .and_then(|mut segments| segments.rfind(|segment| !segment.is_empty())) + .and_then(|segment| AccountDid::new(segment).ok()) + } else { + AccountDid::new(format!("did:web:{host}")).ok() + }; + let Some(requested_did) = requested_did else { + return Ok(not_found()); + }; + if let Some(sec1) = did_overrides.get(&requested_did) { + return Ok(ok_body(did_doc(&requested_did, sec1))); + } + if is_plc { + return Ok(ok_body(did_doc(&requested_did, &plc_pubkey))); + } + match pubkeys.get(&host) { + Some(sec1) => Ok(ok_body(did_doc(&requested_did, sec1))), + None => Ok(not_found()), + } + }) +} + +fn did_doc(did: &AccountDid, sec1: &PublicKeyBytes) -> Bytes { + let multikey = knot_types::crypto::multikey(0xe7, sec1.as_bytes()); + Bytes::from( + serde_json::to_vec(&json!({ + "id": did, + "alsoKnownAs": [], + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": PDS_ENDPOINT + }] + })) + .expect("did doc serializes"), + ) +} + +fn ok_body(body: Bytes) -> HttpResponse { + HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body, + } +} + +fn seed_on_disk(layout: &Layout, did: &RepoDid) { + let repo = layout.create(did).expect("create seed repo"); + write_history(&repo, did); +} + +fn identity() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } +} + +fn write_history(repo: &Repo, did: &RepoDid) { + let identity = identity(); + let main = RefName::new("refs/heads/main").expect("main ref"); + let first_tree = repo + .write_staged_tree( + Oid::from_hex(EMPTY_TREE).expect("empty tree"), + &[StagedChange { + path: knot_types::RepoPath::new("README.md").unwrap(), + action: StagedAction::Put { + content: format!("# {did}\n").into_bytes(), + kind: EntryKind::Blob, + }, + }], + ) + .expect("first tree"); + let root = repo + .write_commit(&NewCommit { + tree: first_tree, + parents: Vec::new(), + author: identity.clone(), + committer: identity.clone(), + message: "root".to_string(), + extra_headers: Vec::new(), + }) + .expect("root commit"); + let second_tree = repo + .write_staged_tree( + first_tree, + &[StagedChange { + path: knot_types::RepoPath::new("src/main.rs").unwrap(), + action: StagedAction::Put { + content: b"fn main() {}\n".to_vec(), + kind: EntryKind::Blob, + }, + }], + ) + .expect("second tree"); + let tip = repo + .write_commit(&NewCommit { + tree: second_tree, + parents: vec![root], + author: identity.clone(), + committer: identity, + message: "add main".to_string(), + extra_headers: Vec::new(), + }) + .expect("second commit"); + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: tip, + }) + .expect("create main"); + repo.set_head(&main).expect("set head"); +} + +pub fn write_pull(repo: &Repo, did: &RepoDid) -> (Oid, Oid) { + let main = RefName::new("refs/heads/main").expect("main ref"); + if repo.find_ref(&main).ok().flatten().is_none() { + write_history(repo, did); + } + let pull = RefName::new("refs/heads/pull").expect("pull ref"); + let base = repo + .find_ref(&main) + .expect("main exists") + .expect("main tip"); + let base_tree = repo.find_commit(base).expect("base commit").tree; + let identity = identity(); + let head_tree = repo + .write_staged_tree( + base_tree, + &[StagedChange { + path: knot_types::RepoPath::new("feature.txt").unwrap(), + action: StagedAction::Put { + content: b"feature\n".to_vec(), + kind: EntryKind::Blob, + }, + }], + ) + .expect("head tree"); + let head = repo + .write_commit(&NewCommit { + tree: head_tree, + parents: vec![base], + author: identity.clone(), + committer: identity, + message: "feature".to_string(), + extra_headers: Vec::new(), + }) + .expect("head commit"); + repo.set_reference(&pull, head).expect("set pull"); + (base, head) +} + +fn register_seed_repo( + meta_path: &std::path::Path, + knot: &KnotId, + secrets: &SealedStore, + owner: &AccountDid, + repo: &RepoDid, +) { + let meta = Repo::open(meta_path).expect("open meta"); + let store = CobStore::new(&meta); + let signer = secrets.signer(knot).expect("knot signer"); + store + .create( + &CobHome::from(knot), + &RegistryChange::Register(Registration { + owner: OwnerDid::from(owner.clone()), + rkey: RepoRkey::new("anemone").expect("rkey"), + name: RepoName::new("anemone").expect("name"), + repo: repo.clone(), + created_at: UnixSeconds::new(1), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + + retired_at: None, + }), + &signer, + UnixSeconds::new(1), + ) + .expect("register seed repo"); +} + +fn subjects(resolved: Resolved>) -> Vec { + match resolved { + Resolved::Ready(subjects) => subjects, + Resolved::Warming => Vec::new(), + } +} + +fn distinct_members(index: &Index) -> Vec { + subjects(index.effective_members()) +} + +fn maintenance_options() -> knot_maintenance::Options { + knot_maintenance::Options { + repack_max_objects: knot_types::ObjectCount::new(1_000_000), + geometric_factor: knot_maintenance::GeometricFactor::full_repack(), + loose_threshold: knot_types::ObjectCount::new(0), + prune_grace: knot_maintenance::PruneGrace::from_secs(NO_PRUNE_EXPIRY_GRACE_SECS), + full_repack_interval: knot_maintenance::FullRepackInterval::from_secs(432_000), + reflog_floor: knot_maintenance::ReflogRetention::from_secs( + NO_REFLOG_EXPIRY_FLOOR_SECS as u64, + ), + commit_graph: true, + multi_pack_index: true, + bitmap: true, + } +} + +struct StateParts { + layout: Layout, + index: Arc, + responder: Responder, + secrets: Arc, + entropy: Arc, + admins: BTreeSet, + admission: AdmissionPolicy, + knot: KnotId, + knot_hostname: KnotHostname, + meta_path: PathBuf, + knot_service_url: KnotServiceUrl, + service_owner: AccountDid, + clock: Arc, + lfs: knot_xrpc::LfsWeb, + plc: Url, + did_document: serde_json::Value, +} + +type FakeState = Arc, Arc>>; + +fn assemble_router(parts: StateParts) -> (Router, FakeState) { + let StateParts { + layout, + index, + responder, + secrets, + entropy, + admins, + admission, + knot, + knot_hostname, + meta_path, + knot_service_url, + service_owner, + clock, + lfs, + plc, + did_document, + } = parts; + let atproto = Arc::new(Atproto::new( + FakeHttp::new(responder), + Arc::clone(&clock), + knot.clone(), + knot_atproto::PlcDirectory::new(plc).expect("plc directory"), + )); + let outbox = knot_xrpc::Outbox::open(&meta_path, Default::default()); + let state = Arc::new(XrpcState { + imports: knot_xrpc::imports::ImportQueue::channel().0, + handover_sessions: Arc::new(knot_xrpc::handover::HandoverSessions::default()), + layout: layout.clone(), + index: Arc::clone(&index), + atproto, + secrets, + entropy, + invite_codes: Default::default(), + admins, + admission, + contribution_policy: knot_types::ContributionPolicy::Anyone, + knot_repo: knot_xrpc::knot_repo(&knot).unwrap(), + knot_did: knot, + knot_hostname, + meta_path, + knot_service_url, + writes: Arc::new(knot_resource::SubjectLimiter::new( + knot_resource::RateLimit { + burst: knot_xrpc::Burst::literal(1_000), + refill: knot_xrpc::RefillMicros::literal(1_000), + }, + )), + social_budget: knot_resource::SocialBudgetBytes::new(1 << 28), + limiter: Arc::new(PreAuthLimiter::with_config(LimitConfig { + rate: None, + per_peer_inflight: Some(PerPeerInflight::new(4096)), + global_inflight: Some(GlobalInflight::new(4096)), + })), + cob_locks: Arc::new(CobLocks::default()), + reservations: Arc::new(Reservations::new( + ReservationTtl::new(1_000_000), + PerActorQuota::new(256), + GlobalQuota::new(256), + )), + proxy_trust: knot_types::ProxyTrust::default(), + byte_limits: ByteLimits { + body: BodyLimit::new(256 * 1024), + response: ResponseLimit::new(8 * 1024 * 1024), + pack: MaxWireBytes::new(1024 * 1024 * 1024), + ..ByteLimits::default() + }, + budgets: Budgets { + tree_last_commit: TreeReadBudget::new(ReadBudget::Unbounded), + blob_last_commit: BlobReadBudget::new(ReadBudget::Unbounded), + languages: LanguagesReadBudget::new(ReadBudget::Unbounded), + }, + git_http: Arc::new(FakeHttp::new(|_request: &HttpRequest| { + Err(NetworkError::Connect( + "simulation serves no git upstream".to_string(), + )) + })), + pack_limits: knot_pack::PackLimits::default(), + service_owner, + legacy_admin_secret: None, + description: Default::default(), + firehose: Arc::new(EventLog::new( + Arc::clone(&clock), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(4096).expect("replay event maximum is nonzero"), + knot_events::ReplayBytes::new(64 << 20).expect("replay byte maximum is nonzero"), + ), + )), + subscriber_gate: Arc::new(SubscriberGate::new( + GlobalSubscriberLimit::new(256), + PerPeerSubscriberLimit::new(64), + )), + maintenance: knot_maintenance::MaintenanceHandle::disabled(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + slots: knot_resource::Slots::testing(8), + lfs, + blobs: Default::default(), + catalog: Arc::new(knot_messages::Catalog::defaults()), + rkeys: Default::default(), + outbox, + }); + let resolver: Arc = { + let index = Arc::clone(&index); + Arc::new(move |target: &knot_pack::RepoTarget| match target { + knot_pack::RepoTarget::Did(did) => match index.owner_of(did) { + Resolved::Ready(Some(_)) => knot_pack::RepoLookup::Hosted(did.clone()), + Resolved::Ready(None) => knot_pack::RepoLookup::Unhosted, + Resolved::Warming => knot_pack::RepoLookup::Unavailable, + }, + knot_pack::RepoTarget::OwnerPath(owner, path) => { + match index.resolve_clone_path(owner, path) { + Resolved::Ready(Some(found)) => knot_pack::RepoLookup::Hosted(found), + Resolved::Ready(None) => knot_pack::RepoLookup::Unhosted, + Resolved::Warming => knot_pack::RepoLookup::Unavailable, + } + } + }) + }; + ( + knot_pack::router(layout, resolver, Arc::clone(&clock) as Arc) + .merge(knot_xrpc::router(Arc::clone(&state))) + .route( + "/.well-known/did.json", + get(move || { + let document = did_document.clone(); + async move { Json(document) } + }), + ), + state, + ) +} + +#[derive(Debug)] +pub enum OpenError { + MasterKeyEnv(String), + MasterKeyDecode, + NoAdmin, + NoRepos, + Config(knot_config::LoadError), + Git(knot_git::GitError), + Secrets(knot_secrets::SecretsError), + Lfs(knot_lfs::LfsError), + Index(knot_index::IndexError), +} + +impl std::fmt::Display for OpenError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::MasterKeyEnv(name) => write!(f, "{name} is not set"), + Self::MasterKeyDecode => write!(f, "master key is not valid base64"), + Self::NoAdmin => write!(f, "config lists no admin"), + Self::NoRepos => write!(f, "target hosts no repos to sample"), + Self::Config(error) => write!(f, "read config: {error}"), + Self::Git(error) => write!(f, "open target repos: {error}"), + Self::Secrets(error) => write!(f, "open sealed key store: {error}"), + Self::Index(error) => write!(f, "rebuild index: {error}"), + Self::Lfs(error) => write!(f, "open lfs store: {error}"), + } + } +} + +impl std::error::Error for OpenError {} + +impl From for OpenError { + fn from(error: knot_git::GitError) -> Self { + Self::Git(error) + } +} + +impl From for OpenError { + fn from(error: knot_secrets::SecretsError) -> Self { + Self::Secrets(error) + } +} + +impl From for OpenError { + fn from(error: knot_lfs::LfsError) -> Self { + Self::Lfs(error) + } +} +impl From for OpenError { + fn from(error: knot_index::IndexError) -> Self { + Self::Index(error) + } +} diff --git a/knot2/crates/knot-sim/src/lib.rs b/knot2/crates/knot-sim/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..ff84a109fcc7b22bcb9a3a5b0031602148fe8792 --- /dev/null +++ b/knot2/crates/knot-sim/src/lib.rs @@ -0,0 +1,26 @@ +mod harness; +mod realdata; +pub mod trace; +mod workload; + +pub use harness::{OpenError, write_pull}; +pub use realdata::run as run_realdata; +pub use trace::{ + OperationIndex, Outcome, Projection, RepoCollaborators, RoundNumber, Snapshot, Step, Trace, +}; + +use std::sync::Arc; + +use harness::Harness; + +pub async fn run(seed: u64, rounds: u32) -> Trace { + let stranger_pool = (rounds as usize) * 2 + 16; + let harness = Arc::new(Harness::build(seed, stranger_pool)); + harness.boot_surfaces().await; + let plan = workload::plan(seed, rounds, harness.subjects.len()); + workload::execute(harness, seed, plan).await +} + +pub fn predict(seed: u64, rounds: u32) -> Projection { + workload::predict(seed, rounds) +} diff --git a/knot2/crates/knot-sim/src/main.rs b/knot2/crates/knot-sim/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..b56f1634e83158aedf88e17e180cba2577d4a14b --- /dev/null +++ b/knot2/crates/knot-sim/src/main.rs @@ -0,0 +1,57 @@ +#[tokio::main(flavor = "multi_thread", worker_threads = 4)] +async fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let dump = args.iter().any(|arg| arg == "dump"); + let in_place = args.iter().any(|arg| arg == "--in-place"); + let target_flag = args.iter().position(|arg| arg == "--target"); + let target = match target_flag { + Some(position) => match args.get(position + 1) { + Some(path) => Some(path.clone()), + None => { + eprintln!("knot-sim: --target needs a path"); + std::process::exit(1); + } + }, + None => None, + }; + let numbers: Vec = args + .iter() + .enumerate() + .filter(|(index, _)| { + !matches!(target_flag, Some(position) if *index == position || *index == position + 1) + }) + .filter_map(|(_, arg)| arg.parse().ok()) + .collect(); + let seed = numbers.first().copied().unwrap_or(1); + let rounds = numbers.get(1).copied().unwrap_or(12) as u32; + let trace = match target { + Some(path) => { + if !in_place { + eprintln!( + "knot-sim: --target writes the knot in place; pass --in-place to accept that" + ); + std::process::exit(1); + } + match knot_sim::run_realdata(std::path::Path::new(&path), seed, rounds).await { + Ok(trace) => trace, + Err(error) => { + eprintln!("knot-sim: {error}"); + std::process::exit(1); + } + } + } + None => knot_sim::run(seed, rounds).await, + }; + if dump { + println!( + "{}", + serde_json::to_string_pretty(&trace).expect("trace serializes") + ); + } + println!( + "knot-sim seed={seed} rounds={rounds} steps={} snapshots={} digest={:016x}", + trace.steps.len(), + trace.snapshots.len(), + trace.digest() + ); +} diff --git a/knot2/crates/knot-sim/src/realdata.rs b/knot2/crates/knot-sim/src/realdata.rs new file mode 100644 index 0000000000000000000000000000000000000000..ef272a13fd632b79be022b0a5748ff588b4be806 --- /dev/null +++ b/knot2/crates/knot-sim/src/realdata.rs @@ -0,0 +1,668 @@ +use std::path::Path; +use std::sync::Arc; + +use axum::body::Bytes; +use futures::future::join_all; +use futures::stream::StreamExt; +use serde_json::json; + +use knot_runtime::{K256Signer, SeededEntropy}; +use knot_types::{ + AccountDid, HttpStatus, KnotHostname, OwnerDid, RecordCollection, RecordRkey, RepoDid, +}; + +use crate::harness::{Harness, SocialProgress, SocialStage}; +use crate::trace::{OperationIndex, Outcome, RoundNumber, Snapshot, Step, Trace, fnv1a}; +use crate::workload::OpSlot; +use crate::workload::{ + Request, Rng, block_collection, body_digest, collaborator_collection, drive_kill, enc, + grant_body, http_call, iso_utc, maintain_outcome, member_collection, nsid, repo_did_of, + signed_post, +}; + +pub(crate) const SAMPLE_TAG: u64 = 0x5a3d_e005; +pub(crate) const SAMPLED_REPOS: SampledRepos = SampledRepos(24); +pub(crate) const SUBJECT_POOL: SubjectPool = SubjectPool(16); +pub(crate) const STRANGER_POOL: StrangerPool = StrangerPool(32); +const ADMIN_SIGNER_TAG: u64 = 0x0ad0_0001; +const DRIVE_TAG: u64 = 0x11c7_0005; +const TICKET_COLLECTION: &str = "org.tangled.track.ticket"; +const CREATE_RECORD_NSID: &str = "com.atproto.repo.createRecord"; +const DELETE_RECORD_NSID: &str = "com.atproto.repo.deleteRecord"; +const SUBMIT_PATCH_NSID: &str = "org.tangled.review.submitPatch"; + +fn ticket_collection() -> RecordCollection { + RecordCollection::new(TICKET_COLLECTION).unwrap() +} + +#[derive(Clone, Copy)] +pub(crate) struct SampledRepos(usize); + +#[derive(Clone, Copy)] +pub(crate) struct SubjectPool(usize); + +#[derive(Clone, Copy)] +pub(crate) struct StrangerPool(usize); + +impl SampledRepos { + pub(crate) fn sample(self, source: &[RepoDid], rng: &Rng) -> Vec { + pick(source, self.0, rng) + } +} + +impl SubjectPool { + pub(crate) fn sample(self, source: &[AccountDid], rng: &Rng) -> Vec { + pick(source, self.0, rng) + } +} + +impl StrangerPool { + pub(crate) fn count(self) -> usize { + self.0 + } +} + +pub struct RealActors { + pub(crate) owner_repos: Vec<(RepoDid, OwnerDid)>, + pub(crate) seed: u64, +} + +pub(crate) fn owner_signer(seed: u64, owner: &OwnerDid) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed ^ fnv1a(owner.as_str().as_bytes()))) +} + +pub(crate) fn admin_signer(seed: u64) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed ^ ADMIN_SIGNER_TAG)) +} + +fn pick(source: &[T], count: usize, rng: &Rng) -> Vec { + if source.is_empty() { + return Vec::new(); + } + (0..count.min(source.len())) + .scan(Vec::::new(), |taken, _| { + let start = rng.below(source.len() as u64) as usize; + let index = (start..source.len()) + .chain(0..start) + .find(|candidate| !taken.contains(candidate)) + .expect("distinct index exists while count <= len"); + taken.push(index); + Some(source[index].clone()) + }) + .collect() +} + +pub async fn run( + target: &Path, + seed: u64, + rounds: u32, +) -> Result { + let (harness, actors) = Harness::open(target, seed)?; + harness.boot_surfaces().await; + Ok(drive(Arc::new(harness), actors, rounds).await) +} + +struct OpResult { + step: Step, + created: Option, + social: Option, +} + +struct SocialOutcome { + repo: RepoDid, + stage: SocialStage, + rkey: Option, +} + +enum Exec { + Http { + op: &'static str, + actor: String, + fault: &'static str, + request: Request, + killed: bool, + capture_create: bool, + social: Option<(RepoDid, SocialStage)>, + }, + Maintain { + repo: RepoDid, + }, +} + +async fn drive(harness: Arc, actors: RealActors, rounds: u32) -> Trace { + let seed = actors.seed; + let rng = Rng::new(seed ^ DRIVE_TAG); + let harness_ref = &harness; + let actors_ref = &actors; + let rng_ref = &rng; + let base: Vec = actors + .owner_repos + .iter() + .map(|(repo, _)| repo.clone()) + .collect(); + let (_created, steps, snapshots) = futures::stream::iter(0..rounds) + .fold( + ( + Vec::::new(), + Vec::::new(), + Vec::::new(), + ), + |(created, mut steps, mut snapshots), round_index| { + let harness = Arc::clone(harness_ref); + let base = &base; + async move { + let round = RoundNumber::new(round_index); + let working: Vec = + base.iter().chain(created.iter()).cloned().collect(); + let execs = if round_index % 2 == 0 { + mutate(&harness, actors_ref, &working, rng_ref, round) + } else { + reads(&working, rng_ref) + }; + let dropped = arm_drops(&harness, &execs); + + let tasks = execs + .into_iter() + .enumerate() + .map(|(position, exec)| { + let harness = Arc::clone(&harness); + let index = OperationIndex::new(position as u32); + tokio::spawn(async move { run_op(&harness, round, index, exec).await }) + }) + .collect::>(); + let results: Vec = join_all(tasks) + .await + .into_iter() + .map(|joined| joined.expect("real-data op task mustn't panic")) + .collect(); + dropped + .iter() + .for_each(|host| harness.faults.clear_host(host)); + + let fresh: Vec = results + .iter() + .filter_map(|result| result.created.clone()) + .collect(); + fresh.iter().for_each(|did| harness.seed_social(did)); + results + .iter() + .filter_map(|result| result.social.as_ref()) + .for_each(|outcome| { + let complete = harness + .social + .lock() + .expect("social lock") + .get_mut(&outcome.repo) + .map(|progress| { + progress.record(outcome.stage, outcome.rkey.clone()) + }) + .unwrap_or(false); + if complete { + harness.seed_social(&outcome.repo); + } + }); + steps.extend(results.into_iter().map(|result| result.step)); + + let mut created = created; + created.extend(fresh); + harness.advance(round_advance(rng_ref)); + let snapshot_repos: Vec = + base.iter().chain(created.iter()).cloned().collect(); + snapshots.push(harness.snapshot(round, &snapshot_repos)); + (created, steps, snapshots) + } + }, + ) + .await; + Trace { + seed, + steps, + snapshots, + } +} + +fn round_advance(rng: &Rng) -> std::time::Duration { + std::time::Duration::from_micros(rng.below(3_000_000)) +} + +fn arm_drops(harness: &Harness, execs: &[Exec]) -> Vec { + let hosts: Vec = execs + .iter() + .filter_map(|exec| match exec { + Exec::Http { + op: "probe", + fault: "drop_identity", + actor, + .. + } => KnotHostname::new(actor).ok(), + _ => None, + }) + .collect(); + hosts.iter().for_each(|host| harness.faults.drop_host(host)); + hosts +} + +fn member_verb(draw: u64) -> (&'static str, RecordCollection, bool) { + match draw { + 0 => ("addMember", member_collection(), true), + 1 => ("removeMember", member_collection(), false), + 2 => ("ban", block_collection(), true), + _ => ("unban", block_collection(), false), + } +} + +fn mutate( + harness: &Harness, + actors: &RealActors, + working: &[RepoDid], + rng: &Rng, + round: RoundNumber, +) -> Vec { + let subjects = &harness.subjects; + let mut execs: Vec = Vec::new(); + + subjects + .iter() + .filter(|_| rng.chance(2, 3)) + .for_each(|subject| { + let (short, collection, creates) = member_verb(rng.below(4)); + let method = nsid(if creates { + CREATE_RECORD_NSID + } else { + DELETE_RECORD_NSID + }); + let skew = rng.chance(1, 5); + let body = grant_body( + harness.knot_repo(), + &collection, + subject, + creates.then(|| iso_utc(harness.now_seconds())).as_ref(), + ); + let request = signed_post( + harness, + &harness.admin.signer, + &harness.admin.did, + &method, + &harness.knot_aud, + body, + OpSlot { + skew, + round, + index: OperationIndex::new(execs.len() as u32), + }, + ); + execs.push(Exec::Http { + op: short, + actor: format!("admin:{short}"), + fault: skew_fault(skew), + request, + killed: false, + capture_create: false, + social: None, + }); + }); + + let collaborated: Vec = if subjects.is_empty() { + Vec::new() + } else { + actors + .owner_repos + .iter() + .filter(|_| rng.chance(1, 2)) + .map(|(repo, owner)| { + let subject = &subjects[rng.below(subjects.len() as u64) as usize]; + let skew = rng.chance(1, 6); + let request = signed_post( + harness, + &owner_signer(actors.seed, owner), + &AccountDid::from(owner.clone()), + &nsid(CREATE_RECORD_NSID), + repo, + grant_body( + repo, + &collaborator_collection(), + subject, + Some(&iso_utc(harness.now_seconds())), + ), + OpSlot { + skew, + round, + index: OperationIndex::new(execs.len() as u32), + }, + ); + execs.push(Exec::Http { + op: "addCollaborator", + actor: format!("owner:{owner}"), + fault: skew_fault(skew), + request, + killed: false, + capture_create: false, + social: None, + }); + repo.clone() + }) + .collect() + }; + + working + .iter() + .filter(|repo| !collaborated.contains(*repo)) + .filter(|_| rng.chance(1, 3)) + .for_each(|repo| execs.push(Exec::Maintain { repo: repo.clone() })); + + (0..rng.below(3)).for_each(|key| { + let name = format!("sim-repo-{}-{}", round.get(), key); + let skew = rng.chance(1, 8); + let request = signed_post( + harness, + &harness.admin.signer, + &harness.admin.did, + &nsid("org.tangled.repo.createRepo"), + &harness.knot_aud, + json!({ "name": name }), + OpSlot { + skew, + round, + index: OperationIndex::new(execs.len() as u32), + }, + ); + execs.push(Exec::Http { + op: "createRepo", + actor: "admin:create".to_string(), + fault: skew_fault(skew), + request, + killed: false, + capture_create: !skew, + social: None, + }); + }); + + if !subjects.is_empty() { + let probes = rng.below(3) as usize; + let slots: Vec = (0..harness.strangers.len()).collect(); + pick(&slots, probes, rng).into_iter().for_each(|slot| { + let stranger = &harness.strangers[slot]; + let drop = rng.chance(1, 2); + let request = signed_post( + harness, + &stranger.signer, + &stranger.did, + &nsid(CREATE_RECORD_NSID), + &harness.knot_aud, + grant_body( + harness.knot_repo(), + &member_collection(), + &subjects[0], + Some(&iso_utc(harness.now_seconds())), + ), + OpSlot { + skew: false, + round, + index: OperationIndex::new(execs.len() as u32), + }, + ); + execs.push(Exec::Http { + op: "probe", + actor: stranger.host.to_string(), + fault: if drop { "drop_identity" } else { "none" }, + request, + killed: false, + capture_create: false, + social: None, + }); + }); + } + + let social_repos: Vec<(RepoDid, SocialProgress)> = { + let social = harness.social.lock().expect("social lock"); + let mut repos: Vec<_> = social + .iter() + .map(|(repo, progress)| (repo.clone(), progress.clone())) + .collect(); + repos.sort_by(|a, b| a.0.cmp(&b.0)); + repos + }; + + social_repos.into_iter().for_each(|(repo, progress)| { + let SocialProgress { + stage, + base, + head, + ticket, + } = progress; + let (signer, did) = actors + .owner_repos + .iter() + .find(|(candidate, _)| candidate == &repo) + .map(|(_, owner)| { + ( + owner_signer(actors.seed, owner), + AccountDid::from(owner.clone()), + ) + }) + .unwrap_or_else(|| (admin_signer(actors.seed), harness.admin.did.clone())); + let (op, method, body) = match stage { + SocialStage::Ticket => ( + "createTicket", + nsid(CREATE_RECORD_NSID), + json!({ + "repo": repo, + "collection": &ticket_collection(), + "record": { + "$type": &ticket_collection(), + "repo": repo, + "title": "sim ticket", + "body": "a ticket from the sim", + "state": "open", + "createdAt": "2026-09-11T00:00:00Z" + } + }), + ), + SocialStage::Patch => ( + "createPatch", + nsid(SUBMIT_PATCH_NSID), + json!({ + "repo": repo, + "patch": { + "ticket": ticket.expect("ticket rkey"), + "base": { "oid": base.to_hex() }, + "head": { "oid": head.to_hex() }, + "sourceRepo": repo + } + }), + ), + }; + let request = signed_post( + harness, + &signer, + &did, + &method, + &repo, + body, + OpSlot { + skew: false, + round, + index: OperationIndex::new(execs.len() as u32), + }, + ); + execs.push(Exec::Http { + op, + actor: format!("owner:{op}"), + fault: "none", + request, + killed: false, + capture_create: false, + social: Some((repo, stage)), + }); + }); + + execs +} + +fn skew_fault(skew: bool) -> &'static str { + if skew { "clock_skew" } else { "none" } +} + +fn reads(working: &[RepoDid], rng: &Rng) -> Vec { + let mut execs: Vec<_> = [ + ("version", "/xrpc/org.tangled.knot.describeKnot"), + ("describeKnot", "/xrpc/org.tangled.knot.describeKnot"), + ("didJson", "/.well-known/did.json"), + ] + .into_iter() + .map(|(op, uri)| anon_read(op, uri, rng.chance(1, 5))) + .collect(); + + working + .iter() + .filter(|_| rng.chance(2, 3)) + .for_each(|repo| { + let killed = rng.chance(1, 5); + let exec = match rng.below(7) { + 0 => repo_read("branches", "org.tangled.git.listBranches", repo, killed), + 1 => anon_read( + "log", + &format!( + "/xrpc/org.tangled.git.listCommits?repo={}&ref=HEAD", + enc(repo) + ), + killed, + ), + 2 => repo_read( + "describeRepo", + "com.atproto.repo.describeRepo", + repo, + killed, + ), + 3 => anon_read( + "infoRefs", + &format!("/{}/info/refs?service=git-upload-pack", repo), + killed, + ), + 4 => repo_read("tree", "org.tangled.git.getTree", repo, killed), + 5 => blob_read(repo, killed), + _ => repo_read("languages", "org.tangled.git.listLanguages", repo, killed), + }; + execs.push(exec); + }); + execs +} + +fn anon_read(op: &'static str, uri: &str, killed: bool) -> Exec { + Exec::Http { + op, + actor: "anon".to_string(), + fault: if killed { "killed" } else { "none" }, + request: Request::get(uri), + killed, + capture_create: false, + social: None, + } +} + +fn repo_read(op: &'static str, method: &str, repo: &RepoDid, killed: bool) -> Exec { + anon_read(op, &format!("/xrpc/{method}?repo={}", enc(repo)), killed) +} + +fn blob_read(repo: &RepoDid, killed: bool) -> Exec { + anon_read( + "blob", + &format!( + "/xrpc/org.tangled.git.getBlob?repo={}&path=README.md&ref=HEAD", + enc(repo) + ), + killed, + ) +} + +async fn run_op( + harness: &Harness, + round: RoundNumber, + index: OperationIndex, + exec: Exec, +) -> OpResult { + match exec { + Exec::Maintain { repo } => OpResult { + step: Step { + round, + index, + op: "maintain", + actor: "knot".to_string(), + fault: "none", + outcome: maintain_outcome(harness, &repo), + }, + created: None, + social: None, + }, + Exec::Http { + op, + actor, + fault, + request, + killed, + capture_create, + social, + } => { + if killed { + drive_kill(harness.router(), request.method, &request.uri, request.body).await; + return OpResult { + step: Step { + round, + index, + op, + actor, + fault: "killed", + outcome: Outcome::Killed, + }, + created: None, + social: None, + }; + } + let (status, body) = http_call( + harness.router(), + request.method, + &request.uri, + request.token.as_deref(), + request.body, + ) + .await; + let created = (capture_create && status == HttpStatus::new(200)) + .then(|| repo_did_of(&body).expect("200 createRepo response must include a did")); + let social = social.and_then(|(repo, stage)| { + (status == HttpStatus::new(200)).then(|| SocialOutcome { + repo, + stage, + rkey: rkey_of(&body), + }) + }); + OpResult { + step: Step { + round, + index, + op, + actor, + fault, + outcome: Outcome::Served { + status, + body: body_digest(&body), + }, + }, + created, + social, + } + } + } +} + +fn rkey_of(body: &Bytes) -> Option { + serde_json::from_slice::(body) + .ok() + .and_then(|value| { + value + .get("uri") + .and_then(serde_json::Value::as_str) + .map(str::to_string) + }) + .and_then(|uri| uri.rsplit('/').next().map(str::to_string)) + .and_then(|rkey| RecordRkey::new(rkey).ok()) +} diff --git a/knot2/crates/knot-sim/src/trace.rs b/knot2/crates/knot-sim/src/trace.rs new file mode 100644 index 0000000000000000000000000000000000000000..c2ec7a41416a56cd431de4846ca0c5b9974e0827 --- /dev/null +++ b/knot2/crates/knot-sim/src/trace.rs @@ -0,0 +1,92 @@ +use knot_types::{AccountDid, HttpStatus, RepoDid, UnixMicros}; +use serde::Serialize; + +const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; +const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + +pub(crate) fn fnv1a(bytes: &[u8]) -> u64 { + bytes.iter().fold(FNV_OFFSET, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(FNV_PRIME) + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] +#[serde(transparent)] +pub struct RoundNumber(u32); + +impl RoundNumber { + pub const fn new(value: u32) -> Self { + Self(value) + } + + pub const fn get(self) -> u32 { + self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] +#[serde(transparent)] +pub struct OperationIndex(u32); + +impl OperationIndex { + pub const fn new(value: u32) -> Self { + Self(value) + } + + pub const fn get(self) -> u32 { + self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Outcome { + Served { status: HttpStatus, body: u64 }, + Killed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Step { + pub round: RoundNumber, + pub index: OperationIndex, + pub op: &'static str, + pub actor: String, + pub fault: &'static str, + pub outcome: Outcome, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct RepoCollaborators { + pub repo: RepoDid, + pub subjects: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Projection { + pub members: Vec, + pub blocked: Vec, + pub collaborators: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Snapshot { + pub round: RoundNumber, + pub clock_micros: UnixMicros, + pub members: Vec, + pub blocked: Vec, + pub repos: Vec, + pub collaborators: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Trace { + pub seed: u64, + pub steps: Vec, + pub snapshots: Vec, +} + +impl Trace { + pub fn digest(&self) -> u64 { + fnv1a(&serde_json::to_vec(self).expect("trace is always serializable")) + } +} diff --git a/knot2/crates/knot-sim/src/workload.rs b/knot2/crates/knot-sim/src/workload.rs new file mode 100644 index 0000000000000000000000000000000000000000..a833be5d614cb19f3d47a4a9d713a10fae40d54d --- /dev/null +++ b/knot2/crates/knot-sim/src/workload.rs @@ -0,0 +1,1021 @@ +use axum::body::Bytes; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use futures::future::join_all; +use futures::stream::StreamExt; +use http::Method; +use http::header::AUTHORIZATION; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet}; +use std::net::SocketAddr; +use std::sync::Arc; +use tower::ServiceExt; + +use knot_runtime::{Entropy, K256Signer, SeededEntropy, Signer}; +use knot_types::{ + AccountDid, Datetime, HttpStatus, KnotHostname, Nsid, RecordCollection, RepoDid, UnixSeconds, +}; + +use crate::harness::{Harness, subject_dids}; +use crate::trace::{OperationIndex, Outcome, Projection, RoundNumber, Step, Trace, fnv1a}; + +const SKEW_BACKDATE_SECS: i64 = 600; +const SKEW_LIFETIME_SECS: i64 = 60; +const CREATE_RECORD_NSID: &str = "com.atproto.repo.createRecord"; +const DELETE_RECORD_NSID: &str = "com.atproto.repo.deleteRecord"; +pub(crate) fn member_collection() -> RecordCollection { + RecordCollection::new("org.tangled.knot.member").expect("literal NSID parses") +} + +pub(crate) fn block_collection() -> RecordCollection { + RecordCollection::new("org.tangled.knot.block").expect("literal NSID parses") +} + +pub(crate) fn collaborator_collection() -> RecordCollection { + RecordCollection::new("org.tangled.graph.collaborator").expect("literal NSID parses") +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct RepoIndex(usize); + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct SubjectIndex(usize); + +#[derive(Clone, Copy)] +struct StrangerIndex(usize); + +#[derive(Clone, Copy)] +enum ReadOp { + Version, + DescribeKnot, + ListMembers, + DidJson, + InfoRefs(RepoIndex), + Branches(RepoIndex), + Log(RepoIndex), + DescribeRepo(RepoIndex), + Tree(RepoIndex), + Blob(RepoIndex), + Languages(RepoIndex), + ListCollaborators(RepoIndex), +} + +impl ReadOp { + fn name(self) -> &'static str { + match self { + ReadOp::Version => "version", + ReadOp::DescribeKnot => "describeKnot", + ReadOp::ListMembers => "listMembers", + ReadOp::DidJson => "didJson", + ReadOp::InfoRefs(_) => "infoRefs", + ReadOp::Branches(_) => "branches", + ReadOp::Log(_) => "log", + ReadOp::DescribeRepo(_) => "describeRepo", + ReadOp::Tree(_) => "tree", + ReadOp::Blob(_) => "blob", + ReadOp::Languages(_) => "languages", + ReadOp::ListCollaborators(_) => "listCollaborators", + } + } +} + +#[derive(Clone, Copy)] +enum AdminOp { + AddMember(SubjectIndex), + RemoveMember(SubjectIndex), + Ban(SubjectIndex), + Unban(SubjectIndex), + CreateRepo(u32), + AddCollaborator(RepoIndex, SubjectIndex), +} + +impl AdminOp { + fn name(self) -> &'static str { + match self { + AdminOp::AddMember(_) => "addMember", + AdminOp::RemoveMember(_) => "removeMember", + AdminOp::Ban(_) => "ban", + AdminOp::Unban(_) => "unban", + AdminOp::CreateRepo(_) => "createRepo", + AdminOp::AddCollaborator(_, _) => "addCollaborator", + } + } +} + +#[derive(Clone, Copy)] +enum Mutation { + Admin { op: AdminOp, skew: bool }, + Probe { stranger: StrangerIndex, drop: bool }, + Maintain { repo: RepoIndex }, +} + +#[derive(Clone, Copy)] +struct Observation { + op: ReadOp, + killed: bool, +} + +#[derive(Clone, Copy)] +enum Planned { + Mutate(Mutation), + Observe(Observation), +} + +impl Planned { + fn creates_a_repo(self) -> bool { + matches!( + self, + Planned::Mutate(Mutation::Admin { + op: AdminOp::CreateRepo(_), + skew: false, + }) + ) + } +} + +enum Phase { + Mutate(Vec), + Observe(Vec), +} + +impl Phase { + fn ops(&self) -> Vec { + match self { + Phase::Mutate(ops) => ops.iter().copied().map(Planned::Mutate).collect(), + Phase::Observe(ops) => ops.iter().copied().map(Planned::Observe).collect(), + } + } +} + +pub(crate) struct Round { + phase: Phase, + advance: std::time::Duration, +} + +pub(crate) struct Rng(SeededEntropy); + +impl Rng { + pub(crate) fn new(seed: u64) -> Self { + Self(SeededEntropy::new(seed ^ 0x57ee_d000)) + } + + pub(crate) fn below(&self, n: u64) -> u64 { + self.0.next_u64() % n.max(1) + } + + pub(crate) fn chance(&self, num: u64, den: u64) -> bool { + assert!(num <= den, "chance numerator exceeds denominator"); + self.below(den) < num + } +} + +pub(crate) fn plan(seed: u64, rounds: u32, subjects: usize) -> Vec { + let rng = Rng::new(seed); + let mut available: u32 = 1; + let mut rkey: u32 = 0; + let mut stranger: usize = 0; + (0..rounds) + .map(|round| { + let (phase, fresh_repos) = match round % 2 { + 0 => { + let (ops, fresh_repos) = + mutate_round(&rng, subjects, available, &mut rkey, &mut stranger); + (Phase::Mutate(ops), fresh_repos) + } + _ => (Phase::Observe(observe_round(&rng, available)), 0), + }; + let advance = std::time::Duration::from_micros(rng.below(3_000_000)); + available += fresh_repos; + Round { phase, advance } + }) + .collect() +} + +#[derive(Default)] +struct Model { + members: BTreeSet, + blocked: BTreeSet, + collaborators: BTreeMap>, +} + +pub(crate) fn predict(seed: u64, rounds: u32) -> Projection { + let subjects = subject_dids(); + let subject_did = |subject: SubjectIndex| subjects[subject.0].clone(); + let plan = plan(seed, rounds, subjects.len()); + let ops = || plan.iter().flat_map(|round| round.phase.ops()); + let repos = 1 + ops().filter(|planned| planned.creates_a_repo()).count(); + let model = ops().fold(Model::default(), |mut model, planned| { + match planned { + Planned::Mutate(Mutation::Admin { op, skew: false }) => match op { + AdminOp::AddMember(subject) => { + model.members.insert(subject_did(subject)); + } + AdminOp::RemoveMember(subject) => { + model.members.remove(&subject_did(subject)); + } + AdminOp::Ban(subject) => { + model.blocked.insert(subject_did(subject)); + } + AdminOp::Unban(subject) => { + model.blocked.remove(&subject_did(subject)); + } + AdminOp::AddCollaborator(repo, subject) => { + model + .collaborators + .entry(repo.0) + .or_default() + .insert(subject_did(subject)); + } + AdminOp::CreateRepo(_) => {} + }, + Planned::Mutate(Mutation::Admin { skew: true, .. }) + | Planned::Mutate(Mutation::Probe { .. }) + | Planned::Mutate(Mutation::Maintain { .. }) + | Planned::Observe(_) => {} + } + model + }); + let mut rosters = model.collaborators; + Projection { + members: model.members.into_iter().collect(), + blocked: model.blocked.into_iter().collect(), + collaborators: (0..repos) + .map(|repo| { + rosters + .remove(&repo) + .unwrap_or_default() + .into_iter() + .collect() + }) + .collect(), + } +} + +fn mutate_round( + rng: &Rng, + subjects: usize, + available: u32, + rkey: &mut u32, + stranger: &mut usize, +) -> (Vec, u32) { + let mut ops: Vec = (0..subjects) + .filter(|_| rng.chance(2, 3)) + .map(|subject| { + let subject = SubjectIndex(subject); + let op = match rng.below(4) { + 0 => AdminOp::AddMember(subject), + 1 => AdminOp::RemoveMember(subject), + 2 => AdminOp::Ban(subject), + _ => AdminOp::Unban(subject), + }; + Mutation::Admin { + op, + skew: rng.chance(1, 5), + } + }) + .collect(); + + (0..available) + .filter(|_| rng.chance(1, 2)) + .for_each(|repo| { + let repo = RepoIndex(repo as usize); + let planned = if rng.chance(1, 2) { + let subject = SubjectIndex(rng.below(subjects as u64) as usize); + Mutation::Admin { + op: AdminOp::AddCollaborator(repo, subject), + skew: rng.chance(1, 6), + } + } else { + Mutation::Maintain { repo } + }; + ops.push(planned); + }); + + let mut fresh_repos = 0; + (0..rng.below(3)).for_each(|_| { + let key = *rkey; + *rkey += 1; + let skew = rng.chance(1, 8); + if !skew { + fresh_repos += 1; + } + ops.push(Mutation::Admin { + op: AdminOp::CreateRepo(key), + skew, + }); + }); + + (0..rng.below(3)).for_each(|_| { + let stranger_index = StrangerIndex(*stranger); + *stranger += 1; + ops.push(Mutation::Probe { + stranger: stranger_index, + drop: rng.chance(1, 2), + }); + }); + + (ops, fresh_repos) +} + +fn observe_round(rng: &Rng, available: u32) -> Vec { + let mut ops: Vec = [ + ReadOp::Version, + ReadOp::DescribeKnot, + ReadOp::ListMembers, + ReadOp::DidJson, + ] + .into_iter() + .map(|op| Observation { + op, + killed: rng.chance(1, 5), + }) + .collect(); + (0..available) + .filter(|_| rng.chance(2, 3)) + .for_each(|repo| { + let repo = RepoIndex(repo as usize); + let op = match rng.below(8) { + 0 => ReadOp::Branches(repo), + 1 => ReadOp::Log(repo), + 2 => ReadOp::DescribeRepo(repo), + 3 => ReadOp::InfoRefs(repo), + 4 => ReadOp::Tree(repo), + 5 => ReadOp::Blob(repo), + 6 => ReadOp::ListCollaborators(repo), + _ => ReadOp::Languages(repo), + }; + ops.push(Observation { + op, + killed: rng.chance(1, 5), + }); + }); + ops +} + +struct OpResult { + step: Step, + created: Option, +} + +pub(crate) async fn execute(harness: Arc, seed: u64, plan: Vec) -> Trace { + let initial = ( + vec![harness.seed_repo.clone()], + Vec::::new(), + Vec::new(), + ); + let harness = &harness; + let (repos, steps, snapshots) = futures::stream::iter(plan.into_iter().enumerate()) + .fold( + initial, + |(repos, mut steps, mut snapshots), (round_index, round)| { + let harness = Arc::clone(harness); + async move { + let round_no = RoundNumber::new(round_index as u32); + let ops = round.phase.ops(); + let drops = arm_drops(&harness, &ops); + let repos = Arc::new(repos); + let tasks = ops + .iter() + .enumerate() + .map(|(index, planned)| { + let harness = Arc::clone(&harness); + let repos = Arc::clone(&repos); + let planned = *planned; + tokio::spawn(async move { + run_op( + &harness, + &repos, + round_no, + OperationIndex::new(index as u32), + planned, + ) + .await + }) + }) + .collect::>(); + let results: Vec = join_all(tasks) + .await + .into_iter() + .map(|joined| joined.expect("sim op task mustn't panic")) + .collect(); + drops + .iter() + .for_each(|host| harness.faults.clear_host(host)); + + let created: Vec = results + .iter() + .filter_map(|result| result.created.clone()) + .collect(); + let planned_creates = ops + .iter() + .filter(|planned| planned.creates_a_repo()) + .count(); + assert_eq!( + planned_creates, + created.len(), + "round {}: {planned_creates} non-skew creates planned but \ + {} materialized, so plan/execute repo indices have drifted apart", + round_no.get(), + created.len() + ); + created.iter().for_each(|did| harness.populate(did)); + steps.extend(results.into_iter().map(|result| result.step)); + let mut repos = Arc::into_inner(repos) + .expect("all op tasks released the round repo snapshot"); + repos.extend(created); + + harness.advance(round.advance); + snapshots.push(harness.snapshot(round_no, &repos)); + (repos, steps, snapshots) + } + }, + ) + .await; + let no_fault_creates = steps + .iter() + .filter(|step| step.op == "createRepo" && step.fault == "none") + .count(); + let materialized = repos.len() - 1; + assert_eq!( + no_fault_creates, materialized, + "no-fault createRepo count {no_fault_creates} doesn't match the {materialized} repos \ + materialized: a planned create silently failed and repo_at would have masked the drift" + ); + Trace { + seed, + steps, + snapshots, + } +} + +fn arm_drops(harness: &Harness, ops: &[Planned]) -> Vec { + let hosts: Vec = ops + .iter() + .filter_map(|planned| match planned { + Planned::Mutate(Mutation::Probe { + stranger, + drop: true, + }) => Some(harness.strangers[stranger.0].host.clone()), + _ => None, + }) + .collect(); + hosts.iter().for_each(|host| harness.faults.drop_host(host)); + hosts +} + +async fn run_op( + harness: &Harness, + repos: &[RepoDid], + round: RoundNumber, + index: OperationIndex, + planned: Planned, +) -> OpResult { + let make = |op: &'static str, actor: String, fault: &'static str, outcome: Outcome| Step { + round, + index, + op, + actor, + fault, + outcome, + }; + + match planned { + Planned::Mutate(Mutation::Maintain { repo }) => { + let repo = repo_at(repos, repo); + OpResult { + step: make( + "maintain", + "knot".to_string(), + "none", + maintain_outcome(harness, repo), + ), + created: None, + } + } + Planned::Observe(Observation { op, killed }) => { + let request = read_request(harness, repos, op); + if killed { + drive_kill(harness.router(), request.method, &request.uri, request.body).await; + return OpResult { + step: make(op.name(), request.actor, "killed", Outcome::Killed), + created: None, + }; + } + let (status, body) = http_call( + harness.router(), + request.method, + &request.uri, + None, + request.body, + ) + .await; + OpResult { + step: make( + op.name(), + request.actor, + "none", + Outcome::Served { + status, + body: body_digest(&body), + }, + ), + created: None, + } + } + Planned::Mutate(Mutation::Admin { op, skew }) => { + let request = admin_request(harness, repos, op, skew, round, index); + let (status, body) = http_call( + harness.router(), + request.method, + &request.uri, + request.token.as_deref(), + request.body, + ) + .await; + let created = match op { + AdminOp::CreateRepo(_) if status == HttpStatus::new(200) => repo_did_of(&body), + _ => None, + }; + OpResult { + step: make( + op.name(), + request.actor, + if skew { "clock_skew" } else { "none" }, + Outcome::Served { + status, + body: body_digest(&body), + }, + ), + created, + } + } + Planned::Mutate(Mutation::Probe { stranger, drop }) => { + let request = probe_request(harness, stranger, round, index); + let (status, body) = http_call( + harness.router(), + request.method, + &request.uri, + request.token.as_deref(), + request.body, + ) + .await; + OpResult { + step: make( + "probe", + request.actor, + if drop { "drop_identity" } else { "none" }, + Outcome::Served { + status, + body: body_digest(&body), + }, + ), + created: None, + } + } + } +} + +pub(crate) struct Request { + pub(crate) method: Method, + pub(crate) uri: String, + pub(crate) token: Option, + pub(crate) body: Bytes, + pub(crate) actor: String, +} + +impl Request { + pub(crate) fn get(uri: impl Into) -> Self { + Self { + method: Method::GET, + uri: uri.into(), + token: None, + body: Bytes::new(), + actor: "anon".to_string(), + } + } +} + +fn read_request(harness: &Harness, repos: &[RepoDid], op: ReadOp) -> Request { + match op { + ReadOp::Version => get("/xrpc/org.tangled.knot.describeKnot"), + ReadOp::DescribeKnot => get("/xrpc/org.tangled.knot.describeKnot"), + ReadOp::ListMembers => get(format!( + "/xrpc/com.atproto.repo.listRecords?repo={}&collection={}", + enc(&harness.knot_aud), + member_collection() + ) + .as_str()), + ReadOp::DidJson => get("/.well-known/did.json"), + ReadOp::InfoRefs(repo) => Request { + method: Method::GET, + uri: format!( + "/{}/info/refs?service=git-upload-pack", + repo_at(repos, repo) + ), + token: None, + body: Bytes::new(), + actor: "anon".to_string(), + }, + ReadOp::Branches(repo) => repo_get("org.tangled.git.listBranches", repo_at(repos, repo)), + ReadOp::Log(repo) => get(format!( + "/xrpc/org.tangled.git.listCommits?repo={}&ref=HEAD", + enc(repo_at(repos, repo)) + ) + .as_str()), + ReadOp::DescribeRepo(repo) => { + repo_get("com.atproto.repo.describeRepo", repo_at(repos, repo)) + } + ReadOp::Tree(repo) => repo_get("org.tangled.git.getTree", repo_at(repos, repo)), + ReadOp::Languages(repo) => repo_get("org.tangled.git.listLanguages", repo_at(repos, repo)), + ReadOp::ListCollaborators(repo) => get(format!( + "/xrpc/com.atproto.repo.listRecords?repo={}&collection={}", + enc(repo_at(repos, repo)), + collaborator_collection() + ) + .as_str()), + ReadOp::Blob(repo) => Request { + method: Method::GET, + uri: format!( + "/xrpc/org.tangled.git.getBlob?repo={}&path=README.md&ref=HEAD", + enc(repo_at(repos, repo)) + ), + token: None, + body: Bytes::new(), + actor: "anon".to_string(), + }, + } +} + +fn admin_request( + harness: &Harness, + repos: &[RepoDid], + op: AdminOp, + skew: bool, + round: RoundNumber, + index: OperationIndex, +) -> Request { + let subjects = &harness.subjects; + let now = iso_utc(harness.now_seconds()); + match op { + AdminOp::AddMember(subject) => admin_post( + harness, + "addMember", + &nsid(CREATE_RECORD_NSID), + &harness.knot_aud, + grant_body( + harness.knot_repo(), + &member_collection(), + &subjects[subject.0], + Some(&now), + ), + OpSlot { skew, round, index }, + ), + AdminOp::RemoveMember(subject) => admin_post( + harness, + "removeMember", + &nsid(DELETE_RECORD_NSID), + &harness.knot_aud, + grant_body( + harness.knot_repo(), + &member_collection(), + &subjects[subject.0], + None, + ), + OpSlot { skew, round, index }, + ), + AdminOp::Ban(subject) => admin_post( + harness, + "ban", + &nsid(CREATE_RECORD_NSID), + &harness.knot_aud, + grant_body( + harness.knot_repo(), + &block_collection(), + &subjects[subject.0], + Some(&now), + ), + OpSlot { skew, round, index }, + ), + AdminOp::Unban(subject) => admin_post( + harness, + "unban", + &nsid(DELETE_RECORD_NSID), + &harness.knot_aud, + grant_body( + harness.knot_repo(), + &block_collection(), + &subjects[subject.0], + None, + ), + OpSlot { skew, round, index }, + ), + AdminOp::CreateRepo(key) => { + let name = format!("repo{key}"); + admin_post( + harness, + "create", + &nsid("org.tangled.repo.createRepo"), + &harness.knot_aud, + json!({ "name": name }), + OpSlot { skew, round, index }, + ) + } + AdminOp::AddCollaborator(repo, subject) => admin_post( + harness, + "addCollaborator", + &nsid(CREATE_RECORD_NSID), + &repo_at(repos, repo), + grant_body( + repo_at(repos, repo), + &collaborator_collection(), + &subjects[subject.0], + Some(&now), + ), + OpSlot { skew, round, index }, + ), + } +} + +fn probe_request( + harness: &Harness, + stranger: StrangerIndex, + round: RoundNumber, + index: OperationIndex, +) -> Request { + let actor = &harness.strangers[stranger.0]; + let mut request = signed_post( + harness, + &actor.signer, + &actor.did, + &nsid(CREATE_RECORD_NSID), + &harness.knot_aud, + grant_body( + harness.knot_repo(), + &member_collection(), + &harness.subjects[0], + Some(&iso_utc(harness.now_seconds())), + ), + OpSlot { + skew: false, + round, + index, + }, + ); + request.actor = actor.host.to_string(); + request +} + +fn get(path: &str) -> Request { + Request::get(path) +} + +fn repo_get(method: &str, repo: &RepoDid) -> Request { + Request::get(format!("/xrpc/{method}?repo={}", enc(repo))) +} + +pub(crate) fn nsid(name: &str) -> Nsid { + Nsid::new_owned(name).expect("the xrpc method is an NSID") +} + +pub(crate) fn signed_post( + harness: &Harness, + signer: &K256Signer, + did: &AccountDid, + method: &Nsid, + aud: &impl AsRef, + body: Value, + OpSlot { skew, round, index }: OpSlot, +) -> Request { + let token = mint( + signer, + did, + aud, + method, + jwt_window(harness, skew), + round, + index, + ); + Request { + method: Method::POST, + uri: format!("/xrpc/{method}"), + token: Some(token), + body: encode_body(body), + actor: String::new(), + } +} + +pub(crate) fn maintain_outcome(harness: &Harness, repo: &RepoDid) -> Outcome { + match harness.maintain(repo) { + Ok(()) => Outcome::Served { + status: HttpStatus::new(200), + body: 0, + }, + Err(error) => Outcome::Served { + status: HttpStatus::new(500), + body: fnv1a(error.to_string().as_bytes()), + }, + } +} + +pub(crate) struct OpSlot { + pub(crate) skew: bool, + pub(crate) round: RoundNumber, + pub(crate) index: OperationIndex, +} + +fn admin_post( + harness: &Harness, + method_short: &str, + method: &Nsid, + aud: &impl AsRef, + body: Value, + OpSlot { skew, round, index }: OpSlot, +) -> Request { + let mut request = signed_post( + harness, + &harness.admin.signer, + &harness.admin.did, + method, + aud, + body, + OpSlot { skew, round, index }, + ); + request.actor = format!("admin:{method_short}"); + request +} + +pub(crate) fn jwt_window(harness: &Harness, skew: bool) -> (UnixSeconds, UnixSeconds) { + let now = harness.now_seconds(); + if skew { + ( + now.saturating_sub_secs(SKEW_BACKDATE_SECS + SKEW_LIFETIME_SECS), + now.saturating_sub_secs(SKEW_BACKDATE_SECS), + ) + } else { + (now, now.saturating_add_secs(60)) + } +} + +pub(crate) fn mint( + signer: &K256Signer, + issuer: &AccountDid, + aud: &impl AsRef, + nsid: &Nsid, + window: (UnixSeconds, UnixSeconds), + round: RoundNumber, + index: OperationIndex, +) -> String { + let header = URL_SAFE_NO_PAD.encode(br#"{"alg":"ES256K","typ":"JWT"}"#); + let payload = URL_SAFE_NO_PAD.encode( + serde_json::to_vec(&json!({ + "iss": issuer, + "aud": aud.as_ref(), + "iat": window.0.get(), + "exp": window.1.get(), + "jti": format!("sim-{}-{}", round.get(), index.get()), + "lxm": nsid, + })) + .expect("claims serialize"), + ); + let signing_input = format!("{header}.{payload}"); + let signature = signer.sign(signing_input.as_bytes()); + format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.as_bytes()) + ) +} + +pub(crate) async fn http_call( + router: axum::Router, + method: Method, + uri: &str, + token: Option<&str>, + body: Bytes, +) -> (HttpStatus, Bytes) { + let mut request = http::Request::builder() + .method(method) + .uri(uri) + .body(axum::body::Body::from(body)) + .expect("request builds"); + if let Some(token) = token { + request.headers_mut().insert( + AUTHORIZATION, + http::HeaderValue::from_str(&format!("Bearer {token}")).expect("bearer header"), + ); + } + request + .extensions_mut() + .insert(axum::extract::ConnectInfo(SocketAddr::from(( + [127, 0, 0, 1], + 4242, + )))); + let response = router + .oneshot(request) + .await + .expect("router must return a response"); + let status = HttpStatus::from(response.status()); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("response body"); + (status, bytes) +} + +pub(crate) async fn drive_kill(router: axum::Router, method: Method, uri: &str, body: Bytes) { + let call = http_call(router, method, uri, None, body); + futures::pin_mut!(call); + tokio::select! { + biased; + _ = &mut call => {} + _ = tokio::task::yield_now() => {} + } +} + +pub(crate) fn repo_did_of(body: &Bytes) -> Option { + serde_json::from_slice::(body) + .ok() + .and_then(|value| value.get("did").and_then(Value::as_str).map(str::to_string)) + .and_then(|did| RepoDid::new(did).ok()) +} + +pub(crate) fn body_digest(body: &Bytes) -> u64 { + match serde_json::from_slice::(body) { + Ok(mut value) => { + canonicalize(&mut value); + fnv1a(&serde_json::to_vec(&value).expect("canonical body serializes")) + } + Err(_) => fnv1a(body), + } +} + +fn canonicalize(value: &mut Value) { + match value { + Value::Array(items) => { + items.iter_mut().for_each(canonicalize); + items.sort_by_cached_key(|item| serde_json::to_string(item).expect("array item")); + } + Value::Object(map) => map.values_mut().for_each(canonicalize), + _ => {} + } +} + +pub(crate) fn encode_body(value: Value) -> Bytes { + Bytes::from(serde_json::to_vec(&value).expect("request body serializes")) +} + +pub(crate) fn enc(value: impl std::fmt::Display) -> String { + value.to_string().replace(':', "%3A") +} + +pub(crate) fn grant_body( + repo: &RepoDid, + collection: &RecordCollection, + rkey: &AccountDid, + created_at: Option<&Datetime>, +) -> Value { + match created_at { + Some(now) => json!({ + "repo": repo, + "collection": collection, + "rkey": rkey, + "record": { "createdAt": now } + }), + None => json!({ + "repo": repo, + "collection": collection, + "rkey": rkey, + }), + } +} + +pub(crate) fn iso_utc(seconds: UnixSeconds) -> Datetime { + let seconds = seconds.get(); + let (days, secs_of_day) = (seconds.div_euclid(86_400), seconds.rem_euclid(86_400)); + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let (month, year) = if mp < 10 { + (mp + 3, yoe + era * 400) + } else { + (mp - 9, yoe + era * 400 + 1) + }; + Datetime::raw_str(format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z", + secs_of_day / 3600, + secs_of_day % 3600 / 60, + secs_of_day % 60 + )) +} + +fn repo_at(repos: &[RepoDid], index: RepoIndex) -> &RepoDid { + repos.get(index.0).unwrap_or_else(|| { + panic!( + "plan/execute repo drift: index {} exceeds {} repos created so far", + index.0, + repos.len() + ) + }) +} diff --git a/knot2/crates/knot-sim/tests/common/mod.rs b/knot2/crates/knot-sim/tests/common/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..b91efbd58d43d581429f03a437b1cae9110416ae --- /dev/null +++ b/knot2/crates/knot-sim/tests/common/mod.rs @@ -0,0 +1,324 @@ +#![allow(dead_code)] + +use std::net::SocketAddr; +use std::num::{NonZeroU32, NonZeroU64}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use bytes::{Buf, Bytes}; +use http::{Method, StatusCode}; +use knot_edge::{ + BodyInactivityTimeout, BurstSize, CertSource, EdgeConfig, EdgeGuards, HeaderTimeout, + IdleTimeout, ListenLimits, MaxInflightRequests, RequestTimeout, RequestsPerSecond, + RequiresFullHandshake, StaticCertPaths, TlsSetup, WriteRequestTimeout, ZeroRttRoutes, +}; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::crypto::aws_lc_rs; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{DigitallySignedStruct, SignatureScheme}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +pub fn nz32(value: u32) -> NonZeroU32 { + NonZeroU32::new(value).unwrap() +} + +pub fn nz64(value: u64) -> NonZeroU64 { + NonZeroU64::new(value).unwrap() +} + +pub fn pkt(payload: &[u8]) -> Vec { + assert!( + payload.len() + 4 <= 0xFFF0, + "pkt-line payload exceeds the 65516-byte maximum" + ); + let mut out = format!("{:04x}", payload.len() + 4).into_bytes(); + out.extend_from_slice(payload); + out +} + +pub fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +pub fn write_self_signed(dir: &Path) -> (PathBuf, PathBuf, Vec) { + let generated = + rcgen::generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()]) + .unwrap(); + let cert_path = dir.join("cert.pem"); + let key_path = dir.join("key.pem"); + std::fs::write(&cert_path, generated.cert.pem()).unwrap(); + std::fs::write(&key_path, generated.signing_key.serialize_pem()).unwrap(); + let der = generated.cert.der().as_ref().to_vec(); + (cert_path, key_path, der) +} + +pub fn edge_config(addr: SocketAddr, cert: PathBuf, key: PathBuf) -> EdgeConfig { + EdgeConfig { + http_addr: knot_edge::PublicBind::new(addr), + limits: ListenLimits::new( + HeaderTimeout::from_millis(nz64(30_000)), + IdleTimeout::from_millis(nz64(120_000)), + nz32(1024), + ), + guards: EdgeGuards::new( + RequestsPerSecond::new(nz32(1_000_000)), + BurstSize::new(nz32(1_000_000)), + MaxInflightRequests::new(nz32(10_000)), + RequestTimeout::from_millis(nz64(120_000)), + BodyInactivityTimeout::from_millis(nz64(120_000)), + WriteRequestTimeout::from_millis(nz64(1_800_000)), + knot_types::ProxyTrust::default(), + ), + tls: Some(TlsSetup { + source: CertSource::Static(StaticCertPaths { + cert_path: knot_edge::CertChainPath::new(cert), + key_path: knot_edge::PrivateKeyPath::new(key), + }), + http3: true, + internal: None, + }), + } +} + +pub struct Edge { + pub addr: SocketAddr, + pub shutdown: CancellationToken, + pub task: JoinHandle>, + pub client: quinn::Endpoint, +} + +async fn probe_identity( + endpoint: &quinn::Endpoint, + addr: SocketAddr, + expected_cert: &[u8], +) -> Option { + let connecting = endpoint.connect(addr, "localhost").ok()?; + let connection = tokio::time::timeout(Duration::from_millis(250), connecting) + .await + .ok()? + .ok()?; + let ours = connection + .peer_identity() + .and_then(|identity| identity.downcast::>>().ok()) + .map(|certs| { + certs + .first() + .is_some_and(|cert| cert.as_ref() == expected_cert) + }) + .unwrap_or(false); + connection.close(0u32.into(), b"probe done"); + Some(ours) +} + +async fn await_ready( + endpoint: &quinn::Endpoint, + addr: SocketAddr, + task: &mut JoinHandle>, + expected_cert: &[u8], +) -> bool { + for _ in 0..200 { + if task.is_finished() { + return false; + } + if let Some(ours) = probe_identity(endpoint, addr, expected_cert).await { + return ours; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + false +} + +pub async fn serve_edge( + certdir: &Path, + build: impl Fn() -> (RequiresFullHandshake, ZeroRttRoutes), +) -> Edge { + for _ in 0..8 { + let addr: SocketAddr = format!("127.0.0.1:{}", free_port()).parse().unwrap(); + let (cert, key, cert_der) = write_self_signed(certdir); + let (app, advertisement) = build(); + let shutdown = CancellationToken::new(); + let mut task = tokio::spawn(knot_edge::serve( + edge_config(addr, cert, key), + app, + advertisement, + shutdown.clone(), + )); + let client = h3_client(); + if await_ready(&client, addr, &mut task, &cert_der).await { + return Edge { + addr, + shutdown, + task, + client, + }; + } + client.close(0u32.into(), b"stand up retry"); + shutdown.cancel(); + let _ = task.await; + } + panic!("couldn't bind a free TCP+UDP port for the edge after several attempts"); +} + +#[derive(Debug)] +struct AcceptAnyServerCert; + +impl ServerCertVerifier for AcceptAnyServerCert { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + aws_lc_rs::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} + +pub fn h3_client() -> quinn::Endpoint { + let mut crypto = + rustls::ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCert)) + .with_no_client_auth(); + crypto.alpn_protocols = vec![b"h3".to_vec()]; + crypto.resumption = rustls::client::Resumption::disabled(); + let quic = quinn::crypto::rustls::QuicClientConfig::try_from(crypto).unwrap(); + let mut endpoint = quinn::Endpoint::client("127.0.0.1:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(quic))); + endpoint +} + +pub async fn drain( + stream: &mut h3::client::RequestStream, Bytes>, +) -> Vec { + let mut out = Vec::new(); + while let Some(mut chunk) = stream.recv_data().await.unwrap() { + out.extend_from_slice(&chunk.copy_to_bytes(chunk.remaining())); + } + out +} + +pub async fn finish_request( + stream: &mut h3::client::RequestStream, Bytes>, +) { + match stream.finish().await { + Ok(()) => (), + Err(h3::error::StreamError::RemoteTerminate { code, .. }) + if code == h3::error::Code::H3_NO_ERROR => {} + Err(error) => panic!("finishing the request stream failed: {error}"), + } +} + +pub async fn h3_request( + edge: &Edge, + method: Method, + uri: String, + headers: &[(&str, &str)], + body: Option, + warmup: Option<&str>, +) -> (StatusCode, Vec) { + let connection = edge + .client + .connect(edge.addr, "localhost") + .unwrap() + .await + .unwrap(); + let quic = connection.clone(); + let (mut driver, mut sender) = h3::client::new(h3_quinn::Connection::new(connection)) + .await + .unwrap(); + let drive = tokio::spawn(async move { + let _ = std::future::poll_fn(|cx| driver.poll_close(cx)).await; + }); + + if let Some(warmup) = warmup { + let request = http::Request::get(warmup).body(()).unwrap(); + let mut stream = sender.send_request(request).await.unwrap(); + finish_request(&mut stream).await; + let _ = stream.recv_response().await.unwrap(); + drain(&mut stream).await; + } + + let request = headers + .iter() + .fold( + http::Request::builder().method(method).uri(uri), + |builder, (name, value)| builder.header(*name, *value), + ) + .body(()) + .unwrap(); + let mut stream = sender.send_request(request).await.unwrap(); + if let Some(body) = body { + stream.send_data(body).await.unwrap(); + } + finish_request(&mut stream).await; + let status = stream.recv_response().await.unwrap().status(); + let out = drain(&mut stream).await; + quic.close(0u32.into(), b"done"); + drive.abort(); + (status, out) +} + +pub fn kill_delays(full: Duration) -> Vec { + let fractions = [0.20, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95]; + std::iter::once(Duration::from_millis(1)) + .chain(std::iter::once(Duration::from_millis(3))) + .chain(fractions.iter().map(|fraction| full.mul_f64(*fraction))) + .chain(std::iter::once(full.mul_f64(2.0))) + .collect() +} + +pub fn spawn_worker(test: &str, envs: &[(&str, &str)]) -> std::process::Child { + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", test, "--nocapture"]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + for (key, value) in envs { + command.env(key, value); + } + command.spawn().expect("spawn chaos worker") +} diff --git a/knot2/crates/knot-sim/tests/h3.rs b/knot2/crates/knot-sim/tests/h3.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb11b099ed9b3f6c004913bc1c3a42d8699454ed --- /dev/null +++ b/knot2/crates/knot-sim/tests/h3.rs @@ -0,0 +1,249 @@ +mod common; + +use std::collections::BTreeSet; +use std::io::Write; +use std::path::Path; +use std::process::Stdio; +use std::sync::Arc; + +use bytes::Bytes; +use common::Edge; +use http::Method; +use knot_edge::RequiresFullHandshake; +use knot_git::Layout; +use knot_pack::{RepoLookup, RepoResolver, RepoTarget}; +use knot_types::{ObjectFormat, RepoDid}; + +const PINNED_DATE: &str = "2026-06-20T12:00:00+00:00"; + +fn git(cwd: &Path, args: &[&str]) -> String { + let out = knot_fixtures::command(cwd) + .args(args) + .env("GIT_AUTHOR_DATE", PINNED_DATE) + .env("GIT_COMMITTER_DATE", PINNED_DATE) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout) + .expect("git stdout is utf-8") + .trim() + .to_string() +} + +fn object_set(dir: &Path) -> BTreeSet { + git( + dir, + &[ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname)", + ], + ) + .lines() + .map(|line| line.trim().to_string()) + .filter(|line| !line.is_empty()) + .collect() +} + +fn seed_bare(work: &Path, bare: &Path, format: ObjectFormat) -> String { + let fmt = format!("--object-format={}", format.capability()); + std::fs::create_dir_all(work).unwrap(); + git(work, &["init", &fmt, "-q", "-b", "main"]); + std::fs::write(work.join("README.md"), "h3 over the simulated quic edge\n").unwrap(); + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", "c1"]); + std::fs::write(work.join("extra.txt"), "a second object\n").unwrap(); + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", "c2"]); + git(work, &["push", "-q", bare.to_str().unwrap(), "main"]); + git(bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); + git(work, &["rev-parse", "HEAD"]) +} + +fn init(dir: &Path, format: ObjectFormat) { + std::fs::create_dir_all(dir).unwrap(); + let fmt = format!("--object-format={}", format.capability()); + git(dir, &["init", &fmt, "-q", dir.to_str().unwrap()]); +} + +fn index_pack(repo: &Path, pack: &[u8]) { + let mut child = knot_fixtures::command(repo) + .args(["index-pack", "--stdin", "--fix-thin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn index-pack"); + child.stdin.take().unwrap().write_all(pack).unwrap(); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "index-pack failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +fn fetch_body(tip: &str) -> Bytes { + let mut body = Vec::new(); + body.extend_from_slice(&common::pkt(b"command=fetch\n")); + body.extend_from_slice(&common::pkt(b"agent=knot/0\n")); + body.extend_from_slice(b"0001"); + body.extend_from_slice(&common::pkt(b"no-progress\n")); + body.extend_from_slice(&common::pkt(b"ofs-delta\n")); + body.extend_from_slice(&common::pkt(format!("want {tip}\n").as_bytes())); + body.extend_from_slice(&common::pkt(b"done\n")); + body.extend_from_slice(b"0000"); + Bytes::from(body) +} + +fn extract_pack(response: &[u8]) -> Vec { + let mut channel = Vec::new(); + let mut pos = 0usize; + while pos + 4 <= response.len() { + let len = std::str::from_utf8(&response[pos..pos + 4]) + .ok() + .and_then(|hex| usize::from_str_radix(hex, 16).ok()) + .unwrap_or(0); + pos += 4; + if len < 4 { + continue; + } + let end = (pos + len - 4).min(response.len()); + let payload = &response[pos..end]; + pos = end; + if payload.first() == Some(&1) { + channel.extend_from_slice(&payload[1..]); + } + } + match channel.windows(4).position(|window| window == b"PACK") { + Some(start) => channel.split_off(start), + None => channel, + } +} + +fn serve_dids() -> Arc { + Arc::new(|target: &RepoTarget| match target { + RepoTarget::Did(did) => RepoLookup::Hosted(did.clone()), + RepoTarget::OwnerPath(_, _) => RepoLookup::Unhosted, + }) +} + +async fn clone_over_h3(edge: &Edge, did: &RepoDid, tip: &str) -> Vec { + let connection = edge + .client + .connect(edge.addr, "localhost") + .unwrap() + .await + .unwrap(); + let quic = connection.clone(); + let (mut driver, mut sender) = h3::client::new(h3_quinn::Connection::new(connection)) + .await + .unwrap(); + let drive = tokio::spawn(async move { + let _ = std::future::poll_fn(|cx| driver.poll_close(cx)).await; + }); + + let warmup = http::Request::get(format!( + "https://localhost/{did}/info/refs?service=git-upload-pack" + )) + .header("git-protocol", "version=2") + .body(()) + .unwrap(); + let mut warm = sender.send_request(warmup).await.unwrap(); + common::finish_request(&mut warm).await; + assert!( + warm.recv_response().await.unwrap().status().is_success(), + "h3 info/refs advertisement must serve over QUIC" + ); + common::drain(&mut warm).await; + + let request = http::Request::builder() + .method(Method::POST) + .uri(format!("https://localhost/{did}/git-upload-pack")) + .header("content-type", "application/x-git-upload-pack-request") + .header("git-protocol", "version=2") + .body(()) + .unwrap(); + let mut stream = sender.send_request(request).await.unwrap(); + stream.send_data(fetch_body(tip)).await.unwrap(); + common::finish_request(&mut stream).await; + let response = stream.recv_response().await.unwrap(); + assert!( + response.status().is_success(), + "h3 upload-pack returned {}", + response.status() + ); + let out = common::drain(&mut stream).await; + quic.close(0u32.into(), b"done"); + drive.abort(); + out +} + +async fn cloned_set( + layout: &Layout, + did: &RepoDid, + tip: &str, + format: ObjectFormat, +) -> BTreeSet { + let certdir = tempfile::tempdir().unwrap(); + let clonedir = tempfile::tempdir().unwrap(); + let edge = common::serve_edge(certdir.path(), || { + let (write_routes, advertisement) = knot_pack::edge_routes(knot_pack::EdgeConfig { + pack_slots: knot_resource::PackSlots::new(4), + ..knot_pack::EdgeConfig::serving( + layout.clone(), + serve_dids(), + Arc::new(knot_runtime::SystemClock), + ) + }); + (RequiresFullHandshake::new(write_routes), advertisement) + }) + .await; + let pack = extract_pack(&clone_over_h3(&edge, did, tip).await); + edge.shutdown.cancel(); + edge.task.abort(); + init(clonedir.path(), format); + index_pack(clonedir.path(), &pack); + object_set(clonedir.path()) +} + +async fn the_live_h3_transport_is_logically_reproducible(format: ObjectFormat, did: &RepoDid) { + let scan = tempfile::tempdir().unwrap(); + let scratch = tempfile::tempdir().unwrap(); + + let layout = Layout::new(scan.path()).with_object_format(format); + layout.create(did).unwrap(); + let bare = layout.repo_path(did).unwrap(); + let tip = seed_bare(&scratch.path().join("work"), &bare, format); + let truth = object_set(&bare); + + let first = cloned_set(&layout, did, &tip, format).await; + let second = cloned_set(&layout, did, &tip, format).await; + + assert_eq!( + first, second, + "{format:?}: two independent live h3 connections must deliver the same object set" + ); + assert_eq!( + first, truth, + "{format:?}: the h3 clone's object set must equal the bare's full object set" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_new_h3_transport_replays_to_the_same_object_set_off_the_recorded_trace() { + the_live_h3_transport_is_logically_reproducible( + ObjectFormat::SHA1, + &RepoDid::new("did:plc:squid").unwrap(), + ) + .await; + the_live_h3_transport_is_logically_reproducible( + ObjectFormat::SHA256, + &RepoDid::new("did:plc:cuttle").unwrap(), + ) + .await; +} diff --git a/knot2/crates/knot-sim/tests/lfs_chaos.rs b/knot2/crates/knot-sim/tests/lfs_chaos.rs new file mode 100644 index 0000000000000000000000000000000000000000..e423ce7b729becc34aa23553e242b8973768f52d --- /dev/null +++ b/knot2/crates/knot-sim/tests/lfs_chaos.rs @@ -0,0 +1,229 @@ +mod common; + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime}; + +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Registration, RegistryChange, RepoRef, RepoRegistryCob, deregister_repo}; +use knot_git::{Layout, Repo}; +use knot_lfs::{ClaimedSize, DiskStore, LfsOid, LfsStore, LfsStorePath, Pool}; +use knot_runtime::OsEntropy; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{KnotId, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds}; + +const REPO_DID: &str = "did:plc:squid"; +const REPO_NAME: &str = "anemone"; +const OWNER_DID: &str = "did:plc:nel"; +const KNOT_DID: &str = "did:web:nel.pet"; +const MEDIA: &[u8] = b"\xff\x00media that mustn't outlive its repo"; +const GRACE: Duration = Duration::from_secs(86_400); +const BACKDATE: Duration = Duration::from_secs(60 * 86_400); + +fn did() -> RepoDid { + RepoDid::new(REPO_DID).unwrap() +} + +fn knot() -> KnotId { + KnotId::new(KNOT_DID).unwrap() +} + +fn master() -> MasterKey { + MasterKey::new([7u8; 32]).unwrap() +} + +fn media_oid() -> LfsOid { + LfsOid::from_digest(knot_types::Sha256Digest::hash(MEDIA)) +} + +struct Paths { + meta: PathBuf, + scan: PathBuf, + store: PathBuf, + keys: PathBuf, +} + +impl Paths { + fn under(root: &Path) -> Self { + Self { + meta: root.join("meta"), + scan: root.join("repos"), + store: root.join("lfs"), + keys: root.join("keys.sealed"), + } + } +} + +fn build_fixture(root: &Path) -> Paths { + let paths = Paths::under(root); + Repo::create(&paths.meta).unwrap(); + Layout::new(&paths.scan).create(&did()).unwrap(); + + let secrets = SealedStore::open(&paths.keys, &master(), Box::new(OsEntropy)).unwrap(); + secrets.ensure(&knot()).unwrap(); + let signer = secrets.signer(&knot()).unwrap(); + let meta = Repo::open(&paths.meta).unwrap(); + CobStore::new(&meta) + .create( + &CobHome::from(&knot()), + &RegistryChange::Register(Registration { + owner: OwnerDid::new(OWNER_DID).unwrap(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: RepoName::new(REPO_NAME).unwrap(), + repo: did(), + created_at: UnixSeconds::new(1), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + + std::fs::create_dir_all(&paths.store).unwrap(); + let store = DiskStore::open(LfsStorePath::new(&paths.store)).unwrap(); + store + .put( + Pool::Lfs, + &did(), + &media_oid(), + ClaimedSize::new(MEDIA.len() as u64), + &mut &MEDIA[..], + ) + .unwrap(); + let object_path = store + .object_file(Pool::Lfs, &did(), &media_oid()) + .unwrap() + .unwrap() + .1; + std::fs::OpenOptions::new() + .write(true) + .open(object_path) + .unwrap() + .set_modified(SystemTime::now() - BACKDATE) + .unwrap(); + paths +} + +fn spawn_worker(paths: &Paths) -> std::process::Child { + common::spawn_worker( + "chaos_delete_worker", + &[ + ("KNOT_CHAOS_ROLE", "delete"), + ("KNOT_CHAOS_META", paths.meta.to_str().unwrap()), + ("KNOT_CHAOS_SCAN", paths.scan.to_str().unwrap()), + ("KNOT_CHAOS_STORE", paths.store.to_str().unwrap()), + ("KNOT_CHAOS_KEYS", paths.keys.to_str().unwrap()), + ], + ) +} + +#[test] +fn chaos_delete_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("delete") { + return; + } + let meta = Repo::open(std::env::var("KNOT_CHAOS_META").unwrap()).unwrap(); + let store = CobStore::new(&meta); + let secrets = SealedStore::open( + std::env::var("KNOT_CHAOS_KEYS").unwrap(), + &master(), + Box::new(OsEntropy), + ) + .unwrap(); + let signer = secrets.signer(&knot()).unwrap(); + let object = store.list::().unwrap()[0]; + deregister_repo( + &store, + &CobHome::from(&knot()), + object, + RepoRef { + owner: OwnerDid::new(OWNER_DID).unwrap(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + }, + did(), + &signer, + UnixSeconds::new(2), + ) + .unwrap(); + Layout::new(std::env::var("KNOT_CHAOS_SCAN").unwrap()) + .remove(&did()) + .unwrap(); + DiskStore::open(LfsStorePath::new( + std::env::var("KNOT_CHAOS_STORE").unwrap(), + )) + .unwrap() + .remove_repo(&did()) + .unwrap(); +} + +#[test] +fn kill9_between_delete_steps_never_strands_the_store_prefix() { + let scratch = tempfile::tempdir().unwrap(); + + let warm = build_fixture(&scratch.path().join("warm")); + let started = Instant::now(); + spawn_worker(&warm).wait().unwrap(); + let full = started.elapsed(); + let warm_store = DiskStore::open(LfsStorePath::new(&warm.store)).unwrap(); + assert_eq!( + warm_store.probe(Pool::Lfs, &did(), &media_oid()).unwrap(), + None, + "an uninterrupted delete removes the store prefix itself" + ); + + let delays = common::kill_delays(full); + + let outcomes: Vec = delays + .iter() + .enumerate() + .map(|(trial, delay)| { + let paths = build_fixture(&scratch.path().join(format!("trial-{trial}"))); + let mut child = spawn_worker(&paths); + std::thread::sleep(*delay); + let _ = child.kill(); + child.wait().unwrap(); + + let index = knot_index::Index::new(paths.meta.clone(), Layout::new(&paths.scan)); + index.rebuild().unwrap_or_else(|error| { + panic!("trial {trial}: registry must rebuild after a killed delete: {error}") + }); + assert_eq!( + index.coverage().registry, + knot_index::Coverage::Ready, + "trial {trial}: a rebuilt projection is ready" + ); + let hosted: HashSet = index.hosted_repos().into_iter().collect(); + let registered = hosted.contains(&did()); + + let store = DiskStore::open(LfsStorePath::new(&paths.store)).unwrap(); + store + .sweep_orphans(&hosted, GRACE, SystemTime::now()) + .unwrap_or_else(|error| { + panic!("trial {trial}: orphan sweep must run after a killed delete: {error}") + }); + let present = store.probe(Pool::Lfs, &did(), &media_oid()).unwrap().is_some(); + match registered { + true => assert!( + present, + "trial {trial}: a still-registered repo's store prefix is never condemned" + ), + false => assert!( + !present, + "trial {trial}: the orphan sweep reclaims the prefix the killed delete left behind" + ), + } + registered + }) + .collect(); + + assert!( + outcomes.iter().any(|registered| *registered), + "some trial must die before the deregister lands, or the chaos window never opened" + ); + assert!( + outcomes.iter().any(|registered| !registered), + "some trial must land the deregister, or the kill delays are all too short" + ); +} diff --git a/knot2/crates/knot-sim/tests/lfs_roundtrip.rs b/knot2/crates/knot-sim/tests/lfs_roundtrip.rs new file mode 100644 index 0000000000000000000000000000000000000000..5951498d4fd50b0fe9ffc0c3ae5c18acde0fa34e --- /dev/null +++ b/knot2/crates/knot-sim/tests/lfs_roundtrip.rs @@ -0,0 +1,1567 @@ +mod common; + +use std::collections::BTreeSet; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Arc; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use bytes::Bytes; +use http::Method; +use knot_atproto::Atproto; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Registration, RegistryChange}; +use knot_edge::RequiresFullHandshake; +use knot_git::{Layout, Repo}; +use knot_lfs::{FreeSpaceFloor, LfsHandle, LfsOid, LfsSize, LfsStore, LfsStorePath, Pool}; +use knot_runtime::{ + FakeHttp, HttpResponse, K256Signer, ManualClock, OsEntropy, Signer, UnixMicros, +}; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{ + AccountDid, ActorId, AdmissionPolicy, Did, KnotHostname, KnotId, KnotServiceUrl, Nsid, + OwnerDid, RepoDid, RepoName, RepoRkey, ServiceDid, UnixSeconds, +}; +use tempfile::TempDir; +use tokio::net::TcpListener; +use tower::ServiceExt; +use url::Url; + +const REPO_DID: &str = "did:plc:squid"; +const REPO_NAME: &str = "anemone"; +const FORK_NAME: &str = "anemone-fork"; +const OWNER_DID: &str = "did:plc:nel"; +const PDS_HOST: &str = "pds.oyster.cafe"; +const KNOT_DID: &str = "did:web:nel.pet"; +const PINNED_DATE: &str = "2026-07-07T12:00:00+00:00"; + +fn repo_did() -> RepoDid { + RepoDid::new(REPO_DID).unwrap() +} + +fn owner() -> AccountDid { + AccountDid::new(OWNER_DID).unwrap() +} + +fn knot_service_did() -> ServiceDid { + ServiceDid::new(KNOT_DID).unwrap() +} + +fn require_git_lfs() -> bool { + let available = Command::new("git-lfs") + .arg("version") + .output() + .map(|out| out.status.success()) + .unwrap_or(false); + match (available, std::env::var("KNOT_LFS_ROUNDTRIP").as_deref()) { + (true, _) => true, + (false, Ok("skip")) => { + eprintln!( + "skipping lfs round trip gate: git-lfs unavailable and KNOT_LFS_ROUNDTRIP=skip" + ); + false + } + (false, _) => panic!( + "the lfs round trip gate found no working git-lfs on PATH. \ + Install git-lfs or set KNOT_LFS_ROUNDTRIP=skip to skip the gate." + ), + } +} + +fn media_bytes() -> Vec { + (0..1_048_576u32) + .map(|n| (n.wrapping_mul(31) % 251) as u8) + .collect() +} + +fn second_media_bytes() -> Vec { + (0..524_288u32) + .map(|n| (n.wrapping_mul(97).wrapping_add(13) % 253) as u8) + .collect() +} + +fn require_scutiger() -> bool { + let available = Command::new("git-lfs-transfer") + .arg("--help") + .output() + .map(|out| out.status.success()) + .unwrap_or(false); + match (available, std::env::var("KNOT_LFS_CONFORMANCE").as_deref()) { + (true, _) => true, + (false, Ok("skip")) => { + eprintln!( + "skipping lfs conformance gate: git-lfs-transfer unavailable and \ + KNOT_LFS_CONFORMANCE=skip" + ); + false + } + (false, _) => panic!( + "the lfs conformance gate found no scutiger git-lfs-transfer on PATH. \ + Install it or set KNOT_LFS_CONFORMANCE=skip to skip the gate." + ), + } +} + +fn git(cwd: &Path, env: &[(String, String)], args: &[&str]) -> (bool, String) { + let mut command = knot_fixtures::command_at(cwd, PINNED_DATE); + command.args(args); + env.iter().for_each(|(key, value)| { + command.env(key, value); + }); + let out = command.output().expect("git runs"); + ( + out.status.success(), + format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ), + ) +} + +fn keygen(dir: &Path) -> (String, String) { + let path = dir.join("client"); + let out = Command::new("ssh-keygen") + .args([ + "-t", + "ed25519", + "-N", + "", + "-C", + "nel@oyster.cafe", + "-f", + path.to_str().unwrap(), + ]) + .output() + .expect("ssh-keygen runs"); + assert!(out.status.success()); + let public_line = std::fs::read_to_string(dir.join("client.pub")) + .unwrap() + .trim() + .to_string(); + (path.to_str().unwrap().to_string(), public_line) +} + +fn actor_signer() -> K256Signer { + K256Signer::from_slice(&[9u8; 32]).unwrap() +} + +fn did_document(did: &Did, pds: &KnotServiceUrl) -> Vec { + let multikey = ActorId::new(knot_types::crypto::multikey( + 0xe7, + actor_signer().public_key().as_bytes(), + )) + .unwrap(); + serde_json::to_vec(&serde_json::json!({ + "id": did.as_str(), + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [{ + "id": format!("{}#atproto", did.as_str()), + "type": "Multikey", + "controller": did.as_str(), + "publicKeyMultibase": multikey.as_str() + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": pds.as_str() + }] + })) + .unwrap() +} + +fn list_records_body(public_line: &str) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "records": [{ + "uri": format!("at://{}/org.tangled.key.sshKey/1", owner()), + "value": { + "$type": "org.tangled.key.sshKey", + "publicKey": public_line, + "name": "laptop", + "createdAt": "2026-07-01T00:00:00Z" + } + }] + })) + .unwrap() +} + +fn fake_http( + published_line: String, + knot_pds: KnotServiceUrl, +) -> FakeHttp< + impl Fn(&knot_runtime::HttpRequest) -> Result + + Send + + Sync, +> { + let foreign_pds = KnotServiceUrl::new(format!("https://{PDS_HOST}")).unwrap(); + FakeHttp::new(move |request: &knot_runtime::HttpRequest| { + let host = request.url.host_str().unwrap_or_default().to_string(); + let path = request.url.path().to_string(); + let body = if host == PDS_HOST { + list_records_body(&published_line) + } else if host == "plc.directory" && request.method == http::Method::POST { + b"{}".to_vec() + } else if host == "plc.directory" && path.starts_with("/did:") { + let did = Did::new_owned(path.trim_start_matches('/')).expect("the plc path is a DID"); + let pds = match did.as_str() == repo_did().as_str() { + true => &knot_pds, + false => &foreign_pds, + }; + did_document(&did, pds) + } else { + return Ok(HttpResponse { + status: http::StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + }); + }; + Ok(HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from(body), + }) + }) +} + +fn nsid(name: &str) -> Nsid { + Nsid::new_owned(name).expect("the xrpc method is an NSID") +} + +fn service_jwt(method: &Nsid, jti: &str) -> String { + service_jwt_as( + &Did::new_owned(OWNER_DID).unwrap(), + &knot_service_did(), + method, + jti, + ) +} + +fn service_jwt_as(iss: &Did, audience: &ServiceDid, method: &Nsid, jti: &str) -> String { + let header = URL_SAFE_NO_PAD.encode(br#"{"alg":"ES256K","typ":"JWT"}"#); + let claims = serde_json::json!({ + "iss": iss, + "aud": audience.as_str(), + "exp": 1_001, + "iat": 999, + "jti": jti, + "lxm": method, + }); + let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()); + let signing_input = format!("{header}.{payload}"); + let signature = actor_signer().sign(signing_input.as_bytes()); + format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.as_bytes()) + ) +} + +struct World { + _scan: TempDir, + lfs: LfsHandle, + ssh_port: u16, + http_base: String, + router: axum::Router, + _imports_stop: tokio::sync::watch::Sender, + layout: Layout, + h3: Option, + _certdir: Option, +} + +async fn spawn_world(published_line: String) -> World { + spawn(published_line, false).await +} + +async fn spawn(published_line: String, with_h3: bool) -> World { + let scan = tempfile::tempdir().unwrap(); + let meta_path = scan.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scan.path().join("repos")); + let repo_did = repo_did(); + layout.create(&repo_did).unwrap(); + + let knot = KnotId::new(KNOT_DID).unwrap(); + let secrets = Arc::new( + SealedStore::open( + scan.path().join("keys.sealed"), + &MasterKey::new([7u8; 32]).unwrap(), + Box::new(OsEntropy), + ) + .unwrap(), + ); + secrets.ensure(&knot).unwrap(); + let knot_signer = secrets.signer(&knot).unwrap(); + + let meta = Repo::open(&meta_path).unwrap(); + CobStore::new(&meta) + .create( + &CobHome::from(&knot), + &RegistryChange::Register(Registration { + owner: OwnerDid::new(OWNER_DID).unwrap(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: RepoName::new(REPO_NAME).unwrap(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(1), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }), + &knot_signer, + UnixSeconds::new(1), + ) + .unwrap(); + + let index = Arc::new(knot_index::Index::new(meta_path.clone(), layout.clone())); + index.rebuild().unwrap(); + + let http_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let http_port = http_listener.local_addr().unwrap().port(); + let http_base = format!("http://127.0.0.1:{http_port}"); + let atproto = Arc::new(Atproto::new( + fake_http( + published_line, + KnotServiceUrl::new(format!("https://127.0.0.1:{http_port}")).unwrap(), + ), + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot.clone(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + )); + + let lfs_store_dir = scan.path().join("lfs"); + std::fs::create_dir_all(&lfs_store_dir).unwrap(); + let lfs = LfsHandle::open( + LfsStorePath::new(&lfs_store_dir), + LfsSize::new(64 * 1024 * 1024), + FreeSpaceFloor::new(0), + ) + .unwrap(); + + let key_dir = scan.path().join("hostkey"); + std::fs::create_dir_all(&key_dir).unwrap(); + let host_key = knot_ssh::load_or_create_host_key(&key_dir.join("host")).unwrap(); + let ssh_state = Arc::new(knot_ssh::SshState::new(knot_ssh::SshConfig { + layout: layout.clone(), + index: Arc::clone(&index), + atproto: Arc::clone(&atproto), + knot_actor: knot_types::ActorId::from_secp256k1(actor_signer().public_key().as_bytes()), + hostname: KnotHostname::new("nel.pet").unwrap(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + admins: BTreeSet::from([owner()]), + admission: AdmissionPolicy::Closed, + max_pack_bytes: knot_xrpc::MaxWireBytes::new(1 << 30), + archive_limit: knot_git::ArchiveLimit::default(), + lfs: lfs.clone(), + lfs_max_transfers: 16, + max_inflight_per_peer: 4, + })); + let ssh_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let ssh_port = ssh_listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let _ = knot_ssh::serve_on_socket(ssh_listener, host_key, ssh_state).await; + }); + + let (imports, jobs) = knot_xrpc::imports::ImportQueue::channel(); + let outbox = knot_xrpc::Outbox::open(&meta_path, Default::default()); + let xrpc_state = Arc::new(knot_xrpc::XrpcState { + imports, + handover_sessions: Arc::new(knot_xrpc::handover::HandoverSessions::default()), + layout: layout.clone(), + index: Arc::clone(&index), + atproto, + secrets, + entropy: Arc::new(OsEntropy), + invite_codes: Default::default(), + admins: BTreeSet::from([owner()]), + admission: AdmissionPolicy::Closed, + contribution_policy: knot_types::ContributionPolicy::Anyone, + writes: Arc::new(knot_resource::SubjectLimiter::new( + knot_resource::RateLimit { + burst: knot_xrpc::Burst::literal(1_000), + refill: knot_xrpc::RefillMicros::literal(1_000), + }, + )), + social_budget: knot_resource::SocialBudgetBytes::new(1 << 28), + knot_repo: knot_xrpc::knot_repo(&knot).unwrap(), + knot_did: knot, + knot_hostname: KnotHostname::new("nel.pet").unwrap(), + meta_path, + knot_service_url: knot_types::KnotServiceUrl::new(http_base.clone()).unwrap(), + limiter: Arc::new(knot_xrpc::PreAuthLimiter::default()), + cob_locks: Arc::new(knot_xrpc::CobLocks::default()), + reservations: Arc::new(knot_xrpc::Reservations::new( + knot_xrpc::ReservationTtl::new(1_000_000), + knot_xrpc::PerActorQuota::new(16), + knot_xrpc::GlobalQuota::new(16), + )), + proxy_trust: knot_types::ProxyTrust::default(), + byte_limits: knot_xrpc::ByteLimits { + pack: knot_xrpc::MaxWireBytes::new(1 << 30), + ..knot_xrpc::ByteLimits::default() + }, + budgets: knot_xrpc::Budgets::default(), + git_http: Arc::new(FakeHttp::new(|_request: &knot_runtime::HttpRequest| { + Err(knot_runtime::NetworkError::Connect( + "no remote upstream is served in this gate".to_string(), + )) + })), + pack_limits: knot_pack::PackLimits::default(), + service_owner: owner(), + legacy_admin_secret: None, + description: Default::default(), + firehose: Arc::new(knot_events::EventLog::new( + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(64).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + )), + subscriber_gate: Arc::new(knot_events::SubscriberGate::new( + knot_events::GlobalSubscriberLimit::new(16), + knot_events::PerPeerSubscriberLimit::new(4), + )), + maintenance: knot_maintenance::MaintenanceHandle::disabled(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + slots: knot_resource::Slots::testing(8), + lfs: knot_xrpc::LfsWeb::new(lfs.clone(), 8), + blobs: Default::default(), + catalog: Arc::new(knot_messages::Catalog::defaults()), + rkeys: Default::default(), + outbox, + }); + + let (_imports_stop, imports_halt) = tokio::sync::watch::channel(false); + tokio::spawn(knot_xrpc::imports::run( + Arc::clone(&xrpc_state), + jobs, + imports_halt, + )); + + let resolver: Arc = { + let index = Arc::clone(&index); + Arc::new(move |target: &knot_pack::RepoTarget| match target { + knot_pack::RepoTarget::Did(did) => match index.owner_of(did) { + knot_index::Resolved::Ready(Some(_)) => knot_pack::RepoLookup::Hosted(did.clone()), + knot_index::Resolved::Ready(None) => knot_pack::RepoLookup::Unhosted, + knot_index::Resolved::Warming => knot_pack::RepoLookup::Unavailable, + }, + knot_pack::RepoTarget::OwnerPath(owner, path) => { + match index.resolve_clone_path(owner, path) { + knot_index::Resolved::Ready(Some(found)) => { + knot_pack::RepoLookup::Hosted(found) + } + knot_index::Resolved::Ready(None) => knot_pack::RepoLookup::Unhosted, + knot_index::Resolved::Warming => knot_pack::RepoLookup::Unavailable, + } + } + }) + }; + let advertiser = knot_xrpc::receive_advertiser(Arc::clone(&xrpc_state)); + let (write_routes, advertisement) = knot_pack::edge_routes(knot_pack::EdgeConfig { + receive: Some(Arc::clone(&advertiser)), + pack_slots: knot_resource::PackSlots::new(4), + ..knot_pack::EdgeConfig::serving( + layout.clone(), + Arc::clone(&resolver), + Arc::new(knot_runtime::SystemClock), + ) + }); + let router = write_routes + .merge(advertisement.into_router()) + .merge(knot_xrpc::router(Arc::clone(&xrpc_state))); + let served = router.clone(); + tokio::spawn(async move { + let _ = axum::serve(http_listener, served).await; + }); + + let (h3, certdir) = match with_h3 { + true => { + let certdir = tempfile::tempdir().unwrap(); + let edge = common::serve_edge(certdir.path(), || { + let (write_routes, advertisement) = knot_pack::edge_routes(knot_pack::EdgeConfig { + receive: Some(Arc::clone(&advertiser)), + pack_slots: knot_resource::PackSlots::new(4), + ..knot_pack::EdgeConfig::serving( + layout.clone(), + Arc::clone(&resolver), + Arc::new(knot_runtime::SystemClock), + ) + }); + let app = RequiresFullHandshake::new( + write_routes.merge(knot_xrpc::router(Arc::clone(&xrpc_state))), + ); + (app, advertisement) + }) + .await; + (Some(edge), Some(certdir)) + } + false => (None, None), + }; + + World { + _scan: scan, + lfs, + ssh_port, + http_base, + router, + _imports_stop, + layout, + h3, + _certdir: certdir, + } +} + +async fn in_git_blocking(task: impl FnOnce() -> T + Send + 'static) -> T { + tokio::task::spawn_blocking(task).await.unwrap() +} + +fn seed_lfs_work(work: &Path, env: &[(String, String)], media: &[u8]) { + std::fs::create_dir_all(work).unwrap(); + let steps: [&[&str]; 2] = [ + &["init", "-q", "-b", "main"], + &["lfs", "install", "--local"], + ]; + steps.iter().for_each(|args| { + let (ok, out) = git(work, env, args); + assert!(ok, "{args:?} failed:\n{out}"); + }); + let (ok, out) = git(work, env, &["lfs", "track", "*.bin"]); + assert!(ok, "lfs track failed:\n{out}"); + let (ok, out) = git(work, env, &["config", "lfs.locksverify", "false"]); + assert!(ok, "config failed:\n{out}"); + std::fs::write(work.join("media.bin"), media).unwrap(); + std::fs::write(work.join("README.md"), "media lives in lfs\n").unwrap(); + let commit: [&[&str]; 2] = [&["add", "-A"], &["commit", "-q", "-m", "media"]]; + commit.iter().for_each(|args| { + let (ok, out) = git(work, env, args); + assert!(ok, "{args:?} failed:\n{out}"); + }); +} + +fn clone_and_pull(base: &Path, url: &str, name: &str, env: &[(String, String)]) -> PathBuf { + let skip_smudge: Vec<(String, String)> = env + .iter() + .cloned() + .chain([("GIT_LFS_SKIP_SMUDGE".to_string(), "1".to_string())]) + .collect(); + let (ok, out) = git(base, &skip_smudge, &["clone", "-q", url, name]); + assert!(ok, "anonymous clone of {url} failed:\n{out}"); + let dst = base.join(name); + let pointer = std::fs::read_to_string(dst.join("media.bin")).unwrap(); + assert!( + pointer.contains("git-lfs.github.com/spec/v1"), + "clone must land the pointer before lfs pull, got:\n{pointer}" + ); + let (ok, out) = git(&dst, env, &["lfs", "install", "--local"]); + assert!(ok, "lfs install in {name} failed:\n{out}"); + let (ok, out) = git(&dst, env, &["lfs", "pull"]); + assert!(ok, "git lfs pull in {name} failed:\n{out}"); + dst +} + +async fn create_fork(world: &World, jti: &str) -> (http::StatusCode, serde_json::Value) { + let token = service_jwt(&nsid("org.tangled.repo.createRepo"), jti); + let body = serde_json::json!({ + "name": FORK_NAME, + "source": { + "$type": "org.tangled.repo.defs#sourceTangled", + "did": repo_did(), + }, + }); + let request = http::Request::builder() + .method("POST") + .uri("/xrpc/org.tangled.repo.createRepo") + .header(http::header::CONTENT_TYPE, "application/json") + .header(http::header::AUTHORIZATION, format!("Bearer {token}")) + .body(axum::body::Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let value = serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); + (status, value) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_lfs_round_trip_gate_holds_over_both_transports_and_the_fork() { + if !require_git_lfs() { + return; + } + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + + let media = media_bytes(); + let media_oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(&media)); + let ssh = format!("ssh -i {key_path} {}", knot_fixtures::UNSHARED_SSH); + let path_env = std::env::var("PATH").unwrap_or_default(); + let home = scratch.path().to_str().unwrap().to_string(); + let env: Vec<(String, String)> = [ + ("GIT_SSH_COMMAND", &ssh), + ("PATH", &path_env), + ("HOME", &home), + ] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + + let work = scratch.path().join("work"); + seed_lfs_work(&work, &env, &media); + + let push_url = format!( + "ssh://git@127.0.0.1:{}/{}/{REPO_NAME}", + world.ssh_port, + owner() + ); + let (ok, out) = { + let work = work.clone(); + let env = env.clone(); + in_git_blocking(move || git(&work, &env, &["push", "-q", &push_url, "main"])).await + }; + assert!(ok, "lfs push over ssh failed:\n{out}"); + + let source_repo = repo_did(); + assert_eq!( + world + .lfs + .store + .probe(Pool::Lfs, &source_repo, &media_oid) + .unwrap(), + Some(LfsSize::new(media.len() as u64)), + "pushed media must be durable in the store" + ); + + let clone_url = format!("{}/{}/{REPO_NAME}", world.http_base, owner()); + let dst = { + let base = scratch.path().to_path_buf(); + let env = env.clone(); + in_git_blocking(move || clone_and_pull(&base, &clone_url, "reader", &env)).await + }; + assert_eq!( + std::fs::read(dst.join("media.bin")).unwrap(), + media, + "anonymous http reader must see byte-identical media" + ); + + let (status, created) = create_fork(&world, "gate-fork-1").await; + assert_eq!( + status, + http::StatusCode::OK, + "fork create failed: {created}" + ); + assert!( + created.get("lfsMissing").is_none(), + "local fork must copy every object, got {created}" + ); + let fork_did = RepoDid::new(created["did"].as_str().unwrap()).unwrap(); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + let forked = loop { + if let Some(size) = world + .lfs + .store + .probe(Pool::Lfs, &fork_did, &media_oid) + .unwrap() + { + break size; + } + assert!( + tokio::time::Instant::now() < deadline, + "the queued fork import never landed its copy of the media; sidecar: {}", + std::fs::read_to_string( + world + .layout + .repo_path(&fork_did) + .unwrap() + .join("tangled.import.json") + ) + .unwrap_or_else(|error| format!("unreadable: {error}")) + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + }; + assert_eq!( + forked, + LfsSize::new(media.len() as u64), + "fork prefix must hold its own copy of the media" + ); + + let fork_url = format!("{}/{}/{FORK_NAME}", world.http_base, owner()); + let fork_dst = { + let base = scratch.path().to_path_buf(); + let env = env.clone(); + in_git_blocking(move || clone_and_pull(&base, &fork_url, "fork-reader", &env)).await + }; + assert_eq!( + std::fs::read(fork_dst.join("media.bin")).unwrap(), + media, + "anonymous clone of the fork must see byte-identical media" + ); +} + +fn seed_many_lfs(work: &Path, env: &[(String, String)], count: usize) { + std::fs::create_dir_all(work).unwrap(); + let steps: [&[&str]; 2] = [ + &["init", "-q", "-b", "main"], + &["lfs", "install", "--local"], + ]; + steps.iter().for_each(|args| { + let (ok, out) = git(work, env, args); + assert!(ok, "{args:?} failed:\n{out}"); + }); + let (ok, out) = git(work, env, &["lfs", "track", "*.bin"]); + assert!(ok, "lfs track failed:\n{out}"); + let (ok, out) = git(work, env, &["config", "lfs.locksverify", "false"]); + assert!(ok, "config failed:\n{out}"); + (0..count).for_each(|index| { + let size = 200 + index * 7; + let bytes: Vec = (0..size) + .map(|n| (n.wrapping_mul(31).wrapping_add(index) % 251) as u8) + .collect(); + std::fs::write(work.join(format!("object-{index}.bin")), bytes).unwrap(); + }); + let commit: [&[&str]; 2] = [&["add", "-A"], &["commit", "-q", "-m", "many media"]]; + commit.iter().for_each(|args| { + let (ok, out) = git(work, env, args); + assert!(ok, "{args:?} failed:\n{out}"); + }); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn many_objects_ride_default_git_lfs_concurrency_over_both_transports() { + if !require_git_lfs() { + return; + } + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + + let ssh = format!("ssh -i {key_path} {}", knot_fixtures::UNSHARED_SSH); + let path_env = std::env::var("PATH").unwrap_or_default(); + let home = scratch.path().to_str().unwrap().to_string(); + let env: Vec<(String, String)> = [ + ("GIT_SSH_COMMAND", &ssh), + ("PATH", &path_env), + ("HOME", &home), + ] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + + let count = 25usize; + let work = scratch.path().join("work"); + seed_many_lfs(&work, &env, count); + + let push_url = format!( + "ssh://git@127.0.0.1:{}/{}/{REPO_NAME}", + world.ssh_port, + owner() + ); + let (ok, out) = { + let work = work.clone(); + let env = env.clone(); + in_git_blocking(move || git(&work, &env, &["push", "-q", &push_url, "main"])).await + }; + assert!( + ok, + "git-lfs at its default concurrency must push {count} objects over ssh without tripping \ + the per-peer connection limit:\n{out}" + ); + + let clone_url = format!("{}/{}/{REPO_NAME}", world.http_base, owner()); + let dst = { + let base = scratch.path().to_path_buf(); + let env = env.clone(); + in_git_blocking(move || { + let skip_smudge: Vec<(String, String)> = env + .iter() + .cloned() + .chain([("GIT_LFS_SKIP_SMUDGE".to_string(), "1".to_string())]) + .collect(); + let (ok, out) = git(&base, &skip_smudge, &["clone", "-q", &clone_url, "reader"]); + assert!(ok, "anonymous clone failed:\n{out}"); + let dst = base.join("reader"); + let (ok, out) = git(&dst, &env, &["lfs", "install", "--local"]); + assert!(ok, "lfs install failed:\n{out}"); + let (ok, out) = git(&dst, &env, &["lfs", "pull"]); + assert!( + ok, + "anonymous http pull of {count} objects mustn't be throttled by the xrpc \ + pre-auth limiter:\n{out}" + ); + dst + }) + .await + }; + (0..count).for_each(|index| { + assert_eq!( + std::fs::read(work.join(format!("object-{index}.bin"))).unwrap(), + std::fs::read(dst.join(format!("object-{index}.bin"))).unwrap(), + "object-{index}.bin must be byte-identical over anonymous http" + ); + }); +} + +fn write_shim(dir: &Path) -> String { + let shim = dir.join("local-ssh.sh"); + std::fs::write( + &shim, + "#!/bin/sh\n\ + while [ \"$#\" -gt 0 ]; do\n\ + case \"$1\" in\n\ + -o|-p) shift 2 ;;\n\ + -*) shift ;;\n\ + *) break ;;\n\ + esac\n\ + done\n\ + shift\n\ + eval exec \"$@\"\n", + ) + .unwrap(); + let mut permissions = std::fs::metadata(&shim).unwrap().permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut permissions, 0o755); + std::fs::set_permissions(&shim, permissions).unwrap(); + shim.to_str().unwrap().to_string() +} + +fn hex_object_files(root: &Path) -> Vec<(String, u64, PathBuf)> { + let entries = match std::fs::read_dir(root) { + Ok(entries) => entries, + Err(_) => return Vec::new(), + }; + entries + .filter_map(Result::ok) + .flat_map(|entry| { + let path = entry.path(); + if path.is_dir() { + return hex_object_files(&path); + } + path.file_name() + .and_then(|name| name.to_str()) + .filter(|name| LfsOid::new(*name).is_ok()) + .map(|name| { + let size = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0); + vec![(name.to_string(), size, path.clone())] + }) + .unwrap_or_default() + }) + .collect() +} + +fn pull_verdict(base: &Path, url: &str, name: &str, env: &[(String, String)]) -> (bool, String) { + let skip_smudge: Vec<(String, String)> = env + .iter() + .cloned() + .chain([("GIT_LFS_SKIP_SMUDGE".to_string(), "1".to_string())]) + .collect(); + let (ok, out) = git(base, &skip_smudge, &["clone", "-q", url, name]); + assert!(ok, "clone of {url} failed:\n{out}"); + let dst = base.join(name); + let (ok, out) = git(&dst, env, &["lfs", "install", "--local"]); + assert!(ok, "lfs install in {name} failed:\n{out}"); + git(&dst, env, &["lfs", "pull"]) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_lfs_stack_is_conformant_with_the_reference_server_and_client() { + if !require_git_lfs() || !require_scutiger() { + return; + } + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + + let media = media_bytes(); + let second = second_media_bytes(); + let media_oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(&media)); + let second_oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(&second)); + let expected: std::collections::BTreeSet<(String, u64)> = [ + (media_oid.as_str().to_string(), media.len() as u64), + (second_oid.as_str().to_string(), second.len() as u64), + ] + .into(); + + let ssh = format!("ssh -i {key_path} {}", knot_fixtures::UNSHARED_SSH); + let path_env = std::env::var("PATH").unwrap_or_default(); + let home = scratch.path().to_str().unwrap().to_string(); + let knot_env: Vec<(String, String)> = [ + ("GIT_SSH_COMMAND", &ssh), + ("PATH", &path_env), + ("HOME", &home), + ] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + let shim = write_shim(scratch.path()); + let reference_env: Vec<(String, String)> = [ + ("GIT_SSH_COMMAND", &shim), + ("PATH", &path_env), + ("HOME", &home), + ] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + + let upstream = scratch.path().join("reference-upstream.git"); + let (ok, out) = git( + scratch.path(), + &reference_env, + &[ + "init", + "-q", + "--bare", + "-b", + "main", + upstream.to_str().unwrap(), + ], + ); + assert!(ok, "reference upstream init failed:\n{out}"); + + let work = scratch.path().join("work"); + seed_lfs_work(&work, &knot_env, &media); + std::fs::write(work.join("extra.bin"), &second).unwrap(); + let commit: [&[&str]; 2] = [&["add", "-A"], &["commit", "-q", "-m", "extra media"]]; + commit.iter().for_each(|args| { + let (ok, out) = git(&work, &knot_env, args); + assert!(ok, "{args:?} failed:\n{out}"); + }); + + let knot_push_url = format!( + "ssh://git@127.0.0.1:{}/{}/{REPO_NAME}", + world.ssh_port, + owner() + ); + let reference_push_url = format!("ssh://ref@localhost{}", upstream.display()); + let pushes = { + let work = work.clone(); + let knot_env = knot_env.clone(); + let reference_env = reference_env.clone(); + let knot_push_url = knot_push_url.clone(); + let reference_push_url = reference_push_url.clone(); + in_git_blocking(move || { + [ + git(&work, &knot_env, &["push", "-q", &knot_push_url, "main"]), + git( + &work, + &reference_env, + &["push", "-q", &reference_push_url, "main"], + ), + ] + }) + .await + }; + pushes.iter().for_each(|(ok, out)| { + assert!(ok, "push failed:\n{out}"); + }); + + let source_repo = repo_did(); + let knot_objects: std::collections::BTreeSet<(String, u64)> = world + .lfs + .store + .enumerate(Pool::Lfs, &source_repo) + .unwrap() + .into_iter() + .map(|object| (object.oid.as_str().to_string(), object.size.get())) + .collect(); + let reference_objects: std::collections::BTreeSet<(String, u64)> = hex_object_files(&upstream) + .into_iter() + .map(|(name, size, _)| (name, size)) + .collect(); + assert_eq!( + knot_objects, expected, + "the knot store holds exactly the pushed object set" + ); + assert_eq!( + knot_objects, reference_objects, + "both servers hold identical object sets after the same push" + ); + + let knot_clone_url = format!("{}/{}/{REPO_NAME}", world.http_base, owner()); + let (knot_dst, reference_dst) = { + let base = scratch.path().to_path_buf(); + let knot_env = knot_env.clone(); + let reference_env = reference_env.clone(); + let knot_clone_url = knot_clone_url.clone(); + let reference_push_url = reference_push_url.clone(); + in_git_blocking(move || { + ( + clone_and_pull(&base, &knot_clone_url, "knot-reader", &knot_env), + clone_and_pull( + &base, + &reference_push_url, + "reference-reader", + &reference_env, + ), + ) + }) + .await + }; + ["media.bin", "extra.bin"].iter().for_each(|file| { + assert_eq!( + std::fs::read(knot_dst.join(file)).unwrap(), + std::fs::read(reference_dst.join(file)).unwrap(), + "{file}: both servers must check out identical media" + ); + }); + assert_eq!(std::fs::read(knot_dst.join("media.bin")).unwrap(), media); + assert_eq!(std::fs::read(knot_dst.join("extra.bin")).unwrap(), second); + + let knot_removed = world + .lfs + .store + .object_file(Pool::Lfs, &source_repo, &second_oid) + .unwrap() + .unwrap() + .1; + std::fs::remove_file(knot_removed).unwrap(); + let removed = hex_object_files(&upstream) + .into_iter() + .filter(|(name, _, _)| name == second_oid.as_str()) + .map(|(_, _, path)| std::fs::remove_file(path).unwrap()) + .count(); + assert!( + removed > 0, + "the reference server holds the object to remove" + ); + + let verdicts = { + let base = scratch.path().to_path_buf(); + let knot_env = knot_env.clone(); + let reference_env = reference_env.clone(); + let knot_push_url = knot_push_url.clone(); + in_git_blocking(move || { + [ + pull_verdict(&base, &knot_clone_url, "knot-missing-http", &knot_env), + pull_verdict(&base, &knot_push_url, "knot-missing-ssh", &knot_env), + pull_verdict( + &base, + &reference_push_url, + "reference-missing", + &reference_env, + ), + ] + }) + .await + }; + let [(http_ok, http_out), (ssh_ok, ssh_out), (reference_ok, _)] = verdicts; + assert!( + !http_ok, + "an http pull of a missing object must fail loudly, never succeed silently:\n{http_out}" + ); + assert!( + !ssh_ok, + "an ssh pull of a missing object must fail loudly, never succeed silently:\n{ssh_out}" + ); + assert!( + reference_ok, + "scutiger 0.3.0 answers noop for a missing download and the client silently \ + succeeds. This pin is the recorded reason knot answers download instead, so its \ + get-object 404 turns the pull into a loud failure. If the reference starts failing \ + loudly too, the divergence note can be retired." + ); +} + +async fn lfs_batch( + world: &World, + auth: Option<&str>, + op: &str, + oid: &LfsOid, + size: u64, +) -> (http::StatusCode, serde_json::Value) { + let body = serde_json::json!({ + "operation": op, + "transfers": ["basic"], + "objects": [{ "oid": oid.as_str(), "size": size }], + "hash_algo": "sha256", + }); + let mut builder = http::Request::builder() + .method("POST") + .uri(format!("/{}/{REPO_NAME}/info/lfs/objects/batch", owner())) + .header(http::header::CONTENT_TYPE, "application/vnd.git-lfs+json"); + if let Some(auth) = auth { + builder = builder.header(http::header::AUTHORIZATION, auth); + } + let request = builder + .body(axum::body::Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null), + ) +} + +async fn lfs_put( + world: &World, + auth: Option<&str>, + oid: &LfsOid, + bytes: Vec, +) -> http::StatusCode { + let mut builder = http::Request::builder() + .method("PUT") + .uri(format!( + "/{}/{REPO_NAME}/info/lfs/objects/{}", + owner(), + oid.as_str() + )) + .header(http::header::CONTENT_LENGTH, bytes.len()); + if let Some(auth) = auth { + builder = builder.header(http::header::AUTHORIZATION, auth); + } + let request = builder.body(axum::body::Body::from(bytes)).unwrap(); + world + .router + .clone() + .oneshot(request) + .await + .unwrap() + .status() +} + +fn basic_auth(token: &str) -> String { + let raw = base64::engine::general_purpose::STANDARD.encode(format!("x-tangled-token:{token}")); + format!("Basic {raw}") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn lfs_http_push_stores_an_object_with_a_push_token() { + let scratch = tempfile::tempdir().unwrap(); + let (_key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + let repo = repo_did(); + + let payload: Vec = (0..4096u32) + .map(|n| (n.wrapping_mul(17) % 251) as u8) + .collect(); + let oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(&payload)); + let size = payload.len() as u64; + + let bearer = format!( + "Bearer {}", + service_jwt(&nsid("org.tangled.git.receivePack"), "lfs-http-push-1") + ); + let (status, body) = lfs_batch(&world, Some(&bearer), "upload", &oid, size).await; + assert_eq!( + status, + http::StatusCode::OK, + "authenticated upload batch: {body}" + ); + let href = body["objects"][0]["actions"]["upload"]["href"] + .as_str() + .unwrap_or_else(|| panic!("expected an upload action, got {body}")); + assert_eq!( + href, + format!( + "{}/{}/{REPO_NAME}/info/lfs/objects/{}", + world.http_base, + owner(), + oid.as_str() + ), + "upload href points at the object route on this knot" + ); + assert_ne!( + body["objects"][0]["authenticated"], + serde_json::Value::Bool(true), + "upload objects mustn't claim authenticated=true, else git-lfs sends the object put with no auth and loops on 401: {body}" + ); + assert!( + world + .lfs + .store + .probe(Pool::Lfs, &repo, &oid) + .unwrap() + .is_none(), + "object must be absent before the put" + ); + + let put = lfs_put(&world, Some(&bearer), &oid, payload.clone()).await; + assert_eq!( + put, + http::StatusCode::OK, + "the same push token must authorize both the batch and the object put" + ); + assert_eq!( + world.lfs.store.probe(Pool::Lfs, &repo, &oid).unwrap(), + Some(LfsSize::new(size)), + "the put object must be durable in the store" + ); + + let get = http::Request::builder() + .method("GET") + .uri(format!( + "/{}/{REPO_NAME}/info/lfs/objects/{}", + owner(), + oid.as_str() + )) + .body(axum::body::Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(get).await.unwrap(); + assert_eq!( + response.status(), + http::StatusCode::OK, + "anonymous download" + ); + let served = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + assert_eq!( + served.as_ref(), + payload.as_slice(), + "an anonymous reader sees the byte-identical object a push stored" + ); + + let payload2: Vec = (0..2048u32) + .map(|n| (n.wrapping_mul(29) % 251) as u8) + .collect(); + let oid2 = LfsOid::from_digest(knot_types::Sha256Digest::hash(&payload2)); + let basic = basic_auth(&service_jwt( + &nsid("org.tangled.git.receivePack"), + "lfs-http-push-2", + )); + let (status, body) = + lfs_batch(&world, Some(&basic), "upload", &oid2, payload2.len() as u64).await; + assert_eq!( + status, + http::StatusCode::OK, + "basic-auth upload batch: {body}" + ); + let put = lfs_put(&world, Some(&basic), &oid2, payload2.clone()).await; + assert_eq!( + put, + http::StatusCode::OK, + "a push token presented as the http basic password must authenticate the put" + ); + assert_eq!( + world.lfs.store.probe(Pool::Lfs, &repo, &oid2).unwrap(), + Some(LfsSize::new(payload2.len() as u64)), + "the basic-authenticated object must be durable too" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn lfs_http_push_rejects_missing_and_mismatched_credentials() { + let scratch = tempfile::tempdir().unwrap(); + let (_key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + + let payload: Vec = (0..1024u32) + .map(|n| (n.wrapping_mul(13) % 251) as u8) + .collect(); + let oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(&payload)); + let size = payload.len() as u64; + + let (status, _) = lfs_batch(&world, None, "upload", &oid, size).await; + assert_eq!( + status, + http::StatusCode::UNAUTHORIZED, + "an unauthenticated upload batch is challenged" + ); + + let wrong_method = format!( + "Bearer {}", + service_jwt(&nsid("org.tangled.repo.createRepo"), "lfs-http-neg-method") + ); + let (status, _) = lfs_batch(&world, Some(&wrong_method), "upload", &oid, size).await; + assert_eq!( + status, + http::StatusCode::UNAUTHORIZED, + "a token bound to another method cannot authorize a push" + ); + + let stranger = format!( + "Bearer {}", + service_jwt_as( + &Did::new_owned(REPO_DID).unwrap(), + &knot_service_did(), + &nsid("org.tangled.git.receivePack"), + "lfs-http-neg-acl", + ) + ); + let (status, _) = lfs_batch(&world, Some(&stranger), "upload", &oid, size).await; + assert_eq!( + status, + http::StatusCode::FORBIDDEN, + "a valid push token from a did that cannot push is refused by the acl" + ); + + let put = lfs_put(&world, None, &oid, payload).await; + assert_eq!( + put, + http::StatusCode::UNAUTHORIZED, + "an unauthenticated object put is challenged" + ); + assert!( + world + .lfs + .store + .probe(Pool::Lfs, &repo_did(), &oid) + .unwrap() + .is_none(), + "no rejected request may leave an object behind" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn git_push_over_http_authenticates_and_lands_the_ref() { + let scratch = tempfile::tempdir().unwrap(); + let (_key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + + let path_env = std::env::var("PATH").unwrap_or_default(); + let home = scratch.path().to_str().unwrap().to_string(); + let env: Vec<(String, String)> = [("PATH", &path_env), ("HOME", &home)] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + let (ok, out) = git(&work, &env, &["init", "-q", "-b", "main"]); + assert!(ok, "init failed:\n{out}"); + std::fs::write(work.join("README.md"), "hello over http\n").unwrap(); + let (ok, out) = git(&work, &env, &["add", "-A"]); + assert!(ok, "add failed:\n{out}"); + let (ok, out) = git(&work, &env, &["commit", "-q", "-m", "init over http"]); + assert!(ok, "commit failed:\n{out}"); + + let url = format!("{}/{}/{REPO_NAME}", world.http_base, owner()); + + let (ok, out) = { + let work = work.clone(); + let env = env.clone(); + let url = url.clone(); + in_git_blocking(move || git(&work, &env, &["push", "-q", &url, "main"])).await + }; + assert!( + !ok, + "an unauthenticated http push must be refused, git reported success:\n{out}" + ); + + let token = service_jwt(&nsid("org.tangled.git.receivePack"), "git-http-push-1"); + let header = format!( + "http.extraHeader=Authorization: Basic {}", + base64::engine::general_purpose::STANDARD.encode(format!("x-tangled-token:{token}")) + ); + let (ok, out) = { + let work = work.clone(); + let env = env.clone(); + let url = url.clone(); + let header = header.clone(); + in_git_blocking(move || git(&work, &env, &["-c", &header, "push", "-q", &url, "main"])) + .await + }; + assert!(ok, "authenticated http push failed:\n{out}"); + + let (ok, refs) = { + let scratch = scratch.path().to_path_buf(); + let env = env.clone(); + let url = url.clone(); + in_git_blocking(move || git(&scratch, &env, &["ls-remote", &url])).await + }; + assert!( + ok && refs.contains("refs/heads/main"), + "the pushed ref must be advertised to an anonymous reader:\n{refs}" + ); + + let wrong = format!( + "http.extraHeader=Authorization: Basic {}", + base64::engine::general_purpose::STANDARD.encode(format!( + "x-tangled-token:{}", + service_jwt(&nsid("org.tangled.repo.createRepo"), "git-http-push-neg") + )) + ); + std::fs::write(work.join("README.md"), "second write\n").unwrap(); + let (ok, out) = git(&work, &env, &["commit", "-q", "-am", "second"]); + assert!(ok, "second commit failed:\n{out}"); + let (ok, out) = { + let work = work.clone(); + let env = env.clone(); + let url = url.clone(); + let wrong = wrong.clone(); + in_git_blocking(move || git(&work, &env, &["-c", &wrong, "push", "-q", &url, "main"])).await + }; + assert!( + !ok, + "a token bound to another method mustn't authorize a push:\n{out}" + ); +} + +fn build_pack(work: &Path, env: &[(String, String)], tip: &str) -> Vec { + let mut command = knot_fixtures::command(work); + command + .args(["pack-objects", "--revs", "--stdout", "--delta-base-offset"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + env.iter().for_each(|(key, value)| { + command.env(key, value); + }); + let mut child = command.spawn().expect("git pack-objects spawns"); + child + .stdin + .take() + .unwrap() + .write_all(format!("{tip}\n").as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "git pack-objects failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + out.stdout +} + +fn build_receive_body(tip: &str, pack: &[u8]) -> Bytes { + let mut command = format!("{} {tip} refs/heads/main", "0".repeat(tip.len())).into_bytes(); + command.push(0); + command.extend_from_slice(b"report-status side-band-64k agent=knot-h3-test/0"); + command.push(b'\n'); + let mut body = common::pkt(&command); + body.extend_from_slice(b"0000"); + body.extend_from_slice(pack); + Bytes::from(body) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn git_push_over_h3_authenticates_and_lands_the_ref() { + let scratch = tempfile::tempdir().unwrap(); + let (_key_path, public_line) = keygen(scratch.path()); + let world = spawn(public_line, true).await; + let edge = world.h3.as_ref().expect("the h3 edge is stood up"); + + let path_env = std::env::var("PATH").unwrap_or_default(); + let home = scratch.path().to_str().unwrap().to_string(); + let env: Vec<(String, String)> = [("PATH", &path_env), ("HOME", &home)] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + let (ok, out) = git(&work, &env, &["init", "-q", "-b", "main"]); + assert!(ok, "init failed:\n{out}"); + std::fs::write(work.join("README.md"), "hello over http3\n").unwrap(); + let (ok, out) = git(&work, &env, &["add", "-A"]); + assert!(ok, "add failed:\n{out}"); + let (ok, out) = git(&work, &env, &["commit", "-q", "-m", "init over http3"]); + assert!(ok, "commit failed:\n{out}"); + let (ok, tip) = git(&work, &env, &["rev-parse", "HEAD"]); + assert!(ok, "rev-parse failed:\n{tip}"); + let tip = tip.trim().to_string(); + let pack = build_pack(&work, &env, &tip); + + let advert_uri = format!( + "https://localhost/{}/{REPO_NAME}/info/refs?service=git-receive-pack", + owner() + ); + let receive_uri = format!("https://localhost/{}/{REPO_NAME}/git-receive-pack", owner()); + let warmup = format!( + "https://localhost/{}/{REPO_NAME}/info/refs?service=git-upload-pack", + owner() + ); + const RECEIVE_CT: &str = "application/x-git-receive-pack-request"; + + let (status, _) = + common::h3_request(edge, Method::GET, advert_uri.clone(), &[], None, None).await; + assert_eq!( + status, + http::StatusCode::UNAUTHORIZED, + "an unauthenticated receive advertisement must be challenged over h3" + ); + + let token = basic_auth(&service_jwt( + &nsid("org.tangled.git.receivePack"), + "git-h3-adv-1", + )); + let (status, advert) = common::h3_request( + edge, + Method::GET, + advert_uri, + &[("authorization", token.as_str())], + None, + None, + ) + .await; + assert_eq!( + status, + http::StatusCode::OK, + "an authenticated receive advertisement is served over h3" + ); + assert!( + String::from_utf8_lossy(&advert).contains("# service=git-receive-pack"), + "the h3 receive advertisement includes the service banner" + ); + + let body = build_receive_body(&tip, &pack); + let (status, _) = common::h3_request( + edge, + Method::POST, + receive_uri.clone(), + &[("content-type", RECEIVE_CT)], + Some(body.clone()), + Some(warmup.as_str()), + ) + .await; + assert_eq!( + status, + http::StatusCode::UNAUTHORIZED, + "an unauthenticated receive-pack post must be challenged over h3" + ); + + let wrong = basic_auth(&service_jwt( + &nsid("org.tangled.repo.createRepo"), + "git-h3-neg", + )); + let (status, _) = common::h3_request( + edge, + Method::POST, + receive_uri.clone(), + &[ + ("content-type", RECEIVE_CT), + ("authorization", wrong.as_str()), + ], + Some(body.clone()), + Some(warmup.as_str()), + ) + .await; + assert_eq!( + status, + http::StatusCode::UNAUTHORIZED, + "a token bound to another method cannot authorize a receive-pack over h3" + ); + + let good = basic_auth(&service_jwt( + &nsid("org.tangled.git.receivePack"), + "git-h3-push-1", + )); + let (status, report) = common::h3_request( + edge, + Method::POST, + receive_uri, + &[ + ("content-type", RECEIVE_CT), + ("authorization", good.as_str()), + ], + Some(body), + Some(warmup.as_str()), + ) + .await; + assert_eq!(status, http::StatusCode::OK, "the authenticated h3 push"); + let report = String::from_utf8_lossy(&report); + assert!( + report.contains("unpack ok"), + "the pack must unpack cleanly over h3:\n{report}" + ); + assert!( + report.contains("ok refs/heads/main"), + "the ref update must be accepted over h3:\n{report}" + ); + + let repo = world.layout.open(&repo_did()).unwrap(); + let landed = repo.references().unwrap().into_iter().any(|record| { + record.name.as_str() == "refs/heads/main" && record.target.to_string() == tip + }); + assert!( + landed, + "the ref pushed over h3 must be durable in the bare repo" + ); +} diff --git a/knot2/crates/knot-sim/tests/reproducible.rs b/knot2/crates/knot-sim/tests/reproducible.rs new file mode 100644 index 0000000000000000000000000000000000000000..4e80820d4621fda3215ee62ceab7248855e8e10d --- /dev/null +++ b/knot2/crates/knot-sim/tests/reproducible.rs @@ -0,0 +1,294 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use futures::future::join_all; +use futures::stream::StreamExt; +use knot_sim::{Outcome, Step, Trace}; + +fn status_of(step: &Step) -> Option { + match step.outcome { + Outcome::Served { status, .. } => Some(status.get()), + Outcome::Killed => None, + } +} + +fn union_has(traces: &[Trace], predicate: impl Fn(&Step) -> bool) -> bool { + traces + .iter() + .any(|trace| trace.steps.iter().any(&predicate)) +} + +const READ_OPS: [&str; 12] = [ + "version", + "describeKnot", + "listMembers", + "listCollaborators", + "didJson", + "infoRefs", + "branches", + "log", + "describeRepo", + "tree", + "blob", + "languages", +]; + +fn every_step_with<'a>( + traces: &'a [Trace], + fault: &'a str, + holds: impl Fn(&Step) -> bool + 'a, +) -> (bool, usize) { + let matching: Vec<&Step> = traces + .iter() + .flat_map(|trace| trace.steps.iter()) + .filter(|step| step.fault == fault) + .collect(); + (matching.iter().all(|step| holds(step)), matching.len()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_same_seed_produces_an_identical_whole_system_trace() { + futures::stream::iter([1u64, 7, 42, 100, 2026]) + .for_each(|seed| async move { + let first = knot_sim::run(seed, 16).await; + let second = knot_sim::run(seed, 16).await; + assert_eq!( + first, second, + "seed {seed} must replay to an identical trace" + ); + assert_eq!( + first.digest(), + second.digest(), + "seed {seed} digest is stable" + ); + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_execution_stays_stable_across_fifty_repeats() { + let baseline = knot_sim::run(7, 18).await.digest(); + let digests: Vec = futures::stream::iter(0..50) + .map(|_| knot_sim::run(7, 18)) + .buffer_unordered(8) + .map(|trace| trace.digest()) + .collect() + .await; + assert!( + digests.iter().all(|digest| *digest == baseline), + "4-thread schedule leaked into trace: {} of 50 replays diverged", + digests.iter().filter(|digest| **digest != baseline).count() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn distinct_seeds_produce_distinct_traces() { + let digests: Vec = join_all((0u64..8).map(|seed| knot_sim::run(seed, 16))) + .await + .iter() + .map(Trace::digest) + .collect(); + let unique: BTreeSet = digests.iter().copied().collect(); + assert_eq!( + unique.len(), + digests.len(), + "every seed must drive different whole-system trace: {digests:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_round_count_extends_the_same_prefix() { + let short = knot_sim::run(7, 10).await; + let long = knot_sim::run(7, 20).await; + assert_ne!(short.digest(), long.digest()); + assert_eq!( + short.steps, + long.steps[..short.steps.len()], + "longer run extends the same prefix the shorter run produced" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_injected_failures_are_correlated_with_their_observable_outcomes() { + let traces = join_all([1u64, 7, 13, 42, 99, 2026].map(|seed| knot_sim::run(seed, 16))).await; + + let (skew_all_401, skew_count) = + every_step_with(&traces, "clock_skew", |step| status_of(step) == Some(401)); + assert!(skew_count > 0, "clock-skew fault must actually fire"); + assert!( + skew_all_401, + "every clock-skewed token must produce 401 expiry rejection, not just one example" + ); + + let (drop_all_503, drop_count) = every_step_with(&traces, "drop_identity", |step| { + status_of(step) == Some(503) + }); + assert!(drop_count > 0, "drop-identity fault must actually fire"); + assert!( + drop_all_503, + "every dropped identity resolution must surface as 503 upstream-unavailable failure" + ); + + let (killed_all_killed, killed_count) = every_step_with(&traces, "killed", |step| { + matches!(step.outcome, Outcome::Killed) && READ_OPS.contains(&step.op) + }); + assert!(killed_count > 0, "kill fault must actually fire"); + assert!( + killed_all_killed, + "killed connection must record a Killed outcome and is only injected on read paths" + ); + + assert!( + union_has(&traces, |step| step.op == "probe" + && step.fault == "none" + && status_of(step) == Some(403)), + "resolved stranger with no fault must be denied 403 by access-control layer" + ); + + [ + "addMember", + "addCollaborator", + "createRepo", + "maintain", + "describeRepo", + "listMembers", + "listCollaborators", + "infoRefs", + "didJson", + "branches", + "log", + "tree", + "blob", + "languages", + ] + .iter() + .for_each(|op| { + assert!( + union_has(&traces, |step| step.op == *op + && status_of(step) == Some(200)), + "{op} path must succeed against the doubles in at least one seed" + ); + }); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_final_projection_matches_an_independent_model() { + let seeds = [1u64, 7, 13, 42, 99, 2026]; + let predicted: Vec = seeds + .iter() + .map(|seed| knot_sim::predict(*seed, 18)) + .collect(); + let runs = join_all(seeds.map(|seed| knot_sim::run(seed, 18))).await; + + seeds + .iter() + .zip(predicted.iter()) + .zip(runs.iter()) + .for_each(|((seed, model), trace)| { + let last = trace.snapshots.last().expect("at least one round"); + assert_eq!( + last.members, model.members, + "seed {seed}: executed member projection must equal the independent model" + ); + assert_eq!( + last.blocked, model.blocked, + "seed {seed}: executed blocklist projection must equal the independent model" + ); + assert_eq!( + last.collaborators + .iter() + .map(|roster| &roster.subjects) + .collect::>(), + model.collaborators.iter().collect::>(), + "seed {seed}: the run must host the repositories the model counts and open each \ + one to exactly the collaborators it predicts" + ); + }); + + assert!( + predicted.iter().any(|model| !model.members.is_empty()), + "oracle is vacuous unless at least one seed predicts a non-empty member set" + ); + assert!( + predicted.iter().any(|model| !model.blocked.is_empty()), + "oracle is vacuous unless at least one seed predicts a non-empty blocklist" + ); + assert!( + predicted + .iter() + .any(|model| model.collaborators.iter().any(|repo| !repo.is_empty())), + "oracle is vacuous unless at least one seed predicts a repo with a collaborator" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn maintenance_never_reports_a_failure() { + let traces = join_all([1u64, 7, 13, 42, 99, 2026].map(|seed| knot_sim::run(seed, 16))).await; + let failures = traces + .iter() + .flat_map(|trace| trace.steps.iter()) + .filter(|step| step.op == "maintain" && status_of(step) == Some(500)) + .count(); + assert_eq!( + failures, 0, + "maintenance must succeed on every repo simulation runs it against" + ); + assert!( + union_has(&traces, |step| step.op == "maintain" + && status_of(step) == Some(200)), + "maintenance path must actually run against a populated repo in at least one seed" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_final_projection_state_is_seed_stable() { + let first = knot_sim::run(2026, 16).await; + let second = knot_sim::run(2026, 16).await; + assert_eq!( + first.snapshots, second.snapshots, + "order-independent COB projections must converge to the same logical state" + ); + let last = first.snapshots.last().expect("at least one round"); + assert!( + last.repos.len() > 1, + "simulation must have minted repos beyond the seed repo: {:?}", + last.repos + ); +} + +const OFFERS: [&str; 2] = ["addMember", "addCollaborator"]; +const ROSTER_READS: [&str; 2] = ["listMembers", "listCollaborators"]; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn roster_read_and_roster_write_land_in_separate_rounds() { + let traces = join_all([1u64, 7, 13, 42, 99, 2026].map(|seed| knot_sim::run(seed, 18))).await; + let rounds: BTreeMap<(u64, u32), BTreeSet<&str>> = traces + .iter() + .flat_map(|trace| { + trace + .steps + .iter() + .map(move |step| ((trace.seed, step.round.get()), step.op)) + }) + .fold(BTreeMap::new(), |mut rounds, (round, op)| { + rounds.entry(round).or_default().insert(op); + rounds + }); + let holds = |ops: &BTreeSet<&str>, names: &[&str]| names.iter().any(|name| ops.contains(name)); + let writes = |ops: &BTreeSet<&str>| holds(ops, &OFFERS) || ops.contains("removeMember"); + + rounds.iter().for_each(|((seed, round), ops)| { + assert!( + !(writes(ops) && holds(ops, &ROSTER_READS)), + "seed {seed} round {round}: a roster listing ran beside a roster write" + ); + }); + + [OFFERS.as_slice(), ROSTER_READS.as_slice()] + .into_iter() + .for_each(|names| { + assert!( + rounds.values().any(|ops| holds(ops, names)), + "the rule is vacuous unless {names:?} run in some round" + ); + }); +} diff --git a/knot2/crates/knot-sim/tests/social_chaos.rs b/knot2/crates/knot-sim/tests/social_chaos.rs new file mode 100644 index 0000000000000000000000000000000000000000..9cdfb33459762ec76e4e08dd1d9f7d1bedf694d5 --- /dev/null +++ b/knot2/crates/knot-sim/tests/social_chaos.rs @@ -0,0 +1,249 @@ +mod common; + +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Instant; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Registration, RegistryChange}; +use knot_git::{Layout, Repo}; +use knot_index::{Coverage, Index, Resolved}; +use knot_record::chain; +use knot_runtime::OsEntropy; +use knot_secrets::{MasterKey, SealedStore}; +use knot_sim::write_pull; +use knot_types::{ + KnotId, ObjectFormat, OwnerDid, RepoDid, RepoName, RepoRkey, TypeName, UnixSeconds, +}; + +const KNOT_HOST: &str = "knot.nel.pet"; +const ADMIN_DID: &str = "did:web:admin.nel.pet"; +const OWNER_DID: &str = "did:plc:nel"; +const REPO_DID: &str = "did:plc:squid"; +const REPO_NAME: &str = "anemone"; +const MASTER_KEY: [u8; 32] = [7u8; 32]; +const SEED: u64 = 1; +const ROUNDS: u32 = 12; + +fn knot() -> KnotId { + KnotId::new(format!("did:web:{KNOT_HOST}")).unwrap() +} + +fn did() -> RepoDid { + RepoDid::new(REPO_DID).unwrap() +} + +fn master() -> MasterKey { + MasterKey::new(MASTER_KEY).unwrap() +} + +fn build_target(root: &Path) { + let repos = root.join("repos"); + std::fs::create_dir_all(&repos).unwrap(); + let layout = Layout::new(&repos) + .with_object_format(ObjectFormat::SHA1) + .reserving_meta(&knot()) + .unwrap(); + layout.bootstrap_meta(&knot()).unwrap(); + let meta_path = layout.meta_path(&knot()).unwrap(); + + let secrets = + SealedStore::open(root.join("sealed-keys"), &master(), Box::new(OsEntropy)).unwrap(); + secrets.ensure(&knot()).unwrap(); + let signer = secrets.signer(&knot()).unwrap(); + + let repo = layout.create(&did()).unwrap(); + write_pull(&repo, &did()); + let mut ticker = jacquard_common::types::tid::Ticker::new(); + futures::executor::block_on(chain::initialize( + &repo, + &did(), + &signer, + &mut ticker, + UnixSeconds::new(1), + )) + .unwrap(); + + let meta = Repo::open(&meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .create( + &CobHome::from(&knot()), + &RegistryChange::Register(Registration { + owner: OwnerDid::new(OWNER_DID).unwrap(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: RepoName::new(REPO_NAME).unwrap(), + repo: did(), + created_at: UnixSeconds::new(1), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + + let config = format!( + "[server]\nhostname = \"{KNOT_HOST}\"\nadmins = [\"{ADMIN_DID}\"]\nssh_host_key_file = \"{keyfile}\"\n\n[repo]\nscan_path = \"{repos}\"\n\n[lfs]\nstore_path = \"{lfs}\"\n\n[secrets]\nsealed_key_file = \"{sealed}\"\nmaster_key_env = \"KNOT_MASTER_KEY\"\n\n[atproto]\nplc_directory = \"https://plc.directory\"\n\n[contact]\nemail = \"knot@{KNOT_HOST}\"\n\n[git]\nobject_format = \"sha1\"\n", + keyfile = root.join("ssh_host_key").display(), + repos = root.join("repos").display(), + lfs = root.join("lfs").display(), + sealed = root.join("sealed-keys").display(), + ); + std::fs::create_dir_all(root.join("lfs")).unwrap(); + std::fs::write(root.join("config.toml"), config).unwrap(); +} + +fn spawn_worker(target: &Path) -> std::process::Child { + let seed = SEED.to_string(); + let rounds = ROUNDS.to_string(); + let master_key = STANDARD.encode(MASTER_KEY); + common::spawn_worker( + "social_chaos_worker", + &[ + ("KNOT_CHAOS_ROLE", "social"), + ("KNOT_CHAOS_TARGET", target.to_str().unwrap()), + ("KNOT_CHAOS_SEED", &seed), + ("KNOT_CHAOS_ROUNDS", &rounds), + ("KNOT_MASTER_KEY", &master_key), + ], + ) +} + +fn boot_target(target: &Path) { + let master_key = STANDARD.encode(MASTER_KEY); + let status = Command::new(env!("CARGO_BIN_EXE_knot-sim")) + .args(["--target", target.to_str().unwrap(), "--in-place", "1", "0"]) + .env("KNOT_MASTER_KEY", master_key) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .expect("spawn boot") + .wait() + .expect("boot process reap"); + assert!(status.success(), "boot command must exit clean"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn social_chaos_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("social") { + return; + } + let target = std::env::var("KNOT_CHAOS_TARGET").unwrap(); + let seed: u64 = std::env::var("KNOT_CHAOS_SEED").unwrap().parse().unwrap(); + let rounds: u32 = std::env::var("KNOT_CHAOS_ROUNDS").unwrap().parse().unwrap(); + knot_sim::run_realdata(Path::new(&target), seed, rounds) + .await + .unwrap(); +} + +fn serving_records(index: &Index, layout: &Layout, repo: &RepoDid, when: &str) -> usize { + index.refresh_social(repo).unwrap(); + let entries = match index.social_entries(repo) { + Resolved::Ready(entries) => entries, + Resolved::Warming => panic!("social projection must be ready after rebuild"), + }; + let git = layout.open(repo).unwrap(); + let store = CobStore::new(&git); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .build() + .unwrap(); + let leaves = runtime.block_on(chain::leaves(&git)).unwrap(); + entries + .iter() + .map(|entry| { + let projection = entry.state.serving().map(|version| version.cid()); + let chain_leaf = leaves.get(&entry.address.mst_key()).copied(); + let type_name = TypeName::new(entry.address.collection().as_str()).unwrap(); + let replayed = knot_cobs::social_state(&store, &type_name, entry.object) + .unwrap() + .and_then(|(state, _)| { + state + .object() + .and_then(|object| object.version()) + .map(|version| version.cid()) + }); + assert_eq!( + projection, + chain_leaf, + "{when}: projection and chain leaves differ for {}", + entry.address.at_uri(repo) + ); + assert_eq!( + projection, + replayed, + "{when}: projection and a fresh replay differ for {}", + entry.address.at_uri(repo) + ); + projection.is_some() as usize + }) + .sum() +} + +#[test] +fn social_chaos() { + let scratch = tempfile::tempdir().unwrap(); + + let warm = scratch.path().join("warm"); + build_target(&warm); + let started = Instant::now(); + spawn_worker(&warm).wait().unwrap(); + let full = started.elapsed(); + + let warm_layout = Layout::new(warm.join("repos")); + let warm_meta = warm_layout.meta_path(&knot()).unwrap(); + let warm_index = Index::new(warm_meta, warm_layout.clone()); + warm_index.rebuild().unwrap(); + let full_records: usize = warm_index + .hosted_repos() + .iter() + .map(|repo| serving_records(&warm_index, &warm_layout, repo, "warm")) + .sum(); + + let delays = common::kill_delays(full); + + let outcomes: Vec = delays + .iter() + .enumerate() + .map(|(trial, delay)| { + let target = scratch.path().join(format!("trial-{trial}")); + build_target(&target); + let mut child = spawn_worker(&target); + std::thread::sleep(*delay); + let _ = child.kill(); + child.wait().unwrap(); + boot_target(&target); + + let layout = Layout::new(target.join("repos")); + let meta = layout.meta_path(&knot()).unwrap(); + let index = Index::new(meta, layout.clone()); + index + .rebuild() + .unwrap_or_else(|error| panic!("trial {trial}: index must rebuild: {error}")); + assert_eq!( + index.coverage().registry, + Coverage::Ready, + "trial {trial}: a rebuilt projection is ready" + ); + let trial_tag = format!("trial {trial}"); + index + .hosted_repos() + .iter() + .map(|repo| serving_records(&index, &layout, repo, &trial_tag)) + .sum() + }) + .collect(); + + assert!( + outcomes.iter().any(|records| *records < full_records), + "some trial must be killed before every social write is stored, or the kill delays are all too long" + ); + assert!( + outcomes.contains(&full_records), + "some trial must reach the full social write set, or the kill delays are all too short" + ); +} diff --git a/knot2/crates/knot-sim/tests/ssh.rs b/knot2/crates/knot-sim/tests/ssh.rs new file mode 100644 index 0000000000000000000000000000000000000000..1e2a901b8bca86a4ccd43e821ed2ed4611920c04 --- /dev/null +++ b/knot2/crates/knot-sim/tests/ssh.rs @@ -0,0 +1,302 @@ +use std::path::Path; +use std::process::Command; +use std::sync::Arc; + +use knot_atproto::Atproto; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Registration, RegistryChange}; +use knot_git::{Layout, Repo}; +use knot_runtime::{ + FakeHttp, HttpResponse, K256Signer, ManualClock, SeededEntropy, Signer, UnixMicros, +}; +use knot_types::{Did, KnotId, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds}; +use tempfile::TempDir; +use tokio::net::TcpListener; +use url::Url; + +const REPO_DID: &str = "did:plc:squid"; +const REPO_NAME: &str = "anemone"; +const OWNER_DID: &str = "did:plc:nel"; +const PDS_HOST: &str = "pds.oyster.cafe"; +const KNOT_DID: &str = "did:web:nel.pet"; +const PINNED_DATE: &str = "2026-06-20T12:00:00+00:00"; + +fn repo_did() -> RepoDid { + RepoDid::new(REPO_DID).unwrap() +} + +fn owner_did() -> OwnerDid { + OwnerDid::new(OWNER_DID).unwrap() +} + +fn knot_id() -> KnotId { + KnotId::new(KNOT_DID).unwrap() +} + +fn git(cwd: &Path, env: &[(&str, &str)], args: &[&str]) -> (bool, String) { + let mut command = knot_fixtures::command_at(cwd, PINNED_DATE); + command.args(args); + env.iter().for_each(|(key, value)| { + command.env(key, value); + }); + let out = command.output().expect("git runs"); + ( + out.status.success(), + format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ), + ) +} + +fn keygen(dir: &Path) -> (String, String) { + let path = dir.join("client"); + let out = Command::new("ssh-keygen") + .args([ + "-t", + "ed25519", + "-N", + "", + "-C", + "nel@oyster.cafe", + "-f", + path.to_str().unwrap(), + ]) + .output() + .expect("ssh-keygen runs"); + assert!(out.status.success()); + let public_line = std::fs::read_to_string(dir.join("client.pub")) + .unwrap() + .trim() + .to_string(); + (path.to_str().unwrap().to_string(), public_line) +} + +fn did_document(signer: &K256Signer, did: &Did) -> Vec { + let multikey = knot_types::crypto::multikey(0xe7, signer.public_key().as_bytes()); + serde_json::to_vec(&serde_json::json!({ + "id": did.as_str(), + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [{ + "id": format!("{}#atproto", did.as_str()), + "type": "Multikey", + "controller": did.as_str(), + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": format!("https://{PDS_HOST}") + }] + })) + .unwrap() +} + +fn list_records_body(public_line: &str) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "records": [{ + "uri": format!("at://{}/org.tangled.key.sshKey/1", owner_did()), + "value": { + "$type": "org.tangled.key.sshKey", + "publicKey": public_line, + "name": "laptop", + "createdAt": "2026-06-08T00:00:00Z" + } + }] + })) + .unwrap() +} + +fn fake_http(published_line: String) -> impl knot_runtime::HttpTransport { + let signer = K256Signer::generate(&SeededEntropy::new(1)); + let owner = Did::new_owned(OWNER_DID).unwrap(); + let repo = Did::new_owned(REPO_DID).unwrap(); + FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap_or_default().to_string(); + let path = request.url.path().to_string(); + let body = if host == PDS_HOST { + list_records_body(&published_line) + } else if path.ends_with(owner.as_str()) { + did_document(&signer, &owner) + } else if path.ends_with(repo.as_str()) { + did_document(&signer, &repo) + } else { + return Ok(HttpResponse { + status: http::StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + }); + }; + Ok(HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from(body), + }) + }) +} + +fn actor_for_seed(seed: u64) -> knot_types::ActorId { + knot_types::ActorId::from_secp256k1( + K256Signer::generate(&SeededEntropy::new(seed)) + .public_key() + .as_bytes(), + ) +} + +struct Server { + _scan: TempDir, + layout: Layout, + repo_did: RepoDid, + port: u16, +} + +async fn spawn(published_line: String) -> Server { + let scan = tempfile::tempdir().unwrap(); + let meta_path = scan.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scan.path().join("repos")); + let repo_did = repo_did(); + layout.create(&repo_did).unwrap(); + + let signer = K256Signer::generate(&SeededEntropy::new(2)); + let meta = Repo::open(&meta_path).unwrap(); + CobStore::new(&meta) + .create( + &CobHome::from(&knot_id()), + &RegistryChange::Register(Registration { + owner: owner_did(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: RepoName::new(REPO_NAME).unwrap(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(1), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + + let index = Arc::new(knot_index::Index::new(meta_path, layout.clone())); + index.rebuild().unwrap(); + + let atproto = Arc::new(Atproto::new( + fake_http(published_line), + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot_id(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + )); + let key_dir = scan.path().join("hostkey"); + std::fs::create_dir_all(&key_dir).unwrap(); + let host_key = knot_ssh::load_or_create_host_key(&key_dir.join("host")).unwrap(); + let lfs = { + let lfs_dir = scan.path().join("lfs"); + std::fs::create_dir_all(&lfs_dir).unwrap(); + knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(&lfs_dir), + knot_lfs::LfsSize::new(64 * 1024 * 1024), + knot_lfs::FreeSpaceFloor::new(0), + ) + .unwrap() + }; + let state = Arc::new(knot_ssh::SshState::new(knot_ssh::SshConfig { + layout: layout.clone(), + index, + atproto, + knot_actor: actor_for_seed(1), + hostname: knot_types::KnotHostname::new("knot.test").unwrap(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + admins: std::collections::BTreeSet::new(), + admission: knot_types::AdmissionPolicy::Closed, + max_pack_bytes: knot_xrpc::MaxWireBytes::new(1 << 30), + archive_limit: knot_git::ArchiveLimit::default(), + lfs, + lfs_max_transfers: 2, + max_inflight_per_peer: 4, + })); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let _ = knot_ssh::serve_on_socket(listener, host_key, state).await; + }); + Server { + _scan: scan, + layout, + repo_did, + port, + } +} + +fn ssh_command(key_path: &str) -> String { + format!("ssh -i {key_path} {}", knot_fixtures::UNSHARED_SSH) +} + +fn seed_work(work: &Path) -> String { + std::fs::create_dir_all(work).unwrap(); + git(work, &[], &["init", "-q", "-b", "main"]); + std::fs::write(work.join("README.md"), "hello over the simulated ssh\n").unwrap(); + git(work, &[], &["add", "-A"]); + git(work, &[], &["commit", "-q", "-m", "initial"]); + let (ok, head) = git(work, &[], &["rev-parse", "HEAD"]); + assert!(ok); + head.trim().to_string() +} + +async fn push_once(scratch: &Path) -> (String, Option) { + let (key_path, public_line) = keygen(scratch); + let server = spawn(public_line).await; + let url = format!( + "ssh://git@127.0.0.1:{}/{}/{REPO_NAME}", + server.port, + owner_did() + ); + let ssh = ssh_command(&key_path); + let work = scratch.join("work"); + let head = seed_work(&work); + + let (ok, out) = tokio::task::spawn_blocking(move || { + git( + &work, + &[("GIT_SSH_COMMAND", &ssh)], + &["push", "-q", &url, "main"], + ) + }) + .await + .unwrap(); + assert!(ok, "simulated ssh server must accept push:\n{out}"); + + let stored = server + .layout + .open(&server.repo_did) + .unwrap() + .find_ref(&knot_types::RefName::new("refs/heads/main").unwrap()) + .unwrap(); + drop(server); + (head, stored) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_simulated_ssh_write_path_lands_a_seed_deterministic_tip() { + let first_dir = tempfile::tempdir().unwrap(); + let (first_head, first_stored) = push_once(first_dir.path()).await; + + let second_dir = tempfile::tempdir().unwrap(); + let (second_head, second_stored) = push_once(second_dir.path()).await; + + let tip = knot_types::Oid::from_hex(&first_head).unwrap(); + assert_eq!( + first_stored, + Some(tip), + "pushed commit must be the repository's main tip" + ); + assert_eq!( + first_head, second_head, + "two independent runs of the simulated ssh push must produce same commit oid" + ); + assert_eq!( + first_stored, second_stored, + "assembled-against-doubles ssh write path is logically reproducible" + ); +} diff --git a/knot2/crates/knot-ssh/Cargo.toml b/knot2/crates/knot-ssh/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..a78a0f1ed16f126f46317ef63a02c59659e8ca68 --- /dev/null +++ b/knot2/crates/knot-ssh/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "knot-ssh" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-runtime = { workspace = true } +knot-resource = { workspace = true } +knot-git = { workspace = true } +knot-lfs = { workspace = true } +knot-pack = { workspace = true } +knot-index = { workspace = true } +knot-acl = { workspace = true } +knot-atproto = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-maintenance = { workspace = true } +knot-postreceive = { workspace = true } +knot-receive = { workspace = true } +knot-messages = { workspace = true } +tracing = { workspace = true } +russh = "0.61" +tokio = { workspace = true } +tokio-util = { workspace = true, features = ["io-util"] } +futures = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +knot-fixtures = { workspace = true } +tempfile = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +http = { workspace = true } +bytes = { workspace = true } +url = { workspace = true } +russh = "0.61" +knot-events = { workspace = true } +serde = { workspace = true } +knot-secrets = { workspace = true } +knot-xrpc = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +tikv-jemallocator = { workspace = true } +tikv-jemalloc-ctl = { workspace = true } diff --git a/knot2/crates/knot-ssh/examples/ephemeral_knot.rs b/knot2/crates/knot-ssh/examples/ephemeral_knot.rs new file mode 100644 index 0000000000000000000000000000000000000000..168484167391ce2895a0707128b63e670b47eccb --- /dev/null +++ b/knot2/crates/knot-ssh/examples/ephemeral_knot.rs @@ -0,0 +1,255 @@ +use std::collections::BTreeSet; +use std::sync::Arc; +use std::time::Duration; + +use knot_atproto::Atproto; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Registration, RegistryChange}; +use knot_git::{Layout, Repo}; +use knot_index::Index; +use knot_runtime::{ + FakeHttp, HttpResponse, K256Signer, ManualClock, SeededEntropy, Signer, UnixMicros, +}; +use knot_types::{ + ActorId, AdmissionPolicy, Did, KnotHostname, KnotId, KnotServiceUrl, OwnerDid, RepoDid, + RepoName, RepoRkey, UnixSeconds, +}; +use tokio::net::TcpListener; +use url::Url; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +const REPO_DID: &str = "did:plc:squid"; +const REPO_NAME: &str = "anemone"; +const OWNER_DID: &str = "did:plc:nel"; +const PDS_HOST: &str = "pds.oyster.cafe"; + +fn apply_decay(ms: isize) { + unsafe { + let _ = tikv_jemalloc_ctl::raw::write(b"arenas.dirty_decay_ms\0", ms); + if let Ok(narenas) = tikv_jemalloc_ctl::raw::read::(b"arenas.narenas\0") { + (0..narenas).for_each(|arena| { + let name = format!("arena.{arena}.dirty_decay_ms\0"); + let _ = tikv_jemalloc_ctl::raw::write(name.as_bytes(), ms); + }); + } + } +} + +async fn govern_decay() { + unsafe { + let _ = tikv_jemalloc_ctl::raw::write(b"background_thread\0", true); + } + let mut ticker = tokio::time::interval(Duration::from_secs(1)); + let mut applied = knot_resource::target_decay(); + apply_decay(applied.ms()); + loop { + ticker.tick().await; + let target = knot_resource::target_decay(); + if target != applied { + apply_decay(target.ms()); + applied = target; + } + } +} + +fn vm_hwm_mib() -> u64 { + std::fs::read_to_string("/proc/self/status") + .unwrap_or_default() + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|kb| kb.parse::().ok()) + .map(|kb| kb / 1024) + .unwrap_or(0) +} + +fn did_document(signer: &K256Signer, did: &Did, pds: &KnotServiceUrl) -> Vec { + let multikey = ActorId::new(knot_types::crypto::multikey( + 0xe7, + signer.public_key().as_bytes(), + )) + .unwrap(); + serde_json::to_vec(&serde_json::json!({ + "id": did.as_str(), + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [{ + "id": format!("{}#atproto", did.as_str()), + "type": "Multikey", + "controller": did.as_str(), + "publicKeyMultibase": multikey.as_str() + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": pds.as_str() + }] + })) + .unwrap() +} + +fn list_records_body(published_line: &str) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "records": [{ + "value": { + "$type": "org.tangled.key.sshKey", + "publicKey": published_line, + "name": "laptop", + "createdAt": "2026-06-08T00:00:00Z" + } + }] + })) + .unwrap() +} + +fn ok_body(body: Vec) -> HttpResponse { + HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from(body), + } +} + +fn not_found() -> HttpResponse { + HttpResponse { + status: http::StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + } +} + +fn fake_http(published_line: String) -> impl knot_runtime::HttpTransport { + let signer = K256Signer::generate(&SeededEntropy::new(1)); + let pds = KnotServiceUrl::new(format!("https://{PDS_HOST}")).unwrap(); + FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap_or_default().to_string(); + let path = request.url.path().to_string(); + let body = if host == PDS_HOST { + list_records_body(&published_line) + } else if path.ends_with(REPO_DID) { + did_document(&signer, &Did::new_owned(REPO_DID).unwrap(), &pds) + } else if path.ends_with(OWNER_DID) { + did_document(&signer, &Did::new_owned(OWNER_DID).unwrap(), &pds) + } else { + return Ok(not_found()); + }; + Ok(ok_body(body)) + }) +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + let mut args = std::env::args().skip(1); + let port: u16 = args + .next() + .expect("usage: ephemeral_knot ") + .parse() + .expect("port"); + let pubkey_path = args.next().expect("client-pubkey-file"); + let published_line = std::fs::read_to_string(&pubkey_path) + .expect("read client pubkey") + .trim() + .to_string(); + + let max_threads = std::env::var("KNOT_MAX_THREADS") + .ok() + .and_then(|value| value.parse::().ok()); + knot_resource::init(knot_resource::Ceilings { + max_threads: max_threads.map(knot_resource::ThreadCount::new), + max_memory: None, + }); + + let scan = tempfile::tempdir().expect("scan tempdir"); + let meta_path = scan.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scan.path().join("repos")); + let repo_did = RepoDid::new(REPO_DID).unwrap(); + layout.create(&repo_did).unwrap(); + + let signer = K256Signer::generate(&SeededEntropy::new(2)); + let meta = Repo::open(&meta_path).unwrap(); + CobStore::new(&meta) + .create( + &CobHome::from(&KnotId::new("did:web:nel.pet").unwrap()), + &RegistryChange::Register(Registration { + owner: OwnerDid::new(OWNER_DID).unwrap(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: RepoName::new(REPO_NAME).unwrap(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(1), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + + retired_at: None, + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + + let index = Arc::new(Index::new(meta_path, layout.clone())); + index.rebuild().unwrap(); + + let atproto = Arc::new(Atproto::new( + fake_http(published_line), + ManualClock::new(UnixMicros::new(1_000_000_000)), + KnotId::new("did:web:nel.pet").unwrap(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + )); + + let key_dir = scan.path().join("hostkey"); + std::fs::create_dir_all(&key_dir).unwrap(); + let host_key = knot_ssh::load_or_create_host_key(&key_dir.join("host")).unwrap(); + + let actor = knot_types::ActorId::from_secp256k1( + K256Signer::generate(&SeededEntropy::new(1)) + .public_key() + .as_bytes(), + ); + let lfs = { + let lfs_dir = scan.path().join("lfs"); + std::fs::create_dir_all(&lfs_dir).unwrap(); + knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(&lfs_dir), + knot_lfs::LfsSize::new(u64::MAX), + knot_lfs::FreeSpaceFloor::new(0), + ) + .unwrap() + }; + let state = Arc::new(knot_ssh::SshState::new(knot_ssh::SshConfig { + layout, + index, + atproto, + knot_actor: actor, + hostname: KnotHostname::new("knot.test").unwrap(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + admins: BTreeSet::new(), + admission: AdmissionPolicy::Closed, + max_pack_bytes: knot_pack::MaxWireBytes::new(1 << 34), + archive_limit: knot_git::ArchiveLimit::default(), + lfs, + lfs_max_transfers: 16, + max_inflight_per_peer: 4, + })); + + let listener = TcpListener::bind(("127.0.0.1", port)).await.unwrap(); + let bound = listener.local_addr().unwrap().port(); + tokio::spawn(govern_decay()); + tokio::spawn(async move { + let _ = knot_ssh::serve_on_socket(listener, host_key, state).await; + }); + + println!("READY pid={} port={}", std::process::id(), bound); + let mut reporter = tokio::time::interval(Duration::from_secs(1)); + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + () = async { + loop { + reporter.tick().await; + println!("VmHWM {} MiB", vm_hwm_mib()); + } + } => {} + } + drop(scan); +} diff --git a/knot2/crates/knot-ssh/src/exec.rs b/knot2/crates/knot-ssh/src/exec.rs new file mode 100644 index 0000000000000000000000000000000000000000..2bc1ba63489de68ddb43c82ae5ece5ddfe553364 --- /dev/null +++ b/knot2/crates/knot-ssh/src/exec.rs @@ -0,0 +1,1100 @@ +use std::net::IpAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use futures::StreamExt; +use knot_acl::{KnotAcl, can_push}; +use knot_index::Resolved; +use knot_lfs::TransferOp; +use knot_pack::{PackError, PackLimits, RepoLookup}; +use knot_resource::SubjectKey; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, ClonePath, ObjectFormat, OwnerDid, RepoDid}; +use russh::Channel; +use russh::server::Msg; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::runtime::Handle; +use tokio::sync::mpsc; + +use crate::SshState; +use crate::identity::Credential; + +const READ_CHUNK: usize = 64 * 1024; +const MAX_UPLOAD_REQUEST: usize = 16 * 1024 * 1024; +const RECEIVE_BODY_DEADLINE: Duration = Duration::from_secs(1800); +const ARCHIVE_REQUEST_DEADLINE: Duration = Duration::from_secs(60); +const LFS_PROGRESS_GRACE: Duration = Duration::from_secs(60); +const LFS_PROGRESS_FLOOR_BYTES_PER_SEC: u64 = 1024; +const LFS_STALL_TIMEOUT: Duration = Duration::from_secs(120); +const CANDIDATE_FANOUT: usize = 4; +const AUTHORIZED_NAMES_SHOWN: usize = 4; + +fn lfs_within_progress_budget(waited: Duration, moved_bytes: u64) -> bool { + waited + <= LFS_PROGRESS_GRACE + Duration::from_secs(moved_bytes / LFS_PROGRESS_FLOOR_BYTES_PER_SEC) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Service { + Upload, + UploadArchive, + Receive, + Lfs(TransferOp), +} + +enum ReadError { + Io(std::io::Error), + Pack(PackError), + TooLarge, + Truncated, + Deadline, +} + +enum RepoRef { + Did(RepoDid), + OwnerPath(OwnerDid, ClonePath), + HandlePath(knot_types::Handle, ClonePath), +} + +enum ResolvedRef { + Did(RepoDid), + OwnerPath(OwnerDid, ClonePath), +} + +fn parse_exec(command: &[u8]) -> Option<(Service, RepoRef)> { + let text = std::str::from_utf8(command).ok()?.trim(); + if let Some(rest) = text.strip_prefix("git-lfs-transfer ") { + let (path, op_token) = rest.trim().rsplit_once(' ')?; + let op = TransferOp::parse(op_token.trim())?; + return Some((Service::Lfs(op), parse_repo_path(path)?)); + } + let (service, rest) = [ + ("git-upload-pack ", Service::Upload), + ("git upload-pack ", Service::Upload), + ("git-upload-archive ", Service::UploadArchive), + ("git upload-archive ", Service::UploadArchive), + ("git-receive-pack ", Service::Receive), + ("git receive-pack ", Service::Receive), + ] + .into_iter() + .find_map(|(prefix, service)| text.strip_prefix(prefix).map(|rest| (service, rest)))?; + Some((service, parse_repo_path(rest)?)) +} + +fn parse_repo_path(raw: &str) -> Option { + let path = raw + .trim() + .trim_matches('\'') + .trim_matches('"') + .trim_start_matches('/'); + match path.split_once('/') { + Some((owner, name)) => { + let candidates = ClonePath::parse(name)?; + match knot_types::OwnerRef::parse(owner)? { + knot_types::OwnerRef::Did(owner) => Some(RepoRef::OwnerPath(owner, candidates)), + knot_types::OwnerRef::Handle(handle) => { + Some(RepoRef::HandlePath(handle, candidates)) + } + } + } + None => Some(RepoRef::Did(RepoDid::new(path).ok()?)), + } +} + +fn resolve_repo_ref( + state: &Arc>, + repo_ref: ResolvedRef, +) -> RepoLookup { + let candidate = match repo_ref { + ResolvedRef::Did(did) => RepoLookup::Hosted(did), + ResolvedRef::OwnerPath(owner, candidates) => RepoLookup::from_resolved( + state.index.resolve_clone_path(&owner, &candidates), + |found| found, + ), + }; + match candidate { + RepoLookup::Hosted(did) => { + RepoLookup::from_resolved(state.index.owner_of(&did), |_| did.clone()) + } + undecided => undecided, + } +} + +pub(crate) async fn run_exec( + state: Arc>, + credential: Credential, + channel: Channel, + command: &[u8], + protocol_v2: bool, + peer: Option, +) { + let Some((service, repo_ref)) = parse_exec(command) else { + fail(channel, &state.catalog.ssh.unsupported_command.text()).await; + return; + }; + let peer_limiter = match &service { + Service::Lfs(_) => &state.lfs.peer_slots, + _ => &state.peer_slots, + }; + let _peer_guard = match peer_limiter.admit(peer, state.atproto.now()) { + Ok(guard) => guard, + Err(rejection) => { + let reason = match rejection { + knot_resource::Rejection::RateLimited => "peer request rate exceeded", + knot_resource::Rejection::Saturated => "peer concurrency limit reached", + }; + tracing::warn!(?peer, reason, "ssh exec rejected"); + return fail(channel, &state.catalog.ssh.too_many_operations.text()).await; + } + }; + let resolved_ref = match repo_ref { + RepoRef::Did(did) => ResolvedRef::Did(did), + RepoRef::OwnerPath(owner, candidates) => ResolvedRef::OwnerPath(owner, candidates), + RepoRef::HandlePath(owner_handle, candidates) => { + let Some(_lookup_permit) = state.lookup_slots.try_acquire() else { + tracing::warn!( + ?peer, + "ssh exec rejected, the lookup budget can't resolve another handle" + ); + return fail(channel, &state.catalog.ssh.too_many_operations.text()).await; + }; + match state + .atproto + .resolve_handle_to_did(&owner_handle) + .await + .ok() + { + Some(did) => ResolvedRef::OwnerPath(did.into(), candidates), + None => { + fail(channel, &state.catalog.ssh.repo_not_found.text()).await; + return; + } + } + } + }; + let repo_did = match resolve_repo_ref(&state, resolved_ref) { + RepoLookup::Hosted(did) => did, + RepoLookup::Unhosted => { + fail(channel, &state.catalog.ssh.repo_not_found.text()).await; + return; + } + RepoLookup::Unavailable => { + fail(channel, &state.catalog.ssh.index_warming.text()).await; + return; + } + }; + let layout = state.layout.clone(); + let did = repo_did.clone(); + let opened = tokio::task::spawn_blocking(move || layout.open(&did).is_ok()) + .await + .unwrap_or(false); + if !opened { + fail(channel, &state.catalog.ssh.repo_not_found.text()).await; + return; + } + match service { + Service::Upload => serve_upload(state, channel, repo_did, protocol_v2).await, + Service::UploadArchive => serve_upload_archive(state, channel, repo_did).await, + Service::Receive => serve_receive(state, credential, channel, repo_did, peer).await, + Service::Lfs(op) => serve_lfs(state, credential, channel, repo_did, op, peer).await, + } +} + +async fn serve_lfs( + state: Arc>, + credential: Credential, + mut channel: Channel, + repo_did: RepoDid, + op: TransferOp, + peer: Option, +) { + let lfs = state.lfs.clone(); + if op == TransferOp::Upload + && let PushAuth::Refused { reason, message } = + authorize_push(&state, &credential, &repo_did, peer).await + { + tracing::warn!(repo = repo_did.as_str(), reason, "ssh lfs upload denied"); + return fail(channel, &message).await; + } + let permit = match Arc::clone(&lfs.slots).acquire_owned().await { + Ok(permit) => permit, + Err(_) => return fail(channel, &state.catalog.ssh.shutting_down.text()).await, + }; + let started = std::time::Instant::now(); + + let (tx, rx) = mpsc::channel::>(8); + let writer = Box::pin(channel.make_writer()); + let handle = lfs.handle.clone(); + let runtime = Handle::current(); + let did = repo_did.clone(); + let catalog = Arc::clone(&state.catalog); + let mut engine = tokio::task::spawn_blocking(move || { + let _permit = permit; + let output = std::io::BufWriter::new(MeteredWrite::new(runtime.clone(), writer)); + knot_lfs::serve_transfer( + handle.store.as_ref(), + handle.admission.as_ref(), + &did, + op, + &catalog.lfs, + MpscRead::new(runtime, rx), + output, + ) + }); + let joined = { + let reader = channel.make_reader(); + tokio::select! { + joined = &mut engine => joined, + () = pump_input(reader, tx) => engine.await, + } + }; + let status = match joined { + Ok(Ok(())) => { + tracing::info!( + repo = repo_did.as_str(), + op = match op { + TransferOp::Upload => "upload", + TransferOp::Download => "download", + }, + duration_ms = started.elapsed().as_millis() as u64, + "ssh lfs transfer finished" + ); + 0 + } + Ok(Err(fault)) => { + tracing::warn!(repo = repo_did.as_str(), %fault, "ssh lfs transfer failed"); + 1 + } + Err(join) => { + tracing::error!(repo = repo_did.as_str(), %join, "ssh lfs transfer task panicked"); + 1 + } + }; + finish(channel, status).await; +} + +async fn pump_input(reader: R, tx: mpsc::Sender>) { + use futures::TryStreamExt; + let _ = tokio_util::io::ReaderStream::with_capacity(reader, READ_CHUNK) + .map_err(|_| ()) + .try_for_each(|chunk| { + let tx = &tx; + async move { + match chunk.is_empty() { + true => Ok(()), + false => tx.send(chunk.to_vec()).await.map_err(|_| ()), + } + } + }) + .await; +} + +fn stalled(direction: &'static str) -> std::io::Error { + std::io::Error::other(format!("lfs {direction} stalled past the idle timeout")) +} + +struct MpscRead { + runtime: Handle, + rx: mpsc::Receiver>, + buffer: Vec, + offset: usize, + waited: Duration, + received: u64, +} + +impl MpscRead { + fn new(runtime: Handle, rx: mpsc::Receiver>) -> Self { + Self { + runtime, + rx, + buffer: Vec::new(), + offset: 0, + waited: Duration::ZERO, + received: 0, + } + } +} + +impl std::io::Read for MpscRead { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.offset >= self.buffer.len() { + let started = std::time::Instant::now(); + let rx = &mut self.rx; + let received = self + .runtime + // I know I know, but these aren't runtime workers here + .block_on(async { tokio::time::timeout(LFS_STALL_TIMEOUT, rx.recv()).await }); + match received { + Ok(Some(chunk)) => { + self.waited += started.elapsed(); + self.received += chunk.len() as u64; + if !lfs_within_progress_budget(self.waited, self.received) { + return Err(std::io::Error::other( + "lfs input trickles below the progress floor", + )); + } + self.buffer = chunk; + self.offset = 0; + } + Ok(None) => return Ok(0), + Err(_) => return Err(stalled("input")), + } + } + let take = out.len().min(self.buffer.len() - self.offset); + out[..take].copy_from_slice(&self.buffer[self.offset..self.offset + take]); + self.offset += take; + Ok(take) + } +} + +struct MeteredWrite { + runtime: Handle, + inner: W, + waited: Duration, + written: u64, +} + +impl MeteredWrite { + fn new(runtime: Handle, inner: W) -> Self { + Self { + runtime, + inner, + waited: Duration::ZERO, + written: 0, + } + } + + fn charge(&mut self, started: std::time::Instant) -> std::io::Result<()> { + self.waited += started.elapsed(); + match lfs_within_progress_budget(self.waited, self.written) { + true => Ok(()), + false => Err(std::io::Error::other( + "lfs output trickles below the progress floor", + )), + } + } +} + +impl std::io::Write for MeteredWrite { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let started = std::time::Instant::now(); + let inner = &mut self.inner; + let wrote = self + .runtime + .block_on(async { tokio::time::timeout(LFS_STALL_TIMEOUT, inner.write(buf)).await }) + .map_err(|_| stalled("output"))??; + self.written += wrote as u64; + self.charge(started).map(|()| wrote) + } + + fn flush(&mut self) -> std::io::Result<()> { + let started = std::time::Instant::now(); + let inner = &mut self.inner; + self.runtime + .block_on(async { tokio::time::timeout(LFS_STALL_TIMEOUT, inner.flush()).await }) + .map_err(|_| stalled("output"))??; + self.charge(started) + } +} + +async fn serve_upload_archive( + state: Arc>, + mut channel: Channel, + repo_did: RepoDid, +) { + let request = { + let mut reader = channel.make_reader(); + tokio::time::timeout(ARCHIVE_REQUEST_DEADLINE, read_archive_request(&mut reader)).await + }; + let request = match request { + Ok(Ok(request)) => request, + Ok(Err(())) => return fail(channel, &state.catalog.ssh.archive_malformed.text()).await, + Err(_) => return fail(channel, &state.catalog.ssh.archive_timeout.text()).await, + }; + + let permit = state.slots.pack.acquire().await; + let (tx, mut rx) = mpsc::channel::>(16); + let layout = state.layout.clone(); + let did = repo_did.clone(); + let archive_limit = state.archive_limit; + let handle = tokio::task::spawn_blocking(move || -> Result<(), PackError> { + let _permit = permit; + let repo = layout.open(&did)?; + let mut sink = |chunk: &[u8]| -> std::io::Result<()> { + tx.blocking_send(chunk.to_vec()) + .map_err(|_| std::io::Error::other("client disconnected")) + }; + knot_pack::upload_archive_streamed(&repo, &request, archive_limit, &mut sink) + }); + + let mut writer = channel.make_writer(); + let mut forward = Ok(()); + while let Some(chunk) = rx.recv().await { + if writer.write_all(&chunk).await.is_err() { + forward = Err(()); + break; + } + } + drop(rx); + let produced = handle.await; + match &produced { + Ok(Err(error)) => { + tracing::warn!(repo = repo_did.as_str(), %error, "upload-archive failed") + } + Err(join) => { + tracing::error!(repo = repo_did.as_str(), %join, "upload-archive task panicked") + } + Ok(Ok(())) => {} + } + match (forward, produced) { + (Ok(()), Ok(Ok(()))) if writer.flush().await.is_ok() => finish(channel, 0).await, + _ => fail(channel, &state.catalog.ssh.archive_failed.text()).await, + } +} + +async fn read_archive_request(reader: &mut R) -> Result, ()> { + let mut buf = Vec::new(); + loop { + if knot_pack::archive_request_complete(&buf).is_some() { + return Ok(buf); + } + match read_chunk(reader, &mut buf, MAX_UPLOAD_REQUEST).await { + Ok(true) => {} + Ok(false) | Err(_) => return Err(()), + } + } +} + +async fn serve_upload( + state: Arc>, + mut channel: Channel, + repo_did: RepoDid, + protocol_v2: bool, +) { + let advert = { + let layout = state.layout.clone(); + let did = repo_did.clone(); + tokio::task::spawn_blocking(move || -> Result, PackError> { + let repo = layout.open(&did)?; + if protocol_v2 { + knot_pack::advertise_upload_ssh(&repo) + } else { + knot_pack::advertise_upload_v0_ssh(&repo) + } + }) + .await + }; + let advert = match advert { + Ok(Ok(bytes)) => bytes, + _ => return fail(channel, &state.catalog.ssh.advertise_failed.text()).await, + }; + + let mut writer = channel.make_writer(); + if writer.write_all(&advert).await.is_err() || writer.flush().await.is_err() { + return; + } + + let outcome = { + let mut reader = channel.make_reader(); + if protocol_v2 { + upload_loop_v2(&state, &repo_did, &mut reader, &mut writer).await + } else { + upload_loop_v0(&state, &repo_did, &mut reader, &mut writer).await + } + }; + let status = match outcome { + Ok(()) => 0, + Err(()) => 1, + }; + finish(channel, status).await; +} + +async fn upload_loop_v2( + state: &Arc>, + repo_did: &RepoDid, + reader: &mut R, + writer: &mut W, +) -> Result<(), ()> +where + H: HttpTransport, + C: Clock, + R: AsyncRead + Unpin, + W: AsyncWriteExt + Unpin, +{ + let mut buf = Vec::new(); + let mut framer = knot_pack::UploadFramer::new(); + loop { + if let Some(len) = framer.advance(&buf) { + let request: Vec = buf.drain(..len).collect(); + stream_upload(state, repo_did, request, writer).await?; + framer = knot_pack::UploadFramer::new(); + continue; + } + match read_chunk(reader, &mut buf, MAX_UPLOAD_REQUEST).await { + Ok(true) => {} + Ok(false) => return Ok(()), + Err(_) => return Err(()), + } + } +} + +async fn upload_loop_v0( + state: &Arc>, + repo_did: &RepoDid, + reader: &mut R, + writer: &mut W, +) -> Result<(), ()> +where + H: HttpTransport, + C: Clock, + R: AsyncRead + Unpin, + W: AsyncWriteExt + Unpin, +{ + let mut buf = Vec::new(); + let mut framer = knot_pack::UploadFramer::new(); + let mut naks_sent = 0usize; + loop { + if let Some(len) = framer.advance(&buf) { + let request: Vec = buf.drain(..len).collect(); + return stream_upload(state, repo_did, request, writer).await; + } + let needed = framer.unanswered_flushes(); + if naks_sent < needed { + let nak = knot_pack::upload_v0_nak(); + if writer.write_all(&nak).await.is_err() || writer.flush().await.is_err() { + return Err(()); + } + naks_sent += 1; + continue; + } + match read_chunk(reader, &mut buf, MAX_UPLOAD_REQUEST).await { + Ok(true) => {} + Ok(false) => return Ok(()), + Err(_) => return Err(()), + } + } +} + +async fn stream_upload( + state: &Arc>, + repo_did: &RepoDid, + request: Vec, + writer: &mut W, +) -> Result<(), ()> +where + H: HttpTransport, + C: Clock, + W: AsyncWriteExt + Unpin, +{ + let permit = state.slots.pack.acquire().await; + let (tx, mut rx) = mpsc::channel::>(16); + let layout = state.layout.clone(); + let did = repo_did.clone(); + let catalog = Arc::clone(&state.catalog); + let knot = state.hostname.clone(); + let handle = tokio::task::spawn_blocking(move || -> Result<(), PackError> { + let _permit = permit; + let repo = layout.open(&did)?; + let mut sink = |chunk: &[u8]| -> std::io::Result<()> { + tx.blocking_send(chunk.to_vec()) + .map_err(|_| std::io::Error::other("client disconnected")) + }; + knot_pack::upload_pack_streamed(&repo, &request, &catalog.fetch, &knot, &mut sink) + }); + + let mut forward = Ok(()); + while let Some(chunk) = rx.recv().await { + if writer.write_all(&chunk).await.is_err() { + forward = Err(()); + break; + } + } + drop(rx); + match (forward, handle.await) { + (Ok(()), Ok(Ok(()))) => writer.flush().await.map_err(|_| ()), + _ => Err(()), + } +} + +async fn serve_receive( + state: Arc>, + credential: Credential, + mut channel: Channel, + repo_did: RepoDid, + peer: Option, +) { + let advert = { + let layout = state.layout.clone(); + let did = repo_did.clone(); + tokio::task::spawn_blocking(move || -> Result<(Vec, ObjectFormat), PackError> { + let repo = layout.open(&did)?; + let bytes = knot_pack::advertise_receive_ssh(&repo)?; + Ok((bytes, repo.object_format())) + }) + .await + }; + let (advert, object_format) = match advert { + Ok(Ok(pair)) => pair, + _ => return fail(channel, &state.catalog.ssh.advertise_failed.text()).await, + }; + + let mut writer = channel.make_writer(); + if writer.write_all(&advert).await.is_err() || writer.flush().await.is_err() { + return; + } + + let committer = match authorize_push(&state, &credential, &repo_did, peer).await { + PushAuth::Allowed(did) => did, + PushAuth::Refused { reason, message } => { + tracing::warn!(repo = repo_did.as_str(), reason, "ssh push denied"); + return fail(channel, &message).await; + } + }; + + let _receive_permit = state.slots.receive.acquire().await; + + let limits = state.limits; + let body = { + let mut reader = channel.make_reader(); + let dir = state.layout.scratch_dir().to_path_buf(); + match tokio::time::timeout( + RECEIVE_BODY_DEADLINE, + read_receive( + &mut reader, + dir, + state.max_pack_bytes, + limits, + object_format, + ), + ) + .await + { + Ok(result) => result, + Err(_) => Err(ReadError::Deadline), + } + }; + let body = match body { + Ok(body) => body, + Err(ReadError::TooLarge) => { + return fail(channel, &state.catalog.ssh.push_too_large.text()).await; + } + Err(ReadError::Deadline) => { + return fail(channel, &state.catalog.ssh.receive_deadline.text()).await; + } + Err(ReadError::Pack(error)) => { + tracing::warn!(repo = repo_did.as_str(), %error, "receive framing failed"); + return fail(channel, &state.catalog.ssh.malformed_pack.text()).await; + } + Err(ReadError::Io(error)) => { + tracing::warn!(repo = repo_did.as_str(), %error, "receive read error"); + return fail(channel, &state.catalog.ssh.receive_read_error.text()).await; + } + Err(ReadError::Truncated) => { + return fail(channel, &state.catalog.ssh.receive_ended_early.text()).await; + } + }; + if body.is_empty() { + return finish(channel, 0).await; + } + + let _pack_permit = state.slots.pack.acquire().await; + let landed = knot_receive::land(knot_receive::Push { + layout: &state.layout, + repo_did: &repo_did, + received: body, + limits: state.limits, + knot_actor: state.knot_actor.clone(), + committer, + index: &state.index, + atproto: &state.atproto, + resolve_slots: &state.slots.resolve, + appview: &state.appview, + maintenance: &state.maintenance, + hostname: &state.hostname, + catalog: Arc::clone(&state.catalog), + ref_surface: state.ref_surface.clone(), + }) + .await; + match landed { + Ok(framed) => { + let _ = writer.write_all(&framed).await; + let _ = writer.flush().await; + finish(channel, 0).await; + } + Err(error) => { + tracing::warn!(repo = repo_did.as_str(), %error, "receive-pack failed"); + fail(channel, &state.catalog.ssh.receive_failed.text()).await; + } + } +} + +pub(crate) async fn run_greeting( + state: Arc>, + credential: Credential, + channel: Channel, +) { + let knot = state.hostname.as_str().to_string(); + let greeting = match greeting_visitor(&state, &credential).await { + Visitor::Named(who) => state.catalog.ssh.greeting.lines(|key| match key { + knot_messages::GreetingKey::User => who.clone(), + knot_messages::GreetingKey::Knot => knot.clone(), + }), + Visitor::Unknown => state + .catalog + .ssh + .greeting_unknown + .lines(|knot_messages::KnotKey::Knot| knot.clone()), + }; + if greeting.is_empty() { + return finish(channel, 0).await; + } + let body = greeting.join("\r\n"); + let _ = channel + .extended_data_bytes(1, format!("{body}\r\n").into_bytes()) + .await; + finish(channel, 0).await; +} + +async fn greeting_visitor( + state: &Arc>, + credential: &Credential, +) -> Visitor { + let did = match credential { + Credential::Identified(did) => did.clone(), + Credential::Offered(key) => { + match state.index.owner_of_key(key, state.atproto.now().seconds()) { + Resolved::Ready(Some(did)) => did, + _ => return Visitor::Unknown, + } + } + }; + match knot_receive::resolve_handle(&state.atproto, &state.slots.resolve, &did).await { + Some(handle) => Visitor::Named(format!("@{}", handle.as_str())), + None => Visitor::Named(did.as_str().to_string()), + } +} + +enum PusherLookup { + Matched(AccountDid), + Unmatched(Vec), + Unavailable, +} + +enum PushAuth { + Allowed(AccountDid), + Refused { + reason: &'static str, + message: String, + }, +} + +enum Visitor { + Named(String), + Unknown, +} + +async fn authorize_push( + state: &Arc>, + credential: &Credential, + repo: &RepoDid, + peer: Option, +) -> PushAuth { + match resolve_pusher(state, credential, repo, peer).await { + PusherLookup::Matched(did) => { + let consent = state.index.collaboration_consent(repo, &did); + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + match can_push(&acl, &consent).is_allowed() { + true => PushAuth::Allowed(did), + false => PushAuth::Refused { + reason: "unauthorized", + message: state.catalog.ssh.push_denied.text(), + }, + } + } + PusherLookup::Unavailable => PushAuth::Refused { + reason: "identity_unavailable", + message: state.catalog.ssh.identity_unavailable.text(), + }, + PusherLookup::Unmatched(candidates) => { + let authorized = describe_authorized(state, &candidates).await; + PushAuth::Refused { + reason: "unregistered_key", + message: state + .catalog + .ssh + .key_not_registered + .line(|knot_messages::AuthorizedKey::Authorized| authorized.clone()), + } + } + } +} + +async fn describe_authorized( + state: &Arc>, + candidates: &[AccountDid], +) -> String { + let names: Vec = futures::stream::iter( + candidates + .iter() + .take(AUTHORIZED_NAMES_SHOWN) + .cloned() + .collect::>(), + ) + .map(|did| { + let state = Arc::clone(state); + async move { + match knot_receive::resolve_handle(&state.atproto, &state.slots.resolve, &did).await { + Some(handle) => format!("@{}", handle.as_str()), + None => did.as_str().to_string(), + } + } + }) + .buffered(CANDIDATE_FANOUT) + .collect() + .await; + match ( + names.as_slice(), + candidates.len().saturating_sub(AUTHORIZED_NAMES_SHOWN), + ) { + ([], _) => "nobody".to_string(), + (shown, 0) => shown.join(", "), + (shown, hidden) => format!("{}, and {hidden} more", shown.join(", ")), + } +} + +fn probe_due(state: &Arc>, did: &AccountDid) -> bool { + state + .probe_pace + .reserve_now(&SubjectKey::new(did), state.atproto.now()) +} + +async fn resolve_pusher( + state: &Arc>, + credential: &Credential, + repo: &RepoDid, + peer: Option, +) -> PusherLookup { + let key = match credential { + Credential::Identified(did) => return PusherLookup::Matched(did.clone()), + Credential::Offered(key) => key, + }; + let owner = match state.index.owner_of(repo) { + Resolved::Ready(Some(owner)) => Some(AccountDid::from(owner)), + _ => None, + }; + { + let index = Arc::clone(&state.index); + let target = repo.clone(); + let _ = tokio::task::spawn_blocking(move || index.ensure_collaborators(&target)).await; + } + let collaborators = state.index.effective_collaborators(repo).ready_or_default(); + let candidates: Vec = owner.into_iter().chain(collaborators).collect(); + let grantable = candidates.clone(); + let now = state.atproto.now().seconds(); + if let Some(publisher) = state.index.keys().publisher_among(&candidates, key, now) { + return PusherLookup::Matched(publisher); + } + let unread: Vec = candidates + .iter() + .filter(|did| !state.index.keys().is_fresh(did, now) || probe_due(state, did)) + .cloned() + .collect(); + if unread.is_empty() { + tracing::debug!( + ?peer, + repo = repo.as_str(), + candidates = candidates.len(), + "push check has every candidate's keys on file, and the candidates don't publish \ + the offered key" + ); + return PusherLookup::Unmatched(grantable); + } + let lease = state.key_ttl.lease_from(now); + let unresolved = Arc::new(AtomicBool::new(false)); + let read: Vec> = futures::stream::iter(unread) + .map(|did| { + let state = Arc::clone(state); + let key = key.clone(); + let unresolved = Arc::clone(&unresolved); + async move { + let _permit = state.slots.resolve.acquire().await; + match state.atproto.resolve_pubkeys(&did).await { + Ok(keys) => { + let matches = keys.contains(&key); + state.index.keys().record(&did, keys, lease); + matches.then_some(did) + } + Err(error) if error.is_gone() => { + tracing::debug!( + did = did.as_str(), + %error, + "push check records an empty key set for a candidate whose DID document is gone" + ); + state.index.keys().record(&did, Vec::new(), lease); + None + } + Err(error) => { + tracing::debug!( + did = did.as_str(), + %error, + "push check couldn't read a candidate's records" + ); + unresolved.store(true, Ordering::Relaxed); + None + } + } + } + }) + .buffered(CANDIDATE_FANOUT) + .collect() + .await; + match read.into_iter().flatten().next() { + Some(did) => PusherLookup::Matched(did), + None if unresolved.load(Ordering::Relaxed) => PusherLookup::Unavailable, + None => PusherLookup::Unmatched(grantable), + } +} + +async fn read_chunk( + reader: &mut R, + buf: &mut Vec, + limit: usize, +) -> Result { + let mut chunk = [0u8; READ_CHUNK]; + let read = reader.read(&mut chunk).await.map_err(ReadError::Io)?; + if read == 0 { + return Ok(false); + } + buf.extend_from_slice(&chunk[..read]); + if buf.len() > limit { + return Err(ReadError::TooLarge); + } + Ok(true) +} + +async fn read_receive( + reader: &mut R, + dir: PathBuf, + limit: knot_pack::MaxWireBytes, + limits: PackLimits, + format: ObjectFormat, +) -> Result { + let (tx, rx) = mpsc::channel::>(8); + let mut framer = + tokio::task::spawn_blocking(move || frame_receive(rx, dir, limit, limits, format)); + let mut chunk = [0u8; READ_CHUNK]; + let mut io_error = None; + loop { + tokio::select! { + biased; + framed = &mut framer => return join_framed(framed, io_error), + read = reader.read(&mut chunk) => match read { + Ok(0) => break, + Ok(read) => { + if tx.send(chunk[..read].to_vec()).await.is_err() { + break; + } + } + Err(error) => { + io_error = Some(error); + break; + } + }, + } + } + drop(tx); + join_framed(framer.await, io_error) +} + +fn join_framed( + framed: Result, tokio::task::JoinError>, + io_error: Option, +) -> Result { + match framed { + Ok(Ok(body)) => Ok(body), + Ok(Err(ReadError::Truncated)) => { + Err(io_error.map(ReadError::Io).unwrap_or(ReadError::Truncated)) + } + Ok(Err(other)) => Err(other), + Err(_) => Err(ReadError::Truncated), + } +} + +fn read_error(error: knot_pack::ReceiveReadError) -> ReadError { + match error { + knot_pack::ReceiveReadError::Io(error) => ReadError::Io(error), + knot_pack::ReceiveReadError::Pack(error) => ReadError::Pack(error), + knot_pack::ReceiveReadError::TooLarge => ReadError::TooLarge, + knot_pack::ReceiveReadError::Truncated => ReadError::Truncated, + } +} + +fn frame_receive( + mut rx: mpsc::Receiver>, + dir: PathBuf, + limit: knot_pack::MaxWireBytes, + limits: PackLimits, + format: ObjectFormat, +) -> Result { + let mut receiver = + knot_pack::PackReceiver::new(&dir, limit, limits, format.kind()).map_err(ReadError::Io)?; + loop { + match rx.blocking_recv() { + Some(chunk) => { + if receiver.write(&chunk).map_err(read_error)? { + return receiver.finish().map_err(read_error); + } + } + None => return receiver.finish().map_err(read_error), + } + } +} + +async fn fail(channel: Channel, message: &str) { + let _ = channel + .extended_data_bytes(1, format!("{message}\n").into_bytes()) + .await; + finish(channel, 1).await; +} + +async fn finish(channel: Channel, status: u32) { + let _ = channel.exit_status(status).await; + let _ = channel.eof().await; + let _ = channel.close().await; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_lfs_progress_budget_spares_slow_links_and_cuts_trickles() { + assert!(lfs_within_progress_budget(Duration::from_secs(59), 0)); + assert!(!lfs_within_progress_budget(Duration::from_secs(61), 0)); + assert!(lfs_within_progress_budget( + Duration::from_secs(50_000), + 5 * 1024 * 1024 * 1024 + )); + assert!(!lfs_within_progress_budget(Duration::from_secs(1_000), 10)); + } + + #[test] + fn the_repo_path_parser_separates_dids_from_handles() { + assert!(matches!( + parse_repo_path("did:plc:nel/squid"), + Some(RepoRef::OwnerPath(..)) + )); + assert!(matches!( + parse_repo_path("nel.pet/squid"), + Some(RepoRef::HandlePath(..)) + )); + assert!(matches!( + parse_repo_path("did:plc:barnacle"), + Some(RepoRef::Did(_)) + )); + assert!(parse_repo_path("did:nonsense/squid").is_none()); + assert!(parse_repo_path("nel.pet").is_none()); + } +} diff --git a/knot2/crates/knot-ssh/src/identity.rs b/knot2/crates/knot-ssh/src/identity.rs new file mode 100644 index 0000000000000000000000000000000000000000..ad30b835dde763266d2708be6efede6a666177fa --- /dev/null +++ b/knot2/crates/knot-ssh/src/identity.rs @@ -0,0 +1,187 @@ +use std::net::IpAddr; +use std::sync::Arc; + +use knot_atproto::ClaimedKeys; +use knot_index::{Coverage, Resolved}; +use knot_resource::AdmitGuard; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, OfferedKey, OwnerRef}; + +use crate::SshState; + +#[derive(Clone)] +pub(crate) enum Credential { + Identified(AccountDid), + Offered(OfferedKey), +} + +pub(crate) struct Asserted { + claim: OwnerRef, + outcome: Claimed, +} + +enum Claimed { + Publishes { + did: AccountDid, + keys: Vec, + }, + Unreadable, +} + +#[derive(Clone)] +pub(crate) enum Verdict { + Identified(AccountDid), + Offered, + Refused, +} + +pub(crate) async fn verify( + state: &Arc>, + claim: Option, + key: &OfferedKey, + peer: Option, + asserted: &mut Option, +) -> Verdict { + match claim { + Some(claim) => match against_claim(state, claim, key, peer, asserted).await { + Some(verdict) => verdict, + None => against_key_set(state, key, peer), + }, + None => against_key_set(state, key, peer), + } +} + +fn against_key_set( + state: &Arc>, + key: &OfferedKey, + peer: Option, +) -> Verdict { + let now = state.atproto.now().seconds(); + match ( + state.index.owner_of_key(key, now), + state.index.keys().coverage(), + ) { + (Resolved::Ready(Some(_)), _) => Verdict::Offered, + (_, Coverage::Warming) => Verdict::Offered, + (_, Coverage::Ready) => match state.index.keys().any_unheld() { + true => Verdict::Offered, + false => { + if miss_worth_a_reread(state, peer) { + state.index.keys().note_miss(); + } + Verdict::Refused + } + }, + } +} + +fn miss_worth_a_reread( + state: &Arc>, + peer: Option, +) -> bool { + peer.is_none_or(|peer| state.miss_pace.reserve_now(&peer, state.atproto.now())) +} + +async fn against_claim( + state: &Arc>, + claim: OwnerRef, + key: &OfferedKey, + peer: Option, + asserted: &mut Option, +) -> Option { + let known = match asserted.take().filter(|known| known.claim == claim) { + Some(known) => known, + None => Asserted { + outcome: resolve_claim(state, &claim, peer).await, + claim, + }, + }; + let verdict = match &known.outcome { + Claimed::Unreadable => None, + Claimed::Publishes { did, keys } if keys.contains(key) => { + Some(Verdict::Identified(did.clone())) + } + Claimed::Publishes { did, keys } => { + tracing::debug!( + ?peer, + did = did.as_str(), + published = keys.len(), + "ssh auth refused a key the asserted account doesn't publish" + ); + Some(Verdict::Refused) + } + }; + *asserted = Some(known); + verdict +} + +async fn resolve_claim( + state: &Arc>, + claim: &OwnerRef, + peer: Option, +) -> Claimed { + // if it's already cached, just reuse it from the cache instead of spending lookup budget + let mut guard = None; + let did = match claim { + OwnerRef::Did(did) => AccountDid::from(did.clone()), + OwnerRef::Handle(handle) => match state.atproto.cached_handle_did(handle).await { + Some(did) => did, + None => { + if !admit_lookup(state, peer, &mut guard) { + return Claimed::Unreadable; + } + match state.atproto.resolve_handle_to_did(handle).await { + Ok(did) => did, + Err(error) => { + tracing::warn!( + ?peer, + handle = handle.as_str(), + %error, + "ssh auth couldn't resolve the handle in the login name" + ); + return Claimed::Unreadable; + } + } + } + }, + }; + let keys = match state.atproto.cached_claimed_pubkeys(&did).await { + Some(keys) => keys, + None => { + if !admit_lookup(state, peer, &mut guard) { + return Claimed::Unreadable; + } + state.atproto.claimed_pubkeys(&did).await + } + }; + match keys { + ClaimedKeys::Published(keys) => Claimed::Publishes { did, keys }, + ClaimedKeys::Unread(error) => { + tracing::warn!( + ?peer, + did = did.as_str(), + %error, + "ssh auth couldn't read the asserted account's published keys" + ); + Claimed::Unreadable + } + } +} + +// a claim only pays once, even when both its handle and its keys need looking up +fn admit_lookup( + state: &Arc>, + peer: Option, + guard: &mut Option, +) -> bool { + if guard.is_some() { + return true; + } + match state.lookup_peers.admit(peer, state.atproto.now()) { + Ok(admitted) => *guard = Some(admitted), + Err(_) => { + tracing::debug!(?peer, "ssh auth couldn't check a claim, peer budget spent"); + } + } + guard.is_some() +} diff --git a/knot2/crates/knot-ssh/src/lib.rs b/knot2/crates/knot-ssh/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..a2f3372bdd029441307375e48c70275bdfc8a818 --- /dev/null +++ b/knot2/crates/knot-ssh/src/lib.rs @@ -0,0 +1,435 @@ +mod exec; +mod identity; +mod proxy; +mod server; + +use std::borrow::Cow; +use std::collections::BTreeSet; +use std::net::SocketAddr; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use knot_atproto::Atproto; +use knot_git::{ArchiveLimit, Layout}; +use knot_index::{Index, KeyTtl}; +use knot_maintenance::MaintenanceHandle; +use knot_pack::{MaxWireBytes, PackLimits}; +use knot_runtime::{Clock, Entropy, HttpTransport, OsEntropy}; +use knot_types::{ + AccountDid, ActorId, AdmissionPolicy, AppviewEndpoint, KnotHostname, TrustedProxies, +}; +use russh::keys::ssh_key::rand_core; +use russh::keys::{Algorithm, PrivateKey, ssh_key}; +use russh::server::{Config, Server as _}; +use russh::{MethodKind, MethodSet, Preferred, compression}; +use tokio::sync::Semaphore; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; + +use knot_resource::{ + Burst, GlobalInflight, LimitConfig, PeerPacer, PerPeerInflight, PreAuthLimiter, RateLimit, + RefillMicros, ResolveSlots, Slots, SubjectPacer, +}; +use server::KnotSshServer; + +const MAX_INFLIGHT_LOOKUPS: usize = 16; +const MAX_PREAUTH_LOOKUPS: usize = 4; +const LOOKUP_BURST_PER_PEER: u32 = 8; +const LOOKUP_REFILL_MICROS: u64 = 500_000; +const LOOKUP_INFLIGHT_PER_PEER: usize = 2; +const PROBE_BURST_PER_ACCOUNT: u32 = 1; +const PROBE_REFILL_MICROS: u64 = 30_000_000; +const MISS_BURST_PER_PEER: u32 = 1; +const MISS_REFILL_MICROS: u64 = 120_000_000; +const INACTIVITY_TIMEOUT: Duration = Duration::from_secs(120); +const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30); +const AUTH_REJECTION_TIME: Duration = Duration::from_millis(250); +const DRAIN_GRACE: Duration = Duration::from_secs(30); +const ACCEPT_BACKOFF: Duration = Duration::from_millis(250); + +#[derive(Debug, thiserror::Error)] +pub enum SshError { + #[error("ssh host key {path}: {message}")] + HostKey { path: String, message: String }, + #[error("ssh server bind or serve: {0}")] + Serve(#[from] std::io::Error), +} + +pub struct SshState { + layout: Layout, + index: Arc, + atproto: Arc>, + knot_actor: ActorId, + hostname: KnotHostname, + appview: AppviewEndpoint, + admins: BTreeSet, + admission: AdmissionPolicy, + limits: PackLimits, + max_pack_bytes: MaxWireBytes, + archive_limit: ArchiveLimit, + slots: Slots, + lookup_slots: ResolveSlots, + lookup_peers: Arc, + probe_pace: SubjectPacer, + miss_pace: PeerPacer, + key_ttl: KeyTtl, + peer_slots: Arc, + proxy_protocol_from: TrustedProxies, + maintenance: MaintenanceHandle, + lfs: LfsRuntime, + catalog: Arc, + ref_surface: Option>, +} + +#[derive(Clone)] +pub(crate) struct LfsRuntime { + pub(crate) handle: knot_lfs::LfsHandle, + pub(crate) slots: Arc, + pub(crate) peer_slots: Arc, +} + +pub struct SshConfig { + pub layout: Layout, + pub index: Arc, + pub atproto: Arc>, + pub knot_actor: ActorId, + pub hostname: KnotHostname, + pub appview: AppviewEndpoint, + pub admins: BTreeSet, + pub admission: AdmissionPolicy, + pub max_pack_bytes: MaxWireBytes, + pub archive_limit: ArchiveLimit, + pub lfs: knot_lfs::LfsHandle, + pub lfs_max_transfers: usize, + pub max_inflight_per_peer: usize, +} + +impl SshState { + pub fn new(config: SshConfig) -> Self { + let SshConfig { + layout, + index, + atproto, + knot_actor, + hostname, + appview, + admins, + admission, + max_pack_bytes, + archive_limit, + lfs: handle, + lfs_max_transfers, + max_inflight_per_peer, + } = config; + Self { + layout, + index, + atproto, + knot_actor, + hostname, + appview, + admins, + admission, + limits: PackLimits::default(), + max_pack_bytes, + archive_limit, + slots: Slots::for_machine(), + lookup_slots: ResolveSlots::new(MAX_INFLIGHT_LOOKUPS), + lookup_peers: Arc::new(PreAuthLimiter::with_config(LimitConfig { + rate: Some(RateLimit { + burst: Burst::literal(LOOKUP_BURST_PER_PEER), + refill: RefillMicros::literal(LOOKUP_REFILL_MICROS), + }), + per_peer_inflight: Some(PerPeerInflight::new(LOOKUP_INFLIGHT_PER_PEER)), + global_inflight: Some(GlobalInflight::new(MAX_PREAUTH_LOOKUPS)), + })), + probe_pace: SubjectPacer::new(RateLimit { + burst: Burst::literal(PROBE_BURST_PER_ACCOUNT), + refill: RefillMicros::literal(PROBE_REFILL_MICROS), + }), + miss_pace: PeerPacer::new(RateLimit { + burst: Burst::literal(MISS_BURST_PER_PEER), + refill: RefillMicros::literal(MISS_REFILL_MICROS), + }), + key_ttl: KeyTtl::DEFAULT, + peer_slots: Arc::new(PreAuthLimiter::with_config(LimitConfig::per_peer_only( + PerPeerInflight::new(max_inflight_per_peer), + ))), + proxy_protocol_from: TrustedProxies::default(), + maintenance: MaintenanceHandle::disabled(), + lfs: LfsRuntime { + handle, + slots: Arc::new(Semaphore::new(lfs_max_transfers)), + peer_slots: Arc::new(PreAuthLimiter::with_config(LimitConfig::per_peer_only( + PerPeerInflight::new(lfs_max_transfers), + ))), + }, + catalog: Arc::new(knot_messages::Catalog::defaults()), + ref_surface: None, + } + } + + pub fn with_catalog(mut self, catalog: Arc) -> Self { + self.catalog = catalog; + self + } + + pub fn with_ref_surface(mut self, surface: Arc) -> Self { + self.ref_surface = Some(surface); + self + } + + pub fn with_slots(mut self, slots: Slots) -> Self { + self.slots = slots; + self + } + + pub fn with_key_ttl(mut self, ttl: KeyTtl) -> Self { + self.key_ttl = ttl; + self + } + + pub fn with_proxy_protocol_from(mut self, proxies: TrustedProxies) -> Self { + self.proxy_protocol_from = proxies; + self + } + + pub fn with_maintenance(mut self, maintenance: MaintenanceHandle) -> Self { + self.maintenance = maintenance; + self + } + + pub fn with_limits(mut self, limits: PackLimits) -> Self { + self.limits = limits; + self + } +} + +fn server_config(host_key: PrivateKey) -> Arc { + Arc::new(Config { + keys: vec![host_key], + methods: MethodSet::from(&[MethodKind::PublicKey][..]), + inactivity_timeout: Some(INACTIVITY_TIMEOUT), + keepalive_interval: Some(KEEPALIVE_INTERVAL), + auth_rejection_time: AUTH_REJECTION_TIME, + preferred: Preferred { + compression: Cow::Borrowed(&[compression::NONE]), + ..Preferred::DEFAULT + }, + ..Config::default() + }) +} + +pub async fn serve( + addr: SocketAddr, + host_key: PrivateKey, + state: Arc>, + shutdown: CancellationToken, +) -> Result<(), SshError> { + let listener = tokio::net::TcpListener::bind(addr).await?; + serve_drained(listener, host_key, state, shutdown).await +} + +pub async fn serve_on_socket( + listener: tokio::net::TcpListener, + host_key: PrivateKey, + state: Arc>, +) -> Result<(), SshError> { + serve_drained(listener, host_key, state, CancellationToken::new()).await +} + +#[doc(hidden)] +pub async fn serve_drained( + listener: tokio::net::TcpListener, + host_key: PrivateKey, + state: Arc>, + shutdown: CancellationToken, +) -> Result<(), SshError> { + let config = server_config(host_key); + let tracker = TaskTracker::new(); + loop { + let accepted = tokio::select! { + () = shutdown.cancelled() => break, + accepted = listener.accept() => accepted, + }; + let (stream, peer) = match accepted { + Ok(pair) => pair, + Err(error) if is_connection_error(&error) => continue, + Err(error) => { + tracing::warn!("ssh accept failed, backing off: {error}"); + tokio::select! { + () = shutdown.cancelled() => break, + () = tokio::time::sleep(ACCEPT_BACKOFF) => {} + } + continue; + } + }; + let config = Arc::clone(&config); + let state = Arc::clone(&state); + let session_tracker = tracker.clone(); + tracker.spawn(async move { + let mut stream = stream; + let mut peer = peer; + if state.proxy_protocol_from.contains(peer.ip()) { + match proxy::read_v1(&mut stream).await { + Ok(Some(client)) => peer.set_ip(client), + Ok(None) => {} + Err(proxy::Malformed) => { + tracing::warn!(%peer, "ssh connection dropped, malformed PROXY header"); + return; + } + } + } + let mut server = KnotSshServer { + state, + tracker: session_tracker, + }; + let handler = server.new_client(Some(peer)); + if let Ok(session) = russh::server::run_stream(config, stream, handler).await { + let _ = session.await; + } + }); + } + tracker.close(); + let _ = tokio::time::timeout(DRAIN_GRACE, tracker.wait()).await; + Ok(()) +} + +fn is_connection_error(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::ConnectionReset + ) +} + +pub fn load_or_create_host_key(path: &Path) -> Result { + let report = |message: String| SshError::HostKey { + path: path.display().to_string(), + message, + }; + let load = || { + ensure_secure_perms(path).map_err(&report)?; + russh::keys::load_secret_key(path, None).map_err(|error| report(error.to_string())) + }; + if path.exists() { + return load(); + } + let key = PrivateKey::random(&mut EntropyRng, Algorithm::Ed25519) + .map_err(|error| report(error.to_string()))?; + match persist_host_key(path, &key)? { + Claim::Won => Ok(key), + Claim::Lost => load(), + } +} + +enum Claim { + Won, + Lost, +} + +fn persist_host_key(path: &Path, key: &PrivateKey) -> Result { + let report = |message: String| SshError::HostKey { + path: path.display().to_string(), + message, + }; + let pem = key + .to_openssh(ssh_key::LineEnding::LF) + .map_err(|error| report(error.to_string()))?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| report(error.to_string()))?; + } + let temp = unique_temp(path); + write_secret(&temp, pem.as_bytes()).map_err(|error| report(error.to_string()))?; + // `hard_link` instead of `rename` so that in case two knots + // are booting at the same time they don't get borked + // if one clobber's the other's key. here the loser reloads + // winner's key. + let claim = match std::fs::hard_link(&temp, path) { + Ok(()) => Claim::Won, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Claim::Lost, + Err(error) => { + let _ = std::fs::remove_file(&temp); + return Err(report(error.to_string())); + } + }; + let _ = std::fs::remove_file(&temp); + if let Some(parent) = path.parent() + && let Ok(dir) = std::fs::File::open(parent) + { + let _ = dir.sync_all(); + } + Ok(claim) +} + +#[cfg(unix)] +fn ensure_secure_perms(path: &Path) -> Result<(), String> { + use std::os::unix::fs::MetadataExt; + let mode = std::fs::metadata(path) + .map_err(|error| error.to_string())? + .mode(); + if mode & 0o077 != 0 { + return Err(format!( + "private host key is group or other accessible at mode {:o}, run chmod 600 on it", + mode & 0o777 + )); + } + Ok(()) +} + +#[cfg(not(unix))] +fn ensure_secure_perms(_path: &Path) -> Result<(), String> { + Ok(()) +} + +fn unique_temp(path: &Path) -> std::path::PathBuf { + use std::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nonce = COUNTER.fetch_add(1, Ordering::Relaxed); + let stem = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("ssh_host_key"); + path.with_file_name(format!(".{stem}.{}.{nonce}.tmp", std::process::id())) +} + +fn write_secret(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path)?; + file.write_all(bytes)?; + file.sync_all() +} + +struct EntropyRng; + +impl rand_core::TryRng for EntropyRng { + type Error = std::convert::Infallible; + + fn try_next_u32(&mut self) -> Result { + let mut bytes = [0u8; 4]; + OsEntropy.fill(&mut bytes); + Ok(u32::from_le_bytes(bytes)) + } + + fn try_next_u64(&mut self) -> Result { + let mut bytes = [0u8; 8]; + OsEntropy.fill(&mut bytes); + Ok(u64::from_le_bytes(bytes)) + } + + fn try_fill_bytes(&mut self, dst: &mut [u8]) -> Result<(), Self::Error> { + OsEntropy.fill(dst); + Ok(()) + } +} + +impl rand_core::TryCryptoRng for EntropyRng {} diff --git a/knot2/crates/knot-ssh/src/proxy.rs b/knot2/crates/knot-ssh/src/proxy.rs new file mode 100644 index 0000000000000000000000000000000000000000..6d70b285b01dbe42e497ab29e8056a6fc78aa681 --- /dev/null +++ b/knot2/crates/knot-ssh/src/proxy.rs @@ -0,0 +1,118 @@ +use std::net::IpAddr; +use std::time::Duration; + +use tokio::io::AsyncReadExt; +use tokio::net::TcpStream; + +const HEADER_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_V1_HEADER: usize = 107; + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Malformed; + +/// Consumes a PROXY protocol v1 header, if the stream starts with one, and +/// returns the client address it names. Leaves the stream untouched otherwise, +/// and when no byte arrives in time, since some ssh clients wait for the server +/// banner before they send their own. +pub(crate) async fn read_v1(stream: &mut TcpStream) -> Result, Malformed> { + let mut first = [0u8; 1]; + match tokio::time::timeout(HEADER_TIMEOUT, stream.peek(&mut first)).await { + Ok(Ok(1)) if first[0] == b'P' => {} + _ => return Ok(None), + } + let mut line = Vec::with_capacity(MAX_V1_HEADER); + let read = async { + while !line.ends_with(b"\r\n") && line.len() < MAX_V1_HEADER { + line.push(stream.read_u8().await.map_err(|_| Malformed)?); + } + Ok(()) + }; + tokio::time::timeout(HEADER_TIMEOUT, read) + .await + .map_err(|_| Malformed)??; + parse_v1(&line) +} + +fn parse_v1(line: &[u8]) -> Result, Malformed> { + let line = std::str::from_utf8(line) + .ok() + .and_then(|line| line.strip_suffix("\r\n")) + .ok_or(Malformed)?; + let mut fields = line.split(' '); + match (fields.next(), fields.next(), fields.next()) { + (Some("PROXY"), Some("UNKNOWN"), _) => Ok(None), + (Some("PROXY"), Some(family @ ("TCP4" | "TCP6")), Some(source)) => { + let source = source.parse::().map_err(|_| Malformed)?; + match (family, source) { + ("TCP4", IpAddr::V4(_)) | ("TCP6", IpAddr::V6(_)) => { + Ok(Some(source.to_canonical())) + } + _ => Err(Malformed), + } + } + _ => Err(Malformed), + } +} + +#[cfg(test)] +mod tests { + use tokio::io::AsyncWriteExt; + + use super::*; + + fn ip(value: &str) -> Option { + Some(value.parse().unwrap()) + } + + #[test] + fn parse_v1_reads_the_source_address() { + assert_eq!( + parse_v1(b"PROXY TCP4 198.51.100.4 203.0.113.1 51234 22\r\n"), + Ok(ip("198.51.100.4")) + ); + assert_eq!( + parse_v1(b"PROXY TCP6 2001:db8::4 2001:db8::1 51234 22\r\n"), + Ok(ip("2001:db8::4")) + ); + assert_eq!(parse_v1(b"PROXY UNKNOWN\r\n"), Ok(None)); + } + + #[test] + fn parse_v1_rejects_malformed_headers() { + for line in [ + &b"PROXY TCP4 198.51.100.4 203.0.113.1 51234 22"[..], + b"PROXY TCP4 2001:db8::4 2001:db8::1 51234 22\r\n", + b"PROXY TCP4 nope 203.0.113.1 51234 22\r\n", + b"PROXY UDP4 198.51.100.4 203.0.113.1 51234 22\r\n", + b"PRAXY TCP4 198.51.100.4 203.0.113.1 51234 22\r\n", + ] { + assert_eq!(parse_v1(line), Err(Malformed), "{line:?}"); + } + } + + async fn pair(sent: &'static [u8]) -> (TcpStream, Vec) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let mut client = TcpStream::connect(addr).await.unwrap(); + client.write_all(sent).await.unwrap(); + client.shutdown().await.unwrap(); + (listener.accept().await.unwrap().0, Vec::new()) + } + + #[tokio::test] + async fn read_v1_consumes_only_the_header() { + let (mut stream, mut rest) = + pair(b"PROXY TCP4 198.51.100.4 203.0.113.1 51234 22\r\nSSH-2.0-OpenSSH\r\n").await; + assert_eq!(read_v1(&mut stream).await, Ok(ip("198.51.100.4"))); + stream.read_to_end(&mut rest).await.unwrap(); + assert_eq!(rest, b"SSH-2.0-OpenSSH\r\n"); + } + + #[tokio::test] + async fn read_v1_leaves_a_plain_ssh_stream_alone() { + let (mut stream, mut rest) = pair(b"SSH-2.0-OpenSSH\r\n").await; + assert_eq!(read_v1(&mut stream).await, Ok(None)); + stream.read_to_end(&mut rest).await.unwrap(); + assert_eq!(rest, b"SSH-2.0-OpenSSH\r\n"); + } +} diff --git a/knot2/crates/knot-ssh/src/server.rs b/knot2/crates/knot-ssh/src/server.rs new file mode 100644 index 0000000000000000000000000000000000000000..6c818c1cb005120fb76222d285f5af11b308e043 --- /dev/null +++ b/knot2/crates/knot-ssh/src/server.rs @@ -0,0 +1,226 @@ +use std::collections::{HashMap, HashSet}; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; + +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{OfferedKey, OwnerRef}; +use russh::keys::ssh_key; +use russh::server::{Auth, Handler, Msg, Server, Session}; +use russh::{Channel, ChannelId}; +use tokio_util::task::TaskTracker; + +use crate::SshState; +use crate::exec::run_exec; +use crate::identity::{self, Asserted, Credential, Verdict}; + +pub(crate) struct KnotSshServer { + pub(crate) state: Arc>, + pub(crate) tracker: TaskTracker, +} + +impl Server for KnotSshServer { + type Handler = KnotSession; + + fn new_client(&mut self, peer: Option) -> Self::Handler { + KnotSession::new( + Arc::clone(&self.state), + self.tracker.clone(), + peer.map(|addr| addr.ip()), + ) + } +} + +pub(crate) struct KnotSession { + state: Arc>, + tracker: TaskTracker, + credential: Option, + asserted: Option, + channels: HashMap>, + protocols: HashSet, + peer: Option, + decided: Option<(String, OfferedKey, Verdict)>, +} + +fn reject() -> Auth { + Auth::Reject { + proceed_with_methods: None, + partial_success: false, + } +} + +impl KnotSession { + fn new(state: Arc>, tracker: TaskTracker, peer: Option) -> Self { + Self { + state, + tracker, + credential: None, + asserted: None, + channels: HashMap::new(), + protocols: HashSet::new(), + peer, + decided: None, + } + } +} + +impl KnotSession { + async fn decide( + &mut self, + user: &str, + public_key: &ssh_key::PublicKey, + ) -> Option<(Verdict, OfferedKey)> { + let key = OfferedKey::from_bytes(public_key.to_bytes().ok()?); + if let Some((asked_as, asked, verdict)) = &self.decided + && asked_as == user + && *asked == key + { + return Some((verdict.clone(), key)); + } + let verdict = identity::verify( + &self.state, + OwnerRef::parse(user), + &key, + self.peer, + &mut self.asserted, + ) + .await; + self.decided = Some((user.to_string(), key.clone(), verdict.clone())); + Some((verdict, key)) + } +} + +impl Handler for KnotSession { + type Error = russh::Error; + + async fn auth_publickey_offered( + &mut self, + user: &str, + public_key: &ssh_key::PublicKey, + ) -> Result { + match self.decide(user, public_key).await { + Some((Verdict::Refused, _)) | None => Ok(reject()), + Some(_) => Ok(Auth::Accept), + } + } + + async fn auth_publickey( + &mut self, + user: &str, + public_key: &ssh_key::PublicKey, + ) -> Result { + match self.decide(user, public_key).await { + Some((Verdict::Identified(did), _)) => { + self.credential = Some(Credential::Identified(did)); + Ok(Auth::Accept) + } + Some((Verdict::Offered, key)) => { + self.credential = Some(Credential::Offered(key)); + Ok(Auth::Accept) + } + Some((Verdict::Refused, _)) | None => Ok(reject()), + } + } + + async fn channel_open_session( + &mut self, + channel: Channel, + _session: &mut Session, + ) -> Result { + self.channels.insert(channel.id(), channel); + Ok(true) + } + + async fn env_request( + &mut self, + channel: ChannelId, + variable_name: &str, + variable_value: &str, + _session: &mut Session, + ) -> Result<(), Self::Error> { + if variable_name == "GIT_PROTOCOL" + && variable_value + .split(':') + .any(|token| token.trim() == "version=2") + { + self.protocols.insert(channel); + } + Ok(()) + } + + async fn channel_eof( + &mut self, + channel: ChannelId, + _session: &mut Session, + ) -> Result<(), Self::Error> { + self.protocols.remove(&channel); + Ok(()) + } + + async fn channel_close( + &mut self, + channel: ChannelId, + _session: &mut Session, + ) -> Result<(), Self::Error> { + self.channels.remove(&channel); + self.protocols.remove(&channel); + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + async fn pty_request( + &mut self, + channel: ChannelId, + _term: &str, + _col_width: u32, + _row_height: u32, + _pix_width: u32, + _pix_height: u32, + _modes: &[(russh::Pty, u32)], + session: &mut Session, + ) -> Result<(), Self::Error> { + session.channel_success(channel)?; + Ok(()) + } + + async fn shell_request( + &mut self, + channel: ChannelId, + session: &mut Session, + ) -> Result<(), Self::Error> { + let (Some(handle), Some(credential)) = + (self.channels.remove(&channel), self.credential.clone()) + else { + session.channel_failure(channel)?; + return Ok(()); + }; + session.channel_success(channel)?; + let state = Arc::clone(&self.state); + self.tracker.spawn(async move { + crate::exec::run_greeting(state, credential, handle).await; + }); + Ok(()) + } + + async fn exec_request( + &mut self, + channel: ChannelId, + data: &[u8], + session: &mut Session, + ) -> Result<(), Self::Error> { + let (Some(handle), Some(credential)) = + (self.channels.remove(&channel), self.credential.clone()) + else { + session.channel_failure(channel)?; + return Ok(()); + }; + session.channel_success(channel)?; + let protocol_v2 = self.protocols.remove(&channel); + let state = Arc::clone(&self.state); + let peer = self.peer; + let command = data.to_vec(); + self.tracker.spawn(async move { + run_exec(state, credential, handle, &command, protocol_v2, peer).await; + }); + Ok(()) + } +} diff --git a/knot2/crates/knot-ssh/tests/ssh_push.rs b/knot2/crates/knot-ssh/tests/ssh_push.rs new file mode 100644 index 0000000000000000000000000000000000000000..67fa824b33a846115c4f441d1836b4f2329a2677 --- /dev/null +++ b/knot2/crates/knot-ssh/tests/ssh_push.rs @@ -0,0 +1,2403 @@ +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use futures::stream::StreamExt; +use knot_atproto::Atproto; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{CollaboratorsChange, Grant, MembersChange, Registration, RegistryChange}; +use knot_fixtures::UNSHARED_SSH; +use knot_git::{ArchiveLimit, Layout, Repo}; +use knot_index::{Index, Resolved}; +use knot_pack::MaxWireBytes; +use knot_runtime::{ + FakeDns, FakeHttp, HttpResponse, K256Signer, ManualClock, SeededEntropy, Signer, UnixMicros, +}; +use knot_types::{ + AccountDid, Did, KnotId, KnotServiceUrl, Oid, OwnerDid, RefName, RepoDid, RepoName, RepoRkey, + UnixSeconds, +}; +use tempfile::TempDir; +use tokio::net::TcpListener; +use url::Url; + +const REPO_DID: &str = "did:plc:squid"; +const REPO_NAME: &str = "anemone"; +const OWNER_DID: &str = "did:plc:nel"; +const TID_REPO_DID: &str = "did:plc:limpet"; +const TID_RKEY: &str = "3mizfnpxii522"; +const TID_REPO_NAME: &str = "periwinkle.cloud"; +const PDS_HOST: &str = "pds.oyster.cafe"; + +fn repo_did() -> RepoDid { + RepoDid::new(REPO_DID).unwrap() +} + +fn repo_name() -> RepoName { + RepoName::new(REPO_NAME).unwrap() +} + +fn owner() -> AccountDid { + AccountDid::new(OWNER_DID).unwrap() +} + +fn owner_did() -> OwnerDid { + OwnerDid::new(OWNER_DID).unwrap() +} + +fn tid_repo_did() -> RepoDid { + RepoDid::new(TID_REPO_DID).unwrap() +} + +fn tid_rkey() -> RepoRkey { + RepoRkey::new(TID_RKEY).unwrap() +} + +fn tid_repo_name() -> RepoName { + RepoName::new(TID_REPO_NAME).unwrap() +} + +fn pds_url() -> KnotServiceUrl { + KnotServiceUrl::new(format!("https://{PDS_HOST}")).unwrap() +} + +fn git(cwd: &Path, env: &[(&str, &str)], args: &[&str]) -> (bool, String) { + let mut command = knot_fixtures::command(cwd); + command.args(args); + env.iter().for_each(|(key, value)| { + command.env(key, value); + }); + let out = command.output().expect("git runs"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out.status.success(), combined) +} + +fn keygen(dir: &Path, name: &str) -> (String, String) { + let path = dir.join(name); + let out = Command::new("ssh-keygen") + .args([ + "-t", + "ed25519", + "-N", + "", + "-C", + "nel@oyster.cafe", + "-f", + path.to_str().unwrap(), + ]) + .output() + .expect("ssh-keygen runs"); + assert!( + out.status.success(), + "ssh-keygen failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let public_line = std::fs::read_to_string(dir.join(format!("{name}.pub"))) + .unwrap() + .trim() + .to_string(); + (path.to_str().unwrap().to_string(), public_line) +} + +fn did_document(signer: &K256Signer, did: &Did, pds: &KnotServiceUrl) -> Vec { + let multikey = knot_types::crypto::multikey(0xe7, signer.public_key().as_bytes()); + serde_json::to_vec(&serde_json::json!({ + "id": did.as_str(), + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [{ + "id": format!("{}#atproto", did.as_str()), + "type": "Multikey", + "controller": did.as_str(), + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": pds.as_str() + }] + })) + .unwrap() +} + +fn list_records_body(lines: &[&str]) -> Vec { + let records: Vec<_> = lines + .iter() + .map(|line| { + serde_json::json!({ + "value": { + "$type": "org.tangled.key.sshKey", + "publicKey": line, + "name": "laptop", + "createdAt": "2026-06-08T00:00:00Z" + } + }) + }) + .collect(); + serde_json::to_vec(&serde_json::json!({ "records": records })).unwrap() +} + +fn ok_body(body: Vec) -> HttpResponse { + HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from(body), + } +} + +fn fake_dns() -> impl knot_runtime::DnsTxtResolver { + FakeDns::new(|name: &str| { + Ok(match name { + "_atproto.nel.pet" => vec![format!("did={}", owner())], + _ => Vec::new(), + }) + }) +} + +fn not_found() -> HttpResponse { + HttpResponse { + status: http::StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + } +} + +fn forever() -> knot_index::KeyLease { + knot_index::KeyTtl::from_secs(u32::MAX.into()).lease_from(UnixSeconds::new(0)) +} + +fn server_error() -> HttpResponse { + HttpResponse { + status: http::StatusCode::INTERNAL_SERVER_ERROR, + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + } +} + +fn fake_http(published_line: String) -> impl knot_runtime::HttpTransport { + let signer = K256Signer::generate(&SeededEntropy::new(1)); + let pds = pds_url(); + let repo = Did::new_owned(REPO_DID).unwrap(); + let tid_repo = Did::new_owned(TID_REPO_DID).unwrap(); + let owner = Did::new_owned(OWNER_DID).unwrap(); + FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap_or_default().to_string(); + let path = request.url.path().to_string(); + let body = if host == pds.authority() { + list_records_body(&[&published_line]) + } else if path.ends_with(repo.as_str()) { + did_document(&signer, &repo, &pds) + } else if path.ends_with(tid_repo.as_str()) { + did_document(&signer, &tid_repo, &pds) + } else if path.ends_with(owner.as_str()) { + did_document(&signer, &owner, &pds) + } else { + return Ok(not_found()); + }; + Ok(ok_body(body)) + }) +} + +#[derive(Default, Clone)] +struct Accounts { + identities: Arc>>>, + unreachable: Arc>>, + listings: Arc, +} + +impl Accounts { + fn publishing(identities: HashMap>) -> Self { + Self { + identities: Arc::new(Mutex::new(identities)), + ..Self::default() + } + } + + fn unreachable(self, dids: HashSet) -> Self { + *self.unreachable.lock().unwrap() = dids; + self + } + + fn restore(&self, did: &AccountDid) { + self.unreachable.lock().unwrap().remove(did); + } + + fn publish(&self, did: &AccountDid, line: String) { + self.identities + .lock() + .unwrap() + .entry(did.clone()) + .or_default() + .push(line); + } + + fn published_by(&self, did: &AccountDid) -> Vec { + self.identities + .lock() + .unwrap() + .get(did) + .cloned() + .unwrap_or_default() + } + + fn listings(&self) -> usize { + self.listings.load(Ordering::SeqCst) + } +} + +fn multi_http(accounts: Accounts) -> impl knot_runtime::HttpTransport { + let signer = K256Signer::generate(&SeededEntropy::new(77)); + let pds_test = KnotServiceUrl::new("https://pds.test").unwrap(); + FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap_or_default().to_string(); + if host == "plc.directory" { + let did = Did::new_owned(request.url.path().trim_start_matches('/')) + .map(|did| did_document(&signer, &did, &pds_test)) + .unwrap_or_default(); + return Ok(ok_body(did)); + } + if host == "pds.test" { + let repo = request + .url + .query_pairs() + .find(|(key, _)| key == "repo") + .and_then(|(_, value)| AccountDid::new(value).ok()); + accounts.listings.fetch_add(1, Ordering::SeqCst); + if repo + .as_ref() + .is_some_and(|did| accounts.unreachable.lock().unwrap().contains(did)) + { + return Ok(server_error()); + } + let lines = repo + .as_ref() + .map(|did| accounts.published_by(did)) + .unwrap_or_default(); + let refs: Vec<&str> = lines.iter().map(String::as_str).collect(); + return Ok(ok_body(list_records_body(&refs))); + } + Ok(not_found()) + }) +} + +fn actor_for_seed(seed: u64) -> knot_types::ActorId { + knot_types::ActorId::from_secp256k1( + K256Signer::generate(&SeededEntropy::new(seed)) + .public_key() + .as_bytes(), + ) +} + +struct Server { + _scan: TempDir, + layout: Layout, + repo_did: RepoDid, + lfs: knot_lfs::LfsHandle, + port: u16, +} + +async fn spawn_server( + published_line: String, + max_pack_bytes: MaxWireBytes, +) -> (Server, Arc) { + spawn_server_with(published_line, max_pack_bytes, true).await +} + +async fn spawn_server_with( + published_line: String, + max_pack_bytes: MaxWireBytes, + warm: bool, +) -> (Server, Arc) { + let (server, index, _, _, _) = spawn_server_core( + published_line, + max_pack_bytes, + ArchiveLimit::default(), + warm, + false, + ) + .await; + (server, index) +} + +async fn spawn_server_core( + published_line: String, + max_pack_bytes: MaxWireBytes, + archive_limit: ArchiveLimit, + warm: bool, + projection: bool, +) -> ( + Server, + Arc, + tokio_util::sync::CancellationToken, + tokio::task::JoinHandle<()>, + Option>>, +) { + let scan = tempfile::tempdir().unwrap(); + let lfs = { + let lfs_dir = scan.path().join("lfs"); + std::fs::create_dir_all(&lfs_dir).unwrap(); + knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(&lfs_dir), + knot_lfs::LfsSize::new(1 << 30), + knot_lfs::FreeSpaceFloor::new(0), + ) + .unwrap() + }; + let meta_path = scan.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scan.path().join("repos")); + let repo_did = repo_did(); + layout.create(&repo_did).unwrap(); + + let signer = K256Signer::generate(&SeededEntropy::new(2)); + let meta = Repo::open(&meta_path).unwrap(); + let store = CobStore::new(&meta); + let home = CobHome::from(&KnotId::new("did:web:nel.pet").unwrap()); + let registry = store + .create( + &home, + &RegistryChange::Register(Registration { + owner: owner_did(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: repo_name(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(1), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap() + .object; + + let tid_repo_did = tid_repo_did(); + layout.create(&tid_repo_did).unwrap(); + store + .update( + &home, + registry, + &RegistryChange::Register(Registration { + owner: owner_did(), + rkey: tid_rkey(), + name: tid_repo_name(), + repo: tid_repo_did, + created_at: UnixSeconds::new(2), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }), + &signer, + UnixSeconds::new(2), + ) + .unwrap(); + + let index = Arc::new(Index::new(meta_path, layout.clone())); + if warm { + index.rebuild().unwrap(); + } + + let atproto = Arc::new( + Atproto::new( + fake_http(published_line), + ManualClock::new(UnixMicros::new(1_000_000_000)), + KnotId::new("did:web:nel.pet").unwrap(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + ) + .with_dns(Arc::new(fake_dns())), + ); + + let key_dir = scan.path().join("hostkey"); + std::fs::create_dir_all(&key_dir).unwrap(); + let host_key = knot_ssh::load_or_create_host_key(&key_dir.join("host")).unwrap(); + + let (xrpc_state, firehose) = if projection { + let knot = KnotId::new("did:web:nel.pet").unwrap(); + let secrets = Arc::new( + knot_secrets::SealedStore::open( + scan.path().join("keys.sealed"), + &knot_secrets::MasterKey::new([7u8; 32]).unwrap(), + Box::new(knot_runtime::OsEntropy), + ) + .unwrap(), + ); + secrets.ensure(&knot).unwrap(); + let outbox = knot_xrpc::Outbox::open(&scan.path().join("meta"), Default::default()); + let state = Arc::new(knot_xrpc::XrpcState { + imports: knot_xrpc::imports::ImportQueue::channel().0, + handover_sessions: Arc::new(knot_xrpc::handover::HandoverSessions::default()), + layout: layout.clone(), + index: Arc::clone(&index), + atproto: Arc::clone(&atproto), + secrets, + entropy: Arc::new(knot_runtime::OsEntropy), + invite_codes: Default::default(), + admins: std::collections::BTreeSet::new(), + admission: knot_types::AdmissionPolicy::Closed, + contribution_policy: knot_types::ContributionPolicy::Anyone, + writes: Arc::new(knot_resource::SubjectLimiter::new( + knot_resource::RateLimit { + burst: knot_xrpc::Burst::literal(1_000), + refill: knot_xrpc::RefillMicros::literal(1_000), + }, + )), + social_budget: knot_resource::SocialBudgetBytes::new(1 << 28), + knot_repo: knot_xrpc::knot_repo(&knot).unwrap(), + knot_did: knot, + knot_hostname: knot_types::KnotHostname::new("knot.test").unwrap(), + meta_path: scan.path().join("meta"), + knot_service_url: knot_types::KnotServiceUrl::new("https://knot.test".to_string()) + .unwrap(), + limiter: Arc::new(knot_xrpc::PreAuthLimiter::with_config( + knot_xrpc::LimitConfig::default(), + )), + cob_locks: Arc::new(knot_xrpc::CobLocks::default()), + reservations: Arc::new(knot_xrpc::Reservations::new( + knot_xrpc::ReservationTtl::new(1_000_000), + knot_xrpc::PerActorQuota::new(16), + knot_xrpc::GlobalQuota::new(16), + )), + proxy_trust: knot_types::ProxyTrust::default(), + byte_limits: knot_xrpc::ByteLimits::default(), + budgets: knot_xrpc::Budgets::default(), + git_http: Arc::new(knot_runtime::FakeHttp::new( + |_request: &knot_runtime::HttpRequest| Ok(not_found()), + )), + pack_limits: knot_pack::PackLimits::default(), + service_owner: owner(), + legacy_admin_secret: None, + description: Default::default(), + subscriber_gate: Arc::new(knot_events::SubscriberGate::new( + knot_events::GlobalSubscriberLimit::new(16), + knot_events::PerPeerSubscriberLimit::new(4), + )), + maintenance: knot_maintenance::MaintenanceHandle::disabled(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + slots: knot_resource::Slots::testing(8), + lfs: knot_xrpc::LfsWeb::new(lfs.clone(), 8), + blobs: Default::default(), + catalog: Arc::new(knot_messages::Catalog::defaults()), + firehose: Arc::new(knot_events::EventLog::new( + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(4096).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + )), + rkeys: Default::default(), + outbox, + }); + let firehose = Arc::clone(&state.firehose); + (Some(state), Some(firehose)) + } else { + (None, None) + }; + let base = knot_ssh::SshState::new(knot_ssh::SshConfig { + layout: layout.clone(), + index: Arc::clone(&index), + atproto, + knot_actor: actor_for_seed(1), + hostname: knot_types::KnotHostname::new("knot.test").unwrap(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + admins: std::collections::BTreeSet::new(), + admission: knot_types::AdmissionPolicy::Closed, + max_pack_bytes, + archive_limit, + lfs: lfs.clone(), + lfs_max_transfers: 2, + max_inflight_per_peer: 4, + }); + let base = match xrpc_state { + Some(xrpc) => base.with_ref_surface(Arc::new(knot_xrpc::RefProjection { state: xrpc })), + None => base, + }; + let state = Arc::new(base); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let shutdown = tokio_util::sync::CancellationToken::new(); + let serve_task = tokio::spawn({ + let shutdown = shutdown.clone(); + async move { + let _ = knot_ssh::serve_drained(listener, host_key, state, shutdown).await; + } + }); + + ( + Server { + _scan: scan, + layout, + repo_did, + lfs, + port, + }, + index, + shutdown, + serve_task, + firehose, + ) +} + +fn ssh_command(key_path: &str) -> String { + format!("ssh -i {key_path} {UNSHARED_SSH}") +} + +async fn git_ssh(cwd: &Path, key: &str, args: &[&str]) -> (bool, String) { + let ssh = ssh_command(key); + let cwd = cwd.to_path_buf(); + let owned: Vec = args.iter().map(|arg| arg.to_string()).collect(); + tokio::task::spawn_blocking(move || { + let argv: Vec<&str> = owned.iter().map(String::as_str).collect(); + git(&cwd, &[("GIT_SSH_COMMAND", &ssh)], &argv) + }) + .await + .unwrap() +} + +async fn push(work: &Path, url: &str, key: &str, refspecs: &[&str]) -> (bool, String) { + let args: Vec<&str> = std::iter::once("push") + .chain(std::iter::once(url)) + .chain(refspecs.iter().copied()) + .collect(); + git_ssh(work, key, &args).await +} + +async fn clone(url: &str, key: &str, dest: &Path) -> (bool, String) { + git_ssh( + Path::new("/tmp"), + key, + &["clone", "-q", url, dest.to_str().unwrap()], + ) + .await +} + +fn seed_work(work: &Path) -> String { + std::fs::create_dir_all(work).unwrap(); + git(work, &[], &["init", "-q", "-b", "main"]); + std::fs::write(work.join("README.md"), "hello over ssh\n").unwrap(); + git(work, &[], &["add", "-A"]); + git(work, &[], &["commit", "-q", "-m", "initial"]); + let (ok, head) = git(work, &[], &["rev-parse", "HEAD"]); + assert!(ok); + head.trim().to_string() +} + +fn seed_commits(work: &Path, count: usize) { + std::fs::create_dir_all(work).unwrap(); + git(work, &[], &["init", "-q", "-b", "main"]); + (0..count).for_each(|i| { + std::fs::write(work.join("log.txt"), format!("line {i}\n")).unwrap(); + git(work, &[], &["add", "-A"]); + git(work, &[], &["commit", "-q", "-m", &format!("c{i}")]); + }); +} + +fn seed_cob( + work: &Path, + signer_seed: u64, + subject: &AccountDid, + home: &CobHome, +) -> (Oid, String, String) { + let repo = Repo::open(work).unwrap(); + let signer = K256Signer::generate(&SeededEntropy::new(signer_seed)); + let created = CobStore::new(&repo) + .create( + home, + &MembersChange::Add(Grant { + subject: subject.clone(), + added_by: owner(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + let cob_ref = format!( + "refs/cobs/org.tangled.knot.member/{}", + created.object.oid().to_hex() + ); + let spec = format!("{cob_ref}:{cob_ref}"); + (created.tip.oid(), cob_ref, spec) +} + +fn main_tip(layout: &Layout, repo: &RepoDid) -> Option { + layout + .open(repo) + .unwrap() + .find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap() +} + +fn ref_names(server: &Server) -> Vec { + server + .layout + .open(&server.repo_did) + .unwrap() + .references() + .unwrap() + .iter() + .map(|record| record.name.as_str().to_string()) + .collect() +} + +struct Fixture { + scratch: TempDir, + server: Server, + index: Arc, + key_path: String, + url: String, + work: PathBuf, +} + +async fn fixture() -> Fixture { + fixture_with_archive_limit(ArchiveLimit::default()).await +} + +async fn fixture_with_archive_limit(archive_limit: ArchiveLimit) -> Fixture { + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path(), "client"); + let (server, index, _, _, _) = spawn_server_core( + public_line, + MaxWireBytes::new(1 << 30), + archive_limit, + true, + false, + ) + .await; + let url = format!( + "ssh://git@127.0.0.1:{}/{}/{}", + server.port, + owner(), + repo_name() + ); + let work = scratch.path().join("work"); + Fixture { + scratch, + server, + index, + key_path, + url, + work, + } +} + +fn fetch_main_exit(clone_dir: &Path, ssh: &str, extra_git: &[&str]) -> Option { + let mut args = vec!["-k", "3", "20", "git"]; + args.extend_from_slice(extra_git); + args.extend_from_slice(&["fetch", "origin", "main"]); + Command::new("timeout") + .args(&args) + .current_dir(clone_dir) + .env("GIT_SSH_COMMAND", ssh) + .status() + .expect("timeout/git runs") + .code() +} + +async fn incremental_fetch_exit( + seed_count: usize, + extra_git: &'static [&'static str], +) -> Option { + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path(), "client"); + let (server, _index) = spawn_server(public_line, MaxWireBytes::new(1 << 30)).await; + let url = format!( + "ssh://git@127.0.0.1:{}/{}/{}", + server.port, + owner(), + repo_name() + ); + + let work = scratch.path().join("work"); + seed_commits(&work, seed_count); + let (ok, out) = push(&work, &url, &key_path, &["main"]).await; + assert!(ok, "seeding push must land:\n{out}"); + + let clone_dir = scratch.path().join("clone"); + let (ok, out) = clone(&url, &key_path, &clone_dir).await; + assert!(ok, "clone over ssh must succeed:\n{out}"); + + git( + &work, + &[], + &["commit", "-q", "--allow-empty", "-m", "advance"], + ); + let (ok, out) = push(&work, &url, &key_path, &["main"]).await; + assert!(ok, "advancing server tip must succeed:\n{out}"); + + let ssh = ssh_command(&key_path); + let exit = tokio::task::spawn_blocking(move || fetch_main_exit(&clone_dir, &ssh, extra_git)) + .await + .unwrap(); + drop(server); + exit +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn incremental_fetch_over_ssh_completes() { + let cases: [(&'static [&'static str], &str); 2] = [ + ( + &["-c", "protocol.version=0"], + "diverged v0 fetch sends more than 32 haves and blocks on an ACK/NAK. Upload loop \ + answers each have-batch flush with a NAK instead of waiting for done, so it never \ + hangs", + ), + ( + &[], + "git forwards GIT_PROTOCOL over ssh, so default fetch path negotiates with the v2 loop", + ), + ]; + futures::stream::iter(cases) + .for_each(|(extra, rationale)| async move { + assert_eq!( + incremental_fetch_exit(50, extra).await, + Some(0), + "{rationale}" + ); + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn addressing_variants_land_for_rkey_and_tid_repos() { + let fx = fixture().await; + let head = seed_work(&fx.work); + let head_oid = Oid::from_hex(&head).unwrap(); + let port = fx.server.port; + let tid_target = tid_repo_did(); + let rkey_variants = [ + format!("ssh://git@127.0.0.1:{port}/{}/{}", owner(), repo_name()), + format!("ssh://git@127.0.0.1:{port}/{}/{}.git", owner(), repo_name()), + format!("ssh://git@127.0.0.1:{port}/nel.pet/{}", repo_name()), + format!("ssh://git@127.0.0.1:{port}/{}", repo_did()), + ]; + let tid_variants = [ + format!("ssh://git@127.0.0.1:{port}/{}/{}", owner(), tid_repo_name()), + format!( + "ssh://git@127.0.0.1:{port}/{}/{}.git", + owner(), + tid_repo_name() + ), + format!("ssh://git@127.0.0.1:{port}/nel.pet/{}", tid_repo_name()), + format!("ssh://git@127.0.0.1:{port}/{}/{}", owner(), tid_rkey()), + ]; + let fx = &fx; + let rkey_target = &fx.server.repo_did; + futures::stream::iter(rkey_variants) + .for_each(|url| async move { + let (ok, out) = push(&fx.work, &url, &fx.key_path, &["main"]).await; + assert!(ok, "addressing {url} must resolve and push:\n{out}"); + assert_eq!( + main_tip(&fx.server.layout, rkey_target), + Some(head_oid), + "{url}: pushed commit must be the repository's main tip" + ); + }) + .await; + let tid_target = &tid_target; + futures::stream::iter(tid_variants) + .for_each(|url| async move { + let (ok, out) = push(&fx.work, &url, &fx.key_path, &["main"]).await; + assert!( + ok, + "a PDS-minted record key leaves the display name as the only human \ + path, so {url} must resolve and push:\n{out}" + ); + assert_eq!( + main_tip(&fx.server.layout, tid_target), + Some(head_oid), + "{url}: pushed commit must be that repository's main tip" + ); + }) + .await; + + let (ok, out) = push( + &fx.work, + &format!("ssh://git@127.0.0.1:{port}/{}/whelk", owner()), + &fx.key_path, + &["main"], + ) + .await; + assert!( + !ok, + "a segment that matches no record key and no name stays unresolvable:\n{out}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_push_while_the_index_is_warming_is_refused() { + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path(), "client"); + let (server, _index) = spawn_server_with(public_line, MaxWireBytes::new(1 << 30), false).await; + let url = format!( + "ssh://git@127.0.0.1:{}/{}/{}", + server.port, + owner(), + repo_name() + ); + + let work = scratch.path().join("work"); + seed_work(&work); + let (ok, out) = push(&work, &url, &key_path, &["main"]).await; + assert!( + !ok, + "warming index must fail closed at the SSH boundary:\n{out}" + ); + assert_eq!( + main_tip(&server.layout, &server.repo_did), + None, + "no ref lands while index is warming" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn unresolvable_targets_refused() { + let fx = fixture().await; + seed_work(&fx.work); + let port = fx.server.port; + + let bad_name = format!("ssh://git@127.0.0.1:{port}/{}/conch", owner()); + let (ok, out) = push(&fx.work, &bad_name, &fx.key_path, &["main"]).await; + assert!( + !ok, + "owner/name with no registry entry must be rejected, not silently routed:\n{out}" + ); + + fx.server + .layout + .create(&RepoDid::new("did:plc:clam").unwrap()) + .unwrap(); + let ghost_url = format!("ssh://git@127.0.0.1:{port}/did:plc:clam"); + let dest = fx.scratch.path().join("ghost"); + let (ok, out) = clone(&ghost_url, &fx.key_path, &dest).await; + assert!( + !ok, + "repo present on disk but absent from registry mustn't be served by bare DID:\n{out}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_authorized_push_over_ssh_succeeds_and_a_clone_reads_it_back() { + let fx = fixture().await; + let head = seed_work(&fx.work); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "authorized push over ssh must succeed:\n{out}"); + assert_eq!( + main_tip(&fx.server.layout, &fx.server.repo_did), + Some(Oid::from_hex(&head).unwrap()), + "pushed commit must be the repository's main tip" + ); + + let clone_dir = fx.scratch.path().join("clone"); + let (ok, out) = clone(&fx.url, &fx.key_path, &clone_dir).await; + assert!(ok, "clone over ssh must succeed:\n{out}"); + assert!( + clone_dir.join("README.md").exists(), + "clone must check out the pushed file" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_client_requesting_ssh_compression_clones_an_incompressible_pack() { + let fx = fixture().await; + std::fs::create_dir_all(&fx.work).unwrap(); + git(&fx.work, &[], &["init", "-q", "-b", "main"]); + let mut state = 0x9e3779b97f4a7c15u64; + let payload: Vec = std::iter::repeat_with(|| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state.to_le_bytes() + }) + .take(32 * 1024) + .flatten() + .collect(); + std::fs::write(fx.work.join("noise.bin"), &payload).unwrap(); + git(&fx.work, &[], &["add", "-A"]); + git(&fx.work, &[], &["commit", "-q", "-m", "noise"]); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "push must succeed:\n{out}"); + + let dest = fx.scratch.path().join("compressed-clone"); + let ssh = format!("{} -o Compression=yes", ssh_command(&fx.key_path)); + let url = fx.url.clone(); + let dest_arg = dest.to_str().unwrap().to_string(); + let (ok, out) = tokio::task::spawn_blocking(move || { + git( + Path::new("/tmp"), + &[("GIT_SSH_COMMAND", &ssh)], + &["clone", "-q", &url, &dest_arg], + ) + }) + .await + .unwrap(); + assert!( + ok, + "clone with ssh compression requested must succeed:\n{out}" + ); + assert_eq!(std::fs::read(dest.join("noise.bin")).unwrap(), payload); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_oversized_push_is_refused_at_the_ssh_boundary() { + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path(), "client"); + let (server, _index) = spawn_server(public_line, MaxWireBytes::new(64)).await; + let url = format!( + "ssh://git@127.0.0.1:{}/{}/{}", + server.port, + owner(), + repo_name() + ); + + let work = scratch.path().join("work"); + seed_work(&work); + let (ok, out) = push(&work, &url, &key_path, &["main"]).await; + assert!( + !ok, + "push larger than the configured limit must be refused:\n{out}" + ); + assert!( + ref_names(&server).is_empty(), + "oversized push mustn't land any ref" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_up_to_date_push_over_ssh_is_accepted() { + let fx = fixture().await; + seed_work(&fx.work); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "first push must land:\n{out}"); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!( + ok, + "up-to-date no-op push must succeed instead of failing with a stream error:\n{out}" + ); + assert!( + out.contains("up-to-date") || out.contains("up to date"), + "git must report branch is up to date:\n{out}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_denied_push_over_ssh_leaves_no_objects_in_the_live_odb() { + let scratch = tempfile::tempdir().unwrap(); + let (_registered_path, registered_line) = keygen(scratch.path(), "registered"); + let (attacker_path, _attacker_line) = keygen(scratch.path(), "attacker"); + let (server, _index) = spawn_server(registered_line, MaxWireBytes::new(1 << 30)).await; + let url = format!( + "ssh://git@127.0.0.1:{}/{}/{}", + server.port, + owner(), + repo_name() + ); + + let work = scratch.path().join("work"); + let head = seed_work(&work); + let (ok, out) = push(&work, &url, &attacker_path, &["main"]).await; + assert!(!ok, "unauthorized push must be rejected:\n{out}"); + + let repo = server.layout.open(&server.repo_did).unwrap(); + assert!( + repo.references().unwrap().is_empty(), + "denied push must create no ref" + ); + assert!( + !repo.contains(Oid::from_hex(&head).unwrap()), + "denied push must migrate no objects into the live odb" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_namespace_policy() { + let fx = fixture().await; + seed_work(&fx.work); + + for (spec, names_itself) in [ + ("main:refs/hidden/feature/main", "refs/hidden/* is reserved"), + ("main:refs/atproto/commit", "atproto commit chain"), + ( + "main:refs/cob-checkpoints/org.tangled.knot.member/beef", + "knot-written collaborative-object snapshots", + ), + ] { + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &[spec]).await; + assert!( + !ok, + "the ssh path screens namespaces too, so {spec} must be refused:\n{out}" + ); + assert!( + out.contains(names_itself), + "git must show the pusher which namespace refused {spec}:\n{out}" + ); + assert!( + ref_names(&fx.server).is_empty(), + "a push refused by policy must leave the ref store empty" + ); + } + + let (ok, out) = push( + &fx.work, + &fx.url, + &fx.key_path, + &["main:refs/notes/commits"], + ) + .await; + assert!( + ok, + "push to any non-reserved namespace must be accepted:\n{out}" + ); + assert!( + ref_names(&fx.server) + .iter() + .any(|name| name == "refs/notes/commits"), + "pushed ref must land" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cob_ref_guard_lifecycle() { + let fx = fixture().await; + seed_work(&fx.work); + let home = CobHome::from(&repo_did()); + let foreign = CobHome::from(&RepoDid::new("did:plc:whelk").unwrap()); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "head must land for the advertisement check:\n{out}"); + + let (owned_tip, owned_ref, owned_spec) = seed_cob( + &fx.work, + 1, + &AccountDid::new("did:plc:limpet").unwrap(), + &home, + ); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &[owned_spec.as_str()]).await; + assert!( + ok, + "COB ref signed by the repository key must verify and land over ssh:\n{out}" + ); + + let (_forged_tip, forged_ref, forged_spec) = seed_cob( + &fx.work, + 9, + &AccountDid::new("did:plc:whelk").unwrap(), + &home, + ); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &[forged_spec.as_str()]).await; + assert!( + !ok, + "COB ref signed by a stranger must be refused at the receive boundary:\n{out}" + ); + + let (_transplant_tip, transplant_ref, transplant_spec) = seed_cob( + &fx.work, + 1, + &AccountDid::new("did:plc:mussel").unwrap(), + &foreign, + ); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &[transplant_spec.as_str()]).await; + assert!( + !ok, + "same key signing for another repo's home must be refused on transplant:\n{out}" + ); + + let landed = ref_names(&fx.server); + assert!( + landed.contains(&owned_ref), + "owner-signed COB ref must be stored: {landed:?}" + ); + assert!( + !landed.contains(&forged_ref), + "stranger-signed COB ref must be absent: {landed:?}" + ); + assert!( + !landed.contains(&transplant_ref), + "transplanted COB ref must be absent: {landed:?}" + ); + + let cob_name = RefName::new(&owned_ref).unwrap(); + let del = format!(":{owned_ref}"); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &[del.as_str()]).await; + assert!(!ok, "deleting a COB ref must be refused:\n{out}"); + assert!( + out.contains("append-only"), + "rejection must name the append-only rule:\n{out}" + ); + assert!( + ref_names(&fx.server).contains(&owned_ref), + "COB ref must survive the refused delete" + ); + + let repo = Repo::open(&fx.work).unwrap(); + CobStore::new(&repo) + .update( + &home, + knot_types::CobId::new(owned_tip), + &MembersChange::Add(Grant { + subject: AccountDid::new("did:plc:bailey").unwrap(), + added_by: owner(), + created_at: UnixSeconds::new(2), + }), + &K256Signer::generate(&SeededEntropy::new(1)), + UnixSeconds::new(2), + ) + .unwrap(); + assert_ne!( + repo.find_ref(&cob_name).unwrap(), + Some(owned_tip), + "local COB ref now points at a new, equally valid tip" + ); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &[owned_spec.as_str()]).await; + assert!( + !ok, + "re-pushing a moved COB ref must be refused instead of silently clobbered:\n{out}" + ); + assert_eq!( + fx.server + .layout + .open(&fx.server.repo_did) + .unwrap() + .find_ref(&cob_name) + .unwrap(), + Some(owned_tip), + "live COB ref must still point at the original tip" + ); + + let (ok, advert) = git_ssh(Path::new("/tmp"), &fx.key_path, &["ls-remote", &fx.url]).await; + assert!(ok, "ls-remote over ssh must succeed:\n{advert}"); + assert!( + advert.contains("refs/heads/main"), + "head must be advertised:\n{advert}" + ); + assert!( + !advert.contains("refs/cobs/"), + "no refs/cobs/* may leak into the ssh advertisement:\n{advert}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_filled_key_set_refuses_an_unregistered_key_and_an_acl_write_reopens_the_check() { + let fx = fixture().await; + let head = seed_work(&fx.work); + let head_oid = Oid::from_hex(&head).unwrap(); + + fx.index.keys().mark_ready(fx.index.generation()); + fx.index.refresh_members().unwrap(); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!( + ok, + "a grant written after the key set was read must reopen the check, or whoever it \ + grants is refused at the handshake until the next fill pass:\n{out}" + ); + assert_eq!( + main_tip(&fx.server.layout, &fx.server.repo_did), + Some(head_oid) + ); + + fx.index.keys().record( + &owner(), + vec![knot_types::OfferedKey::from_bytes(registered_blob(&fx))], + forever(), + ); + fx.index.keys().mark_ready(fx.index.generation()); + + let (unregistered_path, _unregistered_line) = keygen(fx.scratch.path(), "unregistered"); + let two_ids = format!( + "ssh -i {unregistered_path} -i {} {UNSHARED_SSH}", + fx.key_path + ); + let (ok, out) = { + let (work, url) = (fx.work.clone(), fx.url.clone()); + tokio::task::spawn_blocking(move || { + git( + &work, + &[("GIT_SSH_COMMAND", &two_ids)], + &["push", "-q", &url, "main:refs/heads/second"], + ) + }) + .await + .unwrap() + }; + assert!( + ok, + "a filled key set refuses the unregistered key, so the client offers its registered key \ + without the url identifying anybody:\n{out}" + ); + assert_eq!( + fx.server + .layout + .open(&fx.server.repo_did) + .unwrap() + .find_ref(&RefName::new("refs/heads/second").unwrap()) + .unwrap(), + Some(head_oid) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn key_recognition_edge_cases() { + let fx = fixture().await; + let head = seed_work(&fx.work); + let head_oid = Oid::from_hex(&head).unwrap(); + + let (unregistered_path, _unregistered_line) = keygen(fx.scratch.path(), "unregistered"); + let two_ids = format!( + "ssh -i {unregistered_path} -i {} {UNSHARED_SSH}", + fx.key_path + ); + let (ok, out) = { + let (work, url) = (fx.work.clone(), fx.url.clone()); + tokio::task::spawn_blocking(move || { + git( + &work, + &[("GIT_SSH_COMMAND", &two_ids)], + &["push", "-q", &url, "main"], + ) + }) + .await + .unwrap() + }; + assert!( + !ok, + "with the key set still filling, the push is checked against whichever key the client \ + offers first:\n{out}" + ); + assert!( + out.contains("@nel.pet"), + "refusal lists who may push, so the pusher knows which key to offer:\n{out}" + ); + assert!( + out.contains("IdentitiesOnly"), + "refusal states how a multi-key client can offer its registered key:\n{out}" + ); + assert_eq!( + main_tip(&fx.server.layout, &fx.server.repo_did), + None, + "the refused push leaves the repo empty" + ); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "offering only the registered key must succeed:\n{out}"); + assert_eq!( + main_tip(&fx.server.layout, &fx.server.repo_did), + Some(head_oid) + ); + + let blob = russh::keys::ssh_key::PublicKey::from_openssh( + &std::fs::read_to_string(fx.scratch.path().join("client.pub")).unwrap(), + ) + .unwrap() + .to_bytes() + .unwrap(); + fx.index.keys().record( + &AccountDid::new("did:plc:cuttle").unwrap(), + vec![knot_types::OfferedKey::from_bytes(blob)], + forever(), + ); + let (ok, out) = push( + &fx.work, + &fx.url, + &fx.key_path, + &["main:refs/heads/squat-check"], + ) + .await; + assert!( + ok, + "stranger who published the owner's key mustn't deny the owner's push:\n{out}" + ); + assert_eq!( + fx.server + .layout + .open(&fx.server.repo_did) + .unwrap() + .find_ref(&RefName::new("refs/heads/squat-check").unwrap()) + .unwrap(), + Some(head_oid) + ); +} + +#[test] +fn a_group_or_other_readable_host_key_is_refused_on_load() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("host"); + knot_ssh::load_or_create_host_key(&path).unwrap(); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600, + "freshly created host key is 0600" + ); + + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + let refused = knot_ssh::load_or_create_host_key(&path); + assert!( + matches!(refused, Err(knot_ssh::SshError::HostKey { .. })), + "world-readable existing host key must be refused on load: {refused:?}" + ); +} + +async fn launch(host_key_dir: &Path, layout: Layout, index: Arc, accounts: Accounts) -> u16 { + let atproto = Arc::new(Atproto::new( + multi_http(accounts), + ManualClock::new(UnixMicros::new(1_000_000_000)), + KnotId::new("did:web:nel.pet").unwrap(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + )); + std::fs::create_dir_all(host_key_dir).unwrap(); + let host_key = knot_ssh::load_or_create_host_key(&host_key_dir.join("host")).unwrap(); + let lfs = { + let lfs_dir = host_key_dir.join("lfs"); + std::fs::create_dir_all(&lfs_dir).unwrap(); + knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(&lfs_dir), + knot_lfs::LfsSize::new(64 * 1024 * 1024), + knot_lfs::FreeSpaceFloor::new(0), + ) + .unwrap() + }; + let state = Arc::new(knot_ssh::SshState::new(knot_ssh::SshConfig { + layout, + index, + atproto, + knot_actor: actor_for_seed(77), + hostname: knot_types::KnotHostname::new("knot.test").unwrap(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + admins: std::collections::BTreeSet::new(), + admission: knot_types::AdmissionPolicy::Closed, + max_pack_bytes: MaxWireBytes::new(1 << 30), + archive_limit: ArchiveLimit::default(), + lfs, + lfs_max_transfers: 2, + max_inflight_per_peer: 4, + })); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let _ = knot_ssh::serve_on_socket(listener, host_key, state).await; + }); + port +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_handle_in_the_url_identifies_a_visitor_and_lets_a_multi_key_client_find_its_key() { + let fx = fixture().await; + let head = seed_work(&fx.work); + let head_oid = Oid::from_hex(&head).unwrap(); + + let port = fx.server.port; + let greeted_key = fx.key_path.clone(); + let (_ok, out) = + tokio::task::spawn_blocking(move || ssh_bare_as(&greeted_key, "nel.pet", port)) + .await + .unwrap(); + assert!( + out.contains("@nel.pet"), + "an asserted handle identifies the visitor on first contact, with an empty cache:\n{out}" + ); + + let (unregistered_path, _unregistered_line) = keygen(fx.scratch.path(), "unregistered"); + let two_ids = format!( + "ssh -i {unregistered_path} -i {} {UNSHARED_SSH}", + fx.key_path + ); + let identified = format!("ssh://nel.pet@127.0.0.1:{}/{}", fx.server.port, repo_did()); + let (ok, out) = { + let (work, url) = (fx.work.clone(), identified.clone()); + tokio::task::spawn_blocking(move || { + git( + &work, + &[("GIT_SSH_COMMAND", &two_ids)], + &["push", "-q", &url, "main"], + ) + }) + .await + .unwrap() + }; + assert!( + ok, + "a handle in the url lets the knot refuse the unregistered key so the client offers the \ + next key:\n{out}" + ); + assert_eq!( + main_tip(&fx.server.layout, &fx.server.repo_did), + Some(head_oid) + ); + assert_eq!( + fx.index.owner_of_key( + &knot_types::OfferedKey::from_bytes(registered_blob(&fx)), + UnixSeconds::new(0), + ), + Resolved::Ready(None), + "an asserted handle is whatever the client typed, so the keys read for it mustn't enter \ + the set, or anyone can fill the key budget by asserting handles" + ); +} + +// every miss spends a lookup token, and with the clock frozen the budget never refills +const LOOKUP_BURST: usize = 8; + +struct Drained { + scratch: TempDir, + layout: Layout, + repo_did: RepoDid, + accounts: Accounts, + port: u16, + owner_key: String, + late_key: String, +} + +// git mangles a did's colons in an ssh url, so the user goes in through -l instead +async fn push_as(work: &Path, fx: &Drained, user: &AccountDid, key: &str) -> (bool, String) { + let ssh = format!("ssh -i {key} -l {user} {UNSHARED_SSH}"); + let url = format!("ssh://127.0.0.1:{}/{}", fx.port, fx.repo_did); + let work = work.to_path_buf(); + tokio::task::spawn_blocking(move || { + git(&work, &[("GIT_SSH_COMMAND", &ssh)], &["push", &url, "main"]) + }) + .await + .unwrap() +} + +fn late_did() -> AccountDid { + AccountDid::new("did:plc:late").unwrap() +} + +// the owner's keys get cached, then other accounts use up the peer's budget. the key set +// doesn't have either published key, so a push can only get in through its claim +async fn drained_peer() -> Drained { + let scratch = tempfile::tempdir().unwrap(); + let (owner_key, owner_line) = keygen(scratch.path(), "owner"); + let (late_key, late_line) = keygen(scratch.path(), "late"); + let (_, other_line) = keygen(scratch.path(), "other"); + let (layout, repo_did, index) = registered_index(&scratch, knot_index::KeyBudget::DEFAULT); + let other_blob = russh::keys::ssh_key::PublicKey::from_openssh(&other_line) + .unwrap() + .to_bytes() + .unwrap(); + index.keys().record( + &owner(), + vec![knot_types::OfferedKey::from_bytes(other_blob)], + forever(), + ); + index.keys().mark_ready(index.generation()); + + let accounts = Accounts::publishing(HashMap::from([ + (owner(), vec![owner_line]), + (late_did(), vec![late_line]), + ])); + let port = launch( + &scratch.path().join("hostkey"), + layout.clone(), + Arc::clone(&index), + accounts.clone(), + ) + .await; + let mut users = vec![owner().to_string()]; + users.extend((0..LOOKUP_BURST).map(|n| format!("did:plc:stranger{n}"))); + for user in users { + let key = owner_key.clone(); + tokio::task::spawn_blocking(move || ssh_bare_as(&key, &user, port)) + .await + .unwrap(); + } + Drained { + scratch, + layout, + repo_did, + accounts, + port, + owner_key, + late_key, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_cached_claim_still_pushes_when_the_lookup_budget_is_empty() { + let fx = drained_peer().await; + let before = fx.accounts.listings(); + + let work = fx.scratch.path().join("work"); + let head = seed_work(&work); + let (ok, out) = push_as(&work, &fx, &owner(), &fx.owner_key).await; + assert!( + ok, + "the owner's keys are cached, so the claim needs no lookup and no token, and an empty \ + budget mustn't refuse a key the key set hasn't indexed yet:\n{out}" + ); + assert_eq!( + main_tip(&fx.layout, &fx.repo_did), + Some(Oid::from_hex(&head).unwrap()) + ); + assert_eq!( + fx.accounts.listings(), + before, + "a cache hit must not reach the pds" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_uncached_claim_over_budget_is_refused_without_a_lookup() { + let fx = drained_peer().await; + let before = fx.accounts.listings(); + + let work = fx.scratch.path().join("work"); + seed_work(&work); + let (ok, out) = push_as(&work, &fx, &late_did(), &fx.late_key).await; + assert!( + !ok, + "a claim that would need a lookup the budget can't pay for mustn't be answered:\n{out}" + ); + assert_eq!( + fx.accounts.listings(), + before, + "an over-budget peer must not get a lookup" + ); +} + +fn registered_blob(fx: &Fixture) -> Vec { + russh::keys::ssh_key::PublicKey::from_openssh( + &std::fs::read_to_string(fx.scratch.path().join("client.pub")).unwrap(), + ) + .unwrap() + .to_bytes() + .unwrap() +} + +fn registered_index( + scratch: &TempDir, + budget: knot_index::KeyBudget, +) -> (Layout, RepoDid, Arc) { + let meta_path = scratch.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scratch.path().join("repos")); + let repo_did = repo_did(); + layout.create(&repo_did).unwrap(); + + let signer = K256Signer::generate(&SeededEntropy::new(2)); + let meta = Repo::open(&meta_path).unwrap(); + CobStore::new(&meta) + .create( + &CobHome::from(&KnotId::new("did:web:nel.pet").unwrap()), + &RegistryChange::Register(Registration { + owner: owner_did(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: repo_name(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(1), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + + let index = Arc::new(Index::with_key_budget(meta_path, layout.clone(), budget)); + index.rebuild().unwrap(); + index.warm_collaborators(); + (layout, repo_did, index) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_unreachable_pds_reads_as_transient_and_its_keys_get_in_once_it_recovers() { + let scratch = tempfile::tempdir().unwrap(); + let (stale_key, stale_line) = keygen(scratch.path(), "stale"); + let (fresh_key, fresh_line) = keygen(scratch.path(), "fresh"); + let (layout, repo_did, index) = registered_index(&scratch, knot_index::KeyBudget::DEFAULT); + + let accounts = Accounts::publishing(HashMap::from([(owner(), vec![stale_line])])) + .unreachable(HashSet::from([owner()])); + let port = launch( + &scratch.path().join("hostkey"), + layout.clone(), + Arc::clone(&index), + accounts.clone(), + ) + .await; + let url = format!("ssh://git@127.0.0.1:{port}/{}", repo_did); + + let work = scratch.path().join("work"); + let head = seed_work(&work); + let (ok, out) = push(&work, &url, &stale_key, &["main"]).await; + assert!( + !ok, + "a push mustn't be accepted while the owner's records are unreadable:\n{out}" + ); + assert!( + out.contains("retry shortly"), + "an unreadable PDS must read as transient:\n{out}" + ); + assert!( + !out.contains("doesn't match"), + "a transient failure mustn't be reported to the pusher as a wrong key:\n{out}" + ); + assert_eq!( + main_tip(&layout, &repo_did), + None, + "the refused push leaves the repo empty" + ); + + accounts.restore(&owner()); + let (ok, out) = push(&work, &url, &stale_key, &["main"]).await; + assert!( + ok, + "the key the owner publishes must push once its PDS answers again:\n{out}" + ); + + accounts.publish(&owner(), fresh_line); + let (ok, out) = push(&work, &url, &fresh_key, &["main", "--force"]).await; + assert!( + ok, + "a key the owner published after the knot last read the account must get in on the next \ + push, or publishing a second key locks its owner out until a fill pass catches up:\n{out}" + ); + assert_eq!( + main_tip(&layout, &repo_did), + Some(Oid::from_hex(&head).unwrap()) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_account_the_budget_couldnt_fit_still_clears_the_handshake_and_pushes() { + let scratch = tempfile::tempdir().unwrap(); + let (owner_key, owner_line) = keygen(scratch.path(), "owner"); + let (layout, repo_did, index) = + registered_index(&scratch, knot_index::KeyBudget::from_bytes(200)); + + let blob = russh::keys::ssh_key::PublicKey::from_openssh(&owner_line) + .unwrap() + .to_bytes() + .unwrap(); + assert_eq!( + index.keys().record( + &owner(), + vec![knot_types::OfferedKey::from_bytes(blob)], + forever(), + ), + knot_index::KeyRecord::Unheld, + "a 200-byte budget records the read without keeping the key" + ); + index.keys().mark_ready(index.generation()); + + let port = launch( + &scratch.path().join("hostkey"), + layout.clone(), + Arc::clone(&index), + Accounts::publishing(HashMap::from([(owner(), vec![owner_line])])), + ) + .await; + let url = format!("ssh://git@127.0.0.1:{port}/{}", repo_did); + + let work = scratch.path().join("work"); + let head = seed_work(&work); + let (ok, out) = push(&work, &url, &owner_key, &["main"]).await; + assert!( + ok, + "the set can't fit the owner's keys, so the handshake must defer to the push check \ + instead of refusing a key the accounts on file don't publish:\n{out}" + ); + assert_eq!( + main_tip(&layout, &repo_did), + Some(Oid::from_hex(&head).unwrap()) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_collaborator_pushes_its_repo_but_is_denied_on_a_repo_it_doesnt_collaborate_on() { + let scratch = tempfile::tempdir().unwrap(); + let (owner_key, owner_line) = keygen(scratch.path(), "owner"); + let (collab_key, collab_line) = keygen(scratch.path(), "collab"); + + let meta_path = scratch.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scratch.path().join("repos")); + let repo_a = repo_did(); + let repo_b = RepoDid::new("did:plc:clam").unwrap(); + let git_a = layout.create(&repo_a).unwrap(); + layout.create(&repo_b).unwrap(); + + let signer = K256Signer::generate(&SeededEntropy::new(2)); + let meta = Repo::open(&meta_path).unwrap(); + let store = CobStore::new(&meta); + let knot_home = CobHome::from(&KnotId::new("did:web:nel.pet").unwrap()); + let reg = store + .create( + &knot_home, + &RegistryChange::Register(Registration { + owner: owner_did(), + rkey: RepoRkey::new("anemone").unwrap(), + name: RepoName::new("anemone").unwrap(), + repo: repo_a.clone(), + created_at: UnixSeconds::new(1), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + store + .update( + &knot_home, + reg.object, + &RegistryChange::Register(Registration { + owner: owner_did(), + rkey: RepoRkey::new("barnacle").unwrap(), + name: RepoName::new("barnacle").unwrap(), + repo: repo_b.clone(), + created_at: UnixSeconds::new(2), + policy: knot_types::ContributionPolicy::default(), + fostered: false, + retired_at: None, + }), + &signer, + UnixSeconds::new(2), + ) + .unwrap(); + store + .create( + &knot_home, + &MembersChange::Add(Grant { + subject: new_collab(), + added_by: owner(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + CobStore::new(&git_a) + .create( + &CobHome::from(&repo_a), + &CollaboratorsChange::Add(Grant { + subject: new_collab(), + added_by: owner(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + + let index = Arc::new(Index::new(meta_path, layout.clone())); + index.rebuild().unwrap(); + index.warm_collaborators(); + + let identities = HashMap::from([ + (owner(), vec![owner_line]), + (new_collab(), vec![collab_line]), + ]); + let accounts = Accounts::publishing(identities); + let port = launch( + &scratch.path().join("hostkey"), + layout.clone(), + Arc::clone(&index), + accounts.clone(), + ) + .await; + + let work_a = scratch.path().join("work_a"); + let head_a = seed_work(&work_a); + let url_a = format!("ssh://git@127.0.0.1:{port}/{}", repo_a); + let (ok, out) = push(&work_a, &url_a, &collab_key, &["main"]).await; + assert!( + ok, + "collaborator must push the repo it collaborates on:\n{out}" + ); + assert_eq!( + main_tip(&layout, &repo_a), + Some(Oid::from_hex(&head_a).unwrap()), + "collaborator's commit must be repo A's main tip" + ); + + let work_b = scratch.path().join("work_b"); + seed_work(&work_b); + let url_b = format!("ssh://git@127.0.0.1:{port}/{}", repo_b); + let (denied, out) = push(&work_b, &url_b, &collab_key, &["main"]).await; + assert!( + !denied, + "a key that pushes repo A must be denied on repo B, where its owner was never granted:\n{out}" + ); + assert!( + main_tip(&layout, &repo_b).is_none(), + "denied cross-repo push must land nothing on repo B" + ); + + let after_first_denial = accounts.listings(); + let (denied, out) = push(&work_b, &url_b, &collab_key, &["main"]).await; + assert!(!denied, "the second attempt is denied the same way:\n{out}"); + assert_eq!( + accounts.listings(), + after_first_denial, + "repo B's owner was read during the first denial and is on file, so retrying mustn't \ + read that PDS again, or anyone with a key can make the knot fetch from a third party \ + at will:\n{out}" + ); + + let work_owner = scratch.path().join("work_owner_b"); + let head_owner = seed_work(&work_owner); + let (ok, out) = push(&work_owner, &url_b, &owner_key, &["main"]).await; + assert!(ok, "owner must push to repo B:\n{out}"); + assert_eq!( + main_tip(&layout, &repo_b), + Some(Oid::from_hex(&head_owner).unwrap()), + "owner's push to repo B must land, isolating the collaborator's denial as authorization" + ); +} + +const NEW_COLLAB: &str = "did:plc:olaren"; +const COLLAB_BYSTANDER: &str = "did:plc:bailey"; + +fn new_collab() -> AccountDid { + AccountDid::new(NEW_COLLAB).unwrap() +} + +fn collab_bystander() -> AccountDid { + AccountDid::new(COLLAB_BYSTANDER).unwrap() +} + +struct Invitation { + scratch: TempDir, + layout: Layout, + url: String, + head: Oid, + collab_key: String, + bystander_key: String, + accounts: Accounts, +} + +impl Invitation { + fn tip(&self) -> Option { + main_tip(&self.layout, &repo_did()) + } + async fn push(&self, key: &str) -> (bool, String) { + let work = self.scratch.path().join("work"); + push(&work, &self.url, key, &["main"]).await + } +} + +async fn a_new_collaborator(pds_down: bool) -> Invitation { + let scratch = tempfile::tempdir().unwrap(); + let dir = scratch.path(); + let (_owner_key, owner_line) = keygen(dir, "owner"); + let (collab_key, collab_line) = keygen(dir, "collab"); + let (bystander_key, bystander_line) = keygen(dir, "bystander"); + let (layout, repo, index) = registered_index(&scratch, knot_index::KeyBudget::DEFAULT); + let head = Oid::from_hex(&seed_work(&dir.join("work"))).unwrap(); + let now = UnixSeconds::new(1); + let invited = CollaboratorsChange::Add(Grant { + subject: new_collab(), + added_by: owner(), + created_at: now, + }); + let signer = K256Signer::generate(&SeededEntropy::new(2)); + CobStore::new(&layout.open(&repo).unwrap()) + .create(&CobHome::from(&repo), &invited, &signer, now) + .unwrap(); + index.refresh_collaborators(&repo).unwrap(); + + let held = |line: &str| { + let key = russh::keys::ssh_key::PublicKey::from_openssh(line).unwrap(); + vec![knot_types::OfferedKey::from_bytes(key.to_bytes().unwrap())] + }; + let bystander = collab_bystander(); + let keys = index.keys(); + keys.record(&owner(), held(&owner_line), forever()); + keys.record(&bystander, held(&bystander_line), forever()); + + let records = Accounts::publishing(HashMap::from([ + (owner(), vec![owner_line]), + (new_collab(), vec![collab_line]), + ])); + let accounts = match pds_down { + false => records, + true => records.unreachable(HashSet::from([new_collab()])), + }; + let port = launch(dir, layout.clone(), Arc::clone(&index), accounts.clone()).await; + Invitation { + url: format!("ssh://git@127.0.0.1:{port}/{REPO_DID}"), + scratch, + layout, + head, + collab_key, + bystander_key, + accounts, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_new_collaborator_pushes_once_the_knot_reads_their_keys() { + let invitation = a_new_collaborator(false).await; + let (ok, out) = invitation.push(&invitation.collab_key).await; + assert!( + ok, + "the grant list admits this account and the push is what makes the knot read their keys, \ + so the client doesn't need a second call:\n{out}" + ); + assert_eq!(invitation.tip(), Some(invitation.head)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn wide_door_admits_whichever_key_a_new_collaborator_offers_first() { + let invitation = a_new_collaborator(false).await; + let (decoy, _line) = keygen(invitation.scratch.path(), "decoy"); + let offered = format!("{decoy} -i {}", invitation.collab_key); + let (ok, out) = invitation.push(&offered).await; + assert!( + !ok, + "the ssh username is git and the repository arrives in the exec command, so at key \ + identification the knot can't know which grant list to weigh a key against:\n{out}" + ); + assert!( + out.contains("IdentitiesOnly"), + "accepting the offer ends the client's key search, so the refusal has to tell a \ + collaborator holding two keys how to offer their registered one:\n{out}" + ); + assert!( + invitation.tip().is_none(), + "main moved, so the decoy key pushed" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_new_collaborators_first_push_survives_a_strangers_failed_one() { + let invitation = a_new_collaborator(true).await; + let (ok, out) = invitation.push(&invitation.bystander_key).await; + assert!( + !ok, + "bystander's push is refused, and the grant list doesn't list their key anywhere" + ); + assert!( + out.contains("couldn't read the account records"), + "key is on file: session doesn't cost a miss reservation, and fan-out runs, which is what spends the collaborator's pacer while their pds is down:\n{out}" + ); + + invitation.accounts.restore(&new_collab()); + let (ok, out) = invitation.push(&invitation.collab_key).await; + assert!( + ok, + "a collaborator with no keys on file is read on every attempt, so a pacer somebody else \ + spent can't turn their first push into an unregistered key:\n{out}" + ); + assert_eq!(invitation.tip(), Some(invitation.head)); +} + +fn ssh_bare(key_path: &str, port: u16) -> (bool, String) { + ssh_bare_as(key_path, "git", port) +} + +fn ssh_bare_as(key_path: &str, user: &str, port: u16) -> (bool, String) { + let out = Command::new("ssh") + .args(["-i", key_path]) + .args(UNSHARED_SSH.split_whitespace()) + .args(["-p", &port.to_string(), &format!("{user}@127.0.0.1")]) + .output() + .expect("ssh runs"); + ( + out.status.success(), + format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_bare_ssh_session_greets_a_visitor_then_identifies_them_once_they_have_pushed() { + let fx = fixture().await; + let port = fx.server.port; + + let key_path = fx.key_path.clone(); + let (_ok, out) = tokio::task::spawn_blocking(move || ssh_bare(&key_path, port)) + .await + .unwrap(); + assert!(out.contains("knot.test"), "greeting names the knot:\n{out}"); + assert!( + out.contains("ssh key"), + "a visitor the knot can't identify yet learns what a push needs:\n{out}" + ); + + seed_work(&fx.work); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "seeding main must succeed:\n{out}"); + + let key_path = fx.key_path.clone(); + let (_ok, out) = tokio::task::spawn_blocking(move || ssh_bare(&key_path, port)) + .await + .unwrap(); + assert!( + out.contains("@nel.pet"), + "a push teaches the knot the key, so the next greeting uses the handle:\n{out}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_push_to_a_new_branch_offers_a_pull_request_link() { + let fx = fixture().await; + seed_work(&fx.work); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "seeding main must land:\n{out}"); + + git(&fx.work, &[], &["checkout", "-q", "-b", "feature"]); + std::fs::write(fx.work.join("feature.txt"), "work\n").unwrap(); + git(&fx.work, &[], &["add", "-A"]); + git(&fx.work, &[], &["commit", "-q", "-m", "feature work"]); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["feature"]).await; + assert!(ok, "feature-branch push must land:\n{out}"); + assert!( + out.contains("https://tangled.test/nel.pet/anemone/pulls/new"), + "new non-default branch is answered with a pull-request link:\n{out}" + ); + assert!( + out.contains("sourceBranch=feature") && out.contains("targetBranch=main"), + "link points the new branch at the default:\n{out}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_verbose_ci_push_option_reports_a_clean_pipeline() { + let fx = fixture().await; + std::fs::create_dir_all(fx.work.join(".tangled/workflows")).unwrap(); + git(&fx.work, &[], &["init", "-q", "-b", "main"]); + std::fs::write( + fx.work.join(".tangled/workflows/ci.yml"), + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n", + ) + .unwrap(); + git(&fx.work, &[], &["add", "-A"]); + git(&fx.work, &[], &["commit", "-q", "-m", "add ci"]); + + let (ok, out) = push( + &fx.work, + &fx.url, + &fx.key_path, + &["--push-option=verbose-ci", "main"], + ) + .await; + assert!(ok, "push with a push option must land:\n{out}"); + assert!( + out.contains("no diagnostics"), + "verbose-ci reports clean compile over the sideband:\n{out}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn git_archive_remote_over_ssh_streams_a_tar_of_the_tree() { + let fx = fixture().await; + seed_work(&fx.work); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "the seeding push must succeed before archiving:\n{out}"); + + let out_tar = fx.scratch.path().join("archive.tar"); + let (ok, out) = git_ssh( + &fx.work, + &fx.key_path, + &[ + "archive", + "--format=tar", + "--remote", + &fx.url, + "-o", + out_tar.to_str().unwrap(), + "HEAD", + ], + ) + .await; + assert!(ok, "git archive --remote over ssh must succeed:\n{out}"); + + let tar = std::fs::read(&out_tar).unwrap(); + assert!( + knot_fixtures::contains(&tar, b"README.md"), + "archived tar must contain the README.md entry" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn git_archive_remote_over_ssh_honors_the_configured_archive_limit() { + let fx = fixture_with_archive_limit(ArchiveLimit::new(512)).await; + seed_work(&fx.work); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "the seeding push must succeed before archiving:\n{out}"); + + let out_tar = fx.scratch.path().join("archive.tar"); + let (ok, out) = git_ssh( + &fx.work, + &fx.key_path, + &[ + "archive", + "--format=tar", + "--remote", + &fx.url, + "-o", + out_tar.to_str().unwrap(), + "HEAD", + ], + ) + .await; + assert!(!ok, "git archive --remote past the limit must fail:\n{out}"); + assert!( + out.contains("archive exceeds the 512 byte limit"), + "the refusal must reach the client over the ssh channel:\n{out}" + ); +} + +fn pkt(payload: &[u8]) -> Vec { + let mut framed = format!("{:04x}", payload.len() + 4).into_bytes(); + framed.extend_from_slice(payload); + framed +} + +fn pkt_text(line: &str) -> Vec { + pkt(format!("{line}\n").as_bytes()) +} + +fn read_until(reader: &mut impl std::io::Read, needle: &[u8], buffer: &mut Vec) { + std::iter::from_fn(|| { + let mut byte = [0u8; 1]; + match reader.read(&mut byte) { + Ok(0) | Err(_) => None, + Ok(_) => { + buffer.push(byte[0]); + Some(buffer.ends_with(needle)) + } + } + }) + .find(|done| *done) + .expect("the session must answer before closing the stream"); +} + +fn trickled_lfs_upload( + key_path: &str, + port: u16, + body: &[u8], + oid: &str, + midway: std::sync::mpsc::Sender<()>, +) -> (bool, String) { + use std::io::Write; + let mut child = Command::new("ssh") + .args(["-i", key_path]) + .args(UNSHARED_SSH.split_whitespace()) + .args([ + "-p", + &port.to_string(), + "git@127.0.0.1", + &format!("git-lfs-transfer '{}/{}' upload", owner(), repo_name()), + ]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("ssh runs"); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = child.stdout.take().unwrap(); + let mut transcript = Vec::new(); + + read_until(&mut stdout, b"version=1\n0000", &mut transcript); + + let (first, second) = body.split_at(body.len() / 2); + stdin + .write_all(&pkt_text(&format!("put-object {oid}"))) + .unwrap(); + stdin + .write_all(&pkt_text(&format!("size={}", body.len()))) + .unwrap(); + stdin.write_all(b"0001").unwrap(); + first.chunks(32 * 1024).for_each(|chunk| { + stdin.write_all(&pkt(chunk)).unwrap(); + }); + stdin.flush().unwrap(); + midway.send(()).unwrap(); + std::thread::sleep(std::time::Duration::from_millis(900)); + + second.chunks(32 * 1024).for_each(|chunk| { + stdin.write_all(&pkt(chunk)).unwrap(); + }); + stdin.write_all(b"0000").unwrap(); + stdin.flush().unwrap(); + read_until(&mut stdout, b"status 200\n0000", &mut transcript); + + stdin.write_all(&pkt_text("quit")).unwrap(); + stdin.write_all(b"0000").unwrap(); + stdin.flush().unwrap(); + drop(stdin); + use std::io::Read; + let _ = stdout.read_to_end(&mut transcript); + let status = child.wait().expect("ssh exits"); + ( + status.success(), + String::from_utf8_lossy(&transcript).into_owned(), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn shutdown_drains_an_in_flight_lfs_transfer_before_exit() { + use knot_lfs::{LfsStore, Pool}; + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path(), "drain"); + let (server, _index, shutdown, serve_task, _) = spawn_server_core( + public_line, + MaxWireBytes::new(1 << 20), + ArchiveLimit::default(), + true, + false, + ) + .await; + + let body: Vec = (0..1_048_576u32).map(|n| (n % 251) as u8).collect(); + let oid = knot_lfs::LfsOid::from_digest(knot_types::Sha256Digest::hash(&body)); + let (midway_tx, midway_rx) = std::sync::mpsc::channel(); + + let client = { + let key_path = key_path.clone(); + let oid = oid.clone(); + let port = server.port; + tokio::task::spawn_blocking(move || { + trickled_lfs_upload(&key_path, port, &body, oid.as_str(), midway_tx) + }) + }; + + tokio::task::spawn_blocking(move || { + midway_rx + .recv_timeout(std::time::Duration::from_secs(20)) + .expect("the upload must reach its midway point") + }) + .await + .unwrap(); + + shutdown.cancel(); + tokio::time::sleep(std::time::Duration::from_millis(150)).await; + assert!( + !serve_task.is_finished(), + "the listener must keep draining while a transfer is in flight" + ); + + let (ok, transcript) = client.await.unwrap(); + assert!( + ok, + "the in-flight upload must finish cleanly across the shutdown:\n{transcript}" + ); + assert!( + transcript.contains("status 200"), + "the server must acknowledge the drained upload:\n{transcript}" + ); + + tokio::time::timeout(std::time::Duration::from_secs(10), serve_task) + .await + .expect("the drained listener must exit promptly once transfers finish") + .unwrap(); + + let repo_did = repo_did(); + assert_eq!( + server + .lfs + .store + .probe(Pool::Lfs, &repo_did, &oid) + .unwrap() + .map(|size| size.get()), + Some(1_048_576), + "the drained upload must be durable" + ); + + let (connected, _) = { + let key_path = key_path.clone(); + let port = server.port; + tokio::task::spawn_blocking(move || ssh_bare(&key_path, port)) + .await + .unwrap() + }; + assert!( + !connected, + "a connection after shutdown must be refused, the drain only covers in-flight work" + ); +} + +#[derive(serde::Deserialize)] +struct OpProbe { + path: String, +} + +#[derive(serde::Deserialize)] +struct CommitProbe { + ops: Vec, +} + +fn frame_kind_and_ops(frame: &[u8]) -> (&'static str, Vec) { + let mut read = frame; + let header: serde_json::Value = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + match header["t"].as_str().unwrap_or_default() { + "#commit" => { + let payload: CommitProbe = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + let ops = payload.ops.into_iter().map(|op| op.path).collect(); + ("#commit", ops) + } + "#sync" => ("#sync", Vec::new()), + _ => ("#ignored", Vec::new()), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_authorized_ssh_push_projects_the_ref_records_the_http_push_projects() { + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path(), "client"); + let (server, _index, _shutdown, _serve_task, firehose) = spawn_server_core( + public_line, + MaxWireBytes::new(1 << 30), + ArchiveLimit::default(), + true, + true, + ) + .await; + let firehose = firehose.expect("the projection fixture serves a firehose"); + + let url = format!( + "ssh://git@127.0.0.1:{}/{}/{}", + server.port, + owner(), + repo_name() + ); + let work = scratch.path().join("work"); + let head = seed_work(&work); + + let (ok, out) = push(&work, &url, &key_path, &["main"]).await; + assert!(ok, "authorized ssh push must succeed:\n{out}"); + assert_eq!( + main_tip(&server.layout, &server.repo_did), + Some(Oid::from_hex(&head).unwrap()), + "pushed commit must be the repository's main tip" + ); + + let ref_ops: Vec = firehose + .replay( + knot_events::EventCursor::START, + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(4096).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + ) + .events + .iter() + .flat_map(|event| frame_kind_and_ops(&event.frame).1) + .filter(|path| path.starts_with("org.tangled.git.refs/")) + .collect(); + assert_eq!( + ref_ops, + vec!["org.tangled.git.refs/self".to_string()], + "the ssh push projected its ref into an atproto record, exactly as the http push does" + ); +} diff --git a/knot2/crates/knot-storage/Cargo.toml b/knot2/crates/knot-storage/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..d8a2f972b646feb9bbccb29455c9b51c425de3f0 --- /dev/null +++ b/knot2/crates/knot-storage/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "knot-storage" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +async-trait = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +parking_lot = { workspace = true } diff --git a/knot2/crates/knot-storage/src/error.rs b/knot2/crates/knot-storage/src/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..d7ad0cb5e6e4ebe3900d85b338529408bb01e3c3 --- /dev/null +++ b/knot2/crates/knot-storage/src/error.rs @@ -0,0 +1,41 @@ +use knot_types::RefName; + +use crate::{PackLimit, StorableRepoDid}; + +#[derive(Debug, thiserror::Error)] +pub enum StorageError { + #[error("repo {0} not found")] + NotFound(StorableRepoDid), + #[error("another writer won to touch this repo")] + CasConflict, + #[error("the storage lock is held by smth else")] + LockContention, + #[error("storage quota exhausted")] + QuotaExceeded, + #[error("storage corruption: {0}")] + Corruption(String), + #[error("{0} isn't available on this storage backend")] + Unsupported(&'static str), + #[error("backend failure: {0}")] + Backend(String), +} + +#[derive(Debug, thiserror::Error)] +pub enum RefUpdateError { + #[error(transparent)] + Storage(#[from] StorageError), + #[error("{name} moved after was read, update is therefore stale")] + CasConflict { name: RefName }, + #[error("{name} rejected: {reason}")] + Rejected { name: RefName, reason: String }, +} + +#[derive(Debug, thiserror::Error)] +pub enum IngestError { + #[error(transparent)] + Storage(#[from] StorageError), + #[error("incoming pack exceeds limit of {0}")] + Limits(PackLimit), + #[error("incoming pack is malformed: {0}")] + Corrupt(String), +} diff --git a/knot2/crates/knot-storage/src/id.rs b/knot2/crates/knot-storage/src/id.rs new file mode 100644 index 0000000000000000000000000000000000000000..b92c1b320974b493bb8ee7431a39cc4f601629d0 --- /dev/null +++ b/knot2/crates/knot-storage/src/id.rs @@ -0,0 +1,200 @@ +use std::fmt; +use std::str::FromStr; + +use knot_types::{ParseError, RepoDid}; + +knot_types::scalar_newtype! { + pub struct RefEpoch(u64); + pub struct RefGeneration(u64); +} + +mod shard_part { + use std::fmt; + + use knot_types::ParseError; + + #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + pub struct ShardPart(String); + + impl ShardPart { + pub fn new(raw: &str) -> Result { + (!matches!(raw, "." | "..") && !raw.contains(['/', '\\', '\0'])) + .then(|| Self(raw.to_string())) + .ok_or_else(|| ParseError::Invalid { + kind: "shard component", + value: raw.to_string(), + }) + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } + } + + impl fmt::Debug for ShardPart { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("ShardPart").field(&self.0.as_str()).finish() + } + } +} + +use shard_part::ShardPart; + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct StorableRepoDid { + method: ShardPart, + shard: ShardPart, + remainder: ShardPart, +} + +impl StorableRepoDid { + pub fn from_did(did: &RepoDid) -> Result { + let (method, shard, remainder) = + split_shard(did.as_str()).ok_or_else(|| ParseError::Invalid { + kind: "shardable repo DID", + value: did.as_str().to_string(), + })?; + Ok(Self { + method: ShardPart::new(method)?, + shard: ShardPart::new(shard)?, + remainder: ShardPart::new(remainder)?, + }) + } + + pub fn did(&self) -> RepoDid { + RepoDid::new(format!( + "did:{}:{}{}", + self.method.as_str(), + self.shard.as_str(), + self.remainder.as_str(), + )) + .expect("shard components oughtta rebuild into the DID they once were. we can rebuild him.") + } + + pub fn shard_parts(&self) -> (&str, &str, &str) { + ( + self.method.as_str(), + self.shard.as_str(), + self.remainder.as_str(), + ) + } +} + +fn split_shard(did: &str) -> Option<(&str, &str, &str)> { + let mut parts = did.splitn(3, ':'); + parts.next()?; + let method = parts.next()?; + let msid = parts.next()?; + let split = msid + .char_indices() + .nth(2) + .map(|(index, _)| index) + .unwrap_or(msid.len()); + let (shard, remainder) = msid.split_at(split); + Some((method, shard, remainder)) +} + +impl From for RepoDid { + fn from(repo: StorableRepoDid) -> Self { + repo.did() + } +} + +impl FromStr for StorableRepoDid { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + Self::from_did(&RepoDid::new(s)?) + } +} + +impl fmt::Debug for StorableRepoDid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("StorableRepoDid") + .field(&self.did().as_str()) + .finish() + } +} + +impl fmt::Display for StorableRepoDid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(self.did().as_str()) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::*; + + fn id(raw: &str) -> StorableRepoDid { + raw.parse().unwrap() + } + + #[test] + fn how_i_expect_our_stupid_dids_to_look_when_cut_up() { + let repo = id("did:plc:squid"); + assert_eq!(repo.shard_parts(), ("plc", "sq", "uid")); + let repo = id("did:web:oyster.cafe"); + assert_eq!(repo.shard_parts(), ("web", "oy", "ster.cafe")); + } + + #[test] + fn bad_did_byebye() { + [ + "did:plc:....", + "did:web:..", + "did:plc:.", + "did:plc:..anything", + "did:plc:a/b", + "did:web:a\\b", + "did:plc:a\0b", + ] + .into_iter() + .for_each(|raw| { + assert!( + raw.parse::().is_err(), + "{raw} mustn't become a StorableRepoDid or else we have a security hole" + ); + }); + } + + #[test] + fn good_did_goes_in_comes_back_out() { + let repo = id("did:web:oyster.cafe"); + assert_eq!(repo.did().as_str(), "did:web:oyster.cafe"); + assert_eq!(repo.to_string(), "did:web:oyster.cafe"); + assert_eq!( + "did:web:oyster.cafe".parse::().unwrap(), + repo + ); + let set = HashSet::from([repo.clone()]); + assert!(set.contains(&repo)); + assert_eq!(RepoDid::from(repo).as_str(), "did:web:oyster.cafe"); + } + + #[test] + fn did_web_canonicalizes_before_shard() { + let repo = id("did:web:OYSTER.cafe"); + assert_eq!(repo.did().as_str(), "did:web:oyster.cafe"); + assert_eq!(repo.shard_parts(), ("web", "oy", "ster.cafe")); + } + + #[test] + fn from_did_fn_only_should_run_shard_check() { + let did = RepoDid::new("did:plc:limpet").unwrap(); + assert_eq!( + StorableRepoDid::from_did(&did).unwrap().did().as_str(), + "did:plc:limpet" + ); + let hostile = RepoDid::new("did:plc:....").unwrap(); + assert!(StorableRepoDid::from_did(&hostile).is_err()); + } + + #[test] + fn invalid_did_grammar_never_even_makes_it_to_shard_check() { + assert!("not-a-did".parse::().is_err()); + assert!("did:plc:".parse::().is_err()); + } +} diff --git a/knot2/crates/knot-storage/src/kv.rs b/knot2/crates/knot-storage/src/kv.rs new file mode 100644 index 0000000000000000000000000000000000000000..b743c7eb81d5a5f4de750bacd2020de2117cee06 --- /dev/null +++ b/knot2/crates/knot-storage/src/kv.rs @@ -0,0 +1,86 @@ +use crate::StorageError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum RepoKvKey { + ImportJob, + HandoverJob, + HandoverDone, + KeepFailed, + HandoverUnmarked, +} + +pub trait RepoKv: Send + Sync { + fn get(&self, key: RepoKvKey) -> Result>, StorageError>; + + fn put(&self, key: RepoKvKey, value: &[u8]) -> Result<(), StorageError>; + + fn append(&self, key: RepoKvKey, line: &[u8]) -> Result<(), StorageError>; + + fn delete(&self, key: RepoKvKey) -> Result<(), StorageError>; + + fn keys(&self) -> Result, StorageError>; +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use parking_lot::Mutex; + + use super::*; + + struct MapKv(Mutex>>); + + impl RepoKv for MapKv { + fn get(&self, key: RepoKvKey) -> Result>, StorageError> { + let entries = self.0.lock(); + Ok(entries.get(&key).cloned()) + } + + fn put(&self, key: RepoKvKey, value: &[u8]) -> Result<(), StorageError> { + self.0.lock().insert(key, value.to_vec()); + Ok(()) + } + + fn append(&self, key: RepoKvKey, line: &[u8]) -> Result<(), StorageError> { + self.0 + .lock() + .entry(key) + .or_default() + .extend_from_slice(line); + Ok(()) + } + + fn delete(&self, key: RepoKvKey) -> Result<(), StorageError> { + self.0.lock().remove(&key); + Ok(()) + } + + fn keys(&self) -> Result, StorageError> { + Ok(self.0.lock().keys().copied().collect()) + } + } + + #[test] + fn append_should_append_bytes_and_keys_should_list_stored() { + let kv = MapKv(Mutex::new(BTreeMap::new())); + kv.put(RepoKvKey::ImportJob, br#"{"state":"queued"}"#) + .unwrap(); + kv.append(RepoKvKey::KeepFailed, b"3jzfcctlqo2qz did:plc:limpet\n") + .unwrap(); + kv.append(RepoKvKey::KeepFailed, b"3jzfcctlqo2qz did:plc:whelk\n") + .unwrap(); + assert_eq!( + kv.get(RepoKvKey::KeepFailed).unwrap(), + Some(b"3jzfcctlqo2qz did:plc:limpet\n3jzfcctlqo2qz did:plc:whelk\n".to_vec()), + "append... appends onto stored bytes, doesn't ever upsert" + ); + assert_eq!( + kv.keys().unwrap(), + vec![RepoKvKey::ImportJob, RepoKvKey::KeepFailed] + ); + kv.delete(RepoKvKey::ImportJob).unwrap(); + assert_eq!(kv.get(RepoKvKey::ImportJob).unwrap(), None); + assert_eq!(kv.keys().unwrap(), vec![RepoKvKey::KeepFailed]); + } +} diff --git a/knot2/crates/knot-storage/src/lib.rs b/knot2/crates/knot-storage/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..96c8a912c182eff5a57d1d402a9f000bd70698ae --- /dev/null +++ b/knot2/crates/knot-storage/src/lib.rs @@ -0,0 +1,20 @@ +mod error; +mod id; +mod kv; +mod objects; +mod pack; +mod refs; +mod store; + +pub use error::{IngestError, RefUpdateError, StorageError}; +pub use id::{RefEpoch, RefGeneration, StorableRepoDid}; +pub use knot_types::ObjectFormat; +pub use kv::{RepoKv, RepoKvKey}; +pub use objects::{ByteSink, ByteSource, ObjectStore, ObjectType, ReadLimit}; +pub use pack::{ + DeltaDepth, Filter, HaveOids, IngestSession, MaxObjectBytes, MaxTotalBytes, PackBudget, + PackEntry, PackHash, PackLimit, PackLimits, PackQuery, PackStats, PackStore, Selection, + ShallowCut, StagedIngest, StagedSummary, TreeDepth, WantOids, +}; +pub use refs::{AdvertScope, HeadRef, RefClass, RefRecord, RefStore, RefUpdate, ReflogUpdate}; +pub use store::{RepoStores, Storage}; diff --git a/knot2/crates/knot-storage/src/objects.rs b/knot2/crates/knot-storage/src/objects.rs new file mode 100644 index 0000000000000000000000000000000000000000..72b57510554ac656f5a076999f8589a33e565fb4 --- /dev/null +++ b/knot2/crates/knot-storage/src/objects.rs @@ -0,0 +1,131 @@ +use std::io::{Read, Write}; + +use knot_types::Oid; + +use crate::StorageError; + +knot_types::scalar_newtype! { + pub struct ReadLimit(u64); +} + +impl ReadLimit { + pub const ENTIRE: Self = Self(u64::MAX); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ObjectType { + Commit, + Tree, + Blob, + Tag, +} + +pub struct ByteSource(Box); + +impl ByteSource { + pub fn new(inner: Box) -> Self { + Self(inner) + } + + pub fn from_vec(bytes: Vec) -> Self { + Self(Box::new(std::io::Cursor::new(bytes))) + } +} + +impl Read for ByteSource { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.0.read(buf) + } +} + +impl std::fmt::Debug for ByteSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ByteSource").finish_non_exhaustive() + } +} + +pub struct ByteSink(Box); + +impl ByteSink { + pub fn new(inner: Box) -> Self { + Self(inner) + } +} + +impl Write for ByteSink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.0.flush() + } +} + +impl std::fmt::Debug for ByteSink { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ByteSink").finish_non_exhaustive() + } +} + +pub trait ObjectStore: Send + Sync { + fn contains(&self, oid: &Oid) -> Result; + + fn header(&self, oid: &Oid) -> Result, StorageError>; + + fn read(&self, oid: &Oid, limit: ReadLimit) -> Result>, StorageError>; + + fn reader(&self, oid: &Oid) -> Result, StorageError>; + + fn put(&self, kind: ObjectType, bytes: &[u8]) -> Result; + + fn put_batch(&self, objects: &[(ObjectType, Vec)]) -> Result, StorageError> { + objects + .iter() + .map(|(kind, bytes)| self.put(*kind, bytes)) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + struct SharedBuf(Arc>>); + + impl Write for SharedBuf { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[test] + fn byte_source_returns_vec_bytes() { + let mut source = ByteSource::from_vec(b"kelp forest".to_vec()); + let mut out = String::new(); + source.read_to_string(&mut out).unwrap(); + assert_eq!(out, "kelp forest"); + } + + #[test] + fn bytesink_correctly_collects() { + let buffer = Arc::new(parking_lot::Mutex::new(Vec::new())); + let mut sink = ByteSink::new(Box::new(SharedBuf(Arc::clone(&buffer)))); + sink.write_all(b"reef").unwrap(); + sink.write_all(b" and lagoon").unwrap(); + sink.flush().unwrap(); + drop(sink); + assert_eq!(*buffer.lock(), b"reef and lagoon"); + } + + #[test] + fn entire_is_same_as_upper_max_of_read_limit() { + assert_eq!(ReadLimit::ENTIRE.get(), u64::MAX); + } +} diff --git a/knot2/crates/knot-storage/src/pack.rs b/knot2/crates/knot-storage/src/pack.rs new file mode 100644 index 0000000000000000000000000000000000000000..33690b8127890776f520281e17621e9e5abd7fda --- /dev/null +++ b/knot2/crates/knot-storage/src/pack.rs @@ -0,0 +1,424 @@ +use std::collections::HashSet; +use std::fmt; +use std::time::Duration; + +use knot_types::{ObjectCount, Oid, UnixSeconds}; + +use crate::{ + ByteSink, IngestError, RefGeneration, RefUpdate, RefUpdateError, RepoStores, StorageError, +}; + +const MAX_OBJECTS: ObjectCount = ObjectCount::new(16_000_000); +const MAX_OBJECT_BYTES: MaxObjectBytes = MaxObjectBytes::new(512 * 1024 * 1024); +const MAX_TOTAL_BYTES: MaxTotalBytes = MaxTotalBytes::new(16 * 1024 * 1024 * 1024); +const MAX_DELTA_DEPTH: DeltaDepth = DeltaDepth::new(50); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PackLimits { + pub max_objects: ObjectCount, + pub max_object_bytes: MaxObjectBytes, + pub max_total_bytes: MaxTotalBytes, + pub max_delta_depth: DeltaDepth, +} + +impl Default for PackLimits { + fn default() -> Self { + Self { + max_objects: MAX_OBJECTS, + max_object_bytes: MAX_OBJECT_BYTES, + max_total_bytes: MAX_TOTAL_BYTES, + max_delta_depth: MAX_DELTA_DEPTH, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DeltaDepth(usize); + +impl DeltaDepth { + pub const ZERO: Self = Self(0); + + pub const fn new(depth: usize) -> Self { + Self(depth) + } + + pub const fn get(self) -> usize { + self.0 + } + + pub const fn deeper(self) -> Self { + Self(self.0 + 1) + } + + pub const fn exceeds(self, max: DeltaDepth) -> bool { + self.0 > max.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct MaxObjectBytes(u64); + +impl MaxObjectBytes { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn exceeded_by(self, size: u64) -> bool { + size > self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct MaxTotalBytes(u64); + +impl MaxTotalBytes { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn exceeded_by(self, size: u64) -> bool { + size > self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PackLimit { + Objects, + ObjectBytes, + TotalBytes, + DeltaDepth, +} + +impl fmt::Display for PackLimit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + PackLimit::Objects => "object count", + PackLimit::ObjectBytes => "per-object size", + PackLimit::TotalBytes => "total decompressed size", + PackLimit::DeltaDepth => "delta chain depth", + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TreeDepth(u32); + +impl TreeDepth { + pub const fn new(depth: u32) -> Self { + Self(depth) + } + + pub const fn get(self) -> u32 { + self.0 + } + + pub const fn deeper(self) -> Self { + Self(self.0.saturating_add(1)) + } + + pub const fn is_exhausted(self) -> bool { + self.0 == 0 + } + + pub const fn shallower(self) -> Self { + Self(self.0.saturating_sub(1)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Filter { + None, + BlobNone, + BlobLimit(u64), + TreeDepth(TreeDepth), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PackBudget { + max_objects: ObjectCount, + stall: Option, +} + +impl PackBudget { + pub fn new(max_objects: ObjectCount, stall: Duration) -> Self { + Self { + max_objects, + stall: Some(stall), + } + } + + pub fn unbounded() -> Self { + Self { + max_objects: ObjectCount::new(usize::MAX), + stall: None, + } + } + + pub fn max_objects(self) -> ObjectCount { + self.max_objects + } + + pub fn stall(self) -> Option { + self.stall + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct WantOids(Vec); + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HaveOids(Vec); + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ShallowCut(Vec); + +macro_rules! oid_set { + ($name:ident) => { + impl $name { + pub fn new(oids: Vec) -> Self { + Self(oids) + } + + pub fn as_slice(&self) -> &[Oid] { + &self.0 + } + + pub fn iter(&self) -> std::slice::Iter<'_, Oid> { + self.0.iter() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + } + + impl FromIterator for $name { + fn from_iter>(iter: I) -> Self { + Self(iter.into_iter().collect()) + } + } + }; +} + +oid_set!(WantOids); +oid_set!(HaveOids); +oid_set!(ShallowCut); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackHash(Oid); + +impl PackHash { + pub fn new(value: &str) -> Option { + Oid::from_hex(value).ok().map(Self) + } + + pub fn to_hex(&self) -> String { + self.0.to_hex() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackQuery { + pub wants: WantOids, + pub haves: HaveOids, + pub shallow: ShallowCut, + pub filter: Filter, + pub thin: bool, + pub budget: PackBudget, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Selection { + pub send: Vec, + pub client_has: HashSet, + pub thin: bool, +} + +impl Selection { + pub fn count(&self) -> ObjectCount { + ObjectCount::new(self.send.len()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PackStats { + pub objects: ObjectCount, + pub bytes: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackEntry { + pub hash: PackHash, + pub bytes: u64, + pub modified: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StagedSummary { + pub self_contained: bool, + pub present: HashSet, +} + +pub trait PackStore: Send + Sync { + fn begin_ingest(&self, limits: PackLimits) -> Result, StorageError>; + + fn select(&self, query: &PackQuery) -> Result; + + fn write_pack(&self, selection: &Selection, out: ByteSink) -> Result; + + fn packs(&self) -> Result, StorageError>; +} + +pub trait IngestSession: Send { + fn write(&mut self, chunk: &[u8]) -> Result<(), StorageError>; + + fn finish(self: Box) -> Result, IngestError>; +} + +pub trait StagedIngest: Send { + fn stores(&self) -> &RepoStores; + + fn summary(&self) -> &StagedSummary; + + fn apply(self: Box, updates: &[RefUpdate]) -> Result; + + fn abort(self: Box) -> Result<(), StorageError>; +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use crate::ObjectType; + + use super::*; + + // TODO: make these configurable! Keep forgetting. + #[test] + fn default_pack_limits_to_the_maximums() { + let limits = PackLimits::default(); + assert_eq!(limits.max_objects.get(), 16_000_000); + assert_eq!(limits.max_object_bytes.get(), 512 * 1024 * 1024); + assert_eq!(limits.max_total_bytes.get(), 16 * 1024 * 1024 * 1024); + assert_eq!(limits.max_delta_depth.get(), 50); + } + + #[test] + fn limit_labels_match() { + assert_eq!(PackLimit::Objects.to_string(), "object count"); + assert_eq!(PackLimit::ObjectBytes.to_string(), "per-object size"); + assert_eq!(PackLimit::TotalBytes.to_string(), "total decompressed size"); + assert_eq!(PackLimit::DeltaDepth.to_string(), "delta chain depth"); + } + + #[test] + fn selection_only_counts_what_it_sends_in_practice() { + let a = Oid::from_hex("1111111111111111111111111111111111111111").unwrap(); + let b = Oid::from_hex("2222222222222222222222222222222222222222").unwrap(); + let selection = Selection { + send: vec![a, b], + client_has: HashSet::from([a]), + thin: true, + }; + assert_eq!(selection.count(), ObjectCount::new(2)); + } + + #[derive(Default)] + struct MapState { + objects: HashMap)>, + next: u64, + } + + struct MapObjects(parking_lot::Mutex); + + impl MapObjects { + fn fresh(&self) -> Oid { + let mut state = self.0.lock(); + state.next += 1; + Oid::from_hex(&format!("{:040x}", state.next)).unwrap() + } + } + + impl crate::ObjectStore for MapObjects { + fn contains(&self, oid: &Oid) -> Result { + Ok(self.0.lock().objects.contains_key(oid)) + } + + fn header(&self, oid: &Oid) -> Result, StorageError> { + Ok(self + .0 + .lock() + .objects + .get(oid) + .map(|(kind, bytes)| (*kind, bytes.len() as u64))) + } + + fn read( + &self, + oid: &Oid, + _limit: crate::ReadLimit, + ) -> Result>, StorageError> { + Ok(self + .0 + .lock() + .objects + .get(oid) + .map(|(_, bytes)| bytes.clone())) + } + + fn reader(&self, oid: &Oid) -> Result, StorageError> { + Ok(self + .0 + .lock() + .objects + .get(oid) + .map(|(_, bytes)| crate::ByteSource::from_vec(bytes.clone()))) + } + + fn put(&self, kind: ObjectType, bytes: &[u8]) -> Result { + let oid = self.fresh(); + self.0.lock().objects.insert(oid, (kind, bytes.to_vec())); + Ok(oid) + } + } + + #[test] + fn default_put_batch_stores_every_object_in_order() { + use crate::ObjectStore; + + let store = MapObjects(parking_lot::Mutex::new(MapState::default())); + let blob = (ObjectType::Blob, b"kelp".to_vec()); + let tree = (ObjectType::Tree, b"tree".to_vec()); + let dyn_store: &dyn ObjectStore = &store; + let oids = dyn_store.put_batch(&[blob, tree]).unwrap(); + assert_eq!(oids.len(), 2); + assert_ne!(oids[0], oids[1]); + assert_eq!( + dyn_store.read(&oids[0], crate::ReadLimit::ENTIRE).unwrap(), + Some(b"kelp".to_vec()) + ); + assert_eq!( + dyn_store.header(&oids[1]).unwrap(), + Some((ObjectType::Tree, 4)) + ); + } + + fn _every_pack_trait_stays_dyn_callable( + _: &dyn PackStore, + _: &dyn IngestSession, + _: &dyn StagedIngest, + ) { + } +} diff --git a/knot2/crates/knot-storage/src/refs.rs b/knot2/crates/knot-storage/src/refs.rs new file mode 100644 index 0000000000000000000000000000000000000000..516f0e6ac14b950c7e9078f66b5bd6d2611d4e0e --- /dev/null +++ b/knot2/crates/knot-storage/src/refs.rs @@ -0,0 +1,307 @@ +use std::collections::HashSet; + +use knot_types::{Oid, RefName, RefTransition, UnixSeconds}; + +use crate::{RefEpoch, RefGeneration, StorageError}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RefRecord { + pub name: RefName, + pub target: Oid, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HeadRef { + pub name: RefName, + pub target: Oid, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReflogUpdate { + pub name: RefName, + pub old: Option, + pub new: Oid, + pub seconds: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RefUpdate { + Create { name: RefName, new: Oid }, + Update { name: RefName, old: Oid, new: Oid }, + Delete { name: RefName, old: Oid }, +} + +impl RefUpdate { + pub fn name(&self) -> &RefName { + match self { + RefUpdate::Create { name, .. } + | RefUpdate::Update { name, .. } + | RefUpdate::Delete { name, .. } => name, + } + } + + pub fn transition(&self) -> RefTransition { + match self { + RefUpdate::Create { new, .. } => RefTransition::Create { new: *new }, + RefUpdate::Update { old, new, .. } => RefTransition::Advance { + old: *old, + new: *new, + }, + RefUpdate::Delete { old, .. } => RefTransition::Delete { old: *old }, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RefClass { + Public, + Cob, + Checkpoint, + Hidden, + Atproto, + Tngl, +} + +impl RefClass { + pub const ALL: [RefClass; 6] = [ + RefClass::Public, + RefClass::Cob, + RefClass::Checkpoint, + RefClass::Hidden, + RefClass::Atproto, + RefClass::Tngl, + ]; + + pub fn of(name: &RefName) -> RefClass { + RefClass::of_revspec(name.as_str()) + } + + pub fn of_revspec(raw: &str) -> RefClass { + let rest = raw.strip_prefix("refs/").unwrap_or(raw); + match rest.split('/').next().unwrap_or(rest) { + "cobs" => RefClass::Cob, + "cob-checkpoints" => RefClass::Checkpoint, + "hidden" => RefClass::Hidden, + "atproto" => RefClass::Atproto, + "tngl" => RefClass::Tngl, + _ => RefClass::Public, + } + } + + pub fn namespace(self) -> Option<&'static str> { + match self { + RefClass::Public => None, + RefClass::Cob => Some("refs/cobs"), + RefClass::Checkpoint => Some("refs/cob-checkpoints"), + RefClass::Hidden => Some("refs/hidden"), + RefClass::Atproto => Some("refs/atproto"), + RefClass::Tngl => Some("refs/tngl"), + } + } + + pub fn is_public(self) -> bool { + match self { + RefClass::Public => true, + RefClass::Cob + | RefClass::Checkpoint + | RefClass::Hidden + | RefClass::Atproto + | RefClass::Tngl => false, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AdvertScope { + Upload, + Receive, +} + +pub trait RefStore: Send + Sync { + fn find(&self, name: &RefName) -> Result, StorageError>; + + fn list(&self) -> Result, StorageError>; + + fn advertised(&self, scope: AdvertScope) -> Result, StorageError>; + + fn update(&self, updates: &[RefUpdate]) -> Result; + + fn head(&self) -> Result, StorageError>; + + fn set_head(&self, name: &RefName) -> Result<(), StorageError>; + + fn default_branch(&self) -> RefName; + + fn reflogs(&self, since: Option) -> Result, StorageError>; + + fn cache_token(&self) -> (RefEpoch, RefGeneration); + + fn retained_oids( + &self, + since: UnixSeconds, + classes: &[RefClass], + ) -> Result, StorageError> { + retained_from_reflogs(self, since, classes) + } +} + +fn retained_from_reflogs( + store: &S, + since: UnixSeconds, + classes: &[RefClass], +) -> Result, StorageError> { + let mut oids: HashSet = store + .list()? + .into_iter() + .filter(|record| classes.contains(&RefClass::of(&record.name))) + .map(|record| record.target) + .collect(); + store.reflogs(Some(since))?.into_iter().for_each(|update| { + oids.insert(update.new); + oids.extend(update.old); + }); + Ok(oids) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct StubRefs { + refs: Vec, + reflogs: Vec, + } + + impl RefStore for StubRefs { + fn find(&self, name: &RefName) -> Result, StorageError> { + Ok(self + .refs + .iter() + .find(|record| record.name == *name) + .map(|record| record.target)) + } + + fn list(&self) -> Result, StorageError> { + Ok(self.refs.clone()) + } + + fn advertised(&self, _scope: AdvertScope) -> Result, StorageError> { + self.list() + } + + fn update(&self, _updates: &[RefUpdate]) -> Result { + Ok(RefGeneration::new(0)) + } + + fn head(&self) -> Result, StorageError> { + Ok(None) + } + + fn set_head(&self, _name: &RefName) -> Result<(), StorageError> { + Ok(()) + } + + fn default_branch(&self) -> RefName { + RefName::new("refs/heads/main").unwrap() + } + + fn reflogs(&self, since: Option) -> Result, StorageError> { + Ok(self + .reflogs + .iter() + .filter(|update| since.is_none_or(|since| update.seconds >= since)) + .cloned() + .collect()) + } + + fn cache_token(&self) -> (RefEpoch, RefGeneration) { + (RefEpoch::new(0), RefGeneration::new(0)) + } + } + + fn oid(hex: &str) -> Oid { + Oid::from_hex(hex).unwrap() + } + + fn name(raw: &str) -> RefName { + RefName::new(raw).unwrap() + } + + #[test] + fn oids_should_be_unioned_w_class_tips_n_reflog_hist() { + let store = StubRefs { + refs: vec![ + RefRecord { + name: name("refs/heads/main"), + target: oid("1111111111111111111111111111111111111111"), + }, + RefRecord { + name: name("refs/cobs/org.tangled.graph.issue/2"), + target: oid("2222222222222222222222222222222222222222"), + }, + RefRecord { + name: name("refs/hidden/feature"), + target: oid("3333333333333333333333333333333333333333"), + }, + ], + reflogs: vec![ + ReflogUpdate { + name: name("refs/heads/main"), + old: Some(oid("4444444444444444444444444444444444444444")), + new: oid("1111111111111111111111111111111111111111"), + seconds: UnixSeconds::new(200), + }, + ReflogUpdate { + name: name("refs/hidden/feature"), + old: None, + new: oid("6666666666666666666666666666666666666666"), + seconds: UnixSeconds::new(200), + }, + ], + }; + + let retained = store + .retained_oids(UnixSeconds::new(100), &[RefClass::Public, RefClass::Cob]) + .unwrap(); + + assert_eq!( + retained, + HashSet::from([ + oid("1111111111111111111111111111111111111111"), + oid("2222222222222222222222222222222222222222"), + oid("4444444444444444444444444444444444444444"), + oid("6666666666666666666666666666666666666666"), + ]), + "in: reflog history, class tips that are kept for whatever reason. out: entires older than bound, hidden tips." + ); + } + + #[test] + fn ref_store_dyn_callable() { + let store: &dyn RefStore = &StubRefs { + refs: Vec::new(), + reflogs: Vec::new(), + }; + assert_eq!(store.list().unwrap(), Vec::new()); + assert_eq!(store.find(&name("refs/heads/main")).unwrap(), None); + } + + #[test] + fn update_errors_state_ref_and_reason() { + let error = crate::RefUpdateError::Rejected { + name: name("refs/heads/main"), + reason: "directory/file conflict".to_string(), + }; + assert_eq!( + error.to_string(), + "refs/heads/main rejected: directory/file conflict" + ); + let conflict = crate::RefUpdateError::CasConflict { + name: name("refs/tags/v1"), + }; + assert_eq!( + conflict.to_string(), + "refs/tags/v1 moved after was read, update is therefore stale" + ); + } +} diff --git a/knot2/crates/knot-storage/src/store.rs b/knot2/crates/knot-storage/src/store.rs new file mode 100644 index 0000000000000000000000000000000000000000..fbab3c928b351776e0c50112657962e325fe70b3 --- /dev/null +++ b/knot2/crates/knot-storage/src/store.rs @@ -0,0 +1,40 @@ +use std::sync::Arc; + +use knot_types::ObjectFormat; + +use crate::{ObjectStore, PackStore, RefStore, RepoKv, StorableRepoDid, StorageError}; + +pub struct RepoStores { + pub refs: Arc, + pub objects: Arc, + pub packs: Arc, + pub kv: Arc, +} + +#[async_trait::async_trait] +pub trait Storage: Send + Sync { + async fn open(&self, repo: &StorableRepoDid) -> Result; + + async fn create( + &self, + repo: &StorableRepoDid, + format: ObjectFormat, + ) -> Result; + + async fn remove(&self, repo: &StorableRepoDid) -> Result<(), StorageError>; + + async fn probe(&self, repo: &StorableRepoDid) -> Result; + + async fn repos(&self) -> Result, StorageError>; + + async fn recover(&self) -> Result; +} + +#[cfg(test)] +mod tests { + use super::*; + + fn _the_facade_stays_dyn_callable(storage: Arc) -> Arc { + storage + } +} diff --git a/knot2/crates/knot-types/Cargo.toml b/knot2/crates/knot-types/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..0f8c1664d892dfcad68568556469a07025a52b3f --- /dev/null +++ b/knot2/crates/knot-types/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "knot-types" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +cid = { workspace = true } +jacquard-common = { workspace = true } +gix-hash = { workspace = true } +ipld-core = { workspace = true } +http = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +thiserror = { workspace = true } +sha2 = { workspace = true } +trusted-proxies = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/knot2/crates/knot-types/src/changes.rs b/knot2/crates/knot-types/src/changes.rs new file mode 100644 index 0000000000000000000000000000000000000000..86c36b84ed06b09a5bf2efb444eacd5bb2fd3add --- /dev/null +++ b/knot2/crates/knot-types/src/changes.rs @@ -0,0 +1,156 @@ +use std::ops::ControlFlow; + +use crate::ids::RepoPath; + +const MAX_BYTES: usize = 524_288; +const ENTRY_OVERHEAD_BYTES: usize = 48; +const MAX_ENTRIES: usize = 8_192; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Listing { + #[default] + Complete, + Truncated, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChangedFiles { + paths: Vec, + listing: Listing, +} + +impl ChangedFiles { + pub fn none() -> Self { + Self { + paths: Vec::new(), + listing: Listing::Complete, + } + } + + pub fn unknown() -> Self { + Self { + paths: Vec::new(), + listing: Listing::Truncated, + } + } + + pub fn paths(&self) -> &[RepoPath] { + &self.paths + } + + pub fn listing(&self) -> Listing { + self.listing + } + + pub fn into_paths(self) -> Vec { + self.paths + } +} + +#[derive(Debug, Default)] +pub struct ChangedFilesBudget { + paths: Vec, + spent: usize, + listing: Listing, +} + +impl ChangedFilesBudget { + pub fn new() -> Self { + Self::default() + } + + pub fn admit(&mut self, path: RepoPath) -> ControlFlow<()> { + let spent = self.spent + ENTRY_OVERHEAD_BYTES + path.as_str().len(); + match self.listing == Listing::Truncated + || spent > MAX_BYTES + || self.paths.len() == MAX_ENTRIES + { + true => self.truncate(), + false => { + self.spent = spent; + self.paths.push(path); + ControlFlow::Continue(()) + } + } + } + + pub fn truncate(&mut self) -> ControlFlow<()> { + self.listing = Listing::Truncated; + ControlFlow::Break(()) + } + + pub fn finish(self) -> ChangedFiles { + ChangedFiles { + listing: self.listing, + paths: self.paths, + } + } +} + +#[cfg(test)] +mod tests { + use super::{ChangedFiles, ChangedFilesBudget, Listing, MAX_BYTES, MAX_ENTRIES}; + use crate::ids::RepoPath; + use std::ops::ControlFlow; + + fn fill(mut paths: impl Iterator) -> ChangedFiles { + let mut budget = ChangedFilesBudget::new(); + let _ = paths.try_for_each(|path| budget.admit(RepoPath::new(path).unwrap())); + budget.finish() + } + + #[test] + fn a_listing_within_the_budget_stays_complete() { + let changed = fill(["a.txt", "src/deep/main.rs"].into_iter().map(String::from)); + assert_eq!(changed.listing(), Listing::Complete); + assert_eq!(changed.paths().len(), 2); + assert_eq!(ChangedFiles::none().listing(), Listing::Complete); + assert_eq!( + ChangedFiles::unknown().listing(), + Listing::Truncated, + "a listing that couldn't be computed rules no path constraint out" + ); + } + + #[test] + fn the_listing_stops_at_whichever_bound_it_hits_first() { + let wide = fill((0..MAX_ENTRIES * 2).map(|index| format!("f{index}.txt"))); + assert_eq!(wide.listing(), Listing::Truncated); + assert_eq!( + wide.paths().len(), + MAX_ENTRIES, + "the record codec refuses a longer array" + ); + + let deep = fill((0..MAX_ENTRIES).map(|index| format!("{}/f{index}.txt", "d".repeat(256)))); + assert_eq!(deep.listing(), Listing::Truncated); + assert!( + deep.paths().len() < MAX_ENTRIES, + "long paths run out the byte budget before the entry bound: {}", + deep.paths().len() + ); + } + + #[test] + fn truncation_is_sticky_and_keeps_only_the_paths_already_admitted() { + let mut budget = ChangedFilesBudget::new(); + assert_eq!( + budget.admit(RepoPath::new("a.txt").unwrap()), + ControlFlow::Continue(()) + ); + assert_eq!( + budget.admit(RepoPath::new("x".repeat(MAX_BYTES)).unwrap()), + ControlFlow::Break(()), + "a path larger than the whole remaining budget is refused" + ); + assert_eq!(budget.truncate(), ControlFlow::Break(())); + assert_eq!( + budget.admit(RepoPath::new("b.txt").unwrap()), + ControlFlow::Break(()), + "a short path after the break can't reopen the listing" + ); + let changed = budget.finish(); + assert_eq!(changed.paths(), [RepoPath::new("a.txt").unwrap()]); + assert_eq!(changed.listing(), Listing::Truncated); + } +} diff --git a/knot2/crates/knot-types/src/hex.rs b/knot2/crates/knot-types/src/hex.rs new file mode 100644 index 0000000000000000000000000000000000000000..08c48bdb90e6fe4b9444e588d047e225755fbd8d --- /dev/null +++ b/knot2/crates/knot-types/src/hex.rs @@ -0,0 +1,65 @@ +const HEX_DIGITS: &[u8; 16] = b"0123456789abcdef"; + +pub fn lowercase_hex(bytes: &[u8]) -> String { + bytes + .iter() + .flat_map(|byte| { + [ + HEX_DIGITS[usize::from(byte >> 4)], + HEX_DIGITS[usize::from(byte & 0x0f)], + ] + }) + .map(char::from) + .collect() +} + +pub fn decode_hex(text: impl AsRef<[u8]>) -> Option> { + let text = text.as_ref(); + match text.len() % 2 { + 0 => text + .as_chunks::<2>() + .0 + .iter() + .map(|pair| { + let hi = (pair[0] as char).to_digit(16)?; + let lo = (pair[1] as char).to_digit(16)?; + Some((hi * 16 + lo) as u8) + }) + .collect(), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_hex_inverts_an_encoder_that_covers_every_nibble() { + [ + (&[][..], ""), + (&[0x00][..], "00"), + (&[0xff][..], "ff"), + ( + &[0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef][..], + "0123456789abcdef", + ), + ] + .iter() + .for_each(|&(bytes, expected)| { + assert_eq!(lowercase_hex(bytes), expected); + assert_eq!(decode_hex(expected).unwrap(), bytes); + }); + assert_eq!( + decode_hex("abc"), + None, + "An odd digit count can't form whole bytes" + ); + assert_eq!(decode_hex("zz"), None, "A non-hex digit decodes to nothing"); + assert_eq!( + decode_hex("AB").unwrap(), + [0xab], + "Uppercase input still decodes even though the encoder emits lowercase" + ); + } +} diff --git a/knot2/crates/knot-types/src/ids.rs b/knot2/crates/knot-types/src/ids.rs new file mode 100644 index 0000000000000000000000000000000000000000..fd1eb12c6e8437c305f9b7ead33386c49fa1313a --- /dev/null +++ b/knot2/crates/knot-types/src/ids.rs @@ -0,0 +1,1583 @@ +use std::fmt; +use std::str::FromStr; + +use jacquard_common::types::crypto::{PublicKey, multikey}; +use jacquard_common::types::string::{Did as SpecDid, Handle, Nsid, Rkey, Tid}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ParseError { + #[error("Invalid {kind}: {value:?}")] + Invalid { kind: &'static str, value: String }, +} + +impl ParseError { + fn invalid(kind: &'static str, value: impl Into) -> Self { + Self::Invalid { + kind, + value: value.into(), + } + } +} + +#[derive(Clone)] +struct Did(String); + +impl Did { + fn as_str(&self) -> &str { + &self.0 + } +} + +#[macro_export] +macro_rules! string_id { + (@traits $name:ident) => { + $crate::string_id!(@traits_for $name, ParseError); + }; + (@traits_for $name:ident, $err:ty) => { + impl PartialEq for $name { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } + } + + impl Eq for $name {} + + impl PartialOrd for $name { + fn partial_cmp(&self, other: &Self) -> Option<::std::cmp::Ordering> { + Some(self.cmp(other)) + } + } + + impl Ord for $name { + fn cmp(&self, other: &Self) -> ::std::cmp::Ordering { + self.as_str().cmp(other.as_str()) + } + } + + impl ::std::hash::Hash for $name { + fn hash(&self, state: &mut H) { + self.as_str().hash(state); + } + } + + impl fmt::Debug for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple(stringify!($name)).field(&self.as_str()).finish() + } + } + + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Padding, not `write_str`'ing, + // or else every {:>20} in a log line does nothing + f.pad(self.as_str()) + } + } + + impl AsRef for $name { + fn as_ref(&self) -> &str { + self.as_str() + } + } + + impl FromStr for $name { + type Err = $err; + + fn from_str(s: &str) -> Result { + Self::new(s) + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let raw = String::deserialize(deserializer)?; + Self::new(raw).map_err(serde::de::Error::custom) + } + } + }; + ($name:ident, $label:literal, via $parse:path => $inner:ty) => { + #[derive(Clone)] + pub struct $name($inner); + + impl $name { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + match $parse(&value) { + Some(inner) => Ok(Self(inner)), + None => Err(ParseError::invalid($label, value)), + } + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } + } + + $crate::string_id!(@traits $name); + }; + ($name:ident, $label:literal, $parse:path) => { + #[derive(Clone)] + pub struct $name(String); + + impl $name { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + $parse(&value).ok_or_else(|| ParseError::invalid($label, value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + + $crate::string_id!(@traits $name); + }; + ($name:ident, try $new:path, err $err:ty) => { + #[derive(Clone)] + pub struct $name(String); + + impl $name { + pub fn new(value: impl Into) -> Result { + $new(value) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + + $crate::string_id!(@traits_for $name, $err); + }; +} + +// Names + emails come from commit objects that someone lovingly wrote, +// so refusing them probably isn't very good; +// they go back out into the author-line of a commit we write, +// in which a newline would create the rest of the header. +crate::text_newtype! { + pub struct AuthorName(String) => strip_control; + pub struct Email(String) => strip_control; +} + +crate::text_newtype! { + pub struct OriginUrl(String) => verbatim; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct LanguageName(&'static str); + +impl LanguageName { + pub const fn new(name: &'static str) -> Self { + LanguageName(name) + } + + pub fn as_str(&self) -> &'static str { + self.0 + } +} + +impl fmt::Display for LanguageName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(self.0) + } +} + +impl Serialize for LanguageName { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.0) + } +} + +fn parse_did(value: &str) -> Option { + SpecDid::::new_owned(value) + .ok() + .map(|did| canonical_did(did.as_str())) +} + +fn canonical_did(did: &str) -> Did { + match did.strip_prefix("did:web:") { + Some(msid) => { + let (authority, path) = match msid.split_once(':') { + Some((authority, path)) => (authority, Some(path)), + None => (msid, None), + }; + let authority = lowercase_preserving_percent(authority); + Did(match path { + Some(path) => format!("did:web:{authority}:{path}"), + None => format!("did:web:{authority}"), + }) + } + None => Did(did.to_string()), + } +} + +fn lowercase_preserving_percent(authority: &str) -> String { + authority + .chars() + .scan(0u8, |pending, ch| { + let emitted: String = if *pending > 0 { + *pending -= 1; + ch.to_string() + } else if ch == '%' { + *pending = 2; + ch.to_string() + } else { + ch.to_lowercase().collect() + }; + Some(emitted) + }) + .collect() +} + +fn parse_nsid(value: &str) -> Option> { + Nsid::new_owned(value).ok() +} + +fn parse_multikey(value: &str) -> Option { + PublicKey::decode(value) + .is_ok() + .then(|| ActorId(value.to_string())) +} + +fn parse_repo_name(value: &str) -> Option { + is_repo_name(value).then(|| RepoName(value.to_string())) +} + +fn parse_rkey(value: &str) -> Option> { + Rkey::new_owned(value).ok() +} + +fn parse_tid(value: &str) -> Option { + Tid::new(value).ok() +} + +fn parse_label_key(value: &str) -> Option { + let slug = !value.is_empty() + && value.len() <= MAX_LABEL_KEY_BYTES + && value.bytes().all(|byte| { + matches!(byte, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'.' | b'-' | b'_' | b'~' | b':') + }); + slug.then(|| LabelKey(value.to_string())) +} +const MAX_LABEL_KEY_BYTES: usize = 64; + +fn parse_record_collection(value: &str) -> Option> { + let nsid = parse_nsid(value)?; + + // I try to capture why this isn't just me dicking around, in the test + // a_collection_that_would_overrun_an_mst_key_isnt_a_record_collection. + // A tid rkey is always 13 bytes, a did rkey is longer, and an MST is not gonna like a key + // over 256 bytes, so tid-keyed collections get their whole bound here and `MstKey::new` + // stops a did rkey that overruns. + let room = crate::record::MST_KEY_MAX_BYTES - crate::record::RECORD_RKEY_BYTES - 1; + (nsid.as_str().len() <= room).then_some(nsid) +} + +fn parse_did_rkey(value: &str) -> Option { + let did = parse_did(value)?; + Rkey::::new_owned(did.as_str()).ok()?; + (!ported_did_web(value)).then_some(did) +} + +fn ported_did_web(value: &str) -> bool { + matches!(value.strip_prefix("did:web:"), Some(host) if host.contains(':')) +} + +fn parse_ref_name(value: &str) -> Option { + is_ref_name(value).then(|| RefName(value.to_string())) +} + +fn is_bare_host(value: &str) -> bool { + !value.contains([':', '/']) && KnotId::new(format!("did:web:{value}")).is_ok() +} + +fn parse_knot_hostname(value: &str) -> Option { + is_bare_host(value).then(|| KnotHostname(value.to_string())) +} + +fn parse_branch_name(value: &str) -> Option { + (value.len() <= 244 && RefName::new(format!("refs/heads/{value}")).is_ok()) + .then(|| BranchName(value.to_string())) +} + +fn parse_tag_name(value: &str) -> Option { + RefName::new(format!("refs/tags/{value}")) + .is_ok() + .then(|| TagName(value.to_string())) +} + +fn parse_repo_path(value: &str) -> Option { + (!value.is_empty() + && value + .split('/') + .all(|part| !part.is_empty() && part != "." && part != ".." && !part.contains('\0'))) + .then(|| RepoPath(value.to_string())) +} + +fn parse_http_base(value: &str) -> Option { + let trimmed = value.trim_end_matches('/'); + let rest = trimmed + .strip_prefix("https://") + .or_else(|| trimmed.strip_prefix("http://"))?; + let url = url::Url::parse(trimmed).ok()?; + (!rest.starts_with('/') + && url.has_host() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() + && !trimmed.chars().any(|c| c.is_whitespace() || c.is_control())) + .then_some(url) +} + +fn http_base_string(url: url::Url) -> String { + url.as_str().trim_end_matches('/').to_string() +} + +fn parse_service_url(value: &str) -> Option { + parse_http_base(value) + .filter(|url| matches!(url.path(), "" | "/")) + .map(|url| KnotServiceUrl(http_base_string(url))) +} + +fn parse_appview_endpoint(value: &str) -> Option { + parse_http_base(value).map(|url| AppviewEndpoint(http_base_string(url))) +} + +fn parse_push_option(value: &str) -> Option { + (!value.is_empty() && value.len() <= 1024 && !value.contains(['\0', '\n'])) + .then(|| PushOption(value.to_string())) +} + +fn is_repo_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= 100 + && value != "." + && value != ".." + && !value.contains('/') + && !value.contains('\\') + && !value.contains("..") + && value.chars().all(|c| !c.is_control() && !c.is_whitespace()) +} + +fn is_ref_name(value: &str) -> bool { + value.starts_with("refs/") + && !value.ends_with('.') + && !value.contains("..") + && !value.contains("@{") + && value.split('/').all(|component| { + !component.is_empty() && !component.starts_with('.') && !component.ends_with(".lock") + }) + && value.chars().all(|c| { + !c.is_control() + && !matches!(c, ' ' | '~' | '^' | ':' | '?' | '*' | '[' | '\\' | '\u{7f}') + }) +} + +string_id!(RepoDid, "repo DID", via parse_did => Did); +string_id!(OwnerDid, "owner DID", via parse_did => Did); +string_id!(KnotId, "knot DID", via parse_did => Did); +string_id!(AccountDid, "account DID", via parse_did => Did); +string_id!(ServiceDid, "service DID", via parse_did => Did); +string_id!(RepoName, "repo name", parse_repo_name); +string_id!(RepoRkey, "repo record key", via parse_rkey => Rkey); +impl RepoRkey { + pub fn from_repo_name(name: &RepoName) -> Result { + Self::new(name.as_str())?; + Self::new(name.as_str().to_lowercase()) + } +} +string_id!(DidRkey, "DID record key", via parse_did_rkey => Did); +string_id!(UriRkey, "at-uri record key", via parse_rkey => Rkey); +string_id!(RefName, "ref name", parse_ref_name); +string_id!(TypeName, "COB type name", via parse_nsid => Nsid); +string_id!(RecordCollection, "record collection", via parse_record_collection => Nsid); +string_id!(LabelKey, "label def key", parse_label_key); +string_id!(ActorId, "actor public key", parse_multikey); +string_id!(KnotHostname, "knot hostname", parse_knot_hostname); +string_id!(BranchName, "branch name", parse_branch_name); +string_id!(TagName, "tag name", parse_tag_name); +string_id!(RepoPath, "repository path", parse_repo_path); +string_id!(AppviewEndpoint, "appview endpoint", parse_appview_endpoint); +string_id!(KnotServiceUrl, "knot service URL", parse_service_url); +string_id!(PushOption, "push option", parse_push_option); + +impl RepoDid { + pub fn is_plc(&self) -> bool { + self.as_str().starts_with("did:plc:") + } +} + +impl From<&DidRkey> for RepoDid { + fn from(rkey: &DidRkey) -> Self { + Self(rkey.0.clone()) + } +} + +impl RecordRkey { + pub fn minted(tid: Tid) -> Self { + Self::Minted(tid) + } + + pub fn composed(key: LabelKey) -> Self { + Self::Composed(key) + } + + pub fn as_label_key(&self) -> Option { + match self { + Self::Composed(key) => Some(key.clone()), + Self::Minted(tid) => Some(LabelKey(tid.as_str().to_owned())), + Self::Subject(_) => None, + } + } +} + +#[derive(Clone)] +pub enum RecordRkey { + Minted(Tid), + Composed(LabelKey), + Subject(DidRkey), +} + +impl RecordRkey { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + match parse_tid(&value) { + Some(tid) => Ok(Self::Minted(tid)), + None if ported_did_web(&value) => Err(ParseError::invalid("record key", value)), + None => DidRkey::new(value.clone()) + .map(Self::Subject) + .or_else(|_| LabelKey::new(value).map(Self::Composed)), + } + } + + pub fn as_str(&self) -> &str { + match self { + Self::Minted(tid) => tid.as_str(), + Self::Composed(key) => key.as_str(), + Self::Subject(key) => key.as_str(), + } + } +} + +string_id!(@traits RecordRkey); +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] +#[serde(transparent)] +pub struct PushOptions(Vec); + +impl PushOptions { + pub const MAX: usize = 50; + + pub fn new(options: impl IntoIterator) -> Self { + Self(options.into_iter().take(Self::MAX).collect()) + } + + pub fn as_slice(&self) -> &[PushOption] { + &self.0 + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +#[derive(Clone, PartialEq, Eq)] +pub enum OwnerRef { + Did(OwnerDid), + Handle(Handle), +} + +impl OwnerRef { + pub fn parse(segment: &str) -> Option { + match OwnerDid::new(segment) { + Ok(did) => Some(Self::Did(did)), + // Trying a handle only when the segment isn't `did:`-prefixed! + // Not perfect by any means but avoids a network call if DID + // is malformed. + Err(_) if !segment.starts_with("did:") => { + Handle::new_owned(segment).ok().map(Self::Handle) + } + Err(_) => None, + } + } +} + +impl KnotServiceUrl { + pub fn authority(&self) -> &str { + self.0 + .split_once("://") + .map(|(_, rest)| rest) + .unwrap_or(&self.0) + } +} + +impl KnotHostname { + pub fn knot_did(&self) -> KnotId { + KnotId::new(format!("did:web:{}", self.0)).expect("Validated knot hostname forms did:web") + } + + pub fn service_url(&self) -> KnotServiceUrl { + KnotServiceUrl::new(format!("https://{}", self.0)) + .expect("Validated knot hostname forms a knot service URL") + } +} + +impl BranchName { + pub fn head_ref(&self) -> RefName { + RefName::new(format!("refs/heads/{}", self.0)) + .expect("Validated branch name forms refs/heads ref") + } +} + +impl TagName { + pub fn tag_ref(&self) -> RefName { + RefName::new(format!("refs/tags/{}", self.0)) + .expect("Validated tag name forms refs/tags ref") + } +} + +impl RefName { + pub fn branch_name(&self) -> Option { + self.0 + .strip_prefix("refs/heads/") + .map(|name| BranchName(name.to_string())) + } + + pub fn tag_name(&self) -> Option { + self.0 + .strip_prefix("refs/tags/") + .map(|name| TagName(name.to_string())) + } +} + +impl RepoPath { + pub fn components(&self) -> impl Iterator { + self.0.split('/') + } + + pub fn names_dot_git(&self) -> bool { + self.components() + .any(|part| part.eq_ignore_ascii_case(".git")) + } + + pub fn parent(&self) -> Option { + self.0 + .rsplit_once('/') + .map(|(dir, _)| RepoPath(dir.to_string())) + } + + pub fn file_name(&self) -> &str { + self.0 + .rsplit_once('/') + .map(|(_, name)| name) + .unwrap_or(&self.0) + } +} + +impl OwnerDid { + pub fn is(&self, account: &AccountDid) -> bool { + self.as_str() == account.as_str() + } +} + +impl From for AccountDid { + fn from(owner: OwnerDid) -> Self { + AccountDid(owner.0) + } +} + +impl From for OwnerDid { + fn from(account: AccountDid) -> Self { + OwnerDid(account.0) + } +} + +impl From for AccountDid { + fn from(repo: RepoDid) -> Self { + AccountDid(repo.0) + } +} +impl From for AccountDid { + fn from(service: ServiceDid) -> Self { + AccountDid(service.0) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClonePath { + rkeys: Vec, + names: Vec, +} + +impl ClonePath { + pub fn parse(raw: &str) -> Option { + let segments = || std::iter::once(raw).chain(raw.strip_suffix(".git")); + let rkeys: Vec = segments() + .filter_map(|segment| RepoRkey::new(segment).ok()) + .collect(); + let names: Vec = segments() + .filter_map(|segment| RepoName::new(segment).ok()) + .collect(); + (!rkeys.is_empty() || !names.is_empty()).then_some(Self { rkeys, names }) + } + + pub fn rkeys(&self) -> impl Iterator { + self.rkeys.iter() + } + + pub fn names(&self) -> impl Iterator { + self.names.iter() + } +} + +const SECP256K1_MULTICODEC: u64 = 0xe7; + +impl ActorId { + pub fn from_secp256k1(sec1_bytes: &[u8]) -> Self { + Self(multikey(SECP256K1_MULTICODEC, sec1_bytes)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct OfferedKey(Vec); + +impl OfferedKey { + pub fn from_bytes(bytes: impl Into>) -> Self { + Self(bytes.into()) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Oid(gix_hash::ObjectId); + +impl Oid { + pub const fn null() -> Self { + Self(gix_hash::ObjectId::null(gix_hash::Kind::Sha1)) + } + + pub fn is_null(self) -> bool { + self.0.is_null() + } + + pub fn from_hex(hex: &str) -> Result { + gix_hash::ObjectId::from_hex(hex.as_bytes()) + .map(Self) + .map_err(|_| ParseError::invalid("oid", hex)) + } + + pub fn object_id(self) -> gix_hash::ObjectId { + self.0 + } + + pub fn to_hex(self) -> String { + self.0.to_hex().to_string() + } +} + +impl From for Oid { + fn from(value: gix_hash::ObjectId) -> Self { + Self(value) + } +} + +impl From for gix_hash::ObjectId { + fn from(value: Oid) -> Self { + value.0 + } +} + +impl FromStr for Oid { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + Self::from_hex(s) + } +} + +impl fmt::Display for Oid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0.to_hex(), f) + } +} + +impl Serialize for Oid { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.to_hex()) + } +} + +impl<'de> Deserialize<'de> for Oid { + fn deserialize>(deserializer: D) -> Result { + let hex = String::deserialize(deserializer)?; + Self::from_hex(&hex).map_err(serde::de::Error::custom) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum RefTransition { + Create { new: Oid }, + Advance { old: Oid, new: Oid }, + Delete { old: Oid }, +} + +impl RefTransition { + pub fn old_oid(self) -> Option { + match self { + Self::Create { .. } => None, + Self::Advance { old, .. } | Self::Delete { old } => Some(old), + } + } + + pub fn new_oid(self) -> Option { + match self { + Self::Create { new } | Self::Advance { new, .. } => Some(new), + Self::Delete { .. } => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ObjectFormat(gix_hash::Kind); + +impl ObjectFormat { + pub const SHA1: Self = Self(gix_hash::Kind::Sha1); + pub const SHA256: Self = Self(gix_hash::Kind::Sha256); + + pub fn from_kind(kind: gix_hash::Kind) -> Self { + Self(kind) + } + + pub fn kind(self) -> gix_hash::Kind { + self.0 + } + + pub fn from_capability(token: &str) -> Option { + match token { + "sha1" => Some(Self::SHA1), + "sha256" => Some(Self::SHA256), + _ => None, + } + } + + pub fn capability(self) -> &'static str { + match self.0 { + gix_hash::Kind::Sha256 => "sha256", + _ => "sha1", + } + } + + pub fn null_oid(self) -> Oid { + Oid(self.0.null()) + } +} + +impl Default for ObjectFormat { + fn default() -> Self { + Self::SHA1 + } +} + +impl<'de> serde::Deserialize<'de> for ObjectFormat { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let token = String::deserialize(deserializer)?; + Self::from_capability(token.as_str()).ok_or_else(|| { + ::invalid_value( + serde::de::Unexpected::Str(&token), + &"\"sha1\" or \"sha256\"", + ) + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(transparent)] +pub struct UnixSeconds(i64); + +impl UnixSeconds { + pub const fn new(seconds: i64) -> Self { + Self(seconds) + } + + pub const fn get(self) -> i64 { + self.0 + } + + pub const fn saturating_add_secs(self, secs: i64) -> Self { + Self(self.0.saturating_add(secs)) + } + + pub const fn saturating_sub_secs(self, secs: i64) -> Self { + Self(self.0.saturating_sub(secs)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct UnixMicros(u64); + +impl UnixMicros { + pub const fn new(micros: u64) -> Self { + Self(micros) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn next(self) -> Self { + Self(self.0.saturating_add(1)) + } + + pub const fn prev(self) -> Self { + Self(self.0.saturating_sub(1)) + } + + pub const fn seconds(self) -> UnixSeconds { + UnixSeconds((self.0 / 1_000_000) as i64) + } +} + +impl fmt::Display for UnixSeconds { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ObjectCount(usize); + +impl ObjectCount { + pub const fn new(count: usize) -> Self { + Self(count) + } + + pub const fn get(self) -> usize { + self.0 + } + + pub const fn succ(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +impl From for ObjectCount { + fn from(count: u32) -> Self { + Self(count as usize) + } +} + +impl fmt::Display for ObjectCount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +#[derive( + Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, +)] +#[serde(transparent)] +pub struct LanguageBytes(u64); + +impl LanguageBytes { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn saturating_add_bytes(self, bytes: u64) -> Self { + Self(self.0.saturating_add(bytes)) + } +} + +impl fmt::Display for LanguageBytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(transparent)] +pub struct HttpStatus(u16); + +impl HttpStatus { + pub const fn new(code: u16) -> Self { + Self(code) + } + + pub const fn get(self) -> u16 { + self.0 + } + + pub const fn is_success(self) -> bool { + self.0 >= 200 && self.0 < 300 + } + + pub const fn is_server_error(self) -> bool { + self.0 >= 500 && self.0 < 600 + } + + pub const fn is_transient(self) -> bool { + self.0 == 429 || self.is_server_error() + } +} + +impl From for HttpStatus { + fn from(status: http::StatusCode) -> Self { + Self(status.as_u16()) + } +} + +impl fmt::Display for HttpStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct CobId(Oid); + +impl CobId { + pub fn new(oid: Oid) -> Self { + Self(oid) + } + + pub fn oid(self) -> Oid { + self.0 + } +} + +impl fmt::Display for CobId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +impl Serialize for CobId { + fn serialize(&self, serializer: S) -> Result { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for CobId { + fn deserialize>(deserializer: D) -> Result { + Oid::deserialize(deserializer).map(Self) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ChangeId(Oid); + +impl ChangeId { + pub fn new(oid: Oid) -> Self { + Self(oid) + } + + pub fn oid(self) -> Oid { + self.0 + } +} + +impl fmt::Display for ChangeId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +impl Serialize for ChangeId { + fn serialize(&self, serializer: S) -> Result { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for ChangeId { + fn deserialize>(deserializer: D) -> Result { + Oid::deserialize(deserializer).map(Self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dids_validate_and_canonicalize() { + [ + ("did:plc:nel", Some("did:plc:nel")), + ( + "did:plc:7iza6de2dwap2sbkpav7c6c6", + Some("did:plc:7iza6de2dwap2sbkpav7c6c6"), + ), + ("did:web:oyster.cafe", Some("did:web:oyster.cafe")), + ("did:web:OYSTER.cafe", Some("did:web:oyster.cafe")), + ("did:web:Oyster.Cafe", Some("did:web:oyster.cafe")), + ("at://did:plc:nel", Some("did:plc:nel")), + ("not-a-did", None), + ("", None), + ] + .iter() + .for_each(|&(input, expected)| { + assert_eq!( + RepoDid::new(input).ok().as_ref().map(|did| did.as_str()), + expected, + "{input:?}" + ); + }); + } + + #[test] + fn owner_is_account_compares_by_canonical_did() { + [ + ("did:web:OYSTER.cafe", "did:web:oyster.cafe", true), + ("did:plc:nel", "did:plc:nel", true), + ("did:plc:ABC", "did:plc:abc", false), + ] + .iter() + .for_each(|&(owner, account, matches)| { + assert_eq!( + OwnerDid::new(owner) + .unwrap() + .is(&AccountDid::new(account).unwrap()), + matches, + "{owner:?} vs {account:?}" + ); + }); + } + + #[test] + fn repo_names_reject_traversal() { + assert!(RepoName::new("anemone").is_ok()); + assert!(RepoName::new("with-dash_and.dot").is_ok()); + assert!(RepoName::new("../escape").is_err()); + assert!(RepoName::new("a/b").is_err()); + assert!(RepoName::new("..").is_err()); + assert!(RepoName::new("").is_err()); + } + + #[test] + fn repo_rkeys_follow_record_key_rules() { + assert!(RepoRkey::new("anemone").is_ok()); + assert!(RepoRkey::new("3lkz2mvgyx22a").is_ok()); + assert!(RepoRkey::new("with-dash_and.dot~tilde:colon").is_ok()); + assert!(RepoRkey::new(".").is_err()); + assert!(RepoRkey::new("..").is_err()); + assert!(RepoRkey::new("a/b").is_err()); + assert!(RepoRkey::new("space bar").is_err()); + assert!(RepoRkey::new("").is_err()); + assert!(RepoRkey::new("x".repeat(513)).is_err()); + } + + #[test] + fn uri_rkeys_accept_record_key_alphabet_and_reject_dot_and_dotdot() { + assert!( + UriRkey::new("refs~1").is_ok(), + "An escaped git ref should be an rkey" + ); + assert!(UriRkey::new("3lubrptx57d22").is_ok()); + assert!(UriRkey::new(".").is_err()); + assert!(UriRkey::new("..").is_err()); + assert!(UriRkey::new("a/b").is_err()); + } + + #[test] + fn appview_endpoints_require_http_base_url() { + assert_eq!( + AppviewEndpoint::new("https://tangled.test/") + .unwrap() + .as_str(), + "https://tangled.test" + ); + assert_eq!( + AppviewEndpoint::new("https://tangled.test//") + .unwrap() + .as_str(), + "https://tangled.test" + ); + assert!(AppviewEndpoint::new("http://appview.oyster.cafe/base").is_ok()); + assert_eq!( + AppviewEndpoint::new("https://TANGLED.test") + .unwrap() + .as_str(), + "https://tangled.test" + ); + assert_eq!( + AppviewEndpoint::new("https://tangled.test:443") + .unwrap() + .as_str(), + "https://tangled.test" + ); + assert_eq!( + AppviewEndpoint::new("https://tangled.test:8443/base") + .unwrap() + .as_str(), + "https://tangled.test:8443/base" + ); + assert!(AppviewEndpoint::new("tangled.test").is_err()); + assert!(AppviewEndpoint::new("https://tangled.test:+8443").is_err()); + assert!(AppviewEndpoint::new("ftp://tangled.test").is_err()); + assert!(AppviewEndpoint::new("https://").is_err()); + assert!(AppviewEndpoint::new("https:///pulls").is_err()); + assert!(AppviewEndpoint::new("https://tangled.test/a b").is_err()); + assert!(AppviewEndpoint::new("https://nel@tangled.test").is_err()); + assert!(AppviewEndpoint::new("https://nel:hunter2@tangled.test").is_err()); + assert!(AppviewEndpoint::new("https://tangled.test?utm=knot").is_err()); + assert!(AppviewEndpoint::new("https://tangled.test#pulls").is_err()); + assert!(AppviewEndpoint::new("https://tangled.test#").is_err()); + } + + #[test] + fn clone_paths_try_the_exact_segment_before_the_stripped_one() { + let suffixed = ClonePath::parse("anemone.git").unwrap(); + assert_eq!( + suffixed.rkeys().cloned().collect::>(), + vec![ + RepoRkey::new("anemone.git").unwrap(), + RepoRkey::new("anemone").unwrap() + ], + "literal .git rkey wins over conventional suffix interpretation" + ); + assert_eq!( + suffixed.names().cloned().collect::>(), + vec![ + RepoName::new("anemone.git").unwrap(), + RepoName::new("anemone").unwrap() + ] + ); + + let plain = ClonePath::parse("anemone").unwrap(); + assert_eq!( + plain.rkeys().cloned().collect::>(), + vec![RepoRkey::new("anemone").unwrap()] + ); + + let bare = ClonePath::parse(".git").unwrap(); + assert_eq!( + bare.rkeys().cloned().collect::>(), + vec![RepoRkey::new(".git").unwrap()], + "stripping .git from bare suffix leaves nothing valid to try" + ); + + assert!(ClonePath::parse("a/b.git").is_none()); + } + + #[test] + fn clone_paths_keep_segments_only_valid_as_one_of_the_two_kinds() { + let plus = ClonePath::parse("c++").unwrap(); + assert_eq!( + plus.rkeys().count(), + 0, + "a record key allows only [A-Za-z0-9._~:-]" + ); + assert_eq!( + plus.names().cloned().collect::>(), + vec![RepoName::new("c++").unwrap()], + "A repo name accepts the wider charset, so the segment resolves by name" + ); + + let long = "x".repeat(200); + let overlong = ClonePath::parse(&long).unwrap(); + assert_eq!(overlong.names().count(), 0, "a repo name is at most 100"); + assert_eq!( + overlong.rkeys().cloned().collect::>(), + vec![RepoRkey::new(&long).unwrap()] + ); + } + + #[test] + fn repo_paths_reject_traversal_and_expose_structure() { + assert!(RepoPath::new("src/lib.rs").is_ok()); + assert!(RepoPath::new("a b/c.txt").is_ok()); + assert!(RepoPath::new("back\\slash.txt").is_ok()); + assert!(RepoPath::new(".git/config").is_ok()); + assert!(RepoPath::new("../escape").is_err()); + assert!(RepoPath::new("nested/../escape").is_err()); + assert!(RepoPath::new("nested/./here").is_err()); + assert!(RepoPath::new("/absolute").is_err()); + assert!(RepoPath::new("trailing/").is_err()); + assert!(RepoPath::new("double//slash").is_err()); + assert!(RepoPath::new("").is_err()); + assert!(RepoPath::new("nul\0byte").is_err()); + + let path = RepoPath::new("src/deep/lib.rs").unwrap(); + assert_eq!(path.parent().unwrap().as_str(), "src/deep"); + assert_eq!(path.file_name(), "lib.rs"); + assert!(path.parent().unwrap().parent().unwrap().parent().is_none()); + assert_eq!( + path.components().collect::>(), + vec!["src", "deep", "lib.rs"] + ); + assert!(RepoPath::new(".git/hooks").unwrap().names_dot_git()); + assert!(RepoPath::new("dir/.GIT/x").unwrap().names_dot_git()); + assert!(!RepoPath::new("gitless").unwrap().names_dot_git()); + } + + #[test] + fn ref_transitions_expose_their_edge_oids() { + let before = Oid::from_hex(&"1".repeat(40)).unwrap(); + let after = Oid::from_hex(&"2".repeat(40)).unwrap(); + let create = RefTransition::Create { new: after }; + let advance = RefTransition::Advance { + old: before, + new: after, + }; + let delete = RefTransition::Delete { old: before }; + assert_eq!(create.old_oid(), None); + assert_eq!(create.new_oid(), Some(after)); + assert_eq!(advance.old_oid(), Some(before)); + assert_eq!(advance.new_oid(), Some(after)); + assert_eq!(delete.old_oid(), Some(before)); + assert_eq!(delete.new_oid(), None); + } + + #[test] + fn ref_names_follow_git_rules() { + assert!(RefName::new("refs/heads/main").is_ok()); + assert!(RefName::new("refs/heads/feature/x").is_ok()); + assert!(RefName::new("refs/cobs/org.tangled.graph.collaborator/limpet").is_ok()); + assert!(RefName::new("refs/heads/bad..name").is_err()); + assert!(RefName::new("refs/heads/space bar").is_err()); + assert!(RefName::new("trailing.lock").is_err()); + assert!(RefName::new("/leading").is_err()); + assert!(RefName::new("refs/heads/.hidden").is_err()); + assert!(RefName::new("refs/x.lock/y").is_err()); + assert!(RefName::new("refs/heads/ends.").is_err()); + assert!(RefName::new("refs/heads//double").is_err()); + assert!(RefName::new("refs/heads/trailing/").is_err()); + assert!(RefName::new("HEAD").is_err()); + assert!(RefName::new("CONFIG").is_err()); + assert!(RefName::new("config").is_err()); + assert!(RefName::new("FETCH_HEAD").is_err()); + assert!(RefName::new("main").is_err()); + assert!(RefName::new("refs").is_err()); + assert!(RefName::new("").is_err()); + } + + #[test] + fn oid_roundtrips_through_hex() { + let hex = "0123456789abcdef0123456789abcdef01234567"; + let oid = Oid::from_hex(hex).expect("Valid SHA-1 hex"); + assert_eq!(oid.to_hex(), hex); + assert_eq!(oid, Oid::from(oid.object_id())); + assert!(Oid::from_hex("zz").is_err()); + assert!(Oid::null().is_null()); + assert!(!oid.is_null()); + } + + #[test] + fn object_format_round_trips_its_capability_token() { + assert_eq!(ObjectFormat::SHA1.capability(), "sha1"); + assert_eq!(ObjectFormat::SHA256.capability(), "sha256"); + assert_eq!( + ObjectFormat::from_capability("sha1"), + Some(ObjectFormat::SHA1) + ); + assert_eq!( + ObjectFormat::from_capability("sha256"), + Some(ObjectFormat::SHA256) + ); + assert_eq!(ObjectFormat::from_capability("md5"), None); + assert_eq!( + serde_json::from_str::("\"sha256\"").unwrap(), + ObjectFormat::SHA256 + ); + assert!(serde_json::from_str::("\"md5\"").is_err()); + assert_eq!(ObjectFormat::default(), ObjectFormat::SHA1); + } + + #[test] + fn object_format_null_oid_matches_the_hash_width() { + let sha1 = ObjectFormat::SHA1.null_oid(); + let sha256 = ObjectFormat::SHA256.null_oid(); + assert_eq!(sha1.to_hex().len(), 40); + assert_eq!(sha256.to_hex().len(), 64); + assert!(sha1.is_null()); + assert!(sha256.is_null()); + assert_eq!(sha1.object_id().kind(), gix_hash::Kind::Sha1); + assert_eq!(sha256.object_id().kind(), gix_hash::Kind::Sha256); + } + + #[test] + fn type_name_is_an_nsid() { + assert!(TypeName::new("org.tangled.graph.collaborator").is_ok()); + assert!(TypeName::new("not an nsid").is_err()); + } + + #[test] + fn knot_service_url_validates_and_exposes_its_authority() { + let url = KnotServiceUrl::new("https://knot.nel.pet/").unwrap(); + assert_eq!(url.as_str(), "https://knot.nel.pet"); + assert_eq!(url.authority(), "knot.nel.pet"); + assert_eq!( + KnotServiceUrl::new("https://knot.nel.pet:8443") + .unwrap() + .authority(), + "knot.nel.pet:8443" + ); + assert_eq!( + KnotServiceUrl::new("https://knot.nel.pet:443") + .unwrap() + .authority(), + "knot.nel.pet" + ); + assert!(KnotServiceUrl::new("knot.nel.pet").is_err()); + assert!(KnotServiceUrl::new("ftp://knot.nel.pet").is_err()); + assert!(KnotServiceUrl::new("https://knot.nel.pet/base").is_err()); + assert!(KnotServiceUrl::new("https://nel@knot.nel.pet").is_err()); + assert!(KnotServiceUrl::new("https://nel:hunter2@knot.nel.pet").is_err()); + assert!(KnotServiceUrl::new("https://knot.nel.pet?utm=knot").is_err()); + assert!(KnotServiceUrl::new("https://knot.nel.pet#pulls").is_err()); + assert!(KnotServiceUrl::new("https://").is_err()); + assert!(KnotServiceUrl::new("").is_err()); + } + + #[test] + fn knot_hostname_validates_and_forms_did_web() { + let host = KnotHostname::new("oyster.cafe").unwrap(); + assert_eq!(host.knot_did(), KnotId::new("did:web:oyster.cafe").unwrap()); + assert!(KnotHostname::new("").is_err()); + assert!(KnotHostname::new("not a host").is_err()); + assert!(KnotHostname::new("knot.nel.pet:8443").is_err()); + assert!(KnotHostname::new("knot.nel.pet/path").is_err()); + } + + #[test] + fn a_did_record_key_canonicalizes_production_dids_and_refuses_the_rest() { + [ + ( + "did:plc:3fwecdnvtcscjnrx2p4n7alz", + Some("did:plc:3fwecdnvtcscjnrx2p4n7alz"), + ), + ("did:web:KNOT.oyster.cafe", Some("did:web:knot.oyster.cafe")), + ("did:web:localhost%3A3000", None), + ("did:web:oyster.cafe:knots:one", None), + ("not-a-did", None), + ("", None), + (&format!("did:plc:{}", "a".repeat(512)), None), + ] + .into_iter() + .for_each(|(value, canonical)| { + assert_eq!( + DidRkey::new(value).ok().as_ref().map(DidRkey::as_str), + canonical, + "{value:?}" + ); + }); + } + + #[test] + fn every_did_record_key_is_already_a_repository_did() { + let rkey = DidRkey::new("did:web:knot.oyster.cafe").unwrap(); + assert_eq!(RepoDid::from(&rkey).as_str(), rkey.as_str()); + } + + #[test] + fn ported_did_web_record_key_rejected() { + assert!( + RecordRkey::new("did:web:oyster.cafe:3333").is_err(), + "a ported did:web is nobody's record key" + ); + assert!( + RecordRkey::new("did:web:oyster.cafe").is_ok(), + "an unported did:web stays a subject record key" + ); + } + + #[test] + fn push_options_enforce_the_lexicon_bounds_at_construction() { + assert_eq!( + PushOption::new("verbose-ci").unwrap().as_str(), + "verbose-ci" + ); + assert!(PushOption::new("x".repeat(1024)).is_ok()); + assert!(PushOption::new("x".repeat(1025)).is_err()); + assert!(PushOption::new("").is_err()); + assert!(PushOption::new("has\0nul").is_err()); + + let options = PushOptions::new( + (0..PushOptions::MAX + 10) + .map(|index| PushOption::new(format!("option-{index}")).unwrap()), + ); + assert_eq!(options.as_slice().len(), PushOptions::MAX); + assert_eq!(options.as_slice()[0].as_str(), "option-0"); + assert!(PushOptions::default().is_empty()); + } + + #[test] + fn branch_name_validates_and_forms_head_ref() { + let branch = BranchName::new("main").unwrap(); + assert_eq!(branch.head_ref(), RefName::new("refs/heads/main").unwrap()); + assert!(BranchName::new("feature/x").is_ok()); + assert!(BranchName::new("bad..name").is_err()); + assert!(BranchName::new("").is_err()); + } + + #[test] + fn author_text_strips_nul_and_newline_at_construction() { + assert_eq!(AuthorName::new("nel\nbailey").as_str(), "nelbailey"); + assert_eq!(Email::new("nel@oyster.cafe\0").as_str(), "nel@oyster.cafe"); + assert_eq!(AuthorName::new("teq").as_str(), "teq"); + } + + #[test] + fn author_text_strips_nul_and_newline_on_deserialize() { + let author: AuthorName = serde_json::from_str("\"nel\\nbailey\"").unwrap(); + assert_eq!(author.as_str(), "nelbailey"); + let email: Email = serde_json::from_str("\"nel@oyster.cafe\\u0000\"").unwrap(); + assert_eq!(email.as_str(), "nel@oyster.cafe"); + } + + #[test] + fn tag_name_validates_and_forms_tag_ref() { + let tag = TagName::new("v1.2.3").unwrap(); + assert_eq!(tag.tag_ref(), RefName::new("refs/tags/v1.2.3").unwrap()); + assert!(TagName::new("release/v1").is_ok()); + assert!(TagName::new("bad..name").is_err()); + assert!(TagName::new("v1.lock").is_err()); + assert!(TagName::new("").is_err()); + } + + #[test] + fn unix_seconds_arithmetic_saturates_at_the_bounds() { + let base = UnixSeconds::new(1_000); + assert_eq!(base.saturating_add_secs(60), UnixSeconds::new(1_060)); + assert_eq!(base.saturating_sub_secs(60), UnixSeconds::new(940)); + assert_eq!( + UnixSeconds::new(i64::MAX).saturating_add_secs(1), + UnixSeconds::new(i64::MAX) + ); + assert_eq!( + UnixSeconds::new(i64::MIN).saturating_sub_secs(1), + UnixSeconds::new(i64::MIN) + ); + assert_eq!(base.get(), 1_000); + assert_eq!(base.to_string(), "1000"); + } + + #[test] + fn micros_truncate_to_the_second_they_fall_in() { + assert_eq!(UnixMicros::new(1_999_999).seconds(), UnixSeconds::new(1)); + assert_eq!(UnixMicros::new(2_000_000).seconds(), UnixSeconds::new(2)); + assert_eq!(UnixMicros::new(0).seconds(), UnixSeconds::new(0)); + } + + #[test] + fn http_status_classifies_transient_codes() { + assert!(HttpStatus::new(200).is_success()); + assert!(!HttpStatus::new(200).is_transient()); + assert!(HttpStatus::new(429).is_transient()); + assert!(HttpStatus::new(503).is_transient()); + assert!(HttpStatus::new(503).is_server_error()); + assert!(!HttpStatus::new(404).is_transient()); + assert_eq!(HttpStatus::new(404).get(), 404); + } + + #[test] + fn actor_id_wraps_a_secp256k1_multikey() { + let mut compressed = [0u8; 33]; + compressed[0] = 0x02; + let actor = ActorId::from_secp256k1(&compressed); + let decoded = + PublicKey::decode(actor.as_str()).expect("from_secp256k1 yields decodable multikey"); + assert_eq!( + decoded.codec, + jacquard_common::types::crypto::KeyCodec::Secp256k1 + ); + assert_eq!(decoded.bytes.as_ref(), &compressed); + assert!(ActorId::new("not-a-multikey").is_err()); + } +} + +#[cfg(test)] +mod prop_tests { + use super::*; + use proptest::prelude::*; + + const DID: &str = "did:(plc:[a-z2-7]{24}|web:[a-z][a-z0-9-]{0,20}\\.(cafe|pet|dev))"; + + macro_rules! parse_display_identity { + ($name:ident, $ty:ty, $strategy:expr) => { + proptest! { + #[test] + fn $name(raw in $strategy) { + if let Ok(value) = <$ty>::new(raw.clone()) { + prop_assert_eq!(value.as_str(), raw.as_str()); + let reparsed = <$ty>::new(value.to_string()) + .expect("Display output reparses"); + prop_assert_eq!(reparsed, value); + } + } + } + }; + } + + parse_display_identity!(repo_did_identity, RepoDid, DID); + parse_display_identity!(owner_did_identity, OwnerDid, DID); + parse_display_identity!(knot_id_identity, KnotId, DID); + parse_display_identity!(account_did_identity, AccountDid, DID); + parse_display_identity!( + repo_name_identity, + RepoName, + "[A-Za-z0-9][A-Za-z0-9._-]{0,40}" + ); + parse_display_identity!( + repo_rkey_identity, + RepoRkey, + "[A-Za-z0-9][A-Za-z0-9._:~-]{0,40}" + ); + parse_display_identity!( + ref_name_identity, + RefName, + "refs/(heads|tags|cobs)/[a-z][a-z0-9]{0,7}(/[a-z][a-z0-9]{0,7}){0,3}" + ); + parse_display_identity!( + repo_path_identity, + RepoPath, + "[a-z][a-z0-9 ._-]{0,7}(/[a-z][a-z0-9 ._-]{0,7}){0,3}" + ); + parse_display_identity!( + type_name_identity, + TypeName, + "[a-z][a-z0-9]{0,7}(\\.[a-z][a-z0-9]{0,7}){2,4}" + ); + + proptest! { + #[test] + fn oid_hex_identity(hex in "[0-9a-f]{40}") { + let oid = Oid::from_hex(&hex).expect("Forty lowercase hex chars are valid SHA-1"); + prop_assert_eq!(oid.to_hex(), hex.clone()); + prop_assert_eq!(Oid::from_hex(&oid.to_hex()).expect("Reparse"), oid); + } + + #[test] + fn actor_id_identity(tag in 2u8..=3u8, body in prop::collection::vec(any::(), 32..=32)) { + let sec1: Vec = std::iter::once(tag).chain(body).collect(); + let actor = ActorId::from_secp256k1(&sec1); + let encoded = actor.as_str().to_string(); + let reparsed = ActorId::new(encoded.clone()).expect("Multikey output reparses"); + prop_assert_eq!(reparsed.as_str(), encoded.as_str()); + prop_assert_eq!(reparsed, actor); + } + } +} diff --git a/knot2/crates/knot-types/src/lib.rs b/knot2/crates/knot-types/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..139e488b3552bf3d9b8535721bf72b4f7ca22006 --- /dev/null +++ b/knot2/crates/knot-types/src/lib.rs @@ -0,0 +1,42 @@ +#[macro_use] +mod newtype; + +mod changes; +pub use changes::{ChangedFiles, ChangedFilesBudget, Listing}; + +mod ids; +pub use ids::{ + AccountDid, ActorId, AppviewEndpoint, AuthorName, BranchName, ChangeId, ClonePath, CobId, + DidRkey, Email, HttpStatus, KnotHostname, KnotId, KnotServiceUrl, LabelKey, LanguageBytes, + LanguageName, ObjectCount, ObjectFormat, OfferedKey, Oid, OriginUrl, OwnerDid, OwnerRef, + ParseError, PushOption, PushOptions, RecordCollection, RecordRkey, RefName, RefTransition, + RepoDid, RepoName, RepoPath, RepoRkey, ServiceDid, TagName, TypeName, UnixMicros, UnixSeconds, + UriRkey, +}; + +mod record; +pub use record::{ + AddressParseError, BlobCid, MAX_RECORD_BYTES, MstKey, MstKeyTooLong, RecordAddress, RecordBody, + RecordBodyError, RecordCid, Sha256Digest, SubjectNumber, dag_cbor_cid, +}; + +mod policy; +pub use policy::{ + AdmissionPolicy, Blocked, ContributionPermission, ContributionPolicy, Decision, Membership, + RepoPolicy, RepoRole, +}; + +mod hex; +pub use hex::{decode_hex, lowercase_hex}; + +mod net; +pub use net::{PeerKey, ProxyNetError, ProxyTrust, TrustedProxies, comma_separated}; + +pub use jacquard_common::CowStr; +pub use jacquard_common::DefaultStr; +pub use jacquard_common::service_auth; +pub use jacquard_common::types::collection::Collection; +pub use jacquard_common::types::string::{ + AtUri, Cid, Datetime, Did, DidService, Handle, Nsid, Rkey, Tid, +}; +pub use jacquard_common::types::{crypto, did_doc}; diff --git a/knot2/crates/knot-types/src/net.rs b/knot2/crates/knot-types/src/net.rs new file mode 100644 index 0000000000000000000000000000000000000000..18219aaeab8004c74e3caba9cd77e72067adf512 --- /dev/null +++ b/knot2/crates/knot-types/src/net.rs @@ -0,0 +1,309 @@ +use std::net::IpAddr; + +use http::{HeaderMap, HeaderName}; +pub use trusted_proxies::{ProxyNetError, TrustedProxies, comma_separated}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PeerKey { + Relayed(IpAddr), + Socket(IpAddr), + SocketWithIgnoredHeader(IpAddr), + Unidentified, +} + +impl PeerKey { + pub fn address(self) -> Option { + match self { + Self::Relayed(peer) | Self::Socket(peer) | Self::SocketWithIgnoredHeader(peer) => { + Some(peer) + } + Self::Unidentified => None, + } + } + + pub fn ignored_header(self) -> Option { + match self { + Self::SocketWithIgnoredHeader(peer) => Some(peer), + _ => None, + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct ProxyTrust { + header: Option, + proxies: TrustedProxies, +} + +impl ProxyTrust { + pub fn new(header: Option, proxies: TrustedProxies) -> Self { + Self { header, proxies } + } + + pub fn trusts_any_peer(&self) -> bool { + self.header.is_some() && self.proxies.is_empty() + } + + fn trusts(&self, socket: Option) -> bool { + match socket { + _ if self.proxies.is_empty() => true, + Some(socket) => self.proxies.contains(socket), + None => false, + } + } + + pub fn peer_key(&self, headers: &HeaderMap, socket: Option) -> PeerKey { + match (self.relayed_peer(headers, socket), socket) { + (Some(relayed), _) => PeerKey::Relayed(relayed.to_canonical()), + (None, None) => PeerKey::Unidentified, + (None, Some(socket)) => match self.ignores_header_from(headers, socket) { + true => PeerKey::SocketWithIgnoredHeader(socket.to_canonical()), + false => PeerKey::Socket(socket.to_canonical()), + }, + } + } + + pub fn client_peer(&self, headers: &HeaderMap, socket: Option) -> Option { + self.peer_key(headers, socket).address() + } + + pub fn client_peer_of(&self, headers: &HeaderMap, socket: IpAddr) -> IpAddr { + self.peer_key(headers, Some(socket)) + .address() + .unwrap_or(socket.to_canonical()) + } + + fn relayed_peer(&self, headers: &HeaderMap, socket: Option) -> Option { + self.header + .as_ref() + .filter(|_| self.trusts(socket)) + .and_then(|header| { + self.proxies.rightmost_untrusted( + headers + .get_all(header) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')), + ) + }) + } + + fn ignores_header_from(&self, headers: &HeaderMap, socket: IpAddr) -> bool { + self.header + .as_ref() + .is_some_and(|header| headers.contains_key(header)) + && !self.trusts(Some(socket)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn headers(value: Option<&str>) -> HeaderMap { + value + .map(|value| { + let mut map = HeaderMap::new(); + map.insert(forwarded_for(), value.parse().unwrap()); + map + }) + .unwrap_or_default() + } + + fn forwarded_for() -> HeaderName { + HeaderName::from_static("x-forwarded-for") + } + + fn ip(value: &str) -> IpAddr { + value.parse().unwrap() + } + + fn trusting<'a>(entries: impl IntoIterator) -> TrustedProxies { + TrustedProxies::parse(entries).unwrap() + } + + fn relaying<'a>(entries: impl IntoIterator) -> ProxyTrust { + ProxyTrust::new(Some(forwarded_for()), trusting(entries)) + } + + #[test] + fn the_knot_reads_the_rightmost_entry_from_any_peer_with_an_empty_allowlist() { + let anyone = ProxyTrust::new(Some(forwarded_for()), TrustedProxies::default()); + [ + (Some("203.0.113.7, 198.51.100.4"), "198.51.100.4"), + (Some(" 192.0.2.1 "), "192.0.2.1"), + (Some("not-an-ip"), "192.0.2.9"), + (None, "192.0.2.9"), + ] + .iter() + .for_each(|&(header, expected)| { + assert_eq!( + anyone.client_peer_of(&headers(header), ip("192.0.2.9")), + ip(expected), + "{header:?}" + ); + }); + } + + #[test] + fn the_knot_joins_every_line_of_a_repeated_header_into_one_chain() { + let mut map = HeaderMap::new(); + map.append(forwarded_for(), "203.0.113.7".parse().unwrap()); + map.append(forwarded_for(), "198.51.100.4".parse().unwrap()); + assert_eq!( + relaying(["127.0.0.1"]).client_peer(&map, Some(ip("127.0.0.1"))), + Some(ip("198.51.100.4")), + "a proxy that appends a second header line puts the address we want in the last line" + ); + } + + #[test] + fn the_header_applies_only_to_a_peer_on_the_allowlist() { + let relayed = headers(Some("198.51.100.4")); + [ + (&["127.0.0.1"][..], "127.0.0.1", "198.51.100.4", + "a request relayed by the listed proxy is limited by the address the proxy recorded"), + (&["127.0.0.1"], "203.0.113.7", "203.0.113.7", + "a client reaching the knot directly forged the header and must answer for its socket"), + (&["127.0.0.1", "::1"], "::1", "198.51.100.4", + "a second listed entry relays as readily as the first"), + (&["127.0.0.1"], "::ffff:127.0.0.1", "198.51.100.4", + "binding [::] turns an IPv4 proxy into ::ffff:127.0.0.1 and the allowlist must still match it"), + (&["::ffff:127.0.0.1"], "127.0.0.1", "198.51.100.4", + "an operator who writes the mapped form must match a plain IPv4 peer too"), + (&["127.0.0.1"], "::1", "::1", + "that peer answers for the socket it connected from, since the IPv6 loopback is a different address from the IPv4 loopback"), + (&["127.0.0.1"], "::ffff:203.0.113.7", "203.0.113.7", + "an ignored header still keys the peer on its socket, canonical so the warning and the bucket agree"), + ] + .iter() + .for_each(|&(listed, socket, expected, why)| { + assert_eq!( + relaying(listed.iter().copied()).client_peer(&relayed, Some(ip(socket))), + Some(ip(expected)), + "{why}: {listed:?} saw {socket}" + ); + }); + assert_eq!( + relaying(["127.0.0.1"]).client_peer( + &headers(Some("203.0.113.7, not-an-ip")), + Some(ip("127.0.0.1")) + ), + Some(ip("127.0.0.1")), + "the knot keys on the listed proxy's own socket when it can't read past the chain" + ); + } + + #[test] + fn a_caller_with_a_socket_address_gets_the_same_answer_without_an_option() { + let listed = trusting(["127.0.0.1"]); + [ + (ProxyTrust::default(), Some("198.51.100.4")), + ( + ProxyTrust::new(Some(forwarded_for()), listed), + Some("198.51.100.4"), + ), + ( + ProxyTrust::new(Some(forwarded_for()), TrustedProxies::default()), + None, + ), + ] + .into_iter() + .for_each(|(trust, value)| { + let headers = headers(value); + [ip("127.0.0.1"), ip("203.0.113.7"), ip("::ffff:203.0.113.7")] + .into_iter() + .for_each(|socket| { + assert_eq!( + Some(trust.client_peer_of(&headers, socket)), + trust.client_peer(&headers, Some(socket)), + "{trust:?} disagreed with itself for {socket} and header {value:?}" + ); + }); + }); + } + + #[test] + fn client_peer_falls_back_to_the_socket_whenever_the_header_doesnt_apply() { + let socket = ip("203.0.113.7"); + [ + (None, Some("198.51.100.4")), + (Some(forwarded_for()), None), + (Some(forwarded_for()), Some("not-an-ip")), + ] + .into_iter() + .for_each(|(header_name, header_value)| { + let trust = ProxyTrust::new(header_name.clone(), TrustedProxies::default()); + assert_eq!( + trust.client_peer(&headers(header_value), Some(socket)), + Some(socket), + "{header_name:?} with {header_value:?}" + ); + assert_eq!( + trust.client_peer(&headers(header_value), Some(ip("::ffff:203.0.113.7"))), + Some(socket), + "a v4-mapped socket and the plain v4 address are one client, so they share a key" + ); + }); + } + + #[test] + fn the_peer_key_separates_an_ignored_header_from_a_request_that_never_sent_one() { + let listed = ProxyTrust::new(Some(forwarded_for()), trusting(["127.0.0.1"])); + assert_eq!( + listed.peer_key(&headers(Some("198.51.100.4")), Some(ip("203.0.113.7"))), + PeerKey::SocketWithIgnoredHeader(ip("203.0.113.7")), + "an operator has to see the address of an unlisted peer that sent the header" + ); + assert_eq!( + listed.peer_key(&headers(None), Some(ip("203.0.113.7"))), + PeerKey::Socket(ip("203.0.113.7")), + "the allowlist stays untested when a request arrives without the header" + ); + assert_eq!( + listed.peer_key(&headers(Some("198.51.100.4")), Some(ip("127.0.0.1"))), + PeerKey::Relayed(ip("198.51.100.4")), + "the listed proxy relayed this one" + ); + assert_eq!( + listed.peer_key(&headers(Some("198.51.100.4")), None), + PeerKey::Unidentified + ); + assert_eq!( + listed.client_peer(&headers(Some("198.51.100.4")), None), + None, + "the knot won't key on a client until it has a socket to check against the allowlist" + ); + } + + #[test] + fn only_an_ignored_header_has_an_address_to_warn_about() { + assert_eq!( + PeerKey::SocketWithIgnoredHeader(ip("203.0.113.7")).ignored_header(), + Some(ip("203.0.113.7")) + ); + [ + PeerKey::Relayed(ip("198.51.100.4")), + PeerKey::Socket(ip("203.0.113.7")), + PeerKey::Unidentified, + ] + .into_iter() + .for_each(|key| { + assert_eq!( + key.ignored_header(), + None, + "{key:?} isn't a misconfigured allowlist" + ); + }); + } + + #[test] + fn only_a_header_without_an_allowlist_makes_the_knot_trust_any_peer() { + let listed = trusting(["127.0.0.1"]); + assert!( + ProxyTrust::new(Some(forwarded_for()), TrustedProxies::default()).trusts_any_peer() + ); + assert!(!ProxyTrust::new(Some(forwarded_for()), listed).trusts_any_peer()); + assert!(!ProxyTrust::default().trusts_any_peer()); + } +} diff --git a/knot2/crates/knot-types/src/newtype.rs b/knot2/crates/knot-types/src/newtype.rs new file mode 100644 index 0000000000000000000000000000000000000000..ececba4aac3479ff68a0fe9177e11f119fe17c12 --- /dev/null +++ b/knot2/crates/knot-types/src/newtype.rs @@ -0,0 +1,241 @@ +#[macro_export] +macro_rules! scalar_newtype { + ($( + $(#[$meta:meta])* + $vis:vis struct $name:ident($prim:ty) $(=> $mode:ident)?; + )+) => {$( + $(#[$meta])* + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + $vis struct $name($prim); + + impl $name { + $crate::scalar_ctor!($($mode)? , $prim); + + pub const fn get(self) -> $prim { + self.0 + } + } + + $crate::scalar_order!($($mode)? , $name); + )+}; +} + +#[macro_export] +macro_rules! scalar_ctor { + (, $prim:ty) => { + pub const fn new(value: $prim) -> Self { + Self(value) + } + }; + (ordered, $prim:ty) => { + $crate::scalar_ctor!(, $prim); + }; + (sealed, $prim:ty) => { + pub(crate) const fn new(value: $prim) -> Self { + Self(value) + } + }; +} + +#[macro_export] +macro_rules! scalar_order { + (, $name:ident) => {}; + (sealed, $name:ident) => {}; + (ordered, $name:ident) => { + impl ::core::cmp::Ord for $name { + fn cmp(&self, other: &Self) -> ::core::cmp::Ordering { + self.0.cmp(&other.0) + } + } + + impl ::core::cmp::PartialOrd for $name { + fn partial_cmp(&self, other: &Self) -> Option<::core::cmp::Ordering> { + Some(::core::cmp::Ord::cmp(self, other)) + } + } + }; +} +#[macro_export] +macro_rules! text_mode { + (verbatim, $value:expr) => { + $value + }; + (strip_control, $value:expr) => { + match $value { + value if value.contains(['\0', '\n']) => value.replace(['\0', '\n'], ""), + value => value, + } + }; +} + +#[macro_export] +macro_rules! text_newtype { + ($( + $(#[$meta:meta])* + $vis:vis struct $name:ident(String) => $mode:ident; + )+) => { + $crate::text_newtype! {$( + $(#[$meta])* + $vis struct $name(String) => $mode as new; + )+} + }; + ($( + $(#[$meta:meta])* + $vis:vis struct $name:ident(String) => $mode:ident as $ctor:ident; + )+) => {$( + $(#[$meta])* + #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + $vis struct $name(String); + + impl $name { + pub fn $ctor(value: impl Into) -> Self { + Self($crate::text_mode!($mode, value.into())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + + impl ::std::fmt::Debug for $name { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + f.debug_tuple(stringify!($name)).field(&self.0).finish() + } + } + + impl ::std::fmt::Display for $name { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + f.pad(&self.0) + } + } + + impl ::std::convert::AsRef for $name { + fn as_ref(&self) -> &str { + &self.0 + } + } + + impl ::std::borrow::Borrow for $name { + fn borrow(&self) -> &str { + &self.0 + } + } + + impl ::serde::Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } + } + + impl<'de> ::serde::Deserialize<'de> for $name { + fn deserialize>( + deserializer: D, + ) -> Result { + ::deserialize(deserializer).map(Self::$ctor) + } + } + )+}; +} + +#[macro_export] +macro_rules! read_counter { + ($name:ident, $record:ident, $read:ident, $reset:ident, $add:ident, $measure:ident) => { + $crate::scalar_newtype! { + pub struct $name(u64) => sealed; + } + + fn count() -> &'static ::std::thread::LocalKey<::std::cell::Cell> { + ::std::thread_local! { + static COUNT: ::std::cell::Cell = const { ::std::cell::Cell::new(0) }; + } + &COUNT + } + + pub(crate) fn $record() { + count().with(|cell| cell.set(cell.get().wrapping_add(1))); + } + + pub fn $read() -> $name { + $name::new(count().with(::std::cell::Cell::get)) + } + + pub fn $reset() { + count().with(|cell| cell.set(0)); + } + + pub fn $add(reads: $name) { + count().with(|cell| cell.set(cell.get().wrapping_add(reads.get()))); + } + + pub fn $measure(work: impl FnOnce() -> T) -> (T, $name) { + $reset(); + let value = work(); + (value, $read()) + } + }; +} + +#[cfg(test)] +mod tests { + crate::scalar_newtype! { + pub struct Bytes(u64) => ordered; + pub struct Tag(u64); + } + + crate::text_newtype! { + pub struct Header(String) => strip_control; + pub struct Body(String) => verbatim; + } + + crate::text_newtype! { + pub struct Column(String) => strip_control as from_column; + } + + fn sorted(mut values: Vec) -> Vec { + values.sort(); + values + } + + #[test] + fn only_a_newtype_that_declares_an_order_gets_one() { + assert_eq!(Bytes::new(7).get(), 7); + assert!(Bytes::new(7) < Bytes::new(8)); + assert_eq!( + sorted(vec![Bytes::new(8), Bytes::new(7)]), + vec![Bytes::new(7), Bytes::new(8)] + ); + assert_ne!(Tag::new(7), Tag::new(8)); + assert_eq!( + Tag::new(7).get().cmp(&Tag::new(8).get()), + std::cmp::Ordering::Less, + "a plain newtype still compares through the value it wraps, \ + so a caller that wants an ordering declares one instead of inheriting it" + ); + } + + #[test] + fn a_text_newtype_applies_its_mode_through_every_constructor_it_has() { + assert_eq!(Header::new("nel\nolaren\0").as_str(), "nelolaren"); + assert_eq!(Body::new("nel\nolaren\0").as_str(), "nel\nolaren\0"); + assert_eq!( + Column::from_column("nel\nolaren").as_str(), + "nelolaren", + "a type that names its constructor for where the value comes from \ + must still apply its mode" + ); + + let header: Header = serde_json::from_str("\"nel\\nolaren\"").unwrap(); + let body: Body = serde_json::from_str("\"nel\\nolaren\"").unwrap(); + let column: Column = serde_json::from_str("\"nel\\nolaren\"").unwrap(); + assert_eq!( + (header.as_str(), body.as_str(), column.as_str()), + ("nelolaren", "nel\nolaren", "nelolaren"), + "deserialization goes through the constructor, whatever it is named" + ); + assert_eq!(serde_json::to_string(&body).unwrap(), "\"nel\\nolaren\""); + + let map: std::collections::HashMap = + [(Body::new("kelp"), 1)].into_iter().collect(); + assert_eq!(map.get("kelp"), Some(&1), "a text newtype borrows as str"); + } +} diff --git a/knot2/crates/knot-types/src/policy.rs b/knot2/crates/knot-types/src/policy.rs new file mode 100644 index 0000000000000000000000000000000000000000..92c8c7084f9c21375d095c1d9223b20066f25034 --- /dev/null +++ b/knot2/crates/knot-types/src/policy.rs @@ -0,0 +1,300 @@ +use std::fmt; + +use serde::{Deserialize, Serialize}; + +use crate::ids::RepoDid; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AdmissionPolicy { + #[default] + Closed, + Open, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ContributionPolicy { + #[default] + Anyone, + Collaborators, + Members, +} + +impl ContributionPolicy { + pub const fn allows(self, role: RepoRole) -> bool { + match (self, role) { + (_, RepoRole::Owner | RepoRole::Collaborator) => true, + (Self::Anyone, _) => true, + (Self::Members, RepoRole::Member | RepoRole::KnotAdmin(Membership::Member)) => true, + (Self::Members, RepoRole::KnotAdmin(Membership::Outsider) | RepoRole::Stranger) => { + false + } + (Self::Collaborators, _) => false, + } + } + + pub const fn name(self) -> &'static str { + match self { + Self::Anyone => "anyone", + Self::Collaborators => "collaborators", + Self::Members => "members", + } + } +} + +impl fmt::Display for ContributionPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(self.name()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Blocked { + No, + Yes, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ContributionPermission { + blocked: Blocked, + role: RepoRole, + policy: ContributionPolicy, +} + +impl ContributionPermission { + pub const fn new(blocked: Blocked, role: RepoRole, policy: ContributionPolicy) -> Self { + Self { + blocked, + role, + policy, + } + } + + pub const fn role(self) -> RepoRole { + self.role + } + + pub const fn policy(self) -> ContributionPolicy { + self.policy + } + + pub const fn blocked(self) -> Blocked { + self.blocked + } + + pub const fn contributes(self) -> Decision { + match self.blocked { + Blocked::Yes => Decision::Deny, + Blocked::No => Decision::allow_if(self.policy.allows(self.role)), + } + } + + pub const fn moderates(self) -> Decision { + match self.blocked { + Blocked::Yes => Decision::Deny, + Blocked::No => Decision::allow_if(self.role.moderates()), + } + } +} + +impl fmt::Display for ContributionPermission { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let listed = match self.blocked { + Blocked::Yes => " on the blocklist", + Blocked::No => "", + }; + write!( + f, + "{}{listed}, where contribution policy is {}", + self.role, self.policy + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepoPolicy { + repo: RepoDid, + policy: ContributionPolicy, +} + +impl RepoPolicy { + pub const fn new(repo: RepoDid, policy: ContributionPolicy) -> Self { + Self { repo, policy } + } + + pub const fn repo(&self) -> &RepoDid { + &self.repo + } + + pub const fn policy(&self) -> ContributionPolicy { + self.policy + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Membership { + Member, + Outsider, +} + +impl Membership { + pub const fn of(member: bool) -> Self { + if member { Self::Member } else { Self::Outsider } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum RepoRole { + Owner, + Collaborator, + KnotAdmin(Membership), + Member, + Stranger, +} + +impl RepoRole { + pub const fn moderates(self) -> bool { + matches!(self, Self::Owner | Self::Collaborator | Self::KnotAdmin(_)) + } + + pub const fn name(self) -> &'static str { + match self { + Self::Owner => "owner", + Self::Collaborator => "collaborator", + Self::KnotAdmin(Membership::Member) => "knot admin on the member grant list", + Self::KnotAdmin(Membership::Outsider) => "knot admin", + Self::Member => "member", + Self::Stranger => "stranger", + } + } +} + +impl fmt::Display for RepoRole { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(self.name()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[must_use] +pub enum Decision { + Allow, + Deny, +} + +impl Decision { + pub const fn is_allowed(self) -> bool { + matches!(self, Decision::Allow) + } + + pub const fn allow_if(granted: bool) -> Self { + if granted { + Decision::Allow + } else { + Decision::Deny + } + } +} + +#[cfg(test)] +mod tests { + use super::{Blocked, ContributionPermission, ContributionPolicy, Membership, RepoRole}; + + const EVERY_ROLE: [RepoRole; 6] = [ + RepoRole::Owner, + RepoRole::Collaborator, + RepoRole::KnotAdmin(Membership::Member), + RepoRole::KnotAdmin(Membership::Outsider), + RepoRole::Member, + RepoRole::Stranger, + ]; + + #[test] + fn a_knot_admin_moderates_without_being_a_maintainer_of_somebody_else_s_repo() { + let outsider = RepoRole::KnotAdmin(Membership::Outsider); + let enrolled = RepoRole::KnotAdmin(Membership::Member); + assert!( + outsider.moderates() && enrolled.moderates(), + "admin carve-out exists for operator to moderate knot's text" + ); + assert!( + !matches!(outsider, RepoRole::Owner | RepoRole::Collaborator) + && !matches!(enrolled, RepoRole::Owner | RepoRole::Collaborator), + "Admin's authority stops at knot, and maintenance answers to owner" + ); + } + + #[test] + fn each_policy_allows_exactly_the_roles_beneath_it() { + let cases: &[(ContributionPolicy, &[RepoRole])] = &[ + (ContributionPolicy::Anyone, &EVERY_ROLE), + ( + ContributionPolicy::Members, + &[ + RepoRole::Owner, + RepoRole::Collaborator, + RepoRole::KnotAdmin(Membership::Member), + RepoRole::Member, + ], + ), + ( + ContributionPolicy::Collaborators, + &[RepoRole::Owner, RepoRole::Collaborator], + ), + ]; + let every = EVERY_ROLE; + cases.iter().for_each(|(policy, admitted)| { + every.iter().for_each(|role| { + assert_eq!( + policy.allows(*role), + admitted.contains(role), + "{policy:?} disagrees about {role:?}" + ); + }); + }); + } + + #[test] + fn default_policy_accepts_strangers_text() { + assert_eq!( + ContributionPolicy::default(), + ContributionPolicy::Anyone, + "policies nobody chose open repos rather than walling them off" + ); + } + + #[test] + fn a_blocked_subject_doesnt_contribute_or_moderate_whatever_it_would_otherwise_be() { + let every_role = EVERY_ROLE; + let every_policy = [ + ContributionPolicy::Anyone, + ContributionPolicy::Members, + ContributionPolicy::Collaborators, + ]; + every_policy.iter().for_each(|policy| { + every_role.iter().for_each(|role| { + let banned = ContributionPermission::new(Blocked::Yes, *role, *policy); + assert!( + !banned.contributes().is_allowed(), + "Banned {role:?} doesn't write anything under {policy:?}" + ); + assert!( + !banned.moderates().is_allowed(), + "Banned {role:?} doesn't moderate anything under {policy:?}" + ); + let permission = ContributionPermission::new(Blocked::No, *role, *policy); + assert_eq!( + permission.contributes().is_allowed(), + policy.allows(*role), + "Unbanned {role:?} contributes exactly as far as {policy:?} allows" + ); + assert_eq!( + permission.moderates().is_allowed(), + role.moderates(), + "Unbanned {role:?} moderates exactly as its role allows" + ); + }); + }); + } +} diff --git a/knot2/crates/knot-types/src/record.rs b/knot2/crates/knot-types/src/record.rs new file mode 100644 index 0000000000000000000000000000000000000000..59d4dcd64cf133fda262bcef6fc227921374c02a --- /dev/null +++ b/knot2/crates/knot-types/src/record.rs @@ -0,0 +1,610 @@ +use std::fmt; +use std::num::NonZeroU32; + +use cid::Cid as BlockCid; +use cid::multihash::Multihash; +use ipld_core::ipld::Ipld; +use jacquard_common::types::crypto::{DAG_CBOR, SHA2_256}; +use jacquard_common::types::string::AtUri; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::ids::{RecordCollection, RecordRkey, RepoDid}; + +pub(crate) const MST_KEY_MAX_BYTES: usize = 256; + +pub(crate) const RECORD_RKEY_BYTES: usize = 13; + +pub const MAX_RECORD_BYTES: usize = 1_000_000; + +const SHA2_256_BYTES: usize = 32; +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum AddressParseError { + #[error("URI authority has to be a repository DID")] + Authority, + #[error("Collection is missing from the URI or isn't a record collection")] + Collection, + #[error("The URI's record key is missing, or rkey isn't a TID, label key, or DID")] + RecordKey, +} +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct RecordAddress { + collection: RecordCollection, + rkey: RecordRkey, +} + +impl RecordAddress { + pub const fn new(collection: RecordCollection, rkey: RecordRkey) -> Self { + Self { collection, rkey } + } + + pub const fn collection(&self) -> &RecordCollection { + &self.collection + } + + pub const fn rkey(&self) -> &RecordRkey { + &self.rkey + } + + pub fn parse_path(path: &str) -> Result { + let (collection, rkey) = path.split_once('/').ok_or(AddressParseError::Collection)?; + let collection = + RecordCollection::new(collection).map_err(|_| AddressParseError::Collection)?; + let rkey = RecordRkey::new(rkey).map_err(|_| AddressParseError::RecordKey)?; + Ok(Self { collection, rkey }) + } + + pub fn mst_key(&self) -> MstKey { + MstKey(format!( + "{}/{}", + self.collection.as_str(), + self.rkey.as_str() + )) + } + + pub fn at_uri(&self, repo: &RepoDid) -> AtUri { + AtUri::from_parts_owned(repo.as_str(), self.collection.as_str(), self.rkey.as_str()) + .expect("Repo DID, collection NSID and record key compose at-uri") + } + + pub fn parse_at_uri(uri: &AtUri) -> Result<(RepoDid, Self), AddressParseError> { + let repo = + RepoDid::new(uri.authority().as_str()).map_err(|_| AddressParseError::Authority)?; + let collection = uri + .collection() + .and_then(|collection| RecordCollection::new(collection.as_str()).ok()) + .ok_or(AddressParseError::Collection)?; + let rkey = uri + .rkey() + .and_then(|rkey| RecordRkey::new(rkey.as_ref()).ok()) + .ok_or(AddressParseError::RecordKey)?; + Ok((repo, Self { collection, rkey })) + } +} + +impl fmt::Display for RecordAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(self.mst_key().as_str()) + } +} + +#[derive(Debug, thiserror::Error)] +#[error("MST key of {0} bytes is over {MST_KEY_MAX_BYTES} byte limit")] +pub struct MstKeyTooLong(pub usize); + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct MstKey(String); + +impl MstKey { + pub fn new(key: String) -> Result { + if key.len() > MST_KEY_MAX_BYTES { + Err(MstKeyTooLong(key.len())) + } else { + Ok(Self(key)) + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::borrow::Borrow for MstKey { + fn borrow(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for MstKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(&self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct SubjectNumber(NonZeroU32); + +impl SubjectNumber { + pub const FIRST: Self = Self(NonZeroU32::MIN); + + pub const fn new(value: u32) -> Option { + match NonZeroU32::new(value) { + Some(value) => Some(Self(value)), + None => None, + } + } + + pub const fn get(self) -> u32 { + self.0.get() + } + + pub const fn after(self) -> Option { + match self.0.checked_add(1) { + Some(next) => Some(Self(next)), + None => None, + } + } +} + +impl fmt::Display for SubjectNumber { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "#{}", self.0.get()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Sha256Digest([u8; SHA2_256_BYTES]); + +impl Sha256Digest { + pub const fn new(digest: [u8; SHA2_256_BYTES]) -> Self { + Self(digest) + } + + pub fn hash(bytes: &[u8]) -> Self { + Self(Sha256::digest(bytes).into()) + } + + pub const fn bytes(self) -> [u8; SHA2_256_BYTES] { + self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RecordCid([u8; SHA2_256_BYTES]); + +impl RecordCid { + pub fn new(cid: BlockCid) -> Option { + let hash = cid.hash(); + let dag_cbor_sha256 = cid.codec() == DAG_CBOR && hash.code() == SHA2_256; + dag_cbor_sha256 + .then(|| <[u8; SHA2_256_BYTES]>::try_from(hash.digest()).ok()) + .flatten() + .map(Self) + } + + pub fn parse(sent: &str) -> Option { + sent.parse::().ok().and_then(Self::new) + } + + pub fn of(body: &RecordBody) -> Self { + Self(Sha256::digest(body.as_bytes()).into()) + } + + pub fn cid(self) -> BlockCid { + BlockCid::new_v1(DAG_CBOR, sha2_256_multihash(&self.0)) + } +} + +impl Serialize for RecordCid { + fn serialize(&self, serializer: S) -> Result { + self.cid().serialize(serializer) + } +} + +fn sha2_256_multihash(digest: &[u8; SHA2_256_BYTES]) -> Multihash<64> { + Multihash::<64>::wrap(SHA2_256, digest).expect("Sha2-256 digest fits 64 byte multihash") +} + +pub fn dag_cbor_cid(block: &[u8]) -> BlockCid { + BlockCid::new_v1(DAG_CBOR, sha2_256_multihash(&Sha256::digest(block).into())) +} + +impl<'de> Deserialize<'de> for RecordCid { + fn deserialize>(deserializer: D) -> Result { + let cid = BlockCid::deserialize(deserializer)?; + Self::new(cid).ok_or_else(|| { + serde::de::Error::custom("A record CID is dag-cbor hashed with sha2-256") + }) + } +} + +const RAW: u64 = 0x55; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct BlobCid(Sha256Digest); + +impl BlobCid { + pub fn new(cid: BlockCid) -> Option { + let hash = cid.hash(); + let raw_sha256 = cid.codec() == RAW && hash.code() == SHA2_256; + raw_sha256 + .then(|| <[u8; SHA2_256_BYTES]>::try_from(hash.digest()).ok()) + .flatten() + .map(|digest| Self(Sha256Digest::new(digest))) + } + + pub const fn from_digest(digest: Sha256Digest) -> Self { + Self(digest) + } + + pub fn cid(self) -> BlockCid { + BlockCid::new_v1(RAW, sha2_256_multihash(&self.0.bytes())) + } + + pub const fn digest(self) -> Sha256Digest { + self.0 + } +} + +impl Serialize for BlobCid { + fn serialize(&self, serializer: S) -> Result { + match serializer.is_human_readable() { + true => BlobLink { + link: self.cid().to_string(), + } + .serialize(serializer), + false => self.cid().serialize(serializer), + } + } +} + +#[derive(Serialize, Deserialize)] +struct BlobLink { + #[serde(rename = "$link")] + link: String, +} + +impl BlobCid { + fn parse_cid(sent: &str) -> Result { + sent.parse::() + .map_err(|error| format!("`cid` must be a CID, and this one doesn't parse: {error}")) + .and_then(|cid| { + Self::new(cid).ok_or_else(|| { + "A blob CID addresses raw sha2-256 bytes, and this one doesn't.".to_owned() + }) + }) + } +} + +struct BlobCidVisitor; + +impl<'de> serde::de::Visitor<'de> for BlobCidVisitor { + type Value = BlobCid; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("A blob CID as a string or a `$link` map.") + } + + fn visit_str(self, value: &str) -> Result { + BlobCid::parse_cid(value).map_err(E::custom) + } + + fn visit_map>(self, map: A) -> Result { + BlobLink::deserialize(serde::de::value::MapAccessDeserializer::new(map)) + .and_then(|sent| BlobCid::parse_cid(&sent.link).map_err(serde::de::Error::custom)) + } +} + +impl<'de> Deserialize<'de> for BlobCid { + fn deserialize>(deserializer: D) -> Result { + match deserializer.is_human_readable() { + true => deserializer.deserialize_any(BlobCidVisitor), + false => Self::new(BlockCid::deserialize(deserializer)?).ok_or_else(|| { + serde::de::Error::custom("A blob CID is raw bytes hashed with sha2-256.") + }), + } + } +} + +impl fmt::Display for BlobCid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.cid().fmt(f) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum RecordBodyError { + #[error("Record body is dag-cbor and these bytes aren't: {0}")] + NotDagCbor(String), + #[error( + "Record body is canonical dag-cbor, and these {0} bytes aren't canonical encoding of their value" + )] + NotCanonical(usize), + #[error("Record body of {0} bytes is over {MAX_RECORD_BYTES} byte limit")] + Oversized(usize), +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RecordBody(Vec); + +impl RecordBody { + pub fn new(bytes: Vec) -> Result { + if bytes.len() > MAX_RECORD_BYTES { + return Err(RecordBodyError::Oversized(bytes.len())); + } + let value: Ipld = serde_ipld_dagcbor::from_slice(&bytes) + .map_err(|error| RecordBodyError::NotDagCbor(error.to_string()))?; + let canonical = serde_ipld_dagcbor::to_vec(&value) + .map_err(|error| RecordBodyError::NotDagCbor(error.to_string()))?; + if canonical == bytes { + Ok(Self(bytes)) + } else { + Err(RecordBodyError::NotCanonical(bytes.len())) + } + } + + pub fn from_cbor(value: &T) -> Result { + let bytes = serde_ipld_dagcbor::to_vec(value) + .map_err(|error| RecordBodyError::NotDagCbor(error.to_string()))?; + Self::new(bytes) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } + + pub fn into_bytes(self) -> Vec { + self.0 + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn cid(&self) -> RecordCid { + RecordCid::of(self) + } +} + +impl fmt::Display for RecordCid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.cid(), f) + } +} + +#[cfg(test)] +mod tests { + use super::{ + MAX_RECORD_BYTES, MST_KEY_MAX_BYTES, RecordAddress, RecordBody, RecordBodyError, RecordCid, + SubjectNumber, + }; + use crate::ids::{RecordCollection, RecordRkey, RepoDid}; + + fn address() -> RecordAddress { + RecordAddress::new( + RecordCollection::new("org.tangled.repo.issue").unwrap(), + RecordRkey::new("3lubrptx57d22").unwrap(), + ) + } + + #[test] + fn address_round_trips_through_its_at_uri() { + let repo = RepoDid::new("did:plc:squid").unwrap(); + assert_eq!( + RecordAddress::parse_at_uri(&address().at_uri(&repo)), + Ok((repo.clone(), address())), + "parse_at_uri ought to undo at_uri exactly" + ); + assert_eq!( + RecordAddress::parse_at_uri(&crate::AtUri::new_owned("at://did:plc:squid").unwrap()), + Err(super::AddressParseError::Collection), + "An at-uri without a collection shouldn't parse as a record address" + ); + } + + #[test] + fn an_address_renders_the_mst_key_and_the_at_uri_of_one_repo() { + let repo = RepoDid::new("did:plc:squid").unwrap(); + assert_eq!( + address().mst_key().as_str(), + "org.tangled.repo.issue/3lubrptx57d22" + ); + assert_eq!( + address().at_uri(&repo).as_str(), + "at://did:plc:squid/org.tangled.repo.issue/3lubrptx57d22", + "At-uri client resolves is an address under repository DID" + ); + } + + #[test] + fn record_key_parses_as_a_tid_or_a_composed_slug() { + assert!( + RecordRkey::new("3lubrptx57d22").is_ok_and(|rkey| rkey.as_label_key().is_some()), + "a TID-shaped record key can still key a label def" + ); + assert!( + RecordRkey::new("wont-fix.2").is_ok_and(|rkey| rkey.as_label_key().is_some()), + "A label def's record key is the slug the owner composed" + ); + assert!( + RecordRkey::new("self").is_ok_and(|rkey| rkey.as_label_key().is_some()), + "`self` is a slug here, and a DID isn't a slug: the colon isn't in the slug \ + charset" + ); + assert!( + RecordRkey::new("did:plc:squid").is_ok_and(|rkey| rkey.as_label_key().is_none()), + "a grant record is keyed by the subject's DID" + ); + assert!( + RecordRkey::new("did:web:knot.nel.pet:8080").is_err(), + "a ported did:web is a development form that isn't valid as a record key" + ); + assert!(RecordRkey::new("").is_err()); + assert!( + crate::LabelKey::new("documentation").is_ok(), + "a TID-shaped key is still a label key" + ); + let composed = RecordRkey::composed(crate::LabelKey::new("wont-fix.2").unwrap()); + assert!( + RecordRkey::new(composed.as_str()) + .is_ok_and(|reparsed| reparsed.as_label_key().is_some()), + "A composed key re-parses as composed, whatever string boundary the round trip crossed" + ); + } + + #[test] + fn a_subject_number_starts_at_one_and_stops_before_it_repeats_the_last_one() { + assert_eq!(SubjectNumber::FIRST.get(), 1); + assert_eq!(SubjectNumber::FIRST.after(), SubjectNumber::new(2)); + assert!(SubjectNumber::new(7) > SubjectNumber::new(1)); + assert_eq!(SubjectNumber::new(7).unwrap().to_string(), "#7"); + assert_eq!(SubjectNumber::new(0), None, "We start at #1 here"); + assert!( + serde_json::from_str::("0").is_err(), + "`0` from JSON fails the same rule as `SubjectNumber::new`" + ); + assert_eq!( + serde_json::from_str::("7").unwrap(), + SubjectNumber::new(7).unwrap() + ); + assert_eq!( + SubjectNumber::new(u32::MAX).unwrap().after(), + None, + "Count runs out instead of handing the last number out twice" + ); + } + + #[test] + fn a_collection_that_would_overrun_an_mst_key_isnt_a_record_collection() { + let room = MST_KEY_MAX_BYTES - super::RECORD_RKEY_BYTES - 1; + let segments = [ + "a".repeat(57), + "a".repeat(57), + "a".repeat(57), + "a".repeat(56), + ]; + let longest = format!("org.tangled.{}", segments.join(".")); + assert_eq!(longest.len(), room); + let collection = RecordCollection::new(&longest).expect("Longest key still fits"); + assert_eq!( + RecordAddress::new(collection, RecordRkey::new("3lubrptx57d22").unwrap()) + .mst_key() + .as_str() + .len(), + MST_KEY_MAX_BYTES, + "Bound is MST's own maximum key length, and this address is exactly that long" + ); + assert!( + RecordCollection::new(format!("{longest}a")).is_err(), + "objects never end up with one, addresses MST would refuse can't be built" + ); + } + + #[test] + fn a_record_body_parses_before_a_cid_labels_it_dag_cbor() { + let body = RecordBody::new( + serde_ipld_dagcbor::to_vec(&std::collections::BTreeMap::from([("text", "kelp")])) + .unwrap(), + ) + .expect("dag-cbor serializer wrote parses back"); + assert_eq!(body.cid(), RecordCid::of(&body)); + assert!( + matches!( + RecordBody::new(vec![0; MAX_RECORD_BYTES + 1]), + Err(RecordBodyError::Oversized(_)) + ), + "Body over the limit is turned away before it is decoded, whatever its bytes are" + ); + assert!( + RecordBody::new(vec![0xff, 0x00]).is_err(), + "bytes parse before digest runs, label on CID claims them" + ); + assert!( + RecordBody::new([body.as_bytes(), b"trailing"].concat()).is_err(), + "Trailing bytes don't decode; dag-cbor is one complete value" + ); + } + + #[test] + fn address_round_trips_through_mst_path() { + assert_eq!( + RecordAddress::parse_path(address().mst_key().as_str()), + Ok(address()), + "parse_path ought to undo mst_key exactly" + ); + assert!( + RecordAddress::parse_path("org.tangled.repo.issue").is_err(), + "a path without a record key can't be a record address" + ); + assert!( + RecordAddress::parse_path("org.tangled.repo.issue/not-a-tid-or-key!").is_err(), + "a path whose key half isn't an rkey can't be a record address" + ); + } + + fn door_out_of(bytes: &[u8]) -> &'static str { + match RecordBody::new(bytes.to_vec()) { + Ok(_) => "accepted", + Err(RecordBodyError::NotDagCbor(_)) => "not dag-cbor", + Err(RecordBodyError::NotCanonical(_)) => "not canonical", + Err(RecordBodyError::Oversized(_)) => "oversized", + } + } + + #[test] + fn dag_cbor_that_isnt_canonical_is_no_record_body() { + let cases: &[(&str, &str, &[u8])] = &[ + ("not canonical", "Non-minimal integer width", &[0x18, 0x05]), + ( + "not dag-cbor", + "an indefinite-length array", + &[0x9f, 0x01, 0xff], + ), + ( + "not dag-cbor", + "an indefinite-length string", + &[0x7f, 0x61, 0x61, 0xff], + ), + ( + "not canonical", + "map keys out of order", + &[0xa2, 0x61, 0x62, 0x01, 0x61, 0x61, 0x01], + ), + ( + "not dag-cbor", + "Duplicated map key", + &[0xa2, 0x61, 0x61, 0x01, 0x61, 0x61, 0x02], + ), + ]; + cases.iter().for_each(|(door, why, bytes)| { + assert_eq!(door_out_of(bytes), *door, "{why} is {door}"); + }); + assert_eq!( + door_out_of(&[0x05]), + "accepted", + "Canonical spelling of the same integer is the one that stands" + ); + } + + #[test] + fn a_record_cid_with_the_wrong_codec_is_refused() { + let body = RecordBody::new(serde_ipld_dagcbor::to_vec(&"kelp").unwrap()).unwrap(); + let raw = cid::Cid::new_v1(0x55, body.cid().cid().hash().to_owned()); + assert_eq!( + RecordCid::new(raw), + None, + "A raw block is not a record, however well it hashes" + ); + assert_eq!(RecordCid::new(body.cid().cid()), Some(body.cid())); + let encoded = serde_ipld_dagcbor::to_vec(&raw).unwrap(); + assert!( + serde_ipld_dagcbor::from_slice::(&encoded).is_err(), + "Wrong-codec CID from dag-cbor fails the same rule as `RecordCid::new`" + ); + } +} diff --git a/knot2/crates/knot-workflow/Cargo.toml b/knot2/crates/knot-workflow/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..e058bc718ddc97780b3e964612a5dcaef48a5647 --- /dev/null +++ b/knot2/crates/knot-workflow/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "knot-workflow" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +serde = { workspace = true } +serde_norway = { workspace = true } +globset = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } diff --git a/knot2/crates/knot-workflow/src/lib.rs b/knot2/crates/knot-workflow/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..0f9485d8025452a3713a472df02cffad6a4a1fb1 --- /dev/null +++ b/knot2/crates/knot-workflow/src/lib.rs @@ -0,0 +1,524 @@ +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; +use knot_types::{ChangedFiles, Listing, ParseError, RefName}; +use serde::Deserialize; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WorkflowName(String); + +impl WorkflowName { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let valid = + !value.is_empty() && !value.contains('/') && !value.chars().any(char::is_control); + match valid { + true => Ok(Self(value)), + false => Err(ParseError::Invalid { + kind: "workflow name", + value, + }), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +fn engine_is_named(engine: &str) -> Result<(), ParseError> { + match !engine.chars().any(|c| c.is_whitespace() || c.is_control()) { + true => Ok(()), + false => Err(ParseError::Invalid { + kind: "engine reference", + value: engine.to_string(), + }), + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CloneDepth { + Full, + Limited, +} + +impl CloneDepth { + fn parse(depth: i64) -> Result { + match u32::try_from(depth) { + Ok(0) => Ok(Self::Full), + Ok(_) => Ok(Self::Limited), + Err(_) => Err(ParseError::Invalid { + kind: "clone depth", + value: depth.to_string(), + }), + } + } +} + +pub struct RawWorkflow { + pub name: WorkflowName, + pub contents: Vec, +} + +pub enum Trigger { + Push { ref_name: RefName }, +} + +impl Trigger { + fn kind(&self) -> &'static str { + match self { + Trigger::Push { .. } => "push", + } + } +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Diagnostics { + pub errors: Vec, + pub warnings: Vec, +} + +impl Diagnostics { + fn error(&mut self, path: &str, message: impl AsRef) { + self.errors + .push(format!("error: {path}: {}", message.as_ref())); + } + + fn warning(&mut self, path: &str, kind: &str, reason: &str) { + self.warnings + .push(format!("warning: {path}: {kind}: {reason}")); + } + + fn combine(mut self, other: Diagnostics) -> Diagnostics { + self.errors.extend(other.errors); + self.warnings.extend(other.warnings); + self + } + + pub fn is_empty(&self) -> bool { + self.errors.is_empty() && self.warnings.is_empty() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompiledWorkflow { + pub name: WorkflowName, +} + +pub struct Compiled { + pub workflows: Vec, + pub diagnostics: Diagnostics, +} + +pub fn compile(raw: &[RawWorkflow], trigger: &Trigger, changed: &ChangedFiles) -> Compiled { + let (workflows, diagnostics) = raw + .iter() + .map(|workflow| compile_one(workflow, trigger, changed)) + .fold( + (Vec::new(), Diagnostics::default()), + |(mut workflows, diagnostics), (compiled, diag)| { + workflows.extend(compiled); + (workflows, diagnostics.combine(diag)) + }, + ); + Compiled { + workflows, + diagnostics, + } +} + +fn compile_one( + raw: &RawWorkflow, + trigger: &Trigger, + changed: &ChangedFiles, +) -> (Option, Diagnostics) { + let mut diag = Diagnostics::default(); + let parsed = match parse(&raw.contents) { + Ok(parsed) => parsed, + Err(error) => { + diag.error(raw.name.as_str(), error.to_string()); + return (None, diag); + } + }; + let matched = match workflow_matches(&parsed.when, trigger, changed) { + Ok(matched) => matched, + Err(error) => { + diag.error( + raw.name.as_str(), + format!("failed to execute workflow: {error}"), + ); + return (None, diag); + } + }; + if !matched { + diag.warning( + raw.name.as_str(), + "workflow skipped", + &format!("didn't match trigger {}", trigger.kind()), + ); + return (None, diag); + } + let depth = match CloneDepth::parse(parsed.clone.depth) { + Ok(depth) => depth, + Err(error) => { + diag.error(raw.name.as_str(), error.to_string()); + return (None, diag); + } + }; + analyze_clone(&parsed.clone, depth, raw.name.as_str(), &mut diag); + if parsed.engine.is_empty() { + diag.error(raw.name.as_str(), "missing engine"); + return (None, diag); + } + match engine_is_named(&parsed.engine) { + Ok(()) => ( + Some(CompiledWorkflow { + name: raw.name.clone(), + }), + diag, + ), + Err(error) => { + diag.error(raw.name.as_str(), error.to_string()); + (None, diag) + } + } +} + +fn analyze_clone(clone: &CloneOpts, depth: CloneDepth, path: &str, diag: &mut Diagnostics) { + if !clone.skip { + return; + } + [ + ("tags", clone.tags.is_some()), + ("submodules", clone.submodules.is_some()), + ("depth", depth == CloneDepth::Limited), + ] + .into_iter() + .filter(|(_, set)| *set) + .for_each(|(key, _)| { + diag.warning( + path, + "invalid configuration", + &format!("`clone.{key}` has no effect with `clone.skip`"), + ); + }); +} + +fn workflow_matches( + when: &[Constraint], + trigger: &Trigger, + changed: &ChangedFiles, +) -> Result { + if when.is_empty() { + return Ok(true); + } + when.iter() + .map(|constraint| constraint_matches(constraint, trigger, changed)) + .collect::, String>>() + .map(|results| results.contains(&true)) +} + +fn constraint_matches( + constraint: &Constraint, + trigger: &Trigger, + changed: &ChangedFiles, +) -> Result { + match trigger { + Trigger::Push { ref_name } => { + let event = constraint.event.0.iter().any(|kind| kind == "push"); + let reference = if let Some(branch) = ref_name.branch_name() { + glob_set(&constraint.branch.0)?.is_match(branch.as_str()) + } else if let Some(tag) = ref_name.tag_name() { + glob_set(&constraint.tag.0)?.is_match(tag.as_str()) + } else { + false + }; + let globs = glob_set(&constraint.paths.0)?; + let listed = changed + .paths() + .iter() + .any(|path| globs.is_match(path.as_str())); + // We aren't the one deciding whether this workflow runs, remember, + // spindles are, and spindles will decide from the + // atproto record we will emit. + let paths = constraint.paths.0.is_empty() + || listed + || matches!(changed.listing(), Listing::Truncated); + Ok(paths && event && reference) + } + } +} + +fn glob_set(patterns: &[String]) -> Result { + patterns + .iter() + .try_fold(GlobSetBuilder::new(), |mut builder, pattern| { + GlobBuilder::new(pattern) + .literal_separator(true) + .build() + .map(|glob| { + builder.add(glob); + builder + }) + .map_err(|error| error.to_string()) + }) + .and_then(|builder| builder.build().map_err(|error| error.to_string())) +} + +fn parse(contents: &[u8]) -> Result { + serde_norway::from_slice(contents) +} + +#[derive(Debug, Default, Deserialize)] +struct WorkflowFile { + #[serde(default)] + engine: String, + #[serde(default)] + when: Vec, + #[serde(default)] + clone: CloneOpts, +} + +#[derive(Debug, Default, Deserialize)] +struct Constraint { + #[serde(default)] + event: StringList, + #[serde(default)] + branch: StringList, + #[serde(default)] + tag: StringList, + #[serde(default)] + paths: StringList, +} + +#[derive(Debug, Default, Deserialize)] +struct CloneOpts { + #[serde(default)] + skip: bool, + #[serde(default)] + depth: i64, + #[serde(default)] + submodules: Option, + #[serde(default)] + tags: Option, +} + +#[derive(Debug, Default)] +struct StringList(Vec); + +impl<'de> Deserialize<'de> for StringList { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum OneOrMany { + One(String), + Many(Vec), + } + Ok(match OneOrMany::deserialize(deserializer)? { + OneOrMany::One(value) => StringList(vec![value]), + OneOrMany::Many(values) => StringList(values), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn raw(name: &str, contents: &str) -> RawWorkflow { + RawWorkflow { + name: WorkflowName::new(name).unwrap(), + contents: contents.as_bytes().to_vec(), + } + } + + fn changed(values: &[&str]) -> ChangedFiles { + let mut budget = knot_types::ChangedFilesBudget::new(); + let _ = values + .iter() + .try_for_each(|value| budget.admit(knot_types::RepoPath::new(*value).unwrap())); + budget.finish() + } + + fn push(reference: &str) -> Trigger { + Trigger::Push { + ref_name: RefName::new(reference).unwrap(), + } + } + + #[test] + fn workflow_name_round_trips_through_as_str() { + let name = WorkflowName::new("test.yml").unwrap(); + assert_eq!(name.as_str(), "test.yml"); + assert_eq!(name, WorkflowName::new("test.yml".to_string()).unwrap()); + assert_ne!(name, WorkflowName::new("ci.yml").unwrap()); + } + + #[test] + fn a_matching_branch_push_compiles_the_workflow() { + let workflows = [raw( + "test.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: [main]\n", + )]; + let compiled = compile(&workflows, &push("refs/heads/main"), &ChangedFiles::none()); + assert_eq!( + compiled + .workflows + .iter() + .map(|workflow| workflow.name.as_str()) + .collect::>(), + vec!["test.yml"] + ); + assert!( + compiled.diagnostics.is_empty(), + "{:?}", + compiled.diagnostics + ); + } + + #[test] + fn a_paths_constraint_matches_changed_files_and_never_their_parent_directories() { + let changed = changed(&["src/deep/main.rs"]); + let cases: &[(&str, usize)] = &[ + ("", 1), + ("\n paths: ['src/**']", 1), + ("\n paths: ['**/main.rs']", 1), + ("\n paths: ['docs/**']", 0), + ("\n paths: ['src']", 0), + ("\n paths: ['*']", 0), + ("\n paths: ['docs/**', 'src/**']", 1), + ]; + cases.iter().for_each(|(constraint, count)| { + let yaml = format!( + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']{constraint}\n" + ); + let compiled = compile(&[raw("ci.yml", &yaml)], &push("refs/heads/main"), &changed); + assert_eq!(compiled.workflows.len(), *count, "{yaml}"); + }); + + let unmatched_branch = compile( + &[raw( + "ci.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: [release]\n paths: ['[']\n", + )], + &push("refs/heads/main"), + &changed, + ); + assert!( + !unmatched_branch.diagnostics.errors.is_empty(), + "compile reports a malformed paths glob even when the branch already decided the match: {:?}", + unmatched_branch.diagnostics + ); + } + + #[test] + fn a_truncated_listing_assumes_a_paths_match() { + let yaml = "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n paths: ['src/**']\n"; + assert_eq!( + compile( + &[raw("ci.yml", yaml)], + &push("refs/heads/main"), + &changed(&["docs/only.md"]) + ) + .workflows + .len(), + 0, + "a complete listing that misses the globs skips the workflow" + ); + + let assumed = compile( + &[raw("ci.yml", yaml)], + &push("refs/heads/main"), + &ChangedFiles::unknown(), + ); + assert_eq!( + assumed.workflows.len(), + 1, + "a listing the record couldn't hold rules no glob out" + ); + } + + #[test] + fn compile_cases() { + let tag = "engine: nixery.dev/x\nwhen:\n - event: push\n tag: ['v*']\n"; + let single_star = + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['feature/*']\n"; + let cases: &[(&str, &str, usize, Option<&str>)] = &[ + ( + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: [dev]\n", + "refs/heads/main", + 0, + Some("workflow skipped"), + ), + ("engine: nixery.dev/x\n", "refs/heads/anything", 1, None), + ( + "when:\n - event: push\n branch: ['*']\n", + "refs/heads/main", + 0, + Some("missing engine"), + ), + ( + "engine: nixery.dev/x\nclone:\n skip: true\n submodules: true\n", + "refs/heads/main", + 1, + Some("`clone.submodules` has no effect with `clone.skip`"), + ), + ( + "engine: nixery.dev/x\nclone:\n skip: true\n submodules: false\n", + "refs/heads/main", + 1, + Some("`clone.submodules` has no effect with `clone.skip`"), + ), + ( + "engine: nixery.dev/x\nclone:\n skip: true\n tags: true\n", + "refs/heads/main", + 1, + Some("`clone.tags` has no effect with `clone.skip`"), + ), + ( + "engine: nixery.dev/x\nclone:\n skip: true\n", + "refs/heads/main", + 1, + None, + ), + ( + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: '**'\n", + "refs/heads/feature/x", + 1, + None, + ), + (tag, "refs/tags/v1.0", 1, None), + (tag, "refs/heads/v1.0", 0, None), + (single_star, "refs/heads/feature/x", 1, None), + (single_star, "refs/heads/feature/x/y", 0, None), + ]; + cases.iter().for_each(|(yaml, reference, count, diag)| { + let compiled = compile( + &[raw("ci.yml", yaml)], + &push(reference), + &ChangedFiles::none(), + ); + assert_eq!(compiled.workflows.len(), *count, "{yaml}"); + let messages: Vec<&String> = compiled + .diagnostics + .errors + .iter() + .chain(compiled.diagnostics.warnings.iter()) + .collect(); + match diag { + Some(sub) => assert!( + messages.iter().any(|message| message.contains(sub)), + "{sub}: {messages:?}" + ), + None => assert!( + compiled.diagnostics.errors.is_empty() + && !messages + .iter() + .any(|message| message.contains("invalid configuration")), + "{yaml}: {messages:?}" + ), + } + }); + } +} diff --git a/knot2/crates/knot-xrpc/Cargo.toml b/knot2/crates/knot-xrpc/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..506446a0b0d05679ef6cc5872946e82480faf109 --- /dev/null +++ b/knot2/crates/knot-xrpc/Cargo.toml @@ -0,0 +1,67 @@ +[package] +name = "knot-xrpc" +version = "1.20.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-capability = { workspace = true } +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-lfs = { workspace = true } +knot-cache = { workspace = true } +knot-pack = { workspace = true } +knot-resource = { workspace = true } +knot-index = { workspace = true } +knot-acl = { workspace = true } +knot-atproto = { workspace = true } +jacquard-common = { workspace = true } +jacquard-repo = { workspace = true } +cid = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +ipld-core = { workspace = true } +serde_bytes = { workspace = true } +knot-record = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-secrets = { workspace = true } +knot-runtime = { workspace = true } +knot-events = { workspace = true } +knot-langs = { workspace = true } +knot-messages = { workspace = true } +knot-receive = { workspace = true } +knot-maintenance = { workspace = true } +lexicons = { workspace = true } +jacquard-axum = { workspace = true } +tracing = { workspace = true } +axum = { workspace = true, features = ["ws"] } +tower = { workspace = true } +tower-http = { workspace = true, features = ["fs"] } +http-body = { workspace = true } +tokio-util = { workspace = true, features = ["io-util"] } +tokio = { workspace = true } +futures = { workspace = true } +tokio-stream = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +httpdate = { workspace = true } +bytes = { workspace = true } +thiserror = { workspace = true } +http = { workspace = true } +base64 = { workspace = true } +chrono = { workspace = true } +tempfile = { workspace = true } +sha2 = { workspace = true } +subtle = { workspace = true } +zeroize = { workspace = true } +url = { workspace = true } +[dev-dependencies] +knot-fixtures = { workspace = true } +knot-config = { workspace = true } +k256 = { workspace = true } +sha2 = { workspace = true } +tokio-tungstenite = "0.29" +tokio = { workspace = true, features = ["process", "test-util"] } +parking_lot = { workspace = true } +proptest = { workspace = true } diff --git a/knot2/crates/knot-xrpc/src/atproto.rs b/knot2/crates/knot-xrpc/src/atproto.rs new file mode 100644 index 0000000000000000000000000000000000000000..e9a17bae3a7e05f862c4ce09614c82c1c57d35cd --- /dev/null +++ b/knot2/crates/knot-xrpc/src/atproto.rs @@ -0,0 +1,884 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::str::FromStr; +use std::sync::Arc; + +use axum::body::Body; +use axum::extract::State; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::Response; +use bytes::Bytes; +use cid::Cid; +use futures::{StreamExt, TryStreamExt}; +use ipld_core::ipld::Ipld; + +use crate::query::ValidatedQuery; +use jacquard_repo::RepoError; +use jacquard_repo::car::write_car_bytes; +use jacquard_repo::storage::BlockStore; +use knot_git::Repo; +use knot_index::Resolved; +use knot_record::blocks::GitBlocks; +use knot_record::chain::{self, ChainTip}; +use knot_runtime::Signer as _; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{Handle, KnotHostname, RecordCid, RecordCollection, RepoDid, UriRkey}; +use serde::Deserialize; +use serde_json::Value; + +use crate::ResponseLimit; +use crate::XrpcState; +use crate::error::XrpcError; +use crate::query::{Limit, Offset}; +use crate::run_blocking; + +const CAR_CONTENT_TYPE: &str = "application/vnd.ipld.car"; +const LIST_RECORDS_DEFAULT: usize = 50; +const LIST_RECORDS_MAX: usize = 100; +const LIST_REPOS_DEFAULT: usize = 500; +const LIST_REPOS_MAX: usize = 1000; +const GET_BLOCKS_CIDS_MAX: usize = 1000; + +fn repo_not_found(did: &str) -> XrpcError { + XrpcError::named( + StatusCode::BAD_REQUEST, + XrpcError::REPO_NOT_FOUND, + format!("couldn't find repo: {did}"), + ) +} + +fn record_value(bytes: &[u8]) -> Result { + let decoded: Ipld = serde_ipld_dagcbor::from_slice(bytes) + .map_err(|error| XrpcError::internal(error.to_string()))?; + json_of(&decoded).map_err(XrpcError::internal) +} + +fn json_of(ipld: &Ipld) -> Result { + match ipld { + Ipld::Null => Ok(Value::Null), + Ipld::Bool(boolean) => Ok(Value::Bool(*boolean)), + Ipld::Integer(integer) => serde_json::Number::from_i128(*integer) + .map(Value::Number) + .ok_or_else(|| "an integer in the record is out of range for JSON".to_string()), + Ipld::Float(float) => serde_json::Number::from_f64(*float) + .map(Value::Number) + .ok_or_else(|| "a non-finite float in the record doesn't fit JSON".to_string()), + Ipld::String(string) => Ok(Value::String(string.clone())), + Ipld::Link(cid) => Ok(serde_json::json!({ "$link": cid.to_string() })), + Ipld::Bytes(bytes) => { + use base64::Engine as _; + Ok(serde_json::json!({ + "$bytes": base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes) + })) + } + Ipld::List(items) => items + .iter() + .map(json_of) + .collect::, _>>() + .map(Value::Array), + Ipld::Map(fields) => fields + .iter() + .map(|(key, value)| json_of(value).map(|value| (key.clone(), value))) + .collect::, _>>() + .map(Value::Object), + } +} + +fn record_not_found(uri: &str) -> XrpcError { + XrpcError::named( + StatusCode::BAD_REQUEST, + "RecordNotFound", + format!("couldn't locate record: {uri}"), + ) +} + +fn handle_not_found(handle: &str) -> XrpcError { + XrpcError::named( + StatusCode::BAD_REQUEST, + "HandleNotFound", + format!("couldn't resolve handle to a repo here: {handle}"), + ) +} + +fn storage(error: RepoError) -> XrpcError { + XrpcError::internal(error.to_string()) +} + +/// One atproto repository this knot can serve: +/// the knot's own meta-repo under its `did:web`, +/// or a hosted repository under its repo DID. +pub(crate) struct AtprotoRepo { + pub(crate) did: RepoDid, + pub(crate) repo: Repo, +} + +impl AtprotoRepo { + fn blocks(&self, tip: &ChainTip) -> GitBlocks { + GitBlocks::at(&self.repo, tip.blocks) + } +} + +#[derive(Debug, Clone)] +pub(crate) enum RepoSubject { + Did(RepoDid), + Host(KnotHostname), +} + +impl RepoSubject { + pub(crate) fn parse(raw: impl Into) -> Result { + let raw = raw.into(); + if raw.starts_with("did:") { + RepoDid::new(raw).map(Self::Did) + } else { + KnotHostname::new(raw).map(Self::Host) + } + } + + pub(crate) fn as_str(&self) -> &str { + match self { + Self::Did(did) => did.as_str(), + Self::Host(host) => host.as_str(), + } + } +} + +impl serde::Serialize for RepoSubject { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for RepoSubject { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + Self::parse(raw).map_err(serde::de::Error::custom) + } +} + +pub(crate) fn resolve( + state: &XrpcState, + subject: &RepoSubject, +) -> Result { + let self_hosted = match subject { + RepoSubject::Did(did) => *did == state.knot_repo, + RepoSubject::Host(host) => *host == state.knot_hostname, + }; + if self_hosted { + return Ok(AtprotoRepo { + did: state.knot_repo.clone(), + repo: Repo::open(&state.meta_path)?, + }); + } + let RepoSubject::Did(did) = subject else { + return Err(repo_not_found(subject.as_str())); + }; + match state.index.ownership_of(did) { + Resolved::Ready(Some(_)) => Ok(AtprotoRepo { + did: did.clone(), + repo: state.layout.open(did)?, + }), + Resolved::Ready(None) => Err(repo_not_found(subject.as_str())), + Resolved::Warming => Err(crate::reads::warming()), + } +} + +fn tip_of(repo: &Repo, did: &RepoDid) -> Result { + chain::tip(repo)?.ok_or_else(|| repo_not_found(did.as_str())) +} + +pub(crate) fn records_in( + found: &AtprotoRepo, + collection: &str, +) -> Result, XrpcError> { + let tip = tip_of(&found.repo, &found.did)?; + let tree = chain::load_tree(&found.repo, &tip); + let store = found.blocks(&tip); + let prefix = format!("{collection}/"); + futures::executor::block_on(async { + let mut records = Vec::new(); + for (key, cid) in tree.leaves().await.map_err(storage)? { + let Some(rkey) = key + .as_str() + .strip_prefix(&prefix) + .and_then(|rkey| UriRkey::new(rkey.to_owned()).ok()) + else { + continue; + }; + let Some(bytes) = store.get(&cid).await.map_err(storage)? else { + continue; + }; + records.push((rkey, record_value(&bytes)?)); + } + Ok(records) + }) +} + +fn car(response_limit: ResponseLimit, bytes: Vec) -> Result { + if bytes.len() > response_limit.get() { + return Err(XrpcError::request_too_large(format!( + "the repository export is {} bytes over the {} byte response limit", + bytes.len(), + response_limit.get() + ))); + } + let mut response = Response::new(Body::from(bytes)); + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static(CAR_CONTENT_TYPE), + ); + Ok(response) +} + +async fn serve_car( + found: AtprotoRepo, + response_limit: ResponseLimit, + collect: R, +) -> Result +where + R: FnOnce(&AtprotoRepo, &ChainTip) -> Result, XrpcError> + Send + 'static, +{ + run_blocking(move || { + let tip = tip_of(&found.repo, &found.did)?; + let commit_bytes = chain::commit_bytes(&found.repo, &tip)?; + let commit_cid = chain::commit_cid(&commit_bytes)?; + let mut blocks = collect(&found, &tip)?; + blocks.insert(commit_cid, Bytes::from(commit_bytes)); + let bytes = futures::executor::block_on(write_car_bytes(commit_cid, blocks)) + .map_err(|error| XrpcError::internal(error.to_string()))?; + car(response_limit, bytes) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct RepoExportParams { + pub(crate) did: RepoSubject, + #[serde(default)] + pub(crate) since: Option, +} + +const EXPORT_FRAME_WINDOW: usize = 5_000; + +pub(crate) async fn sync_get_repo( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let found = resolve(&state, ¶ms.did)?; + let response_limit = state.byte_limits.response; + serve_car(found, response_limit, move |found, tip| { + match ¶ms.since { + Some(since) => { + frames_since_blocks(found, tip, since.tid().clone(), response_limit.get()) + } + None => whole_tree_blocks(found, tip), + } + }) + .await +} + +fn whole_tree_blocks( + found: &AtprotoRepo, + tip: &ChainTip, +) -> Result, XrpcError> { + let tree = chain::load_tree(&found.repo, tip); + let store = found.blocks(tip); + let root = tip.data; + futures::executor::block_on(async move { + let mut nodes = tree.collect_node_cids().await?; + nodes.push(root); + let leaves: Vec = tree + .leaves() + .await? + .into_iter() + .map(|(_, cid)| cid) + .collect(); + futures::stream::iter(nodes) + .map(Ok) + .and_then(|cid| block_or_corrupt(&store, cid)) + .chain( + futures::stream::iter(leaves) + .map(Ok) + .try_filter_map(|cid| block_if_present(&store, cid)), + ) + .try_collect::>() + .await + }) + .map_err(|error: RepoError| XrpcError::internal(error.to_string())) +} + +fn frames_since_blocks( + found: &AtprotoRepo, + tip: &ChainTip, + since: jacquard_common::types::tid::Tid, + limit: usize, +) -> Result, XrpcError> { + let traversed = chain::frames_after( + &found.repo, + tip.git, + knot_types::UnixMicros::new(since.timestamp()), + EXPORT_FRAME_WINDOW, + ); + match traversed.end { + chain::TraversalEnd::Limited => { + return Err(XrpcError::invalid_request( + "`since` is older than this knot's export window. Please fetch the whole repo \ + without `since` instead, thanks.", + )); + } + chain::TraversalEnd::Broken => { + return Err(XrpcError::internal( + "The chain wouldn't traverse back to `since`, so the export diff is \ + unavailable. Please fetch the whole repo instead.", + )); + } + chain::TraversalEnd::Exhausted => (), + } + let mut blocks = BTreeMap::new(); + traversed.found.iter().try_for_each(|(rev, frame)| { + let (_header, message) = crate::firehose::commit_message(frame).ok_or_else(|| { + XrpcError::internal(format!( + "A stored frame at {rev} wouldn't parse as a commit." + )) + })?; + let car = knot_record::car_blocks(&message.blocks).map_err(|error| { + XrpcError::internal(format!( + "A stored frame wouldn't read back as a CAR: {error}" + )) + })?; + blocks.extend(car.blocks); + if knot_record::chain::blocks_bytes(&blocks) > limit { + return Err(XrpcError::request_too_large( + "This response exceeds the configured maximum size. Please narrow the request, \ + thanks.", + )); + } + Ok::<_, XrpcError>(()) + })?; + Ok(blocks) +} + +#[derive(Deserialize)] +pub(crate) struct DidParams { + pub(crate) did: RepoSubject, +} + +pub(crate) async fn sync_get_latest_commit( + State(state): State>>, + axum::extract::Query(params): axum::extract::Query, +) -> Result { + let found = resolve(&state, ¶ms.did)?; + let response_limit = state.byte_limits.response; + run_blocking(move || { + let tip = tip_of(&found.repo, &found.did)?; + let cid = chain::commit_cid(&chain::commit_bytes(&found.repo, &tip)?)?; + crate::reads::json( + serde_json::json!({ "cid": cid.to_string(), "rev": tip.rev.to_string() }), + response_limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct SyncRecordParams { + pub(crate) did: RepoSubject, + pub(crate) collection: RecordCollection, + pub(crate) rkey: UriRkey, +} + +pub(crate) async fn sync_get_record( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let found = resolve(&state, ¶ms.did)?; + let key = format!("{}/{}", params.collection, params.rkey); + serve_car(found, state.byte_limits.response, |found, tip| { + let tree = chain::load_tree(&found.repo, tip); + let store = found.blocks(tip); + futures::executor::block_on(async move { + let leaves = tree.leaves().await?; + let mut keys: Vec<&str> = leaves.iter().map(|(key, _)| key.as_str()).collect(); + keys.sort_unstable(); + let mut wanted = neighbors_of(&keys, &key); + wanted.push(key.clone()); + let mut blocks = BTreeMap::new(); + for probe in wanted { + tree.blocks_for_path(&probe, &mut blocks).await?; + } + if let Some(cid) = tree.get(&key).await? { + blocks.extend([block_or_corrupt(&store, cid).await?]); + } + Ok::, RepoError>(blocks) + }) + .map_err(|error| XrpcError::internal(error.to_string())) + }) + .await +} + +fn neighbors_of(sorted: &[&str], key: &str) -> Vec { + let position = sorted.partition_point(|existing| *existing < key); + [ + position + .checked_sub(1) + .and_then(|before| sorted.get(before)), + sorted.get(position), + ] + .into_iter() + .flatten() + .map(|neighbor| (*neighbor).to_owned()) + .collect() +} + +async fn block_or_corrupt( + store: &GitBlocks, + cid: Cid, +) -> jacquard_repo::error::Result<(Cid, Bytes)> { + let bytes = store.get(&cid).await?.ok_or_else(|| { + RepoError::storage(std::io::Error::other(format!( + "the chain names block {cid} and the block store doesn't hold it" + ))) + })?; + Ok((cid, bytes)) +} + +async fn block_if_present( + store: &GitBlocks, + cid: Cid, +) -> jacquard_repo::error::Result> { + let block = store.get(&cid).await?.map(|bytes| (cid, bytes)); + if block.is_none() { + tracing::error!( + %cid, + "the chain has a leaf whose block is missing from every store, so the car goes \ + out without it; the next reconcile takes its leaf out" + ); + } + Ok(block) +} + +pub(crate) struct BlocksQuery { + pub(crate) did: RepoSubject, + pub(crate) cids: Vec, +} + +impl axum::extract::FromRequestParts for BlocksQuery { + type Rejection = XrpcError; + + async fn from_request_parts( + parts: &mut http::request::Parts, + _state: &S, + ) -> Result { + let pairs = url::form_urlencoded::parse(parts.uri.query().unwrap_or_default().as_bytes()); + let did = pairs + .clone() + .find(|(key, _)| key == "did") + .and_then(|(_, value)| RepoSubject::parse(value.to_string()).ok()) + .ok_or_else(|| { + XrpcError::invalid_request("did must be a repository DID or a hostname") + })?; + let cids: Vec = pairs + .filter(|(key, _)| key == "cids") + .map(|(_, value)| Cid::from_str(&value)) + .collect::>() + .map_err(|_| XrpcError::invalid_request("a requested cid doesn't parse"))?; + if cids.len() > GET_BLOCKS_CIDS_MAX { + return Err(XrpcError::invalid_request(format!( + "at most {GET_BLOCKS_CIDS_MAX} cids may be requested per call" + ))); + } + Ok(Self { did, cids }) + } +} + +pub(crate) async fn sync_get_blocks( + State(state): State>>, + params: BlocksQuery, +) -> Result { + let found = resolve(&state, ¶ms.did)?; + let wanted = params.cids; + serve_car(found, state.byte_limits.response, |found, tip| { + let store = found.blocks(tip); + futures::executor::block_on(async move { + let mut blocks = BTreeMap::new(); + for cid in &wanted { + if let Some(bytes) = store.get(cid).await? { + blocks.insert(*cid, bytes); + } + } + Ok::, RepoError>(blocks) + }) + .map_err(|error| XrpcError::internal(error.to_string())) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct ListReposParams { + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: Offset, +} + +pub(crate) async fn sync_list_repos( + State(state): State>>, + axum::extract::Query(params): axum::extract::Query, +) -> Result { + if matches!( + state.index.coverage().registry, + knot_index::Coverage::Warming + ) { + return Err(crate::reads::warming()); + } + let dids: Vec = std::iter::once(RepoSubject::Did(state.knot_repo.clone())) + .chain(state.index.hosted_repos().into_iter().map(RepoSubject::Did)) + .collect(); + let cursor = crate::query::next_cursor( + params.cursor, + params.limit, + crate::query::Total::new(dids.len()), + ); + let page: Vec = dids + .into_iter() + .skip(params.cursor.get()) + .take(params.limit.get()) + .collect(); + let response_limit = state.byte_limits.response; + run_blocking(move || { + let repos: Vec = page + .iter() + .filter_map(|did| { + resolve(&state, did) + .and_then(|found| { + tip_of(&found.repo, &found.did).and_then(|tip| { + let head = chain::commit_cid(&chain::commit_bytes(&found.repo, &tip)?)?; + Ok(serde_json::json!({ + "did": did, + "head": head.to_string(), + "rev": tip.rev.to_string(), + "active": true, + })) + }) + }) + .map_err(|error| { + tracing::warn!( + did = did.as_str(), + %error, + "listRepos left a repo out: it failed to open or read" + ); + error + }) + .ok() + }) + .collect(); + crate::reads::json(ListReposOut { repos, cursor }, response_limit) + }) + .await +} + +#[derive(serde::Serialize)] +struct ListReposOut { + repos: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, +} + +#[derive(serde::Serialize)] +struct RepoStatusOut { + did: RepoDid, + active: bool, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + rev: Option, +} + +pub(crate) async fn sync_get_repo_status( + State(state): State>>, + axum::extract::Query(params): axum::extract::Query, +) -> Result { + let response_limit = state.byte_limits.response; + match resolve(&state, ¶ms.did) { + Ok(found) => { + run_blocking(move || { + let rev = chain::tip(&found.repo)?.map(|tip| tip.rev.to_string()); + crate::reads::json( + RepoStatusOut { + did: found.did, + active: true, + status: None, + rev, + }, + response_limit, + ) + }) + .await + } + Err(error) if error.is_repo_not_found() => { + let RepoSubject::Did(did) = ¶ms.did else { + return Err(error); + }; + if crate::plc::is_deleted(&state, did).await? { + crate::reads::json( + RepoStatusOut { + did: did.clone(), + active: false, + status: Some(crate::plc::STATUS_DELETED), + rev: None, + }, + response_limit, + ) + } else { + Err(error) + } + } + Err(error) => Err(error), + } +} + +#[derive(Deserialize)] +pub(crate) struct RepoRecordParams { + pub(crate) repo: RepoSubject, + pub(crate) collection: RecordCollection, + pub(crate) rkey: UriRkey, + #[serde(default)] + pub(crate) cid: Option, +} + +pub(crate) async fn repo_get_record( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let found = resolve(&state, ¶ms.repo)?; + let key = format!("{}/{}", params.collection, params.rkey); + let uri = format!("at://{}/{key}", found.did); + let wanted = params.cid; + let response_limit = state.byte_limits.response; + run_blocking(move || { + let tip = tip_of(&found.repo, &found.did)?; + let tree = chain::load_tree(&found.repo, &tip); + let store = found.blocks(&tip); + let (cid, value) = futures::executor::block_on(async { + let cid = tree + .get(&key) + .await + .map_err(storage)? + .ok_or_else(|| record_not_found(&uri))?; + if wanted + .and_then(crate::out::CidString::record) + .is_some_and(|wanted| RecordCid::new(cid) != Some(wanted)) + { + return Err(record_not_found(&uri)); + } + let bytes = store.get(&cid).await.map_err(storage)?.ok_or_else(|| { + tracing::error!( + %uri, + %cid, + "the chain has a record whose block is missing from every store, so the \ + read answers as it does for a record that left; the next reconcile \ + takes its leaf out" + ); + record_not_found(&uri) + })?; + let value = record_value(&bytes)?; + Result::<(Cid, Value), XrpcError>::Ok((cid, value)) + })?; + crate::reads::json( + serde_json::json!({ + "uri": uri, + "cid": cid.to_string(), + "value": value, + }), + response_limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct ListRecordsParams { + repo: RepoSubject, + collection: RecordCollection, + #[serde(default)] + limit: Limit, + cursor: Option, + #[serde(default)] + reverse: bool, +} + +pub(crate) async fn repo_list_records( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let found = resolve(&state, ¶ms.repo)?; + let prefix = format!("{}/", params.collection); + let limit = params.limit; + let reverse = params.reverse; + let after = params.cursor; + let response_limit = state.byte_limits.response; + run_blocking(move || { + let tip = tip_of(&found.repo, &found.did)?; + let tree = chain::load_tree(&found.repo, &tip); + let store = found.blocks(&tip); + let (records, more) = futures::executor::block_on(async { + let mut matching: Vec<(UriRkey, Cid)> = tree + .leaves() + .await + .map_err(storage)? + .into_iter() + .filter_map(|(key, cid)| { + let rkey = UriRkey::new(key.as_str().strip_prefix(&prefix)?.to_owned()).ok()?; + Some((rkey, cid)) + }) + .collect(); + matching.sort_by(|a, b| a.0.cmp(&b.0)); + if reverse { + matching.reverse(); + } + let start = after.as_ref().map_or(0, |after| { + matching.partition_point(|(rkey, _)| { + if reverse { + rkey.as_str() >= after.as_str() + } else { + rkey.as_str() <= after.as_str() + } + }) + }); + let candidates: Vec<(UriRkey, Cid)> = matching + .into_iter() + .skip(start) + .take(limit.get() + 1) + .collect(); + let more = candidates.len() > limit.get(); + let page: Vec<(UriRkey, Cid, Value)> = futures::stream::iter(candidates) + .map(Ok) + .try_filter_map(|(rkey, cid)| { + let store = &store; + let prefix = &prefix; + async move { + let Some(bytes) = store.get(&cid).await.map_err(storage)? else { + tracing::error!( + key = %format!("{prefix}{rkey}"), + %cid, + "the chain has a record whose block is missing from every \ + store, so the page leaves it out; the next reconcile \ + takes its leaf out" + ); + return Ok(None); + }; + let value = record_value(&bytes)?; + Ok::<_, XrpcError>(Some((rkey, cid, value))) + } + }) + .try_collect() + .await?; + Result::<(Vec<(UriRkey, Cid, Value)>, bool), XrpcError>::Ok((page, more)) + })?; + let next = records + .iter() + .take(limit.get()) + .next_back() + .filter(|_| more) + .map(|(rkey, _, _)| rkey.clone()); + let items: Vec = records + .into_iter() + .take(limit.get()) + .map(|(rkey, cid, value)| { + serde_json::json!({ + "uri": format!("at://{}/{prefix}{rkey}", found.did), + "cid": cid.to_string(), + "value": value, + }) + }) + .collect(); + crate::reads::json( + serde_json::json!({ "records": items, "cursor": next }), + response_limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct RepoQuery { + repo: RepoSubject, +} + +pub(crate) async fn repo_describe_repo( + State(state): State>>, + axum::extract::Query(params): axum::extract::Query, +) -> Result { + let found = resolve(&state, ¶ms.repo)?; + let knot = found.did == state.knot_repo; + let handle = if knot { + state.knot_hostname.as_str().to_owned() + } else { + "handle.invalid".to_owned() + }; + let did_doc = if knot { + let key = state.secrets.signer(&state.knot_did)?.public_key(); + knot_atproto::knot_did_document(&state.knot_did, &key, &state.knot_service_url) + } else { + // todo: decide if this should be filled in, the lexicon requires the field so it is {} for now + serde_json::json!({}) + }; + let response_limit = state.byte_limits.response; + run_blocking(move || { + let collections = chain::tip(&found.repo)? + .map(|tip| chain::load_tree(&found.repo, &tip)) + .map(|tree| { + futures::executor::block_on(async { + let names: BTreeSet = tree + .leaves() + .await? + .iter() + .filter_map(|(key, _)| key.as_str().split('/').next().map(String::from)) + .collect(); + Ok::, RepoError>(names.into_iter().collect()) + }) + }) + .transpose() + .map_err(storage)? + .unwrap_or_default(); + let body = serde_json::json!({ + "did": found.did, + "handle": handle, + "handleIsCorrect": false, + "collections": collections, + "didDoc": did_doc, + }); + crate::reads::json(body, response_limit) + }) + .await +} + +pub(crate) async fn describe_server( + State(state): State>>, +) -> Result { + crate::reads::json( + serde_json::json!({ + "did": state.knot_did.as_str(), + "availableUserDomains": [], + }), + state.byte_limits.response, + ) +} + +#[derive(Deserialize)] +pub(crate) struct HandleParams { + handle: String, +} + +pub(crate) async fn resolve_handle( + State(state): State>>, + axum::extract::Query(params): axum::extract::Query, +) -> Result { + let text = params.handle; + let handle = Handle::new(text.clone()).map_err(|_| handle_not_found(&text))?; + if handle.as_str() == KnotHostname::as_str(&state.knot_hostname) { + return crate::reads::json( + serde_json::json!({ "did": state.knot_did.as_str() }), + state.byte_limits.response, + ); + } + Err(handle_not_found(handle.as_str())) +} diff --git a/knot2/crates/knot-xrpc/src/blobs.rs b/knot2/crates/knot-xrpc/src/blobs.rs new file mode 100644 index 0000000000000000000000000000000000000000..328bf65fafb44af064e36d47e1c428ebe8abc72e --- /dev/null +++ b/knot2/crates/knot-xrpc/src/blobs.rs @@ -0,0 +1,707 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::{Arc, Mutex}; + +use axum::body::Bytes; +use axum::extract::State; +use axum::http::{HeaderName, HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use http::HeaderMap; +use ipld_core::ipld::Ipld; +use jacquard_common::types::tid::Tid; +use knot_git::{Layout, Repo}; +use knot_lfs::{ClaimedSize, LfsError, LfsOid, LfsStore, Pool, RootsDenied, UploadAdmission}; +use knot_record::attachment::{ + Attachment, AttachmentMime, AttachmentSize, AttachmentTooBig, OCTET_STREAM, SNIFF_BYTES, +}; +use knot_record::chain; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{BlobCid, Decision, RecordAddress, RepoDid, Sha256Digest}; +use serde::Deserialize; +use serde_json::json; + +use crate::XrpcState; +use crate::error::XrpcError; +use crate::query::{Limit, Offset, Since, ValidatedQuery}; +use crate::run_blocking; + +pub(crate) const UPLOAD_BLOB_ROUTE: &str = "/xrpc/com.atproto.repo.uploadBlob"; +pub(crate) const GET_BLOB_ROUTE: &str = "/xrpc/com.atproto.sync.getBlob"; +pub(crate) const LIST_BLOBS_ROUTE: &str = "/xrpc/com.atproto.sync.listBlobs"; + +const BLOB_CSP: &str = "default-src 'none'; sandbox"; +const INDEX_FRAME_WINDOW: usize = 50_000; +const LIST_BLOBS_DEFAULT: usize = 500; +const LIST_BLOBS_MAX: usize = 1000; + +fn blob_not_found() -> XrpcError { + XrpcError::named( + StatusCode::NOT_FOUND, + "BlobNotFound", + "No such blob for this repo.", + ) +} + +struct UploadedAttachment { + digest: Sha256Digest, + mime: AttachmentMime, + size: AttachmentSize, +} + +impl UploadedAttachment { + fn parse(body: &Bytes, hint: &str) -> Result { + if body.is_empty() { + return Err(XrpcError::invalid_request( + "This upload's body is empty; please send the attachment bytes as the body.", + )); + } + let size = AttachmentSize::new(body.len() as u64).map_err( + |AttachmentTooBig { sent, max_bytes }| { + XrpcError::request_too_large(format!( + "An attachment is at most {max_bytes} bytes, and this upload sends {sent}." + )) + }, + )?; + Ok(Self { + digest: Sha256Digest::hash(body), + mime: AttachmentMime::detected(body, hint), + size, + }) + } + + fn attachment(&self) -> Attachment { + Attachment::new( + BlobCid::from_digest(self.digest), + self.mime.clone(), + self.size, + ) + } + + fn oid(&self) -> LfsOid { + LfsOid::from_digest(self.digest) + } + + fn claimed(&self) -> ClaimedSize { + ClaimedSize::new(self.size.get()) + } +} + +fn sniffed_kind(store: &knot_lfs::DiskStore, repo: &RepoDid, oid: &LfsOid) -> Option<&'static str> { + let mut body = store.read(Pool::Attachments, repo, oid).ok()?; + let mut prefix = Vec::new(); + let mut limited = std::io::Read::take(&mut body, SNIFF_BYTES as u64); + std::io::Read::read_to_end(&mut limited, &mut prefix).ok()?; + knot_record::attachment::sniffed(&prefix) +} + +pub(crate) async fn upload_blob( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let repo = state.audience_repo(&headers).await?; + let session = crate::social::session(&state, &headers, &method, repo).await?; + let contributor = session.require_contribution("uploads")?; + store_upload(&state, contributor.session.repo.clone(), &headers, body).await +} + +pub(crate) async fn store_upload( + state: &XrpcState, + upload_repo: RepoDid, + headers: &HeaderMap, + body: Bytes, +) -> Result { + let lfs = &state.lfs; + let hint = headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or(""); + let uploaded = UploadedAttachment::parse(&body, hint)?; + let claimed = uploaded.claimed(); + let permit = lfs + .handle + .admission + .admit(claimed) + .map_err(|error| match error { + LfsError::SizeLimitExceeded { declared, limit } => { + XrpcError::request_too_large(format!( + "This knot's LFS object limit is {limit} bytes, and this attachment is \ + {declared}. Please try a smaller upload, thanks." + )) + } + LfsError::FreeSpaceDenied { free, floor } => XrpcError::named( + StatusCode::SERVICE_UNAVAILABLE, + "BlobStoreFull", + format!( + "The blob store's disk is down to {free} bytes, under the {floor}-byte \ + floor. Please try again once there's room, thanks." + ), + ), + other => XrpcError::internal(other.to_string()), + })?; + let oid = uploaded.oid(); + let stored = Arc::clone(&lfs.handle.store); + run_blocking(move || { + let outcome = stored.put( + Pool::Attachments, + &upload_repo, + &oid, + claimed, + &mut &body[..], + ); + drop(permit); + outcome.map_err(|error| XrpcError::internal(error.to_string())) + }) + .await?; + let response = json!({ + "blob": serde_json::to_value(uploaded.attachment()).expect("an attachment serializes") + }); + crate::reads::json(response, state.byte_limits.response) +} + +#[derive(Deserialize)] +pub(crate) struct GetBlobParams { + did: crate::atproto::RepoSubject, + cid: BlobCid, +} + +pub(crate) async fn get_blob( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let found = crate::atproto::resolve(&state, ¶ms.did)?; + let lfs = &state.lfs; + let oid = LfsOid::from_digest(params.cid.digest()); + let stored = Arc::clone(&lfs.handle.store); + let repo = found.did.clone(); + let bytes = run_blocking(move || { + let read = stored + .read(Pool::Attachments, &repo, &oid) + .map_err(|error| match error { + LfsError::NotFound { .. } => blob_not_found(), + other => XrpcError::internal(other.to_string()), + })?; + let mut body = read; + let mut bytes = Vec::new(); + std::io::Read::read_to_end(&mut body, &mut bytes) + .map_err(|error| XrpcError::internal(error.to_string()))?; + Ok::<_, XrpcError>(bytes) + }) + .await?; + let mime = knot_record::attachment::sniffed(&bytes).unwrap_or(OCTET_STREAM); + Ok(( + StatusCode::OK, + [ + ( + header::CONTENT_TYPE, + HeaderValue::from_str(mime).expect("a sniffed mime is a header value"), + ), + (header::CONTENT_LENGTH, HeaderValue::from(bytes.len())), + ( + HeaderName::from_static("x-content-type-options"), + HeaderValue::from_static("nosniff"), + ), + ( + HeaderName::from_static("content-security-policy"), + HeaderValue::from_static(BLOB_CSP), + ), + ], + bytes, + ) + .into_response()) +} + +#[derive(Deserialize)] +pub(crate) struct ListBlobsParams { + did: crate::atproto::RepoSubject, + #[serde(default)] + since: Option, + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: Offset, +} + +pub(crate) async fn list_blobs( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let found = crate::atproto::resolve(&state, ¶ms.did)?; + let index = index_of(&state, &found.did).await?; + match index.coverage { + Coverage::Whole => (), + Coverage::Short(shortness) => return Err(index_refused(shortness)), + } + let eligible: Vec<&(Tid, LfsOid)> = index + .listed + .iter() + .filter(|(rev, _)| { + params + .since + .as_ref() + .is_none_or(|since| rev.newer_than(since.tid())) + }) + .collect(); + let total = eligible.len(); + let taken = params.cursor.get().saturating_add(params.limit.get()); + let page: Vec = eligible + .into_iter() + .skip(params.cursor.get()) + .take(params.limit.get()) + .map(|(_, oid)| BlobCid::from_digest(oid.digest()).to_string()) + .collect(); + let response = json!({ + "cids": page, + "cursor": (taken < total).then(|| taken.to_string()), + }); + crate::reads::json(response, state.byte_limits.response) +} + +fn index_refused(shortness: Shortness) -> XrpcError { + let cause = match shortness { + Shortness::Windowed => format!( + "This repo's history is longer than the {INDEX_FRAME_WINDOW}-frame blob index \ + window" + ), + Shortness::Untraversable => { + "This repo's history stopped traversing or a stored frame wouldn't parse".to_owned() + } + }; + XrpcError::named( + StatusCode::SERVICE_UNAVAILABLE, + "BlobIndexUnavailable", + format!("{cause}, so `listBlobs` fails and the attachment sweep keeps every blob."), + ) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Coverage { + Whole, + Short(Shortness), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Shortness { + Windowed, + Untraversable, +} + +impl Coverage { + fn degrade(&mut self) { + if *self == Coverage::Whole { + *self = Coverage::Short(Shortness::Untraversable); + } + } +} + +type Live = BTreeMap)>; + +pub struct AttachmentIndex { + pub listed: Vec<(Tid, LfsOid)>, + pub referenced: HashSet, + pub coverage: Coverage, + live: Live, +} + +impl AttachmentIndex { + fn derived(live: Live, coverage: Coverage) -> Self { + let mut newest: BTreeMap = BTreeMap::new(); + let mut referenced = HashSet::new(); + live.values().for_each(|(rev, blobs)| { + blobs.iter().for_each(|blob| { + let oid = LfsOid::from_digest(blob.digest()); + referenced.insert(oid.clone()); + if newest.get(&oid).is_none_or(|seen| rev.newer_than(seen)) { + newest.insert(oid, rev.clone()); + } + }); + }); + let mut listed: Vec<(Tid, LfsOid)> = + newest.into_iter().map(|(oid, rev)| (rev, oid)).collect(); + listed.sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); + Self { + listed, + referenced, + coverage, + live, + } + } +} + +fn blob_ref(blob: &Ipld) -> Option { + match blob { + Ipld::Map(fields) => match fields.get("ref") { + Some(Ipld::Link(cid)) => BlobCid::new(*cid), + _ => None, + }, + _ => None, + } +} + +fn blob_cids_of(record: &Ipld) -> Vec { + let Ipld::Map(fields) = record else { + return Vec::new(); + }; + let markup_blobs = fields + .get("body") + .and_then(|body| match body { + Ipld::Map(inner) => inner.get("blobs"), + _ => None, + }) + .into_iter(); + let artifact = fields.get("artifact").and_then(blob_ref); + let listed = fields + .get("blobs") + .into_iter() + .chain(markup_blobs) + .flat_map(|listed| match listed { + Ipld::List(items) => items.iter(), + _ => [].iter(), + }) + .filter_map(|item| match item { + Ipld::Map(blob) => blob + .get("image") + .or_else(|| blob.get("video")) + .and_then(blob_ref), + _ => None, + }); + artifact.into_iter().chain(listed).collect() +} + +fn apply_frame( + rev: &Tid, + frame: &Bytes, + live: &mut Live, + coverage: &mut Coverage, +) -> Result<(), XrpcError> { + let Some((_header, message)) = crate::firehose::commit_message(frame) else { + tracing::warn!( + %rev, + "A stored frame wouldn't parse, so the blob index treats the history as untraversable." + ); + coverage.degrade(); + return Ok(()); + }; + let car = knot_record::car_blocks(&message.blocks).map_err(|error| { + XrpcError::internal(format!( + "A stored frame wouldn't read back as a CAR: {error}" + )) + })?; + message.ops.into_iter().for_each(|op| { + let address = match RecordAddress::parse_path(&op.path) { + Ok(address) => address, + Err(_) => { + tracing::warn!( + %rev, + path = %op.path, + "A stored frame's path doesn't parse as a record address, so the blob index \ + treats the history as untraversable." + ); + coverage.degrade(); + return; + } + }; + match op.action { + crate::firehose::RepoAction::Delete => { + live.remove(&address); + } + _ => { + let Some(cid) = op.cid else { + return; + }; + let blobs = car + .blocks + .get(&cid) + .and_then(|bytes| serde_ipld_dagcbor::from_slice::(bytes).ok()) + .map(|record| blob_cids_of(&record)) + .unwrap_or_else(|| { + coverage.degrade(); + Vec::new() + }); + live.insert(address, (rev.clone(), blobs)); + } + } + }); + Ok(()) +} + +fn fold_attachments(git: &Repo) -> Result { + let tip = chain::tip(git)? + .ok_or_else(|| XrpcError::internal("A repo serving blobs has a chain tip."))?; + let traversal = chain::framed_revs( + git, + tip.git, + knot_types::UnixMicros::new(0), + INDEX_FRAME_WINDOW, + ); + let mut coverage = match traversal.end { + chain::TraversalEnd::Exhausted => Coverage::Whole, + chain::TraversalEnd::Limited => Coverage::Short(Shortness::Windowed), + chain::TraversalEnd::Broken => Coverage::Short(Shortness::Untraversable), + }; + let mut live = Live::new(); + traversal + .found + .into_iter() + .rev() + .try_for_each(|(_, oid)| -> Result<(), XrpcError> { + match chain::frame_of(git, oid) { + Ok(Some((rev, frame))) => apply_frame(&rev, &frame, &mut live, &mut coverage), + Ok(None) => { + coverage.degrade(); + Ok(()) + } + Err(error) => { + tracing::warn!( + %error, + "A chain wouldn't yield a frame, so the blobs stay unindexed." + ); + coverage.degrade(); + Ok(()) + } + } + })?; + Ok(AttachmentIndex::derived(live, coverage)) +} + +fn advance_attachments( + git: &Repo, + base_tip: knot_types::Oid, + base: &AttachmentIndex, + tip: knot_types::Oid, +) -> Result, XrpcError> { + let traversal = chain::frames_to(git, tip, base_tip, INDEX_FRAME_WINDOW); + if !traversal.reached { + return Ok(None); + } + let mut live = base.live.clone(); + let mut coverage = base.coverage; + traversal + .found + .into_iter() + .rev() + .try_for_each(|(rev, frame)| apply_frame(&rev, &frame, &mut live, &mut coverage))?; + Ok(Some(AttachmentIndex::derived(live, coverage))) +} + +#[derive(Default)] +pub struct BlobViews { + cached: Mutex)>>, +} + +impl BlobViews { + pub fn get(&self, git: &Repo, repo: &RepoDid) -> Result, XrpcError> { + let tip = chain::tip(git)? + .ok_or_else(|| XrpcError::internal("A repo serving blobs has a chain tip."))? + .git; + let cached = { + let guard = self.cached.lock().expect("blob view cache lock poisoned"); + guard + .get(repo) + .map(|(seen, index)| (*seen, Arc::clone(index))) + }; + match cached { + Some((seen, index)) if seen == tip => Ok(index), + Some((seen, index)) => { + let advanced = advance_attachments(git, seen, &index, tip)?; + let index = match advanced { + Some(advanced) => Arc::new(advanced), + None => Arc::new(fold_attachments(git)?), + }; + let mut cached = self.cached.lock().expect("blob view cache lock poisoned"); + let stale = matches!(cached.get(repo), Some((current, _)) if *current != seen); + if !stale { + cached.insert(repo.clone(), (tip, Arc::clone(&index))); + } + Ok(index) + } + None => { + let index = Arc::new(fold_attachments(git)?); + self.cached + .lock() + .expect("blob view cache lock poisoned") + .insert(repo.clone(), (tip, Arc::clone(&index))); + Ok(index) + } + } + } + pub fn live_roots(&self, git: &Repo, repo: &RepoDid) -> Result, RootsDenied> { + let index = self + .get(git, repo) + .map_err(|_| RootsDenied::Untraversable)?; + match index.coverage { + Coverage::Whole => Ok(index.referenced.clone()), + Coverage::Short(Shortness::Windowed) => Err(RootsDenied::Windowed), + Coverage::Short(Shortness::Untraversable) => Err(RootsDenied::Untraversable), + } + } +} + +pub struct RepoAttachments { + layout: Layout, + views: Arc, +} + +impl RepoAttachments { + pub fn new(layout: Layout, views: Arc) -> Self { + Self { layout, views } + } +} + +impl knot_maintenance::AttachmentRoots for RepoAttachments { + fn referenced(&self, repo: &RepoDid) -> Result, RootsDenied> { + let git = self.layout.open(repo).map_err(RootsDenied::Unopened)?; + self.views.live_roots(&git, repo) + } +} + +async fn index_of( + state: &Arc>, + repo: &RepoDid, +) -> Result, XrpcError> { + let git = state.layout.open(repo)?; + let wanted = repo.clone(); + let state = Arc::clone(state); + run_blocking(move || state.blobs.get(&git, &wanted)).await +} + +async fn admit_attachments( + state: &Arc>, + session: &crate::social::Session, + sent: &[Attachment], +) -> Result<(), XrpcError> { + if sent.is_empty() { + return Ok(()); + } + let lfs = &state.lfs; + let can_contribute = session.permission().contributes(); + let store = Arc::clone(&lfs.handle.store); + let repo = session.repo.clone(); + let blobs = sent.to_vec(); + run_blocking(move || { + blobs.iter().try_for_each(|blob| { + let oid = LfsOid::from_digest(blob.cid().digest()); + let probed = store + .probe(Pool::Attachments, &repo, &oid) + .map_err(|error| XrpcError::internal(error.to_string()))?; + match probed { + Some(size) if size.get() == blob.size().get() => (), + Some(size) => { + return Err(XrpcError::invalid_request(format!( + "The blob {} is stored as {} bytes, but the record claims {}. Please \ + re-upload it and copy the size `uploadBlob` returned, thanks.", + blob.cid(), + size.get(), + blob.size().get() + ))); + } + None => { + return Err(XrpcError::invalid_request(format!( + "The blob {} isn't in this repo's blob store. Please upload it \ + with `com.atproto.repo.uploadBlob` before the record that references \ + it, thanks.", + blob.cid() + ))); + } + } + match sniffed_kind(&store, &repo, &oid) { + Some(reads) if reads == blob.mime().as_str() => (), + Some(reads) => { + return Err(XrpcError::invalid_request(format!( + "The blob {} reads as {reads} on this knot, but the record claims {}. \ + Please use the `mimeType` from the upload, thanks.", + blob.cid(), + blob.mime().as_str() + ))); + } + None => (), + } + match can_contribute { + Decision::Allow => { + store + .touch(Pool::Attachments, &repo, &oid) + .map_err(|error| XrpcError::internal(error.to_string()))?; + } + Decision::Deny => (), + } + Ok(()) + }) + }) + .await +} + +pub(crate) async fn session_admitting( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + repo: RepoDid, + sent: &[Attachment], +) -> Result { + let session = crate::social::session(state, headers, method, repo).await?; + admit_attachments(state, &session, sent).await?; + Ok(session) +} + +pub(crate) async fn session_admitting_artifact( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + repo: RepoDid, + sent: &knot_record::artifact::ArtifactBlob, +) -> Result { + let session = crate::social::session(state, headers, method, repo).await?; + let lfs = &state.lfs; + let can_contribute = session.permission().contributes(); + let store = Arc::clone(&lfs.handle.store); + let repo = session.repo.clone(); + let blob = sent.clone(); + run_blocking(move || { + let oid = LfsOid::from_digest(blob.cid().digest()); + let probed = store + .probe(Pool::Attachments, &repo, &oid) + .map_err(|error| XrpcError::internal(error.to_string()))?; + match probed { + Some(size) if size.get() == blob.size() => (), + Some(size) => { + return Err(XrpcError::invalid_request(format!( + "The blob {} is stored as {} bytes, but the record claims {}. Please \ + re-upload it and copy the size `uploadBlob` returned, thanks.", + blob.cid(), + size.get(), + blob.size() + ))); + } + None => { + return Err(XrpcError::invalid_request(format!( + "The blob {} isn't in this repo's blob store. Please upload it \ + with `com.atproto.repo.uploadBlob` before the record that references \ + it, thanks.", + blob.cid() + ))); + } + } + match sniffed_kind(&store, &repo, &oid) { + Some(reads) if reads == blob.mime() => (), + Some(reads) => { + return Err(XrpcError::invalid_request(format!( + "The blob {} reads as {reads} on this knot, but the record claims {}. \ + Please use the `mimeType` from the upload, thanks.", + blob.cid(), + blob.mime() + ))); + } + None => (), + } + match can_contribute { + Decision::Allow => { + store + .touch(Pool::Attachments, &repo, &oid) + .map_err(|error| XrpcError::internal(error.to_string()))?; + } + Decision::Deny => (), + } + Ok(()) + }) + .await?; + Ok(session) +} diff --git a/knot2/crates/knot-xrpc/src/blocklist.rs b/knot2/crates/knot-xrpc/src/blocklist.rs new file mode 100644 index 0000000000000000000000000000000000000000..0fa7b3d0aad11526f2b61938f9f75e06fb3d39dc --- /dev/null +++ b/knot2/crates/knot-xrpc/src/blocklist.rs @@ -0,0 +1,80 @@ +use std::sync::Arc; + +use axum::response::Response; + +use knot_cobs::{BlocklistChange, BlocklistCob, Grant, Removal}; +use knot_index::Index; +use knot_record::grant::GrantCollection; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, UnixSeconds}; + +use crate::XrpcState; +use crate::cob::Authorized; +use crate::error::XrpcError; +use crate::members::{commit_grant_list, require_admin}; + +pub(crate) async fn block_add( + state: &Arc>, + actor: &AccountDid, + subject: AccountDid, +) -> Result<(), XrpcError> { + require_admin(state, actor, "only knot admin may ban accounts")?; + + ban_account(state, subject, actor.clone()).await.map(|_| ()) +} + +pub(crate) async fn ban_account( + state: &Arc>, + subject: AccountDid, + added_by: AccountDid, +) -> Result { + if state.admins.contains(&subject) { + return Err(XrpcError::forbidden("admin cannot be banned")); + } + + let now = state.now(); + let grant = Grant { + subject, + added_by, + created_at: now, + }; + commit_blocklist(state, BlocklistChange::Add(grant), now).await +} + +pub(crate) async fn block_remove( + state: &Arc>, + actor: &AccountDid, + subject: AccountDid, +) -> Result<(), XrpcError> { + require_admin(state, actor, "only knot admin may unban accounts")?; + + unban_account(state, subject).await.map(|_| ()) +} + +pub(crate) async fn unban_account( + state: &Arc>, + subject: AccountDid, +) -> Result { + commit_blocklist( + state, + BlocklistChange::Remove(Removal { subject }), + state.now(), + ) + .await +} + +async fn commit_blocklist( + state: &Arc>, + change: BlocklistChange, + now: UnixSeconds, +) -> Result { + commit_grant_list::( + state, + Authorized::by_decree(change), + Index::refresh_blocklist, + crate::grants::sync_blocks, + GrantCollection::Block, + now, + ) + .await +} diff --git a/knot2/crates/knot-xrpc/src/body.rs b/knot2/crates/knot-xrpc/src/body.rs new file mode 100644 index 0000000000000000000000000000000000000000..bde4b10b98827f867af0d4f9cca90ec533435ead --- /dev/null +++ b/knot2/crates/knot-xrpc/src/body.rs @@ -0,0 +1,55 @@ +use serde::de::{self, Deserializer}; +use serde::ser::Serializer; +use serde::{Deserialize, Serialize}; + +use url::Url; + +#[derive(Clone, Debug)] +pub(crate) struct SourceUrl(Url); + +impl SourceUrl { + pub(crate) fn from_url(url: Url) -> Result { + match url.scheme() == "https" && url.has_host() { + true => Ok(Self(url)), + false => Err("source must be an https url"), + } + } + + pub(crate) fn as_str(&self) -> &str { + self.0.as_str() + } + + pub(crate) fn as_url(&self) -> &Url { + &self.0 + } +} + +impl<'de> Deserialize<'de> for SourceUrl { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + parse_source_url(&raw).map_err(de::Error::custom) + } +} + +impl Serialize for SourceUrl { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.0.as_str()) + } +} + +fn parse_source_url(raw: &str) -> Result { + let url = Url::parse(raw).map_err(|_| "source must be a valid url")?; + SourceUrl::from_url(url) +} + +#[derive(Clone)] +pub(crate) struct Source { + pub(crate) url: SourceUrl, + pub(crate) kind: crate::forks::SourceKind, +} + +knot_types::text_newtype! { + pub(crate) struct Patch(String) => verbatim; + pub(crate) struct CommitMessage(String) => verbatim; + pub(crate) struct CommitBody(String) => verbatim; +} diff --git a/knot2/crates/knot-xrpc/src/branches.rs b/knot2/crates/knot-xrpc/src/branches.rs new file mode 100644 index 0000000000000000000000000000000000000000..1f827b545f027ddd41ff0158e6114473e7d45a97 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/branches.rs @@ -0,0 +1,125 @@ +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use http::HeaderMap; +use serde::{Deserialize, Serialize}; + +use knot_git::{GitError, RefUpdate}; +use knot_record::git_refs; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AtUri, BranchName, RecordAddress, RecordRkey, RepoDid}; + +use crate::error::XrpcError; +use crate::out::CidString; +use crate::reads::require_hosted; +use crate::{XrpcState, decode, ok_empty, run_blocking}; + +pub(crate) const DELETE_ROUTE: &str = "/xrpc/org.tangled.git.deleteBranch"; +pub(crate) const SET_DEFAULT_ROUTE: &str = "/xrpc/org.tangled.git.setDefaultBranch"; + +#[derive(Deserialize)] +struct SetDefaultBranchInput { + repo: RepoDid, + branch: BranchName, +} + +#[derive(Serialize)] +struct SetDefaultBranchOutput { + uri: AtUri, + cid: CidString, +} + +#[derive(Deserialize)] +struct DeleteBranchInput { + repo: RepoDid, + branch: BranchName, +} + +const BRANCH_DENIED: &str = "only repository owner or a collaborator may change its branches"; + +pub(crate) fn set_default_head( + repo: &knot_git::Repo, + default_branch: &BranchName, +) -> Result<(), XrpcError> { + let refname = default_branch.head_ref(); + let target_exists = repo.find_ref(&refname)?.is_some(); + let has_branches = !repo.branches()?.is_empty(); + if !target_exists && has_branches { + return Err(XrpcError::not_found("no such branch to set as default")); + } + repo.set_head(&refname).map_err(XrpcError::from) +} + +pub(crate) async fn set_default_branch( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let input: SetDefaultBranchInput = decode(&body)?; + let actor = state + .authenticate_write_for_repo(&headers, &method, &input.repo) + .await?; + let repo_did = require_hosted(&state, input.repo)?; + crate::authorize_settings(&state, &actor, &repo_did).await?; + let worker = Arc::clone(&state); + let output = run_blocking(move || { + let repo = worker.layout.open(&repo_did)?; + // TODO: ensure `git.refs/self` record exists after the move. + set_default_head(&repo, &input.branch)?; + crate::refrecords::sync_refs(&worker, &repo, &repo_did, &[]); + let rkey = RecordRkey::new(git_refs::SELF_RKEY).expect("the refs self key is a valid rkey"); + Ok(SetDefaultBranchOutput { + uri: RecordAddress::new(git_refs::git_refs_collection(), rkey).at_uri(&repo_did), + cid: git_refs::desired_record(&repo)? + .into_values() + .next() + .ok_or_else(|| XrpcError::internal("the refs record went missing after the move"))? + .cid() + .into(), + }) + }) + .await?; + Ok(Json(output).into_response()) +} + +pub(crate) async fn delete_branch( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate_write(&headers, &method).await?; + let input: DeleteBranchInput = decode(&body)?; + let repo_did = require_hosted(&state, input.repo)?; + crate::authorize_push(&state, &actor, &repo_did, BRANCH_DENIED).await?; + let refname = input.branch.head_ref(); + + let layout = state.layout.clone(); + let target = repo_did.clone(); + let update = run_blocking(move || { + let repo = layout.open(&target)?; + if repo.default_branch().as_ref() == Some(&refname) { + return Err(XrpcError::invalid_request( + "default branch cannot be deleted until a different default is set", + )); + } + let old = repo + .find_ref(&refname)? + .ok_or_else(|| XrpcError::not_found("no such branch"))?; + let update = RefUpdate::Delete { name: refname, old }; + repo.update_ref(&update).map_err(|error| match error { + GitError::AtomicRefs(_) => XrpcError::conflict("branch changed during deletion, retry"), + other => XrpcError::internal(other.to_string()), + })?; + Ok(update) + }) + .await?; + + crate::refrecords::sync_after(&state, &repo_did, update).await; + + Ok(ok_empty()) +} diff --git a/knot2/crates/knot-xrpc/src/cob.rs b/knot2/crates/knot-xrpc/src/cob.rs new file mode 100644 index 0000000000000000000000000000000000000000..8fe8f530a16c2d06103493b76f022ce60670237d --- /dev/null +++ b/knot2/crates/knot-xrpc/src/cob.rs @@ -0,0 +1,49 @@ +use knot_cob::{Checkpoint, CobHome, CobStore, Evaluate, KnotAuthored, Materialized}; +use knot_cobs::{GrantChange, GrantList}; +use knot_runtime::Signer; +use knot_types::{AccountDid, UnixSeconds}; + +use crate::error::XrpcError; + +pub(crate) struct Authorized(C); + +impl Authorized { + pub(crate) fn by_decree(change: C) -> Self { + Self(change) + } + + pub(crate) fn subject(&self) -> &AccountDid { + self.0.subject() + } +} + +pub(crate) fn grant_list_apply( + store: &CobStore, + home: &CobHome, + Authorized(change): Authorized, + signer: &dyn Signer, + now: UnixSeconds, +) -> Result<(), XrpcError> +where + E: Checkpoint + Evaluate, + E::Change: KnotAuthored + Clone + GrantChange, +{ + match store.list::().map_err(XrpcError::from)?.as_slice() { + [] => match GrantList::empty().would_change(&change) { + true => store + .create_materialized(home, &Materialized::bare(change.clone()), signer, now) + .map(|_| ()) + .map_err(XrpcError::from), + false => Ok(()), + }, + [object] => store + .update_maybe_checkpointed::(home, *object, signer, now, |list| { + Ok(list.would_change(&change).then(|| change.clone())) + }) + .map(|_| ()), + many => Err(XrpcError::internal(format!( + "{} collaborative objects of one type share namespace", + many.len() + ))), + } +} diff --git a/knot2/crates/knot-xrpc/src/collaborators.rs b/knot2/crates/knot-xrpc/src/collaborators.rs new file mode 100644 index 0000000000000000000000000000000000000000..2377eb3a343bd08a722a82ea4b636c358b01cfb7 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/collaborators.rs @@ -0,0 +1,136 @@ +use std::sync::Arc; + +use axum::response::Response; + +use knot_acl::{KnotAcl, can_manage_collaborators}; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{CollaboratorsChange, CollaboratorsCob, Grant, Removal}; +use knot_index::Resolved; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, RepoDid, UnixSeconds}; + +use crate::cob::{Authorized, grant_list_apply}; +use crate::error::XrpcError; +use crate::{XrpcState, ok_empty, run_blocking}; + +fn registered_owner( + state: &XrpcState, + repo: &RepoDid, +) -> Result { + match state.index.owner_of(repo) { + Resolved::Ready(Some(owner)) => Ok(owner), + Resolved::Ready(None) => Err(XrpcError::not_found( + "repository isn't registered on this knot", + )), + Resolved::Warming => Err(crate::reads::warming()), + } +} + +fn require_owner( + state: &XrpcState, + actor: &AccountDid, + repo: &RepoDid, +) -> Result { + let owner = registered_owner(state, repo)?; + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if can_manage_collaborators(&acl, actor, repo).is_allowed() { + Ok(owner) + } else { + Err(XrpcError::forbidden( + "only repository owner may manage collaborators", + )) + } +} + +pub(crate) async fn collaborator_add( + state: &Arc>, + actor: &AccountDid, + repo: &RepoDid, + subject: AccountDid, +) -> Result<(), XrpcError> { + let owner = require_owner(state, actor, repo)?; + let now = state.now(); + commit_collaborators( + state, + repo.clone(), + owner, + Authorized::by_decree(CollaboratorsChange::Add(Grant { + subject, + added_by: actor.clone(), + created_at: now, + })), + now, + ) + .await + .map(|_| ()) +} + +async fn commit_collaborators( + state: &Arc>, + repo: RepoDid, + owner: knot_types::OwnerDid, + change: Authorized, + now: UnixSeconds, +) -> Result { + let signer = state.secrets.signer(&state.knot_did)?; + let layout = state.layout.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let surfaces = Arc::clone(state); + run_blocking(move || { + let _guard = cob_locks.repo(&repo); + owner_unmoved(&index, &repo, &owner)?; + if owner.is(change.subject()) { + return Ok(()); + } + let git = layout.open(&repo)?; + grant_list_apply::( + &CobStore::new(&git), + &CobHome::from(&repo), + change, + &signer, + now, + )?; + index.refresh_collaborators(&repo)?; + crate::grants::noted( + crate::grants::sync_collaborators(&surfaces, &git, &repo), + knot_record::grant::GrantCollection::Collaborator, + ); + Ok(()) + }) + .await?; + + Ok(ok_empty()) +} + +fn owner_unmoved( + index: &knot_index::Index, + repo: &RepoDid, + owner: &knot_types::OwnerDid, +) -> Result<(), XrpcError> { + match index.owner_of(repo) { + Resolved::Ready(Some(current)) if current == *owner => Ok(()), + _ => Err(XrpcError::conflict( + "repository is no longer registered to owner who authorized this request", + )), + } +} + +pub(crate) async fn collaborator_remove( + state: &Arc>, + actor: &AccountDid, + repo: &RepoDid, + subject: AccountDid, +) -> Result<(), XrpcError> { + let owner = require_owner(state, actor, repo)?; + + commit_collaborators( + state, + repo.clone(), + owner, + Authorized::by_decree(CollaboratorsChange::Remove(Removal { subject })), + state.now(), + ) + .await + .map(|_| ()) +} diff --git a/knot2/crates/knot-xrpc/src/error.rs b/knot2/crates/knot-xrpc/src/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..e2d125c8fa81feca1d49752e0eae7a2b506c24ed --- /dev/null +++ b/knot2/crates/knot-xrpc/src/error.rs @@ -0,0 +1,568 @@ +use axum::Json; +use axum::response::{IntoResponse, Response}; +use http::StatusCode; +use knot_types::{AtUri, RecordCid}; +use serde_json::json; + +#[derive(Debug, Clone)] +enum Detail { + None, + Bucket(knot_resource::Bucket), + AtVersion { uri: AtUri, cid: RecordCid }, +} + +#[derive(Debug, Clone)] +pub struct XrpcError { + status: StatusCode, + error: &'static str, + message: String, + detail: Detail, +} + +impl XrpcError { + fn new(status: StatusCode, error: &'static str, message: impl Into) -> Self { + Self { + status, + error, + message: message.into(), + detail: Detail::None, + } + } + pub(crate) const REPO_NOT_FOUND: &str = "RepoNotFound"; + + pub(crate) fn is_repo_not_found(&self) -> bool { + self.error == Self::REPO_NOT_FOUND + } + + pub(crate) fn is_not_found(&self) -> bool { + self.error == "NotFound" + } + + pub fn invalid_request(message: impl Into) -> Self { + Self::new(StatusCode::BAD_REQUEST, "InvalidRequest", message) + } + + pub(crate) fn named( + status: StatusCode, + error: &'static str, + message: impl Into, + ) -> Self { + Self::new(status, error, message) + } + + pub fn auth_required(message: impl Into) -> Self { + Self::new(StatusCode::UNAUTHORIZED, "AuthenticationRequired", message) + } + + pub fn forbidden(message: impl Into) -> Self { + Self::new(StatusCode::FORBIDDEN, "Forbidden", message) + } + + pub fn not_found(message: impl Into) -> Self { + Self::new(StatusCode::NOT_FOUND, "NotFound", message) + } + + pub fn conflict(message: impl Into) -> Self { + Self::new(StatusCode::CONFLICT, "Conflict", message) + } + + pub(crate) fn throttled(message: impl Into) -> Self { + Self::new(StatusCode::TOO_MANY_REQUESTS, "RateLimitExceeded", message) + } + + pub fn request_too_large(message: impl Into) -> Self { + Self::new(StatusCode::PAYLOAD_TOO_LARGE, "RequestTooLarge", message) + } + + pub fn warming(message: impl Into) -> Self { + Self::new( + StatusCode::SERVICE_UNAVAILABLE, + "ProjectionWarming", + message, + ) + } + + pub fn upstream_unavailable(message: impl Into) -> Self { + Self::new( + StatusCode::SERVICE_UNAVAILABLE, + "UpstreamUnavailable", + message, + ) + } + + pub fn rate_limited(bucket: knot_resource::Bucket, message: impl Into) -> Self { + Self { + detail: Detail::Bucket(bucket), + ..Self::new(StatusCode::TOO_MANY_REQUESTS, "RateLimitExceeded", message) + } + } + + pub fn stale(uri: AtUri, cid: RecordCid, message: impl Into) -> Self { + Self { + detail: Detail::AtVersion { uri, cid }, + ..Self::new(StatusCode::CONFLICT, "InvalidSwap", message) + } + } + + pub fn stored_but_stale(message: impl Into) -> Self { + Self::new( + StatusCode::INTERNAL_SERVER_ERROR, + "StoredProjectionStale", + message, + ) + } + + pub fn overloaded(message: impl Into) -> Self { + Self::new(StatusCode::SERVICE_UNAVAILABLE, "Overloaded", message) + } + + pub fn bad_gateway(message: impl Into) -> Self { + Self::new(StatusCode::BAD_GATEWAY, "UpstreamFailure", message) + } + + pub fn internal(message: impl Into) -> Self { + Self::new(StatusCode::INTERNAL_SERVER_ERROR, "InternalError", message) + } + + pub(crate) fn status(&self) -> StatusCode { + self.status + } + + pub(crate) fn from_status(status: StatusCode, message: impl Into) -> Self { + let error = match status { + StatusCode::BAD_REQUEST => "InvalidRequest", + StatusCode::UNAUTHORIZED => "AuthenticationRequired", + StatusCode::FORBIDDEN => "Forbidden", + StatusCode::NOT_FOUND => "NotFound", + StatusCode::CONFLICT => "Conflict", + StatusCode::PAYLOAD_TOO_LARGE => "RequestTooLarge", + StatusCode::UNSUPPORTED_MEDIA_TYPE => "UnsupportedMediaType", + StatusCode::TOO_MANY_REQUESTS => "RateLimitExceeded", + StatusCode::SERVICE_UNAVAILABLE => "Overloaded", + StatusCode::BAD_GATEWAY => "UpstreamFailure", + _ => return Self::internal(message), + }; + Self::new(status, error, message) + } +} + +impl std::fmt::Display for XrpcError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {}", self.error, self.message) + } +} + +impl From for XrpcError { + fn from(error: knot_git::GitError) -> Self { + use knot_git::GitError; + let message = error.to_string(); + match error { + GitError::AlreadyExists(_) => Self::conflict(message), + GitError::AtomicRefs(_) => Self::conflict(message), + GitError::UnsafeRepoDid(_) | GitError::ReservedDid(_) => Self::invalid_request(message), + GitError::DepthExceeded(_) => Self::invalid_request(message), + GitError::ArchiveTooLarge { .. } | GitError::BlameObjectTooLarge { .. } => { + Self::request_too_large(message) + } + GitError::Selection(_) | GitError::BlameTimedOut => Self::overloaded(message), + // Every oid passed to the object database here came from a ref this + // knot already resolved or a tree it already read, so a miss means + // the repository is missing an object it references. A handler that + // looks up something the caller named reports its own named error + // before it ever gets an oid. + GitError::ObjectNotFound(_) + | GitError::Open { .. } + | GitError::Create { .. } + | GitError::Remove { .. } + | GitError::Reference { .. } + | GitError::Fsync { .. } + | GitError::Write { .. } + | GitError::RevWalk(_) + | GitError::RemoveObject { .. } + | GitError::Corrupt { .. } + | GitError::ObjectType { .. } + | GitError::Decode(_) + | GitError::Backend(_) + | GitError::Staging(_) + | GitError::Config { .. } + | GitError::Maintenance(_) => Self::internal(message), + } + } +} + +impl From for XrpcError { + fn from(error: knot_record::chain::ChainError) -> Self { + use knot_record::chain::ChainError; + let message = error.to_string(); + match error { + ChainError::ChunkTooLarge => Self::request_too_large(message), + ChainError::NotEnabled | ChainError::RecordAbsent { .. } => Self::not_found(message), + ChainError::RecordExists { uri, current } + | ChainError::RecordMoved { uri, current, .. } => Self::stale(uri, current, message), + ChainError::AlreadyMaterialized => Self::conflict(message), + ChainError::RecordContested { .. } | ChainError::TooManyOps { .. } => { + Self::invalid_request(message) + } + ChainError::Record(_) + | ChainError::Block(_) + | ChainError::Git(_) + | ChainError::LeafForeign { .. } + | ChainError::PartlyApplied { .. } + | ChainError::Malformed(_) => Self::internal(message), + } + } +} + +impl From for XrpcError { + fn from(error: knot_git::ApplyError) -> Self { + use knot_git::ApplyError; + match error { + ApplyError::TooLarge => Self::request_too_large(error.to_string()), + ApplyError::Git(inner) => inner.into(), + } + } +} + +impl From for XrpcError { + fn from(error: crate::materialize::EnableError) -> Self { + match error { + crate::materialize::EnableError::Chain(chain) => chain.into(), + other => Self::internal(other.to_string()), + } + } +} + +impl From for XrpcError { + fn from(error: knot_cob::CobError) -> Self { + use knot_cob::CobError; + let message = error.to_string(); + match error { + CobError::Contended(_) | CobError::StaleTip { .. } => Self::conflict(message), + CobError::StaleRecord { uri, current, .. } => Self::stale(uri, current, message), + CobError::Impersonation { .. } + | CobError::CreateRejected { .. } + | CobError::AppendRejected { .. } => Self::forbidden(message), + CobError::ChangeTooLarge { .. } => Self::request_too_large(message), + _ => Self::internal(message), + } + } +} + +impl From for XrpcError { + fn from(error: knot_record::RecordError) -> Self { + Self::internal(error.to_string()) + } +} + +impl From for XrpcError { + fn from(error: knot_cobs::SettleError) -> Self { + match error { + knot_cobs::SettleError::Cob(cob) => cob.into(), + knot_cobs::SettleError::Stored(message) => Self::internal(message), + } + } +} + +impl From for XrpcError { + fn from(error: knot_types::RecordBodyError) -> Self { + use knot_types::RecordBodyError; + match error { + RecordBodyError::Oversized(_) => Self::request_too_large(error.to_string()), + other => Self::internal(other.to_string()), + } + } +} + +impl From for XrpcError { + fn from(error: knot_index::IndexError) -> Self { + use knot_index::IndexError; + match error { + IndexError::Git(git) => git.into(), + IndexError::Cob(cob) => cob.into(), + other => Self::internal(other.to_string()), + } + } +} + +impl From for XrpcError { + fn from(error: knot_secrets::SecretsError) -> Self { + use knot_secrets::SecretsError; + match error { + SecretsError::Occupied(_) => Self::conflict(error.to_string()), + other => Self::internal(other.to_string()), + } + } +} + +impl From for XrpcError { + fn from(error: knot_cobs::RegistryError) -> Self { + use knot_cobs::RegistryError; + match error { + RegistryError::AlreadyRegistered { .. } + | RegistryError::RepoMismatch { .. } + | RegistryError::RkeyTaken { .. } + | RegistryError::OwnerMoved { .. } => Self::conflict(error.to_string()), + RegistryError::NotRegistered { .. } | RegistryError::NotHosted { .. } => { + Self::not_found(error.to_string()) + } + RegistryError::Cob(cob) => cob.into(), + } + } +} + +impl From for XrpcError { + fn from(error: knot_atproto::AtprotoError) -> Self { + use knot_atproto::AtprotoError; + if error.is_transient() { + return Self::upstream_unavailable(error.to_string()); + } + match error { + AtprotoError::Resolve(_) => Self::invalid_request(error.to_string()), + AtprotoError::PlcSubmit { .. } => Self::bad_gateway(error.to_string()), + other => Self::internal(other.to_string()), + } + } +} + +impl From for XrpcError { + fn from(error: knot_atproto::IdentityError) -> Self { + Self::internal(error.to_string()) + } +} + +impl From for XrpcError { + fn from(error: knot_pack::PackError) -> Self { + Self::from_status(error.http_status(), error.to_string()) + } +} + +impl From for XrpcError { + fn from(error: knot_pack::FetchError) -> Self { + use knot_pack::FetchError; + let message = error.to_string(); + match error { + FetchError::Url(_) => Self::invalid_request(message), + FetchError::Network(_) => Self::upstream_unavailable(message), + FetchError::Status(_) | FetchError::Protocol(_) | FetchError::Remote(_) => { + Self::bad_gateway(message) + } + FetchError::PackTooLarge { .. } => Self::request_too_large(message), + FetchError::Pack(pack) => pack.into(), + } + } +} + +impl IntoResponse for XrpcError { + fn into_response(self) -> Response { + match self.status { + StatusCode::FORBIDDEN | StatusCode::TOO_MANY_REQUESTS => tracing::warn!( + status = self.status.as_u16(), + error = self.error, + message = %self.message, + "request rejected" + ), + StatusCode::UNAUTHORIZED => tracing::debug!( + error = self.error, + message = %self.message, + "request unauthenticated" + ), + _ => {} + } + let body = match self.detail { + Detail::Bucket(bucket) => json!({ + "error": self.error, + "message": self.message, + "bucket": bucket.name(), + }), + Detail::AtVersion { uri, cid } => json!({ + "error": self.error, + "message": self.message, + "uri": uri.as_str(), + "cid": cid.to_string(), + }), + Detail::None => json!({ "error": self.error, "message": self.message }), + }; + (self.status, Json(body)).into_response() + } +} + +#[cfg(test)] +mod tests { + use super::XrpcError; + use axum::response::IntoResponse; + use http::StatusCode; + + #[test] + fn each_tag_maps_to_its_status_in_the_class_and_through_into_response() { + let cases: &[(XrpcError, StatusCode, &str)] = &[ + ( + XrpcError::invalid_request("x"), + StatusCode::BAD_REQUEST, + "InvalidRequest", + ), + ( + XrpcError::auth_required("x"), + StatusCode::UNAUTHORIZED, + "AuthenticationRequired", + ), + ( + XrpcError::forbidden("x"), + StatusCode::FORBIDDEN, + "Forbidden", + ), + (XrpcError::not_found("x"), StatusCode::NOT_FOUND, "NotFound"), + (XrpcError::conflict("x"), StatusCode::CONFLICT, "Conflict"), + ( + XrpcError::request_too_large("x"), + StatusCode::PAYLOAD_TOO_LARGE, + "RequestTooLarge", + ), + ( + XrpcError::rate_limited(knot_resource::Bucket::ActorWrites, "x"), + StatusCode::TOO_MANY_REQUESTS, + "RateLimitExceeded", + ), + ( + XrpcError::warming("x"), + StatusCode::SERVICE_UNAVAILABLE, + "ProjectionWarming", + ), + ( + XrpcError::upstream_unavailable("x"), + StatusCode::SERVICE_UNAVAILABLE, + "UpstreamUnavailable", + ), + ( + XrpcError::overloaded("x"), + StatusCode::SERVICE_UNAVAILABLE, + "Overloaded", + ), + ( + XrpcError::bad_gateway("x"), + StatusCode::BAD_GATEWAY, + "UpstreamFailure", + ), + ( + XrpcError::internal("x"), + StatusCode::INTERNAL_SERVER_ERROR, + "InternalError", + ), + ]; + cases.iter().for_each(|(error, status, tag)| { + assert_eq!((error.status, error.error), (*status, *tag)); + assert_eq!( + error.clone().into_response().status(), + *status, + "into_response serves the mapped status for {tag}" + ); + }); + } + + #[test] + fn a_domain_error_maps_to_the_status_that_names_whose_fault_it_is() { + let oid = knot_types::Oid::from_hex(&"a".repeat(40)).unwrap(); + let cases: Vec<(XrpcError, StatusCode, &str)> = vec![ + ( + knot_git::GitError::ObjectNotFound(oid).into(), + StatusCode::INTERNAL_SERVER_ERROR, + "these oids come from refs and trees this knot resolved itself, \ + and every read lexicon names its own not-found error for what the caller asked for", + ), + ( + knot_git::GitError::Corrupt { + oid, + message: "truncated".to_string(), + } + .into(), + StatusCode::INTERNAL_SERVER_ERROR, + "a corrupt repository on this knot isn't the caller's fault to fix", + ), + ( + knot_git::GitError::AtomicRefs("lost".to_string()).into(), + StatusCode::CONFLICT, + "losing a ref race is a conflict", + ), + ( + knot_git::GitError::AlreadyExists("/scallop".into()).into(), + StatusCode::CONFLICT, + "creating a repository that exists is a conflict", + ), + ( + knot_pack::FetchError::Remote("gone".to_string()).into(), + StatusCode::BAD_GATEWAY, + "a fetch failure at the upstream during fork sync is the upstream's fault", + ), + ( + knot_git::ApplyError::Git(knot_git::GitError::AtomicRefs("lost".to_string())) + .into(), + StatusCode::CONFLICT, + "wrapping a git error in ApplyError mustn't downgrade it to a generic fault", + ), + ( + knot_index::IndexError::Git(knot_git::GitError::AlreadyExists("/whelk".into())) + .into(), + StatusCode::CONFLICT, + "wrapping a git error in IndexError mustn't downgrade it either", + ), + ]; + cases.iter().for_each(|(error, status, why)| { + assert_eq!(error.status(), *status, "{why}"); + }); + } + + #[tokio::test] + async fn a_shed_request_names_the_bucket_it_came_from_and_a_clock_answer_names_none() { + let limited = XrpcError::rate_limited( + knot_resource::Bucket::ActorWrites, + "this account is writing too fast", + ) + .into_response(); + assert_eq!(limited.status(), StatusCode::TOO_MANY_REQUESTS); + let body = axum::body::to_bytes(limited.into_body(), 4096) + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + json["bucket"], "actor-writes", + "the bucket field separates the limiter answer from the lease answer, without reading prose" + ); + + let conflict = XrpcError::conflict("Acceptance lease expired").into_response(); + let body = axum::body::to_bytes(conflict.into_body(), 4096) + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert!( + json.get("bucket").is_none(), + "the absence of the bucket field means the other kind of answer, only the limiter answer has one" + ); + } + + #[tokio::test] + async fn a_stale_write_answers_the_uri_and_cid_a_client_rebases_onto() { + let uri = knot_types::AtUri::from_parts_owned( + "did:plc:squid", + "org.tangled.track.ticket", + "3lubrptx57d22", + ) + .unwrap(); + let body = + knot_types::RecordBody::new(serde_ipld_dagcbor::to_vec(&"kelp").unwrap()).unwrap(); + let stale = XrpcError::from(knot_cob::CobError::StaleRecord { + object: knot_types::CobId::new(knot_types::Oid::null()), + uri: uri.clone(), + current: body.cid(), + }) + .into_response(); + assert_eq!(stale.status(), StatusCode::CONFLICT); + let bytes = axum::body::to_bytes(stale.into_body(), 4096).await.unwrap(); + let json: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(json["error"], "InvalidSwap"); + assert_eq!( + json["uri"], + uri.as_str(), + "Record's at-uri is field, not substring of prose" + ); + assert_eq!(json["cid"], body.cid().to_string()); + } +} diff --git a/knot2/crates/knot-xrpc/src/firehose.rs b/knot2/crates/knot-xrpc/src/firehose.rs new file mode 100644 index 0000000000000000000000000000000000000000..08c10e179f80977a62b1f623eb0ed313c7b476ba --- /dev/null +++ b/knot2/crates/knot-xrpc/src/firehose.rs @@ -0,0 +1,1632 @@ +use std::collections::{BTreeMap, VecDeque}; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use axum::extract::ws::{CloseFrame, Message, WebSocket, WebSocketUpgrade}; +use axum::extract::{ConnectInfo, Query, State}; +use axum::response::{IntoResponse, Response}; +use bytes::Bytes; +use cid::Cid; +use futures::{SinkExt, StreamExt}; +use http::HeaderMap; +use jacquard_common::types::tid::Tid; +use knot_events::{ + BatchEnd, EventCursor, EventLog, ReplayBounds, ReplayBytes, ReplayEvents, Replayed, +}; +use knot_git::{Layout, RefUpdate, Repo}; +use knot_record::chain::{self, CommitChunk, Emit, MAX_COMMIT_BLOCKS_BYTES, Op, blocks_bytes}; +use knot_runtime::{Clock, HttpTransport, Signer}; +use knot_types::{ + AccountDid, Datetime, Oid, PushOption, PushOptions, RefName, RepoDid, UnixMicros, UnixSeconds, +}; +use serde::{Deserialize, Serialize}; + +use crate::XrpcState; +use crate::error::XrpcError; +use crate::out::datetime_of; +use Source::{Chains, Ring}; + +const DRAIN_BATCH: usize = 100; +const MAX_REPLAY_INDEX_FRAMES: usize = 100_000; +const META_REPLAY_SHARE: usize = MAX_REPLAY_INDEX_FRAMES / 2; +const DRAIN_BYTES: usize = 4 << 20; +const KEEPALIVE: Duration = Duration::from_secs(30); +const WRITE_DEADLINE: Duration = Duration::from_secs(10); +const TRY_AGAIN_LATER: u16 = 1013; +const MAX_PUSH_EDITS: usize = 200; +const MAX_PUSH_OPTIONS: usize = 10; + +pub struct FirehoseEmit<'a, C: Clock> { + log: &'a EventLog, + did: &'a RepoDid, + now: UnixSeconds, + reserved: Vec, + frames: Vec>, +} + +impl<'a, C: Clock> FirehoseEmit<'a, C> { + pub fn new(log: &'a EventLog, did: &'a RepoDid, now: UnixSeconds) -> Self { + Self { + log, + did, + now, + reserved: Vec::new(), + frames: Vec::new(), + } + } + + pub fn finish(self) -> Result<(), UnstampedFrames> { + let (stamped, framed) = (self.reserved.len(), self.frames.len()); + if stamped > framed { + tracing::warn!( + stamped, + framed, + "chain write stopped after stamping revisions it never committed, withdrawing their reservations, knot bug, please report" + ); + } + self.reserved + .into_iter() + .zip(self.frames) + .for_each(|(reservation, frame)| match frame { + Some(frame) => reservation.fulfill_frame(frame), + None => drop(reservation), + }); + if framed > stamped { + return Err(UnstampedFrames { stamped, framed }); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error( + "chain write committed {framed} commits, but stamped only {stamped} revisions, frames past last stamp have no sequence number, and don't go out, knot bug, please report" +)] +pub struct UnstampedFrames { + pub stamped: usize, + pub framed: usize, +} + +impl From for XrpcError { + fn from(error: UnstampedFrames) -> Self { + Self::internal(error.to_string()) + } +} + +impl Emit for FirehoseEmit<'_, C> { + fn rev(&mut self, prev: Option<&Tid>) -> Tid { + if let Some(prev) = prev { + self.log.raise_floor(UnixMicros::new(prev.timestamp())); + } + let reservation = self.log.reserve(); + let tid = Tid::from_time(reservation.micros().get(), 0); + self.reserved.push(reservation); + tid + } + + fn frame(&self, chunk: &CommitChunk) -> Option { + commit_frame( + self.did, + chunk, + UnixMicros::new(chunk.rev.timestamp()), + self.now, + ) + } + + fn committed(&mut self, frame: Option) { + self.frames.push(frame); + } +} +pub fn fulfill_identity(reservation: knot_events::Reservation, did: &RepoDid, now: UnixSeconds) { + let seq = reservation.micros(); + fulfill( + reservation, + FrameKind::Identity, + &IdentityMessage { + seq, + did: did.clone(), + time: datetime_of(now), + }, + ); +} + +pub fn fulfill_account( + reservation: knot_events::Reservation, + did: &RepoDid, + active: bool, + status: Option<&'static str>, + now: UnixSeconds, +) { + let seq = reservation.micros(); + fulfill( + reservation, + FrameKind::Account, + &AccountMessage { + seq, + did: did.clone(), + time: datetime_of(now), + active, + status, + }, + ); +} + +pub fn publish_account( + log: &EventLog, + did: &RepoDid, + active: bool, + status: Option<&'static str>, + now: UnixSeconds, +) { + fulfill_account(log.reserve(), did, active, status, now); +} + +pub(crate) fn account_deleted_frame(did: &RepoDid, deleted_at: UnixMicros) -> Bytes { + frame( + FrameKind::Account, + &AccountMessage { + seq: deleted_at, + did: did.clone(), + time: datetime_of(deleted_at.seconds()), + active: false, + status: Some("deleted"), + }, + ) +} + +pub fn publish_sync( + log: &EventLog, + did: &RepoDid, + rev: &Tid, + commit_bytes: &[u8], + now: UnixSeconds, +) { + let commit = match chain::commit_cid(commit_bytes) { + Ok(cid) => (cid, Bytes::copy_from_slice(commit_bytes)), + Err(error) => { + tracing::error!(%error, did = did.as_str(), "missing cid skips the sync frame of the commit this knot signed"); + return; + } + }; + let car = knot_record::car_bytes(commit.0, &BTreeMap::from([commit])); + let reservation = log.reserve(); + let seq = reservation.micros(); + fulfill( + reservation, + FrameKind::Sync, + &SyncMessage { + seq, + did: did.clone(), + rev: rev.clone(), + blocks: car, + time: datetime_of(now), + }, + ); +} + +pub fn publish_push( + log: &EventLog, + did: &RepoDid, + pusher: &AccountDid, + applied: &[RefUpdate], + options: &PushOptions, + signer: &dyn Signer, + now: UnixSeconds, +) { + let options = options.as_slice(); + if applied.len() > MAX_PUSH_EDITS || options.len() > MAX_PUSH_OPTIONS { + tracing::warn!( + repo = did.as_str(), + edits = applied.len(), + options = options.len(), + "a push overran what its event can carry, so the frame names only the refs and options that fit" + ); + } + let reservation = log.reserve(); + let seq = reservation.micros(); + let unsigned = UnsignedPushEventMessage { + repo: did, + pusher, + edits: applied + .iter() + .take(MAX_PUSH_EDITS) + .map(RefEdit::of) + .collect(), + time: datetime_of(now), + options: (!options.is_empty()).then(|| options.iter().take(MAX_PUSH_OPTIONS).collect()), + }; + let sig = signer.sign(&encode(&unsigned)).as_bytes().to_vec(); + fulfill( + reservation, + FrameKind::Push, + &PushEventMessage { + unsigned: &unsigned, + seq, + sig, + }, + ); +} + +fn fulfill(reservation: knot_events::Reservation, t: FrameKind, message: &M) { + reservation.fulfill_frame(frame(t, message)); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +enum FrameKind { + #[serde(rename = "#commit")] + Commit, + #[serde(rename = "#sync")] + Sync, + #[serde(rename = "#identity")] + Identity, + #[serde(rename = "#account")] + Account, + #[serde(rename = "#info")] + Info, + #[serde(rename = "org.tangled.git.pushEvent")] + Push, +} + +#[derive(Debug, Clone, Copy)] +enum ErrorCode { + FutureCursor, +} + +impl ErrorCode { + const fn as_str(self) -> &'static str { + match self { + Self::FutureCursor => "FutureCursor", + } + } +} + +#[derive(Debug, Clone, Copy)] +enum InfoCode { + OutdatedCursor, + AgedNonCommitCursor, +} + +impl InfoCode { + const fn as_str(self) -> &'static str { + match self { + Self::OutdatedCursor => "OutdatedCursor", + Self::AgedNonCommitCursor => "AgedNonCommitCursor", + } + } +} +#[derive(Serialize, Deserialize)] +pub(crate) struct Header { + op: i64, + t: FrameKind, +} + +#[derive(Serialize)] +struct ErrorHeader { + op: i64, +} + +#[derive(Serialize)] +struct FrameError<'a> { + error: &'static str, + message: &'a str, +} + +#[derive(Serialize)] +struct FrameInfo<'a> { + name: &'static str, + message: &'a str, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) enum RepoAction { + #[serde(rename = "create")] + Create, + #[serde(rename = "update")] + Update, + #[serde(rename = "delete")] + Delete, +} + +#[derive(Serialize, Deserialize)] +pub(crate) struct RepoOp { + pub(crate) action: RepoAction, + pub(crate) path: String, + pub(crate) cid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + prev: Option, +} +#[derive(Serialize, Deserialize)] +pub(crate) struct CommitMessage { + pub(crate) seq: UnixMicros, + rebase: bool, + #[serde(rename = "tooBig")] + too_big: bool, + pub(crate) repo: RepoDid, + pub(crate) commit: Cid, + pub(crate) rev: Tid, + since: Option, + #[serde(with = "serde_bytes")] + pub(crate) blocks: Vec, + pub(crate) ops: Vec, + blobs: Vec, + #[serde(rename = "prevData", skip_serializing_if = "Option::is_none")] + prev_data: Option, + time: Datetime, +} + +#[derive(Serialize)] +struct IdentityMessage { + seq: UnixMicros, + did: RepoDid, + time: Datetime, +} + +#[derive(Serialize)] +struct AccountMessage<'a> { + seq: UnixMicros, + did: RepoDid, + time: Datetime, + active: bool, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option<&'a str>, +} + +#[derive(Serialize)] +struct SyncMessage { + seq: UnixMicros, + did: RepoDid, + rev: Tid, + #[serde(with = "serde_bytes")] + blocks: Vec, + time: Datetime, +} + +#[derive(Serialize)] +struct RefEdit<'a> { + name: &'a RefName, + new: Option<&'a Oid>, + old: Option<&'a Oid>, +} + +impl<'a> RefEdit<'a> { + fn of(update: &'a RefUpdate) -> Self { + match update { + RefUpdate::Create { name, new } => Self { + name, + new: Some(new), + old: None, + }, + RefUpdate::Update { name, old, new } => Self { + name, + new: Some(new), + old: Some(old), + }, + RefUpdate::Delete { name, old } => Self { + name, + new: None, + old: Some(old), + }, + } + } +} + +#[derive(Serialize)] +struct UnsignedPushEventMessage<'a> { + repo: &'a RepoDid, + pusher: &'a AccountDid, + edits: Vec>, + time: Datetime, + #[serde(skip_serializing_if = "Option::is_none")] + options: Option>, +} + +#[derive(Serialize)] +struct PushEventMessage<'a> { + #[serde(flatten)] + unsigned: &'a UnsignedPushEventMessage<'a>, + seq: UnixMicros, + #[serde(with = "serde_bytes")] + sig: Vec, +} + +fn encode(value: &T) -> Bytes { + Bytes::from(serde_ipld_dagcbor::to_vec(value).expect("frame values encode as dag-cbor")) +} + +fn glued(header: &H, payload: &P) -> Bytes { + let mut out = encode(header).to_vec(); + out.extend_from_slice(&encode(payload)); + Bytes::from(out) +} + +fn frame(t: FrameKind, payload: &impl Serialize) -> Bytes { + glued(&Header { op: 1, t }, payload) +} +pub(crate) fn commit_message(frame: &[u8]) -> Option<(Header, CommitMessage)> { + let mut read = frame; + let header: Header = serde_ipld_dagcbor::de::from_reader_once(&mut read).ok()?; + if header.t != FrameKind::Commit { + return None; + } + let message = serde_ipld_dagcbor::from_slice::(read).ok()?; + Some((header, message)) +} + +#[cfg(test)] +pub(crate) fn commit_blocks(frame: &[u8]) -> Vec { + commit_message(frame) + .and_then(|(_, message)| knot_record::car_blocks(&message.blocks).ok()) + .map(|car| car.blocks.keys().copied().collect()) + .unwrap_or_default() +} + +fn error_frame(code: ErrorCode, message: &str) -> Bytes { + glued( + &ErrorHeader { op: -1 }, + &FrameError { + error: code.as_str(), + message, + }, + ) +} + +fn info_frame(code: InfoCode, message: &str) -> Bytes { + frame( + FrameKind::Info, + &FrameInfo { + name: code.as_str(), + message, + }, + ) +} + +fn commit_frame( + did: &RepoDid, + chunk: &CommitChunk, + seq: UnixMicros, + now: UnixSeconds, +) -> Option { + let commit_cid = match chain::commit_cid(&chunk.commit) { + Ok(cid) => cid, + Err(error) => { + tracing::error!(%error, did = did.as_str(), "missing cid skips the frame of the chunk this knot signed"); + return None; + } + }; + let mut blocks = chunk.blocks.clone(); + blocks.insert(commit_cid, Bytes::from(chunk.commit.clone())); + let car = knot_record::car_bytes(commit_cid, &blocks); + Some(frame( + FrameKind::Commit, + &CommitMessage { + seq, + rebase: false, + too_big: blocks_bytes(&chunk.blocks) > MAX_COMMIT_BLOCKS_BYTES, + repo: did.clone(), + commit: commit_cid, + rev: chunk.rev.clone(), + since: chunk.prev_rev.clone(), + blocks: car, + ops: chunk + .ops + .iter() + .map(|op| RepoOp { + action: match op { + Op::Create { .. } => RepoAction::Create, + Op::Update { .. } => RepoAction::Update, + Op::Delete { .. } => RepoAction::Delete, + }, + path: op.key().as_str().to_owned(), + cid: op.cid(), + prev: op.prev(), + }) + .collect(), + blobs: Vec::new(), + prev_data: chunk.prev_data, + time: datetime_of(now), + }, + )) +} + +#[derive(serde::Deserialize)] +pub(crate) struct SubscribeQuery { + cursor: Option, +} + +pub(crate) async fn subscribe_repos( + State(state): State>>, + ConnectInfo(socket_peer): ConnectInfo, + headers: HeaderMap, + Query(query): Query, + upgrade: WebSocketUpgrade, +) -> Response { + let asked = match query.cursor.as_deref() { + None | Some("") => None, + Some(raw) => match raw.parse::() { + Ok(seq) => Some(seq), + Err(_) => { + return XrpcError::invalid_request("cursor must be an integer").into_response(); + } + }, + }; + let peer = state.proxy_trust.client_peer_of(&headers, socket_peer.ip()); + let Some(permit) = state.subscriber_gate.try_admit(peer) else { + return XrpcError::overloaded( + "knot is serving its maximum number of event subscribers, retry shortly", + ) + .into_response(); + }; + let current = latest(&state.firehose).micros(); + let asked_seq = asked.map(|seq| UnixMicros::new(seq.max(0) as u64)); + if asked_seq.is_some_and(|asked_cursor| future_cursor(&state.firehose, asked_cursor)) { + let rejection = error_frame(ErrorCode::FutureCursor, "Cursor is ahead of this stream"); + return upgrade.on_upgrade(move |socket| async move { + close_with(socket, rejection).await; + }); + } + let (meta_path, layout, hosted, log) = ( + state.meta_path.clone(), + state.layout.clone(), + state.index.hosted_repos(), + Arc::clone(&state.firehose), + ); + upgrade.on_upgrade(move |socket| async move { + let _permit = permit; + let (cursor, replay, opening) = match asked_seq { + None => (current, ReplayFeed::empty(), Vec::new()), + Some(asked_cursor) => { + let replay = tokio::task::spawn_blocking({ + let (meta_path, layout, hosted) = + (meta_path.clone(), layout.clone(), hosted.clone()); + move || replay_index(&meta_path, &layout, &hosted, asked_cursor) + }) + .await + .unwrap_or_else(|error| { + tracing::warn!( + %error, + "the replay index wouldn't build, so this subscriber starts at the head" + ); + ReplayFeed::empty() + }); + let mut opening = Vec::new(); + if replay.capped { + opening.push(vec![Message::Binary(info_frame( + InfoCode::OutdatedCursor, + "the replay index keeps a bounded number of frames, and this cursor reached below them; replay starts at the oldest frame kept", + ))]); + } + if log.lost_before(EventCursor::of_micros(asked_cursor)) { + opening.push(vec![Message::Binary(info_frame( + InfoCode::AgedNonCommitCursor, + "non-commit events this cursor asked for have left the replay window; every commit still replays from the durable chains", + ))]); + } + let cursor = replay + .first_seq() + .map_or(asked_cursor, |oldest| oldest.prev().max(asked_cursor)); + (cursor, replay, opening) + } + }; + stream(socket, log, cursor, replay, opening).await; + }) +} + +fn replay_index( + meta_path: &Path, + layout: &Layout, + hosted: &[RepoDid], + cursor: UnixMicros, +) -> ReplayFeed { + let mut feed = ReplayFeed::empty(); + match Repo::open(meta_path) { + Ok(meta) => { + feed.paths.push(meta_path.to_path_buf()); + index_repo(&mut feed, 0, &meta, cursor, META_REPLAY_SHARE); + match crate::plc::deleted_accounts(&meta) { + Ok(accounts) => accounts.into_iter().for_each(|(did, micros)| { + if micros > cursor { + feed.index.push_back(IndexEntry::Synthesized { + micros, + frame: account_deleted_frame(&did, micros), + }); + } + }), + Err(error) => { + tracing::warn!(%error, "deletion tombstones wouldn't read, so deleted repos stay unannounced on replay") + } + } + } + Err(error) => { + tracing::warn!(%error, "the meta repo wouldn't open, so its commits stay unreplayed") + } + } + for did in hosted { + match layout.repo_path(did) { + Ok(path) => match Repo::open(&path) { + Ok(repo) => { + let slot = feed.paths.len(); + feed.paths.push(path); + index_repo(&mut feed, slot, &repo, cursor, MAX_REPLAY_INDEX_FRAMES); + } + Err(error) => { + tracing::warn!(repo = did.as_str(), %error, "a repo wouldn't open, so its commits stay unreplayed") + } + }, + Err(error) => { + tracing::warn!(repo = did.as_str(), %error, "a repo path wouldn't resolve, so its commits stay unreplayed") + } + } + } + let mut index: Vec = feed.index.into(); + index.sort_unstable_by_key(IndexEntry::micros); + feed.index = index.into(); + feed +} + +fn index_repo(feed: &mut ReplayFeed, slot: usize, repo: &Repo, cursor: UnixMicros, cap: usize) { + match chain::tip(repo) { + Ok(Some(tip)) => { + if UnixMicros::new(tip.rev.timestamp()) > cursor { + let budget = cap.saturating_sub(feed.index.len()); + let walked = chain::framed_revs(repo, tip.git, cursor, budget); + feed.capped |= walked.end == chain::TraversalEnd::Limited; + feed.index + .extend(walked.found.into_iter().map(|(micros, oid)| { + IndexEntry::Chain(FrameAt { + micros, + repo: slot, + oid, + }) + })); + } + } + Ok(None) => {} + Err(error) => { + tracing::warn!(%error, "a chain wouldn't open, so its commits stay unreplayed") + } + } +} + +#[derive(Clone)] +struct FrameAt { + micros: UnixMicros, + repo: usize, + oid: Oid, +} + +#[derive(Clone)] +enum IndexEntry { + Chain(FrameAt), + Synthesized { micros: UnixMicros, frame: Bytes }, +} + +impl IndexEntry { + fn micros(&self) -> UnixMicros { + match self { + Self::Chain(at) => at.micros, + Self::Synthesized { micros, .. } => *micros, + } + } +} + +struct ReplayFeed { + paths: Vec, + index: VecDeque, + buffer: VecDeque<(UnixMicros, Bytes)>, + capped: bool, + poisoned: bool, +} + +impl ReplayFeed { + fn empty() -> Self { + Self { + paths: Vec::new(), + index: VecDeque::new(), + buffer: VecDeque::new(), + capped: false, + poisoned: false, + } + } + + fn first_seq(&self) -> Option { + self.index.front().map(IndexEntry::micros) + } + + fn peek(&self) -> Option<&(UnixMicros, Bytes)> { + self.buffer.front() + } + + fn pop(&mut self) -> Option<(UnixMicros, Bytes)> { + self.buffer.pop_front() + } + + fn needs_fetch(&self) -> bool { + self.buffer.is_empty() && !self.index.is_empty() + } + + fn exhausted(&self) -> bool { + self.buffer.is_empty() && self.index.is_empty() + } + + async fn fetch(&mut self) { + let mut taken = Vec::new(); + while taken.len() < DRAIN_BATCH { + match self.index.pop_front() { + Some(at) => taken.push(at), + None => break, + } + } + if taken.is_empty() { + return; + } + let paths = self.paths.clone(); + let fetched = tokio::task::spawn_blocking(move || { + let mut slots = taken + .iter() + .filter_map(|entry| match entry { + IndexEntry::Chain(at) => Some(at.repo), + IndexEntry::Synthesized { .. } => None, + }) + .collect::>(); + slots.sort_unstable(); + slots.dedup(); + let opened = slots + .into_iter() + .filter_map(|slot| match Repo::open(&paths[slot]) { + Ok(repo) => Some((slot, repo)), + Err(error) => { + tracing::warn!(%error, "a repo wouldn't reopen for replay"); + None + } + }) + .collect::>(); + let mut loaded = Vec::new(); + let mut spent = 0usize; + let mut split = taken.len(); + let mut failed = false; + for (position, at) in taken.iter().enumerate() { + if spent > DRAIN_BYTES && !loaded.is_empty() { + split = position; + break; + } + let (micros, frame) = match at { + IndexEntry::Synthesized { micros, frame } => (*micros, frame.clone()), + IndexEntry::Chain(chain_at) => { + let read = opened + .iter() + .find(|(slot, _)| *slot == chain_at.repo) + .map(|(_, repo)| chain::frame_blob(repo, chain_at.oid)); + match read { + Some(Ok(Some(frame))) => { + spent += frame.len(); + loaded.push((chain_at.micros, frame)); + continue; + } + Some(Ok(None)) => { + tracing::error!(seq = chain_at.micros.get(), "a promised commit frame is gone from its chain"); + } + Some(Err(error)) => { + tracing::error!(seq = chain_at.micros.get(), %error, "a stored frame wouldn't read back"); + } + None => { + tracing::error!(seq = chain_at.micros.get(), "a repo wouldn't reopen for its promised frame"); + } + } + split = position; + failed = true; + break; + } + }; + loaded.push((micros, frame)); + } + (loaded, taken.split_off(split), failed) + }) + .await; + match fetched { + Ok((loaded, rest, failed)) => { + self.buffer.extend(loaded); + self.poisoned |= failed; + for at in rest.into_iter().rev() { + self.index.push_front(at); + } + } + Err(error) => { + tracing::error!(%error, "the replay fetch wouldn't run; ending the replay"); + self.poisoned = true; + } + } + } +} + +async fn stream( + socket: WebSocket, + log: Arc>, + cursor: UnixMicros, + replay: ReplayFeed, + opening: Vec>, +) { + let (sink, input) = socket.split(); + let head = log.subscribe(); + let batches = futures::stream::iter(opening).chain(pump(log, cursor, replay, head)); + let served = batches.fold(Ok(sink), |sink, batch| async move { + let mut sink = sink?; + let sent = tokio::time::timeout( + WRITE_DEADLINE, + sink.send_all(&mut futures::stream::iter( + batch.into_iter().map(Ok::), + )), + ) + .await; + matches!(sent, Ok(Ok(()))).then_some(sink).ok_or(()) + }); + tokio::select! { + _ = served => {} + () = until_closed(input) => {} + } +} + +struct Pumping { + log: Arc>, + replay: ReplayFeed, + ring: VecDeque<(UnixMicros, Bytes)>, + ring_end: BatchEnd, + cursor: UnixMicros, + ring_served: bool, + reported_lost: bool, + head: tokio::sync::watch::Receiver, +} + +fn future_cursor(log: &EventLog, cursor: UnixMicros) -> bool { + cursor > log.last_issued() +} + +fn latest(log: &EventLog) -> EventCursor { + *log.subscribe().borrow() +} + +fn drain_bounds() -> ReplayBounds { + ReplayBounds::new( + ReplayEvents::new(DRAIN_BATCH).expect("drain event maximum is nonzero"), + ReplayBytes::new(DRAIN_BYTES).expect("drain byte maximum is nonzero"), + ) +} + +fn pump( + log: Arc>, + cursor: UnixMicros, + replay: ReplayFeed, + head: tokio::sync::watch::Receiver, +) -> impl futures::Stream> { + futures::stream::unfold( + Pumping { + log, + replay, + ring: VecDeque::new(), + ring_end: BatchEnd::Bounded, + cursor, + ring_served: false, + reported_lost: false, + head, + }, + |mut state| async move { + if state.replay.poisoned && state.replay.buffer.is_empty() { + return None; + } + if state.reported_lost { + return None; + } + if state.ring_served { + let here = EventCursor::of_micros(state.cursor); + if state.log.lost_before(here) { + state.reported_lost = true; + return Some(( + vec![Message::Binary(info_frame( + InfoCode::AgedNonCommitCursor, + "frames this cursor still owed have left the replay window; resume live", + ))], + state, + )); + } + } + if state.ring.is_empty() && state.ring_end == BatchEnd::Bounded { + let Replayed { events, end } = state + .log + .replay(EventCursor::of_micros(state.cursor), drain_bounds()); + let stamped = events + .into_iter() + .map(|event| (event.created.micros(), event.frame.clone())); + state.ring = stamped.collect(); + state.ring_end = end; + } + if state.replay.needs_fetch() { + state.replay.fetch().await; + } + let horizon = state.log.pending_horizon(); + let held = state + .replay + .peek() + .is_some_and(|(micros, _)| horizon.is_some_and(|horizon| *micros >= horizon)); + let mut batch = Vec::new(); + let mut bytes = 0usize; + while batch.len() < DRAIN_BATCH { + let peeked = if held { None } else { state.replay.peek() }; + let next = match (peeked, state.ring.front()) { + (Some((micros, _)), Some((ring_micros, _))) if *ring_micros <= *micros => Ring, + (Some(_), _) => Chains, + (None, Some(_)) => Ring, + (None, None) => break, + }; + let (micros, message) = match next { + Chains => { + let (micros, frame) = state.replay.pop().expect("peeked a replay frame"); + (micros, Message::Binary(frame)) + } + Ring => { + let (micros, frame) = state.ring.pop_front().expect("peeked a ring event"); + state.ring_served = true; + (micros, Message::Binary(frame)) + } + }; + if micros <= state.cursor { + continue; + } + state.cursor = micros; + bytes += match &message { + Message::Binary(frame) => frame.len(), + _ => 0, + }; + if bytes > DRAIN_BYTES && !batch.is_empty() { + break; + } + batch.push(message); + } + if !batch.is_empty() { + return Some((batch, state)); + } + if held { + state.ring_end = BatchEnd::Bounded; + return match tokio::time::timeout(KEEPALIVE, state.head.changed()).await { + Err(_) => Some((vec![Message::Ping(Bytes::new())], state)), + Ok(_) => Some((Vec::new(), state)), + }; + } + let drained = state.replay.exhausted() && state.ring.is_empty(); + if drained && state.ring_end == BatchEnd::CaughtUp { + let caught_up = EventCursor::of_micros(state.cursor); + if *state.head.borrow_and_update() <= caught_up { + return match tokio::time::timeout(KEEPALIVE, state.head.changed()).await { + Err(_) => Some((vec![Message::Ping(Bytes::new())], state)), + Ok(_) => Some((Vec::new(), state)), + }; + } + state.ring_end = BatchEnd::Bounded; + return Some((Vec::new(), state)); + } + if drained { + state.ring_end = BatchEnd::Bounded; + } + Some((Vec::new(), state)) + }, + ) +} + +enum Source { + Chains, + Ring, +} + +async fn until_closed(input: futures::stream::SplitStream) { + input + .filter(|received| futures::future::ready(!matches!(received, Ok(Message::Close(_))))) + .for_each(|_| futures::future::ready(())) + .await; +} + +async fn close_with(socket: WebSocket, frame: Bytes) { + let (mut sink, _input) = socket.split(); + let _ = sink.send(Message::Binary(frame)).await; + let _ = sink + .send(Message::Close(Some(CloseFrame { + code: TRY_AGAIN_LATER, + reason: "reconnect with a fresh cursor".into(), + }))) + .await; +} + +pub const SEQ_REF: &str = "refs/atproto/seq"; + +fn seq_ref() -> RefName { + RefName::new(SEQ_REF).expect("the seq ref name is a valid ref name") +} + +pub fn replay_floor(meta: &Repo, layout: &Layout, hosted: &[RepoDid]) -> UnixMicros { + let mut floor = read_seq(meta).unwrap_or_else(|error| { + tracing::warn!(%error, "the seq floor wouldn't read, so the firehose starts at the chains"); + UnixMicros::new(0) + }); + if let Ok(Some(tip)) = chain::tip(meta) { + floor = floor.max(UnixMicros::new(tip.rev.timestamp())); + } + for did in hosted { + match layout + .repo_path(did) + .map_err(|error| error.to_string()) + .and_then(|path| Repo::open(&path).map_err(|error| error.to_string())) + { + Ok(repo) => match chain::tip(&repo) { + Ok(Some(tip)) => floor = floor.max(UnixMicros::new(tip.rev.timestamp())), + Ok(None) => {} + Err(error) => tracing::warn!( + repo = did.as_str(), + %error, + "a chain wouldn't open for the seq floor, so its revs sit below it" + ), + }, + Err(error) => tracing::warn!( + repo = did.as_str(), + %error, + "a repo wouldn't open for the seq floor, so its revs sit below it" + ), + } + } + floor +} + +pub fn persist_seq(log: &EventLog, meta_path: &Path) { + let issued = log.last_issued(); + let meta = match Repo::open(meta_path) { + Ok(meta) => meta, + Err(error) => { + tracing::error!(%error, "the seq floor couldn't persist, the meta repo won't open"); + return; + } + }; + let persisted = read_seq(&meta).unwrap_or(UnixMicros::new(0)); + if issued <= persisted { + return; + } + let name = seq_ref(); + let blob = match meta.git().write_blob(issued.get().to_string().as_bytes()) { + Ok(blob) => blob.detach(), + Err(error) => { + tracing::warn!(%error, "the seq floor blob wouldn't write"); + return; + } + }; + let update = match meta.find_ref(&name) { + Ok(Some(old)) => RefUpdate::Update { + name, + old, + new: knot_types::Oid::from(blob), + }, + Ok(None) => RefUpdate::Create { + name, + new: knot_types::Oid::from(blob), + }, + Err(error) => { + tracing::warn!(%error, "the seq floor ref wouldn't read"); + return; + } + }; + if let Err(error) = meta.update_ref(&update) { + tracing::warn!(%error, "the seq floor wouldn't advance"); + } +} + +fn read_seq(meta: &Repo) -> Result { + let oid = meta + .find_ref(&seq_ref()) + .map_err(|error| error.to_string())? + .ok_or("no seq ref yet")?; + let object = meta + .git() + .find_object(oid.object_id()) + .map_err(|error| error.to_string())?; + std::str::from_utf8(&object.detach().data) + .map_err(|error| error.to_string())? + .trim() + .parse::() + .map(UnixMicros::new) + .map_err(|error| error.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::types::tid::Ticker; + use knot_runtime::ManualClock; + + const DID: &str = "did:web:knot.example"; + + #[derive(serde::Deserialize)] + struct OpProbe { + action: String, + path: String, + cid: Option, + } + + #[derive(serde::Deserialize)] + struct CommitProbe { + seq: u64, + rebase: bool, + #[serde(rename = "tooBig")] + too_big: bool, + repo: String, + commit: Cid, + rev: String, + ops: Vec, + } + + fn log() -> EventLog { + EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), + ReplayBounds::new( + ReplayEvents::new(64).unwrap(), + ReplayBytes::new(16 << 20).unwrap(), + ), + ) + } + + type Rig = ( + tempfile::TempDir, + knot_git::Layout, + knot_types::RepoDid, + Repo, + knot_runtime::K256Signer, + ); + + fn rig() -> Rig { + let dir = tempfile::tempdir().unwrap(); + let layout = knot_git::Layout::new(dir.path()); + let did = knot_types::RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + let signer = knot_runtime::K256Signer::from_slice(&[9u8; 32]).unwrap(); + (dir, layout, did, repo, signer) + } + + const COLLECTION: &str = "org.tangled.test.record"; + + fn collection() -> knot_types::RecordCollection { + knot_types::RecordCollection::new(COLLECTION).unwrap() + } + + #[derive(serde::Serialize)] + struct Fixture { + #[serde(rename = "$type")] + record_type: &'static str, + #[serde(rename = "createdAt")] + created_at: i64, + } + + fn records(count: usize) -> BTreeMap { + (0..count) + .map(|n| { + ( + knot_types::MstKey::new(format!("{COLLECTION}/record{n}")).unwrap(), + knot_types::RecordBody::new( + serde_ipld_dagcbor::to_vec(&Fixture { + record_type: COLLECTION, + created_at: n as i64, + }) + .unwrap(), + ) + .unwrap(), + ) + }) + .collect() + } + + fn did() -> knot_types::RepoDid { + knot_types::RepoDid::new(DID).unwrap() + } + + fn header_t(frame: &[u8]) -> String { + let mut read = frame; + let header: serde_json::Value = + serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + header["t"].as_str().unwrap().to_owned() + } + + type Pump = std::pin::Pin>; + + async fn drained>>( + pumped: &mut Pump, + millis: u64, + ) -> Vec { + let deadline = tokio::time::Instant::now() + Duration::from_millis(millis); + let mut seen = Vec::new(); + while let Ok(batch) = tokio::time::timeout_at(deadline, pumped.as_mut().next()).await { + match batch { + Some(batch) => seen.extend(batch), + None => break, + } + } + seen + } + + async fn next_binary>>(pumped: &mut Pump) -> Bytes { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + for message in pumped + .as_mut() + .next() + .await + .expect("the pump keeps serving") + { + if let Message::Binary(frame) = message { + return frame; + } + } + } + }) + .await + .expect("a frame arrives within the timeout") + } + + fn binary_quiet(seen: &[Message]) -> bool { + seen.iter() + .all(|message| !matches!(message, Message::Binary(_))) + } + + #[test] + fn an_emitted_chunk_frames_with_the_rev_itself_as_the_seq() { + let (_dir, _layout, _did, repo, signer) = rig(); + let log = log(); + let did = did(); + let mut emit = FirehoseEmit::new(&log, &did, UnixSeconds::new(1_000)); + let chunks = futures::executor::block_on(chain::materialize( + &repo, + &did, + &chain::in_tree(records(1)), + &signer, + &mut emit, + UnixSeconds::new(1), + )) + .unwrap(); + emit.finish().unwrap(); + let [event] = &log.replay(EventCursor::START, drain_bounds()).events[..] else { + panic!("the emitted chunk reaches the ring"); + }; + let mut read = &event.frame[..]; + let header: serde_json::Value = + serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + assert_eq!(header["op"], 1); + assert_eq!(header["t"], "#commit"); + let payload: CommitProbe = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + assert_eq!(payload.repo, DID); + assert_eq!( + payload.seq, + chunks[0].rev.timestamp(), + "the seq is the microsecond field of the commit's own rev" + ); + assert_eq!( + payload.seq, + event.created.micros().get(), + "the ring cursor and the seq agree" + ); + assert_eq!(payload.rev, chunks[0].rev.to_string()); + assert_eq!( + payload.commit, + chain::commit_cid(&chunks[0].commit).unwrap() + ); + assert_eq!(payload.ops[0].action, "create"); + assert_eq!(payload.ops[0].path, format!("{COLLECTION}/record0")); + assert_eq!(payload.ops[0].cid, chunks[0].ops[0].cid()); + assert!(!payload.rebase && !payload.too_big); + let tip = chain::tip(&repo).unwrap().unwrap(); + assert_eq!( + chain::frame_blob(&repo, tip.git).unwrap().as_deref(), + Some(&event.frame[..]), + "the chain stores the same bytes the ring serves" + ); + } + + #[test] + fn unframeable_chunk_is_skipped_and_cursor_stands() { + let log = log(); + let did = did(); + let mut emit = FirehoseEmit::new(&log, &did, UnixSeconds::new(1_000)); + let broken = CommitChunk { + commit: vec![0xff, 0xff, 0xff, 0xff], + rev: Emit::rev(&mut emit, None), + prev_rev: None, + data: knot_record::cid_of(b"x"), + prev_data: None, + ops: Vec::new(), + blocks: BTreeMap::new(), + }; + assert!( + Emit::frame(&emit, &broken).is_none(), + "Emit::frame answers None for a chunk missing its cid" + ); + emit.finish().unwrap(); + assert!( + log.replay(EventCursor::START, drain_bounds()) + .events + .is_empty(), + "an unframeable chunk publishes nothing" + ); + assert_eq!( + *log.subscribe().borrow(), + EventCursor::START, + "a skipped frame advances nothing" + ); + } + + #[tokio::test] + async fn the_seq_climbs_back_above_the_chains_after_a_hard_kill() { + let (_dir, layout, did, repo, signer) = rig(); + let first = log(); + { + let mut emit = FirehoseEmit::new(&first, &did, UnixSeconds::new(1)); + let chunks = chain::materialize( + &repo, + &did, + &chain::in_tree(records(1)), + &signer, + &mut emit, + UnixSeconds::new(1), + ) + .await + .unwrap(); + emit.finish().unwrap(); + publish_account(&first, &did, false, Some("deleted"), UnixSeconds::new(1)); + persist_seq(&first, repo.path()); + assert!( + chunks[0].rev.timestamp() < first.last_issued().get(), + "a non-commit event sits above the commit seqs" + ); + } + drop(first); + let floor = replay_floor(&repo, &layout, std::slice::from_ref(&did)); + assert!( + floor.get() >= chain::tip(&repo).unwrap().unwrap().rev.timestamp(), + "the floor covers every rev the chains hold" + ); + let stepped_back = EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000 - 10_000_000)), + drain_bounds(), + ) + .with_boot_floor(floor); + let persisted = stepped_back.last_issued(); + assert!( + !future_cursor(&stepped_back, persisted), + "a cursor at the persisted high-water mark is not the future" + ); + assert!( + future_cursor(&stepped_back, persisted.next()), + "one past the high-water mark is the future" + ); + let mut emit = FirehoseEmit::new(&stepped_back, &did, UnixSeconds::new(2)); + chain::sync( + &repo, + &did, + std::slice::from_ref(&collection()), + &chain::in_tree(records(2)), + &signer, + &mut emit, + UnixSeconds::new(2), + ) + .await + .unwrap(); + emit.finish().unwrap(); + let replayed = stepped_back.replay(EventCursor::START, drain_bounds()); + let last = replayed + .events + .last() + .expect("the changed records published"); + assert!( + last.created.micros().get() > floor.get(), + "no seq is ever reissued, not even under a backward clock" + ); + } + + #[tokio::test] + async fn a_pending_reservation_gates_chain_frames_behind_it() { + let (_dir, layout, did, repo, signer) = rig(); + let did = &did; + let log = Arc::new(log()); + let beneath = log.reserve(); + let reserved = std::thread::scope(|scope| { + let log = log.as_ref(); + scope + .spawn(move || { + let mut emit = FirehoseEmit::new(log, did, UnixSeconds::new(1)); + futures::executor::block_on(chain::materialize( + &repo, + did, + &chain::in_tree(records(1)), + &signer, + &mut emit, + UnixSeconds::new(1), + )) + .unwrap(); + std::mem::take(&mut emit.reserved) + }) + .join() + .unwrap() + }); + let replay = replay_index( + Path::new("/nonexistent-meta"), + &layout, + std::slice::from_ref(did), + UnixMicros::new(0), + ); + let mut pumped = Box::pin(pump( + Arc::clone(&log), + UnixMicros::new(0), + replay, + log.subscribe(), + )); + assert!( + binary_quiet(&drained(&mut pumped, 300).await), + "no frame crosses the horizon while a reservation beneath it is pending" + ); + fulfill_identity(beneath, did, UnixSeconds::new(1)); + assert_eq!(header_t(&next_binary(&mut pumped).await), "#identity"); + assert!( + binary_quiet(&drained(&mut pumped, 300).await), + "the commit stays behind its own pending reservation" + ); + drop(reserved); + assert_eq!(header_t(&next_binary(&mut pumped).await), "#commit"); + } + + #[tokio::test] + async fn a_lost_stored_frame_ends_the_replay_without_serving_its_seq() { + let (_dir, _layout, _did, repo, signer) = rig(); + let path = repo.path().to_path_buf(); + tokio::task::spawn_blocking(move || { + futures::executor::block_on(chain::materialize( + &Repo::open(&path).unwrap(), + &did(), + &chain::in_tree(records(1)), + &signer, + &mut Ticker::new(), + UnixSeconds::new(1), + )) + .unwrap(); + }) + .await + .unwrap(); + let tip = chain::tip(&repo).unwrap().unwrap(); + assert!( + chain::unframed_tip(&repo).unwrap().is_some(), + "the ticker write leaves an unframed tip" + ); + let mut feed = ReplayFeed { + paths: vec![repo.path().to_path_buf()], + index: VecDeque::from([IndexEntry::Chain(FrameAt { + micros: UnixMicros::new(tip.rev.timestamp()), + repo: 0, + oid: tip.git, + })]), + buffer: VecDeque::new(), + capped: false, + poisoned: false, + }; + feed.fetch().await; + assert!(feed.poisoned, "an unreadable frame poisons the feed"); + assert!( + feed.pop().is_none(), + "no frame answers at a seq whose stored commit wouldn't read" + ); + assert!( + !feed.index.is_empty(), + "the unread frame stays queued for the consumer's next attempt" + ); + } + + #[test] + fn push_events_verify_against_the_signing_key_over_the_sigless_encoding() { + let log = log(); + let signer = knot_runtime::K256Signer::from_slice(&[7u8; 32]).unwrap(); + let did = RepoDid::new("did:plc:squid").unwrap(); + let pusher = AccountDid::new("did:plc:octopus").unwrap(); + let applied = [ + RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: Oid::from_hex("1111111111111111111111111111111111111111").unwrap(), + }, + RefUpdate::Delete { + name: RefName::new("refs/heads/gone").unwrap(), + old: Oid::from_hex("2222222222222222222222222222222222222222").unwrap(), + }, + ]; + let options = PushOptions::new([PushOption::new("verbose-ci").unwrap()]); + publish_push( + &log, + &did, + &pusher, + &applied, + &options, + &signer, + UnixSeconds::new(1_700_000_000), + ); + + let [event] = &log.replay(EventCursor::START, drain_bounds()).events[..] else { + panic!("the push event reaches the ring"); + }; + let mut read = &event.frame[..]; + let header: Header = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + assert_eq!(header.t, FrameKind::Push); + let ipld_core::ipld::Ipld::Map(mut payload) = + serde_ipld_dagcbor::from_slice::(read).unwrap() + else { + panic!("a push event payload is a map"); + }; + let Some(ipld_core::ipld::Ipld::Bytes(sig)) = payload.remove("sig") else { + panic!("a push event carries sig bytes"); + }; + assert_eq!(sig.len(), 64, "secp256k1 signatures are 64 compact bytes"); + assert!( + matches!( + payload.remove("seq"), + Some(ipld_core::ipld::Ipld::Integer(_)) + ), + "a push event still carries seq on the wire, just outside the signature" + ); + assert!( + payload.contains_key("options"), + "the signed bytes cover the push options" + ); + + let unsigned = serde_ipld_dagcbor::to_vec(&ipld_core::ipld::Ipld::Map(payload)).unwrap(); + assert!( + knot_runtime::verify( + &signer.public_key(), + &unsigned, + &knot_runtime::Signature::from_bytes(sig), + ), + "sig must cover the dag-cbor of this very message with the seq and sig keys \ + absent, which is what a consumer reconstructs" + ); + } + + #[test] + fn push_event_signatures_survive_a_relay_reassigning_seq() { + let log = log(); + let signer = knot_runtime::K256Signer::from_slice(&[7u8; 32]).unwrap(); + publish_push( + &log, + &RepoDid::new("did:plc:squid").unwrap(), + &AccountDid::new("did:plc:octopus").unwrap(), + &[RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: Oid::from_hex("1111111111111111111111111111111111111111").unwrap(), + }], + &PushOptions::new([]), + &signer, + UnixSeconds::new(1_700_000_000), + ); + + let [event] = &log.replay(EventCursor::START, drain_bounds()).events[..] else { + panic!("the push event reaches the ring"); + }; + let mut read = &event.frame[..]; + let _: Header = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + let ipld_core::ipld::Ipld::Map(mut payload) = + serde_ipld_dagcbor::from_slice::(read).unwrap() + else { + panic!("a push event payload is a map"); + }; + + // what a relay does to a frame before passing it on + payload.insert("seq".to_owned(), ipld_core::ipld::Ipld::Integer(9_001)); + + let Some(ipld_core::ipld::Ipld::Bytes(sig)) = payload.remove("sig") else { + panic!("a push event carries sig bytes"); + }; + payload.remove("seq"); + assert!( + knot_runtime::verify( + &signer.public_key(), + &serde_ipld_dagcbor::to_vec(&ipld_core::ipld::Ipld::Map(payload)).unwrap(), + &knot_runtime::Signature::from_bytes(sig), + ), + "a re-sequenced frame must still verify, or the event cannot cross a relay" + ); + } +} diff --git a/knot2/crates/knot-xrpc/src/forks.rs b/knot2/crates/knot-xrpc/src/forks.rs new file mode 100644 index 0000000000000000000000000000000000000000..39af8d8c6e4a7eb3b4b5fe5b49637a1520a80fb3 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/forks.rs @@ -0,0 +1,365 @@ +use std::path::Path; + +use serde::{Deserialize, Serialize}; +use url::Url; + +use knot_git::{Filter, Haves, RefUpdate, Repo, Wants}; +use knot_pack::{FetchError, HaveOids, MaxWireBytes, UpstreamRefs, WantOids}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{ObjectFormat, Oid, OriginUrl, OwnerDid, RefName, RepoDid, RepoName, RepoRkey}; + +use crate::body::SourceUrl; +use crate::error::XrpcError; +use crate::query::RepoArg; +use crate::reads::{HostedRepo, repo_not_found, require_hosted, resolve_repo}; +use crate::{XrpcState, run_blocking}; + +#[derive(Clone)] +pub(crate) enum Upstream { + Local(HostedRepo), + Remote(Url), +} + +pub(crate) enum LocalPath { + Did(RepoDid), + Named { owner: OwnerDid, name: RepoName }, +} + +impl LocalPath { + fn from_segments(segments: &[&str]) -> Result { + match segments { + [did] => RepoDid::new(*did) + .map(Self::Did) + .map_err(|_| "path isn't a DID"), + [owner, name] => OwnerDid::new(*owner) + .map_err(|_| "owner segment isn't a DID") + .and_then(|owner| Self::named(owner, name)), + _ => Err("path must be /did or /owner/name"), + } + } + + fn named(owner: OwnerDid, name: &str) -> Result { + let name = name.strip_suffix(".git").unwrap_or(name); + RepoName::new(name) + .map(|name| Self::Named { owner, name }) + .map_err(|_| "name segment isn't a valid repo name") + } +} + +pub(crate) fn resolve_by_name( + state: &XrpcState, + owner: &OwnerDid, + name: &RepoName, +) -> Result { + let rkey = RepoRkey::new(name.as_str()).map_err(|_| repo_not_found())?; + let owner = owner.clone(); + resolve_repo(state, &RepoArg::OwnerRkey { owner, rkey }) +} + +fn resolve_local( + state: &XrpcState, + path: &LocalPath, +) -> Result { + match path { + LocalPath::Did(did) => require_hosted(state, did.clone()), + LocalPath::Named { owner, name } => resolve_by_name(state, owner, name), + } +} + +pub(crate) fn resolve_upstream( + state: &XrpcState, + source: &SourceUrl, +) -> Result { + let url = source.as_url(); + let authority = state.knot_authority(); + let host = url.host_str().unwrap_or_default(); + let same_knot = match url.port() { + None => authority == host, + Some(port) => authority.strip_prefix(host).is_some_and(|rest| { + rest.strip_prefix(':') + .is_some_and(|digits| digits.parse::().is_ok_and(|parsed| parsed == port)) + }), + }; + if same_knot { + let segments: Vec<&str> = url + .path_segments() + .map(|segments| segments.filter(|segment| !segment.is_empty()).collect()) + .unwrap_or_default(); + return LocalPath::from_segments(&segments) + .map_err(|reason| XrpcError::invalid_request(format!("fork source {reason}"))) + .and_then(|path| resolve_local(state, &path)) + .map(Upstream::Local); + } + Ok(Upstream::Remote(url.clone())) +} + +pub(crate) struct ForkSource { + pub(crate) origin: SourceUrl, + pub(crate) upstream: Upstream, + pub(crate) refs: UpstreamRefs, +} + +impl ForkSource { + pub(crate) async fn resolve( + state: &XrpcState, + origin: &SourceUrl, + ) -> Result { + let upstream = resolve_upstream(state, origin)?; + let prefixes = ["HEAD", "refs/heads/", "refs/tags/"] + .iter() + .map(|prefix| prefix.to_string()) + .collect(); + let refs = upstream_refs(state, &upstream, prefixes).await?; + Ok(Self { + origin: origin.clone(), + upstream, + refs, + }) + } +} + +pub(crate) fn map_fetch(error: FetchError) -> XrpcError { + match error { + FetchError::PackTooLarge { limit } => XrpcError::request_too_large(format!( + "upstream exceeded this knot's limit of {} bytes", + limit.get() + )), + FetchError::Pack(inner) => { + XrpcError::bad_gateway(format!("upstream sent an unusable pack: {inner}")) + } + other => XrpcError::bad_gateway(other.to_string()), + } +} + +fn pack_too_large(byte_limit: MaxWireBytes) -> XrpcError { + XrpcError::request_too_large(format!( + "upstream pack exceeds this knot's limit of {} bytes", + byte_limit.get() + )) +} + +pub(crate) async fn upstream_refs( + state: &XrpcState, + upstream: &Upstream, + prefixes: Vec, +) -> Result { + match upstream { + Upstream::Local(did) => { + let layout = state.layout.clone(); + let did = did.clone(); + run_blocking(move || { + let repo = layout.open(&did)?; + let prefixes: Vec<&str> = prefixes.iter().map(String::as_str).collect(); + knot_pack::local_refs(&repo, &prefixes).map_err(XrpcError::from) + }) + .await + } + Upstream::Remote(url) => { + let prefixes: Vec<&str> = prefixes.iter().map(String::as_str).collect(); + knot_pack::remote_refs(state.git_http.as_ref(), url, &prefixes) + .await + .map_err(map_fetch) + } + } +} + +pub(crate) async fn fetch_pack_into( + state: &XrpcState, + upstream: &Upstream, + wants: WantOids, + haves: HaveOids, + // must be what the upstream advertised, or a sha256 source refuses the request + object_format: ObjectFormat, + byte_limit: MaxWireBytes, + pack_path: &Path, +) -> Result<(), XrpcError> { + match upstream { + Upstream::Local(did) => { + let layout = state.layout.clone(); + let did = did.clone(); + let pack_path = pack_path.to_path_buf(); + run_blocking(move || { + let repo = layout.open(&did)?; + let mut out = std::fs::File::create(&pack_path) + .map_err(|error| XrpcError::internal(error.to_string()))?; + knot_pack::local_pack_to(&repo, &wants, &haves, byte_limit, &mut out).map_err( + |error| match error { + FetchError::PackTooLarge { .. } => pack_too_large(byte_limit), + other => XrpcError::internal(other.to_string()), + }, + ) + }) + .await + } + Upstream::Remote(url) => { + let mut out = std::fs::File::create(pack_path) + .map_err(|error| XrpcError::internal(error.to_string()))?; + knot_pack::remote_pack_streamed( + state.git_http.as_ref(), + url, + &wants, + &haves, + object_format, + byte_limit, + &mut out, + ) + .await + .map_err(map_fetch) + } + } +} + +pub(crate) fn connected( + repo: &Repo, + wants: Wants<'_>, + haves: Haves<'_>, +) -> Result { + repo.select_pack_objects_filtered(wants, haves, Filter::None, knot_pack::selection_budget()) + .map(|selection| selection.send.iter().all(|oid| repo.contains(*oid))) + .map_err(XrpcError::from) +} + +pub(crate) fn ingest_pack( + repo: &Repo, + pack: &Path, + wants: &[Oid], + haves: &[Oid], +) -> Result<(), XrpcError> { + let staging = + knot_git::Staging::new(repo).map_err(|error| XrpcError::internal(error.to_string()))?; + knot_pack::ingest_pack_file( + &staging.repo().objects_dir(), + pack, + &knot_pack::PackLimits::default(), + repo.object_format().kind(), + ) + .map_err(|error| XrpcError::bad_gateway(format!("fork source pack is unusable: {error}")))?; + if !connected(staging.repo(), Wants::new(wants), Haves::new(haves))? { + return Err(XrpcError::bad_gateway( + "fork source sent an incomplete pack", + )); + } + staging + .migrate_into(repo) + .map_err(|error| XrpcError::internal(error.to_string())) +} + +pub(crate) async fn pull_objects( + state: &XrpcState, + upstream: &Upstream, + repo: &RepoDid, + wants: Vec, + haves: Vec, + object_format: ObjectFormat, +) -> Result<(), XrpcError> { + let scratch = state.layout.scratch_dir().to_path_buf(); + std::fs::create_dir_all(&scratch).map_err(|error| XrpcError::internal(error.to_string()))?; + let pack = tempfile::Builder::new() + .prefix("pull-pack.") + .tempfile_in(&scratch) + .map_err(|error| XrpcError::internal(error.to_string()))?; + fetch_pack_into( + state, + upstream, + WantOids::new(wants.clone()), + HaveOids::new(haves.clone()), + object_format, + MaxWireBytes::new(state.byte_limits.import_pack.get() as usize), + pack.path(), + ) + .await?; + let layout = state.layout.clone(); + let target = repo.clone(); + run_blocking(move || ingest_pack(&layout.open(&target)?, pack.path(), &wants, &haves)).await +} + +pub(crate) async fn fetch_commit( + state: &XrpcState, + repo: &RepoDid, + source: &RepoDid, + commit: Oid, +) -> Result<(), XrpcError> { + let origin = crate::repos::source_of( + state, + &knot_record::manifest::ManifestSource::Tangled { + did: source.clone(), + }, + ) + .await? + .url; + let upstream = resolve_upstream(state, &origin)?; + let layout = state.layout.clone(); + let target = repo.clone(); + let (haves, object_format) = run_blocking(move || { + let repo = layout.open(&target)?; + let haves: Vec = repo + .branches()? + .into_iter() + .map(|branch| branch.target) + .collect(); + Ok((haves, repo.object_format())) + }) + .await?; + pull_objects(state, &upstream, repo, vec![commit], haves, object_format).await +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub(crate) enum SourceKind { + Fork, + Import, +} + +pub(crate) fn apply_source_refs( + repo: &Repo, + refs: &UpstreamRefs, + origin: &SourceUrl, + kind: SourceKind, +) -> Result<(), XrpcError> { + let placed = repo.references()?; + let placed: std::collections::HashMap<&RefName, Oid> = placed + .iter() + .map(|have| (&have.name, have.target)) + .collect(); + let updates: Vec = refs + .refs + .iter() + .filter_map(|record| match placed.get(&record.name) { + None => Some(RefUpdate::Create { + name: record.name.clone(), + new: record.target, + }), + Some(&old) => (old != record.target).then(|| RefUpdate::Update { + name: record.name.clone(), + old, + new: record.target, + }), + }) + .collect(); + if !updates.is_empty() { + repo.update_refs(&updates)?; + } + let head = refs + .head_symref + .as_ref() + .filter(|head| head.as_str().starts_with("refs/heads/")) + .or_else(|| { + refs.head_target.and_then(|target| { + refs.refs + .iter() + .find(|record| { + record.name.as_str().starts_with("refs/heads/") && record.target == target + }) + .map(|record| &record.name) + }) + }); + if let Some(head) = head { + repo.set_head(head)?; + } + // the appview's fork links read the upstream, so an import must not set it + if kind == SourceKind::Fork { + repo.set_origin_url(&OriginUrl::new(origin.as_str())) + .map_err(XrpcError::from)?; + } + Ok(()) +} diff --git a/knot2/crates/knot-xrpc/src/foster.rs b/knot2/crates/knot-xrpc/src/foster.rs new file mode 100644 index 0000000000000000000000000000000000000000..75a2f69901684fe5a191ebb4338055a0b41e6c28 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/foster.rs @@ -0,0 +1,1534 @@ +use std::collections::BTreeMap; +use std::path::Path; +use std::str::FromStr; +use std::sync::Arc; + +use axum::Json; +use axum::Router; +use axum::body::Bytes; +use axum::extract::{DefaultBodyLimit, State}; +use axum::middleware::from_fn_with_state; +use axum::response::{IntoResponse, Response}; +use axum::routing::post; +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use cid::Cid; +use http::{HeaderMap, HeaderValue, StatusCode}; +use ipld_core::ipld::Ipld; +use serde::{Deserialize, Serialize}; + +use knot_capability::Capability; +use knot_cob::{ + ChangePayload, CobHome, CobStore, Emits, HistoryModel, KnotAuthored, Materialized, Record, +}; +use knot_cobs::{Registry, RepoRef, RepoRegistryCob, Retired, deregister_repo, retire_repo}; +use knot_git::Repo; +use knot_index::Resolved; +use knot_record::chain; +use knot_record::manifest::ManifestMeta; +use knot_runtime::{ + Clock, HttpLimits, HttpRequest, HttpTransport, PublicKeyBytes, ReqwestHttp, Signature, Signer, +}; +use knot_types::crypto::{KeyCodec, PublicKey as CryptoKey}; +use knot_types::{ + AccountDid, ActorId, CobId, ContributionPolicy, MstKey, Nsid, OwnerDid, RecordAddress, + RecordBody, RecordBodyError, RecordCid, RecordCollection, RecordRkey, RepoDid, RepoName, + RepoRkey, ServiceDid, SubjectNumber, UnixSeconds, +}; +use url::Url; + +use crate::XrpcState; +use crate::error::XrpcError; +use crate::handover::{get_json, knot_identity}; +use crate::query::ValidatedQuery; +use crate::{decode, enforce_pre_auth_limit, run_blocking}; + +pub const SEED_ROUTE: &str = "/xrpc/org.tangled.temp.server.seedRepo"; +pub const UPLOAD_BLOB_ROUTE: &str = "/xrpc/org.tangled.temp.server.uploadBlob"; +pub const ACK_ROUTE: &str = "/xrpc/org.tangled.temp.server.ackHandover"; +pub const REMINT_ROUTE: &str = "/xrpc/org.tangled.temp.server.reMintShadow"; + +pub(crate) const ACK_CONTEXT: &str = "org.tangled.temp.handover.ack.v1"; + +const MAX_RECORDS_PER_REQUEST: usize = 200; +const LIAR_UNMARK_ROUTE: &str = "/xrpc/org.tangled.foster.adminUnmarkLiarAccount"; +const LIAR_UNMARK_LXM: &str = "org.tangled.foster.adminUnmarkLiarAccount"; +const LIAR_UNMARK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +const LIAR_RELEASED: &str = "tangled.handover.unmarked"; + +#[derive(Clone, Debug)] +pub struct LiarHydrant { + url: Url, + http: ReqwestHttp, +} + +impl LiarHydrant { + pub fn new(raw: &str) -> Result { + let url = Url::parse(raw).map_err(|_| format!("liar hydrant url {raw} isn't a url"))?; + if !hydrant_is_trusted(&url) { + return Err(format!( + "liar hydrant url {raw} must be https, or http to a private address. a foster \ + only reports its retires over tls, or over a network it shares with the hydrant" + )); + } + let http = ReqwestHttp::new(HttpLimits { + block_private_addresses: false, + ..HttpLimits::default() + }) + .map_err(|error| format!("the liar hydrant client wouldn't build: {error}"))?; + Ok(Self { url, http }) + } + + fn unmark_url(&self) -> Url { + let mut url = self.url.clone(); + url.path_segments_mut() + .expect("an http url always has segments") + .extend(LIAR_UNMARK_ROUTE.trim_start_matches('/').split('/')); + url + } + + fn audience(&self) -> ServiceDid { + let host = self.url.host_str().unwrap_or_default(); + let port = self + .url + .port() + .map(|port| format!("%3A{port}")) + .unwrap_or_default(); + ServiceDid::new(format!("did:web:{host}{port}")) + .expect("a url host mints a did:web service did") + } +} + +fn hydrant_is_trusted(url: &Url) -> bool { + match url.scheme() { + "https" => true, + "http" => match url.host() { + Some(url::Host::Domain(name)) => name.eq_ignore_ascii_case("localhost"), + Some(url::Host::Ipv4(ip)) => ip.is_loopback() || ip.is_private(), + Some(url::Host::Ipv6(ip)) => ip.is_loopback() || ip.is_unique_local(), + None => false, + }, + _ => false, + } +} + +#[derive(Deserialize)] +struct SeedInput { + #[serde(rename = "repoDid")] + repo_did: RepoDid, + manifest: serde_json::Value, + records: Vec, + #[serde(default)] + deletes: Vec, + #[serde(default, rename = "objectFormat")] + object_format: Option, +} + +#[derive(Deserialize)] +struct SeedDelete { + collection: RecordCollection, + rkey: RecordRkey, +} + +#[derive(Deserialize)] +struct ManifestDeclaration { + declaration: Declared, + #[serde(rename = "contributionPolicy", default)] + contribution_policy: Option, +} + +#[derive(Deserialize)] +struct Declared { + owner: OwnerDid, + name: RepoName, +} + +#[derive(Deserialize)] +struct SeedRecord { + collection: RecordCollection, + rkey: RecordRkey, + author: AccountDid, + #[serde(rename = "createdAt")] + created_at: UnixSeconds, + #[serde(rename = "subjectNumber", default)] + subject_number: Option, + record: serde_json::Value, +} + +#[derive(Serialize)] +struct SeedOutput { + did: RepoDid, + key: ActorId, + results: Vec, +} + +#[derive(Serialize)] +struct RecordResult { + collection: RecordCollection, + rkey: RecordRkey, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +#[derive(Deserialize)] +struct AckInput { + context: String, + #[serde(rename = "repoDid")] + repo_did: RepoDid, + records: RecordCount, + #[serde(rename = "ackedAt")] + acked_at: UnixSeconds, + signature: String, + #[serde(default)] + superseded: Vec, +} + +#[derive(Serialize)] +pub(crate) struct AckSigned<'a> { + pub(crate) context: &'static str, + #[serde(rename = "repoDid")] + pub(crate) repo_did: &'a RepoDid, + pub(crate) records: RecordCount, + #[serde(rename = "ackedAt")] + pub(crate) acked_at: UnixSeconds, + #[serde(skip_serializing_if = "nothing_superseded")] + pub(crate) superseded: &'a [RecordAddress], +} + +fn nothing_superseded(superseded: &[RecordAddress]) -> bool { + superseded.is_empty() +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(transparent)] +pub struct RecordCount(usize); + +impl RecordCount { + pub(crate) const fn new(records: usize) -> Self { + Self(records) + } + + pub const fn get(self) -> usize { + self.0 + } +} + +struct CheckedRecord { + address: RecordAddress, + author: AccountDid, + created_at: UnixSeconds, + number: Option, + body: RecordBody, +} + +#[derive(Debug, thiserror::Error)] +enum SeedAdmission { + #[error("git-refs & manifest collections reserved for knot's own writes")] + Reserved, + #[error( + "createdAt {0} is before epoch or more than a whole-ass day ahead of the knot, what the helly" + )] + Stamp(UnixSeconds), + #[error("the record's $type isn't {0}, so it doesn't belong in that collection")] + TypeMismatch(String), + #[error("{0} never settles on an upgraded knot, so a shadow of it could never come home")] + Unsettleable(String), + #[error("the record doesn't decode as its collection's shape: {0}")] + Undecodable(String), + #[error("a string in the record contains a NUL byte, which no at:// record allows")] + Nul, + #[error("the ticket's title is blank, and a blank ticket can't come home as a real record")] + BlankTitle, + #[error(transparent)] + Body(#[from] RecordBodyError), +} + +impl CheckedRecord { + fn new(record: SeedRecord, now: UnixSeconds) -> Result { + if record.collection.as_str() == knot_record::git_refs::GIT_REFS_COLLECTION + || record.collection.as_str() == knot_record::manifest::MANIFEST_COLLECTION + { + return Err(SeedAdmission::Reserved); + } + if record.created_at < UnixSeconds::new(0) + || record.created_at > now.saturating_add_secs(86_400) + { + return Err(SeedAdmission::Stamp(record.created_at)); + } + if record.record["$type"].as_str() != Some(record.collection.as_str()) { + return Err(SeedAdmission::TypeMismatch( + record.collection.as_str().to_owned(), + )); + } + if contains_nul(&record.record) { + return Err(SeedAdmission::Nul); + } + if record.collection.as_str() == knot_record::ticket::TICKET_COLLECTION + && record.record["title"]["$type"].as_str() == Some(knot_record::ticket::INLINE_MARKUP) + && record.record["title"]["text"] + .as_str() + .is_some_and(|text| text.trim().is_empty()) + { + return Err(SeedAdmission::BlankTitle); + } + let body = RecordBody::from_cbor(&json_to_ipld(&record.record))?; + match knot_cobs::validate_stored(&record.collection, &body) { + None => { + return Err(SeedAdmission::Unsettleable( + record.collection.as_str().to_owned(), + )); + } + Some(Err(why)) => return Err(SeedAdmission::Undecodable(why)), + Some(Ok(())) => {} + } + Ok(Self { + address: RecordAddress::new(record.collection, record.rkey), + author: record.author, + created_at: record.created_at, + number: record.subject_number, + body, + }) + } +} + +const BASE64_STANDARD_INDIFFERENT: base64::engine::GeneralPurpose = + base64::engine::GeneralPurpose::new( + &base64::alphabet::STANDARD, + base64::engine::GeneralPurposeConfig::new() + .with_decode_padding_mode(base64::engine::DecodePaddingMode::Indifferent), + ); + +fn contains_nul(value: &serde_json::Value) -> bool { + match value { + serde_json::Value::String(text) => text.contains('\0'), + serde_json::Value::Array(items) => items.iter().any(contains_nul), + serde_json::Value::Object(fields) => fields.values().any(contains_nul), + _ => false, + } +} + +fn json_to_ipld(value: &serde_json::Value) -> Ipld { + use base64::Engine as _; + use serde_json::Value; + match value { + Value::Null => Ipld::Null, + Value::Bool(b) => Ipld::Bool(*b), + Value::Number(n) => { + if let Some(i) = n.as_i64() { + Ipld::Integer(i128::from(i)) + } else if let Some(f) = n.as_f64() { + Ipld::Float(f) + } else { + Ipld::Null + } + } + Value::String(s) => Ipld::String(s.clone()), + Value::Array(arr) => Ipld::List(arr.iter().map(json_to_ipld).collect()), + Value::Object(obj) => { + if let Some(Value::String(link)) = obj.get("$link") + && obj.len() == 1 + && let Ok(cid) = Cid::from_str(link) + { + return Ipld::Link(cid); + } + if let Some(Value::String(b64)) = obj.get("$bytes") + && obj.len() == 1 + && let Ok(bytes) = BASE64_STANDARD_INDIFFERENT.decode(b64) + { + return Ipld::Bytes(bytes); + } + Ipld::Map( + obj.iter() + .map(|(k, v)| (k.clone(), json_to_ipld(v))) + .collect(), + ) + } + } +} + +#[derive(Clone, Serialize, Deserialize)] +pub(crate) struct Seeded { + pub(crate) address: RecordAddress, + pub(crate) author: AccountDid, + pub(crate) produces: RecordCid, + pub(crate) created_at: UnixSeconds, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) number: Option, +} + +#[derive(Clone, Serialize, Deserialize)] +pub(crate) struct SeedErasure { + pub(crate) from: RecordCid, + pub(crate) erased_at: UnixSeconds, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub(crate) enum SeedChange { + Seed(Seeded), + Erase(SeedErasure), +} + +impl ChangePayload for SeedChange { + const TYPE: &'static str = "org.tangled.temp.seed"; + + fn address(&self) -> Option { + match self { + Self::Seed(seed) => Some(seed.address.clone()), + Self::Erase(_) => None, + } + } + + fn emits(&self) -> Emits { + match self { + Self::Seed(seed) => Emits::Version(seed.produces), + Self::Erase(_) => Emits::Tombstone, + } + } + + fn stamped_at(&self) -> Option { + match self { + Self::Seed(seed) => Some(seed.created_at), + Self::Erase(erase) => Some(erase.erased_at), + } + } +} + +impl KnotAuthored for SeedChange {} + +struct Foster { + state: Arc>, + skip_knot_service: bool, + skip_origin_checks: bool, + liar: Option, +} + +pub fn router( + state: Arc>, + skip_knot_service: bool, + skip_origin_checks: bool, + liar: Option, +) -> Router { + let limits = state.byte_limits.patch.get(); + let limiter = Arc::clone(&state); + Router::new() + .route(SEED_ROUTE, post(seed_repo::)) + .route(ACK_ROUTE, post(ack_handover::)) + .route(REMINT_ROUTE, post(remint_shadow::)) + .route( + UPLOAD_BLOB_ROUTE, + post(upload_blob::).layer(DefaultBodyLimit::max( + knot_record::attachment::MAX_UPLOAD_BYTES as usize + 1024, + )), + ) + .layer(DefaultBodyLimit::max(limits)) + .layer(from_fn_with_state(limiter, enforce_pre_auth_limit::)) + .with_state(Arc::new(Foster { + state, + skip_knot_service, + skip_origin_checks, + liar, + })) +} + +async fn seed_repo( + State(foster): State>>, + headers: HeaderMap, + body: Bytes, +) -> Result { + let state = &foster.state; + state.legacy_admin_authorize(&headers)?; + let input: SeedInput = decode(&body)?; + if input.records.len() > MAX_RECORDS_PER_REQUEST { + return Err(XrpcError::invalid_request(format!( + "Too many records (max {MAX_RECORDS_PER_REQUEST})" + ))); + } + let SeedInput { + repo_did, + mut manifest, + records, + deletes, + object_format, + } = input; + if let serde_json::Value::Object(fields) = &mut manifest { + fields.insert( + "contributionPolicy".to_owned(), + serde_json::Value::String(ContributionPolicy::Anyone.name().to_owned()), + ); + } + let declared = + serde_json::from_value::(manifest.clone()).map_err(|error| { + XrpcError::invalid_request(format!( + "the manifest's declaration names no owner and name: {error}" + )) + })?; + let Declared { owner, name } = declared.declaration; + let policy = declared + .contribution_policy + .and_then(|policy| serde_json::from_value(policy).ok()); + let manifest = RecordBody::from_cbor(&json_to_ipld(&manifest)) + .map_err(|error| XrpcError::invalid_request(error.to_string()))?; + let rkey = RepoRkey::from_repo_name(&name) + .map_err(|error| XrpcError::invalid_request(error.to_string()))?; + if let Some(record) = read_registry_record(state, &repo_did).await? + && record.retired_at.is_some() + { + return Err(XrpcError::conflict( + "the repo's shadow was retired by a handover ack, so it no longer accepts seeds", + )); + } + let (repo_did, key) = match hosted(state, &owner, &rkey)? { + Some(existing) => { + if existing != repo_did { + return Err(XrpcError::conflict( + "that owner and name is already fostered under a different repo DID", + )); + } + let actor = AccountDid::from(owner.clone()); + match crate::social::replace_manifest( + state, + actor.clone(), + existing.clone(), + manifest.clone(), + state.now(), + ) + .await + { + // an interrupted seed can register the repo without ever opening its manifest + Err(error) if error.is_not_found() => { + crate::social::open_manifest( + state, + actor, + existing.clone(), + manifest, + state.now(), + ) + .await? + } + other => other?, + } + let public = state.secrets.public_key(&state.knot_did)?; + (existing, ActorId::from_secp256k1(public.as_bytes())) + } + None => { + foster_repo( + state, + owner, + name, + rkey, + repo_did, + manifest, + policy, + object_format, + ) + .await? + } + }; + let results = commit_seeded(state, &repo_did, records, deletes).await?; + Ok(( + http::StatusCode::OK, + Json(SeedOutput { + did: repo_did, + key, + results, + }), + ) + .into_response()) +} + +#[derive(Deserialize)] +struct UploadBlobParams { + #[serde(rename = "repoDid")] + repo_did: RepoDid, +} + +async fn upload_blob( + State(foster): State>>, + headers: HeaderMap, + ValidatedQuery(params): ValidatedQuery, + body: Bytes, +) -> Result { + let state = &foster.state; + state.legacy_admin_authorize(&headers)?; + match state.index.rkey_of(¶ms.repo_did) { + Resolved::Ready(Some(_)) => {} + Resolved::Ready(None) => { + return Err(XrpcError::not_found("No such repository on this knot")); + } + Resolved::Warming => return Err(warming()), + } + crate::blobs::store_upload(state, params.repo_did, &headers, body).await +} + +async fn ack_handover( + State(foster): State>>, + body: Bytes, +) -> Result { + let state = &foster.state; + let input: AckInput = decode(&body)?; + if input.context != ACK_CONTEXT { + return Err(XrpcError::invalid_request( + "the ack context isn't the handover ack context", + )); + } + if input.acked_at.get() < 0 { + return Err(XrpcError::invalid_request( + "the ack's ackedAt predates the epoch", + )); + } + + let Some(record) = read_registry_record(state, &input.repo_did) + .await? + .filter(|record| record.fostered) + else { + return Err(XrpcError::not_found("the repo isn't fostered on this knot")); + }; + if let Some(retired_at) = record.retired_at + && retired_at >= input.acked_at + { + return Ok((StatusCode::OK, Json(serde_json::json!({}))).into_response()); + } + + let origin = if foster.skip_origin_checks { + tracing::warn!( + repo = %input.repo_did, + "the foster_ack_skip_origin_checks knob is set, so the ack retires the shadow without the origin checks" + ); + None + } else { + origin_knot_url(state, &input.repo_did, foster.skip_knot_service).await? + }; + if let Some(origin) = origin { + let key = origin_key(state, &origin).await?; + let signed = AckSigned { + context: ACK_CONTEXT, + repo_did: &input.repo_did, + records: input.records, + acked_at: input.acked_at, + superseded: &input.superseded, + }; + let bytes = serde_ipld_dagcbor::to_vec(&signed) + .map_err(|error| XrpcError::internal(error.to_string()))?; + let signature = Signature::from_bytes( + URL_SAFE_NO_PAD + .decode(&input.signature) + .map_err(|_| XrpcError::invalid_request("the ack signature isn't base64url"))?, + ); + if !knot_runtime::verify(&key, &bytes, &signature) { + return Err(XrpcError::forbidden( + "the ack signature doesn't verify against the origin knot's key", + )); + } + spot_check_record(state, &origin, &input.repo_did, &input.superseded).await?; + } + retire(state, &input.repo_did, input.acked_at).await?; + if let Some(liar) = &foster.liar + && !liar_released(state, liar, &input.repo_did).await + { + tracing::warn!( + repo = %input.repo_did, + "the ack retired the shadow but the liar hydrant wouldn't take the unmark, so the retire GC retries it" + ); + } + Ok((StatusCode::OK, Json(serde_json::json!({}))).into_response()) +} + +async fn origin_knot_url( + state: &XrpcState, + repo_did: &RepoDid, + skip_missing_knot_service: bool, +) -> Result, XrpcError> { + let url = match state.atproto.resolve_knot_endpoint(repo_did).await { + Ok(url) => url, + Err(error) + if skip_missing_knot_service + && matches!( + &error, + knot_atproto::AtprotoError::Resolve(knot_atproto::ResolveError::MissingKnot) + ) => + { + tracing::warn!( + repo = %repo_did, + "the repo's did document declares no tangled_knot service endpoint, so the ack retires the shadow without the origin checks" + ); + return Ok(None); + } + Err(error) => { + return Err(match error { + knot_atproto::AtprotoError::Resolve(_) => { + XrpcError::invalid_request(error.to_string()) + } + other => other.into(), + }); + } + }; + if !crate::handover::url_is_trusted(&url) { + return Err(XrpcError::invalid_request(format!( + "the repo's knot endpoint {} must be https, or http on loopback", + url.as_str() + ))); + } + let mut origin = url; + origin.set_path("/"); + origin.set_query(None); + origin.set_fragment(None); + Ok(Some(origin)) +} + +async fn origin_key( + state: &XrpcState, + origin: &Url, +) -> Result { + let multibase = knot_identity( + state, + origin, + "origin knot", + &Capability::SocialWrites, + "doesn't serve COBs, so it can't be the origin of a retired shadow", + ) + .await?; + let key = CryptoKey::decode_owned(multibase) + .map_err(|_| XrpcError::bad_gateway("the origin knot's multikey doesn't parse as a key"))?; + if key.codec != KeyCodec::Secp256k1 { + return Err(XrpcError::bad_gateway( + "the origin knot's #atproto key isn't secp256k1", + )); + } + Ok(PublicKeyBytes::from_bytes(key.bytes.as_ref().to_vec())) +} + +async fn spot_check_record( + state: &XrpcState, + origin: &Url, + repo: &RepoDid, + superseded: &[RecordAddress], +) -> Result<(), XrpcError> { + let layout = state.layout.clone(); + let repo = repo.clone(); + let versions = run_blocking({ + let repo = repo.clone(); + move || { + let git = layout.open(&repo)?; + let store = CobStore::new(&git); + seeded_versions(&store) + } + }) + .await?; + let Some(version) = versions + .iter() + .find(|version| !superseded.contains(&version.address)) + else { + return Ok(()); + }; + let mut url = origin_url(origin, "xrpc/com.atproto.repo.getRecord")?; + url.query_pairs_mut() + .append_pair("repo", repo.as_str()) + .append_pair("collection", version.address.collection().as_str()) + .append_pair("rkey", version.address.rkey().as_str()); + let served = get_json(state.git_http.as_ref(), url, "origin knot").await?; + let cid = served["cid"] + .as_str() + .and_then(RecordCid::parse) + .ok_or_else(|| { + XrpcError::bad_gateway("the origin knot served getRecord without a cid it can name") + })?; + if cid != version.produces { + return Err(XrpcError::bad_gateway(format!( + "the origin knot serves {} at {cid} while the shadow recorded {}", + version.address.at_uri(&repo), + version.produces.cid() + ))); + } + Ok(()) +} + +async fn retire( + state: &Arc>, + repo: &RepoDid, + at: UnixSeconds, +) -> Result<(), XrpcError> { + let signer = state.secrets.signer(&state.knot_did)?; + let meta_path = state.meta_path.clone(); + let cob_locks = Arc::clone(&state.cob_locks); + let knot_did = state.knot_did.clone(); + let repo = repo.clone(); + run_blocking(move || { + let _guard = cob_locks.meta(); + let home = CobHome::from(&knot_did); + with_registry(&meta_path, |store, object, registry| { + let record = registry + .record_of(&repo) + .ok_or_else(|| XrpcError::not_found("the repo isn't registered on this knot"))?; + let change = Retired { + owner: record.owner.clone(), + rkey: record.rkey.clone(), + repo: repo.clone(), + at, + }; + retire_repo(store, &home, object, change, &signer, at) + .map(|_| ()) + .map_err(XrpcError::from) + })? + .ok_or_else(|| XrpcError::not_found("no git repos registered on this knot")) + }) + .await +} + +async fn read_registry_record( + state: &XrpcState, + repo: &RepoDid, +) -> Result, XrpcError> { + let meta_path = state.meta_path.clone(); + let repo = repo.clone(); + run_blocking(move || { + with_registry(&meta_path, |_, _, registry| { + Ok(registry.record_of(&repo).cloned()) + }) + .map(Option::flatten) + }) + .await +} + +fn with_registry( + meta_path: &Path, + f: impl FnOnce(&CobStore, CobId, &Registry) -> Result, +) -> Result, XrpcError> { + let meta = Repo::open(meta_path)?; + let store = CobStore::new(&meta); + let Some(object) = crate::repos::registry_object(&store)? else { + return Ok(None); + }; + let (registry, _) = store.materialize::(object)?; + f(&store, object, ®istry).map(Some) +} + +pub(crate) fn origin_url(base: &Url, path: &str) -> Result { + let mut url = base.clone(); + url.path_segments_mut() + .map_err(|_| XrpcError::bad_gateway("the origin knot's endpoint can't take a path"))? + .extend(path.split('/')); + Ok(url) +} + +pub(crate) fn origin_root(base: &Url, path: &str) -> Url { + let mut url = base.clone(); + url.set_path(path); + url.set_query(None); + url.set_fragment(None); + url +} + +fn hosted( + state: &XrpcState, + owner: &OwnerDid, + rkey: &RepoRkey, +) -> Result, XrpcError> { + match state.index.resolve_repo(owner, rkey) { + Resolved::Ready(Some(existing)) => match state.index.rkey_of(&existing) { + Resolved::Ready(Some(canonical)) => Ok((canonical == *rkey).then_some(existing)), + Resolved::Ready(None) => Ok(None), + Resolved::Warming => Err(warming()), + }, + Resolved::Warming => Err(warming()), + Resolved::Ready(None) => Ok(None), + } +} + +fn warming() -> XrpcError { + XrpcError::warming( + "registry projection is still getting warmed by lewis' comforting hands, please retry shortly", + ) +} + +#[allow(clippy::too_many_arguments)] +async fn foster_repo( + state: &Arc>, + owner: OwnerDid, + name: RepoName, + rkey: RepoRkey, + repo_did: RepoDid, + manifest: RecordBody, + policy: Option, + object_format: Option, +) -> Result<(RepoDid, ActorId), XrpcError> { + let meta = ManifestMeta { + contribution_policy: Some(policy.unwrap_or(state.contribution_policy)), + spindle: None, + description: None, + website: None, + topics: None, + source: None, + }; + crate::repos::seed_repo( + state, + AccountDid::from(owner.clone()), + owner, + name, + meta, + rkey, + repo_did, + crate::repos::Seeding::Fostered { + manifest, + object_format, + }, + None, + None, + state.now(), + ) + .await +} + +async fn commit_seeded( + state: &Arc>, + repo_did: &RepoDid, + records: Vec, + deletes: Vec, +) -> Result, XrpcError> { + let now = state.now(); + let checked = records + .into_iter() + .map(|record| { + let collection = record.collection.clone(); + let rkey = record.rkey.clone(); + let outcome = CheckedRecord::new(record, now); + (collection, rkey, outcome) + }) + .collect(); + seed_checked(state, repo_did, checked, deletes).await +} + +async fn seed_checked( + state: &Arc>, + repo_did: &RepoDid, + checked: Vec<( + RecordCollection, + RecordRkey, + Result, + )>, + deletes: Vec, +) -> Result, XrpcError> { + let signer = state.secrets.signer(&state.knot_did)?; + let state = Arc::clone(state); + let repo = repo_did.clone(); + run_blocking(move || { + let _guard = state.cob_locks.repo(&repo); + let _guard = state.index.guard_social(&repo)?; + let git = state.layout.open(&repo)?; + let store = CobStore::new(&git); + let home = CobHome::from(&repo); + let mut standing = futures::executor::block_on(chain::leaves(&git))?; + let mut objects = fostered_objects(&store)?; + let mut results = checked + .into_iter() + .map(|(collection, rkey, outcome)| { + let outcome = outcome + .map_err(|admission| XrpcError::invalid_request(admission.to_string())) + .and_then(|checked| { + seed_one( + &state, + &git, + &store, + &home, + &repo, + &signer, + &mut standing, + &mut objects, + &checked, + ) + }); + RecordResult { + collection, + rkey, + error: outcome.err().map(|error| error.to_string()), + } + }) + .collect::>(); + results.extend(erase_records( + &state, &git, &store, &home, &repo, &signer, deletes, + )?); + Ok(results) + }) + .await +} + +fn erase_records( + state: &XrpcState, + git: &Repo, + store: &CobStore, + home: &CobHome, + repo: &RepoDid, + signer: &dyn Signer, + deletes: Vec, +) -> Result, XrpcError> { + if deletes.is_empty() { + return Ok(Vec::new()); + } + let standing = futures::executor::block_on(chain::leaves(git))?; + let objects = fostered_objects(store)?; + let now = state.now(); + Ok(deletes + .into_iter() + .map(|delete| { + let collection = delete.collection.clone(); + let address = RecordAddress::new(delete.collection, delete.rkey.clone()); + let outcome = erase_one( + state, git, store, home, repo, signer, &standing, &objects, &address, now, + ); + RecordResult { + collection, + rkey: delete.rkey, + error: outcome.err().map(|error| error.to_string()), + } + }) + .collect()) +} + +pub(crate) fn fostered_objects( + store: &CobStore, +) -> Result, XrpcError> { + let mut objects = BTreeMap::new(); + for (type_name, object) in store.cobs()? { + if type_name.as_str() == SeedChange::TYPE + && let Some(address) = store.root_address::(object)? + { + objects.entry(address).or_insert(object); + } + } + Ok(objects) +} + +pub(crate) struct SeededVersion { + pub(crate) address: RecordAddress, + pub(crate) object: CobId, + pub(crate) produces: RecordCid, + pub(crate) seeded: Seeded, +} + +pub(crate) fn seed_type_name() -> knot_types::TypeName { + knot_types::TypeName::new(SeedChange::TYPE).expect("the seed type is a valid type name") +} + +pub(crate) fn seeded_versions(store: &CobStore) -> Result, XrpcError> { + let seed_type = seed_type_name(); + fostered_objects(store)? + .into_iter() + .map(|(address, object)| { + let changes = store.changes_of(&seed_type, object, HistoryModel::Convergent, None)?; + let tip = changes.changes.last().ok_or_else(|| { + XrpcError::internal(format!("the seeded graph for {address:?} has no changes")) + })?; + let SeedChange::Seed(seeded) = SeedChange::decode(tip.payload()) + .map_err(|error| XrpcError::internal(error.to_string()))? + else { + return Ok(None); + }; + Ok(Some(SeededVersion { + address, + object, + produces: seeded.produces, + seeded, + })) + }) + .collect::>, XrpcError>>() + .map(|versions| versions.into_iter().flatten().collect()) +} + +pub(crate) fn seeds_of(store: &CobStore, object: CobId) -> Result, XrpcError> { + store + .changes_of(&seed_type_name(), object, HistoryModel::Convergent, None)? + .changes + .iter() + .filter_map(|change| match SeedChange::decode(change.payload()) { + Ok(SeedChange::Seed(seeded)) => Some(Ok(seeded)), + Ok(SeedChange::Erase(_)) => None, + Err(error) => Some(Err(XrpcError::internal(error.to_string()))), + }) + .collect() +} + +pub(crate) fn history_of(type_name: &knot_types::TypeName) -> Option { + (type_name.as_str() == SeedChange::TYPE) + .then_some(HistoryModel::Convergent) + .or_else(|| knot_cobs::history_model_of(type_name)) +} + +#[allow(clippy::too_many_arguments)] +fn seed_one( + state: &XrpcState, + git: &Repo, + store: &CobStore<'_>, + home: &CobHome, + repo: &RepoDid, + signer: &dyn Signer, + standing: &mut BTreeMap, + objects: &mut BTreeMap, + checked: &CheckedRecord, +) -> Result<(), XrpcError> { + let body = &checked.body; + let address = checked.address.clone(); + let key = address.mst_key(); + let produces = body.cid(); + let current = standing.get(&key).copied(); + if current == Some(produces.cid()) { + return Ok(()); + } + let op = match current { + None => chain::RecordOp::Create { + record: body.clone(), + }, + Some(current) => chain::RecordOp::Update { + record: body.clone(), + from: RecordCid::new(current) + .ok_or_else(|| XrpcError::internal("A liveleaf isn't a record version"))?, + }, + }; + let materialized = Materialized { + change: SeedChange::Seed(Seeded { + address: address.clone(), + author: checked.author.clone(), + produces, + created_at: checked.created_at, + number: checked.number, + }), + record: Record::version(body.clone()), + }; + match objects.entry(address.clone()) { + std::collections::btree_map::Entry::Occupied(existing) => { + store.update_materialized( + home, + *existing.get(), + &materialized, + signer, + checked.created_at, + )?; + } + std::collections::btree_map::Entry::Vacant(slot) => { + let created = + store.create_materialized(home, &materialized, signer, checked.created_at)?; + slot.insert(created.object); + } + } + let mut emit = crate::FirehoseEmit::new(&state.firehose, repo, checked.created_at); + futures::executor::block_on(chain::apply( + git, + repo, + &[(address, op)], + signer, + &mut emit, + checked.created_at, + )) + .map_err(|error| { + XrpcError::internal(format!( + "Chain apply failed, however we did manage to store the change, and the next boot should hopefully materialize it: {error}" + )) + })?; + standing.insert(key, produces.cid()); + emit.finish().map_err(|error| { + XrpcError::internal(format!( + "Firehose announcement unfortunately failed, but the literal record got saved: {error}" + )) + })?; + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn erase_one( + state: &XrpcState, + git: &Repo, + store: &CobStore<'_>, + home: &CobHome, + repo: &RepoDid, + signer: &dyn Signer, + leaves: &BTreeMap, + objects: &BTreeMap, + address: &RecordAddress, + now: UnixSeconds, +) -> Result<(), XrpcError> { + let object = objects + .get(address) + .copied() + .ok_or_else(|| XrpcError::not_found("No such seeded record at that address"))?; + let from = leaves + .get(&address.mst_key()) + .copied() + .and_then(RecordCid::new) + .ok_or_else(|| XrpcError::not_found("Seeded record was already erased"))?; + store.update_materialized( + home, + object, + &Materialized { + change: SeedChange::Erase(SeedErasure { + from, + erased_at: now, + }), + record: Record::Tombstone, + }, + signer, + now, + )?; + let mut emit = crate::FirehoseEmit::new(&state.firehose, repo, now); + futures::executor::block_on(chain::apply( + git, + repo, + &[(address.clone(), chain::RecordOp::Delete { from })], + signer, + &mut emit, + now, + )) + .map_err(|error| { + XrpcError::internal(format!( + "Chain apply failed, however we managed to store the erasure, next boot should hopefully materialize it: {error}" + )) + })?; + emit.finish().map_err(|error| { + XrpcError::internal(format!( + "Firehose announcement unfortunately failed, but erasure got saved: {error}" + )) + })?; + Ok(()) +} + +#[derive(Deserialize)] +struct RemintInput { + #[serde(rename = "repoDid")] + repo_did: RepoDid, + #[serde(rename = "objectFormat")] + object_format: knot_types::ObjectFormat, +} + +async fn remint_shadow( + State(foster): State>>, + headers: HeaderMap, + body: Bytes, +) -> Result { + let state = &foster.state; + state.legacy_admin_authorize(&headers)?; + let input: RemintInput = decode(&body)?; + let Some(record) = read_registry_record(state, &input.repo_did) + .await? + .filter(|record| record.fostered) + else { + return Err(XrpcError::not_found( + "That repo isn't fostered on this knot", + )); + }; + if record.retired_at.is_some() { + return Err(XrpcError::conflict( + "That repo's shadow was retired by a handover ack shebang thing, so we're not gonna re-mint", + )); + } + let repo = input.repo_did.clone(); + let snapshot = { + let layout = state.layout.clone(); + let locks = Arc::clone(&state.cob_locks); + let repo = repo.clone(); + run_blocking(move || { + let _guard = locks.repo(&repo); + let git = layout.open(&repo)?; + let store = CobStore::new(&git); + Ok((manifest_body(&store)?, seeded_snapshot(&store)?)) + }) + .await? + }; + wipe_shadow(state, &repo, &record).await?; + let (repo_did, key) = foster_repo( + state, + record.owner.clone(), + record.name.clone(), + record.rkey.clone(), + repo, + snapshot.0, + Some(record.policy), + Some(input.object_format), + ) + .await?; + let checked = snapshot + .1 + .into_iter() + .map(|checked| { + ( + checked.address.collection().clone(), + checked.address.rkey().clone(), + Ok(checked), + ) + }) + .collect(); + let results = seed_checked(state, &repo_did, checked, Vec::new()).await?; + Ok(( + http::StatusCode::OK, + Json(SeedOutput { + did: repo_did, + key, + results, + }), + ) + .into_response()) +} + +fn manifest_body(store: &CobStore) -> Result { + let manifest_type = knot_types::TypeName::new(knot_record::manifest::MANIFEST_COLLECTION) + .expect("manifest collection is valid type name"); + for (type_name, object) in store.cobs()? { + if type_name != manifest_type { + continue; + } + let history = knot_cobs::history_model_of(&manifest_type) + .unwrap_or(knot_cob::HistoryModel::Convergent); + let delta = store.changes_of(&manifest_type, object, history, None)?; + if let Some(body) = delta + .changes + .iter() + .rev() + .find_map(|change| change.record.clone().into_body()) + { + return Ok(body); + } + } + Err(XrpcError::internal( + "Fostered repo's manifest never opened, whoops", + )) +} + +fn seeded_snapshot(store: &CobStore) -> Result, XrpcError> { + let seed_type = seed_type_name(); + seeded_versions(store)? + .into_iter() + .map(|version| { + let body = + crate::social::tree_body(store, &seed_type, version.object, version.produces)? + .ok_or_else(|| { + XrpcError::internal( + "one record in this seeded recordset hasn't got any body data", + ) + })?; + Ok(CheckedRecord { + address: version.address, + author: version.seeded.author, + created_at: version.seeded.created_at, + number: version.seeded.number, + body, + }) + }) + .collect() +} + +async fn wipe_shadow( + state: &Arc>, + repo: &RepoDid, + record: &knot_cobs::RepoRecord, +) -> Result<(), XrpcError> { + let signer = state.secrets.signer(&state.knot_did)?; + let layout = state.layout.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let meta_path = state.meta_path.clone(); + let repo = repo.clone(); + let target = RepoRef { + owner: record.owner.clone(), + rkey: record.rkey.clone(), + }; + let home = CobHome::from(&state.knot_did); + let now = state.now(); + run_blocking(move || { + let _repo_guard = cob_locks.repo(&repo); + let _meta_guard = cob_locks.meta(); + with_registry(&meta_path, |store, object, _| { + layout.remove(&repo).map_err(|error| { + XrpcError::internal(format!( + "Shadow couldn't be removed from disk, so it'll be tried again next sweep: {error}" + )) + })?; + deregister_repo(store, &home, object, target, repo.clone(), &signer, now)?; + index.refresh_registry().map_err(|error| { + XrpcError::internal(format!( + "Shadow was de-registered out, but registry projection refresh failed: {error}" + )) + })?; + Ok(()) + })? + .ok_or_else(|| XrpcError::not_found("No git repos registered on this knot")) + }) + .await +} + +pub async fn run_retire_gc( + state: Arc>, + shutdown: tokio_util::sync::CancellationToken, + liar: Option, +) { + let mut tick = tokio::time::interval(std::time::Duration::from_secs(5)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let _first = tick.tick().await; + loop { + tokio::select! { + () = shutdown.cancelled() => return, + _ = tick.tick() => gc_tick(&state, &liar).await, + } + } +} + +pub async fn gc_tick( + state: &Arc>, + liar: &Option, +) { + let now = state.now(); + const RETIRE_GRACE_SECS: u64 = 3_600; + let meta_path = state.meta_path.clone(); + let candidates = run_blocking(move || { + with_registry(&meta_path, |_, _, registry| { + Ok(registry + .records() + .filter_map(|(repo, record)| { + record.retired_at.and_then(|at| { + u64::try_from(now.get().saturating_sub(at.get())) + .is_ok_and(|age| age >= RETIRE_GRACE_SECS) + .then(|| repo.clone()) + }) + }) + .collect::>()) + }) + }) + .await + .unwrap_or_else(|error: XrpcError| { + tracing::error!(%error, "retire GC couldn't read the registry"); + None + }) + .unwrap_or_default(); + for repo in candidates { + if let Some(liar) = liar + && !liar_released(state, liar, &repo).await + { + continue; + } + if let Err(error) = gc_repo(state, &repo).await { + tracing::error!(repo = %repo, %error, "retire GC couldn't remove the shadow"); + } + } +} + +async fn liar_released( + state: &XrpcState, + liar: &LiarHydrant, + repo: &RepoDid, +) -> bool { + if state + .layout + .repo_path(repo) + .is_ok_and(|path| path.join(LIAR_RELEASED).exists()) + { + return true; + } + match tokio::time::timeout(LIAR_UNMARK_TIMEOUT, release_liar(state, liar, repo)).await { + Ok(Ok(())) => { + if let Ok(path) = state.layout.repo_path(repo) + && let Err(error) = std::fs::write(path.join(LIAR_RELEASED), b"") + { + tracing::warn!( + repo = %repo, + %error, + "the liar release marker wouldn't write, so the next sweep asks the liar hydrant again" + ); + } + true + } + Ok(Err(error)) => { + tracing::warn!( + repo = %repo, + %error, + "the liar hydrant refused the unmark, so this shadow outlives its grace window" + ); + false + } + Err(_) => { + tracing::warn!( + repo = %repo, + "the liar hydrant unmark timed out, so this shadow outlives its grace window" + ); + false + } + } +} + +async fn release_liar( + state: &XrpcState, + liar: &LiarHydrant, + repo: &RepoDid, +) -> Result<(), String> { + let signer = state + .secrets + .signer(&state.knot_did) + .map_err(|error| error.to_string())?; + let method = Nsid::new_static(LIAR_UNMARK_LXM).expect("the unmark nsid is a literal nsid"); + let token = knot_atproto::mint_service_jwt( + &signer, + &state.knot_did, + &liar.audience(), + &method, + state.now(), + ); + let body = serde_json::json!({"dids": [repo.as_str()]}) + .to_string() + .into_bytes(); + let mut request = HttpRequest::post(liar.unmark_url(), Bytes::from(body)); + let value = HeaderValue::from_str(&format!("Bearer {token}")) + .map_err(|error| format!("the unmark token wouldn't become a header: {error}"))?; + request.headers.insert(http::header::AUTHORIZATION, value); + let response = liar + .http + .execute(request) + .await + .map_err(|error| format!("the liar hydrant is unreachable: {error}"))?; + response + .status + .is_success() + .then_some(()) + .ok_or_else(|| format!("the liar hydrant returned {}", response.status)) +} + +async fn gc_repo( + state: &Arc>, + repo: &RepoDid, +) -> Result<(), XrpcError> { + let signer = state.secrets.signer(&state.knot_did)?; + let layout = state.layout.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let meta_path = state.meta_path.clone(); + let lfs_store = Arc::clone(&state.lfs.handle.store); + let deleted = repo.clone(); + let home = CobHome::from(&state.knot_did); + let now = state.now(); + run_blocking(move || { + let _repo_guard = cob_locks.repo(&deleted); + let _meta_guard = cob_locks.meta(); + with_registry(&meta_path, |store, object, registry| { + let record = registry + .record_of(&deleted) + .ok_or_else(|| XrpcError::not_found("the repo isn't registered on this knot"))?; + let target = RepoRef { + owner: record.owner.clone(), + rkey: record.rkey.clone(), + }; + layout + .remove(&deleted) + .map_err(|error| XrpcError::internal(format!("repo {deleted} couldn't be removed from disk, so it stays registered for the next sweep: {error}")))?; + deregister_repo(store, &home, object, target, deleted.clone(), &signer, now)?; + index.refresh_registry().map_err(|error| { + XrpcError::internal(format!( + "repo {deleted} was deregistered but registry projection refresh failed: {error}" + )) + })?; + if let Err(error) = lfs_store.remove_repo(&deleted) { + tracing::warn!( + repo = %deleted, + %error, + "lfs prefix removal on retire GC failed, orphan sweep will reclaim" + ); + } + Ok(()) + })? + .ok_or_else(|| XrpcError::not_found("no git repos registered on this knot")) + }) + .await +} diff --git a/knot2/crates/knot-xrpc/src/git.rs b/knot2/crates/knot-xrpc/src/git.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb0080e4d868dee6d6036dfcfc6af68d08bedc37 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/git.rs @@ -0,0 +1,99 @@ +use http::StatusCode; + +use knot_types::{Oid, RefName, RepoDid}; + +use crate::error::XrpcError; + +pub(crate) fn encode_ref_component(raw: &str) -> String { + let mut out = String::with_capacity(raw.len()); + for c in raw.chars() { + match c { + '%' => out.push_str("%25"), + ':' => out.push_str("%3a"), + '~' => out.push_str("%7e"), + other => out.push(other), + } + } + out +} + +pub(crate) const KEEP_PREFIX: &str = "refs/tngl/keep/"; + +pub(crate) fn keep_ref( + record_did: &RepoDid, + address: &knot_types::RecordAddress, + commit: Oid, +) -> Result { + RefName::new(format!( + "{KEEP_PREFIX}{}/{}/{}/{}", + encode_ref_component(record_did.as_str()), + address.collection().as_str(), + encode_ref_component(address.rkey().as_str()), + commit, + )) + .map_err(|_e| { + XrpcError::from_status( + StatusCode::BAD_REQUEST, + "record doesn't map to a usable ref name", + ) + }) +} + +#[cfg(test)] +mod tests { + use super::{encode_ref_component, keep_ref}; + use knot_types::{RefName, RepoDid}; + + const OID: &str = "81158e200ff7e520a20b5e7505960cee4f35e4b3"; + + fn keep_ref_for( + did: &RepoDid, + collection: &str, + rkey: &str, + ) -> Result { + let address = knot_types::RecordAddress::new( + knot_types::RecordCollection::new(collection).unwrap(), + knot_types::RecordRkey::new(rkey).unwrap(), + ); + keep_ref(did, &address, knot_types::Oid::from_hex(OID).unwrap()) + .map(|name| name.as_str().to_string()) + } + + #[test] + fn keep_ref_for_a_did_and_tid_rkey_is_a_valid_ref_name() { + assert_eq!( + keep_ref_for( + &RepoDid::new("did:plc:squid").unwrap(), + "org.tangled.review.patch", + "3jzfcijpj2z2a" + ) + .unwrap(), + "refs/tngl/keep/did%3aplc%3asquid/org.tangled.review.patch/3jzfcijpj2z2a/81158e200ff7e520a20b5e7505960cee4f35e4b3" + ); + } + + #[test] + fn keep_ref_for_a_rkey_that_is_itself_a_did_is_valid() { + let name = keep_ref_for( + &RepoDid::new("did:plc:squid").unwrap(), + "org.tangled.snippet.string", + "3jzfcijpj2z2a", + ) + .unwrap(); + assert!(RefName::new(name).is_ok()); + } + + #[test] + fn every_reserved_byte_in_a_component_is_percent_encoded() { + assert_eq!(encode_ref_component("a:b~c%d"), "a%3ab%7ec%25d"); + } + #[test] + fn encoding_is_injective() { + assert_ne!( + encode_ref_component("did:web:knot:conch"), + encode_ref_component("did:web:knot-conch") + ); + assert_ne!(encode_ref_component("a%3ab"), encode_ref_component("a:b")); + assert_eq!(encode_ref_component("a%3ab"), "a%253ab"); + } +} diff --git a/knot2/crates/knot-xrpc/src/grants.rs b/knot2/crates/knot-xrpc/src/grants.rs new file mode 100644 index 0000000000000000000000000000000000000000..5fb8a87008e38ba71e2f3db7c8b3971b28a894c0 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/grants.rs @@ -0,0 +1,143 @@ +use std::sync::Arc; + +use knot_git::Repo; +use knot_index::{Entry, Resolved}; +use knot_record::chain; +use knot_record::grant::{self, GrantCollection}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, RepoDid}; + +use crate::XrpcState; + +#[derive(Debug, thiserror::Error)] +pub enum SyncError { + #[error(transparent)] + Chain(#[from] chain::ChainError), + #[error(transparent)] + Frames(#[from] crate::firehose::UnstampedFrames), + #[error("the signing key wouldn't load for a grant sync: {0}")] + Key(#[from] knot_secrets::SecretsError), +} + +fn sync_grants_for( + state: &Arc>, + git: &Repo, + did: &RepoDid, + entries: Resolved>, + collection: GrantCollection, +) -> Result { + let entries = match entries { + Resolved::Ready(entries) => entries, + Resolved::Warming => { + tracing::warn!( + surface = collection.surface(), + "the projection is warming, so its records stay as they are rather than \ + syncing an empty grant list" + ); + return Ok(0); + } + }; + let now = state.now(); + let signer = state.secrets.signer(&state.knot_did)?; + let records = chain::in_tree(grant::bodies( + entries + .iter() + .map(|(subject, entry)| (subject, entry.created_at)), + collection, + )); + let mut emit = crate::firehose::FirehoseEmit::new(&state.firehose, did, now); + let chunks = futures::executor::block_on(chain::settle( + git, + did, + std::slice::from_ref(&collection.records()), + &records, + &signer, + &mut emit, + now, + ))?; + emit.finish()?; + Ok(chunks.len()) +} + +pub(crate) fn sync_members( + state: &Arc>, + meta: &Repo, +) -> Result { + sync_grants_for( + state, + meta, + &state.knot_repo, + state.index.member_entries(), + GrantCollection::Member, + ) +} + +pub(crate) fn sync_collaborators( + state: &Arc>, + git: &Repo, + repo: &RepoDid, +) -> Result { + sync_grants_for( + state, + git, + repo, + state.index.collaborator_entries(repo), + GrantCollection::Collaborator, + ) +} + +pub(crate) fn sync_blocks( + state: &Arc>, + meta: &Repo, +) -> Result { + sync_grants_for( + state, + meta, + &state.knot_repo, + state.index.blocklist_entries(), + GrantCollection::Block, + ) +} + +pub fn sweep_members(state: &Arc>) { + let _guard = state.cob_locks.meta(); + match Repo::open(&state.meta_path) { + Ok(meta) => noted(sync_members(state, &meta), GrantCollection::Member), + Err(error) => tracing::warn!(%error, "the grant sweep couldn't open the members chain"), + } +} + +pub fn sweep_collaborators( + state: &Arc>, + repo: &RepoDid, +) { + let _guard = state.cob_locks.repo(repo); + match state.layout.open(repo) { + Ok(git) => noted( + sync_collaborators(state, &git, repo), + GrantCollection::Collaborator, + ), + Err(error) => tracing::warn!( + repo = repo.as_str(), + %error, + "the grant sweep couldn't open a collaborator chain" + ), + } +} + +pub(crate) fn noted(outcome: Result, collection: GrantCollection) { + match outcome { + Ok(0) => {} + Ok(commits) => tracing::info!( + commits, + surface = collection.surface(), + "grant records synced" + ), + Err(error) => tracing::warn!( + %error, + surface = collection.surface(), + "grant records wouldn't sync, the grant list stays put, and a later grant write, \ + sweep, or restart retries" + ), + } +} diff --git a/knot2/crates/knot-xrpc/src/handover.rs b/knot2/crates/knot-xrpc/src/handover.rs new file mode 100644 index 0000000000000000000000000000000000000000..402b690fec9ce42f5a18a1d5d6d85ac5036f3350 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/handover.rs @@ -0,0 +1,1424 @@ +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::path::Path; +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use http::{HeaderMap, StatusCode}; +use ipld_core::ipld::Ipld; +use knot_acl::{KnotAcl, can_admin_knot}; +use knot_capability::Capability; +use knot_cob::{COBS_PREFIX, CobHome, CobStore, cob_ref_name, parse_cob_ref}; +use knot_git::{RefUpdate, Repo}; +use knot_index::Resolved; +use knot_lfs::{ClaimedSize, LfsOid, LfsStore, Pool, UploadAdmission}; +use knot_record::chain; +use knot_runtime::{Clock, HttpRequest, HttpTransport, Signer}; +use knot_types::{ + ActorId, BlobCid, ChangeId, CobId, ObjectFormat, Oid, RecordAddress, RecordCid, RepoDid, + Sha256Digest, Tid, TypeName, UnixMicros, UnixSeconds, +}; +use serde::{Deserialize, Serialize}; +use tokio_util::sync::CancellationToken; +use url::Url; + +use crate::XrpcState; +use crate::error::XrpcError; +use crate::forks::{self, Upstream}; +use crate::{decode, run_blocking}; + +pub const TRIGGER_ROUTE: &str = "/xrpc/org.tangled.temp.server.handoverRepo"; +pub const ACK_ROUTE: &str = "/xrpc/org.tangled.temp.server.ackHandover"; +pub const CANCEL_ROUTE: &str = "/xrpc/org.tangled.temp.server.cancelHandover"; +pub(crate) const SIDECAR: &str = "tangled.handover.json"; +pub(crate) const SETTLED: &str = "tangled.handover.done"; + +const FOSTER_DOWN_PARK_MIN_ATTEMPTS: usize = 20; +const FOSTER_DOWN_PARK_DENOMINATOR: usize = 2; +const QUARANTINE_AFTER_FAILURES: usize = 3; +const RETRY_ROUNDS: usize = 200; +const RETRY_PAUSE_SECS: u64 = 15; +const FOSTER_DOWN_BACKOFF_SECS: [u64; 3] = [15, 60, 300]; +const ACK_BACKOFF_SECS: [u64; 3] = [1, 4, 0]; +const KNOT_DOC_POLLS: usize = 12; +const CONFIRM_PAGES_AT_MOST: usize = 1_000; + +#[derive(Default)] +pub struct HandoverSessions { + running: std::sync::Mutex>, +} + +pub(crate) type IdentityCache = Arc>>; + +impl HandoverSessions { + fn locked(&self) -> std::sync::MutexGuard<'_, BTreeMap> { + self.running + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub fn acquire(self: &Arc, repo: &RepoDid) -> Option { + let mut running = self.locked(); + (!running.contains_key(repo)).then(|| { + let token = CancellationToken::new(); + running.insert(repo.clone(), token.clone()); + HandoverLease { + sessions: Arc::clone(self), + repo: repo.clone(), + token, + } + }) + } + + fn stop(&self, repo: &RepoDid) -> Option { + self.locked().get(repo).cloned() + } +} + +pub struct HandoverLease { + sessions: Arc, + repo: RepoDid, + token: CancellationToken, +} + +impl Drop for HandoverLease { + fn drop(&mut self) { + self.sessions.locked().remove(&self.repo); + } +} + +pub(crate) fn pending(state: &XrpcState, repo: &RepoDid) -> bool { + state + .layout + .repo_path(repo) + .is_ok_and(|path| path.join(SIDECAR).exists() && !path.join(SETTLED).exists()) +} + +fn mark_settled(repo_path: &Path) -> std::io::Result<()> { + std::fs::write(repo_path.join(SETTLED), b"") +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct FosterKnot(Url); + +impl FosterKnot { + pub fn new(raw: &str) -> Result { + let url = Url::parse(raw).map_err(|_| format!("foster url {raw} isn't a url"))?; + if !url_is_trusted(&url) { + return Err(format!( + "foster url {raw} must be https, or http on loopback. a foster knot is \ + trusted over tls or not at all" + )); + } + Ok(Self(url)) + } + + fn url(&self, path: &str) -> Url { + let mut url = self.0.clone(); + url.path_segments_mut() + .expect("an http url always has segments") + .extend(path.split('/')); + url + } +} + +pub(crate) fn url_is_trusted(url: &Url) -> bool { + matches!(url.scheme(), "https") || matches!(url.scheme(), "http") && host_is_loopback(url) +} + +fn host_is_loopback(url: &Url) -> bool { + match url.host() { + Some(url::Host::Domain(name)) => name.eq_ignore_ascii_case("localhost"), + Some(url::Host::Ipv4(ip)) => ip.is_loopback(), + Some(url::Host::Ipv6(ip)) => ip.is_loopback(), + None => false, + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub(crate) struct HandoverJob { + repo: RepoDid, + foster: FosterKnot, + triggered_at: UnixSeconds, + #[serde(default, skip_serializing_if = "Option::is_none")] + error: Option, + #[serde(default)] + consecutive_failures: usize, +} + +impl HandoverJob { + fn read(repo_path: &Path) -> Option> { + std::fs::read(repo_path.join(SIDECAR)) + .ok() + .map(|bytes| serde_json::from_slice(&bytes)) + } + + fn write(&self, repo_path: &Path) -> std::io::Result<()> { + let staged = tempfile::Builder::new() + .prefix(&format!("{SIDECAR}.tmp.")) + .tempfile_in(repo_path)?; + std::fs::write(staged.path(), serde_json::to_vec(self)?)?; + std::fs::rename(staged.path(), repo_path.join(SIDECAR)) + } +} + +#[derive(Deserialize)] +struct HandoverInput { + #[serde(rename = "repoDid")] + repo_did: RepoDid, + #[serde(rename = "fosterUrl")] + foster_url: String, +} + +pub(crate) async fn trigger( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if !can_admin_knot(&acl, &actor).is_allowed() { + return Err(XrpcError::forbidden( + "only a knot admin may pull a repo back from its foster knot", + )); + } + let input: HandoverInput = decode(&body)?; + crate::reads::require_hosted(&state, input.repo_did.clone())?; + let foster = FosterKnot::new(&input.foster_url).map_err(XrpcError::invalid_request)?; + let repo_path = state + .layout + .repo_path(&input.repo_did) + .map_err(|error| XrpcError::internal(error.to_string()))?; + if repo_path.join(SETTLED).exists() { + return Ok(( + StatusCode::OK, + Json(serde_json::json!({ + "repoDid": input.repo_did, + "state": "done", + })), + ) + .into_response()); + } + let lease = state + .handover_sessions + .acquire(&input.repo_did) + .ok_or_else(|| { + XrpcError::conflict("a handover for this repo is already running on this knot") + })?; + HandoverJob { + repo: input.repo_did.clone(), + foster: foster.clone(), + triggered_at: state.now(), + error: None, + consecutive_failures: 0, + } + .write(&repo_path) + .map_err(|error| XrpcError::internal(format!("writing the handover job: {error}")))?; + tokio::spawn(drive( + Arc::clone(&state), + input.repo_did.clone(), + lease, + Arc::new(std::sync::Mutex::new(BTreeMap::new())), + )); + Ok(( + StatusCode::OK, + Json(serde_json::json!({ + "repoDid": input.repo_did, + "state": "running", + })), + ) + .into_response()) +} + +#[derive(Deserialize)] +struct CancelInput { + #[serde(rename = "repoDid")] + repo_did: RepoDid, +} + +pub(crate) async fn cancel( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if !can_admin_knot(&acl, &actor).is_allowed() { + return Err(XrpcError::forbidden( + "only a knot admin may abandon a handover job", + )); + } + let input: CancelInput = decode(&body)?; + if let Some(running) = state.handover_sessions.stop(&input.repo_did) { + running.cancel(); + } + let repo_path = state + .layout + .repo_path(&input.repo_did) + .map_err(|error| XrpcError::internal(error.to_string()))?; + let sidecar = repo_path.join(SIDECAR); + if !sidecar.is_file() { + return Err(XrpcError::not_found(format!( + "{} has no handover job to abandon", + input.repo_did + ))); + } + std::fs::remove_file(&sidecar) + .map_err(|error| XrpcError::internal(format!("removing the handover job: {error}")))?; + mark_settled(&repo_path).map_err(|error| { + XrpcError::internal(format!("settling the abandoned handover: {error}")) + })?; + tracing::info!( + repo = %input.repo_did, + "the handover job is abandoned and won't be staged again. a live pull stops at its next step" + ); + Ok(( + StatusCode::OK, + Json(serde_json::json!({ "repoDid": input.repo_did })), + ) + .into_response()) +} + +async fn drive( + state: Arc>, + repo: RepoDid, + lease: HandoverLease, + identities: IdentityCache, +) -> Result { + tokio::select! { + () = lease.token.cancelled() => { + tracing::info!(repo = %repo, "the handover was canceled. its job is abandoned"); + Ok(crate::foster::RecordCount::new(0)) + } + outcome = finish(&state, &repo, &identities) => { + match &outcome { + Ok(records) => tracing::info!(repo = %repo, records = records.get(), "the handover finished. every fostered record of this repo is home"), + Err(error) => tracing::error!(repo = %repo, %error, "the handover didn't complete. its job stays for a replay"), + }; + outcome + } + } +} + +pub(crate) async fn finish( + state: &Arc>, + repo: &RepoDid, + identities: &IdentityCache, +) -> Result { + let repo_path = state + .layout + .repo_path(repo) + .map_err(|error| XrpcError::internal(error.to_string()))?; + let job = match HandoverJob::read(&repo_path) { + Some(Ok(job)) => job, + Some(Err(error)) => { + return Err(XrpcError::internal(format!( + "the handover sidecar for {repo} wouldn't decode: {error}" + ))); + } + None => { + return Err(XrpcError::not_found(format!( + "{repo} has no handover pending" + ))); + } + }; + match handover(state, repo, &job.foster, identities).await { + Ok(records) => { + std::fs::remove_file(repo_path.join(SIDECAR)).map_err(|error| { + XrpcError::internal(format!("retiring the finished handover job: {error}")) + })?; + mark_settled(&repo_path).map_err(|error| { + XrpcError::internal(format!("settling the finished handover: {error}")) + })?; + let state = Arc::clone(state); + let repo = repo.clone(); + tokio::spawn(async move { + crate::keep_sweep::sweep_repo(&state, &repo).await; + }); + Ok(records) + } + Err(error) => { + HandoverJob { + consecutive_failures: job.consecutive_failures + 1, + error: Some(error.to_string()), + ..job + } + .write(&repo_path) + .map_err(|write_error| { + XrpcError::internal(format!("recording the failed handover: {write_error}")) + })?; + Err(error) + } + } +} + +/// Converts and drops the seeds a settled handover left behind, the way a finished handover +/// would have. `None` when the repo has no handover settled or no seeds left. +pub async fn repair( + state: &Arc>, + repo: &RepoDid, +) -> Result, XrpcError> { + let settled = Arc::clone(state); + let target = repo.clone(); + let converted = run_blocking(move || { + let repo_path = settled + .layout + .repo_path(&target) + .map_err(|error| XrpcError::internal(error.to_string()))?; + if !repo_path.join(SETTLED).exists() || repo_path.join(SIDECAR).exists() { + return Ok(None); + } + let _guard = settled.cob_locks.repo(&target); + let git = settled.layout.open(&target)?; + if crate::foster::fostered_objects(&CobStore::new(&git))?.is_empty() { + return Ok(None); + } + let signer = settled.secrets.signer(&settled.knot_did)?; + let converted = crate::social::convert_seeded(&git, &target, &signer)?; + crate::social::reconcile(&settled, &git, &target, &signer, settled.now())?; + confirm_local_leaves(&git, &converted)?; + announce_rebuilt(&settled, &git, &target, settled.now()); + Ok(Some(crate::foster::RecordCount::new(converted.len()))) + }) + .await?; + if converted.is_some() { + prune_seeded(state, repo).await?; + } + Ok(converted) +} + +/// [`repair`] over every native repo, logging each outcome. +pub async fn repair_all(state: Arc>) { + let (mut repaired, mut failed) = (0usize, 0usize); + for repo in state.index.native_repos() { + match repair(&state, &repo).await { + Ok(Some(records)) => { + repaired += 1; + tracing::info!(repo = %repo, records = records.get(), "the settled handover's seeds are converted and dropped"); + } + Ok(None) => {} + Err(error) => { + failed += 1; + tracing::warn!(repo = %repo, %error, "the settled handover's seeds wouldn't convert"); + } + } + } + tracing::info!(repaired, failed, "the handover repair finished"); +} + +async fn handover( + state: &Arc>, + repo: &RepoDid, + foster: &FosterKnot, + identities: &IdentityCache, +) -> Result { + let url = foster.url(repo.as_str()); + let (object_format, advertised) = match advertised_cobs(state, &url).await? { + Shadow::Absent => { + tracing::info!( + repo = %repo, + "the foster knot doesn't shadow this repo, so its foster era is settled" + ); + return Ok(crate::foster::RecordCount::new(0)); + } + Shadow::Hosted(object_format, advertised) => (object_format, advertised), + }; + if advertised.is_empty() { + return Err(XrpcError::bad_gateway(format!( + "the foster knot hosts {repo} with no cobs to its name, so an empty shadow \ + can't hand anything back" + ))); + } + let key = foster_key(state, foster, identities).await?; + let upstream = Upstream::Remote(url); + + let layout = state.layout.clone(); + let target = repo.clone(); + let wanted = advertised.clone(); + let (local_format, local, missing) = run_blocking(move || { + let git = layout.open(&target)?; + let local: BTreeMap = git + .references()? + .into_iter() + .map(|record| (record.name.as_str().to_owned(), record.target)) + .collect(); + let missing: BTreeSet = wanted + .iter() + .filter(|(type_name, object, tip)| { + let name = cob_ref_name(type_name, *object).expect("a served cob ref name parses"); + local.get(name.as_str()) != Some(tip) + }) + .map(|(_, _, tip)| *tip) + .collect(); + Ok::<_, XrpcError>((git.object_format(), local, missing)) + }) + .await?; + if local_format != object_format { + return Err(XrpcError::invalid_request( + "the foster knot and this knot disagree on the repo's object format", + )); + } + let wants: Vec = missing.into_iter().collect(); + let haves: Vec = local.values().copied().collect(); + if !wants.is_empty() { + forks::pull_objects(state, &upstream, repo, wants, haves, object_format).await?; + } + raise_floor_to_foster_rev(state, foster, repo).await; + let settled = Arc::clone(state); + let target = repo.clone(); + let graphs = advertised.clone(); + let placed = local; + let (converted, seeded) = run_blocking(move || { + let _guard = settled.cob_locks.repo(&target); + let git = settled.layout.open(&target)?; + let store = CobStore::new(&git); + let home = CobHome::from(&target); + for (type_name, object, tip) in &graphs { + let history = crate::foster::history_of(type_name).ok_or_else(|| { + XrpcError::bad_gateway(format!( + "the foster knot serves {} cobs whose history model this knot doesn't know", + type_name.as_str() + )) + })?; + store.verify_type_from_tip( + &home, + type_name, + *object, + ChangeId::new(*tip), + history, + &key, + )?; + } + apply_cob_refs(&git, &graphs, &placed)?; + let signer = settled.secrets.signer(&settled.knot_did)?; + let converted = crate::social::convert_seeded(&git, &target, &signer)?; + crate::social::reconcile(&settled, &git, &target, &signer, settled.now())?; + confirm_local_leaves(&git, &converted)?; + announce_rebuilt(&settled, &git, &target, settled.now()); + Ok((converted, crate::foster::seeded_versions(&store)?)) + }) + .await?; + transfer_blobs(state, foster, repo, &converted).await?; + confirm_foster_serving(state, foster, repo, &seeded).await?; + prune_seeded(state, repo).await?; + let records = crate::foster::RecordCount::new(seeded.len()); + let superseded: Vec = converted + .iter() + .filter(|converted| matches!(converted.settled, knot_cobs::Settled::Superseded)) + .map(|converted| converted.version.address.clone()) + .collect(); + send_ack(state, foster, repo, records, &superseded, state.now()).await?; + Ok(records) +} + +async fn prune_seeded( + state: &Arc>, + repo: &RepoDid, +) -> Result<(), XrpcError> { + let layout = state.layout.clone(); + let target = repo.clone(); + let settled = Arc::clone(state); + run_blocking(move || { + let _guard = settled.cob_locks.repo(&target); + let git = layout.open(&target)?; + let store = CobStore::new(&git); + let seed_type = crate::foster::seed_type_name(); + let updates = crate::foster::fostered_objects(&store)? + .into_values() + .filter_map(|object| seeded_ref_delete(&git, &seed_type, object).transpose()) + .collect::, XrpcError>>()?; + git.with_ref_txn(|txn| txn.update_refs(&updates)) + .map_err(XrpcError::from) + }) + .await +} + +fn seeded_ref_delete( + git: &Repo, + seed_type: &TypeName, + object: CobId, +) -> Result, XrpcError> { + let name = cob_ref_name(seed_type, object).expect("a served cob ref name parses"); + Ok(git + .find_ref(&name)? + .map(|old| RefUpdate::Delete { name, old })) +} + +enum Shadow { + Absent, + Hosted(ObjectFormat, Vec<(TypeName, CobId, Oid)>), +} + +async fn advertised_cobs( + state: &XrpcState, + url: &Url, +) -> Result { + let refs = match knot_pack::remote_refs(state.git_http.as_ref(), url, &[COBS_PREFIX]).await { + Ok(refs) => refs, + Err(knot_pack::FetchError::Status(status)) if status.get() == 404 => { + return Ok(Shadow::Absent); + } + Err(other) => return Err(forks::map_fetch(other)), + }; + let parsed = refs + .refs + .iter() + .filter(|record| record.name.as_str().starts_with(COBS_PREFIX)) + .map(|record| match parse_cob_ref(&record.name) { + Some((type_name, object)) => Ok((type_name, object, record.target)), + None => Err(XrpcError::bad_gateway(format!( + "the foster knot serves a ref outside the cob ref rules: {}", + record.name.as_str() + ))), + }) + .collect::, XrpcError>>()?; + Ok(Shadow::Hosted(refs.object_format, parsed)) +} + +fn apply_cob_refs( + git: &Repo, + graphs: &[(TypeName, CobId, Oid)], + placed: &BTreeMap, +) -> Result<(), XrpcError> { + let updates = graphs + .iter() + .filter_map(|(type_name, object, tip)| { + let name = cob_ref_name(type_name, *object).expect("a served cob ref name parses"); + match placed.get(name.as_str()) { + None => Some(Ok(RefUpdate::Create { name, new: *tip })), + Some(&old) if old == *tip => None, + Some(&old) => Some(git.merge_base(old, *tip).map_err(XrpcError::from).and_then( + |base| match base { + Some(base) if base == old => Ok(RefUpdate::Update { + name, + old, + new: *tip, + }), + _ => Err(XrpcError::bad_gateway(format!( + "the foster knot serves {name} at {tip}, which isn't \ + ahead of the local {old}" + ))), + }, + )), + } + }) + .collect::, XrpcError>>()?; + if !updates.is_empty() { + git.update_refs(&updates)?; + } + Ok(()) +} + +fn confirm_local_leaves( + git: &Repo, + converted: &[crate::social::Converted], +) -> Result<(), XrpcError> { + let leaves = futures::executor::block_on(chain::leaves(git))?; + converted.iter().try_for_each( + |crate::social::Converted { + version, settled, .. + }| { + if matches!(settled, knot_cobs::Settled::Superseded) { + return Ok(()); + } + (leaves.get(&version.address.mst_key()) == Some(&version.produces.cid())) + .then_some(()) + .ok_or_else(|| { + XrpcError::internal(format!( + "the rebuilt chain doesn't serve {} at {} the way the foster knot's \ + last change recorded it", + version.address.mst_key().as_str(), + version.produces.cid() + )) + }) + }, + ) +} + +fn announce_rebuilt( + state: &Arc>, + git: &Repo, + did: &RepoDid, + now: UnixSeconds, +) { + let rebuilt = chain::tip(git).and_then(|tip| match tip { + Some(tip) => chain::commit_bytes(git, &tip).map(|commit| Some((tip, commit))), + None => Ok(None), + }); + match rebuilt { + Ok(Some((tip, commit))) => { + crate::firehose::publish_sync(&state.firehose, did, &tip.rev, &commit, now) + } + Ok(None) => {} + Err(error) => tracing::warn!( + repo = did.as_str(), + %error, + "the rebuilt tip wouldn't read, so its sync frame is skipped" + ), + } +} + +pub(crate) fn advertises(described: &serde_json::Value, capability: &Capability) -> bool { + described["capabilities"] + .as_array() + .is_some_and(|capabilities| { + capabilities + .iter() + .any(|token| token.as_str() == Some(capability.token())) + }) +} + +pub(crate) async fn knot_identity( + state: &XrpcState, + base: &Url, + who: &str, + capability: &Capability, + lacking: &str, +) -> Result { + let describe_url = crate::foster::origin_url(base, "xrpc/org.tangled.knot.describeKnot")?; + let described = get_json(state.git_http.as_ref(), describe_url, who).await?; + let did = described["did"] + .as_str() + .ok_or_else(|| XrpcError::bad_gateway("describeKnot leaves out the knot's own did"))? + .to_owned(); + if !advertises(&described, capability) { + return Err(XrpcError::invalid_request(format!("{did} {lacking}"))); + } + let document_url = crate::foster::origin_root(base, ".well-known/did.json"); + let document = get_json(state.git_http.as_ref(), document_url, who).await?; + if document["id"].as_str() != Some(did.as_str()) { + return Err(XrpcError::bad_gateway(format!( + "the did document served by {base} belongs to a different knot than describeKnot reported" + ))); + } + let wanted = format!("{did}#{}", knot_atproto::ATPROTO_METHOD); + let multibase = document["verificationMethod"] + .as_array() + .and_then(|methods| { + methods + .iter() + .find(|method| method["id"].as_str() == Some(wanted.as_str())) + }) + .and_then(|method| method["publicKeyMultibase"].as_str()) + .ok_or_else(|| { + XrpcError::bad_gateway(format!("{did} publishes no multikey under #atproto")) + })? + .to_owned(); + Ok(multibase) +} + +async fn foster_key( + state: &Arc>, + foster: &FosterKnot, + identities: &IdentityCache, +) -> Result { + let cached = identities + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .get(&foster.0) + .cloned(); + match cached { + Some(actor) => Ok(actor), + None => { + let multibase = knot_identity( + state, + &foster.0, + "foster knot", + &Capability::Foster, + "doesn't take fostered repos", + ) + .await?; + let actor = ActorId::new(multibase).map_err(|_| { + XrpcError::bad_gateway("the foster knot's multikey doesn't parse as a key") + })?; + identities + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(foster.0.clone(), actor.clone()); + Ok(actor) + } + } +} + +async fn foster_latest_rev( + state: &XrpcState, + foster: &FosterKnot, + repo: &RepoDid, +) -> Result { + let mut url = crate::foster::origin_url(&foster.0, "xrpc/com.atproto.sync.getLatestCommit")?; + url.query_pairs_mut().append_pair("did", repo.as_str()); + let served = get_json(state.git_http.as_ref(), url, "foster knot").await?; + served["rev"] + .as_str() + .and_then(|rev| rev.parse::().ok()) + .ok_or_else(|| { + XrpcError::bad_gateway( + "the foster knot serves getLatestCommit without a rev it can name", + ) + }) +} + +async fn raise_floor_to_foster_rev( + state: &XrpcState, + foster: &FosterKnot, + repo: &RepoDid, +) { + match foster_latest_rev(state, foster, repo).await { + Ok(rev) => state.firehose.raise_floor(UnixMicros::new(rev.timestamp())), + Err(error) => tracing::warn!( + repo = repo.as_str(), + %error, + "the foster knot's latest rev wouldn't learn, so revs minted after the pull back may sit at or below the foster era" + ), + } +} + +async fn confirm_foster_serving( + state: &Arc>, + foster: &FosterKnot, + repo: &RepoDid, + seeded: &[crate::foster::SeededVersion], +) -> Result<(), XrpcError> { + let collections: BTreeSet<_> = seeded + .iter() + .map(|version| version.address.collection()) + .collect(); + let mut served: BTreeMap = BTreeMap::new(); + for collection in collections { + let mut cursor: Option = None; + let mut asked: BTreeSet = BTreeSet::new(); + loop { + if asked.len() >= CONFIRM_PAGES_AT_MOST { + return Err(XrpcError::bad_gateway(format!( + "the foster knot pages past {CONFIRM_PAGES_AT_MOST} pages of {collection} \ + records without ending, so its shadow can't be confirmed" + ))); + } + let mut url = foster.url("xrpc/com.atproto.repo.listRecords"); + url.query_pairs_mut() + .append_pair("repo", repo.as_str()) + .append_pair("collection", collection.as_str()) + .append_pair("limit", "100"); + if let Some(at) = &cursor { + url.query_pairs_mut().append_pair("cursor", at); + } + let page = get_json(state.git_http.as_ref(), url, "foster knot").await?; + let records = page["records"].as_array().ok_or_else(|| { + XrpcError::bad_gateway(format!( + "the foster knot returned a listRecords page for {collection} with no \ + records list" + )) + })?; + let empty = records.is_empty(); + for record in records { + let address = record["uri"].as_str().and_then(|uri| { + uri.strip_prefix(&format!("at://{}/", repo)) + .and_then(|path| RecordAddress::parse_path(path).ok()) + }); + let address = address.ok_or_else(|| { + XrpcError::bad_gateway(format!( + "the foster knot's listRecords page for {collection} includes a record \ + outside this repo: {}", + record["uri"] + )) + })?; + let cid = record["cid"].as_str().and_then(RecordCid::parse).ok_or_else(|| { + XrpcError::bad_gateway(format!( + "the foster knot's listRecords page serves {address:?} without a cid it can name" + )) + })?; + served.insert(address, cid); + } + cursor = page["cursor"] + .as_str() + .filter(|at| !at.is_empty()) + .map(str::to_owned); + if cursor.is_none() || empty { + break; + } + if cursor + .as_deref() + .is_some_and(|at| !asked.insert(at.to_owned())) + { + return Err(XrpcError::bad_gateway(format!( + "the foster knot keeps returning the same page of {collection} records, so \ + its shadow can't be confirmed" + ))); + } + } + } + for version in seeded { + let cid = served.get(&version.address).ok_or_else(|| { + XrpcError::bad_gateway(format!( + "the foster knot's records no longer include {}, so its shadow can't be confirmed", + version.address.at_uri(repo) + )) + })?; + if cid != &version.produces { + return Err(XrpcError::internal(format!( + "the foster knot serves {} at {cid} while the knot's graph recorded {}", + version.address.at_uri(repo), + version.produces.cid() + ))); + } + } + Ok(()) +} + +fn collect_blob_cids(value: &Ipld, found: &mut BTreeSet) -> Result<(), String> { + match value { + Ipld::Map(fields) => { + if matches!(fields.get("$type"), Some(Ipld::String(kind)) if kind.as_str() == "blob") + && let Some(Ipld::Link(link)) = fields.get("ref") + { + found.insert(BlobCid::new(*link).ok_or_else(|| link.to_string())?); + } + fields + .values() + .try_for_each(|field| collect_blob_cids(field, found))?; + } + Ipld::List(items) => items + .iter() + .try_for_each(|item| collect_blob_cids(item, found))?, + _ => {} + } + Ok(()) +} + +async fn transfer_blobs( + state: &Arc>, + foster: &FosterKnot, + repo: &RepoDid, + converted: &[crate::social::Converted], +) -> Result<(), XrpcError> { + let mut wanted = BTreeSet::new(); + for converted in converted { + if matches!(converted.settled, knot_cobs::Settled::Superseded) { + continue; + } + let body: Ipld = + serde_ipld_dagcbor::from_slice(converted.body.as_bytes()).map_err(|error| { + XrpcError::internal(format!("a settled record body wouldn't decode: {error}")) + })?; + collect_blob_cids(&body, &mut wanted).map_err(|cid| { + XrpcError::bad_gateway(format!("a blob ref the store can't take: {cid}")) + })?; + } + if wanted.is_empty() { + return Ok(()); + } + let lfs = &state.lfs; + for cid in wanted { + let oid = LfsOid::from_digest(cid.digest()); + if probe_blob(Arc::clone(&lfs.handle.store), repo, &oid) + .await? + .is_some() + { + continue; + } + let mut url = foster.url("xrpc/com.atproto.sync.getBlob"); + url.query_pairs_mut() + .append_pair("did", repo.as_str()) + .append_pair("cid", &cid.to_string()); + let bytes = fetch_blob(state, &url, &cid).await?; + let claimed = ClaimedSize::new(bytes.len() as u64); + let permit = lfs.handle.admission.admit(claimed).map_err(|error| { + XrpcError::internal(format!( + "the blob store can't take the transferred bytes: {error}" + )) + })?; + let stored = Arc::clone(&lfs.handle.store); + let target = repo.clone(); + let owned = oid.clone(); + run_blocking(move || { + let outcome = stored.put(Pool::Attachments, &target, &owned, claimed, &mut &bytes[..]); + drop(permit); + outcome.map_err(|error| XrpcError::internal(error.to_string())) + }) + .await?; + if probe_blob(Arc::clone(&lfs.handle.store), repo, &oid) + .await? + .is_none() + { + return Err(XrpcError::internal(format!( + "the blob store didn't keep the bytes it just stored for {cid}" + ))); + } + } + Ok(()) +} + +async fn probe_blob( + store: Arc, + repo: &RepoDid, + oid: &LfsOid, +) -> Result, XrpcError> { + let target = repo.clone(); + let oid = oid.clone(); + run_blocking(move || { + store + .probe(Pool::Attachments, &target, &oid) + .map_err(|error| XrpcError::internal(error.to_string())) + }) + .await +} + +async fn fetch_blob( + state: &Arc>, + url: &Url, + cid: &BlobCid, +) -> Result { + let response = state + .git_http + .execute(HttpRequest::get(url.clone())) + .await + .map_err(|error| { + XrpcError::bad_gateway(format!( + "the foster knot's bytes for blob {cid} are unreachable: {error}" + )) + })?; + if response.status == StatusCode::TOO_MANY_REQUESTS { + return Err(XrpcError::throttled(format!( + "the foster knot is rate-limiting the blob fetch for {cid}" + ))); + } + if !response.status.is_success() { + return Err(XrpcError::bad_gateway(format!( + "the foster knot returned {} for blob {cid}", + response.status.as_u16() + ))); + } + if BlobCid::from_digest(Sha256Digest::hash(&response.body)) != *cid { + return Err(XrpcError::bad_gateway(format!( + "the bytes the foster knot served for {cid} don't hash to that cid" + ))); + } + Ok(response.body) +} + +#[derive(Serialize)] +struct AckBody<'a> { + #[serde(flatten)] + signed: crate::foster::AckSigned<'a>, + signature: String, +} + +async fn send_ack( + state: &Arc>, + foster: &FosterKnot, + repo: &RepoDid, + records: crate::foster::RecordCount, + superseded: &[RecordAddress], + now: UnixSeconds, +) -> Result<(), XrpcError> { + let signer = state.secrets.signer(&state.knot_did)?; + let signed = crate::foster::AckSigned { + context: crate::foster::ACK_CONTEXT, + repo_did: repo, + records, + acked_at: now, + superseded, + }; + let bytes = serde_ipld_dagcbor::to_vec(&signed) + .map_err(|error| XrpcError::internal(error.to_string()))?; + let ack = AckBody { + signed, + signature: URL_SAFE_NO_PAD.encode(signer.sign(&bytes).as_bytes()), + }; + let body: Bytes = serde_json::to_vec(&ack) + .map_err(|error| XrpcError::internal(error.to_string()))? + .into(); + let url = foster.url(ACK_ROUTE.trim_start_matches('/')); + let mut rejected = None; + let mut unreachable = None; + for (attempt, backoff) in ACK_BACKOFF_SECS.iter().enumerate() { + match state + .git_http + .execute(HttpRequest::post(url.clone(), body.clone())) + .await + { + Ok(response) if response.status.is_success() => return Ok(()), + Ok(response) => { + rejected = Some((response.status, response.body)); + unreachable = None; + } + Err(error) => { + unreachable = Some(error.to_string()); + rejected = None; + } + } + let semantic = rejected.as_ref().is_some_and(|(status, _)| { + *status != StatusCode::TOO_MANY_REQUESTS && status.as_u16() < 500 + }); + if semantic { + if attempt > 0 || knot_doc_is_ours(state, repo).await { + break; + } + await_knot_doc(state, repo).await; + } else { + tokio::time::sleep(std::time::Duration::from_secs(*backoff)).await; + } + } + Err(match rejected { + Some((StatusCode::TOO_MANY_REQUESTS, _)) => { + XrpcError::throttled("the foster knot is rate-limiting the ack") + } + Some((status, body)) => XrpcError::conflict(format!( + "the foster knot rejected the ack with {}: {}", + status.as_u16(), + String::from_utf8_lossy(&body) + )), + None => XrpcError::bad_gateway(format!( + "the foster knot wouldn't take the ack: {}", + unreachable.expect("every failed ack attempt leaves its reason") + )), + }) +} + +async fn knot_doc_is_ours( + state: &XrpcState, + repo: &RepoDid, +) -> bool { + state + .atproto + .resolve_knot_endpoint(repo) + .await + .is_ok_and(|endpoint| { + endpoint.origin().ascii_serialization() == state.knot_service_url.as_str() + }) +} + +async fn await_knot_doc(state: &XrpcState, repo: &RepoDid) { + for _ in 0..KNOT_DOC_POLLS { + if knot_doc_is_ours(state, repo).await { + return; + } + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + tracing::warn!( + repo = %repo, + "the repo's did document still doesn't point at this knot, so the rejected ack stands" + ) +} + +pub(crate) async fn get_json( + http: &dyn HttpTransport, + url: Url, + who: &str, +) -> Result { + let response = http + .execute(HttpRequest::get(url)) + .await + .map_err(|error| XrpcError::bad_gateway(format!("the {who} is unreachable: {error}")))?; + if response.status == StatusCode::TOO_MANY_REQUESTS { + return Err(XrpcError::throttled(format!( + "the {who} is rate-limiting the handover" + ))); + } + if !response.status.is_success() { + return Err(XrpcError::bad_gateway(format!( + "the {who} returned {}", + response.status.as_u16() + ))); + } + serde_json::from_slice(&response.body).map_err(|error| { + XrpcError::bad_gateway(format!("the {who} returned non-json bytes: {error}")) + }) +} + +pub fn recover(state: &XrpcState) -> Vec { + crate::imports::child_dirs(state.layout.scratch_dir()) + .into_iter() + .flat_map(|method| crate::imports::child_dirs(&method)) + .flat_map(|shard| crate::imports::child_dirs(&shard)) + .filter(|repo_path| !repo_path.join(SETTLED).exists()) + .filter_map(|repo_path| match HandoverJob::read(&repo_path) { + Some(Ok(job)) => { + if job.consecutive_failures >= QUARANTINE_AFTER_FAILURES { + tracing::info!( + repo = %job.repo, + consecutive_failures = job.consecutive_failures, + "the handover keeps failing, so its repo sits out this boot's replay; an admin trigger still drives it" + ); + return None; + } + matches!(state.index.owner_of(&job.repo), Resolved::Ready(Some(_))) + .then_some(job.repo) + } + Some(Err(error)) => { + tracing::error!(path = %repo_path.display(), %error, "couldn't decode the handover sidecar; leaving it alone"); + None + } + None => None, + }) + .collect() +} + +fn stage_all(state: &XrpcState, foster: &FosterKnot) -> usize { + state + .index + .native_repos() + .into_iter() + .filter_map(|repo| { + let path = state.layout.repo_path(&repo).ok()?; + (!(path.join(SIDECAR).exists() || path.join(SETTLED).exists())).then_some((repo, path)) + }) + .filter(|(repo, path)| { + HandoverJob { + repo: repo.clone(), + foster: foster.clone(), + triggered_at: state.now(), + error: None, + consecutive_failures: 0, + } + .write(path) + .map_err( + |error| tracing::error!(repo = %repo, %error, "couldn't stage the handover job"), + ) + .is_ok() + }) + .count() +} + +pub async fn run_boot( + state: Arc>, + shutdown: CancellationToken, + auto: Option, + boot_width: usize, +) { + let identities: IdentityCache = Arc::new(std::sync::Mutex::new(BTreeMap::new())); + if let Some(foster) = &auto { + match foster_key(&state, foster, &identities).await { + Ok(_) => { + let staged = stage_all(&state, foster); + if staged > 0 { + tracing::info!( + repos = staged, + "a configured foster knot means every hosted repo gets its shadowed records pulled home" + ); + } + } + Err(error) => tracing::error!( + %error, + "social.foster_url doesn't answer as a foster knot, so no handovers are staged this boot" + ), + } + } + let pending = recover(&state); + if pending.is_empty() { + return; + } + tracing::info!( + repos = pending.len(), + "handover jobs that a crash left half-finished will now replay" + ); + replay( + pending, + Drain::default(), + &state, + &identities, + &shutdown, + boot_width, + ) + .await +} + +#[derive(Default)] +struct Drain { + attempts: usize, + foster_down: usize, + pauses: usize, + rounds: usize, + parked: bool, +} + +async fn replay( + queue: Vec, + drain: Drain, + state: &Arc>, + identities: &IdentityCache, + shutdown: &CancellationToken, + boot_width: usize, +) { + if queue.is_empty() || drain.parked || shutdown.is_cancelled() { + return; + } + let (next, drain) = drain_pass(queue, drain, state, identities, shutdown, boot_width).await; + if drain.parked || next.is_empty() || drain.rounds >= RETRY_ROUNDS { + return; + } + tokio::select! { + () = tokio::time::sleep(std::time::Duration::from_secs(RETRY_PAUSE_SECS)) => {} + () = shutdown.cancelled() => return, + } + Box::pin(replay( + next, + Drain { + rounds: drain.rounds + 1, + ..drain + }, + state, + identities, + shutdown, + boot_width, + )) + .await +} + +async fn drain_pass( + queue: Vec, + drain: Drain, + state: &Arc>, + identities: &IdentityCache, + shutdown: &CancellationToken, + boot_width: usize, +) -> (Vec, Drain) { + use futures::StreamExt as _; + futures::stream::iter(queue.chunks(boot_width)) + .fold( + (Vec::new(), drain), + |(mut unfinished, drain), batch| async move { + if drain.parked || shutdown.is_cancelled() { + return (unfinished, drain); + } + let (mut batch_unfinished, drain) = + drain_batch(batch, drain, state, identities, shutdown).await; + unfinished.append(&mut batch_unfinished); + (unfinished, drain) + }, + ) + .await +} + +async fn drain_batch( + batch: &[RepoDid], + drain: Drain, + state: &Arc>, + identities: &IdentityCache, + shutdown: &CancellationToken, +) -> (Vec, Drain) { + let running = batch + .iter() + .filter_map(|repo| { + let lease = state.handover_sessions.acquire(repo)?; + Some(tokio::spawn(drive( + Arc::clone(state), + repo.clone(), + lease, + Arc::clone(identities), + ))) + }) + .collect::>(); + let folded = batch + .iter() + .cloned() + .zip(futures::future::join_all(running).await) + .fold(BatchFold::default(), |fold, (repo, outcome)| { + fold.absorbing(repo, outcome) + }); + let drain = Drain { + attempts: drain.attempts + folded.attempts, + foster_down: drain.foster_down + folded.foster_down, + ..drain + }; + if drain.attempts < FOSTER_DOWN_PARK_MIN_ATTEMPTS + || drain.foster_down * FOSTER_DOWN_PARK_DENOMINATOR <= drain.attempts + { + return (folded.unfinished, drain); + } + let Some(pause) = FOSTER_DOWN_BACKOFF_SECS.get(drain.pauses) else { + tracing::error!( + attempts = drain.attempts, + foster_down = drain.foster_down, + "the foster knot keeps answering with gateway errors, so the remaining handovers wait for the next boot" + ); + return ( + folded.unfinished, + Drain { + parked: true, + ..drain + }, + ); + }; + tracing::error!( + attempts = drain.attempts, + foster_down = drain.foster_down, + pause_secs = pause, + "the foster knot keeps answering with gateway errors; the handovers pause before they resume" + ); + tokio::select! { + () = tokio::time::sleep(std::time::Duration::from_secs(*pause)) => {} + () = shutdown.cancelled() => { + return (folded.unfinished, Drain { parked: true, ..drain }); + } + } + ( + folded.unfinished, + Drain { + pauses: drain.pauses + 1, + ..drain + }, + ) +} + +#[derive(Default)] +struct BatchFold { + unfinished: Vec, + attempts: usize, + foster_down: usize, +} + +impl BatchFold { + fn absorbing( + mut self, + repo: RepoDid, + outcome: Result, tokio::task::JoinError>, + ) -> Self { + match outcome { + Ok(Ok(_)) => self.attempts += 1, + Ok(Err(error)) => { + if error.status() == StatusCode::BAD_GATEWAY { + self.foster_down += 1; + } + self.attempts += 1; + self.unfinished.push(repo); + } + Err(error) => { + tracing::error!(%error, "a handover replay task died"); + } + } + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_foster_url_takes_https_or_loopback_http_with_a_host() { + assert!(FosterKnot::new("https://foster.nel.pet").is_ok()); + assert!(FosterKnot::new("http://127.0.0.1:7433").is_ok()); + assert!(FosterKnot::new("http://localhost:9110").is_ok()); + assert!(FosterKnot::new("http://foster.nel.pet").is_err()); + assert!(FosterKnot::new("ftp://foster.nel.pet").is_err()); + assert!(FosterKnot::new("not a url").is_err()); + assert!(FosterKnot::new("https://").is_err()); + } + + #[test] + fn a_repo_url_hangs_off_the_foster_base() { + let foster = FosterKnot::new("https://foster.nel.pet/").unwrap(); + let repo = RepoDid::new("did:plc:feralpearl").unwrap(); + assert_eq!( + foster.url(repo.as_str()).as_str(), + "https://foster.nel.pet/did:plc:feralpearl" + ); + } +} diff --git a/knot2/crates/knot-xrpc/src/imports.rs b/knot2/crates/knot-xrpc/src/imports.rs new file mode 100644 index 0000000000000000000000000000000000000000..b5fac0feadec95ce17d69e69b1c533dba311155d --- /dev/null +++ b/knot2/crates/knot-xrpc/src/imports.rs @@ -0,0 +1,505 @@ +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use axum::extract::State; +use axum::response::Response; +use http::HeaderMap; +use knot_git::RefRecord; +use knot_index::Resolved; +use knot_pack::UpstreamRefs; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{ObjectFormat, Oid, RefName, RepoDid, UnixSeconds}; +use serde::{Deserialize, Serialize}; +use tokio::sync::{mpsc, watch}; +use tokio::task::JoinSet; + +use crate::XrpcState; +use crate::body::SourceUrl; +use crate::error::XrpcError; +use crate::forks::{ForkSource, SourceKind}; +use crate::query::ValidatedQuery; +use crate::reads::{json, repo_not_found, warming}; + +pub(crate) const SIDECAR: &str = "tangled.import.json"; +pub(crate) const STATUS_ROUTE: &str = "/xrpc/org.tangled.repo.getImportStatus"; + +// the real error quotes upstream urls and knot internals, so only owners and admins get it +pub(crate) const GENERIC_REASON: &str = + "the import from the source didn't finish; the repo owner can see details"; + +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)] +#[serde(rename_all = "lowercase")] +pub(crate) enum ImportState { + Queued, + Fetching, + Present, + Partial, + Failed, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +struct StoredRef { + name: RefName, + target: Oid, +} + +// the sidecar stores the capability token ("sha1" or "sha256"), not the knot-types kind +mod format_serde { + use knot_types::ObjectFormat; + use serde::{Deserialize, Deserializer, Serializer}; + + pub(crate) fn serialize( + format: &ObjectFormat, + serializer: S, + ) -> Result { + serializer.serialize_str(format.capability()) + } + + pub(crate) fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result { + let token = String::deserialize(deserializer)?; + ObjectFormat::from_capability(&token) + .ok_or_else(|| serde::de::Error::custom(format!("unsupported object format {token}"))) + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +struct StoredRefs { + #[serde(with = "format_serde")] + object_format: ObjectFormat, + head_symref: Option, + head_target: Option, + refs: Vec, +} + +impl StoredRefs { + fn capture(refs: &UpstreamRefs) -> Self { + Self { + object_format: refs.object_format, + head_symref: refs.head_symref.clone(), + head_target: refs.head_target, + refs: refs + .refs + .iter() + .map(|record| StoredRef { + name: record.name.clone(), + target: record.target, + }) + .collect(), + } + } +} + +impl From for UpstreamRefs { + fn from(stored: StoredRefs) -> Self { + Self { + object_format: stored.object_format, + head_symref: stored.head_symref, + head_target: stored.head_target, + refs: stored + .refs + .into_iter() + .map(|record| RefRecord { + name: record.name, + target: record.target, + }) + .collect(), + } + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub(crate) struct ImportJob { + pub(crate) repo: RepoDid, + pub(crate) source: SourceUrl, + pub(crate) kind: SourceKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + refs: Option, + pub(crate) state: ImportState, + pub(crate) queued_at: UnixSeconds, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) finished_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) error: Option, +} + +impl ImportJob { + // the sidecar is written only by this module, so a sidecar that doesn't decode was + // truncated or edited outside; callers log it instead of treating the job as absent + pub(crate) fn read(repo_path: &Path) -> Option> { + std::fs::read(repo_path.join(SIDECAR)) + .ok() + .map(|bytes| serde_json::from_slice(&bytes)) + } + + fn write(&self, repo_path: &Path) -> std::io::Result<()> { + let staged = repo_path.join(format!("{SIDECAR}.tmp")); + std::fs::write(&staged, serde_json::to_vec(self)?)?; + std::fs::rename(&staged, repo_path.join(SIDECAR)) + } + + fn status(&self) -> &'static str { + match self.state { + ImportState::Queued | ImportState::Fetching => "fetching", + ImportState::Present => "present", + ImportState::Partial => "partial", + ImportState::Failed => "failed", + } + } +} + +#[derive(Deserialize)] +pub(crate) struct StatusParams { + pub(crate) repo: RepoDid, +} + +#[derive(Serialize)] +struct StatusOut { + #[serde(skip_serializing_if = "Option::is_none")] + status: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option, +} + +pub(crate) async fn import_status( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = params.repo; + let owner = match state.index.owner_of(&did) { + Resolved::Ready(Some(owner)) => owner, + Resolved::Ready(None) => return Err(repo_not_found()), + Resolved::Warming => return Err(warming()), + }; + let caller = state + .authenticate_optional_for_repo(&headers, &method, &did) + .await?; + let trusted = caller + .as_ref() + .is_some_and(|caller| caller.as_str() == owner.as_str() || state.admins.contains(caller)); + let job = state + .layout + .repo_path(&did) + .ok() + .and_then(|path| ImportJob::read(&path)) + .and_then(Result::ok); + let out = StatusOut { + status: job.as_ref().map(ImportJob::status), + reason: job.and_then(|job| job.error).map(|reason| match trusted { + true => reason, + false => GENERIC_REASON.to_string(), + }), + }; + json(out, state.byte_limits.response) +} +pub struct ImportQueue { + wake: mpsc::UnboundedSender, + processed: AtomicUsize, +} + +impl ImportQueue { + pub fn channel() -> (Arc, mpsc::UnboundedReceiver) { + let (wake, jobs) = mpsc::unbounded_channel(); + ( + Arc::new(Self { + wake, + processed: AtomicUsize::new(0), + }), + jobs, + ) + } + + pub(crate) fn wake(&self, repo: RepoDid) { + let _ = self.wake.send(repo); + } + + // how many wakes the runner has finished processing; tests await this instead of + // guessing how long a fetch takes + pub fn processed(&self) -> usize { + self.processed.load(Ordering::SeqCst) + } +} + +pub(crate) fn stage( + state: &XrpcState, + repo: &RepoDid, + source: &SourceUrl, + kind: SourceKind, + refs: &UpstreamRefs, +) -> Result<(), crate::XrpcError> { + let repo_path = state + .layout + .repo_path(repo) + .map_err(|error| crate::XrpcError::internal(error.to_string()))?; + let job = ImportJob { + repo: repo.clone(), + source: source.clone(), + kind, + refs: Some(StoredRefs::capture(refs)), + state: ImportState::Queued, + queued_at: state.now(), + finished_at: None, + error: None, + }; + job.write(&repo_path) + .map_err(|error| crate::XrpcError::internal(format!("writing import job: {error}"))) +} + +// a partial repo is missing objects, so re-staging the fetch retries it instead of erroring +fn retryable(state: ImportState) -> bool { + matches!(state, ImportState::Failed | ImportState::Partial) +} + +pub(crate) async fn retry_failed( + state: &Arc>, + repo: &RepoDid, + source: &SourceUrl, + kind: SourceKind, +) -> Result { + let repo_path = state + .layout + .repo_path(repo) + .map_err(|error| crate::XrpcError::internal(error.to_string()))?; + { + let _guard = state.cob_locks.repo(repo); + if !ImportJob::read(&repo_path).is_some_and(|job| job.is_ok_and(|job| retryable(job.state))) + { + return Ok(false); + } + } + let upstream = ForkSource::resolve(state, source).await?; + { + let _guard = state.cob_locks.repo(repo); + if !ImportJob::read(&repo_path).is_some_and(|job| job.is_ok_and(|job| retryable(job.state))) + { + return Ok(false); + } + stage(state, repo, source, kind, &upstream.refs)?; + } + state.imports.wake(repo.clone()); + Ok(true) +} + +async fn populate_from( + state: &Arc>, + repo: &RepoDid, + upstream: &ForkSource, + kind: SourceKind, +) -> Result, crate::XrpcError> { + let layout = state.layout.clone(); + let placed = repo.clone(); + let wanted = upstream.refs.object_format; + let held = crate::run_blocking(move || { + layout + .open(&placed) + .map(|repo| repo.object_format()) + .map_err(|error| crate::XrpcError::internal(error.to_string())) + }) + .await?; + if held != wanted { + return Err(crate::XrpcError::invalid_request( + "the source changed its object format after this repository was created", + )); + } + crate::repos::populate(state, repo, upstream, kind).await +} + +// the job keeps its previous on-disk state when this write fails, so the log line is the +// only trace of the divergence +fn persist(job: &ImportJob, repo_path: &Path) { + if let Err(error) = job.write(repo_path) { + tracing::error!( + repo = %job.repo, + %error, + "couldn't record the import state; the sidecar keeps its previous on-disk state" + ); + } +} + +fn fail(repo_path: &Path, job: &mut ImportJob, reason: String, now: UnixSeconds) { + job.state = ImportState::Failed; + job.error = Some(reason); + job.finished_at = Some(now); + persist(job, repo_path); +} + +pub(crate) fn hosted(state: &XrpcState, repo: &RepoDid) -> bool { + matches!(state.index.owner_of(repo), Resolved::Ready(Some(_))) +} + +pub(crate) async fn run_one( + state: &Arc>, + repo: &RepoDid, +) { + let Ok(repo_path) = state.layout.repo_path(repo) else { + return; + }; + let mut job = { + let _guard = state.cob_locks.repo(repo); + if !hosted(state, repo) { + return; + } + let mut job = match ImportJob::read(&repo_path) { + Some(Ok(job)) => job, + Some(Err(error)) => { + tracing::error!( + repo = %repo, + %error, + "couldn't decode the import sidecar; the job keeps its previous state" + ); + return; + } + None => return, + }; + if job.state != ImportState::Queued { + return; + } + job.state = ImportState::Fetching; + job.error = None; + if let Err(error) = job.write(&repo_path) { + tracing::error!( + repo = %job.repo, + %error, + "couldn't stage the import fetch; the wake is spent and the job stays queued until the next restart" + ); + return; + } + job + }; + let source = job.source.clone(); + let upstream = match job.refs.take() { + Some(refs) => crate::forks::resolve_upstream(state, &source).map(|upstream| ForkSource { + origin: source.clone(), + upstream, + refs: refs.into(), + }), + None => ForkSource::resolve(state, &source).await, + }; + let outcome = match upstream { + Ok(upstream) => populate_from(state, repo, &upstream, job.kind).await, + Err(error) => Err(error), + }; + let _guard = state.cob_locks.repo(repo); + if !hosted(state, repo) { + return; + } + let now = state.now(); + let landed = match outcome { + Ok(missing) => { + job.finished_at = Some(now); + if missing.is_empty() { + job.state = ImportState::Present; + job.error = None; + } else { + job.state = ImportState::Partial; + job.error = Some(format!("{} lfs objects are still missing", missing.len())); + } + persist(&job, &repo_path); + true + } + Err(error) => { + fail(&repo_path, &mut job, error.to_string(), now); + false + } + }; + drop(_guard); + if landed { + // the arrived refs go onto the record surface the way a push would + let state = Arc::clone(state); + let placed = repo.clone(); + let _ = crate::run_blocking(move || { + let git = state.layout.open(&placed)?; + crate::refrecords::sync_refs(&state, &git, &placed, &[]); + Ok(()) + }) + .await; + } +} + +pub(crate) fn child_dirs(path: &Path) -> Vec { + std::fs::read_dir(path) + .into_iter() + .flatten() + .flatten() + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .collect() +} + +pub(crate) fn recover_jobs(state: &XrpcState) -> Vec { + let now = state.now(); + let mut queued = Vec::new(); + for repo_path in child_dirs(state.layout.scratch_dir()) + .into_iter() + .flat_map(|method| child_dirs(&method)) + .flat_map(|shard| child_dirs(&shard)) + { + let mut job = match ImportJob::read(&repo_path) { + Some(Ok(job)) => job, + Some(Err(error)) => { + tracing::error!( + path = %repo_path.display(), + %error, + "couldn't decode the import sidecar; leaving it alone" + ); + continue; + } + None => continue, + }; + match job.state { + ImportState::Queued => queued.push(job.repo.clone()), + ImportState::Fetching => fail( + &repo_path, + &mut job, + "the knot restarted while this repository was being fetched".to_string(), + now, + ), + ImportState::Present | ImportState::Partial | ImportState::Failed => {} + } + } + queued +} + +pub(crate) fn recover(state: &XrpcState) { + recover_jobs(state) + .into_iter() + .for_each(|repo| state.imports.wake(repo)); +} + +const MAX_CONCURRENT_IMPORTS: usize = 4; + +pub async fn run( + state: Arc>, + mut wake: mpsc::UnboundedReceiver, + mut shutdown: watch::Receiver, +) { + recover(&state); + let mut workers = JoinSet::new(); + loop { + tokio::select! { + repo = wake.recv(), if workers.len() < MAX_CONCURRENT_IMPORTS => match repo { + Some(repo) => { + let state = Arc::clone(&state); + let queue = Arc::clone(&state.imports); + workers.spawn(async move { + run_one(&state, &repo).await; + queue.processed.fetch_add(1, Ordering::SeqCst); + }); + } + None => { + workers.shutdown().await; + return; + } + }, + _ = workers.join_next(), if !workers.is_empty() => {} + _ = shutdown.changed() => { + workers.abort_all(); + return; + }, + } + } +} diff --git a/knot2/crates/knot-xrpc/src/invite_codes.rs b/knot2/crates/knot-xrpc/src/invite_codes.rs new file mode 100644 index 0000000000000000000000000000000000000000..e0dbd02a20a6f4e7d6cde00c9e7a584a61491627 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/invite_codes.rs @@ -0,0 +1,175 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use axum::body::Bytes; +use axum::extract::State; +use axum::response::Response; +use http::{HeaderMap, StatusCode}; +use serde::Deserialize; + +use knot_cobs::{Grant, MembersChange}; +use knot_index::Resolved; +use knot_runtime::{Clock, Entropy, HttpTransport}; +use knot_types::{KnotHostname, UnixSeconds}; + +use crate::cob::Authorized; +use crate::error::XrpcError; +use crate::members::commit_members; +use crate::{XrpcState, current_member, decode, ok_empty}; + +pub(crate) const REDEEM_ROUTE: &str = "/xrpc/org.tangled.temp.server.redeemInviteCode"; + +pub const LANDING_PATH: &str = "/invite"; + +const TTL_SECS: i64 = 2 * 24 * 60 * 60; + +const CODE_BYTES: usize = 16; + +#[derive(Debug, Default)] +pub struct InviteCodes { + live: Mutex>, +} + +impl InviteCodes { + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.live + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + pub fn mint(&self, entropy: &dyn Entropy, now: UnixSeconds) -> (String, UnixSeconds) { + let mut bytes = [0u8; CODE_BYTES]; + entropy.fill(&mut bytes); + let code: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); + let expires_at = now.saturating_add_secs(TTL_SECS); + let mut live = self.lock(); + live.retain(|_, expiry| *expiry > now); + live.insert(code.clone(), expires_at); + (code, expires_at) + } + + pub fn is_live(&self, code: &str, now: UnixSeconds) -> bool { + self.lock().get(code).is_some_and(|expiry| *expiry > now) + } + + fn redeem(&self, code: &str, now: UnixSeconds) -> bool { + self.lock().remove(code).is_some_and(|expiry| expiry > now) + } +} + +pub(crate) fn invite_url(hostname: &KnotHostname, code: &str) -> String { + format!("https://{}{LANDING_PATH}/{code}", hostname.as_str()) +} + +#[derive(Deserialize)] +struct CodeInput { + code: String, +} + +pub(crate) async fn redeem_invite_code( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate_write(&headers, &method).await?; + let CodeInput { code } = decode(&body)?; + + let now = state.now(); + if !state.invite_codes.redeem(&code, now) { + return Err(XrpcError::named( + StatusCode::BAD_REQUEST, + "InviteNotFound", + "this invite code is unknown, already spent, or expired", + )); + } + + match current_member(&state, &actor).await? { + Resolved::Warming => { + return Err(crate::reads::warming_for("member")); + } + Resolved::Ready(true) => return Ok(ok_empty()), + Resolved::Ready(false) => {} + } + let change = MembersChange::Add(Grant { + subject: actor.clone(), + added_by: state.service_owner.clone(), + created_at: now, + }); + + tracing::info!(route = REDEEM_ROUTE, %actor, "an invite code was redeemed"); + commit_members(&state, Authorized::by_decree(change), now).await +} + +#[cfg(test)] +mod tests { + use super::*; + + use knot_runtime::SeededEntropy; + + fn at(seconds: i64) -> UnixSeconds { + UnixSeconds::new(seconds) + } + + fn entropy() -> SeededEntropy { + SeededEntropy::new(7) + } + + #[test] + fn a_code_redeems_once_and_never_again() { + let links = InviteCodes::default(); + let (code, expires_at) = links.mint(&entropy(), at(1_000)); + + assert_eq!( + expires_at, + at(1_000 + TTL_SECS), + "a fresh code expires two days out" + ); + assert!(links.is_live(&code, at(1_000)), "a fresh code is live"); + assert!(links.redeem(&code, at(1_000)), "the first redeem spends it"); + assert!( + !links.redeem(&code, at(1_000)), + "a spent code must not admit a second account" + ); + assert!(!links.is_live(&code, at(1_000)), "a spent code is not live"); + } + + #[test] + fn a_code_past_its_expiry_neither_reads_as_live_nor_redeems() { + let links = InviteCodes::default(); + let (code, expires_at) = links.mint(&entropy(), at(1_000)); + let expired = expires_at.saturating_add_secs(1); + + assert!(!links.is_live(&code, expired)); + assert!(!links.redeem(&code, expired)); + } + + #[test] + fn minting_sweeps_codes_that_have_already_expired() { + let links = InviteCodes::default(); + // one source across both mints: a fresh SeededEntropy would replay the + // same bytes and the "stale" code would be the one just reinserted + let source = entropy(); + let (stale, stale_expiry) = links.mint(&source, at(1_000)); + let later = stale_expiry.saturating_add_secs(1); + + let (fresh, _) = links.mint(&source, later); + assert!( + !links.lock().contains_key(&stale), + "minting left an expired code in the map" + ); + assert!(links.is_live(&fresh, later)); + } + + #[test] + fn two_codes_from_one_source_differ_and_are_full_length() { + let links = InviteCodes::default(); + let source = entropy(); + let (first, _) = links.mint(&source, at(1)); + let (second, _) = links.mint(&source, at(1)); + + assert_ne!(first, second, "a repeated code would admit twice"); + assert_eq!(first.len(), CODE_BYTES * 2, "the code is hex over 128 bits"); + assert!(first.chars().all(|c| c.is_ascii_hexdigit())); + } +} diff --git a/knot2/crates/knot-xrpc/src/keep_sweep.rs b/knot2/crates/knot-xrpc/src/keep_sweep.rs new file mode 100644 index 0000000000000000000000000000000000000000..4a17011d0836b9622840889b724ad35e427d1b87 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/keep_sweep.rs @@ -0,0 +1,241 @@ +use std::collections::BTreeSet; +use std::io::Write as _; +use std::sync::Arc; + +use futures::StreamExt as _; +use knot_record::pull::{PATCH_COLLECTION, patch_collection}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{Oid, RecordAddress, RecordRkey, RepoDid}; +use tokio_util::sync::CancellationToken; + +use crate::error::XrpcError; +use crate::{XrpcState, run_blocking}; + +// " " per line. a patch head listed here is never retried +const FAILED: &str = "tangled.keep.failed"; + +struct Unpinned { + address: RecordAddress, + head: Oid, + fork: Option, +} + +pub async fn run( + state: Arc>, + shutdown: CancellationToken, + width: usize, +) { + let repos: Vec = state + .index + .native_repos() + .into_iter() + .filter(|repo| { + state + .layout + .repo_path(repo) + .is_ok_and(|path| path.join(crate::handover::SETTLED).exists()) + }) + .collect(); + tracing::info!( + repos = repos.len(), + "the keep sweep pins the head of every patch in the settled repos" + ); + let total = repos.len(); + let (swept, pinned, failed) = futures::stream::iter(repos) + .map(|repo| { + let state = &state; + let shutdown = &shutdown; + async move { + if shutdown.is_cancelled() { + return None; + } + Some(sweep_repo(state, &repo).await) + } + }) + .buffer_unordered(width) + .fold((0, 0, 0), |(swept, pinned, failed), outcome| async move { + let Some((repo_pinned, repo_failed)) = outcome else { + return (swept, pinned, failed); + }; + let swept = swept + 1; + if swept % 100 == 0 { + tracing::info!(swept, total, "the keep sweep is still going"); + } + (swept, pinned + repo_pinned, failed + repo_failed) + }) + .await; + tracing::info!(swept, total, pinned, failed, "the keep sweep finished"); +} + +pub(crate) async fn sweep_repo( + state: &Arc>, + repo: &RepoDid, +) -> (usize, usize) { + let unpinned = match unpinned(state, repo).await { + Ok(unpinned) => unpinned, + Err(error) => { + tracing::warn!(repo = %repo, %error, "the keep sweep couldn't read the repo's patches"); + return (0, 0); + } + }; + if unpinned.is_empty() { + return (0, 0); + } + tracing::info!(repo = %repo, patches = unpinned.len(), "the keep sweep pins unpinned patch heads"); + let (mut pinned, mut failed) = (0, 0); + for patch in unpinned { + match pin(state, repo, &patch).await { + Ok(fetched) => { + pinned += 1; + tracing::info!( + repo = %repo, + patch = %patch.address.rkey().as_str(), + head = %patch.head, + fork = ?patch.fork.as_ref().map(RepoDid::as_str), + fetched, + "the keep sweep pinned a patch head" + ); + } + Err(PinError::Retry(error)) => { + failed += 1; + tracing::warn!( + repo = %repo, + patch = %patch.address.rkey().as_str(), + head = %patch.head, + %error, + "the keep sweep couldn't pin a patch head, so it'll try again on the next boot" + ); + } + Err(PinError::Gone(error)) => { + failed += 1; + tracing::warn!( + repo = %repo, + patch = %patch.address.rkey().as_str(), + head = %patch.head, + %error, + "the keep sweep can never pin this head, so it won't try again" + ); + remember_failure(state, repo, &patch) + .inspect_err(|error| { + tracing::error!(repo = %repo, %error, "the keep sweep couldn't record a failed head") + }) + .ok(); + } + } + } + tracing::info!(repo = %repo, pinned, failed, "the keep sweep finished the repo"); + (pinned, failed) +} + +async fn unpinned( + state: &Arc>, + repo: &RepoDid, +) -> Result, XrpcError> { + let state = Arc::clone(state); + let repo = repo.clone(); + run_blocking(move || { + let path = state + .layout + .repo_path(&repo) + .map_err(|error| XrpcError::internal(error.to_string()))?; + let failed = std::fs::read_to_string(path.join(FAILED)).unwrap_or_default(); + let failed: BTreeSet<&str> = failed.lines().collect(); + let found = crate::atproto::AtprotoRepo { + did: repo.clone(), + repo: state.layout.open(&repo)?, + }; + let mut unpinned = Vec::new(); + for (rkey, value) in crate::atproto::records_in(&found, PATCH_COLLECTION)? { + let Some(head) = value["head"]["oid"] + .as_str() + .and_then(|head| Oid::from_hex(head).ok()) + else { + continue; + }; + if failed.contains(format!("{} {head}", rkey.as_str()).as_str()) { + continue; + } + let Ok(rkey) = RecordRkey::new(rkey.as_str()) else { + continue; + }; + let address = RecordAddress::new(patch_collection(), rkey); + let name = crate::git::keep_ref(&repo, &address, head)?; + if found.repo.find_ref(&name)? == Some(head) { + continue; + } + let fork = value["source"]["repo"] + .as_str() + .and_then(|source| RepoDid::new(source).ok()) + .filter(|source| *source != repo); + unpinned.push(Unpinned { + address, + head, + fork, + }); + } + Ok(unpinned) + }) + .await +} + +enum PinError { + Gone(XrpcError), + Retry(XrpcError), +} + +async fn pin( + state: &Arc>, + repo: &RepoDid, + patch: &Unpinned, +) -> Result { + let head = patch.head; + let local = { + let state = Arc::clone(state); + let repo = repo.clone(); + run_blocking(move || Ok(state.layout.open(&repo)?.contains(head))) + .await + .map_err(PinError::Retry)? + }; + if !local { + let Some(fork) = &patch.fork else { + return Err(PinError::Gone(XrpcError::not_found(format!( + "the head {head} isn't in this repo and the patch has no fork to fetch it from" + )))); + }; + crate::forks::fetch_commit(state, repo, fork, head) + .await + .map_err(PinError::Retry)?; + } + let state = Arc::clone(state); + let repo = repo.clone(); + let address = patch.address.clone(); + run_blocking(move || { + let _guard = state.cob_locks.repo(&repo); + let git = state.layout.open(&repo)?; + git.find_commit(head)?; + git.set_reference(&crate::git::keep_ref(&repo, &address, head)?, head)?; + Ok(!local) + }) + .await + .map_err(PinError::Retry) +} + +fn remember_failure( + state: &XrpcState, + repo: &RepoDid, + patch: &Unpinned, +) -> std::io::Result<()> { + let path = state + .layout + .repo_path(repo) + .map_err(|error| std::io::Error::other(error.to_string()))?; + writeln!( + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path.join(FAILED))?, + "{} {}", + patch.address.rkey().as_str(), + patch.head + ) +} diff --git a/knot2/crates/knot-xrpc/src/legacy_admin.rs b/knot2/crates/knot-xrpc/src/legacy_admin.rs new file mode 100644 index 0000000000000000000000000000000000000000..0575a5453b2fedfb3571c0df9d7409796ef842db --- /dev/null +++ b/knot2/crates/knot-xrpc/src/legacy_admin.rs @@ -0,0 +1,219 @@ +use std::sync::Arc; + +use axum::extract::{DefaultBodyLimit, State}; +use axum::middleware::from_fn_with_state; +use axum::response::{IntoResponse, Response}; +use axum::routing::post; +use axum::{Json, Router}; +use http::HeaderMap; +use serde_json::json; +use subtle::ConstantTimeEq; +use zeroize::Zeroizing; + +use knot_runtime::{Clock, HttpTransport}; +use knot_types::RepoDid; +use serde::Deserialize; + +use crate::error::XrpcError; +use crate::invite_codes::invite_url; +use crate::{XrpcState, basic_credentials, enforce_pre_auth_limit}; + +pub const CREATE_INVITE_CODE_ROUTE: &str = "/xrpc/org.tangled.temp.server.createInviteCode"; +pub const REPAIR_HANDOVER_ROUTE: &str = "/xrpc/org.tangled.temp.server.repairHandover"; + +const BASIC_USER: &str = "admin"; + +#[derive(Debug, thiserror::Error)] +#[error("legacy admin secret mustn't be empty")] +pub struct EmptySecret; + +#[derive(Clone)] +pub struct LegacyAdminSecret(Zeroizing); + +pub(crate) enum LegacyAdminAttempt { + NotBasic, + Admitted, + Denied, +} + +impl LegacyAdminSecret { + pub fn new(value: &str) -> Result { + let value = value.trim(); + match value.is_empty() { + true => Err(EmptySecret), + false => Ok(Self(Zeroizing::new(value.to_string()))), + } + } + + pub(crate) fn attempt(&self, headers: &HeaderMap) -> LegacyAdminAttempt { + let Some(value) = headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + else { + return LegacyAdminAttempt::NotBasic; + }; + let Some((scheme, _)) = value.split_once(' ') else { + return LegacyAdminAttempt::NotBasic; + }; + if !scheme.eq_ignore_ascii_case("Basic") { + return LegacyAdminAttempt::NotBasic; + } + if basic_credentials(value).is_some_and(|credentials| { + credentials.user.matches(BASIC_USER) + && bool::from(credentials.password.as_bytes().ct_eq(self.0.as_bytes())) + }) { + LegacyAdminAttempt::Admitted + } else { + LegacyAdminAttempt::Denied + } + } + + pub(crate) fn authorize(&self, headers: &HeaderMap) -> Result<(), XrpcError> { + match self.attempt(headers) { + LegacyAdminAttempt::Admitted => Ok(()), + _ => Err(XrpcError::auth_required("invalid admin creds")), + } + } +} + +struct LegacyAdmin { + state: Arc>, +} + +pub fn router(state: Arc>) -> Router { + let limits = state.byte_limits.body.get(); + let limiter = Arc::clone(&state); + Router::new() + .route(CREATE_INVITE_CODE_ROUTE, post(create_invite_code::)) + .route(REPAIR_HANDOVER_ROUTE, post(repair_handover::)) + .layer(DefaultBodyLimit::max(limits)) + .layer(from_fn_with_state(limiter, enforce_pre_auth_limit::)) + .with_state(Arc::new(LegacyAdmin { state })) +} + +async fn create_invite_code( + State(admin): State>>, + headers: HeaderMap, +) -> Result { + admin.state.legacy_admin_authorize(&headers)?; + let state = &admin.state; + let (code, expires_at) = state.invite_codes.mint(&*state.entropy, state.now()); + tracing::info!( + route = CREATE_INVITE_CODE_ROUTE, + %expires_at, + "minted an invite code" + ); + Ok(Json(json!({ + "code": code, + "url": invite_url(&state.knot_hostname, &code), + "expiresAt": crate::out::rfc3339(expires_at, 0), + })) + .into_response()) +} + +#[derive(Deserialize)] +struct RepairInput { + repo: Option, +} + +/// One repo is repaired in the request; without one, every repo is swept in the background. +async fn repair_handover( + State(admin): State>>, + headers: HeaderMap, + body: axum::body::Bytes, +) -> Result { + admin.state.legacy_admin_authorize(&headers)?; + let input: RepairInput = match body.is_empty() { + true => RepairInput { repo: None }, + false => crate::decode(&body)?, + }; + let Some(repo) = input.repo else { + tokio::spawn(crate::handover::repair_all(Arc::clone(&admin.state))); + return Ok(Json(json!({ "sweeping": true })).into_response()); + }; + let records = crate::handover::repair(&admin.state, &repo).await?; + Ok(Json(json!({ + "repo": repo, + "records": records.map(crate::foster::RecordCount::get), + })) + .into_response()) +} + +#[cfg(test)] +mod tests { + use super::*; + + use base64::Engine; + + fn header(value: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + http::header::AUTHORIZATION, + value.parse().expect("header is ascii"), + ); + headers + } + + fn basic(scheme: &str, user: &str, password: &str) -> HeaderMap { + let encoded = base64::engine::general_purpose::STANDARD + .encode(format!("{user}:{password}").as_bytes()); + header(&format!("{scheme} {encoded}")) + } + + #[test] + fn a_trimmed_secret_admits_only_the_admin_user_sending_it_exactly() { + assert!(LegacyAdminSecret::new(" ").is_err()); + let secret = LegacyAdminSecret::new("\tnekomilk2\n").expect("secret is non-empty"); + + let admitted = + ["Basic", "basic", "BASIC"].map(|scheme| basic(scheme, "admin", "nekomilk2")); + let refused = [ + basic("Basic", "admin", "\tnekomilk2\n"), + basic("Basic", "admin", "nope"), + basic("Basic", "root", "nekomilk2"), + basic("Basic", "admin", ""), + header("Basic !!!not-base64!!!"), + header("Bearer nekomilk2"), + header("nekomilk2"), + HeaderMap::new(), + ]; + assert!( + admitted + .iter() + .all(|headers| secret.authorize(headers).is_ok()), + "authorize admits the trimmed secret under any case of the Basic scheme" + ); + assert!( + refused + .iter() + .all(|headers| secret.authorize(headers).is_err()), + "authorize refuses an untrimmed, wrong, empty or malformed credential" + ); + } + + #[test] + fn attempt_can_tell_a_basic_header_from_bearer_or_missing() { + let secret = LegacyAdminSecret::new("nekomilk2").expect("secret is non-empty"); + + assert!(matches!( + secret.attempt(&basic("Basic", "admin", "nekomilk2")), + LegacyAdminAttempt::Admitted + )); + assert!(matches!( + secret.attempt(&basic("Basic", "admin", "nope")), + LegacyAdminAttempt::Denied + )); + assert!(matches!( + secret.attempt(&header("Basic !!!not-base64!!!")), + LegacyAdminAttempt::Denied + )); + assert!(matches!( + secret.attempt(&header("Bearer nekomilk2")), + LegacyAdminAttempt::NotBasic + )); + assert!(matches!( + secret.attempt(&HeaderMap::new()), + LegacyAdminAttempt::NotBasic + )); + } +} diff --git a/knot2/crates/knot-xrpc/src/lfs.rs b/knot2/crates/knot-xrpc/src/lfs.rs new file mode 100644 index 0000000000000000000000000000000000000000..b429a6479d7d39ea1252213dbfd7edfe6103caa5 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/lfs.rs @@ -0,0 +1,1129 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::rejection::BytesRejection; +use axum::extract::{DefaultBodyLimit, Path, Request, State}; +use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use knot_lfs::{ + BATCH_MEDIA_TYPE, BatchAction, BatchActions, BatchObject, BatchObjectError, BatchOperation, + BatchRequest, BatchResponse, BatchResponseObject, ClaimedSize, LfsError, LfsHandle, LfsOid, + LfsSize, LfsStore, MAX_BATCH_OBJECTS, Pool, UploadAdmission, +}; +use knot_pack::{HaveOids, SocketPeer, WantOids}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{HttpStatus, KnotServiceUrl, RepoDid}; +use serde_json::json; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch}; +use tower::ServiceExt; +use tower_http::services::ServeFile; +use url::Url; + +use crate::XrpcState; +use crate::forks::Upstream; + +pub const MAX_BATCH_BYTES: usize = 1024 * 1024; + +const IMMUTABLE_CACHE: &str = "public, max-age=31536000, immutable"; + +const READINESS_PROBE_TIMEOUT: Duration = Duration::from_secs(5); + +const PUSH_DENIED: &str = "you aren't authorized to push to this repository"; + +struct Readiness { + result: watch::Sender>, + probing: Mutex, +} + +pub struct LfsWeb { + pub handle: LfsHandle, + downloads: Arc, + readiness: Arc, +} + +impl LfsWeb { + pub fn new(handle: LfsHandle, max_downloads: usize) -> Self { + let (result, _rx) = watch::channel(None); + Self { + handle, + downloads: Arc::new(Semaphore::new(max_downloads)), + readiness: Arc::new(Readiness { + result, + probing: Mutex::new(false), + }), + } + } + + pub async fn ready(&self) -> bool { + let mut rx = self.readiness.result.subscribe(); + let launch = { + let mut probing = self.readiness.probing.lock().unwrap(); + match *probing { + false => { + *probing = true; + self.readiness.result.send_replace(None); + true + } + true => false, + } + }; + if launch { + Self::spawn_probe(Arc::clone(&self.readiness), Arc::clone(&self.handle.store)); + } + let settled = async { + match rx.wait_for(Option::is_some).await { + Ok(seen) => seen.unwrap_or(false), + Err(_) => false, + } + }; + tokio::time::timeout(READINESS_PROBE_TIMEOUT + Duration::from_secs(1), settled) + .await + .unwrap_or(false) + } + + fn spawn_probe(readiness: Arc, store: Arc) { + tokio::spawn(async move { + let mut guard = ProbeGuard { + readiness, + outcome: false, + }; + let joined = tokio::task::spawn_blocking(move || store.probe_ready()).await; + let ok = matches!(joined, Ok(Ok(()))); + if !ok { + tracing::warn!("lfs store readiness probe failed, reporting unready"); + } + guard.outcome = ok; + }); + } +} + +struct ProbeGuard { + readiness: Arc, + outcome: bool, +} + +impl Drop for ProbeGuard { + fn drop(&mut self) { + *self.readiness.probing.lock().unwrap() = false; + self.readiness.result.send_replace(Some(self.outcome)); + } +} + +pub(crate) fn routes() -> Router>> { + let batch = Router::new() + .route( + "/{did}/{name}/info/lfs/objects/batch", + post(batch_named::), + ) + .route("/{did}/info/lfs/objects/batch", post(batch_did::)) + .layer(DefaultBodyLimit::max(MAX_BATCH_BYTES)); + let objects = Router::new() + .route( + "/{did}/{name}/info/lfs/objects/{oid}", + get(object_named::).put(object_upload_named::), + ) + .route( + "/{did}/info/lfs/objects/{oid}", + get(object_did::).put(object_upload_did::), + ) + .layer(DefaultBodyLimit::disable()); + batch.merge(objects) +} + +fn fail(status: StatusCode, message: &str) -> Response { + ( + status, + [( + header::CONTENT_TYPE, + HeaderValue::from_static(BATCH_MEDIA_TYPE), + )], + json!({ "message": message }).to_string(), + ) + .into_response() +} + +fn lfs_error(error: crate::XrpcError) -> Box { + Box::new(fail(error.status(), &error.to_string())) +} + +fn resolve_did( + state: &XrpcState, + segment: &crate::RepoDidSegment, +) -> Result> { + crate::resolve_repo_did(state, segment).map_err(lfs_error) +} + +async fn resolve_named( + state: &XrpcState, + owner: &crate::OwnerSegment, + name: &crate::RepoNameSegment, +) -> Result> { + crate::resolve_repo_named(state, owner, name) + .await + .map_err(lfs_error) +} + +#[derive(serde::Deserialize)] +#[serde(transparent)] +struct OidSegment(String); + +impl OidSegment { + fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(serde::Deserialize)] +struct RepoObjectParams { + did: crate::OwnerSegment, + name: crate::RepoNameSegment, + oid: OidSegment, +} + +#[derive(serde::Deserialize)] +struct DidObjectParams { + did: crate::RepoDidSegment, + oid: OidSegment, +} + +async fn batch_named( + State(state): State>>, + Path(crate::RepoPathParams { did, name }): Path, + peer: SocketPeer, + headers: HeaderMap, + body: Result, +) -> Response { + let repo = match resolve_named(&state, &did, &name).await { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_batch( + &state, + repo, + &format!("{}/{}", did.as_str(), name.as_str()), + peer, + &headers, + body, + ) + .await +} + +async fn batch_did( + State(state): State>>, + Path(did): Path, + peer: SocketPeer, + headers: HeaderMap, + body: Result, +) -> Response { + let repo = match resolve_did(&state, &did) { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_batch(&state, repo, did.as_str(), peer, &headers, body).await +} + +async fn serve_batch( + state: &XrpcState, + repo: RepoDid, + path_prefix: &str, + peer: SocketPeer, + headers: &HeaderMap, + body: Result, +) -> Response { + let lfs = &state.lfs; + let body = match body { + Ok(body) => body, + Err(rejection) => return fail(rejection.status(), &rejection.body_text()), + }; + let request: BatchRequest = match serde_json::from_slice(&body) { + Ok(request) => request, + Err(error) => { + return fail( + StatusCode::UNPROCESSABLE_ENTITY, + &format!("invalid batch request: {error}"), + ); + } + }; + if request.objects.len() > MAX_BATCH_OBJECTS { + return fail( + StatusCode::UNPROCESSABLE_ENTITY, + &format!("batch exceeds {MAX_BATCH_OBJECTS} objects"), + ); + } + if !request.transfers.is_empty() + && !request + .transfers + .iter() + .any(knot_lfs::TransferAdapter::is_basic) + { + return fail( + StatusCode::UNPROCESSABLE_ENTITY, + "no mutually supported transfer adapter, this server serves basic", + ); + } + if let Some(algo) = &request.hash_algo + && !algo.is_sha256() + { + return fail( + StatusCode::CONFLICT, + &format!("unsupported hash algorithm {:?}", algo.as_str()), + ); + } + let base = &state.knot_service_url; + let objects: Result, Box> = match request.operation { + BatchOperation::Upload => { + match crate::receive::authorized_pusher(state, peer, headers, &repo, PUSH_DENIED) + .await + .map_err(|error| Box::new(challenge(error))) + { + Ok(_actor) => match probe_all(lfs, &repo, &request.objects).await { + Ok(stored) => request + .objects + .iter() + .zip(stored) + .map(|(object, present)| upload_action(base, path_prefix, object, present)) + .collect(), + Err(response) => Err(response), + }, + Err(response) => return *response, + } + } + BatchOperation::Download => match probe_all(lfs, &repo, &request.objects).await { + Ok(stored) => request + .objects + .iter() + .zip(stored) + .map(|(object, stored)| downloadable(base, path_prefix, object, stored)) + .collect(), + Err(response) => Err(response), + }, + }; + let objects = match objects { + Ok(objects) => objects, + Err(response) => return *response, + }; + let response = BatchResponse { + transfer: knot_lfs::TransferAdapter::Basic, + objects, + hash_algo: Some(knot_lfs::HashAlgo::Sha256), + }; + ( + StatusCode::OK, + [( + header::CONTENT_TYPE, + HeaderValue::from_static(BATCH_MEDIA_TYPE), + )], + serde_json::to_string(&response).expect("batch response always serializes"), + ) + .into_response() +} + +fn challenge(error: crate::XrpcError) -> Response { + if error.status() == StatusCode::UNAUTHORIZED { + unauthorized(&error.to_string()) + } else { + error.into_response() + } +} + +fn unauthorized(message: &str) -> Response { + ( + StatusCode::UNAUTHORIZED, + [ + (header::WWW_AUTHENTICATE, crate::BASIC_CHALLENGE), + ( + header::CONTENT_TYPE, + HeaderValue::from_static(BATCH_MEDIA_TYPE), + ), + ], + json!({ "message": message }).to_string(), + ) + .into_response() +} + +fn upload_action( + base: &KnotServiceUrl, + path_prefix: &str, + object: &BatchObject, + present: Option, +) -> Result> { + Ok(match present { + Some(size) => BatchResponseObject { + oid: object.oid.clone(), + size: ClaimedSize::new(size.get()), + authenticated: Some(true), + actions: None, + error: None, + }, + None => BatchResponseObject { + oid: object.oid.clone(), + size: object.size, + // I know what you're thinking about putting `authenticated: true`, + // but trust me TM git-lfs thinks + // "the href already has credentials on it" + // and omits the Authorization header from the following PUT req. + // PUT needs auth so the client would 401, re-run the batch, + // repeat. + // + // Having this be `None` makes git-lfs re-send the header it + // used on the batch-call in the first place. + authenticated: None, + actions: Some(BatchActions { + download: None, + upload: Some(BatchAction { + href: object_href(base, path_prefix, &object.oid)?, + }), + }), + error: None, + }, + }) +} + +async fn probe_all( + lfs: &LfsWeb, + repo: &RepoDid, + objects: &[BatchObject], +) -> Result>, Box> { + let store = Arc::clone(&lfs.handle.store); + let target = repo.clone(); + let oids: Vec = objects.iter().map(|object| object.oid.clone()).collect(); + tokio::task::spawn_blocking(move || { + oids.iter() + .map(|oid| store.probe(Pool::Lfs, &target, oid)) + .collect::, _>>() + }) + .await + .map_err(|_| { + Box::new(fail( + StatusCode::INTERNAL_SERVER_ERROR, + "store probe failed", + )) + })? + .map_err(|error| { + tracing::warn!(repo = repo.as_str(), %error, "lfs store probe failed"); + Box::new(fail( + StatusCode::INTERNAL_SERVER_ERROR, + "store probe failed", + )) + }) +} + +fn downloadable( + base: &KnotServiceUrl, + path_prefix: &str, + object: &BatchObject, + stored: Option, +) -> Result> { + Ok(match stored { + Some(size) => BatchResponseObject { + oid: object.oid.clone(), + size: ClaimedSize::new(size.get()), + authenticated: Some(true), + actions: Some(BatchActions { + download: Some(BatchAction { + href: object_href(base, path_prefix, &object.oid)?, + }), + upload: None, + }), + error: None, + }, + None => BatchResponseObject { + oid: object.oid.clone(), + size: object.size, + authenticated: None, + actions: None, + error: Some(BatchObjectError { + code: HttpStatus::new(404), + message: "object not found".to_string(), + }), + }, + }) +} + +fn object_href( + base: &KnotServiceUrl, + path_prefix: &str, + oid: &LfsOid, +) -> Result> { + Url::parse(&format!( + "{}/{path_prefix}/info/lfs/objects/{oid}", + base.as_str() + )) + .map_err(|error| { + Box::new(fail( + StatusCode::INTERNAL_SERVER_ERROR, + &format!("cannot derive object href: {error}"), + )) + }) +} + +async fn object_named( + State(state): State>>, + Path(RepoObjectParams { did, name, oid }): Path, + request: Request, +) -> Response { + let repo = match resolve_named(&state, &did, &name).await { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_object(&state, repo, &oid, request).await +} + +async fn object_did( + State(state): State>>, + Path(DidObjectParams { did, oid }): Path, + request: Request, +) -> Response { + let repo = match resolve_did(&state, &did) { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_object(&state, repo, &oid, request).await +} + +async fn serve_object( + state: &XrpcState, + repo: RepoDid, + oid_raw: &OidSegment, + mut request: Request, +) -> Response { + let lfs = &state.lfs; + let Ok(oid) = LfsOid::new(oid_raw.as_str()) else { + return fail(StatusCode::NOT_FOUND, "object not found"); + }; + let located = { + let store = Arc::clone(&lfs.handle.store); + let target = repo.clone(); + let oid = oid.clone(); + tokio::task::spawn_blocking(move || store.object_file(Pool::Lfs, &target, &oid)).await + }; + let (size, path) = match located { + Ok(Ok(Some((size, path)))) => (size, path), + Ok(Ok(None)) => return fail(StatusCode::NOT_FOUND, "object not found"), + Ok(Err(error)) => { + tracing::warn!(repo = repo.as_str(), oid = oid.as_str(), %error, "lfs store read failed"); + return fail(StatusCode::INTERNAL_SERVER_ERROR, "store read failed"); + } + Err(_) => return fail(StatusCode::INTERNAL_SERVER_ERROR, "store read failed"), + }; + let etag = format!("\"{oid}\""); + if crate::reads::etag_matches(request.headers(), &etag) { + return not_modified(&etag); + } + crate::reads::reconcile_if_range(&mut request, &etag, None); + let permit = match Arc::clone(&lfs.downloads).acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + return fail( + StatusCode::SERVICE_UNAVAILABLE, + "server is shutting down, retry shortly", + ); + } + }; + let served = ServeFile::new(path) + .oneshot(request) + .await + .map(|response| response.map(Body::new)); + let mut response = match served { + Ok(response) => response, + Err(error) => match error {}, + }; + tracing::info!( + repo = repo.as_str(), + oid = oid.as_str(), + size = size.get(), + status = response.status().as_u16(), + "lfs object served over http" + ); + if response.status().is_success() { + let headers = response.headers_mut(); + headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/octet-stream"), + ); + headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_static(IMMUTABLE_CACHE), + ); + if let Ok(value) = HeaderValue::from_str(&etag) { + headers.insert(header::ETAG, value); + } + } + response.map(|body| { + Body::new(PermitBody { + body, + _permit: permit, + }) + }) +} + +async fn object_upload_named( + State(state): State>>, + Path(RepoObjectParams { did, name, oid }): Path, + peer: SocketPeer, + request: Request, +) -> Response { + let repo = match resolve_named(&state, &did, &name).await { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_object_upload(&state, repo, &oid, peer, request).await +} + +async fn object_upload_did( + State(state): State>>, + Path(DidObjectParams { did, oid }): Path, + peer: SocketPeer, + request: Request, +) -> Response { + let repo = match resolve_did(&state, &did) { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_object_upload(&state, repo, &oid, peer, request).await +} + +async fn serve_object_upload( + state: &XrpcState, + repo: RepoDid, + oid_raw: &OidSegment, + peer: SocketPeer, + request: Request, +) -> Response { + let lfs = &state.lfs; + let Ok(oid) = LfsOid::new(oid_raw.as_str()) else { + return fail(StatusCode::NOT_FOUND, "object not found"); + }; + let denied = + crate::receive::authorized_pusher(state, peer, request.headers(), &repo, PUSH_DENIED) + .await + .map_err(|error| Box::new(challenge(error))); + if let Err(response) = denied { + return *response; + } + let Some(size) = content_length(request.headers()) else { + return fail( + StatusCode::LENGTH_REQUIRED, + "content-length is required for an lfs object upload", + ); + }; + let permit = match lfs.handle.admission.admit(size) { + Ok(permit) => permit, + Err(error) => return store_fault(&repo, &oid, error), + }; + let store = Arc::clone(&lfs.handle.store); + let target = repo.clone(); + let object = oid.clone(); + let landed = { + use futures::TryStreamExt; + let reader = tokio_util::io::StreamReader::new( + request + .into_body() + .into_data_stream() + .map_err(std::io::Error::other), + ); + tokio::task::spawn_blocking(move || { + let mut body = tokio_util::io::SyncIoBridge::new(reader); + let outcome = store.put(Pool::Lfs, &target, &object, size, &mut body); + drop(permit); + outcome + }) + .await + }; + match landed { + Ok(Ok(())) => { + tracing::info!( + repo = repo.as_str(), + oid = oid.as_str(), + size = size.get(), + "lfs object stored over http" + ); + StatusCode::OK.into_response() + } + Ok(Err(error)) => store_fault(&repo, &oid, error), + Err(_) => fail(StatusCode::INTERNAL_SERVER_ERROR, "upload task died"), + } +} + +fn content_length(headers: &HeaderMap) -> Option { + headers + .get(header::CONTENT_LENGTH)? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .map(ClaimedSize::new) +} + +fn store_fault(repo: &RepoDid, oid: &LfsOid, error: LfsError) -> Response { + let status = match &error { + LfsError::HashMismatch { .. } | LfsError::SizeMismatch { .. } => { + StatusCode::UNPROCESSABLE_ENTITY + } + LfsError::SizeLimitExceeded { .. } => StatusCode::PAYLOAD_TOO_LARGE, + LfsError::FreeSpaceDenied { .. } => StatusCode::INSUFFICIENT_STORAGE, + LfsError::BodyRead { .. } => StatusCode::BAD_REQUEST, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }; + if status.is_server_error() { + tracing::warn!(repo = repo.as_str(), oid = oid.as_str(), %error, "lfs object upload failed"); + } + fail(status, &error.to_string()) +} + +fn not_modified(etag: &str) -> Response { + let mut response = StatusCode::NOT_MODIFIED.into_response(); + let headers = response.headers_mut(); + headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_static(IMMUTABLE_CACHE), + ); + if let Ok(value) = HeaderValue::from_str(etag) { + headers.insert(header::ETAG, value); + } + response +} + +pub(crate) fn mirror_fork_objects( + state: Arc>, + upstream: Upstream, + fork: RepoDid, + wants: WantOids, + haves: HaveOids, +) -> futures::future::BoxFuture<'static, Result, crate::XrpcError>> { + Box::pin(mirror_fork_objects_inner( + state, upstream, fork, wants, haves, + )) +} + +async fn mirror_fork_objects_inner( + state: Arc>, + upstream: Upstream, + fork: RepoDid, + wants: WantOids, + haves: HaveOids, +) -> Result, crate::XrpcError> { + let state = &state; + let upstream = &upstream; + let fork = ⋔ + let needed = { + let layout = state.layout.clone(); + let fork = fork.clone(); + let store = Arc::clone(&state.lfs.handle.store); + tokio::task::spawn_blocking(move || -> Result, String> { + let repo = layout.open(&fork).map_err(|error| error.to_string())?; + knot_lfs::scan_pointers( + &repo, + knot_git::Wants::new(wants.as_slice()), + knot_git::Haves::new(haves.as_slice()), + ) + .map_err(|error| error.to_string())? + .into_iter() + .map(|(oid, size)| match store.probe(Pool::Lfs, &fork, &oid) { + Ok(None) => Ok(Some((oid, size))), + Ok(Some(_)) => Ok(None), + Err(fault) => Err(fault.to_string()), + }) + .filter_map(Result::transpose) + .collect() + }) + .await + }; + let needed = match needed { + Ok(Ok(needed)) => needed, + Ok(Err(fault)) => { + tracing::warn!( + repo = fork.as_str(), + fault, + "lfs pointer scan failed on fork" + ); + return Err(crate::XrpcError::internal("lfs pointer scan failed")); + } + Err(_) => { + return Err(crate::XrpcError::internal("lfs pointer scan task died")); + } + }; + let lfs = &state.lfs; + if needed.is_empty() { + return Ok(Vec::new()); + } + let store = Arc::clone(&lfs.handle.store); + let admission = Arc::clone(&lfs.handle.admission); + + let missing = match upstream { + Upstream::Local(hosted) => { + let source = hosted.clone().into_did(); + let fork = fork.clone(); + let store = Arc::clone(&store); + let admission = Arc::clone(&admission); + tokio::task::spawn_blocking(move || { + needed + .into_iter() + .filter_map(|(oid, size)| { + let copied = admission.admit(size).and_then(|_permit| { + store.read(Pool::Lfs, &source, &oid).and_then(|mut body| { + store.put(Pool::Lfs, &fork, &oid, size, &mut body) + }) + }); + match copied { + Ok(()) => None, + Err(fault) => { + tracing::warn!( + source = source.as_str(), + oid = oid.as_str(), + %fault, + "lfs fork copy skipped an object" + ); + Some(oid) + } + } + }) + .collect() + }) + .await + .map_err(|_| crate::XrpcError::internal("lfs fork copy task died"))? + } + Upstream::Remote(url) => match remote_batch_url(url) { + Some(batch_url) => { + use futures::StreamExt; + let chunks: Vec> = needed + .chunks(REMOTE_BATCH_CHUNK) + .map(<[(LfsOid, ClaimedSize)]>::to_vec) + .collect(); + futures::stream::iter(chunks) + .then(|chunk| { + fetch_remote_chunk( + Arc::clone(state), + Arc::clone(&store), + Arc::clone(&admission), + fork.clone(), + batch_url.clone(), + chunk, + ) + }) + .concat() + .await + } + None => needed.into_iter().map(|(oid, _)| oid).collect(), + }, + }; + if !missing.is_empty() { + tracing::warn!( + repo = fork.as_str(), + count = missing.len(), + "fork upstream couldn't serve every referenced lfs object" + ); + } + Ok(missing) +} + +const REMOTE_BATCH_CHUNK: usize = 100; + +fn remote_batch_url(origin: &Url) -> Option { + let mut origin = origin.clone(); + origin.set_query(None); + origin.set_fragment(None); + let base = origin.as_str().trim_end_matches('/'); + let base = match base.ends_with(".git") { + true => base.to_string(), + false => format!("{base}.git"), + }; + Url::parse(&format!("{base}/info/lfs/objects/batch")).ok() +} + +async fn fetch_remote_chunk( + state: Arc>, + store: Arc, + admission: Arc, + fork: RepoDid, + batch_url: Url, + chunk: Vec<(LfsOid, ClaimedSize)>, +) -> Vec { + use futures::StreamExt; + let all_missing = || chunk.iter().map(|(oid, _)| oid.clone()).collect::>(); + let request_body = BatchRequest { + operation: BatchOperation::Download, + transfers: vec![knot_lfs::TransferAdapter::Basic], + reference: None, + objects: chunk + .iter() + .map(|(oid, size)| knot_lfs::BatchObject { + oid: oid.clone(), + size: *size, + }) + .collect(), + hash_algo: Some(knot_lfs::HashAlgo::Sha256), + }; + let body = match serde_json::to_vec(&request_body) { + Ok(body) => body, + Err(_) => return all_missing(), + }; + let mut request = knot_runtime::HttpRequest::post(batch_url.clone(), body.into()); + request.headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static(BATCH_MEDIA_TYPE), + ); + request + .headers + .insert(header::ACCEPT, HeaderValue::from_static(BATCH_MEDIA_TYPE)); + let response = match state.git_http.execute(request).await { + Ok(response) if response.status.is_success() => response, + Ok(response) => { + tracing::warn!( + url = batch_url.as_str(), + status = response.status.as_u16(), + "upstream lfs batch refused" + ); + return all_missing(); + } + Err(fault) => { + tracing::warn!(url = batch_url.as_str(), %fault, "upstream lfs batch failed"); + return all_missing(); + } + }; + let parsed: BatchResponse = match serde_json::from_slice(&response.body) { + Ok(parsed) => parsed, + Err(fault) => { + tracing::warn!(url = batch_url.as_str(), %fault, "upstream lfs batch unparsable"); + return all_missing(); + } + }; + let declared: BTreeMap = chunk.into_iter().collect(); + let unanswered = unanswered_oids(&declared, &parsed.objects); + let tagged: Vec<(BatchResponseObject, ClaimedSize)> = parsed + .objects + .into_iter() + .filter_map(|object| { + declared + .get(&object.oid) + .copied() + .map(|size| (object, size)) + }) + .collect(); + let failed: Vec = futures::stream::iter(tagged) + .then(|(object, size)| { + fetch_remote_object( + Arc::clone(&state), + Arc::clone(&store), + Arc::clone(&admission), + fork.clone(), + object, + size, + ) + }) + .filter_map(std::future::ready) + .collect() + .await; + unanswered.into_iter().chain(failed).collect() +} + +fn unanswered_oids( + declared: &BTreeMap, + answered: &[BatchResponseObject], +) -> Vec { + let answered: BTreeSet<&LfsOid> = answered.iter().map(|object| &object.oid).collect(); + declared + .keys() + .filter(|oid| !answered.contains(oid)) + .cloned() + .collect() +} + +fn href_is_fetchable(url: &Url) -> bool { + let scheme_ok = matches!(url.scheme(), "http" | "https"); + let host_ok = match url.host() { + Some(url::Host::Ipv4(ip)) => !knot_runtime::is_blocked_ip(ip.into()), + Some(url::Host::Ipv6(ip)) => !knot_runtime::is_blocked_ip(ip.into()), + Some(url::Host::Domain(_)) => true, + None => false, + }; + scheme_ok && host_ok +} + +async fn fetch_remote_object( + state: Arc>, + store: Arc, + admission: Arc, + fork: RepoDid, + object: BatchResponseObject, + declared: ClaimedSize, +) -> Option { + let Some(action) = object.actions.and_then(|actions| actions.download) else { + return Some(object.oid); + }; + if !href_is_fetchable(&action.href) { + tracing::warn!( + oid = object.oid.as_str(), + href = action.href.as_str(), + "lfs fork download href isn't a public http target" + ); + return Some(object.oid); + } + let permit = match admission.admit(declared) { + Ok(permit) => permit, + Err(fault) => { + tracing::warn!(oid = object.oid.as_str(), %fault, "lfs fork download refused by admission"); + return Some(object.oid); + } + }; + let streamed = match state + .git_http + .execute_streamed(knot_runtime::HttpRequest::get(action.href)) + .await + { + Ok(streamed) if streamed.status.is_success() => streamed, + _ => return Some(object.oid), + }; + let landed = { + use futures::TryStreamExt; + let reader = + tokio_util::io::StreamReader::new(streamed.body.map_err(std::io::Error::other)); + let oid = object.oid.clone(); + tokio::task::spawn_blocking(move || { + let mut body = tokio_util::io::SyncIoBridge::new(reader); + let outcome = store.put(Pool::Lfs, &fork, &oid, declared, &mut body); + drop(permit); + outcome + }) + .await + }; + match landed { + Ok(Ok(())) => None, + Ok(Err(fault)) => { + tracing::warn!(oid = object.oid.as_str(), %fault, "lfs fork download failed"); + Some(object.oid) + } + Err(_) => Some(object.oid), + } +} + +struct PermitBody { + body: Body, + _permit: OwnedSemaphorePermit, +} + +impl http_body::Body for PermitBody { + type Data = Bytes; + type Error = axum::Error; + + fn poll_frame( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll, Self::Error>>> { + std::pin::Pin::new(&mut self.body).poll_frame(cx) + } + + fn is_end_stream(&self) -> bool { + self.body.is_end_stream() + } + + fn size_hint(&self) -> http_body::SizeHint { + self.body.size_hint() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_remote_batch_endpoint_matches_git_lfs_derivation() { + let plain = Url::parse("https://nel.pet/did:web:witchcraft.systems/anemone").unwrap(); + assert_eq!( + remote_batch_url(&plain).unwrap().as_str(), + "https://nel.pet/did:web:witchcraft.systems/anemone.git/info/lfs/objects/batch" + ); + let suffixed = Url::parse("https://nel.pet/did:plc:cuttle.git").unwrap(); + assert_eq!( + remote_batch_url(&suffixed).unwrap().as_str(), + "https://nel.pet/did:plc:cuttle.git/info/lfs/objects/batch" + ); + let trailing = Url::parse("https://nel.pet/did:plc:cuttle/").unwrap(); + assert_eq!( + remote_batch_url(&trailing).unwrap().as_str(), + "https://nel.pet/did:plc:cuttle.git/info/lfs/objects/batch" + ); + let decorated = Url::parse("https://nel.pet/did:plc:cuttle?ref=main#readme").unwrap(); + assert_eq!( + remote_batch_url(&decorated).unwrap().as_str(), + "https://nel.pet/did:plc:cuttle.git/info/lfs/objects/batch" + ); + } + + #[test] + fn oids_absent_from_upstream_batch_count_as_missing() { + let stored = LfsOid::from_digest(knot_types::Sha256Digest::hash(b"stored")); + let ignored = LfsOid::from_digest(knot_types::Sha256Digest::hash(b"ignored")); + let declared: BTreeMap = [ + (stored.clone(), ClaimedSize::new(4)), + (ignored.clone(), ClaimedSize::new(7)), + ] + .into_iter() + .collect(); + let answered = vec![BatchResponseObject { + oid: stored, + size: ClaimedSize::new(4), + authenticated: None, + actions: None, + error: None, + }]; + assert_eq!(unanswered_oids(&declared, &answered), vec![ignored.clone()]); + assert_eq!( + unanswered_oids(&declared, &[]).len(), + 2, + "an empty upstream response must leave every oid missing" + ); + } + + #[test] + fn a_stale_if_range_drops_the_range_for_a_full_response() { + use axum::http::Request as HttpRequest; + let etag = "\"6c17f2007cbe934aee6e309b28b2fba3c119d98be6ea4156da3aa3173456ad16\""; + + let mut matching = HttpRequest::builder() + .header(header::IF_RANGE, etag) + .header(header::RANGE, "bytes=0-9") + .body(Body::empty()) + .unwrap(); + crate::reads::reconcile_if_range(&mut matching, etag, None); + assert!(matching.headers().get(header::IF_RANGE).is_none()); + assert!( + matching.headers().get(header::RANGE).is_some(), + "a matching validator keeps the range for a 206" + ); + + let mut stale = HttpRequest::builder() + .header(header::IF_RANGE, "\"stale\"") + .header(header::RANGE, "bytes=0-9") + .body(Body::empty()) + .unwrap(); + crate::reads::reconcile_if_range(&mut stale, etag, None); + assert!(stale.headers().get(header::IF_RANGE).is_none()); + assert!( + stale.headers().get(header::RANGE).is_none(), + "a stale validator drops the range so the client gets the whole object" + ); + } + + #[test] + fn revalidation_matches_the_oid_etag() { + let etag = "\"6c17f2007cbe934aee6e309b28b2fba3c119d98be6ea4156da3aa3173456ad16\""; + let holds = |value: &str| { + let mut headers = HeaderMap::new(); + headers.insert(header::IF_NONE_MATCH, HeaderValue::from_str(value).unwrap()); + crate::reads::etag_matches(&headers, etag) + }; + assert!(holds(etag)); + assert!(holds(&format!("W/{etag}"))); + assert!(holds(&format!("\"other\", {etag}"))); + assert!(holds("*")); + assert!(!holds("\"other\"")); + let absent = HeaderMap::new(); + assert!(!crate::reads::etag_matches(&absent, etag)); + } +} diff --git a/knot2/crates/knot-xrpc/src/lib.rs b/knot2/crates/knot-xrpc/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..8f746c871eb247c2cbca30913841e564a5d8908f --- /dev/null +++ b/knot2/crates/knot-xrpc/src/lib.rs @@ -0,0 +1,888 @@ +mod atproto; +mod blobs; +mod blocklist; +mod body; +mod branches; +mod cob; +mod collaborators; +mod error; +mod firehose; +mod forks; +pub mod foster; +mod git; +pub mod grants; +pub mod handover; +pub mod imports; +pub mod invite_codes; +pub mod keep_sweep; +pub mod legacy_admin; +mod lfs; +mod locks; +pub mod materialize; +mod members; +pub mod migrate; +mod mint; +mod out; +pub mod outbox; +pub mod plc; +mod query; +mod reads; +mod receive; +mod refrecords; +mod repos; +mod reservations; +mod service; +mod social; + +#[cfg(test)] +mod tests; + +pub use blobs::{BlobViews, RepoAttachments}; +pub use error::XrpcError; +pub use firehose::{FirehoseEmit, persist_seq, replay_floor as firehose_floor}; +pub use invite_codes::InviteCodes; +pub use knot_pack::MaxWireBytes; +pub use knot_resource::{ + Burst, GlobalInflight, LimitConfig, PerPeerInflight, PreAuthLimiter, RateLimit, RefillMicros, +}; +pub use lfs::LfsWeb; +pub use locks::CobLocks; +pub use outbox::{Outbox, OutboxLimits}; +pub use receive::advertiser as receive_advertiser; +pub use refrecords::RefProjection; +pub use reservations::{GlobalQuota, PerActorQuota, ReservationTtl, Reservations}; +pub use service::{Contact, KnotDescription, Links}; +pub use social::{RecordKeys, RepoMethod}; + +use std::collections::BTreeSet; +use std::net::IpAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use axum::Json; +use axum::Router; +use axum::body::Bytes; +use axum::extract::{DefaultBodyLimit, FromRequestParts, MatchedPath, Request, State}; +use axum::middleware::{Next, from_fn_with_state}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use http::request::Parts; +use http::{HeaderMap, HeaderValue, StatusCode, header::AUTHORIZATION}; +use serde::de::DeserializeOwned; +use serde_json::json; + +use knot_atproto::{Atproto, AtprotoError, ServiceJwt}; + +use crate::legacy_admin::LegacyAdminSecret; +use knot_events::{EventLog, SubscriberGate}; +pub use knot_git::ArchiveLimit; +use knot_git::Layout; +use knot_index::{Index, Resolved}; +use knot_maintenance::MaintenanceHandle; +use knot_resource::Slots; +use knot_runtime::{Clock, Entropy, HttpTransport}; +use knot_secrets::SealedStore; +use knot_types::{ + AccountDid, AdmissionPolicy, AppviewEndpoint, ClonePath, ContributionPolicy, KnotHostname, + KnotId, KnotServiceUrl, Nsid, OwnerDid, OwnerRef, RepoDid, UnixSeconds, +}; + +use base64::Engine; +use knot_pack::SocketPeer; +use knot_resource::{ + AdmitGuard, Rejection, SocialBudgetBytes, SocialCharge, SubjectKey, SubjectLimiter, +}; + +pub(crate) const PUSH_NSID: &str = "org.tangled.git.receivePack"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ReadBudget { + Within(Duration), + Unbounded, +} + +impl ReadBudget { + pub fn deadline(self) -> Option { + match self { + ReadBudget::Within(budget) => Some(Instant::now() + budget), + ReadBudget::Unbounded => None, + } + } +} + +// `XrpcState` keeps a bunch of these side by side, +// some usize & some u64. +// Within each group every one of them typechecked in every other one's slot. +knot_types::scalar_newtype! { + pub struct BodyLimit(usize); + pub struct PatchLimit(usize); + pub struct PatchDecompressedLimit(u64); + pub struct ResponseLimit(usize); + pub struct ForkPackLimit(u64); + pub struct ImportPackLimit(u64); + pub struct TreeReadBudget(ReadBudget); + pub struct BlobReadBudget(ReadBudget); + pub struct LanguagesReadBudget(ReadBudget); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ByteLimits { + pub body: BodyLimit, + pub patch: PatchLimit, + pub patch_decompressed: PatchDecompressedLimit, + pub response: ResponseLimit, + pub archive: ArchiveLimit, + pub fork_pack: ForkPackLimit, + // imports run unattended from a background queue, so they cap packs clearly + // under the fork limit to keep several of them from exhausting memory at once + pub import_pack: ImportPackLimit, + pub pack: MaxWireBytes, +} + +impl Default for ByteLimits { + fn default() -> Self { + Self { + body: BodyLimit::new(64 * 1024), + patch: PatchLimit::new(16 * 1024 * 1024), + patch_decompressed: PatchDecompressedLimit::new(128 * 1024 * 1024), + response: ResponseLimit::new(5 * 1024 * 1024), + archive: ArchiveLimit::default(), + fork_pack: ForkPackLimit::new(1024 * 1024 * 1024), + import_pack: ImportPackLimit::new(256 * 1024 * 1024), + pack: MaxWireBytes::new(8 * 1024 * 1024 * 1024), + } + } +} + +const BINARY_RESPONSE_SHARE: u64 = 4; +const BINARY_WIRE_COPIES: u64 = 3; + +impl ByteLimits { + pub fn binary_patch(self) -> knot_git::BinaryBudget { + knot_git::BinaryBudget::new( + self.response.get() as u64 / BINARY_RESPONSE_SHARE / BINARY_WIRE_COPIES, + ) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Budgets { + pub tree_last_commit: TreeReadBudget, + pub blob_last_commit: BlobReadBudget, + pub languages: LanguagesReadBudget, +} + +impl Default for Budgets { + fn default() -> Self { + Self { + tree_last_commit: TreeReadBudget::new(ReadBudget::Within(Duration::from_millis(300))), + blob_last_commit: BlobReadBudget::new(ReadBudget::Within(Duration::from_millis(2_000))), + languages: LanguagesReadBudget::new(ReadBudget::Within(Duration::from_millis(1_000))), + } + } +} + +pub struct XrpcState { + pub layout: Layout, + pub index: Arc, + pub atproto: Arc>, + pub secrets: Arc, + pub entropy: Arc, + pub invite_codes: Arc, + pub admins: BTreeSet, + pub admission: AdmissionPolicy, + pub contribution_policy: ContributionPolicy, + pub knot_did: KnotId, + pub knot_repo: RepoDid, + pub knot_hostname: KnotHostname, + pub meta_path: PathBuf, + pub knot_service_url: KnotServiceUrl, + pub writes: Arc, + pub social_budget: SocialBudgetBytes, + pub limiter: Arc, + pub cob_locks: Arc, + pub reservations: Arc, + pub proxy_trust: knot_types::ProxyTrust, + pub byte_limits: ByteLimits, + pub budgets: Budgets, + pub git_http: Arc, + pub pack_limits: knot_pack::PackLimits, + pub service_owner: AccountDid, + pub legacy_admin_secret: Option, + pub description: KnotDescription, + pub subscriber_gate: Arc, + pub maintenance: MaintenanceHandle, + pub appview: AppviewEndpoint, + pub slots: Slots, + pub imports: Arc, + pub handover_sessions: Arc, + pub lfs: LfsWeb, + pub blobs: Arc, + pub catalog: Arc, + pub firehose: Arc>, + pub rkeys: RecordKeys, + pub outbox: Arc, +} + +pub fn knot_repo(knot_did: &KnotId) -> Result { + knot_types::DidRkey::new(knot_did.as_str()).map(|rkey| RepoDid::from(&rkey)) +} + +impl XrpcState { + pub fn now(&self) -> UnixSeconds { + UnixSeconds::new((self.atproto.now().get() / 1_000_000) as i64) + } + + pub(crate) fn knot_authority(&self) -> &str { + self.knot_service_url.authority() + } + + pub(crate) async fn authenticate( + &self, + headers: &HeaderMap, + method: &Method, + ) -> Result { + let token = bearer(headers)?; + self.atproto + .verify_service_jwt(&token, method.nsid()) + .await + .map_err(map_verify_error) + } + + pub(crate) async fn authenticate_for_repo( + &self, + headers: &HeaderMap, + method: &Method, + repo: &RepoDid, + ) -> Result { + let token = bearer(headers)?; + self.atproto + .verify_service_jwt_for_repo(&token, method.nsid(), repo) + .await + .map_err(map_verify_error) + } + + // a token that doesn't verify is a 401, not an anonymous read + pub(crate) async fn authenticate_optional_for_repo( + &self, + headers: &HeaderMap, + method: &Method, + repo: &RepoDid, + ) -> Result, XrpcError> { + match headers.contains_key(AUTHORIZATION) { + true => self + .authenticate_for_repo(headers, method, repo) + .await + .map(Some), + false => Ok(None), + } + } + + pub(crate) async fn authenticate_write( + &self, + headers: &HeaderMap, + method: &Method, + ) -> Result { + let actor = self.authenticate(headers, method).await?; + admit_write(self, &actor)?; + Ok(actor) + } + + pub(crate) async fn authenticate_write_for_repo( + &self, + headers: &HeaderMap, + method: &Method, + repo: &RepoDid, + ) -> Result { + let actor = self.authenticate_for_repo(headers, method, repo).await?; + admit_write(self, &actor)?; + Ok(actor) + } + + pub(crate) fn legacy_admin_authorize(&self, headers: &HeaderMap) -> Result<(), XrpcError> { + let secret = self + .legacy_admin_secret + .as_ref() + .ok_or_else(|| XrpcError::auth_required("invalid admin credentials"))?; + secret.authorize(headers) + } + + pub(crate) fn legacy_admin_actor( + &self, + headers: &HeaderMap, + ) -> Result, XrpcError> { + let Some(secret) = self.legacy_admin_secret.as_ref() else { + return Ok(None); + }; + match secret.attempt(headers) { + legacy_admin::LegacyAdminAttempt::Admitted => Ok(Some(self.service_owner.clone())), + legacy_admin::LegacyAdminAttempt::Denied => { + Err(XrpcError::auth_required("invalid admin credentials")) + } + legacy_admin::LegacyAdminAttempt::NotBasic => Ok(None), + } + } + + pub(crate) async fn audience_repo(&self, headers: &HeaderMap) -> Result { + let token = bearer(headers)?; + knot_atproto::token_audience(&token) + .map_err(knot_atproto::AtprotoError::from) + .map_err(map_verify_error) + } + + pub(crate) async fn authenticate_push( + &self, + headers: &HeaderMap, + ) -> Result { + let token = push_credential(headers)?; + let method = Nsid::new_owned(PUSH_NSID).expect("Push NSID is always a valid NSID"); + self.atproto + .verify_service_jwt_guarded( + &token, + &method, + knot_atproto::ReplayGuard::ReusableUntilExpiry, + ) + .await + .map_err(map_verify_error) + } +} + +fn map_verify_error(error: AtprotoError) -> XrpcError { + if error.is_transient() { + XrpcError::upstream_unavailable(error.to_string()) + } else { + XrpcError::auth_required(error.to_string()) + } +} + +pub(crate) struct Method(Nsid); + +impl Method { + fn nsid(&self) -> &Nsid { + &self.0 + } + + #[cfg(test)] + pub(crate) fn from_nsid(nsid: &str) -> Self { + Self(Nsid::new_owned(nsid).expect("Test route NSID parses")) + } +} + +impl FromRequestParts for Method { + type Rejection = XrpcError; + + async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + let matched = MatchedPath::from_request_parts(parts, state) + .await + .map_err(|_| XrpcError::internal("XRPC handler reached without a matched route"))?; + let nsid = matched + .as_str() + .strip_prefix("/xrpc/") + .ok_or_else(|| XrpcError::internal("XRPC route paths are prefixed with /xrpc/"))?; + Nsid::new_owned(nsid) + .map(Self) + .map_err(|_| XrpcError::internal("Route NSID is always a valid NSID")) + } +} + +pub fn router(state: Arc>) -> Router { + Router::new() + .route( + invite_codes::REDEEM_ROUTE, + post(invite_codes::redeem_invite_code::), + ) + .route( + social::CREATE_RECORD_ROUTE, + post(social::create_record::), + ) + .route(social::PUT_RECORD_ROUTE, post(social::put_record::)) + .route( + social::DELETE_RECORD_ROUTE, + post(social::delete_record::), + ) + .route( + social::LIST_RECORD_VERSIONS_ROUTE, + get(social::list_record_versions::), + ) + .route( + social::SUBMIT_PATCH_ROUTE, + post(social::submit_patch::), + ) + .route(repos::CREATE_ROUTE, post(repos::create_repo::)) + .route(repos::DELETE_ROUTE, post(repos::delete_repo::)) + .route(handover::TRIGGER_ROUTE, post(handover::trigger::)) + .route(handover::CANCEL_ROUTE, post(handover::cancel::)) + .route(repos::RESERVE_ROUTE, post(repos::reserve_key::)) + .route(mint::MINT_TOKEN_ROUTE, post(mint::mint_token::)) + .route( + service::RECEIVE_PACK_ROUTE, + post(service::receive_pack::), + ) + .route( + branches::DELETE_ROUTE, + post(branches::delete_branch::), + ) + .route( + branches::SET_DEFAULT_ROUTE, + post(branches::set_default_branch::), + ) + .route(reads::TREE_ROUTE, get(reads::repo_tree::)) + .route(reads::LOG_ROUTE, get(reads::repo_log::)) + .route(reads::BRANCHES_ROUTE, get(reads::repo_branches::)) + .route(reads::BRANCH_ROUTE, get(reads::repo_branch::)) + .route(reads::TAGS_ROUTE, get(reads::repo_tags::)) + .route(reads::TAG_ROUTE, get(reads::repo_tag::)) + .route(reads::BLOB_ROUTE, get(reads::repo_blob::)) + .route(reads::ENTRY_ROUTE, get(reads::repo_entry::)) + .route(reads::BLAME_ROUTE, get(reads::repo_blame::)) + .route(reads::DIFF_ROUTE, get(reads::repo_diff::)) + .route(reads::ARCHIVE_ROUTE, get(reads::repo_archive::)) + .route(reads::LANGUAGES_ROUTE, get(reads::repo_languages::)) + .route(reads::LIST_REFS_ROUTE, get(reads::git_list_refs::)) + .route(imports::STATUS_ROUTE, get(imports::import_status::)) + .route(service::VERSION_ROUTE, get(service::version::)) + .route( + "/xrpc/com.atproto.sync.getRepo", + get(atproto::sync_get_repo::), + ) + .route( + "/xrpc/com.atproto.sync.getLatestCommit", + get(atproto::sync_get_latest_commit::), + ) + .route( + "/xrpc/com.atproto.sync.getRecord", + get(atproto::sync_get_record::), + ) + .route( + "/xrpc/com.atproto.sync.getBlocks", + get(atproto::sync_get_blocks::), + ) + .route( + "/xrpc/com.atproto.sync.listRepos", + get(atproto::sync_list_repos::), + ) + .route( + "/xrpc/com.atproto.sync.getRepoStatus", + get(atproto::sync_get_repo_status::), + ) + .route( + "/xrpc/com.atproto.repo.getRecord", + get(atproto::repo_get_record::), + ) + .route( + "/xrpc/com.atproto.repo.listRecords", + get(atproto::repo_list_records::), + ) + .route( + "/xrpc/com.atproto.repo.describeRepo", + get(atproto::repo_describe_repo::), + ) + .route( + "/xrpc/com.atproto.server.describeServer", + get(atproto::describe_server::), + ) + .route( + "/xrpc/com.atproto.identity.resolveHandle", + get(atproto::resolve_handle::), + ) + .route( + "/xrpc/com.atproto.sync.subscribeRepos", + get(firehose::subscribe_repos::), + ) + .route( + blobs::UPLOAD_BLOB_ROUTE, + post(blobs::upload_blob::).layer(DefaultBodyLimit::max( + knot_record::attachment::MAX_UPLOAD_BYTES as usize + 1024, + )), + ) + .route(blobs::GET_BLOB_ROUTE, get(blobs::get_blob::)) + .route(blobs::LIST_BLOBS_ROUTE, get(blobs::list_blobs::)) + .layer(DefaultBodyLimit::max(state.byte_limits.body.get())) + .layer(from_fn_with_state( + Arc::clone(&state), + enforce_pre_auth_limit::, + )) + .merge(lfs::routes::()) + .merge(receive::routes::()) + .route(service::HEALTH_ROUTE, get(service::health::)) + .with_state(state) +} + +pub(crate) async fn enforce_pre_auth_limit( + State(state): State>>, + socket: SocketPeer, + request: Request, + next: Next, +) -> Response { + let peer = state + .proxy_trust + .client_peer(request.headers(), socket.ip()); + match admit_pre_auth(&state, peer) { + Ok(guard) => { + let response = next.run(request).await; + drop(guard); + response + } + Err(error) => error.into_response(), + } +} + +pub(crate) fn admit_pre_auth( + state: &XrpcState, + peer: Option, +) -> Result { + state + .limiter + .admit(peer, state.atproto.now()) + .map_err(|rejection| match rejection { + Rejection::RateLimited => XrpcError::rate_limited( + knot_resource::Bucket::Peer, + "Too many pre-authentication requests, please retry shortly", + ), + Rejection::Saturated => { + XrpcError::overloaded("Knot is shedding pre-authentication load, retry shortly") + } + }) +} + +pub(crate) fn admit_write( + state: &XrpcState, + actor: &AccountDid, +) -> Result<(), XrpcError> { + state + .writes + .admit(&SubjectKey::new(actor), state.atproto.now()) + .map_err(|rejection| match rejection { + Rejection::RateLimited => XrpcError::rate_limited( + knot_resource::Bucket::ActorWrites, + "This account is writing faster than knot takes writes, please retry shortly", + ), + Rejection::Saturated => { + XrpcError::overloaded("Knot already tracks as many writers as it can take, please retry shortly, thank you for your patience") + } + }) +} + +pub fn admit_social(budget: SocialBudgetBytes, charge: SocialCharge) -> Result<(), XrpcError> { + if budget.allows(charge) { + return Ok(()); + } + Err(XrpcError::rate_limited( + knot_resource::Bucket::RepoSocial, + format!( + "this repository is at {} bytes of serving record text and this write leaves {} more, over its {} byte budget; editing or deleting a record frees the bytes of the version it replaces", + charge.current.get(), + charge.adding.get(), + budget.get() + ), + )) +} + +pub(crate) const BASIC_CHALLENGE: HeaderValue = HeaderValue::from_static("Basic realm=\"knot\""); + +fn strip_bearer(value: &str) -> Option<&str> { + let (scheme, rest) = value.split_once(' ')?; + scheme.eq_ignore_ascii_case("Bearer").then_some(rest) +} + +fn bearer(headers: &HeaderMap) -> Result { + headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(strip_bearer) + .map(str::trim) + .and_then(|token| ServiceJwt::new(token).ok()) + .ok_or_else(|| XrpcError::auth_required("Missing or malformed Bearer authorization header")) +} + +pub(crate) struct BasicUser(String); + +impl BasicUser { + pub(crate) fn matches(&self, expected: &str) -> bool { + use subtle::ConstantTimeEq as _; + bool::from(self.0.as_bytes().ct_eq(expected.as_bytes())) + } +} + +pub(crate) struct BasicPassword(String); + +impl BasicPassword { + pub(crate) fn as_bytes(&self) -> &[u8] { + self.0.as_bytes() + } +} + +pub(crate) struct BasicCredentials { + pub(crate) user: BasicUser, + pub(crate) password: BasicPassword, +} + +pub(crate) fn basic_credentials(value: &str) -> Option { + let (scheme, rest) = value.split_once(' ')?; + if !scheme.eq_ignore_ascii_case("Basic") { + return None; + } + let decoded = base64::engine::general_purpose::STANDARD + .decode(rest.trim()) + .ok()?; + let text = String::from_utf8(decoded).ok()?; + let (user, password) = text.split_once(':')?; + Some(BasicCredentials { + user: BasicUser(user.to_string()), + password: BasicPassword(password.to_string()), + }) +} + +fn strip_basic(value: &str) -> Option { + basic_credentials(value) + .map(|credentials| credentials.password.0) + .filter(|password| !password.is_empty()) +} + +fn push_credential(headers: &HeaderMap) -> Result { + let value = headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| XrpcError::auth_required("Missing authorization header"))?; + strip_bearer(value) + .map(str::trim) + .map(str::to_string) + .or_else(|| strip_basic(value)) + .and_then(|token| ServiceJwt::new(token).ok()) + .ok_or_else(|| { + XrpcError::auth_required("Authorization isn't a bearer token or basic credential") + }) +} + +pub(crate) fn decode(body: &Bytes) -> Result { + serde_json::from_slice(body) + .map_err(|error| XrpcError::invalid_request(format!("Invalid request body: {error}"))) +} + +pub(crate) fn ok_empty() -> Response { + (StatusCode::OK, Json(json!({}))).into_response() +} + +pub(crate) async fn project_collaborators( + state: &XrpcState, + repo: &RepoDid, +) -> Result<(), XrpcError> { + let index = Arc::clone(&state.index); + let target = repo.clone(); + run_blocking(move || index.ensure_collaborators(&target).map_err(XrpcError::from)).await +} + +pub(crate) async fn current_member( + state: &XrpcState, + actor: &AccountDid, +) -> Result, XrpcError> { + let index = Arc::clone(&state.index); + run_blocking(move || index.refresh_members().map_err(XrpcError::from)).await?; + Ok(state.index.effective_member(actor)) +} + +pub(crate) async fn authorize_push( + state: &XrpcState, + actor: &AccountDid, + repo: &RepoDid, + denied: &str, +) -> Result<(), XrpcError> { + project_collaborators(state, repo).await?; + let consent = state.index.collaboration_consent(repo, actor); + let acl = knot_acl::KnotAcl::new(&state.admins, state.admission, &state.index); + if knot_acl::can_push(&acl, &consent).is_allowed() { + Ok(()) + } else { + Err(XrpcError::forbidden(denied)) + } +} + +pub(crate) async fn authorize_settings( + state: &XrpcState, + actor: &AccountDid, + repo: &RepoDid, +) -> Result<(), XrpcError> { + social::settings_authority(social::permission(state, actor, repo).await?) +} + +pub(crate) async fn authorize_create( + state: &XrpcState, + actor: &AccountDid, + denied: &str, +) -> Result<(), XrpcError> { + let acl = knot_acl::KnotAcl::new(&state.admins, state.admission, &state.index); + let consent = match knot_acl::needs_membership(&acl, actor) { + true => state.index.member_status(actor), + false => knot_index::MemberStatus::unread(actor), + }; + if knot_acl::can_create_repo(&acl, &consent).is_allowed() { + Ok(()) + } else { + Err(XrpcError::forbidden(denied)) + } +} + +pub(crate) async fn authenticate_and_authorize_push( + state: &XrpcState, + socket: SocketPeer, + headers: &HeaderMap, + repo: &RepoDid, + denied: &str, +) -> Result { + let peer = state.proxy_trust.client_peer(headers, socket.ip()); + let guard = admit_pre_auth(state, peer)?; + let actor = state.authenticate_push(headers).await?; + guard.refund(); + authorize_push(state, &actor, repo, denied).await?; + Ok(actor) +} + +pub(crate) async fn run_blocking(task: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + match tokio::task::spawn_blocking(task).await { + Ok(result) => result, + Err(_) => Err(XrpcError::internal("Blocking task failed to complete")), + } +} + +#[derive(serde::Deserialize)] +#[serde(transparent)] +pub(crate) struct OwnerSegment(String); + +#[derive(serde::Deserialize)] +#[serde(transparent)] +pub(crate) struct RepoNameSegment(String); + +impl OwnerSegment { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl RepoNameSegment { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(serde::Deserialize)] +#[serde(transparent)] +pub(crate) struct RepoDidSegment(String); + +impl RepoDidSegment { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(serde::Deserialize)] +pub(crate) struct RepoPathParams { + pub(crate) did: OwnerSegment, + pub(crate) name: RepoNameSegment, +} + +pub(crate) fn resolve_repo_did( + state: &XrpcState, + segment: &RepoDidSegment, +) -> Result { + let raw = segment.as_str(); + let trimmed = raw.strip_suffix(".git").unwrap_or(raw); + let did = RepoDid::new(trimmed).map_err(|_| XrpcError::not_found("Repository not found"))?; + match state.index.owner_of(&did) { + Resolved::Ready(Some(_)) => Ok(did), + Resolved::Ready(None) => Err(XrpcError::not_found("Repository not found")), + Resolved::Warming => Err(crate::reads::warming()), + } +} + +pub(crate) async fn resolve_repo_named( + state: &XrpcState, + owner: &OwnerSegment, + name: &RepoNameSegment, +) -> Result { + let owner = resolve_owner_segment(state, owner).await?; + let path = ClonePath::parse(name.as_str()) + .ok_or_else(|| XrpcError::not_found("Repository not found"))?; + match state.index.resolve_clone_path(&owner, &path) { + Resolved::Ready(Some(did)) => Ok(did), + Resolved::Ready(None) => Err(XrpcError::not_found("Repository not found")), + Resolved::Warming => Err(crate::reads::warming()), + } +} + +async fn resolve_owner_segment( + state: &XrpcState, + owner: &OwnerSegment, +) -> Result { + let not_found = || XrpcError::not_found("Repository not found"); + match OwnerRef::parse(owner.as_str()).ok_or_else(not_found)? { + OwnerRef::Did(did) => Ok(did), + OwnerRef::Handle(handle) => state + .atproto + .resolve_handle_to_did(&handle) + .await + .map(OwnerDid::from) + .map_err(|_| not_found()), + } +} + +#[cfg(test)] +mod credential_tests { + use super::push_credential; + use base64::Engine; + use http::{HeaderMap, HeaderValue, header::AUTHORIZATION}; + + fn with(value: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, HeaderValue::from_str(value).unwrap()); + headers + } + + fn basic(user_pass: &str) -> String { + format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(user_pass) + ) + } + + #[test] + fn a_bearer_token_is_taken_verbatim() { + assert_eq!( + push_credential(&with("Bearer jwt.abc.def")) + .unwrap() + .as_str(), + "jwt.abc.def" + ); + assert_eq!( + push_credential(&with("bearer jwt.abc.def")) + .unwrap() + .as_str(), + "jwt.abc.def" + ); + } + + #[test] + fn a_basic_credential_yields_the_password_after_the_first_colon() { + assert_eq!( + push_credential(&with(&basic("x-tangled-token:jwt.abc.def"))) + .unwrap() + .as_str(), + "jwt.abc.def", + "RFC 7617 puts the token in the password half, so the username stays colon-free" + ); + } + + #[test] + fn malformed_or_empty_credentials_are_rejected() { + assert!(push_credential(&HeaderMap::new()).is_err()); + assert!(push_credential(&with("Bearer ")).is_err()); + assert!(push_credential(&with(&basic("x-tangled-token:"))).is_err()); + assert!(push_credential(&with(&basic("no-colon"))).is_err()); + assert!(push_credential(&with("Basic !!!not-base64")).is_err()); + assert!(push_credential(&with("Digest whatever")).is_err()); + } +} diff --git a/knot2/crates/knot-xrpc/src/locks.rs b/knot2/crates/knot-xrpc/src/locks.rs new file mode 100644 index 0000000000000000000000000000000000000000..718174e77a56aefd146599408a857e861db9c2f1 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/locks.rs @@ -0,0 +1,38 @@ +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::sync::{Mutex, MutexGuard}; + +use knot_types::RepoDid; + +const REPO_SHARDS: usize = 64; + +pub struct CobLocks { + meta: Mutex<()>, + repos: Vec>, +} + +impl Default for CobLocks { + fn default() -> Self { + Self { + meta: Mutex::new(()), + repos: (0..REPO_SHARDS).map(|_| Mutex::new(())).collect(), + } + } +} + +impl CobLocks { + pub fn meta(&self) -> MutexGuard<'_, ()> { + self.meta + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub fn repo(&self, repo: &RepoDid) -> MutexGuard<'_, ()> { + let mut hasher = DefaultHasher::new(); + repo.as_str().hash(&mut hasher); + let shard = (hasher.finish() as usize) % self.repos.len(); + self.repos[shard] + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} diff --git a/knot2/crates/knot-xrpc/src/materialize.rs b/knot2/crates/knot-xrpc/src/materialize.rs new file mode 100644 index 0000000000000000000000000000000000000000..43c3545e634e876674a03e4f3414171bf406d25d --- /dev/null +++ b/knot2/crates/knot-xrpc/src/materialize.rs @@ -0,0 +1,225 @@ +use std::sync::Arc; + +use knot_cob::CobHome; +use knot_git::Repo; +use knot_index::Resolved; +use knot_record::chain::{self, ChainError}; +use knot_runtime::{Clock, HttpTransport, Signer}; +use knot_types::{RepoDid, UnixSeconds}; + +use crate::XrpcState; + +fn announce_unframed( + state: &Arc>, + git: &Repo, + did: &RepoDid, + now: UnixSeconds, +) { + match chain::unframed_tip(git) { + Ok(Some(tip)) => { + crate::firehose::publish_sync(&state.firehose, did, &tip.rev, &tip.commit, now) + } + Ok(None) => {} + Err(error) => tracing::warn!( + repo = did.as_str(), + %error, + "a tip wouldn't read, so its sync frame is skipped" + ), + } +} + +pub fn enable_surfaces(state: &Arc>) -> usize { + let now = state.now(); + let signer = match state.secrets.signer(&state.knot_did) { + Ok(signer) => signer, + Err(error) => { + tracing::error!(%error, "the signing key wouldn't load, so the atproto surfaces stay off"); + return 1; + } + }; + let knot_did = state.knot_repo.clone(); + let mut commits = 0; + let mut failed = 0; + let mut policies = 0; + { + let _guard = state.cob_locks.meta(); + match Repo::open(&state.meta_path) { + Ok(meta) => { + crate::grants::noted( + crate::grants::sync_members(state, &meta), + knot_record::grant::GrantCollection::Member, + ); + crate::grants::noted( + crate::grants::sync_blocks(state, &meta), + knot_record::grant::GrantCollection::Block, + ); + announce_unframed(state, &meta, &knot_did, now); + } + Err(error) => { + tracing::error!(%error, "the members chain failed to enable"); + failed += 1; + } + } + } + for did in state.index.hosted_repos() { + match state.layout.open(&did) { + Ok(git) => { + { + let _guard = state.cob_locks.repo(&did); + let (round, failure) = reconcile_social(state, &git, &did, &signer, now); + commits += round; + failed += failure; + match reconcile_policy(state, &git, &did, &signer) { + Ok(appended) => policies += usize::from(appended), + Err(()) => failed += 1, + } + crate::grants::noted( + crate::grants::sync_collaborators(state, &git, &did), + knot_record::grant::GrantCollection::Collaborator, + ); + announce_unframed(state, &git, &did, now); + } + commits += crate::refrecords::sync_refs(state, &git, &did, &[]); + } + Err(error) => { + tracing::warn!(repo = did.as_str(), %error, "a collaborator chain failed to enable"); + failed += 1; + } + } + } + if policies > 0 { + match state.index.refresh_registry() { + Ok(()) => {} + Err(error) => { + tracing::warn!( + %error, + "the registry projection wouldn't refresh after policy rows were reconciled" + ); + failed += 1; + } + } + } + tracing::info!(commits, failed, policies, "atproto surfaces enabled"); + failed +} + +fn reconcile_social( + state: &Arc>, + git: &Repo, + did: &RepoDid, + signer: &dyn Signer, + now: UnixSeconds, +) -> (usize, usize) { + match crate::social::reconcile(state, git, did, signer, now) { + Ok(reconciled) => { + if reconciled.commits > 0 { + tracing::warn!( + repo = did.as_str(), + commits = reconciled.commits, + "social records had drifted from their projection and were reconciled at boot" + ); + } + (reconciled.commits, 0) + } + Err(error) => { + tracing::error!( + repo = did.as_str(), + %error, + "social records wouldn't reconcile at boot; the next restart retries" + ); + (0, 1) + } + } +} + +fn reconcile_policy( + state: &Arc>, + git: &Repo, + did: &RepoDid, + signer: &dyn Signer, +) -> Result { + let declared = match crate::social::declared_policy(state, git, did, signer) { + Ok(declared) => declared.unwrap_or(state.contribution_policy), + Err(error) => { + tracing::warn!( + repo = did.as_str(), + %error, + "the manifest record wouldn't read at boot, so the registry's policy row stays put" + ); + return Ok(false); + } + }; + let (owner, row) = match ( + state.index.owner_of(did), + state.index.contribution_policy(did), + ) { + (Resolved::Ready(Some(owner)), Resolved::Ready(Some(row))) => (owner, row), + (Resolved::Warming, _) | (_, Resolved::Warming) => { + tracing::warn!( + repo = did.as_str(), + "the registry projection is still warming at boot, so the policy row stays put" + ); + return Ok(false); + } + _ => { + tracing::warn!( + repo = did.as_str(), + "the registry row for this repo went missing, so the policy row stays put" + ); + return Ok(false); + } + }; + let current = row.policy(); + if current == declared { + return Ok(false); + } + let meta = match Repo::open(&state.meta_path) { + Ok(meta) => meta, + Err(error) => { + tracing::warn!( + repo = did.as_str(), + %error, + "the meta repo wouldn't open, so the policy row stays put" + ); + return Err(()); + } + }; + let _guard = state.cob_locks.meta(); + match crate::repos::set_policy_in_registry( + &meta, + &CobHome::from(&state.knot_did), + owner, + did, + declared, + signer, + state.now(), + ) { + Ok(()) => { + tracing::info!( + repo = did.as_str(), + from = current.name(), + to = declared.name(), + "the registry's contribution policy row now follows the manifest record" + ); + Ok(true) + } + Err(error) => { + tracing::warn!( + repo = did.as_str(), + %error, + "the registry's contribution policy row wouldn't follow the manifest record" + ); + Err(()) + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum EnableError { + #[error(transparent)] + Chain(#[from] ChainError), + #[error(transparent)] + Frames(#[from] crate::firehose::UnstampedFrames), + #[error(transparent)] + Body(#[from] knot_types::RecordBodyError), +} diff --git a/knot2/crates/knot-xrpc/src/members.rs b/knot2/crates/knot-xrpc/src/members.rs new file mode 100644 index 0000000000000000000000000000000000000000..436031bffd50d075149417b8eb4b031ce9f6781d --- /dev/null +++ b/knot2/crates/knot-xrpc/src/members.rs @@ -0,0 +1,125 @@ +use std::sync::Arc; + +use axum::response::Response; + +use knot_acl::{KnotAcl, can_admin_knot}; +use knot_cob::{Checkpoint, CobHome, CobStore, Evaluate, KnotAuthored}; +use knot_cobs::{Grant, GrantChange, GrantList, MembersChange, MembersCob}; +use knot_git::Repo; +use knot_index::{Index, IndexError}; +use knot_record::grant::GrantCollection; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, UnixSeconds}; + +use crate::cob::{Authorized, grant_list_apply}; +use crate::error::XrpcError; +use crate::grants::SyncError; +use crate::{XrpcState, ok_empty, run_blocking}; + +pub(crate) fn require_admin( + state: &Arc>, + actor: &AccountDid, + denied: &'static str, +) -> Result<(), XrpcError> { + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + match can_admin_knot(&acl, actor).is_allowed() { + true => Ok(()), + false => Err(XrpcError::forbidden(denied)), + } +} + +pub(crate) async fn member_add( + state: &Arc>, + actor: &AccountDid, + subject: AccountDid, +) -> Result<(), XrpcError> { + require_admin(state, actor, "only knot admin may add members")?; + + offer_membership( + state, + Grant { + subject, + added_by: actor.clone(), + created_at: state.now(), + }, + ) + .await + .map(|_| ()) +} + +pub(crate) async fn offer_membership( + state: &Arc>, + offer: Grant, +) -> Result { + if state.admins.contains(&offer.subject) { + return Ok(ok_empty()); + } + + let now = offer.created_at; + commit_members(state, Authorized::by_decree(MembersChange::Add(offer)), now).await +} + +pub(crate) async fn commit_members( + state: &Arc>, + change: Authorized, + now: UnixSeconds, +) -> Result { + commit_grant_list::( + state, + change, + Index::refresh_members, + crate::grants::sync_members, + GrantCollection::Member, + now, + ) + .await +} + +type SyncFn = fn(&Arc>, &Repo) -> Result; + +pub(crate) async fn commit_grant_list( + state: &Arc>, + change: Authorized, + refresh: fn(&Index) -> Result<(), IndexError>, + sync: SyncFn, + collection: GrantCollection, + now: UnixSeconds, +) -> Result +where + E: Checkpoint + Evaluate, + E::Change: KnotAuthored + Clone + GrantChange + Send + 'static, +{ + let signer = state.secrets.signer(&state.knot_did)?; + let meta_path = state.meta_path.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let home = CobHome::from(&state.knot_did); + let surfaces = Arc::clone(state); + run_blocking(move || { + let _guard = cob_locks.meta(); + let meta = Repo::open(&meta_path)?; + grant_list_apply::(&CobStore::new(&meta), &home, change, &signer, now)?; + refresh(&index)?; + crate::grants::noted(sync(&surfaces, &meta), collection); + Ok(()) + }) + .await?; + + Ok(ok_empty()) +} + +pub(crate) async fn member_remove( + state: &Arc>, + actor: &AccountDid, + subject: AccountDid, +) -> Result<(), XrpcError> { + require_admin(state, actor, "only knot admin may remove members")?; + + commit_members( + state, + Authorized::by_decree(MembersChange::Remove(knot_cobs::Removal { subject })), + state.now(), + ) + .await + .map(|_| ()) +} diff --git a/knot2/crates/knot-xrpc/src/migrate.rs b/knot2/crates/knot-xrpc/src/migrate.rs new file mode 100644 index 0000000000000000000000000000000000000000..b22f5ea1896e1056c268a7d167f5aba3ef7ff87f --- /dev/null +++ b/knot2/crates/knot-xrpc/src/migrate.rs @@ -0,0 +1,184 @@ +use std::collections::BTreeMap; +use std::sync::Arc; + +use knot_git::{RefClass, RefUpdate, Repo}; +use knot_record::chain; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{RefName, RepoDid, UnixSeconds}; + +use crate::XrpcState; +use crate::git::KEEP_PREFIX; + +#[derive(Debug, thiserror::Error)] +pub enum MigrateError { + #[error(transparent)] + Cob(#[from] knot_cob::CobError), + #[error(transparent)] + Chain(#[from] chain::ChainError), + #[error(transparent)] + Git(#[from] knot_git::GitError), + #[error(transparent)] + Frames(#[from] crate::firehose::UnstampedFrames), + #[error("the signing key wouldn't load for the legacy sweep: {0}")] + Key(#[from] knot_secrets::SecretsError), +} + +const LEGACY_COLLECTIONS: [&str; 4] = [ + "sh.tangled.git.ref", + "sh.tangled.knot.member", + "sh.tangled.knot.block", + "sh.tangled.repo.collaborator", +]; + +pub fn legacy_meta_refs(meta: &Repo) -> Result { + Ok(knot_cob::migrate_legacy_refs(meta)?) +} + +pub fn legacy_state( + state: &Arc>, +) -> Result { + let meta = Repo::open(&state.meta_path)?; + let mut swept = knot_cob::migrate_legacy_refs(&meta)?; + swept += sweep_legacy_records(state, &meta, &state.knot_repo)?; + swept += state + .index + .hosted_repos() + .into_iter() + .map(|repo| noted(state, repo)) + .sum::(); + Ok(swept) +} + +fn noted(state: &Arc>, repo: RepoDid) -> usize { + match swept_repo(state, &repo) { + Ok(count) => count, + Err(error) => { + tracing::warn!( + repo = %repo, + %error, + "the legacy sweep left this repo alone; its records and refs stay as they are" + ); + 0 + } + } +} + +fn swept_repo( + state: &Arc>, + repo: &RepoDid, +) -> Result { + let git = state.layout.open(repo)?; + Ok(knot_cob::migrate_legacy_refs(&git)? + + normalize_keep_refs(&git)? + + drop_hidden_refs(&git)? + + sweep_legacy_records(state, &git, repo)?) +} + +fn sweep_legacy_records( + state: &Arc>, + git: &Repo, + did: &RepoDid, +) -> Result { + if chain::tip(git)?.is_none() { + return Ok(0); + } + let now: UnixSeconds = state.now(); + let records = chain::in_tree(BTreeMap::new()); + let collections: Vec<_> = LEGACY_COLLECTIONS + .iter() + .map(|nsid| knot_types::RecordCollection::new(*nsid).expect("literal NSID parses")) + .collect(); + let signer = state.secrets.signer(&state.knot_did)?; + let mut emit = crate::firehose::FirehoseEmit::new(&state.firehose, did, now); + let chunks = futures::executor::block_on(chain::settle( + git, + did, + &collections, + &records, + &signer, + &mut emit, + now, + ))?; + emit.finish()?; + Ok(chunks.len()) +} + +fn normalize_keep_refs(git: &Repo) -> Result { + let renamed: Vec<_> = git + .references()? + .into_iter() + .filter_map(|record| { + let modern = normalized_keep_name(&record.name)?; + Some((record.name, modern, record.target)) + }) + .collect(); + let count = renamed.len(); + git.rename_references(renamed)?; + Ok(count) +} + +fn drop_hidden_refs(git: &Repo) -> Result { + let deletes: Vec<_> = git + .references()? + .into_iter() + .filter(|record| RefClass::of(&record.name) == RefClass::Hidden) + .map(|record| RefUpdate::Delete { + name: record.name, + old: record.target, + }) + .collect(); + if !deletes.is_empty() { + git.update_refs(&deletes)?; + } + Ok(deletes.len()) +} + +fn normalized_keep_name(name: &RefName) -> Option { + let rest = name.as_str().strip_prefix(KEEP_PREFIX)?; + let components: Vec<_> = rest + .split('/') + .map(modernized_component) + .collect::>()?; + let joined = components.join("/"); + (joined != rest) + .then(|| format!("{KEEP_PREFIX}{joined}")) + .and_then(|modern| RefName::new(modern).ok()) +} + +fn modernized_component(component: &str) -> Option { + let decoded = percent_decode(component)?; + let modern = match knot_types::TypeName::new(&decoded) { + Ok(type_name) => knot_cob::modernized_type_of(&type_name) + .map(|modern| modern.as_str().to_string()) + .unwrap_or_else(|| { + decoded + .strip_prefix("sh.tangled.") + .map(|rest| format!("org.tangled.{rest}")) + .unwrap_or(decoded.clone()) + }), + Err(_) => decoded, + }; + Some(crate::git::encode_ref_component(&modern)) +} + +fn percent_decode(raw: &str) -> Option { + let bytes = raw.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut at = 0; + while at < bytes.len() { + match bytes[at] { + b'%' => { + let pair = bytes.get(at + 1..at + 3)?; + let high = (pair[0] as char).to_digit(16)?; + let low = (pair[1] as char).to_digit(16)?; + out.push((high * 16 + low) as u8); + at += 3; + } + byte => { + out.push(byte); + at += 1; + } + } + } + String::from_utf8(out).ok() +} diff --git a/knot2/crates/knot-xrpc/src/mint.rs b/knot2/crates/knot-xrpc/src/mint.rs new file mode 100644 index 0000000000000000000000000000000000000000..efd22ddc2cc660dcfbbc7d780dce22e8206f3f81 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/mint.rs @@ -0,0 +1,88 @@ +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::State; +use axum::http::HeaderMap; +use axum::response::{IntoResponse, Response}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{DidService, Nsid, RepoDid}; + +use crate::XrpcState; +use crate::authorize_push; +use crate::decode; +use crate::error::XrpcError; +use crate::reads::require_hosted; +use knot_atproto::{JwtNonce, RepoTokenExpiry}; + +pub(crate) const MINT_TOKEN_ROUTE: &str = "/xrpc/org.tangled.repo.mintToken"; + +const MINT_DENIED: &str = "A repo token grants the same authority as a push, and \ + you aren't on the roster; please ask the repo owner to add you as a collaborator, \ + or a knot admin to act for you."; + +#[derive(serde::Deserialize)] +struct MintTokenInput { + repo: RepoDid, + aud: DidService, + lxm: Nsid, + exp: i64, +} + +#[derive(serde::Serialize)] +struct MintTokenOutput { + token: knot_atproto::ServiceJwt, +} + +pub(crate) async fn mint_token( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let MintTokenInput { + repo, + aud, + lxm, + exp, + } = decode(&body)?; + let repo = require_hosted(&state, repo)?; + let actor = state + .authenticate_write_for_repo(&headers, &method, &repo) + .await?; + authorize_push(&state, &actor, &repo, MINT_DENIED).await?; + + let now = state.now(); + let expires_at = RepoTokenExpiry::mint(exp, now) + .map_err(|rejected| XrpcError::invalid_request(rejected.to_string()))?; + + let mut entropy = [0u8; 16]; + state.entropy.fill(&mut entropy); + let nonce = JwtNonce::from_entropy(entropy); + + let signer = state.secrets.signer(&state.knot_did)?; + let token = knot_atproto::mint_repo_token( + &signer, + knot_atproto::RepoTokenRequest { + repo: &repo, + audience: &aud, + method: &lxm, + nonce: &nonce, + controller: &actor, + now, + expires_at, + }, + ); + + tracing::info!( + route = MINT_TOKEN_ROUTE, + repo = repo.as_str(), + actor = actor.as_str(), + jti = nonce.as_str(), + aud = aud.as_str(), + lxm = lxm.as_str(), + "minted a repo token" + ); + + Ok(Json(MintTokenOutput { token }).into_response()) +} diff --git a/knot2/crates/knot-xrpc/src/out.rs b/knot2/crates/knot-xrpc/src/out.rs new file mode 100644 index 0000000000000000000000000000000000000000..b0bd2ccb19ad40d1d5b8f25296e1864e4b6ee3e3 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/out.rs @@ -0,0 +1,96 @@ +use knot_types::{Datetime, RecordCid, UnixSeconds}; +use serde::{Deserialize, Serialize, Serializer}; +use std::fmt; +use std::str::FromStr; + +fn zoned(seconds: i64, offset_seconds: i32) -> chrono::DateTime { + let offset = chrono::FixedOffset::east_opt(offset_seconds) + .unwrap_or_else(|| chrono::FixedOffset::east_opt(0).expect("Zero offset is valid")); + chrono::DateTime::from_timestamp(seconds, 0) + .unwrap_or_default() + .with_timezone(&offset) +} + +pub(crate) fn rfc3339(at: UnixSeconds, offset_seconds: i32) -> String { + zoned(at.get(), offset_seconds).to_rfc3339_opts(chrono::SecondsFormat::Secs, true) +} + +pub(crate) fn datetime_of(at: UnixSeconds) -> Datetime { + Datetime::new( + chrono::DateTime::from_timestamp(at.get(), 0) + .unwrap_or_default() + .fixed_offset(), + ) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CidString(cid::Cid); + +impl CidString { + #[cfg(test)] + pub(crate) fn cid(&self) -> cid::Cid { + self.0 + } + + pub(crate) fn record(self) -> Option { + RecordCid::new(self.0) + } +} +impl From for CidString { + fn from(cid: cid::Cid) -> Self { + Self(cid) + } +} + +impl From for CidString { + fn from(cid: RecordCid) -> Self { + Self(cid.cid()) + } +} + +impl Serialize for CidString { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0.to_string()) + } +} + +impl fmt::Display for CidString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(&self.0.to_string()) + } +} +impl<'de> Deserialize<'de> for CidString { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + cid::Cid::from_str(&raw) + .map(Self) + .map_err(serde::de::Error::custom) + } +} + +pub(crate) fn fold_subject(message: &str) -> String { + message + .split("\n\n") + .next() + .unwrap_or_default() + .lines() + .map(str::trim_end) + .filter(|line| !line.is_empty()) + .collect::>() + .join(" ") +} + +#[cfg(test)] +mod tests { + use super::rfc3339; + use knot_types::UnixSeconds; + + const A_JUNE_INSTANT: UnixSeconds = UnixSeconds::new(1_717_236_600); + + #[test] + fn rfc3339_renders_the_commits_own_offset_independent_of_the_host_zone() { + assert!(rfc3339(A_JUNE_INSTANT, 7200).ends_with("+02:00")); + assert!(rfc3339(A_JUNE_INSTANT, -18000).ends_with("-05:00")); + assert!(rfc3339(A_JUNE_INSTANT, 0).ends_with('Z')); + } +} diff --git a/knot2/crates/knot-xrpc/src/outbox.rs b/knot2/crates/knot-xrpc/src/outbox.rs new file mode 100644 index 0000000000000000000000000000000000000000..4d32048cacfa19f474c2ab9011c2593dc6787ba5 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/outbox.rs @@ -0,0 +1,1514 @@ +use std::collections::{HashMap, HashSet, VecDeque}; +use std::hash::Hash; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use knot_atproto::{Atproto, AtprotoError, PointerAuthorizer, ServiceAuth, SpindleAnswer}; +use knot_resource::{FileMode, atomic_write_bytes}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{RepoDid, ServiceDid, UnixSeconds}; +use serde::{Deserialize, Serialize}; +use tokio::sync::{Notify, oneshot}; +use tokio::task::JoinSet; +use tokio_util::sync::CancellationToken; +use tokio_util::task::AbortOnDropHandle; + +use crate::XrpcState; + +const FILE: &str = "spindle-outbox.json"; + +#[derive(Debug, Clone, Copy)] +pub struct OutboxLimits { + pub backoff_base: Duration, + pub backoff_cap: Duration, + pub max_age: Duration, + pub max_entries: usize, + pub max_per_spindle: usize, + pub max_per_repo: usize, + pub give_up_after: u32, + pub max_in_flight_per_spindle: usize, + pub delivery_timeout: Duration, +} + +impl Default for OutboxLimits { + fn default() -> Self { + Self { + backoff_base: Duration::from_secs(1), + backoff_cap: Duration::from_secs(60), + max_age: Duration::from_secs(3 * 24 * 60 * 60), + max_entries: 1_000, + // a tenant's dead spindle can only crowd out its own events + max_per_spindle: 200, + // a repo the spindle keeps failing can only crowd out its own events + max_per_repo: 100, + // about an hour at the backoff cap, and only if the spindle takes other repos' events meanwhile + give_up_after: 60, + max_in_flight_per_spindle: 4, + // longer than the http client's own timeouts because it only catches what slips past them + delivery_timeout: Duration::from_secs(120), + } + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub(crate) struct Entry { + id: u64, + pub(crate) spindle: ServiceDid, + pub(crate) repo: RepoDid, + pub(crate) ref_name: String, + pub(crate) body: serde_json::Value, + queued_at: UnixSeconds, +} + +struct Backoff { + failures: u32, + retry_at: Instant, + since: Instant, +} + +struct Inner { + entries: VecDeque, + next_id: u64, + // keyed by the lane's head entry, so its next entry starts over + backoff: HashMap, + spindles: HashMap, + delivered: HashMap, + in_flight: HashMap, + // only an entry's first attempt answers, a retry lands after the push output is gone + pushers: HashMap>>, + #[cfg(test)] + writes: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + /// no answer from the spindle itself. all its repos wait unless others still get through + Unreachable, + /// the spindle answered but didn't take the event. only its repo waits + Refused, + /// a 4xx other than 408 and 429, which a retry can't fix + Rejected, +} + +#[derive(Debug)] +pub(crate) struct Failure { + error: String, + kind: Kind, +} + +impl Failure { + pub(crate) fn unreachable(error: impl Into) -> Self { + Self { + error: error.into(), + kind: Kind::Unreachable, + } + } +} + +impl From for Failure { + fn from(error: AtprotoError) -> Self { + let kind = match &error { + AtprotoError::NotifyEvent { status, .. } => match status.get() { + 408 | 429 | 502..=504 => Kind::Unreachable, + 400..=499 => Kind::Rejected, + _ => Kind::Refused, + }, + _ => Kind::Unreachable, + }; + Self { + error: error.to_string(), + kind, + } + } +} + +/// reloaded from disk in open, so a restart picks the queue back up +pub struct Outbox { + path: PathBuf, + limits: OutboxLimits, + inner: Mutex, + wake: Notify, +} + +fn backoff_delay(limits: &OutboxLimits, failures: u32) -> Duration { + let doublings = failures.saturating_sub(1).min(31); + limits + .backoff_base + .saturating_mul(1u32 << doublings) + .min(limits.backoff_cap) +} + +fn waiting(backoff: Option<&Backoff>, now: Instant) -> bool { + backoff.is_some_and(|backoff| backoff.retry_at > now) +} + +fn record_failure( + limits: &OutboxLimits, + backoff: &mut HashMap, + key: K, + since: Instant, + now: Instant, +) -> (u32, Instant) { + let backoff = backoff.entry(key).or_insert(Backoff { + failures: 0, + retry_at: now, + since, + }); + backoff.failures = backoff.failures.saturating_add(1); + backoff.retry_at = now + backoff_delay(limits, backoff.failures); + (backoff.failures, backoff.since) +} + +// one warning about every ten minutes at the backoff cap +fn worth_warning(failures: u32) -> bool { + failures == 1 || failures.is_multiple_of(10) +} + +impl Outbox { + pub fn open(dir: &Path, limits: OutboxLimits) -> Arc { + let path = dir.join(FILE); + let entries: VecDeque = match std::fs::read(&path) { + Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|error| { + tracing::error!(path = %path.display(), %error, "couldn't decode the spindle outbox, starting empty"); + VecDeque::new() + }), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => VecDeque::new(), + Err(error) => { + tracing::error!(path = %path.display(), %error, "couldn't read the spindle outbox, starting empty"); + VecDeque::new() + } + }; + let next_id = entries.iter().map(|entry| entry.id + 1).max().unwrap_or(0); + Arc::new(Self { + path, + limits, + inner: Mutex::new(Inner { + entries, + next_id, + backoff: HashMap::new(), + spindles: HashMap::new(), + delivered: HashMap::new(), + in_flight: HashMap::new(), + pushers: HashMap::new(), + #[cfg(test)] + writes: 0, + }), + wake: Notify::new(), + }) + } + + /// one push's events go in under one write, so a many-ref push costs one fsync + pub(crate) fn enqueue_all( + &self, + spindle: &ServiceDid, + repo: &RepoDid, + events: Vec<(String, serde_json::Value)>, + now: UnixSeconds, + ) -> Vec>> { + if events.is_empty() { + return Vec::new(); + } + let mut inner = self.inner.lock().expect("outbox lock isn't poisoned"); + inner.pushers.retain(|_, pusher| !pusher.is_closed()); + let mut replies = Vec::new(); + for (ref_name, body) in events { + self.make_room(&mut inner, spindle, repo); + let id = inner.next_id; + inner.next_id += 1; + let (pusher, reply) = oneshot::channel(); + inner.pushers.insert(id, pusher); + replies.push(reply); + inner.entries.push_back(Entry { + id, + spindle: spindle.clone(), + repo: repo.clone(), + ref_name, + body, + queued_at: now, + }); + } + Self::prune_backoff(&mut inner); + self.persist(&mut inner); + drop(inner); + self.wake.notify_one(); + replies + } + + /// an entry in flight is never the one evicted, so its settle still finds it + fn make_room(&self, inner: &mut Inner, spindle: &ServiceDid, repo: &RepoDid) { + let in_spindle = |entry: &Entry| &entry.spindle == spindle; + let in_lane = |entry: &Entry| in_spindle(entry) && &entry.repo == repo; + let anywhere = |_: &Entry| true; + loop { + let held = |pick: &dyn Fn(&Entry) -> bool| { + inner.entries.iter().filter(|entry| pick(entry)).count() + }; + let (pick, why): (&dyn Fn(&Entry) -> bool, &str) = + if held(&in_lane) >= self.limits.max_per_repo { + (&in_lane, "its repo has too many events queued") + } else if held(&in_spindle) >= self.limits.max_per_spindle { + (&in_spindle, "its spindle has too many events queued") + } else if inner.entries.len() >= self.limits.max_entries { + (&anywhere, "the spindle outbox is full") + } else { + return; + }; + let Some(at) = inner + .entries + .iter() + .position(|entry| pick(entry) && !inner.in_flight.contains_key(&entry.id)) + else { + return; + }; + if let Some(dropped) = inner.entries.remove(at) { + tracing::warn!( + repo = dropped.repo.as_str(), + ref_name = dropped.ref_name.as_str(), + spindle = dropped.spindle.as_str(), + "{why}, so its oldest push event is dropped" + ); + } + } + } + + fn prune_backoff(inner: &mut Inner) { + let queued: HashSet = inner.entries.iter().map(|entry| entry.id).collect(); + inner.backoff.retain(|id, _| queued.contains(id)); + let spindles: HashSet<&ServiceDid> = + inner.entries.iter().map(|entry| &entry.spindle).collect(); + inner + .spindles + .retain(|spindle, _| spindles.contains(spindle)); + inner + .delivered + .retain(|spindle, _| spindles.contains(spindle)); + } + + fn drop_entry(&self, inner: &mut Inner, id: u64) { + inner.entries.retain(|queued| queued.id != id); + Self::prune_backoff(inner); + self.persist(inner); + } + + pub fn queued(&self) -> Vec<(ServiceDid, serde_json::Value)> { + let inner = self.inner.lock().expect("outbox lock isn't poisoned"); + inner + .entries + .iter() + .map(|entry| (entry.spindle.clone(), entry.body.clone())) + .collect() + } + + fn persist(&self, inner: &mut Inner) { + #[cfg(test)] + { + inner.writes += 1; + } + let written = serde_json::to_vec(&inner.entries) + .map_err(|error| error.to_string()) + .and_then(|bytes| { + atomic_write_bytes(&self.path, &bytes, FileMode::Inherited) + .map_err(|error| error.to_string()) + }); + if let Err(error) = written { + tracing::error!(path = %self.path.display(), %error, "couldn't write the spindle outbox, so a restart forgets what's queued"); + } + } + + fn expire(&self, inner: &mut Inner, now: UnixSeconds) { + let oldest_kept = now.saturating_sub_secs(self.limits.max_age.as_secs() as i64); + let before = inner.entries.len(); + let in_flight = &inner.in_flight; + inner.entries.retain(|entry| { + let fresh = entry.queued_at >= oldest_kept || in_flight.contains_key(&entry.id); + if !fresh { + tracing::warn!( + repo = entry.repo.as_str(), + ref_name = entry.ref_name.as_str(), + spindle = entry.spindle.as_str(), + "giving up on a push event the spindle never took" + ); + } + fresh + }); + if inner.entries.len() != before { + Self::prune_backoff(inner); + self.persist(inner); + } + } + + // only the oldest entry per lane goes out because a repo's events stay in order + fn claim_due(&self, now: Instant, wall: UnixSeconds) -> Vec { + let mut inner = self.inner.lock().expect("outbox lock isn't poisoned"); + self.expire(&mut inner, wall); + let mut busy: HashMap<&ServiceDid, usize> = HashMap::new(); + for entry in &inner.entries { + if inner.in_flight.contains_key(&entry.id) { + *busy.entry(&entry.spindle).or_default() += 1; + } + } + let mut seen = HashSet::new(); + let mut due = Vec::new(); + for entry in &inner.entries { + let lane = (&entry.spindle, &entry.repo); + if !seen.insert(lane) || inner.in_flight.contains_key(&entry.id) { + continue; + } + let waiting = waiting(inner.backoff.get(&entry.id), now) + || waiting(inner.spindles.get(&entry.spindle), now); + let busy = busy.entry(&entry.spindle).or_default(); + if !waiting && *busy < self.limits.max_in_flight_per_spindle { + *busy += 1; + due.push(entry.clone()); + } + } + for entry in &due { + inner.in_flight.insert(entry.id, now); + } + due + } + + /// a claimed entry that never settles keeps its lane blocked for good + fn settle(&self, entry: &Entry, result: Result, Failure>, now: Instant) { + let mut inner = self.inner.lock().expect("outbox lock isn't poisoned"); + let claimed_at = inner.in_flight.remove(&entry.id).unwrap_or(now); + if let Some(pusher) = inner.pushers.remove(&entry.id) { + let _ = pusher.send(result.as_ref().ok().cloned().flatten()); + } + if !inner.entries.iter().any(|queued| queued.id == entry.id) { + return; + } + let Failure { error, kind } = match result { + Ok(_) => { + inner.spindles.remove(&entry.spindle); + inner.delivered.insert(entry.spindle.clone(), now); + self.drop_entry(&mut inner, entry.id); + return; + } + Err(failure) => failure, + }; + if kind != Kind::Unreachable { + inner.spindles.remove(&entry.spindle); + } + if kind == Kind::Rejected { + tracing::warn!( + repo = entry.repo.as_str(), + ref_name = entry.ref_name.as_str(), + spindle = entry.spindle.as_str(), + %error, + "the spindle rejected the push event, so it's dropped" + ); + self.drop_entry(&mut inner, entry.id); + return; + } + let since = inner + .backoff + .get(&entry.id) + .map_or(claimed_at, |backoff| backoff.since); + let others_went_through = inner + .delivered + .get(&entry.spindle) + .is_some_and(|at| *at >= since); + // an unreachable spindle that took other repos' events meanwhile is only failing this repo + if kind == Kind::Unreachable && !others_went_through { + let (failures, since) = record_failure( + &self.limits, + &mut inner.spindles, + entry.spindle.clone(), + now, + now, + ); + if worth_warning(failures) { + tracing::warn!( + spindle = entry.spindle.as_str(), + attempts = failures, + failing_for_secs = (now - since).as_secs(), + %error, + "the spindle can't be reached, so its push events wait" + ); + } + return; + } + let (failures, since) = + record_failure(&self.limits, &mut inner.backoff, entry.id, since, now); + if failures >= self.limits.give_up_after && others_went_through { + tracing::error!( + repo = entry.repo.as_str(), + ref_name = entry.ref_name.as_str(), + spindle = entry.spindle.as_str(), + attempts = failures, + failing_for_secs = (now - since).as_secs(), + queued_at = entry.queued_at.get(), + %error, + "the spindle keeps failing this push event while taking others, so it's dropped" + ); + self.drop_entry(&mut inner, entry.id); + } else if worth_warning(failures) { + tracing::warn!( + repo = entry.repo.as_str(), + ref_name = entry.ref_name.as_str(), + spindle = entry.spindle.as_str(), + attempts = failures, + failing_for_secs = (now - since).as_secs(), + %error, + "the spindle didn't take the push event, retrying" + ); + } + } + + /// anything already due was offered to claim_due. only later instants matter + fn next_retry(&self, now: Instant) -> Option { + let inner = self.inner.lock().expect("outbox lock isn't poisoned"); + inner + .backoff + .values() + .chain(inner.spindles.values()) + .map(|backoff| backoff.retry_at) + .filter(|at| *at > now) + .min() + } +} + +pub(crate) async fn deliver( + atproto: &Atproto, + auth: &dyn PointerAuthorizer, + entry: &Entry, +) -> Result, AtprotoError> { + let answer = atproto + .notify_spindle(auth, &entry.spindle, entry.body.to_string().into()) + .await?; + let (pipelines, message) = match answer { + SpindleAnswer::Scheduled { pipelines, message } => (pipelines, message), + SpindleAnswer::Nothing => (Vec::new(), None), + SpindleAnswer::Malformed(problem) => { + tracing::warn!( + spindle = entry.spindle.as_str(), + problem = problem.as_str(), + "the spindle's answer to the push wasn't readable, counting it delivered" + ); + (Vec::new(), None) + } + }; + tracing::info!( + repo = entry.repo.as_str(), + ref_name = entry.ref_name.as_str(), + spindle = entry.spindle.as_str(), + ?pipelines, + "the spindle heard of the push" + ); + Ok(message) +} + +/// settles a delivery that hangs or panics, otherwise its lane stays claimed +async fn guarded( + delivery: impl Future, Failure>> + Send + 'static, + timeout: Duration, +) -> Result, Failure> { + let task = AbortOnDropHandle::new(tokio::spawn(delivery)); + match tokio::time::timeout(timeout, task).await { + Ok(Ok(result)) => result, + Ok(Err(error)) => Err(Failure { + error: format!("the delivery died: {error}"), + kind: Kind::Refused, + }), + Err(_) => Err(Failure::unreachable("the spindle didn't answer in time")), + } +} + +async fn attempt(state: Arc>, entry: Entry) { + let delivery = { + let (state, entry) = (Arc::clone(&state), entry.clone()); + async move { + let signer = state.secrets.signer(&state.knot_did).map_err(|error| { + Failure::unreachable(format!("the signing key wouldn't load: {error}")) + })?; + let auth = ServiceAuth::new(&signer, state.entropy.as_ref()); + deliver(&state.atproto, &auth, &entry) + .await + .map_err(Failure::from) + } + }; + let result = guarded(delivery, state.outbox.limits.delivery_timeout).await; + state.outbox.settle(&entry, result, Instant::now()); + state.outbox.wake.notify_one(); +} + +/// spawned, so a slow spindle can't stall the loop for the others +pub async fn run( + state: Arc>, + shutdown: CancellationToken, +) { + let outbox = Arc::clone(&state.outbox); + let mut deliveries = JoinSet::new(); + loop { + let (now, wall) = (Instant::now(), state.now()); + for entry in outbox.claim_due(now, wall) { + deliveries.spawn(attempt(Arc::clone(&state), entry)); + } + let next = outbox.next_retry(now); + let idle = async { + match next { + Some(at) => tokio::time::sleep_until(at.into()).await, + None => std::future::pending().await, + } + }; + tokio::select! { + () = outbox.wake.notified() => {} + () = idle => {} + _ = deliveries.join_next(), if !deliveries.is_empty() => {} + () = shutdown.cancelled() => return, + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use knot_atproto::{PlcDirectory, PointerAuth}; + use knot_runtime::{FakeHttp, HttpRequest, HttpResponse, ManualClock, NetworkError}; + use knot_types::{HttpStatus, KnotId, UnixMicros}; + + use super::*; + + struct NoAuth; + + impl PointerAuthorizer for NoAuth { + fn authorize( + &self, + _request: &mut HttpRequest, + _ctx: &PointerAuth<'_>, + ) -> Result<(), AtprotoError> { + Ok(()) + } + } + + const SPINDLE: &str = "did:web:spindle.test"; + const OTHER: &str = "did:web:other.test"; + const START_SECS: i64 = 1_000_000; + + type Responder = Box Result + Send + Sync>; + + fn limits() -> OutboxLimits { + OutboxLimits { + backoff_base: Duration::from_secs(1), + backoff_cap: Duration::from_secs(5), + max_age: Duration::from_secs(100), + max_entries: 10, + max_per_spindle: 10, + max_per_repo: 10, + give_up_after: 100, + max_in_flight_per_spindle: 10, + delivery_timeout: Duration::from_secs(5), + } + } + + fn spindle(did: &str) -> ServiceDid { + ServiceDid::new(did).unwrap() + } + + fn enqueue(outbox: &Outbox, to: &str, ref_name: &str, now: i64) { + enqueue_all(outbox, to, &[ref_name], now); + } + + fn enqueue_all( + outbox: &Outbox, + to: &str, + ref_names: &[&str], + now: i64, + ) -> Vec>> { + enqueue_repo(outbox, to, "did:plc:squid", ref_names, now) + } + + fn enqueue_repo( + outbox: &Outbox, + to: &str, + repo: &str, + ref_names: &[&str], + now: i64, + ) -> Vec>> { + let events = ref_names + .iter() + .map(|name| { + ( + name.to_string(), + serde_json::json!({ "event": { "ref": name } }), + ) + }) + .collect(); + outbox.enqueue_all( + &spindle(to), + &RepoDid::new(repo).unwrap(), + events, + UnixSeconds::new(now), + ) + } + + fn answered(status: u16) -> Failure { + Failure::from(AtprotoError::NotifyEvent { + spindle: spindle(SPINDLE), + status: HttpStatus::new(status), + }) + } + + fn refs(due: &[Entry]) -> Vec<&str> { + due.iter().map(|entry| entry.ref_name.as_str()).collect() + } + + fn doc(host: &str) -> HttpResponse { + let body = serde_json::json!({ + "id": format!("did:web:{host}"), + "service": [{ + "id": "#tangled_spindle", + "type": "TangledSpindle", + "serviceEndpoint": format!("https://{host}"), + }], + }); + reply(200, body.to_string()) + } + + fn reply(status: u16, body: String) -> HttpResponse { + HttpResponse { + status: http::StatusCode::from_u16(status).unwrap(), + headers: http::HeaderMap::new(), + body: body.into(), + } + } + + struct Spindles { + atproto: Atproto, Arc>, + attempts: Arc, + taken: Arc>>, + clock: Arc, + } + + fn spindles(failures: usize, down_host: Option<&'static str>) -> Spindles { + let attempts = Arc::new(AtomicUsize::new(0)); + let taken = Arc::new(Mutex::new(Vec::new())); + let clock = Arc::new(ManualClock::new(UnixMicros::new( + START_SECS as u64 * 1_000_000, + ))); + let responder: Responder = { + let (attempts, taken) = (Arc::clone(&attempts), Arc::clone(&taken)); + Box::new(move |request| { + let host = request.url.host_str().unwrap().to_string(); + if request.method == http::Method::GET { + return Ok(doc(&host)); + } + let seen = attempts.fetch_add(1, Ordering::SeqCst); + if seen < failures || down_host == Some(host.as_str()) { + return Ok(reply(503, String::new())); + } + let body: serde_json::Value = + serde_json::from_slice(request.body.as_ref().unwrap()).unwrap(); + let ref_name = body["event"]["ref"].as_str().unwrap().to_string(); + let answer = serde_json::json!({ "ci": { "pipelines": ["3kabc"], "message": format!("follow {ref_name}") } }); + taken.lock().unwrap().push((host, ref_name)); + Ok(reply(200, answer.to_string())) + }) + }; + let atproto = Atproto::new( + FakeHttp::new(responder), + Arc::clone(&clock), + KnotId::new("did:web:knot.test").unwrap(), + PlcDirectory::new(url::Url::parse("https://plc.directory/").unwrap()).unwrap(), + ); + Spindles { + atproto, + attempts, + taken, + clock, + } + } + + fn outbox(dir: &Path) -> Arc { + Outbox::open(dir, limits()) + } + + async fn pump(outbox: &Outbox, world: &Spindles, now: Instant) -> Option { + loop { + let due = outbox.claim_due(now, world.atproto.now().seconds()); + if due.is_empty() { + return outbox.next_retry(now); + } + let results = futures::future::join_all( + due.iter() + .map(|entry| deliver(&world.atproto, &NoAuth, entry)), + ) + .await; + for (entry, result) in due.iter().zip(results) { + outbox.settle(entry, result.map_err(Failure::from), now); + } + } + } + + fn queued_refs(outbox: &Outbox) -> Vec { + outbox + .queued() + .iter() + .map(|(_, body)| body["event"]["ref"].as_str().unwrap().to_string()) + .collect() + } + + #[tokio::test] + async fn the_pusher_hears_what_the_spindle_said_on_the_first_attempt() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + let world = spindles(0, None); + let mut replies = enqueue_all(&outbox, SPINDLE, &["refs/heads/main"], START_SECS); + pump(&outbox, &world, Instant::now()).await; + assert_eq!( + replies.remove(0).await.unwrap(), + Some("follow refs/heads/main".to_string()) + ); + assert_eq!(world.taken.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn a_failed_first_attempt_tells_the_pusher_nothing_and_the_retry_still_delivers_once() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + let world = spindles(1, None); + let mut replies = enqueue_all(&outbox, SPINDLE, &["refs/heads/main"], START_SECS); + let t0 = Instant::now(); + pump(&outbox, &world, t0).await; + assert_eq!(replies.remove(0).await.unwrap(), None); + pump(&outbox, &world, t0 + Duration::from_secs(1)).await; + assert_eq!(world.attempts.load(Ordering::SeqCst), 2); + assert_eq!(world.taken.lock().unwrap().len(), 1); + assert!(outbox.queued().is_empty()); + } + + #[test] + fn backoff_doubles_up_to_the_cap() { + let delays: Vec = (1..=6) + .map(|failures| backoff_delay(&limits(), failures).as_secs()) + .collect(); + assert_eq!(delays, [1, 2, 4, 5, 5, 5]); + assert_eq!(backoff_delay(&limits(), u32::MAX), limits().backoff_cap); + } + + #[tokio::test] + async fn a_queued_event_survives_a_restart_and_is_then_delivered() { + let dir = tempfile::tempdir().unwrap(); + enqueue(&outbox(dir.path()), SPINDLE, "refs/heads/main", START_SECS); + let world = spindles(0, None); + + let reopened = outbox(dir.path()); + assert_eq!( + reopened.queued().len(), + 1, + "the entry is still on disk after a reopen" + ); + pump(&reopened, &world, Instant::now()).await; + + assert_eq!( + *world.taken.lock().unwrap(), + [("spindle.test".to_string(), "refs/heads/main".to_string())] + ); + assert!( + outbox(dir.path()).queued().is_empty(), + "delivery is persisted too" + ); + } + + #[tokio::test] + async fn a_failing_spindle_is_retried_with_growing_delays_until_it_answers_2xx() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + enqueue(&outbox, SPINDLE, "refs/heads/main", START_SECS); + let world = spindles(3, None); + let t0 = Instant::now(); + let secs = Duration::from_secs; + + let next = pump(&outbox, &world, t0).await; + assert_eq!(next, Some(t0 + secs(1))); + pump(&outbox, &world, t0 + secs(0)).await; + assert_eq!( + world.attempts.load(Ordering::SeqCst), + 1, + "no retry before its delay" + ); + + let next = pump(&outbox, &world, t0 + secs(1)).await; + assert_eq!(next, Some(t0 + secs(3))); + let next = pump(&outbox, &world, t0 + secs(3)).await; + assert_eq!(next, Some(t0 + secs(7))); + assert_eq!(world.attempts.load(Ordering::SeqCst), 3); + + let next = pump(&outbox, &world, t0 + secs(7)).await; + assert_eq!(next, None); + assert_eq!(world.attempts.load(Ordering::SeqCst), 4); + assert!(outbox.queued().is_empty()); + assert_eq!(world.taken.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn an_event_the_spindle_never_took_is_dropped_after_the_age_limit() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + enqueue(&outbox, SPINDLE, "refs/heads/old", START_SECS); + enqueue(&outbox, SPINDLE, "refs/heads/young", START_SECS + 50); + let world = spindles(0, None); + // the old one is 120s old, past the 100s max age, the young one is 70s + world.clock.advance(Duration::from_secs(120)); + pump(&outbox, &world, Instant::now()).await; + assert_eq!( + *world.taken.lock().unwrap(), + [("spindle.test".to_string(), "refs/heads/young".to_string())], + "only the old event is given up on" + ); + } + + #[test] + fn a_full_outbox_drops_its_oldest_event() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + max_entries: 2, + ..limits() + }, + ); + for name in ["refs/heads/a", "refs/heads/b", "refs/heads/c"] { + enqueue(&outbox, SPINDLE, name, START_SECS); + } + let kept = queued_refs(&outbox); + assert_eq!(kept, ["refs/heads/b", "refs/heads/c"]); + } + + #[tokio::test] + async fn a_stuck_spindle_blocks_only_its_own_events_and_keeps_them_in_order() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + enqueue(&outbox, SPINDLE, "refs/heads/first", START_SECS); + enqueue(&outbox, SPINDLE, "refs/heads/second", START_SECS); + enqueue(&outbox, OTHER, "refs/heads/elsewhere", START_SECS); + let world = spindles(0, Some("spindle.test")); + let t0 = Instant::now(); + + pump(&outbox, &world, t0).await; + assert_eq!( + *world.taken.lock().unwrap(), + [("other.test".to_string(), "refs/heads/elsewhere".to_string())], + "the healthy spindle still gets its event" + ); + assert_eq!( + world.attempts.load(Ordering::SeqCst), + 2, + "the stuck spindle was tried once, on its oldest event only" + ); + let left = queued_refs(&outbox); + assert_eq!(left, ["refs/heads/first", "refs/heads/second"]); + } + + #[test] + fn a_spindle_with_a_delivery_in_flight_doesnt_hold_up_another_one() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + enqueue(&outbox, SPINDLE, "refs/heads/slow-1", START_SECS); + enqueue(&outbox, SPINDLE, "refs/heads/slow-2", START_SECS); + enqueue(&outbox, OTHER, "refs/heads/fast-1", START_SECS); + enqueue(&outbox, OTHER, "refs/heads/fast-2", START_SECS); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + + let first = outbox.claim_due(t0, wall); + assert_eq!(refs(&first), ["refs/heads/slow-1", "refs/heads/fast-1"]); + outbox.settle(&first[1], Ok(None), t0); + + let second = outbox.claim_due(t0, wall); + assert_eq!( + refs(&second), + ["refs/heads/fast-2"], + "the fast spindle moves on while the slow one is still in flight and keeps its order" + ); + } + + #[test] + fn a_repo_the_spindle_keeps_failing_doesnt_hold_up_another_repos_pushes() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + let (stuck, squid) = ("did:plc:stuck", "did:plc:squid"); + enqueue_repo(&outbox, SPINDLE, stuck, &["refs/heads/stuck"], START_SECS); + enqueue_repo(&outbox, SPINDLE, squid, &["refs/heads/a"], START_SECS); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + + let first = outbox.claim_due(t0, wall); + assert_eq!(refs(&first), ["refs/heads/stuck", "refs/heads/a"]); + outbox.settle(&first[0], Err(answered(500)), t0); + outbox.settle(&first[1], Ok(None), t0); + + enqueue_repo(&outbox, SPINDLE, squid, &["refs/heads/b"], START_SECS); + assert_eq!( + refs(&outbox.claim_due(t0, wall)), + ["refs/heads/b"], + "the other repo's next push goes out while the stuck one backs off" + ); + assert_eq!(outbox.next_retry(t0), Some(t0 + Duration::from_secs(1))); + } + + #[test] + fn a_repos_events_wait_on_its_earlier_ones_in_flight_or_backing_off() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + enqueue(&outbox, SPINDLE, "refs/heads/first", START_SECS); + enqueue(&outbox, SPINDLE, "refs/heads/second", START_SECS); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + + let first = outbox.claim_due(t0, wall); + assert_eq!(refs(&first), ["refs/heads/first"]); + assert!(outbox.claim_due(t0, wall).is_empty(), "in flight"); + outbox.settle(&first[0], Err(Failure::unreachable("down")), t0); + assert!(outbox.claim_due(t0, wall).is_empty(), "backing off"); + + let retry = outbox.claim_due(t0 + Duration::from_secs(1), wall); + assert_eq!(refs(&retry), ["refs/heads/first"]); + outbox.settle(&retry[0], Ok(None), t0); + assert_eq!(refs(&outbox.claim_due(t0, wall)), ["refs/heads/second"]); + } + + #[test] + fn an_outbox_file_from_before_per_repo_ordering_still_loads() { + let dir = tempfile::tempdir().unwrap(); + let old_entry = |id: u64, repo: &str, ref_name: &str| { + serde_json::json!({ + "id": id, + "spindle": SPINDLE, + "repo": repo, + "ref_name": ref_name, + "body": { "event": { "ref": ref_name } }, + "queued_at": START_SECS, + }) + }; + let old = serde_json::json!([ + old_entry(6, "did:plc:stuck", "refs/heads/stuck"), + old_entry(7, "did:plc:squid", "refs/heads/main"), + ]); + std::fs::write(dir.path().join(FILE), old.to_string()).unwrap(); + let outbox = outbox(dir.path()); + let due = outbox.claim_due(Instant::now(), UnixSeconds::new(START_SECS)); + assert_eq!( + refs(&due), + ["refs/heads/stuck", "refs/heads/main"], + "a stuck head from before the restart doesn't hold up the other repo" + ); + assert_eq!(outbox.inner.lock().unwrap().next_id, 8); + } + + #[test] + fn only_an_answer_from_the_spindle_itself_is_about_the_event() { + let kind = |error: AtprotoError| Failure::from(error).kind; + for (status, expected) in [ + (500, Kind::Refused), + (501, Kind::Refused), + (502, Kind::Unreachable), + (503, Kind::Unreachable), + (504, Kind::Unreachable), + (408, Kind::Unreachable), + (429, Kind::Unreachable), + (400, Kind::Rejected), + (404, Kind::Rejected), + ] { + assert_eq!(answered(status).kind, expected, "status {status}"); + } + let document_gone = knot_atproto::ResolveError::Status { + status: HttpStatus::new(404), + }; + assert_eq!(kind(document_gone.into()), Kind::Unreachable); + let timeout = NetworkError::Timeout("slow".to_string()); + assert_eq!(kind(timeout.into()), Kind::Unreachable); + } + + #[test] + fn an_unreachable_spindle_holds_all_its_repos_and_gives_up_on_none() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + give_up_after: 2, + ..limits() + }, + ); + enqueue_repo(&outbox, SPINDLE, "did:plc:a", &["refs/heads/a"], START_SECS); + enqueue(&outbox, OTHER, "refs/heads/elsewhere", START_SECS); + let (mut now, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + + let first = outbox.claim_due(now, wall); + outbox.settle(&first[0], Err(Failure::unreachable("down")), now); + outbox.settle(&first[1], Ok(None), now); + enqueue_repo(&outbox, SPINDLE, "did:plc:b", &["refs/heads/b"], START_SECS); + assert!( + outbox.claim_due(now, wall).is_empty(), + "b waits on the spindle too" + ); + for _ in 0..10 { + now = outbox.next_retry(now).unwrap(); + for entry in outbox.claim_due(now, wall) { + outbox.settle(&entry, Err(Failure::unreachable("down")), now); + } + } + assert_eq!(queued_refs(&outbox), ["refs/heads/a", "refs/heads/b"]); + } + + #[test] + fn a_spindle_refusing_everything_holds_each_repo_back_alone_and_gives_up_on_none() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + give_up_after: 2, + ..limits() + }, + ); + enqueue_repo(&outbox, SPINDLE, "did:plc:a", &["refs/heads/a"], START_SECS); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + let a = outbox.claim_due(t0, wall); + outbox.settle(&a[0], Err(answered(500)), t0); + + enqueue_repo(&outbox, SPINDLE, "did:plc:b", &["refs/heads/b"], START_SECS); + let b = outbox.claim_due(t0, wall); + assert_eq!(refs(&b), ["refs/heads/b"], "a's refusal doesn't hold up b"); + outbox.settle(&b[0], Err(answered(500)), t0); + + let mut now = t0; + for _ in 0..10 { + now = outbox.next_retry(now).unwrap(); + for entry in outbox.claim_due(now, wall) { + outbox.settle(&entry, Err(answered(500)), now); + } + } + assert_eq!( + queued_refs(&outbox), + ["refs/heads/a", "refs/heads/b"], + "nothing went through, so it may be the spindle and not the repos" + ); + } + + #[test] + fn a_push_event_the_spindle_keeps_refusing_while_taking_others_is_given_up() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + give_up_after: 3, + ..limits() + }, + ); + let stuck = "did:plc:stuck"; + enqueue_repo( + &outbox, + SPINDLE, + stuck, + &["refs/heads/1", "refs/heads/2"], + START_SECS, + ); + enqueue_repo( + &outbox, + SPINDLE, + "did:plc:squid", + &["refs/heads/a"], + START_SECS, + ); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + + let first = outbox.claim_due(t0, wall); + outbox.settle(&first[0], Err(answered(500)), t0); + outbox.settle(&first[1], Ok(None), t0); + let mut now = t0; + for _ in 0..2 { + now = outbox.next_retry(now).unwrap(); + let retry = outbox.claim_due(now, wall); + assert_eq!(refs(&retry), ["refs/heads/1"]); + outbox.settle(&retry[0], Err(answered(500)), now); + } + assert_eq!(queued_refs(&outbox), ["refs/heads/2"]); + let next = outbox.claim_due(now, wall); + assert_eq!( + refs(&next), + ["refs/heads/2"], + "the repo's next event starts over" + ); + outbox.settle(&next[0], Err(answered(500)), now); + assert_eq!(queued_refs(&outbox), ["refs/heads/2"]); + } + + #[test] + fn a_repo_the_spindle_times_out_on_doesnt_pause_its_other_repos() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + give_up_after: 3, + ..limits() + }, + ); + enqueue_repo( + &outbox, + SPINDLE, + "did:plc:slow", + &["refs/heads/slow"], + START_SECS, + ); + enqueue(&outbox, SPINDLE, "refs/heads/a", START_SECS); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + let secs = Duration::from_secs; + + let first = outbox.claim_due(t0, wall); + outbox.settle(&first[1], Ok(None), t0 + secs(1)); + outbox.settle(&first[0], Err(answered(504)), t0 + secs(2)); + enqueue(&outbox, SPINDLE, "refs/heads/b", START_SECS); + let next = outbox.claim_due(t0 + secs(2), wall); + assert_eq!(refs(&next), ["refs/heads/b"], "the spindle isn't paused"); + outbox.settle(&next[0], Ok(None), t0 + secs(2)); + + let mut now = t0 + secs(2); + for _ in 0..2 { + now = outbox.next_retry(now).unwrap(); + let retry = outbox.claim_due(now, wall); + assert_eq!(refs(&retry), ["refs/heads/slow"]); + outbox.settle(&retry[0], Err(Failure::unreachable("timed out")), now); + } + assert!( + outbox.queued().is_empty(), + "the slow repo's event is given up" + ); + } + + #[test] + fn a_repo_isnt_charged_for_the_time_its_spindle_was_down() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + give_up_after: 2, + ..limits() + }, + ); + enqueue_repo(&outbox, SPINDLE, "did:plc:a", &["refs/heads/a"], START_SECS); + enqueue_repo(&outbox, SPINDLE, "did:plc:b", &["refs/heads/b"], START_SECS); + let (mut now, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + for _ in 0..5 { + for entry in outbox.claim_due(now, wall) { + outbox.settle(&entry, Err(Failure::unreachable("down")), now); + } + now = outbox.next_retry(now).unwrap(); + } + let back = outbox.claim_due(now, wall); + assert_eq!(refs(&back), ["refs/heads/a", "refs/heads/b"]); + outbox.settle(&back[1], Ok(None), now); + outbox.settle(&back[0], Err(answered(500)), now); + assert_eq!(queued_refs(&outbox), ["refs/heads/a"]); + } + + #[test] + fn a_repos_next_event_doesnt_inherit_a_dropped_ones_failures() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + give_up_after: 3, + ..limits() + }, + ); + let stuck = "did:plc:stuck"; + enqueue_repo( + &outbox, + SPINDLE, + stuck, + &["refs/heads/1", "refs/heads/2"], + START_SECS, + ); + enqueue(&outbox, SPINDLE, "refs/heads/a", START_SECS); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + + let first = outbox.claim_due(t0, wall); + outbox.settle(&first[0], Err(answered(500)), t0); + outbox.settle(&first[1], Ok(None), t0); + let now = outbox.next_retry(t0).unwrap(); + let retry = outbox.claim_due(now, wall); + outbox.settle(&retry[0], Err(answered(500)), now); + let now = outbox.next_retry(now).unwrap(); + let rejected = outbox.claim_due(now, wall); + outbox.settle(&rejected[0], Err(answered(404)), now); + + let next = outbox.claim_due(now, wall); + assert_eq!(refs(&next), ["refs/heads/2"], "no wait left over from 1"); + outbox.settle(&next[0], Err(answered(500)), now); + assert_eq!(queued_refs(&outbox), ["refs/heads/2"], "nor its attempts"); + } + + #[test] + fn a_full_repo_evicts_its_oldest_event_that_isnt_in_flight() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + max_per_repo: 2, + ..limits() + }, + ); + enqueue_all( + &outbox, + SPINDLE, + &["refs/heads/1", "refs/heads/2"], + START_SECS, + ); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + let first = outbox.claim_due(t0, wall); + enqueue(&outbox, SPINDLE, "refs/heads/3", START_SECS); + assert_eq!(queued_refs(&outbox), ["refs/heads/1", "refs/heads/3"]); + outbox.settle(&first[0], Ok(None), t0); + assert_eq!(refs(&outbox.claim_due(t0, wall)), ["refs/heads/3"]); + } + + #[test] + fn a_claimed_event_outlives_its_age_until_it_settles() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + enqueue_all( + &outbox, + SPINDLE, + &["refs/heads/1", "refs/heads/2"], + START_SECS, + ); + let t0 = Instant::now(); + outbox.claim_due(t0, UnixSeconds::new(START_SECS)); + assert!( + outbox + .claim_due(t0, UnixSeconds::new(START_SECS + 1_000)) + .is_empty() + ); + assert_eq!(queued_refs(&outbox), ["refs/heads/1"]); + } + + #[test] + fn an_outbox_over_lowered_limits_shrinks_back_under_them() { + let dir = tempfile::tempdir().unwrap(); + let flood = [ + "refs/heads/1", + "refs/heads/2", + "refs/heads/3", + "refs/heads/4", + ]; + enqueue_all(&outbox(dir.path()), SPINDLE, &flood, START_SECS); + let reopened = Outbox::open( + dir.path(), + OutboxLimits { + max_per_repo: 2, + ..limits() + }, + ); + enqueue(&reopened, SPINDLE, "refs/heads/5", START_SECS); + assert_eq!(queued_refs(&reopened), ["refs/heads/4", "refs/heads/5"]); + } + + #[tokio::test(start_paused = true)] + async fn a_dropped_delivery_stops_its_task() { + let held = Arc::new(()); + let task_holds = Arc::clone(&held); + let delivery = guarded( + async move { + let _held = task_holds; + std::future::pending().await + }, + Duration::from_secs(5), + ); + let _ = tokio::time::timeout(Duration::from_secs(1), delivery).await; + tokio::task::yield_now().await; + assert_eq!(Arc::strong_count(&held), 1); + } + + #[test] + fn a_spindle_only_gets_so_many_deliveries_at_once() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + max_in_flight_per_spindle: 2, + ..limits() + }, + ); + for repo in ["did:plc:a", "did:plc:b", "did:plc:c"] { + enqueue_repo(&outbox, SPINDLE, repo, &["refs/heads/main"], START_SECS); + } + enqueue(&outbox, OTHER, "refs/heads/elsewhere", START_SECS); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + let first = outbox.claim_due(t0, wall); + let repos: Vec<&str> = first.iter().map(|entry| entry.repo.as_str()).collect(); + assert_eq!(repos, ["did:plc:a", "did:plc:b", "did:plc:squid"]); + outbox.settle(&first[0], Ok(None), t0); + let next = outbox.claim_due(t0, wall); + assert_eq!(next[0].repo.as_str(), "did:plc:c"); + } + + #[tokio::test(start_paused = true)] + async fn a_delivery_that_hangs_settles_as_unreachable() { + let hung = guarded(std::future::pending(), Duration::from_secs(5)).await; + assert_eq!(hung.unwrap_err().kind, Kind::Unreachable); + } + + #[tokio::test] + async fn a_delivery_that_panics_settles_as_refused() { + let died = guarded( + async { panic!("a bug in the delivery") }, + Duration::from_secs(5), + ) + .await; + assert_eq!(died.unwrap_err().kind, Kind::Refused); + } + + #[test] + fn a_retry_already_claimed_doesnt_set_the_next_wakeup() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + enqueue(&outbox, SPINDLE, "refs/heads/main", START_SECS); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + let first = outbox.claim_due(t0, wall); + outbox.settle(&first[0], Err(Failure::unreachable("down")), t0); + let retry_at = t0 + Duration::from_secs(1); + assert_eq!(outbox.next_retry(t0), Some(retry_at)); + + assert_eq!(outbox.claim_due(retry_at, wall).len(), 1); + assert_eq!( + outbox.next_retry(retry_at), + None, + "a past instant would wake run() in a loop until the delivery returns" + ); + } + + #[test] + fn a_pushs_events_cost_one_queue_write() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + enqueue_all( + &outbox, + SPINDLE, + &["refs/heads/a", "refs/heads/b", "refs/heads/c"], + START_SECS, + ); + assert_eq!(outbox.inner.lock().unwrap().writes, 1); + assert_eq!(queued_refs(&outbox).len(), 3); + } + + #[test] + fn a_flooding_spindle_only_evicts_its_own_events() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + max_per_spindle: 3, + ..limits() + }, + ); + enqueue(&outbox, OTHER, "refs/heads/theirs", START_SECS); + let flood = [ + "refs/heads/1", + "refs/heads/2", + "refs/heads/3", + "refs/heads/4", + "refs/heads/5", + ]; + enqueue_all(&outbox, SPINDLE, &flood, START_SECS); + assert_eq!( + queued_refs(&outbox), + [ + "refs/heads/theirs", + "refs/heads/3", + "refs/heads/4", + "refs/heads/5" + ] + ); + } + + #[test] + fn a_flooding_repo_only_evicts_its_own_events() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + max_per_spindle: 4, + max_per_repo: 2, + ..limits() + }, + ); + enqueue_repo( + &outbox, + SPINDLE, + "did:plc:other", + &["refs/heads/theirs"], + START_SECS, + ); + let flood = [ + "refs/heads/1", + "refs/heads/2", + "refs/heads/3", + "refs/heads/4", + ]; + enqueue_all(&outbox, SPINDLE, &flood, START_SECS); + assert_eq!( + queued_refs(&outbox), + ["refs/heads/theirs", "refs/heads/3", "refs/heads/4"] + ); + } + + #[tokio::test] + async fn a_4xx_answer_drops_the_event_but_503_429_and_408_keep_it() { + for (status, kept) in [ + (400, false), + (404, false), + (408, true), + (429, true), + (503, true), + ] { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + enqueue(&outbox, SPINDLE, "refs/heads/main", START_SECS); + let entry = outbox + .claim_due(Instant::now(), UnixSeconds::new(START_SECS)) + .remove(0); + outbox.settle(&entry, Err(answered(status)), Instant::now()); + assert_eq!(outbox.queued().len() == 1, kept, "status {status}"); + } + } + + #[test] + fn backoff_is_forgotten_once_a_spindle_has_nothing_queued() { + let dir = tempfile::tempdir().unwrap(); + let outbox = outbox(dir.path()); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + let backoff_len = || { + let inner = outbox.inner.lock().unwrap(); + inner.backoff.len() + inner.spindles.len() + }; + enqueue(&outbox, SPINDLE, "refs/heads/expires", START_SECS); + let entry = outbox.claim_due(t0, wall).remove(0); + outbox.settle(&entry, Err(Failure::unreachable("down")), t0); + assert_eq!(backoff_len(), 1); + outbox.claim_due(t0, UnixSeconds::new(START_SECS + 1_000)); + assert_eq!(backoff_len(), 0, "expiry took its last entry"); + } + + #[test] + fn backoff_is_forgotten_when_eviction_takes_a_spindles_last_event() { + let dir = tempfile::tempdir().unwrap(); + let outbox = Outbox::open( + dir.path(), + OutboxLimits { + max_entries: 2, + ..limits() + }, + ); + let (t0, wall) = (Instant::now(), UnixSeconds::new(START_SECS)); + enqueue(&outbox, SPINDLE, "refs/heads/old", START_SECS); + let entry = outbox.claim_due(t0, wall).remove(0); + outbox.settle(&entry, Err(Failure::unreachable("down")), t0); + enqueue_all( + &outbox, + OTHER, + &["refs/heads/1", "refs/heads/2"], + START_SECS, + ); + assert!(outbox.inner.lock().unwrap().spindles.is_empty()); + } +} diff --git a/knot2/crates/knot-xrpc/src/plc.rs b/knot2/crates/knot-xrpc/src/plc.rs new file mode 100644 index 0000000000000000000000000000000000000000..70549da376e61e26d7f4973d3cb5b04a036761e4 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/plc.rs @@ -0,0 +1,557 @@ +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use knot_atproto::AtprotoError; +use knot_atproto::RepoTarget; +use knot_git::Repo; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::RefName; +use knot_types::RepoDid; +use knot_types::UnixMicros; + +use crate::XrpcState; +use crate::error::XrpcError; + +const PENDING_REF: &str = "refs/atproto/plc/pending"; +const DELETED_REF: &str = "refs/atproto/deleted"; + +fn pending_ref() -> RefName { + RefName::new(PENDING_REF).expect("the pending queue's ref name parses") +} + +fn deleted_ref() -> RefName { + RefName::new(DELETED_REF).expect("the tombstone list's ref name parses") +} + +pub const STATUS_DELETED: &str = "deleted"; +const PLC_TOMBSTONE_TYPE: &str = "plc_tombstone"; +const SWEEP_INTERVAL: Duration = Duration::from_secs(5); +const SUBMIT_GAP: Duration = Duration::from_millis(250); +const BACKOFF_BASE_MICROS: u64 = 30_000_000; +const BACKOFF_CAP_MICROS: u64 = 60 * 60 * 1_000_000; +pub(crate) const MAX_ATTEMPTS: u32 = 12; + +#[derive(Debug, Serialize, Deserialize)] +pub struct PendingGenesis { + pub did: RepoDid, + pub operation: serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct DeletedAccount { + did: String, + deleted_at_micros: u64, +} + +fn load Deserialize<'de>>(repo: &Repo, name: &RefName) -> Result, XrpcError> { + let Some(oid) = repo.find_ref(name).map_err(XrpcError::from)? else { + return Ok(None); + }; + let object = repo + .git() + .find_object(oid.object_id()) + .map_err(|error| XrpcError::internal(error.to_string()))?; + serde_json::from_slice(&object.detach().data) + .map(Some) + .map_err(|error| XrpcError::internal(error.to_string())) +} + +fn store(repo: &Repo, name: &RefName, value: Option<&T>) -> Result<(), XrpcError> { + let new = value + .map(|value| { + serde_json::to_vec(value) + .map_err(|error| XrpcError::internal(error.to_string())) + .and_then(|bytes| { + repo.git() + .write_blob(&bytes) + .map(|blob| knot_types::Oid::from(blob.detach())) + .map_err(|error| XrpcError::internal(error.to_string())) + }) + }) + .transpose()?; + let update = match (new, repo.find_ref(name).map_err(XrpcError::from)?) { + (Some(new), Some(old)) => knot_git::RefUpdate::Update { + name: name.clone(), + old, + new, + }, + (Some(new), None) => knot_git::RefUpdate::Create { + name: name.clone(), + new, + }, + (None, None) => return Ok(()), + (None, Some(old)) => knot_git::RefUpdate::Delete { + name: name.clone(), + old, + }, + }; + repo.update_ref(&update).map_err(XrpcError::from) +} + +fn edit_list(repo: &Repo, name: &RefName, edit: F) -> Result<(), XrpcError> +where + T: for<'de> Deserialize<'de> + Serialize + Default, + F: FnOnce(&mut T) -> bool, +{ + let mut entries = load::(repo, name)?.unwrap_or_default(); + if edit(&mut entries) { + store(repo, name, Some(&entries)) + } else { + Ok(()) + } +} + +pub fn pending_entries(meta: &Repo) -> Result, XrpcError> { + Ok(load(meta, &pending_ref())?.unwrap_or_default()) +} + +pub fn enqueue_pending_genesis(meta: &Repo, entry: PendingGenesis) -> Result<(), XrpcError> { + edit_list( + meta, + &pending_ref(), + move |entries: &mut Vec| { + entries.retain(|existing| existing.did != entry.did); + entries.push(entry); + true + }, + ) +} + +pub fn drop_pending_genesis(meta: &Repo, did: &RepoDid) -> Result { + edit_list(meta, &pending_ref(), |entries: &mut Vec| { + let before = entries.len(); + entries.retain(|entry| entry.did != *did); + entries.len() != before + }) + .map(|()| true) +} + +pub fn record_deletion( + meta: &Repo, + did: &RepoDid, + deleted_at: UnixMicros, +) -> Result<(), XrpcError> { + edit_list( + meta, + &deleted_ref(), + |accounts: &mut Vec| { + accounts.retain(|account| account.did != did.as_str()); + accounts.push(DeletedAccount { + did: did.as_str().to_owned(), + deleted_at_micros: deleted_at.get(), + }); + true + }, + ) +} + +pub fn deleted_dids(meta: &Repo) -> Result, XrpcError> { + load::>(meta, &deleted_ref())? + .unwrap_or_default() + .into_iter() + .map(|account| RepoDid::new(account.did)) + .collect::>() + .map_err(|_| XrpcError::internal("Deletion tombstone has a did that won't parse")) +} + +pub fn deleted_accounts(meta: &Repo) -> Result, XrpcError> { + load::>(meta, &deleted_ref())? + .unwrap_or_default() + .into_iter() + .map(|account| { + RepoDid::new(account.did).map(|did| (did, UnixMicros::new(account.deleted_at_micros))) + }) + .collect::>() + .map_err(|_| XrpcError::internal("Deletion tombstone has a did that won't parse")) +} + +async fn with_meta( + state: &XrpcState, + query: Q, +) -> Result +where + T: Send + 'static, + Q: FnOnce(&Repo) -> Result + Send + 'static, +{ + let cob_locks = Arc::clone(&state.cob_locks); + let meta_path = state.meta_path.clone(); + crate::run_blocking(move || { + let _guard = cob_locks.meta(); + query(&Repo::open(&meta_path)?) + }) + .await +} + +async fn clear_pending(state: &XrpcState, did: &RepoDid) { + let wanted = did.clone(); + let _ = with_meta(state, move |meta| drop_pending_genesis(meta, &wanted)).await; +} + +pub(crate) async fn is_deleted( + state: &XrpcState, + did: &RepoDid, +) -> Result { + let wanted = did.clone(); + with_meta(state, move |meta| { + deleted_dids(meta).map(|set| set.contains(&wanted)) + }) + .await +} + +#[derive(Clone, Copy)] +pub(crate) struct Waiter { + attempts: u32, + due_at_micros: UnixMicros, +} + +#[derive(Default)] +pub(crate) struct SweepMemory { + pub(crate) settled: HashSet, + pub(crate) waiting: HashMap, +} + +impl SweepMemory { + fn is_due(&self, did: &RepoDid, now_micros: UnixMicros) -> bool { + self.waiting + .get(did) + .is_none_or(|waiter| now_micros >= waiter.due_at_micros) + } + + #[cfg(test)] + pub(crate) fn is_given_up(&self, did: &str) -> bool { + self.waiting.iter().any(|(key, waiter)| { + key.as_str() == did && waiter.due_at_micros == UnixMicros::new(u64::MAX) + }) + } + + fn schedule(&mut self, did: &RepoDid, now_micros: UnixMicros) -> u32 { + let waiter = self.waiting.entry(did.clone()).or_insert(Waiter { + attempts: 0, + due_at_micros: UnixMicros::new(0), + }); + waiter.attempts = waiter.attempts.saturating_add(1); + waiter.due_at_micros = UnixMicros::new( + now_micros.get().saturating_add( + BACKOFF_BASE_MICROS + .saturating_mul(1u64 << waiter.attempts.saturating_sub(1).min(7)) + .min(BACKOFF_CAP_MICROS), + ), + ); + let attempts = waiter.attempts; + if attempts >= MAX_ATTEMPTS { + tracing::error!( + repo = %did, + "plc reconciler gave up on this repository, after {MAX_ATTEMPTS} failed attempts" + ); + self.give_up_on(did); + } + attempts + } + fn give_up_on(&mut self, did: &RepoDid) { + self.waiting.insert( + did.clone(), + Waiter { + attempts: MAX_ATTEMPTS + 1, + due_at_micros: UnixMicros::new(u64::MAX), + }, + ); + } + + fn settle(&mut self, did: &RepoDid) { + self.settled.insert(did.clone()); + self.waiting.remove(did); + } + + fn clear(&mut self, did: &RepoDid) { + self.waiting.remove(did); + } +} + +enum Outcome { + Settled, + Submitted, + Retriable(String), + GivenUp(String), +} + +pub async fn run_sweeper( + state: Arc>, + shutdown: tokio_util::sync::CancellationToken, + sweep_width: usize, +) { + let mut tick = tokio::time::interval(SWEEP_INTERVAL); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let _first = tick.tick().await; + let mut memory = SweepMemory::default(); + loop { + tokio::select! { + () = shutdown.cancelled() => return, + _ = tick.tick() => sweep_tick(&state, &mut memory, sweep_width).await, + } + } +} + +pub(crate) async fn sweep_tick( + state: &XrpcState, + memory: &mut SweepMemory, + sweep_width: usize, +) { + let now_micros = state.atproto.now(); + let candidates: Vec = state + .index + .native_repos() + .into_iter() + .filter(RepoDid::is_plc) + .filter(|did| !memory.settled.contains(did)) + .filter(|did| memory.is_due(did, now_micros)) + .collect(); + if candidates.is_empty() { + return; + } + let knot_public = match state.secrets.public_key(&state.knot_did) { + Ok(public) => public, + Err(error) => { + tracing::error!( + %error, + "the knot signer is unavailable, so this sweep ends before the first submission" + ); + return; + } + }; + let target = RepoTarget::new(&knot_public, &state.knot_service_url); + let target = ⌖ + use futures::StreamExt as _; + let mut reconciles = futures::stream::iter(candidates) + .map(|did| async move { + let outcome = reconcile(state, target, &did).await; + (did, outcome) + }) + .buffered(sweep_width); + while let Some((did, outcome)) = reconciles.next().await { + match outcome { + Outcome::Settled => { + memory.settle(&did); + tracing::info!(repo = %did, "plc document matches the desired account shape"); + } + Outcome::Submitted => { + memory.clear(&did); + tracing::info!( + repo = %did, + "plc operation accepted; verification lands on the next pass" + ); + } + Outcome::Retriable(message) => { + if memory.schedule(&did, now_micros) <= 2 { + tracing::warn!(repo = %did, %message, "plc submission will retry"); + } + } + Outcome::GivenUp(message) => { + memory.give_up_on(&did); + tracing::error!(repo = %did, %message, "plc reconciler gave up on this repository"); + } + } + tokio::time::sleep(SUBMIT_GAP).await; + } +} + +async fn reconcile( + state: &XrpcState, + target: &RepoTarget, + did: &RepoDid, +) -> Outcome { + let last_op = match state.atproto.last_plc_operation(did).await { + Ok(Some(op)) => op, + Ok(None) => return submit_genesis(state, did).await, + Err(error) => return Outcome::Retriable(error.to_string()), + }; + if last_op.get("type").and_then(Value::as_str) == Some(PLC_TOMBSTONE_TYPE) { + return Outcome::GivenUp(format!( + "{did} was tombstoned in the directory; its history is closed" + )); + } + match target.satisfied_by(&last_op) { + Ok(true) => { + clear_pending(state, did).await; + Outcome::Settled + } + Ok(false) => { + let signer = match signer_for(state, target.knot_did_key(), &last_op, did) { + Ok(signer) => signer, + Err(outcome) => return outcome, + }; + match knot_atproto::update_operation(&signer, &last_op, target) { + Ok(update) => post(state, did, &update).await, + Err(error) => Outcome::GivenUp(error.to_string()), + } + } + Err(error) => Outcome::GivenUp(error.to_string()), + } +} + +async fn submit_genesis( + state: &XrpcState, + did: &RepoDid, +) -> Outcome { + let wanted = did.clone(); + match with_meta(state, move |meta| { + Ok(pending_entries(meta)? + .into_iter() + .find(|entry| entry.did == wanted)) + }) + .await + { + Err(error) => Outcome::Retriable(format!("couldn't read the pending queue: {error}")), + Ok(None) => { + Outcome::GivenUp("directory has no operations and genesis isn't queued".to_owned()) + } + Ok(Some(entry)) => post(state, did, &entry.operation).await, + } +} + +async fn post( + state: &XrpcState, + did: &RepoDid, + operation: &Value, +) -> Outcome { + match state.atproto.submit_plc_operation(did, operation).await { + Ok(()) => Outcome::Submitted, + Err(AtprotoError::PlcSubmit { status, .. }) if !status.is_transient() => { + Outcome::GivenUp(format!( + "the directory rejected this operation with HTTP {status}; \ + resubmitting identical bytes can't converge" + )) + } + Err(error) => Outcome::Retriable(error.to_string()), + } +} + +fn signer_for( + state: &XrpcState, + knot_did_key: &str, + last_op: &Value, + did: &RepoDid, +) -> Result { + let rotation_names_knot = last_op + .get("rotationKeys") + .and_then(Value::as_array) + .is_some_and(|keys| keys.iter().any(|key| key.as_str() == Some(knot_did_key))); + if rotation_names_knot { + return state + .secrets + .signer(&state.knot_did) + .map_err(|_| Outcome::GivenUp("Sealed knot signer is missing".to_owned())); + } + knot_types::KnotId::new(did.as_str()) + .ok() + .and_then(|id| { + state + .secrets + .signer(knot_secrets::SealedKeyId::from(&id)) + .ok() + }) + .ok_or_else(|| { + Outcome::GivenUp(format!( + "{did} rotates on its migration per-repo key and none is sealed; \ + point [atproto].repo_signing_keys at the migration archive" + )) + }) +} + +#[derive(serde::Deserialize)] +struct ArchivedKey { + repo_did: String, + key_type: String, + secret_key_hex: String, +} + +pub fn ingest_legacy_keys( + secrets: &knot_secrets::SealedStore, + archive: &Path, +) -> Result { + let bytes = std::fs::read(archive) + .map_err(|error| XrpcError::internal(format!("reading {}: {error}", archive.display())))?; + let entries: Vec = + serde_json::from_slice(&bytes).map_err(|error| XrpcError::internal(error.to_string()))?; + let provided = entries + .into_iter() + .filter_map(|entry| match archived_scalar(&entry) { + Ok(pair) => Some(pair), + Err(reason) => { + tracing::warn!( + repo_did = %entry.repo_did, + %reason, + "skipped a row in the migration key archive" + ); + None + } + }); + secrets + .ingest(provided) + .map_err(|error| XrpcError::internal(error.to_string())) +} + +fn archived_scalar( + entry: &ArchivedKey, +) -> Result<(knot_secrets::SealedKeyId, [u8; 32]), &'static str> { + if entry.key_type != "k256" { + return Err("unsupported key type"); + } + let scalar = knot_types::decode_hex(&entry.secret_key_hex).ok_or("the secret key isn't hex")?; + let scalar: [u8; 32] = scalar + .try_into() + .map_err(|_| "the secret key isn't a 32-byte scalar")?; + let id = knot_types::KnotId::new(&entry.repo_did) + .map_err(|_| "the repo DID isn't a knot identity")?; + Ok((knot_secrets::SealedKeyId::from(&id), scalar)) +} + +#[cfg(test)] +mod backoff_tests { + use super::*; + + #[test] + fn backoff_doubles_to_3600_seconds_and_the_sweep_gives_up_on_the_twelfth_failure() { + let mut memory = SweepMemory::default(); + let did = RepoDid::new("did:plc:nelpet").unwrap(); + for want_seconds in [30u64, 60, 120, 240, 480, 960, 1920, 3600, 3600, 3600, 3600] { + memory.schedule(&did, UnixMicros::new(0)); + assert_eq!( + memory.waiting[&did].due_at_micros.get() / 1_000_000, + want_seconds + ); + } + memory.schedule(&did, UnixMicros::new(0)); + assert!( + memory.is_given_up(did.as_str()), + "Sweep gives up on repository at twelfth failure" + ); + } + + #[test] + fn the_sweep_gives_up_on_a_repository_after_five_hours_of_failing_at_every_due_time() { + let mut memory = SweepMemory::default(); + let did = RepoDid::new("did:plc:nelpet").unwrap(); + let due_at_last_retry = (1..MAX_ATTEMPTS).fold(UnixMicros::new(0), |now, _| { + memory.schedule(&did, now); + memory.waiting[&did].due_at_micros + }); + assert_eq!( + ( + memory.is_given_up(did.as_str()), + due_at_last_retry.get() / 1_000_000 + ), + (false, 18_210), + "the eleven delays sum to five hours, so the test asserts the arithmetic instead of sleeping" + ); + memory.schedule(&did, due_at_last_retry); + assert!( + memory.is_given_up(did.as_str()), + "Twelfth failure is where sweep gives up on repo" + ); + } +} diff --git a/knot2/crates/knot-xrpc/src/query.rs b/knot2/crates/knot-xrpc/src/query.rs new file mode 100644 index 0000000000000000000000000000000000000000..72393a58e39c3cf3f1ca988e13b3b4237bbe07de --- /dev/null +++ b/knot2/crates/knot-xrpc/src/query.rs @@ -0,0 +1,256 @@ +use axum::extract::{FromRequestParts, Query}; +use http::request::Parts; +use knot_types::{OwnerDid, RepoDid, RepoPath, RepoRkey}; +use serde::de::{self, Deserialize, DeserializeOwned, Deserializer}; + +use crate::error::XrpcError; + +pub(crate) struct ValidatedQuery(pub(crate) T); + +impl FromRequestParts for ValidatedQuery +where + T: DeserializeOwned, + S: Send + Sync, +{ + type Rejection = XrpcError; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + Query::::try_from_uri(&parts.uri) + .map(|query| ValidatedQuery(query.0)) + .map_err(|rejection| XrpcError::invalid_request(rejection.body_text())) + } +} + +// Each route's default and its limit are included in the type, +// such that a limit that is ok on one endpoint +// can't be spent on another one with a lower roof. +#[derive(Clone, Copy)] +pub(crate) struct Limit(usize); + +impl Limit { + pub(crate) fn get(self) -> usize { + self.0 + } + + #[cfg(test)] + pub(crate) const fn literal(value: usize) -> Self { + Self(value) + } +} + +impl Default for Limit { + fn default() -> Self { + Limit(DEFAULT) + } +} + +impl<'de, const DEFAULT: usize, const MAX: usize> Deserialize<'de> for Limit { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + if raw.is_empty() { + return Ok(Limit(DEFAULT)); + } + let value = raw + .parse::() + .map_err(|_| de::Error::custom("limit must be an integer"))?; + Ok(Limit(usize::try_from(value).unwrap_or(0).min(MAX).max(1))) + } +} + +knot_types::scalar_newtype! { + #[derive(Default)] + pub(crate) struct Offset(usize); +} + +impl<'de> Deserialize<'de> for Offset { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + if raw.is_empty() { + return Ok(Offset::new(0)); + } + raw.parse::() + .map(Offset::new) + .map_err(|_| de::Error::custom("cursor must be an integer")) + } +} + +pub(crate) struct Since(jacquard_common::types::tid::Tid); + +impl Since { + pub(crate) fn tid(&self) -> &jacquard_common::types::tid::Tid { + &self.0 + } +} + +impl<'de> Deserialize<'de> for Since { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + jacquard_common::types::tid::Tid::new(&raw) + .map(Self) + .map_err(|_| de::Error::custom("`since` must be a TID, the rev of an export.")) + } +} + +knot_types::scalar_newtype! { + pub(crate) struct Total(usize); +} + +pub(crate) struct Cursor(usize); + +impl Cursor { + pub(crate) fn new(end: usize) -> Self { + Self(end) + } +} + +impl serde::Serialize for Cursor { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0.to_string()) + } +} + +pub(crate) fn next_cursor( + offset: Offset, + limit: Limit, + total: Total, +) -> Option { + offset + .get() + .checked_add(limit.get()) + .filter(|&end| end < total.get()) + .map(Cursor::new) +} +pub(crate) enum RepoArg { + Did(RepoDid), + OwnerRkey { owner: OwnerDid, rkey: RepoRkey }, +} + +impl<'de> Deserialize<'de> for RepoArg { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + if !raw.starts_with("did:") { + return Err(de::Error::custom( + "missing or invalid repo parameter, expected repo DID", + )); + } + Ok(match raw.split_once('/') { + None => RepoArg::Did(RepoDid::new(raw).map_err(de::Error::custom)?), + Some((owner, rkey)) => RepoArg::OwnerRkey { + owner: OwnerDid::new(owner).map_err(de::Error::custom)?, + rkey: RepoRkey::new(rkey).map_err(de::Error::custom)?, + }, + }) + } +} + +const MAX_REVSPEC_BYTES: usize = 4096; + +#[derive(Clone, Default)] +pub(crate) struct Revspec(String); + +impl Revspec { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl<'de> Deserialize<'de> for Revspec { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + match raw.len() <= MAX_REVSPEC_BYTES && !raw.chars().any(char::is_control) { + true => Ok(Self(raw)), + false => Err(de::Error::custom( + "`rev` must be a short plain ref. Please send a TID or a branch name, thanks.", + )), + } + } +} + +#[derive(Default)] +pub(crate) struct BranchArg(Option); + +impl BranchArg { + pub(crate) fn get(&self) -> Option<&knot_types::BranchName> { + self.0.as_ref() + } +} + +impl<'de> serde::Deserialize<'de> for BranchArg { + fn deserialize>(deserializer: D) -> Result { + match String::deserialize(deserializer)? { + raw if raw.is_empty() => Ok(Self(None)), + raw => knot_types::BranchName::new(raw) + .map(|name| Self(Some(name))) + .map_err(de::Error::custom), + } + } +} + +#[derive(Default)] +pub(crate) struct TagArg(Option); + +impl TagArg { + pub(crate) fn get(&self) -> Option<&knot_types::TagName> { + self.0.as_ref() + } +} + +impl<'de> serde::Deserialize<'de> for TagArg { + fn deserialize>(deserializer: D) -> Result { + match String::deserialize(deserializer)? { + raw if raw.is_empty() => Ok(Self(None)), + raw => { + let short = raw.strip_prefix("refs/tags/").unwrap_or(&raw); + knot_types::TagName::new(short) + .map(|name| Self(Some(name))) + .map_err(de::Error::custom) + } + } + } +} + +#[derive(Default)] +pub(crate) enum TreePath { + #[default] + Root, + At(RepoPath), + Outside(String), +} + +impl TreePath { + pub(crate) fn as_str(&self) -> &str { + match self { + TreePath::Root => "", + TreePath::At(path) => path.as_str(), + TreePath::Outside(raw) => raw, + } + } + + pub(crate) fn dir(&self) -> Option> { + match self { + TreePath::Root => Some(None), + TreePath::At(path) => Some(Some(path)), + TreePath::Outside(_) => None, + } + } + + pub(crate) fn file(&self) -> Option<&RepoPath> { + match self { + TreePath::At(path) => Some(path), + TreePath::Root | TreePath::Outside(_) => None, + } + } +} + +impl<'de> Deserialize<'de> for TreePath { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + Ok(match raw.is_empty() { + true => TreePath::Root, + false => match RepoPath::new(&raw) { + Ok(path) => TreePath::At(path), + Err(_) => TreePath::Outside(raw), + }, + }) + } +} diff --git a/knot2/crates/knot-xrpc/src/reads.rs b/knot2/crates/knot-xrpc/src/reads.rs new file mode 100644 index 0000000000000000000000000000000000000000..074355d87df89267b8abbba61854fe403f306530 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/reads.rs @@ -0,0 +1,1823 @@ +use std::collections::BTreeSet; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use axum::body::Body; +use axum::extract::{Query, Request, State}; +use axum::response::{IntoResponse, Response}; +use http::{HeaderMap, HeaderValue, StatusCode, header}; +use serde::de::{self, Deserializer}; +use serde::{Deserialize, Serialize, Serializer}; +use sha2::{Digest, Sha256}; +use tokio::time::timeout; +use tower::ServiceExt; +use tower_http::services::ServeFile; + +use knot_git::{ + ArchiveFormat, BinaryBudget, BinaryDiff, BranchInfo, BranchTip, Commit, EntryKind, FilePatch, + GitError, Identity, Layout, LineOp, LogLimit, LogSkip, PatchBody, PatchStatus, RefClass, Repo, + SizedEntry, TagInfo, +}; +use knot_index::Resolved; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{ + AuthorName, BranchName, Email, Oid, RefName, RepoDid, RepoPath, TagName, UnixSeconds, +}; + +use crate::error::XrpcError; +use crate::out::{fold_subject, rfc3339}; +use crate::query::{ + BranchArg, Limit, Offset, RepoArg, Revspec, TagArg, Total, TreePath, ValidatedQuery, + next_cursor, +}; +use crate::{ResponseLimit, XrpcState, run_blocking}; + +pub(crate) const TREE_ROUTE: &str = "/xrpc/org.tangled.git.getTree"; +pub(crate) const LOG_ROUTE: &str = "/xrpc/org.tangled.git.listCommits"; +pub(crate) const BRANCHES_ROUTE: &str = "/xrpc/org.tangled.git.listBranches"; +pub(crate) const BRANCH_ROUTE: &str = "/xrpc/org.tangled.git.getBranch"; +pub(crate) const TAGS_ROUTE: &str = "/xrpc/org.tangled.git.listTags"; +pub(crate) const TAG_ROUTE: &str = "/xrpc/org.tangled.git.getTag"; +pub(crate) const BLOB_ROUTE: &str = "/xrpc/org.tangled.git.getBlob"; +pub(crate) const ENTRY_ROUTE: &str = "/xrpc/org.tangled.git.getEntry"; +pub(crate) const BLAME_ROUTE: &str = "/xrpc/org.tangled.git.getBlame"; +pub(crate) const DIFF_ROUTE: &str = "/xrpc/org.tangled.git.getDiff"; +pub(crate) const ARCHIVE_ROUTE: &str = "/xrpc/org.tangled.git.getArchive"; +pub(crate) const LANGUAGES_ROUTE: &str = "/xrpc/org.tangled.git.listLanguages"; +pub(crate) const LIST_REFS_ROUTE: &str = "/xrpc/org.tangled.git.listRefs"; + +const DEFAULT_PAGE: usize = 50; +const MAX_PAGE: usize = 100; +const LIST_REFS_DEFAULT: usize = 100; +const LIST_REFS_MAX: usize = 1000; +const MAX_BLOB_BYTES: u64 = 25 * 1024 * 1024; +const MAX_BLAME_OBJECT_BYTES: u64 = 1024 * 1024; +const MAX_BLAME_RUNS: usize = 50_000; +const BLAME_TIMEOUT: Duration = Duration::from_secs(5); +const RAW_CSP: &str = "default-src 'none'; style-src 'unsafe-inline'; sandbox"; + +pub(crate) fn repo_not_found() -> XrpcError { + XrpcError::named( + StatusCode::NOT_FOUND, + XrpcError::REPO_NOT_FOUND, + "repository not found on this knot", + ) +} + +fn ref_not_found() -> XrpcError { + XrpcError::named( + StatusCode::NOT_FOUND, + "RefNotFound", + "git reference not found", + ) +} + +fn blob_too_large() -> XrpcError { + XrpcError::named( + StatusCode::PAYLOAD_TOO_LARGE, + "BlobTooLarge", + "file is too large to serve", + ) +} + +fn blame_runs_too_large(runs: usize) -> bool { + runs > MAX_BLAME_RUNS +} + +fn blame_too_large() -> XrpcError { + XrpcError::named( + StatusCode::PAYLOAD_TOO_LARGE, + "BlameTooLarge", + "blame exceeds resource limits", + ) +} + +fn blame_timeout() -> XrpcError { + XrpcError::named( + StatusCode::SERVICE_UNAVAILABLE, + "BlameTimeout", + "blame exceeded its computation time budget", + ) +} + +fn names_reserved(refspec: &str) -> bool { + match RefClass::of_revspec(refspec) { + RefClass::Cob | RefClass::Checkpoint | RefClass::Atproto | RefClass::Tngl => true, + RefClass::Public | RefClass::Hidden => false, + } +} + +pub(crate) fn warming_for(projection: &str) -> XrpcError { + XrpcError::warming(format!( + "{projection} projection is still warming, please retry shortly" + )) +} + +pub(crate) fn warming() -> XrpcError { + warming_for("registry") +} + +#[derive(Clone)] +pub(crate) struct HostedRepo(RepoDid); + +impl HostedRepo { + fn registered(did: RepoDid) -> Self { + Self(did) + } + + pub(crate) fn into_did(self) -> RepoDid { + self.0 + } +} + +impl std::ops::Deref for HostedRepo { + type Target = RepoDid; + + fn deref(&self) -> &RepoDid { + &self.0 + } +} + +pub(crate) fn require_hosted( + state: &XrpcState, + repo: RepoDid, +) -> Result { + match state.index.owner_of(&repo) { + Resolved::Ready(Some(_)) => Ok(HostedRepo::registered(repo)), + Resolved::Ready(None) => Err(repo_not_found()), + Resolved::Warming => Err(warming()), + } +} + +pub(crate) fn resolve_repo( + state: &XrpcState, + repo: &RepoArg, +) -> Result { + match repo { + RepoArg::Did(did) => require_hosted(state, did.clone()), + RepoArg::OwnerRkey { owner, rkey } => match state.index.resolve_repo(owner, rkey) { + Resolved::Ready(Some(did)) => Ok(HostedRepo::registered(did)), + Resolved::Ready(None) => Err(repo_not_found()), + Resolved::Warming => Err(warming()), + }, + } +} + +pub(crate) fn open(layout: &Layout, did: &RepoDid) -> Result { + layout + .open(did) + .map_err(|error| XrpcError::internal(format!("cannot open repository: {error}"))) +} + +fn commit_for(repo: &Repo, refspec: &Revspec) -> Result { + let refspec = refspec.as_str(); + if names_reserved(refspec) { + return Err(ref_not_found()); + } + let oid = match refspec.is_empty() { + true => repo.head().map(|head| head.target), + false => repo.resolve_revision(refspec), + } + .ok_or_else(ref_not_found)?; + let commit = repo.peel_to_commit(oid).map_err(|_| ref_not_found())?; + match repo.reachable_from_public(commit) { + Ok(true) => Ok(commit), + Ok(false) => Err(ref_not_found()), + Err(error) => Err(error.into()), + } +} + +struct LimitWriter { + buf: Vec, + limit: usize, +} + +impl std::io::Write for LimitWriter { + fn write(&mut self, data: &[u8]) -> std::io::Result { + if self.buf.len() + data.len() > self.limit { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "response exceeds configured maximum size", + )); + } + self.buf.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +pub(crate) fn json(value: impl Serialize, limit: ResponseLimit) -> Result { + let mut writer = LimitWriter { + buf: Vec::new(), + limit: limit.get(), + }; + match serde_json::to_writer(&mut writer, &value) { + Ok(()) => Ok(( + StatusCode::OK, + [(header::CONTENT_TYPE, "application/json")], + writer.buf, + ) + .into_response()), + Err(error) if error.is_io() => Err(XrpcError::request_too_large( + "response exceeds configured maximum size", + )), + Err(error) => Err(XrpcError::internal(format!( + "failed to serialize response: {error}" + ))), + } +} + +#[derive(Serialize)] +struct SignatureOut { + name: AuthorName, + email: Email, + when: String, +} + +impl SignatureOut { + fn of(identity: &Identity) -> Self { + Self { + name: identity.name.clone(), + email: identity.email.clone(), + when: rfc3339(identity.time, identity.offset_seconds), + } + } + + fn zero() -> Self { + Self { + name: AuthorName::new(""), + email: Email::new(""), + when: rfc3339(UnixSeconds::new(0), 0), + } + } +} + +// org.tangled.git.defs#commit +#[derive(Serialize)] +struct CommitOut { + hash: Oid, + author: SignatureOut, + committer: SignatureOut, + message: String, + tree: Oid, +} + +impl CommitOut { + fn of(commit: &Commit) -> Self { + Self { + hash: commit.id, + author: SignatureOut::of(&commit.author), + committer: SignatureOut::of(&commit.committer), + message: commit.message.clone(), + tree: commit.tree, + } + } +} + +fn commit_out_of(repo: &Repo, id: Oid) -> Option { + repo.find_commit(id) + .ok() + .map(|commit| CommitOut::of(&commit)) +} + +#[derive(Deserialize)] +pub(crate) struct TreeParams { + repo: RepoDid, + #[serde(rename = "ref", default)] + refspec: Revspec, + #[serde(default)] + path: TreePath, +} + +#[derive(Serialize)] +struct TreeEntryOut { + name: String, + mode: String, + size: i64, + #[serde(rename = "lastCommit", skip_serializing_if = "Option::is_none")] + last_commit: Option, +} + +#[derive(Serialize)] +struct TreeOut { + entries: Vec, + #[serde(rename = "lastCommit", skip_serializing_if = "Option::is_none")] + last_commit: Option, +} + +pub(crate) async fn repo_tree( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = require_hosted(&state, params.repo)?; + let layout = state.layout.clone(); + let limit = state.byte_limits.response; + let tree_deadline = state.budgets.tree_last_commit.get().deadline(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let commit = commit_for(&repo, ¶ms.refspec)?; + let path_not_found = || { + XrpcError::named( + StatusCode::NOT_FOUND, + "PathNotFound", + "path not found in repository tree", + ) + }; + let dir = params.path.dir().ok_or_else(path_not_found)?; + let entries = repo + .tree_entries_at(commit, dir)? + .ok_or_else(path_not_found)?; + let names: Vec = entries.iter().map(|entry| entry.name.clone()).collect(); + let attributed = repo + .last_commits(commit, dir, &names, tree_deadline) + .unwrap_or_default(); + let files: Vec = entries + .iter() + .map(|entry| TreeEntryOut { + name: entry.name.clone(), + mode: entry.kind.mode_octal().to_string(), + size: entry.size as i64, + last_commit: attributed + .get(&entry.name) + .and_then(|last| commit_out_of(&repo, last.id)), + }) + .collect(); + let newest = attributed.values().max_by_key(|last| (last.time, last.id)); + let last_commit = newest.and_then(|last| commit_out_of(&repo, last.id)); + json( + TreeOut { + entries: files, + last_commit, + }, + limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct LogParams { + repo: RepoDid, + #[serde(rename = "ref", default)] + refspec: Revspec, + #[serde(default)] + path: TreePath, + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: CommitCursor, +} + +// the listCommits cursor is the commit id of the last commit of the +// previous page, so pages resume right after it instead of counting +// offsets that drift while history moves +#[derive(Default)] +struct CommitCursor(Option); + +impl<'de> Deserialize<'de> for CommitCursor { + fn deserialize>(deserializer: D) -> Result { + match String::deserialize(deserializer)? { + raw if raw.is_empty() => Ok(Self(None)), + raw => Oid::from_hex(&raw) + .map(Some) + .map(Self) + .map_err(|_| de::Error::custom("cursor must be a commit id of this repository")), + } + } +} + +#[derive(Serialize)] +struct LogOut { + commits: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, +} + +// does this commit change anything under `path` (a file or a directory)? +// merge commits compare against their first parent only, which matches +// how single-commit patches get rendered +fn commit_touches(repo: &Repo, commit: &Commit, path: &TreePath) -> Result { + let path = match path { + TreePath::Root => return Ok(true), + TreePath::Outside(_) => return Ok(false), + TreePath::At(path) => path, + }; + let patches = repo.commit_patches( + knot_git::PatchRange { + base: commit.parents.first().copied(), + head: commit.id, + }, + &mut BinaryBudget::Omit, + )?; + Ok(patches.iter().any(|patch| { + patch.path.as_str() == path.as_str() + || patch + .path + .as_str() + .strip_prefix(path.as_str()) + .is_some_and(|rest| rest.starts_with('/')) + })) +} + +pub(crate) async fn repo_log( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + if params.refspec.as_str().is_empty() { + return Err(XrpcError::invalid_request("missing ref parameter")); + } + let did = require_hosted(&state, params.repo)?; + let limit = params.limit.get(); + let layout = state.layout.clone(); + let response_limit = state.byte_limits.response; + run_blocking(move || { + let repo = open(&layout, &did)?; + let start = commit_for(&repo, ¶ms.refspec)?; + // one candidate beyond the page tells whether a cursor is due + let candidates: Vec = match params.cursor.0 { + None => repo.log_window_bounded( + start, + LogSkip::new(0), + LogLimit::new(limit.saturating_add(1)), + )?, + Some(cursor) => { + let resume = repo.find_commit(cursor).map_err(|_| { + XrpcError::invalid_request("cursor must be a commit id of this repository") + })?; + repo.log_past_bounded(start, resume.id, LogLimit::new(limit.saturating_add(1)))? + .ok_or_else(|| { + XrpcError::invalid_request( + "cursor must sit in the ref's history, not beside it", + ) + })? + } + }; + let mut commits: Vec = Vec::new(); + for commit in candidates.iter().take(limit) { + if commit_touches(&repo, commit, ¶ms.path)? { + commits.push(CommitOut::of(commit)); + } + } + let cursor = (candidates.len() > limit) + .then(|| { + commits + .last() + .map(|last| last.hash) + .or_else(|| candidates.get(limit - 1).map(|commit| commit.id)) + }) + .flatten(); + json(LogOut { commits, cursor }, response_limit) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct BranchesParams { + repo: RepoDid, + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: Offset, +} + +#[derive(Serialize)] +struct BranchInfoOut { + name: BranchName, + oid: Oid, + message: String, + author: SignatureOut, + committer: SignatureOut, +} + +impl BranchInfoOut { + fn of(branch: &BranchInfo) -> Self { + match &branch.tip { + BranchTip::Commit(commit) => Self { + name: branch.name.clone(), + oid: commit.id, + message: commit.message.clone(), + author: SignatureOut::of(&commit.author), + committer: SignatureOut::of(&commit.committer), + }, + // a ref pointing past a commit has no identities to cite, + // and listing it at all is already a courtesy + BranchTip::Opaque { id, message, .. } => Self { + name: branch.name.clone(), + oid: *id, + message: message.trim_end().to_owned(), + author: SignatureOut::zero(), + committer: SignatureOut::zero(), + }, + } + } +} + +#[derive(Serialize)] +struct BranchesOut { + branches: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, + total: usize, + #[serde(rename = "defaultBranch", skip_serializing_if = "Option::is_none")] + default_branch: Option, +} + +pub(crate) async fn repo_branches( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = require_hosted(&state, params.repo)?; + let offset = params.cursor; + let limit = params.limit; + let layout = state.layout.clone(); + let response_limit = state.byte_limits.response; + run_blocking(move || { + let repo = open(&layout, &did)?; + let mut branches = repo.branch_list()?; + branches.sort_by(|a, b| { + b.tip + .created_at() + .cmp(&a.tip.created_at()) + .then_with(|| a.name.cmp(&b.name)) + }); + let total = branches.len(); + let default = repo.default_branch().and_then(|name| name.branch_name()); + let window: Vec = branches + .iter() + .skip(offset.get()) + .take(limit.get()) + .map(BranchInfoOut::of) + .collect(); + json( + BranchesOut { + branches: window, + cursor: next_cursor(offset, limit, Total::new(total)), + total, + default_branch: default, + }, + response_limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct BranchParams { + repo: RepoDid, + #[serde(default)] + name: BranchArg, +} + +#[derive(Serialize)] +struct BranchOut { + name: BranchName, + hash: Oid, + message: String, + author: SignatureOut, + #[serde(rename = "totalCommits")] + total_commits: usize, +} + +pub(crate) async fn repo_branch( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = require_hosted(&state, params.repo)?; + let Some(name) = params.name.get().cloned() else { + return Err(XrpcError::invalid_request("missing name parameter")); + }; + let layout = state.layout.clone(); + let limit = state.byte_limits.response; + run_blocking(move || { + let repo = open(&layout, &did)?; + let branch_not_found = + || XrpcError::named(StatusCode::NOT_FOUND, "BranchNotFound", "branch not found"); + let target = repo + .find_ref(&name.head_ref()) + .ok() + .flatten() + .ok_or_else(branch_not_found)?; + let commit = repo.find_commit(target).map_err(|_| branch_not_found())?; + // a zero-width window still walks the graph once and returns + // the full count, without decoding a single commit + let (_, total_commits) = repo.log_window(target, LogSkip::new(0), LogLimit::new(0))?; + json( + BranchOut { + name, + hash: target, + message: commit.message.clone(), + author: SignatureOut::of(&commit.author), + total_commits, + }, + limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct TagsParams { + repo: RepoDid, + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: Offset, +} + +// org.tangled.git.listTags#annotation +#[derive(Serialize)] +struct AnnotationOut { + #[serde(skip_serializing_if = "Option::is_none")] + tagger: Option, + message: String, +} + +#[derive(Serialize)] +struct TagInfoOut { + name: TagName, + oid: Oid, + message: String, + author: SignatureOut, + #[serde(skip_serializing_if = "Option::is_none")] + annotation: Option, +} + +impl TagInfoOut { + fn of(repo: &Repo, info: &TagInfo) -> Result { + let annotation = info.annotated.as_ref().map(|annotated| AnnotationOut { + tagger: annotated.tagger.as_ref().map(SignatureOut::of), + message: info.message.trim_end().to_owned(), + }); + // the lexicon's oid and message are the peeled commit's; only + // a tag pointing somewhere stranger falls back to the ref itself + let target = match &info.annotated { + Some(annotated) => annotated.target, + None => info.id, + }; + let (oid, message, author) = match repo + .peel_to_commit(target) + .ok() + .and_then(|oid| repo.find_commit(oid).ok()) + { + Some(commit) => (commit.id, commit.message, SignatureOut::of(&commit.author)), + None => (info.id, info.message.clone(), SignatureOut::zero()), + }; + Ok(Self { + name: info.name.clone(), + oid, + message, + author, + annotation, + }) + } +} + +#[derive(Serialize)] +struct TagsOut { + tags: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, + total: usize, +} + +pub(crate) async fn repo_tags( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = require_hosted(&state, params.repo)?; + let offset = params.cursor; + let limit = params.limit; + let layout = state.layout.clone(); + let response_limit = state.byte_limits.response; + run_blocking(move || { + let repo = open(&layout, &did)?; + let mut tags = repo.tag_list()?; + tags.sort_by(|a, b| { + b.created_at + .cmp(&a.created_at) + .then_with(|| a.name.cmp(&b.name)) + }); + let total = tags.len(); + let window: Vec = tags + .iter() + .skip(offset.get()) + .take(limit.get()) + .map(|info| TagInfoOut::of(&repo, info)) + .collect::, _>>()?; + json( + TagsOut { + tags: window, + cursor: next_cursor(offset, limit, Total::new(total)), + total, + }, + response_limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct TagParams { + repo: RepoDid, + #[serde(default)] + tag: TagArg, +} + +pub(crate) async fn repo_tag( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = require_hosted(&state, params.repo)?; + let Some(name) = params.tag.get().cloned() else { + return Err(XrpcError::invalid_request("missing tag parameter")); + }; + let layout = state.layout.clone(); + let limit = state.byte_limits.response; + run_blocking(move || { + let repo = open(&layout, &did)?; + let info = repo + .tag_list()? + .into_iter() + .find(|tag| tag.name == name) + .ok_or_else(ref_not_found)?; + json(TagInfoOut::of(&repo, &info)?, limit) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct BlobParams { + repo: RepoDid, + #[serde(rename = "ref", default)] + refspec: Revspec, + #[serde(default)] + path: TreePath, +} + +pub(crate) fn etag_matches(headers: &HeaderMap, etag: &str) -> bool { + headers + .get_all(header::IF_NONE_MATCH) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .map(str::trim) + .any(|candidate| { + candidate == "*" || candidate.strip_prefix("W/").unwrap_or(candidate) == etag + }) +} + +fn quoted_etag(digest: &[u8]) -> String { + format!("\"{}\"", knot_types::lowercase_hex(digest)) +} + +// the lexicon pins this query to raw bytes, so every blob rides +// application/octet-stream behind nosniff and a sandbox, whatever +// the content sniffs as +fn serve_blob(headers: &HeaderMap, contents: Vec) -> Response { + let etag = quoted_etag(&Sha256::digest(&contents)); + if etag_matches(headers, &etag) { + return StatusCode::NOT_MODIFIED.into_response(); + } + ( + StatusCode::OK, + [ + (header::ETAG, etag), + (header::CONTENT_TYPE, "application/octet-stream".to_string()), + (header::CACHE_CONTROL, "no-cache".to_string()), + (header::X_CONTENT_TYPE_OPTIONS, "nosniff".to_string()), + (header::CONTENT_SECURITY_POLICY, RAW_CSP.to_string()), + ], + contents, + ) + .into_response() +} + +pub(crate) async fn repo_blob( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, + headers: HeaderMap, +) -> Result { + let did = require_hosted(&state, params.repo)?; + if params.path.as_str().is_empty() { + return Err(XrpcError::invalid_request("missing path parameter")); + } + let layout = state.layout.clone(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let commit = commit_for(&repo, ¶ms.refspec)?; + let file_not_found = || { + XrpcError::named( + StatusCode::NOT_FOUND, + "FileNotFound", + "file not found at specified path", + ) + }; + let file_path = params.path.file().ok_or_else(file_not_found)?; + let entry = repo + .entry_at(commit, file_path)? + .filter(|entry| { + matches!( + entry.kind, + EntryKind::Blob | EntryKind::BlobExecutable | EntryKind::Link + ) + }) + .ok_or_else(file_not_found)?; + if repo.blob_size(entry.oid).map_err(|_| file_not_found())? > MAX_BLOB_BYTES { + return Err(blob_too_large()); + } + let contents = repo.read_blob(entry.oid).map_err(|_| file_not_found())?; + Ok(serve_blob(&headers, contents)) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct EntryParams { + repo: RepoDid, + #[serde(rename = "ref", default)] + refspec: Revspec, + #[serde(default)] + path: TreePath, +} + +// org.tangled.git.defs#submodule +#[derive(Serialize)] +struct SubmoduleOut { + name: String, + url: String, + #[serde(skip_serializing_if = "Option::is_none")] + branch: Option, +} + +#[derive(Serialize)] +struct EntryOut { + name: String, + mode: String, + oid: Oid, + size: i64, + #[serde(rename = "lastCommit", skip_serializing_if = "Option::is_none")] + last_commit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + submodule: Option, +} + +fn entry_not_found() -> XrpcError { + XrpcError::named( + StatusCode::NOT_FOUND, + "EntryNotFound", + "entry not found at specified path", + ) +} + +pub(crate) async fn repo_entry( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = require_hosted(&state, params.repo)?; + if params.path.as_str().is_empty() { + return Err(XrpcError::invalid_request("missing path parameter")); + } + let layout = state.layout.clone(); + let limit = state.byte_limits.response; + let entry_deadline = state.budgets.blob_last_commit.get().deadline(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let commit = commit_for(&repo, ¶ms.refspec)?; + let file_path = params.path.file().ok_or_else(entry_not_found)?; + let dir = file_path.parent(); + let name = file_path.file_name().to_string(); + let submodule = repo + .submodules(commit) + .unwrap_or_default() + .into_iter() + .find(|submodule| submodule.path.as_str() == file_path.as_str()); + let (entry, submodule_out) = match submodule { + Some(submodule) => { + let entry = repo + .entry_at(commit, file_path)? + .filter(|entry| entry.kind == EntryKind::Commit) + .ok_or_else(entry_not_found)?; + ( + SizedEntry { + name: name.clone(), + oid: entry.oid, + kind: EntryKind::Commit, + size: 0, + }, + Some(SubmoduleOut { + name: submodule.name, + url: submodule.url.to_string(), + branch: submodule.branch, + }), + ) + } + None => { + let entries = repo + .tree_entries_at(commit, dir.as_ref())? + .ok_or_else(entry_not_found)?; + let entry = entries + .into_iter() + .find(|entry| entry.name == name) + .ok_or_else(entry_not_found)?; + (entry, None) + } + }; + let last_commit = repo + .last_commits( + commit, + dir.as_ref(), + std::slice::from_ref(&name), + entry_deadline, + ) + .ok() + .and_then(|attributed| attributed.get(&name).cloned()) + .and_then(|last| commit_out_of(&repo, last.id)); + json( + EntryOut { + name: entry.name.clone(), + mode: entry.kind.mode_octal().to_string(), + oid: entry.oid, + size: entry.size as i64, + last_commit, + submodule: submodule_out, + }, + limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct BlameParams { + repo: RepoDid, + #[serde(rename = "ref", default)] + refspec: Revspec, + #[serde(default)] + path: TreePath, +} + +#[derive(Serialize)] +struct BlameAuthorOut { + name: AuthorName, + email: Email, +} + +#[derive(Serialize)] +struct BlameCommitOut { + sha: Oid, + subject: String, + #[serde(rename = "authoredAt")] + authored_at: String, + author: BlameAuthorOut, +} + +#[derive(Serialize)] +struct BlameRunOut { + start: u32, + count: u32, + commit: Oid, +} + +#[derive(Serialize)] +struct BlameOut { + commit: Oid, + commits: Vec, + runs: Vec, +} + +pub(crate) async fn repo_blame( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + if params.refspec.as_str().is_empty() { + return Err(XrpcError::invalid_request("missing ref parameter")); + } + let did = require_hosted(&state, params.repo)?; + if params.path.as_str().is_empty() { + return Err(XrpcError::invalid_request("missing path parameter")); + } + let layout = state.layout.clone(); + let limit = state.byte_limits.response; + let blame_slots = state.slots.blame.clone(); + let deadline = Instant::now() + BLAME_TIMEOUT; + let task = async move { + let permit = blame_slots.acquire().await; + run_blocking(move || { + let _permit = permit; + let file_not_found = || { + XrpcError::named( + StatusCode::NOT_FOUND, + "FileNotFound", + "file not found at specified path", + ) + }; + let repo = open(&layout, &did)?; + let target = commit_for(&repo, ¶ms.refspec)?; + let path = params.path.file().ok_or_else(file_not_found)?; + let entry = repo + .entry_at(target, path)? + .filter(|entry| { + matches!( + entry.kind, + EntryKind::Blob | EntryKind::BlobExecutable | EntryKind::Link + ) + }) + .ok_or_else(file_not_found)?; + if repo.blob_size(entry.oid).map_err(|_| file_not_found())? > MAX_BLAME_OBJECT_BYTES { + return Err(blame_too_large()); + } + + let hunks = + match repo.blame_file_bounded(target, path, MAX_BLAME_OBJECT_BYTES, deadline) { + Ok(hunks) => hunks, + Err(GitError::BlameObjectTooLarge { .. }) => return Err(blame_too_large()), + Err(GitError::BlameTimedOut) => return Err(blame_timeout()), + Err(error) => return Err(error.into()), + }; + if blame_runs_too_large(hunks.len()) { + return Err(blame_too_large()); + } + let commits = hunks + .iter() + .map(|hunk| hunk.commit) + .collect::>() + .into_iter() + .map(|id| -> Result { + if Instant::now() >= deadline { + return Err(blame_timeout()); + } + let commit = repo.find_commit(id)?; + Ok(BlameCommitOut { + sha: commit.id, + subject: fold_subject(&commit.message), + authored_at: rfc3339(commit.author.time, commit.author.offset_seconds), + author: BlameAuthorOut { + name: commit.author.name, + email: commit.author.email, + }, + }) + }) + .collect::, _>>()?; + let runs = hunks + .into_iter() + .map(|hunk| BlameRunOut { + start: hunk.start, + count: hunk.lines, + commit: hunk.commit, + }) + .collect(); + json( + BlameOut { + commit: target, + commits, + runs, + }, + limit, + ) + }) + .await + }; + timeout(BLAME_TIMEOUT, task) + .await + .map_err(|_| blame_timeout())? +} + +#[derive(Deserialize)] +pub(crate) struct DiffParams { + repo: RepoDid, + #[serde(default)] + base: Revspec, + #[serde(default)] + head: Revspec, + #[serde(default)] + paths: PathsArg, +} + +// the query extractor's urlencoded deserializer can't surface repeated +// keys as a sequence, so the lexicon's `paths` array rides one +// comma-separated parameter +#[derive(Default)] +struct PathsArg(Vec); + +impl<'de> Deserialize<'de> for PathsArg { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + Ok(Self( + raw.split(',') + .map(str::trim) + .filter(|path| !path.is_empty()) + .map(str::to_string) + .collect(), + )) + } +} + +// org.tangled.git.defs#diffSrc +#[derive(Serialize)] +struct DiffSrcOut { + #[serde(serialize_with = "serialize_diff_path")] + path: Option, + oid: Oid, + size: i64, + #[serde(rename = "isBinary")] + is_binary: bool, + #[serde(rename = "isSubmodule")] + is_submodule: bool, +} + +fn serialize_diff_path( + path: &Option, + serializer: S, +) -> Result { + serializer.serialize_str(path.as_ref().map(RepoPath::as_str).unwrap_or("")) +} + +fn diff_src( + repo: &Repo, + path: Option<&RepoPath>, + oid: Oid, + kind: Option, + is_binary: bool, +) -> DiffSrcOut { + DiffSrcOut { + path: path.cloned(), + oid, + size: kind + .filter(|kind| { + matches!( + kind, + EntryKind::Blob | EntryKind::BlobExecutable | EntryKind::Link + ) + }) + .and_then(|_| repo.blob_size(oid).ok()) + .map(|size| size as i64) + .unwrap_or(0), + is_binary, + is_submodule: kind == Some(EntryKind::Commit), + } +} + +// org.tangled.git.defs#linePair, 0-based on both sides +#[derive(Serialize)] +struct LinePairOut { + #[serde(skip_serializing_if = "Option::is_none")] + lhs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + rhs: Option, +} + +// org.tangled.git.defs#diffHunk +#[derive(Serialize)] +struct DiffHunkOut { + #[serde(rename = "novelLhs")] + novel_lhs: Vec, + #[serde(rename = "novelRhs")] + novel_rhs: Vec, + lines: Vec, +} + +fn diff_hunk(hunk: &knot_git::Hunk) -> DiffHunkOut { + let mut novel_lhs = Vec::new(); + let mut novel_rhs = Vec::new(); + let mut lines = Vec::new(); + let mut lhs = hunk.old_start.get().saturating_sub(1); + let mut rhs = hunk.new_start.get().saturating_sub(1); + for line in &hunk.lines { + match line.op { + LineOp::Context => { + lines.push(LinePairOut { + lhs: Some(lhs), + rhs: Some(rhs), + }); + lhs += 1; + rhs += 1; + } + LineOp::Delete => { + novel_lhs.push(lhs); + lines.push(LinePairOut { + lhs: Some(lhs), + rhs: None, + }); + lhs += 1; + } + LineOp::Add => { + novel_rhs.push(rhs); + lines.push(LinePairOut { + lhs: None, + rhs: Some(rhs), + }); + rhs += 1; + } + } + } + DiffHunkOut { + novel_lhs, + novel_rhs, + lines, + } +} + +// org.tangled.git.defs#byteChanges +#[derive(Serialize)] +struct ByteChangesOut { + lhs: u64, + rhs: u64, +} + +// org.tangled.git.defs#fileDiff +#[derive(Serialize)] +struct FileDiffOut { + #[serde(rename = "lhsSrc")] + lhs_src: DiffSrcOut, + #[serde(rename = "rhsSrc")] + rhs_src: DiffSrcOut, + hunks: Vec, + #[serde(rename = "hasByteChanges", skip_serializing_if = "Option::is_none")] + has_byte_changes: Option, + #[serde(rename = "hasSyntacticChanges")] + has_syntactic_changes: bool, +} + +fn file_diff(repo: &Repo, patch: &FilePatch) -> FileDiffOut { + let old_path = match patch.status { + PatchStatus::Added => None, + _ => Some(&patch.path), + }; + let new_path = match patch.status { + PatchStatus::Deleted => None, + _ => Some(&patch.path), + }; + let is_binary = patch.is_binary(); + FileDiffOut { + lhs_src: diff_src(repo, old_path, patch.old_oid, patch.old_kind, is_binary), + rhs_src: diff_src(repo, new_path, patch.new_oid, patch.new_kind, is_binary), + hunks: patch.hunks().iter().map(diff_hunk).collect(), + has_byte_changes: match &patch.body { + PatchBody::Binary(BinaryDiff::Encoded { sizes, .. }) + | PatchBody::Binary(BinaryDiff::Omitted(sizes)) => Some(ByteChangesOut { + lhs: sizes.old, + rhs: sizes.new, + }), + PatchBody::Binary(BinaryDiff::Unchanged(_)) | PatchBody::Text(_) => None, + }, + has_syntactic_changes: patch + .hunks() + .iter() + .any(|hunk| hunk.added().get() > 0 || hunk.deleted().get() > 0), + } +} + +// org.tangled.git.defs#diffStat +#[derive(Serialize)] +struct DiffStatOut { + insertions: i64, + deletions: i64, + #[serde(rename = "filesChanged")] + files_changed: i64, +} + +#[derive(Serialize)] +struct DiffOut { + diffs: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + stats: Option, +} + +pub(crate) async fn repo_diff( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + if params.base.as_str().is_empty() { + return Err(XrpcError::invalid_request("missing base parameter")); + } + if params.head.as_str().is_empty() { + return Err(XrpcError::invalid_request("missing head parameter")); + } + let did = require_hosted(&state, params.repo)?; + let layout = state.layout.clone(); + let limit = state.byte_limits.response; + run_blocking(move || { + let repo = open(&layout, &did)?; + let base = commit_for(&repo, ¶ms.base)?; + let head = commit_for(&repo, ¶ms.head)?; + let mut patches = repo.commit_patches( + knot_git::PatchRange { + base: Some(base), + head, + }, + &mut BinaryBudget::Omit, + )?; + if !params.paths.0.is_empty() { + patches.retain(|patch| { + params.paths.0.iter().any(|filter| { + patch.path.as_str() == filter + || patch + .path + .as_str() + .strip_prefix(filter) + .is_some_and(|rest| rest.starts_with('/')) + }) + }); + } + let stats = DiffStatOut { + insertions: patches + .iter() + .flat_map(|patch| patch.hunks().iter()) + .map(|hunk| hunk.added().get() as i64) + .sum(), + deletions: patches + .iter() + .flat_map(|patch| patch.hunks().iter()) + .map(|hunk| hunk.deleted().get() as i64) + .sum(), + files_changed: patches.len() as i64, + }; + let diffs: Vec = patches + .iter() + .map(|patch| file_diff(&repo, patch)) + .collect(); + json( + DiffOut { + diffs, + stats: Some(stats), + }, + limit, + ) + }) + .await +} + +#[derive(Clone, Copy)] +struct ArchiveFormatArg(ArchiveFormat); + +impl Default for ArchiveFormatArg { + fn default() -> Self { + ArchiveFormatArg(ArchiveFormat::TarGz) + } +} + +impl ArchiveFormatArg { + fn format(self) -> ArchiveFormat { + self.0 + } + + fn name(self) -> &'static str { + match self.0 { + ArchiveFormat::Zip => "zip", + _ => "tar.gz", + } + } + + fn content_type(self) -> &'static str { + match self.0 { + ArchiveFormat::Zip => "application/zip", + _ => "application/gzip", + } + } +} + +impl<'de> Deserialize<'de> for ArchiveFormatArg { + fn deserialize>(deserializer: D) -> Result { + match String::deserialize(deserializer)?.as_str() { + "tar.gz" => Ok(ArchiveFormatArg(ArchiveFormat::TarGz)), + "zip" => Ok(ArchiveFormatArg(ArchiveFormat::Zip)), + _ => Err(de::Error::custom( + "only tar.gz and zip formats are supported", + )), + } + } +} + +// the lexicon makes prefix a required parameter, so a missing or +// cleaning-to-empty value is rejected instead of falling back to a +// derived name +struct ArchivePrefixArg(knot_git::ArchivePrefix); + +impl<'de> Deserialize<'de> for ArchivePrefixArg { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + let cleaned = raw + .split('/') + .filter(|component| !component.is_empty() && *component != ".") + .collect::>() + .join("/"); + match cleaned.is_empty() { + true => Err(de::Error::custom("archive prefix mustn't be empty")), + false => knot_git::ArchivePrefix::new(cleaned) + .ok() + .filter(|prefix| { + !prefix.as_str().contains('\\') && !prefix.as_str().contains(char::is_control) + }) + .map(ArchivePrefixArg) + .ok_or_else(|| { + de::Error::custom(format!( + "archive prefix must stay inside the archive root within {} bytes, and mustn't contain a backslash or a control character", + knot_git::ArchivePrefix::MAX_BYTES + )) + }), + } + } +} + +#[derive(Deserialize)] +pub(crate) struct ArchiveParams { + repo: RepoDid, + #[serde(rename = "ref", default)] + refspec: Revspec, + #[serde(default)] + format: ArchiveFormatArg, + prefix: ArchivePrefixArg, +} + +fn sanitize_filename(name: &str) -> String { + name.replace( + |c: char| c.is_ascii_control() || matches!(c, '"' | '\'' | '/'), + "-", + ) +} + +fn rfc5987_encode(name: &str) -> String { + name.bytes() + .map(|byte| match byte { + b'A'..=b'Z' + | b'a'..=b'z' + | b'0'..=b'9' + | b'!' + | b'#' + | b'$' + | b'&' + | b'+' + | b'-' + | b'.' + | b'^' + | b'_' + | b'`' + | b'|' + | b'~' => String::from(byte as char), + _ => format!("%{byte:02X}"), + }) + .collect() +} + +fn content_disposition(filename: &str) -> String { + let safe = sanitize_filename(filename); + let ascii: String = safe + .chars() + .map(|c| match c.is_ascii() { + true => c, + false => '-', + }) + .collect(); + match safe == ascii { + true => format!("attachment; filename=\"{ascii}\""), + false => format!( + "attachment; filename=\"{ascii}\"; filename*=UTF-8''{}", + rfc5987_encode(&safe) + ), + } +} + +fn archive_etag(did: &RepoDid, commit: Oid, format: ArchiveFormat, prefix: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(did.as_str().as_bytes()); + hasher.update(b"\0"); + hasher.update(commit.to_hex().as_bytes()); + hasher.update(b"\0"); + hasher.update(ArchiveFormatArg(format).name().as_bytes()); + hasher.update(b"\0"); + hasher.update(prefix.as_bytes()); + quoted_etag(&hasher.finalize()) +} + +fn pinned_modified(modified_secs: i64) -> std::time::SystemTime { + std::time::UNIX_EPOCH + std::time::Duration::from_secs(modified_secs.max(0) as u64) +} + +pub(crate) fn reconcile_if_range(request: &mut Request, etag: &str, modified_secs: Option) { + let Some(value) = request + .headers() + .get(header::IF_RANGE) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .map(str::to_string) + else { + return; + }; + let resumes = match ( + value.starts_with('"') || value.starts_with("W/"), + modified_secs, + ) { + (true, _) => value.starts_with('"') && value == etag, + (false, Some(secs)) => { + httpdate::parse_http_date(&value).is_ok_and(|client| client == pinned_modified(secs)) + } + (false, None) => false, + }; + let headers = request.headers_mut(); + headers.remove(header::IF_RANGE); + if !resumes { + headers.remove(header::RANGE); + } +} + +pub(crate) async fn repo_archive( + State(state): State>>, + mut request: Request, +) -> Result { + let params = Query::::try_from_uri(request.uri()) + .map_err(|rejection| XrpcError::invalid_request(rejection.body_text()))? + .0; + if params.refspec.as_str().is_empty() { + return Err(XrpcError::invalid_request("missing ref parameter")); + } + let did = require_hosted(&state, params.repo)?; + let format = params.format; + let format_name = format.name(); + let archive_prefix = params.prefix.0.clone(); + + let (resolved, modified_secs) = run_blocking({ + let layout = state.layout.clone(); + let did = did.clone(); + let refspec = params.refspec.clone(); + move || { + let repo = open(&layout, &did)?; + let commit = commit_for(&repo, &refspec)?; + let modified_secs = repo + .find_commit(commit) + .map(|commit| commit.committer.time.get()) + .unwrap_or(0); + Ok((commit, modified_secs)) + } + }) + .await?; + + let link = { + let mut query = url::form_urlencoded::Serializer::new(String::new()); + query.append_pair("format", format_name); + query.append_pair("prefix", archive_prefix.as_str()); + query.append_pair("ref", &resolved.to_hex()); + query.append_pair("repo", did.as_str()); + format!( + "<{}/xrpc/org.tangled.git.getArchive?{}>; rel=\"immutable\"", + state.knot_service_url.as_str(), + query.finish() + ) + }; + + let etag = archive_etag(&did, resolved, format.format(), archive_prefix.as_str()); + let disposition = content_disposition(&format!("{}.{format_name}", archive_prefix.as_str())); + if etag_matches(request.headers(), &etag) { + return Ok(( + StatusCode::NOT_MODIFIED, + [ + (header::ETAG, etag), + (header::LINK, link), + (header::CACHE_CONTROL, "no-cache".to_string()), + ], + ) + .into_response()); + } + + let temp = run_blocking({ + let layout = state.layout.clone(); + let did = did.clone(); + let archive_limit = state.byte_limits.archive; + let tree_prefix = archive_prefix.into_tree_prefix(); + move || { + let repo = open(&layout, &did)?; + let tree = repo.peel_to_tree(resolved)?; + let temp = tempfile::NamedTempFile::new() + .map_err(|error| XrpcError::internal(format!("cannot spool archive: {error}")))?; + let mut file = temp + .reopen() + .map_err(|error| XrpcError::internal(format!("cannot spool archive: {error}")))?; + repo.write_archive( + tree, + format.format(), + Some(&tree_prefix), + archive_limit, + &mut file, + ) + .map_err(|error| { + match matches!(error, knot_git::GitError::ArchiveTooLarge { .. }) { + true => XrpcError::named( + StatusCode::PAYLOAD_TOO_LARGE, + "ArchiveTooLarge", + format!("archive exceeds the size limit: {error}"), + ), + false => XrpcError::named( + StatusCode::BAD_REQUEST, + "ArchiveError", + format!("failed to create archive: {error}"), + ), + } + })?; + temp.as_file() + .set_modified(pinned_modified(modified_secs)) + .map_err(|error| XrpcError::internal(error.to_string()))?; + Ok(temp) + } + }) + .await?; + + let content_type = format.content_type(); + + reconcile_if_range(&mut request, &etag, Some(modified_secs)); + let serve_response = ServeFile::new(temp.path()) + .oneshot(request) + .await + .unwrap_or_else(|error| match error {}); + drop(temp); + + let mut response = serve_response.map(Body::new); + let headers = response.headers_mut(); + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type)); + let header_value = |value: &str| { + HeaderValue::from_str(value).map_err(|error| XrpcError::internal(error.to_string())) + }; + headers.insert(header::CONTENT_DISPOSITION, header_value(&disposition)?); + headers.insert(header::LINK, header_value(&link)?); + headers.insert(header::ETAG, header_value(&etag)?); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache")); + Ok(response) +} + +#[derive(Deserialize)] +pub(crate) struct LanguagesParams { + repo: RepoDid, + #[serde(rename = "ref", default)] + refspec: Revspec, +} + +#[derive(Serialize)] +struct LanguageOut { + name: knot_types::LanguageName, + size: knot_types::LanguageBytes, + percentage: i64, +} + +#[derive(Serialize)] +struct LanguagesOut { + #[serde(rename = "ref")] + refspec: String, + languages: Vec, + #[serde(rename = "totalSize", skip_serializing_if = "Option::is_none")] + total_size: Option, + #[serde(rename = "totalFiles", skip_serializing_if = "Option::is_none")] + total_files: Option, +} + +pub(crate) async fn repo_languages( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = require_hosted(&state, params.repo)?; + let layout = state.layout.clone(); + let limit = state.byte_limits.response; + let languages_deadline = state.budgets.languages.get().deadline(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let commit = commit_for(&repo, ¶ms.refspec)?; + let sizes = knot_langs::analyze(&repo, commit, languages_deadline)?; + let total: u64 = sizes.values().map(|size| size.get()).sum(); + let mut languages: Vec = sizes + .iter() + .filter(|(_, size)| size.get() > 0) + .map(|(name, size)| LanguageOut { + name: *name, + size: *size, + percentage: ((size.get() as f64) / (total as f64) * 100.0).round() as i64, + }) + .collect(); + languages.sort_by(|a, b| b.size.cmp(&a.size).then_with(|| a.name.cmp(&b.name))); + let count = languages.len() as i64; + json( + LanguagesOut { + refspec: params.refspec.as_str().to_string(), + languages, + total_size: (total > 0).then_some(total), + total_files: (total > 0).then_some(count), + }, + limit, + ) + }) + .await +} + +#[derive(Serialize)] +struct DefaultBranchWire { + #[serde(rename = "ref")] + name: RefName, + #[serde(skip_serializing_if = "Option::is_none")] + head: Option, +} + +#[derive(Deserialize)] +pub(crate) struct ListRefsParams { + repo: RepoDid, + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: Offset, +} + +#[derive(Serialize)] +struct RefWire { + #[serde(rename = "ref")] + name: RefName, + sha: Oid, +} + +#[derive(Serialize)] +struct ListRefsOut { + refs: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, + #[serde(rename = "defaultBranch", skip_serializing_if = "Option::is_none")] + default_branch: Option, +} + +pub(crate) async fn git_list_refs( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let offset = params.cursor; + let limit = params.limit; + let did = require_hosted(&state, params.repo)?; + let layout = state.layout.clone(); + let response_limit = state.byte_limits.response; + run_blocking(move || { + let repo = open(&layout, &did)?; + let mut refs: Vec<_> = repo + .references()? + .into_iter() + .filter(|record| RefClass::of(&record.name).is_public()) + .collect(); + refs.sort_by(|a, b| a.name.as_str().cmp(b.name.as_str())); + let total = refs.len(); + let window: Vec = refs + .iter() + .skip(offset.get()) + .take(limit.get()) + .map(|record| RefWire { + name: record.name.clone(), + sha: record.target, + }) + .collect(); + let cursor = next_cursor(offset, limit, Total::new(total)); + let default_branch = repo.head().map(|head| DefaultBranchWire { + name: head.name, + head: Some(head.target), + }); + json( + ListRefsOut { + refs: window, + cursor, + default_branch, + }, + response_limit, + ) + }) + .await +} + +#[cfg(test)] +mod tests { + use super::{MAX_BLAME_RUNS, blame_runs_too_large, content_disposition, rfc5987_encode}; + + #[test] + fn blame_run_limit_is_inclusive() { + assert!(!blame_runs_too_large(MAX_BLAME_RUNS)); + assert!(blame_runs_too_large(MAX_BLAME_RUNS + 1)); + } + + #[test] + fn content_disposition_quotes_dashes_quotes_and_adds_an_encoded_form_for_non_ascii() { + let cases: &[(&str, &str)] = &[ + ( + "squid-main.tar.gz", + "attachment; filename=\"squid-main.tar.gz\"", + ), + ( + "squid-a\"b.tar.gz", + "attachment; filename=\"squid-a-b.tar.gz\"", + ), + ( + "squid-café.zip", + "attachment; filename=\"squid-caf-.zip\"; filename*=UTF-8''squid-caf%C3%A9.zip", + ), + ]; + cases.iter().for_each(|(name, expected)| { + assert_eq!(content_disposition(name), *expected); + }); + } + + #[test] + fn rfc5987_percent_encodes_outside_the_attr_char_set() { + assert_eq!(rfc5987_encode("a b:c"), "a%20b%3Ac"); + assert_eq!(rfc5987_encode("plain-._~"), "plain-._~"); + } +} diff --git a/knot2/crates/knot-xrpc/src/receive.rs b/knot2/crates/knot-xrpc/src/receive.rs new file mode 100644 index 0000000000000000000000000000000000000000..128ad73dd1126df00fe277d2d027d4401b32b19d --- /dev/null +++ b/knot2/crates/knot-xrpc/src/receive.rs @@ -0,0 +1,297 @@ +use std::future::Future; +use std::io::Read; +use std::pin::Pin; +use std::sync::Arc; + +use axum::Router; +use axum::body::Body; +use axum::extract::{DefaultBodyLimit, Path, State}; +use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::post; +use futures::TryStreamExt; +use knot_messages::{ErrorKey, HttpMessages}; +use knot_pack::{PackError, PackReceiver, ReceivedPack, SocketPeer}; +use knot_receive::{Push, land}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, ActorId, ObjectFormat, RepoDid}; +use tokio_util::io::{StreamReader, SyncIoBridge}; + +use crate::{XrpcError, XrpcState, authenticate_and_authorize_push, run_blocking}; + +const READ_CHUNK: usize = 64 * 1024; +const ADVERTISEMENT: &str = "application/x-git-receive-pack-advertisement"; +const RESULT: &str = "application/x-git-receive-pack-result"; + +pub(crate) fn routes() -> Router>> { + Router::new() + .route( + "/{did}/{name}/git-receive-pack", + post(receive_named::), + ) + .route("/{did}/git-receive-pack", post(receive_did::)) + .layer(DefaultBodyLimit::disable()) +} + +pub fn advertiser( + state: Arc>, +) -> Arc { + Arc::new(ReceiveGate { state }) +} + +struct ReceiveGate { + state: Arc>, +} + +impl knot_pack::ReceiveAdvertiser for ReceiveGate { + fn advertise( + &self, + repo: RepoDid, + peer: SocketPeer, + headers: HeaderMap, + ) -> Pin + Send + '_>> { + Box::pin(async move { serve_advertisement(&self.state, repo, peer, &headers).await }) + } +} + +async fn serve_advertisement( + state: &Arc>, + repo: RepoDid, + peer: SocketPeer, + headers: &HeaderMap, +) -> Response { + let denied = state.catalog.http.push_denied.text(); + if let Err(response) = authorized_pusher(state, peer, headers, &repo, &denied) + .await + .map_err(|error| Box::new(challenge(error))) + { + return *response; + } + let layout = state.layout.clone(); + let target = repo.clone(); + let missing = state.catalog.http.repo_not_found.text(); + let advert = run_blocking(move || { + let repo = layout + .open(&target) + .map_err(|_| XrpcError::not_found(missing))?; + knot_pack::advertise_receive(&repo).map_err(map_pack) + }) + .await; + match advert { + Ok(body) => git_response(ADVERTISEMENT, body), + Err(error) => error.into_response(), + } +} + +async fn receive_named( + State(state): State>>, + Path(crate::RepoPathParams { did, name }): Path, + peer: SocketPeer, + headers: HeaderMap, + body: Body, +) -> Response { + let repo = match crate::resolve_repo_named(&state, &did, &name).await { + Ok(repo) => repo, + Err(error) => return error.into_response(), + }; + serve_receive(&state, repo, peer, &headers, body).await +} + +async fn receive_did( + State(state): State>>, + Path(did): Path, + peer: SocketPeer, + headers: HeaderMap, + body: Body, +) -> Response { + let repo = match crate::resolve_repo_did(&state, &did) { + Ok(repo) => repo, + Err(error) => return error.into_response(), + }; + serve_receive(&state, repo, peer, &headers, body).await +} + +async fn serve_receive( + state: &Arc>, + repo_did: RepoDid, + peer: SocketPeer, + headers: &HeaderMap, + body: Body, +) -> Response { + let denied = state.catalog.http.push_denied.text(); + let pusher = match authorized_pusher(state, peer, headers, &repo_did, &denied) + .await + .map_err(|error| Box::new(challenge(error))) + { + Ok(pusher) => pusher, + Err(response) => return *response, + }; + + let knot_actor = match state.secrets.public_key(&state.knot_did) { + Ok(public) => ActorId::from_secp256k1(public.as_bytes()), + Err(error) => return XrpcError::from(error).into_response(), + }; + + let format = { + let layout = state.layout.clone(); + let target = repo_did.clone(); + let missing = state.catalog.http.repo_not_found.text(); + match run_blocking(move || { + layout + .open(&target) + .map(|repo| repo.object_format()) + .map_err(|_| XrpcError::not_found(missing)) + }) + .await + { + Ok(format) => format, + Err(error) => return error.into_response(), + } + }; + + let _receive_permit = state.slots.receive.acquire().await; + + let received = { + let scratch = state.layout.scratch_dir().to_path_buf(); + let limits = state.pack_limits; + let limit = state.byte_limits.pack; + let catalog = Arc::clone(&state.catalog); + let reader = StreamReader::new(body.into_data_stream().map_err(std::io::Error::other)); + run_blocking(move || { + drain_pack( + SyncIoBridge::new(reader), + &scratch, + limit, + limits, + format, + &catalog.http, + ) + }) + .await + }; + let received = match received { + Ok(received) => received, + Err(error) => return error.into_response(), + }; + if received.is_empty() { + return git_response(RESULT, Vec::new()); + } + + let landed = land(Push { + layout: &state.layout, + repo_did: &repo_did, + received, + limits: state.pack_limits, + knot_actor, + committer: pusher.clone(), + index: &state.index, + atproto: &state.atproto, + resolve_slots: &state.slots.resolve, + appview: &state.appview, + maintenance: &state.maintenance, + hostname: &state.knot_hostname, + catalog: Arc::clone(&state.catalog), + ref_surface: Some(Arc::new(crate::RefProjection { + state: Arc::clone(state), + })), + }) + .await; + match landed { + Ok(framed) => git_response(RESULT, framed), + Err(error) => { + tracing::warn!(repo = repo_did.as_str(), %error, "HTTP receive-pack failed"); + error.into_response() + } + } +} + +fn drain_pack( + mut reader: R, + scratch: &std::path::Path, + limit: knot_pack::MaxWireBytes, + limits: knot_pack::PackLimits, + format: ObjectFormat, + messages: &HttpMessages, +) -> Result { + let mut receiver = PackReceiver::new(scratch, limit, limits, format.kind()) + .map_err(|error| XrpcError::internal(format!("Receive staging failed: {error}")))?; + let mut buffer = [0u8; READ_CHUNK]; + loop { + let read = reader + .read(&mut buffer) + .map_err(|error| XrpcError::invalid_request(format!("Receive read error: {error}")))?; + if read == 0 { + break; + } + if receiver + .write(&buffer[..read]) + .map_err(|error| map_receive_read(error, messages))? + { + break; + } + } + receiver + .finish() + .map_err(|error| map_receive_read(error, messages)) +} + +pub(crate) async fn authorized_pusher( + state: &XrpcState, + peer: SocketPeer, + headers: &HeaderMap, + repo: &RepoDid, + denied: &str, +) -> Result { + authenticate_and_authorize_push(state, peer, headers, repo, denied).await +} + +fn challenge(error: XrpcError) -> Response { + if error.status() == StatusCode::UNAUTHORIZED { + ( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, crate::BASIC_CHALLENGE)], + error.to_string(), + ) + .into_response() + } else { + error.into_response() + } +} + +fn git_response(content_type: &'static str, body: Vec) -> Response { + ( + StatusCode::OK, + [ + (header::CONTENT_TYPE, HeaderValue::from_static(content_type)), + ( + header::CACHE_CONTROL, + HeaderValue::from_static("no-cache, max-age=0, must-revalidate"), + ), + ], + body, + ) + .into_response() +} + +fn map_pack(error: PackError) -> XrpcError { + XrpcError::named(error.http_status(), "PackError", error.to_string()) +} + +fn map_receive_read(error: knot_pack::ReceiveReadError, messages: &HttpMessages) -> XrpcError { + match error { + knot_pack::ReceiveReadError::TooLarge => { + XrpcError::request_too_large(messages.push_too_large.text()) + } + knot_pack::ReceiveReadError::Truncated => { + XrpcError::invalid_request(messages.receive_ended_early.text()) + } + knot_pack::ReceiveReadError::Pack(error) => XrpcError::invalid_request( + messages + .malformed_pack + .line(|ErrorKey::Error| error.to_string()), + ), + knot_pack::ReceiveReadError::Io(error) => { + XrpcError::internal(format!("Receive I/O error: {error}")) + } + } +} diff --git a/knot2/crates/knot-xrpc/src/refrecords.rs b/knot2/crates/knot-xrpc/src/refrecords.rs new file mode 100644 index 0000000000000000000000000000000000000000..e50534b5ff4a2ec453eb54cd6da2d921c57b7341 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/refrecords.rs @@ -0,0 +1,338 @@ +use std::sync::Arc; + +use jacquard_repo::storage::BlockStore; +use knot_git::{RefClass, RefUpdate, Repo}; +use knot_record::blocks::GitBlocks; +use knot_record::chain; +use knot_record::git_refs; +use knot_record::manifest::{MANIFEST_COLLECTION, MANIFEST_RKEY}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, ObjectFormat, PushOptions, RepoDid, ServiceDid}; + +use crate::error::XrpcError; +use crate::run_blocking; +use crate::{XrpcState, firehose}; + +pub fn sync_refs( + state: &Arc>, + git: &Repo, + did: &RepoDid, + applied: &[RefUpdate], +) -> usize { + let _guard = state.cob_locks.repo(did); + let now = state.now(); + let signer = match state.secrets.signer(&state.knot_did) { + Ok(signer) => signer, + Err(error) => { + tracing::error!(%error, "the signing key wouldn't load, so this repository's ref records stay unwritten"); + return 0; + } + }; + let refs_self = match git_refs::desired_record(git) { + Ok(record) => record, + Err(error) => { + tracing::error!( + repo = did.as_str(), + %error, + "the refs record wouldn't build, a restart resyncs it" + ); + return 0; + } + }; + let mut emit = firehose::FirehoseEmit::new(&state.firehose, did, now); + let chunks = match futures::executor::block_on(async { + match (chain::tip(git), refs_self.is_empty()) { + (Ok(None), true) => Ok(Vec::new()), + (Ok(None), false) => { + let seeded = chain::in_tree(refs_self); + chain::materialize(git, did, &seeded, &signer, &mut emit, now).await + } + (Ok(Some(_)), _) => { + chain::sync( + git, + did, + std::slice::from_ref(&git_refs::git_refs_collection()), + &chain::in_tree(refs_self), + &signer, + &mut emit, + now, + ) + .await + } + (Err(error), _) => Err(error), + } + }) { + Ok(chunks) => match emit.finish() { + Ok(()) => chunks, + Err(error) => { + out_of_step(did, &error); + return 0; + } + }, + Err(error) => { + drop(emit); + out_of_step(did, &error); + return 0; + } + }; + let catch_up = applied.is_empty() && !chunks.is_empty(); + if chunks.first().is_some_and(|first| first.prev_rev.is_none()) || catch_up { + let last = chunks + .last() + .expect("a non-empty chunk list has a last chunk"); + firehose::publish_sync(&state.firehose, did, &last.rev, &last.commit, now); + } + chunks.len() +} + +fn out_of_step(did: &RepoDid, error: &dyn std::fmt::Display) { + tracing::error!( + repo = did.as_str(), + %error, + "chain fell out of step with refs, next push, or restart, resyncs that mr chain" + ); +} + +pub(crate) async fn sync_after( + state: &Arc>, + did: &RepoDid, + update: RefUpdate, +) { + let state = Arc::clone(state); + let layout = state.layout.clone(); + let target = did.clone(); + if let Err(error) = run_blocking(move || -> Result<(), XrpcError> { + let repo = layout.open(&target)?; + sync_refs(&state, &repo, &target, std::slice::from_ref(&update)); + Ok(()) + }) + .await + { + tracing::warn!(repo = did.as_str(), %error, "the deleted branch's repository wouldn't open for its ref-record pass"); + } +} + +pub struct RefProjection { + pub state: Arc>, +} + +impl knot_receive::RefSurface for RefProjection { + fn project( + &self, + repo: &Repo, + repo_did: &RepoDid, + applied: &[RefUpdate], + pusher: &AccountDid, + options: &PushOptions, + ) -> Vec { + sync_refs(&self.state, repo, repo_did, applied); + // queued, so it doesn't need the signer, the outbox signs per attempt + let replies = notify_spindle(&self.state, repo, repo_did, applied, pusher, options); + match self.state.secrets.signer(&self.state.knot_did) { + Ok(signer) => firehose::publish_push( + &self.state.firehose, + repo_did, + pusher, + applied, + options, + &signer, + self.state.now(), + ), + Err(error) => tracing::error!( + repo = repo_did.as_str(), + %error, + "the signing key wouldn't load, so this push goes unannounced" + ), + } + replies + } +} + +fn notify_spindle( + state: &Arc>, + repo: &Repo, + did: &RepoDid, + applied: &[RefUpdate], + pusher: &AccountDid, + options: &PushOptions, +) -> Vec { + let spindle = match get_repo_spindle(repo) { + Ok(Some(spindle)) => spindle, + Ok(None) => return Vec::new(), + Err(error) => { + tracing::warn!(repo = did.as_str(), %error, "the manifest wouldn't read, so the spindle doesn't hear of this push"); + return Vec::new(); + } + }; + let events: Vec<_> = applied + .iter() + .filter(|update| RefClass::of(update.name()) == RefClass::Public) + .filter_map(|update| { + let body = bounded_push_event_body(repo.object_format(), did, update, pusher, options)?; + Some((update.name().as_str().to_string(), body)) + }) + .collect(); + state.outbox.enqueue_all(&spindle, did, events, state.now()) +} + +/// real events are a few hundred bytes, so this only bites ref names and push options built to bloat the queue +const MAX_EVENT_BYTES: usize = 8 * 1024; + +fn bounded_push_event_body( + format: ObjectFormat, + did: &RepoDid, + update: &RefUpdate, + pusher: &AccountDid, + options: &PushOptions, +) -> Option { + let fits = |body: &serde_json::Value| body.to_string().len() <= MAX_EVENT_BYTES; + let body = push_event_body(format, did, update, pusher, options); + if fits(&body) { + return Some(body); + } + tracing::warn!( + repo = did.as_str(), + ref_name = update.name().as_str(), + "the push options are too big to queue, so the spindle gets the push without them" + ); + Some(push_event_body( + format, + did, + update, + pusher, + &PushOptions::default(), + )) + .filter(fits) +} + +fn push_event_body( + format: ObjectFormat, + did: &RepoDid, + update: &RefUpdate, + pusher: &AccountDid, + options: &PushOptions, +) -> serde_json::Value { + let transition = update.transition(); + let null = format.null_oid(); + let mut event = serde_json::json!({ + "$type": "org.tangled.event.push", + "repo": did, + "ref": update.name(), + "oldSha": transition.old_oid().unwrap_or(null).to_hex(), + "newSha": transition.new_oid().unwrap_or(null).to_hex(), + "actor": pusher, + }); + if !options.is_empty() { + event["pushOptions"] = serde_json::json!(options); + } + serde_json::json!({ "repo": did, "event": event }) +} + +fn get_repo_spindle(repo: &Repo) -> Result, String> { + #[derive(serde::Deserialize)] + struct Manifest { + spindle: Option, + } + let Some(tip) = chain::tip(repo).map_err(|error| error.to_string())? else { + return Ok(None); + }; + let key = format!("{MANIFEST_COLLECTION}/{MANIFEST_RKEY}"); + futures::executor::block_on(async { + let Some(cid) = chain::load_tree(repo, &tip) + .get(&key) + .await + .map_err(|error| error.to_string())? + else { + return Ok(None); + }; + let Some(bytes) = GitBlocks::at(repo, tip.blocks) + .get(&cid) + .await + .map_err(|error| error.to_string())? + else { + return Ok(None); + }; + serde_ipld_dagcbor::from_slice::(&bytes) + .map(|manifest| manifest.spindle) + .map_err(|error| error.to_string()) + }) +} + +#[cfg(test)] +mod tests { + use knot_types::{Oid, PushOption, RefName}; + + use super::*; + + fn body(options: &[&str]) -> serde_json::Value { + let update = RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: Oid::from_hex(&"ab".repeat(20)).unwrap(), + }; + let options = PushOptions::new(options.iter().map(|o| PushOption::new(*o).unwrap())); + push_event_body( + ObjectFormat::SHA1, + &RepoDid::new("did:plc:squid").unwrap(), + &update, + &AccountDid::new("did:plc:pusher").unwrap(), + &options, + ) + } + + #[test] + fn push_event_names_the_pusher() { + let event = &body(&[])["event"]; + assert_eq!(event["actor"], "did:plc:pusher"); + assert_eq!(event["ref"], "refs/heads/main"); + } + + #[test] + fn push_event_omits_empty_push_options() { + assert!(body(&[])["event"].get("pushOptions").is_none()); + } + + #[test] + fn push_event_carries_push_options_in_order() { + let event = &body(&["ci-skip", "foo=bar"])["event"]; + assert_eq!( + event["pushOptions"], + serde_json::json!(["ci-skip", "foo=bar"]) + ); + } + + fn bounded(ref_name: &str, options: &[&str]) -> Option { + let update = RefUpdate::Create { + name: RefName::new(ref_name).unwrap(), + new: Oid::from_hex(&"ab".repeat(20)).unwrap(), + }; + let options = PushOptions::new(options.iter().map(|o| PushOption::new(*o).unwrap())); + bounded_push_event_body( + ObjectFormat::SHA1, + &RepoDid::new("did:plc:squid").unwrap(), + &update, + &AccountDid::new("did:plc:pusher").unwrap(), + &options, + ) + } + + #[test] + fn an_ordinary_push_event_keeps_its_options() { + let event = &bounded("refs/heads/main", &["ci.skip"]).unwrap()["event"]; + assert_eq!(event["pushOptions"][0], "ci.skip"); + } + + #[test] + fn bloated_push_options_are_dropped_but_the_event_stays() { + let big = "x".repeat(1024); + let options = vec![big.as_str(); 20]; + let event = &bounded("refs/heads/main", &options).unwrap()["event"]; + assert!(event.get("pushOptions").is_none()); + assert_eq!(event["ref"], "refs/heads/main"); + } + + #[test] + fn a_ref_name_that_alone_blows_the_budget_is_skipped() { + let name = format!("refs/heads/{}", "a".repeat(MAX_EVENT_BYTES)); + assert!(bounded(&name, &[]).is_none()); + } +} diff --git a/knot2/crates/knot-xrpc/src/repos.rs b/knot2/crates/knot-xrpc/src/repos.rs new file mode 100644 index 0000000000000000000000000000000000000000..057135b4ca549df326803cbb3d80ac439259da72 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/repos.rs @@ -0,0 +1,865 @@ +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use http::HeaderMap; +use serde::{Deserialize, Serialize}; + +use knot_acl::{KnotAcl, can_admin_knot, can_delete_repo}; +use knot_atproto::{PreparedRepoDid, RecordPresence}; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{ + Registration, RegistryChange, Rename, RepoRef, RepoRegistryCob, deregister_repo, register_repo, +}; +use knot_git::{GitError, Layout, Repo}; +use knot_index::Resolved; +use knot_record::manifest::{ + ManifestMeta, ManifestRecord, ManifestSource, RepoDescription, RepoTopics, +}; +use knot_runtime::{Clock, HttpTransport, Signer}; +use knot_types::{ + AccountDid, ActorId, BranchName, CobId, ContributionPolicy, OwnerDid, RecordBody, RefName, + RepoDid, RepoName, RepoRkey, ServiceDid, UnixSeconds, +}; + +use crate::error::XrpcError; +use crate::reservations::ReserveDecision; +use crate::{XrpcState, decode, ok_empty, run_blocking}; + +pub(crate) const CREATE_ROUTE: &str = "/xrpc/org.tangled.repo.createRepo"; +pub(crate) const DELETE_ROUTE: &str = "/xrpc/org.tangled.repo.deleteRepo"; +pub(crate) const RESERVE_ROUTE: &str = "/xrpc/org.tangled.repo.reserveKey"; + +#[derive(Deserialize)] +struct CreateInput { + name: RepoName, + did: Option, + #[serde(rename = "defaultBranch")] + default_branch: Option, + spindle: Option, + description: Option, + website: Option, + topics: Option, + source: Option, +} + +#[derive(Serialize)] +struct CreateOutput { + did: RepoDid, + key: ActorId, +} + +#[derive(Deserialize)] +struct DeleteInput { + repo: RepoDid, + #[serde(default)] + force: bool, +} + +#[derive(Deserialize)] +struct ReserveInput { + #[serde(rename = "repo")] + repo_did: RepoDid, +} + +#[derive(Serialize)] +struct ReserveOutput { + #[serde(rename = "repo")] + repo_did: RepoDid, + key: ActorId, +} + +pub(crate) async fn reserve_key( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate_write(&headers, &method).await?; + crate::authorize_create( + &state, + &actor, + "only knot admin or member may reserve a repository key", + ) + .await?; + let ReserveInput { repo_did } = decode(&body)?; + if repo_did == state.knot_repo { + return Err(XrpcError::invalid_request( + "repo mustn't be knot's own identity", + )); + } + match state.index.owner_of(&repo_did) { + Resolved::Ready(Some(_)) => { + return Err(XrpcError::conflict( + "that repo DID is already hosted on this knot", + )); + } + Resolved::Warming => { + return Err(XrpcError::warming( + "registry projection is still warming, please retry shortly", + )); + } + Resolved::Ready(None) => {} + } + + let now = state.now(); + state.reservations.prune(now); + + match state.reservations.try_reserve(&repo_did, &actor, now) { + ReserveDecision::HeldByOther => { + return Err(XrpcError::conflict( + "that repo DID is reserved by another account", + )); + } + ReserveDecision::PerActorFull => { + return Err(XrpcError::rate_limited( + knot_resource::Bucket::ActorReservations, + "you are holding maximum number of reserved repository keys awaiting creation", + )); + } + ReserveDecision::GlobalFull => { + return Err(XrpcError::rate_limited( + knot_resource::Bucket::KnotReservations, + "knot is holding maximum number of reserved repository keys awaiting creation", + )); + } + ReserveDecision::Fresh | ReserveDecision::Renewed => {} + } + + let public = match state.secrets.public_key(&state.knot_did) { + Ok(public) => public, + Err(error) => { + state.reservations.release(&repo_did); + return Err(error.into()); + } + }; + let key = ActorId::from_secp256k1(public.as_bytes()); + + Ok((http::StatusCode::OK, Json(ReserveOutput { repo_did, key })).into_response()) +} + +pub(crate) async fn create_repo( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate_write(&headers, &method).await?; + crate::authorize_create( + &state, + &actor, + "only knot admin or member may create repositories", + ) + .await?; + + let input: CreateInput = decode(&body)?; + let head = input + .default_branch + .as_ref() + .map(|branch| branch.head_ref()); + let rkey = RepoRkey::from_repo_name(&input.name) + .map_err(|error| XrpcError::invalid_request(error.to_string()))?; + let source = match &input.source { + Some(source) => Some(source_of(&state, source).await?), + None => None, + }; + let meta = ManifestMeta { + contribution_policy: Some(state.contribution_policy), + spindle: input.spindle, + description: input.description, + website: input.website, + topics: input.topics, + source: input.source, + }; + + let owner = OwnerDid::from(actor.clone()); + match state.index.resolve_repo(&owner, &rkey) { + Resolved::Ready(Some(existing)) => match state.index.rkey_of(&existing) { + Resolved::Ready(Some(canonical)) if canonical == rkey => { + if let Some(source) = &source + && crate::imports::retry_failed(&state, &existing, &source.url, source.kind) + .await? + { + let public = state.secrets.public_key(&state.knot_did)?; + let key = ActorId::from_secp256k1(public.as_bytes()); + return Ok(( + http::StatusCode::OK, + Json(CreateOutput { did: existing, key }), + ) + .into_response()); + } + return Err(XrpcError::conflict( + "repository with that record key already exists for this owner", + )); + } + Resolved::Warming => { + return Err(XrpcError::warming( + "registry projection is still warming, please retry shortly", + )); + } + _ => {} + }, + Resolved::Warming => { + return Err(XrpcError::warming( + "registry projection is still warming, please retry shortly", + )); + } + Resolved::Ready(None) => {} + } + + let fork = match &source { + Some(source) => Some(( + crate::forks::ForkSource::resolve(&state, &source.url).await?, + source.kind, + )), + None => None, + }; + + let (repo_did, provisioning) = provision_repo_did(&state, &actor, &rkey, input.did).await?; + let now = state.now(); + let genesis = match provisioning { + Provisioned::Minted(prepared) => Some(crate::plc::PendingGenesis { + did: repo_did.clone(), + operation: prepared.operation().clone(), + }), + Provisioned::Reserved => None, + }; + let (repo_did, key) = seed_repo( + &state, + actor, + owner, + input.name, + meta, + rkey, + repo_did, + Seeding::Native { genesis }, + fork, + head, + now, + ) + .await?; + + Ok(( + http::StatusCode::OK, + Json(CreateOutput { did: repo_did, key }), + ) + .into_response()) +} + +pub(crate) enum Seeding { + Native { + genesis: Option, + }, + Fostered { + manifest: RecordBody, + object_format: Option, + }, +} +#[allow(clippy::too_many_arguments)] +pub(crate) async fn seed_repo( + state: &Arc>, + actor: AccountDid, + owner: OwnerDid, + name: RepoName, + meta: ManifestMeta, + rkey: RepoRkey, + repo_did: RepoDid, + seeding: Seeding, + fork: Option<(crate::forks::ForkSource, crate::forks::SourceKind)>, + head: Option, + now: UnixSeconds, +) -> Result<(RepoDid, ActorId), XrpcError> { + let knot_signer = state.secrets.signer(&state.knot_did)?; + let key = ActorId::from_secp256k1(knot_signer.public_key().as_bytes()); + + let policy = meta + .contribution_policy + .unwrap_or(state.contribution_policy); + let manifest = match &seeding { + Seeding::Fostered { manifest, .. } => manifest.clone(), + _ => ManifestRecord::new(owner.clone(), name.clone(), meta, now, actor.clone())?.body()?, + }; + let registration = Registration { + owner, + rkey, + name, + repo: repo_did.clone(), + created_at: now, + policy, + fostered: matches!(seeding, Seeding::Fostered { .. }), + retired_at: None, + }; + + let lfs_store = Arc::clone(&state.lfs.handle.store); + let object_format = match (&seeding, &fork) { + (_, Some((fork, _))) => Some(fork.refs.object_format), + (Seeding::Fostered { object_format, .. }, None) => *object_format, + _ => None, + }; + let layout = state.layout.clone(); + let placed = repo_did.clone(); + let lfs = lfs_store.clone(); + run_blocking(move || { + let repo = match object_format { + Some(format) => layout.create_with_format(&placed, format), + None => layout.create(&placed), + } + .map_err(|error| match error { + GitError::AlreadyExists(_) => XrpcError::conflict( + "that repo DID already has a directory on this knot: a hosted repo or crash leftovers", + ), + GitError::ReservedDid(_) => { + XrpcError::invalid_request("Repo DID mustn't be knot's own DID") + } + other => XrpcError::internal(other.to_string()), + })?; + let outcome = stage_repo(&repo, head.as_ref()); + if outcome.is_err() { + rollback_local(&layout, lfs.as_ref(), &placed); + } + outcome + }) + .await?; + + if let Some((fork, kind)) = &fork + && let Err(error) = crate::imports::stage(state, &repo_did, &fork.origin, *kind, &fork.refs) + { + let layout = state.layout.clone(); + let placed = repo_did.clone(); + let lfs = lfs_store.clone(); + let _ = run_blocking(move || { + rollback_local(&layout, lfs.as_ref(), &placed); + Ok(()) + }) + .await; + return Err(error); + } + + let native = matches!(seeding, Seeding::Native { .. }); + let reserved = matches!(seeding, Seeding::Native { genesis: None }); + let genesis_op = match &seeding { + Seeding::Native { + genesis: Some(entry), + } => Some(entry.operation.clone()), + _ => None, + }; + let queued_import = fork.is_some(); + let layout = state.layout.clone(); + let cob_locks = Arc::clone(&state.cob_locks); + let meta_path = state.meta_path.clone(); + let placed = repo_did.clone(); + let lfs = lfs_store.clone(); + let home = CobHome::from(&state.knot_did); + run_blocking(move || { + let _guard = cob_locks.meta(); + let was_queued = matches!(seeding, Seeding::Native { genesis: Some(_) }); + let outcome = Repo::open(&meta_path) + .map_err(XrpcError::from) + .and_then(|meta| { + let queued = match seeding { + Seeding::Native { + genesis: Some(entry), + } => crate::plc::enqueue_pending_genesis(&meta, entry), + _ => Ok(()), + }; + let registered = queued.and_then(|()| { + register( + &CobStore::new(&meta), + &home, + registration, + &knot_signer, + now, + ) + }); + if registered.is_err() + && was_queued + && let Err(cleanup) = crate::plc::drop_pending_genesis(&meta, &placed) + { + tracing::warn!( + repo = %placed, + %cleanup, + "DoA genesis rows remain queued along with the rolled back git repo" + ); + } + registered + }); + if outcome.is_err() { + rollback_local(&layout, lfs.as_ref(), &placed); + } + if let Err(error) = &outcome + && !reserved + { + tracing::error!( + repo = %placed, + %error, + "Git repo registration failed; the newly minted repo was rolled back" + ); + } + outcome + }) + .await?; + + if reserved { + state.reservations.release(&repo_did); + } + + if let Some(operation) = &genesis_op + && let Err(error) = state + .atproto + .submit_plc_operation(&repo_did, operation) + .await + { + tracing::warn!(repo = %repo_did, %error, "genesis didn't post at birth; the plc sweeper retries it"); + } + + // the import sidecar is already queued on disk, so any error past this point + // must wake the import queue instead of stranding a Queued job until restart + let outcome: Result<(), XrpcError> = async { + let index = Arc::clone(&state.index); + let refreshed = repo_did.clone(); + run_blocking(move || { + index.refresh_registry().map_err(|error| { + XrpcError::internal(format!( + "repo {refreshed} was created and registered but registry projection refresh failed: {error}" + )) + }) + }) + .await?; + + let knot_signer = state.secrets.signer(&state.knot_did)?; + let layout = state.layout.clone(); + let cob_locks = Arc::clone(&state.cob_locks); + let firehose = Arc::clone(&state.firehose); + let did = repo_did.clone(); + let initialized = repo_did.clone(); + let chunk = run_blocking(move || -> Result { + let identity = native.then(|| firehose.reserve()); + let account = native.then(|| firehose.reserve()); + let _guard = cob_locks.repo(&initialized); + let git = layout.open(&initialized)?; + let mut emit = crate::FirehoseEmit::new(&firehose, &did, now); + let tail = futures::executor::block_on(knot_record::chain::initialize( + &git, + &did, + &knot_signer, + &mut emit, + now, + ))?; + let tail = match knot_record::git_refs::desired_record(&git) { + Ok(record) => match futures::executor::block_on(knot_record::chain::sync( + &git, + &did, + std::slice::from_ref(&knot_record::git_refs::git_refs_collection()), + &knot_record::chain::in_tree(record), + &knot_signer, + &mut emit, + now, + )) { + Ok(chunks) => chunks.into_iter().last().unwrap_or(tail), + Err(error) => { + tracing::warn!(repo = %did, %error, "a new repo's refs record wouldn't project at birth; the boot pass or its first push resyncs it"); + tail + } + }, + Err(error) => { + tracing::warn!(repo = %did, %error, "a new repo's refs record wouldn't build; the boot pass or its first push resyncs it"); + tail + } + }; + emit.finish()?; + if let Some(identity) = identity { + crate::firehose::fulfill_identity(identity, &did, now); + } + if let Some(account) = account { + crate::firehose::fulfill_account(account, &did, true, None, now); + } + Ok(tail) + }) + .await?; + crate::firehose::publish_sync(&state.firehose, &repo_did, &chunk.rev, &chunk.commit, now); + crate::social::open_manifest(state, actor, repo_did.clone(), manifest, now).await + } + .await; + if queued_import { + state.imports.wake(repo_did.clone()); + } + outcome?; + + Ok((repo_did, key)) +} + +pub(crate) async fn source_of( + state: &XrpcState, + source: &ManifestSource, +) -> Result { + let (url, kind) = match source { + ManifestSource::Tangled { did } => { + // TODO: short-circuit to knot_service_url when the source DID is hosted on this knot + let identity = state + .atproto + .resolve_identity(&AccountDid::from(did.clone())) + .await?; + let mut url = identity.pds.url().clone(); + url.path_segments_mut() + .map_err(|()| XrpcError::invalid_request("source repo's knot url can't be a base"))? + .pop_if_empty() + .push(did.as_str()); + (url, crate::forks::SourceKind::Fork) + } + ManifestSource::External { uri } => (uri.clone(), crate::forks::SourceKind::Import), + }; + crate::body::SourceUrl::from_url(url) + .map(|url| crate::body::Source { url, kind }) + .map_err(XrpcError::invalid_request) +} + +pub(crate) async fn populate( + state: &Arc>, + repo_did: &RepoDid, + fork: &crate::forks::ForkSource, + kind: crate::forks::SourceKind, +) -> Result, XrpcError> { + let tips = fork.refs.tips(); + let pack = match tips.is_empty() { + true => None, + false => { + let scratch = state.layout.scratch_dir().to_path_buf(); + std::fs::create_dir_all(&scratch) + .map_err(|error| XrpcError::internal(error.to_string()))?; + let pack = tempfile::Builder::new() + .prefix("import-pack.") + .tempfile_in(&scratch) + .map_err(|error| XrpcError::internal(error.to_string()))?; + crate::forks::fetch_pack_into( + state, + &fork.upstream, + knot_pack::WantOids::new(tips.clone()), + knot_pack::HaveOids::default(), + fork.refs.object_format, + knot_pack::MaxWireBytes::new(state.byte_limits.import_pack.get() as usize), + pack.path(), + ) + .await?; + Some(pack) + } + }; + let layout = state.layout.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let placed = repo_did.clone(); + let origin = fork.origin.clone(); + let refs = fork.refs.clone(); + let staged_tips = tips.clone(); + run_blocking(move || { + let _guard = cob_locks.repo(&placed); + if !matches!(index.owner_of(&placed), Resolved::Ready(Some(_))) { + return Err(XrpcError::conflict("repository was deleted during import")); + } + let repo = layout.open(&placed)?; + if let Some(pack) = &pack { + crate::forks::ingest_pack(&repo, pack.path(), &staged_tips, &[])?; + } + crate::forks::apply_source_refs(&repo, &refs, &origin, kind) + }) + .await?; + let missing = crate::lfs::mirror_fork_objects( + Arc::clone(state), + fork.upstream.clone(), + repo_did.clone(), + knot_pack::WantOids::new(tips), + knot_pack::HaveOids::default(), + ) + .await?; + if !crate::imports::hosted(state, repo_did) { + let _ = state.lfs.handle.store.remove_repo(repo_did); + return Err(XrpcError::conflict("repository was deleted during import")); + } + Ok(missing) +} + +fn rollback_local(layout: &Layout, lfs: &knot_lfs::DiskStore, repo_did: &RepoDid) { + if let Err(error) = lfs.remove_repo(repo_did) { + tracing::error!(repo = %repo_did, %error, "couldn't roll back lfs prefix"); + } + if let Err(error) = layout.remove(repo_did) { + tracing::error!(repo = %repo_did, %error, "couldn't roll back on-disk repo :3"); + } +} + +enum Provisioned { + Minted(PreparedRepoDid), + Reserved, +} + +async fn provision_repo_did( + state: &XrpcState, + actor: &knot_types::AccountDid, + rkey: &RepoRkey, + provided: Option, +) -> Result<(RepoDid, Provisioned), XrpcError> { + match provided { + Some(did) => { + match state.index.owner_of(&did) { + Resolved::Ready(Some(_)) => { + return Err(XrpcError::conflict( + "that repo DID is already hosted on this knot", + )); + } + Resolved::Warming => { + return Err(XrpcError::warming( + "registry projection is still warming, please retry shortly", + )); + } + Resolved::Ready(None) => {} + } + if !state.reservations.holder_is(&did, actor, state.now()) { + return Err(XrpcError::invalid_request( + "reserve this did for your own account via org.tangled.repo.reserveKey before creating it", + )); + } + let knot_public = state.secrets.public_key(&state.knot_did)?; + state + .atproto + .verify_account_document(&did, &knot_public, &state.knot_service_url) + .await + .map_err(|error| match error { + knot_atproto::AtprotoError::Resolve(_) => XrpcError::invalid_request( + "repoDid document must publish #atproto verification method naming \ + this knot's key and #atproto_pds service pointing at it", + ), + other => other.into(), + })?; + Ok((did, Provisioned::Reserved)) + } + None => { + let knot_signer = state.secrets.signer(&state.knot_did)?; + let owner = OwnerDid::from(actor.clone()); + let nonce = knot_atproto::MintNonce::mint(&*state.entropy, &owner, rkey); + let prepared = + knot_atproto::prepare_repo_did(&knot_signer, &state.knot_service_url, &nonce) + .map_err(XrpcError::from)?; + Ok((prepared.did.clone(), Provisioned::Minted(prepared))) + } + } +} + +fn stage_repo(repo: &Repo, head: Option<&RefName>) -> Result<(), XrpcError> { + if let Some(refname) = head { + repo.set_head(refname)?; + } + Ok(()) +} + +pub(crate) fn registry_object(store: &CobStore) -> Result, XrpcError> { + match store + .list::() + .map_err(XrpcError::from)? + .as_slice() + { + [] => Ok(None), + [object] => Ok(Some(*object)), + many => Err(XrpcError::internal(format!( + "{} repo registry objects share meta-repo", + many.len() + ))), + } +} + +fn register( + store: &CobStore, + home: &CobHome, + registration: Registration, + signer: &dyn Signer, + now: UnixSeconds, +) -> Result<(), XrpcError> { + match registry_object(store)? { + None => store + .create(home, &RegistryChange::Register(registration), signer, now) + .map(|_| ()) + .map_err(XrpcError::from), + Some(object) => register_repo(store, home, object, registration, signer, now) + .map(|_| ()) + .map_err(XrpcError::from), + } +} + +pub(crate) async fn delete_repo( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate_write(&headers, &method).await?; + let DeleteInput { + repo: repo_did, + force, + } = decode(&body)?; + + let RepoRef { owner: did, rkey } = match state.index.ownership_of(&repo_did) { + Resolved::Ready(Some(found)) => found, + Resolved::Ready(None) => return Ok(ok_empty()), + Resolved::Warming => { + return Err(XrpcError::warming( + "registry projection is still warming, please retry shortly", + )); + } + }; + + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if !can_delete_repo(&acl, &actor, &repo_did).is_allowed() { + return Err(XrpcError::forbidden( + "only repository owner or a knot admin may delete it", + )); + } + + if force { + if !can_admin_knot(&acl, &actor).is_allowed() { + return Err(XrpcError::forbidden( + "only knot admin may force a delete past the PDS record check", + )); + } + } else { + let owner = AccountDid::from(did.clone()); + match state.atproto.repo_record_present(&owner, &rkey).await { + Ok(RecordPresence::Present) => { + return Err(XrpcError::conflict( + "org.tangled.repo.declaration record still exists on the owner's PDS. Remove it there first or force the delete.", + )); + } + Ok(RecordPresence::Absent) => {} + Err(error) => { + tracing::warn!( + repo = %repo_did, + %error, + "proceeding w/ best-effort delete despite unconfirmed owner PDS record :p" + ); + } + } + } + + let now = state.now(); + let now_micros = state.atproto.now(); + let knot_signer = state.secrets.signer(&state.knot_did)?; + let layout = state.layout.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let meta_path = state.meta_path.clone(); + let target = RepoRef { owner: did, rkey }; + let deleted = repo_did.clone(); + let home = CobHome::from(&state.knot_did); + let lfs_store = Arc::clone(&state.lfs.handle.store); + run_blocking(move || { + let _repo_guard = cob_locks.repo(&deleted); + let _meta_guard = cob_locks.meta(); + let meta = Repo::open(&meta_path)?; + let store = CobStore::new(&meta); + crate::plc::record_deletion(&meta, &deleted, now_micros)?; + crate::plc::drop_pending_genesis(&meta, &deleted)?; + deregister(&store, &home, target, &deleted, &knot_signer, now)?; + let removal = layout.remove(&deleted); + index.refresh_registry().map_err(|error| { + XrpcError::internal(format!( + "repo {deleted} was deregistered but registry projection refresh failed: {error}" + )) + })?; + if let Err(error) = lfs_store.remove_repo(&deleted) { + tracing::warn!( + repo = %deleted, + %error, + "lfs prefix removal on delete failed, orphan sweep will reclaim" + ); + } + removal.map_err(|error| { + XrpcError::internal(format!( + "repo {deleted} was deregistered but its on-disk directory couldn't be removed: {error}" + )) + }) + }) + .await?; + + crate::firehose::publish_account( + &state.firehose, + &repo_did, + false, + Some("deleted"), + state.now(), + ); + Ok(ok_empty()) +} + +fn deregister( + store: &CobStore, + home: &CobHome, + target: RepoRef, + expected: &RepoDid, + signer: &dyn Signer, + now: UnixSeconds, +) -> Result<(), XrpcError> { + match registry_object(store)? { + None => Ok(()), + Some(object) => deregister_repo(store, home, object, target, expected.clone(), signer, now) + .map(|_| ()) + .map_err(XrpcError::from), + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn rename_in_registry( + meta: &Repo, + home: &CobHome, + owner: OwnerDid, + repo: &RepoDid, + rkey: RepoRkey, + name: RepoName, + signer: &dyn Signer, + now: UnixSeconds, +) -> Result<(), XrpcError> { + let store = CobStore::new(meta); + let object = registry_object(&store)? + .ok_or_else(|| XrpcError::not_found("no repositories are registered on this knot"))?; + knot_cobs::rename_repo( + &store, + home, + object, + Rename { + owner, + rkey, + name, + repo: repo.clone(), + }, + signer, + now, + ) + .map(|_| ()) + .map_err(XrpcError::from) +} + +pub(crate) fn set_policy_in_registry( + meta: &Repo, + home: &CobHome, + owner: OwnerDid, + repo: &RepoDid, + policy: ContributionPolicy, + signer: &dyn Signer, + now: UnixSeconds, +) -> Result<(), XrpcError> { + let store = CobStore::new(meta); + let object = registry_object(&store)? + .ok_or_else(|| XrpcError::not_found("no repositories are registered on this knot"))?; + knot_cobs::set_repo_policy( + &store, + home, + object, + knot_cobs::Policy { + owner, + repo: repo.clone(), + policy, + }, + signer, + now, + ) + .map(|_| ()) + .map_err(XrpcError::from) +} diff --git a/knot2/crates/knot-xrpc/src/reservations.rs b/knot2/crates/knot-xrpc/src/reservations.rs new file mode 100644 index 0000000000000000000000000000000000000000..f55aa808950b30691c6b596d00680e90cc1a59bb --- /dev/null +++ b/knot2/crates/knot-xrpc/src/reservations.rs @@ -0,0 +1,198 @@ +use knot_cache::{Admitted, Expiring, GroupQuota, Quotas, Rejected, TotalQuota}; +use knot_types::{AccountDid, RepoDid, UnixMicros, UnixSeconds}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ReserveDecision { + Fresh, + Renewed, + HeldByOther, + PerActorFull, + GlobalFull, +} + +knot_types::scalar_newtype! { + pub struct ReservationTtl(i64); + pub struct PerActorQuota(usize); + pub struct GlobalQuota(usize); +} + +pub struct Reservations { + held: Expiring, + ttl: ReservationTtl, +} + +fn micros(seconds: UnixSeconds) -> UnixMicros { + UnixMicros::new((seconds.get().max(0) as u64).saturating_mul(1_000_000)) +} + +impl Reservations { + pub fn new(ttl: ReservationTtl, per_actor: PerActorQuota, global: GlobalQuota) -> Self { + Self { + held: Expiring::new(Quotas { + per_group: GroupQuota::new(per_actor.get()), + total: TotalQuota::new(global.get()), + }), + ttl, + } + } + + pub(crate) fn prune(&self, now: UnixSeconds) -> Vec { + self.held.prune(micros(now)) + } + + pub(crate) fn try_reserve( + &self, + repo: &RepoDid, + actor: &AccountDid, + now: UnixSeconds, + ) -> ReserveDecision { + let expires_at = micros(now.saturating_add_secs(self.ttl.get())); + match self.held.admit_or_renew( + repo.clone(), + actor.clone(), + actor.clone(), + expires_at, + micros(now), + ) { + Ok(Admitted::Inserted) => ReserveDecision::Fresh, + Ok(Admitted::Occupied(holder)) if &holder == actor => ReserveDecision::Renewed, + Ok(Admitted::Occupied(_)) => ReserveDecision::HeldByOther, + Err(Rejected::Group) => ReserveDecision::PerActorFull, + Err(Rejected::Total) => ReserveDecision::GlobalFull, + } + } + + pub(crate) fn holder_is(&self, repo: &RepoDid, actor: &AccountDid, now: UnixSeconds) -> bool { + self.held + .get(repo, micros(now)) + .is_some_and(|holder| &holder == actor) + } + + pub(crate) fn release(&self, repo: &RepoDid) { + self.held.remove(repo); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn actor(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:web:{suffix}")).unwrap() + } + + fn repo(suffix: &str) -> RepoDid { + RepoDid::new(format!("did:web:{suffix}.olaren.dev")).unwrap() + } + + fn at(seconds: i64) -> UnixSeconds { + UnixSeconds::new(seconds) + } + + #[test] + fn a_reservation_binds_to_its_actor_and_blocks_a_stranger() { + let reservations = Reservations::new( + ReservationTtl::new(3_600), + PerActorQuota::new(8), + GlobalQuota::new(64), + ); + assert_eq!( + reservations.try_reserve(&repo("squid"), &actor("nel.pet"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("squid"), &actor("olaren.dev"), at(1)), + ReserveDecision::HeldByOther, + "different account cannot take a live reservation" + ); + assert!(reservations.holder_is(&repo("squid"), &actor("nel.pet"), at(1))); + assert!(!reservations.holder_is(&repo("squid"), &actor("olaren.dev"), at(1))); + } + + #[test] + fn re_reserving_by_the_same_actor_renews_the_lease() { + let reservations = Reservations::new( + ReservationTtl::new(100), + PerActorQuota::new(8), + GlobalQuota::new(64), + ); + assert_eq!( + reservations.try_reserve(&repo("squid"), &actor("nel.pet"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("squid"), &actor("nel.pet"), at(50)), + ReserveDecision::Renewed + ); + assert!( + reservations.holder_is(&repo("squid"), &actor("nel.pet"), at(140)), + "renewal pushed the expiry out from the second call instead of the first" + ); + } + + #[test] + fn the_per_actor_limit_bounds_one_account_without_touching_another() { + let reservations = Reservations::new( + ReservationTtl::new(3_600), + PerActorQuota::new(2), + GlobalQuota::new(64), + ); + assert_eq!( + reservations.try_reserve(&repo("a"), &actor("nel.pet"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("b"), &actor("nel.pet"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("c"), &actor("nel.pet"), at(0)), + ReserveDecision::PerActorFull, + "one account is held to its per-actor budget" + ); + assert_eq!( + reservations.try_reserve(&repo("c"), &actor("olaren.dev"), at(0)), + ReserveDecision::Fresh, + "different account keeps its own budget" + ); + } + + #[test] + fn the_global_limit_bounds_the_total_across_accounts() { + let reservations = Reservations::new( + ReservationTtl::new(3_600), + PerActorQuota::new(64), + GlobalQuota::new(2), + ); + assert_eq!( + reservations.try_reserve(&repo("a"), &actor("nel.pet"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("b"), &actor("olaren.dev"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("c"), &actor("teq.dev"), at(0)), + ReserveDecision::GlobalFull + ); + } + + #[test] + fn an_expired_reservation_is_pruned_and_frees_its_slot() { + let reservations = Reservations::new( + ReservationTtl::new(100), + PerActorQuota::new(8), + GlobalQuota::new(64), + ); + reservations.try_reserve(&repo("squid"), &actor("nel.pet"), at(0)); + assert!(reservations.prune(at(50)).is_empty(), "not yet expired"); + let pruned = reservations.prune(at(150)); + assert_eq!(pruned, vec![repo("squid")], "expired lease is reaped"); + assert_eq!( + reservations.try_reserve(&repo("squid"), &actor("olaren.dev"), at(160)), + ReserveDecision::Fresh, + "once the lease lapses a different account may claim DID" + ); + } +} diff --git a/knot2/crates/knot-xrpc/src/service.rs b/knot2/crates/knot-xrpc/src/service.rs new file mode 100644 index 0000000000000000000000000000000000000000..2e5cfaa853d284c47205ce3170aa9b40d533a278 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/service.rs @@ -0,0 +1,93 @@ +use std::sync::Arc; + +use axum::Json; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use serde::Serialize; + +use knot_capability::Capability; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::KnotId; + +use crate::XrpcState; +use crate::error::XrpcError; +pub(crate) const VERSION_ROUTE: &str = "/xrpc/org.tangled.knot.describeKnot"; +pub(crate) const RECEIVE_PACK_ROUTE: &str = "/xrpc/org.tangled.git.receivePack"; +pub(crate) const HEALTH_ROUTE: &str = "/xrpc/_health"; + +const WIRE_VERSION: &str = "v1.20.0"; + +#[derive(Serialize)] +struct VersionWire<'a> { + did: KnotId, + version: &'static str, + capabilities: Vec<&'static str>, + #[serde(flatten)] + description: &'a KnotDescription, +} + +/// The operator-supplied part of `org.tangled.knot.describeKnot`. +#[derive(Debug, Clone, Default, Serialize)] +pub struct KnotDescription { + pub links: Links, + pub contact: Contact, +} + +#[derive(Debug, Clone, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Links { + #[serde(skip_serializing_if = "Option::is_none")] + pub privacy_policy: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub terms_of_service: Option, +} + +#[derive(Debug, Clone, Default, Serialize)] +pub struct Contact { + pub email: String, +} + +#[derive(Serialize)] +struct HealthWire { + version: String, +} + +pub(crate) async fn health( + State(state): State>>, +) -> Response { + if !state.lfs.ready().await { + return ( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "lfs store is unreachable or not writable", + ) + .into_response(); + } + Json(HealthWire { + version: format!("knot {}", env!("CARGO_PKG_VERSION")), + }) + .into_response() +} + +pub(crate) async fn version( + State(state): State>>, +) -> Response { + Json(VersionWire { + did: state.knot_did.clone(), + version: WIRE_VERSION, + capabilities: Capability::served(state.legacy_admin_secret.is_some()) + .into_iter() + .map(|capability| capability.token()) + .collect(), + description: &state.description, + }) + .into_response() +} + +pub(crate) async fn receive_pack( + State(state): State>>, + headers: http::HeaderMap, + method: crate::Method, +) -> Result { + state.authenticate_write(&headers, &method).await?; + Ok(axum::http::StatusCode::OK.into_response()) +} diff --git a/knot2/crates/knot-xrpc/src/social.rs b/knot2/crates/knot-xrpc/src/social.rs new file mode 100644 index 0000000000000000000000000000000000000000..2fee388c0dc50ba5f0fddec87cc586529c31e742 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/social.rs @@ -0,0 +1,3289 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use cid::Cid; +use http::HeaderMap; +use jacquard_common::types::tid::Ticker; +use knot_acl::{KnotAcl, RepoAccess}; +use knot_cob::{ + Approved, ChangePayload, CobError, CobHome, CobStore, Materialized, Record, WriteContext, +}; +use knot_cobs::{ + Artifact, Comment, Creation, Editing, Erasure, LabelDef, LabelDefChange, Manifest, + ManifestChange, Opening, Patch, Reaction, SocialBody, SocialChange, SocialCob, SocialObject, + Ticket, TicketChange, +}; +use knot_git::{CommitRange, Repo}; +use knot_index::{Resolved, SocialEntry}; +use knot_record::Prose; +use knot_record::artifact::{ArtifactBlob, ArtifactRecord, artifact_collection}; +use knot_record::chain::{self, RecordOp}; +use knot_record::comment::{ + COMMIT_EMBED_TYPE, CommentMarkup, CommentRecord, CommentRef, comment_collection, +}; +use knot_record::label::{ + LabelColor, LabelDef as RecordLabelDef, LabelDefRecord, LabelName, LabelToken, + label_def_collection, +}; +use knot_record::manifest::{ + MANIFEST_RKEY, ManifestMeta, ManifestRecord, ManifestSource, RepoDescription, RepoTopics, + manifest_collection, +}; +use knot_record::pull::{PatchRecord, PatchSource, patch_collection}; +use knot_record::reaction::{ReactionKind, ReactionRecord, reaction_collection}; +use knot_record::ticket::{ + MAX_ASSIGNEES, MarkdownBody, TargetBranch, TicketRecord, TicketState, + TicketTitle as RecordTicketTitle, ticket_collection, +}; +use knot_resource::StructureBytes; +use knot_runtime::{Clock, HttpTransport, Signer}; +use knot_types::{ + AccountDid, AtUri, BranchName, ChangeId, CobId, ContributionPermission, ContributionPolicy, + Decision, MstKey, Oid, OwnerDid, RecordAddress, RecordBody, RecordCid, RecordCollection, + RecordRkey, RepoDid, RepoName, RepoRkey, ServiceDid, SubjectNumber, Tid, TypeName, UnixSeconds, +}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +use crate::error::XrpcError; +use crate::query::ValidatedQuery; +use crate::{XrpcState, admit_social, decode, project_collaborators, run_blocking}; + +pub const CREATE_RECORD_ROUTE: &str = "/xrpc/com.atproto.repo.createRecord"; +pub(crate) const PUT_RECORD_ROUTE: &str = "/xrpc/com.atproto.repo.putRecord"; +pub const DELETE_RECORD_ROUTE: &str = "/xrpc/com.atproto.repo.deleteRecord"; + +#[derive(Debug, Clone, Copy)] +pub enum RepoMethod { + Create, + Delete, +} + +impl RepoMethod { + pub fn route(self) -> &'static str { + match self { + Self::Create => CREATE_RECORD_ROUTE, + Self::Delete => DELETE_RECORD_ROUTE, + } + } +} + +pub(crate) const LIST_RECORD_VERSIONS_ROUTE: &str = "/xrpc/org.tangled.repo.listRecordVersions"; + +const NOT_STATE_AUTHORITY: &str = "State changes belong to the ticket's author or a maintainer"; +const NOT_ASSIGN_AUTHORITY: &str = "Assignees belong to a maintainer"; +const STILL_MOVING: &str = "This repository is still moving to this knot, so it can't take new issues or pulls yet. Please try again in a few minutes."; + +pub(crate) const COLLABORATE_REMEDY: &str = "Please ask the repo owner to add you as a collaborator, or to open the contribution policy to members or anyone, thanks."; + +pub struct RecordKeys(Mutex); + +impl Default for RecordKeys { + fn default() -> Self { + Self(Mutex::new(Ticker::new())) + } +} + +impl RecordKeys { + fn mint(&self) -> RecordRkey { + RecordRkey::minted( + self.0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .next(None), + ) + } +} + +#[derive(Deserialize)] +struct CreateRecordInput { + repo: RepoDid, + collection: RecordCollection, + #[serde(default)] + rkey: Option, + record: serde_json::Value, +} + +#[derive(Deserialize)] +struct SwapInput { + repo: RepoDid, + collection: RecordCollection, + rkey: RecordRkey, + #[serde(rename = "swapRecord", default)] + swap_record: Option, + #[serde(rename = "swapCommit", default)] + swap_commit: Option, +} + +impl SwapInput { + fn edit_of( + self, + editable: &RecordCollection, + ) -> Result<(RepoDid, RecordRkey, RecordCid), XrpcError> { + if self.swap_commit.is_some() { + return Err(no_swap_commit()); + } + if &self.collection != editable { + return Err(rejected_collection(&self.collection)); + } + let from = self + .swap_record + .and_then(|swap| swap.record()) + .ok_or_else(|| { + XrpcError::invalid_request( + "swapRecord must be the dag-cbor CID of a record version", + ) + })?; + Ok((self.repo, self.rkey, from)) + } +} + +#[derive(Deserialize)] +struct PutRecordInput { + #[serde(flatten)] + swap: SwapInput, + record: serde_json::Value, +} + +fn rejected_collection(collection: &RecordCollection) -> XrpcError { + XrpcError::invalid_request(format!( + "{collection} isn't a collection this knot takes contributed records for" + )) +} + +fn no_swap_commit() -> XrpcError { + XrpcError::invalid_request( + "this knot compares the record version in swapRecord and never the commit in swapCommit", + ) +} + +fn typed( + record: serde_json::Value, + collection: &RecordCollection, +) -> Result { + serde_json::from_value(record).map_err(|error| { + XrpcError::invalid_request(format!("Record for {collection} is unusable: {error}")) + }) +} + +fn kind_matches( + declared: Option<&TypeName>, + collection: &RecordCollection, +) -> Result<(), XrpcError> { + match declared { + Some(declared) if declared.as_str() != collection.as_str() => { + Err(XrpcError::invalid_request(format!( + "the record's $type is {declared}, and this collection is {collection}" + ))) + } + _ => Ok(()), + } +} + +fn prose(title: String, body: Option) -> Result { + let unusable = |what: &'static str| { + move |error| XrpcError::invalid_request(format!("{what} is unusable: {error}")) + }; + Ok(Prose::new( + RecordTicketTitle::new(title).map_err(unusable("title"))?, + body.map(MarkdownBody::new) + .transpose() + .map_err(unusable("body"))?, + )) +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum TicketBody { + Text(String), + Markup(MarkupInput), +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum TicketTitle { + Text(String), + Inline(InlineTitle), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct InlineTitle { + #[serde(rename = "$type", default)] + kind: Option, + text: String, +} + +impl TicketTitle { + fn parse(self) -> Result { + match self { + Self::Text(text) => Ok(text), + Self::Inline(inline) => { + if let Some(declared) = &inline.kind + && declared != knot_record::ticket::INLINE_MARKUP + { + return Err(XrpcError::invalid_request(format!( + "a ticket's title is {}, and this one declares {declared}", + knot_record::ticket::INLINE_MARKUP + ))); + } + Ok(inline.text) + } + } + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TicketDraft { + #[serde(rename = "$type", default)] + kind: Option, + #[serde(default)] + repo: Option, + title: TicketTitle, + #[serde(default)] + body: Option, + #[serde(default)] + state: Option, + #[serde(rename = "tangledLabels", default)] + tangled_labels: Option>, + #[serde(default)] + assignees: Option>, + #[serde(rename = "targetBranch", default)] + target_branch: Option, + #[serde(default)] + labels: Option, + #[serde(rename = "createdAt")] + _created_at: serde::de::IgnoredAny, + #[serde(rename = "x-tngl-editor", default)] + _editor: Option, +} + +struct ParsedTicket { + prose: Prose, + state: Option, + tangled_labels: Option>, + assignees: Option>, + target_branch: Option, + blobs: Option>, +} + +impl TicketDraft { + fn parse( + self, + written_to: &RepoDid, + collection: &RecordCollection, + ) -> Result { + kind_matches(self.kind.as_ref(), collection)?; + if self.labels.is_some() { + return Err(XrpcError::invalid_request( + "labels are atproto self-labels this knot doesn't store; please leave labels out", + )); + } + if let Some(echoed) = &self.repo { + on_this_knot(echoed, written_to, "record")?; + } + let title = self.title.parse()?; + let target_branch = self.target_branch; + let assignees = self.assignees.filter(|assignees| !assignees.is_empty()); + if let Some(assignees) = &assignees { + if assignees.len() > MAX_ASSIGNEES { + return Err(XrpcError::invalid_request(format!( + "{} assignees are over the {MAX_ASSIGNEES} limit", + assignees.len() + ))); + } + if assignees.iter().collect::>().len() != assignees.len() { + return Err(XrpcError::invalid_request("assignees lists someone twice")); + } + } + let (body, blobs) = match self.body { + Some(TicketBody::Text(text)) => (Some(text), None), + Some(TicketBody::Markup(markup)) => { + let markup = markup.parse("Ticket body's")?; + let attachments = markup.attachments(); + let blobs = (!attachments.is_empty()).then(|| attachments.to_vec()); + (Some(markup.text().as_str().to_owned()), blobs) + } + None => (None, None), + }; + Ok(ParsedTicket { + prose: prose(title, body)?, + state: self.state, + tangled_labels: self.tangled_labels, + assignees, + target_branch, + blobs, + }) + } +} + +#[derive(Deserialize)] +struct StrongRefInput { + uri: AtUri, + cid: StrongCid, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum StrongCid { + Bare(crate::out::CidString), + Linked { + #[serde(rename = "$link")] + link: crate::out::CidString, + }, +} + +impl StrongCid { + fn cid(self) -> crate::out::CidString { + match self { + Self::Bare(cid) | Self::Linked { link: cid } => cid, + } + } +} + +impl StrongRefInput { + fn record(self, what: &'static str) -> Result { + self.cid.cid().record().ok_or_else(|| { + XrpcError::invalid_request(format!( + "a record CID is dag-cbor sha2-256, and {what}'s strongRef cid isn't" + )) + }) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct MarkupInput { + #[serde(rename = "$type", default)] + kind: Option, + text: String, + #[serde(default)] + blobs: Option>, +} + +impl MarkupInput { + fn parse(self, whose: &'static str) -> Result { + kind_matches( + self.kind.as_ref(), + &knot_record::comment::markup_collection(), + )?; + let unusable = |what: &'static str| { + move |error| XrpcError::invalid_request(format!("{whose} {what} is unusable: {error}")) + }; + let bare = + self.text.trim().is_empty() && self.blobs.as_ref().is_none_or(|blobs| blobs.is_empty()); + if bare { + return Err(XrpcError::invalid_request(format!( + "{whose} text is unusable: empty, with no attachments" + ))); + } + let text = MarkdownBody::new(self.text).map_err(unusable("text"))?; + Ok(CommentMarkup::new(text, self.blobs)) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct CommentDraft { + #[serde(rename = "$type", default)] + kind: Option, + subject: StrongRefInput, + #[serde(rename = "replyTo", default)] + reply_to: Option, + body: MarkupInput, + #[serde(default)] + embed: Option, + #[serde(rename = "createdAt")] + _created_at: serde::de::IgnoredAny, + #[serde(rename = "x-tngl-editor", default)] + _editor: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct CommitEmbedInput { + #[serde(rename = "$type", default)] + kind: Option, + repo: RepoDid, + commit: GitOidInput, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct GitOidInput { + #[serde(rename = "$type", default)] + _kind: Option, + oid: Oid, +} + +struct ParsedComment { + subject: CommentRef, + reply_to: Option, + embed: Option, + markup: CommentMarkup, +} + +impl CommentDraft { + fn parse(self, written_to: &RepoDid) -> Result { + kind_matches(self.kind.as_ref(), &comment_collection())?; + let embed = self + .embed + .map(|embed| -> Result { + if let Some(kind) = embed.kind.filter(|kind| kind.as_str() != COMMIT_EMBED_TYPE) { + return Err(XrpcError::invalid_request(format!( + "this knot takes {COMMIT_EMBED_TYPE} as a comment embed, and the embed's \ + $type is {kind}" + ))); + } + on_this_knot(&embed.repo, written_to, "embedded commit")?; + Ok(embed.commit.oid) + }) + .transpose()?; + let subject = any_target(&self.subject.uri)?; + if subject.address.collection() != &ticket_collection() + && subject.address.collection() != &patch_collection() + { + return Err(XrpcError::invalid_request(format!( + "this knot takes tickets and patches as comment subjects; the uri's collection is {}", + subject.address.collection() + ))); + } + on_this_knot(&subject.repo, written_to, "subject")?; + let reply_to = self + .reply_to + .map(|reply| -> Result { + let target = target(&reply.uri, &comment_collection())?; + on_this_knot(&target.repo, written_to, "reply")?; + Ok(CommentRef::new(target.address, reply.record("the reply")?)) + }) + .transpose()?; + Ok(ParsedComment { + subject: CommentRef::new(subject.address, self.subject.record("the subject")?), + reply_to, + embed, + markup: self.body.parse("Comment body's")?, + }) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ReactionDraft { + #[serde(rename = "$type", default)] + kind: Option, + subject: StrongRefInput, + reaction: ReactionKind, + #[serde(rename = "createdAt")] + _created_at: serde::de::IgnoredAny, +} + +impl ReactionDraft { + fn parse(self, written_to: &RepoDid) -> Result<(CommentRef, ReactionKind), XrpcError> { + kind_matches(self.kind.as_ref(), &reaction_collection())?; + let subject = any_target(&self.subject.uri)?; + on_this_knot(&subject.repo, written_to, "subject")?; + Ok(( + CommentRef::new(subject.address, self.subject.record("the subject")?), + self.reaction, + )) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ArtifactDraft { + #[serde(rename = "$type", default)] + kind: Option, + subject: GitOidInput, + name: String, + artifact: ArtifactBlob, + #[serde(rename = "createdAt")] + _created_at: serde::de::IgnoredAny, +} + +impl ArtifactDraft { + fn parse(self) -> Result<(Oid, String, ArtifactBlob), XrpcError> { + kind_matches(self.kind.as_ref(), &artifact_collection())?; + Ok((self.subject.oid, self.name, self.artifact)) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct LabelDefDraft { + #[serde(rename = "$type", default)] + kind: Option, + name: LabelName, + #[serde(default)] + color: Option, + #[serde(rename = "createdAt")] + _created_at: serde::de::IgnoredAny, + #[serde(rename = "x-tngl-editor", default)] + _editor: Option, +} + +impl LabelDefDraft { + fn parse(self, collection: &RecordCollection) -> Result { + kind_matches(self.kind.as_ref(), collection)?; + Ok(RecordLabelDef::new(self.name, self.color)) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct GrantDraft { + #[serde(rename = "$type", default)] + kind: Option, + #[serde(rename = "createdAt", default)] + _created_at: Option, +} + +impl GrantDraft { + fn parse(self, collection: &RecordCollection) -> Result<(), XrpcError> { + kind_matches(self.kind.as_ref(), collection) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct OidDraft { + oid: Oid, +} + +struct ParsedPatch { + ticket: RecordRkey, + range: CommitRange, + source: Option, +} + +fn on_this_knot(repo: &RepoDid, written_to: &RepoDid, what: &'static str) -> Result<(), XrpcError> { + if repo == written_to { + return Ok(()); + } + Err(XrpcError::invalid_request(format!( + "the write is for {written_to}, but the {what}'s repo is {repo}" + ))) +} + +#[derive(Serialize)] +enum ValidationStatus { + #[serde(rename = "valid")] + Valid, +} + +#[derive(Serialize, Clone)] +struct CommitHead { + cid: crate::out::CidString, + rev: Tid, +} + +#[derive(Serialize)] +struct Written { + uri: AtUri, + cid: crate::out::CidString, + commit: CommitHead, + #[serde(rename = "validationStatus")] + validation_status: ValidationStatus, + // missing when the answer replays a version instead of writing one + #[serde(rename = "x-tngl-change", skip_serializing_if = "Option::is_none")] + change: Option, +} + +impl Written { + fn by(self, change: ChangeId) -> Self { + Self { + change: Some(change), + ..self + } + } +} + +#[derive(Serialize)] +struct Deleted { + commit: CommitHead, +} + +fn wrote(repo: &RepoDid, address: &RecordAddress, cid: RecordCid, head: &CommitHead) -> Written { + Written { + uri: address.at_uri(repo), + cid: cid.into(), + commit: head.clone(), + validation_status: ValidationStatus::Valid, + change: None, + } +} + +struct Target { + repo: RepoDid, + address: RecordAddress, +} + +fn target(uri: &AtUri, collection: &RecordCollection) -> Result { + let target = any_target(uri)?; + if target.address.collection() != collection { + return Err(XrpcError::invalid_request(format!( + "this procedure takes {collection}; the uri's collection is {}", + target.address.collection() + ))); + } + Ok(target) +} + +fn any_target(uri: &AtUri) -> Result { + let (repo, address) = RecordAddress::parse_at_uri(uri) + .map_err(|error| XrpcError::invalid_request(error.to_string()))?; + Ok(Target { repo, address }) +} + +fn ready(resolved: Resolved, projection: &'static str) -> Result { + match resolved { + Resolved::Ready(value) => Ok(value), + Resolved::Warming => Err(XrpcError::warming(format!( + "{projection} projection is still warming, please retry shortly" + ))), + } +} + +pub(crate) async fn permission( + state: &XrpcState, + actor: &AccountDid, + repo: &RepoDid, +) -> Result { + let policy = ready(state.index.contribution_policy(repo), "registry")? + .ok_or_else(|| XrpcError::not_found("No such repository on this knot"))?; + project_collaborators(state, repo).await?; + let collaboration = state.index.collaboration_consent(repo, actor); + let membership = state.index.member_status(actor); + let access = RepoAccess::of(&collaboration, &membership, &policy) + .ok_or_else(|| XrpcError::internal("Consents answered with mismatched subjects"))?; + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + ready(knot_acl::permission_of(&acl, &access), "grant list") +} + +async fn warm_social( + state: &XrpcState, + repo: &RepoDid, +) -> Result<(), XrpcError> { + let index = Arc::clone(&state.index); + let target = repo.clone(); + run_blocking(move || index.ensure_social(&target).map_err(XrpcError::from)).await +} + +fn next_number( + state: &XrpcState, + repo: &RepoDid, + collection: &RecordCollection, +) -> Result { + ready(state.index.next_subject_number(repo, collection), "social")? + .ok_or_else(|| XrpcError::internal("This repository has numbered past what fits")) +} + +fn object_at( + state: &XrpcState, + repo: &RepoDid, + address: &RecordAddress, +) -> Result { + ready(state.index.social_object_at(repo, address), "social")? + .ok_or_else(|| XrpcError::not_found(format!("no record at {}", address.at_uri(repo)))) +} + +fn rejected(error: CobError) -> XrpcError { + match error { + CobError::CreateRejected { permission, .. } + if !permission.policy().allows(permission.role()) => + { + let sentence = error.to_string(); + if permission.moderates().is_allowed() { + XrpcError::forbidden(sentence) + } else { + XrpcError::forbidden(format!("{sentence}. {COLLABORATE_REMEDY}")) + } + } + other => other.into(), + } +} + +fn drifted(error: &chain::ChainError) -> bool { + matches!( + error, + chain::ChainError::RecordMoved { .. } + | chain::ChainError::RecordExists { .. } + | chain::ChainError::RecordAbsent { .. } + ) +} + +struct Ledger<'a, H, C> { + state: &'a Arc>, + git: &'a Repo, + repo: &'a RepoDid, + signer: &'a dyn Signer, + now: UnixSeconds, +} + +impl Ledger<'_, H, C> { + fn charge_for(&self, adding: StructureBytes, replaced: Option) -> Result<(), XrpcError> { + let current = ready(self.state.index.social_serving_bytes(self.repo), "social")?; + let freed = replaced + .map(|object| { + ready( + self.state.index.social_serving_bytes_of(self.repo, object), + "social", + ) + }) + .transpose()? + .flatten(); + admit_social( + self.state.social_budget, + knot_resource::SocialCharge { + current: StructureBytes::new( + current + .get() + .saturating_sub(freed.map_or(0, StructureBytes::get)), + ), + adding, + }, + ) + } + + fn publish(&self, address: &RecordAddress, op: RecordOp) -> Result { + let mut emit = + crate::firehose::FirehoseEmit::new(&self.state.firehose, self.repo, self.now); + let applied = futures::executor::block_on(chain::apply( + self.git, + self.repo, + &[(address.clone(), op)], + self.signer, + &mut emit, + self.now, + )); + match applied { + Ok(chunks) => { + emit.finish()?; + self.head_of(&chunks) + } + Err(drift) if drifted(&drift) => { + drop(emit); + tracing::warn!( + repo = %self.repo, + uri = %address.at_uri(self.repo), + %drift, + "Chain stood where the projection didn't expect it, so the write reconciled the chain to the projection before answering" + ); + reconcile(self.state, self.git, self.repo, self.signer, self.now) + .and_then(|_| self.head()) + } + Err(other) => Err(other.into()), + } + } + + fn head_of(&self, chunks: &[chain::CommitChunk]) -> Result { + match chunks.last() { + Some(chunk) => Ok(CommitHead { + cid: chain::commit_cid(&chunk.commit)?.into(), + rev: chunk.rev.clone(), + }), + None => self.head(), + } + } + + fn head(&self) -> Result { + let tip = chain::tip(self.git)?.ok_or_else(|| { + XrpcError::internal("A repository that took a social write has a chain tip") + })?; + Ok(CommitHead { + cid: chain::commit_cid(&chain::commit_bytes(self.git, &tip)?)?.into(), + rev: tip.rev.clone(), + }) + } + + fn placed(body: &RecordBody) -> (Record, RecordBody) { + (Record::version(body.clone()), body.clone()) + } + + fn settled( + &self, + object: CobId, + type_name: &knot_types::TypeName, + work: impl FnOnce() -> Result, + ) -> Result { + work() + .and_then(|done| { + self.state + .index + .refresh_social_object(self.repo, type_name, object) + .map_err(XrpcError::from)?; + Ok(done) + }) + .inspect_err(|error| self.repair(error)) + } + + fn repair(&self, cause: &XrpcError) { + match reconcile(self.state, self.git, self.repo, self.signer, self.now) { + Ok(reconciled) => tracing::warn!( + repo = %self.repo, + %cause, + commits = reconciled.commits, + "A social write failed after its change was committed, so the chain was \ + reconciled to the projection" + ), + Err(error) => tracing::error!( + repo = %self.repo, + %cause, + %error, + "A social write failed after its change was committed and the reconciler failed too; the next restart reconciles" + ), + } + } + + fn write( + &self, + cx: &WriteContext<'_>, + address: &RecordAddress, + change: SocialChange, + doing: Doing<'_>, + ) -> Result<(CobId, ChangeId, CommitHead), XrpcError> { + let (record, op, amending, replaced, verb) = match doing { + Doing::Open { body } => { + let (record, placed) = Self::placed(body); + ( + record, + RecordOp::Create { record: placed }, + None, + None, + None, + ) + } + Doing::Edit { object, from, body } => { + let (record, placed) = Self::placed(body); + ( + record, + RecordOp::Update { + record: placed, + from, + }, + Some(object), + Some(object), + Some("edit"), + ) + } + Doing::Erase { object, from } => ( + Record::Tombstone, + RecordOp::Delete { from }, + Some(object), + None, + Some("erase"), + ), + }; + let adding = match &op { + RecordOp::Delete { .. } => None, + RecordOp::Create { .. } | RecordOp::Update { .. } => Some(StructureBytes::new( + record.bytes().map_or(0, <[u8]>::len) as u64, + )), + }; + let approved = Approved::signed_by(cx, Materialized { change, record })?; + if let Some(adding) = adding { + self.charge_for(adding, replaced)?; + } + let store = CobStore::new(self.git); + let (object, tip) = match amending { + None => store + .create_approved::>( + &CobHome::from(self.repo), + &approved, + self.signer, + self.now, + ) + .map_err(rejected) + .map(|created| (created.object, created.tip))?, + Some(object) => ( + object, + store + .append_approved::>( + &CobHome::from(self.repo), + object, + &approved, + self.signer, + self.now, + ) + .map_err(|error| match (verb, error) { + (Some(verb), CobError::AppendRejected { permission, .. }) => { + append_denial(B::OPENING, verb, address.collection(), permission) + } + (_, other) => other.into(), + })?, + ), + }; + let head = self.settled(object, &SocialChange::::type_name(), || { + self.publish(address, op) + })?; + Ok((object, tip, head)) + } +} + +pub(crate) struct Reconciled { + pub(crate) commits: usize, +} +fn collection_type(collection: &RecordCollection) -> knot_types::TypeName { + knot_types::TypeName::new(collection.as_str()).expect("Record collection is a valid type name") +} + +pub(crate) fn tree_body( + store: &CobStore, + type_name: &knot_types::TypeName, + object: CobId, + version: RecordCid, +) -> Result, XrpcError> { + let delta = store.changes_of(type_name, object, knot_cob::HistoryModel::Convergent, None)?; + Ok(delta + .changes + .into_iter() + .rev() + .filter_map(|change| change.record.into_body()) + .find(|body| body.cid() == version)) +} + +fn resolve_due( + store: &CobStore, + type_name: &knot_types::TypeName, + object: CobId, + version: RecordCid, + leaf: Option, + context: Option<(&RepoDid, &RecordAddress)>, +) -> Result, XrpcError> { + if leaf == Some(version.cid()) { + return Ok(Some(chain::Due::Keep(version))); + } + match tree_body(store, type_name, object, version)? { + Some(body) => Ok(Some(chain::Due::Write(body))), + None => { + if let Some((repo, address)) = context { + tracing::error!( + repo = %repo, + uri = %address.at_uri(repo), + %version, + "A live social record is missing body in the literal actual change tree; the chain will keep the live leaf and leave body out" + ); + } + Ok(leaf.and_then(RecordCid::new).map(chain::Due::Keep)) + } + } +} + +#[derive(Default)] +struct Kept { + due: BTreeMap, + extras: BTreeMap, +} + +impl Kept { + fn taking(mut self, key: MstKey, due: chain::Due) -> Self { + self.due.insert(key, due); + self + } + + fn holding(mut self, cid: Cid, bytes: Bytes) -> Self { + self.extras.insert(cid, bytes); + self + } +} + +impl chain::Records for Kept { + fn due(&self) -> &BTreeMap { + &self.due + } + + fn extra_blocks(&self) -> BTreeMap { + self.extras.clone() + } +} + +pub(crate) struct Converted { + pub(crate) version: crate::foster::SeededVersion, + pub(crate) body: RecordBody, + pub(crate) settled: knot_cobs::Settled, +} + +pub(crate) fn convert_seeded( + git: &Repo, + repo: &RepoDid, + signer: &dyn Signer, +) -> Result, XrpcError> { + let store = CobStore::new(git); + let home = CobHome::from(repo); + let seeded = crate::foster::seeded_versions(&store)?; + let seed_type = crate::foster::seed_type_name(); + let mut settled = Vec::with_capacity(seeded.len()); + let mut numbers: BTreeMap = BTreeMap::new(); + let mut spent: BTreeMap> = + BTreeMap::new(); + let mut homes: BTreeMap = BTreeMap::new(); + for (type_name, object) in store.cobs()? { + if let Some(address) = knot_cobs::social_root_address(&store, &type_name, object)? { + homes.entry(address).or_insert(object); + } + if let Some((state, _)) = knot_cobs::social_state(&store, &type_name, object)? + && let Some(live) = state.object() + { + spent + .entry(live.address().collection().clone()) + .or_default() + .insert(live.number(), live.address().clone()); + } + } + for version in seeded { + let seeds = crate::foster::seeds_of(&store, version.object)?; + let first = seeds + .first() + .ok_or_else(|| XrpcError::internal("a seeded record has no seed"))?; + let (author, created_at, seeded_number) = + (first.author.clone(), first.created_at, first.number); + let first_body = tree_body(&store, &seed_type, version.object, first.produces)? + .ok_or_else(|| XrpcError::internal("a seeded record has no body"))?; + let edits = seeds + .windows(2) + .map(|pair| { + let body = tree_body(&store, &seed_type, version.object, pair[1].produces)? + .ok_or_else(|| XrpcError::internal("a seeded record has no body"))?; + Ok(( + Editing { + author: pair[1].author.clone(), + from: pair[0].produces, + produces: pair[1].produces, + edited_at: pair[1].created_at, + }, + body, + )) + }) + .collect::, XrpcError>>()?; + let body = tree_body(&store, &seed_type, version.object, version.produces)? + .ok_or_else(|| XrpcError::internal("a seeded record has no body"))?; + let taken = spent + .entry(version.address.collection().clone()) + .or_default(); + let number = match seeded_number + .filter(|number| taken.get(number).is_none_or(|at| at == &version.address)) + { + _ if version.address.collection() == &ticket_collection() => { + let rkey = version.address.rkey().as_str(); + let number = rkey + .parse() + .ok() + .and_then(SubjectNumber::new) + .filter(|number| number.get().to_string() == rkey) + .ok_or_else(|| { + XrpcError::bad_gateway(format!( + "a seeded ticket's rkey {rkey} isn't its subject number" + )) + })?; + if taken.get(&number).is_some_and(|at| at != &version.address) { + return Err(XrpcError::conflict(format!( + "a seeded ticket at rkey {rkey} wants {number}, which is already spent" + ))); + } + number + } + Some(number) => number, + None => { + let mut number = numbers + .get(version.address.collection()) + .copied() + .unwrap_or(SubjectNumber::FIRST); + while taken.contains_key(&number) { + number = number.after().ok_or_else(|| { + XrpcError::internal("a collection has numbered past what fits") + })?; + } + numbers.insert( + version.address.collection().clone(), + number.after().ok_or_else(|| { + XrpcError::internal("a collection has numbered past what fits") + })?, + ); + number + } + }; + taken.insert(number, version.address.clone()); + let collection = version.address.collection().clone(); + let parent = seeded_parent(&version.address, repo, &first_body)?; + let creation = Creation { + address: version.address.clone(), + author, + number, + parent, + created_at, + produces: first.produces, + }; + let outcome = knot_cobs::settle_stored( + &store, + &home, + repo, + signer, + &collection, + homes.get(&version.address).copied(), + creation, + first_body, + edits, + )? + .ok_or_else(|| { + XrpcError::bad_gateway(format!( + "the foster knot serves {collection} under an unknown history model" + )) + })?; + settled.push(Converted { + version, + body, + settled: outcome, + }); + } + Ok(settled) +} +fn seeded_parent( + address: &RecordAddress, + repo: &RepoDid, + body: &RecordBody, +) -> Result, XrpcError> { + match address.collection() { + collection + if *collection == comment_collection() || *collection == reaction_collection() => + { + let subject = knot_cobs::stored_subject(body).map_err(|error| { + XrpcError::bad_gateway(format!( + "the foster knot serves {} whose subject won't decode: {error}", + address.at_uri(repo) + )) + })?; + Ok(Some(subject.address().clone())) + } + _ => Ok(None), + } +} + +pub(crate) fn reconcile( + state: &Arc>, + git: &Repo, + repo: &RepoDid, + signer: &dyn Signer, + now: UnixSeconds, +) -> Result { + state.index.refresh_social(repo)?; + let entries = ready(state.index.social_entries(repo), "social")?; + let serving: Vec<(&SocialEntry, RecordCid)> = entries + .iter() + .filter_map(|entry| entry.state.serving().map(|version| (entry, version))) + .collect(); + let store = CobStore::new(git); + let leaves = futures::executor::block_on(chain::leaves(git))?; + let mut collections: Vec = knot_cobs::social_collections().to_vec(); + let mut records = serving + .iter() + .try_fold(Kept::default(), |kept, (entry, version)| { + let key = entry.address.mst_key(); + let leaf = leaves.get(&key).copied(); + let due = resolve_due( + &store, + &collection_type(entry.address.collection()), + entry.object, + *version, + leaf, + Some((repo, &entry.address)), + )?; + Ok::(match due { + Some(due) => kept.taking(key, due), + None => kept, + }) + })?; + let (fostered, extras) = fostered_state(&store, git, &leaves)?; + let missing: Vec = fostered + .iter() + .map(|(_, collection, _)| collection.clone()) + .filter(|collection| !collections.contains(collection)) + .collect(); + collections.extend(missing); + let erased: Vec = entries + .iter() + .filter(|entry| entry.state.erased()) + .map(|entry| entry.address.mst_key()) + .collect(); + let unheld: Vec<_> = fostered + .into_iter() + .filter(|(key, _, _)| !records.due.contains_key(key) && !erased.contains(key)) + .collect(); + records = unheld + .into_iter() + .fold(records, |records, (key, _, due)| records.taking(key, due)); + records = extras + .into_iter() + .fold(records, |records, (cid, bytes)| records.holding(cid, bytes)); + if chain::tip(git)?.is_none() && chain::Records::due(&records).is_empty() { + return Ok(Reconciled { commits: 0 }); + } + let mut emit = crate::firehose::FirehoseEmit::new(&state.firehose, repo, now); + let chunks = futures::executor::block_on(chain::settle( + git, + repo, + &collections, + &records, + signer, + &mut emit, + now, + ))?; + emit.finish()?; + Ok(Reconciled { + commits: chunks.len(), + }) +} + +type FosteredDue = (MstKey, RecordCollection, chain::Due); +type FosteredState = (Vec, BTreeMap); +type FosteredGraph = (Option, BTreeMap); + +fn fostered_state( + store: &CobStore, + git: &Repo, + leaves: &BTreeMap, +) -> Result { + let seed_type = crate::foster::seed_type_name(); + let held = chain::tip(git)?.map(|tip| knot_record::blocks::GitBlocks::at(git, tip.blocks)); + let graphs = crate::foster::seeded_versions(store)? + .into_iter() + .map(|version| fostered_graph(store, &seed_type, held.as_ref(), leaves, version)) + .collect::, XrpcError>>()?; + Ok(graphs.into_iter().fold( + (Vec::new(), BTreeMap::new()), + |(mut dues, mut extras), (due, graph_extras)| { + dues.extend(due); + extras.extend(graph_extras); + (dues, extras) + }, + )) +} + +fn fostered_graph( + store: &CobStore, + seed_type: &knot_types::TypeName, + held: Option<&knot_record::blocks::GitBlocks>, + leaves: &BTreeMap, + version: crate::foster::SeededVersion, +) -> Result { + let key = version.address.mst_key(); + let bodies: Vec = store + .changes_of( + seed_type, + version.object, + knot_cob::HistoryModel::Convergent, + None, + )? + .changes + .into_iter() + .filter_map(|change| change.record.into_body()) + .collect(); + let leaf = leaves.get(&key).copied(); + let due = if leaf == Some(version.produces.cid()) { + Some(chain::Due::Keep(version.produces)) + } else { + bodies + .iter() + .find(|body| body.cid() == version.produces) + .map(|body| chain::Due::Write(body.clone())) + .or_else(|| leaf.and_then(RecordCid::new).map(chain::Due::Keep)) + }; + let extras = bodies + .into_iter() + .filter(|body| body.cid() != version.produces) + .map(|body| (body.cid().cid(), Bytes::copy_from_slice(body.as_bytes()))) + .filter(|(cid, _)| !already_held(held, cid)) + .collect(); + Ok(( + due.map(|due| (key, version.address.collection().clone(), due)), + extras, + )) +} + +fn already_held(held: Option<&knot_record::blocks::GitBlocks>, cid: &Cid) -> bool { + use jacquard_repo::BlockStore as _; + held.is_some_and(|blocks| { + futures::executor::block_on(async { blocks.has(cid).await }).unwrap_or(false) + }) +} + +enum Doing<'a> { + Open { + body: &'a RecordBody, + }, + Edit { + object: CobId, + from: RecordCid, + body: &'a RecordBody, + }, + Erase { + object: CobId, + from: RecordCid, + }, +} + +#[derive(Clone)] +pub(crate) struct Session { + pub(crate) actor: AccountDid, + pub(crate) repo: RepoDid, + permission: ContributionPermission, + pub(crate) now: UnixSeconds, +} + +impl Session { + pub(crate) fn permission(&self) -> ContributionPermission { + self.permission + } + + pub(crate) fn require_contribution(self, act: &'static str) -> Result { + match self.permission.contributes() { + Decision::Allow => Ok(Contributor { session: self }), + Decision::Deny => Err(XrpcError::forbidden(format!( + "`{act}` belong to contributors this repo's policy allows, and it leaves \ + you out; {COLLABORATE_REMEDY}" + ))), + } + } +} + +pub(crate) struct Contributor { + pub(crate) session: Session, +} + +fn require_native( + state: &XrpcState, + repo: &RepoDid, +) -> Result<(), XrpcError> { + if ready(state.index.is_fostered(repo), "registry")? { + return Err(XrpcError::forbidden( + "this repo is only fostered here. its records change on its origin knot", + )); + } + Ok(()) +} + +pub(crate) async fn session( + state: &XrpcState, + headers: &HeaderMap, + method: &crate::Method, + repo: RepoDid, +) -> Result { + let actor = state + .authenticate_write_for_repo(headers, method, &repo) + .await?; + require_native(state, &repo)?; + let permission = permission(state, &actor, &repo).await?; + Ok(Session { + actor, + repo, + permission, + now: state.now(), + }) +} + +/// The manifest's settings write runs in more than one ledger pass, so the +/// critical section lives here and `on_repo` keeps answering with a body. +async fn in_ledger( + state: &Arc>, + session: Session, + work: F, +) -> Result +where + H: HttpTransport, + C: Clock, + T: Send + 'static, + F: FnOnce(&Ledger<'_, H, C>, &WriteContext<'_>) -> Result + Send + 'static, +{ + let signer = state.secrets.signer(&state.knot_did)?; + let state = Arc::clone(state); + let Session { + actor, + repo, + permission, + now, + } = session; + run_blocking(move || { + let _guard = state.cob_locks.repo(&repo); + let _guard = state.index.guard_social(&repo)?; + let git = state.layout.open(&repo)?; + let ledger = Ledger { + state: &state, + git: &git, + repo: &repo, + signer: &signer, + now, + }; + let cx = WriteContext::new(&actor, &repo, permission); + work(&ledger, &cx) + }) + .await +} + +async fn on_repo( + state: &Arc>, + session: Session, + work: F, +) -> Result +where + H: HttpTransport, + C: Clock, + T: Serialize + Send + 'static, + F: FnOnce(&Ledger<'_, H, C>, &WriteContext<'_>) -> Result + Send + 'static, +{ + in_ledger(state, session, work) + .await + .map(|written| Json(written).into_response()) +} + +pub(crate) async fn create_record( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let CreateRecordInput { + repo, + collection, + rkey, + record, + } = decode::(&body)?; + if rkey.is_some() + && [ + ticket_collection(), + comment_collection(), + reaction_collection(), + artifact_collection(), + ] + .contains(&collection) + { + return Err(XrpcError::invalid_request(format!( + "this knot mints the record key for {collection}; please leave rkey out" + ))); + } + if collection == ticket_collection() { + ticket_open(&state, &headers, &method, repo, record).await + } else if collection == comment_collection() { + comment_open(&state, &headers, &method, repo, record).await + } else if collection == reaction_collection() { + reaction_open(&state, &headers, &method, repo, record).await + } else if collection == label_def_collection() { + label_def_open(&state, &headers, &method, repo, rkey, record).await + } else if collection == artifact_collection() { + artifact_open(&state, &headers, &method, repo, record).await + } else if grant_of(&collection).is_some() { + grant_open(&state, &headers, &method, repo, collection, rkey, record).await + } else { + Err(rejected_collection(&collection)) + } +} + +pub(crate) async fn open_manifest( + state: &Arc>, + actor: AccountDid, + repo: RepoDid, + record: RecordBody, + now: UnixSeconds, +) -> Result<(), XrpcError> { + let permission = permission(state, &actor, &repo).await?; + let session = Session { + actor, + repo, + permission, + now, + }; + let address = RecordAddress::new( + manifest_collection(), + RecordRkey::new(MANIFEST_RKEY).expect("Manifest rkey is a valid record key"), + ); + open_at(state, session, address, None, record, Manifest, |_, _| { + Ok(None) + }) + .await + .map(drop) +} + +pub(crate) async fn replace_manifest( + state: &Arc>, + actor: AccountDid, + repo: RepoDid, + record: RecordBody, + now: UnixSeconds, +) -> Result<(), XrpcError> { + let permission = permission(state, &actor, &repo).await?; + let signer = state.secrets.signer(&state.knot_did)?; + let state = Arc::clone(state); + run_blocking(move || { + let _repo_guard = state.cob_locks.repo(&repo); + let _social_guard = state.index.guard_social(&repo)?; + let git = state.layout.open(&repo)?; + let ledger = Ledger { + state: &state, + git: &git, + repo: &repo, + signer: &signer, + now, + }; + let cx = WriteContext::new(&actor, &repo, permission); + let address = RecordAddress::new( + manifest_collection(), + RecordRkey::new(MANIFEST_RKEY).expect("Manifest rkey is a valid record key"), + ); + let (object, live) = standing(&ledger, &address)?; + let from = live + .version() + .ok_or_else(|| XrpcError::internal("The live manifest is missing its version"))?; + let produces = record.cid(); + if from == produces { + return Ok(()); + } + let change: ManifestChange = SocialChange::Edit { + editing: Editing { + author: actor.clone(), + from, + produces, + edited_at: now, + }, + body: Manifest, + }; + ledger.write( + &cx, + &address, + change, + Doing::Edit { + object, + from, + body: &record, + }, + )?; + Ok(()) + }) + .await +} + +pub(crate) fn declared_policy( + state: &Arc>, + git: &Repo, + repo: &RepoDid, + signer: &dyn Signer, +) -> Result, XrpcError> { + let ledger = Ledger { + state, + git, + repo, + signer, + now: state.now(), + }; + let address = RecordAddress::new( + manifest_collection(), + RecordRkey::new(MANIFEST_RKEY).expect("Manifest rkey is a valid record key"), + ); + let (object, live) = standing(&ledger, &address)?; + Ok(stored_manifest_of(&ledger, &address, object, &live)?.contribution_policy) +} + +async fn open( + state: &Arc>, + session: Session, + collection: RecordCollection, + parent: Option, + record: RecordBody, + body: B, + admit: F, +) -> Result +where + B: SocialBody + Send + 'static, + H: HttpTransport, + C: Clock, + F: FnOnce(&Ledger<'_, H, C>, &WriteContext<'_>) -> Result, XrpcError> + + Send + + 'static, +{ + open_at( + state, + session, + RecordAddress::new(collection.clone(), state.rkeys.mint()), + parent, + record, + body, + admit, + ) + .await +} + +async fn open_at( + state: &Arc>, + session: Session, + address: RecordAddress, + parent: Option, + record: RecordBody, + body: B, + admit: F, +) -> Result +where + B: SocialBody + Send + 'static, + H: HttpTransport, + C: Clock, + F: FnOnce(&Ledger<'_, H, C>, &WriteContext<'_>) -> Result, XrpcError> + + Send + + 'static, +{ + let collection = address.collection().clone(); + open_keyed( + state, + session, + collection, + |_| Ok(address), + parent, + record, + body, + admit, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn open_keyed( + state: &Arc>, + session: Session, + collection: RecordCollection, + key: K, + parent: Option, + record: RecordBody, + body: B, + admit: F, +) -> Result +where + B: SocialBody + Send + 'static, + H: HttpTransport, + C: Clock, + F: FnOnce(&Ledger<'_, H, C>, &WriteContext<'_>) -> Result, XrpcError> + + Send + + 'static, + K: FnOnce(SubjectNumber) -> Result + Send + 'static, +{ + let produces = record.cid(); + on_repo(state, session, move |ledger, cx| { + if let Some(answered) = admit(ledger, cx)? { + return Ok(answered); + } + + let number = next_number(ledger.state, ledger.repo, &collection)?; + let address = key(number)?; + let change = SocialChange::Open { + creation: Creation { + address: address.clone(), + author: cx.actor().clone(), + number, + parent, + created_at: ledger.now, + produces, + }, + body, + }; + let (_, tip, head) = ledger.write(cx, &address, change, Doing::Open { body: &record })?; + Ok(wrote(ledger.repo, &address, produces, &head).by(tip)) + }) + .await +} + +fn require_label_defs( + ledger: &Ledger<'_, H, C>, + tokens: &[LabelToken], +) -> Result<(), XrpcError> { + tokens.iter().try_for_each(|token| { + let key = token.key(); + let address = RecordAddress::new(label_def_collection(), RecordRkey::composed(key.clone())); + let object = ready( + ledger.state.index.social_object_at(ledger.repo, &address), + "social", + )? + .ok_or_else(|| { + XrpcError::invalid_request(format!( + "tangledLabels cites {key}, which isn't a label def on this repository" + )) + })?; + serving_object(ledger.git, &address, object)?.ok_or_else(|| { + XrpcError::invalid_request(format!( + "tangledLabels cites {key}, whose label def was deleted" + )) + })?; + Ok(()) + }) +} + +fn vet_ticket_scope( + ledger: &Ledger<'_, H, C>, + target_branch: Option<&TargetBranch>, + tokens: Option<&[LabelToken]>, +) -> Result<(), XrpcError> { + if let Some(branch) = target_branch + && ledger.git.find_ref(&branch.head_ref())?.is_none() + { + return Err(XrpcError::invalid_request(format!( + "a ticket's targetBranch opens onto a branch in this repository, and {} \ + isn't one of them", + branch + ))); + } + if let Some(tokens) = tokens { + require_label_defs(ledger, tokens)?; + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn ticket_body_of( + prose: Prose, + state: TicketState, + tangled_labels: Option>, + assignees: Option>, + target_branch: Option, + blobs: Option>, + created_at: UnixSeconds, + editor: AccountDid, +) -> Result<(RecordBody, Ticket), XrpcError> { + let keys = tangled_labels + .as_ref() + .map(|tokens| tokens.iter().map(|token| token.key().clone()).collect()); + let record = TicketRecord::new( + prose, + state, + tangled_labels, + assignees.clone(), + target_branch, + blobs, + created_at, + editor, + )? + .body()?; + Ok(( + record, + Ticket { + state, + tangled_labels: keys, + assignees, + }, + )) +} + +async fn ticket_open( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + repo: RepoDid, + record: serde_json::Value, +) -> Result { + let collection = ticket_collection(); + let parsed = typed::(record, &collection)?.parse(&repo, &collection)?; + let wanted = parsed.state.unwrap_or(TicketState::Open); + if wanted != TicketState::Open { + return Err(XrpcError::invalid_request( + "a ticket opens in the open state; close or complete it with putRecord", + )); + } + let session = crate::blobs::session_admitting( + state, + headers, + method, + repo, + parsed.blobs.as_deref().unwrap_or(&[]), + ) + .await?; + let branch = parsed.target_branch.clone(); + let tokens = parsed.tangled_labels.clone(); + let assigning = parsed.assignees.is_some(); + let (record, ticket) = ticket_body_of( + parsed.prose, + wanted, + parsed.tangled_labels, + parsed.assignees, + parsed.target_branch, + parsed.blobs, + session.now, + session.actor.clone(), + )?; + open_keyed( + state, + session, + collection.clone(), + move |number| { + RecordRkey::new(number.get().to_string()) + .map(|rkey| RecordAddress::new(collection, rkey)) + .map_err(|error| { + XrpcError::internal(format!("a ticket number isn't a record key: {error}")) + }) + }, + None, + record, + ticket, + move |ledger, cx| { + // a ticket opened before the foster's tickets come home would take a number one of them holds + if crate::handover::pending(ledger.state, ledger.repo) { + return Err(XrpcError::conflict(STILL_MOVING)); + } + if assigning && !cx.permission().moderates().is_allowed() { + return Err(XrpcError::forbidden(NOT_ASSIGN_AUTHORITY)); + } + vet_ticket_scope(ledger, branch.as_ref(), tokens.as_deref())?; + Ok(None) + }, + ) + .await +} + +async fn comment_open( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + repo: RepoDid, + record: serde_json::Value, +) -> Result { + let collection = comment_collection(); + let draft = typed::(record, &collection)?.parse(&repo)?; + let session = + crate::blobs::session_admitting(state, headers, method, repo, draft.markup.attachments()) + .await?; + let subject = draft.subject.clone(); + let reply = draft.reply_to.clone(); + let record = CommentRecord::new( + session.repo.clone(), + draft.subject, + draft.markup, + draft.reply_to, + draft.embed, + session.now, + session.actor.clone(), + )? + .body()?; + let body = Comment { + subject: subject.cid(), + reply: reply.clone(), + }; + open( + state, + session, + comment_collection(), + Some(subject.address().clone()), + record, + body, + move |ledger, _cx| { + standing_version(ledger, &subject, "subject")?; + reply + .as_ref() + .map(|reply| standing_version(ledger, reply, "reply")) + .transpose()?; + Ok(None) + }, + ) + .await +} + +fn standing_version( + ledger: &Ledger<'_, H, C>, + reference: &CommentRef, + what: &'static str, +) -> Result<(), XrpcError> { + let serving = live_record_at(ledger, reference.address())? + .version() + .ok_or_else(|| XrpcError::internal(format!("The current {what} is missing its version")))?; + if serving != reference.cid() { + return Err(XrpcError::invalid_request(format!( + "The {what}'s strongRef CID is {}, but the record at {} now has version {serving}", + reference.cid(), + reference.address().at_uri(ledger.repo), + ))); + } + Ok(()) +} + +async fn reaction_open( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + repo: RepoDid, + record: serde_json::Value, +) -> Result { + let (subject, reaction) = + typed::(record, &reaction_collection())?.parse(&repo)?; + let session = session(state, headers, method, repo).await?; + let record = ReactionRecord::new( + session.repo.clone(), + subject.clone(), + reaction, + session.now, + session.actor.clone(), + )? + .body()?; + open( + state, + session, + reaction_collection(), + Some(subject.address().clone()), + record, + Reaction { + subject: subject.cid(), + kind: reaction, + }, + move |ledger, cx| { + standing_version(ledger, &subject, "subject")?; + if !cx.permission().contributes().is_allowed() { + return Ok(None); + } + if let Some(object) = ready( + ledger.state.index.social_live_reaction( + ledger.repo, + subject.address(), + cx.actor(), + reaction, + ), + "social", + )? { + let address = ready( + ledger.state.index.social_address_of(ledger.repo, object), + "social", + )? + .ok_or_else(|| { + XrpcError::internal("The matching reaction is missing its address") + })?; + let version = live_record_at(ledger, &address)?.version().ok_or_else(|| { + XrpcError::internal("The matching reaction should have a version and doesn't") + })?; + return Ok(Some(wrote(ledger.repo, &address, version, &ledger.head()?))); + } + Ok(None) + }, + ) + .await +} +async fn artifact_open( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + repo: RepoDid, + record: serde_json::Value, +) -> Result { + let (oid, name, artifact) = typed::(record, &artifact_collection())?.parse()?; + let session = + crate::blobs::session_admitting_artifact(state, headers, method, repo, &artifact).await?; + let record = + ArtifactRecord::new(oid, name, artifact, session.now, session.actor.clone())?.body()?; + open( + state, + session, + artifact_collection(), + None, + record, + Artifact, + |_, _| Ok(None), + ) + .await +} + +fn noun_of(collection: &RecordCollection) -> &'static str { + match collection.as_str() { + knot_record::ticket::TICKET_COLLECTION => "ticket", + knot_record::comment::COMMENT_COLLECTION => "comment", + knot_record::reaction::REACTION_COLLECTION => "reaction", + knot_record::label::LABEL_DEF_COLLECTION => "label def", + knot_record::pull::PATCH_COLLECTION => "patch", + knot_record::grant::MEMBER_COLLECTION => "member", + knot_record::grant::COLLABORATOR_COLLECTION => "collaborator", + knot_record::grant::BLOCK_COLLECTION => "block", + knot_record::manifest::MANIFEST_COLLECTION => "manifest", + knot_record::artifact::ARTIFACT_COLLECTION => "artifact", + _ => "record", + } +} +fn standing( + ledger: &Ledger<'_, H, C>, + address: &RecordAddress, +) -> Result<(CobId, SocialObject), XrpcError> { + let object = object_at(ledger.state, ledger.repo, address)?; + let live = serving_object(ledger.git, address, object)? + .ok_or_else(|| deleted(ledger, address, XrpcError::not_found))?; + Ok((object, live)) +} + +fn describe(reply: Option<&CommentRef>, repo: &RepoDid) -> String { + reply.map_or_else( + || "without a reply".to_string(), + |reply| format!("{} at {}", reply.cid(), reply.address().at_uri(repo)), + ) +} + +fn live_record_at( + ledger: &Ledger<'_, H, C>, + address: &RecordAddress, +) -> Result { + let object = ready( + ledger.state.index.social_object_at(ledger.repo, address), + "social", + )? + .ok_or_else(|| { + XrpcError::invalid_request(format!( + "{} isn't a record on this knot", + address.at_uri(ledger.repo) + )) + })?; + serving_object(ledger.git, address, object)? + .ok_or_else(|| deleted(ledger, address, XrpcError::invalid_request)) +} + +fn deleted( + ledger: &Ledger<'_, H, C>, + address: &RecordAddress, + fail: fn(String) -> XrpcError, +) -> XrpcError { + fail(format!( + "{} at {} was deleted", + noun_of(address.collection()), + address.at_uri(ledger.repo) + )) +} + +fn serving_object( + git: &Repo, + address: &RecordAddress, + object: CobId, +) -> Result, XrpcError> { + Ok(state_at(git, address, object)? + .0 + .object() + .filter(|live| !live.is_erased()) + .cloned()) +} + +fn state_at( + git: &Repo, + address: &RecordAddress, + object: CobId, +) -> Result<(knot_cobs::SocialState, ChangeId), XrpcError> { + let type_name = TypeName::new(address.collection().as_str()) + .expect("Record collection is a valid type name"); + knot_cobs::social_state(&CobStore::new(git), &type_name, object) + .map_err(XrpcError::from)? + .ok_or_else(|| { + XrpcError::internal(format!( + "this knot didn't register a builder for {type_name}" + )) + }) +} + +fn change_of( + ledger: &Ledger<'_, H, C>, + object: CobId, + named: &'static str, + missing: &'static str, + root: bool, + read: fn(SocialChange) -> Result, +) -> Result +where + B: SocialBody, + H: HttpTransport, + C: Clock, +{ + let type_name = TypeName::new(B::COLLECTION) + .unwrap_or_else(|_| panic!("{named} collection is a valid type name")); + let mut changes = CobStore::new(ledger.git) + .changes_of(&type_name, object, knot_cob::HistoryModel::Convergent, None)? + .changes + .into_iter(); + let change = if root { changes.next() } else { changes.last() } + .ok_or_else(|| XrpcError::internal(missing))?; + match as ChangePayload>::decode(change.payload()) { + Ok(change) => read(change), + Err(error) => Err(XrpcError::internal(error.to_string())), + } +} + +fn opened_comment( + ledger: &Ledger<'_, H, C>, + object: CobId, +) -> Result { + change_of( + ledger, + object, + "Comment", + "A comment object's root change went missing", + true, + |change| match change { + SocialChange::Open { body, .. } => Ok(body), + _ => Err(XrpcError::internal( + "A comment object's root change isn't the change that created it", + )), + }, + ) +} + +fn opened_ticket( + ledger: &Ledger<'_, H, C>, + object: CobId, +) -> Result { + change_of( + ledger, + object, + "Ticket", + "A ticket object's changes went missing", + false, + |change| match change { + SocialChange::Open { body, .. } | SocialChange::Edit { body, .. } => Ok(body), + SocialChange::Erase { .. } => Err(XrpcError::internal( + "A live ticket's latest change is an erasure", + )), + }, + ) +} +pub(crate) async fn put_record( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let PutRecordInput { swap, record } = decode::(&body)?; + let SwapInput { collection, .. } = &swap; + match collection.as_str() { + knot_record::reaction::REACTION_COLLECTION => Err(XrpcError::invalid_request( + "createRecord makes a reaction and deleteRecord removes it; a reaction is never edited", + )), + knot_record::ticket::TICKET_COLLECTION => { + ticket_put(&state, &headers, &method, swap, record).await + } + knot_record::comment::COMMENT_COLLECTION => { + comment_put(&state, &headers, &method, swap, record).await + } + knot_record::label::LABEL_DEF_COLLECTION => { + label_def_put(&state, &headers, &method, swap, record).await + } + knot_record::pull::PATCH_COLLECTION => Err(XrpcError::invalid_request( + "submitPatch makes a patch and deleteRecord removes it; a patch is immutable, so \ + resubmit a new patch with submitPatch", + )), + knot_record::grant::MEMBER_COLLECTION + | knot_record::grant::COLLABORATOR_COLLECTION + | knot_record::grant::BLOCK_COLLECTION => Err(XrpcError::invalid_request( + "a grant is never edited; delete and recreate", + )), + knot_record::manifest::MANIFEST_COLLECTION => { + manifest_put(&state, &headers, &method, swap, record).await + } + knot_record::artifact::ARTIFACT_COLLECTION => Err(XrpcError::invalid_request( + "an artifact is never edited; delete it and create a new one", + )), + _ => Err(rejected_collection(&swap.collection)), + } +} + +async fn ticket_put( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + swap: SwapInput, + record: serde_json::Value, +) -> Result { + let (repo, rkey, from) = swap.edit_of(&ticket_collection())?; + let collection = ticket_collection(); + let parsed = typed::(record, &collection)?.parse(&repo, &collection)?; + let session = crate::blobs::session_admitting( + state, + headers, + method, + repo, + parsed.blobs.as_deref().unwrap_or(&[]), + ) + .await?; + let address = RecordAddress::new(collection, rkey); + on_repo(state, session, move |ledger, cx| { + let (object, live) = standing(ledger, &address)?; + let opened = opened_ticket(ledger, object)?; + let wanted = parsed.state.unwrap_or(opened.state); + if parsed.state.is_some_and(|sent| sent != opened.state) + && live.author() != cx.actor() + && !cx.permission().moderates().is_allowed() + { + return Err(XrpcError::forbidden(NOT_STATE_AUTHORITY)); + } + let assigned = |assignees: &Option>| { + assignees.iter().flatten().cloned().collect::>() + }; + if assigned(&parsed.assignees) != assigned(&opened.assignees) + && !cx.permission().moderates().is_allowed() + { + return Err(XrpcError::forbidden(NOT_ASSIGN_AUTHORITY)); + } + vet_ticket_scope( + ledger, + parsed.target_branch.as_ref(), + parsed.tangled_labels.as_deref(), + )?; + let (record, body) = ticket_body_of( + parsed.prose, + wanted, + parsed.tangled_labels, + parsed.assignees, + parsed.target_branch, + parsed.blobs, + live.created_at(), + cx.actor().clone(), + )?; + let produces = record.cid(); + if live.version() == Some(produces) { + if parsed.state.is_some_and(|sent| sent == opened.state) { + return Err(XrpcError::conflict(format!( + "ticket at {} is already {wanted}", + address.at_uri(ledger.repo) + ))); + } + if from == produces { + return Ok(wrote(ledger.repo, &address, produces, &ledger.head()?)); + } + } + let change: TicketChange = SocialChange::Edit { + editing: Editing { + author: cx.actor().clone(), + from, + produces, + edited_at: ledger.now, + }, + body, + }; + let (_, tip, head) = ledger.write( + cx, + &address, + change, + Doing::Edit { + object, + from, + body: &record, + }, + )?; + Ok(wrote(ledger.repo, &address, produces, &head).by(tip)) + }) + .await +} + +async fn comment_put( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + swap: SwapInput, + record: serde_json::Value, +) -> Result { + let (repo, rkey, from) = swap.edit_of(&comment_collection())?; + let collection = comment_collection(); + let draft = typed::(record, &collection)?.parse(&repo)?; + let session = + crate::blobs::session_admitting(state, headers, method, repo, draft.markup.attachments()) + .await?; + let address = RecordAddress::new(collection, rkey); + on_repo(state, session, move |ledger, cx| { + let (object, live) = standing(ledger, &address)?; + if live.parent() != Some(draft.subject.address()) { + let opened_on = live.parent().map_or_else( + || "wasn't created on a subject".to_string(), + |parent| format!("was created on {}", parent.at_uri(ledger.repo)), + ); + return Err(XrpcError::invalid_request(format!( + "a comment's subject is set at creation: {} {}, and the edit's subject is {}", + address.at_uri(ledger.repo), + opened_on, + draft.subject.address().at_uri(ledger.repo), + ))); + } + let opened = opened_comment(ledger, object)?; + if opened.subject != draft.subject.cid() { + return Err(XrpcError::invalid_request(format!( + "the edit's subject is {}, but this comment was created on {}; subject is set \ + at creation", + draft.subject.cid(), + opened.subject, + ))); + } + if opened.reply != draft.reply_to { + return Err(XrpcError::invalid_request(format!( + "reply is set at creation: the comment was created {}, the edit's reply is {}", + describe(opened.reply.as_ref(), ledger.repo), + describe(draft.reply_to.as_ref(), ledger.repo), + ))); + } + let record = CommentRecord::new( + ledger.repo.clone(), + draft.subject, + draft.markup, + draft.reply_to, + draft.embed, + live.created_at(), + cx.actor().clone(), + )? + .body()?; + let produces = record.cid(); + let change: knot_cobs::CommentChange = SocialChange::Edit { + editing: Editing { + author: cx.actor().clone(), + from, + produces, + edited_at: ledger.now, + }, + body: opened, + }; + let (_, tip, head) = ledger.write( + cx, + &address, + change, + Doing::Edit { + object, + from, + body: &record, + }, + )?; + Ok(wrote(ledger.repo, &address, produces, &head).by(tip)) + }) + .await +} + +pub(crate) async fn delete_record( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let swap = decode::(&body)?; + let SwapInput { collection, .. } = &swap; + match collection.as_str() { + knot_record::ticket::TICKET_COLLECTION => { + erase_of::(&state, &headers, &method, swap).await + } + knot_record::comment::COMMENT_COLLECTION => { + erase_of::(&state, &headers, &method, swap).await + } + knot_record::reaction::REACTION_COLLECTION => { + erase_of::(&state, &headers, &method, swap).await + } + knot_record::label::LABEL_DEF_COLLECTION => { + erase_of::(&state, &headers, &method, swap).await + } + knot_record::pull::PATCH_COLLECTION => { + erase_of::(&state, &headers, &method, swap).await + } + knot_record::artifact::ARTIFACT_COLLECTION => { + erase_of::(&state, &headers, &method, swap).await + } + knot_record::grant::MEMBER_COLLECTION + | knot_record::grant::COLLABORATOR_COLLECTION + | knot_record::grant::BLOCK_COLLECTION => { + grant_erase(&state, &headers, &method, swap).await + } + _ => Err(rejected_collection(&swap.collection)), + } +} + +async fn erase_of( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + swap: SwapInput, +) -> Result +where + B: SocialBody + Send + 'static, + H: HttpTransport, + C: Clock, +{ + let collection = SocialCob::::collection(); + let (repo, rkey, from) = swap.edit_of(&collection)?; + let address = RecordAddress::new(collection, rkey); + let session = session(state, headers, method, repo).await?; + on_repo(state, session, move |ledger, cx| { + let (object, _) = standing(ledger, &address)?; + let change: SocialChange = SocialChange::Erase { + erasure: Erasure { + author: cx.actor().clone(), + from, + erased_at: ledger.now, + }, + }; + let (_, _, head) = ledger.write(cx, &address, change, Doing::Erase { object, from })?; + Ok(Deleted { commit: head }) + }) + .await +} + +fn grant_of(collection: &RecordCollection) -> Option { + [ + knot_record::grant::GrantCollection::Member, + knot_record::grant::GrantCollection::Collaborator, + knot_record::grant::GrantCollection::Block, + ] + .into_iter() + .find(|grant| grant.nsid() == collection.as_str()) +} + +fn grant_subject( + collection: &RecordCollection, + rkey: Option, +) -> Result { + let rkey = rkey.ok_or_else(|| { + XrpcError::invalid_request(format!( + "a {}'s record key is the subject's DID; send that DID as the rkey", + noun_of(collection) + )) + })?; + AccountDid::new(rkey.as_str()).map_err(|_| { + XrpcError::invalid_request(format!( + "a {}'s record key is the subject's DID, and {} isn't one", + noun_of(collection), + rkey.as_str() + )) + }) +} + +/// Grants live on the knot's own chain for members and blocks, and on the +/// repository's chain for collaborators, so the repo in the request decides +/// which audience the token is minted for. +async fn grant_open( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + repo: RepoDid, + collection: RecordCollection, + rkey: Option, + record: serde_json::Value, +) -> Result { + let subject = grant_subject(&collection, rkey)?; + typed::(record, &collection)?.parse(&collection)?; + let grant = grant_of(&collection).ok_or_else(|| rejected_collection(&collection))?; + knot_record::grant::record_key(&subject, grant).map_err(|error| { + XrpcError::invalid_request(format!( + "the subject's DID doesn't fit as a record key under {}: {error}", + collection.as_str() + )) + })?; + let actor = grant_actor(state, headers, method, &repo, &collection).await?; + match collection.as_str() { + knot_record::grant::MEMBER_COLLECTION => { + crate::members::member_add(state, &actor, subject).await + } + knot_record::grant::BLOCK_COLLECTION => { + crate::blocklist::block_add(state, &actor, subject).await + } + _ => crate::collaborators::collaborator_add(state, &actor, &repo, subject).await, + }?; + Ok(crate::ok_empty()) +} + +async fn grant_erase( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + swap: SwapInput, +) -> Result { + let SwapInput { + repo, + collection, + rkey, + swap_record: _, + swap_commit, + } = swap; + if swap_commit.is_some() { + return Err(no_swap_commit()); + } + let subject = grant_subject(&collection, Some(rkey))?; + let actor = grant_actor(state, headers, method, &repo, &collection).await?; + match collection.as_str() { + knot_record::grant::MEMBER_COLLECTION => { + crate::members::member_remove(state, &actor, subject).await + } + knot_record::grant::BLOCK_COLLECTION => { + crate::blocklist::block_remove(state, &actor, subject).await + } + _ => crate::collaborators::collaborator_remove(state, &actor, &repo, subject).await, + }?; + Ok(crate::ok_empty()) +} + +async fn grant_actor( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + repo: &RepoDid, + collection: &RecordCollection, +) -> Result { + if collection.as_str() == knot_record::grant::COLLABORATOR_COLLECTION { + let actor = state + .authenticate_write_for_repo(headers, method, repo) + .await?; + require_native(state, repo)?; + Ok(actor) + } else { + on_this_knot(repo, &state.knot_repo, "record")?; + if let Some(actor) = state.legacy_admin_actor(headers)? { + tracing::warn!( + collection = collection.as_str(), + actor = actor.as_str(), + "legacy shared-secret admin path is still in use" + ); + return Ok(actor); + } + state.authenticate_write(headers, method).await + } +} + +const NOT_LABEL_AUTHORITY: &str = "label defs belong to the repository's owner alone"; + +fn append_denial( + opening: Opening, + verb: &str, + collection: &RecordCollection, + permission: ContributionPermission, +) -> XrpcError { + match opening { + Opening::Ownership => XrpcError::forbidden(NOT_LABEL_AUTHORITY), + Opening::Contribution if verb == "erase" => XrpcError::forbidden(format!( + "Sorry, only a collaborator can erase this {}", + noun_of(collection) + )), + _ if !permission.contributes().is_allowed() => XrpcError::forbidden(format!( + "Sorry, you're {permission}, so you can't {verb} this {}; {COLLABORATE_REMEDY}", + noun_of(collection) + )), + _ => XrpcError::forbidden(format!( + "Sorry, only the author or a moderator can {verb} this {}", + noun_of(collection) + )), + } +} + +async fn label_def_open( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + repo: RepoDid, + rkey: Option, + record: serde_json::Value, +) -> Result { + let collection = label_def_collection(); + let slug = rkey.and_then(|rkey| rkey.as_label_key()).ok_or_else(|| { + XrpcError::invalid_request( + "a label def's record key is the slug the owner chooses; send that slug \ + as the rkey", + ) + })?; + let def = typed::(record, &collection)?.parse(&collection)?; + let session = session(state, headers, method, repo).await?; + if !Opening::Ownership.decide(session.permission).is_allowed() { + return Err(XrpcError::forbidden(NOT_LABEL_AUTHORITY)); + } + let record = LabelDefRecord::new(def, session.now, session.actor.clone())?.body()?; + let address = RecordAddress::new(collection, RecordRkey::composed(slug)); + open_at( + state, + session, + address.clone(), + None, + record, + LabelDef, + move |ledger, _| match object_at(ledger.state, ledger.repo, &address) { + Ok(object) => match serving_object(ledger.git, &address, object) { + Ok(Some(_)) => Err(XrpcError::conflict(format!( + "a label def already stands at {}", + address.at_uri(ledger.repo) + ))), + Ok(None) => Err(XrpcError::conflict(format!( + "a label def was erased at {}, and an erased def's slug stays \ + retired while the versions stay in the change graph", + address.at_uri(ledger.repo) + ))), + Err(interrupted) => Err(interrupted), + }, + Err(missing) if missing.status() == http::StatusCode::NOT_FOUND => Ok(None), + Err(other) => Err(other), + }, + ) + .await +} + +async fn label_def_put( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + swap: SwapInput, + record: serde_json::Value, +) -> Result { + let collection = label_def_collection(); + let (repo, rkey, from) = swap.edit_of(&collection)?; + let def = typed::(record, &collection)?.parse(&collection)?; + let address = RecordAddress::new(collection, rkey); + let session = session(state, headers, method, repo).await?; + on_repo(state, session, move |ledger, cx| { + let (object, live) = standing(ledger, &address)?; + let record = LabelDefRecord::new(def, live.created_at(), cx.actor().clone())?.body()?; + let produces = record.cid(); + if live.version() == Some(produces) && from == produces { + return Ok(wrote(ledger.repo, &address, produces, &ledger.head()?)); + } + let change: LabelDefChange = SocialChange::Edit { + editing: Editing { + author: cx.actor().clone(), + from, + produces, + edited_at: ledger.now, + }, + body: LabelDef, + }; + let (_, tip, head) = ledger.write( + cx, + &address, + change, + Doing::Edit { + object, + from, + body: &record, + }, + )?; + Ok(wrote(ledger.repo, &address, produces, &head).by(tip)) + }) + .await +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ManifestDeclaration { + owner: OwnerDid, + name: RepoName, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ManifestDraft { + #[serde(rename = "$type", default)] + kind: Option, + declaration: ManifestDeclaration, + #[serde(default)] + spindle: Option, + #[serde(default)] + description: Option, + #[serde(default)] + website: Option, + #[serde(default)] + topics: Option, + #[serde(default)] + source: Option, + #[serde(rename = "contributionPolicy", default)] + contribution_policy: Option, + #[serde(rename = "createdAt")] + _created_at: serde::de::IgnoredAny, + #[serde(rename = "x-tngl-editor", default)] + _editor: Option, +} + +/// The live manifest read off the chain, decoded leniently the way the +/// stored ticket is: the knot wrote it, so an unknown field is a newer knot. +#[derive(Deserialize)] +struct StoredManifest { + declaration: StoredDeclaration, + #[serde(rename = "contributionPolicy", default)] + contribution_policy: Option, +} + +#[derive(Deserialize)] +struct StoredDeclaration { + owner: OwnerDid, + name: RepoName, +} + +fn stored_manifest_of( + ledger: &Ledger<'_, H, C>, + address: &RecordAddress, + object: CobId, + live: &SocialObject, +) -> Result { + let version = live + .version() + .ok_or_else(|| XrpcError::internal("A live manifest is missing its version"))?; + let body = tree_body( + &CobStore::new(ledger.git), + &collection_type(address.collection()), + object, + version, + )? + .ok_or_else(|| XrpcError::internal("A live manifest is missing its record bytes"))?; + serde_ipld_dagcbor::from_slice(body.as_bytes()) + .map_err(|error| XrpcError::internal(format!("The manifest record doesn't parse: {error}"))) +} + +const SETTINGS_DENIED: &str = "settings belong to the repository's owner alone"; + +pub(crate) fn settings_authority(permission: ContributionPermission) -> Result<(), XrpcError> { + Opening::Ownership + .decide(permission) + .is_allowed() + .then_some(()) + .ok_or_else(|| XrpcError::forbidden(SETTINGS_DENIED)) +} + +#[derive(Default, Clone)] +struct ManifestEffects { + rename: Option<(RepoRkey, RepoName)>, + policy: Option, +} + +struct ManifestPlan { + /// The draft already matches the live record, so the plan has no effects + /// and no write left to do. + reply: Option, + effects: ManifestEffects, + object: CobId, + from: RecordCid, + record: RecordBody, +} + +/// The settings write validates the whole draft against the live record +/// before any of it takes effect: the rename and the policy are applied to +/// the registry, and only then is the manifest record itself written with +/// the new values. +fn manifest_plan( + ledger: &Ledger<'_, H, C>, + cx: &WriteContext<'_>, + address: &RecordAddress, + from: RecordCid, + draft: ManifestDraft, +) -> Result { + settings_authority(cx.permission())?; + let (object, live) = standing(ledger, address)?; + let stored = stored_manifest_of(ledger, address, object, &live)?; + if stored.declaration.owner != draft.declaration.owner { + return Err(XrpcError::invalid_request(format!( + "the repository's owner is set at creation: the manifest declares {}, and the \ + edit declares {}", + stored.declaration.owner, draft.declaration.owner, + ))); + } + let mut effects = ManifestEffects::default(); + if stored.declaration.name != draft.declaration.name { + let name = draft.declaration.name.clone(); + let rkey = RepoRkey::new(name.as_str().to_lowercase()) + .map_err(|error| XrpcError::invalid_request(error.to_string()))?; + match ready( + ledger + .state + .index + .resolve_repo(&stored.declaration.owner, &rkey), + "registry", + )? { + Some(existing) if existing != *ledger.repo => { + return Err(XrpcError::conflict(format!( + "the record key {rkey} already belongs to another repository" + ))); + } + _ => effects.rename = Some((rkey, name)), + } + } + match (stored.contribution_policy, draft.contribution_policy) { + (Some(_), None) => { + return Err(XrpcError::invalid_request( + "clearing the manifest's contributionPolicy isn't supported; set it to a policy", + )); + } + (old, Some(new)) if old != Some(new) => effects.policy = Some(new), + _ => {} + } + let record = ManifestRecord::new( + stored.declaration.owner.clone(), + draft.declaration.name.clone(), + ManifestMeta { + spindle: draft.spindle, + description: draft.description, + website: draft.website, + topics: draft.topics, + source: draft.source, + contribution_policy: draft.contribution_policy, + }, + live.created_at(), + cx.actor().clone(), + )? + .body()?; + let produces = record.cid(); + if live.version() == Some(produces) && from == produces { + return Ok(ManifestPlan { + reply: Some(wrote(ledger.repo, address, produces, &ledger.head()?)), + effects: ManifestEffects::default(), + object, + from, + record, + }); + } + Ok(ManifestPlan { + reply: None, + effects, + object, + from, + record, + }) +} + +async fn manifest_put( + state: &Arc>, + headers: &HeaderMap, + method: &crate::Method, + swap: SwapInput, + record: serde_json::Value, +) -> Result { + let (repo, rkey, from) = swap.edit_of(&manifest_collection())?; + if rkey.as_str() != MANIFEST_RKEY { + return Err(XrpcError::invalid_request(format!( + "the manifest's record key is `{MANIFEST_RKEY}`, and this knot won't write it as \ + {rkey}", + ))); + } + let collection = manifest_collection(); + let draft = typed::(record, &collection)?; + kind_matches(draft.kind.as_ref(), &collection)?; + let address = RecordAddress::new(collection, rkey); + let session = session(state, headers, method, repo).await?; + let signer = state.secrets.signer(&state.knot_did)?; + let state = Arc::clone(state); + let Session { + actor, + repo, + permission, + now, + } = session; + let home = CobHome::from(&state.knot_did); + let planned = address.clone(); + let written = run_blocking(move || -> Result { + let _repo_guard = state.cob_locks.repo(&repo); + let _meta_guard = state.cob_locks.meta(); + let _social_guard = state.index.guard_social(&repo)?; + let git = state.layout.open(&repo)?; + let meta = Repo::open(&state.meta_path)?; + let ledger = Ledger { + state: &state, + git: &git, + repo: &repo, + signer: &signer, + now, + }; + let cx = WriteContext::new(&actor, &repo, permission); + let plan = manifest_plan(&ledger, &cx, &planned, from, draft)?; + let ManifestPlan { + reply, + effects, + object, + from, + record, + } = plan; + if let Some(reply) = reply { + return Ok(reply); + } + let ManifestEffects { rename, policy } = effects; + if rename.is_some() || policy.is_some() { + let owner = match state.index.owner_of(&repo) { + Resolved::Ready(Some(owner)) => owner, + Resolved::Ready(None) => { + return Err(XrpcError::not_found("no such repository on this knot")); + } + Resolved::Warming => { + return Err(XrpcError::warming( + "registry projection is still warming, please retry shortly", + )); + } + }; + if let Some((rkey, name)) = rename { + crate::repos::rename_in_registry( + &meta, + &home, + owner.clone(), + &repo, + rkey, + name, + &signer, + now, + )?; + } + if let Some(policy) = policy { + crate::repos::set_policy_in_registry( + &meta, &home, owner, &repo, policy, &signer, now, + )?; + } + } + let produces = record.cid(); + let change: ManifestChange = SocialChange::Edit { + editing: Editing { + author: actor.clone(), + from, + produces, + edited_at: now, + }, + body: Manifest, + }; + let (_, _, head) = ledger.write( + &cx, + &address, + change, + Doing::Edit { + object, + from, + body: &record, + }, + )?; + state.index.refresh_registry().map_err(|error| { + XrpcError::internal(format!( + "repo {repo} took new settings but registry projection refresh failed: {error}" + )) + })?; + Ok(wrote(&repo, &address, produces, &head)) + }) + .await?; + Ok(Json(written).into_response()) +} + +fn require_commit( + ledger: &Ledger<'_, H, C>, + oid: Oid, + what: &str, +) -> Result<(), XrpcError> { + ledger.git.find_commit(oid).map(|_| ()).map_err(|_| { + XrpcError::invalid_request(format!( + "the patch's {what} {oid} isn't a commit in this repository" + )) + }) +} + +fn require_ancestor( + ledger: &Ledger<'_, H, C>, + base: Oid, + head: Oid, +) -> Result<(), XrpcError> { + if ledger.git.merge_base(base, head).map_err(XrpcError::from)? != Some(base) { + return Err(XrpcError::invalid_request(format!( + "a patch is the range {}..{}, and its base isn't an ancestor of its head", + base, head, + ))); + } + Ok(()) +} + +// the branch may have moved past the head since, but the head has to be on it +fn require_on_branch( + ledger: &Ledger<'_, H, C>, + branch: &BranchName, + head: Oid, +) -> Result<(), XrpcError> { + let tip = ledger.git.find_ref(&branch.head_ref())?.ok_or_else(|| { + XrpcError::invalid_request(format!( + "the patch's source branch {branch} isn't a branch of this repository" + )) + })?; + require_commit(ledger, head, "head")?; + if ledger.git.merge_base(head, tip).map_err(XrpcError::from)? != Some(head) { + return Err(XrpcError::invalid_request(format!( + "the patch's head {head} isn't on its source branch {branch}" + ))); + } + Ok(()) +} + +fn pinned( + ledger: &Ledger<'_, H, C>, + address: &RecordAddress, + range: CommitRange, +) { + [range.base, range.head] + .into_iter() + .try_for_each(|commit| { + let name = crate::git::keep_ref(ledger.repo, address, commit)?; + if ledger + .git + .find_ref(&name)? + .is_some_and(|pinned| pinned == commit) + { + return Ok(()); + } + ledger + .git + .set_reference(&name, commit) + .map_err(XrpcError::from) + }) + .inspect_err(|error| { + tracing::warn!( + repo = %ledger.repo, + %error, + "A patch's keep refs wouldn't write after the open" + ) + }) + .ok(); +} + +fn patch_ticket( + ledger: &Ledger<'_, H, C>, + ticket_address: &RecordAddress, +) -> Result<(), XrpcError> { + if live_record_at(ledger, ticket_address)?.parent().is_some() { + return Err(XrpcError::invalid_request(format!( + "a patch's ticket must be a top-level ticket, and {} isn't", + ticket_address.at_uri(ledger.repo) + ))); + } + Ok(()) +} + +async fn open_patch( + state: &Arc>, + session: Session, + parsed: ParsedPatch, +) -> Result { + let collection = patch_collection(); + let ticket_address = RecordAddress::new(ticket_collection(), parsed.ticket.clone()); + let record = PatchRecord::new( + parsed.ticket.clone(), + parsed.range, + parsed.source, + session.now, + session.actor.clone(), + )? + .body()?; + let produces = record.cid(); + let key = RecordRkey::new(produces.to_string()).map_err(|error| { + XrpcError::internal(format!( + "a patch's record key is its record's CID, and {produces} wouldn't parse as one: \ + {error}" + )) + })?; + let address = RecordAddress::new(collection.clone(), key); + open_at( + state, + session, + address.clone(), + Some(ticket_address.clone()), + record, + Patch, + move |ledger, _cx| match object_at(ledger.state, ledger.repo, &address) { + Ok(object) => match serving_object(ledger.git, &address, object) { + Ok(Some(live)) => { + let serving = live.version().ok_or_else(|| { + XrpcError::internal("a serving patch is missing its version") + })?; + if serving != produces { + return Err(XrpcError::internal(format!( + "the patch at {} serves version {serving} under the key of \ + {produces}, and a patch's key is its own record's CID", + address.at_uri(ledger.repo) + ))); + } + Ok(Some(wrote( + ledger.repo, + &address, + produces, + &ledger.head()?, + ))) + } + Ok(None) => Err(XrpcError::conflict(format!( + "a patch was erased at {}, and an erased patch's key stays retired", + address.at_uri(ledger.repo) + ))), + Err(interrupted) => Err(interrupted), + }, + Err(missing) if missing.status() == http::StatusCode::NOT_FOUND => { + patch_ticket(ledger, &ticket_address)?; + require_commit(ledger, parsed.range.base, "base")?; + require_commit(ledger, parsed.range.head, "head")?; + require_ancestor(ledger, parsed.range.base, parsed.range.head)?; + pinned(ledger, &address, parsed.range); + Ok(None) + } + Err(other) => Err(other), + }, + ) + .await +} + +pub(crate) const SUBMIT_PATCH_ROUTE: &str = "/xrpc/org.tangled.review.submitPatch"; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct SubmitPatchInput { + repo: RepoDid, + patch: SubmitPatchDraft, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct SubmitPatchDraft { + ticket: RecordRkey, + base: OidDraft, + head: OidDraft, + source_repo: RepoDid, + #[serde(default)] + source_branch: Option, +} + +pub(crate) async fn submit_patch( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let SubmitPatchInput { repo, patch } = decode::(&body)?; + let session = session(&state, &headers, &method, repo.clone()).await?; + if matches!(Patch::OPENING.decide(session.permission()), Decision::Deny) { + return Err(rejected(CobError::CreateRejected { + actor: session.actor.clone(), + type_name: as ChangePayload>::type_name(), + permission: session.permission(), + })); + } + let range = CommitRange { + base: patch.base.oid, + head: patch.head.oid, + }; + let ticket_address = RecordAddress::new(ticket_collection(), patch.ticket.clone()); + let same_repo = patch.source_repo == repo; + let source_branch = patch.source_branch.clone(); + let should_fetch = in_ledger(&state, session.clone(), move |ledger, _| { + patch_ticket(ledger, &ticket_address)?; + require_commit(ledger, range.base, "base")?; + if !same_repo { + return Ok(!ledger.git.contains(range.head)); + } + if let Some(branch) = &source_branch { + require_on_branch(ledger, branch, range.head)?; + } + Ok(false) + }) + .await?; + if should_fetch { + crate::forks::fetch_commit(&state, &repo, &patch.source_repo, range.head).await?; + } + open_patch( + &state, + session, + ParsedPatch { + ticket: patch.ticket, + range, + source: patch.source_branch.map(|branch| PatchSource { + repo: patch.source_repo, + branch, + }), + }, + ) + .await +} + +const COB_VERSIONS_DEFAULT: usize = 100; + +const COB_VERSIONS_MAX: usize = 500; + +// not `query::Order`, which defaults to desc while this list reads oldest first +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) enum VersionOrder { + #[default] + OldestFirst, + NewestFirst, +} + +impl<'de> Deserialize<'de> for VersionOrder { + fn deserialize>(deserializer: D) -> Result { + match String::deserialize(deserializer)?.as_str() { + "" | "asc" => Ok(Self::OldestFirst), + "desc" => Ok(Self::NewestFirst), + _ => Err(serde::de::Error::custom("order must be 'asc' or 'desc'")), + } + } +} + +#[derive(Deserialize)] +pub(crate) struct VersionsQuery { + pub(crate) uri: AtUri, + #[serde(default)] + pub(crate) limit: crate::query::Limit, + #[serde(default)] + pub(crate) cursor: crate::query::Offset, + #[serde(default)] + pub(crate) order: VersionOrder, +} + +#[derive(Serialize)] +struct Version { + #[serde(skip_serializing_if = "Option::is_none")] + cid: Option, + #[serde(rename = "changeId")] + change_id: ChangeId, + editor: AccountDid, + #[serde(rename = "createdAt")] + created_at: knot_types::Datetime, +} + +#[derive(Serialize)] +struct Versions { + uri: AtUri, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, + total: usize, + versions: Vec, +} + +pub(crate) async fn list_record_versions( + State(state): State>>, + ValidatedQuery(query): ValidatedQuery, +) -> Result { + let Target { repo, address } = any_target(&query.uri)?; + let collection = address.collection().clone(); + if !knot_cobs::social_collections().contains(&collection) { + return Err(XrpcError::invalid_request(format!( + "{collection} isn't a collection this knot keeps version chains for" + ))); + } + ready(state.index.ownership_of(&repo), "registry")? + .ok_or_else(|| XrpcError::not_found("No such repository on this knot"))?; + warm_social(&state, &repo).await?; + let object = object_at(&state, &repo, &address)?; + let index_state = Arc::clone(&state); + let listing = address.at_uri(&repo); + let response_limit = state.byte_limits.response; + let (limit, cursor) = (query.limit, query.cursor); + let newest_first = query.order == VersionOrder::NewestFirst; + let type_name = knot_types::TypeName::new(collection.as_str()) + .map_err(|_| XrpcError::invalid_request("The URI's collection isn't a type name"))?; + let (versions, total) = run_blocking(move || { + let git = index_state.layout.open(&repo)?; + let store = CobStore::new(&git); + let delta = + store.changes_of(&type_name, object, knot_cob::HistoryModel::Convergent, None)?; + let changes = &delta.changes; + let total = crate::query::Total::new(changes.len()); + let end = cursor.get().saturating_add(limit.get()).min(changes.len()); + let nth = |at: usize| { + if newest_first { + changes.len() - 1 - at + } else { + at + } + }; + (cursor.get()..end) + .map(|at| &changes[nth(at)]) + .map(|change| { + let payload = knot_cobs::decode_social(change.payload()).map_err(CobError::from)?; + let emission = knot_cobs::Social::emission(&payload); + Ok(Version { + cid: emission.produces().map(crate::out::CidString::from), + change_id: change.id, + editor: emission.author().clone(), + created_at: crate::out::datetime_of(emission.at()), + }) + }) + .collect::, XrpcError>>() + .map(|versions| (versions, total)) + }) + .await?; + crate::reads::json( + Versions { + uri: listing, + cursor: crate::query::next_cursor(cursor, limit, total), + total: total.get(), + versions, + }, + response_limit, + ) +} diff --git a/knot2/crates/knot-xrpc/src/tests.rs b/knot2/crates/knot-xrpc/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..568b0ce790d65d189ba93f2fe10aed054fe51ad2 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/tests.rs @@ -0,0 +1,7857 @@ +use parking_lot::Mutex; +use parking_lot::RwLock; +use std::collections::{BTreeSet, HashMap}; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + +use axum::body::Bytes; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use futures::StreamExt; +use http::{HeaderMap, HeaderValue, StatusCode, header::AUTHORIZATION}; +use serde_json::json; +use tempfile::TempDir; + +use knot_atproto::Atproto; +use knot_cobs::{CollaboratorsChange, CollaboratorsCob, Grant, MembersChange, MembersCob}; +use knot_git::{Layout, Repo}; +use knot_index::{Index, Resolved}; +use knot_record::ticket::TicketState; +use knot_runtime::{ + FakeHttp, HttpRequest, HttpResponse, HttpTransport, K256Signer, ManualClock, NetworkError, + OsEntropy, SeededEntropy, Signer, UnixMicros, +}; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{ + AccountDid, ActorId, AdmissionPolicy, AtUri, AuthorName, BranchName, ContributionPolicy, Did, + Email, KnotHostname, KnotId, KnotServiceUrl, Nsid, OriginUrl, OwnerDid, RecordCollection, + RecordRkey, RepoDid, RepoName, RepoRkey, ServiceDid, UnixSeconds, +}; + +use crate::{RepoMethod, XrpcState}; + +const KNOT_HOST: &str = "knot.nel.pet"; +const ADMIN_HOST: &str = "admin.nel.pet"; +const MEMBER_HOST: &str = "member.nel.pet"; +const STRANGER_HOST: &str = "stranger.nel.pet"; +const PASSERBY_HOST: &str = "passerby.nel.pet"; + +const CREATE: &str = "org.tangled.repo.createRepo"; +const RESERVE: &str = "org.tangled.repo.reserveKey"; +const DELETE: &str = "org.tangled.repo.deleteRepo"; +const DELETE_BRANCH: &str = "org.tangled.git.deleteBranch"; +const SET_DEFAULT_BRANCH: &str = "org.tangled.git.setDefaultBranch"; +const MANIFEST: &str = "org.tangled.repo.manifest"; +const MEMBER: &str = "org.tangled.knot.member"; +const BLOCK: &str = "org.tangled.knot.block"; +const COLLABORATOR: &str = "org.tangled.graph.collaborator"; + +fn member_collection() -> RecordCollection { + RecordCollection::new(MEMBER).unwrap() +} + +fn block_collection() -> RecordCollection { + RecordCollection::new(BLOCK).unwrap() +} + +fn collaborator_collection() -> RecordCollection { + RecordCollection::new(COLLABORATOR).unwrap() +} + +type Responder = Box Result + Send + Sync>; +type SharedState = Arc, Arc>>; + +static JTI: AtomicU64 = AtomicU64::new(0); + +fn knot_did() -> KnotId { + KnotId::new(format!("did:web:{KNOT_HOST}")).unwrap() +} + +fn knot_service_did() -> ServiceDid { + ServiceDid::new(format!("did:web:{KNOT_HOST}")).unwrap() +} + +fn account(host: &str) -> AccountDid { + AccountDid::new(format!("did:web:{host}")).unwrap() +} + +struct Actor { + signer: K256Signer, + did: AccountDid, +} + +fn actor(seed: u64, host: &str) -> Actor { + Actor { + signer: signer(seed), + did: account(host), + } +} + +fn signer(seed: u64) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed)) +} + +fn did_web_doc(did: &Did, sec1: &[u8], pds: &KnotServiceUrl) -> Bytes { + let multikey = knot_types::crypto::multikey(0xe7, sec1); + Bytes::from( + serde_json::to_vec(&json!({ + "id": did.as_str(), + "alsoKnownAs": [], + "verificationMethod": [{ + "id": format!("{}#atproto", did.as_str()), + "type": "Multikey", + "controller": did.as_str(), + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": pds.as_str() + }] + })) + .unwrap(), + ) +} +fn repo_account_doc(did: &Did, multikey: &ActorId, pds: &KnotServiceUrl) -> Bytes { + Bytes::from( + serde_json::to_vec(&json!({ + "id": did.as_str(), + "verificationMethod": [{ + "id": format!("{}#atproto", did.as_str()), + "type": "Multikey", + "controller": did.as_str(), + "publicKeyMultibase": multikey.as_str() + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": pds.as_str() + }] + })) + .unwrap(), + ) +} + +fn nsid(name: &str) -> Nsid { + Nsid::new_owned(name).expect("the xrpc method is an NSID") +} + +fn mint(actor: &Actor, nsid: &Nsid) -> String { + mint_for(actor, nsid, &knot_service_did()) +} + +fn mint_for(actor: &Actor, nsid: &Nsid, audience: &ServiceDid) -> String { + let jti = JTI.fetch_add(1, Ordering::Relaxed); + let header = URL_SAFE_NO_PAD.encode(br#"{"alg":"ES256K","typ":"JWT"}"#); + let payload = URL_SAFE_NO_PAD.encode( + serde_json::to_vec(&json!({ + "iss": actor.did.as_str(), + "aud": audience.as_str(), + "exp": 1_100, + "iat": 999, + "jti": format!("nonce-{jti}"), + "lxm": nsid.as_str(), + })) + .unwrap(), + ); + let signing_input = format!("{header}.{payload}"); + let signature = actor.signer.sign(signing_input.as_bytes()); + format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.as_bytes()) + ) +} + +fn bearer(token: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {token}")).unwrap(), + ); + headers +} + +fn body(value: serde_json::Value) -> Bytes { + Bytes::from(serde_json::to_vec(&value).unwrap()) +} + +fn into_response(result: Result) -> Response { + match result { + Ok(response) => response, + Err(error) => error.into_response(), + } +} + +async fn json_of(response: Response) -> serde_json::Value { + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() +} + +async fn repo_status(world: &World, did: &RepoDid) -> serde_json::Value { + json_of( + crate::atproto::sync_get_repo_status( + State(Arc::clone(&world.state)), + axum::extract::Query(crate::atproto::DidParams { + did: crate::atproto::RepoSubject::Did(did.clone()), + }), + ) + .await + .unwrap(), + ) + .await +} + +async fn call( + world: &World, + handler: F, + actor: &Actor, + nsid: &Nsid, + value: serde_json::Value, +) -> Response +where + F: FnOnce(State, HeaderMap, crate::Method, Bytes) -> Fut, + Fut: std::future::Future>, +{ + let token = mint(actor, nsid); + into_response( + handler( + world.state(), + bearer(&token), + crate::Method::from_nsid(nsid.as_str()), + body(value), + ) + .await, + ) +} + +async fn as_member( + world: &World, + handler: F, + nsid: &Nsid, + value: serde_json::Value, +) -> Response +where + F: FnOnce(State, HeaderMap, crate::Method, Bytes) -> Fut, + Fut: std::future::Future>, +{ + call(world, handler, &world.member, nsid, value).await +} + +async fn as_admin( + world: &World, + handler: F, + nsid: &Nsid, + value: serde_json::Value, +) -> Response +where + F: FnOnce(State, HeaderMap, crate::Method, Bytes) -> Fut, + Fut: std::future::Future>, +{ + call(world, handler, &world.admin, nsid, value).await +} + +async fn as_stranger( + world: &World, + handler: F, + nsid: &Nsid, + value: serde_json::Value, +) -> Response +where + F: FnOnce(State, HeaderMap, crate::Method, Bytes) -> Fut, + Fut: std::future::Future>, +{ + call(world, handler, &world.stranger, nsid, value).await +} + +fn member_owner() -> OwnerDid { + OwnerDid::new(format!("did:web:{MEMBER_HOST}")).unwrap() +} + +fn resolve(world: &World, name: &RepoName) -> Resolved> { + world + .state + .index + .resolve_repo(&member_owner(), &RepoRkey::new(name.as_str()).unwrap()) +} + +fn bootstrap( + dir: &TempDir, + rebuild: bool, + object_format: knot_types::ObjectFormat, +) -> (Layout, Arc, PathBuf) { + let scan_path = dir.path().join("repos"); + std::fs::create_dir_all(&scan_path).unwrap(); + let knot = knot_did(); + let layout = Layout::new(&scan_path) + .with_object_format(object_format) + .reserving_meta(&knot) + .unwrap(); + layout.bootstrap_meta(&knot).unwrap(); + let meta_path = layout.meta_path(&knot).unwrap(); + let index = Arc::new(Index::new(meta_path.clone(), layout.clone())); + if rebuild { + index.rebuild().unwrap(); + } + (layout, index, meta_path) +} + +fn generous_writes() -> knot_resource::RateLimit { + knot_resource::RateLimit { + burst: crate::Burst::literal(1_000), + refill: crate::RefillMicros::literal(1_000), + } +} + +struct WriteSide { + rate: knot_resource::RateLimit, + social_budget: knot_resource::SocialBudgetBytes, +} + +fn state_from( + dir: &TempDir, + boot: (Layout, Arc, PathBuf), + responder: Responder, + admission: AdmissionPolicy, + reservations: Arc, + git_http: Arc, + WriteSide { + rate: writes, + social_budget, + }: WriteSide, +) -> (SharedState, Arc) { + let (layout, index, meta_path) = boot; + let knot = knot_did(); + let knot_url = knot_types::KnotServiceUrl::new(format!("https://{KNOT_HOST}")).unwrap(); + let clock = Arc::new(ManualClock::new(UnixMicros::new(1_000_000_000))); + let atproto = Arc::new(Atproto::new( + FakeHttp::new(responder), + Arc::clone(&clock), + knot.clone(), + knot_atproto::PlcDirectory::new(url::Url::parse("https://plc.directory/").unwrap()) + .unwrap(), + )); + let secrets = Arc::new( + SealedStore::open( + dir.path().join("keys.sealed"), + &MasterKey::new([7u8; 32]).unwrap(), + Box::new(OsEntropy), + ) + .unwrap(), + ); + secrets.ensure(&knot).unwrap(); + let lfs = { + let lfs_dir = dir.path().join("lfs"); + std::fs::create_dir_all(&lfs_dir).unwrap(); + crate::LfsWeb::new( + knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(&lfs_dir), + knot_lfs::LfsSize::new(64 * 1024 * 1024), + knot_lfs::FreeSpaceFloor::new(0), + ) + .unwrap(), + 8, + ) + }; + let outbox = crate::Outbox::open(&meta_path, Default::default()); + let state = Arc::new(XrpcState { + blobs: Default::default(), + imports: crate::imports::ImportQueue::channel().0, + handover_sessions: Arc::new(crate::handover::HandoverSessions::default()), + layout, + index, + atproto, + secrets, + entropy: Arc::new(OsEntropy), + invite_codes: Default::default(), + admins: BTreeSet::from([account(ADMIN_HOST)]), + admission, + contribution_policy: knot_types::ContributionPolicy::Anyone, + knot_repo: crate::knot_repo(&knot).unwrap(), + knot_did: knot, + knot_hostname: KnotHostname::new(KNOT_HOST).unwrap(), + meta_path, + knot_service_url: knot_url, + writes: Arc::new(knot_resource::SubjectLimiter::new(writes)), + social_budget, + limiter: Arc::new(crate::PreAuthLimiter::default()), + cob_locks: Arc::new(crate::CobLocks::default()), + reservations, + proxy_trust: knot_types::ProxyTrust::default(), + byte_limits: crate::ByteLimits::default(), + budgets: crate::Budgets::default(), + git_http, + pack_limits: knot_pack::PackLimits::default(), + service_owner: account(ADMIN_HOST), + legacy_admin_secret: None, + description: Default::default(), + subscriber_gate: Arc::new(knot_events::SubscriberGate::new( + knot_events::GlobalSubscriberLimit::new(16), + knot_events::PerPeerSubscriberLimit::new(8), + )), + maintenance: knot_maintenance::MaintenanceHandle::disabled(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + slots: knot_resource::Slots::testing(8), + lfs, + catalog: Arc::new(knot_messages::Catalog::defaults()), + firehose: Arc::new(knot_events::EventLog::new( + Arc::clone(&clock), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(1024).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + )), + rkeys: Default::default(), + outbox, + }); + (state, clock) +} + +fn world_responder( + pubkeys: HashMap>, + repo_docs: Arc>>, + plc_docs: Arc>>, + pds_records: Arc>>, + reads: Arc, +) -> Responder { + let doc_url = KnotServiceUrl::new(format!("https://{KNOT_HOST}")).unwrap(); + let plc_key = ActorId::new(knot_types::crypto::multikey(0xe7, &pubkeys[MEMBER_HOST])).unwrap(); + Box::new(move |request: &HttpRequest| { + if request.method == http::Method::POST { + return Ok(HttpResponse { + status: StatusCode::OK, + headers: http::HeaderMap::new(), + body: Bytes::new(), + }); + } + if request.url.path().ends_with("com.atproto.repo.getRecord") { + reads.fetch_add(1, Ordering::Relaxed); + let rkey = request + .url + .query_pairs() + .find(|(key, _)| key == "rkey") + .map(|(_, value)| value.into_owned()) + .unwrap_or_default(); + let answer = pds_records.lock().get(&rkey).copied(); + return Ok(HttpResponse { + status: answer.unwrap_or(StatusCode::BAD_REQUEST), + headers: http::HeaderMap::new(), + body: match answer { + Some(_) => Bytes::new(), + None => Bytes::from_static(b"{\"error\":\"RecordNotFound\"}"), + }, + }); + } + let host = request.url.host_str().unwrap_or_default(); + if host == "plc.directory" { + let did = request.url.path().trim_start_matches('/'); + return Ok(match plc_docs.lock().get(did) { + Some(pds) => reply( + StatusCode::OK, + repo_account_doc( + &Did::new_owned(did).unwrap(), + &plc_key, + &KnotServiceUrl::new(pds.as_str()).unwrap(), + ), + ), + None => reply(StatusCode::NOT_FOUND, Bytes::new()), + }); + } + if let Some(multikey) = repo_docs.lock().get(host).cloned() { + return Ok(HttpResponse { + status: StatusCode::OK, + headers: http::HeaderMap::new(), + body: repo_account_doc( + &Did::new_owned(format!("did:web:{host}")).unwrap(), + &ActorId::new(multikey).unwrap(), + &doc_url, + ), + }); + } + match pubkeys.get(host) { + Some(sec1) => Ok(HttpResponse { + status: StatusCode::OK, + headers: http::HeaderMap::new(), + body: did_web_doc( + &Did::new_owned(format!("did:web:{host}")).unwrap(), + sec1, + &doc_url, + ), + }), + None => Ok(HttpResponse { + status: StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: Bytes::new(), + }), + } + }) +} + +struct World { + _dir: TempDir, + layout: Layout, + state: SharedState, + admin: Actor, + member: Actor, + stranger: Actor, + passerby: Actor, + repo_docs: Arc>>, + plc_docs: Arc>>, + pds_records: Arc>>, + clock: Arc, + reads: Arc, +} + +impl World { + fn new() -> Self { + Self::build( + 256, + 256, + AdmissionPolicy::Closed, + None, + knot_types::ObjectFormat::SHA1, + generous_writes(), + ) + } + + fn open() -> Self { + Self::build( + 256, + 256, + AdmissionPolicy::Open, + None, + knot_types::ObjectFormat::SHA1, + generous_writes(), + ) + } + + fn with_pending_limit(limit: usize) -> Self { + Self::build( + limit, + limit, + AdmissionPolicy::Closed, + None, + knot_types::ObjectFormat::SHA1, + generous_writes(), + ) + } + + fn with_limits(global: usize, per_actor: usize) -> Self { + Self::build( + global, + per_actor, + AdmissionPolicy::Closed, + None, + knot_types::ObjectFormat::SHA1, + generous_writes(), + ) + } + + fn with_write_burst(burst: u32) -> Self { + Self::build( + 256, + 256, + AdmissionPolicy::Closed, + None, + knot_types::ObjectFormat::SHA1, + knot_resource::RateLimit { + burst: crate::Burst::of(burst).expect("Test burst allows at least one write"), + refill: crate::RefillMicros::literal(1_000_000), + }, + ) + } + + fn with_git_http( + git_http: Arc, + object_format: knot_types::ObjectFormat, + ) -> Self { + Self::build( + 256, + 256, + AdmissionPolicy::Closed, + Some(git_http), + object_format, + generous_writes(), + ) + } + + fn with_social_budget(social_budget: knot_resource::SocialBudgetBytes) -> Self { + Self::assemble( + 256, + 256, + AdmissionPolicy::Closed, + None, + knot_types::ObjectFormat::SHA1, + generous_writes(), + social_budget, + ) + } + + fn build( + global: usize, + per_actor: usize, + admission: AdmissionPolicy, + git_http: Option>, + object_format: knot_types::ObjectFormat, + writes: knot_resource::RateLimit, + ) -> Self { + Self::assemble( + global, + per_actor, + admission, + git_http, + object_format, + writes, + knot_resource::SocialBudgetBytes::new(1 << 28), + ) + } + + fn assemble( + global: usize, + per_actor: usize, + admission: AdmissionPolicy, + git_http: Option>, + object_format: knot_types::ObjectFormat, + writes: knot_resource::RateLimit, + social_budget: knot_resource::SocialBudgetBytes, + ) -> Self { + let dir = tempfile::tempdir().unwrap(); + let (layout, index, meta_path) = bootstrap(&dir, true, object_format); + let admin = actor(1, ADMIN_HOST); + let member = actor(2, MEMBER_HOST); + let stranger = actor(3, STRANGER_HOST); + let passerby = actor(4, PASSERBY_HOST); + let repo_docs: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let pds_records: Arc>> = + Arc::new(Mutex::new(HashMap::new())); + let pubkeys = [ + (ADMIN_HOST, &admin), + (MEMBER_HOST, &member), + (STRANGER_HOST, &stranger), + (PASSERBY_HOST, &passerby), + ] + .into_iter() + .map(|(host, actor)| { + ( + host.to_string(), + actor.signer.public_key().as_bytes().to_vec(), + ) + }) + .collect(); + let reads = Arc::new(AtomicUsize::new(0)); + let plc_docs = Arc::new(Mutex::new(HashMap::new())); + let responder = world_responder( + pubkeys, + Arc::clone(&repo_docs), + Arc::clone(&plc_docs), + Arc::clone(&pds_records), + Arc::clone(&reads), + ); + let (state, clock) = state_from( + &dir, + (layout.clone(), index, meta_path), + responder, + admission, + Arc::new(crate::Reservations::new( + crate::ReservationTtl::new(1_000_000), + crate::PerActorQuota::new(per_actor), + crate::GlobalQuota::new(global), + )), + git_http.unwrap_or_else(no_git_upstream), + WriteSide { + rate: writes, + social_budget, + }, + ); + Self { + _dir: dir, + layout, + state, + admin, + member, + stranger, + passerby, + repo_docs, + plc_docs, + pds_records, + clock, + reads, + } + } + + fn state(&self) -> State { + State(Arc::clone(&self.state)) + } + + fn with_legacy_admin_secret(mut self, secret: &str) -> Self { + Arc::get_mut(&mut self.state) + .expect("za warudo does the only state handling at constructoin") + .legacy_admin_secret = + Some(crate::legacy_admin::LegacyAdminSecret::new(secret).unwrap()); + self + } + + fn publish_pds_record(&self, rkey: &RecordRkey) { + self.respond_pds_record(rkey, StatusCode::OK); + } + + fn respond_pds_record(&self, rkey: &RecordRkey, status: StatusCode) { + self.pds_records + .lock() + .insert(rkey.as_str().to_string(), status); + } + fn publish_repo_doc(&self, host: &str, multikey: &str) { + self.repo_docs + .lock() + .insert(host.to_string(), multikey.to_string()); + } + + fn publish_plc_doc(&self, did: &Did, pds: &KnotServiceUrl) { + self.plc_docs + .lock() + .insert(did.as_str().to_string(), pds.as_str().to_string()); + } + + fn advance(&self, secs: u64) { + self.clock.advance(std::time::Duration::from_secs(secs)); + } + + fn record_reads(&self) -> usize { + self.reads.load(Ordering::Relaxed) + } + + fn migrated(&self, host: &str) -> Grant { + Grant { + subject: account(host), + added_by: self.admin.did.clone(), + created_at: UnixSeconds::new(1_000), + } + } + + fn append(&self, repo: Option<&RepoDid>, change: &E::Change) + where + E: knot_cob::Evaluate, + E::Change: knot_cob::KnotAuthored, + { + let git = match repo { + None => Repo::open(&self.state.meta_path).unwrap(), + Some(repo) => self.layout.open(repo).unwrap(), + }; + let home = match repo { + None => knot_cob::CobHome::from(&self.state.knot_did), + Some(repo) => knot_cob::CobHome::from(repo), + }; + let store = knot_cob::CobStore::new(&git); + let signer = self.state.secrets.signer(&self.state.knot_did).unwrap(); + let at = UnixSeconds::new(1_000); + match store.list::().unwrap().as_slice() { + [] => _ = store.create(&home, change, &signer, at).unwrap(), + [object] => _ = store.update(&home, *object, change, &signer, at).unwrap(), + many => panic!( + "{} grant list objects under namespace that allows one", + many.len() + ), + } + } + + fn grandfather(&self, host: &str) { + self.append::(None, &MembersChange::Add(self.migrated(host))); + self.refresh(None); + } + + fn grandfather_collaborator(&self, repo: &RepoDid, host: &str) { + self.append::(Some(repo), &CollaboratorsChange::Add(self.migrated(host))); + self.refresh(Some(repo)); + } + + fn seed(&self, repo: Option<&RepoDid>, host: &str) { + let grant = self.migrated(host); + match repo { + None => self.append::(repo, &MembersChange::Add(grant)), + Some(_) => self.append::(repo, &CollaboratorsChange::Add(grant)), + } + } + + fn refresh(&self, repo: Option<&RepoDid>) { + match repo { + None => self.state.index.refresh_members().unwrap(), + Some(repo) => self.state.index.refresh_collaborators(repo).unwrap(), + } + } + + fn on_grant_list(&self, repo: Option<&RepoDid>, subject: &AccountDid) -> Resolved { + self.refresh(repo); + match repo { + None => self.state.index.effective_member(subject), + Some(repo) => self.state.index.effective_collaborator(repo, subject), + } + } +} + +fn build_state(responder: Responder, rebuild: bool) -> (TempDir, SharedState) { + let (dir, state, _clock) = build_state_with_clock(responder, rebuild); + (dir, state) +} + +fn build_state_with_clock( + responder: Responder, + rebuild: bool, +) -> (TempDir, SharedState, Arc) { + let dir = tempfile::tempdir().unwrap(); + let boot = bootstrap(&dir, rebuild, knot_types::ObjectFormat::SHA1); + let (state, clock) = state_from( + &dir, + boot, + responder, + AdmissionPolicy::Closed, + Arc::new(crate::Reservations::new( + crate::ReservationTtl::new(1_000_000), + crate::PerActorQuota::new(256), + crate::GlobalQuota::new(256), + )), + no_git_upstream(), + WriteSide { + rate: generous_writes(), + social_budget: knot_resource::SocialBudgetBytes::new(1 << 28), + }, + ); + (dir, state, clock) +} + +fn no_git_upstream() -> Arc { + Arc::new(FakeHttp::new(|_request: &HttpRequest| { + Err(NetworkError::Connect( + "no git upstream is served in this test".to_string(), + )) + })) +} + +fn doc_responder( + sec1: Vec, + post_status: impl Fn() -> StatusCode + Send + Sync + 'static, +) -> Responder { + let pds = KnotServiceUrl::new(format!("https://{KNOT_HOST}")).unwrap(); + Box::new(move |request: &HttpRequest| { + let post = request.method == http::Method::POST; + let host = request.url.host_str().unwrap_or_default(); + Ok(HttpResponse { + status: if post { post_status() } else { StatusCode::OK }, + headers: http::HeaderMap::new(), + body: if post { + Bytes::new() + } else { + did_web_doc( + &Did::new_owned(format!("did:web:{host}")).unwrap(), + &sec1, + &pds, + ) + }, + }) + }) +} + +#[derive(Default)] +struct PlcFake { + reject_posts: bool, + ops: Vec<(String, serde_json::Value)>, + post_count: usize, + tombstones: std::collections::HashSet, + last_override: Option, +} + +fn reply(status: StatusCode, body: Bytes) -> HttpResponse { + HttpResponse { + status, + headers: http::HeaderMap::new(), + body, + } +} + +fn plc_stage(fake: Arc>, sec1: Vec) -> Responder { + let pds = KnotServiceUrl::new(format!("https://{KNOT_HOST}")).unwrap(); + Box::new(move |request: &HttpRequest| { + let host = request.url.host_str().unwrap_or_default(); + if host != "plc.directory" { + return Ok(reply( + StatusCode::OK, + did_web_doc( + &Did::new_owned(format!("did:web:{host}")).unwrap(), + &sec1, + &pds, + ), + )); + } + let mut fake = fake.lock(); + if request.method == http::Method::POST { + fake.post_count += 1; + if fake.reject_posts { + return Ok(reply(StatusCode::BAD_REQUEST, Bytes::new())); + } + let did = request.url.path().trim_start_matches('/').to_owned(); + let operation = + serde_json::from_slice(request.body.as_ref().expect("Post body")).unwrap(); + fake.ops.push((did, operation)); + return Ok(reply(StatusCode::OK, Bytes::new())); + } + let path = request.url.path().to_owned(); + match path + .strip_suffix("/log/last") + .map(|did| did.trim_start_matches('/')) + { + None => Ok(reply(StatusCode::NOT_FOUND, Bytes::new())), + Some(did) => { + if fake.tombstones.contains(did) { + return Ok(reply( + StatusCode::OK, + Bytes::from( + serde_json::to_vec(&json!({ + "type": "plc_tombstone", + "sig": "AAAA", + })) + .unwrap(), + ), + )); + } + let last = match &fake.last_override { + Some(op) => Some(op.clone()), + None => fake + .ops + .iter() + .rev() + .find(|(entry_did, _)| entry_did == did) + .map(|(_, op)| op.clone()), + }; + match last { + None => Ok(reply(StatusCode::NOT_FOUND, Bytes::new())), + Some(op) => Ok(reply( + StatusCode::OK, + Bytes::from(serde_json::to_vec(&op).unwrap()), + )), + } + } + } + }) +} + +struct SweptRepo { + _dir: TempDir, + state: SharedState, + clock: Arc, + did: RepoDid, + fake: Arc>, + create_status: StatusCode, +} + +async fn minted_repo(fake: PlcFake) -> SweptRepo { + let admin = actor(1, ADMIN_HOST); + let key = admin.signer.public_key().as_bytes().to_vec(); + let fake = Arc::new(Mutex::new(fake)); + let responder = plc_stage(Arc::clone(&fake), key); + let (_dir, state, clock) = build_state_with_clock(responder, true); + let create_status = into_response( + crate::repos::create_repo( + State(Arc::clone(&state)), + bearer(&mint(&admin, &nsid(CREATE))), + crate::Method::from_nsid(CREATE), + body(json!({ "name": "anemone" })), + ) + .await, + ) + .status(); + let owner = OwnerDid::new(format!("did:web:{ADMIN_HOST}")).unwrap(); + let did = match state + .index + .resolve_repo(&owner, &RepoRkey::new("anemone").unwrap()) + { + Resolved::Ready(Some(did)) => did, + other => panic!("Repo wasn't registered despite the queue: {other:?}"), + }; + fake.lock().post_count = 0; + SweptRepo { + _dir, + state, + clock, + did, + fake, + create_status, + } +} + +fn grant_body( + repo: &RepoDid, + collection: &RecordCollection, + subject: &AccountDid, +) -> serde_json::Value { + json!({ + "repo": repo, + "collection": collection, + "rkey": subject, + "record": { + "$type": collection, + "createdAt": "2026-09-11T00:00:00Z", + }, + }) +} + +async fn offer(world: &World, repo: Option<&RepoDid>, subject: &AccountDid) -> StatusCode { + match repo { + None => offer_grant(world, &world.admin, &member_collection(), subject).await, + Some(repo) => offer_collaborator_as(world, &world.member, repo, subject).await, + } +} + +async fn offer_grant( + world: &World, + actor: &Actor, + collection: &RecordCollection, + subject: &AccountDid, +) -> StatusCode { + let knot_repo = world.state.knot_repo.clone(); + call( + world, + crate::social::create_record, + actor, + &nsid(CREATE_RECORD), + grant_body(&knot_repo, collection, subject), + ) + .await + .status() +} + +async fn offer_block(world: &World, actor: &Actor, subject: &AccountDid) -> StatusCode { + offer_grant(world, actor, &block_collection(), subject).await +} + +async fn revoke( + world: &World, + actor: &Actor, + collection: &RecordCollection, + subject: &AccountDid, +) -> StatusCode { + let knot_repo = world.state.knot_repo.clone(); + call( + world, + crate::social::delete_record, + actor, + &nsid(DELETE_RECORD), + json!({ "repo": knot_repo, "collection": collection, "rkey": subject.as_str() }), + ) + .await + .status() +} + +async fn offer_collaborator_as( + world: &World, + actor: &Actor, + repo: &RepoDid, + subject: &AccountDid, +) -> StatusCode { + on_repo( + world, + crate::social::create_record, + actor, + &nsid(CREATE_RECORD), + repo, + grant_body(repo, &collaborator_collection(), subject), + ) + .await + .0 +} + +async fn revoke_collaborator( + world: &World, + actor: &Actor, + repo: &RepoDid, + subject: &AccountDid, +) -> StatusCode { + on_repo( + world, + crate::social::delete_record, + actor, + &nsid(DELETE_RECORD), + repo, + json!({ "repo": repo, "collection": COLLABORATOR, "rkey": subject.as_str() }), + ) + .await + .0 +} + +async fn create_repo_helper(world: &World, name: &str) -> RepoDid { + assert_eq!( + as_member( + world, + crate::repos::create_repo, + &nsid(CREATE), + json!({ "name": name }) + ) + .await + .status(), + StatusCode::OK + ); + match resolve(world, &RepoName::new(name).unwrap()) { + Resolved::Ready(Some(did)) => did, + other => panic!("Repo {name} wasn't registered: {other:?}"), + } +} + +async fn create_status(world: &World, actor: &Actor, value: serde_json::Value) -> StatusCode { + call( + world, + crate::repos::create_repo, + actor, + &nsid(CREATE), + value, + ) + .await + .status() +} + +async fn reserve_status(world: &World, actor: &Actor, did: &RepoDid) -> StatusCode { + call( + world, + crate::repos::reserve_key, + actor, + &nsid(RESERVE), + json!({ "repo": did }), + ) + .await + .status() +} + +async fn reserve_repo_key(world: &World, did: &RepoDid) -> String { + let response = as_member( + world, + crate::repos::reserve_key, + &nsid(RESERVE), + json!({ "repo": did }), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let key = json_of(response).await["key"].as_str().unwrap().to_string(); + let host = did.as_str().strip_prefix("did:web:").unwrap(); + world.publish_repo_doc(host, &key); + key +} + +fn manifest_collection() -> RecordCollection { + RecordCollection::new(MANIFEST).unwrap() +} + +fn manifest_rkey() -> RecordRkey { + RecordRkey::new("self").unwrap() +} + +async fn live_manifest( + world: &World, + repo: &RepoDid, +) -> (serde_json::Value, crate::out::CidString) { + let (status, served) = record_at( + world, + repo, + &at_of(repo, manifest_collection(), &manifest_rkey()), + None, + ) + .await; + assert_eq!(status, StatusCode::OK, "the manifest serves: {served}"); + (served["value"].clone(), cid_of(&served)) +} + +async fn put_manifest_as( + world: &World, + actor: &Actor, + repo: &RepoDid, + edit: impl FnOnce(&mut serde_json::Value), +) -> (StatusCode, serde_json::Value) { + let (mut record, from) = live_manifest(world, repo).await; + edit(&mut record); + put_as( + world, + actor, + repo, + &manifest_collection(), + manifest_rkey(), + json!({ "swapRecord": from }), + record, + ) + .await +} + +async fn set_policy_as( + world: &World, + actor: &Actor, + repo: &RepoDid, + policy: knot_types::ContributionPolicy, +) -> StatusCode { + put_manifest_as(world, actor, repo, |record| { + record["contributionPolicy"] = json!(policy); + }) + .await + .0 +} + +async fn rename_repo_as( + world: &World, + actor: &Actor, + repo: &RepoDid, + rkey: &RepoRkey, +) -> StatusCode { + put_manifest_as(world, actor, repo, |record| { + record["declaration"]["name"] = json!(rkey.as_str()); + }) + .await + .0 +} + +async fn set_default_branch_as( + world: &World, + actor: &Actor, + repo: &RepoDid, + branch: &BranchName, +) -> StatusCode { + on_repo( + world, + crate::branches::set_default_branch, + actor, + &nsid(SET_DEFAULT_BRANCH), + repo, + json!({ "repo": repo, "branch": branch }), + ) + .await + .0 +} + +#[tokio::test] +async fn a_repository_takes_the_knots_default_policy_and_its_owner_or_the_operator_moves_it() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo = create_repo_helper(&world, "anemone").await; + let policy_now = || { + world + .state + .index + .contribution_policy(&repo) + .map(|found| found.map(|record| record.policy())) + }; + + assert_eq!( + policy_now(), + Resolved::Ready(Some(ContributionPolicy::Anyone)), + "Fresh repository takes the policy the operator configured" + ); + assert_eq!( + set_policy_as( + &world, + &world.stranger, + &repo, + ContributionPolicy::Collaborators + ) + .await, + StatusCode::FORBIDDEN, + "Stranger doesn't decide who may write to somebody else's repository" + ); + assert_eq!( + set_policy_as( + &world, + &world.member, + &repo, + ContributionPolicy::Collaborators + ) + .await, + StatusCode::OK + ); + assert_eq!( + policy_now(), + Resolved::Ready(Some(ContributionPolicy::Collaborators)), + "Projection answers new policy on the very next read" + ); + assert_eq!( + set_policy_as(&world, &world.admin, &repo, ContributionPolicy::Anyone).await, + StatusCode::FORBIDDEN, + "settings are the manifest's, and the manifest belongs to the repository's owner alone" + ); + assert_eq!( + policy_now(), + Resolved::Ready(Some(ContributionPolicy::Collaborators)), + "a refused settings write doesn't move the policy" + ); + assert_eq!( + set_policy_as(&world, &world.member, &repo, ContributionPolicy::Anyone).await, + StatusCode::OK, + "re-sending the current policy doesn't write a change. It still returns 200" + ); +} + +#[tokio::test] +async fn a_second_write_inside_one_account_s_burst_answers_429_named_by_bucket() { + let world = World::with_write_burst(1); + world.grandfather(MEMBER_HOST); + let repo = create_repo_helper(&world, "anemone").await; + let (status, body) = put_manifest_as(&world, &world.member, &repo, |record| { + record["contributionPolicy"] = json!(ContributionPolicy::Collaborators); + }) + .await; + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "create spent member's one token: next write is over the burst: {body}" + ); + assert_eq!(body["bucket"], "actor-writes"); + assert_eq!( + create_status(&world, &world.admin, json!({ "name": "anemone2" })).await, + StatusCode::OK, + "The limit is the writer's, leaving another account's token untouched" + ); +} + +const CREATE_RECORD: &str = "com.atproto.repo.createRecord"; +const PUT_RECORD: &str = "com.atproto.repo.putRecord"; +const DELETE_RECORD: &str = "com.atproto.repo.deleteRecord"; +const TICKET: &str = "org.tangled.track.ticket"; +const CLOSED: &str = "closed"; + +async fn on_repo( + world: &World, + handler: F, + actor: &Actor, + nsid: &Nsid, + repo: &RepoDid, + value: serde_json::Value, +) -> (StatusCode, serde_json::Value) +where + F: FnOnce(State, HeaderMap, crate::Method, Bytes) -> Fut, + Fut: Future>, +{ + let token = mint_for(actor, nsid, &ServiceDid::new(repo.as_str()).unwrap()); + let response = into_response( + handler( + world.state(), + bearer(&token), + crate::Method::from_nsid(nsid.as_str()), + body(value), + ) + .await, + ); + let status = response.status(); + (status, json_of(response).await) +} + +fn merged(mut base: serde_json::Value, extra: serde_json::Value) -> serde_json::Value { + match (&mut base, extra) { + (serde_json::Value::Object(fields), serde_json::Value::Object(extra)) => { + fields.extend(extra); + base + } + _ => base, + } +} + +fn retype(mut record: serde_json::Value, kind: &str) -> serde_json::Value { + record["$type"] = json!(kind); + record +} + +fn ticket_record(repo: &RepoDid, title: &str, text: &str) -> serde_json::Value { + json!({ + "$type": TICKET, + "repo": repo, + "title": title, + "body": text, + "createdAt": "2026-09-11T00:00:00Z" + }) +} + +async fn create_as( + world: &World, + actor: &Actor, + repo: &RepoDid, + collection: &RecordCollection, + rkey: Option, + record: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + let mut value = json!({ "repo": repo, "collection": collection, "record": record }); + if let Some(rkey) = rkey { + value["rkey"] = json!(rkey); + } + on_repo( + world, + crate::social::create_record, + actor, + &nsid(CREATE_RECORD), + repo, + value, + ) + .await +} + +async fn submit_patch_as( + world: &World, + actor: &Actor, + repo: &RepoDid, + patch: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + on_repo( + world, + crate::social::submit_patch, + actor, + &nsid("org.tangled.review.submitPatch"), + repo, + json!({ "repo": repo, "patch": patch }), + ) + .await +} + +async fn put_as( + world: &World, + actor: &Actor, + repo: &RepoDid, + collection: &RecordCollection, + rkey: RecordRkey, + swap: serde_json::Value, + record: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + let value = merged( + json!({ "repo": repo, "collection": collection, "rkey": rkey, "record": record }), + swap, + ); + on_repo( + world, + crate::social::put_record, + actor, + &nsid(PUT_RECORD), + repo, + value, + ) + .await +} + +async fn delete_as( + world: &World, + actor: &Actor, + repo: &RepoDid, + collection: &RecordCollection, + rkey: RecordRkey, + swap: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + let value = merged( + json!({ "repo": repo, "collection": collection, "rkey": rkey }), + swap, + ); + on_repo( + world, + crate::social::delete_record, + actor, + &nsid(DELETE_RECORD), + repo, + value, + ) + .await +} + +struct Ticket<'a> { + world: &'a World, + repo: &'a RepoDid, +} + +impl<'a> Ticket<'a> { + fn on(world: &'a World, repo: &'a RepoDid) -> Self { + Self { world, repo } + } + + async fn open( + &self, + actor: &Actor, + title: &str, + text: &str, + ) -> (StatusCode, serde_json::Value) { + create_as( + self.world, + actor, + self.repo, + &RecordCollection::new(TICKET).unwrap(), + None, + ticket_record(self.repo, title, text), + ) + .await + } + + async fn opened(&self, actor: &Actor, title: &str) -> (AtUri, crate::out::CidString) { + let written = ok(self.open(actor, title, "a forest").await); + (uri_of(&written), cid_of(&written)) + } + + async fn create( + &self, + actor: &Actor, + record: serde_json::Value, + ) -> (StatusCode, serde_json::Value) { + create_as( + self.world, + actor, + self.repo, + &RecordCollection::new(TICKET).unwrap(), + None, + record, + ) + .await + } + + async fn refused_create(&self, record: serde_json::Value) -> serde_json::Value { + failed( + self.create(&self.world.stranger, record).await, + StatusCode::BAD_REQUEST, + ) + } + + async fn put( + &self, + actor: &Actor, + uri: &AtUri, + from: &crate::out::CidString, + record: serde_json::Value, + ) -> (StatusCode, serde_json::Value) { + put_as( + self.world, + actor, + self.repo, + &RecordCollection::new(TICKET).unwrap(), + rkey_of(uri), + json!({ "swapRecord": from }), + record, + ) + .await + } + + async fn edit( + &self, + actor: &Actor, + uri: &AtUri, + from: &crate::out::CidString, + title: &str, + ) -> (StatusCode, serde_json::Value) { + self.put( + actor, + uri, + from, + ticket_record(self.repo, title, "kelp forest"), + ) + .await + } + + async fn set_state( + &self, + actor: &Actor, + uri: &AtUri, + from: &crate::out::CidString, + state: TicketState, + ) -> (StatusCode, serde_json::Value) { + let mut record = ticket_record(self.repo, "kelp", "a forest"); + record["state"] = json!(state); + self.put(actor, uri, from, record).await + } + + async fn erase( + &self, + actor: &Actor, + uri: &AtUri, + from: &crate::out::CidString, + ) -> (StatusCode, serde_json::Value) { + delete_as( + self.world, + actor, + self.repo, + &RecordCollection::new(TICKET).unwrap(), + rkey_of(uri), + json!({ "swapRecord": from }), + ) + .await + } +} + +struct TicketDraft { + record: serde_json::Value, +} + +impl TicketDraft { + fn fresh(repo: &RepoDid) -> Self { + Self { + record: ticket_record(repo, "kelp", "a forest"), + } + } + + fn labeled(mut self) -> Self { + self.record["labels"] = + json!({ "$type": "com.atproto.label.defs#selfLabels", "values": [] }); + self + } + + fn misdeclared_title(mut self) -> Self { + self.record["title"] = json!({ "$type": "org.tangled.markup.markdown", "text": "kelp" }); + self + } + + fn flooded_body(mut self) -> Self { + self.record["body"] = json!({ + "$type": "org.tangled.markup.markdown", + "text": "k".repeat(knot_record::ticket::MAX_BODY_BYTES + 1), + }); + self + } + + fn dense_body(mut self) -> Self { + self.record["body"] = json!({ + "$type": "org.tangled.markup.markdown", + "text": "e\u{301}".repeat(knot_record::ticket::MAX_BODY_GRAPHEMES + 1), + }); + self + } + + fn citing(mut self, key: &str) -> Self { + self.record["tangledLabels"] = json!([{ "key": key }]); + self + } + + fn on(mut self, branch: &str) -> Self { + self.record["targetBranch"] = json!(branch); + self + } +} + +fn rkey_of(uri: &AtUri) -> RecordRkey { + RecordRkey::new(uri.rkey().expect("An AT-URI ends in a record key").as_ref()).unwrap() +} + +fn record_of( + repo: &RepoDid, + uri: &AtUri, + cid: Option, +) -> crate::atproto::RepoRecordParams { + crate::atproto::RepoRecordParams { + repo: crate::atproto::RepoSubject::Did(repo.clone()), + collection: knot_types::RecordCollection::new( + uri.collection() + .expect("An AT-URI has a collection") + .as_str(), + ) + .unwrap(), + rkey: knot_types::UriRkey::new(rkey_of(uri).as_str()).unwrap(), + cid, + } +} + +async fn record_at( + world: &World, + repo: &RepoDid, + uri: &AtUri, + cid: Option<&crate::out::CidString>, +) -> (StatusCode, serde_json::Value) { + let response = into_response( + crate::atproto::repo_get_record( + world.state(), + crate::query::ValidatedQuery(record_of(repo, uri, cid.cloned())), + ) + .await, + ); + let status = response.status(); + (status, json_of(response).await) +} + +async fn records_in(world: &World, repo: &RepoDid, collection: &RecordCollection) -> Vec { + let params: crate::atproto::ListRecordsParams = + serde_json::from_value(json!({ "repo": repo, "collection": collection })).unwrap(); + let listed = json_of( + crate::atproto::repo_list_records(world.state(), crate::query::ValidatedQuery(params)) + .await + .unwrap(), + ) + .await; + listed["records"] + .as_array() + .unwrap() + .iter() + .map(|record| record["uri"].as_str().unwrap().to_owned()) + .collect() +} + +async fn block_served(world: &World, repo: &RepoDid, cid: &crate::out::CidString) -> bool { + let response = crate::atproto::sync_get_blocks( + world.state(), + crate::atproto::BlocksQuery { + did: crate::atproto::RepoSubject::Did(repo.clone()), + cids: vec![cid.cid()], + }, + ) + .await + .unwrap(); + let car = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + knot_record::car_blocks(&car) + .unwrap() + .blocks + .keys() + .any(|block| block.to_string() == cid.to_string()) +} + +async fn versions_page( + world: &World, + uri: &AtUri, + paging: Option<(usize, usize)>, +) -> serde_json::Value { + versions_ordered(world, uri, paging, Default::default()).await +} + +const NEWEST_FIRST: crate::social::VersionOrder = crate::social::VersionOrder::NewestFirst; + +async fn versions_ordered( + world: &World, + uri: &AtUri, + paging: Option<(usize, usize)>, + order: crate::social::VersionOrder, +) -> serde_json::Value { + let (limit, cursor) = paging.map_or_else(Default::default, |(limit, cursor)| { + ( + crate::query::Limit::literal(limit), + crate::query::Offset::new(cursor), + ) + }); + json_of(into_response( + crate::social::list_record_versions( + world.state(), + crate::query::ValidatedQuery(crate::social::VersionsQuery { + uri: uri.clone(), + limit, + cursor, + order, + }), + ) + .await, + )) + .await +} + +async fn names_its_change(world: &World, written: &serde_json::Value) { + let newest = versions_ordered(world, &uri_of(written), Some((1, 0)), NEWEST_FIRST).await; + assert!(written["x-tngl-change"].is_string(), "{written}"); + assert_eq!( + written["x-tngl-change"], newest["versions"][0]["changeId"], + "a write ought to name the change the chain lists for the version it made" + ); +} + +fn replayed_blocks(world: &World) -> BTreeSet { + world + .state + .firehose + .replay( + knot_events::EventCursor::START, + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(256).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + ) + .events + .into_iter() + .flat_map(|event| crate::firehose::commit_blocks(&event.frame)) + .collect() +} + +fn ok((status, body): (StatusCode, serde_json::Value)) -> serde_json::Value { + assert_eq!(status, StatusCode::OK, "{body}"); + body +} + +fn failed((status, body): (StatusCode, serde_json::Value), want: StatusCode) -> serde_json::Value { + assert_eq!(status, want, "{body}"); + body +} + +fn message_contains(body: &serde_json::Value, fragments: &[&str]) { + let message = body["message"].as_str().unwrap_or_default(); + assert!( + fragments.iter().all(|fragment| message.contains(fragment)), + "The answer oughtn't miss any of {fragments:?}: {body}" + ); +} + +async fn ticket_world() -> (World, RepoDid) { + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo = create_repo_helper(&world, "anemone").await; + (world, repo) +} + +fn uri_of(body: &serde_json::Value) -> AtUri { + AtUri::new_owned(body["uri"].as_str().unwrap()).unwrap() +} + +fn cid_of(body: &serde_json::Value) -> crate::out::CidString { + serde_json::from_value(body["cid"].clone()).unwrap() +} + +#[tokio::test] +async fn only_a_collaborator_erases_a_ticket() { + let (world, repo) = ticket_world().await; + let ticket = Ticket::on(&world, &repo); + let written = ok(ticket.open(&world.stranger, "kelp", "a forest").await); + let (uri, cid) = (uri_of(&written), cid_of(&written)); + + let refused = failed( + ticket.erase(&world.stranger, &uri, &cid).await, + StatusCode::FORBIDDEN, + ); + message_contains(&refused, &["only a collaborator can erase this ticket"]); + ok(ticket.erase(&world.member, &uri, &cid).await); +} + +#[tokio::test] +async fn stranger_opens_tickets_until_policy_closes_and_reads_collaborators_remedy() { + let (world, repo) = ticket_world().await; + let ticket = Ticket::on(&world, &repo); + + let written = ok(ticket.open(&world.stranger, "kelp", "a forest").await); + let uri = uri_of(&written); + assert!( + uri.as_str() + .starts_with(&format!("at://{}/{TICKET}/", repo.as_str())) + ); + assert!(written["cid"].as_str().unwrap().starts_with("bafyrei")); + assert!( + written["commit"]["cid"] + .as_str() + .unwrap() + .starts_with("bafyrei"), + "The answer oughtta include the signed commit stock clients expect" + ); + assert!(!written["commit"]["rev"].as_str().unwrap().is_empty()); + assert_eq!(written["validationStatus"], "valid"); + + let record = ok(record_at(&world, &repo, &uri, None).await); + assert_eq!( + record["cid"], written["cid"], + "getRecord answers the version createRecord wrote" + ); + assert_eq!(record["value"]["$type"], TICKET); + assert_eq!(record["value"]["title"]["text"], "kelp"); + assert_eq!(record["value"]["body"]["text"], "a forest"); + assert_eq!( + record["value"]["x-tngl-editor"], + world.stranger.did.as_str(), + "the editor ought to be the authenticated writer, never the repo DID" + ); + assert_eq!( + records_in(&world, &repo, &RecordCollection::new(TICKET).unwrap()).await, + vec![uri.as_str().to_owned()] + ); + + assert_eq!( + set_policy_as( + &world, + &world.member, + &repo, + ContributionPolicy::Collaborators + ) + .await, + StatusCode::OK + ); + let rejected = failed( + ticket.open(&world.stranger, "urchin", "").await, + StatusCode::FORBIDDEN, + ); + message_contains( + &rejected, + &["stranger", "collaborators", "add you as a collaborator"], + ); + ok(ticket.open(&world.member, "urchin", "").await); +} + +#[tokio::test] +async fn stock_routes_dispatch_on_collection_and_reject_foreign_collections() { + let (world, repo) = ticket_world().await; + let (uri, cid) = Ticket::on(&world, &repo) + .opened(&world.stranger, "kelp") + .await; + + let minted = failed( + create_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(TICKET).unwrap(), + Some(rkey_of(&uri)), + ticket_record(&repo, "again", ""), + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&minted, &["mints the record key"]); + + futures::stream::iter([ + ( + "cafe.oyster.gripe", + Some("3lubrptx57d22".to_owned()), + StatusCode::BAD_REQUEST, + vec!["isn't a collection this knot takes"], + ), + ("cafe.oyster.gripe", None, StatusCode::BAD_REQUEST, vec![]), + ( + "org.tangled.knot.member", + Some("did:plc:whelk".to_owned()), + StatusCode::BAD_REQUEST, + vec!["the write is for", "but the record's repo is"], + ), + ]) + .for_each(|(collection, rkey, want, fragments)| { + let (world, repo) = (&world, &repo); + async move { + let record = match collection { + "org.tangled.knot.member" => { + json!({ "$type": collection, "createdAt": "2026-09-11T00:00:00Z" }) + } + other => retype(ticket_record(repo, "no", ""), other), + }; + let rejected = failed( + create_as( + world, + &world.stranger, + repo, + &RecordCollection::new(collection).unwrap(), + rkey.map(|rkey| RecordRkey::new(rkey).unwrap()), + record, + ) + .await, + want, + ); + message_contains(&rejected, &fragments); + } + }) + .await; + + let refused = create_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(COLLABORATOR).unwrap(), + Some(RecordRkey::new("did:plc:whelk").unwrap()), + json!({ "$type": COLLABORATOR, "createdAt": "2026-09-11T00:00:00Z" }), + ) + .await; + assert_eq!( + refused.0, + StatusCode::FORBIDDEN, + "a collaborator grant is the repository owner's to give: {refused:?}" + ); + + let wrong_type = failed( + create_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(TICKET).unwrap(), + None, + retype(ticket_record(&repo, "kelp", ""), "org.tangled.review.patch"), + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&wrong_type, &["$type"]); + + let swap_commit = failed( + delete_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(TICKET).unwrap(), + rkey_of(&uri), + json!({ "swapCommit": cid }), + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&swap_commit, &["swapRecord"]); + + let no_swap = failed( + delete_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(TICKET).unwrap(), + rkey_of(&uri), + json!({}), + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&no_swap, &["swapRecord"]); + + let edit_state = failed( + put_as( + &world, + &world.member, + &repo, + &RecordCollection::new("org.tangled.feed.reaction").unwrap(), + rkey_of(&uri), + json!({ "swapRecord": cid }), + json!({ + "$type": "org.tangled.feed.reaction", + "subject": { "uri": uri, "cid": cid }, + "reaction": "❤️", + "createdAt": "2026-09-11T00:00:00Z" + }), + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&edit_state, &["never edited"]); +} + +#[tokio::test] +async fn a_record_with_one_version_lists_the_same_either_way() { + let (world, repo) = ticket_world().await; + let written = ok(Ticket::on(&world, &repo) + .open(&world.stranger, "kelp", "a forest") + .await); + let uri = uri_of(&written); + + let oldest = versions_page(&world, &uri, None).await; + let newest = versions_ordered(&world, &uri, None, NEWEST_FIRST).await; + + assert_eq!(oldest["versions"], newest["versions"]); + assert_eq!(oldest["versions"].as_array().unwrap().len(), 1); + assert_eq!( + (oldest["total"].clone(), newest["total"].clone()), + (1.into(), 1.into()) + ); + assert!(oldest.get("cursor").is_none() && newest.get("cursor").is_none()); + assert_eq!(oldest["versions"][0]["changeId"], written["x-tngl-change"]); +} + +#[test] +fn versions_query_reads_oldest_first_unless_told_newest() { + use crate::social::VersionOrder::{NewestFirst, OldestFirst}; + let order_of = |query: &str| { + let uri = format!( + "http://knot/xrpc/sh.tangled.repo.listCobVersions?uri=at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/{TICKET}/3labc{query}" + ); + axum::extract::Query::::try_from_uri(&uri.parse().unwrap()) + .map(|parsed| parsed.0.order) + }; + + assert_eq!(order_of("").unwrap(), OldestFirst); + assert_eq!(order_of("&order=asc").unwrap(), OldestFirst); + assert_eq!(order_of("&order=").unwrap(), OldestFirst); + assert_eq!(order_of("&order=desc").unwrap(), NewestFirst); + assert!(order_of("&order=sideways").is_err()); +} + +#[tokio::test] +async fn rewrites_chain_editors_and_erasure_tombstones_record_and_bytes() { + let (world, repo) = ticket_world().await; + let ticket = Ticket::on(&world, &repo); + let first = ok(ticket.open(&world.stranger, "kelp", "a forest").await); + let uri = uri_of(&first); + let v1 = cid_of(&first); + assert!( + block_served(&world, &repo, &v1).await, + "before the delete, getBlocks should serve the leaf from the tree" + ); + assert!( + replayed_blocks(&world).contains(&v1.cid()), + "Before the delete, replay off the ring oughtta serve the leaf" + ); + + let second = ok(ticket.edit(&world.stranger, &uri, &v1, "kelp, again").await); + let v2 = cid_of(&second); + assert_ne!(v1, v2); + + let stale = failed( + ticket + .edit(&world.stranger, &uri, &v1, "kelp, thrice") + .await, + StatusCode::CONFLICT, + ); + assert_eq!(stale["error"], "InvalidSwap"); + assert_eq!(stale["uri"], uri.as_str()); + assert_eq!( + stale["cid"], + v2.to_string(), + "the 409 ought to return the version that stands" + ); + failed( + ticket + .edit(&world.passerby, &uri, &v2, "kelp, by the passerby") + .await, + StatusCode::FORBIDDEN, + ); + assert_eq!( + records_in(&world, &repo, &RecordCollection::new(TICKET).unwrap()) + .await + .len(), + 1 + ); + + let same = ok(ticket.edit(&world.stranger, &uri, &v2, "kelp, again").await); + assert_eq!( + same["cid"], + v2.to_string(), + "Without writing, an edit that doesn't change anything oughtta answer the standing version" + ); + assert_eq!(same["commit"], second["commit"]); + assert!( + same.get("x-tngl-change").is_none(), + "An answer that replays a version, not writes one, has no change to name" + ); + + let third = ok(ticket + .edit(&world.member, &uri, &v2, "kelp, by the owner") + .await); + let v3 = cid_of(&third); + let record = ok(record_at(&world, &repo, &uri, None).await); + assert_eq!(record["cid"], v3.to_string()); + assert_eq!(record["value"]["title"]["text"], "kelp, by the owner"); + assert_eq!( + record["value"]["x-tngl-editor"], + world.member.did.as_str(), + "The version ought to answer with the maintainer who wrote it" + ); + let requested = ok(record_at(&world, &repo, &uri, Some(&v3)).await); + assert_eq!( + requested["cid"], + v3.to_string(), + "A CID matching the standing version should serve it" + ); + let superseded = failed( + record_at(&world, &repo, &uri, Some(&v2)).await, + StatusCode::BAD_REQUEST, + ); + assert_eq!( + superseded["error"], "RecordNotFound", + "A CID from a superseded version must answer as a record that left" + ); + + failed( + delete_as( + &world, + &world.admin, + &repo, + &RecordCollection::new(TICKET).unwrap(), + rkey_of(&uri), + json!({ "swapRecord": "bafyreidoesnotparse" }), + ) + .await, + StatusCode::BAD_REQUEST, + ); + let deleted = ok(ticket.erase(&world.member, &uri, &v3).await); + assert!( + deleted.get("uri").is_none() && deleted.get("cid").is_none(), + "Tombstone answer oughtta be the stock commit alone" + ); + assert!( + deleted["commit"]["cid"] + .as_str() + .unwrap() + .starts_with("bafyrei") + ); + + let (status, gone) = record_at(&world, &repo, &uri, None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(gone["error"], "RecordNotFound"); + assert!( + records_in(&world, &repo, &RecordCollection::new(TICKET).unwrap()) + .await + .is_empty() + ); + assert!( + block_served(&world, &repo, &v1).await, + "The block tree still keeps the prose git can't delete" + ); + let versions = versions_page(&world, &uri, None).await; + let chain = versions["versions"].as_array().unwrap(); + assert_eq!( + chain.len(), + 4, + "with the tombstone last, the chain must list every version" + ); + assert_eq!(chain[0]["cid"], v1.to_string()); + assert_eq!( + chain[0]["editor"], + world.stranger.did.as_str(), + "Chain lists the opener first" + ); + assert_eq!(chain[1]["cid"], v2.to_string()); + assert_eq!(chain[1]["editor"], world.stranger.did.as_str()); + assert_eq!(chain[2]["cid"], v3.to_string()); + assert_eq!(chain[2]["editor"], world.member.did.as_str()); + assert!(chain[3].get("cid").is_none()); + assert_eq!(chain[3]["editor"], world.member.did.as_str()); + assert_eq!(versions["total"], 4); + for (written, listed) in [ + (&first, &chain[0]), + (&second, &chain[1]), + (&third, &chain[2]), + ] { + assert_eq!( + written["x-tngl-change"], listed["changeId"], + "A write ought to name the change the chain lists for the version it made" + ); + } + + let newest = versions_ordered(&world, &uri, Some((2, 0)), NEWEST_FIRST).await; + let newest_chain = newest["versions"].as_array().unwrap(); + assert_eq!( + (newest_chain[0].clone(), newest_chain[1].clone()), + (chain[3].clone(), chain[2].clone()), + "desc starts at the tombstone and walks back" + ); + assert_eq!( + (newest["total"].clone(), newest["cursor"].clone()), + (4.into(), "2".into()) + ); + let oldest = versions_ordered(&world, &uri, Some((2, 2)), NEWEST_FIRST).await; + assert_eq!(oldest["versions"][1], chain[0]); + assert!(oldest.get("cursor").is_none()); + let beyond = versions_ordered(&world, &uri, Some((2, 9)), NEWEST_FIRST).await; + assert_eq!( + ( + beyond["versions"].as_array().unwrap().len(), + beyond["total"].clone() + ), + (0, 4.into()), + "a cursor past the end is an empty page that still says how many there are" + ); + assert!(beyond.get("cursor").is_none()); + + let first = versions_page(&world, &uri, Some((2, 0))).await; + assert_eq!(first["versions"].as_array().unwrap().len(), 2); + assert_eq!(first["cursor"], "2"); + let last = versions_page(&world, &uri, Some((2, 2))).await; + assert_eq!( + ( + last["versions"].as_array().unwrap().len(), + last["cursor"].clone() + ), + (2, serde_json::Value::Null), + "a paged chain ought to be the whole chain in the same order, and the last page hands \ + back no cursor" + ); + + let again = failed( + ticket.erase(&world.member, &uri, &v1).await, + StatusCode::NOT_FOUND, + ); + assert_eq!(again["message"], format!("ticket at {uri} was deleted")); + assert!( + replayed_blocks(&world).contains(&v1.cid()), + "Replay serves the stored frame, whose create commit still includes the leaf" + ); +} + +async fn car_blocks_of(response: Response) -> std::collections::BTreeMap { + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + knot_record::car_blocks(&body) + .unwrap() + .blocks + .into_iter() + .map(|(cid, bytes)| (cid.to_string(), bytes)) + .collect() +} + +async fn sync_record_blocks( + world: &World, + repo: &RepoDid, + uri: &AtUri, +) -> std::collections::BTreeMap { + let response = crate::atproto::sync_get_record( + world.state(), + crate::query::ValidatedQuery(crate::atproto::SyncRecordParams { + did: crate::atproto::RepoSubject::Did(repo.clone()), + collection: knot_types::RecordCollection::new( + uri.collection() + .expect("An AT-URI has a collection") + .as_str(), + ) + .unwrap(), + rkey: knot_types::UriRkey::new(rkey_of(uri).as_str()).unwrap(), + }), + ) + .await + .unwrap(); + car_blocks_of(response).await +} + +async fn repo_export_blocks( + world: &World, + repo: &RepoDid, + since: Option, +) -> std::collections::BTreeMap { + let since = since.map(|sent| { + serde_json::from_value::(serde_json::Value::String(sent)) + .expect("a rev string parses as a TID") + }); + let response = crate::atproto::sync_get_repo( + world.state(), + crate::query::ValidatedQuery(crate::atproto::RepoExportParams { + did: crate::atproto::RepoSubject::Did(repo.clone()), + since, + }), + ) + .await + .unwrap(); + car_blocks_of(response).await +} + +fn tip_rev(world: &World, repo: &RepoDid) -> String { + let git = world.layout.open(repo).unwrap(); + knot_record::chain::tip(&git) + .unwrap() + .expect("ticket_world enables the chain") + .rev + .to_string() +} + +fn cob_version_bytes( + world: &World, + repo: &RepoDid, + uri: &AtUri, +) -> Vec<(Option, Option>)> { + let git = world.layout.open(repo).unwrap(); + let address = knot_types::RecordAddress::new( + knot_types::RecordCollection::new( + uri.collection() + .expect("An AT-URI has a collection") + .as_str(), + ) + .unwrap(), + rkey_of(uri), + ); + let object = match world.state.index.social_object_at(repo, &address) { + knot_index::Resolved::Ready(Some(object)) => object, + other => panic!("the written issue ought to project to an object, got {other:?}"), + }; + let type_name = knot_types::TypeName::new(TICKET).unwrap(); + let delta = knot_cob::CobStore::new(&git) + .changes_of(&type_name, object, knot_cob::HistoryModel::Convergent, None) + .unwrap(); + delta + .changes + .iter() + .map(|change| { + let payload = knot_cobs::decode_social(change.payload()).unwrap(); + let emission = knot_cobs::Social::emission(&payload); + ( + emission.produces().map(|cid| cid.to_string()), + change.record.bytes().map(<[u8]>::to_vec), + ) + }) + .collect() +} + +#[tokio::test] +async fn deleted_prose_stays_off_every_presenting_route_and_reachable_from_history() { + let (world, repo) = ticket_world().await; + let before = tip_rev(&world, &repo); + let ticket = Ticket::on(&world, &repo); + let first = ok(ticket.open(&world.stranger, "kelp", "a forest").await); + let uri = uri_of(&first); + let v1 = cid_of(&first); + let birth = first["commit"]["rev"].as_str().unwrap().to_owned(); + let second = ok(ticket.edit(&world.stranger, &uri, &v1, "kelp, again").await); + let v2 = cid_of(&second); + ok(ticket.erase(&world.member, &uri, &v2).await); + + let gone = failed( + record_at(&world, &repo, &uri, None).await, + StatusCode::BAD_REQUEST, + ); + assert_eq!(gone["error"], "RecordNotFound"); + let by_old_cid = failed( + record_at(&world, &repo, &uri, Some(&v1)).await, + StatusCode::BAD_REQUEST, + ); + assert_eq!( + by_old_cid["error"], "RecordNotFound", + "The deleted version's own CID mustn't resurrect it." + ); + assert!( + records_in(&world, &repo, &RecordCollection::new(TICKET).unwrap()) + .await + .is_empty(), + "listRecords must drop the row." + ); + let proof = sync_record_blocks(&world, &repo, &uri).await; + assert!( + !proof.is_empty(), + "sync.getRecord returns a proof of absence." + ); + assert!( + !proof.contains_key(&v1.to_string()) && !proof.contains_key(&v2.to_string()), + "sync.getRecord mustn't return either version's block." + ); + let chain = versions_page(&world, &uri, None).await; + let rows = chain["versions"].as_array().unwrap(); + assert_eq!(rows.len(), 3, "one create, one edit, one tombstone"); + rows.iter().for_each(|row| { + let keys = row.as_object().unwrap(); + assert!( + keys.keys().all(|key| key == "cid" + || key == "changeId" + || key == "editor" + || key == "createdAt"), + "Version row is metadata alone, not prose: {row}" + ); + }); + assert!( + rows[2].get("cid").is_none(), + "The tombstone row has no `cid`." + ); + + let mut served = repo_export_blocks(&world, &repo, None).await; + assert!( + !served.contains_key(&v1.to_string()) && !served.contains_key(&v2.to_string()), + "The current export omits the deleted blocks." + ); + served = repo_export_blocks(&world, &repo, Some(before)).await; + assert!( + served.contains_key(&v1.to_string()) && served.contains_key(&v2.to_string()), + "Diffing the export from before the create still returns both versions, the way \ + reflog keeps a reverted commit." + ); + let replay_since_create = repo_export_blocks(&world, &repo, Some(birth)).await; + assert!( + !replay_since_create.contains_key(&v1.to_string()), + "`since` at the create's own rev excludes that rev, matching the firehose cursor." + ); + + assert!( + block_served(&world, &repo, &v1).await && block_served(&world, &repo, &v2).await, + "getBlocks keeps serving both versions by CID." + ); + let blocks = crate::atproto::sync_get_blocks( + world.state(), + crate::atproto::BlocksQuery { + did: crate::atproto::RepoSubject::Did(repo.clone()), + cids: vec![v1.cid()], + }, + ) + .await + .unwrap(); + let served_v1 = car_blocks_of(blocks).await; + let change_tree_v1 = cob_version_bytes(&world, &repo, &uri) + .into_iter() + .find_map(|(cid, bytes)| (cid == Some(v1.to_string())).then_some(bytes)) + .flatten() + .expect("the change graph keeps the version's record blob"); + assert_eq!( + served_v1.get(&v1.to_string()).map(Bytes::as_ref), + Some(change_tree_v1.as_slice()), + "`refs/cobs/*` serves the same bytes `getBlocks` serves, so a mirror fetch reads them." + ); + assert!( + replayed_blocks(&world).contains(&v1.cid()), + "Firehose replay from before the create still includes the leaf." + ); +} + +#[tokio::test] +async fn an_edit_echoing_the_live_state_still_writes() { + let (world, repo) = ticket_world().await; + let ticket = Ticket::on(&world, &repo); + let (uri, v1) = ticket.opened(&world.stranger, "kelp").await; + let (_, _def) = wontfix(&world, &repo).await; + + let mut conformant = ticket_record(&repo, "kelp", "a forest"); + conformant["state"] = json!("open"); + conformant["tangledLabels"] = json!([{ "key": "wontfix" }]); + let edited = ok(ticket + .put(&world.stranger, &uri, &v1, conformant.clone()) + .await); + let v2 = cid_of(&edited); + let served = ok(record_at(&world, &repo, &uri, None).await); + assert_eq!( + served["value"]["tangledLabels"], + json!([{ "key": "wontfix" }]), + "a record echoing the live state can still change the rest" + ); + + let refused = failed( + ticket.put(&world.stranger, &uri, &v2, conformant).await, + StatusCode::CONFLICT, + ); + message_contains(&refused, &["already open"]); + + let mut without_repo = ticket_record(&repo, "urchin", "a shoal"); + without_repo + .as_object_mut() + .expect("a ticket draft is an object") + .remove("repo"); + let bare = ok(ticket.create(&world.stranger, without_repo).await); + let bare_uri = uri_of(&bare); + assert!( + bare_uri + .collection() + .is_some_and(|collection| collection.as_str() == TICKET), + "a draft without the dropped repo field still opens a ticket" + ); +} + +#[tokio::test] +async fn a_served_ticket_round_trips_through_put_record() { + let (world, repo) = ticket_world().await; + let ticket = Ticket::on(&world, &repo); + let (uri, v1) = ticket.opened(&world.stranger, "kelp").await; + let served = ok(record_at(&world, &repo, &uri, None).await); + let echoed = served["value"].clone(); + assert_eq!( + echoed["title"]["$type"], "org.tangled.markup.markdown#inline", + "the served title is the lexicon's inline union" + ); + let refused = failed( + ticket.put(&world.stranger, &uri, &v1, echoed.clone()).await, + StatusCode::CONFLICT, + ); + message_contains(&refused, &["already open"]); + + let mut retitled = echoed; + retitled["title"]["text"] = json!("kelp, again"); + ok(ticket.put(&world.stranger, &uri, &v1, retitled).await); + assert_eq!( + ok(record_at(&world, &repo, &uri, None).await)["value"]["title"]["text"], + serde_json::json!("kelp, again"), + "a ticket echoed back with a new title writes the new title" + ); + + let malformed = vec![ + ( + TicketDraft::fresh(&repo).labeled(), + vec!["self-labels", "leave labels out"], + ), + ( + TicketDraft::fresh(&repo).misdeclared_title(), + vec!["a ticket's title is org.tangled.markup.markdown#inline"], + ), + ( + TicketDraft::fresh(&repo).flooded_body(), + vec!["Ticket body's text is unusable"], + ), + ( + TicketDraft::fresh(&repo).dense_body(), + vec![ + "Ticket body's text is unusable", + "graphemes is over the 65536 grapheme limit", + ], + ), + ]; + for (draft, fragments) in malformed { + message_contains(&ticket.refused_create(draft.record).await, &fragments); + } +} + +#[tokio::test] +async fn assignees_belong_to_a_maintainer_and_round_trip() { + let (world, repo) = ticket_world().await; + let ticket = Ticket::on(&world, &repo); + let assigned = |title: &str, assignees: serde_json::Value| { + let mut record = ticket_record(&repo, title, "a shoal"); + record["assignees"] = assignees; + record + }; + + let twice = json!([world.stranger.did, world.stranger.did]); + message_contains( + &ticket.refused_create(assigned("kelp", twice)).await, + &["assignees lists someone twice"], + ); + let crowd: Vec = ('a'..='k') + .map(|letter| format!("did:plc:{}", letter.to_string().repeat(24))) + .collect(); + message_contains( + &ticket.refused_create(assigned("kelp", json!(crowd))).await, + &["11 assignees are over the 10 limit"], + ); + + let rejected = failed( + ticket + .create( + &world.stranger, + assigned("kelp", json!([world.stranger.did])), + ) + .await, + StatusCode::FORBIDDEN, + ); + assert_eq!(rejected["message"], "Assignees belong to a maintainer"); + + let emptied = ok(ticket + .create(&world.stranger, assigned("kelp", json!([]))) + .await); + let served = ok(record_at(&world, &repo, &uri_of(&emptied), None).await); + assert_eq!( + served["value"].get("assignees"), + None, + "an empty assignee list is stored as absent" + ); + + let both = json!([world.stranger.did, world.admin.did]); + let opened = ok(ticket + .create(&world.member, assigned("urchin", both.clone())) + .await); + let uri = uri_of(&opened); + let served = ok(record_at(&world, &repo, &uri, None).await); + assert_eq!(served["value"]["assignees"], both); + + let mut echoed = served["value"].clone(); + echoed["title"]["text"] = json!("urchin, again"); + echoed["assignees"] = json!([world.admin.did, world.stranger.did]); + let retitled = ok(ticket + .put(&world.member, &uri, &cid_of(&opened), echoed.clone()) + .await); + + echoed + .as_object_mut() + .expect("a ticket record is an object") + .remove("assignees"); + let dropped = failed( + ticket + .put(&world.stranger, &uri, &cid_of(&retitled), echoed.clone()) + .await, + StatusCode::FORBIDDEN, + ); + assert_eq!(dropped["message"], "Assignees belong to a maintainer"); + + ok(ticket + .put(&world.member, &uri, &cid_of(&retitled), echoed) + .await); + assert_eq!( + ok(record_at(&world, &repo, &uri, None).await)["value"].get("assignees"), + None, + "a maintainer can unassign everyone" + ); +} + +#[tokio::test] +async fn state_transition_belongs_to_author_or_maintainer_and_a_repeat_gets_409() { + let (world, repo) = ticket_world().await; + let ticket = Ticket::on(&world, &repo); + let (uri, v1) = ticket.opened(&world.stranger, "kelp").await; + + failed( + ticket + .set_state(&world.stranger, &uri, &v1, TicketState::Open) + .await, + StatusCode::CONFLICT, + ); + + let closed = ok(ticket + .set_state(&world.stranger, &uri, &v1, TicketState::Closed) + .await); + let v2 = cid_of(&closed); + let record = ok(record_at(&world, &repo, &uri, None).await); + assert_eq!(record["value"]["state"], CLOSED); + + failed( + ticket + .set_state(&world.stranger, &uri, &v2, TicketState::Closed) + .await, + StatusCode::CONFLICT, + ); + + let reopened = ok(ticket + .set_state(&world.admin, &uri, &v2, TicketState::Open) + .await); + let v3 = cid_of(&reopened); + let reauthored = ok(ticket + .set_state(&world.member, &uri, &v3, TicketState::Open) + .await); + assert_ne!( + cid_of(&reauthored), + v3, + "a maintainer re-sending an open ticket's own state re-authors the record rather than \ + refusing the echo" + ); + + let (owners, owners_v1) = ticket.opened(&world.member, "urchin").await; + let rejected = failed( + ticket + .set_state(&world.stranger, &owners, &owners_v1, TicketState::Closed) + .await, + StatusCode::FORBIDDEN, + ); + assert_eq!( + rejected["message"], + "State changes belong to the ticket's author or a maintainer" + ); + + assert_eq!( + set_policy_as( + &world, + &world.member, + &repo, + ContributionPolicy::Collaborators + ) + .await, + StatusCode::OK + ); + let closed = ok(ticket + .set_state( + &world.member, + &uri, + &cid_of(&reauthored), + TicketState::Closed, + ) + .await); + assert_eq!( + ok(record_at(&world, &repo, &uri, None).await)["value"]["state"], + CLOSED, + "a maintainer's state write lands under a policy the owner closed" + ); + let shut = failed( + ticket + .set_state(&world.stranger, &uri, &cid_of(&closed), TicketState::Open) + .await, + StatusCode::FORBIDDEN, + ); + message_contains( + &shut, + &["stranger", "collaborators", "add you as a collaborator"], + ); +} + +#[tokio::test] +async fn racing_creates_take_two_numbers_and_racing_closes_settle_one_state() { + let (world, repo) = ticket_world().await; + let ticket = Ticket::on(&world, &repo); + let (kelp, kelp_v1) = ticket.opened(&world.stranger, "kelp").await; + + let (urchin, whelk) = tokio::join!( + ticket.open(&world.stranger, "urchin", ""), + ticket.open(&world.member, "whelk", ""), + ); + assert_eq!( + [urchin.0, whelk.0], + [StatusCode::OK, StatusCode::OK], + "{} / {}", + urchin.1, + whelk.1 + ); + assert_eq!( + world + .state + .index + .next_subject_number(&repo, &knot_record::ticket::ticket_collection()), + Resolved::Ready(knot_types::SubjectNumber::new(4)), + "#2 and #3 both go because each create must read the number under the lock the other one \ + bumped" + ); + + let (left, right) = tokio::join!( + ticket.set_state(&world.stranger, &kelp, &kelp_v1, TicketState::Closed), + ticket.set_state(&world.member, &kelp, &kelp_v1, TicketState::Closed), + ); + let mut statuses = [left.0, right.0]; + statuses.sort(); + assert_eq!( + statuses, + [StatusCode::OK, StatusCode::CONFLICT], + "one close oughtta be stored and the other ought to read it under the lock: {} / {}", + left.1, + right.1 + ); + let settled = ok(record_at(&world, &repo, &kelp, None).await); + assert_eq!( + settled["value"]["state"], CLOSED, + "the close that won the race is the state that stands" + ); +} + +fn ticket_body_bytes(world: &World, actor: &Actor, title: &str, text: &str) -> u64 { + let prose = knot_record::Prose::new( + knot_record::ticket::TicketTitle::new(title).unwrap(), + Some(knot_record::ticket::MarkdownBody::new(text).unwrap()), + ); + let record = knot_record::ticket::TicketRecord::new( + prose, + knot_record::ticket::TicketState::Open, + None, + None, + None, + None, + world.state.now(), + actor.did.clone(), + ) + .unwrap(); + record.body().unwrap().as_bytes().len() as u64 +} + +#[tokio::test] +async fn social_budget_429s_over_serving_bytes_and_edits_or_deletes_free_them() { + let probe = World::new(); + probe.grandfather(MEMBER_HOST); + let probed = create_repo_helper(&probe, "anemone").await; + let manifest_bytes = match probe.state.index.social_serving_bytes(&probed) { + Resolved::Ready(bytes) => bytes.get(), + Resolved::Warming => panic!("social projection is warm after create"), + }; + let opened_bytes = ticket_body_bytes(&probe, &probe.member, "kelp", "a forest"); + let edited_bytes = ticket_body_bytes(&probe, &probe.member, "kelp", "kelp forest"); + assert!(opened_bytes < edited_bytes); + + let world = World::with_social_budget(knot_resource::SocialBudgetBytes::new( + manifest_bytes + edited_bytes, + )); + world.grandfather(MEMBER_HOST); + let repo = create_repo_helper(&world, "anemone").await; + + let ticket = Ticket::on(&world, &repo); + let (uri, v1) = ticket.opened(&world.member, "kelp").await; + let edited = ok(ticket.edit(&world.member, &uri, &v1, "kelp").await); + let v2 = cid_of(&edited); + assert_ne!(v1, v2, "a different body mints a different version"); + + let rejected = failed( + ticket.open(&world.member, "kelp", "a forest").await, + StatusCode::TOO_MANY_REQUESTS, + ); + assert_eq!(rejected["error"], "RateLimitExceeded"); + assert_eq!( + rejected["bucket"], "repo-social", + "The edited version still occupies the budget until it is freed" + ); + let message = rejected["message"].as_str().unwrap(); + assert!( + message.contains("serving record text") + && message.contains("frees the bytes of the version it replaces") + && !message.contains("pool"), + "The 429 oughtta explain how to free the bytes: {rejected}" + ); + + ok(ticket.erase(&world.member, &uri, &v2).await); + let reopened = ok(ticket.open(&world.member, "kelp", "a forest").await); + assert!( + reopened["uri"] + .as_str() + .unwrap() + .contains("/org.tangled.track.ticket/"), + "Erasure frees the bytes of the version it retracts" + ); +} +#[tokio::test] +async fn admission_screens_repo_creation() { + let closed = World::new(); + let make = || json!({ "name": "anemone" }); + assert_eq!( + create_status(&closed, &closed.stranger, make()).await, + StatusCode::FORBIDDEN, + "A closed knot denies a stranger" + ); + + let open = World::open(); + assert_eq!( + create_status(&open, &open.stranger, make()).await, + StatusCode::OK + ); + assert!( + matches!( + open.state.index.resolve_repo( + &OwnerDid::new(format!("did:web:{STRANGER_HOST}")).unwrap(), + &RepoRkey::new("anemone").unwrap() + ), + Resolved::Ready(Some(_)) + ), + "An open knot registers the stranger's repo without membership" + ); +} + +#[tokio::test] +async fn blocklist_lifecycle() { + let world = World::open(); + let subject = account(STRANGER_HOST); + let make = || json!({ "name": "anemone" }); + + assert_eq!( + offer_block(&world, &world.admin, &subject).await, + StatusCode::OK + ); + assert!(matches!( + world.state.index.is_blocked(&account(STRANGER_HOST)), + Resolved::Ready(true) + )); + assert_eq!( + create_status(&world, &world.stranger, make()).await, + StatusCode::FORBIDDEN, + "a banned account cannot create" + ); + + assert_eq!( + revoke(&world, &world.admin, &block_collection(), &subject).await, + StatusCode::OK + ); + assert!(matches!( + world.state.index.is_blocked(&account(STRANGER_HOST)), + Resolved::Ready(false) + )); + assert_eq!( + create_status(&world, &world.stranger, make()).await, + StatusCode::OK, + "an unban restores creation" + ); + + assert_eq!( + offer_block(&world, &world.admin, &account(ADMIN_HOST)).await, + StatusCode::FORBIDDEN, + "an admin cannot be banned" + ); + assert_eq!( + offer_block(&world, &world.stranger, &account(MEMBER_HOST)).await, + StatusCode::FORBIDDEN, + "a non-admin cannot ban" + ); +} + +#[tokio::test] +async fn member_lifecycle() { + let world = World::new(); + let member = account(MEMBER_HOST); + + assert_eq!(offer(&world, None, &member).await, StatusCode::OK); + assert_eq!( + world.state.index.effective_member(&member), + Resolved::Ready(true), + "the knot holds the list, so adding is the whole membership" + ); + assert_eq!( + offer(&world, None, &member).await, + StatusCode::OK, + "the knot refused a second offer for a member it already added" + ); + + assert_eq!( + revoke(&world, &world.admin, &member_collection(), &member).await, + StatusCode::OK + ); + assert_eq!( + world.state.index.effective_member(&account(MEMBER_HOST)), + Resolved::Ready(false), + "removed member is gone on the very next read" + ); + assert_eq!( + revoke(&world, &world.admin, &member_collection(), &member).await, + StatusCode::OK + ); +} + +fn mint_invite(world: &World) -> String { + let state = world.state(); + let (code, _) = state.invite_codes.mint(&*state.entropy, state.now()); + code +} + +async fn redeem(world: &World, actor: &Actor, code: &str) -> StatusCode { + call( + world, + crate::invite_codes::redeem_invite_code, + actor, + // derived, never spelled twice: these tests call the handler directly, so + // a token minted for the wrong nsid would still agree with itself + &nsid( + crate::invite_codes::REDEEM_ROUTE + .strip_prefix("/xrpc/") + .expect("xrpc route paths are prefixed"), + ), + json!({ "code": code }), + ) + .await + .status() +} + +#[tokio::test] +async fn redeeming_an_invite_link_admits_the_token_issuer_and_spends_the_code() { + let world = World::new(); + let code = mint_invite(&world); + let stranger = account(STRANGER_HOST); + + assert_eq!( + world.state.index.effective_member(&stranger), + Resolved::Ready(false), + "the stranger starts outside the knot" + ); + assert_eq!(redeem(&world, &world.stranger, &code).await, StatusCode::OK); + assert_eq!( + world.state.index.effective_member(&stranger), + Resolved::Ready(true), + "redeeming admits the account that signed the token, with no acceptance round trip" + ); + + assert_eq!( + redeem(&world, &world.passerby, &code).await, + StatusCode::BAD_REQUEST, + "a spent code mustn't admit a second account" + ); + assert_eq!( + world.state.index.effective_member(&account(PASSERBY_HOST)), + Resolved::Ready(false) + ); +} + +#[tokio::test] +async fn an_unknown_code_admits_nobody() { + let world = World::new(); + assert_eq!( + redeem(&world, &world.stranger, "not-a-code").await, + StatusCode::BAD_REQUEST + ); + assert_eq!( + world.state.index.effective_member(&account(STRANGER_HOST)), + Resolved::Ready(false) + ); +} + +#[tokio::test] +async fn a_link_redeemed_by_a_member_the_admin_had_already_added_is_a_no_op() { + let world = World::new(); + let member = account(MEMBER_HOST); + assert_eq!(offer(&world, None, &member).await, StatusCode::OK); + + let code = mint_invite(&world); + assert_eq!( + redeem(&world, &world.member, &code).await, + StatusCode::OK, + "a redemption by an account already on the grant list must answer ok, not conflict" + ); + assert_eq!( + world.state.index.effective_member(&member), + Resolved::Ready(true) + ); +} + +#[tokio::test] +async fn redeeming_twice_as_an_existing_member_is_a_no_op() { + let world = World::new(); + let first = mint_invite(&world); + let second = mint_invite(&world); + assert_eq!( + redeem(&world, &world.stranger, &first).await, + StatusCode::OK + ); + assert_eq!( + redeem(&world, &world.stranger, &second).await, + StatusCode::OK, + "an account already inside stays inside rather than conflicting" + ); + assert_eq!( + world.state.index.effective_member(&account(STRANGER_HOST)), + Resolved::Ready(true) + ); +} + +#[tokio::test] +async fn member_offer_auth_outcomes() { + struct Case { + headers: HeaderMap, + subject: &'static str, + status: StatusCode, + why: &'static str, + } + + let world = World::new(); + let knot_repo = world.state.knot_repo.clone(); + let lowercase = { + let token = mint(&world.admin, &nsid(CREATE_RECORD)); + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("bearer {token}")).unwrap(), + ); + headers + }; + let cases = vec![ + Case { + headers: bearer(&mint(&world.member, &nsid(CREATE_RECORD))), + subject: "did:web:olaren.dev", + status: StatusCode::FORBIDDEN, + why: "a non-admin can't offer membership", + }, + Case { + headers: HeaderMap::new(), + subject: "did:web:olaren.dev", + status: StatusCode::UNAUTHORIZED, + why: "a request without a token is unauthorized", + }, + Case { + headers: bearer(&mint(&world.admin, &nsid(CREATE_RECORD))), + subject: "not-a-did", + status: StatusCode::BAD_REQUEST, + why: "a record key that isn't a DID is rejected before the grant runs", + }, + Case { + headers: lowercase, + subject: "did:web:olaren.dev", + status: StatusCode::OK, + why: "the bearer scheme is matched case-insensitively per RFC 7235", + }, + ]; + + let world_ref = &world; + futures::stream::iter(cases) + .for_each(|case| { + let knot_repo = knot_repo.clone(); + async move { + let payload = match AccountDid::new(case.subject) { + Ok(subject) => grant_body(&knot_repo, &member_collection(), &subject), + Err(_) => json!({ + "repo": knot_repo, + "collection": member_collection(), + "rkey": case.subject, + "record": { + "$type": member_collection(), + "createdAt": "2026-09-11T00:00:00Z", + }, + }), + }; + let status = into_response( + crate::social::create_record( + world_ref.state(), + case.headers, + crate::Method::from_nsid(CREATE_RECORD), + body(payload), + ) + .await, + ) + .status(); + assert_eq!(status, case.status, "{}", case.why); + } + }) + .await; +} + +#[tokio::test] +async fn a_transient_upstream_identity_failure_is_a_503_named_upstream_unavailable() { + let responder: Responder = Box::new(|_request: &HttpRequest| { + Ok(HttpResponse { + status: StatusCode::INTERNAL_SERVER_ERROR, + headers: http::HeaderMap::new(), + body: Bytes::new(), + }) + }); + let (_dir, state) = build_state(responder, true); + let admin = actor(1, ADMIN_HOST); + let token = mint(&admin, &nsid(CREATE_RECORD)); + let knot_repo = crate::knot_repo(&state.knot_did).unwrap(); + let response = into_response( + crate::social::create_record( + State(state), + bearer(&token), + crate::Method::from_nsid(CREATE_RECORD), + body(grant_body( + &knot_repo, + &member_collection(), + &account(MEMBER_HOST), + )), + ) + .await, + ); + assert_eq!( + response.status(), + StatusCode::SERVICE_UNAVAILABLE, + "transient issuer-doc failure is 503, distinct from a bad token's 401" + ); + assert_eq!( + json_of(response).await["error"], + "UpstreamUnavailable", + "transient upstream identity failure is named distinctly from a warming projection" + ); +} + +#[tokio::test] +async fn concurrent_first_member_adds_converge_to_a_single_cob_object() { + use knot_cob::CobStore; + use knot_cobs::MembersCob; + use knot_git::Repo; + + let world = World::new(); + let first = account("witchcraft.systems"); + let second = account("isabelroses.com"); + let (left, right) = tokio::join!(offer(&world, None, &first), offer(&world, None, &second),); + assert_eq!(left, StatusCode::OK); + assert_eq!(right, StatusCode::OK); + + let meta = Repo::open(world.state.meta_path.clone()).unwrap(); + assert_eq!( + CobStore::new(&meta).list::().unwrap().len(), + 1, + "concurrent first adds serialize onto one singleton members COB, never splitting it" + ); + let hosts = ["witchcraft.systems", "isabelroses.com"]; + hosts.iter().for_each(|host| { + assert_eq!( + world.on_grant_list(None, &account(host)), + Resolved::Ready(true), + "both concurrent adds wrote to the same grant list, and {host} is on it" + ); + }); +} + +#[tokio::test] +async fn re_adding_a_member_under_warming_appends_no_redundant_change() { + use knot_cob::CobStore; + use knot_cobs::{Grant, MembersChange, MembersCob}; + use knot_git::Repo; + + let admin = actor(1, ADMIN_HOST); + let responder = doc_responder(admin.signer.public_key().as_bytes().to_vec(), || { + StatusCode::OK + }); + let (_dir, state) = build_state(responder, false); + + let now = state.now(); + let knot_signer = state.secrets.signer(&state.knot_did).unwrap(); + let subject = account(MEMBER_HOST); + let meta = Repo::open(state.meta_path.clone()).unwrap(); + let created = CobStore::new(&meta) + .create( + &knot_cob::CobHome::from(&state.knot_did), + &MembersChange::Add(Grant { + subject: subject.clone(), + added_by: account(ADMIN_HOST), + created_at: now, + }), + &knot_signer, + now, + ) + .unwrap(); + + let token = mint(&admin, &nsid(CREATE_RECORD)); + let knot_repo = crate::knot_repo(&state.knot_did).unwrap(); + assert_eq!( + into_response( + crate::social::create_record( + State(Arc::clone(&state)), + bearer(&token), + crate::Method::from_nsid(CREATE_RECORD), + body(grant_body(&knot_repo, &member_collection(), &subject)), + ) + .await + ) + .status(), + StatusCode::OK + ); + + let meta = Repo::open(state.meta_path.clone()).unwrap(); + let delta = CobStore::new(&meta) + .changes_since::(created.object, Some(created.tip)) + .unwrap(); + assert!( + delta.changes.is_empty(), + "re-adding existing member doesn't append change, even while projection is warming" + ); +} + +#[tokio::test] +async fn create_mints_a_did_plc_repo_and_refuses_a_duplicate_name() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + + let repo_did = create_repo_helper(&world, "anemone").await; + assert!( + repo_did.as_str().starts_with("did:plc:"), + "knot minted a did:plc identity for the repo" + ); + assert!( + world.layout.open(&repo_did).is_ok(), + "bare repo exists on disk under its minted DID" + ); + assert_eq!( + world.state.secrets.len(), + 1, + "only the shared knot key is sealed" + ); + + assert_eq!( + create_status(&world, &world.member, json!({ "name": "anemone" })).await, + StatusCode::CONFLICT, + "a second repo of the same name is refused, never silently overwritten" + ); + assert_eq!( + resolve(&world, &RepoName::new("anemone").unwrap()), + Resolved::Ready(Some(repo_did.clone())), + "the original repo still owns the name" + ); + assert!( + world.layout.open(&repo_did).is_ok(), + "the original repo is untouched on disk" + ); +} + +#[tokio::test] +async fn create_and_reserve_reject_bad_identities() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + + assert_eq!( + create_status( + &world, + &world.member, + json!({ "name": "a", "did": "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa" }) + ) + .await, + StatusCode::BAD_REQUEST, + "a did:plc can't be brought; the knot mints those itself" + ); + + assert_eq!( + create_status( + &world, + &world.member, + json!({ "name": "evil", "did": format!("did:web:{KNOT_HOST}") }) + ) + .await, + StatusCode::BAD_REQUEST, + "the knot's own DID as a repoDid is a client error instead of a 500" + ); + assert_eq!( + world.state.index.effective_member(&account(MEMBER_HOST)), + Resolved::Ready(true), + "the meta-repo is intact" + ); + + assert_eq!( + create_status( + &world, + &world.member, + json!({ "name": "uni", "did": "did:web:uni.olaren.dev" }) + ) + .await, + StatusCode::BAD_REQUEST, + "a did:web create without a prior reserveKey is refused" + ); + assert_eq!( + resolve(&world, &RepoName::new("uni").unwrap()), + Resolved::Ready(None), + "nothing was registered for the unproven did:web" + ); + + assert_eq!( + reserve_status( + &world, + &world.member, + &RepoDid::new(format!("did:web:{KNOT_HOST}")).unwrap(), + ) + .await, + StatusCode::BAD_REQUEST, + "the knot's own DID cannot have a repo key reserved against it" + ); + + let unpublished = RepoDid::new("did:web:conch.olaren.dev").unwrap(); + assert_eq!( + reserve_status(&world, &world.member, &unpublished).await, + StatusCode::OK + ); + let impostor = knot_types::crypto::multikey(0xe7, signer(99).public_key().as_bytes()); + world.publish_repo_doc("conch.olaren.dev", &impostor); + assert_eq!( + create_status( + &world, + &world.member, + json!({ "name": "conch", "did": &unpublished }) + ) + .await, + StatusCode::BAD_REQUEST, + "a did:web whose document publishes a different key fails the control proof" + ); + assert!( + world.layout.open(&unpublished).is_err(), + "the repo was never created on disk" + ); + + let victim = RepoDid::new("did:web:victim.olaren.dev").unwrap(); + let reserved = reserve_repo_key(&world, &victim).await; + assert_eq!( + reserve_repo_key(&world, &victim).await, + reserved, + "re-reserving returns the same key so an already-published document stays valid" + ); + assert_eq!( + create_status( + &world, + &world.admin, + json!({ "name": "victim", "did": &victim }) + ) + .await, + StatusCode::BAD_REQUEST, + "only the account that reserved the did:web may create it" + ); + assert!( + world.layout.open(&victim).is_err(), + "the hijack attempt created no repo on disk" + ); + + let before = world.state.secrets.len(); + assert_eq!( + create_status( + &world, + &world.member, + json!({ "name": "doomed", "defaultBranch": "bad..name" }) + ) + .await, + StatusCode::BAD_REQUEST + ); + assert_eq!( + world.state.secrets.len(), + before, + "an invalid branch is rejected before any key is minted, sealed, or DID submitted" + ); +} + +#[tokio::test] +async fn a_byo_did_web_repo_is_accepted_and_its_key_is_returned() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + let did = "did:web:nautilus.olaren.dev"; + let reserved_key = reserve_repo_key(&world, &RepoDid::new(did).unwrap()).await; + + let response = as_member( + &world, + crate::repos::create_repo, + &nsid(CREATE), + json!({ "name": "nautilus", "did": did }), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let output = json_of(response).await; + assert_eq!(output["did"].as_str(), Some(did)); + + let repo_did = RepoDid::new(did).unwrap(); + assert!( + world.layout.open(&repo_did).is_ok(), + "bring-your-own did:web repo is on disk" + ); + let knot_public = world + .state + .secrets + .public_key(&world.state.knot_did) + .unwrap(); + let expected_key = knot_types::crypto::multikey(0xe7, knot_public.as_bytes()); + assert_eq!( + expected_key, reserved_key, + "reserve returns the knot key the owner publishes in their did:web document" + ); + assert_eq!( + output["key"].as_str(), + Some(expected_key.as_str()), + "create returns the knot-held key the owner published in their own did:web document" + ); + + assert_eq!( + on_repo( + &world, + crate::social::create_record, + &world.member, + &nsid(CREATE_RECORD), + &repo_did, + grant_body( + &repo_did, + &collaborator_collection(), + &account("witchcraft.systems") + ), + ) + .await + .0, + StatusCode::OK + ); + assert_eq!( + world + .state + .index + .effective_collaborator(&repo_did, &account("witchcraft.systems")), + Resolved::Ready(true), + "the collaborator COB signed by the knot-held repo key lands and is visible" + ); + + let squid = "did:web:squid.olaren.dev"; + let victim = RepoDid::new(squid).unwrap(); + world.layout.create(&victim).unwrap(); + reserve_repo_key(&world, &victim).await; + assert_eq!( + create_status( + &world, + &world.member, + json!({ "name": "anemone", "did": squid }) + ) + .await, + StatusCode::CONFLICT + ); + assert!( + world.layout.open(&victim).is_ok(), + "a colliding create mustn't delete the repository already on disk" + ); +} + +#[tokio::test] +async fn create_posts_the_genesis_at_birth() { + let repo = minted_repo(PlcFake::default()).await; + let snapshot = repo.fake.lock(); + assert_eq!(snapshot.ops.len(), 1); + assert_eq!(snapshot.ops[0].0, repo.did.as_str()); + assert!(snapshot.ops[0].1["prev"].is_null()); +} + +#[tokio::test] +async fn a_rejected_genesis_stays_queued_and_the_queue_lands_it() { + let repo = minted_repo(PlcFake { + reject_posts: true, + ..PlcFake::default() + }) + .await; + assert_eq!( + repo.create_status, + StatusCode::OK, + "a directory outage mustn't fail the create; submission is queued" + ); + { + let meta = Repo::open(&repo.state.meta_path).unwrap(); + let pending = crate::plc::pending_entries(&meta).unwrap(); + assert_eq!(pending.len(), 1, "the genesis op sits in the pending queue"); + assert_eq!(pending[0].did, repo.did); + } + + let mut memory = crate::plc::SweepMemory::default(); + repo.fake.lock().reject_posts = false; + for _ in 0..4 { + repo.clock.advance(std::time::Duration::from_secs(31)); + crate::plc::sweep_tick(&repo.state, &mut memory, 1).await; + } + + let snapshot = repo.fake.lock(); + assert_eq!( + snapshot.post_count, 1, + "only the genesis should POST; ops: {:?}", + snapshot.ops + ); + drop(snapshot); + assert_eq!(memory.settled.len(), 1, "the document settles"); + let meta = Repo::open(&repo.state.meta_path).unwrap(); + let pending = crate::plc::pending_entries(&meta).unwrap(); + assert!(pending.is_empty(), "queue must drain; entries: {pending:?}"); +} +#[tokio::test] +async fn deleting_a_repo_records_the_deactivation_for_get_repo_status() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo = create_repo_helper(&world, "reef").await; + + let live = repo_status(&world, &repo).await; + assert_eq!(live["active"], json!(true)); + assert!(live.get("status").is_none()); + + as_member( + &world, + crate::repos::delete_repo, + &nsid(DELETE), + json!({ "repo": repo.as_str() }), + ) + .await; + + let deleted = repo_status(&world, &repo).await; + assert_eq!(deleted["did"], json!(repo.as_str())); + assert_eq!(deleted["active"], json!(false)); + assert_eq!(deleted["status"], json!("deleted")); +} +#[tokio::test] +async fn resolve_by_name_matches_the_rkey_case_sensitively() { + let admin = actor(1, ADMIN_HOST); + let key = admin.signer.public_key().as_bytes().to_vec(); + let responder = doc_responder(key, || StatusCode::OK); + let (_dir, state) = build_state(responder, true); + let owner = OwnerDid::new(format!("did:web:{ADMIN_HOST}")).unwrap(); + + assert_eq!( + into_response( + crate::repos::create_repo( + State(Arc::clone(&state)), + bearer(&mint(&admin, &nsid(CREATE))), + crate::Method::from_nsid(CREATE), + body(json!({ "name": "anemone" })), + ) + .await + ) + .status(), + StatusCode::OK + ); + + assert!( + crate::forks::resolve_by_name(&*state, &owner, &RepoName::new("anemone").unwrap()).is_ok(), + "the exact rkey resolves" + ); + assert!( + crate::forks::resolve_by_name(&*state, &owner, &RepoName::new("Anemone").unwrap()).is_err(), + "rkeys are case-sensitive, so a differently-cased name mustn't resolve at all" + ); +} + +#[tokio::test] +async fn reserve_key_refuses_once_the_pending_limit_is_reached() { + let world = World::with_pending_limit(2); + world.grandfather(MEMBER_HOST); + + assert_eq!( + reserve_status( + &world, + &world.member, + &RepoDid::new("did:web:p0.olaren.dev").unwrap(), + ) + .await, + StatusCode::OK, + "first reservation is within the limit" + ); + assert_eq!( + reserve_status( + &world, + &world.member, + &RepoDid::new("did:web:p1.olaren.dev").unwrap(), + ) + .await, + StatusCode::OK, + "second reservation reaches the limit" + ); + assert_eq!( + reserve_status( + &world, + &world.member, + &RepoDid::new("did:web:p2.olaren.dev").unwrap(), + ) + .await, + StatusCode::TOO_MANY_REQUESTS, + "member cannot grow the sealed store without bound past the pending-reservation limit" + ); +} + +#[tokio::test] +async fn one_account_cannot_exhaust_the_global_reservation_budget() { + let world = World::with_limits(256, 2); + world.grandfather(MEMBER_HOST); + + let world_ref = &world; + futures::stream::iter([ + RepoDid::new("did:web:m0.olaren.dev").unwrap(), + RepoDid::new("did:web:m1.olaren.dev").unwrap(), + ]) + .for_each(|did| async move { + assert_eq!( + reserve_status(world_ref, &world_ref.member, &did).await, + StatusCode::OK + ); + }) + .await; + assert_eq!( + reserve_status( + &world, + &world.member, + &RepoDid::new("did:web:m2.olaren.dev").unwrap(), + ) + .await, + StatusCode::TOO_MANY_REQUESTS, + "member is held to its per-actor reservation budget" + ); + assert_eq!( + reserve_status( + &world, + &world.admin, + &RepoDid::new("did:web:admin0.olaren.dev").unwrap(), + ) + .await, + StatusCode::OK, + "a different account keeps its own budget while global capacity remains" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_reserve_key_calls_all_succeed() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + + let handles: Vec<_> = (0..24) + .map(|i| { + let state = world.state(); + let token = mint(&world.member, &nsid(RESERVE)); + let payload = body(json!({ "repo": format!("did:web:r{i}.olaren.dev") })); + tokio::spawn(async move { + into_response( + crate::repos::reserve_key( + state, + bearer(&token), + crate::Method::from_nsid(RESERVE), + payload, + ) + .await, + ) + .status() + }) + }) + .collect(); + + futures::future::join_all(handles) + .await + .into_iter() + .for_each(|result| { + assert_eq!( + result.unwrap(), + StatusCode::OK, + "concurrent reserveKey mustn't race the in-memory reservation map" + ) + }); +} + +#[tokio::test] +async fn collaborator_lifecycle() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo_did = create_repo_helper(&world, "scallop").await; + let collaborator = account(STRANGER_HOST); + let repo = Some(&repo_did); + + assert_eq!(offer(&world, repo, &collaborator).await, StatusCode::OK); + assert_eq!( + world + .state + .index + .effective_collaborator(&repo_did, &collaborator), + Resolved::Ready(true) + ); + assert_eq!( + revoke_collaborator(&world, &world.member, &repo_did, &collaborator).await, + StatusCode::OK + ); + assert_eq!( + world + .state + .index + .effective_collaborator(&repo_did, &collaborator), + Resolved::Ready(false), + "removed collaborator is gone on the very next read" + ); + assert_eq!( + revoke_collaborator(&world, &world.member, &repo_did, &collaborator).await, + StatusCode::OK + ); +} + +#[tokio::test] +async fn repo_management_is_owner_or_collaborator_gated() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo_did = create_repo_helper(&world, "squid").await; + + assert_eq!( + offer_collaborator_as(&world, &world.admin, &repo_did, &account("isabelroses.com")).await, + StatusCode::FORBIDDEN, + "collaborator management is the repo owner's right instead of a knot admin's" + ); + assert_eq!( + set_default_branch_as( + &world, + &world.stranger, + &repo_did, + &BranchName::new("trunk").unwrap() + ) + .await, + StatusCode::FORBIDDEN, + "a stranger can't move the default branch" + ); + assert_eq!( + as_stranger( + &world, + crate::branches::delete_branch, + &nsid(DELETE_BRANCH), + json!({ "repo": repo_did, "branch": "trunk" }) + ) + .await + .status(), + StatusCode::FORBIDDEN, + "a stranger cannot delete a branch" + ); + assert_eq!( + as_stranger( + &world, + crate::repos::delete_repo, + &nsid(DELETE), + json!({ "repo": repo_did }) + ) + .await + .status(), + StatusCode::FORBIDDEN, + "neither owner nor a knot admin, so delete is refused" + ); + + assert_eq!( + rename_repo_as( + &world, + &world.stranger, + &repo_did, + &RepoRkey::new("stolen").unwrap(), + ) + .await, + StatusCode::FORBIDDEN + ); + assert_eq!( + world.state.index.rkey_of(&repo_did), + Resolved::Ready(Some(RepoRkey::new("squid").unwrap())), + "the canonical rkey is untouched by the rejected rename" + ); + + world.grandfather_collaborator(&repo_did, STRANGER_HOST); + assert_eq!( + rename_repo_as( + &world, + &world.stranger, + &repo_did, + &RepoRkey::new("periwinkle").unwrap(), + ) + .await, + StatusCode::FORBIDDEN, + "a rename is a manifest write now, and the manifest is the owner's alone" + ); + assert_eq!( + rename_repo_as( + &world, + &world.member, + &repo_did, + &RepoRkey::new("periwinkle").unwrap(), + ) + .await, + StatusCode::OK, + "the owner renames through the manifest" + ); +} + +#[tokio::test] +async fn the_owner_moves_the_default_branch() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo_did = create_repo_helper(&world, "mussel").await; + + assert_eq!( + set_default_branch_as( + &world, + &world.member, + &repo_did, + &BranchName::new("trunk").unwrap() + ) + .await, + StatusCode::OK + ); + + let repo = world.layout.open(&repo_did).unwrap(); + assert_eq!( + repo.default_branch().map(|name| name.as_str().to_string()), + Some("refs/heads/trunk".to_string()), + "HEAD now points at the requested branch" + ); +} + +#[tokio::test] +async fn set_default_branch_rejections() { + use knot_cob::CobStore; + use knot_cobs::{CollaboratorsChange, Grant}; + use knot_git::RefUpdate; + use knot_types::RefName; + + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo_did = create_repo_helper(&world, "mussel").await; + + let git = world.layout.open(&repo_did).unwrap(); + let now = world.state.now(); + let signer = world.state.secrets.signer(&world.state.knot_did).unwrap(); + let created = CobStore::new(&git) + .create( + &knot_cob::CobHome::from(&repo_did), + &CollaboratorsChange::Add(Grant { + subject: account("olaren.dev"), + added_by: account("olaren.dev"), + created_at: now, + }), + &signer, + now, + ) + .unwrap(); + git.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: created.tip.oid(), + }) + .unwrap(); + + let (live, from) = live_manifest(&world, &repo_did).await; + let ghost_repo = RepoDid::new("did:web:squid.oyster.cafe").unwrap(); + assert_eq!( + put_as( + &world, + &world.member, + &ghost_repo, + &manifest_collection(), + manifest_rkey(), + json!({ "swapRecord": from }), + live, + ) + .await + .0, + StatusCode::NOT_FOUND, + "we'll reject a repo DID that this knot doesn't host" + ); + assert_eq!( + set_default_branch_as( + &world, + &world.member, + &repo_did, + &BranchName::new("ghost").unwrap() + ) + .await, + StatusCode::NOT_FOUND, + "a populated repo rejects a default pointing at a branch that doesn't exist" + ); +} + +#[tokio::test] +async fn delete_branch_removes_a_branch_and_refuses_the_default() { + use knot_cob::CobStore; + use knot_cobs::{CollaboratorsChange, Grant}; + use knot_git::RefUpdate; + use knot_types::RefName; + + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo_did = create_repo_helper(&world, "periwinkle").await; + let git = world.layout.open(&repo_did).unwrap(); + let now = world.state.now(); + let signer = world.state.secrets.signer(&world.state.knot_did).unwrap(); + let created = CobStore::new(&git) + .create( + &knot_cob::CobHome::from(&repo_did), + &CollaboratorsChange::Add(Grant { + subject: account("olaren.dev"), + added_by: account("olaren.dev"), + created_at: now, + }), + &signer, + now, + ) + .unwrap(); + let oid = created.tip.oid(); + ["refs/heads/main", "refs/heads/trunk"] + .into_iter() + .for_each(|name| { + git.update_ref(&RefUpdate::Create { + name: RefName::new(name).unwrap(), + new: oid, + }) + .unwrap(); + }); + git.set_head(&RefName::new("refs/heads/main").unwrap()) + .unwrap(); + + assert_eq!( + as_member( + &world, + crate::branches::delete_branch, + &nsid(DELETE_BRANCH), + json!({ "repo": repo_did, "branch": "trunk" }) + ) + .await + .status(), + StatusCode::OK, + "a non-default branch is deleted" + ); + assert!( + git.find_ref(&RefName::new("refs/heads/trunk").unwrap()) + .unwrap() + .is_none(), + "trunk is gone" + ); + assert_eq!( + as_member( + &world, + crate::branches::delete_branch, + &nsid(DELETE_BRANCH), + json!({ "repo": repo_did, "branch": "main" }) + ) + .await + .status(), + StatusCode::BAD_REQUEST, + "the current default branch cannot be deleted" + ); +} + +#[tokio::test] +async fn delete_repo_lifecycle_and_guards() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + + let plain = create_repo_helper(&world, "whelk").await; + assert_eq!( + as_member( + &world, + crate::repos::delete_repo, + &nsid(DELETE), + json!({ "repo": plain }) + ) + .await + .status(), + StatusCode::OK + ); + assert!( + world.layout.open(&plain).is_err(), + "bare repo is removed from disk" + ); + assert_eq!( + resolve(&world, &RepoName::new("whelk").unwrap()), + Resolved::Ready(None), + "repo is deregistered" + ); + + let guarded = create_repo_helper(&world, "conch").await; + world.publish_pds_record(&RecordRkey::new("conch").unwrap()); + let delete_conch = || json!({ "repo": guarded }); + assert_eq!( + as_member( + &world, + crate::repos::delete_repo, + &nsid(DELETE), + delete_conch() + ) + .await + .status(), + StatusCode::CONFLICT, + "the guard rejects the delete while the repo's declaration record is still on the owner's PDS" + ); + assert!( + world.layout.open(&guarded).is_ok(), + "a refused delete left the repo intact on disk" + ); + + let force_conch = || json!({ "repo": guarded, "force": true }); + assert_eq!( + as_member( + &world, + crate::repos::delete_repo, + &nsid(DELETE), + force_conch() + ) + .await + .status(), + StatusCode::FORBIDDEN, + "force is an admin-only escape hatch instead of the owner's" + ); + assert_eq!( + as_admin( + &world, + crate::repos::delete_repo, + &nsid(DELETE), + force_conch() + ) + .await + .status(), + StatusCode::OK, + "a knot admin forces the delete past the lingering PDS record" + ); + assert!( + world.layout.open(&guarded).is_err(), + "forced delete removed the repo from disk" + ); +} + +#[tokio::test] +async fn rename_alias_lifecycle() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + + let repo_a = create_repo_helper(&world, "alpha").await; + assert_eq!( + rename_repo_as( + &world, + &world.member, + &repo_a, + &RepoRkey::new("alphanew").unwrap() + ) + .await, + StatusCode::OK + ); + assert_eq!( + resolve(&world, &RepoName::new("alphanew").unwrap()), + Resolved::Ready(Some(repo_a.clone())), + "the new rkey resolves on the very next read" + ); + assert_eq!( + resolve(&world, &RepoName::new("alpha").unwrap()), + Resolved::Ready(Some(repo_a.clone())), + "the prior rkey keeps resolving as an alias" + ); + assert_eq!( + world.state.index.rkey_of(&repo_a), + Resolved::Ready(Some(RepoRkey::new("alphanew").unwrap())), + "the new rkey is canonical" + ); + + let repo_a2 = create_repo_helper(&world, "alpha").await; + assert_ne!(repo_a2, repo_a, "a brand-new repo DID was minted"); + assert_eq!( + resolve(&world, &RepoName::new("alpha").unwrap()), + Resolved::Ready(Some(repo_a2.clone())), + "the reused rkey now resolves to the new repo" + ); + assert_eq!( + resolve(&world, &RepoName::new("alphanew").unwrap()), + Resolved::Ready(Some(repo_a.clone())), + "the renamed repo keeps its canonical rkey" + ); + + let repo_b = create_repo_helper(&world, "beta").await; + assert_eq!( + rename_repo_as( + &world, + &world.member, + &repo_b, + &RepoRkey::new("alpha").unwrap() + ) + .await, + StatusCode::CONFLICT, + "a rename cannot take the canonical rkey of another live repo" + ); + assert_eq!( + resolve(&world, &RepoName::new("alpha").unwrap()), + Resolved::Ready(Some(repo_a2)), + "the contested rkey still belongs to its original repo" + ); + + let repo_g = create_repo_helper(&world, "gamma").await; + assert_eq!( + rename_repo_as( + &world, + &world.member, + &repo_g, + &RepoRkey::new("gammanew").unwrap() + ) + .await, + StatusCode::OK + ); + assert_eq!( + as_member( + &world, + crate::repos::delete_repo, + &nsid(DELETE), + json!({ "repo": repo_g }) + ) + .await + .status(), + StatusCode::OK + ); + assert!( + world.layout.open(&repo_g).is_err(), + "a renamed repo is deleted through its new rkey" + ); + assert_eq!( + resolve(&world, &RepoName::new("gamma").unwrap()), + Resolved::Ready(None), + "deletion drops the retained alias along with the repo" + ); + + let ghost = RepoDid::new("did:web:ghost.nel.pet").unwrap(); + let (live, from) = live_manifest(&world, &repo_a).await; + let mut renamed = live; + renamed["declaration"]["name"] = json!("kelp"); + assert_eq!( + put_as( + &world, + &world.member, + &ghost, + &manifest_collection(), + manifest_rkey(), + json!({ "swapRecord": from }), + renamed, + ) + .await + .0, + StatusCode::NOT_FOUND, + "renaming a repo this knot doesn't host is 404 instead of 403" + ); +} + +#[tokio::test] +async fn a_manifest_write_against_a_warming_registry_is_unavailable_not_forbidden() { + let admin = actor(1, ADMIN_HOST); + let responder = doc_responder(admin.signer.public_key().as_bytes().to_vec(), || { + StatusCode::OK + }); + let (_dir, state) = build_state(responder, false); + + let ghost = RepoDid::new("did:web:squid.nel.pet").unwrap(); + let token = mint_for( + &admin, + &nsid(PUT_RECORD), + &ServiceDid::new(ghost.as_str()).unwrap(), + ); + let swap = crate::out::CidString::from(knot_types::dag_cbor_cid(b"warming")); + assert_eq!( + into_response( + crate::social::put_record( + State(Arc::clone(&state)), + bearer(&token), + crate::Method::from_nsid(PUT_RECORD), + body(json!({ + "repo": ghost, + "collection": MANIFEST, + "rkey": "self", + "swapRecord": swap, + "record": { + "$type": MANIFEST, + "declaration": { "owner": "did:web:squid.nel.pet", "name": "kelp" }, + "createdAt": "2026-09-11T00:00:00Z", + }, + })), + ) + .await + ) + .status(), + StatusCode::SERVICE_UNAVAILABLE, + "a warming registry is a retryable 503, never a permanent 403" + ); +} + +#[tokio::test] +async fn the_router_sheds_a_pre_auth_flood_from_one_peer() { + use axum::body::Body; + use axum::extract::ConnectInfo; + use std::net::SocketAddr; + use tower::ServiceExt; + + let world = World::new(); + let app = crate::router(Arc::clone(&world.state)); + let peer = SocketAddr::from(([203, 0, 113, 7], 5555)); + + let statuses: Vec = futures::stream::iter(0..22) + .then(|_| { + let app = app.clone(); + async move { + let mut request = http::Request::builder() + .method("POST") + .uri(crate::service::RECEIVE_PACK_ROUTE) + .body(Body::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(peer)); + app.oneshot(request).await.unwrap().status() + } + }) + .collect() + .await; + + assert!( + statuses[..20] + .iter() + .all(|status| *status == StatusCode::UNAUTHORIZED), + "per-peer burst is admitted and then fails auth on the missing token, got {statuses:?}" + ); + assert!( + statuses[20..] + .iter() + .all(|status| *status == StatusCode::TOO_MANY_REQUESTS), + "past the burst the router sheds the flood before it can reach the resolver, got {statuses:?}" + ); +} + +#[tokio::test] +async fn the_router_binds_each_route_to_its_matched_method_scope() { + use axum::body::Body; + use axum::extract::ConnectInfo; + use std::net::SocketAddr; + use tower::ServiceExt; + + let world = World::new(); + let app = crate::router(Arc::clone(&world.state)); + let peer = SocketAddr::from(([203, 0, 113, 23], 5555)); + + let knot_repo = world.state.knot_repo.clone(); + let post = |token: String| { + let mut request = http::Request::builder() + .method("POST") + .uri(crate::social::CREATE_RECORD_ROUTE) + .header(http::header::AUTHORIZATION, format!("Bearer {token}")) + .body(Body::from( + serde_json::to_vec(&grant_body( + &knot_repo, + &member_collection(), + &account(MEMBER_HOST), + )) + .unwrap(), + )) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(peer)); + request + }; + + let matched = mint(&world.admin, &nsid(CREATE_RECORD)); + assert_eq!( + app.clone().oneshot(post(matched)).await.unwrap().status(), + StatusCode::OK, + "a token whose lxm is the route's own method authenticates" + ); + + let sibling = mint(&world.admin, &nsid(PUT_RECORD)); + assert_eq!( + app.oneshot(post(sibling)).await.unwrap().status(), + StatusCode::UNAUTHORIZED, + "a token minted for a sibling method is rejected at the createRecord route" + ); +} + +#[tokio::test] +async fn the_http_push_surface_sheds_a_bogus_credential_flood_from_one_peer() { + use axum::body::Body; + use axum::extract::ConnectInfo; + use base64::Engine as _; + use std::net::SocketAddr; + use tower::ServiceExt; + + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo = create_repo_helper(&world, "kelp").await; + let app = crate::router(Arc::clone(&world.state)); + let peer = SocketAddr::from(([203, 0, 113, 11], 5555)); + let credential = format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode("git:not-a-service-jwt") + ); + + let statuses: Vec = futures::stream::iter(0..22) + .then(|_| { + let app = app.clone(); + let uri = format!("/{}/git-receive-pack", repo.as_str()); + let credential = credential.clone(); + async move { + let mut request = http::Request::builder() + .method("POST") + .uri(uri) + .header(http::header::AUTHORIZATION, credential) + .body(Body::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(peer)); + app.oneshot(request).await.unwrap().status() + } + }) + .collect() + .await; + + assert!( + statuses[..20] + .iter() + .all(|status| *status == StatusCode::UNAUTHORIZED), + "bogus credentials inside the burst fail authentication, got {statuses:?}" + ); + assert!( + statuses[20..] + .iter() + .all(|status| *status == StatusCode::TOO_MANY_REQUESTS), + "past the burst the push surface sheds the flood before it can reach the resolver, got {statuses:?}" + ); +} + +#[tokio::test] +async fn health_is_unauthenticated_and_exempt_from_shedding() { + use axum::body::Body; + use axum::extract::ConnectInfo; + use std::net::SocketAddr; + use tower::ServiceExt; + + let world = World::new(); + let app = crate::router(Arc::clone(&world.state)); + let peer = SocketAddr::from(([203, 0, 113, 9], 5555)); + let health = || { + let mut request = http::Request::builder() + .method("GET") + .uri(crate::service::HEALTH_ROUTE) + .body(Body::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(peer)); + request + }; + + let statuses = futures::future::join_all((0..30).map(|_| { + let app = app.clone(); + async move { app.oneshot(health()).await.unwrap() } + })) + .await; + assert!( + statuses + .iter() + .all(|response| response.status() == StatusCode::OK), + "health stays 200 even past the pre-auth burst, got {:?}", + statuses.iter().map(|r| r.status()).collect::>() + ); + + let wire = json_of(app.oneshot(health()).await.unwrap()).await; + assert!( + wire["version"] + .as_str() + .is_some_and(|v| v.starts_with("knot ")), + "health reports a knot version, got {wire}" + ); +} + +mod patch_endpoints { + use super::*; + + use knot_git::{EntryKind, Identity, NewCommit, RefUpdate, StagedAction, StagedChange}; + use knot_types::{Oid, RefName, UnixSeconds}; + + const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + + fn seed_main(world: &World, repo_did: &RepoDid, files: &[(&str, &str)]) -> Oid { + let repo = world.layout.open(repo_did).unwrap(); + let staged: Vec = files + .iter() + .map(|(path, content)| StagedChange { + path: knot_types::RepoPath::new(*path).unwrap(), + action: StagedAction::Put { + content: content.as_bytes().to_vec(), + kind: EntryKind::Blob, + }, + }) + .collect(); + let tree = repo + .write_staged_tree(Oid::from_hex(EMPTY_TREE).unwrap(), &staged) + .unwrap(); + let nel = Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_000), + offset_seconds: 0, + }; + let commit = repo + .write_commit(&NewCommit { + tree, + parents: Vec::new(), + author: nel.clone(), + committer: nel, + message: "base".to_string(), + extra_headers: Vec::new(), + }) + .unwrap(); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: commit, + }) + .unwrap(); + commit + } + + fn ident() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_000), + offset_seconds: 0, + } + } + + fn advance(world: &World, repo_did: &RepoDid, parent: Oid, path: &str, content: &str) -> Oid { + let repo = world.layout.open(repo_did).unwrap(); + let base_tree = repo.find_commit(parent).unwrap().tree; + let staged = vec![StagedChange { + path: knot_types::RepoPath::new(path).unwrap(), + action: StagedAction::Put { + content: content.as_bytes().to_vec(), + kind: EntryKind::Blob, + }, + }]; + let tree = repo.write_staged_tree(base_tree, &staged).unwrap(); + repo.write_commit(&NewCommit { + tree, + parents: vec![parent], + author: ident(), + committer: ident(), + message: "advance".to_string(), + extra_headers: Vec::new(), + }) + .unwrap() + } + + #[tokio::test] + async fn patch_keeps_its_source_and_refuses_a_bad_branch() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo = create_repo_helper(&world, "scallop").await; + let base = seed_main(&world, &repo, &[("reef.txt", "old line\n")]); + let head = advance(&world, &repo, base, "reef.txt", "new line\n"); + let (ticket, _) = Ticket::on(&world, &repo) + .opened(&world.member, "kelp") + .await; + let patch = |branch: &str| { + json!({ + "ticket": rkey_of(&ticket), + "base": { "oid": base.to_hex() }, + "head": { "oid": head.to_hex() }, + "sourceRepo": repo, + "sourceBranch": branch + }) + }; + + let branch_at = |name: &str, tip: Oid| { + world + .layout + .open(&repo) + .unwrap() + .update_ref(&RefUpdate::Create { + name: RefName::new(format!("refs/heads/{name}")).unwrap(), + new: tip, + }) + .unwrap(); + }; + branch_at("feat/kelp", head); + let written = ok(submit_patch_as(&world, &world.member, &repo, patch("feat/kelp")).await); + let record = ok(record_at(&world, &repo, &uri_of(&written), None).await); + assert_eq!(record["value"]["source"]["repo"], repo.as_str()); + assert_eq!(record["value"]["source"]["branch"], "feat/kelp"); + + branch_at( + "feat/moved-on", + advance(&world, &repo, head, "reef.txt", "newer line\n"), + ); + ok(submit_patch_as(&world, &world.member, &repo, patch("feat/moved-on")).await); + + for (branch, wants) in [ + ("a..b", "branch"), + ("feat/nowhere", "isn't a branch of this repository"), + ("main", "isn't on its source branch"), + ] { + let refused = failed( + submit_patch_as(&world, &world.member, &repo, patch(branch)).await, + StatusCode::BAD_REQUEST, + ); + assert!( + refused["message"].as_str().unwrap().contains(wants), + "{branch}: expected {wants}, got {refused}" + ); + } + } + + #[tokio::test] + async fn submit_patch_refuses_a_missing_ticket_base_or_head() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo = create_repo_helper(&world, "scallop").await; + let base = seed_main(&world, &repo, &[("reef.txt", "old line\n")]); + let head = advance(&world, &repo, base, "reef.txt", "new line\n"); + let (ticket, _) = Ticket::on(&world, &repo) + .opened(&world.member, "kelp") + .await; + let ghost = Oid::from_hex(&"e".repeat(40)).unwrap(); + let patch = |ticket: &str, base: Oid, head: Oid| { + json!({ + "ticket": ticket, + "base": { "oid": base.to_hex() }, + "head": { "oid": head.to_hex() }, + "sourceRepo": repo + }) + }; + + for (patch, wants) in [ + ( + patch("3lkelpkelpk22", base, head), + "isn't a record on this knot", + ), + (patch(rkey_of(&ticket).as_str(), ghost, head), "base"), + (patch(rkey_of(&ticket).as_str(), base, ghost), "head"), + ( + patch(rkey_of(&ticket).as_str(), head, base), + "isn't an ancestor", + ), + ] { + let refused = failed( + submit_patch_as(&world, &world.member, &repo, patch).await, + StatusCode::BAD_REQUEST, + ); + assert!( + refused["message"].as_str().unwrap().contains(wants), + "expected {wants}, got {refused}" + ); + } + } + + #[tokio::test] + async fn a_repo_did_addresses_a_repo_whose_rkey_is_its_lowercased_name() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + assert_eq!( + as_member( + &world, + crate::repos::create_repo, + &nsid(CREATE), + json!({ "name": "Periwinkle" }) + ) + .await + .status(), + StatusCode::OK + ); + assert!(matches!( + resolve(&world, &RepoName::new("Periwinkle").unwrap()), + Resolved::Ready(None) + )); + assert!( + matches!( + resolve(&world, &RepoName::new("periwinkle").unwrap()), + Resolved::Ready(Some(_)) + ), + "Repo wasn't registered under its lowercased name" + ); + } +} + +mod fork_endpoints { + use super::*; + + use knot_git::{EntryKind, Identity, NewCommit, RefUpdate, Repo, StagedAction, StagedChange}; + use knot_runtime::HttpTransport; + use knot_types::{Oid, RefName, UnixSeconds}; + + const EMPTY_TREE_SHA1: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + const EMPTY_TREE_SHA256: &str = + "6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321"; + + fn empty_tree(repo: &Repo) -> Oid { + let hex = match repo.object_format() { + knot_types::ObjectFormat::SHA256 => EMPTY_TREE_SHA256, + _ => EMPTY_TREE_SHA1, + }; + Oid::from_hex(hex).unwrap() + } + + fn ident(time: i64) -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(time), + offset_seconds: 0, + } + } + + fn put_commit(repo: &Repo, parent: Option, path: &str, content: &str, time: i64) -> Oid { + let base_tree = match parent { + Some(parent) => repo.find_commit(parent).unwrap().tree, + None => empty_tree(repo), + }; + let staged = vec![StagedChange { + path: knot_types::RepoPath::new(path).unwrap(), + action: StagedAction::Put { + content: content.as_bytes().to_vec(), + kind: EntryKind::Blob, + }, + }]; + let tree = repo.write_staged_tree(base_tree, &staged).unwrap(); + repo.write_commit(&NewCommit { + tree, + parents: parent.into_iter().collect(), + author: ident(time), + committer: ident(time), + message: format!("put {path}"), + extra_headers: Vec::new(), + }) + .unwrap() + } + + fn advance(repo: &Repo, branch: &RefName, path: &str, content: &str, time: i64) -> Oid { + let old = repo.find_ref(branch).unwrap(); + let new = put_commit(repo, old, path, content, time); + let update = match old { + Some(old) => RefUpdate::Update { + name: branch.clone(), + old, + new, + }, + None => RefUpdate::Create { + name: branch.clone(), + new, + }, + }; + repo.update_ref(&update).unwrap(); + new + } + + fn main_ref() -> RefName { + RefName::new("refs/heads/main").unwrap() + } + + fn master_ref() -> RefName { + RefName::new("refs/heads/master").unwrap() + } + + fn member_did() -> OwnerDid { + OwnerDid::new(format!("did:web:{MEMBER_HOST}")).unwrap() + } + + fn source_url(rkey: &RepoRkey) -> String { + format!( + "https://{KNOT_HOST}/did:web:{MEMBER_HOST}/{}", + rkey.as_str() + ) + } + + const IMPORT_STATUS: &str = "org.tangled.repo.getImportStatus"; + + async fn import_status_as( + world: &World, + headers: HeaderMap, + did: &RepoDid, + ) -> Result { + crate::imports::import_status( + State(Arc::clone(&world.state)), + headers, + crate::Method::from_nsid(IMPORT_STATUS), + crate::query::ValidatedQuery(crate::imports::StatusParams { repo: did.clone() }), + ) + .await + } + + async fn import_status_for(world: &World, actor: &Actor, did: &RepoDid) -> serde_json::Value { + let audience = ServiceDid::new(did.as_str()).unwrap(); + let token = mint_for(actor, &nsid(IMPORT_STATUS), &audience); + json_of(import_status_as(world, bearer(&token), did).await.unwrap()).await + } + + async fn anonymous_import_status(world: &World, did: &RepoDid) -> serde_json::Value { + json_of( + import_status_as(world, HeaderMap::new(), did) + .await + .unwrap(), + ) + .await + } + + async fn described_content(world: &World, did: &RepoDid) -> Option { + anonymous_import_status(world, did).await["status"] + .as_str() + .map(str::to_string) + } + + async fn failed_import(world: &World) -> RepoDid { + let _ = kelp_source(world, &[]).await; + let did = import_of(world, &source_url(&RepoRkey::new("kelp").unwrap())).await; + rewrite_sidecar(world, &did, |job| { + job["source"] = json!("https://barnacle.nel.pet/did:plc:nowhere/uni"); + job["refs"] = serde_json::Value::Null; + }); + crate::imports::run_one(&world.state, &did).await; + did + } + + async fn create_repo_from(world: &World, source: serde_json::Value, name: &str) -> RepoDid { + assert_eq!( + as_member( + world, + crate::repos::create_repo, + &nsid(CREATE), + json!({ "name": name, "source": source }) + ) + .await + .status(), + StatusCode::OK + ); + match world + .state + .index + .resolve_repo(&member_did(), &RepoRkey::new(name.to_lowercase()).unwrap()) + { + Resolved::Ready(Some(did)) => did, + other => panic!("{name} wasn't registered under its lowercased name: {other:?}"), + } + } + + fn tangled(did: &Did) -> serde_json::Value { + json!({ "$type": knot_record::manifest::SOURCE_TANGLED, "did": did.as_str() }) + } + + fn external(uri: &str) -> serde_json::Value { + json!({ "$type": knot_record::manifest::SOURCE_EXTERNAL, "uri": uri }) + } + + async fn fork_repo(world: &World, source: &Did, pds: &KnotServiceUrl, name: &str) -> RepoDid { + world.publish_plc_doc(source, pds); + let did = create_repo_from(world, tangled(source), name).await; + crate::imports::run_one(&world.state, &did).await; + did + } + + fn import_world() -> World { + let world = World::new(); + world.grandfather(MEMBER_HOST); + world + } + + // the kelp source repo every import test fetches from, with the commits on main + async fn kelp_source(world: &World, commits: &[(&str, &str)]) -> (RepoDid, Vec) { + let did = create_repo_helper(world, "kelp").await; + let repo = world.layout.open(&did).unwrap(); + let tips = commits + .iter() + .enumerate() + .map(|(i, (path, content))| { + advance(&repo, &main_ref(), path, content, 1_000 + i as i64) + }) + .collect(); + (did, tips) + } + + async fn import_of(world: &World, source: &str) -> RepoDid { + create_repo_from(world, external(source), "uni").await + } + + fn read_job(world: &World, did: &RepoDid) -> crate::imports::ImportJob { + crate::imports::ImportJob::read(&world.state.layout.repo_path(did).unwrap()) + .unwrap() + .unwrap() + } + + fn rewrite_sidecar(world: &World, did: &RepoDid, edit: impl FnOnce(&mut serde_json::Value)) { + let sidecar = world + .state + .layout + .repo_path(did) + .unwrap() + .join(crate::imports::SIDECAR); + let mut job: serde_json::Value = + serde_json::from_slice(&std::fs::read(&sidecar).unwrap()).unwrap(); + edit(&mut job); + std::fs::write(&sidecar, serde_json::to_vec(&job).unwrap()).unwrap(); + } + + async fn recreate_import(world: &World, source: &str) -> StatusCode { + as_member( + world, + crate::repos::create_repo, + &nsid(CREATE), + json!({ "name": "uni", "source": external(source) }), + ) + .await + .status() + } + + #[tokio::test] + async fn an_import_lands_after_create_answers_and_says_so() { + let world = import_world(); + let (_, tips) = kelp_source( + &world, + &[("reef.txt", "kelp forest\n"), ("tide.txt", "rock pool\n")], + ) + .await; + let tip = tips[1]; + + let did = import_of(&world, &source_url(&RepoRkey::new("kelp").unwrap())).await; + let job = read_job(&world, &did); + assert_eq!( + job.state, + crate::imports::ImportState::Queued, + "create answers before the fetch" + ); + assert_eq!( + world + .layout + .open(&did) + .unwrap() + .find_ref(&main_ref()) + .unwrap(), + None, + "nothing has arrived yet" + ); + assert_eq!( + described_content(&world, &did).await.as_deref(), + Some("fetching") + ); + + crate::imports::run_one(&world.state, &did).await; + assert_eq!( + world + .layout + .open(&did) + .unwrap() + .find_ref(&main_ref()) + .unwrap(), + Some(tip) + ); + assert_eq!( + described_content(&world, &did).await.as_deref(), + Some("present") + ); + } + + #[tokio::test] + async fn an_import_without_an_advertised_symref_uses_the_upstream_head_target() { + let world = import_world(); + let (source_did, _) = kelp_source(&world, &[]).await; + let source = world.layout.open(&source_did).unwrap(); + source.set_head(&master_ref()).unwrap(); + let tip = advance(&source, &master_ref(), "reef.txt", "kelp forest\n", 1_000); + + let did = import_of(&world, &source_url(&RepoRkey::new("kelp").unwrap())).await; + rewrite_sidecar(&world, &did, |job| { + job["refs"]["head_symref"] = serde_json::Value::Null; + }); + + crate::imports::run_one(&world.state, &did).await; + let imported = world.layout.open(&did).unwrap(); + assert_eq!(imported.head().unwrap().name, master_ref()); + assert_eq!(imported.find_ref(&master_ref()).unwrap(), Some(tip)); + } + + #[tokio::test] + async fn a_failed_import_can_be_recreated_and_updates_existing_refs() { + let world = import_world(); + let (source_did, _) = kelp_source(&world, &[("reef.txt", "kelp forest\n")]).await; + let source = world.layout.open(&source_did).unwrap(); + let source_url = source_url(&RepoRkey::new("kelp").unwrap()); + let did = import_of(&world, &source_url).await; + let old = world + .layout + .open(&did) + .unwrap() + .find_ref(&main_ref()) + .unwrap(); + let new = advance(&source, &main_ref(), "tide.txt", "rock pool\n", 1_001); + assert_ne!(old, Some(new)); + + rewrite_sidecar(&world, &did, |job| { + job["state"] = json!("failed"); + job["error"] = json!("upstream disconnected"); + }); + + assert_eq!( + as_member( + &world, + crate::repos::create_repo, + &nsid(CREATE), + json!({ "name": "uni", "source": external(&source_url) }) + ) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + read_job(&world, &did).state, + crate::imports::ImportState::Queued + ); + + crate::imports::run_one(&world.state, &did).await; + assert_eq!( + world + .layout + .open(&did) + .unwrap() + .find_ref(&main_ref()) + .unwrap(), + Some(new) + ); + assert_eq!( + described_content(&world, &did).await.as_deref(), + Some("present") + ); + } + + #[tokio::test] + async fn a_partial_import_can_be_recreated_and_re_stages_the_fetch() { + let world = import_world(); + let (_, tips) = kelp_source( + &world, + &[("reef.txt", "kelp forest\n"), ("tide.txt", "rock pool\n")], + ) + .await; + let new = tips[1]; + let source_url = source_url(&RepoRkey::new("kelp").unwrap()); + let did = import_of(&world, &source_url).await; + + rewrite_sidecar(&world, &did, |job| { + job["state"] = json!("partial"); + job["error"] = json!("3 lfs objects are still missing"); + }); + + assert_eq!( + recreate_import(&world, &source_url).await, + StatusCode::OK, + "a partial import is retryable, not a conflict" + ); + assert_eq!( + read_job(&world, &did).state, + crate::imports::ImportState::Queued + ); + + crate::imports::run_one(&world.state, &did).await; + assert_eq!( + world + .layout + .open(&did) + .unwrap() + .find_ref(&main_ref()) + .unwrap(), + Some(new) + ); + assert_eq!( + described_content(&world, &did).await.as_deref(), + Some("present") + ); + } + + #[tokio::test] + async fn import_status_hides_the_reason_from_anyone_but_the_owner_and_admins() { + let world = import_world(); + let did = failed_import(&world).await; + + let anonymous = anonymous_import_status(&world, &did).await; + assert_eq!(anonymous["status"], json!("failed")); + assert_eq!( + anonymous["reason"], + json!(crate::imports::GENERIC_REASON), + "an anonymous caller gets the stable reason, never the raw import error" + ); + let stranger = import_status_for(&world, &world.stranger, &did).await; + assert_eq!(stranger["reason"], json!(crate::imports::GENERIC_REASON)); + + let raw = read_job(&world, &did).error.unwrap(); + assert_ne!(raw, crate::imports::GENERIC_REASON); + for (who, actor) in [("owner", &world.member), ("admin", &world.admin)] { + let seen = import_status_for(&world, actor, &did).await; + assert_eq!(seen["status"], json!("failed"), "{who}"); + assert_eq!( + seen["reason"], + json!(raw), + "the {who} reads the reason verbatim" + ); + } + } + + #[tokio::test] + async fn import_status_refuses_credentials_that_dont_verify() { + let world = import_world(); + let did = failed_import(&world).await; + let mut malformed = HeaderMap::new(); + malformed.insert(AUTHORIZATION, HeaderValue::from_static("Basic nope")); + let error = import_status_as(&world, malformed, &did).await.unwrap_err(); + assert_eq!(error.into_response().status(), StatusCode::UNAUTHORIZED); + + let knot_token = mint(&world.member, &nsid(IMPORT_STATUS)); + let error = import_status_as(&world, bearer(&knot_token), &did) + .await + .unwrap_err(); + assert_eq!( + error.into_response().status(), + StatusCode::UNAUTHORIZED, + "the token is addressed to the repo, not the knot" + ); + } + + #[tokio::test] + async fn import_status_folds_queued_into_fetching_and_keeps_a_partial_reason() { + let world = import_world(); + let _ = kelp_source(&world, &[]).await; + let did = import_of(&world, &source_url(&RepoRkey::new("kelp").unwrap())).await; + assert_eq!( + read_job(&world, &did).state, + crate::imports::ImportState::Queued + ); + let queued = anonymous_import_status(&world, &did).await; + assert_eq!(queued, json!({ "status": "fetching" })); + + rewrite_sidecar(&world, &did, |job| { + job["state"] = json!("partial"); + job["error"] = json!("3 lfs objects are still missing"); + }); + let owner = import_status_for(&world, &world.member, &did).await; + assert_eq!( + owner, + json!({ "status": "partial", "reason": "3 lfs objects are still missing" }) + ); + } + + #[tokio::test] + async fn an_import_of_a_source_without_commits_lands_present_and_empty() { + let world = import_world(); + let _ = kelp_source(&world, &[]).await; + let did = import_of(&world, &source_url(&RepoRkey::new("kelp").unwrap())).await; + + crate::imports::run_one(&world.state, &did).await; + assert_eq!( + import_status_for(&world, &world.member, &did).await, + json!({ "status": "present" }) + ); + let imported = world.layout.open(&did).unwrap(); + assert_eq!(imported.find_ref(&main_ref()).unwrap(), None); + knot_pack::advertise_upload_v0(&imported).unwrap(); + knot_pack::advertise_upload(&imported).unwrap(); + } + + #[tokio::test] + async fn an_import_of_lfs_pointers_the_source_cant_serve_is_partial() { + let world = import_world(); + let pointer = |digit: char| { + format!( + "version https://git-lfs.github.com/spec/v1\noid sha256:{}\nsize 12\n", + digit.to_string().repeat(64) + ) + }; + let (first, second) = (pointer('a'), pointer('b')); + let (_, tips) = kelp_source(&world, &[("reef.bin", &first), ("tide.bin", &second)]).await; + let did = import_of(&world, &source_url(&RepoRkey::new("kelp").unwrap())).await; + + crate::imports::run_one(&world.state, &did).await; + assert_eq!( + world + .layout + .open(&did) + .unwrap() + .find_ref(&main_ref()) + .unwrap(), + Some(tips[1]) + ); + assert_eq!( + import_status_for(&world, &world.member, &did).await, + json!({ + "status": "partial", + "reason": "2 lfs objects are still missing" + }) + ); + } + + #[tokio::test] + async fn import_status_omits_both_for_a_repo_not_made_from_a_source() { + let world = import_world(); + let did = create_repo_helper(&world, "kelp").await; + assert_eq!(anonymous_import_status(&world, &did).await, json!({})); + assert_eq!( + import_status_for(&world, &world.member, &did).await, + json!({}) + ); + } + + #[tokio::test] + async fn import_status_of_an_unknown_repo_is_repo_not_found() { + let world = import_world(); + let ghost = RepoDid::new("did:web:squid.nel.pet").unwrap(); + let response = import_status_as(&world, HeaderMap::new(), &ghost) + .await + .unwrap_err() + .into_response(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(json_of(response).await["error"], json!("RepoNotFound")); + } + + #[tokio::test] + async fn an_import_refuses_a_plain_http_source() { + let world = import_world(); + assert_eq!( + as_member( + &world, + crate::repos::create_repo, + &nsid(CREATE), + json!({ "name": "uni", "source": external("http://kelp.example/uni") }) + ) + .await + .status(), + StatusCode::BAD_REQUEST + ); + } + + #[tokio::test] + async fn queued_imports_are_recovered_without_scanning_git_internals() { + let world = import_world(); + let _ = kelp_source(&world, &[]).await; + let did = import_of(&world, &source_url(&RepoRkey::new("kelp").unwrap())).await; + let repo_path = world.state.layout.repo_path(&did).unwrap(); + let nested = repo_path.join("objects/aa"); + std::fs::create_dir_all(&nested).unwrap(); + let mut decoy = read_job(&world, &did); + decoy.state = crate::imports::ImportState::Fetching; + std::fs::write( + nested.join(crate::imports::SIDECAR), + serde_json::to_vec(&decoy).unwrap(), + ) + .unwrap(); + + assert_eq!(crate::imports::recover_jobs(&world.state), vec![did]); + assert_eq!( + crate::imports::ImportJob::read(&nested) + .unwrap() + .unwrap() + .state, + crate::imports::ImportState::Fetching, + "fixed-depth recovery must not descend into bare git storage" + ); + } + + #[tokio::test] + async fn a_failed_fetch_stays_on_the_record() { + let world = import_world(); + let did = failed_import(&world).await; + let after = read_job(&world, &did); + assert_eq!(after.state, crate::imports::ImportState::Failed); + assert!( + after.error.is_some(), + "the reason has to survive the worker" + ); + assert_eq!( + described_content(&world, &did).await.as_deref(), + Some("failed") + ); + } + + #[tokio::test] + async fn a_fetch_interrupted_by_a_restart_is_reported_not_resumed() { + let world = import_world(); + let _ = kelp_source(&world, &[]).await; + let did = import_of(&world, &source_url(&RepoRkey::new("kelp").unwrap())).await; + rewrite_sidecar(&world, &did, |job| job["state"] = json!("fetching")); + + crate::imports::recover(&world.state); + + let after = read_job(&world, &did); + assert_eq!( + after.state, + crate::imports::ImportState::Failed, + "a fetch this process did not finish is not resumed later" + ); + assert!( + after.error.is_some(), + "the reason has to say the knot restarted" + ); + assert_eq!( + described_content(&world, &did).await.as_deref(), + Some("failed") + ); + assert_eq!( + world + .layout + .open(&did) + .unwrap() + .find_ref(&main_ref()) + .unwrap(), + None, + "nothing may arrive from a run that never finished" + ); + } + + #[tokio::test] + async fn an_import_takes_the_source_refs_but_keeps_no_origin() { + let world = import_world(); + let (_, tips) = kelp_source( + &world, + &[("reef.txt", "kelp forest\n"), ("tide.txt", "rock pool\n")], + ) + .await; + let tip = tips[1]; + + let imported_did = import_of(&world, &source_url(&RepoRkey::new("kelp").unwrap())).await; + crate::imports::run_one(&world.state, &imported_did).await; + let imported = world.layout.open(&imported_did).unwrap(); + assert_eq!( + imported.find_ref(&main_ref()).unwrap(), + Some(tip), + "an import takes the source's refs" + ); + assert_eq!( + imported.origin_url(), + None, + "an import must keep no upstream, or it reads as a fork everywhere" + ); + } + + struct ForkWorld { + world: World, + source_did: RepoDid, + fork_did: RepoDid, + tip: Oid, + } + + async fn forked_world() -> ForkWorld { + let world = World::new(); + world.grandfather(MEMBER_HOST); + let source_did = create_repo_helper(&world, "kelp").await; + let source = world.layout.open(&source_did).unwrap(); + advance(&source, &main_ref(), "reef.txt", "kelp forest\n", 1_000); + let tip = advance(&source, &main_ref(), "tide.txt", "rock pool\n", 1_001); + [ + "refs/tags/v1", + "refs/cobs/org.tangled.graph.collaborator/limpet", + "refs/hidden/feature/main", + ] + .into_iter() + .for_each(|name| { + source + .update_ref(&RefUpdate::Create { + name: RefName::new(name).unwrap(), + new: tip, + }) + .unwrap(); + }); + let fork_did = fork_repo( + &world, + &Did::new_owned(source_did.as_str()).unwrap(), + &KnotServiceUrl::new(format!("https://{KNOT_HOST}")).unwrap(), + "uni", + ) + .await; + ForkWorld { + world, + source_did, + fork_did, + tip, + } + } + + #[tokio::test] + async fn a_member_forks_a_repo_hosted_on_this_knot() { + let setup = forked_world().await; + assert_ne!(setup.fork_did, setup.source_did); + + let fork = setup.world.layout.open(&setup.fork_did).unwrap(); + assert_eq!(fork.find_ref(&main_ref()).unwrap(), Some(setup.tip)); + assert_eq!( + fork.find_ref(&RefName::new("refs/tags/v1").unwrap()) + .unwrap(), + Some(setup.tip) + ); + assert_eq!(fork.default_branch().unwrap().as_str(), "refs/heads/main"); + assert_eq!( + fork.origin_url(), + Some(OriginUrl::new(format!( + "https://{KNOT_HOST}/{}", + setup.source_did + ))) + ); + assert!( + fork.references().unwrap().iter().all(|record| { + (!record.name.as_str().starts_with("refs/cobs/") + || record + .name + .as_str() + .starts_with("refs/cobs/org.tangled.repo.manifest/")) + && !record.name.as_str().starts_with("refs/hidden/") + }), + "a fork must copy only heads and tags, never cob or hidden refs" + ); + + let entry = fork + .entry_at(setup.tip, &knot_types::RepoPath::new("tide.txt").unwrap()) + .unwrap() + .unwrap(); + assert_eq!(fork.read_blob(entry.oid).unwrap(), b"rock pool\n"); + } + + #[tokio::test] + async fn submit_patch_fetches_the_head_from_a_fork() { + let setup = forked_world().await; + let world = &setup.world; + world.publish_plc_doc( + &Did::new_owned(setup.fork_did.as_str()).unwrap(), + &KnotServiceUrl::new(format!("https://{KNOT_HOST}")).unwrap(), + ); + let fork = world.layout.open(&setup.fork_did).unwrap(); + let head = advance(&fork, &main_ref(), "kelp.txt", "holdfast\n", 2_000); + assert!(!world.layout.open(&setup.source_did).unwrap().contains(head)); + let (ticket, _) = Ticket::on(world, &setup.source_did) + .opened(&world.member, "kelp") + .await; + + let written = ok(submit_patch_as( + world, + &world.member, + &setup.source_did, + json!({ + "ticket": rkey_of(&ticket), + "base": { "oid": setup.tip.to_hex() }, + "head": { "oid": head.to_hex() }, + "sourceRepo": setup.fork_did, + "sourceBranch": "main" + }), + ) + .await); + assert!(world.layout.open(&setup.source_did).unwrap().contains(head)); + let record = ok(record_at(world, &setup.source_did, &uri_of(&written), None).await); + assert_eq!(record["value"]["source"]["repo"], setup.fork_did.as_str()); + } + + #[tokio::test] + async fn forking_a_source_this_knot_does_not_host_is_not_found() { + const GHOST: &str = "did:plc:whelkwhelkwhelkwhelkwhel"; + let ghost = Did::new_owned(GHOST).unwrap(); + let world = World::new(); + world.grandfather(MEMBER_HOST); + world.publish_plc_doc( + &ghost, + &KnotServiceUrl::new(format!("https://{KNOT_HOST}")).unwrap(), + ); + assert_eq!( + as_member( + &world, + crate::repos::create_repo, + &nsid(CREATE), + json!({ "name": "uni", "source": tangled(&ghost) }) + ) + .await + .status(), + StatusCode::NOT_FOUND + ); + assert!(matches!( + world + .state + .index + .resolve_repo(&member_did(), &RepoRkey::new("uni").unwrap()), + Resolved::Ready(None) + )); + } + + #[tokio::test] + async fn a_fork_over_http_takes_the_upstream_object_format() { + let upstream_dir = tempfile::tempdir().unwrap(); + let upstream_path = upstream_dir.path().join("uni.git"); + let upstream = + Repo::create_with_format(&upstream_path, knot_types::ObjectFormat::SHA1).unwrap(); + upstream.set_head(&main_ref()).unwrap(); + advance(&upstream, &main_ref(), "reef.txt", "kelp forest\n", 1_000); + let tip = advance(&upstream, &main_ref(), "tide.txt", "rock pool\n", 1_001); + + let served = Arc::new(RwLock::new(upstream_path.clone())); + let path = Arc::clone(&served); + let discoveries = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&discoveries); + let git_http: Arc = + Arc::new(FakeHttp::new(move |request: &HttpRequest| { + let repo = Repo::open(path.read().clone()).unwrap(); + let body = if request.url.path().ends_with("/info/refs") { + counted.fetch_add(1, Ordering::SeqCst); + knot_pack::advertise_upload(&repo).unwrap() + } else { + knot_pack::upload_pack(&repo, request.body.as_deref().unwrap_or_default()) + .unwrap() + }; + Ok(HttpResponse { + status: StatusCode::OK, + headers: http::HeaderMap::new(), + body: body.into(), + }) + })); + + let world = World::with_git_http(git_http, knot_types::ObjectFormat::SHA256); + world.grandfather(MEMBER_HOST); + let plain = create_repo_helper(&world, "kelp").await; + assert_eq!( + world.layout.open(&plain).unwrap().object_format(), + knot_types::ObjectFormat::SHA256, + "a repo with no upstream still uses this knot's configured format" + ); + + let squid = Did::new_owned("did:plc:squidsquidsquidsquidsqui").unwrap(); + let remote = format!("https://barnacle.nel.pet/{}", squid.as_str()); + let fork_did = fork_repo( + &world, + &squid, + &KnotServiceUrl::new("https://barnacle.nel.pet").unwrap(), + "uni", + ) + .await; + let fork = world.layout.open(&fork_did).unwrap(); + assert_eq!( + fork.object_format(), + knot_types::ObjectFormat::SHA1, + "a fork of a sha1 upstream must be sha1 so the upstream's objects ingest" + ); + assert_eq!(fork.find_ref(&main_ref()).unwrap(), Some(tip)); + assert_eq!( + discoveries.load(Ordering::SeqCst), + 1, + "create and its worker share one upstream ref discovery" + ); + assert_eq!(fork.origin_url(), Some(OriginUrl::new(remote))); + assert_eq!( + fork.read_blob( + fork.entry_at(tip, &knot_types::RepoPath::new("tide.txt").unwrap()) + .unwrap() + .unwrap() + .oid + ) + .unwrap(), + b"rock pool\n" + ); + } +} + +mod legacy_admin_route { + use super::*; + use crate::legacy_admin::CREATE_INVITE_CODE_ROUTE; + use tower::ServiceExt; + + const SECRET: &str = "nekomilk2"; + + fn basic(user: &str, password: &str) -> String { + format!( + "Basic {}", + base64::engine::general_purpose::STANDARD + .encode(format!("{user}:{password}").as_bytes()) + ) + } + + fn router(world: &World) -> axum::Router { + crate::router(Arc::clone(&world.state)) + .merge(crate::legacy_admin::router(Arc::clone(&world.state))) + } + + async fn invite(router: &axum::Router, user: &str, password: &str) -> StatusCode { + let request = http::Request::builder() + .method("POST") + .uri(CREATE_INVITE_CODE_ROUTE) + .header(AUTHORIZATION, basic(user, password)) + .body(axum::body::Body::empty()) + .unwrap(); + router.clone().oneshot(request).await.unwrap().status() + } + + async fn grant( + world: &World, + method: RepoMethod, + collection: &RecordCollection, + user: &str, + password: &str, + subject: &AccountDid, + ) -> StatusCode { + let body = match method { + RepoMethod::Create => grant_body(&world.state.knot_repo, collection, subject), + RepoMethod::Delete => json!({ + "repo": world.state.knot_repo, + "collection": collection, + "rkey": subject, + }), + }; + let request = http::Request::builder() + .method("POST") + .uri(method.route()) + .header(AUTHORIZATION, basic(user, password)) + .header(http::header::CONTENT_TYPE, "application/json") + .body(axum::body::Body::from(body.to_string())) + .unwrap(); + router(world).oneshot(request).await.unwrap().status() + } + + #[tokio::test] + async fn the_invite_code_route_admits_only_the_configured_credentials() { + let world = World::new().with_legacy_admin_secret(SECRET); + let router = router(&world); + + assert_eq!(invite(&router, "admin", SECRET).await, StatusCode::OK); + assert_eq!( + invite(&router, "admin", "nope").await, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + invite(&router, "root", SECRET).await, + StatusCode::UNAUTHORIZED + ); + } + + #[tokio::test] + async fn create_record_admits_the_shared_secret_for_members() { + let world = World::new().with_legacy_admin_secret(SECRET); + let subject = account(MEMBER_HOST); + + assert_eq!( + grant( + &world, + RepoMethod::Create, + &member_collection(), + "admin", + "nope", + &subject + ) + .await, + StatusCode::UNAUTHORIZED, + "the knot 401s a wrong secret before the grant runs" + ); + assert_eq!( + world.state.index.effective_member(&subject), + Resolved::Ready(false), + "a refused call grants nothing" + ); + + assert_eq!( + grant( + &world, + RepoMethod::Create, + &member_collection(), + "admin", + SECRET, + &subject + ) + .await, + StatusCode::OK + ); + assert_eq!( + world.state.index.effective_member(&subject), + Resolved::Ready(true) + ); + let Resolved::Ready(members) = world.state.index.member_entries() else { + panic!("The member grant list is warm in this test"); + }; + assert_eq!( + members + .iter() + .find(|(who, _)| *who == subject) + .expect("Member is on the grant list") + .1 + .added_by, + world.state.service_owner, + "the shared secret doesn't take an actor, so `added_by` is the service owner" + ); + + assert_eq!( + grant( + &world, + RepoMethod::Delete, + &member_collection(), + "admin", + SECRET, + &subject + ) + .await, + StatusCode::OK + ); + assert_eq!( + world.state.index.effective_member(&subject), + Resolved::Ready(false) + ); + } + + #[tokio::test] + async fn create_record_admits_the_shared_secret_for_blocks() { + let world = World::open().with_legacy_admin_secret(SECRET); + let stranger = account(STRANGER_HOST); + + assert_eq!( + grant( + &world, + RepoMethod::Create, + &block_collection(), + "admin", + SECRET, + &stranger + ) + .await, + StatusCode::OK + ); + assert_eq!( + world.state.index.is_blocked(&stranger), + Resolved::Ready(true) + ); + assert_eq!( + grant( + &world, + RepoMethod::Delete, + &block_collection(), + "admin", + SECRET, + &stranger + ) + .await, + StatusCode::OK + ); + assert_eq!( + world.state.index.is_blocked(&stranger), + Resolved::Ready(false) + ); + assert_eq!( + grant( + &world, + RepoMethod::Create, + &block_collection(), + "admin", + SECRET, + &account(ADMIN_HOST) + ) + .await, + StatusCode::FORBIDDEN, + "the secret doesn't lift the admin carve-out" + ); + } + + #[tokio::test] + async fn the_legacy_route_sheds_a_pre_auth_flood_from_one_peer() { + use axum::extract::ConnectInfo; + use std::net::SocketAddr; + + let world = World::new().with_legacy_admin_secret(SECRET); + let router = crate::legacy_admin::router(Arc::clone(&world.state)); + let peer = SocketAddr::from(([203, 0, 113, 9], 5555)); + + let statuses: Vec = futures::stream::iter(0..22) + .then(|_| { + let router = router.clone(); + async move { + let mut request = http::Request::builder() + .method("POST") + .uri(CREATE_INVITE_CODE_ROUTE) + .body(axum::body::Body::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(peer)); + router.oneshot(request).await.unwrap().status() + } + }) + .collect() + .await; + + assert!( + statuses[..20] + .iter() + .all(|status| *status == StatusCode::UNAUTHORIZED), + "the knot admits the per-peer burst and then fails it on the missing credentials, got {statuses:?}" + ); + assert!( + statuses[20..] + .iter() + .all(|status| *status == StatusCode::TOO_MANY_REQUESTS), + "past the burst the knot sheds the guess flood before it reaches the secret comparison, got {statuses:?}" + ); + } + + #[test] + fn every_wire_copy_of_an_embedded_payload_fits_a_quarter_of_the_response() { + let limits = crate::ByteLimits::default(); + assert_eq!(limits.response.get(), 5 * 1024 * 1024); + let response = limits.response.get() as u64; + assert_eq!( + limits.binary_patch(), + knot_git::BinaryBudget::new(response / 4 / 3), + "a compare serialises the series twice and the combined patch once" + ); + let per_pass = match limits.binary_patch() { + knot_git::BinaryBudget::Spend { remaining, .. } => remaining, + knot_git::BinaryBudget::Omit => panic!("binary_patch spends, it doesn't omit"), + }; + assert!( + per_pass * 3 <= response / 4, + "three copies of {per_pass} bytes must stay inside a quarter of {response}" + ); + } +} + +mod grant_lists { + use super::*; + + const DENIED: &str = "you don't have a live collaboration on this repository"; + const GRACE: u64 = 21_600; + + #[derive(Debug, Clone, Copy)] + enum WhichGrantList { + Knot, + Repository, + } + + struct Fixture { + world: World, + from: String, + repo: Option, + subject: AccountDid, + } + + impl Fixture { + async fn opened(which: WhichGrantList) -> Self { + let world = World::new(); + let repo = match which { + WhichGrantList::Knot => None, + WhichGrantList::Repository => { + world.grandfather(MEMBER_HOST); + Some(create_repo_helper(&world, "nautilus").await) + } + }; + let subject = account(STRANGER_HOST); + world.seed(repo.as_ref(), STRANGER_HOST); + let fixture = Self { + world, + from: format!("{which:?}"), + repo, + subject, + }; + assert_eq!( + fixture.on_grant_list(), + Resolved::Ready(true), + "{}: the fixture must open on the grant list", + fixture.from + ); + fixture + } + + fn repo(&self) -> Option<&RepoDid> { + self.repo.as_ref() + } + + fn on_grant_list(&self) -> Resolved { + self.world.on_grant_list(self.repo(), &self.subject) + } + + async fn allowed(&self) -> bool { + let state = &self.world.state; + match self.repo() { + Some(repo) => crate::authorize_push(state, &self.subject, repo, DENIED).await, + None => crate::authorize_create(state, &self.subject, DENIED).await, + } + .is_ok() + } + } + + async fn every_grant_list(check: impl AsyncFn(Fixture)) { + check(Fixture::opened(WhichGrantList::Knot).await).await; + check(Fixture::opened(WhichGrantList::Repository).await).await; + } + + #[tokio::test] + async fn a_second_add_leaves_the_entry_and_the_admins_revocation_clears_it() { + every_grant_list(async |fixture: Fixture| { + let from = &fixture.from; + let world = &fixture.world; + + let repeated = offer(world, fixture.repo(), &fixture.subject).await; + assert_eq!( + repeated, + StatusCode::OK, + "{from}: an operator adding again keeps the entry it already made rather \ + than appending another" + ); + assert_eq!( + fixture.on_grant_list(), + Resolved::Ready(true), + "{from}: a second add leaves the entry alone" + ); + + let revoked = match fixture.repo() { + None => revoke(world, &world.admin, &member_collection(), &fixture.subject).await, + Some(repo) => { + revoke_collaborator(world, &world.member, repo, &fixture.subject).await + } + }; + assert_eq!( + revoked, + StatusCode::OK, + "{from}: a revocation is one of the two changes an acl consumer reads" + ); + assert_eq!( + fixture.on_grant_list(), + Resolved::Ready(false), + "{from}: the revocation has to leave a Removal behind" + ); + }) + .await; + } + + #[tokio::test] + async fn the_knot_authorizes_from_the_grant_list_and_never_a_pds() { + every_grant_list(async |fixture: Fixture| { + let from = &fixture.from; + assert_eq!( + (fixture.allowed().await, fixture.world.record_reads()), + (true, 0), + "{from}: the knot authorizes from the grant list with zero pds reads" + ); + fixture.world.advance(GRACE * 2); + assert_eq!( + (fixture.allowed().await, fixture.world.record_reads()), + (true, 0), + "{from}: an entry never expires, and the knot still hasn't read a pds" + ); + }) + .await; + } + + mod materialization { + use super::*; + use knot_cob::CobStore; + + fn members_changes(world: &World) -> Vec { + let meta = Repo::open(world.state.meta_path.clone()).unwrap(); + let store = CobStore::new(&meta); + let object = store.list::().unwrap()[0]; + store + .changes_since::(object, None) + .unwrap() + .changes + } + + #[tokio::test] + async fn an_offered_member_becomes_a_record_on_the_knot_account() { + let world = World::new(); + let state = world.state(); + let (code, _) = state.invite_codes.mint(&*state.entropy, state.now()); + assert_eq!( + offer(&world, None, &account(MEMBER_HOST)).await, + StatusCode::OK + ); + assert_eq!(redeem(&world, &world.stranger, &code).await, StatusCode::OK); + + let meta_path = world.state.meta_path.clone(); + let mut leaves = leaves_at(move || Repo::open(meta_path).unwrap()).await; + leaves.sort(); + assert_eq!( + leaves, + vec![ + format!("org.tangled.knot.member/{}", account(MEMBER_HOST).as_str()), + format!("org.tangled.knot.member/{}", world.stranger.did.as_str()), + ], + "both the offer and the redemption become member records on the knot account" + ); + assert!( + members_changes(&world) + .iter() + .all(|change| change.record.bytes().is_none()), + "the record is a projection, and the member change log doesn't store it" + ); + } + + const FIXTURE_COLLECTION: &str = "org.tangled.track.ticket"; + + #[derive(serde::Serialize)] + struct Fixture { + #[serde(rename = "$type")] + record_type: &'static str, + #[serde(rename = "createdAt")] + created_at: i64, + } + + async fn materialize_meta(world: &World, rkeys: &[&str]) { + let meta = Repo::open(world.state.meta_path.clone()).unwrap(); + let signer = world.state.secrets.signer(&world.state.knot_did).unwrap(); + let did = knot_types::RepoDid::new(world.state.knot_did.as_str()).unwrap(); + let records: std::collections::BTreeMap = + rkeys + .iter() + .map(|rkey| { + ( + knot_types::MstKey::new(format!("{FIXTURE_COLLECTION}/{rkey}")) + .unwrap(), + knot_types::RecordBody::new( + serde_ipld_dagcbor::to_vec(&Fixture { + record_type: FIXTURE_COLLECTION, + created_at: 1_000, + }) + .unwrap(), + ) + .unwrap(), + ) + }) + .collect(); + knot_record::chain::materialize( + &meta, + &did, + &knot_record::chain::in_tree(records), + &signer, + &mut jacquard_common::types::tid::Ticker::new(), + UnixSeconds::new(2_000), + ) + .await + .unwrap(); + } + + fn framed_events(world: &World) -> Vec> { + world + .state + .firehose + .replay( + knot_events::EventCursor::START, + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(64).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + ) + .events + } + + async fn leaves_at(open: impl FnOnce() -> Repo + Send + 'static) -> Vec { + tokio::task::spawn_blocking(move || { + futures::executor::block_on(knot_record::chain::leaves(&open())) + .unwrap() + .into_keys() + .map(|key| key.as_str().to_owned()) + .collect() + }) + .await + .unwrap() + } + + #[tokio::test] + async fn every_member_of_the_grant_list_gets_a_record_on_the_knot_account() { + let world = World::new(); + world.seed(None, "limpet.nel.pet"); + world.seed(None, "whelk.nel.pet"); + world.refresh(None); + let surfaces = world.state.clone(); + tokio::task::spawn_blocking(move || crate::materialize::enable_surfaces(&surfaces)) + .await + .unwrap(); + let meta_path = world.state.meta_path.clone(); + assert_eq!( + leaves_at(move || Repo::open(meta_path).unwrap()) + .await + .len(), + 2, + "each member of the grant list gets one record on the knot account" + ); + } + + #[tokio::test] + async fn a_failed_chain_sync_still_announces_and_refreshes() { + use axum::body::Body; + use axum::extract::ConnectInfo; + use std::net::SocketAddr; + use tower::ServiceExt; + + let world = World::new(); + let app = crate::router(Arc::clone(&world.state)); + let knot_repo = world.state.knot_repo.clone(); + let post = |method: &'static str, subject: &AccountDid| { + let mut request = http::Request::builder() + .method("POST") + .uri(match method { + CREATE_RECORD => crate::social::CREATE_RECORD_ROUTE, + _ => crate::social::DELETE_RECORD_ROUTE, + }) + .header( + http::header::AUTHORIZATION, + format!("Bearer {}", mint(&world.admin, &nsid(method))), + ) + .body(Body::from( + serde_json::to_vec(&match method { + CREATE_RECORD => grant_body(&knot_repo, &member_collection(), subject), + _ => json!({ + "repo": knot_repo, + "collection": MEMBER, + "rkey": subject.as_str(), + }), + }) + .unwrap(), + )) + .unwrap(); + request + .extensions_mut() + .insert(ConnectInfo(SocketAddr::from(([203, 0, 113, 31], 5555)))); + app.clone().oneshot(request) + }; + + let holds = + |world: &World, subject: &AccountDid| match world.state.index.member_entries() { + knot_index::Resolved::Ready(entries) => entries + .iter() + .any(|(did, _)| did.as_str() == subject.as_str()), + knot_index::Resolved::Warming => panic!("The members projection is warming"), + }; + let limpet = AccountDid::new("did:plc:limpet").unwrap(); + let response = post(CREATE_RECORD, &limpet).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK, "the healthy add lands"); + assert!(holds(&world, &limpet)); + + let meta = Repo::open(world.state.meta_path.clone()).unwrap(); + let junk = meta.git().write_blob(b"not a commit").unwrap().detach(); + let chain_ref = knot_types::RefName::new(knot_record::chain::CHAIN_REF).unwrap(); + let healthy = knot_record::chain::tip(&meta).unwrap().unwrap().git; + meta.update_ref(&knot_git::RefUpdate::Update { + name: chain_ref, + old: healthy, + new: knot_types::Oid::from(junk), + }) + .unwrap(); + + let whelk = AccountDid::new("did:plc:whelk").unwrap(); + let response = post(CREATE_RECORD, &whelk).await.unwrap(); + assert_eq!( + response.status(), + StatusCode::OK, + "an add stands even when the chain is unreadable" + ); + assert!( + holds(&world, &whelk), + "the projection refreshes even with a broken chain ref" + ); + + let response = post(DELETE_RECORD, &whelk).await.unwrap(); + assert_eq!( + response.status(), + StatusCode::OK, + "the removal stands even when the chain is unreadable" + ); + assert!( + !holds(&world, &whelk), + "the projection drops the removed member" + ); + } + #[tokio::test] + async fn a_legacy_repository_with_social_records_gets_its_chain_at_boot() { + let world = World::new(); + world.grandfather(MEMBER_HOST); + let repo = create_repo_helper(&world, "anemone").await; + let ticket = Ticket::on(&world, &repo); + let (uri, _version) = ticket.opened(&world.member, "kelp").await; + + let git = world.layout.open(&repo).unwrap(); + let chain_ref = knot_types::RefName::new(knot_record::chain::CHAIN_REF).unwrap(); + let healthy = knot_record::chain::tip(&git).unwrap().unwrap().git; + git.update_ref(&knot_git::RefUpdate::Delete { + name: chain_ref, + old: healthy, + }) + .unwrap(); + assert!( + knot_record::chain::tip(&git).unwrap().is_none(), + "the fixture now matches a repository from before chains existed" + ); + + let surfaces = world.state.clone(); + tokio::task::spawn_blocking(move || crate::materialize::enable_surfaces(&surfaces)) + .await + .unwrap(); + + let leaves = leaves_at(move || world.layout.open(&repo).unwrap()).await; + let record = uri.as_str().rsplit('/').next().unwrap().to_owned(); + assert!( + leaves + .iter() + .any(|key| key == &format!("{TICKET}/{record}")), + "boot materializes the repository's record chain, and the social record that \ + predates chains is on it, among {leaves:?}" + ); + } + + #[tokio::test] + async fn a_frameless_tip_from_before_the_upgrade_gets_a_sync_frame_at_boot() { + let world = World::new(); + materialize_meta(&world, &["did:plc:limpet"]).await; + let meta = Repo::open(world.state.meta_path.clone()).unwrap(); + let tip = knot_record::chain::tip(&meta).unwrap().unwrap(); + assert!( + knot_record::chain::unframed_tip(&meta).unwrap().is_some(), + "the enable helper writes its chain with no frame blobs" + ); + crate::materialize::enable_surfaces(&world.state); + let frames = framed_events(&world); + let [event] = &frames[..] else { + panic!("A single sync frame covers the frameless tip"); + }; + let mut read = &event.frame[..]; + let header: serde_json::Value = + serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + assert_eq!(header["t"], "#sync"); + #[derive(serde::Deserialize)] + struct SyncProbe { + did: String, + rev: String, + #[serde(with = "serde_bytes")] + blocks: Vec, + } + let payload: SyncProbe = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + assert_eq!( + payload.rev, + tip.rev.to_string(), + "the sync frame asserts the tip the upgrade inherited" + ); + assert_eq!(payload.did, world.state.knot_did.as_str()); + assert!( + !payload.blocks.is_empty(), + "the sync frame carries the commit block in its CAR" + ); + crate::materialize::enable_surfaces(&world.state); + assert_eq!( + framed_events(&world).len(), + 2, + "each boot reasserts an unframed tip, and no commit accompanies it" + ); + } + } +} + +#[tokio::test] +async fn sweep_gives_up_on_tombstoned_repository_before_first_post() { + let repo = minted_repo(PlcFake::default()).await; + repo.fake + .lock() + .tombstones + .insert(repo.did.as_str().to_owned()); + + let mut memory = crate::plc::SweepMemory::default(); + repo.clock.advance(std::time::Duration::from_secs(31)); + crate::plc::sweep_tick(&repo.state, &mut memory, 1).await; + + assert_eq!( + repo.fake.lock().post_count, + 0, + "the sweep reads the tombstone and stops before posting" + ); + assert_eq!(memory.settled.len(), 0); + assert!(memory.is_given_up(repo.did.as_str())); +} + +#[tokio::test] +async fn sweep_gives_up_on_legacy_rotation_missing_from_sealed_archive() { + let foreign_rotation = "did:key:z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK"; + let legacy = json!({ + "type": "plc_operation", + "rotationKeys": [foreign_rotation], + "verificationMethods": {"atproto": foreign_rotation}, + "alsoKnownAs": [], + "services": {"atproto_pds": { + "type": "AtprotoPersonalDataServer", + "endpoint": format!("https://{KNOT_HOST}"), + }}, + "prev": null, + "sig": "AAAA", + }); + let repo = minted_repo(PlcFake { + last_override: Some(legacy), + ..PlcFake::default() + }) + .await; + + let mut memory = crate::plc::SweepMemory::default(); + repo.clock.advance(std::time::Duration::from_secs(31)); + crate::plc::sweep_tick(&repo.state, &mut memory, 1).await; + + assert_eq!( + repo.fake.lock().post_count, + 0, + "the sweep signs only from the sealed archive" + ); + assert_eq!(memory.settled.len(), 0); + assert!(memory.is_given_up(repo.did.as_str())); +} + +#[tokio::test] +async fn sweep_gives_up_on_a_permanently_rejected_repository_after_one_post() { + let repo = minted_repo(PlcFake { + reject_posts: true, + ..PlcFake::default() + }) + .await; + + let mut memory = crate::plc::SweepMemory::default(); + for _ in 0..2 { + repo.clock.advance(std::time::Duration::from_secs(31)); + crate::plc::sweep_tick(&repo.state, &mut memory, 1).await; + } + + assert_eq!( + repo.fake.lock().post_count, + 1, + "Sweep gives up at first permanent rejection. Second due pass skips that repository" + ); + assert_eq!(memory.settled.len(), 0); + assert!(memory.is_given_up(repo.did.as_str())); +} + +#[tokio::test] +async fn a_sweep_as_wide_as_the_candidates_lands_every_genesis_in_one_pass() { + let admin = actor(1, ADMIN_HOST); + let key = admin.signer.public_key().as_bytes().to_vec(); + let fake = Arc::new(Mutex::new(PlcFake { + reject_posts: true, + ..PlcFake::default() + })); + let responder = plc_stage(Arc::clone(&fake), key); + let (_dir, state, clock) = build_state_with_clock(responder, true); + let owner = OwnerDid::new(format!("did:web:{ADMIN_HOST}")).unwrap(); + let mut dids = Vec::new(); + for name in ["anemone", "conch", "scallop"] { + let status = into_response( + crate::repos::create_repo( + State(Arc::clone(&state)), + bearer(&mint(&admin, &nsid(CREATE))), + crate::Method::from_nsid(CREATE), + body(json!({ "name": name })), + ) + .await, + ) + .status(); + assert_eq!( + status, + StatusCode::OK, + "{name} registers despite the outage" + ); + match state + .index + .resolve_repo(&owner, &RepoRkey::new(name).unwrap()) + { + Resolved::Ready(Some(did)) => dids.push(did), + other => panic!("Repo wasn't registered despite the queue: {other:?}"), + } + } + fake.lock().reject_posts = false; + fake.lock().post_count = 0; + + let mut memory = crate::plc::SweepMemory::default(); + clock.advance(std::time::Duration::from_secs(31)); + crate::plc::sweep_tick(&state, &mut memory, dids.len()).await; + assert_eq!( + fake.lock().post_count, + dids.len(), + "a sweep as wide as the candidates posts every genesis in the first pass" + ); + + clock.advance(std::time::Duration::from_secs(31)); + crate::plc::sweep_tick(&state, &mut memory, dids.len()).await; + assert_eq!( + memory.settled.len(), + dids.len(), + "every wide-sweep submission settles on the next pass" + ); +} + +const COMMENT: &str = "org.tangled.feed.comment"; + +const LEGACY_COMMENT: &str = "sh.tangled.repo.issue.comment"; + +const MARKUP: &str = "org.tangled.markup.markdown"; + +const REACTION: &str = "org.tangled.feed.reaction"; + +fn comment_record( + subject: &AtUri, + cid: &crate::out::CidString, + text: &str, +) -> serde_json::Value { + json!({ + "$type": COMMENT, + "subject": { "uri": subject, "cid": cid }, + "body": { "$type": MARKUP, "text": text }, + "createdAt": "2026-09-11T00:00:00Z" + }) +} + +fn comment_record_replying( + subject: &AtUri, + cid: &crate::out::CidString, + reply: &AtUri, + reply_cid: &crate::out::CidString, + text: &str, +) -> serde_json::Value { + let mut record = comment_record(subject, cid, text); + record["replyTo"] = json!({ "uri": reply, "cid": reply_cid }); + record +} + +struct Contribution<'a> { + world: &'a World, + repo: &'a RepoDid, +} + +impl<'a> Contribution<'a> { + fn on(world: &'a World, repo: &'a RepoDid) -> Self { + Self { world, repo } + } + + async fn opened(&self, actor: &Actor, title: &str) -> Opened<'_> { + let (uri, cid) = Ticket::on(self.world, self.repo).opened(actor, title).await; + Opened { + contribution: self, + uri, + cid, + } + } + + async fn comment( + &self, + actor: &Actor, + subject: &AtUri, + cid: &crate::out::CidString, + text: &str, + ) -> (StatusCode, serde_json::Value) { + self.create(actor, comment_record(subject, cid, text)).await + } + + async fn replying( + &self, + actor: &Actor, + subject: &AtUri, + cid: &crate::out::CidString, + reply: &AtUri, + reply_cid: &crate::out::CidString, + text: &str, + ) -> (StatusCode, serde_json::Value) { + self.create( + actor, + comment_record_replying(subject, cid, reply, reply_cid, text), + ) + .await + } + + async fn create( + &self, + actor: &Actor, + record: serde_json::Value, + ) -> (StatusCode, serde_json::Value) { + create_as( + self.world, + actor, + self.repo, + &RecordCollection::new(COMMENT).unwrap(), + None, + record, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn edit( + &self, + actor: &Actor, + uri: &AtUri, + from: &crate::out::CidString, + subject: &AtUri, + subject_cid: &crate::out::CidString, + text: &str, + ) -> (StatusCode, serde_json::Value) { + self.put(actor, uri, from, comment_record(subject, subject_cid, text)) + .await + } + + async fn put( + &self, + actor: &Actor, + uri: &AtUri, + from: &crate::out::CidString, + record: serde_json::Value, + ) -> (StatusCode, serde_json::Value) { + put_as( + self.world, + actor, + self.repo, + &RecordCollection::new(COMMENT).unwrap(), + rkey_of(uri), + json!({ "swapRecord": from }), + record, + ) + .await + } + + async fn react( + &self, + actor: &Actor, + subject: &AtUri, + subject_cid: &crate::out::CidString, + kind: &str, + ) -> (StatusCode, serde_json::Value) { + create_as( + self.world, + actor, + self.repo, + &RecordCollection::new(REACTION).unwrap(), + None, + json!({ + "$type": REACTION, + "subject": { "uri": subject, "cid": subject_cid }, + "reaction": kind, + "createdAt": "2026-09-11T00:00:00Z" + }), + ) + .await + } + + async fn erase( + &self, + actor: &Actor, + collection: &RecordCollection, + uri: &AtUri, + from: &crate::out::CidString, + ) -> (StatusCode, serde_json::Value) { + delete_as( + self.world, + actor, + self.repo, + collection, + rkey_of(uri), + json!({ "swapRecord": from }), + ) + .await + } +} + +struct Opened<'a> { + contribution: &'a Contribution<'a>, + uri: AtUri, + cid: crate::out::CidString, +} + +impl<'a> Opened<'a> { + async fn comment(&self, actor: &Actor, text: &str) -> (StatusCode, serde_json::Value) { + self.contribution + .comment(actor, &self.uri, &self.cid, text) + .await + } + + async fn replying( + &self, + actor: &Actor, + reply: &AtUri, + reply_cid: &crate::out::CidString, + text: &str, + ) -> (StatusCode, serde_json::Value) { + self.contribution + .replying(actor, &self.uri, &self.cid, reply, reply_cid, text) + .await + } + + async fn react(&self, actor: &Actor, kind: &str) -> (StatusCode, serde_json::Value) { + self.contribution + .react(actor, &self.uri, &self.cid, kind) + .await + } + + async fn edit( + &self, + actor: &Actor, + uri: &AtUri, + from: &crate::out::CidString, + text: &str, + ) -> (StatusCode, serde_json::Value) { + self.contribution + .edit(actor, uri, from, &self.uri, &self.cid, text) + .await + } +} + +#[tokio::test] +async fn comment_opens_edits_and_deletes_along_its_version_chain() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + + let written = ok(issue.comment(&world.stranger, "a drifter").await); + names_its_change(&world, &written).await; + let comment_uri = uri_of(&written); + let v1 = cid_of(&written); + assert!( + comment_uri + .as_str() + .starts_with(&format!("at://{}/{COMMENT}/", repo.as_str())) + ); + assert_eq!( + records_in(&world, &repo, &RecordCollection::new(COMMENT).unwrap()).await, + vec![comment_uri.as_str().to_owned()] + ); + let record = ok(record_at(&world, &repo, &comment_uri, None).await); + assert_eq!(record["value"]["$type"], COMMENT); + assert_eq!(record["value"]["subject"]["uri"], issue.uri.as_str()); + assert_eq!( + record["value"]["subject"]["cid"], + issue.cid.to_string(), + "subject strongRef must name the exact issue versoon the comment was made on" + ); + assert_eq!(record["value"]["body"]["$type"], MARKUP); + assert_eq!(record["value"]["body"]["text"], "a drifter"); + assert_eq!( + record["value"]["x-tngl-editor"], + world.stranger.did.as_str() + ); + assert!(record["value"].get("replyTo").is_none()); + let versions = versions_page(&world, &comment_uri, None).await; + assert_eq!(versions["versions"].as_array().unwrap().len(), 1); + + let reply = ok(issue + .replying(&world.stranger, &comment_uri, &v1, "on the drifter") + .await); + let reply_uri = uri_of(&reply); + let reply_record = ok(record_at(&world, &repo, &reply_uri, None).await); + assert_eq!( + reply_record["value"]["replyTo"]["uri"], + comment_uri.as_str() + ); + + let edited = ok(issue + .edit(&world.stranger, &comment_uri, &v1, "the drifter") + .await); + names_its_change(&world, &edited).await; + let v2 = cid_of(&edited); + assert_ne!(v1, v2); + + let stale = failed( + issue + .edit(&world.stranger, &comment_uri, &v1, "the late drifter") + .await, + StatusCode::CONFLICT, + ); + assert_eq!(stale["error"], "InvalidSwap"); + + let maintained = ok(issue + .edit(&world.member, &comment_uri, &v2, "the drifter, kept") + .await); + let v3 = cid_of(&maintained); + assert_eq!( + ok(record_at(&world, &repo, &comment_uri, None).await)["value"]["x-tngl-editor"], + world.member.did.as_str() + ); + + let owned_by_member = ok(issue.comment(&world.member, "owner's note").await); + let owned_uri = uri_of(&owned_by_member); + let owned_v1 = cid_of(&owned_by_member); + let denied = failed( + issue + .edit(&world.stranger, &owned_uri, &owned_v1, "rewritten") + .await, + StatusCode::FORBIDDEN, + ); + assert!(denied["message"].as_str().unwrap().contains("author")); + + let deleted = ok(contribution + .erase( + &world.member, + &RecordCollection::new(COMMENT).unwrap(), + &comment_uri, + &v3, + ) + .await); + assert!( + deleted.get("uri").is_none() && deleted.get("cid").is_none(), + "the delete response should probably be the commit alone, without uri or cid" + ); + let (status, gone) = record_at(&world, &repo, &comment_uri, None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(gone["error"], "RecordNotFound"); + assert!( + block_served(&world, &repo, &v1).await, + "the old version's block oughtta stay in storage after the api stops returning it" + ); + let versions = versions_page(&world, &comment_uri, None).await; + let chain = versions["versions"].as_array().unwrap(); + assert_eq!(chain.len(), 4); + assert_eq!(chain[0]["cid"], v1.to_string()); + assert_eq!(chain[2]["cid"], v3.to_string()); + assert!(chain[3].get("cid").is_none()); +} + +#[tokio::test] +async fn a_served_comment_round_trips_through_put_record() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + let written = ok(issue.comment(&world.stranger, "a drifter").await); + let comment_uri = uri_of(&written); + let v1 = cid_of(&written); + let served = ok(record_at(&world, &repo, &comment_uri, None).await); + let echoed = served["value"].clone(); + assert_eq!( + echoed["body"]["$type"], MARKUP, + "the served body is the lexicon's markdown union" + ); + let verbatim = ok(contribution + .put(&world.stranger, &comment_uri, &v1, echoed.clone()) + .await); + assert_eq!( + cid_of(&verbatim), + v1, + "a comment echoed back verbatim serves the record that already exists" + ); + + let mut rewritten = echoed; + rewritten["body"]["text"] = json!("the drifter"); + ok(contribution + .put(&world.stranger, &comment_uri, &v1, rewritten) + .await); + assert_eq!( + ok(record_at(&world, &repo, &comment_uri, None).await)["value"]["body"]["text"], + serde_json::json!("the drifter"), + "a comment echoed back with the union shape kept writes the new text" + ); +} + +#[tokio::test] +async fn comment_rejects_subjects_this_knot_doesnt_serve() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + let ticket = Ticket::on(&world, &repo); + + let missing = at_of( + &repo, + knot_types::RecordCollection::new(TICKET).unwrap(), + &RecordRkey::new("3lubrptx57d33").unwrap(), + ); + let rejected = failed( + contribution + .comment(&world.stranger, &missing, &issue.cid, "nope") + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&rejected, &["isn't a record on this knot"]); + + let edited_issue = ok(ticket + .edit(&world.stranger, &issue.uri, &issue.cid, "kelp, again") + .await); + let current = cid_of(&edited_issue); + let stale_subject = failed( + issue.comment(&world.stranger, "nope").await, + StatusCode::BAD_REQUEST, + ); + message_contains(&stale_subject, &["strongRef CID is", "now has version"]); + assert!( + stale_subject["message"] + .as_str() + .unwrap() + .contains(¤t.to_string()) + ); + + let on_state = at_of( + &repo, + knot_types::RecordCollection::new("cafe.oyster.gripe").unwrap(), + &RecordRkey::new("3lubrptx57d44").unwrap(), + ); + let wrong_collection = failed( + contribution + .comment(&world.stranger, &on_state, ¤t, "nope") + .await, + StatusCode::BAD_REQUEST, + ); + message_contains( + &wrong_collection, + &["takes tickets and patches as comment subjects"], + ); + + let legacy = failed( + create_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(LEGACY_COMMENT).unwrap(), + None, + retype( + comment_record(&issue.uri, ¤t, "old world"), + LEGACY_COMMENT, + ), + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&legacy, &["isn't a collection this knot takes"]); + + let minted = failed( + create_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(COMMENT).unwrap(), + Some(RecordRkey::new("3lubrptx57d55").unwrap()), + comment_record(&issue.uri, ¤t, "mine"), + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&minted, &["mints the record key"]); + + let empty = failed( + contribution + .comment(&world.stranger, &issue.uri, ¤t, "") + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&empty, &["unusable"]); + + let replied_wrong = failed( + contribution + .replying( + &world.stranger, + &issue.uri, + ¤t, + &issue.uri, + ¤t, + "nope", + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&replied_wrong, &["the uri's collection is", "takes"]); + + let closed = ok(ticket + .set_state(&world.stranger, &issue.uri, ¤t, TicketState::Closed) + .await); + let closed_cid = cid_of(&closed); + ok(contribution + .comment(&world.stranger, &issue.uri, &closed_cid, "still fine") + .await); +} + +fn at_of(repo: &RepoDid, collection: RecordCollection, rkey: &RecordRkey) -> AtUri { + knot_types::RecordAddress::new(collection, rkey.clone()).at_uri(repo) +} + +#[tokio::test] +async fn reaction_answers_one_live_record_per_actor_subject_and_kind() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + + let liked = ok(issue.react(&world.stranger, "❤️").await); + names_its_change(&world, &liked).await; + let like_uri = uri_of(&liked); + let like_v1 = cid_of(&liked); + assert!( + like_uri + .as_str() + .starts_with(&format!("at://{}/{REACTION}/", repo.as_str())) + ); + let record = ok(record_at(&world, &repo, &like_uri, None).await); + assert_eq!(record["value"]["$type"], REACTION); + assert_eq!(record["value"]["subject"]["uri"], issue.uri.as_str()); + assert_eq!(record["value"]["reaction"], "❤️"); + assert_eq!( + record["value"]["x-tngl-editor"], + world.stranger.did.as_str() + ); + + let again = ok(issue.react(&world.stranger, "❤️").await); + assert!( + again.get("x-tngl-change").is_none(), + "a deduped reaction replays the first one, so no change was made" + ); + assert_eq!(again["uri"], like_uri.as_str()); + assert_eq!(again["cid"], like_v1.to_string()); + assert_eq!( + again["commit"]["rev"], liked["commit"]["rev"], + "the second identical react should return the first commit" + ); + assert_eq!( + records_in(&world, &repo, &RecordCollection::new(REACTION).unwrap()) + .await + .len(), + 1 + ); + + let rocket = ok(issue.react(&world.stranger, "🚀").await); + let rocket_uri = uri_of(&rocket); + assert_ne!(rocket_uri, like_uri); + let members = ok(issue.react(&world.member, "❤️").await); + let members_uri = uri_of(&members); + assert_ne!(members_uri, like_uri); + assert_ne!(members_uri, rocket_uri); + assert_eq!( + records_in(&world, &repo, &RecordCollection::new(REACTION).unwrap()) + .await + .len(), + 3 + ); + + let put = failed( + put_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(REACTION).unwrap(), + rkey_of(&like_uri), + json!({ "swapRecord": like_v1 }), + json!({ + "$type": REACTION, + "subject": { "uri": issue.uri, "cid": issue.cid }, + "reaction": "🚀" + }), + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&put, &["never edited"]); + + let erased = ok(contribution + .erase( + &world.member, + &RecordCollection::new(REACTION).unwrap(), + &like_uri, + &like_v1, + ) + .await); + assert!(erased.get("uri").is_none()); + let (status, gone) = record_at(&world, &repo, &like_uri, None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(gone["error"], "RecordNotFound"); + + let reminted = ok(issue.react(&world.stranger, "❤️").await); + assert_ne!( + uri_of(&reminted), + like_uri, + "creating again after a delete has to produce a new record" + ); + + let denied = failed( + contribution + .erase( + &world.stranger, + &RecordCollection::new(REACTION).unwrap(), + &members_uri, + &cid_of(&members), + ) + .await, + StatusCode::FORBIDDEN, + ); + assert!(denied["message"].as_str().unwrap().contains("author")); +} + +#[tokio::test] +async fn reaction_opens_only_on_live_subjects() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + let ticket = Ticket::on(&world, &repo); + let liked = ok(issue.react(&world.member, "❤️").await); + let like_uri = uri_of(&liked); + + let edited = ok(ticket + .edit(&world.stranger, &issue.uri, &issue.cid, "kelp, again") + .await); + let v2 = cid_of(&edited); + let stale = failed( + issue.react(&world.stranger, "👍").await, + StatusCode::BAD_REQUEST, + ); + message_contains(&stale, &["strongRef CID", &format!("now has version {v2}")]); + + let missing = at_of( + &repo, + knot_types::RecordCollection::new(TICKET).unwrap(), + &RecordRkey::new("3lubrptx57d33").unwrap(), + ); + let rejected = failed( + contribution + .react(&world.stranger, &missing, &v2, "👍") + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&rejected, &["isn't a record on this knot"]); + + ok(ticket.erase(&world.member, &issue.uri, &v2).await); + let erased = failed( + contribution + .react(&world.member, &issue.uri, &v2, "❤️") + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&erased, &["was deleted"]); + assert_eq!( + ok(record_at(&world, &repo, &like_uri, None).await)["cid"], + cid_of(&liked).to_string(), + "the reaction should stay readable after its issue is deleted" + ); +} + +#[tokio::test] +async fn comment_outlives_its_subject_on_record_reads() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + + let written = ok(issue.comment(&world.stranger, "a drifter").await); + let comment_uri = uri_of(&written); + let comment_v1 = cid_of(&written); + let (_, ticket_address) = knot_types::RecordAddress::parse_at_uri(&issue.uri).unwrap(); + + ok(Ticket::on(&world, &repo) + .erase(&world.member, &issue.uri, &issue.cid) + .await); + + assert_eq!( + ok(record_at(&world, &repo, &comment_uri, None).await)["cid"], + comment_v1.to_string(), + "the comment must stay readable after its issue is deleted" + ); + assert_eq!( + records_in(&world, &repo, &RecordCollection::new(COMMENT).unwrap()).await, + vec![comment_uri.as_str().to_owned()] + ); + assert!( + records_in(&world, &repo, &RecordCollection::new(TICKET).unwrap()) + .await + .is_empty() + ); + assert_eq!( + versions_page(&world, &comment_uri, None).await["versions"] + .as_array() + .unwrap() + .len(), + 1 + ); + assert_eq!( + world.state.index.social_children_of(&repo, &ticket_address), + knot_index::Resolved::Ready(Vec::new()), + "the index shouldn't list children of a deleted issue" + ); +} + +#[tokio::test] +async fn comment_embeds_a_commit_in_its_repo() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + + let embed = json!({ + "$type": "org.tangled.embed.commit", + "repo": repo.as_str(), + "commit": { "$type": "org.tangled.git.oid", "oid": "a".repeat(40) }, + }); + let mut pinned = comment_record(&issue.uri, &issue.cid, "on this commit"); + pinned["embed"] = embed; + let written = ok(contribution.create(&world.stranger, pinned).await); + let record = ok(record_at(&world, &repo, &uri_of(&written), None).await); + assert_eq!( + record["value"]["embed"], + json!({ + "$type": "org.tangled.embed.commit", + "repo": repo.as_str(), + "commit": { "oid": "a".repeat(40) }, + }) + ); +} + +#[tokio::test] +async fn comment_rejects_extras_it_cant_store() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + + let rejected_with = |record: serde_json::Value| { + let world = &world; + let contribution = &contribution; + async move { + failed( + contribution.create(&world.stranger, record).await, + StatusCode::BAD_REQUEST, + ) + } + }; + + let mut elsewhere = comment_record(&issue.uri, &issue.cid, "on a foreign commit"); + elsewhere["embed"] = json!({ + "$type": "org.tangled.embed.commit", + "repo": "did:plc:elsewhere", + "commit": { "oid": "a".repeat(40) }, + }); + message_contains(&rejected_with(elsewhere).await, &["embedded commit"]); + + let mut malformed = comment_record(&issue.uri, &issue.cid, "with a malformed blob"); + malformed["body"]["blobs"] = json!([{ + "$type": "org.tangled.markup.embed.image", + "image": { + "ref": { "$link": "not-a-cid" }, + "mimeType": "image/png", + "size": 5 + } + }]); + message_contains(&rejected_with(malformed).await, &["unusable"]); +} + +#[tokio::test] +async fn comment_subject_and_reply_are_fixed_at_open() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + + let drifter = ok(issue.comment(&world.stranger, "a drifter").await); + let drifter_uri = uri_of(&drifter); + let drifter_v1 = cid_of(&drifter); + let reply = ok(issue + .replying(&world.stranger, &drifter_uri, &drifter_v1, "on the drifter") + .await); + let reply_uri = uri_of(&reply); + let reply_v1 = cid_of(&reply); + let never: crate::out::CidString = + serde_json::from_value(json!(knot_types::dag_cbor_cid(b"never minted").to_string())) + .unwrap(); + + for (uri, from, record, want) in [ + ( + &reply_uri, + &reply_v1, + { + let mut record = comment_record(&issue.uri, &issue.cid, "on the drifter, kept"); + record["subject"]["cid"] = json!(never.to_string()); + record + }, + "subject is set at creation", + ), + ( + &reply_uri, + &reply_v1, + comment_record_replying( + &issue.uri, + &issue.cid, + &drifter_uri, + &never, + "on the drifter, kept", + ), + "reply is set at creation", + ), + ( + &reply_uri, + &reply_v1, + comment_record(&issue.uri, &issue.cid, "on the drifter, kept"), + "reply is set at creation", + ), + ( + &drifter_uri, + &drifter_v1, + comment_record_replying( + &issue.uri, + &issue.cid, + &reply_uri, + &reply_v1, + "the drifter, kept", + ), + "reply is set at creation", + ), + ] { + message_contains( + &failed( + contribution.put(&world.stranger, uri, from, record).await, + StatusCode::BAD_REQUEST, + ), + &[want], + ); + } + + let kept = ok(contribution + .put( + &world.stranger, + &reply_uri, + &reply_v1, + comment_record_replying( + &issue.uri, + &issue.cid, + &drifter_uri, + &drifter_v1, + "on the drifter, kept", + ), + ) + .await); + assert_ne!(cid_of(&kept), reply_v1); + + let drifter_v2 = cid_of(&ok(issue + .edit( + &world.stranger, + &drifter_uri, + &drifter_v1, + "the drifter, kept", + ) + .await)); + let stale = failed( + issue + .replying(&world.stranger, &drifter_uri, &drifter_v1, "on the drifter") + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&stale, &["reply's strongRef CID is", "now has version"]); + assert!( + stale["message"] + .as_str() + .unwrap() + .contains(&drifter_v2.to_string()) + ); + + ok(issue + .replying(&world.stranger, &drifter_uri, &drifter_v2, "on the drifter") + .await); +} + +#[tokio::test] +async fn reaction_accepts_any_live_record_as_subject() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + + let drifter = ok(issue.comment(&world.stranger, "a drifter").await); + let drifter_uri = uri_of(&drifter); + let drifter_cid = cid_of(&drifter); + + ok(contribution + .react(&world.member, &drifter_uri, &drifter_cid, "👀") + .await); + let on_comment = ok(contribution + .react(&world.stranger, &drifter_uri, &drifter_cid, "👍") + .await); + let reaction_uri = uri_of(&on_comment); + let reaction_cid = cid_of(&on_comment); + ok(contribution + .react(&world.member, &reaction_uri, &reaction_cid, "❤️") + .await); + + assert_eq!( + records_in(&world, &repo, &RecordCollection::new(REACTION).unwrap()) + .await + .len(), + 3 + ); + assert_eq!( + ok(record_at(&world, &repo, &reaction_uri, None).await)["value"]["subject"]["uri"], + drifter_uri.as_str(), + "a reaction can target any existing record in the repository" + ); +} + +#[tokio::test] +async fn no_op_edit_by_stranger_is_still_forbidden() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + + let owned = ok(issue.comment(&world.member, "owner's note").await); + let owned_uri = uri_of(&owned); + let owned_v1 = cid_of(&owned); + + let denied = failed( + issue + .edit(&world.stranger, &owned_uri, &owned_v1, "owner's note") + .await, + StatusCode::FORBIDDEN, + ); + assert!(denied["message"].as_str().unwrap().contains("author")); + assert_eq!( + versions_page(&world, &owned_uri, None).await["versions"] + .as_array() + .unwrap() + .len(), + 1, + "the rejected edit shouldn't add a version" + ); + + let minted = ok(issue + .edit(&world.member, &owned_uri, &owned_v1, "owner's note") + .await); + assert_eq!(cid_of(&minted), owned_v1); + let page = versions_page(&world, &owned_uri, None).await; + let chain = page["versions"].as_array().unwrap(); + assert_eq!(chain.len(), 2); + assert_eq!( + chain[1]["cid"], + owned_v1.to_string(), + "an edit that doesn't change anything still adds a version with the same bytes" + ); +} + +#[tokio::test] +async fn reaction_dedup_wont_answer_when_policy_rejects() { + let (world, repo) = ticket_world().await; + let contribution = Contribution::on(&world, &repo); + let issue = contribution.opened(&world.stranger, "kelp").await; + + let liked = ok(issue.react(&world.stranger, "❤️").await); + let like_uri = uri_of(&liked); + + assert_eq!( + set_policy_as( + &world, + &world.member, + &repo, + ContributionPolicy::Collaborators + ) + .await, + StatusCode::OK + ); + let shut_out = failed( + issue.react(&world.stranger, "❤️").await, + StatusCode::FORBIDDEN, + ); + assert!( + shut_out["message"] + .as_str() + .unwrap() + .contains("contribution policy is collaborators") + ); + + assert_eq!( + set_policy_as(&world, &world.member, &repo, ContributionPolicy::Anyone).await, + StatusCode::OK + ); + let again = ok(issue.react(&world.stranger, "❤️").await); + assert_eq!(again["uri"], like_uri.as_str()); + assert_eq!( + records_in(&world, &repo, &RecordCollection::new(REACTION).unwrap()) + .await + .len(), + 1, + "the rejected second react shouldn't create a duplicate" + ); +} + +const LABEL_DEF: &str = "org.tangled.label.definition"; + +fn def_body(name: &str, color: Option<&str>) -> serde_json::Value { + let mut body = json!({ + "$type": LABEL_DEF, + "name": name, + "createdAt": "2026-09-01T00:00:00Z", + }); + if let Some(color) = color { + body["color"] = json!(color); + } + body +} + +async fn define_label( + world: &World, + repo: &RepoDid, + slug: &str, + name: &str, + color: Option<&str>, +) -> (AtUri, crate::out::CidString) { + let written = ok(create_as( + world, + &world.member, + repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + Some(RecordRkey::new(slug).unwrap()), + def_body(name, color), + ) + .await); + names_its_change(world, &written).await; + (uri_of(&written), cid_of(&written)) +} + +async fn wontfix(world: &World, repo: &RepoDid) -> (AtUri, crate::out::CidString) { + define_label(world, repo, "wontfix", "wontfix", Some("#79ffe1")).await +} + +#[tokio::test] +async fn label_defs_belong_to_the_owner_and_go_by_their_slug() { + let (world, repo) = ticket_world().await; + + let refused = failed( + create_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + Some(RecordRkey::new("wontfix").unwrap()), + def_body("wontfix", Some("#79ffe1")), + ) + .await, + StatusCode::FORBIDDEN, + ); + message_contains(&refused, &["label defs belong to the repository's owner"]); + + let refused_policy = failed( + create_as( + &world, + &world.admin, + &repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + Some(RecordRkey::new("wontfix").unwrap()), + def_body("wontfix", Some("#79ffe1")), + ) + .await, + StatusCode::FORBIDDEN, + ); + message_contains( + &refused_policy, + &["label defs belong to the repository's owner"], + ); + + let no_slug = failed( + create_as( + &world, + &world.member, + &repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + None, + def_body("wontfix", Some("#79ffe1")), + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&no_slug, &["slug"]); + + let (uri, cid) = wontfix(&world, &repo).await; + assert_eq!( + uri.as_str(), + format!("at://{repo}/{LABEL_DEF}/wontfix"), + "the def's record key is the slug the owner sent" + ); + assert_eq!( + records_in(&world, &repo, &RecordCollection::new(LABEL_DEF).unwrap()).await, + vec![uri.as_str().to_owned()] + ); + let served = ok(record_at(&world, &repo, &uri, None).await); + assert_eq!(served["value"]["name"], "wontfix"); + assert_eq!(served["value"]["color"], "#79FFE1"); + assert_eq!(served["value"]["x-tngl-editor"], world.member.did.as_str()); + + let duplicate = failed( + create_as( + &world, + &world.member, + &repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + Some(RecordRkey::new("wontfix").unwrap()), + def_body("wontfix2", Some("#79ffe1")), + ) + .await, + StatusCode::CONFLICT, + ); + message_contains(&duplicate, &["already stands"]); + + let edited = ok(put_as( + &world, + &world.member, + &repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + RecordRkey::new("wontfix").unwrap(), + json!({ "swapRecord": cid }), + def_body("never_happening", Some("#ff0000")), + ) + .await); + assert_eq!(uri_of(&edited).as_str(), uri.as_str()); + let renamed = ok(record_at(&world, &repo, &uri, None).await); + assert_eq!(renamed["value"]["name"], "never_happening"); + + let outsider = failed( + put_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + RecordRkey::new("wontfix").unwrap(), + json!({ "swapRecord": cid_of(&edited) }), + def_body("taken_over", Some("#79ffe1")), + ) + .await, + StatusCode::FORBIDDEN, + ); + message_contains( + &outsider, + &["label defs belong to the repository's owner alone"], + ); + + ok(delete_as( + &world, + &world.member, + &repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + RecordRkey::new("wontfix").unwrap(), + json!({ "swapRecord": cid_of(&edited) }), + ) + .await); + assert_eq!( + records_in(&world, &repo, &RecordCollection::new(LABEL_DEF).unwrap()).await, + Vec::::new(), + "the erased def stops serving, the way every social record does" + ); +} + +#[tokio::test] +async fn erased_def_slug_stays_retired() { + let (world, repo) = ticket_world().await; + let (_, cid) = wontfix(&world, &repo).await; + ok(delete_as( + &world, + &world.member, + &repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + RecordRkey::new("wontfix").unwrap(), + json!({ "swapRecord": cid }), + ) + .await); + let refused = failed( + create_as( + &world, + &world.member, + &repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + Some(RecordRkey::new("wontfix").unwrap()), + def_body("wontfix", Some("#79ffe1")), + ) + .await, + StatusCode::CONFLICT, + ); + message_contains(&refused, &["was erased at", "slug stays retired"]); + assert_eq!( + records_in(&world, &repo, &RecordCollection::new(LABEL_DEF).unwrap()).await, + Vec::::new(), + "the erased def doesn't serve, and the slug stays retired: one record address is \ + one object for life" + ); +} +#[tokio::test] +async fn tickets_cite_label_defs_the_repository_actually_serves() { + let (world, repo) = ticket_world().await; + let ticket = Ticket::on(&world, &repo); + let (uri, v1) = ticket.opened(&world.stranger, "kelp").await; + + message_contains( + &ticket + .refused_create(TicketDraft::fresh(&repo).citing("wontfix").record) + .await, + &["wontfix", "isn't a label def on this repository"], + ); + + let refused = failed( + ticket + .put( + &world.stranger, + &uri, + &v1, + TicketDraft::fresh(&repo).citing("ghost").record, + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&refused, &["ghost", "isn't a label def on this repository"]); + + let (_, def_cid) = wontfix(&world, &repo).await; + ok(delete_as( + &world, + &world.member, + &repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + RecordRkey::new("wontfix").unwrap(), + json!({ "swapRecord": def_cid }), + ) + .await); + message_contains( + &ticket + .refused_create(TicketDraft::fresh(&repo).citing("wontfix").record) + .await, + &["wontfix", "whose label def was deleted"], + ); +} + +#[tokio::test] +async fn a_stranger_probing_a_taken_slug_hears_ownership_first() { + let (world, repo) = ticket_world().await; + wontfix(&world, &repo).await; + let response = create_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(LABEL_DEF).unwrap(), + Some(RecordRkey::new("wontfix").unwrap()), + def_body("wontfix", Some("#79ffe1")), + ) + .await; + assert_eq!(response.0, StatusCode::FORBIDDEN); + message_contains( + &response.1, + &["label defs belong to the repository's owner alone"], + ); +} + +mod pulls { + use super::*; + use knot_git::{EntryKind, NewCommit, RefUpdate, StagedAction, StagedChange}; + use knot_types::{Oid, RefName, UnixSeconds}; + + const PULL: &str = "org.tangled.review.patch"; + const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + + fn main_ref() -> RefName { + RefName::new("refs/heads/main").unwrap() + } + + struct Pulls<'a> { + world: &'a World, + repo: &'a RepoDid, + } + + impl<'a> Pulls<'a> { + fn on(world: &'a World, repo: &'a RepoDid) -> Self { + Self { world, repo } + } + + fn git(&self) -> knot_git::Repo { + self.world.layout.open(self.repo).unwrap() + } + + fn set_main(&self, tip: Oid) { + self.git() + .update_ref(&RefUpdate::Create { + name: main_ref(), + new: tip, + }) + .unwrap(); + } + + fn commit(&self, parents: &[Oid], staged: &[StagedChange], message: &str) -> Oid { + let repo = self.git(); + let base_tree = parents + .last() + .map(|parent| repo.find_commit(*parent).unwrap().tree) + .unwrap_or_else(|| Oid::from_hex(EMPTY_TREE).unwrap()); + let who = knot_git::Identity { + name: knot_types::AuthorName::new("nel"), + email: knot_types::Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_000), + offset_seconds: 0, + }; + repo.write_commit(&NewCommit { + tree: repo.write_staged_tree(base_tree, staged).unwrap(), + parents: parents.to_vec(), + author: who.clone(), + committer: who, + message: message.to_string(), + extra_headers: Vec::new(), + }) + .unwrap() + } + + fn commit_with(&self, parents: &[Oid], files: &[(&str, &str)], message: &str) -> Oid { + let staged: Vec = files + .iter() + .map(|(path, content)| StagedChange { + path: knot_types::RepoPath::new(*path).unwrap(), + action: StagedAction::Put { + content: content.as_bytes().to_vec(), + kind: EntryKind::Blob, + }, + }) + .collect(); + self.commit(parents, &staged, message) + } + + async fn ticket( + &self, + actor: &Actor, + title: &str, + ) -> (AtUri, crate::out::CidString) { + let written = ok(Ticket::on(self.world, self.repo) + .open(actor, title, "a forest") + .await); + (uri_of(&written), cid_of(&written)) + } + + fn record(&self, ticket: &AtUri, base: Oid, head: Oid) -> serde_json::Value { + json!({ + "ticket": rkey_of(ticket), + "base": { "oid": base.to_hex() }, + "head": { "oid": head.to_hex() }, + "sourceRepo": self.repo + }) + } + + async fn open( + &self, + actor: &Actor, + record: serde_json::Value, + ) -> (StatusCode, serde_json::Value) { + submit_patch_as(self.world, actor, self.repo, record).await + } + + async fn refused_open(&self, record: serde_json::Value) -> serde_json::Value { + failed( + self.open(&self.world.stranger, record).await, + StatusCode::BAD_REQUEST, + ) + } + + async fn at(&self, uri: &AtUri) -> serde_json::Value { + ok(record_at(self.world, self.repo, uri, None).await) + } + } + + async fn kelp_patch() -> ( + World, + RepoDid, + Oid, + Oid, + AtUri, + AtUri, + crate::out::CidString, + ) { + let (world, repo) = ticket_world().await; + let pulls = Pulls::on(&world, &repo); + let base = pulls.commit_with(&[], &[("kelp.txt", "forest")], "base"); + let head = pulls.commit_with(&[base], &[("kelp.txt", "reef")], "head"); + pulls.set_main(base); + let (ticket, _) = pulls.ticket(&world.stranger, "kelp").await; + let written = ok(pulls + .open(&world.stranger, pulls.record(&ticket, base, head)) + .await); + ( + world, + repo, + base, + head, + ticket, + uri_of(&written), + cid_of(&written), + ) + } + + #[tokio::test] + async fn opened_patch_writes_keep_refs_for_its_commits() { + let (world, repo, base, head, _ticket, uri, _) = kelp_patch().await; + let pulls = Pulls::on(&world, &repo); + let record = pulls.at(&uri).await; + assert_eq!(record["value"]["base"]["oid"], base.to_hex()); + assert_eq!(record["value"]["head"]["oid"], head.to_hex()); + assert_eq!( + record["value"]["x-tngl-editor"], + world.stranger.did.as_str() + ); + let address = knot_types::RecordAddress::new( + knot_types::RecordCollection::new(PULL).unwrap(), + rkey_of(&uri), + ); + for pinned in [base, head] { + let keep = crate::git::keep_ref(&repo, &address, pinned).unwrap(); + assert_eq!( + pulls.git().find_ref(&keep).unwrap(), + Some(pinned), + "an accepted patch's commits stay pinned by a keep ref" + ); + } + } + + #[tokio::test] + async fn opening_rejects_ghost_heads_crossed_ranges_and_non_tickets() { + let (world, repo) = ticket_world().await; + let pulls = Pulls::on(&world, &repo); + let base = pulls.commit_with(&[], &[("kelp.txt", "forest")], "base"); + pulls.set_main(base); + let (ticket, _) = pulls.ticket(&world.stranger, "kelp").await; + + let ghost = Oid::from_hex(&"f".repeat(40)).unwrap(); + let refused = pulls.refused_open(pulls.record(&ticket, base, ghost)).await; + message_contains( + &refused, + &["head", &ghost.to_hex(), "isn't a commit in this repository"], + ); + + let right = pulls.commit_with(&[], &[("rock.txt", "grey")], "right"); + let refused = pulls.refused_open(pulls.record(&ticket, base, right)).await; + message_contains( + &refused, + &["isn't an ancestor of its head", &right.to_hex()], + ); + + let missing = AtUri::new_owned(format!("at://{repo}/{TICKET}/3lubrptx5zzzz")).unwrap(); + let refused = pulls.refused_open(pulls.record(&missing, base, base)).await; + message_contains(&refused, &["isn't a record on this knot"]); + + let mut with_blobs = pulls.record(&ticket, base, base); + with_blobs["blobs"] = json!([]); + message_contains( + &pulls.refused_open(with_blobs).await, + &["unknown field", "blobs"], + ); + } + + #[tokio::test] + async fn a_patch_is_keyed_by_its_own_record_and_never_edited() { + let (world, repo, base, head, ticket, uri, cid) = kelp_patch().await; + let pulls = Pulls::on(&world, &repo); + assert_eq!( + rkey_of(&uri).as_str(), + cid.to_string(), + "the patch's record key is the CID of its own record" + ); + + let again = pulls.commit_with(&[head], &[("kelp.txt", "atoll")], "again"); + let refused = failed( + put_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(PULL).unwrap(), + rkey_of(&uri), + json!({ "swapRecord": cid }), + pulls.record(&ticket, base, again), + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&refused, &["a patch is immutable"]); + + let refused = failed( + create_as( + &world, + &world.stranger, + &repo, + &RecordCollection::new(PULL).unwrap(), + None, + json!({ + "$type": PULL, + "ticket": rkey_of(&ticket), + "base": { "oid": base.to_hex() }, + "head": { "oid": again.to_hex() }, + "createdAt": "2026-09-11T00:00:00Z" + }), + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains(&refused, &["isn't a collection this knot takes"]); + + let resubmitted = ok(pulls + .open(&world.stranger, pulls.record(&ticket, base, again)) + .await); + let resubmitted_uri = uri_of(&resubmitted); + let resubmitted_cid = cid_of(&resubmitted); + assert_ne!( + resubmitted_uri, uri, + "resubmission is a new patch record, not an edit" + ); + assert_eq!( + rkey_of(&resubmitted_uri).as_str(), + resubmitted_cid.to_string(), + "the new patch is keyed by its own CID too" + ); + assert_eq!( + pulls.at(&resubmitted_uri).await["value"]["head"]["oid"], + again.to_hex() + ); + + let repeated = ok(pulls + .open(&world.stranger, pulls.record(&ticket, base, again)) + .await); + assert_eq!( + uri_of(&repeated), + resubmitted_uri, + "creating the same patch twice serves the one record both times" + ); + } + + #[tokio::test] + async fn a_ticket_targets_a_branch_that_exists() { + let (world, repo) = ticket_world().await; + let pulls = Pulls::on(&world, &repo); + let base = pulls.commit_with(&[], &[("kelp.txt", "forest")], "base"); + pulls.set_main(base); + pulls + .git() + .update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/reef").unwrap(), + new: base, + }) + .unwrap(); + + let tickets = Ticket::on(&world, &repo); + let ticket = uri_of(&ok(tickets + .create(&world.stranger, TicketDraft::fresh(&repo).on("reef").record) + .await)); + let created = ok(record_at(&world, &repo, &ticket, None).await); + assert_eq!(created["value"]["targetBranch"], "reef"); + + for (draft, fragments) in [ + ( + TicketDraft::fresh(&repo).on("refs/heads/reef").record, + vec!["without the refs/ prefix"], + ), + ( + TicketDraft::fresh(&repo).on("ghost").record, + vec!["opens onto a branch", "ghost isn't one"], + ), + ] { + message_contains(&tickets.refused_create(draft).await, &fragments); + } + let retargeted = failed( + tickets + .put( + &world.stranger, + &ticket, + &cid_of(&created), + TicketDraft::fresh(&repo).on("no-such-branch").record, + ) + .await, + StatusCode::BAD_REQUEST, + ); + message_contains( + &retargeted, + &["targetBranch opens onto a branch", "no-such-branch"], + ); + } +} diff --git a/knot2/crates/knot-xrpc/tests/atproto.rs b/knot2/crates/knot-xrpc/tests/atproto.rs new file mode 100644 index 0000000000000000000000000000000000000000..dde1706f492b7a41b1812265a994a90af090f01f --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/atproto.rs @@ -0,0 +1,673 @@ +mod common; + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use axum::body::Body; +use cid::Cid; +use common::{World, fixture_collection, knot_host, nsid, owner, percent_encode, post_authed}; +use futures::StreamExt; +use http::{HeaderValue, StatusCode, header}; +use jacquard_repo::Mst; +use jacquard_repo::car::parse_car_bytes; +use jacquard_repo::storage::MemoryBlockStore; +use knot_git::Repo; +use knot_types::{DidRkey, RecordRkey, RepoDid, RepoName, UnixSeconds}; +use tower::ServiceExt; + +const FIVE_SECONDS: Duration = Duration::from_secs(5); + +fn knot_repo(world: &World) -> RepoDid { + RepoDid::new(world.state.knot_did.as_str()).unwrap() +} + +async fn get(world: &World, path_and_query: &str) -> (StatusCode, HeaderValue, Vec) { + let request = http::Request::builder() + .method("GET") + .uri(path_and_query) + .body(Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let content_type = response + .headers() + .get(header::CONTENT_TYPE) + .cloned() + .unwrap_or_else(|| HeaderValue::from_static("")); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap() + .to_vec(); + (status, content_type, body) +} + +async fn get_json(world: &World, path_and_query: &str) -> serde_json::Value { + let (status, content_type, body) = get(world, path_and_query).await; + assert_eq!(status, StatusCode::OK, "{path_and_query}"); + assert!( + content_type + .to_str() + .unwrap() + .starts_with("application/json"), + "{path_and_query} serves json" + ); + serde_json::from_slice(&body).unwrap() +} + +async fn error_of(world: &World, path_and_query: &str) -> (StatusCode, serde_json::Value) { + let (status, _content_type, body) = get(world, path_and_query).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{path_and_query}"); + (status, serde_json::from_slice(&body).unwrap()) +} + +fn published_world() -> World { + let world = World::new(); + world.publish_meta( + &[DidRkey::new("did:plc:limpet").unwrap()], + UnixSeconds::new(1_000), + ); + world.publish_meta( + &[ + DidRkey::new("did:plc:limpet").unwrap(), + DidRkey::new("did:plc:whelk").unwrap(), + ], + UnixSeconds::new(1_100), + ); + world +} + +async fn cold_tree(car: &[u8]) -> (Cid, BTreeMap) { + let parsed = parse_car_bytes(car).await.unwrap(); + (parsed.root, parsed.blocks) +} + +async fn cold_loaded_tree(car: &[u8]) -> Mst { + let (head, blocks) = cold_tree(car).await; + let commit = jacquard_repo::commit::Commit::::from_cbor( + blocks.get(&head).expect("the commit block rides the car"), + ) + .unwrap(); + Mst::load( + Arc::new(MemoryBlockStore::new_from_blocks(blocks)), + *commit.data(), + None, + ) +} + +async fn root_data_of(world: &World) -> Cid { + let meta = Repo::open(&world.state.meta_path).unwrap(); + knot_record::chain::tip(&meta).unwrap().unwrap().data +} + +type Ws = + tokio_tungstenite::WebSocketStream>; +#[derive(serde::Deserialize)] +struct CommitProbe { + seq: u64, + rebase: bool, + repo: String, + rev: String, + ops: Vec, +} + +#[derive(serde::Deserialize)] +struct SyncProbe { + seq: u64, + did: String, + rev: String, + #[serde(with = "serde_bytes")] + blocks: Vec, +} + +fn frame_header(frame: &[u8]) -> (serde_json::Value, &[u8]) { + let mut read = frame; + let header: serde_json::Value = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + (header, read) +} + +fn frame_payload(rest: &[u8]) -> T { + serde_ipld_dagcbor::from_slice(rest).unwrap() +} + +async fn next_binary(ws: &mut Ws) -> Vec { + let received = tokio::time::timeout(FIVE_SECONDS, ws.next()) + .await + .expect("a frame arrives within the timeout") + .expect("the stream stays open") + .expect("the frame is readable"); + match received { + tokio_tungstenite::tungstenite::Message::Binary(bytes) => bytes.to_vec(), + other => panic!("the frame is binary, not {other:?}"), + } +} + +async fn drain_binaries(ws: &mut Ws) -> Vec> { + let mut seen = Vec::new(); + while let Ok(Some(Ok(frame))) = tokio::time::timeout(FIVE_SECONDS, ws.next()).await { + if let tokio_tungstenite::tungstenite::Message::Binary(bytes) = frame { + seen.push(bytes.to_vec()); + } + } + seen +} + +#[tokio::test] +async fn an_unmodified_cold_client_verifies_a_record_off_the_proof_car() { + let world = published_world(); + let knot = knot_repo(&world); + let collection = fixture_collection(); + let key = format!("{collection}/did:plc:limpet"); + let (status, content_type, body) = get( + &world, + &format!( + "/xrpc/com.atproto.sync.getRecord?did={knot}&collection={collection}&rkey=did%3Aplc%3Alimpet" + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(content_type, "application/vnd.ipld.car"); + let latest = get_json( + &world, + &format!("/xrpc/com.atproto.sync.getLatestCommit?did={knot}"), + ) + .await; + let head: Cid = latest["cid"].as_str().unwrap().parse().unwrap(); + let (root, blocks) = cold_tree(&body).await; + assert_eq!(root, head, "the car roots at the signed commit"); + let commit = jacquard_repo::commit::Commit::::from_cbor( + blocks.get(&head).expect("the commit block rides the car"), + ) + .unwrap(); + assert_eq!( + commit.data(), + &root_data_of(&world).await, + "the commit's data cid names the tree the car proves" + ); + let found = cold_loaded_tree(&body) + .await + .get(&key) + .await + .unwrap() + .expect("the record is present"); + let record: serde_json::Value = + serde_ipld_dagcbor::from_slice(blocks.get(&found).unwrap()).unwrap(); + assert_eq!(record["createdAt"], "1970-01-01T00:18:20Z"); + assert_eq!(record["x-tngl-editor"], owner().as_str()); + assert_eq!( + record["$type"], + collection.as_str(), + "the stored bytes carry their own $type, so a cold car reader can type the record" + ); + + let (_status, _type, absent) = get( + &world, + &format!( + "/xrpc/com.atproto.sync.getRecord?did={knot}&collection={collection}&rkey=did%3Aplc%3Aabsent" + ), + ) + .await; + assert!( + cold_loaded_tree(&absent) + .await + .get(&format!("{collection}/did:plc:absent")) + .await + .unwrap() + .is_none(), + "non-inclusion is a proof, not an error" + ); + let (_, error) = error_of( + &world, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={knot}&collection={collection}&rkey=did%3Aplc%3Aabsent" + ), + ) + .await; + assert_eq!(error["error"], "RecordNotFound"); + + let described = get_json( + &world, + &format!("/xrpc/com.atproto.repo.describeRepo?repo={knot}"), + ) + .await; + assert_eq!( + described["handle"], + knot_host().as_str(), + "the knot's own handle is its hostname" + ); + assert!( + described["didDoc"].is_object(), + "the knot's own repo carries a did doc" + ); + assert_eq!(described["collections"], serde_json::json!([&collection])); + + let (other_repo, _bare, _kept) = common::empty_repo(&world, &RepoName::new("squid").unwrap()); + let described_other = get_json( + &world, + &format!("/xrpc/com.atproto.repo.describeRepo?repo={other_repo}"), + ) + .await; + assert_eq!( + described_other["didDoc"], + serde_json::json!({}), + "a non-knot repo still sends the required field, empty for now" + ); + + let described = get_json(&world, "/xrpc/com.atproto.server.describeServer").await; + assert_eq!(described["did"], knot.as_str()); + let resolved = get_json( + &world, + &format!( + "/xrpc/com.atproto.identity.resolveHandle?handle={}", + knot_host() + ), + ) + .await; + assert_eq!(resolved["did"], knot.as_str()); + let (_, error) = error_of( + &world, + "/xrpc/com.atproto.identity.resolveHandle?handle=stranger.example", + ) + .await; + assert_eq!(error["error"], "HandleNotFound"); +} + +#[tokio::test] +async fn get_repo_serves_the_whole_tree_and_get_blocks_serves_named_cids() { + let world = published_world(); + let knot = knot_repo(&world); + let latest = get_json( + &world, + &format!("/xrpc/com.atproto.sync.getLatestCommit?did={knot}"), + ) + .await; + let head: Cid = latest["cid"].as_str().unwrap().parse().unwrap(); + + let (status, _content_type, body) = get( + &world, + &format!("/xrpc/com.atproto.sync.getRepo?did={knot}"), + ) + .await; + assert_eq!(status, StatusCode::OK); + let (root, _blocks) = cold_tree(&body).await; + assert_eq!(root, head); + assert_eq!( + cold_loaded_tree(&body).await.leaves().await.unwrap().len(), + 2, + "the whole grant list exports" + ); + for wanted in [head, "bafkqaaa".parse::().unwrap()] { + let (status, _content_type, body) = get( + &world, + &format!("/xrpc/com.atproto.sync.getBlocks?did={knot}&cids={wanted}"), + ) + .await; + assert_eq!(status, StatusCode::OK); + let (root, blocks) = cold_tree(&body).await; + assert_eq!(root, head, "the car still roots at the commit"); + assert_eq!( + blocks.contains_key(&wanted), + wanted == head, + "a named block is served and a cid too short to shard reads as absent, not a panic" + ); + } +} + +#[tokio::test] +async fn a_removal_takes_the_record_off_the_surface() { + let world = published_world(); + let knot = knot_repo(&world); + let collection = fixture_collection(); + world.publish_meta( + &[DidRkey::new("did:plc:whelk").unwrap()], + UnixSeconds::new(1_200), + ); + let listed = get_json( + &world, + &format!("/xrpc/com.atproto.repo.listRecords?repo={knot}&collection={collection}"), + ) + .await; + assert_eq!( + listed["records"][0]["uri"], + format!("at://{knot}/{collection}/did:plc:whelk"), + "the record the sync left out is gone, and whelk is the only row left" + ); +} + +#[tokio::test] +async fn hosted_repositories_read_over_com_atproto_until_a_chain_exists() { + let world = World::new(); + let collection = fixture_collection(); + let (repo, bare, _kept) = common::empty_repo(&world, &RepoName::new("squid").unwrap()); + world.publish( + &repo, + std::path::Path::new(&bare), + &[RecordRkey::new("did:plc:limpet").unwrap()], + UnixSeconds::new(1_000), + ); + let listed = get_json( + &world, + &format!( + "/xrpc/com.atproto.repo.listRecords?repo={}&collection={collection}", + percent_encode(repo.as_str()) + ), + ) + .await; + assert_eq!( + listed["records"][0]["uri"], + format!("at://{}/{collection}/did:plc:limpet", repo.as_str()), + "a repo with a chain lists its own records" + ); + let repos = get_json(&world, "/xrpc/com.atproto.sync.listRepos").await; + assert!( + repos["repos"] + .as_array() + .unwrap() + .iter() + .any(|row| row["did"] == repo.as_str()), + "a repo with a chain is listed" + ); + + let (lonely, _name, _kept) = common::empty_repo(&world, &RepoName::new("lonely").unwrap()); + let did = percent_encode(lonely.as_str()); + let (_, error) = error_of( + &world, + &format!("/xrpc/com.atproto.sync.getLatestCommit?did={did}"), + ) + .await; + assert_eq!(error["error"], "RepoNotFound"); + let repos = get_json(&world, "/xrpc/com.atproto.sync.listRepos").await; + assert!( + !repos["repos"] + .as_array() + .unwrap() + .iter() + .any(|row| row["did"] == lonely.as_str()), + "a registered repo with no chain has no head to name" + ); + let described = get_json( + &world, + &format!("/xrpc/com.atproto.sync.getRepoStatus?did={did}"), + ) + .await; + assert_eq!(described["active"], true); + assert!( + described.get("rev").is_none(), + "a repo with no chain carries no rev rather than a null one" + ); +} + +#[tokio::test] +async fn an_evicted_cursor_answers_an_out_of_cursor_info_frame() { + let world = published_world(); + let knot = knot_repo(&world); + let addr = common::serve(&world).await; + let (mut ws, _response) = tokio_tungstenite::connect_async(format!( + "ws://{addr}/xrpc/com.atproto.sync.subscribeRepos?cursor=0" + )) + .await + .unwrap(); + let first = next_binary(&mut ws).await; + let (header, rest) = frame_header(&first); + let payload: CommitProbe = frame_payload(rest); + assert_eq!(header["op"], 1); + assert_eq!( + header["t"], "#commit", + "a zero cursor against a window that lost nothing replays without an info frame" + ); + assert_eq!(payload.repo, knot.as_str()); + assert!(!payload.rebase); + assert!(payload.seq > 0); + let second = next_binary(&mut ws).await; + let (header, _rest) = frame_header(&second); + assert_eq!(header["t"], "#commit", "both records reach the stream"); + + let mut rkeys = vec![ + DidRkey::new("did:plc:limpet").unwrap(), + DidRkey::new("did:plc:whelk").unwrap(), + ]; + (0..10).for_each(|n| { + rkeys.push(DidRkey::new(format!("did:plc:subject{n}")).unwrap()); + world.publish_meta(&rkeys, UnixSeconds::new(2_000 + n)); + }); + let (mut ws, _response) = tokio_tungstenite::connect_async(format!( + "ws://{addr}/xrpc/com.atproto.sync.subscribeRepos?cursor=0" + )) + .await + .unwrap(); + let notified = next_binary(&mut ws).await; + let (header, rest) = frame_header(¬ified); + assert_eq!(header["t"], "#info"); + let info: serde_json::Value = frame_payload(rest); + assert_eq!(info["name"], "AgedNonCommitCursor"); + let seqs = drain_binaries(&mut ws) + .await + .iter() + .map(|bytes| frame_payload::(frame_header(bytes).1).seq) + .collect::>(); + assert_eq!( + seqs.len(), + 12, + "every commit the ring evicted still replays off the chains" + ); + assert!( + seqs.windows(2).all(|pair| pair[0] < pair[1]), + "the replay arrives in seq order" + ); +} + +#[tokio::test] +async fn account_creation_announces_identity_account_and_an_empty_commit() { + let world = World::administered(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + let addr = common::serve(&world).await; + let (mut ws, _response) = tokio_tungstenite::connect_async(format!( + "ws://{addr}/xrpc/com.atproto.sync.subscribeRepos" + )) + .await + .unwrap(); + let (status, body) = post_authed( + &world, + &nsid("org.tangled.repo.createRepo"), + &owner(), + serde_json::json!({ "name": "limpet" }), + ) + .await; + assert_eq!(status, StatusCode::OK, "create failed: {body}"); + let repo_did = body["did"].as_str().unwrap().to_owned(); + let identity = next_binary(&mut ws).await; + let (header, rest) = frame_header(&identity); + assert_eq!(header["t"], "#identity"); + let payload: serde_json::Value = frame_payload(rest); + assert_eq!(payload["did"], repo_did); + let identity_seq = payload["seq"].as_u64().unwrap(); + + let account = next_binary(&mut ws).await; + let (header, rest) = frame_header(&account); + assert_eq!(header["t"], "#account"); + let payload: serde_json::Value = frame_payload(rest); + assert_eq!(payload["did"], repo_did); + assert_eq!(payload["active"], true); + let account_seq = payload["seq"].as_u64().unwrap(); + + let commit = next_binary(&mut ws).await; + let (header, rest) = frame_header(&commit); + assert_eq!(header["t"], "#commit"); + let payload: CommitProbe = frame_payload(rest); + assert_eq!(payload.repo, repo_did); + assert_eq!( + payload.ops.len(), + 0, + "a fresh account's first commit is empty" + ); + assert!( + identity_seq < account_seq && account_seq < payload.seq, + "creation goes identity, account, commit" + ); + let mut last_seq = payload.seq; + let mut last_rev = payload.rev; + loop { + let frame = next_binary(&mut ws).await; + let (header, rest) = frame_header(&frame); + match header["t"].as_str() { + Some("#commit") => { + let payload: CommitProbe = frame_payload(rest); + assert_eq!(payload.repo, repo_did); + assert!( + last_seq < payload.seq, + "creation's commits stay in seq order" + ); + last_seq = payload.seq; + last_rev = payload.rev; + } + Some("#sync") => { + let payload: SyncProbe = frame_payload(rest); + assert!( + last_seq < payload.seq, + "the sync assertion follows the commit it names" + ); + assert_eq!(payload.did, repo_did); + assert_eq!( + payload.rev, last_rev, + "the sync frame asserts the last commit creation announced" + ); + assert!( + !payload.blocks.is_empty(), + "the sync frame carries that commit in its CAR" + ); + break; + } + other => panic!("creation announced an unexpected frame: {other:?}"), + } + } + + let (status, body) = post_authed( + &world, + &nsid("org.tangled.repo.deleteRepo"), + &owner(), + serde_json::json!({ "repo": repo_did, "force": true }), + ) + .await; + assert_eq!(status, StatusCode::OK, "delete failed: {body}"); + let closed = drain_binaries(&mut ws) + .await + .iter() + .filter(|bytes| frame_header(bytes).0["t"] == "#account") + .map(|bytes| frame_payload::(frame_header(bytes).1)) + .next_back() + .expect("an account frame closed the lifecycle"); + assert_eq!(closed["did"], repo_did); + assert_eq!(closed["active"], false); + assert_eq!(closed["status"], "deleted"); +} +#[tokio::test] +async fn subscribers_behind_a_proxy_are_counted_by_their_own_address() { + use tokio_tungstenite::tungstenite::client::IntoClientRequest; + + let world = World::behind_proxy(); + let addr = common::serve(&world).await; + let subscribe = |client: &str| { + let mut request = format!("ws://{addr}/xrpc/com.atproto.sync.subscribeRepos") + .into_client_request() + .unwrap(); + request + .headers_mut() + .insert("x-forwarded-for", HeaderValue::from_str(client).unwrap()); + tokio_tungstenite::connect_async(request) + }; + + // the test world admits 4 subscribers per address + let mut held = Vec::new(); + for client in 1..=5 { + held.push(subscribe(&format!("198.51.100.{client}")).await.unwrap()); + } + for _ in 0..4 { + held.push(subscribe("198.51.100.200").await.unwrap()); + } + let refused = match subscribe("198.51.100.200").await.unwrap_err() { + tokio_tungstenite::tungstenite::Error::Http(response) => response, + other => panic!("the handshake answers http, not {other:?}"), + }; + assert_eq!(refused.status(), StatusCode::SERVICE_UNAVAILABLE); +} + +#[tokio::test] +async fn a_non_integer_cursor_is_rejected_in_the_xrpc_error_shape() { + let addr = common::serve(&published_world()).await; + let error = tokio_tungstenite::connect_async(format!( + "ws://{addr}/xrpc/com.atproto.sync.subscribeRepos?cursor=soon" + )) + .await + .unwrap_err(); + let response = match error { + tokio_tungstenite::tungstenite::Error::Http(response) => response, + other => panic!("the handshake answers http, not {other:?}"), + }; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = response.into_body().unwrap_or_default(); + let error: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(error["error"], "InvalidRequest"); + assert_eq!(error["message"], "cursor must be an integer"); +} + +#[tokio::test] +async fn a_future_cursor_answers_an_error_frame_and_a_try_again_close() { + let addr = common::serve(&published_world()).await; + let (mut ws, _response) = tokio_tungstenite::connect_async(format!( + "ws://{addr}/xrpc/com.atproto.sync.subscribeRepos?cursor=9000000000000000000" + )) + .await + .unwrap(); + let refused = next_binary(&mut ws).await; + let (header, rest) = frame_header(&refused); + let refusal: serde_json::Value = frame_payload(rest); + assert_eq!(header["op"], -1); + assert!( + header.get("t").is_none(), + "an error frame carries no type field" + ); + assert_eq!(refusal["error"], "FutureCursor"); + let closed = tokio::time::timeout(FIVE_SECONDS, ws.next()) + .await + .expect("the close arrives within the timeout") + .expect("the stream ends") + .expect("the close is readable"); + match closed { + tokio_tungstenite::tungstenite::Message::Close(Some(frame)) => assert_eq!( + u16::from(frame.code), + 1013, + "the client is told to try again later" + ), + other => panic!("the stream closes with a code, not {other:?}"), + } +} + +async fn refusal(world: &World, query: String) { + let (_, error) = error_of(world, &query).await; + assert_eq!(error["error"], "InvalidRequest", "{query}"); +} + +#[tokio::test] +async fn malformed_collections_rkeys_and_cid_lists_answer_invalid_request() { + let world = published_world(); + let knot = knot_repo(&world); + let collection = fixture_collection(); + [ + format!( + "/xrpc/com.atproto.sync.getRecord?did={knot}&collection=not%20an%20nsid&rkey=did%3Aplc%3Alimpet" + ), + format!( + "/xrpc/com.atproto.sync.getRecord?did={knot}&collection={collection}&rkey=has%20spaces" + ), + format!( + "/xrpc/com.atproto.repo.getRecord?repo={knot}&collection=all+wrong&rkey=did%3Aplc%3Alimpet" + ), + format!("/xrpc/com.atproto.repo.listRecords?repo={knot}&collection=also+wrong"), + format!( + "/xrpc/com.atproto.sync.getBlocks?did={knot}&{}", + (0..1001).map(|n| format!("cids=bafkqaa{n}")).collect::>().join("&") + ), + ] + .into_iter() + .for_each(|query| futures::executor::block_on(refusal(&world, query))); +} diff --git a/knot2/crates/knot-xrpc/tests/atproto_interop.rs b/knot2/crates/knot-xrpc/tests/atproto_interop.rs new file mode 100644 index 0000000000000000000000000000000000000000..f6c5587c9e134a5fdd91a16e73f5fe30939f67b8 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/atproto_interop.rs @@ -0,0 +1,526 @@ +mod common; + +use common::{ + World, commit_file, empty_repo, enable_chain, fixture_collection, http_push, knot_host, nsid, + owner, seeded_feature_branch, serve_pushes, sh_git, staged_source, +}; +use http::StatusCode; +use knot_types::{AccountDid, AtUri, Did, DidRkey, RecordRkey, RepoName, UnixSeconds}; + +const KNOT_DID: &str = "did:web:knot.nel.pet"; +const RKEY: &str = "did:plc:limpet"; +const BOB_DID: &str = "did:plc:bobfixture"; +const ATTESTATION_RKEY: &str = "self"; +const FOSTER_SECRET: &str = "foster-pearl"; +const FOSTER_OWNER: &str = "did:plc:boltfulboy"; +const FOSTER_RKEY: &str = "1"; +const FOSTER_REPO_DID: &str = "did:plc:feralpearl"; +const FOSTER_CREATED_AT: &str = "1970-01-01T00:18:20Z"; + +async fn go_test(run: &str, env: &[(&str, String)]) { + let mut command = tokio::process::Command::new("go"); + command + .arg("test") + .arg("-tags=interop") + .arg("./knot2/interop/") + .arg("-run") + .arg(run) + .arg("-count=1") + .arg("-v") + .current_dir(env!("CARGO_MANIFEST_DIR").to_owned() + "/../../..") + .envs(env.iter().map(|(key, value)| (*key, value))); + let go = command.output().await.expect("go test launches"); + assert!( + go.status.success(), + "go test -run {run} failed:\n{}{}", + String::from_utf8_lossy(&go.stdout), + String::from_utf8_lossy(&go.stderr), + ); +} + +fn fostering_env( + addr: std::net::SocketAddr, + source: &tempfile::TempDir, + sha: &str, +) -> Vec<(&'static str, String)> { + vec![ + ("KNOT_TEST_BASE_URL", format!("http://{addr}")), + ("KNOT_TEST_ADDR", format!("127.0.0.1:{}", addr.port())), + ("KNOT_TEST_FOSTER_SECRET", FOSTER_SECRET.to_owned()), + ( + "KNOT_TEST_FOSTER_SOURCE", + source.path().to_string_lossy().into_owned(), + ), + ("KNOT_TEST_SHA", sha.to_owned()), + ("KNOT_TEST_FOSTER_REPO_DID", FOSTER_REPO_DID.to_owned()), + ( + "KNOT_TEST_PUSH_TOKEN", + common::service_jwt( + &nsid("org.tangled.git.receivePack"), + &AccountDid::new(FOSTER_OWNER).unwrap(), + ), + ), + ] +} + +#[tokio::test] +async fn indigo_client_reads_and_verifies_whole_knot() { + let world = World::new(); + world.publish_meta(&[DidRkey::new(RKEY).unwrap()], UnixSeconds::new(1_000)); + world.publish_meta( + &[ + DidRkey::new(RKEY).unwrap(), + DidRkey::new("did:plc:whelk").unwrap(), + ], + UnixSeconds::new(1_100), + ); + + let addr = common::serve(&world).await; + + go_test( + "TestKnotServesComAtprotoReads", + &[ + ("KNOT_TEST_BASE_URL", format!("http://{addr}")), + ("KNOT_TEST_DID", KNOT_DID.to_owned()), + ("KNOT_TEST_HANDLE", knot_host().to_string()), + ("KNOT_TEST_COLLECTION", fixture_collection().to_string()), + ("KNOT_TEST_RKEY", RKEY.to_owned()), + ], + ) + .await; +} + +#[tokio::test] +async fn an_indigo_client_consumes_the_firehose() { + let world = World::administered(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + let addr = common::serve(&world).await; + + go_test( + "TestKnotServesSubscribeRepos", + &[ + ("KNOT_TEST_BASE_URL", format!("http://{addr}")), + ( + "KNOT_TEST_CREATE_TOKEN", + common::service_jwt(&nsid("org.tangled.repo.createRepo"), &owner()), + ), + ( + "KNOT_TEST_DELETE_TOKEN", + common::service_jwt(&nsid("org.tangled.repo.deleteRepo"), &owner()), + ), + ], + ) + .await; +} + +#[tokio::test] +async fn a_knotfeed_consumer_reads_the_ref_records() { + let world = World::new(); + let (did, _bare, work) = empty_repo(&world, &RepoName::new("conch").unwrap()); + commit_file( + work.path(), + "README.md", + b"hello\n", + "first", + "2026-06-01T12:30:00+02:00", + ); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &did, &owner(), &["main"]).await; + let sha = sh_git(work.path(), &["rev-parse", "HEAD"]); + + go_test( + "TestKnotfeedConsumerReadsRefRecords", + &[ + ("KNOT_TEST_ADDR", format!("127.0.0.1:{}", addr.port())), + ("KNOT_TEST_REPO_DID", did.as_str().to_owned()), + ("KNOT_TEST_SHA", sha.trim().to_owned()), + ], + ) + .await; +} + +#[tokio::test] +async fn indigo_client_reads_tickets_and_patches_in_the_org_tangled_shapes() { + let world = World::new(); + let (did, base, head) = seeded_feature_branch(&world, &RepoName::new("conch").unwrap()); + enable_chain(&world, &did); + let (ticket_rkey, ticket_cid) = common::create_ticket(&world, &did, "kelp", "a forest").await; + + let (patch_rkey, patch_cid) = + common::create_patch(&world, &did, &ticket_rkey, &base, &head).await; + + let addr = common::serve(&world).await; + go_test( + "TestKnotServesTicketsAndPatchesInTheOrgTangledShapes", + &[ + ("KNOT_TEST_BASE_URL", format!("http://{addr}")), + ("KNOT_TEST_REPO_DID", did.as_str().to_owned()), + ("KNOT_TEST_TICKET_RKEY", ticket_rkey.to_string()), + ("KNOT_TEST_TICKET_CID", ticket_cid.cid().to_string()), + ("KNOT_TEST_TICKET_TITLE", "kelp".to_owned()), + ("KNOT_TEST_EDITOR", owner().to_string()), + ("KNOT_TEST_PATCH_RKEY", patch_rkey.to_string()), + ("KNOT_TEST_PATCH_CID", patch_cid.cid().to_string()), + ], + ) + .await; +} + +#[tokio::test] +async fn indigo_client_resolves_bobs_issue_and_matches_his_attestation_from_the_local_plc() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("conch").unwrap()); + enable_chain(&world, &did); + let (ticket_rkey, ticket_cid) = common::create_ticket(&world, &did, "kelp", "a forest").await; + + let ticket_uri = format!("at://{did}/{}/{}", common::ticket_collection(), ticket_rkey); + + let addr = common::serve(&world).await; + let plc_addr = common::serve_local_plc(&world).await; + let pds_addr = common::serve_bob_pds( + &Did::new_owned(BOB_DID).unwrap(), + &RecordRkey::new(ATTESTATION_RKEY).unwrap(), + &AtUri::new_owned(ticket_uri).unwrap(), + &ticket_cid, + ) + .await; + + go_test( + "TestIndigoClientResolvesBobsIssueAndMatchesHisAttestation", + &[ + ("KNOT_TEST_BASE_URL", format!("http://{addr}")), + ("KNOT_TEST_PLC_URL", format!("http://{plc_addr}")), + ("KNOT_TEST_PDS_URL", format!("http://{pds_addr}")), + ("KNOT_TEST_REPO_DID", did.as_str().to_owned()), + ("KNOT_TEST_TICKET_RKEY", ticket_rkey.to_string()), + ("KNOT_TEST_TICKET_CID", ticket_cid.cid().to_string()), + ("KNOT_TEST_EDITOR", owner().to_string()), + ("KNOT_TEST_BOB_DID", BOB_DID.to_owned()), + ], + ) + .await; +} + +#[tokio::test] +async fn foster_functionality_doesnt_activate_if_no_secret() { + let world = World::new(); + let credentials = common::basic_auth("admin", "denied-anyway"); + let seed = serde_json::json!({ + "repoDid": FOSTER_REPO_DID, + "manifest": { + "$type": "org.tangled.repo.manifest", + "declaration": { "owner": FOSTER_OWNER, "name": "boltfulboy" }, + }, + "records": [], + }); + let (status, body) = common::post_raw( + &world, + "/xrpc/org.tangled.temp.server.seedRepo", + credentials.clone(), + seed, + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + let (_, unknown) = common::post_raw( + &world, + "/xrpc/org.tangled.temp.server.noSuchRoute", + credentials, + serde_json::json!({}), + ) + .await; + assert_eq!( + body, unknown, + "unmounted seed route looks same as any arbitrary unknown xrpc" + ); + + let described = common::get_json(&world, "/xrpc/org.tangled.knot.describeKnot").await; + assert_eq!( + described["capabilities"], + serde_json::json!([ + "tangled1.0", + "knot-acl", + "repo-did-input", + "atproto-firehose", + "knot-repo-token-mint", + "knot-social-writes", + ]), + "no foster secret means same capabilities as default knot" + ); +} + +async fn go_foster_leg(run: &str) { + let world = World::fostering(FOSTER_SECRET); + let (source, sha) = staged_source(); + let addr = serve_pushes(&world).await; + go_test(run, &fostering_env(addr, &source, &sha)).await; +} + +#[tokio::test] +async fn seeded_knot_fosters_foreign_repo_verbatim() { + go_foster_leg("TestSeededKnotFostersForeignRepoVerbatim").await; +} + +#[tokio::test] +async fn fostered_repos_announce_to_firehose() { + go_foster_leg("TestFosteredReposAnnounceToFirehose").await; +} + +#[tokio::test] +async fn a_fostering_knot_says_so_in_its_capabilities() { + let world = World::fostering(FOSTER_SECRET); + let described = common::get_json(&world, "/xrpc/org.tangled.knot.describeKnot").await; + assert_eq!( + described["capabilities"], + serde_json::json!([ + "tangled1.0", + "knot-acl", + "repo-did-input", + "atproto-firehose", + "knot-repo-token-mint", + "knot-social-writes", + "knot-foster", + ]), + "foster secret set means knot-foster in capabilities" + ); +} + +fn foster_seed_body(repo_did: &str) -> serde_json::Value { + serde_json::json!({ + "repoDid": repo_did, + "manifest": { + "$type": "org.tangled.repo.manifest", + "declaration": { "owner": FOSTER_OWNER, "name": "boltfulboy" }, + }, + "records": [{ + "collection": "org.tangled.track.ticket", + "rkey": FOSTER_RKEY, + "author": FOSTER_OWNER, + "createdAt": 1100, + "record": { + "$type": "org.tangled.track.ticket", + "title": {"$type": "org.tangled.markup.markdown#inline", "text": "a feral pearl"}, + "state": "open", + "createdAt": FOSTER_CREATED_AT, + "x-tngl-editor": FOSTER_OWNER, + }, + }], + }) +} + +#[tokio::test] +async fn reseeding_under_a_different_repo_did_is_refused() { + let world = World::fostering(FOSTER_SECRET); + let credentials = common::basic_auth("admin", FOSTER_SECRET); + let seed = foster_seed_body(FOSTER_REPO_DID); + let (status, _) = common::post_raw( + &world, + "/xrpc/org.tangled.temp.server.seedRepo", + credentials.clone(), + seed, + ) + .await; + assert_eq!(status, StatusCode::OK); + let impostor = foster_seed_body("did:plc:somebodyelses"); + let (status, body) = common::post_raw( + &world, + "/xrpc/org.tangled.temp.server.seedRepo", + credentials, + impostor, + ) + .await; + assert_eq!( + status, + StatusCode::CONFLICT, + "fostered owner+name has unshakable *real* repo DID: {}", + String::from_utf8_lossy(&body) + ); +} + +#[tokio::test] +async fn fostered_manifests_are_stored_open_whatever_they_declared() { + let world = World::fostering(FOSTER_SECRET); + let credentials = common::basic_auth("admin", FOSTER_SECRET); + let mut seed = foster_seed_body(FOSTER_REPO_DID); + seed["manifest"]["contributionPolicy"] = serde_json::json!("collaborators"); + let (status, body) = common::post_raw( + &world, + "/xrpc/org.tangled.temp.server.seedRepo", + credentials, + seed, + ) + .await; + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + let manifest = common::get_json( + &world, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={FOSTER_REPO_DID}&collection=org.tangled.repo.manifest&rkey=self" + ), + ) + .await; + assert_eq!( + manifest["value"]["contributionPolicy"], "anyone", + "fostered shadows open their doors no matter what seeds declared" + ); +} + +#[derive(serde::Deserialize)] +struct FirehoseOp { + action: String, + path: String, +} + +#[derive(serde::Deserialize)] +struct FirehoseCommit { + repo: String, + ops: Vec, +} + +fn firehose_commit(frame: &[u8]) -> Option { + let mut read = frame; + let header: serde_json::Value = serde_ipld_dagcbor::de::from_reader_once(&mut read).ok()?; + match header["t"].as_str() == Some("#commit") { + true => serde_ipld_dagcbor::de::from_reader_once(&mut read).ok(), + false => None, + } +} + +#[tokio::test] +async fn a_fostered_record_survives_a_restart() { + let world = World::fostering(FOSTER_SECRET); + let credentials = common::basic_auth("admin", FOSTER_SECRET); + let seed = serde_json::json!({ + "repoDid": FOSTER_REPO_DID, + "manifest": { + "$type": "org.tangled.repo.manifest", + "declaration": { "owner": FOSTER_OWNER, "name": "boltfulboy" }, + }, + "records": [{ + "collection": "org.tangled.track.ticket", + "rkey": FOSTER_RKEY, + "author": FOSTER_OWNER, + "createdAt": 1100, + "record": { + "$type": "org.tangled.track.ticket", + "title": {"$type": "org.tangled.markup.markdown#inline", "text": "a feral pearl"}, + "state": "open", + "createdAt": FOSTER_CREATED_AT, + "x-tngl-editor": FOSTER_OWNER, + }, + }], + }); + let expected = seed["records"][0]["record"].clone(); + let (status, raw) = common::post_raw( + &world, + "/xrpc/org.tangled.temp.server.seedRepo", + credentials, + seed, + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "seed req must succeed: {}", + String::from_utf8_lossy(&raw) + ); + let seeded: serde_json::Value = serde_json::from_slice(&raw).unwrap(); + assert!( + seeded["results"][0]["error"].is_null(), + "verbatim record seed cleand" + ); + let did = seeded["did"].as_str().unwrap().to_owned(); + assert_eq!( + did, FOSTER_REPO_DID, + "seed resp gives the supplied repo DID instead of creating a new one mistakenly" + ); + + let state = std::sync::Arc::clone(&world.state); + let failed = + tokio::task::spawn_blocking(move || knot_xrpc::materialize::enable_surfaces(&state)) + .await + .unwrap(); + assert_eq!( + failed, 0, + "boot materializion-pass reconciles all hosted repos" + ); + + let fetched = common::get_json( + &world, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={did}&collection=org.tangled.track.ticket&rkey={FOSTER_RKEY}" + ), + ) + .await; + assert_eq!( + fetched["value"], expected, + "fostered record stays verbatim after restart" + ); + + let listed = common::get_json( + &world, + &format!( + "/xrpc/com.atproto.repo.listRecords?repo={did}&collection=org.tangled.track.ticket&limit=50" + ), + ) + .await; + let records = listed["records"].as_array().unwrap(); + assert_eq!( + records.len(), + 1, + "restart shouldn't wipe out fostered records from listing" + ); + assert_eq!( + records[0]["uri"], + format!("at://{did}/org.tangled.track.ticket/{FOSTER_RKEY}") + ); + + let bounds = knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(1024).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ); + let replayed = world + .state + .firehose + .replay(knot_events::EventCursor::START, bounds); + for event in &replayed.events { + let mut read: &[u8] = &event.frame; + let Ok(header) = + serde_ipld_dagcbor::de::from_reader_once::(&mut read) + else { + continue; + }; + match header["t"].as_str() { + Some("#identity") | Some("#account") => { + let Ok(payload) = + serde_ipld_dagcbor::de::from_reader_once::(&mut read) + else { + continue; + }; + assert_ne!( + payload["did"].as_str(), + Some(did.as_str()), + "foster knots should never ever ever announce ownership of an identity of a repo it fosters" + ); + } + _ => {} + } + } + let path = format!("org.tangled.track.ticket/{FOSTER_RKEY}"); + let mut announced = false; + for event in &replayed.events { + let Some(commit) = firehose_commit(&event.frame) else { + continue; + }; + if commit.repo != did { + continue; + } + for op in &commit.ops { + assert_ne!( + op.action, "delete", + "knot restart shouldn't emit firehose delete for {}", + op.path + ); + announced |= op.action == "create" && op.path == path; + } + } + assert!(announced, "a seeded record announced itself before restart"); +} diff --git a/knot2/crates/knot-xrpc/tests/blob_properties.proptest-regressions b/knot2/crates/knot-xrpc/tests/blob_properties.proptest-regressions new file mode 100644 index 0000000000000000000000000000000000000000..bd2aeefe4b0059651cd9f996c999c1c7aa8cb408 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/blob_properties.proptest-regressions @@ -0,0 +1,7 @@ +# Here lie some seeds for failure cases that I have already generated with the proptester. +# The proptester will start by reading these and running them before generating any new cases! +# +# Please don't mess with these existing cases unless they become unnecessary. +cc 54ac38076efa69c847c604527a729aee1b2f4c713d51d99bcfe9647a749c28bf # which condenses to ops = [Upload(13)] +cc c671907a763ce7005394279834692fc14ee75200aa4f1fdc35792e9a2bf3fbcc # condenses to ops = [Upload(16), Upload(6), Upload(0), Upload(1), Upload(2), Upload(3), Upload(4)] +cc 32078ad37be8c22b598242f5235d8c065b64f88097a37bfe9db908fc017ac3b9 # shrinks to ops = [Upload(2), Create(2), Delete(0)] diff --git a/knot2/crates/knot-xrpc/tests/blob_properties.rs b/knot2/crates/knot-xrpc/tests/blob_properties.rs new file mode 100644 index 0000000000000000000000000000000000000000..851900b337a34a09037c90032584ff1482da6177 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/blob_properties.rs @@ -0,0 +1,381 @@ +mod common; + +use std::collections::BTreeMap; + +use axum::body::Body; +use common::{World, empty_repo, enable_chain, get, nsid, owner, post_as, service_jwt_for}; +use futures::{StreamExt, TryStreamExt}; +use http::{Request, StatusCode, header}; +use jacquard_common::types::tid::Tid; +use knot_types::{BlobCid, RecordRkey, RepoDid, RepoName, ServiceDid, Sha256Digest}; +use proptest::prelude::*; +use serde_json::json; +use tower::ServiceExt; + +const MAGICS: [(&[u8], &str); 8] = [ + ( + &[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a], + "image/png", + ), + (&[0xFF, 0xD8, 0xFF, 0xE0], "image/jpeg"), + (b"GIF87a", "image/gif"), + ( + &[b'R', b'I', b'F', b'F', 1, 2, 3, 4, b'W', b'E', b'B', b'P'], + "image/webp", + ), + (&[b'B', b'M', 1, 2, 3, 4], "image/bmp"), + (b" Vec { + let (magic, _) = MAGICS[(token % MAGICS.len() as u16) as usize]; + let mut body = magic.to_vec(); + body.extend_from_slice(&token.to_be_bytes()); + body.extend(std::iter::repeat_n(0x5a, (token % 7) as usize)); + body +} + +fn mime_of(token: u16) -> &'static str { + MAGICS[(token % MAGICS.len() as u16) as usize].1 +} + +fn blob_cid(bytes: &[u8]) -> BlobCid { + BlobCid::from_digest(Sha256Digest::hash(bytes)) +} + +fn commit_cid_of(response: &serde_json::Value) -> cid::Cid { + cid::Cid::try_from( + response["cid"] + .as_str() + .expect("the write response has a commit"), + ) + .expect("the commit CID parses") +} + +#[derive(Clone, Debug)] +enum Op { + Upload(u16), + Create(u16), + Edit(u16, u16), + Delete(u16), +} + +fn op_strategy() -> impl Strategy { + prop_oneof![ + (0u16..24).prop_map(Op::Upload), + (0u16..24, 0u16..24).prop_map(|(a, b)| Op::Create(if b % 4 == 0 { a } else { a % 5 })), + (0u16..24, 0u16..24, 0u16..24).prop_map(|(record, a, b)| { + Op::Edit(record % 6, if b % 4 == 0 { a } else { a % 5 }) + }), + (0u16..24).prop_map(|record| Op::Delete(record % 6)), + ] +} + +#[derive(Clone)] +struct LiveRecord { + swap: cid::Cid, + rev: Tid, + blob: u16, +} + +struct Model { + uploaded: BTreeMap>, + live: Vec<(RecordRkey, LiveRecord)>, +} + +impl Model { + fn blob_json(&self, token: u16) -> serde_json::Value { + let blob = json!({ + "ref": { "$link": blob_cid(&self.uploaded[&token]).to_string() }, + "mimeType": mime_of(token), + "size": self.uploaded[&token].len(), + }); + match mime_of(token) { + "video/mp4" => json!({ + "$type": "org.tangled.markup.embed.video", + "video": blob + }), + _ => json!({ + "$type": "org.tangled.markup.embed.image", + "image": blob + }), + } + } + + fn ticket_json(&self, repo: &RepoDid, title: &str, token: u16) -> serde_json::Value { + json!({ + "$type": common::ticket_collection(), + "repo": repo, + "title": title, + "body": { + "$type": "org.tangled.markup.markdown", + "text": title, + "blobs": [self.blob_json(token)], + }, + "createdAt": "2026-10-01T00:00:00Z", + }) + } + + fn embed_cid(blob: &serde_json::Value) -> BlobCid { + let inner = if blob["image"].is_null() { + &blob["video"] + } else { + &blob["image"] + }; + common::blob_cid_of(&inner["ref"]["$link"]) + } + + fn embeddable(token: u16) -> bool { + let mime = mime_of(token); + mime == "video/mp4" || mime.starts_with("image/") + } + + fn at(&self, index: usize) -> Option<(&RecordRkey, &LiveRecord)> { + self.live.get(index).map(|(rkey, record)| (rkey, record)) + } + + fn expected_listing(&self) -> Vec { + let mut latest: BTreeMap = BTreeMap::new(); + self.live + .iter() + .for_each(|(_, record)| match latest.get(&record.blob) { + Some(seen) if seen.as_str() >= record.rev.as_str() => (), + _ => { + latest.insert(record.blob, &record.rev); + } + }); + let mut ordered: Vec<(&Tid, u16)> = latest + .into_iter() + .map(|(token, rev)| (rev, token)) + .collect(); + ordered.sort_by(|(a, _), (b, _)| a.as_str().cmp(b.as_str())); + ordered + .into_iter() + .map(|(_, token)| blob_cid(&self.uploaded[&token])) + .collect() + } +} + +async fn upload(world: &World, repo: &RepoDid, body: &[u8]) -> (StatusCode, serde_json::Value) { + let request = Request::post("/xrpc/com.atproto.repo.uploadBlob") + .header( + header::AUTHORIZATION, + format!( + "Bearer {}", + service_jwt_for( + &nsid("com.atproto.repo.uploadBlob"), + &owner(), + &ServiceDid::new(repo.as_str()).unwrap(), + ) + ), + ) + .header(header::CONTENT_TYPE, "application/octet-stream") + .body(Body::from(body.to_vec())) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null), + ) +} + +async fn drive(ops: Vec) -> Result<(), TestCaseError> { + let world = World::unshed(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("anemone").unwrap()); + enable_chain(&world, &did); + let model = Model { + uploaded: BTreeMap::new(), + live: Vec::new(), + }; + + let model = futures::stream::iter(ops) + .map(Ok) + .try_fold(model, |mut model, op| { + let (world, did) = (&world, &did); + async move { + match op { + Op::Upload(token) => { + let body = body_of(token); + let (status, response) = upload(world, did, &body).await; + prop_assert_eq!(status, StatusCode::OK, "{}", response); + model.uploaded.entry(token).or_insert(body); + } + Op::Create(token) => { + if !model.uploaded.contains_key(&token) || !Model::embeddable(token) { + return Ok(model); + } + let (status, written) = post_as( + world, + &nsid("com.atproto.repo.createRecord"), + &owner(), + &ServiceDid::new(did.as_str()).unwrap(), + json!({ + "repo": did, + "collection": common::ticket_collection(), + "record": model.ticket_json(did, "kelp under test", token), + }), + ) + .await; + prop_assert_eq!(status, StatusCode::OK, "{}", written); + model.live.push(( + common::rkey_of(&written), + LiveRecord { + swap: commit_cid_of(&written), + rev: common::commit_rev_of(&written), + blob: token, + }, + )); + } + Op::Edit(index, token) => { + let (rkey, swap) = match model.at(index as usize) { + Some((rkey, record)) => (rkey.clone(), record.swap), + None => return Ok(model), + }; + if !model.uploaded.contains_key(&token) || !Model::embeddable(token) { + return Ok(model); + } + let (status, written) = post_as( + world, + &nsid("com.atproto.repo.putRecord"), + &owner(), + &ServiceDid::new(did.as_str()).unwrap(), + json!({ + "repo": did, + "collection": common::ticket_collection(), + "rkey": rkey.to_string(), + "swapRecord": swap.to_string(), + "record": model.ticket_json(did, "kelp under test, again", token), + }), + ) + .await; + prop_assert_eq!(status, StatusCode::OK, "{}", written); + let replaced = common::rkey_of(&written); + if let Some((_, record)) = + model.live.iter_mut().find(|(rkey, _)| *rkey == replaced) + { + record.swap = commit_cid_of(&written); + record.rev = common::commit_rev_of(&written); + record.blob = token; + } + } + Op::Delete(index) => { + let (rkey, swap) = match model.at(index as usize) { + Some((rkey, record)) => (rkey.clone(), record.swap), + None => return Ok(model), + }; + let (status, response) = post_as( + world, + &nsid("com.atproto.repo.deleteRecord"), + &owner(), + &ServiceDid::new(did.as_str()).unwrap(), + json!({ + "repo": did, + "collection": common::ticket_collection(), + "rkey": rkey.to_string(), + "swapRecord": swap.to_string(), + }), + ) + .await; + prop_assert_eq!(status, StatusCode::OK, "{}", response); + model.live.retain(|(key, _)| key != &rkey); + } + } + + let (status, _headers, listed) = get( + world, + &format!("/xrpc/com.atproto.sync.listBlobs?did={did}"), + ) + .await; + let listed: serde_json::Value = serde_json::from_slice(&listed).unwrap_or_default(); + prop_assert_eq!(status, StatusCode::OK); + let served: Vec = listed["cids"] + .as_array() + .map(|cids| cids.iter().map(common::blob_cid_of).collect()) + .unwrap_or_default(); + prop_assert_eq!( + served, + model.expected_listing(), + "The listing holds one blob per live record, oldest reference first." + ); + Ok(model) + } + }) + .await?; + + futures::stream::iter(&model.uploaded) + .map(Ok) + .try_for_each(|(token, bytes)| { + let (world, did) = (&world, &did); + async move { + let (status, headers, served) = get( + world, + &format!( + "/xrpc/com.atproto.sync.getBlob?did={did}&cid={}", + blob_cid(bytes) + ), + ) + .await; + prop_assert_eq!(status, StatusCode::OK, "Blob {} serves.", token); + prop_assert_eq!(served.as_ref(), bytes.as_slice()); + prop_assert_eq!(headers.get(header::CONTENT_TYPE).unwrap(), mime_of(*token)); + Ok(()) + } + }) + .await?; + + futures::stream::iter(&model.live) + .map(Ok) + .try_for_each(|(rkey, record)| { + let (world, did, uploaded) = (&world, &did, &model.uploaded); + async move { + let (status, _headers, response) = get( + world, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={did}&collection=org.tangled.track.ticket&rkey={rkey}" + ), + ) + .await; + let response: serde_json::Value = + serde_json::from_slice(&response).unwrap_or_default(); + prop_assert_eq!(status, StatusCode::OK, "{}", response); + prop_assert_eq!( + response["value"]["body"]["blobs"] + .as_array() + .map(|blobs| blobs.iter().map(Model::embed_cid).collect::>()), + Some(vec![blob_cid(&uploaded[&record.blob])]), + "{} has blob {} until it is deleted.", + rkey, + record.blob + ); + Ok(()) + } + }) + .await?; + + let git = world.layout.open(&did).unwrap(); + let warm = world.state.blobs.get(&git, &did).unwrap(); + let cold = knot_xrpc::BlobViews::default().get(&git, &did).unwrap(); + prop_assert_eq!(&warm.listed, &cold.listed); + prop_assert_eq!(&warm.referenced, &cold.referenced); + prop_assert_eq!(warm.coverage, cold.coverage); + Ok(()) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(24))] + #[test] + fn blob_listings_and_serves_follow_random_write_history( + ops in proptest::collection::vec(op_strategy(), 0..24) + ) { + tokio::runtime::Runtime::new().unwrap().block_on(drive(ops))?; + } +} diff --git a/knot2/crates/knot-xrpc/tests/blobs.rs b/knot2/crates/knot-xrpc/tests/blobs.rs new file mode 100644 index 0000000000000000000000000000000000000000..24a5f3371eb5695e08eb22194b50bf010b48f5fc --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/blobs.rs @@ -0,0 +1,661 @@ +mod common; + +use axum::body::Body; +use common::post_json; +use futures::StreamExt; +use http::{Request, StatusCode, header}; +use serde_json::json; +use tower::ServiceExt; + +use common::{ + World, empty_repo, enable_chain, get, nsid, owner, post_as, seeded_feature_branch, + service_jwt_for, +}; +use knot_types::{AccountDid, RepoDid, RepoName, ServiceDid}; + +const PNG: &[u8] = &[ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, + 0x89, +]; + +const SECOND: &[u8] = &[ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x01, +]; +const STRANGER: &str = "did:web:stranger.nel.pet"; + +fn stranger() -> AccountDid { + AccountDid::new(STRANGER).unwrap() +} + +fn blob_cid_link(bytes: &[u8]) -> knot_types::BlobCid { + knot_types::BlobCid::from_digest(knot_types::Sha256Digest::hash(bytes)) +} +fn blob_ref_json(cid: knot_types::BlobCid, mime: &str, size: u64) -> serde_json::Value { + json!({ + "$type": "blob", + "ref": { "$link": cid.to_string() }, + "mimeType": mime, + "size": size, + }) +} + +async fn upload_as( + world: &World, + repo: &RepoDid, + body: &[u8], + hint: &str, +) -> (StatusCode, serde_json::Value) { + let request = Request::post("/xrpc/com.atproto.repo.uploadBlob") + .header( + header::AUTHORIZATION, + format!( + "Bearer {}", + service_jwt_for( + &nsid("com.atproto.repo.uploadBlob"), + &owner(), + &ServiceDid::new(repo.as_str()).unwrap(), + ) + ), + ) + .header(header::CONTENT_TYPE, hint) + .body(Body::from(body.to_vec())) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let value = serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); + (status, value) +} + +async fn upload(world: &World, repo: &RepoDid, body: &[u8]) -> (StatusCode, serde_json::Value) { + upload_as(world, repo, body, "image/png").await +} + +async fn get_json(world: &World, path_and_query: &str) -> (StatusCode, serde_json::Value) { + let (status, _headers, bytes) = get(world, path_and_query).await; + let value = serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); + (status, value) +} + +async fn create_with_blob( + world: &World, + actor: Option<&AccountDid>, + repo: &RepoDid, + blob: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + let record = json!({ + "repo": repo, + "collection": common::ticket_collection(), + "record": { + "$type": common::ticket_collection(), + "repo": repo, + "title": "kelp with a picture", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "a forest", + "blobs": [json!({ + "$type": "org.tangled.markup.embed.image", + "image": blob + })], + }, + "state": "open", + "createdAt": "2026-09-30T00:00:00Z", + }, + }); + match actor { + Some(actor) => { + post_as( + world, + &nsid("com.atproto.repo.createRecord"), + actor, + &ServiceDid::new(repo.as_str()).unwrap(), + record, + ) + .await + } + None => post_json(world, "/xrpc/com.atproto.repo.createRecord", record).await, + } +} + +fn aged_blob_path(world: &World, repo: &RepoDid, body: &[u8]) -> std::path::PathBuf { + let oid = knot_lfs::LfsOid::from_digest(knot_types::Sha256Digest::hash(body)); + let store = &world.state.lfs.handle.store; + store + .object_file(knot_lfs::Pool::Attachments, repo, &oid) + .unwrap() + .unwrap() + .1 +} + +#[tokio::test] +async fn attachment_round_trips_from_upload_to_getblob_and_listblobs() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("anemone").unwrap()); + enable_chain(&world, &did); + let repo = did.as_str().to_owned(); + + let (status, uploaded) = upload(&world, &did, PNG).await; + assert_eq!(status, StatusCode::OK, "{uploaded}"); + assert_eq!(uploaded["blob"]["$type"], "blob"); + assert_eq!(uploaded["blob"]["mimeType"], "image/png"); + assert_eq!(uploaded["blob"]["size"], PNG.len() as u64); + assert_eq!( + common::blob_cid_of(&uploaded["blob"]["ref"]["$link"]), + blob_cid_link(PNG) + ); + + let blob = uploaded["blob"].clone(); + let (status, written) = create_with_blob(&world, Some(&owner()), &did, blob).await; + assert_eq!(status, StatusCode::OK, "{written}"); + let uri = written["uri"].as_str().unwrap().to_owned(); + let rkey = uri.rsplit('/').next().unwrap().to_owned(); + + let (status, _headers, _bytes) = get( + &world, + &format!("/xrpc/com.atproto.repo.getRecord?repo={repo}&collection=org.tangled.track.ticket&rkey={rkey}"), + ) + .await; + assert_eq!(status, StatusCode::OK); + + let (status, headers, served) = get( + &world, + &format!( + "/xrpc/com.atproto.sync.getBlob?did={repo}&cid={}", + blob_cid_link(PNG) + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(served.as_ref(), PNG); + assert_eq!(headers.get(header::CONTENT_TYPE).unwrap(), "image/png"); + assert_eq!(headers.get("x-content-type-options").unwrap(), "nosniff"); + assert!( + headers + .get("content-security-policy") + .unwrap() + .to_str() + .unwrap() + .starts_with("default-src 'none'") + ); + + let (status, listed) = get_json( + &world, + &format!("/xrpc/com.atproto.sync.listBlobs?did={repo}"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + common::listed_cids(&listed), + vec![blob_cid_link(PNG)], + "{listed}" + ); + + futures::stream::iter([ + ( + blob_ref_json(blob_cid_link(b"never uploaded anywhere"), "image/png", 9), + "isn't in this repo's blob store", + ), + ( + blob_ref_json(blob_cid_link(PNG), "image/png", 3), + "is stored as", + ), + ( + blob_ref_json(blob_cid_link(PNG), "image/webp", PNG.len() as u64), + "reads as", + ), + ]) + .for_each(|(blob, refused)| { + let (world, did) = (&world, &did); + async move { + let (status, response) = create_with_blob(world, Some(&owner()), did, blob).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{response}"); + assert!( + response["message"].as_str().unwrap().contains(refused), + "{response}" + ); + } + }) + .await; +} + +#[tokio::test] +async fn upload_bounds_take_any_bytes_under_one_mime_budget() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("anemone").unwrap()); + let mut boundary = vec![0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a]; + boundary.resize(1_000_000, 0); + let (status, accepted) = upload(&world, &did, &boundary).await; + assert_eq!(status, StatusCode::OK, "{accepted}"); + assert_eq!(accepted["blob"]["size"], 1_000_000_u64, "{accepted}"); + assert_eq!(accepted["blob"]["mimeType"], "image/png", "{accepted}"); +} + +#[tokio::test] +async fn unsniffable_bytes_take_the_hint_except_html() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("anemone").unwrap()); + let mystery = b"an unsniffable body"; + let (status, trusted) = upload_as(&world, &did, mystery, "application/x-whelk-report").await; + assert_eq!(status, StatusCode::OK, "{trusted}"); + assert_eq!( + trusted["blob"]["mimeType"], "application/x-whelk-report", + "{trusted}" + ); + let (status, refused_html) = upload_as(&world, &did, mystery, "text/html; charset=utf-8").await; + assert_eq!(status, StatusCode::OK, "{refused_html}"); + assert_eq!( + refused_html["blob"]["mimeType"], "application/octet-stream", + "{refused_html}" + ); + let (status, bare) = upload_as(&world, &did, b"mystery bytes", "").await; + assert_eq!(status, StatusCode::OK, "{bare}"); + assert_eq!( + bare["blob"]["mimeType"], "application/octet-stream", + "{bare}" + ); +} + +#[tokio::test] +async fn pdf_svg_and_mp4_uploads_take_the_sniffed_mime() { + async fn upload_returns_mime(world: &World, repo: &RepoDid, body: &[u8], mime: &str) { + let (status, response) = upload(world, repo, body).await; + assert_eq!(status, StatusCode::OK, "{response}"); + assert_eq!(response["blob"]["mimeType"], mime, "{response}"); + } + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("anemone").unwrap()); + upload_returns_mime( + &world, + &did, + b"%PDF-1.7\n%an academic paper\n", + "application/pdf", + ) + .await; + upload_returns_mime( + &world, + &did, + b"", + "image/svg+xml", + ) + .await; + upload_returns_mime( + &world, + &did, + &[0, 0, 0, 32, b'f', b't', b'y', b'p', b'i', b's', b'o', b'm'], + "video/mp4", + ) + .await; +} + +#[tokio::test] +async fn getblob_returns_blob_not_found_for_unknown_cid() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("anemone").unwrap()); + enable_chain(&world, &did); + let repo = did.as_str().to_owned(); + let (status, response) = get_json( + &world, + &format!( + "/xrpc/com.atproto.sync.getBlob?did={repo}&cid={}", + blob_cid_link(b"never uploaded anywhere") + ), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND, "{response}"); + assert_eq!(response["error"], "BlobNotFound", "{response}"); +} + +#[tokio::test] +async fn listblobs_pages_and_applies_since_at_each_records_rev() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("anemone").unwrap()); + enable_chain(&world, &did); + let repo = did.as_str().to_owned(); + + let links: Vec<(jacquard_common::types::tid::Tid, knot_types::BlobCid)> = + futures::stream::iter([PNG, SECOND]) + .then(|body| { + let (world, did) = (&world, &did); + async move { + let (status, uploaded) = upload(world, did, body).await; + assert_eq!(status, StatusCode::OK, "{uploaded}"); + let (status, written) = + create_with_blob(world, Some(&owner()), did, uploaded["blob"].clone()) + .await; + assert_eq!(status, StatusCode::OK, "{written}"); + (common::commit_rev_of(&written), blob_cid_link(body)) + } + }) + .collect() + .await; + + let (status, both) = get_json( + &world, + &format!("/xrpc/com.atproto.sync.listBlobs?did={repo}"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + common::listed_cids(&both), + vec![links[0].1, links[1].1], + "The full listing is oldest reference first." + ); + + let (status, page) = get_json( + &world, + &format!("/xrpc/com.atproto.sync.listBlobs?did={repo}&limit=1"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(common::listed_cids(&page), vec![links[0].1], "{page}"); + let cursor = page["cursor"].as_str().unwrap().to_owned(); + let (status, rest) = get_json( + &world, + &format!("/xrpc/com.atproto.sync.listBlobs?did={repo}&limit=1&cursor={cursor}"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(common::listed_cids(&rest), vec![links[1].1], "{rest}"); + assert!(rest["cursor"].is_null(), "The last page has no cursor."); + + let (status, since) = get_json( + &world, + &format!( + "/xrpc/com.atproto.sync.listBlobs?did={repo}&since={}", + links[0].0 + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + common::listed_cids(&since), + vec![links[1].1], + "`since` at the first write's own rev excludes that rev, matching the firehose cursor." + ); + let (status, empty) = get_json( + &world, + &format!( + "/xrpc/com.atproto.sync.listBlobs?did={repo}&since={}", + links[1].0 + ), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + empty["cids"], + json!([]), + "`since` at the newest rev returns an empty page." + ); +} + +async fn tcp_call( + addr: std::net::SocketAddr, + method: &str, + path_and_query: &str, + bearer: Option<&str>, + content_type: Option<&str>, + body: &[u8], +) -> (u16, Vec<(String, String)>, Vec) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let mut request = + format!("{method} {path_and_query} HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\n"); + if let Some(token) = bearer { + request.push_str(&format!("Authorization: Bearer {token}\r\n")); + } + if let Some(mime) = content_type { + request.push_str(&format!("Content-Type: {mime}\r\n")); + } + request.push_str(&format!("Content-Length: {}\r\n\r\n", body.len())); + + let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap(); + stream.write_all(request.as_bytes()).await.unwrap(); + stream.write_all(body).await.unwrap(); + let mut raw = Vec::new(); + stream.read_to_end(&mut raw).await.unwrap(); + + let split = raw + .windows(4) + .position(|window| window == b"\r\n\r\n") + .expect("the response starts with a header block"); + let head = String::from_utf8_lossy(&raw[..split]).to_string(); + let mut lines = head.split("\r\n"); + let status: u16 = lines + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .parse() + .unwrap(); + let headers = lines + .map(|line| { + let (name, value) = line.split_once(':').expect("a header line"); + (name.trim().to_ascii_lowercase(), value.trim().to_owned()) + }) + .collect(); + (status, headers, raw[split + 4..].to_vec()) +} + +fn header_of<'a>(headers: &'a [(String, String)], name: &str) -> &'a str { + &headers + .iter() + .find(|(key, _)| key == name) + .unwrap_or_else(|| panic!("the {name} header is present")) + .1 +} + +#[tokio::test] +async fn attachments_round_trip_over_real_tcp_against_live_listener() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("conch").unwrap()); + enable_chain(&world, &did); + let repo = did.as_str(); + let addr = common::serve(&world).await; + + let (status, _headers, uploaded) = tcp_call( + addr, + "POST", + "/xrpc/com.atproto.repo.uploadBlob", + Some(&service_jwt_for( + &nsid("com.atproto.repo.uploadBlob"), + &owner(), + &ServiceDid::new(repo).unwrap(), + )), + Some("image/png"), + PNG, + ) + .await; + assert_eq!(status, 200); + let uploaded: serde_json::Value = serde_json::from_slice(&uploaded).unwrap(); + let link = common::blob_cid_of(&uploaded["blob"]["ref"]["$link"]); + assert_eq!(uploaded["blob"]["mimeType"], "image/png"); + assert_eq!(link, blob_cid_link(PNG)); + + let (status, headers, served) = tcp_call( + addr, + "GET", + &format!("/xrpc/com.atproto.sync.getBlob?did={repo}&cid={link}"), + None, + None, + b"", + ) + .await; + assert_eq!(status, 200); + assert_eq!(served, PNG); + assert_eq!(header_of(&headers, "content-type"), "image/png"); + assert_eq!(header_of(&headers, "x-content-type-options"), "nosniff"); + assert_eq!( + header_of(&headers, "content-security-policy"), + "default-src 'none'; sandbox" + ); +} + +#[tokio::test] +async fn refused_writers_leave_stored_blobs_grace_window_untouched() { + let world = World::under(knot_types::ContributionPolicy::Collaborators); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("anemone").unwrap()); + enable_chain(&world, &did); + let (status, uploaded) = upload(&world, &did, PNG).await; + assert_eq!(status, StatusCode::OK, "{uploaded}"); + + let path = aged_blob_path(&world, &did, PNG); + let aged = std::time::UNIX_EPOCH + std::time::Duration::from_secs(3_600); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_modified(aged) + .unwrap(); + + let blob = uploaded["blob"].clone(); + let (status, refused) = create_with_blob(&world, None, &did, blob.clone()).await; + assert_eq!(status, StatusCode::UNAUTHORIZED, "{refused}"); + let (status, refused) = create_with_blob(&world, Some(&stranger()), &did, blob).await; + assert_eq!(status, StatusCode::FORBIDDEN, "{refused}"); + assert_eq!( + std::fs::metadata(&path).unwrap().modified().unwrap(), + aged, + "Refused writes never touch a stored blob's mtime." + ); +} + +#[tokio::test] +async fn listblobs_keeps_shared_blob_at_latest_referencing_record() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("anemone").unwrap()); + enable_chain(&world, &did); + let repo = did.as_str().to_owned(); + + let (status, uploaded) = upload(&world, &did, PNG).await; + assert_eq!(status, StatusCode::OK, "{uploaded}"); + let blob = uploaded["blob"].clone(); + let (status, first) = create_with_blob(&world, Some(&owner()), &did, blob.clone()).await; + assert_eq!(status, StatusCode::OK, "{first}"); + let (status, second) = create_with_blob(&world, Some(&owner()), &did, blob).await; + assert_eq!(status, StatusCode::OK, "{second}"); + + let at_first = common::commit_rev_of(&first); + let (status, listed) = get_json( + &world, + &format!("/xrpc/com.atproto.sync.listBlobs?did={repo}&since={at_first}"), + ) + .await; + assert_eq!(status, StatusCode::OK, "{listed}"); + assert_eq!( + common::listed_cids(&listed), + vec![blob_cid_link(PNG)], + "Writing the record after the cursor keeps the blob listed." + ); +} + +#[tokio::test] +async fn pull_attachments_live_on_the_ticket_not_the_patch() { + let world = World::new(); + let (did, base, head) = seeded_feature_branch(&world, &RepoName::new("whelk").unwrap()); + enable_chain(&world, &did); + let repo = did.as_str().to_owned(); + let (status, uploaded) = upload(&world, &did, PNG).await; + assert_eq!(status, StatusCode::OK, "{uploaded}"); + + let (status, ticket) = + create_with_blob(&world, Some(&owner()), &did, uploaded["blob"].clone()).await; + assert_eq!(status, StatusCode::OK, "{ticket}"); + let ticket_rkey = common::rkey_of(&ticket); + + common::create_patch(&world, &did, &ticket_rkey, &base, &head).await; + + let (status, listed) = get_json( + &world, + &format!("/xrpc/com.atproto.sync.listBlobs?did={repo}"), + ) + .await; + assert_eq!(status, StatusCode::OK, "{listed}"); + assert_eq!( + common::listed_cids(&listed) + .into_iter() + .filter(|cid| *cid == blob_cid_link(PNG)) + .count(), + 1, + "The ticket that fronts the pull lists its blob: {listed}" + ); + + let (status, served) = get_json( + &world, + &format!("/xrpc/com.atproto.repo.getRecord?repo={repo}&collection=org.tangled.track.ticket&rkey={ticket_rkey}"), + ) + .await; + assert_eq!(status, StatusCode::OK, "{served}"); + assert_eq!( + common::blob_cid_of(&served["value"]["body"]["blobs"][0]["image"]["ref"]["$link"]), + blob_cid_link(PNG), + "The ticket record lists the exact blob it was written with: {served}" + ); +} + +#[tokio::test] +async fn an_artifact_keeps_its_blob_listed_and_past_the_sweep() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("anemone").unwrap()); + enable_chain(&world, &did); + let repo = did.as_str().to_owned(); + + let (status, uploaded) = upload_as(&world, &did, PNG, "application/octet-stream").await; + assert_eq!(status, StatusCode::OK, "{uploaded}"); + let (status, written) = post_as( + &world, + &nsid("com.atproto.repo.createRecord"), + &owner(), + &ServiceDid::new(&repo).unwrap(), + json!({ + "repo": repo, + "collection": "org.tangled.repo.artifact", + "record": { + "$type": "org.tangled.repo.artifact", + "subject": { "$type": "org.tangled.git.oid", "oid": "1".repeat(40) }, + "name": "kelp-1.0.png", + "artifact": uploaded["blob"], + "createdAt": "2026-09-30T00:00:00Z", + }, + }), + ) + .await; + assert_eq!(status, StatusCode::OK, "{written}"); + + let (status, listed) = get_json( + &world, + &format!("/xrpc/com.atproto.sync.listBlobs?did={repo}"), + ) + .await; + assert_eq!(status, StatusCode::OK, "{listed}"); + assert_eq!(common::listed_cids(&listed), vec![blob_cid_link(PNG)]); + + let path = aged_blob_path(&world, &did, PNG); + let aged = std::time::UNIX_EPOCH + std::time::Duration::from_secs(3_600); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_modified(aged) + .unwrap(); + let git = world.state.layout.open(&did).unwrap(); + let roots = world.state.blobs.live_roots(&git, &did).unwrap(); + let swept = world + .state + .lfs + .handle + .store + .sweep_unreferenced( + &did, + &roots, + std::time::Duration::from_secs(14 * 86_400), + std::time::SystemTime::now(), + ) + .unwrap(); + assert_eq!(swept.objects, 0, "An artifact's blob is referenced."); + assert!(path.exists()); +} diff --git a/knot2/crates/knot-xrpc/tests/common/mod.rs b/knot2/crates/knot-xrpc/tests/common/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..d8228cfe5776cbffe9fbdb5c8a41507071fa2425 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/common/mod.rs @@ -0,0 +1,1546 @@ +#![allow(dead_code)] + +pub async fn serve(world: &World) -> std::net::SocketAddr { + serve_router(world.router.clone()).await +} + +pub async fn serve_router(router: axum::Router) -> std::net::SocketAddr { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve( + listener, + router.into_make_service_with_connect_info::(), + ) + .await + .unwrap(); + }); + addr +} + +fn knot_multikey(world: &World) -> ActorId { + let signer = world.state.secrets.signer(&world.state.knot_did).unwrap(); + let sec1 = knot_runtime::Signer::public_key(&signer) + .as_bytes() + .to_vec(); + ActorId::new(knot_types::crypto::multikey(0xe7, &sec1)).unwrap() +} + +pub async fn serve_local_plc(world: &World) -> std::net::SocketAddr { + let multikey = knot_multikey(world); + let endpoint = world.state.knot_service_url.clone(); + serve_router(axum::Router::new().route( + "/{did}", + axum::routing::get( + move |axum::extract::Path(did): axum::extract::Path| { + let multikey = multikey.clone(); + let endpoint = endpoint.clone(); + async move { + let did = Did::new_owned(did).expect("the plc request path is a DID"); + let doc = did_doc(&did, &multikey, &endpoint, None); + let mut headers = axum::http::HeaderMap::new(); + headers.insert( + axum::http::header::CONTENT_TYPE, + axum::http::HeaderValue::from_static("application/did+json"), + ); + (headers, axum::Json(doc)) + } + }, + ), + )) + .await +} + +pub async fn serve_bob_pds( + bob_did: &Did, + attestation_rkey: &RecordRkey, + subject_uri: &AtUri, + subject_cid: &RecordCid, +) -> std::net::SocketAddr { + let subject_cid = *subject_cid; + let collection = RecordCollection::new("org.tangled.graph.attestation").unwrap(); + let uri = format!( + "at://{}/{}/{}", + bob_did.as_str(), + collection.as_str(), + attestation_rkey.as_str() + ); + let value = serde_json::json!({ + "$type": collection.as_str(), + "createdAt": "2026-09-11T00:00:00Z", + "subject": { "uri": subject_uri.as_str(), "cid": subject_cid.cid().to_string() } + }); + serve_router(axum::Router::new().route( + "/xrpc/com.atproto.repo.getRecord", + axum::routing::get(move || { + let uri = uri.clone(); + let value = value.clone(); + async move { + axum::Json(serde_json::json!({ + "uri": uri, + "cid": subject_cid.cid().to_string(), + "value": value, + })) + } + }), + )) + .await +} + +pub fn pack_edge(world: &World) -> Router { + let resolver: Arc = { + let index = Arc::clone(&world.state.index); + Arc::new(move |target: &knot_pack::RepoTarget| match target { + knot_pack::RepoTarget::Did(did) => { + knot_pack::RepoLookup::from_resolved(index.owner_of(did), |_| did.clone()) + } + knot_pack::RepoTarget::OwnerPath(owner, path) => knot_pack::RepoLookup::from_resolved( + index.resolve_clone_path(owner, path), + |found| found, + ), + }) + }; + let (write, advertisement) = knot_pack::edge_routes(knot_pack::EdgeConfig { + receive: Some(knot_xrpc::receive_advertiser(Arc::clone(&world.state))), + ..knot_pack::EdgeConfig::serving( + world.layout.clone(), + resolver, + Arc::new(knot_runtime::SystemClock), + ) + }); + advertisement + .into_router() + .merge(write) + .merge(world.router.clone()) +} + +pub async fn serve_pushes(world: &World) -> std::net::SocketAddr { + serve_router(pack_edge(world)).await +} + +pub async fn http_push( + work: &Path, + addr: std::net::SocketAddr, + did: &knot_types::RepoDid, + actor: &knot_types::AccountDid, + refspecs: &[&str], +) -> (bool, String) { + let header = format!( + "http.extraHeader=Authorization: Basic {}", + base64::engine::general_purpose::STANDARD.encode(format!( + "x-tangled-token:{}", + service_jwt(&nsid("org.tangled.git.receivePack"), actor) + )) + ); + let url = format!("http://{addr}/{did}"); + let mut args = vec![ + "-c".to_string(), + header, + "push".to_string(), + "-q".to_string(), + ]; + args.push(url); + args.extend(refspecs.iter().map(|spec| spec.to_string())); + let work = work.to_path_buf(); + tokio::task::spawn_blocking(move || { + let output = std::process::Command::new("git") + .args(&args) + .env("GIT_TERMINAL_PROMPT", "0") + .current_dir(&work) + .output() + .expect("git is available"); + ( + output.status.success(), + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ), + ) + }) + .await + .unwrap() +} + +pub fn percent_encode(value: &str) -> String { + value + .as_bytes() + .iter() + .fold(String::with_capacity(value.len()), |mut out, byte| { + match byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + true => out.push(*byte as char), + false => out.push_str(&format!("%{byte:02X}")), + } + out + }) +} + +use std::collections::BTreeSet; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::sync::Arc; + +use std::sync::atomic::{AtomicU64, Ordering}; + +use axum::Router; +use axum::body::{Body, Bytes, to_bytes}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use http::{HeaderMap, Request, StatusCode, header}; +use k256::ecdsa::signature::Signer; +use k256::ecdsa::{Signature, SigningKey}; +use tower::ServiceExt; + +use knot_atproto::Atproto; +use knot_cob::{ChangePayload, CobHome, CobStore, Evaluate}; +use knot_cobs::{ + BlocklistChange, BlocklistCob, CollaboratorsChange, CollaboratorsCob, Grant, MembersChange, + MembersCob, Policy, Registration, RegistryChange, Removal, RepoRegistryCob, register_repo, + set_repo_policy, +}; +use knot_git::{Layout, Repo}; +use knot_index::Index; +use knot_runtime::{ + FakeHttp, HttpRequest, HttpResponse, ManualClock, NetworkError, OsEntropy, UnixMicros, +}; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{ + AccountDid, ActorId, AtUri, ContributionPolicy, Did, DidRkey, KnotHostname, KnotId, + KnotServiceUrl, Nsid, ObjectFormat, Oid, OwnerDid, RecordAddress, RecordCid, RecordCollection, + RecordRkey, RepoDid, RepoName, RepoRkey, ServiceDid, UnixSeconds, +}; +use knot_xrpc::{ + ArchiveLimit, Budgets, ByteLimits, CobLocks, GlobalQuota, LimitConfig, PerActorQuota, + PreAuthLimiter, Reservations, ResponseLimit, XrpcState, +}; + +const KNOT_HOST: &str = "knot.nel.pet"; +const OWNER: &str = "did:web:olaren.dev"; +const FIXTURE_COLLECTION: &str = "org.tangled.test.record"; +const TICKET_COLLECTION: &str = "org.tangled.track.ticket"; + +pub fn knot_host() -> KnotHostname { + KnotHostname::new(KNOT_HOST).unwrap() +} + +pub fn knot_service_did() -> ServiceDid { + ServiceDid::new(format!("did:web:{KNOT_HOST}")).unwrap() +} + +pub fn owner() -> AccountDid { + AccountDid::new(OWNER).unwrap() +} + +fn owner_did() -> OwnerDid { + OwnerDid::new(OWNER).unwrap() +} + +pub fn nsid(name: &str) -> Nsid { + Nsid::new_owned(name).expect("the xrpc method is an NSID") +} + +pub fn fixture_collection() -> RecordCollection { + RecordCollection::new(FIXTURE_COLLECTION).unwrap() +} + +pub fn ticket_collection() -> RecordCollection { + RecordCollection::new(TICKET_COLLECTION).unwrap() +} + +pub fn member_collection() -> RecordCollection { + RecordCollection::new("org.tangled.knot.member").unwrap() +} + +pub fn collaborator_collection() -> RecordCollection { + RecordCollection::new("org.tangled.graph.collaborator").unwrap() +} + +pub fn grant_body( + repo: &RepoDid, + collection: &RecordCollection, + subject: &AccountDid, + created_at: &str, +) -> serde_json::Value { + serde_json::json!({ + "repo": repo, + "collection": collection, + "rkey": subject.as_str(), + "record": { + "$type": collection, + "createdAt": created_at, + }, + }) +} + +pub fn grant_delete_body( + repo: &RepoDid, + collection: &RecordCollection, + subject: &AccountDid, +) -> serde_json::Value { + serde_json::json!({ + "repo": repo, + "collection": collection, + "rkey": subject.as_str(), + }) +} + +pub fn blob_cid_of(value: &serde_json::Value) -> knot_types::BlobCid { + let sent = value.as_str().expect("a blob CID is a string"); + knot_types::BlobCid::new(cid::Cid::try_from(sent).expect("the CID parses")) + .expect("the CID is raw sha2-256") +} + +pub fn record_cid_of(value: &serde_json::Value) -> RecordCid { + let sent = value.as_str().expect("a record CID is a string"); + RecordCid::new(cid::Cid::try_from(sent).expect("the CID parses")) + .expect("the CID is dag-cbor sha2-256") +} + +pub fn listed_cids(response: &serde_json::Value) -> Vec { + response["cids"] + .as_array() + .map(|cids| cids.iter().map(blob_cid_of).collect()) + .unwrap_or_default() +} + +pub fn commit_rev_of(response: &serde_json::Value) -> jacquard_common::types::tid::Tid { + jacquard_common::types::tid::Tid::new( + response["commit"]["rev"] + .as_str() + .expect("the write response has a rev"), + ) + .expect("the rev parses as a TID") +} + +pub fn rkey_of(response: &serde_json::Value) -> RecordRkey { + RecordRkey::new( + response["uri"] + .as_str() + .expect("the write response has a URI") + .rsplit('/') + .next() + .expect("a URI ends in the rkey"), + ) + .expect("a URI's tail is an rkey") +} + +pub async fn create_ticket( + world: &World, + repo: &RepoDid, + title: &str, + body: &str, +) -> (RecordRkey, RecordCid) { + let collection = ticket_collection(); + let (status, written) = post_as( + world, + &nsid("com.atproto.repo.createRecord"), + &owner(), + &ServiceDid::new(repo.as_str()).unwrap(), + serde_json::json!({ + "repo": repo, + "collection": &collection, + "record": { + "$type": &collection, + "repo": repo, + "title": title, + "body": body, + "state": "open", + "createdAt": "2026-09-11T00:00:00Z" + } + }), + ) + .await; + assert_eq!(status, StatusCode::OK, "{written}"); + (rkey_of(&written), record_cid_of(&written["cid"])) +} + +pub async fn create_patch( + world: &World, + repo: &RepoDid, + ticket: &RecordRkey, + base: &Oid, + head: &Oid, +) -> (RecordRkey, RecordCid) { + let (status, written) = post_as( + world, + &nsid("org.tangled.review.submitPatch"), + &owner(), + &ServiceDid::new(repo.as_str()).unwrap(), + serde_json::json!({ + "repo": repo, + "patch": { + "ticket": ticket, + "base": { "oid": base.to_hex() }, + "head": { "oid": head.to_hex() }, + "sourceRepo": repo + } + }), + ) + .await; + assert_eq!(status, StatusCode::OK, "{written}"); + (rkey_of(&written), record_cid_of(&written["cid"])) +} + +#[derive(serde::Serialize)] +struct FixtureRecord<'a> { + #[serde(rename = "$type")] + record_type: &'a RecordCollection, + #[serde(rename = "createdAt")] + created_at: String, + #[serde(rename = "x-tngl-editor")] + editor: &'a AccountDid, +} + +pub fn fixture_record(created_at: UnixSeconds) -> knot_types::RecordBody { + let editor = owner(); + knot_types::RecordBody::new( + serde_ipld_dagcbor::to_vec(&FixtureRecord { + record_type: &fixture_collection(), + created_at: chrono::DateTime::from_timestamp(created_at.get(), 0) + .unwrap() + .to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + editor: &editor, + }) + .unwrap(), + ) + .unwrap() +} + +pub type Responder = Box Result + Send + Sync>; + +pub struct World { + _dir: tempfile::TempDir, + // keeps the import runner's shutdown sender alive for as long as the world is + _shutdown: tokio::sync::watch::Sender, + pub layout: Layout, + pub lfs_dir: std::path::PathBuf, + pub router: Router, + pub state: Arc, Arc>>, + clock: Arc, + plc_pds: Arc>>, + plc_knots: Arc>>, +} + +struct WorldSetup { + admins: BTreeSet, + policy: knot_types::ContributionPolicy, + legacy_admin_secret: Option, + git_http: Option>, + liar: Option, + proxy_trust: knot_types::ProxyTrust, +} + +impl Default for WorldSetup { + fn default() -> Self { + Self { + admins: BTreeSet::new(), + policy: knot_types::ContributionPolicy::Anyone, + legacy_admin_secret: None, + git_http: None, + liar: None, + proxy_trust: knot_types::ProxyTrust::default(), + } + } +} + +impl World { + pub fn publish_plc_pds(&self, did: &Did, pds: &KnotServiceUrl) { + self.plc_pds + .lock() + .unwrap() + .insert(did.clone(), pds.clone()); + } + + pub fn publish_plc_knot(&self, did: &str, knot: &str) { + self.plc_knots + .lock() + .unwrap() + .insert(Did::new_owned(did).unwrap(), knot.to_string()); + } + + pub fn advance(&self, delta: std::time::Duration) { + self.clock.advance(delta); + } + + pub fn new() -> Self { + Self::build(true, ByteLimits::default(), ObjectFormat::SHA1) + } + + pub fn unshed() -> Self { + Self::build_with_limits( + true, + ByteLimits::default(), + ObjectFormat::SHA1, + LimitConfig::unmetered(), + ) + } + + pub fn sha256() -> Self { + Self::build(true, ByteLimits::default(), ObjectFormat::SHA256) + } + + pub fn warming() -> Self { + Self::build(false, ByteLimits::default(), ObjectFormat::SHA1) + } + + pub fn with_response_limit(response: ResponseLimit) -> Self { + Self::build( + true, + ByteLimits { + response, + ..ByteLimits::default() + }, + ObjectFormat::SHA1, + ) + } + + pub fn with_archive_limit(archive: ArchiveLimit) -> Self { + Self::build( + true, + ByteLimits { + archive, + ..ByteLimits::default() + }, + ObjectFormat::SHA1, + ) + } + + fn tailored(setup: WorldSetup) -> Self { + Self::assemble( + true, + ByteLimits::default(), + ObjectFormat::SHA1, + LimitConfig::default(), + setup, + ) + } + + pub fn administered() -> Self { + Self::tailored(WorldSetup { + admins: BTreeSet::from([owner()]), + ..WorldSetup::default() + }) + } + + pub fn fostering(secret: &str) -> Self { + Self::tailored(WorldSetup { + legacy_admin_secret: knot_xrpc::legacy_admin::LegacyAdminSecret::new(secret).ok(), + ..WorldSetup::default() + }) + } + + pub fn fostering_with_grace(secret: &str) -> Self { + let limits = knot_runtime::HttpLimits { + block_private_addresses: false, + ..knot_runtime::HttpLimits::default() + }; + Self::tailored(WorldSetup { + legacy_admin_secret: knot_xrpc::legacy_admin::LegacyAdminSecret::new(secret).ok(), + git_http: Some(Arc::new(knot_runtime::ReqwestHttp::new(limits).unwrap())), + ..WorldSetup::default() + }) + } + + pub fn fostering_liar( + secret: knot_xrpc::legacy_admin::LegacyAdminSecret, + liar: knot_xrpc::foster::LiarHydrant, + ) -> Self { + let limits = knot_runtime::HttpLimits { + block_private_addresses: false, + ..knot_runtime::HttpLimits::default() + }; + Self::tailored(WorldSetup { + legacy_admin_secret: Some(secret), + git_http: Some(Arc::new(knot_runtime::ReqwestHttp::new(limits).unwrap())), + liar: Some(liar), + ..WorldSetup::default() + }) + } + + pub fn pulling(git_http: Arc) -> Self { + Self::tailored(WorldSetup { + admins: BTreeSet::from([owner()]), + git_http: Some(git_http), + ..WorldSetup::default() + }) + } + + pub fn behind_proxy() -> Self { + Self::tailored(WorldSetup { + proxy_trust: knot_types::ProxyTrust::new( + Some(http::HeaderName::from_static("x-forwarded-for")), + knot_types::TrustedProxies::default(), + ), + ..WorldSetup::default() + }) + } + + pub fn under(policy: knot_types::ContributionPolicy) -> Self { + Self::tailored(WorldSetup { + policy, + ..WorldSetup::default() + }) + } + + fn build(rebuilt: bool, byte_limits: ByteLimits, object_format: ObjectFormat) -> Self { + Self::build_with_limits(rebuilt, byte_limits, object_format, LimitConfig::default()) + } + + fn build_with_limits( + rebuilt: bool, + byte_limits: ByteLimits, + object_format: ObjectFormat, + limits: LimitConfig, + ) -> Self { + Self::assemble( + rebuilt, + byte_limits, + object_format, + limits, + WorldSetup::default(), + ) + } + + fn assemble( + rebuilt: bool, + byte_limits: ByteLimits, + object_format: ObjectFormat, + limits: LimitConfig, + setup: WorldSetup, + ) -> Self { + let WorldSetup { + admins, + policy, + legacy_admin_secret, + git_http, + liar, + proxy_trust, + } = setup; + let dir = tempfile::tempdir().unwrap(); + let scan_path = dir.path().join("repos"); + std::fs::create_dir_all(&scan_path).unwrap(); + let knot = KnotId::new(format!("did:web:{KNOT_HOST}")).unwrap(); + let layout = Layout::new(&scan_path) + .with_object_format(object_format) + .reserving_meta(&knot) + .unwrap(); + layout.bootstrap_meta(&knot).unwrap(); + let meta_path = layout.meta_path(&knot).unwrap(); + let index = Arc::new(Index::new(meta_path.clone(), layout.clone())); + if rebuilt { + index.rebuild().unwrap(); + } + + let plc_pds = Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())); + let plc_knots = Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())); + let pds_of = Arc::clone(&plc_pds); + let knots_of = Arc::clone(&plc_knots); + let responder: Responder = Box::new(move |request| { + let host = request.url.host_str().unwrap_or_default(); + let did = match host { + "plc.directory" => Some(request.url.path().trim_start_matches('/').to_string()), + host if request.url.path().ends_with("/.well-known/did.json") => { + Some(format!("did:web:{host}")) + } + _ => None, + } + .and_then(|did| Did::new_owned(did).ok()); + let body = match did { + Some(did) => { + let knot = knots_of.lock().unwrap().get(&did).cloned(); + did_doc_at( + &did, + &pds_of + .lock() + .unwrap() + .get(&did) + .cloned() + .unwrap_or_else(default_pds), + knot.as_deref(), + ) + } + None => Bytes::new(), + }; + Ok(HttpResponse { + status: StatusCode::OK, + headers: http::HeaderMap::new(), + body, + }) + }); + let clock = Arc::new(ManualClock::new(UnixMicros::new(1_000_000_000))); + let atproto = Arc::new(Atproto::new( + FakeHttp::new(responder), + Arc::clone(&clock), + knot.clone(), + knot_atproto::PlcDirectory::new(url::Url::parse("https://plc.directory/").unwrap()) + .unwrap(), + )); + let secrets = Arc::new( + SealedStore::open( + dir.path().join("keys.sealed"), + &MasterKey::new([7u8; 32]).unwrap(), + Box::new(OsEntropy), + ) + .unwrap(), + ); + secrets.ensure(&knot).unwrap(); + + let lfs_store = dir.path().join("lfs"); + let lfs = { + std::fs::create_dir_all(&lfs_store).unwrap(); + knot_xrpc::LfsWeb::new( + knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(&lfs_store), + knot_lfs::LfsSize::new(64 * 1024 * 1024), + knot_lfs::FreeSpaceFloor::new(0), + ) + .unwrap(), + 8, + ) + }; + + let (imports, import_jobs) = knot_xrpc::imports::ImportQueue::channel(); + let shutdown = tokio::sync::watch::channel(false); + let fostering = legacy_admin_secret.is_some(); + let firehose = Arc::new(knot_events::EventLog::new( + Arc::clone(&clock), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(if fostering { 64 } else { 8 }).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + )); + let outbox = knot_xrpc::Outbox::open(&meta_path, Default::default()); + let state = Arc::new(XrpcState { + imports, + handover_sessions: Arc::new(knot_xrpc::handover::HandoverSessions::default()), + layout: layout.clone(), + index, + atproto, + secrets, + entropy: Arc::new(OsEntropy), + invite_codes: Default::default(), + admins, + admission: knot_types::AdmissionPolicy::Closed, + contribution_policy: policy, + knot_repo: knot_xrpc::knot_repo(&knot).unwrap(), + knot_did: knot, + knot_hostname: KnotHostname::new(KNOT_HOST).unwrap(), + meta_path, + knot_service_url: knot_types::KnotServiceUrl::new(format!("https://{KNOT_HOST}")) + .unwrap(), + writes: Arc::new(knot_resource::SubjectLimiter::new( + knot_resource::RateLimit { + burst: knot_xrpc::Burst::literal(1_000), + refill: knot_xrpc::RefillMicros::literal(1_000), + }, + )), + social_budget: knot_resource::SocialBudgetBytes::new(1 << 28), + limiter: Arc::new(PreAuthLimiter::with_config(limits)), + cob_locks: Arc::new(CobLocks::default()), + reservations: Arc::new(Reservations::new( + knot_xrpc::ReservationTtl::new(1_000_000), + PerActorQuota::new(16), + GlobalQuota::new(16), + )), + proxy_trust, + byte_limits, + budgets: Budgets::default(), + git_http: git_http.unwrap_or_else(|| { + Arc::new(FakeHttp::new(|_request: &HttpRequest| { + Err(NetworkError::Connect( + "no git upstream is served in this test".to_string(), + )) + })) + }), + pack_limits: knot_pack::PackLimits::default(), + service_owner: AccountDid::new(OWNER).unwrap(), + legacy_admin_secret, + description: Default::default(), + subscriber_gate: Arc::new(knot_events::SubscriberGate::new( + knot_events::GlobalSubscriberLimit::new(16), + knot_events::PerPeerSubscriberLimit::new(4), + )), + maintenance: knot_maintenance::MaintenanceHandle::disabled(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + slots: knot_resource::Slots::testing(8), + lfs, + blobs: Default::default(), + catalog: Arc::new(knot_messages::Catalog::defaults()), + firehose, + rkeys: Default::default(), + outbox, + }); + // the runner lives like it does in the server, so wakes from tests are processed + tokio::spawn(knot_xrpc::imports::run( + Arc::clone(&state), + import_jobs, + shutdown.1, + )); + let did_document = knot_atproto::knot_did_document( + &state.knot_did, + &knot_runtime::Signer::public_key(&state.secrets.signer(&state.knot_did).unwrap()), + &state.knot_service_url, + ); + let router = match fostering { + true => knot_xrpc::router(Arc::clone(&state)).merge(knot_xrpc::foster::router( + Arc::clone(&state), + false, + false, + liar, + )), + false => knot_xrpc::router(Arc::clone(&state)), + } + .route( + "/.well-known/did.json", + axum::routing::get(move || { + let document = did_document.clone(); + async move { axum::Json(document) } + }), + ); + Self { + _dir: dir, + _shutdown: shutdown.0, + layout, + lfs_dir: lfs_store, + router, + state, + clock, + plc_pds, + plc_knots, + } + } + + pub fn register(&self, did: &RepoDid, name: &RepoName, rkey: &RepoRkey) { + let meta = Repo::open(&self.state.meta_path).unwrap(); + let store = CobStore::new(&meta); + let home = CobHome::from(&self.state.knot_did); + let signer = self.state.secrets.signer(&self.state.knot_did).unwrap(); + let registration = Registration { + owner: owner_did(), + rkey: rkey.clone(), + name: name.clone(), + repo: did.clone(), + created_at: UnixSeconds::new(1_000), + policy: self.state.contribution_policy, + fostered: false, + + retired_at: None, + }; + match store.list::().unwrap().as_slice() { + [] => { + store + .create( + &home, + &RegistryChange::Register(registration), + &signer, + UnixSeconds::new(1_000), + ) + .unwrap(); + } + [object] => { + register_repo( + &store, + &home, + *object, + registration, + &signer, + UnixSeconds::new(1_000), + ) + .unwrap(); + } + many => panic!("{} registry objects", many.len()), + } + self.state.index.refresh_registry().unwrap(); + } + + pub fn set_policy(&self, did: &RepoDid, policy: ContributionPolicy) { + let meta = Repo::open(&self.state.meta_path).unwrap(); + let store = CobStore::new(&meta); + let home = CobHome::from(&self.state.knot_did); + let signer = self.state.secrets.signer(&self.state.knot_did).unwrap(); + match store.list::().unwrap().as_slice() { + [] => panic!("policy is set on registries that register no repos"), + [object] => { + set_repo_policy( + &store, + &home, + *object, + Policy { + owner: owner_did(), + repo: did.clone(), + policy, + }, + &signer, + UnixSeconds::new(1_000), + ) + .unwrap(); + } + many => panic!("{} registry objects", many.len()), + } + self.state.index.refresh_registry().unwrap(); + } + + pub fn add_member(&self, subject: &AccountDid, added_by: &AccountDid, at: UnixSeconds) { + self.member(MembersChange::Add(grant(subject, added_by, at)), at); + } + + pub fn remove_member(&self, subject: &AccountDid, at: UnixSeconds) { + self.member( + MembersChange::Remove(Removal { + subject: subject.clone(), + }), + at, + ); + } + + pub fn ban(&self, subject: &AccountDid, at: UnixSeconds) { + self.on_meta::(BlocklistChange::Add(grant(subject, &owner(), at)), at); + self.state.index.refresh_blocklist().unwrap(); + } + + pub fn add_collaborator( + &self, + repo: &RepoDid, + subject: &AccountDid, + added_by: &AccountDid, + at: UnixSeconds, + ) { + let change = CollaboratorsChange::Add(grant(subject, added_by, at)); + self.collaborator(repo, change, at); + } + + pub fn remove_collaborator(&self, repo: &RepoDid, subject: &AccountDid, at: UnixSeconds) { + self.collaborator( + repo, + CollaboratorsChange::Remove(Removal { + subject: subject.clone(), + }), + at, + ); + } + + fn member(&self, change: MembersChange, at: UnixSeconds) { + self.on_meta::(change, at); + self.state.index.refresh_members().unwrap(); + } + + fn collaborator(&self, repo: &RepoDid, change: CollaboratorsChange, at: UnixSeconds) { + self.append::( + self.layout.open(repo).unwrap(), + CobHome::from(repo), + change, + at, + ); + self.state.index.refresh_collaborators(repo).unwrap(); + } + + pub fn publish_meta(&self, rkeys: &[DidRkey], at: UnixSeconds) { + let did = RepoDid::new(self.state.knot_did.as_str()).unwrap(); + let path = self.state.meta_path.clone(); + let rkeys = rkeys + .iter() + .map(|rkey| RecordRkey::Subject(rkey.clone())) + .collect::>(); + self.publish(&did, &path, &rkeys, at); + } + + pub fn publish(&self, did: &RepoDid, path: &Path, rkeys: &[RecordRkey], at: UnixSeconds) { + let now = at; + let records: std::collections::BTreeMap = rkeys + .iter() + .map(|rkey| { + ( + RecordAddress::new(fixture_collection(), rkey.clone()).mst_key(), + fixture_record(at), + ) + }) + .collect(); + let path = path.to_path_buf(); + std::thread::scope(|scope| { + scope + .spawn(move || { + let git = Repo::open(&path).unwrap(); + let signer = self.state.secrets.signer(&self.state.knot_did).unwrap(); + let mut emit = knot_xrpc::FirehoseEmit::new(&self.state.firehose, did, now); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime + .block_on(knot_record::chain::settle( + &git, + did, + std::slice::from_ref(&fixture_collection()), + &knot_record::chain::in_tree(records), + &signer, + &mut emit, + now, + )) + .unwrap(); + emit.finish().unwrap(); + }) + .join() + .unwrap(); + }); + } + + fn on_meta(&self, change: E::Change, at: UnixSeconds) + where + E::Change: knot_cob::KnotAuthored, + { + let meta = Repo::open(&self.state.meta_path).unwrap(); + self.append::(meta, CobHome::from(&self.state.knot_did), change, at); + } + + fn append(&self, git: Repo, home: CobHome, change: E::Change, at: UnixSeconds) + where + E::Change: knot_cob::KnotAuthored, + { + let store = CobStore::new(&git); + let signer = self.state.secrets.signer(&self.state.knot_did).unwrap(); + let kind = ::TYPE; + match store.list::().unwrap().as_slice() { + [] => _ = store.create(&home, &change, &signer, at).unwrap(), + [object] => _ = store.update(&home, *object, &change, &signer, at).unwrap(), + many => panic!("{} {kind} objects", many.len()), + } + } +} + +fn grant(subject: &AccountDid, added_by: &AccountDid, at: UnixSeconds) -> Grant { + Grant { + subject: subject.clone(), + added_by: added_by.clone(), + created_at: at, + } +} + +fn actor_key() -> SigningKey { + SigningKey::from_bytes(&[9u8; 32].into()).unwrap() +} + +fn did_doc( + did: &Did, + multikey: &ActorId, + endpoint: &KnotServiceUrl, + knot: Option<&str>, +) -> serde_json::Value { + let mut services = vec![serde_json::json!({"id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", "serviceEndpoint": endpoint.as_str()})]; + if let Some(knot) = knot { + services.push(serde_json::json!({"id": "#tangled_knot", + "type": "TangledKnot", "serviceEndpoint": knot})); + } + serde_json::json!({ + "id": did.as_str(), + "alsoKnownAs": [], + "verificationMethod": [{ + "id": format!("{}#atproto", did.as_str()), + "type": "Multikey", + "controller": did.as_str(), + "publicKeyMultibase": multikey.as_str() + }], + "service": services + }) +} + +fn did_doc_at(did: &Did, endpoint: &KnotServiceUrl, knot: Option<&str>) -> Bytes { + let sec1 = actor_key() + .verifying_key() + .to_encoded_point(true) + .as_bytes() + .to_vec(); + let multikey = ActorId::new(knot_types::crypto::multikey(0xe7, &sec1)).unwrap(); + let doc = did_doc(did, &multikey, endpoint, knot); + Bytes::from(serde_json::to_vec(&doc).unwrap()) +} + +fn default_pds() -> KnotServiceUrl { + KnotServiceUrl::new("https://pds.oyster.cafe").unwrap() +} + +static JTI: AtomicU64 = AtomicU64::new(0); + +pub fn service_jwt(method: &Nsid, actor: &AccountDid) -> String { + service_jwt_for(method, actor, &knot_service_did()) +} + +pub fn service_jwt_for(method: &Nsid, actor: &AccountDid, audience: &ServiceDid) -> String { + let nonce = JTI.fetch_add(1, Ordering::SeqCst); + let header = URL_SAFE_NO_PAD.encode(br#"{"alg":"ES256K","typ":"JWT"}"#); + let claims = serde_json::json!({ + "iss": actor.as_str(), + "aud": audience.as_str(), + "exp": 1_001, + "iat": 999, + "jti": format!("nonce-{nonce}"), + "lxm": method.as_str(), + }); + let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()); + let signing_input = format!("{header}.{payload}"); + let signature: Signature = actor_key().sign(signing_input.as_bytes()); + format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.to_bytes()) + ) +} + +async fn oneshot_json(world: &World, request: Request) -> (StatusCode, serde_json::Value) { + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + match body.is_empty() { + true => (status, serde_json::Value::Null), + false => (status, serde_json::from_slice(&body).unwrap()), + } +} + +pub async fn post_authed( + world: &World, + method: &Nsid, + actor: &AccountDid, + value: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + post_as(world, method, actor, &knot_service_did(), value).await +} + +pub async fn post_as( + world: &World, + method: &Nsid, + actor: &AccountDid, + audience: &ServiceDid, + value: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + let token = service_jwt_for(method, actor, audience); + let request = http::Request::builder() + .method("POST") + .uri(format!("/xrpc/{}", method.as_str())) + .header(header::CONTENT_TYPE, "application/json") + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .body(Body::from(serde_json::to_vec(&value).unwrap())) + .unwrap(); + oneshot_json(world, request).await +} + +pub fn enable_chain(world: &World, did: &RepoDid) { + let now = UnixSeconds::new(1_000); + let git = Repo::open(world.layout.repo_path(did).unwrap()).unwrap(); + let signer = world.state.secrets.signer(&world.state.knot_did).unwrap(); + let mut emit = knot_xrpc::FirehoseEmit::new(&world.state.firehose, did, now); + let chain = knot_record::chain::initialize(&git, did, &signer, &mut emit, now); + futures::executor::block_on(chain).unwrap(); + emit.finish().unwrap(); +} + +pub fn repo_did_document(world: &World, did: &RepoDid) -> serde_json::Value { + did_doc( + &Did::new_owned(did.as_str()).unwrap(), + &knot_multikey(world), + &world.state.knot_service_url, + None, + ) +} + +pub async fn post_json( + world: &World, + path: &str, + value: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + let request = http::Request::builder() + .method("POST") + .uri(path) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_vec(&value).unwrap())) + .unwrap(); + oneshot_json(world, request).await +} + +pub fn git_run(cwd: &Path, when: &str, author: (&str, &str), args: &[&str]) -> String { + let output = knot_fixtures::command_at(cwd, when) + .args(args) + .env("GIT_AUTHOR_NAME", author.0) + .env("GIT_AUTHOR_EMAIL", author.1) + .env("GIT_COMMITTER_NAME", author.0) + .env("GIT_COMMITTER_EMAIL", author.1) + .output() + .expect("git is available"); + assert!( + output.status.success(), + "git {args:?} failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().trim().to_string() +} + +pub fn sh_git_at(cwd: &Path, when: &str, args: &[&str]) -> String { + git_run(cwd, when, ("nel", "nel@oyster.cafe"), args) +} + +pub fn sh_git(cwd: &Path, args: &[&str]) -> String { + sh_git_at(cwd, "2026-06-01T12:30:00+02:00", args) +} + +pub fn commit_file(work: &Path, file: &str, contents: &[u8], message: &str, when: &str) { + let target = work.join(file); + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(target, contents).unwrap(); + sh_git_at(work, when, &["add", "-A"]); + sh_git_at(work, when, &["commit", "-q", "-m", message]); +} + +pub fn seeded(world: &World, name: &RepoName) -> (RepoDid, tempfile::TempDir) { + let did = RepoDid::new(format!("did:plc:{name}fixture")).unwrap(); + let object_format = world.layout.create(&did).unwrap().object_format(); + world.register(&did, name, &RepoRkey::new(name.as_str()).unwrap()); + let bare = world.layout.repo_path(&did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + sh_git(work, &init_args(object_format)); + commit_file( + work, + "README.md", + b"# coral\n\nhello\n", + "first", + "2026-06-01T12:30:00+02:00", + ); + commit_file( + work, + "src/main.rs", + b"fn main() {\n println!(\"reef\");\n}\n", + "add main", + "2026-06-01T12:31:00+02:00", + ); + commit_file( + work, + "logo.png", + b"\x89PNG\r\n\x1a\n0000binarybytes\x00\x01", + "add logo", + "2026-06-01T12:32:00+02:00", + ); + sh_git(work, &["tag", "lightweight"]); + sh_git_at( + work, + "2026-06-01T12:32:30+02:00", + &["tag", "-a", "v1.0.0", "-m", "release one"], + ); + commit_file( + work, + "README.md", + b"# coral\n\nhello reef\n", + "update readme", + "2026-06-01T12:33:00+02:00", + ); + sh_git( + work, + &["push", "-q", "--tags", bare.to_str().unwrap(), "main"], + ); + (did, work_dir) +} + +fn init_args(object_format: ObjectFormat) -> Vec<&'static str> { + match object_format == ObjectFormat::SHA256 { + true => vec!["init", "-q", "--object-format=sha256", "-b", "main"], + false => vec!["init", "-q", "-b", "main"], + } +} + +pub fn empty_repo(world: &World, name: &RepoName) -> (RepoDid, String, tempfile::TempDir) { + let did = RepoDid::new(format!("did:plc:{name}fixture")).unwrap(); + let object_format = world.layout.create(&did).unwrap().object_format(); + world.register(&did, name, &RepoRkey::new(name.as_str()).unwrap()); + let bare = world + .layout + .repo_path(&did) + .unwrap() + .to_str() + .unwrap() + .to_string(); + let work_dir = tempfile::tempdir().unwrap(); + sh_git(work_dir.path(), &init_args(object_format)); + (did, bare, work_dir) +} + +pub fn staged_source() -> (tempfile::TempDir, String) { + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + sh_git(work, &init_args(ObjectFormat::SHA1)); + commit_file( + work, + "README.md", + b"# boltfulboy\n\nhello\n", + "first", + "2026-06-01T12:30:00+02:00", + ); + commit_file( + work, + "src/main.rs", + b"fn main() {\n println!(\"reef\");\n}\n", + "add main", + "2026-06-01T12:31:00+02:00", + ); + sh_git_at( + work, + "2026-06-01T12:32:30+02:00", + &["tag", "-a", "v1.0.0", "-m", "release one"], + ); + let tip = sh_git(work, &["rev-parse", "HEAD"]); + (work_dir, tip) +} + +fn shell_bytes(seed: u8) -> Vec { + let stream = std::iter::successors(Some(seed as u32), |state| { + Some(state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223)) + }) + .map(|state| (state >> 16) as u8); + std::iter::once(0).chain(stream).take(4096).collect() +} + +fn at(clock: &str) -> String { + format!("2026-06-01T{clock}+02:00") +} + +pub fn seeded_feature_branch(world: &World, name: &RepoName) -> (RepoDid, Oid, Oid) { + let (did, bare, work_dir) = empty_repo(world, name); + let work = work_dir.path(); + let commits = |list: Vec<(&str, Vec, &str, &str)>| { + list.into_iter().for_each(|(file, bytes, message, minute)| { + commit_file(work, file, &bytes, message, &at(minute)) + }); + }; + commits(vec![ + ("reef.txt", b"one\ntwo\n".into(), "base", "12:30:00"), + ("shell.bin", shell_bytes(0), "add shell", "12:30:10"), + ("anchor.bin", shell_bytes(211), "add anchor", "12:30:20"), + ("hull.bin", shell_bytes(159), "add hull", "12:30:30"), + ]); + sh_git(work, &["checkout", "-q", "-b", "feature"]); + std::fs::set_permissions( + work.join("hull.bin"), + std::fs::Permissions::from_mode(0o755), + ) + .unwrap(); + sh_git_at(work, &at("12:30:40"), &["add", "-A"]); + sh_git_at( + work, + &at("12:30:40"), + &["commit", "-q", "-m", "make hull runnable"], + ); + commits(vec![ + ( + "reef.txt", + b"one\nTWO\n".into(), + "capitalize two\n\nbecause waves", + "12:31:00", + ), + ("kelp.txt", b"frond\n".into(), "add kelp", "12:31:10"), + ("shell.bin", shell_bytes(97), "reshape shell", "12:31:20"), + ("pearl.bin", vec![0, 255, 12, 0, 9], "add pearl", "12:31:30"), + ( + "deep water.bin", + shell_bytes(43), + "add deep water", + "12:31:40", + ), + ]); + std::fs::remove_file(work.join("anchor.bin")).unwrap(); + sh_git_at(work, &at("12:32:00"), &["add", "-A"]); + sh_git_at( + work, + &at("12:32:00"), + &["commit", "-q", "-m", "drop anchor"], + ); + sh_git(work, &["push", "-q", &bare, "main", "feature"]); + let tip = |name: &str| Oid::from_hex(&sh_git(work, &["rev-parse", name])).unwrap(); + (did, tip("main"), tip("feature")) +} + +pub async fn get_with_headers( + world: &World, + path_and_query: &str, + headers: HeaderMap, +) -> (StatusCode, HeaderMap, Bytes) { + let mut request = http::Request::builder() + .method("GET") + .uri(path_and_query) + .body(Body::empty()) + .unwrap(); + request.headers_mut().extend(headers); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let response_headers = response.headers().clone(); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + (status, response_headers, body) +} + +pub async fn get(world: &World, path_and_query: &str) -> (StatusCode, HeaderMap, Bytes) { + get_with_headers(world, path_and_query, HeaderMap::new()).await +} + +pub async fn post_raw( + world: &World, + path: &str, + headers: HeaderMap, + body: serde_json::Value, +) -> (StatusCode, Bytes) { + let mut request = http::Request::builder() + .method("POST") + .uri(path) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + request.headers_mut().extend(headers); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + ( + status, + to_bytes(response.into_body(), usize::MAX).await.unwrap(), + ) +} + +pub fn basic_auth(user: &str, secret: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + let encoded = base64::engine::general_purpose::STANDARD.encode(format!("{user}:{secret}")); + headers.insert( + header::AUTHORIZATION, + format!("Basic {encoded}").parse().unwrap(), + ); + headers +} + +pub async fn get_json(world: &World, path_and_query: &str) -> serde_json::Value { + let (status, _, body) = get(world, path_and_query).await; + assert_eq!( + status, + StatusCode::OK, + "GET {path_and_query} failed: {}", + String::from_utf8_lossy(&body) + ); + serde_json::from_slice(&body).unwrap() +} + +pub async fn get_error(world: &World, path_and_query: &str) -> (StatusCode, String) { + let (status, _, body) = get(world, path_and_query).await; + assert!(!status.is_success(), "GET {path_and_query} unexpectedly ok"); + let value: serde_json::Value = serde_json::from_slice(&body).unwrap(); + (status, value["error"].as_str().unwrap().to_string()) +} + +pub fn ref_list(value: &serde_json::Value, key: &str) -> Vec { + value[key] + .as_array() + .unwrap() + .iter() + .map(|entry| entry["ref"].as_str().unwrap().to_string()) + .collect() +} + +pub fn repo_dids(value: &serde_json::Value) -> Vec { + value["repos"] + .as_array() + .unwrap() + .iter() + .map(|entry| entry["did"].as_str().unwrap().to_string()) + .collect() +} + +pub async fn archive_full(world: &World, did: &RepoDid, prefix: &str) -> (String, String, Bytes) { + let (status, headers, body) = get( + world, + &format!("/xrpc/org.tangled.git.getArchive?repo={did}&ref=main&prefix={prefix}"), + ) + .await; + assert_eq!(status, StatusCode::OK); + let etag = headers + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap() + .to_string(); + let last_modified = headers + .get(header::LAST_MODIFIED) + .expect("the archive includes the commit's time for date revalidation") + .to_str() + .unwrap() + .to_string(); + (etag, last_modified, body) +} + +fn immutable_target<'a>(headers: &'a HeaderMap, origin: &KnotServiceUrl) -> &'a str { + headers[header::LINK] + .to_str() + .unwrap() + .trim_start_matches('<') + .split('>') + .next() + .unwrap() + .strip_prefix(origin.as_str()) + .expect("the immutable link points at this knot") +} + +pub async fn assert_immutable_round_trip( + world: &World, + headers: &HeaderMap, + full: &Bytes, + etag: &str, +) { + let immutable = immutable_target(headers, &world.state.knot_service_url); + + let (status, immutable_headers, immutable_body) = get(world, immutable).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + &immutable_body, full, + "following the immutable link regenerates the very bytes it was attached to" + ); + assert_eq!( + immutable_headers[header::ETAG], + etag, + "the immutable link shares the etag of the response that advertised it" + ); + assert_eq!( + immutable_target(&immutable_headers, &world.state.knot_service_url), + immutable, + "the immutable response advertises the same link, so one commit has one archive URL" + ); + assert_eq!( + immutable_headers[header::CONTENT_DISPOSITION], + headers[header::CONTENT_DISPOSITION], + "the prefix in the URL shapes the filename on both responses" + ); + + let mut conditional = HeaderMap::new(); + conditional.insert(header::IF_NONE_MATCH, etag.parse().unwrap()); + let (status, revalidated, body) = get_with_headers(world, immutable, conditional).await; + assert_eq!(status, StatusCode::NOT_MODIFIED); + assert!(body.is_empty()); + assert_eq!( + immutable_target(&revalidated, &world.state.knot_service_url), + immutable, + "the 304 advertises the immutable link too, because a proxy that only revalidates never sees the 200" + ); +} + +pub async fn assert_warming(world: &World, path: &str, expected_error: Option<&str>) { + let (status, error) = get_error(world, path).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "path {path}"); + if let Some(expected) = expected_error { + assert_eq!(error, expected, "path {path}"); + } +} + +pub async fn assert_post_rejected( + world: &World, + method: &Nsid, + actor: &AccountDid, + value: serde_json::Value, +) { + let (status, body) = post_authed(world, method, actor, value).await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "{}: {body}", + method.as_str() + ); + assert_eq!(body["error"], "InvalidRequest", "{}", method.as_str()); +} diff --git a/knot2/crates/knot-xrpc/tests/grant_records.rs b/knot2/crates/knot-xrpc/tests/grant_records.rs new file mode 100644 index 0000000000000000000000000000000000000000..ad7de7ca31d21a724877e2ba90745be4a1fd4002 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/grant_records.rs @@ -0,0 +1,224 @@ +mod common; + +use common::{World, nsid, owner}; +use futures::StreamExt; + +use knot_record::grant::GrantCollection; +use knot_types::{AccountDid, RecordCollection, RepoDid, RepoName}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct CommitLine { + repo: String, + ops: Vec, +} + +#[derive(Deserialize)] +struct OpLine { + action: String, + path: String, +} + +type Ws = + tokio_tungstenite::WebSocketStream>; + +#[derive(Deserialize)] +struct FrameHeader { + t: String, +} + +fn commit_of(bytes: &[u8]) -> Option { + let mut read = bytes; + let header: FrameHeader = serde_ipld_dagcbor::de::from_reader_once(&mut read).ok()?; + if header.t != "#commit" { + return None; + } + serde_ipld_dagcbor::from_slice(read).ok() +} + +async fn commits(ws: &mut Ws) -> Vec { + let mut seen = Vec::new(); + while let Ok(Some(next)) = + tokio::time::timeout(std::time::Duration::from_secs(5), ws.next()).await + { + if let tokio_tungstenite::tungstenite::Message::Binary(bytes) = next.unwrap() { + seen.extend(commit_of(&bytes)); + } + } + seen +} + +fn ops(frames: &[CommitLine], repo: &RepoDid) -> Vec<(String, String)> { + frames + .iter() + .filter(|commit| commit.repo == repo.as_str()) + .flat_map(|commit| commit.ops.iter()) + .map(|op| (op.action.clone(), op.path.clone())) + .collect() +} + +async fn records_of( + world: &World, + repo: &RepoDid, + collection: &RecordCollection, +) -> Vec { + let (status, _headers, body) = common::get( + world, + format!( + "/xrpc/com.atproto.repo.listRecords?repo={}&collection={}", + common::percent_encode(repo.as_str()), + collection.as_str() + ) + .as_str(), + ) + .await; + assert_eq!(status, 200, "listRecords serves {collection} on {repo}"); + let parsed: serde_json::Value = serde_json::from_slice(&body).unwrap(); + parsed["records"].as_array().cloned().unwrap_or_default() +} + +async fn offer_member( + world: &World, + subject: &AccountDid, +) -> (http::StatusCode, serde_json::Value) { + common::post_authed( + world, + &nsid("com.atproto.repo.createRecord"), + &owner(), + common::grant_body( + &world.state.knot_repo, + &GrantCollection::Member.records(), + subject, + "2026-09-11T00:00:00Z", + ), + ) + .await +} + +async fn offer_collaborator( + world: &World, + repo: &RepoDid, + subject: &AccountDid, +) -> (http::StatusCode, serde_json::Value) { + common::post_as( + world, + &nsid("com.atproto.repo.createRecord"), + &owner(), + &knot_types::ServiceDid::new(repo.as_str()).unwrap(), + common::grant_body( + repo, + &GrantCollection::Collaborator.records(), + subject, + "2026-09-11T00:00:00Z", + ), + ) + .await +} + +async fn revoke_collaborator( + world: &World, + repo: &RepoDid, + subject: &AccountDid, +) -> (http::StatusCode, serde_json::Value) { + common::post_as( + world, + &nsid("com.atproto.repo.deleteRecord"), + &owner(), + &knot_types::ServiceDid::new(repo.as_str()).unwrap(), + common::grant_delete_body(repo, &GrantCollection::Collaborator.records(), subject), + ) + .await +} + +#[tokio::test] +async fn the_offer_is_a_record_the_generic_surface_serves() { + let world = World::administered(); + let (repo, _bare, _work) = common::empty_repo(&world, &RepoName::new("squid").unwrap()); + let addr = common::serve(&world).await; + let (mut ws, _response) = tokio_tungstenite::connect_async(format!( + "ws://{addr}/xrpc/com.atproto.sync.subscribeRepos" + )) + .await + .unwrap(); + + let knot_repo = RepoDid::new(world.state.knot_did.as_str()).unwrap(); + let member = AccountDid::new("did:plc:limpet").unwrap(); + let collaborator = AccountDid::new("did:plc:whelk").unwrap(); + let member_collection = GrantCollection::Member.records(); + let collaborator_collection = GrantCollection::Collaborator.records(); + + let (status, body) = offer_member(&world, &member).await; + assert_eq!(status, 200, "createRecord served the member offer: {body}"); + let (status, body) = offer_collaborator(&world, &repo, &collaborator).await; + assert_eq!(status, 200, "createRecord served the offer: {body}"); + + let during = commits(&mut ws).await; + assert_eq!( + ops(&during, &knot_repo), + vec![("create".into(), format!("{member_collection}/{member}"))], + "the member offer emits 1 commit on the knot account: a create of the member record" + ); + assert_eq!( + ops(&during, &repo), + vec![( + "create".into(), + format!("{collaborator_collection}/{collaborator}") + )], + "the collaborator offer emits 1 commit on the repository: a create of the record" + ); + + let members = records_of(&world, &knot_repo, &member_collection).await; + assert_eq!( + members.len(), + 1, + "one member record, keyed by the offered member's DID" + ); + assert_eq!( + members[0]["uri"].as_str().unwrap(), + format!("at://{knot_repo}/{member_collection}/{member}") + ); + assert_eq!( + members[0]["value"]["$type"].as_str().unwrap(), + member_collection.as_str() + ); + assert!( + members[0]["value"]["createdAt"].as_str().is_some(), + "the record has the offer's createdAt" + ); + + let (status, _headers, body) = common::get( + &world, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={}&collection={}&rkey={}", + common::percent_encode(knot_repo.as_str()), + member_collection.as_str(), + common::percent_encode(member.as_str()) + ), + ) + .await; + assert_eq!(status, 200, "getRecord returns the member record"); + let fetched: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + fetched["value"]["$type"].as_str().unwrap(), + member_collection.as_str(), + "getRecord finds the record by its DID rkey" + ); + + let (status, _body) = revoke_collaborator(&world, &repo, &collaborator).await; + assert_eq!(status, 200, "deleteRecord revokes the collaborator"); + assert_eq!( + ops(&commits(&mut ws).await, &repo), + vec![( + "delete".into(), + format!("{collaborator_collection}/{collaborator}") + )], + "deleteRecord emits 1 commit on the repository: a delete of the collaborator record" + ); + assert_eq!( + records_of(&world, &repo, &collaborator_collection) + .await + .len(), + 0, + "listRecords returns zero collaborator records after the removal" + ); +} diff --git a/knot2/crates/knot-xrpc/tests/handover.rs b/knot2/crates/knot-xrpc/tests/handover.rs new file mode 100644 index 0000000000000000000000000000000000000000..306262334c5db8fbbb69648807d3a34cd91b1b79 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/handover.rs @@ -0,0 +1,1521 @@ +mod common; + +use std::collections::BTreeMap; +use std::sync::Arc; + +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use common::{World, nsid, owner, pack_edge, post_authed, serve_router}; +use http::StatusCode; +use knot_cob::{ChangePayload, CobHome, CobStore, KnotAuthored}; +use knot_git::RefUpdate; +use knot_runtime::{HttpLimits, ReqwestHttp, Signature, Signer as _}; +use knot_types::{ + AccountDid, Oid, RefName, RepoDid, RepoName, RepoRkey, ServiceDid, Tid, UnixSeconds, +}; +use parking_lot::Mutex; +use tokio_util::sync::CancellationToken; + +const FOSTER_SECRET: &str = "foster-pearl"; +const FOSTER_OWNER: &str = "did:plc:boltfulboy"; +const REPO: &str = "did:plc:feralpearl"; +const COLLECTION: &str = "org.tangled.track.ticket"; +const RKEY: &str = "1"; +const CREATED_AT: &str = "1970-01-01T00:18:20Z"; +const ACK_PATH: &str = "/xrpc/org.tangled.temp.server.ackHandover"; +const HANDOVER: &str = "org.tangled.temp.server.handoverRepo"; +const CANCEL: &str = "org.tangled.temp.server.cancelHandover"; + +#[derive(Clone)] +struct AckSink { + bodies: Arc>>, + refuse_first: usize, +} + +impl AckSink { + fn len(&self) -> usize { + self.bodies.lock().len() + } +} + +fn ok_sink() -> AckSink { + AckSink { + bodies: Arc::new(Mutex::new(Vec::new())), + refuse_first: 0, + } +} + +fn ack_edge(sink: AckSink) -> axum::Router { + axum::Router::new().route( + ACK_PATH, + axum::routing::post(move |body: axum::body::Bytes| async move { + let posted: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let seen = sink.len(); + sink.bodies.lock().push(posted); + let status = if sink.refuse_first > seen { + StatusCode::INTERNAL_SERVER_ERROR + } else { + StatusCode::OK + }; + (status, "{}") + }), + ) +} + +fn did_edge(document: serde_json::Value) -> axum::Router { + axum::Router::new().route( + "/.well-known/did.json", + axum::routing::get(move || async move { axum::Json(document.clone()) }), + ) +} + +async fn serve_foster(world: &World, sink: AckSink) -> std::net::SocketAddr { + serve_router(ack_edge(sink).fallback_service(pack_edge(world))).await +} + +fn latest_commit_edge(rev: Tid) -> axum::Router { + axum::Router::new().route( + "/xrpc/com.atproto.sync.getLatestCommit", + axum::routing::get(move || async move { + axum::Json(serde_json::json!({ + "cid": "bafyreigbotv7hck4dnibcnf7oqny3je5yl7mlmoynqunmxm4naq6vcl5e", + "rev": rev.to_string(), + })) + }), + ) +} + +async fn serve_foster_with_rev(world: &World, sink: AckSink, rev: Tid) -> std::net::SocketAddr { + serve_router( + ack_edge(sink) + .merge(latest_commit_edge(rev)) + .fallback_service(pack_edge(world)), + ) + .await +} + +async fn serve_stuck_foster(world: &World, sink: AckSink) -> std::net::SocketAddr { + serve_router(ack_edge(sink).fallback_service( + pack_edge(world).layer(axum::middleware::from_fn( + |request: axum::http::Request, + next: axum::middleware::Next| async move { + let listing = request.uri().path().ends_with("com.atproto.repo.listRecords"); + let response = next.run(request).await; + if !listing { + return response; + } + let (mut parts, body) = response.into_parts(); + let bytes = axum::body::to_bytes(body, 1 << 20).await.unwrap(); + let mut page: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + page["cursor"] = serde_json::json!("0"); + parts.headers.remove(http::header::CONTENT_LENGTH); + axum::response::Response::from_parts( + parts, + axum::body::Body::from(serde_json::to_vec(&page).unwrap()), + ) + }, + )), + )) + .await +} + +fn fostering() -> World { + World::fostering(FOSTER_SECRET) +} + +fn upgrading() -> World { + let outbound = ReqwestHttp::new(HttpLimits { + block_private_addresses: false, + ..HttpLimits::default() + }) + .unwrap(); + World::pulling(Arc::new(outbound)) +} + +fn repo_did() -> RepoDid { + RepoDid::new(REPO).unwrap() +} + +fn ticket_record(text: &str) -> serde_json::Value { + serde_json::json!({ + "collection": COLLECTION, + "rkey": RKEY, + "author": FOSTER_OWNER, + "createdAt": 1100, + "record": { + "$type": COLLECTION, + "title": {"$type": "org.tangled.markup.markdown#inline", "text": text}, + "state": "open", + "createdAt": CREATED_AT, + "x-tngl-editor": FOSTER_OWNER, + }, + }) +} + +async fn post_seed(world: &World, records: serde_json::Value) -> (StatusCode, serde_json::Value) { + let (status, body) = common::post_raw( + world, + "/xrpc/org.tangled.temp.server.seedRepo", + common::basic_auth("admin", FOSTER_SECRET), + serde_json::json!({ + "repoDid": REPO, + "manifest": {"$type": "org.tangled.repo.manifest", + "declaration": { "owner": FOSTER_OWNER, "name": "feralpearl" }}, + "records": records, + }), + ) + .await; + let parsed = serde_json::from_slice(&body).unwrap_or(serde_json::Value::Null); + (status, parsed) +} + +async fn seed_foster_record(world: &World) { + let origin_addr = serve_router(fake_origin_edge()).await; + world.publish_plc_knot(REPO, &format!("http://{origin_addr}")); + let (status, body) = + post_seed(world, serde_json::json!([ticket_record("a feral pearl")])).await; + assert_eq!(status, StatusCode::OK, "seed failed: {body}"); +} + +fn host_locally(world: &World) { + let repo = repo_did(); + world.layout.create(&repo).unwrap(); + world.register( + &repo, + &RepoName::new("feralpearl").unwrap(), + &RepoRkey::new("feralpearl").unwrap(), + ); +} + +fn local_knot() -> World { + let upgrade = upgrading(); + host_locally(&upgrade); + upgrade +} + +fn fake_origin_edge() -> axum::Router { + axum::Router::new().route( + "/xrpc/org.tangled.knot.describeKnot", + axum::routing::get(|| async { + axum::Json(serde_json::json!({"did": "did:web:origin.test", "capabilities": []})) + }), + ) +} + +async fn handover_pair( + records: serde_json::Value, + refuse_first: usize, +) -> (World, std::net::SocketAddr, AckSink, World) { + handover_pair_with_rev(records, refuse_first, None).await +} + +async fn handover_pair_with_rev( + records: serde_json::Value, + refuse_first: usize, + foster_rev: Option, +) -> (World, std::net::SocketAddr, AckSink, World) { + let foster = fostering(); + let origin_addr = serve_router(fake_origin_edge()).await; + foster.publish_plc_knot(REPO, &format!("http://{origin_addr}")); + let (status, body) = post_seed(&foster, records).await; + assert_eq!(status, StatusCode::OK, "seed failed: {body}"); + let sink = AckSink { + refuse_first, + ..ok_sink() + }; + let foster_addr = match foster_rev { + Some(rev) => serve_foster_with_rev(&foster, sink.clone(), rev).await, + None => serve_foster(&foster, sink.clone()).await, + }; + let upgrade = upgrading(); + host_locally(&upgrade); + (foster, foster_addr, sink, upgrade) +} + +async fn until(world: &World, ready: impl Fn(&World) -> bool) { + for _ in 0..400 { + if ready(world) { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + panic!( + "the handover never reached the expected state: {}", + sidecar(world) + ); +} + +async fn settled(world: &World) -> serde_json::Value { + until(world, |world| { + let done = !sidecar_path(world).exists(); + let idle = world.state.handover_sessions.acquire(&repo_did()).is_some(); + done || (!sidecar(world)["error"].is_null() && idle) + }) + .await; + match sidecar_path(world).exists() { + false => serde_json::json!({"state": "done"}), + true => serde_json::json!({"state": "running", "error": sidecar(world)["error"]}), + } +} + +async fn post_trigger( + world: &World, + actor: &AccountDid, + foster_addr: std::net::SocketAddr, +) -> (StatusCode, serde_json::Value) { + post_authed( + world, + &nsid(HANDOVER), + actor, + serde_json::json!({ + "repoDid": REPO, + "fosterUrl": format!("http://{foster_addr}"), + }), + ) + .await +} + +async fn trigger(world: &World, foster_addr: std::net::SocketAddr) -> serde_json::Value { + let (status, _) = post_trigger(world, &owner(), foster_addr).await; + assert_eq!(status, StatusCode::OK, "the trigger route takes the job"); + settled(world).await +} + +async fn cancel(world: &World, actor: &AccountDid) -> (StatusCode, serde_json::Value) { + post_authed( + world, + &nsid(CANCEL), + actor, + serde_json::json!({ "repoDid": REPO }), + ) + .await +} + +fn sidecar_path(world: &World) -> std::path::PathBuf { + world + .layout + .repo_path(&repo_did()) + .unwrap() + .join("tangled.handover.json") +} + +fn sidecar(world: &World) -> serde_json::Value { + serde_json::from_slice(&std::fs::read(sidecar_path(world)).unwrap()).unwrap() +} + +fn assert_pending(world: &World) { + assert!(sidecar_path(world).exists(), "the failed job stays pending"); +} + +fn assert_retired(world: &World) { + assert!( + !sidecar_path(world).exists(), + "a finished job leaves no sidecar" + ); +} + +fn local_cob_refs(world: &World) -> BTreeMap { + let repo = repo_did(); + let git = world.layout.open(&repo).unwrap(); + git.references() + .unwrap() + .into_iter() + .filter(|record| record.name.as_str().starts_with("refs/cobs/")) + .map(|record| (record.name.as_str().to_owned(), record.target.to_string())) + .collect() +} + +async fn assert_cid_parity(foster: &World, upgrade: &World, collection: &str, rkey: &str) { + let path = + format!("/xrpc/com.atproto.repo.getRecord?repo={REPO}&collection={collection}&rkey={rkey}"); + let fetched = common::get_json(foster, &path).await; + let local = common::get_json(upgrade, &path).await; + assert_eq!( + local["cid"], fetched["cid"], + "the {rkey} record serves the foster knot's last cid" + ); + assert_eq!(local["value"], fetched["value"]); +} + +#[derive(serde::Serialize)] +struct AckSignedView<'a> { + context: &'a str, + #[serde(rename = "repoDid")] + repo_did: &'a str, + records: u64, + #[serde(rename = "ackedAt")] + acked_at: i64, +} + +fn ack_signature_checks_out(world: &World, ack: &serde_json::Value) -> bool { + let view = AckSignedView { + context: "org.tangled.temp.handover.ack.v1", + repo_did: ack["repoDid"].as_str().unwrap(), + records: ack["records"].as_u64().unwrap(), + acked_at: ack["ackedAt"].as_i64().unwrap(), + }; + let signed = serde_ipld_dagcbor::to_vec(&view).unwrap(); + let signature = Signature::from_bytes( + URL_SAFE_NO_PAD + .decode(ack["signature"].as_str().unwrap()) + .unwrap(), + ); + let public = world + .state + .secrets + .public_key(&world.state.knot_did) + .unwrap(); + knot_runtime::verify(&public, &signed, &signature) +} + +#[tokio::test] +async fn an_upgrading_knot_pulls_its_records_back_and_goes_green() { + let (foster, foster_addr, sink, upgrade) = + handover_pair(serde_json::json!([ticket_record("a feral pearl")]), 1).await; + let outcome = trigger(&upgrade, foster_addr).await; + + assert_eq!(outcome["state"], "done", "handover outcome: {outcome}"); + assert_eq!( + sink.bodies.lock()[0]["records"], + 1, + "one seeded record comes home" + ); + assert_retired(&upgrade); + assert_cid_parity(&foster, &upgrade, COLLECTION, RKEY).await; + assert!( + !local_cob_refs(&upgrade).is_empty(), + "the cob refs came over with the objects" + ); + assert_eq!( + sink.len(), + 2, + "a transiently rejected ack is retried in the session, which still ends green" + ); + let ack = &sink.bodies.lock()[0]; + assert_eq!(ack["repoDid"], REPO); + assert_eq!(ack["records"], 1); + assert!( + ack_signature_checks_out(&upgrade, ack), + "the ack is signed by the upgrading knot's own key" + ); +} + +#[tokio::test] +async fn a_foster_that_repeats_its_listrecords_page_cant_loop_the_confirm() { + let foster = fostering(); + let origin_addr = serve_router(fake_origin_edge()).await; + foster.publish_plc_knot(REPO, &format!("http://{origin_addr}")); + let (status, body) = + post_seed(&foster, serde_json::json!([ticket_record("a feral pearl")])).await; + assert_eq!(status, StatusCode::OK, "seed failed: {body}"); + let foster_addr = serve_stuck_foster(&foster, ok_sink()).await; + let upgrade = local_knot(); + let outcome = trigger(&upgrade, foster_addr).await; + + assert_eq!(outcome["state"], "running", "{outcome}"); + assert!( + outcome["error"] + .as_str() + .unwrap() + .contains("keeps returning the same page"), + "the repeating page has to end the session, not loop it: {outcome}" + ); + assert_pending(&upgrade); +} + +#[tokio::test] +async fn a_foster_key_that_didnt_sign_the_graphs_stops_the_handover() { + let foster = fostering(); + seed_foster_record(&foster).await; + let sink = ok_sink(); + let knot = foster.state.knot_did.clone(); + let service = foster.state.knot_service_url.clone(); + let stranger = knot_runtime::K256Signer::generate(&knot_runtime::OsEntropy); + let lying = knot_atproto::knot_did_document(&knot, &stranger.public_key(), &service); + let addr = serve_router(did_edge(lying).fallback_service(pack_edge(&foster))).await; + + let upgrade = local_knot(); + let outcome = trigger(&upgrade, addr).await; + + assert_eq!( + outcome["state"], "running", + "a lying did document can't finish the handover: {outcome}" + ); + assert!( + outcome["error"] + .as_str() + .unwrap() + .contains("isn't validly signed"), + "the failure points at the unverified change: {outcome}" + ); + assert_pending(&upgrade); + assert!( + local_cob_refs(&upgrade).is_empty(), + "unverified graphs never touch the local refs" + ); + assert_eq!(sink.len(), 0, "no ack without green"); +} + +#[tokio::test] +async fn a_crashed_handover_replays_to_green() { + let (foster, foster_addr, sink, upgrade) = + handover_pair(serde_json::json!([ticket_record("a feral pearl")]), 3).await; + let first = trigger(&upgrade, foster_addr).await; + assert_eq!( + first["state"], "running", + "the refused ack leaves it pending" + ); + assert_pending(&upgrade); + + let second = trigger(&upgrade, foster_addr).await; + assert_eq!(second["state"], "done", "the replay finishes: {second}"); + assert_eq!(sink.bodies.lock()[3]["records"], 1); + assert_retired(&upgrade); + assert_cid_parity(&foster, &upgrade, COLLECTION, RKEY).await; + assert_eq!(sink.len(), 4, "the ack is resent, not remembered"); + assert!( + sink.bodies + .lock() + .iter() + .all(|ack| ack_signature_checks_out(&upgrade, ack)), + "every ack is signed by the upgrading knot's key" + ); +} + +#[tokio::test] +async fn only_a_knot_admin_may_trigger_a_handover() { + let foster = fostering(); + let foster_addr = serve_foster(&foster, ok_sink()).await; + let upgrade = local_knot(); + let stranger = AccountDid::new("did:plc:visitor").unwrap(); + let (status, _) = post_trigger(&upgrade, &stranger, foster_addr).await; + assert_eq!(status, StatusCode::FORBIDDEN); + assert!( + !sidecar_path(&upgrade).exists(), + "a refused trigger stages no job" + ); +} + +#[tokio::test] +async fn a_repo_the_foster_knot_doesnt_host_settles_without_an_ack() { + let foster = fostering(); + let sink = ok_sink(); + let foster_addr = serve_foster(&foster, sink.clone()).await; + let upgrade = local_knot(); + let outcome = trigger(&upgrade, foster_addr).await; + assert_eq!(outcome["state"], "done", "{outcome}"); + assert_retired(&upgrade); + assert!( + upgrade + .layout + .repo_path(&repo_did()) + .unwrap() + .join("tangled.handover.done") + .exists(), + "an real, non-shadowed repo is happily in the right place forever, like never queried for again" + ); + assert_eq!( + sink.len(), + 0, + "lack of records returned back to us, so we don't ack" + ); + let git = upgrade.layout.open(&repo_did()).unwrap(); + assert_eq!( + CobStore::new(&git).cobs().unwrap().len(), + 0, + "no cobs were invented on the local side" + ); +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct Stranger { + note: String, +} + +impl ChangePayload for Stranger { + const TYPE: &'static str = "org.tangled.test.stranger"; +} + +impl KnotAuthored for Stranger {} + +fn plant_stranger_cob(world: &World) { + let repo = repo_did(); + let git = world.layout.open(&repo).unwrap(); + let store = CobStore::new(&git); + let home = CobHome::from(&repo); + let signer = world.state.secrets.signer(&world.state.knot_did).unwrap(); + store + .create( + &home, + &Stranger { + note: "an odd barnacle".to_string(), + }, + &signer, + UnixSeconds::new(1100), + ) + .unwrap(); +} + +#[tokio::test] +async fn a_crashed_handover_replays_on_boot() { + let (_foster, foster_addr, sink, upgrade) = + handover_pair(serde_json::json!([ticket_record("a feral pearl")]), 3).await; + let first = trigger(&upgrade, foster_addr).await; + assert_eq!( + first["state"], "running", + "the refused ack leaves it pending" + ); + + let stray = RepoDid::new("did:plc:straypearl").unwrap(); + upgrade.layout.create(&stray).unwrap(); + let stray_sidecar = upgrade + .layout + .repo_path(&stray) + .unwrap() + .join("tangled.handover.json"); + std::fs::write(&stray_sidecar, b"{").unwrap(); + assert_eq!( + knot_xrpc::handover::recover(&upgrade.state), + vec![repo_did()], + "recover finds the running job in the repo dir and skips the undecodable one" + ); + + knot_xrpc::handover::run_boot( + Arc::clone(&upgrade.state), + CancellationToken::new(), + None, + 4, + ) + .await; + + assert_retired(&upgrade); + assert_eq!(sink.bodies.lock()[3]["records"], 1); + assert!(knot_xrpc::handover::recover(&upgrade.state).is_empty()); + assert_eq!(sink.len(), 4, "the boot replay resends the ack"); + assert!( + stray_sidecar.exists(), + "an undecodable sidecar is left alone" + ); +} + +#[tokio::test] +async fn a_stuck_handover_job_can_be_canceled_by_an_admin() { + let (_foster, foster_addr, _, upgrade) = + handover_pair(serde_json::json!([ticket_record("a feral pearl")]), 3).await; + let first = trigger(&upgrade, foster_addr).await; + assert_eq!( + first["state"], "running", + "the refused ack leaves the job stuck" + ); + + let (status, outcome) = cancel(&upgrade, &owner()).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(outcome["repoDid"], REPO); + assert_retired(&upgrade); + assert!(knot_xrpc::handover::recover(&upgrade.state).is_empty()); + + let (status, _) = cancel(&upgrade, &owner()).await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "canceling twice finds no job" + ); +} + +#[tokio::test] +async fn a_reserved_handover_returns_409_to_a_second_trigger() { + let (_foster, foster_addr, _, upgrade) = + handover_pair(serde_json::json!([ticket_record("a feral pearl")]), 0).await; + let reservation = upgrade + .state + .handover_sessions + .acquire(&repo_did()) + .unwrap(); + let (status, _) = post_trigger(&upgrade, &owner(), foster_addr).await; + assert_eq!(status, StatusCode::CONFLICT); + let (status, _) = cancel(&upgrade, &owner()).await; + assert_eq!(status, StatusCode::NOT_FOUND, "cancel finds no sidecar"); + drop(reservation); + + let outcome = trigger(&upgrade, foster_addr).await; + assert_eq!( + outcome["state"], "done", + "once the reservation drops the handover runs: {outcome}" + ); +} + +#[tokio::test] +async fn a_decoy_verification_method_cant_misdirect_the_foster_key() { + let foster = fostering(); + seed_foster_record(&foster).await; + let sink = ok_sink(); + let knot = foster.state.knot_did.clone(); + let service = foster.state.knot_service_url.clone(); + let real = foster.state.secrets.public_key(&knot).unwrap(); + let stranger = knot_runtime::K256Signer::generate(&knot_runtime::OsEntropy); + let mut document = + serde_json::to_value(knot_atproto::knot_did_document(&knot, &real, &service)).unwrap(); + let mut other = serde_json::to_value(knot_atproto::knot_did_document( + &knot, + &stranger.public_key(), + &service, + )) + .unwrap(); + other["verificationMethod"][0]["id"] = serde_json::json!(format!("{}#rotation", knot.as_str())); + let mut methods = other["verificationMethod"].as_array().unwrap().clone(); + methods.append(document["verificationMethod"].as_array_mut().unwrap()); + *document.get_mut("verificationMethod").unwrap() = serde_json::Value::Array(methods); + let addr = serve_router( + ack_edge(sink.clone()) + .fallback_service(did_edge(document).fallback_service(pack_edge(&foster))), + ) + .await; + + let upgrade = local_knot(); + let outcome = trigger(&upgrade, addr).await; + assert_eq!( + outcome["state"], "done", + "the key comes from the #atproto method, so the decoy is skipped: {outcome}" + ); + assert_eq!(sink.len(), 1); +} + +#[tokio::test] +async fn a_cob_type_with_an_unknown_history_model_blocks_the_handover() { + let (foster, foster_addr, _, upgrade) = + handover_pair(serde_json::json!([ticket_record("a feral pearl")]), 0).await; + plant_stranger_cob(&foster); + let outcome = trigger(&upgrade, foster_addr).await; + assert_eq!( + outcome["state"], "running", + "an unverifiable history model can't go green: {outcome}" + ); + assert!( + outcome["error"].as_str().unwrap().contains("history model"), + "the error reports the type this knot can't vouch for: {outcome}" + ); + assert!( + local_cob_refs(&upgrade).is_empty(), + "no refs are applied while a served type can't be trusted" + ); +} + +#[tokio::test] +async fn empty_seed_still_fosters_manifest() { + let foster = fostering(); + let (status, body) = post_seed(&foster, serde_json::json!([])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["did"], REPO, "the shadow hosts the supplied repo DID"); + assert_eq!( + body["results"], + serde_json::json!([]), + "no records, no results" + ); +} + +#[tokio::test] +async fn an_empty_shadow_cant_go_green() { + let foster = fostering(); + host_locally(&foster); + let sink = ok_sink(); + let foster_addr = serve_foster(&foster, sink.clone()).await; + + let upgrade = local_knot(); + let outcome = trigger(&upgrade, foster_addr).await; + assert_eq!(outcome["state"], "running", "{outcome}"); + assert!( + outcome["error"].as_str().unwrap().contains("empty shadow"), + "the failure points at the empty shell: {outcome}" + ); + assert!(local_cob_refs(&upgrade).is_empty()); + assert_eq!(sink.len(), 0, "no ack without records"); +} + +#[tokio::test] +async fn a_social_write_onto_a_shadow_repo_is_refused() { + let foster = fostering(); + seed_foster_record(&foster).await; + let repo = repo_did(); + let collection = common::ticket_collection(); + let (status, body) = common::post_as( + &foster, + &nsid("com.atproto.repo.createRecord"), + &owner(), + &ServiceDid::new(repo.as_str()).unwrap(), + serde_json::json!({ + "repo": repo.as_str(), + "collection": collection.as_str(), + "record": { + "$type": collection.as_str(), + "repo": repo.as_str(), + "title": "stray", + "body": "a social write onto a shadow repo", + "state": "open", + "createdAt": "2026-09-11T00:00:00Z" + } + }), + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN, "{body}"); + assert!( + body.to_string().contains("only fostered here"), + "the 403 comes from the fostered-repo check itself: {body}" + ); +} + +async fn ticketed_pair(refuse_first: usize) -> (World, std::net::SocketAddr, AckSink, World) { + handover_pair( + serde_json::json!([ticket_record("a feral pearl")]), + refuse_first, + ) + .await +} + +fn ticket_path() -> String { + format!("/xrpc/com.atproto.repo.getRecord?repo={REPO}&collection={COLLECTION}&rkey={RKEY}") +} + +async fn ticket_cid(world: &World) -> String { + common::get_json(world, &ticket_path()).await["cid"] + .as_str() + .unwrap() + .to_string() +} + +async fn delete_ticket(world: &World) { + let (status, body) = common::post_as( + world, + &nsid("com.atproto.repo.deleteRecord"), + &owner(), + &ServiceDid::new(REPO).unwrap(), + serde_json::json!({ + "repo": REPO, "rkey": RKEY, "collection": common::ticket_collection(), + "swapRecord": ticket_cid(world).await, + }), + ) + .await; + assert_eq!(status, StatusCode::OK, "deleteRecord failed: {body}"); +} + +async fn edit_ticket(world: &World) { + let (status, body) = common::post_as( + world, + &nsid("com.atproto.repo.putRecord"), + &owner(), + &ServiceDid::new(REPO).unwrap(), + serde_json::json!({ + "repo": REPO, "rkey": RKEY, "collection": common::ticket_collection(), + "swapRecord": ticket_cid(world).await, + "record": {"$type": COLLECTION, + "title": {"$type": "org.tangled.markup.markdown#inline", "text": "an edited pearl"}, + "state": "completed", "createdAt": CREATED_AT}, + }), + ) + .await; + assert_eq!(status, StatusCode::OK, "putRecord failed: {body}"); +} + +#[tokio::test] +async fn edit_to_pulled_ticket_succeeds() { + let (_foster, foster_addr, _, upgrade) = ticketed_pair(0).await; + let outcome = trigger(&upgrade, foster_addr).await; + assert_eq!(outcome["state"], "done", "{outcome}"); + + edit_ticket(&upgrade).await; +} + +#[tokio::test] +async fn pulled_back_repos_mint_revs_above_foster_era() { + let fostered = Tid::from_time(2_000_000_000_000_000, 0); + let (_foster, foster_addr, _, upgrade) = handover_pair_with_rev( + serde_json::json!([ticket_record("a feral pearl")]), + 0, + Some(fostered.clone()), + ) + .await; + let outcome = trigger(&upgrade, foster_addr).await; + assert_eq!(outcome["state"], "done", "{outcome}"); + + edit_ticket(&upgrade).await; + + let latest = common::get_json( + &upgrade, + &format!( + "/xrpc/com.atproto.sync.getLatestCommit?did={}", + common::percent_encode(REPO), + ), + ) + .await; + let minted: Tid = latest["rev"].as_str().unwrap().parse().unwrap(); + assert!( + minted.timestamp() > fostered.timestamp(), + "home knot minted {minted} at or below foster rev {fostered}" + ); +} + +#[tokio::test] +async fn deleted_pulled_ticket_stays_deleted_through_reconcile() { + let (_foster, foster_addr, _, upgrade) = ticketed_pair(0).await; + let outcome = trigger(&upgrade, foster_addr).await; + assert_eq!(outcome["state"], "done", "{outcome}"); + + delete_ticket(&upgrade).await; + + let state = std::sync::Arc::clone(&upgrade.state); + let surfaces = knot_xrpc::materialize::enable_surfaces; + let failed = tokio::task::spawn_blocking(move || surfaces(&state)) + .await + .unwrap(); + assert_eq!(failed, 0, "the reconcile pass after a delete stays green"); + + let (status, error) = common::get_error(&upgrade, &ticket_path()).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{error}"); + assert_eq!(error, "RecordNotFound", "the deleted record stays deleted"); +} + +#[tokio::test] +async fn delete_during_crashed_handover_doesnt_wedge_replay() { + let (_foster, foster_addr, sink, upgrade) = ticketed_pair(3).await; + let first = trigger(&upgrade, foster_addr).await; + assert_eq!( + first["state"], "running", + "the rejected ack leaves it pending" + ); + + delete_ticket(&upgrade).await; + + let second = trigger(&upgrade, foster_addr).await; + assert_eq!( + second["state"], "done", + "the replay converges past the delete: {second}" + ); + assert_eq!(sink.len(), 4, "the ack goes out once the delete settles"); + assert_retired(&upgrade); + + let (status, error) = common::get_error(&upgrade, &ticket_path()).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{error}"); + assert_eq!( + error, "RecordNotFound", + "the origin's delete outlives the handover" + ); +} + +#[tokio::test] +async fn edit_during_crashed_handover_survives_replay() { + let (foster, foster_addr, sink, upgrade) = ticketed_pair(3).await; + let first = trigger(&upgrade, foster_addr).await; + assert_eq!( + first["state"], "running", + "the rejected ack leaves it pending" + ); + + edit_ticket(&upgrade).await; + + let second = trigger(&upgrade, foster_addr).await; + assert_eq!( + second["state"], "done", + "the replay goes green over the edit: {second}" + ); + assert_eq!(sink.len(), 4, "the ack goes out once the edit settles"); + assert_retired(&upgrade); + + let fetched = common::get_json(&upgrade, &ticket_path()).await; + assert_eq!( + fetched["value"]["title"]["text"], "an edited pearl", + "the origin's edit outlives the handover" + ); + let shadow = common::get_json(&foster, &ticket_path()).await; + assert_ne!( + fetched["cid"], shadow["cid"], + "the origin serves the edit, not the shadow's version" + ); +} + +#[tokio::test] +async fn finished_handover_drops_seed_refs() { + let (foster, foster_addr, _sink, upgrade) = ticketed_pair(0).await; + let outcome = trigger(&upgrade, foster_addr).await; + assert_eq!(outcome["state"], "done", "{outcome}"); + assert_cid_parity(&foster, &upgrade, COLLECTION, RKEY).await; + + let refs = local_cob_refs(&upgrade); + assert!( + refs.keys().all(|name| !name.contains("temp.seed")), + "the transport refs go away once the handover is green: {refs:?}" + ); + assert!( + !refs.is_empty(), + "the repo's own manifest and ticket cobs stay behind" + ); +} + +#[tokio::test] +async fn a_settled_handover_that_kept_its_seeds_is_repaired() { + let (foster, foster_addr, _sink, upgrade) = ticketed_pair(0).await; + assert_eq!(trigger(&upgrade, foster_addr).await["state"], "done"); + let healthy = local_cob_refs(&upgrade); + let served = ticket_cid(&upgrade).await; + + let seeds = local_cob_refs(&foster) + .into_iter() + .filter(|(name, _)| name.contains("temp.seed")) + .map(|(name, target)| RefUpdate::Create { + name: RefName::new(&name).unwrap(), + new: Oid::from_hex(&target).unwrap(), + }); + let tickets = healthy + .iter() + .filter(|(name, _)| name.contains(COLLECTION)) + .map(|(name, target)| RefUpdate::Delete { + name: RefName::new(name).unwrap(), + old: Oid::from_hex(target).unwrap(), + }); + let broken: Vec = seeds.chain(tickets).collect(); + assert!(broken.len() >= 2, "{broken:?}"); + let git = upgrade.layout.open(&repo_did()).unwrap(); + git.update_refs(&broken).unwrap(); + + let repaired = knot_xrpc::handover::repair(&upgrade.state, &repo_did()) + .await + .unwrap(); + assert_eq!(repaired.map(|records| records.get()), Some(1)); + let refs = local_cob_refs(&upgrade); + assert!( + refs.keys().all(|name| !name.contains("temp.seed")), + "the seeds are dropped: {refs:?}" + ); + assert!( + refs.keys().any(|name| name.contains(COLLECTION)), + "the seeded ticket has its cob back: {refs:?}" + ); + assert_eq!(ticket_cid(&upgrade).await, served); + assert!( + knot_xrpc::handover::repair(&upgrade.state, &repo_did()) + .await + .unwrap() + .is_none(), + "a repaired repo has nothing left to repair" + ); +} + +fn regress_foster_refs(world: &World, prior: &BTreeMap) { + let git = world.layout.open(&repo_did()).unwrap(); + let back = git + .references() + .unwrap() + .into_iter() + .filter_map(|record| { + let name = record.name.as_str().to_owned(); + let old = prior.get(&name)?; + (*old != record.target.to_string()).then(|| RefUpdate::Update { + name: RefName::new(&name).unwrap(), + old: record.target, + new: Oid::from_hex(old).unwrap(), + }) + }) + .collect::>(); + git.update_refs(&back).unwrap(); +} + +#[tokio::test] +async fn a_regressed_foster_ref_cant_pull_the_local_ref_backwards() { + let (foster, foster_addr, _, upgrade) = + handover_pair(serde_json::json!([ticket_record("a feral pearl")]), 0).await; + assert_eq!(trigger(&upgrade, foster_addr).await["state"], "done"); + let after_first = local_cob_refs(&foster); + + let (status, body) = post_seed( + &foster, + serde_json::json!([ticket_record("a feraler pearl")]), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + let outcome = trigger(&upgrade, foster_addr).await; + assert_eq!(outcome["state"], "done", "{outcome}"); + let after_second = local_cob_refs(&foster); + assert!( + after_second != after_first, + "the reseed must move the foster knot's cob refs" + ); + + regress_foster_refs(&foster, &after_first); + std::fs::remove_file(settled_path(&upgrade, &repo_did())).unwrap(); + let outcome = trigger(&upgrade, foster_addr).await; + assert_eq!(outcome["state"], "running", "{outcome}"); + assert!( + outcome["error"] + .as_str() + .unwrap() + .contains("while the knot's graph recorded"), + "the green check catches the regressed foster ref: {outcome}" + ); + assert!( + after_second + .iter() + .filter(|(name, _)| !name.contains("temp.seed")) + .all(|(name, target)| local_cob_refs(&upgrade).get(name) == Some(target)), + "the local refs keep the newer tips" + ); +} + +fn hanging_ack_edge() -> axum::Router { + axum::Router::new().route( + ACK_PATH, + axum::routing::post(|| async { std::future::pending::<()>().await }), + ) +} + +#[tokio::test] +async fn a_canceled_live_pull_stops_and_releases_the_repo() { + let foster = fostering(); + seed_foster_record(&foster).await; + let addr = serve_router(hanging_ack_edge().fallback_service(pack_edge(&foster))).await; + let upgrade = local_knot(); + let (status, _) = post_trigger(&upgrade, &owner(), addr).await; + assert_eq!(status, StatusCode::OK); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + let (status, outcome) = cancel(&upgrade, &owner()).await; + assert_eq!(status, StatusCode::OK, "{outcome}"); + until(&upgrade, |world| { + world.state.handover_sessions.acquire(&repo_did()).is_some() + }) + .await; + assert_retired(&upgrade); +} + +fn host_extra(world: &World, did: &str, name: &str) -> RepoDid { + let repo = RepoDid::new(did).unwrap(); + world.layout.create(&repo).unwrap(); + world.register( + &repo, + &RepoName::new(name).unwrap(), + &RepoRkey::new(name).unwrap(), + ); + repo +} + +fn settled_path(world: &World, repo: &RepoDid) -> std::path::PathBuf { + world + .layout + .repo_path(repo) + .unwrap() + .join("tangled.handover.done") +} + +fn foster_knot(addr: std::net::SocketAddr) -> knot_xrpc::handover::FosterKnot { + knot_xrpc::handover::FosterKnot::new(&format!("http://{addr}")).unwrap() +} + +#[tokio::test] +async fn a_configured_foster_url_pulls_every_hosted_repo_home_and_never_asks_again() { + let (foster, foster_addr, sink, upgrade) = + handover_pair(serde_json::json!([ticket_record("a feral pearl")]), 0).await; + let unshadowed = host_extra(&upgrade, "did:plc:straypearl", "straypearl"); + + knot_xrpc::handover::run_boot( + Arc::clone(&upgrade.state), + CancellationToken::new(), + Some(foster_knot(foster_addr)), + 4, + ) + .await; + + assert!(settled_path(&upgrade, &repo_did()).exists()); + assert!( + settled_path(&upgrade, &unshadowed).exists(), + "a repo the foster knot doesn't shadow is settled without an ack" + ); + assert_eq!(sink.len(), 1); + assert_eq!(sink.bodies.lock()[0]["records"], 1); + assert_cid_parity(&foster, &upgrade, COLLECTION, RKEY).await; + assert!(knot_xrpc::handover::recover(&upgrade.state).is_empty()); + + knot_xrpc::handover::run_boot( + Arc::clone(&upgrade.state), + CancellationToken::new(), + Some(foster_knot(foster_addr)), + 4, + ) + .await; + assert_eq!( + sink.len(), + 1, + "a happily done repo is never pulled/acked a second time" + ); +} + +#[tokio::test(start_paused = true)] +async fn a_down_foster_knot_records_every_repo_and_bounds_the_replay() { + let upgrade = upgrading(); + let repos: Vec<_> = ["conch", "scallop", "abalone", "tuna", "winkle", "clam"] + .iter() + .map(|name| host_extra(&upgrade, &format!("did:plc:{name}"), name)) + .collect(); + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let dead = listener.local_addr().unwrap(); + drop(listener); + + knot_xrpc::handover::run_boot( + Arc::clone(&upgrade.state), + CancellationToken::new(), + Some(foster_knot(dead)), + 4, + ) + .await; + assert!( + repos.iter().all(|repo| { + !upgrade + .layout + .repo_path(repo) + .unwrap() + .join("tangled.handover.json") + .exists() + }), + "a foster knot that can't be reached stages no pull" + ); + + for repo in &repos { + std::fs::write( + upgrade + .layout + .repo_path(repo) + .unwrap() + .join("tangled.handover.json"), + serde_json::json!({ + "repo": repo.as_str(), + "foster": format!("http://{dead}"), + "triggered_at": 1100 + }) + .to_string(), + ) + .unwrap(); + } + assert_eq!( + knot_xrpc::handover::recover(&upgrade.state).len(), + repos.len(), + "the staged jobs decode and belong to this knot" + ); + + knot_xrpc::handover::run_boot( + Arc::clone(&upgrade.state), + CancellationToken::new(), + None, + 4, + ) + .await; + + let failed = repos + .iter() + .filter(|repo| { + std::fs::read( + upgrade + .layout + .repo_path(repo) + .unwrap() + .join("tangled.handover.json"), + ) + .map(|bytes| { + serde_json::from_slice::(&bytes).unwrap()["error"].is_string() + }) + .unwrap_or(false) + }) + .count(); + assert_eq!( + failed, 6, + "every repo records the gateway error and the retry rounds bound the boot without parking" + ); +} + +#[tokio::test(start_paused = true)] +async fn a_boot_width_as_wide_as_the_queue_attempts_every_repo_in_one_pass() { + let upgrade = upgrading(); + let repos: Vec<_> = ["conch", "scallop", "abalone", "tuna", "winkle", "clam"] + .iter() + .map(|name| host_extra(&upgrade, &format!("did:plc:{name}"), name)) + .collect(); + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let dead = listener.local_addr().unwrap(); + drop(listener); + + for repo in &repos { + std::fs::write( + upgrade + .layout + .repo_path(repo) + .unwrap() + .join("tangled.handover.json"), + serde_json::json!({ + "repo": repo.as_str(), + "foster": format!("http://{dead}"), + "triggered_at": 1100 + }) + .to_string(), + ) + .unwrap(); + } + + knot_xrpc::handover::run_boot( + Arc::clone(&upgrade.state), + CancellationToken::new(), + None, + repos.len(), + ) + .await; + + let failed = repos + .iter() + .filter(|repo| { + std::fs::read( + upgrade + .layout + .repo_path(repo) + .unwrap() + .join("tangled.handover.json"), + ) + .map(|bytes| { + serde_json::from_slice::(&bytes).unwrap()["error"].is_string() + }) + .unwrap_or(false) + }) + .count(); + assert_eq!( + failed, + repos.len(), + "a boot width as wide as the queue sweeps every repo into the first pass" + ); +} + +#[tokio::test] +async fn a_foster_url_without_the_foster_capability_stages_no_pull() { + let upgrade = upgrading(); + let hosted = host_extra(&upgrade, "did:plc:conch", "conch"); + let plain = serve_router(axum::Router::new().route( + "/xrpc/org.tangled.knot.describeKnot", + axum::routing::get(|| async { + axum::Json(serde_json::json!({ + "did": "did:web:plain.test", + "capabilities": ["knot-acl"], + })) + }), + )) + .await; + + knot_xrpc::handover::run_boot( + Arc::clone(&upgrade.state), + CancellationToken::new(), + Some(foster_knot(plain)), + 4, + ) + .await; + + assert!( + !upgrade + .layout + .repo_path(&hosted) + .unwrap() + .join("tangled.handover.json") + .exists(), + "a knot that doesn't take fostered repos stages no pull" + ); + assert!( + !settled_path(&upgrade, &hosted).exists(), + "no repo settles against a url that isn't a foster knot" + ); + assert!(knot_xrpc::handover::recover(&upgrade.state).is_empty()); +} + +#[tokio::test] +async fn a_seeded_link_and_bytes_encode_as_dag_cbor_not_as_maps() { + use ipld_core::ipld::Ipld; + const BLOB: &str = "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e"; + let record = serde_json::json!({ + "$type": COLLECTION, + "state": "open", + "image": { + "$type": "blob", + "ref": {"$link": BLOB}, + "mimeType": "image/png", + "size": 11, + }, + "nonce": {"$bytes": "cGVhcmw"}, + }); + let foster = fostering(); + let (status, body) = post_seed( + &foster, + serde_json::json!([{ + "collection": COLLECTION, + "rkey": RKEY, + "author": FOSTER_OWNER, + "createdAt": 1200, + "record": record, + }]), + ) + .await; + assert_eq!(status, StatusCode::OK, "the seed must succeed: {body}"); + assert!(body["results"][0]["error"].is_null(), "{body}"); + + let ipld = Ipld::Map(BTreeMap::from([ + ("$type".to_owned(), Ipld::String(COLLECTION.to_owned())), + ("state".to_owned(), Ipld::String("open".to_owned())), + ( + "image".to_owned(), + Ipld::Map(BTreeMap::from([ + ("$type".to_owned(), Ipld::String("blob".to_owned())), + ("ref".to_owned(), Ipld::Link(BLOB.parse().unwrap())), + ("mimeType".to_owned(), Ipld::String("image/png".to_owned())), + ("size".to_owned(), Ipld::Integer(11)), + ])), + ), + ("nonce".to_owned(), Ipld::Bytes(b"pearl".to_vec())), + ])); + let expected = knot_types::RecordBody::from_cbor(&ipld).unwrap().cid(); + + let fetched = common::get_json( + &foster, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={REPO}&collection={COLLECTION}&rkey={RKEY}" + ), + ) + .await; + assert_eq!(fetched["cid"], expected.to_string(), "{fetched}"); + assert_eq!(fetched["value"]["image"]["ref"]["$link"], BLOB); + assert_eq!(fetched["value"]["nonce"]["$bytes"], "cGVhcmw"); +} + +#[tokio::test] +async fn un_acked_shadow_will_mint_n_erase_via_admin() { + let world = fostering(); + seed_foster_record(&world).await; + assert_eq!( + world.layout.open(&repo_did()).unwrap().object_format(), + knot_types::ObjectFormat::SHA1, + "it must shadow in the layout's default format" + ); + let (status, raw) = common::post_raw( + &world, + "/xrpc/org.tangled.temp.server.reMintShadow", + common::basic_auth("admin", FOSTER_SECRET), + serde_json::json!({ "repoDid": REPO, "objectFormat": "sha256" }), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "re-mint must succeed: {}", + String::from_utf8_lossy(&raw) + ); + assert_eq!( + world.layout.open(&repo_did()).unwrap().object_format(), + knot_types::ObjectFormat::SHA256, + "shadow storage was re-minted in origin's format" + ); + let fetched = common::get_json(&world, &ticket_path()).await; + assert_eq!( + fetched["value"]["title"]["text"], "a feral pearl", + "our feral ahh records survived the re-mint" + ); + let (status, raw) = common::post_raw( + &world, + "/xrpc/org.tangled.temp.server.seedRepo", + common::basic_auth("admin", FOSTER_SECRET), + serde_json::json!({ + "repoDid": REPO, + "manifest": { + "$type": "org.tangled.repo.manifest", + "declaration": { "owner": FOSTER_OWNER, "name": "feralpearl" } + }, + "records": [], + "deletes": [{ "collection": COLLECTION, "rkey": RKEY }], + }), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "admin-erase must succeed: {}", + String::from_utf8_lossy(&raw) + ); + let body: serde_json::Value = serde_json::from_slice(&raw).unwrap_or(serde_json::Value::Null); + assert!(body["results"][0]["error"].is_null(), "{body}"); + let (status, _, gone) = common::get(&world, &ticket_path()).await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "erased record no longer served: {}", + String::from_utf8_lossy(&gone) + ); + let surfaces = std::sync::Arc::clone(&world.state); + let failed = + tokio::task::spawn_blocking(move || knot_xrpc::materialize::enable_surfaces(&surfaces)) + .await + .unwrap(); + assert_eq!(failed, 0, "boot pass reconciles state"); + let (status, _, _) = common::get(&world, &ticket_path()).await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "restart doesn't resurrect the erased record" + ); + let bounds = knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(1024).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ); + let replayed = world + .state + .firehose + .replay(knot_events::EventCursor::START, bounds); + let erased = replayed.events.iter().any(|event| { + let frame = String::from_utf8_lossy(&event.frame); + frame.contains("delete") && frame.contains(&format!("{COLLECTION}/{RKEY}")) + }); + assert!(erased, "erase announced itself as delete over firehose"); +} + +#[tokio::test] +async fn a_repo_still_moving_refuses_new_tickets_until_the_fostered_ones_are_home() { + let mut fostered = ticket_record("the fostered pearl"); + fostered["rkey"] = serde_json::json!("1"); + let (_foster, foster_addr, _, upgrade) = handover_pair(serde_json::json!([fostered]), 0).await; + let state = Arc::clone(&upgrade.state); + tokio::task::spawn_blocking(move || knot_xrpc::materialize::enable_surfaces(&state)) + .await + .unwrap(); + std::fs::write( + sidecar_path(&upgrade), + serde_json::to_vec(&serde_json::json!({ + "repo": REPO, "foster": format!("http://{foster_addr}/"), "triggered_at": 0, + })) + .unwrap(), + ) + .unwrap(); + let open = |title: &'static str| { + let upgrade = &upgrade; + async move { + common::post_as( + upgrade, + &nsid("com.atproto.repo.createRecord"), + &owner(), + &ServiceDid::new(REPO).unwrap(), + serde_json::json!({ + "repo": REPO, + "collection": COLLECTION, + "record": {"$type": COLLECTION, "repo": REPO, "title": title, "body": "", + "state": "open", "createdAt": "2026-09-11T00:00:00Z"}, + }), + ) + .await + } + }; + + let (status, body) = open("too early").await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + + let outcome = trigger(&upgrade, foster_addr).await; + assert_eq!(outcome["state"], "done", "{outcome}"); + let (status, body) = open("after the move").await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert!( + body["uri"].as_str().unwrap().ends_with("/2"), + "the new ticket numbers after the fostered one: {body}" + ); +} diff --git a/knot2/crates/knot-xrpc/tests/lfs.rs b/knot2/crates/knot-xrpc/tests/lfs.rs new file mode 100644 index 0000000000000000000000000000000000000000..7455dc1bd50b60f6346af729aee2b55e72b168da --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/lfs.rs @@ -0,0 +1,303 @@ +mod common; + +use axum::body::Body; +use http::{Request, StatusCode, header}; +use sha2::{Digest, Sha256}; +use tower::ServiceExt; + +use knot_lfs::{LfsOid, LfsStore, Pool}; +use knot_types::{RepoDid, RepoName}; + +use common::{World, empty_repo, get, owner}; + +const MEDIA: &[u8] = b"\xff\x00heavy media bytes that live outside the odb"; + +fn seeded_object(world: &World, repo: &RepoDid) -> (LfsOid, usize) { + let oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(MEDIA)); + world + .state + .lfs + .handle + .store + .put( + Pool::Lfs, + repo, + &oid, + knot_lfs::ClaimedSize::new(MEDIA.len() as u64), + &mut &MEDIA[..], + ) + .unwrap(); + (oid, MEDIA.len()) +} + +fn absent_oid() -> LfsOid { + LfsOid::from_digest(knot_types::Sha256Digest::hash(b"never uploaded anywhere")) +} + +async fn post_batch(world: &World, path: &str, body: String) -> (StatusCode, serde_json::Value) { + let request = Request::post(path) + .header(header::CONTENT_TYPE, "application/vnd.git-lfs+json") + .body(Body::from(body)) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let value = serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); + (status, value) +} + +fn batch_body(operation: &str, oids: &[(String, u64)]) -> String { + let objects: Vec = oids + .iter() + .map(|(oid, size)| serde_json::json!({"oid": oid, "size": size})) + .collect(); + serde_json::json!({ + "operation": operation, + "transfers": ["basic", "ssh"], + "objects": objects, + "hash_algo": "sha256", + }) + .to_string() +} + +#[tokio::test] +async fn the_batch_download_surface_answers_every_addressing_form() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("barnacle").unwrap()); + let (oid, size) = seeded_object(&world, &did); + let missing = absent_oid(); + + let body = batch_body( + "download", + &[ + (oid.as_str().to_string(), size as u64), + (missing.as_str().to_string(), 9), + ], + ); + let paths = [ + format!("/{did}/info/lfs/objects/batch"), + format!("/{did}.git/info/lfs/objects/batch"), + format!("/{}/barnacle/info/lfs/objects/batch", owner()), + format!("/{}/barnacle.git/info/lfs/objects/batch", owner()), + ]; + futures::future::join_all(paths.iter().map(|path| { + let world = &world; + let body = body.clone(); + let oid = oid.clone(); + async move { + let (status, json) = post_batch(world, path, body).await; + assert_eq!(status, StatusCode::OK, "batch at {path}"); + assert_eq!(json["transfer"], "basic"); + assert_eq!(json["hash_algo"], "sha256"); + let objects = json["objects"].as_array().unwrap(); + assert_eq!(objects.len(), 2); + assert_eq!(objects[0]["oid"], oid.as_str()); + assert_eq!(objects[0]["size"], size as u64); + assert_eq!(objects[0]["authenticated"], true); + let href = objects[0]["actions"]["download"]["href"].as_str().unwrap(); + assert!( + href.ends_with(&format!("/info/lfs/objects/{oid}")), + "href {href}" + ); + assert!(href.starts_with("https://"), "href {href}"); + assert!(objects[0].get("error").is_none()); + assert_eq!(objects[1]["error"]["code"], 404); + assert!(objects[1].get("actions").is_none()); + } + })) + .await; + + let upload = batch_body("upload", &[(oid.as_str().to_string(), size as u64)]); + let (status, _) = post_batch(&world, &format!("/{did}/info/lfs/objects/batch"), upload).await; + assert_eq!( + status, + StatusCode::UNAUTHORIZED, + "an unauthenticated HTTP upload batch is challenged for credentials, never answered" + ); +} + +#[tokio::test] +async fn the_object_route_streams_ranges_and_stays_anonymous() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("limpet").unwrap()); + let (oid, size) = seeded_object(&world, &did); + + let request = Request::get(format!("/{did}/info/lfs/objects/{oid}")) + .body(Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "application/octet-stream" + ); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "public, max-age=31536000, immutable" + ); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + assert_eq!(&bytes[..], MEDIA); + + let request = Request::get(format!("/{}/limpet.git/info/lfs/objects/{oid}", owner())) + .header(header::RANGE, "bytes=3-6") + .body(Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + assert_eq!(&bytes[..], &MEDIA[3..=6]); + + let request = Request::get(format!("/{did}/info/lfs/objects/{oid}")) + .header(header::IF_NONE_MATCH, format!("\"{oid}\"")) + .body(Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + assert_eq!( + response.headers()[header::ETAG], + format!("\"{oid}\"").as_str() + ); + let _ = size; +} + +#[tokio::test] +async fn missing_and_hostile_objects_get_typed_404s() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("scallop").unwrap()); + seeded_object(&world, &did); + + let absent = absent_oid(); + let cases = [ + format!("/{did}/info/lfs/objects/{absent}"), + format!("/{did}/info/lfs/objects/deadbeef"), + format!("/{did}/info/lfs/objects/..%2f..%2fetc%2fpasswd"), + format!("/did:plc:nowhere/info/lfs/objects/{absent}"), + ]; + futures::future::join_all(cases.iter().map(|path| { + let world = &world; + async move { + let request = Request::get(path).body(Body::empty()).unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND, "GET {path}"); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "application/vnd.git-lfs+json", + "GET {path}" + ); + } + })) + .await; + + let request = Request::get(format!("/{did}/info/lfs/objects/{absent}")) + .header(header::IF_NONE_MATCH, "*") + .body(Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "an absent object must 404 even when revalidated" + ); +} + +#[tokio::test] +async fn readiness_reflects_the_lfs_store() { + let world = World::new(); + let (status, _, _) = get(&world, "/xrpc/_health").await; + assert_eq!(status, StatusCode::OK, "a writable store reports ready"); + + let incoming = world.lfs_dir.join(".incoming"); + std::fs::remove_dir(&incoming).unwrap(); + let (status, _, body) = get(&world, "/xrpc/_health").await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an unwritable store reports unready: {}", + String::from_utf8_lossy(&body) + ); + + std::fs::create_dir_all(&incoming).unwrap(); + let (status, _, _) = get(&world, "/xrpc/_health").await; + assert_eq!(status, StatusCode::OK, "a recovered store reports ready"); +} + +#[tokio::test] +async fn hostile_batches_get_clean_typed_rejections() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("conch").unwrap()); + let (oid, size) = seeded_object(&world, &did); + let base = format!("/{did}/info/lfs/objects/batch"); + + let (status, _) = post_batch(&world, &base, "not json at all".to_string()).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + + let traversal = serde_json::json!({ + "operation": "download", + "objects": [{"oid": "../../../../etc/passwd", "size": 1}], + }) + .to_string(); + let (status, _) = post_batch(&world, &base, traversal).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + + let oversized_list: Vec<(String, u64)> = (0..1001) + .map(|index| { + let digest = Sha256::digest(index.to_string().as_bytes()); + ( + LfsOid::from_digest(knot_types::Sha256Digest::new(digest.into())) + .as_str() + .to_string(), + 1, + ) + }) + .collect(); + let (status, _) = post_batch(&world, &base, batch_body("download", &oversized_list)).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + + let foreign_algo = serde_json::json!({ + "operation": "download", + "objects": [{"oid": oid.as_str(), "size": size}], + "hash_algo": "sha1", + }) + .to_string(); + let (status, _) = post_batch(&world, &base, foreign_algo).await; + assert_eq!(status, StatusCode::CONFLICT); + + let no_common_transfer = serde_json::json!({ + "operation": "download", + "transfers": ["tus"], + "objects": [{"oid": oid.as_str(), "size": size}], + }) + .to_string(); + let (status, _) = post_batch(&world, &base, no_common_transfer).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + + let over_limit = format!( + r#"{{"operation":"download","objects":[],"pad":"{}"}}"#, + "a".repeat(1024 * 1024 + 1) + ); + let (status, _) = post_batch(&world, &base, over_limit).await; + assert_eq!( + status, + StatusCode::PAYLOAD_TOO_LARGE, + "a batch body over the limit is refused before buffering" + ); + + let absurd_size = serde_json::json!({ + "operation": "download", + "objects": [{"oid": oid.as_str(), "size": u64::MAX}], + }) + .to_string(); + let (status, json) = post_batch(&world, &base, absurd_size).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + json["objects"][0]["size"], size as u64, + "a download answer reports the stored size, ignoring the client's absurd claim" + ); + assert!(json["objects"][0]["actions"]["download"]["href"].is_string()); +} diff --git a/knot2/crates/knot-xrpc/tests/lfs_soak.rs b/knot2/crates/knot-xrpc/tests/lfs_soak.rs new file mode 100644 index 0000000000000000000000000000000000000000..7dfece5b734b3529abb5b458d4b02c425b8bd723 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/lfs_soak.rs @@ -0,0 +1,159 @@ +mod common; + +use axum::body::Body; +use futures::StreamExt; +use http::{Request, StatusCode, header}; +use knot_lfs::{LfsOid, LfsStore, Pool}; +use knot_types::{RepoDid, RepoName}; +use sha2::{Digest, Sha256}; +use tower::ServiceExt; + +use common::{World, empty_repo}; + +const OBJECT_BYTES: usize = 8 * 1024 * 1024; +const OBJECTS: usize = 4; +const DOWNLOADERS: usize = 12; +const ROUNDS: usize = 4; +const PAGE_BYTES: u64 = 4096; +const PEAK_CEILING: u64 = 512 * 1024 * 1024; +const GROWTH_SLACK: u64 = 64 * 1024 * 1024; + +fn rss_bytes() -> u64 { + let statm = std::fs::read_to_string("/proc/self/statm").expect("/proc/self/statm is readable"); + statm + .split_whitespace() + .nth(1) + .and_then(|pages| pages.parse::().ok()) + .map(|pages| pages * PAGE_BYTES) + .expect("statm lists the resident page count") +} + +fn incompressible(len: usize, seed: u64) -> Vec { + let mut state = seed | 1; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() +} + +fn seed_objects(world: &World, repo: &RepoDid) -> Vec { + let store = &world.state.lfs.handle.store; + (0..OBJECTS) + .map(|index| { + let body = incompressible(OBJECT_BYTES, 0x5eed_0000 + index as u64); + let oid = LfsOid::from_digest(knot_types::Sha256Digest::hash(&body)); + store + .put( + Pool::Lfs, + repo, + &oid, + knot_lfs::ClaimedSize::new(body.len() as u64), + &mut &body[..], + ) + .unwrap(); + oid + }) + .collect() +} + +async fn download(world: &World, did: &RepoDid, oid: &LfsOid, tag: &str) { + let request = Request::get(format!("/{did}/info/lfs/objects/{oid}")) + .body(Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{tag}"); + let (streamed, hasher) = response + .into_body() + .into_data_stream() + .fold( + (0u64, Sha256::new()), + |(streamed, mut hasher), chunk| async move { + let chunk = chunk.unwrap(); + hasher.update(&chunk); + (streamed + chunk.len() as u64, hasher) + }, + ) + .await; + assert_eq!(streamed, OBJECT_BYTES as u64, "{tag}"); + assert_eq!( + LfsOid::from_digest(knot_types::Sha256Digest::new(hasher.finalize().into())), + oid.clone(), + "{tag}: downloaded bytes must hash to the requested oid" + ); +} + +async fn batch(world: &World, did: &RepoDid, oids: &[LfsOid], tag: &str) { + let objects: Vec = oids + .iter() + .map(|oid| serde_json::json!({"oid": oid.as_str(), "size": OBJECT_BYTES})) + .collect(); + let body = serde_json::json!({"operation": "download", "objects": objects}).to_string(); + let request = Request::post(format!("/{did}/info/lfs/objects/batch")) + .header(header::CONTENT_TYPE, "application/vnd.git-lfs+json") + .body(Body::from(body)) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + assert_eq!( + status, + StatusCode::OK, + "{tag}: {}", + String::from_utf8_lossy(&body) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn sustained_anonymous_downloads_stay_bounded_and_leak_nothing() { + let world = World::unshed(); + let (did, _bare, _work) = empty_repo(&world, &RepoName::new("nautilus").unwrap()); + let oids = seed_objects(&world, &did); + + let storm = |round: usize| { + let world = &world; + let did = &did; + let oids = &oids; + async move { + futures::future::join_all((0..DOWNLOADERS).map(|task| { + let oid = oids[task % OBJECTS].clone(); + async move { + let tag = format!("round {round} task {task}"); + batch(world, did, oids, &tag).await; + download(world, did, &oid, &tag).await; + } + })) + .await; + } + }; + + storm(0).await; + let settled = rss_bytes(); + + let peaks: Vec = futures::stream::iter(1..ROUNDS) + .then(|round| { + let storm = &storm; + async move { + storm(round).await; + rss_bytes() + } + }) + .collect() + .await; + + let peak = peaks.iter().copied().max().unwrap_or(settled); + assert!( + peak < PEAK_CEILING, + "concurrent downloads peaked at {peak} bytes, ceiling {PEAK_CEILING}" + ); + let last = *peaks.last().unwrap_or(&settled); + assert!( + last <= settled + GROWTH_SLACK, + "rss grew from {settled} to {last} across rounds, downloads are leaking" + ); +} diff --git a/knot2/crates/knot-xrpc/tests/mint.rs b/knot2/crates/knot-xrpc/tests/mint.rs new file mode 100644 index 0000000000000000000000000000000000000000..1a447c08d0ab6b0c1de693076687f4608e5593f5 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/mint.rs @@ -0,0 +1,217 @@ +mod common; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use http::StatusCode; + +use knot_atproto::ServiceJwt; +use knot_types::service_auth::{PublicKey, parse_jwt, verify_signature}; +use knot_types::{AccountDid, DidService, Nsid, RepoDid, RepoName, ServiceDid, UnixSeconds}; + +use common::{World, empty_repo, owner, post_as, post_authed}; + +fn mint_method() -> Nsid { + Nsid::new_owned("org.tangled.repo.mintToken").expect("the mint method NSID parses") +} + +fn stranger() -> AccountDid { + AccountDid::new("did:plc:stranger").expect("the stranger DID parses") +} + +fn collaborator() -> AccountDid { + AccountDid::new("did:plc:limpet").expect("the collaborator DID parses") +} + +fn foreign_repo() -> RepoDid { + RepoDid::new("did:plc:not_hosted_anywhere").expect("the foreign repo DID parses") +} + +fn spindle() -> DidService { + DidService::new_owned("did:web:spindle.test").expect("the spindle fixture DID parses") +} + +fn secret_method() -> Nsid { + Nsid::new_owned("org.tangled.secret.addSecret").expect("the secret method NSID parses") +} + +fn mint_body(repo: &RepoDid, exp: i64) -> serde_json::Value { + serde_json::json!({ + "repo": repo, + "aud": spindle(), + "lxm": secret_method(), + "exp": exp, + }) +} + +fn mint_as( + world: &World, + repo: &RepoDid, + actor: &AccountDid, + exp: i64, +) -> (StatusCode, serde_json::Value) { + let body = mint_body(repo, exp); + futures::executor::block_on(post_as( + world, + &mint_method(), + actor, + &ServiceDid::new(repo.as_str()).unwrap(), + body, + )) +} + +fn token_in(response: &serde_json::Value) -> ServiceJwt { + ServiceJwt::new( + response["token"] + .as_str() + .expect("The mint returns a token."), + ) + .expect("This minted token has three JWT segments.") +} + +fn claims_of(token: &ServiceJwt) -> serde_json::Value { + let payload = token + .as_str() + .split('.') + .nth(1) + .expect("This token has a payload segment."); + serde_json::from_slice( + &URL_SAFE_NO_PAD + .decode(payload) + .expect("This payload is base64url."), + ) + .expect("This payload is JSON.") +} + +#[tokio::test] +async fn owner_mint_should_be_signed_by_knot_key_for_repo() { + let world = World::new(); + let (repo, _bare, _work) = empty_repo(&world, &RepoName::new("kelp").unwrap()); + + let (status, response) = mint_as(&world, &repo, &owner(), 1_030); + assert_eq!(status, StatusCode::OK, "{response}"); + let token = token_in(&response); + + let parsed = parse_jwt(token.as_str()).expect("This minted token parsez as a JWT."); + let signer = world.state.secrets.signer(&world.state.knot_did).unwrap(); + let key = PublicKey::from_k256_bytes(knot_runtime::Signer::public_key(&signer).as_bytes()) + .expect("The knot key converts to a verifying key."); + verify_signature(&parsed, &key).expect("This repo token verifies under the knot's key."); + + let claims = claims_of(&token); + assert_eq!(claims["iss"], repo.as_str()); + assert_eq!(claims["aud"], spindle().as_str()); + assert_eq!(claims["lxm"], secret_method().as_str()); + assert_eq!(claims["iat"], 1_000); + assert_eq!(claims["exp"], 1_030); + assert_eq!(claims["act"]["sub"], owner().as_str()); + assert!(!claims["jti"].as_str().unwrap().is_empty()); + + let (_, second) = mint_as(&world, &repo, &owner(), 1_030); + assert_ne!( + claims_of(&token_in(&second))["jti"], + claims["jti"], + "Every mint gets its own JTI." + ); +} + +#[tokio::test] +async fn stranger_mint_ought_to_return_403_mentioning_roster() { + let world = World::new(); + let (repo, _bare, _work) = empty_repo(&world, &RepoName::new("kelp").unwrap()); + + let (status, response) = mint_as(&world, &repo, &stranger(), 1_030); + assert_eq!(status, StatusCode::FORBIDDEN, "{response}"); + let message = response["message"] + .as_str() + .expect("This 403 has a message."); + assert!(message.contains("roster"), "{message}"); + assert!(message.contains("collaborator"), "{message}"); +} + +#[tokio::test] +async fn freshly_revoked_collaborator_must_not_be_able_to_mint() { + let world = World::new(); + let (repo, _bare, _work) = empty_repo(&world, &RepoName::new("kelp").unwrap()); + let collaborator = collaborator(); + world.add_collaborator(&repo, &collaborator, &owner(), UnixSeconds::new(1_000)); + + let (admitted, _) = mint_as(&world, &repo, &collaborator, 1_030); + assert_eq!(admitted, StatusCode::OK); + + world.remove_collaborator(&repo, &collaborator, UnixSeconds::new(1_001)); + let (status, response) = mint_as(&world, &repo, &collaborator, 1_030); + assert_eq!(status, StatusCode::FORBIDDEN, "{response}"); + assert!( + response["message"] + .as_str() + .expect("This 403 has a message.") + .contains("roster"), + "{response}" + ); +} + +#[tokio::test] +async fn out_of_bounds_exp_should_be_refused() { + let world = World::new(); + let (repo, _bare, _work) = empty_repo(&world, &RepoName::new("kelp").unwrap()); + + for (exp, want) in [(999, "future"), (1_100, "60")] { + let (status, response) = mint_as(&world, &repo, &owner(), exp); + assert_eq!(status, StatusCode::BAD_REQUEST, "{response}"); + let message = response["message"].as_str().unwrap_or_default(); + assert!(message.contains(want), "exp {exp}: {message}"); + } +} + +#[tokio::test] +async fn mint_without_lxm_ought_to_be_refused() { + let world = World::new(); + let (repo, _bare, _work) = empty_repo(&world, &RepoName::new("kelp").unwrap()); + + let body = serde_json::json!({ + "repo": repo, + "aud": spindle(), + "exp": 1_030, + }); + + let (status, response) = futures::executor::block_on(post_as( + &world, + &mint_method(), + &owner(), + &ServiceDid::new(repo.as_str()).unwrap(), + body, + )); + assert_eq!(status, StatusCode::BAD_REQUEST, "{response}"); +} + +#[tokio::test] +async fn calling_session_should_be_addressed_to_repo() { + let world = World::new(); + let (repo, _bare, _work) = empty_repo(&world, &RepoName::new("kelp").unwrap()); + + let body = mint_body(&repo, 1_030); + let (status, response) = + futures::executor::block_on(post_authed(&world, &mint_method(), &owner(), body)); + assert_eq!(status, StatusCode::UNAUTHORIZED, "{response}"); +} + +#[tokio::test] +async fn mint_for_unhosted_repo_should_return_404() { + let world = World::new(); + let _ = empty_repo(&world, &RepoName::new("kelp").unwrap()); + + let body = serde_json::json!({ + "repo": foreign_repo(), + "aud": spindle(), + "lxm": secret_method(), + "exp": 1_030, + }); + let (status, response) = futures::executor::block_on(post_as( + &world, + &mint_method(), + &owner(), + &ServiceDid::new(foreign_repo().as_str()).unwrap(), + body, + )); + assert_eq!(status, StatusCode::NOT_FOUND, "{response}"); +} diff --git a/knot2/crates/knot-xrpc/tests/policy_reconcile.rs b/knot2/crates/knot-xrpc/tests/policy_reconcile.rs new file mode 100644 index 0000000000000000000000000000000000000000..fbeaeaa06507b857a810ec3845d4c16e769719a6 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/policy_reconcile.rs @@ -0,0 +1,96 @@ +mod common; + +use common::{World, get_json, nsid, owner, post_as, post_authed}; +use knot_index::Resolved; +use knot_types::{ContributionPolicy, RepoDid, ServiceDid, UnixSeconds}; + +async fn create_repo(world: &World, name: &str) -> RepoDid { + let (status, created) = post_authed( + world, + &nsid("org.tangled.repo.createRepo"), + &owner(), + serde_json::json!({ "name": name, "defaultBranch": "main" }), + ) + .await; + assert_eq!(status, http::StatusCode::OK, "create failed: {created}"); + RepoDid::new(created["did"].as_str().unwrap()).unwrap() +} + +fn reconciled_policy(world: &World, did: &RepoDid) -> ContributionPolicy { + match world.state.index.contribution_policy(did) { + Resolved::Ready(Some(row)) => row.policy(), + Resolved::Ready(None) => panic!("registry row for {did} went missing"), + Resolved::Warming => panic!("registry projection is still warming after boot"), + } +} + +#[tokio::test] +async fn drifted_registry_rows_reopen_to_manifest_policy_at_boot() { + let world = World::new(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + let did = create_repo(&world, "anemone").await; + + world.set_policy(&did, ContributionPolicy::Collaborators); + let state = std::sync::Arc::clone(&world.state); + let failed = + tokio::task::spawn_blocking(move || knot_xrpc::materialize::enable_surfaces(&state)) + .await + .unwrap(); + assert_eq!(failed, 0, "boot pass reconciles state"); + + assert_eq!( + reconciled_policy(&world, &did), + ContributionPolicy::Anyone, + "manifests that never narrowed reopen registry rows that wandered" + ); +} + +#[tokio::test] +async fn deliberate_narrowing_in_manifests_survives_boot_reconcile() { + let world = World::new(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + let did = create_repo(&world, "conch").await; + + let live = get_json( + &world, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={did}&collection=org.tangled.repo.manifest&rkey=self" + ), + ) + .await; + let mut record = live["value"].clone(); + record["contributionPolicy"] = serde_json::json!("members"); + let (status, body) = post_as( + &world, + &nsid("com.atproto.repo.putRecord"), + &owner(), + &ServiceDid::new(did.as_str()).unwrap(), + serde_json::json!({ + "repo": did, + "collection": "org.tangled.repo.manifest", + "rkey": "self", + "record": record, + "swapRecord": live["cid"], + }), + ) + .await; + assert_eq!( + status, + http::StatusCode::OK, + "owners narrow repos through their manifests: {body}" + ); + + world.set_policy(&did, ContributionPolicy::Collaborators); + let state = std::sync::Arc::clone(&world.state); + let failed = + tokio::task::spawn_blocking(move || knot_xrpc::materialize::enable_surfaces(&state)) + .await + .unwrap(); + assert_eq!(failed, 0, "boot pass reconciles state"); + + assert_eq!( + reconciled_policy(&world, &did), + ContributionPolicy::Members, + "boot pass takes manifest's word for it and keeps owner's narrowing" + ); +} diff --git a/knot2/crates/knot-xrpc/tests/projection.rs b/knot2/crates/knot-xrpc/tests/projection.rs new file mode 100644 index 0000000000000000000000000000000000000000..cdb24bd4e9e9b97e4905dc0c40fc0c314438f28e --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/projection.rs @@ -0,0 +1,357 @@ +mod common; + +use std::collections::BTreeMap; + +use common::{ + World, commit_file, create_patch, create_ticket, get_json, http_push, nsid, owner, post_as, + post_authed, serve_pushes, sh_git, ticket_collection, +}; +use knot_index::Resolved; +use knot_record::chain; +use knot_record::git_refs; +use knot_record::grant::{self, GrantCollection}; +use knot_types::{AccountDid, MstKey, Oid, RepoDid, ServiceDid, UnixSeconds}; + +async fn assert_projection(world: &World, did: &RepoDid) { + let git = world.layout.open(did).unwrap(); + let leaves = chain::leaves(&git).await.unwrap(); + + world.state.index.refresh_social(did).unwrap(); + let entries = match world.state.index.social_entries(did) { + Resolved::Ready(entries) => entries, + Resolved::Warming => panic!("social projection must be ready"), + }; + let mut desired: BTreeMap = BTreeMap::new(); + for entry in &entries { + if let Some(serving) = entry.state.serving() { + desired.insert(entry.address.mst_key(), serving.cid()); + } + } + + for (key, body) in git_refs::desired_record(&git).unwrap() { + desired.insert(key, body.cid().cid()); + } + + world.state.index.refresh_collaborators(did).unwrap(); + let collaborators = match world.state.index.collaborator_entries(did) { + Resolved::Ready(entries) => entries, + Resolved::Warming => panic!("collaborator projection must be ready"), + }; + for (subject, entry) in &collaborators { + let key = grant::record_key(subject, GrantCollection::Collaborator).unwrap(); + let bodies = grant::bodies([(subject, entry.created_at)], GrantCollection::Collaborator); + desired.insert(key.clone(), bodies[&key].cid().cid()); + } + + assert_eq!(leaves, desired, "MST projection mismatch for {did}"); +} + +async fn create_repo(world: &World, name: &str) -> RepoDid { + let (status, created) = post_authed( + world, + &nsid("org.tangled.repo.createRepo"), + &owner(), + serde_json::json!({ "name": name, "defaultBranch": "main" }), + ) + .await; + assert_eq!(status, http::StatusCode::OK, "create failed: {created}"); + RepoDid::new(created["did"].as_str().unwrap()).unwrap() +} + +async fn push_main(world: &World, did: &RepoDid, work: &std::path::Path) { + let addr = serve_pushes(world).await; + let (ok, detail) = http_push(work, addr, did, &owner(), &["main"]).await; + assert!(ok, "push failed: {detail}"); +} + +#[tokio::test] +async fn the_full_lifecycle_keeps_the_mst_projection_exact() { + let world = World::new(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + + let did = create_repo(&world, "anemone").await; + assert_projection(&world, &did).await; + + let work = tempfile::tempdir().unwrap(); + sh_git(work.path(), &["init", "-q", "-b", "main"]); + commit_file( + work.path(), + "README.md", + b"# anemone\n", + "first", + "2026-06-01T12:30:00+02:00", + ); + push_main(&world, &did, work.path()).await; + assert_projection(&world, &did).await; + + let main_sha = Oid::from_hex(&sh_git(work.path(), &["rev-parse", "HEAD"])).unwrap(); + let refs = get_json( + &world, + &format!( + "/xrpc/org.tangled.git.listRefs?repo={}", + common::percent_encode(did.as_str()) + ), + ) + .await; + assert_eq!( + refs["refs"] + .as_array() + .unwrap() + .iter() + .map(|entry| entry["ref"].as_str().unwrap()) + .collect::>(), + ["refs/heads/main"], + "the read surface lists the pushed branch" + ); + + let (ticket, ticket_cid) = create_ticket(&world, &did, "kelp", "a forest").await; + assert_projection(&world, &did).await; + + sh_git(work.path(), &["checkout", "-q", "-b", "feature"]); + commit_file( + work.path(), + "src/lib.rs", + b"pub fn reef() {}\n", + "add reef", + "2026-06-01T12:31:00+02:00", + ); + let feature_sha = Oid::from_hex(&sh_git(work.path(), &["rev-parse", "HEAD"])).unwrap(); + let addr = serve_pushes(&world).await; + let (ok, detail) = http_push(work.path(), addr, &did, &owner(), &["feature"]).await; + assert!(ok, "feature push failed: {detail}"); + assert_projection(&world, &did).await; + + create_patch(&world, &did, &ticket, &main_sha, &feature_sha).await; + assert_projection(&world, &did).await; + + let collection = ticket_collection(); + let (status, completed) = post_as( + &world, + &nsid("com.atproto.repo.putRecord"), + &owner(), + &ServiceDid::new(did.as_str()).unwrap(), + serde_json::json!({ + "repo": did, + "collection": &collection, + "rkey": ticket, + "swapRecord": ticket_cid.to_string(), + "record": { + "$type": &collection, + "repo": did, + "title": "kelp", + "body": "a forest", + "state": "completed", + "createdAt": "2026-09-11T00:00:00Z" + } + }), + ) + .await; + assert_eq!( + status, + http::StatusCode::OK, + "completing failed: {completed}" + ); + assert_projection(&world, &did).await; + + let (status, deleted) = post_authed( + &world, + &nsid("org.tangled.git.deleteBranch"), + &owner(), + serde_json::json!({ "repo": did, "branch": "feature" }), + ) + .await; + assert_eq!( + status, + http::StatusCode::OK, + "deleteBranch failed: {deleted}" + ); + assert_projection(&world, &did).await; + + let refs = get_json( + &world, + &format!( + "/xrpc/org.tangled.git.listRefs?repo={}", + common::percent_encode(did.as_str()) + ), + ) + .await; + assert_eq!( + refs["refs"] + .as_array() + .unwrap() + .iter() + .map(|entry| entry["ref"].as_str().unwrap()) + .collect::>(), + ["refs/heads/main"], + "the deleted branch is gone from the read surface" + ); + + let collaborator = AccountDid::new("did:plc:limpet").unwrap(); + let (status, offered) = post_as( + &world, + &nsid("com.atproto.repo.createRecord"), + &owner(), + &ServiceDid::new(did.as_str()).unwrap(), + common::grant_body( + &did, + &GrantCollection::Collaborator.records(), + &collaborator, + "2026-09-11T00:00:00Z", + ), + ) + .await; + assert_eq!(status, http::StatusCode::OK, "offer failed: {offered}"); + assert_projection(&world, &did).await; + + let (status, revoked) = post_as( + &world, + &nsid("com.atproto.repo.deleteRecord"), + &owner(), + &ServiceDid::new(did.as_str()).unwrap(), + common::grant_delete_body( + &did, + &GrantCollection::Collaborator.records(), + &collaborator, + ), + ) + .await; + assert_eq!(status, http::StatusCode::OK, "revoke failed: {revoked}"); + assert_projection(&world, &did).await; +} + +#[tokio::test] +async fn branch_churn_keeps_the_refs_record_exact() { + let world = World::new(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + + let did = create_repo(&world, "coral").await; + let work = tempfile::tempdir().unwrap(); + sh_git(work.path(), &["init", "-q", "-b", "main"]); + commit_file( + work.path(), + "README.md", + b"# coral\n", + "first", + "2026-06-01T12:30:00+02:00", + ); + push_main(&world, &did, work.path()).await; + assert_projection(&world, &did).await; + + let addr = serve_pushes(&world).await; + for branch in ["one", "two", "three"] { + sh_git(work.path(), &["branch", branch, "main"]); + let (ok, detail) = http_push(work.path(), addr, &did, &owner(), &[branch]).await; + assert!(ok, "push {branch} failed: {detail}"); + assert_projection(&world, &did).await; + } + + sh_git(work.path(), &["tag", "v1", "main"]); + let (ok, detail) = http_push(work.path(), addr, &did, &owner(), &["v1"]).await; + assert!(ok, "tag push failed: {detail}"); + assert_projection(&world, &did).await; + + let (status, deleted) = post_authed( + &world, + &nsid("org.tangled.git.deleteBranch"), + &owner(), + serde_json::json!({ "repo": did, "branch": "two" }), + ) + .await; + assert_eq!( + status, + http::StatusCode::OK, + "deleteBranch failed: {deleted}" + ); + assert_projection(&world, &did).await; + + let refs = get_json( + &world, + &format!( + "/xrpc/org.tangled.git.listRefs?repo={}", + common::percent_encode(did.as_str()) + ), + ) + .await; + let listed = refs["refs"] + .as_array() + .unwrap() + .iter() + .map(|entry| entry["ref"].as_str().unwrap()) + .collect::>(); + assert_eq!( + listed, + [ + "refs/heads/main", + "refs/heads/one", + "refs/heads/three", + "refs/tags/v1" + ], + "the read surface lists exactly the live refs" + ); +} + +#[tokio::test] +async fn keep_refs_stay_out_of_the_read_surface_and_the_projection() { + let world = World::new(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + + let did = create_repo(&world, "kelp").await; + let work = tempfile::tempdir().unwrap(); + sh_git(work.path(), &["init", "-q", "-b", "main"]); + commit_file( + work.path(), + "README.md", + b"# kelp\n", + "first", + "2026-06-01T12:30:00+02:00", + ); + push_main(&world, &did, work.path()).await; + + let main_sha = Oid::from_hex(&sh_git(work.path(), &["rev-parse", "HEAD"])).unwrap(); + sh_git(work.path(), &["checkout", "-q", "-b", "feature"]); + commit_file( + work.path(), + "src/lib.rs", + b"pub fn kelp() {}\n", + "add kelp", + "2026-06-01T12:31:00+02:00", + ); + let feature_sha = Oid::from_hex(&sh_git(work.path(), &["rev-parse", "HEAD"])).unwrap(); + let addr = serve_pushes(&world).await; + let (ok, detail) = http_push(work.path(), addr, &did, &owner(), &["feature"]).await; + assert!(ok, "feature push failed: {detail}"); + + let (ticket, _) = create_ticket(&world, &did, "kelp", "a forest").await; + let (_patch, _) = create_patch(&world, &did, &ticket, &main_sha, &feature_sha).await; + + let refs = get_json( + &world, + &format!( + "/xrpc/org.tangled.git.listRefs?repo={}", + common::percent_encode(did.as_str()) + ), + ) + .await; + let listed = refs["refs"] + .as_array() + .unwrap() + .iter() + .map(|entry| entry["ref"].as_str().unwrap()) + .collect::>(); + assert_eq!( + listed, + ["refs/heads/feature", "refs/heads/main"], + "the keep refs the patch wrote stay out of listRefs" + ); + assert_projection(&world, &did).await; + + let bare = world.layout.repo_path(&did).unwrap(); + let keep = sh_git( + &bare, + &["for-each-ref", "--format=%(refname)", "refs/tngl/keep"], + ); + assert!( + !keep.is_empty(), + "the patch wrote keep refs into the bare repo" + ); +} diff --git a/knot2/crates/knot-xrpc/tests/reads.rs b/knot2/crates/knot-xrpc/tests/reads.rs new file mode 100644 index 0000000000000000000000000000000000000000..be3bd04e612e41e1a4f2c794e412e851a080c859 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/reads.rs @@ -0,0 +1,1989 @@ +mod common; + +use std::time::{Duration, Instant}; + +use futures::StreamExt; +use futures::stream; +use http::{HeaderMap, StatusCode, header}; +use tokio_tungstenite::tungstenite; + +use knot_capability::Capability; +use knot_index::Resolved; +use knot_types::{AccountDid, Nsid, Oid, RepoDid, RepoName, UnixSeconds}; + +use knot_xrpc::{ArchiveLimit, ResponseLimit}; + +use common::{ + World, archive_full, assert_immutable_round_trip, assert_post_rejected, assert_warming, + commit_file, empty_repo, get, get_error, get_json, get_with_headers, nsid, owner, post_as, + post_authed, ref_list, repo_dids, seeded, sh_git, sh_git_at, +}; + +#[tokio::test] +async fn the_seeded_read_surface_renders_each_lexicon_shape_once() { + let world = World::new(); + let (did, work) = seeded(&world, &RepoName::new("coral").unwrap()); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + let parent = sh_git(work.path(), &["rev-parse", "HEAD~2"]); + let tagged_commit = sh_git(work.path(), &["rev-parse", "v1.0.0^{commit}"]); + + let tree = get_json( + &world, + &format!("/xrpc/org.tangled.git.getTree?repo={did}&ref=main"), + ) + .await; + let entries = tree["entries"].as_array().unwrap(); + let listed: Vec<&str> = entries + .iter() + .map(|entry| entry["name"].as_str().unwrap()) + .collect(); + assert_eq!(listed, vec!["README.md", "logo.png", "src"]); + let readme = &entries[0]; + assert_eq!(readme["mode"], "0100644"); + assert_eq!( + readme["size"].as_i64().unwrap(), + b"# coral\n\nhello reef\n".len() as i64 + ); + assert_eq!( + readme["lastCommit"]["hash"].as_str().unwrap(), + head, + "README was last touched by the head commit" + ); + assert_eq!(readme["lastCommit"]["message"], "update readme\n"); + assert_eq!(entries[2]["mode"], "0040000"); + assert_eq!( + tree["lastCommit"]["hash"].as_str().unwrap(), + head, + "the tree's own lastCommit is the newest commit touching any entry" + ); + assert_eq!(tree["lastCommit"]["author"]["name"], "nel"); + assert_eq!( + tree["lastCommit"]["author"]["when"], + "2026-06-01T12:33:00+02:00" + ); + + let sub = get_json( + &world, + &format!("/xrpc/org.tangled.git.getTree?repo={did}&ref=main&path=src"), + ) + .await; + assert_eq!(sub["entries"][0]["name"], "main.rs"); + assert_eq!( + get_error( + &world, + &format!("/xrpc/org.tangled.git.getTree?repo={did}&ref=main&path=nope"), + ) + .await, + (StatusCode::NOT_FOUND, "PathNotFound".to_string()) + ); + + let entry = get_json( + &world, + &format!("/xrpc/org.tangled.git.getEntry?repo={did}&ref=main&path=README.md"), + ) + .await; + assert_eq!(entry["name"], "README.md"); + assert_eq!(entry["mode"], "0100644"); + assert_eq!( + entry["size"].as_i64().unwrap(), + b"# coral\n\nhello reef\n".len() as i64 + ); + assert!( + entry["oid"].as_str().unwrap().len() == 40, + "the entry's blob oid rides as hex" + ); + assert_eq!(entry["lastCommit"]["message"], "update readme\n"); + + let log = get_json( + &world, + &format!("/xrpc/org.tangled.git.listCommits?repo={did}&ref=main"), + ) + .await; + let commits = log["commits"].as_array().unwrap(); + assert_eq!(commits.len(), 4); + assert!( + log.get("cursor").is_none() || log["cursor"].is_null(), + "a page that reaches the oldest commit hands back no cursor" + ); + let first = &commits[0]; + assert_eq!(first["hash"], head.as_str()); + assert_eq!(first["author"]["name"], "nel"); + assert_eq!(first["author"]["email"], "nel@oyster.cafe"); + assert_eq!(first["author"]["when"], "2026-06-01T12:33:00+02:00"); + assert_eq!(first["committer"]["name"], "nel"); + assert_eq!(first["message"], "update readme\n"); + assert!(first["tree"].as_str().unwrap().len() == 40); + let second_page = get_json( + &world, + &format!("/xrpc/org.tangled.git.listCommits?repo={did}&ref=main&limit=2"), + ) + .await; + let two = second_page["commits"].as_array().unwrap(); + assert_eq!(two.len(), 2); + assert_eq!(two[0]["hash"], head.as_str()); + let cursor = second_page["cursor"].as_str().unwrap().to_string(); + assert_eq!( + cursor, + two[1]["hash"].as_str().unwrap(), + "the cursor is the commit id of the last commit in the page" + ); + let rest = get_json( + &world, + &format!("/xrpc/org.tangled.git.listCommits?repo={did}&ref=main&limit=2&cursor={cursor}"), + ) + .await; + let older = rest["commits"].as_array().unwrap(); + assert_eq!(older.len(), 2); + assert_eq!(older[0]["hash"], parent.as_str()); + + let branches = get_json( + &world, + &format!("/xrpc/org.tangled.git.listBranches?repo={did}"), + ) + .await; + let listed = branches["branches"].as_array().unwrap(); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0]["name"], "main"); + assert_eq!(listed[0]["oid"], head.as_str()); + assert_eq!(listed[0]["message"], "update readme\n"); + assert_eq!(listed[0]["author"]["name"], "nel"); + assert_eq!(listed[0]["committer"]["email"], "nel@oyster.cafe"); + assert_eq!(branches["total"].as_i64(), Some(1)); + assert_eq!(branches["defaultBranch"], "main"); + let branch = get_json( + &world, + &format!("/xrpc/org.tangled.git.getBranch?repo={did}&name=main"), + ) + .await; + assert_eq!(branch["name"], "main"); + assert_eq!(branch["hash"], head.as_str()); + assert_eq!(branch["message"], "update readme\n"); + assert_eq!(branch["author"]["name"], "nel"); + assert_eq!(branch["totalCommits"].as_i64(), Some(4)); + assert_eq!( + get_error( + &world, + &format!("/xrpc/org.tangled.git.getBranch?repo={did}&name=mangrove"), + ) + .await, + (StatusCode::NOT_FOUND, "BranchNotFound".to_string()) + ); + + let tags = get_json( + &world, + &format!("/xrpc/org.tangled.git.listTags?repo={did}"), + ) + .await; + let tag_list = tags["tags"].as_array().unwrap(); + assert_eq!(tag_list.len(), 2); + assert_eq!(tags["total"].as_i64(), Some(2)); + let annotated = tag_list.iter().find(|tag| tag["name"] == "v1.0.0").unwrap(); + assert_eq!( + annotated["oid"], + tagged_commit.as_str(), + "an annotated tag reports the commit it peels to" + ); + assert_eq!(annotated["message"], "add logo\n"); + assert_eq!(annotated["author"]["name"], "nel"); + assert_eq!( + annotated["annotation"]["message"], "release one", + "the tag's own message rides the annotation" + ); + assert_eq!(annotated["annotation"]["tagger"]["name"], "nel"); + let lightweight = tag_list + .iter() + .find(|tag| tag["name"] == "lightweight") + .unwrap(); + assert!(lightweight.get("annotation").is_none()); + assert_eq!(lightweight["oid"], tagged_commit.as_str()); + let single = get_json( + &world, + &format!("/xrpc/org.tangled.git.getTag?repo={did}&tag=v1.0.0"), + ) + .await; + assert_eq!(single["name"], "v1.0.0"); + assert_eq!(single["oid"], tagged_commit.as_str()); + assert_eq!(single["annotation"]["message"], "release one"); + assert_eq!( + get_error( + &world, + &format!("/xrpc/org.tangled.git.getTag?repo={did}&tag=v9.9.9"), + ) + .await, + (StatusCode::NOT_FOUND, "RefNotFound".to_string()) + ); + + let (status, headers, body) = get( + &world, + &format!("/xrpc/org.tangled.git.getBlob?repo={did}&ref=main&path=README.md"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + headers.get(header::CONTENT_TYPE).unwrap(), + "application/octet-stream", + "the lexicon pins the blob to raw bytes" + ); + assert_eq!( + headers.get(header::X_CONTENT_TYPE_OPTIONS).unwrap(), + "nosniff" + ); + assert_eq!( + headers.get(header::CONTENT_SECURITY_POLICY).unwrap(), + "default-src 'none'; style-src 'unsafe-inline'; sandbox" + ); + assert_eq!(body, "# coral\n\nhello reef\n"); + let etag = headers + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap() + .to_string(); + + let mut cached = HeaderMap::new(); + cached.insert( + header::IF_NONE_MATCH, + http::HeaderValue::from_str(&etag).unwrap(), + ); + let (status, _, _) = get_with_headers( + &world, + &format!("/xrpc/org.tangled.git.getBlob?repo={did}&ref=main&path=README.md"), + cached, + ) + .await; + assert_eq!(status, StatusCode::NOT_MODIFIED); + + let mut weak = HeaderMap::new(); + weak.insert( + header::IF_NONE_MATCH, + http::HeaderValue::from_str(&format!("W/{etag}")).unwrap(), + ); + let (status, _, _) = get_with_headers( + &world, + &format!("/xrpc/org.tangled.git.getBlob?repo={did}&ref=main&path=README.md"), + weak, + ) + .await; + assert_eq!( + status, + StatusCode::NOT_MODIFIED, + "weak validator must revalidate too" + ); + + let logo = get( + &world, + &format!("/xrpc/org.tangled.git.getBlob?repo={did}&ref=main&path=logo.png"), + ) + .await; + assert_eq!(logo.0, StatusCode::OK); + assert!(logo.2.starts_with(b"\x89PNG")); + + let blame = get_json( + &world, + &format!("/xrpc/org.tangled.git.getBlame?repo={did}&ref=main&path=README.md"), + ) + .await; + assert_eq!(blame["commit"], head); + let blame_commits = blame["commits"].as_array().unwrap(); + let runs = blame["runs"].as_array().unwrap(); + assert_eq!(blame_commits.len(), 2); + assert_eq!(runs.len(), 2); + assert_eq!(runs[0]["start"], 1); + assert_eq!(runs[0]["count"], 2); + let first_sha = runs[0]["commit"].as_str().unwrap(); + let first_commit = blame_commits + .iter() + .find(|commit| commit["sha"] == first_sha) + .unwrap(); + assert_eq!(first_commit["subject"], "first"); + assert_eq!(first_commit["author"]["name"], "nel"); + assert_eq!(runs[1]["start"], 3); + assert_eq!(runs[1]["count"], 1); + assert_eq!(runs[1]["commit"], head); + let head_commit = blame_commits + .iter() + .find(|commit| commit["sha"] == head) + .unwrap(); + assert_eq!(head_commit["subject"], "update readme"); + assert_eq!(head_commit["authoredAt"], "2026-06-01T12:33:00+02:00"); + + assert_eq!( + get_error( + &world, + &format!("/xrpc/org.tangled.git.getBlob?repo={did}&ref=main&path=ghost.txt"), + ) + .await, + (StatusCode::NOT_FOUND, "FileNotFound".to_string()) + ); +} + +#[tokio::test] +async fn tree_directory_last_commit_is_the_newest_touching_commit() { + let world = World::new(); + let (did, bare, work) = empty_repo(&world, &RepoName::new("periwinkle").unwrap()); + let work = work.path(); + commit_file( + work, + "src/a.rs", + b"fn a() {}\n", + "add a", + "2026-06-01T12:30:00+02:00", + ); + let older = sh_git(work, &["rev-parse", "HEAD"]); + commit_file( + work, + "src/b.rs", + b"fn b() {}\n", + "add b", + "2026-06-01T12:31:00+02:00", + ); + let newer = sh_git(work, &["rev-parse", "HEAD"]); + commit_file( + work, + "README.md", + b"# periwinkle\n", + "doc", + "2026-06-01T12:33:00+02:00", + ); + let head = sh_git(work, &["rev-parse", "HEAD"]); + sh_git(work, &["push", "-q", &bare, "main"]); + + let value = get_json( + &world, + &format!("/xrpc/org.tangled.git.getTree?repo={did}&ref=main"), + ) + .await; + let entries = value["entries"].as_array().unwrap(); + let src = entries + .iter() + .find(|entry| entry["name"] == "src") + .expect("src directory is listed"); + let reported = src["lastCommit"]["hash"].as_str().unwrap(); + assert_eq!( + reported, newer, + "the newest commit that touches the subtree is reported, not the oldest" + ); + assert_ne!(reported, older, "not the first commit that created subtree"); + assert_ne!( + reported, head, + "head commit only touched README, never the src subtree" + ); +} + +#[tokio::test] +async fn languages_timeout_yields_a_partial_answer_not_an_error() { + let world = World::new(); + let (did, work) = seeded(&world, &RepoName::new("scallop").unwrap()); + let head = Oid::from_hex(&sh_git(work.path(), &["rev-parse", "HEAD"])).unwrap(); + let repo = world.layout.open(&did).unwrap(); + + let full = + knot_langs::analyze(&repo, head, Some(Instant::now() + Duration::from_secs(60))).unwrap(); + assert!( + full.values().any(|size| size.get() > 0), + "generous budget detects code" + ); + + let expired = Instant::now() + .checked_sub(Duration::from_secs(1)) + .unwrap_or_else(Instant::now); + let partial = knot_langs::analyze(&repo, head, Some(expired)).unwrap(); + assert!( + partial.is_empty(), + "exhausted budget breaks the walk and returns the partial map gathered so far, never an error" + ); +} + +#[tokio::test] +async fn diff_reports_structured_fragments_and_stats() { + let world = World::new(); + let (did, work) = seeded(&world, &RepoName::new("whelk").unwrap()); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + let base = sh_git(work.path(), &["rev-parse", "HEAD~1"]); + + let value = get_json( + &world, + &format!("/xrpc/org.tangled.git.getDiff?repo={did}&base={base}&head={head}"), + ) + .await; + let stats = &value["stats"]; + assert_eq!(stats["filesChanged"].as_i64(), Some(1)); + assert_eq!(stats["insertions"].as_i64(), Some(1)); + assert_eq!(stats["deletions"].as_i64(), Some(1)); + let diff = &value["diffs"][0]; + assert_eq!(diff["lhsSrc"]["path"], "README.md"); + assert_eq!(diff["rhsSrc"]["path"], "README.md"); + assert_eq!(diff["lhsSrc"]["isBinary"], false); + assert_eq!(diff["lhsSrc"]["isSubmodule"], false); + assert!(diff["lhsSrc"]["size"].as_i64().unwrap() > 0); + assert_eq!(diff["hasSyntacticChanges"], true); + let hunk = &diff["hunks"][0]; + assert_eq!(hunk["novelLhs"], serde_json::json!([2])); + assert_eq!(hunk["novelRhs"], serde_json::json!([2])); + assert_eq!( + hunk["lines"], + serde_json::json!([ + { "lhs": 0, "rhs": 0 }, + { "lhs": 1, "rhs": 1 }, + { "lhs": 2 }, + { "rhs": 2 }, + ]), + "0-based line pairs: context lines first, then one deleted hello and one added hello reef" + ); + + assert_eq!( + get_error( + &world, + &format!("/xrpc/org.tangled.git.getDiff?repo={did}&base=nope&head={head}"), + ) + .await, + (StatusCode::NOT_FOUND, "RefNotFound".to_string()) + ); +} + +#[tokio::test] +async fn archive_conditional_and_range_semantics() { + let world = World::new(); + let (did, _work) = seeded(&world, &RepoName::new("nautilus").unwrap()); + let path = format!("/xrpc/org.tangled.git.getArchive?repo={did}&ref=main&prefix=nautilus-main"); + + let (status, headers, full) = get(&world, &path).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + headers.get(header::CONTENT_TYPE).unwrap(), + "application/gzip" + ); + assert_eq!( + headers + .get(header::CONTENT_DISPOSITION) + .unwrap() + .to_str() + .unwrap(), + "attachment; filename=\"nautilus-main.tar.gz\"" + ); + let link = headers.get(header::LINK).unwrap().to_str().unwrap(); + assert!(link.contains("rel=\"immutable\"")); + assert!(link.contains("/xrpc/org.tangled.git.getArchive?")); + assert!(link.contains("prefix=nautilus-main")); + assert_eq!(headers.get(header::ACCEPT_RANGES).unwrap(), "bytes"); + let last_modified = headers + .get(header::LAST_MODIFIED) + .expect("the archive file's mtime is the commit's time, backing date revalidation") + .to_str() + .unwrap() + .to_string(); + let etag = headers + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap() + .to_string(); + assert!( + etag.starts_with('"') && etag.ends_with('"'), + "a strong etag is quoted" + ); + assert_eq!(&full[..2], &[0x1f, 0x8b]); + + let mut range = HeaderMap::new(); + range.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + let (status, range_headers, partial) = get_with_headers(&world, &path, range).await; + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert_eq!( + range_headers + .get(header::CONTENT_RANGE) + .unwrap() + .to_str() + .unwrap(), + format!("bytes 0-3/{}", full.len()) + ); + assert_eq!( + partial.as_ref(), + &full[..4], + "a resumed range regenerates byte for byte" + ); + + let mut conditional = HeaderMap::new(); + conditional.insert(header::IF_NONE_MATCH, etag.parse().unwrap()); + let (status, cond_headers, conditional_body) = + get_with_headers(&world, &path, conditional).await; + assert_eq!(status, StatusCode::NOT_MODIFIED); + assert_eq!( + cond_headers.get(header::ETAG).unwrap().to_str().unwrap(), + etag + ); + assert!(conditional_body.is_empty(), "a 304 has no body"); + + let (status, _) = get_error( + &world, + &format!("/xrpc/org.tangled.git.getArchive?repo={did}&ref=main&prefix=p&format=tar.bz2"), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + + let mut etag_match = HeaderMap::new(); + etag_match.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + etag_match.insert(header::IF_RANGE, etag.parse().unwrap()); + let (status, if_range_headers, partial) = get_with_headers(&world, &path, etag_match).await; + assert_eq!( + status, + StatusCode::PARTIAL_CONTENT, + "a matching content etag resumes the range" + ); + assert_eq!( + if_range_headers + .get(header::CONTENT_RANGE) + .unwrap() + .to_str() + .unwrap(), + format!("bytes 0-3/{}", full.len()) + ); + assert_eq!(partial.as_ref(), &full[..4]); + + let mut etag_stale = HeaderMap::new(); + etag_stale.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + etag_stale.insert(header::IF_RANGE, "\"0000\"".parse().unwrap()); + let (status, _, body) = get_with_headers(&world, &path, etag_stale).await; + assert_eq!( + status, + StatusCode::OK, + "a stale content etag falls back to the full body" + ); + assert_eq!(body, full, "the full archive comes back byte for byte"); + + let mut weak = HeaderMap::new(); + weak.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + weak.insert(header::IF_RANGE, format!("W/{etag}").parse().unwrap()); + let (status, _, body) = get_with_headers(&world, &path, weak).await; + assert_eq!( + status, + StatusCode::OK, + "a weak validator never serves a range, per strong-comparison rules" + ); + assert_eq!(body, full); + + let mut date_match = HeaderMap::new(); + date_match.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + date_match.insert(header::IF_RANGE, last_modified.parse().unwrap()); + let (status, date_headers, partial) = get_with_headers(&world, &path, date_match).await; + assert_eq!( + status, + StatusCode::PARTIAL_CONTENT, + "a date matching the commit-time last-modified resumes the range" + ); + assert_eq!( + date_headers + .get(header::CONTENT_RANGE) + .unwrap() + .to_str() + .unwrap(), + format!("bytes 0-3/{}", full.len()) + ); + assert_eq!(partial.as_ref(), &full[..4]); + + let mut date_stale = HeaderMap::new(); + date_stale.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + date_stale.insert( + header::IF_RANGE, + "Wed, 21 Oct 2015 07:28:00 GMT".parse().unwrap(), + ); + let (status, _, body) = get_with_headers(&world, &path, date_stale).await; + assert_eq!( + status, + StatusCode::OK, + "a date that doesn't match the commit-time last-modified falls back to the full body" + ); + assert_eq!(body, full); + + assert_immutable_round_trip(&world, &headers, &full, &etag).await; +} + +#[tokio::test] +async fn archive_etag_distinguishes_prefixes_that_share_a_commit() { + let world = World::new(); + let (did, work) = seeded(&world, &RepoName::new("scallop").unwrap()); + let bare = world.layout.repo_path(&did).unwrap(); + sh_git(work.path(), &["branch", "release", "main"]); + sh_git( + work.path(), + &["push", "-q", bare.to_str().unwrap(), "refs/heads/release"], + ); + + let (main_etag, _main_last_modified, main_body) = + archive_full(&world, &did, "scallop-main").await; + let (status, release_headers, release_body) = get( + &world, + &format!("/xrpc/org.tangled.git.getArchive?repo={did}&ref=release&prefix=scallop-release"), + ) + .await; + assert_eq!(status, StatusCode::OK); + let release_etag = release_headers + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap() + .to_string(); + assert_ne!( + main_etag, release_etag, + "two prefixes over one commit must give different strong etags" + ); + assert_ne!( + main_body, release_body, + "the archives use different top-level directories and differ byte for byte" + ); + + let mut conditional = HeaderMap::new(); + conditional.insert(header::IF_NONE_MATCH, main_etag.parse().unwrap()); + let (status, _, _) = get_with_headers( + &world, + &format!("/xrpc/org.tangled.git.getArchive?repo={did}&ref=release&prefix=scallop-release"), + conditional, + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "the main prefix's etag mustn't satisfy a conditional request for the release archive" + ); +} + +#[tokio::test] +async fn archive_link_advertises_the_prefix_it_served() { + let world = World::new(); + let (did, _work) = seeded(&world, &RepoName::new("periwinkle").unwrap()); + let query = + |suffix: &str| format!("/xrpc/org.tangled.git.getArchive?repo={did}&ref=main{suffix}"); + + assert_eq!( + get_error(&world, &query("")).await, + (StatusCode::BAD_REQUEST, "InvalidRequest".to_string()), + "the lexicon makes prefix a required parameter" + ); + + for (suffix, prefix, filename) in [ + ("&prefix=kelp/uni", "kelp%2Funi", "kelp-uni.tar.gz"), + ("&prefix=/kelp//./uni/", "kelp%2Funi", "kelp-uni.tar.gz"), + ] { + let (status, headers, full) = get(&world, &query(suffix)).await; + assert_eq!(status, StatusCode::OK); + let link = headers[header::LINK].to_str().unwrap(); + assert!( + link.contains(&format!("prefix={prefix}")), + "the link repeats the cleaned prefix the knot served, percent-encoded: {suffix} gave {link}" + ); + assert_eq!( + headers[header::CONTENT_DISPOSITION], + format!("attachment; filename=\"{filename}\""), + "the filename follows the prefix with the separator flattened" + ); + let etag = headers[header::ETAG].to_str().unwrap().to_string(); + assert_immutable_round_trip(&world, &headers, &full, &etag).await; + } + + for prefix in ["u".repeat(256), "kelp%5Cuni".to_string()] { + assert_eq!( + get_error(&world, &query(&format!("&prefix={prefix}"))).await, + (StatusCode::BAD_REQUEST, "InvalidRequest".to_string()), + "prefix {prefix}" + ); + } +} + +#[tokio::test] +async fn archive_serves_a_sha256_repo_with_a_stable_etag() { + let world = World::sha256(); + let (did, _work) = seeded(&world, &RepoName::new("nautilus").unwrap()); + + let (status, headers, full) = get( + &world, + &format!("/xrpc/org.tangled.git.getArchive?repo={did}&ref=main&prefix=nautilus-main"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + headers.get(header::CONTENT_TYPE).unwrap(), + "application/gzip" + ); + assert_eq!(&full[..2], &[0x1f, 0x8b]); + let etag = headers + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap() + .to_string(); + + assert_immutable_round_trip(&world, &headers, &full, &etag).await; + + let mut conditional = HeaderMap::new(); + conditional.insert(header::IF_NONE_MATCH, etag.parse().unwrap()); + let (status, _, body) = get_with_headers( + &world, + &format!("/xrpc/org.tangled.git.getArchive?repo={did}&ref=main&prefix=nautilus-main"), + conditional, + ) + .await; + assert_eq!( + status, + StatusCode::NOT_MODIFIED, + "conditional revalidation works under sha256" + ); + assert!(body.is_empty()); +} + +#[tokio::test] +async fn languages_detect_rust_and_markdown_stays_out() { + let world = World::new(); + let (did, _work) = seeded(&world, &RepoName::new("uni").unwrap()); + + let value = get_json( + &world, + &format!("/xrpc/org.tangled.git.listLanguages?repo={did}&ref=main"), + ) + .await; + assert_eq!(value["ref"], "main"); + let languages = value["languages"].as_array().unwrap(); + assert_eq!( + languages.len(), + 1, + "only Rust counts: markdown is prose, png is binary" + ); + assert_eq!(languages[0]["name"], "Rust"); + assert_eq!(languages[0]["percentage"].as_i64(), Some(100)); + assert!(languages[0]["size"].as_i64().unwrap() > 0); + assert_eq!(value["totalFiles"].as_i64(), Some(1)); + assert!(value["totalSize"].as_i64().is_some_and(|total| total > 0)); +} + +#[tokio::test] +async fn import_status_is_served_and_empty_for_a_repo_not_made_from_a_source() { + let world = World::new(); + let (did, _work) = seeded(&world, &RepoName::new("uni").unwrap()); + + let value = get_json( + &world, + &format!("/xrpc/org.tangled.repo.getImportStatus?repo={did}"), + ) + .await; + assert_eq!(value, serde_json::json!({})); + assert_eq!( + get_error( + &world, + "/xrpc/org.tangled.repo.getImportStatus?repo=did:plc:unregistered", + ) + .await, + (StatusCode::NOT_FOUND, "RepoNotFound".to_string()) + ); + let (status, error) = get_error(&world, "/xrpc/org.tangled.repo.getImportStatus").await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(error, "InvalidRequest"); +} + +#[tokio::test] +async fn repo_reads_fail_closed_on_unknown_repos_refs_and_params() { + let world = World::new(); + let (did, _work) = seeded(&world, &RepoName::new("cuttle").unwrap()); + + let refs = get_json( + &world, + &format!("/xrpc/org.tangled.git.listRefs?repo={did}"), + ) + .await; + assert_eq!(refs["defaultBranch"]["ref"], "refs/heads/main"); + + assert_eq!( + get_error( + &world, + "/xrpc/org.tangled.git.getTree?repo=did:plc:unregistered", + ) + .await, + (StatusCode::NOT_FOUND, "RepoNotFound".to_string()) + ); + assert_eq!( + get_error( + &world, + &format!("/xrpc/org.tangled.git.listCommits?repo={did}&ref=mangrove"), + ) + .await, + (StatusCode::NOT_FOUND, "RefNotFound".to_string()) + ); + let (status, error) = get_error(&world, "/xrpc/org.tangled.git.getTree").await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!( + error, "InvalidRequest", + "a structurally unsound query gets the lexicon error shape instead of the runtime's default plaintext" + ); +} + +#[tokio::test] +async fn list_refs_reports_paginates_and_drains() { + let world = World::new(); + let (did, work) = seeded(&world, &RepoName::new("whelk").unwrap()); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + + let value = get_json( + &world, + &format!("/xrpc/org.tangled.git.listRefs?repo={did}"), + ) + .await; + assert_eq!( + ref_list(&value, "refs"), + vec![ + "refs/heads/main", + "refs/tags/lightweight", + "refs/tags/v1.0.0" + ] + ); + let main = value["refs"] + .as_array() + .unwrap() + .iter() + .find(|entry| entry["ref"] == "refs/heads/main") + .unwrap(); + assert_eq!(main["sha"], head); + assert_eq!(value["defaultBranch"]["ref"], "refs/heads/main"); + assert_eq!(value["defaultBranch"]["head"], head); + assert!(value["cursor"].is_null()); + + let first = get_json( + &world, + &format!("/xrpc/org.tangled.git.listRefs?repo={did}&limit=2"), + ) + .await; + assert_eq!(ref_list(&first, "refs").len(), 2); + let cursor = first["cursor"].as_str().unwrap().to_string(); + let second = get_json( + &world, + &format!("/xrpc/org.tangled.git.listRefs?repo={did}&limit=2&cursor={cursor}"), + ) + .await; + assert_eq!(ref_list(&second, "refs").len(), 1); + assert!(second["cursor"].is_null()); + + let refs = get_json( + &world, + &format!( + "/xrpc/org.tangled.git.listRefs?repo={did}&cursor={}", + usize::MAX + ), + ) + .await; + assert!(ref_list(&refs, "refs").is_empty()); + assert!(refs["cursor"].is_null()); +} + +#[tokio::test] +async fn the_cob_ref_namespace_is_invisible_across_every_read() { + let world = World::new(); + let (did, work) = seeded(&world, &RepoName::new("anemone").unwrap()); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + let bare = world.layout.repo_path(&did).unwrap(); + let cob = "refs/cobs/org.tangled.graph.collaborator/x"; + sh_git(bare.as_path(), &["update-ref", cob, &head]); + + let value = get_json( + &world, + &format!("/xrpc/org.tangled.git.listRefs?repo={did}"), + ) + .await; + assert!( + ref_list(&value, "refs") + .iter() + .all(|name| !name.starts_with("refs/cobs/")), + "reserved cob ref leaked into listRefs" + ); + + let w = &world; + let d = &did; + let probe_routes: &[(&str, &str)] = &[ + ("org.tangled.git.listCommits", "&ref={cob}"), + ("org.tangled.git.getTree", "&ref={cob}"), + ("org.tangled.git.getBlob", "&ref={cob}&path=README.md"), + ("org.tangled.git.getDiff", "&base={cob}&head={head}"), + ("org.tangled.git.getArchive", "&ref={cob}&prefix=p"), + ("org.tangled.git.listLanguages", "&ref={cob}"), + ]; + stream::iter(probe_routes) + .for_each(|(route, suffix)| { + let head = head.clone(); + async move { + let query = suffix.replace("{cob}", cob).replace("{head}", &head); + let (status, _) = get_error(w, &format!("/xrpc/{route}?repo={d}&{query}")).await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "{route} mustn't resolve reserved cobs ref" + ); + } + }) + .await; + let (status, _) = get_error( + &world, + &format!( + "/xrpc/org.tangled.git.listCommits?repo={did}&ref=cobs/org.tangled.graph.collaborator/x" + ), + ) + .await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "reserved namespace shorthand mustn't resolve either" + ); + + commit_file( + work.path(), + "secret.txt", + b"hidden\n", + "secret", + "2026-06-01T12:40:00+02:00", + ); + let hidden = sh_git(work.path(), &["rev-parse", "HEAD"]); + sh_git( + work.path(), + &[ + "push", + "-q", + bare.to_str().unwrap(), + "HEAD:refs/cobs/org.tangled.graph.collaborator/secret", + ], + ); + + let hidden_routes: &[(&str, &str)] = &[ + ("org.tangled.git.listCommits", "&ref={hidden}"), + ("org.tangled.git.getTree", "&ref={hidden}&path=secret.txt"), + ("org.tangled.git.getDiff", "&base={head}&head={hidden}"), + ("org.tangled.git.getArchive", "&ref={hidden}&prefix=p"), + ("org.tangled.git.listLanguages", "&ref={hidden}"), + ]; + stream::iter(hidden_routes) + .for_each(|(route, suffix)| { + let hidden = hidden.clone(); + let head = head.clone(); + async move { + let query = suffix.replace("{hidden}", &hidden).replace("{head}", &head); + let (status, _) = get_error(w, &format!("/xrpc/{route}?repo={d}&{query}")).await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "{route} mustn't serve a commit reachable only through cob ref" + ); + } + }) + .await; + let (status, _) = get_error( + &world, + &format!("/xrpc/org.tangled.git.getBlob?repo={did}&ref={hidden}&path=secret.txt"), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + let still_public = get_json( + &world, + &format!("/xrpc/org.tangled.git.listCommits?repo={did}&ref={head}"), + ) + .await; + assert_eq!(still_public["commits"].as_array().unwrap().len(), 4); +} + +#[tokio::test] +async fn the_list_reads_reject_malformed_paging_params() { + let world = World::new(); + let (did, _work) = seeded(&world, &RepoName::new("barnacle").unwrap()); + + let queries: Vec = vec![ + format!("/xrpc/org.tangled.git.listRefs?repo={did}&limit=abc"), + format!("/xrpc/org.tangled.git.listRefs?repo={did}&cursor=notanint"), + format!("/xrpc/org.tangled.git.listBranches?repo={did}&limit=abc"), + ]; + let w = &world; + stream::iter(queries.iter()) + .for_each(|query| async move { + let (status, error) = get_error(w, query).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "query {query}"); + assert_eq!(error, "InvalidRequest", "query {query}"); + }) + .await; + + let clamped = get_json( + &world, + &format!("/xrpc/org.tangled.git.listRefs?repo={did}&limit=5000"), + ) + .await; + assert_eq!( + ref_list(&clamped, "refs").len(), + 3, + "an oversize limit clamps to the max instead of erroring" + ); +} + +#[tokio::test] +async fn list_repos_lists_hosted_repos_with_pagination() { + let world = World::new(); + let (mussel, _a) = seeded(&world, &RepoName::new("mussel").unwrap()); + let (nautilus, _b) = seeded(&world, &RepoName::new("nautilus").unwrap()); + let (scallop, _c) = seeded(&world, &RepoName::new("scallop").unwrap()); + common::enable_chain(&world, &world.state.knot_repo); + common::enable_chain(&world, &mussel); + common::enable_chain(&world, &nautilus); + common::enable_chain(&world, &scallop); + + let desc = get_json(&world, "/xrpc/com.atproto.sync.listRepos").await; + let listed = repo_dids(&desc); + assert_eq!( + listed[1..], + vec![ + mussel.as_str().to_string(), + nautilus.as_str().to_string(), + scallop.as_str().to_string(), + ][..], + "registry order, behind the knot's own account" + ); + assert_eq!(desc["repos"][0]["did"], world.state.knot_repo.as_str()); + assert_eq!(desc["repos"][1]["active"], true); + + let page = get_json(&world, "/xrpc/com.atproto.sync.listRepos?limit=3").await; + assert_eq!(page["repos"].as_array().unwrap().len(), 3); + let cursor = page["cursor"].as_str().unwrap().to_string(); + let rest = get_json( + &world, + &format!("/xrpc/com.atproto.sync.listRepos?limit=3&cursor={cursor}"), + ) + .await; + assert_eq!( + repo_dids(&rest), + vec![scallop.as_str().to_string()], + "the cursor resumes where the page stopped" + ); + assert!(rest["cursor"].is_null()); +} + +#[tokio::test] +async fn every_projection_read_fails_closed_while_warming() { + let world = World::warming(); + let did = RepoDid::new("did:plc:limpetfixture").unwrap(); + let cases: Vec<(String, Option<&str>)> = vec![ + ( + "/xrpc/com.atproto.sync.listRepos".to_string(), + Some("ProjectionWarming"), + ), + ( + format!("/xrpc/com.atproto.repo.describeRepo?repo={did}"), + Some("ProjectionWarming"), + ), + ( + format!( + "/xrpc/com.atproto.repo.listRecords?repo={did}&collection=org.tangled.knot.member" + ), + Some("ProjectionWarming"), + ), + ]; + let w = &world; + stream::iter(cases.iter()) + .for_each(|(path, expected)| async move { + assert_warming(w, path, *expected).await; + }) + .await; +} + +#[tokio::test] +async fn branch_tips_render_edge_shapes() { + let world = World::new(); + let (did, work) = seeded(&world, &RepoName::new("trochus").unwrap()); + let bare = world.layout.repo_path(&did).unwrap(); + let bare_str = bare.to_str().unwrap().to_string(); + + sh_git(work.path(), &["checkout", "-q", "-b", "side", "HEAD~1"]); + commit_file( + work.path(), + "side.txt", + b"side\n", + "side work", + "2026-06-01T12:34:00+02:00", + ); + sh_git(work.path(), &["checkout", "-q", "main"]); + sh_git_at( + work.path(), + "2026-06-01T12:35:00+02:00", + &["merge", "-q", "--no-ff", "-m", "merge side", "side"], + ); + sh_git(work.path(), &["push", "-q", &bare_str, "main"]); + let merge_tip = sh_git(work.path(), &["rev-parse", "HEAD"]); + + let tag_object = sh_git(work.path(), &["rev-parse", "v1.0.0"]); + std::fs::write(bare.join("refs/heads/tagtip"), format!("{tag_object}\n")).unwrap(); + let root_commit = sh_git(work.path(), &["rev-list", "--max-parents=0", "HEAD"]); + std::fs::write(bare.join("refs/heads/roottip"), format!("{root_commit}\n")).unwrap(); + + let value = get_json( + &world, + &format!("/xrpc/org.tangled.git.listBranches?repo={did}"), + ) + .await; + let branches = value["branches"].as_array().unwrap(); + assert_eq!(branches.len(), 3); + assert_eq!(value["total"].as_i64(), Some(3)); + + let branch = |name: &str| { + branches + .iter() + .find(|branch| branch["name"] == name) + .unwrap() + }; + + let main = branch("main"); + assert_eq!( + main["oid"], merge_tip, + "the merge tip is reported as the branch's oid" + ); + assert_eq!(main["message"], "merge side\n"); + assert_eq!(main["author"]["name"], "nel"); + assert_eq!(main["committer"]["name"], "nel"); + + let root = branch("roottip"); + assert_eq!(root["oid"], root_commit); + assert_eq!(root["message"], "first\n"); + + let tagtip = branch("tagtip"); + assert_eq!( + tagtip["oid"], tag_object, + "a non-commit tip still reports the ref's own target" + ); + assert_eq!(tagtip["message"], "release one"); + assert_eq!( + tagtip["author"], + serde_json::json!({ + "name": "", + "email": "", + "when": "1970-01-01T00:00:00Z", + }), + "a ref pointing past a commit has no identities to cite" + ); +} + +#[tokio::test] +async fn a_submodule_path_in_the_tree_is_path_not_found() { + let world = World::new(); + let (did, work) = seeded(&world, &RepoName::new("razorclam").unwrap()); + let bare = world + .layout + .repo_path(&did) + .unwrap() + .to_str() + .unwrap() + .to_string(); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + sh_git( + work.path(), + &[ + "update-index", + "--add", + "--cacheinfo", + &format!("160000,{head},vendor/dep"), + ], + ); + sh_git(work.path(), &["commit", "-q", "-m", "add gitlink"]); + sh_git(work.path(), &["push", "-q", &bare, "main"]); + + assert_eq!( + get_error( + &world, + &format!("/xrpc/org.tangled.git.getTree?repo={did}&ref=main&path=vendor/dep"), + ) + .await, + (StatusCode::NOT_FOUND, "PathNotFound".to_string()) + ); +} + +#[tokio::test] +async fn blame_rejects_a_blob_past_its_work_limit() { + let world = World::new(); + let (did, bare, work) = empty_repo(&world, &RepoName::new("whelk").unwrap()); + commit_file( + work.path(), + "big.txt", + &vec![b'a'; 1024 * 1024 + 1], + "big file", + "2026-06-01T12:30:00+02:00", + ); + sh_git(work.path(), &["push", "-q", &bare, "main"]); + + let (status, error) = get_error( + &world, + &format!("/xrpc/org.tangled.git.getBlame?repo={did}&ref=main&path=big.txt"), + ) + .await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error, "BlameTooLarge"); +} + +#[tokio::test] +async fn a_blob_past_the_serving_limit_is_refused_as_blob_too_large() { + let world = World::new(); + let (did, bare, work) = empty_repo(&world, &RepoName::new("auger").unwrap()); + commit_file( + work.path(), + "small.txt", + b"hello\n", + "small file", + "2026-06-01T12:30:00+02:00", + ); + commit_file( + work.path(), + "big.bin", + &vec![0u8; 25 * 1024 * 1024 + 1], + "big file", + "2026-06-01T12:31:00+02:00", + ); + sh_git(work.path(), &["push", "-q", &bare, "main"]); + + let (status, error) = get_error( + &world, + &format!("/xrpc/org.tangled.git.getBlob?repo={did}&ref=main&path=big.bin"), + ) + .await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!( + error, "BlobTooLarge", + "twenty five mebibytes is the limit itself, not a bound derived from the response budget" + ); + + let (status, _, body) = get( + &world, + &format!("/xrpc/org.tangled.git.getBlob?repo={did}&ref=main&path=small.txt"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "hello\n", "a blob under the limit still serves"); +} + +#[tokio::test] +async fn blame_accepts_many_short_lines_below_the_byte_limit() { + let world = World::new(); + let (did, bare, work) = empty_repo(&world, &RepoName::new("limpet").unwrap()); + commit_file( + work.path(), + "many.txt", + "x\n".repeat(100_001).as_bytes(), + "many lines", + "2026-06-01T12:30:00+02:00", + ); + sh_git(work.path(), &["push", "-q", &bare, "main"]); + + let blame = get_json( + &world, + &format!("/xrpc/org.tangled.git.getBlame?repo={did}&ref=main&path=many.txt"), + ) + .await; + assert_eq!(blame["runs"].as_array().unwrap().len(), 1); + assert_eq!(blame["runs"][0]["count"], 100_001); +} + +#[tokio::test] +async fn blame_rejects_a_large_historical_blob_behind_a_small_target() { + let world = World::new(); + let (did, bare, work) = empty_repo(&world, &RepoName::new("barnacle").unwrap()); + let large = "x\n".repeat(524_289); + commit_file( + work.path(), + "history.txt", + large.as_bytes(), + "large ancestor", + "2026-06-01T12:30:00+02:00", + ); + commit_file( + work.path(), + "history.txt", + b"x\nsentinel\n", + "small target", + "2026-06-01T12:31:00+02:00", + ); + sh_git(work.path(), &["push", "-q", &bare, "main"]); + + let (status, error) = get_error( + &world, + &format!("/xrpc/org.tangled.git.getBlame?repo={did}&ref=main&path=history.txt"), + ) + .await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error, "BlameTooLarge"); +} + +#[tokio::test] +async fn archive_rejects_traversal_prefixes_and_sanitizes_the_filename() { + let world = World::new(); + let (did, work) = seeded(&world, &RepoName::new("cockle").unwrap()); + let bare = world + .layout + .repo_path(&did) + .unwrap() + .to_str() + .unwrap() + .to_string(); + + let (status, error) = get_error( + &world, + &format!("/xrpc/org.tangled.git.getArchive?repo={did}&ref=main&prefix=../evil"), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(error, "InvalidRequest"); + + sh_git(work.path(), &["branch", "a\"b"]); + sh_git(work.path(), &["push", "-q", &bare, "refs/heads/a\"b"]); + let (status, headers, _) = get( + &world, + &format!("/xrpc/org.tangled.git.getArchive?repo={did}&ref=a%22b&prefix=cockle"), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "a quote in the ref name mustn't break the headers" + ); + let disposition = headers + .get(header::CONTENT_DISPOSITION) + .unwrap() + .to_str() + .unwrap(); + assert_eq!( + disposition, "attachment; filename=\"cockle.tar.gz\"", + "the filename comes from the caller's prefix, so header quoting stays sound" + ); +} + +#[tokio::test] +async fn an_archive_larger_than_the_configured_limit_is_refused() { + let world = World::with_archive_limit(ArchiveLimit::new(64)); + let (did, _work) = seeded(&world, &RepoName::new("murex").unwrap()); + + let (status, error) = get_error( + &world, + &format!("/xrpc/org.tangled.git.getArchive?repo={did}&ref=main&prefix=murex"), + ) + .await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error, "ArchiveTooLarge"); +} + +#[tokio::test] +async fn an_oversized_read_response_is_refused() { + let world = World::with_response_limit(ResponseLimit::new(256)); + let (did, _work) = seeded(&world, &RepoName::new("clam").unwrap()); + + let (status, error) = get_error( + &world, + &format!("/xrpc/org.tangled.git.listCommits?repo={did}&ref=main"), + ) + .await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error, "RequestTooLarge"); + + let (status, error) = get_error( + &world, + &format!("/xrpc/org.tangled.git.getBlame?repo={did}&ref=main&path=README.md"), + ) + .await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error, "RequestTooLarge"); + + let small = get_json( + &world, + &format!("/xrpc/org.tangled.git.getBranch?repo={did}&name=main"), + ) + .await; + assert_eq!(small["name"], "main"); +} + +#[tokio::test] +async fn languages_omit_files_with_zero_size() { + let world = World::new(); + let (did, bare, work) = empty_repo(&world, &RepoName::new("topshell").unwrap()); + commit_file( + work.path(), + "lib.rs", + b"", + "empty rust file", + "2026-06-01T12:30:00+02:00", + ); + sh_git(work.path(), &["push", "-q", &bare, "main"]); + + let value = get_json( + &world, + &format!("/xrpc/org.tangled.git.listLanguages?repo={did}&ref=main"), + ) + .await; + assert_eq!( + value["languages"], + serde_json::json!([]), + "zero-byte file mustn't surface as a language with a NaN percentage" + ); + assert!(value.get("totalSize").is_none()); + assert!(value.get("totalFiles").is_none()); +} + +#[tokio::test] +async fn list_records_serves_the_grant_collections() { + let world = World::administered(); + let member = AccountDid::new("did:plc:limpet").unwrap(); + let knot_repo = RepoDid::new(world.state.knot_did.as_str()).unwrap(); + let (status, body) = post_authed( + &world, + &nsid("com.atproto.repo.createRecord"), + &owner(), + common::grant_body( + &knot_repo, + &common::member_collection(), + &member, + "1970-01-01T00:16:40Z", + ), + ) + .await; + assert_eq!(status, StatusCode::OK, "the member offer failed: {body}"); + + let members = get_json( + &world, + &format!( + "/xrpc/com.atproto.repo.listRecords?repo={}&collection=org.tangled.knot.member", + common::percent_encode(knot_repo.as_str()), + ), + ) + .await; + let records = members["records"].as_array().unwrap(); + assert_eq!(records.len(), 1); + assert_eq!( + records[0]["uri"], + format!("at://{knot_repo}/org.tangled.knot.member/{member}"), + "the member record is keyed by the offered member's DID" + ); + assert_eq!(records[0]["value"]["$type"], "org.tangled.knot.member"); + assert!(records[0]["value"]["createdAt"].as_str().is_some()); + + let (repo, _bare, _work) = empty_repo(&world, &RepoName::new("squid").unwrap()); + let collaborator = AccountDid::new("did:plc:whelk").unwrap(); + let (status, body) = post_as( + &world, + &nsid("com.atproto.repo.createRecord"), + &owner(), + &knot_types::ServiceDid::new(repo.as_str()).unwrap(), + common::grant_body( + &repo, + &common::collaborator_collection(), + &collaborator, + "1970-01-01T00:33:20Z", + ), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "the collaborator offer failed: {body}" + ); + let collaborators = get_json( + &world, + &format!( + "/xrpc/com.atproto.repo.listRecords?repo={}&collection=org.tangled.graph.collaborator", + common::percent_encode(repo.as_str()), + ), + ) + .await; + let records = collaborators["records"].as_array().unwrap(); + assert_eq!(records.len(), 1); + assert_eq!( + records[0]["uri"], + format!("at://{repo}/org.tangled.graph.collaborator/{collaborator}"), + "collaborator records are scoped to their repository" + ); +} + +async fn offer_member(world: &World, member: &AccountDid) { + let knot_repo = RepoDid::new(world.state.knot_did.as_str()).unwrap(); + let (status, body) = post_authed( + world, + &nsid("com.atproto.repo.createRecord"), + &owner(), + common::grant_body( + &knot_repo, + &common::member_collection(), + member, + "2026-09-11T00:00:00Z", + ), + ) + .await; + assert_eq!(status, StatusCode::OK, "the member offer failed: {body}"); +} + +async fn offer_collaborator(world: &World, repo: &RepoDid, collaborator: &AccountDid) { + let (status, body) = post_as( + world, + &nsid("com.atproto.repo.createRecord"), + &owner(), + &knot_types::ServiceDid::new(repo.as_str()).unwrap(), + common::grant_body( + repo, + &common::collaborator_collection(), + collaborator, + "2026-09-11T00:00:00Z", + ), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "the collaborator offer failed: {body}" + ); +} + +async fn listing(world: &World, repo: &RepoDid, query: &str) -> serde_json::Value { + get_json( + world, + &format!( + "/xrpc/com.atproto.repo.listRecords?repo={}&{query}", + common::percent_encode(repo.as_str()), + ), + ) + .await +} + +fn subjects(page: &serde_json::Value) -> Vec { + page["records"] + .as_array() + .unwrap() + .iter() + .map(|record| { + record["uri"] + .as_str() + .unwrap() + .rsplit('/') + .next() + .unwrap() + .to_string() + }) + .collect() +} + +#[tokio::test] +async fn the_member_listing_walks_its_cursor_in_rkey_order() { + let world = World::administered(); + let knot_repo = RepoDid::new(world.state.knot_did.as_str()).unwrap(); + let whelk = AccountDid::new("did:plc:whelk").unwrap(); + let limpet = AccountDid::new("did:plc:limpet").unwrap(); + let scallop = AccountDid::new("did:plc:scallop").unwrap(); + offer_member(&world, &whelk).await; + offer_member(&world, &limpet).await; + offer_member(&world, &scallop).await; + + let whole = listing(&world, &knot_repo, "collection=org.tangled.knot.member").await; + assert_eq!( + subjects(&whole), + [limpet.as_str(), scallop.as_str(), whelk.as_str()], + "records are keyed by the member's DID, so the page orders on that key, not on when each grant was added" + ); + assert!( + whole["cursor"].is_null(), + "a page holding every record ends without a cursor" + ); + + let page = listing( + &world, + &knot_repo, + "collection=org.tangled.knot.member&limit=2", + ) + .await; + assert_eq!(subjects(&page), [limpet.as_str(), scallop.as_str()]); + let cursor = page["cursor"].as_str().unwrap(); + let rest = listing( + &world, + &knot_repo, + &format!( + "collection=org.tangled.knot.member&limit=2&cursor={}", + common::percent_encode(cursor), + ), + ) + .await; + assert_eq!( + subjects(&rest), + [whelk.as_str()], + "the cursor resumes strictly past the last key it served" + ); + assert!(rest["cursor"].is_null(), "a drained listing has no cursor"); + + let flipped = listing( + &world, + &knot_repo, + "collection=org.tangled.knot.member&reverse=true&limit=2", + ) + .await; + assert_eq!( + subjects(&flipped), + [whelk.as_str(), scallop.as_str()], + "reverse turns the page around without breaking the order" + ); +} + +#[tokio::test] +async fn the_grant_listings_reject_malformed_params_and_clamp_the_limit() { + let world = World::administered(); + let knot_repo = RepoDid::new(world.state.knot_did.as_str()).unwrap(); + offer_member(&world, &AccountDid::new("did:plc:limpet").unwrap()).await; + let member_query = format!( + "/xrpc/com.atproto.repo.listRecords?repo={}&collection=org.tangled.knot.member", + common::percent_encode(knot_repo.as_str()), + ); + + let malformed = [ + format!("{member_query}&limit=abc"), + format!("{member_query}&cursor=not%40a%20rkey"), + format!("{member_query}&reverse=maybe"), + "/xrpc/com.atproto.repo.listRecords?repo=did:plc:not%20a%20did&collection=org.tangled.knot.member" + .to_string(), + format!( + "/xrpc/com.atproto.repo.listRecords?repo={}&collection=nope", + common::percent_encode(knot_repo.as_str()), + ), + ]; + let w = &world; + stream::iter(malformed.iter()) + .for_each(|query| async move { + let (status, error) = get_error(w, query).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "query {query}"); + assert_eq!(error, "InvalidRequest", "query {query}"); + }) + .await; + + let clamped = get_json(&world, &format!("{member_query}&limit=5000")).await; + assert_eq!( + subjects(&clamped).len(), + 1, + "an oversize limit clamps to the max instead of erroring" + ); + let floor = get_json(&world, &format!("{member_query}&limit=0")).await; + assert_eq!( + subjects(&floor).len(), + 1, + "a zero limit clamps up to one record" + ); +} + +#[tokio::test] +async fn grant_listings_stay_scoped_to_their_own_account() { + let world = World::administered(); + let knot_repo = RepoDid::new(world.state.knot_did.as_str()).unwrap(); + let member = AccountDid::new("did:plc:limpet").unwrap(); + offer_member(&world, &member).await; + let (squid, _squid_bare, _squid_work) = empty_repo(&world, &RepoName::new("squid").unwrap()); + let (clam, _clam_bare, _clam_work) = empty_repo(&world, &RepoName::new("clam").unwrap()); + let lyna = AccountDid::new("did:plc:lyna").unwrap(); + let bailey = AccountDid::new("did:plc:bailey").unwrap(); + offer_collaborator(&world, &squid, &lyna).await; + offer_collaborator(&world, &clam, &bailey).await; + + let members = listing(&world, &knot_repo, "collection=org.tangled.knot.member").await; + assert_eq!(subjects(&members), [member.as_str()]); + assert_eq!( + members["records"][0]["uri"], + format!("at://{knot_repo}/org.tangled.knot.member/{member}"), + "member records serve under the knot's own did" + ); + assert!( + members["records"][0]["value"]["createdAt"] + .as_str() + .is_some(), + "the grant record keeps its createdAt" + ); + + assert_eq!( + subjects(&listing(&world, &squid, "collection=org.tangled.graph.collaborator").await), + [lyna.as_str()] + ); + assert_eq!( + subjects(&listing(&world, &clam, "collection=org.tangled.graph.collaborator").await), + [bailey.as_str()], + "collaborator records never cross repositories" + ); + assert_eq!( + subjects( + &listing( + &world, + &knot_repo, + "collection=org.tangled.graph.collaborator" + ) + .await + ), + Vec::::new(), + "the knot account holds no collaborator records" + ); + assert_eq!( + subjects(&listing(&world, &squid, "collection=org.tangled.knot.member").await), + Vec::::new(), + "a repository holds no member records" + ); +} + +#[tokio::test] +async fn a_banned_member_keeps_its_entry_on_the_members_grant_list() { + let world = World::new(); + world.add_member( + &AccountDid::new("did:plc:whelk").unwrap(), + &owner(), + UnixSeconds::new(1_000), + ); + world.ban( + &AccountDid::new("did:plc:whelk").unwrap(), + UnixSeconds::new(4_000), + ); + assert_eq!( + world + .state + .index + .effective_member(&AccountDid::new("did:plc:whelk").unwrap()), + Resolved::Ready(true), + "the blocklist runs its own grant list, and a ban doesn't remove whelk's entry on the members grant list" + ); +} + +#[tokio::test] +async fn service_metadata_endpoints_answer() { + let world = World::new(); + let described = get_json(&world, "/xrpc/org.tangled.knot.describeKnot").await; + assert_eq!(described["version"], "v1.20.0"); + assert_eq!(described["did"], world.state.knot_did.as_str()); + let served = [ + Capability::Tangled1, + Capability::KnotAcl, + Capability::RepoDidInput, + Capability::AtprotoFirehose, + Capability::RepoTokenMint, + Capability::SocialWrites, + ] + .map(Capability::token); + assert_eq!(described["capabilities"], serde_json::json!(served)); + assert_eq!(described["links"], serde_json::json!({})); + assert_eq!(described["contact"], serde_json::json!({ "email": "" })); +} + +async fn serve_router(world: &World) -> std::net::SocketAddr { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = world.router.clone(); + tokio::spawn(async move { + axum::serve( + listener, + router.into_make_service_with_connect_info::(), + ) + .await + .unwrap(); + }); + addr +} + +fn refused(result: Result) { + match result { + Err(tungstenite::Error::Http(response)) => { + assert_eq!(response.status().as_u16(), 503); + } + Err(other) => panic!("expected an http rejection: {other}"), + Ok(_) => panic!("a subscriber past the limit connected"), + } +} + +#[tokio::test] +async fn a_subscriber_beyond_the_firehose_limit_is_refused() { + let world = World::new(); + let addr = serve_router(&world).await; + let saturated: Vec<_> = (0..16u8) + .map(|octet| { + world + .state + .subscriber_gate + .try_admit(std::net::IpAddr::V4(std::net::Ipv4Addr::new( + 10, 0, 0, octet, + ))) + .expect("distinct peers fill the global limit") + }) + .collect(); + refused( + tokio_tungstenite::connect_async(format!( + "ws://{addr}/xrpc/com.atproto.sync.subscribeRepos" + )) + .await, + ); + drop(saturated); +} + +#[tokio::test] +async fn one_peer_opens_four_subscriptions_and_fifth_is_refused() { + let world = World::new(); + let addr = serve_router(&world).await; + let subscriptions: Vec<_> = stream::iter(0..4) + .then(|_| async { + tokio_tungstenite::connect_async(format!( + "ws://{addr}/xrpc/com.atproto.sync.subscribeRepos" + )) + .await + .expect("a connection within the per-peer limit is admitted") + .0 + }) + .collect() + .await; + refused( + tokio_tungstenite::connect_async(format!( + "ws://{addr}/xrpc/com.atproto.sync.subscribeRepos" + )) + .await, + ); + drop(subscriptions); +} + +#[tokio::test] +async fn set_default_branch_moves_head_and_the_refs_record_and_rejects_a_missing_one() { + let world = World::new(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + let (status, created) = post_authed( + &world, + &nsid("org.tangled.repo.createRepo"), + &owner(), + serde_json::json!({ "name": "coral", "defaultBranch": "main" }), + ) + .await; + assert_eq!(status, StatusCode::OK, "create failed: {created}"); + let did = RepoDid::new(created["did"].as_str().unwrap()).unwrap(); + + let work = tempfile::tempdir().unwrap(); + sh_git(work.path(), &["init", "-q", "-b", "main"]); + commit_file( + work.path(), + "README.md", + b"# coral\n", + "first", + "2026-06-01T12:30:00+02:00", + ); + let bare = world.layout.repo_path(&did).unwrap(); + sh_git(work.path(), &["push", "-q", bare.to_str().unwrap(), "main"]); + sh_git(work.path(), &["branch", "release", "main"]); + sh_git( + work.path(), + &["push", "-q", bare.to_str().unwrap(), "refs/heads/release"], + ); + + let (status, body) = post_as( + &world, + &nsid("org.tangled.git.setDefaultBranch"), + &owner(), + &knot_types::ServiceDid::new(did.as_str()).unwrap(), + serde_json::json!({ "repo": did, "branch": "release" }), + ) + .await; + assert_eq!(status, StatusCode::OK, "setDefaultBranch failed: {body}"); + + let repo = world.layout.open(&did).unwrap(); + assert_eq!( + repo.default_branch().unwrap().as_str(), + "refs/heads/release", + "the default head moved to the requested branch" + ); + let record = get_json( + &world, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={}&collection=org.tangled.git.refs&rkey=self", + common::percent_encode(did.as_str()), + ), + ) + .await; + assert_eq!( + record["value"]["defaultBranch"], "release", + "the refs record follows HEAD" + ); + assert_eq!( + (&body["uri"], &body["cid"]), + (&record["uri"], &record["cid"]), + "the reply names the refs record version it left behind" + ); + + let (status, body) = post_as( + &world, + &nsid("org.tangled.git.setDefaultBranch"), + &owner(), + &knot_types::ServiceDid::new(did.as_str()).unwrap(), + serde_json::json!({ "repo": did, "branch": "ghost" }), + ) + .await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "a default pointing at a branch the repo lacks: {body}" + ); +} + +#[tokio::test] +async fn delete_branch_removes_a_non_default_branch_then_reports_it_gone() { + let world = World::new(); + let (did, work) = seeded(&world, &RepoName::new("kelp").unwrap()); + let bare = world.layout.repo_path(&did).unwrap(); + sh_git(work.path(), &["branch", "feature", "main"]); + sh_git( + work.path(), + &["push", "-q", bare.to_str().unwrap(), "refs/heads/feature"], + ); + + let (status, _) = post_authed( + &world, + &nsid("org.tangled.git.deleteBranch"), + &owner(), + serde_json::json!({ "repo": did, "branch": "feature" }), + ) + .await; + assert_eq!(status, StatusCode::OK); + + let (status, body) = post_authed( + &world, + &nsid("org.tangled.git.deleteBranch"), + &owner(), + serde_json::json!({ "repo": did, "branch": "feature" }), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND, "second delete: {body}"); +} + +#[tokio::test] +async fn bad_post_bodies_are_invalid_request() { + let world = World::new(); + let (_kelp, _wk) = seeded(&world, &RepoName::new("kelp").unwrap()); + + let cases: &[(Nsid, serde_json::Value)] = &[( + nsid("org.tangled.git.deleteBranch"), + serde_json::json!({ + "repo": "did:plc:kelpfixture", + "branch": "bad branch", + }), + )]; + let w = &world; + stream::iter(cases) + .for_each(|(method, value)| async move { + assert_post_rejected(w, method, &owner(), value.clone()).await; + }) + .await; +} + +#[tokio::test] +async fn create_mints_a_did_plc_repo_with_the_requested_default_branch() { + let world = World::new(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + let (status, body) = post_authed( + &world, + &nsid("org.tangled.repo.createRepo"), + &owner(), + serde_json::json!({ "name": "squid", "defaultBranch": "trunk" }), + ) + .await; + assert_eq!(status, StatusCode::OK, "create failed: {body}"); + let repo_did = body["did"].as_str().unwrap(); + assert!( + repo_did.starts_with("did:plc:"), + "minted a did:plc: {repo_did}" + ); + let did = RepoDid::new(repo_did).unwrap(); + let repo = world.layout.open(&did).unwrap(); + assert_eq!( + repo.default_branch().unwrap().as_str(), + "refs/heads/trunk", + "the requested default branch became HEAD" + ); +} diff --git a/knot2/crates/knot-xrpc/tests/ref_records.rs b/knot2/crates/knot-xrpc/tests/ref_records.rs new file mode 100644 index 0000000000000000000000000000000000000000..085aeda40639aa81b502d9004c9d3a5db6c7a008 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/ref_records.rs @@ -0,0 +1,613 @@ +mod common; + +use common::{ + World, commit_file, empty_repo, get_json, http_push, owner, percent_encode, serve_pushes, + sh_git, +}; +use knot_events::{EventCursor, ReplayBounds, ReplayBytes, ReplayEvents}; +use knot_record::{chain, git_refs}; +use knot_types::{ + Did, KnotServiceUrl, RecordAddress, RecordCollection, RecordRkey, RepoDid, RepoName, + UnixSeconds, +}; +use serde::Deserialize; +use serde_json::Value; +use std::sync::Arc; + +use base64::Engine as _; + +fn seed(work: &std::path::Path, contents: &[u8]) { + commit_file( + work, + "README.md", + contents, + "first", + "2026-06-01T12:30:00+02:00", + ); +} + +async fn snapshot(world: &World, did: &RepoDid) -> Value { + get_json( + world, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={}&collection={}&rkey=self", + percent_encode(did.as_str()), + git_refs::GIT_REFS_COLLECTION, + ), + ) + .await +} + +/// The snapshot's packed refs are dag-cbor bytes, so JSON serves them +/// base64-wrapped. The test unpacks them back into " " lines. +fn packed_lines(record: &Value) -> Vec { + let wrapped = record["value"]["refs"]["$bytes"] + .as_str() + .expect("the refs field is the wrapped bytes"); + let bytes = base64::engine::general_purpose::STANDARD_NO_PAD + .decode(wrapped) + .expect("the knot wrote standard base64"); + String::from_utf8(bytes) + .expect("the packed refs are utf-8") + .lines() + .map(str::to_owned) + .collect() +} + +fn sha_of(lines: &[String], refname: &str) -> String { + lines + .iter() + .find_map(|line| line.strip_suffix(&format!(" {refname}"))) + .unwrap_or_else(|| panic!("no {refname} in {lines:?}")) + .to_owned() +} + +#[derive(Debug, Deserialize)] +struct OpProbe { + action: String, + path: String, +} + +#[derive(Deserialize)] +struct CommitProbe { + #[allow(dead_code)] + seq: u64, + ops: Vec, +} + +fn decoded(frame: &[u8]) -> (&'static str, Option) { + let mut read = frame; + let header: Value = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + match header["t"].as_str().unwrap() { + "#commit" => { + let payload: CommitProbe = serde_ipld_dagcbor::de::from_reader_once(&mut read).unwrap(); + ("#commit", Some(payload)) + } + "#sync" => ("#sync", None), + _ => ("#ignored", None), + } +} + +fn frames(world: &World) -> Vec<(&'static str, Option)> { + world + .state + .firehose + .replay( + EventCursor::START, + ReplayBounds::new( + ReplayEvents::new(4096).unwrap(), + ReplayBytes::new(16 << 20).unwrap(), + ), + ) + .events + .iter() + .map(|event| decoded(&event.frame)) + .filter(|(kind, _)| *kind != "#ignored") + .collect() +} + +#[tokio::test] +async fn a_push_stores_the_refs_snapshot_the_knot_serves_over_com_atproto() { + let world = World::new(); + let (did, _bare, work) = empty_repo(&world, &RepoName::new("squid").unwrap()); + seed(work.path(), b"hello\n"); + sh_git(work.path(), &["tag", "v1"]); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &did, &owner(), &["main", "v1"]).await; + + let record = snapshot(&world, &did).await; + assert_eq!( + record["value"]["$type"], + git_refs::GIT_REFS_COLLECTION, + "the stored bytes include their own type" + ); + assert_eq!( + record["value"]["defaultBranch"], "main", + "the snapshot reports the default branch" + ); + let lines = packed_lines(&record); + assert_eq!( + lines, + vec![ + format!( + "{} refs/heads/main", + sh_git(work.path(), &["rev-parse", "refs/heads/main"]) + ), + format!( + "{} refs/tags/v1", + sh_git(work.path(), &["rev-parse", "refs/tags/v1"]) + ), + ], + "the packed refs list every live ref, sorted by name" + ); + + let kinds = frames(&world) + .into_iter() + .map(|(kind, _)| kind) + .collect::>(); + assert!( + kinds.contains(&"#commit"), + "the push's ops ride #commit frames" + ); + assert!( + kinds.contains(&"#sync"), + "a chain born from this push closes with #sync" + ); + + sh_git(work.path(), &["branch", "feature"]); + http_push(work.path(), addr, &did, &owner(), &["feature"]).await; + let lines = packed_lines(&snapshot(&world, &did).await); + assert_eq!( + lines.len(), + 3, + "the branch is written into the snapshot before anything deletes it" + ); + + http_push(work.path(), addr, &did, &owner(), &[":feature"]).await; + let lines = packed_lines(&snapshot(&world, &did).await); + assert_eq!( + lines.len(), + 2, + "the snapshot no longer lists the deleted branch" + ); + assert!( + lines + .iter() + .all(|line| !line.ends_with("refs/heads/feature")), + "the repo serves exactly the refs in the packed refs" + ); +} + +#[tokio::test] +async fn the_refs_snapshot_appears_on_the_firehose_as_a_create_then_updates() { + let world = World::new(); + let (did, _bare, work) = empty_repo(&world, &RepoName::new("auger").unwrap()); + seed(work.path(), b"hello\n"); + sh_git(work.path(), &["branch", "feature"]); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &did, &owner(), &["main", "feature"]).await; + + let snapshot_ops = |world: &World| { + frames(world) + .into_iter() + .filter_map(|(_, payload)| payload) + .flat_map(|payload| payload.ops) + .filter(|op| op.path == format!("{}/self", git_refs::GIT_REFS_COLLECTION)) + .map(|op| op.action) + .collect::>() + }; + assert_eq!( + snapshot_ops(&world), + ["create"], + "the push's #commit frame includes the snapshot record's create" + ); + + http_push(work.path(), addr, &did, &owner(), &[":feature"]).await; + assert_eq!( + snapshot_ops(&world), + ["create", "update"], + "a branch delete is another update to the snapshot record, never a delete op" + ); +} + +#[tokio::test] +async fn repushing_identical_refs_is_a_no_op() { + let world = World::new(); + let (did, _bare, work) = empty_repo(&world, &RepoName::new("anemone").unwrap()); + seed(work.path(), b"first\n"); + sh_git(work.path(), &["checkout", "-q", "-b", "feature"]); + commit_file( + work.path(), + "kelp.txt", + b"frond\n", + "second", + "2026-06-01T12:31:00+02:00", + ); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &did, &owner(), &["main", "feature"]).await; + + let latest_path = format!( + "/xrpc/com.atproto.sync.getLatestCommit?did={}", + percent_encode(did.as_str()), + ); + let settled = get_json(&world, &latest_path).await; + let record = snapshot(&world, &did).await; + let frames_before = frames(&world).len(); + + http_push(work.path(), addr, &did, &owner(), &["main", "feature"]).await; + assert_eq!( + get_json(&world, &latest_path).await, + settled, + "a no-op re-push, the shape git push --mirror sends for every current ref, won't move a commit" + ); + assert_eq!( + snapshot(&world, &did).await, + record, + "the stored snapshot bytes are what the first push wrote" + ); + assert_eq!( + frames(&world).len(), + frames_before, + "a no-op re-push doesn't emit firehose frames" + ); +} + +#[tokio::test] +async fn the_legacy_sweep_deletes_old_ref_records_and_survives_its_chunking() { + let world = World::new(); + let (did, _bare, work) = empty_repo(&world, &RepoName::new("atoll").unwrap()); + seed(work.path(), b"first\n"); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &did, &owner(), &["main"]).await; + let snapshot_before = snapshot(&world, &did).await; + + let legacy = RecordCollection::new("sh.tangled.git.ref").unwrap(); + let ops: Vec<(RecordAddress, chain::RecordOp)> = (0..205) + .map(|nth| { + ( + RecordAddress::new( + legacy.clone(), + RecordRkey::new(format!("legacy~{nth:03}")).unwrap(), + ), + chain::RecordOp::Create { + record: knot_types::RecordBody::new( + serde_ipld_dagcbor::to_vec(&serde_json::json!({ + "$type": "sh.tangled.git.ref", + "sha": "0".repeat(40), + "createdAt": "2026-06-01T12:30:00Z", + })) + .unwrap(), + ) + .unwrap(), + }, + ) + }) + .collect(); + let layout = world.layout.clone(); + let chunks = tokio::task::spawn_blocking({ + let state = Arc::clone(&world.state); + let did = did.clone(); + move || { + let git = layout.open(&did).unwrap(); + let signer = state.secrets.signer(&state.knot_did).unwrap(); + let now = UnixSeconds::new(2_000); + let mut emit = knot_xrpc::FirehoseEmit::new(&state.firehose, &did, now); + let chunks = futures::executor::block_on(chain::apply( + &git, &did, &ops, &signer, &mut emit, now, + )) + .unwrap(); + emit.finish().unwrap(); + chunks + } + }) + .await + .unwrap(); + assert_eq!( + chunks.len(), + 2, + "205 creates split past the 200 ops each commit allows" + ); + let seeded = get_json( + &world, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={}&collection=sh.tangled.git.ref&rkey=legacy~000", + percent_encode(did.as_str()), + ), + ) + .await; + assert_eq!(seeded["value"]["$type"], "sh.tangled.git.ref"); + + let swept = tokio::task::spawn_blocking({ + let state = Arc::clone(&world.state); + move || knot_xrpc::migrate::legacy_state(&state) + }) + .await + .unwrap() + .unwrap(); + assert_eq!( + swept, 2, + "the sweep's deletes split across 2 chain commits of their own" + ); + let empty = get_json( + &world, + &format!( + "/xrpc/com.atproto.repo.listRecords?repo={}&collection=sh.tangled.git.ref", + percent_encode(did.as_str()), + ), + ) + .await; + assert!( + empty["records"].as_array().unwrap().is_empty(), + "the legacy collection serves an empty page after the sweep" + ); + let (status, error) = common::get_error( + &world, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={}&collection=sh.tangled.git.ref&rkey=legacy~000", + percent_encode(did.as_str()), + ), + ) + .await; + assert_eq!( + (status, error.as_str()), + (http::StatusCode::BAD_REQUEST, "RecordNotFound") + ); + assert_eq!( + snapshot(&world, &did).await, + snapshot_before, + "the sweep won't touch the refs snapshot the push wrote" + ); + let deletes = frames(&world) + .into_iter() + .filter_map(|(_, payload)| payload) + .flat_map(|payload| payload.ops) + .filter(|op| op.action == "delete" && op.path.starts_with("sh.tangled.git.ref/")) + .count(); + assert_eq!( + deletes, 205, + "the sweep will emit a firehose delete for every legacy record, across the chunk boundary" + ); + + let frames_after_sweep = frames(&world).len(); + let again = tokio::task::spawn_blocking({ + let state = Arc::clone(&world.state); + move || knot_xrpc::migrate::legacy_state(&state) + }) + .await + .unwrap() + .unwrap(); + assert_eq!(again, 0, "a second sweep is a no-op"); + assert_eq!( + frames(&world).len(), + frames_after_sweep, + "the second sweep doesn't emit frames" + ); + assert_eq!( + snapshot(&world, &did).await, + snapshot_before, + "the snapshot is unchanged after the second sweep" + ); +} + +#[tokio::test] +async fn the_legacy_sweep_drops_leftover_hidden_refs() { + let world = World::new(); + let (did, bare, work) = empty_repo(&world, &RepoName::new("lagoon").unwrap()); + let bare = std::path::Path::new(&bare); + seed(work.path(), b"first\n"); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &did, &owner(), &["main"]).await; + + let repo = world.layout.open(&did).unwrap(); + let main_ref = knot_types::RefName::new("refs/heads/main").unwrap(); + let main = repo.find_ref(&main_ref).unwrap().unwrap(); + repo.update_ref(&knot_git::RefUpdate::Create { + name: knot_types::RefName::new("refs/hidden/feature/main").unwrap(), + new: main, + }) + .unwrap(); + + let sweep = || { + let state = Arc::clone(&world.state); + tokio::task::spawn_blocking(move || knot_xrpc::migrate::legacy_state(&state)) + }; + sweep().await.unwrap().unwrap(); + + let hidden = sh_git( + bare, + &["for-each-ref", "--format=%(refname)", "refs/hidden"], + ); + assert!(hidden.is_empty(), "hidden refs are swept: {hidden}"); + assert_eq!( + world + .layout + .open(&did) + .unwrap() + .find_ref(&main_ref) + .unwrap(), + Some(main), + "public refs are untouched" + ); + + assert_eq!( + sweep().await.unwrap().unwrap(), + 0, + "a second sweep is a no-op" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_boot_pass_catches_up_refs_that_moved_without_a_record() { + let world = World::new(); + let (did, _bare, work) = empty_repo(&world, &RepoName::new("coral").unwrap()); + seed(work.path(), b"first\n"); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &did, &owner(), &["main"]).await; + + let repo = world.layout.open(&did).unwrap(); + let main = repo + .find_ref(&knot_types::RefName::new("refs/heads/main").unwrap()) + .unwrap() + .unwrap(); + repo.update_ref(&knot_git::RefUpdate::Create { + name: knot_types::RefName::new("refs/heads/ghost").unwrap(), + new: main, + }) + .unwrap(); + let state = std::sync::Arc::clone(&world.state); + tokio::task::spawn_blocking(move || knot_xrpc::materialize::enable_surfaces(&state)) + .await + .unwrap(); + + let lines = packed_lines(&snapshot(&world, &did).await); + assert_eq!(sha_of(&lines, "refs/heads/ghost"), main.to_hex()); + assert_eq!(sha_of(&lines, "refs/heads/main"), main.to_hex()); + assert_eq!( + frames(&world).last().map(|(kind, _)| kind), + Some(&"#sync"), + "recovery closes with a #sync frame" + ); + + let settled = chain::tip(&world.layout.open(&did).unwrap()) + .unwrap() + .unwrap(); + let state = std::sync::Arc::clone(&world.state); + tokio::task::spawn_blocking(move || knot_xrpc::materialize::enable_surfaces(&state)) + .await + .unwrap(); + assert_eq!( + chain::tip(&world.layout.open(&did).unwrap()) + .unwrap() + .unwrap() + .git, + settled.git, + "a second boot without drift doesn't move the tip" + ); +} + +#[tokio::test] +async fn created_fork_serves_its_upstream_refs_at_birth() { + let world = World::administered(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + let (source_did, _bare, work) = empty_repo(&world, &RepoName::new("anchorage").unwrap()); + seed(work.path(), b"base\n"); + sh_git(work.path(), &["tag", "v1"]); + let addr = serve_pushes(&world).await; + http_push(work.path(), addr, &source_did, &owner(), &["main", "v1"]).await; + + world.publish_plc_pds( + &Did::new_owned(source_did.as_str()).unwrap(), + &KnotServiceUrl::new("https://knot.nel.pet").unwrap(), + ); + let (status, created) = common::post_authed( + &world, + &common::nsid("org.tangled.repo.createRepo"), + &owner(), + serde_json::json!({ + "name": "reef", + "source": { + "$type": "org.tangled.repo.defs#sourceTangled", + "did": &source_did, + }, + }), + ) + .await; + assert_eq!( + status, + http::StatusCode::OK, + "fork create failed: {created}" + ); + let fork_did = RepoDid::new( + created["did"] + .as_str() + .expect("a created repo returns its did"), + ) + .unwrap(); + + // the fork populates through the background importer, so the birth records are written + // once its fetch finishes; the timeout only bounds the failure, never the success + let before = world.state.imports.processed(); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while world.state.imports.processed() == before { + tokio::task::yield_now().await; + } + }) + .await + .expect("the importer never finished the fork's fetch"); + let record = snapshot(&world, &fork_did).await; + assert_eq!( + record["value"]["defaultBranch"], "main", + "the fork's snapshot reports the default branch it took from the upstream" + ); + let lines = packed_lines(&record); + assert_eq!( + sha_of(&lines, "refs/heads/main"), + sh_git(work.path(), &["rev-parse", "refs/heads/main"]), + "the seeded snapshot reports the upstream tip" + ); + assert_eq!( + sha_of(&lines, "refs/tags/v1"), + sh_git(work.path(), &["rev-parse", "refs/tags/v1"]), + "the seeded snapshot includes the upstream's tag" + ); +} + +#[tokio::test] +async fn create_repo_writes_its_manifest_into_the_repo_account() { + let world = World::new(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + let (status, created) = common::post_authed( + &world, + &common::nsid("org.tangled.repo.createRepo"), + &owner(), + serde_json::json!({ + "name": "Lagoon", + "description": "a shallow lagoon", + "topics": ["reef", "tide"], + "website": "https://lagoon.example/", + }), + ) + .await; + assert_eq!(status, http::StatusCode::OK, "create failed: {created}"); + let did = created["did"].as_str().unwrap(); + + let record = get_json( + &world, + &format!( + "/xrpc/com.atproto.repo.getRecord?repo={did}&collection=org.tangled.repo.manifest&rkey=self" + ), + ) + .await; + let value = &record["value"]; + assert_eq!(value["$type"], "org.tangled.repo.manifest"); + assert_eq!(value["declaration"]["owner"], owner().as_str()); + assert_eq!(value["declaration"]["name"], "Lagoon"); + assert_eq!(value["description"], "a shallow lagoon"); + assert_eq!(value["topics"], serde_json::json!(["reef", "tide"])); + assert_eq!(value["website"], "https://lagoon.example/"); + + assert!( + frames(&world) + .iter() + .any(|(_, payload)| payload.as_ref().is_some_and(|payload| { + payload + .ops + .iter() + .any(|op| op.action == "create" && op.path == "org.tangled.repo.manifest/self") + })), + "the manifest's create rides the firehose" + ); +} + +#[tokio::test] +async fn the_legacy_create_route_is_gone() { + let world = World::new(); + let (status, _) = common::post_authed( + &world, + &common::nsid("sh.tangled.repo.create"), + &owner(), + serde_json::json!({ "rkey": "lagoon", "name": "lagoon" }), + ) + .await; + assert_eq!(status, http::StatusCode::NOT_FOUND); +} diff --git a/knot2/crates/knot-xrpc/tests/retire.rs b/knot2/crates/knot-xrpc/tests/retire.rs new file mode 100644 index 0000000000000000000000000000000000000000..5ab5f3f7ddc0f440ea52aa8ff331cc84eac8c281 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/retire.rs @@ -0,0 +1,503 @@ +mod common; + +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use common::{World, basic_auth, get_json, serve_router}; +use http::StatusCode; +use knot_cob::CobStore; +use knot_cobs::RepoRegistryCob; +use knot_runtime::{K256Signer, OsEntropy, Signer as _}; +use knot_types::{KnotId, KnotServiceUrl, ObjectFormat, RepoDid, UnixSeconds}; + +const FOSTER_SECRET: &str = "foster-pearl"; +const FOSTER_OWNER: &str = "did:plc:boltfulboy"; +const REPO: &str = "did:plc:feralpearl"; +const COLLECTION: &str = "org.tangled.track.ticket"; +const RKEY: &str = "1"; +const ACK_PATH: &str = "/xrpc/org.tangled.temp.server.ackHandover"; +const ORIGIN_DID: &str = "did:web:origin.test"; +const ACK_CONTEXT: &str = "org.tangled.temp.handover.ack.v1"; + +fn repo_did() -> RepoDid { + RepoDid::new(REPO).unwrap() +} + +fn ticket_record(text: &str) -> serde_json::Value { + serde_json::json!({"collection": COLLECTION, "rkey": RKEY, "author": FOSTER_OWNER, "createdAt": 1100, + "record": {"$type": COLLECTION, + "title": {"$type": "org.tangled.markup.markdown#inline", "text": text}, + "state": "open", "createdAt": "1970-01-01T00:18:20Z", "x-tngl-editor": FOSTER_OWNER}}) +} + +async fn post_seed(world: &World, records: serde_json::Value) -> (StatusCode, serde_json::Value) { + let (status, body) = common::post_raw( + world, + "/xrpc/org.tangled.temp.server.seedRepo", + basic_auth("admin", FOSTER_SECRET), + serde_json::json!({ + "repoDid": REPO, "records": records, + "manifest": {"$type": "org.tangled.repo.manifest", + "declaration": { "owner": FOSTER_OWNER, "name": "feralpearl" }}, + }), + ) + .await; + let parsed = serde_json::from_slice(&body).unwrap_or(serde_json::Value::Null); + (status, parsed) +} + +async fn seed_foster_record(world: &World) { + let (status, body) = + post_seed(world, serde_json::json!([ticket_record("a feral pearl")])).await; + assert_eq!(status, StatusCode::OK, "seed failed: {body}"); +} + +fn json_get(payload: serde_json::Value) -> axum::routing::MethodRouter { + axum::routing::get(move || { + let payload = payload.clone(); + async move { axum::Json(payload) } + }) +} + +fn knot_origin(signer: &K256Signer, capabilities: &[&str]) -> axum::Router { + let document = knot_atproto::knot_did_document( + &KnotId::new(ORIGIN_DID).unwrap(), + &signer.public_key(), + &KnotServiceUrl::new("https://origin.test").unwrap(), + ); + axum::Router::new() + .route( + "/xrpc/org.tangled.knot.describeKnot", + json_get(serde_json::json!({"did": ORIGIN_DID, "capabilities": capabilities})), + ) + .route("/.well-known/did.json", json_get(document)) +} + +fn knot_social_origin(signer: &K256Signer) -> axum::Router { + knot_origin(signer, &["knot-social-writes"]) +} + +async fn serve_origin(world: &World, signer: &K256Signer) -> String { + let path = + format!("/xrpc/com.atproto.repo.getRecord?repo={REPO}&collection={COLLECTION}&rkey={RKEY}"); + let fetched = get_json(world, &path).await; + let cid = fetched["cid"].as_str().unwrap().to_string(); + let origin = knot_social_origin(signer).route( + "/xrpc/com.atproto.repo.getRecord", + json_get( + serde_json::json!({"uri": format!("at://{REPO}/{COLLECTION}/{RKEY}"), + "cid": cid, "value": fetched["value"].clone()}), + ), + ); + format!("http://{}", serve_router(origin).await) +} + +#[tokio::test] +async fn a_seed_declaring_sha256_mints_a_sha256_shadow_on_a_sha1_knot() { + let world = World::fostering(FOSTER_SECRET); + let (status, body) = common::post_raw( + &world, + "/xrpc/org.tangled.temp.server.seedRepo", + basic_auth("admin", FOSTER_SECRET), + serde_json::json!({ + "repoDid": REPO, "records": [], "objectFormat": "sha256", + "manifest": {"$type": "org.tangled.repo.manifest", + "declaration": { "owner": FOSTER_OWNER, "name": "feralpearl" }}, + }), + ) + .await; + assert_eq!(status, StatusCode::OK, "seed failed: {body:?}"); + let shadow = world.layout.open(&repo_did()).unwrap(); + assert_eq!(shadow.object_format(), ObjectFormat::SHA256); +} + +#[derive(serde::Serialize)] +struct AckSignedView<'a> { + context: &'a str, + #[serde(rename = "repoDid")] + repo_did: &'a str, + records: u64, + #[serde(rename = "ackedAt")] + acked_at: i64, +} + +fn signed_ack(signer: &K256Signer, acked_at: i64) -> serde_json::Value { + let view = AckSignedView { + context: ACK_CONTEXT, + repo_did: REPO, + records: 1, + acked_at, + }; + let bytes = serde_ipld_dagcbor::to_vec(&view).unwrap(); + let signature = URL_SAFE_NO_PAD.encode(signer.sign(&bytes).as_bytes()); + serde_json::json!({"context": ACK_CONTEXT, "repoDid": REPO, "records": 1, + "ackedAt": acked_at, "signature": signature}) +} + +async fn post_ack(world: &World, ack: serde_json::Value) -> (StatusCode, serde_json::Value) { + let (status, body) = common::post_raw(world, ACK_PATH, http::HeaderMap::new(), ack).await; + let parsed = serde_json::from_slice(&body).unwrap_or(serde_json::Value::Null); + (status, parsed) +} + +fn retired_at(world: &World) -> Option { + let meta = knot_git::Repo::open(&world.state.meta_path).unwrap(); + let store = CobStore::new(&meta); + let object = store.list::().unwrap()[0]; + let (registry, _) = store.materialize::(object).unwrap(); + registry + .record_of(&repo_did()) + .and_then(|record| record.retired_at.map(UnixSeconds::get)) +} + +async fn seeded_origin() -> (World, K256Signer, String) { + let foster = World::fostering_with_grace(FOSTER_SECRET); + seed_foster_record(&foster).await; + let signer = K256Signer::generate(&OsEntropy); + let origin_url = serve_origin(&foster, &signer).await; + foster.publish_plc_knot(REPO, &origin_url); + (foster, signer, origin_url) +} + +#[tokio::test] +async fn valid_ack_retires_shadow() { + let (foster, signer, _) = seeded_origin().await; + + let (status, body) = post_ack(&foster, signed_ack(&signer, 1100)).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + retired_at(&foster), + Some(1100), + "retiredAt follows the ack's ackedAt" + ); +} + +#[tokio::test] +async fn retired_shadow_returns_409_to_further_seeds() { + let (foster, signer, _) = seeded_origin().await; + let (status, body) = post_ack(&foster, signed_ack(&signer, 1100)).await; + assert_eq!(status, StatusCode::OK, "{body}"); + + let (status, body) = post_seed( + &foster, + serde_json::json!([ticket_record("a second pearl")]), + ) + .await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); +} + +#[tokio::test] +async fn forged_ack_rejected() { + let (foster, _, _) = seeded_origin().await; + + let stranger = K256Signer::generate(&OsEntropy); + let (status, _) = post_ack(&foster, signed_ack(&stranger, 1100)).await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "the ack's signer isn't the foster key" + ); + assert_eq!( + retired_at(&foster), + None, + "a rejected ack doesn't retire the repo" + ); +} + +#[tokio::test] +async fn plain_http_origin_off_loopback_rejected() { + let (foster, signer, origin_url) = seeded_origin().await; + let off_loopback = origin_url.replace("127.0.0.1", "foster.nel.pet"); + foster.publish_plc_knot(REPO, &off_loopback); + + let (status, body) = post_ack(&foster, signed_ack(&signer, 1100)).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}"); + assert_eq!( + retired_at(&foster), + None, + "no retire over plain http off loopback" + ); +} + +#[tokio::test] +async fn data_url_origin_rejected() { + let foster = World::fostering_with_grace(FOSTER_SECRET); + foster.publish_plc_knot(REPO, "data:text/plain,a-knot"); + seed_foster_record(&foster).await; + let signer = K256Signer::generate(&OsEntropy); + + let (status, body) = post_ack(&foster, signed_ack(&signer, 1100)).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}"); + assert_eq!( + retired_at(&foster), + None, + "a data url origin doesn't retire the shadow or panic the knot" + ); +} + +#[tokio::test] +async fn pre_epoch_ack_rejected() { + let (foster, signer, _) = seeded_origin().await; + + let (status, body) = post_ack(&foster, signed_ack(&signer, -1)).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}"); + assert_eq!( + retired_at(&foster), + None, + "a pre-epoch ackedAt can't retire anything" + ); +} + +#[tokio::test] +async fn old_ack_after_newer_is_no_op() { + let (foster, signer, _) = seeded_origin().await; + + let (status, body) = post_ack(&foster, signed_ack(&signer, 20)).await; + assert_eq!( + status, + StatusCode::OK, + "the newer ack retires first: {body}" + ); + + let (status, _) = post_ack(&foster, signed_ack(&signer, 10)).await; + assert_eq!( + status, + StatusCode::OK, + "an old ack is a no-op, not an error" + ); + assert_eq!( + retired_at(&foster), + Some(20), + "an old ack can't retire a fresh shadow" + ); +} + +#[tokio::test] +async fn replay_after_retire_returns_200() { + let (foster, signer, _) = seeded_origin().await; + + let ack = signed_ack(&signer, 1100); + let (status, body) = post_ack(&foster, ack.clone()).await; + assert_eq!( + status, + StatusCode::OK, + "the first ack retires the shadow: {body}" + ); + + let (status, _) = post_ack(&foster, ack).await; + assert_eq!( + status, + StatusCode::OK, + "a replay after retire is a 200, not an error" + ); + assert_eq!( + retired_at(&foster), + Some(1100), + "the replay doesn't move the retired state" + ); +} + +#[tokio::test] +async fn seed_from_knot_social_origin_is_welcome() { + let foster = World::fostering_with_grace(FOSTER_SECRET); + let signer = K256Signer::generate(&OsEntropy); + let origin = serve_router(knot_social_origin(&signer)).await; + foster.publish_plc_knot(REPO, &format!("http://{origin}")); + + let (status, body) = + post_seed(&foster, serde_json::json!([ticket_record("a feral pearl")])).await; + assert_eq!( + status, + StatusCode::OK, + "the PLC flip can point at the knot-social origin before the seed drain ends, so \ + seeding mustn't mind it: {body}" + ); +} + +#[tokio::test] +async fn ack_from_origin_without_cobs_is_refused() { + let foster = World::fostering_with_grace(FOSTER_SECRET); + seed_foster_record(&foster).await; + let signer = K256Signer::generate(&OsEntropy); + let origin = serve_router(knot_origin(&signer, &[])).await; + foster.publish_plc_knot(REPO, &format!("http://{origin}")); + + let (status, body) = post_ack(&foster, signed_ack(&signer, 1100)).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}"); + assert!( + body["message"] + .as_str() + .is_some_and(|message| message.contains("doesn't serve COBs")), + "the retire check lives at the ack, not the seed: {body}" + ); + assert_eq!( + retired_at(&foster), + None, + "a knot that can't serve the records can't retire the shadow" + ); +} + +#[tokio::test] +async fn gc_takes_retired_repo_past_grace_window() { + let (foster, signer, _) = seeded_origin().await; + + let (status, body) = post_ack(&foster, signed_ack(&signer, 0)).await; + assert_eq!( + status, + StatusCode::OK, + "retire before the grace window: {body}" + ); + + foster.advance(std::time::Duration::from_secs(3_601)); + knot_xrpc::foster::gc_tick(&foster.state, &None).await; + + let meta = knot_git::Repo::open(&foster.state.meta_path).unwrap(); + let store = CobStore::new(&meta); + let object = store.list::().unwrap()[0]; + let (registry, _) = store.materialize::(object).unwrap(); + assert!( + registry.record_of(&repo_did()).is_none(), + "the retired shadow is deregistered" + ); + assert!( + !foster.layout.repo_path(&repo_did()).unwrap().exists(), + "the on-disk shadow is removed" + ); + let tombstones = knot_xrpc::plc::deleted_dids(&meta).unwrap(); + assert!( + !tombstones.contains(&repo_did()), + "the repo is alive on the origin, so the retire GC writes no PLC tombstone" + ); +} + +#[derive(Default)] +struct LiarCalls { + failing: std::sync::atomic::AtomicBool, + seen: std::sync::Mutex, serde_json::Value)>>, +} + +fn liar_hydrant(calls: &std::sync::Arc) -> axum::Router { + let calls = std::sync::Arc::clone(calls); + axum::Router::new().route( + "/xrpc/org.tangled.foster.adminUnmarkLiarAccount", + axum::routing::post(move |headers: http::HeaderMap, body: String| { + let calls = std::sync::Arc::clone(&calls); + async move { + let auth = headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let payload = serde_json::from_str(&body).unwrap_or(serde_json::Value::Null); + calls.seen.lock().unwrap().push((auth, payload)); + match calls.failing.load(std::sync::atomic::Ordering::SeqCst) { + true => (StatusCode::INTERNAL_SERVER_ERROR, String::new()), + false => (StatusCode::OK, r#"{"unmarked":1}"#.to_owned()), + } + } + }), + ) +} + +async fn seeded_liar_origin( + calls: &std::sync::Arc, +) -> (World, K256Signer, knot_xrpc::foster::LiarHydrant) { + let addr = serve_router(liar_hydrant(calls)).await; + let liar = knot_xrpc::foster::LiarHydrant::new(&format!("http://{addr}")) + .expect("a loopback hydrant url is trusted"); + let secret = knot_xrpc::legacy_admin::LegacyAdminSecret::new(FOSTER_SECRET) + .expect("the test secret parses"); + let foster = World::fostering_liar(secret, liar.clone()); + seed_foster_record(&foster).await; + let signer = K256Signer::generate(&OsEntropy); + let origin_url = serve_origin(&foster, &signer).await; + foster.publish_plc_knot(REPO, &origin_url); + (foster, signer, liar) +} + +fn registry_still_serves(world: &World) -> bool { + let meta = knot_git::Repo::open(&world.state.meta_path).unwrap(); + let store = CobStore::new(&meta); + let object = store.list::().unwrap()[0]; + let (registry, _) = store.materialize::(object).unwrap(); + registry.record_of(&repo_did()).is_some() +} + +#[tokio::test] +async fn ack_releases_the_liar_resolution_and_gc_wipes_the_shadow() { + let calls = std::sync::Arc::new(LiarCalls::default()); + let (foster, signer, liar) = seeded_liar_origin(&calls).await; + + let (status, body) = post_ack(&foster, signed_ack(&signer, 0)).await; + assert_eq!(status, StatusCode::OK, "{body}"); + + let seen = calls.seen.lock().unwrap().clone(); + assert_eq!(seen.len(), 1, "the retiring ack sends exactly one unmark"); + let (auth, payload) = &seen[0]; + assert_eq!(payload["dids"][0].as_str(), Some(REPO)); + let claims = auth + .as_deref() + .unwrap_or_default() + .strip_prefix("Bearer ") + .and_then(|token| token.split('.').nth(1)) + .and_then(|encoded| URL_SAFE_NO_PAD.decode(encoded).ok()) + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .unwrap_or_default(); + assert_eq!( + claims["lxm"].as_str(), + Some("org.tangled.foster.adminUnmarkLiarAccount"), + "the unmark token references its method" + ); + assert_eq!( + claims["iss"].as_str(), + Some(foster.state.knot_did.as_str()), + "the foster signs the unmark as itself" + ); + + foster.advance(std::time::Duration::from_secs(3_601)); + knot_xrpc::foster::gc_tick(&foster.state, &Some(liar)).await; + + assert!( + !registry_still_serves(&foster), + "the unmarked shadow is deregistered" + ); + assert!( + !foster.layout.repo_path(&repo_did()).unwrap().exists(), + "the unmarked shadow leaves disk" + ); +} + +#[tokio::test] +async fn gc_keeps_the_shadow_until_the_liar_takes_the_unmark() { + let calls = std::sync::Arc::new(LiarCalls { + failing: std::sync::atomic::AtomicBool::new(true), + ..LiarCalls::default() + }); + let (foster, signer, liar) = seeded_liar_origin(&calls).await; + + let (status, body) = post_ack(&foster, signed_ack(&signer, 0)).await; + assert_eq!(status, StatusCode::OK, "{body}"); + + foster.advance(std::time::Duration::from_secs(3_601)); + knot_xrpc::foster::gc_tick(&foster.state, &Some(liar.clone())).await; + + assert!( + registry_still_serves(&foster), + "a shadow the liar still points at survives its grace window" + ); + assert!( + foster.layout.repo_path(&repo_did()).unwrap().exists(), + "a shadow the liar still points at stays on disk" + ); + assert!( + calls.seen.lock().unwrap().len() >= 2, + "the ack and the sweep both asked the liar hydrant" + ); + + calls + .failing + .store(false, std::sync::atomic::Ordering::SeqCst); + knot_xrpc::foster::gc_tick(&foster.state, &Some(liar)).await; + + assert!( + !registry_still_serves(&foster), + "the shadow goes once the liar takes the unmark" + ); +} diff --git a/knot2/crates/knot-xrpc/tests/spindle_outbox.rs b/knot2/crates/knot-xrpc/tests/spindle_outbox.rs new file mode 100644 index 0000000000000000000000000000000000000000..1dffa18e759963f4b73075d764f024cbc918903f --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/spindle_outbox.rs @@ -0,0 +1,90 @@ +mod common; + +use common::{World, commit_file, http_push, nsid, owner, post_authed, serve_pushes, sh_git}; +use knot_types::{RepoDid, UnixSeconds}; + +const SPINDLE: &str = "did:web:spindle.test"; + +async fn create_repo_on_spindle(world: &World) -> RepoDid { + let (status, created) = post_authed( + world, + &nsid("org.tangled.repo.createRepo"), + &owner(), + serde_json::json!({ "name": "anemone", "defaultBranch": "main", "spindle": SPINDLE }), + ) + .await; + assert_eq!(status, http::StatusCode::OK, "create failed: {created}"); + RepoDid::new(created["did"].as_str().unwrap()).unwrap() +} + +fn queued_refs(world: &World) -> Vec<(String, Vec)> { + world + .state + .outbox + .queued() + .into_iter() + .map(|(spindle, body)| { + assert_eq!(spindle.as_str(), SPINDLE); + let options = body["event"]["pushOptions"] + .as_array() + .map(|options| { + options + .iter() + .map(|option| option.as_str().unwrap().to_string()) + .collect() + }) + .unwrap_or_default(); + (body["event"]["ref"].as_str().unwrap().to_string(), options) + }) + .collect() +} + +async fn seeded() -> (World, RepoDid, tempfile::TempDir) { + let world = World::new(); + world.add_member(&owner(), &owner(), UnixSeconds::new(1_000)); + let did = create_repo_on_spindle(&world).await; + let work = tempfile::tempdir().unwrap(); + sh_git(work.path(), &["init", "-q", "-b", "main"]); + commit_file( + work.path(), + "README.md", + b"# anemone\n", + "first", + "2026-06-01T12:30:00+02:00", + ); + (world, did, work) +} + +#[tokio::test] +async fn a_push_is_queued_for_the_spindle_even_when_it_asks_to_skip_ci() { + let (world, did, work) = seeded().await; + let addr = serve_pushes(&world).await; + + let (ok, detail) = http_push(work.path(), addr, &did, &owner(), &["main"]).await; + assert!(ok, "push failed: {detail}"); + commit_file( + work.path(), + "more.md", + b"more\n", + "second", + "2026-06-01T12:31:00+02:00", + ); + let (ok, detail) = http_push( + work.path(), + addr, + &did, + &owner(), + &["--push-option=ci-skip", "main"], + ) + .await; + assert!(ok, "skip-ci push failed: {detail}"); + + // the knot doesn't act on ci-skip, it still queues the push with the option + assert_eq!( + queued_refs(&world), + [ + ("refs/heads/main".to_string(), vec![]), + ("refs/heads/main".to_string(), vec!["ci-skip".to_string()]), + ] + ); +} diff --git a/knot2/example.toml b/knot2/example.toml new file mode 100644 index 0000000000000000000000000000000000000000..c655c06cccd55479baa6d04f93958ed937ad7991 --- /dev/null +++ b/knot2/example.toml @@ -0,0 +1,845 @@ +[server] +# Can also be specified via environment variable `KNOT_HOSTNAME`. +# Required! This value must be specified. +#hostname = + +# Can also be specified via environment variable `KNOT_ADMINS`. +# Required! This value must be specified. +#admins = + +# Can also be specified via environment variable `KNOT_LISTEN_ADDR`. +# Default value: "[::]:5555" +#listen_addr = "[::]:5555" + +# Can also be specified via environment variable `KNOT_LISTEN_HEADER_TIMEOUT_MS`. +# Default value: 10000 +#listen_header_timeout_ms = 10000 + +# Can also be specified via environment variable `KNOT_LISTEN_IDLE_TIMEOUT_MS`. +# Default value: 60000 +#listen_idle_timeout_ms = 60000 + +# Can also be specified via environment variable `KNOT_LISTEN_MAX_CONNECTIONS`. +# Default value: 1024 +#listen_max_connections = 1024 + +# Can also be specified via environment variable `KNOT_LISTEN_RATE_LIMIT_PER_SECOND`. +# Default value: 50 +#listen_rate_limit_per_second = 50 + +# Can also be specified via environment variable `KNOT_LISTEN_RATE_LIMIT_BURST`. +# Default value: 200 +#listen_rate_limit_burst = 200 + +# Can also be specified via environment variable `KNOT_LISTEN_MAX_INFLIGHT_REQUESTS`. +# Default value: 1024 +#listen_max_inflight_requests = 1024 + +# Can also be specified via environment variable `KNOT_LISTEN_REQUEST_TIMEOUT_MS`. +# Default value: 60000 +#listen_request_timeout_ms = 60000 + +# Can also be specified via environment variable `KNOT_LISTEN_BODY_TIMEOUT_MS`. +# Default value: 30000 +#listen_body_timeout_ms = 30000 + +# Can also be specified via environment variable `KNOT_LISTEN_WRITE_REQUEST_TIMEOUT_MS`. +# Default value: 1800000 +#listen_write_request_timeout_ms = 1800000 + +# Can also be specified via environment variable `KNOT_INTERNAL_LISTEN_ADDR`. +# Default value: "[::1]:5444" +#internal_listen_addr = "[::1]:5444" + +# Can also be specified via environment variable `KNOT_SSH_LISTEN_ADDR`. +# Default value: "[::]:2222" +#ssh_listen_addr = "[::]:2222" + +# Can also be specified via environment variable `KNOT_SSH_HOST_KEY_FILE`. +# Required! This value must be specified. +#ssh_host_key_file = + +# Can also be specified via environment variable `KNOT_SSH_MAX_PACK_BYTES`. +# Default value: 8589934592 +#ssh_max_pack_bytes = 8589934592 + +# Maximum number of SSH git operations that one client address can run at the same time. +# +# Can also be specified via environment variable `KNOT_SSH_MAX_INFLIGHT_PER_PEER`. +# +# Default value: 4 +#ssh_max_inflight_per_peer = 4 + +# Addresses allowed to prefix an ssh connection with a PROXY protocol v1 header, +# each a bare IP without a port or a CIDR block, +# for ex the address of a TCP proxy such as nginx `stream` with `proxy_protocol on`. +# The knot will rate-limit the connection by the source address in the header. +# The knot never reads the header from any other address. +# +# Can also be specified via environment variable `KNOT_SSH_PROXY_PROTOCOL_FROM`. +# +# Default value: [] +#ssh_proxy_protocol_from = [] + +# Can also be specified via environment variable `KNOT_APPVIEW_ENDPOINT`. +# Default value: "https://tangled.org" +#appview_endpoint = "https://tangled.org" + +[tls] +# Can also be specified via environment variable `KNOT_TLS_CERT_PATH`. +#cert_path = + +# Can also be specified via environment variable `KNOT_TLS_KEY_PATH`. +#key_path = + +# Can also be specified via environment variable `KNOT_TLS_HTTP3`. +# Default value: true +#http3 = true + +# Can also be specified via environment variable `KNOT_TLS_ACME_ENABLED`. +# Default value: false +#acme_enabled = false + +# Can also be specified via environment variable `KNOT_TLS_ACME_CACHE_DIR`. +#acme_cache_dir = + +# Can also be specified via environment variable `KNOT_TLS_ACME_CONTACT`. +#acme_contact = + +# Can also be specified via environment variable `KNOT_TLS_ACME_STAGING`. +# Default value: false +#acme_staging = false + +# Can also be specified via environment variable `KNOT_TLS_MTLS_ENABLED`. +# Default value: false +#mtls_enabled = false + +# Can also be specified via environment variable `KNOT_TLS_MTLS_CLIENT_CA_PATH`. +#mtls_client_ca_path = + +# Can also be specified via environment variable `KNOT_TLS_MTLS_ADMIN_SPKI_PIN`. +#mtls_admin_spki_pin = + +[acl] +# Can also be specified via environment variable `KNOT_ADMISSION`. +# Default value: "closed" +#admission = "closed" + +# Can also be specified via environment variable `KNOT_LEGACY_ADMIN_SECRET_ENV`. +#legacy_admin_secret_env = + +[repo] +# Can also be specified via environment variable `KNOT_SCAN_PATH`. +# Required! This value must be specified. +#scan_path = + +# Can also be specified via environment variable `KNOT_DEFAULT_BRANCH`. +# Default value: "main" +#default_branch = "main" + +# Can also be specified via environment variable `KNOT_CONTRIBUTION_POLICY`. +# Default value: "anyone" +#contribution_policy = "anyone" + +# Can also be specified via environment variable `KNOT_REPO_WRITE_BURST`. +# Default value: 30 +#write_burst = 30 + +# Can also be specified via environment variable `KNOT_REPO_WRITE_REFILL_MS`. +# Default value: 500 +#write_refill_ms = 500 + +[git] +# Can also be specified via environment variable `KNOT_GIT_OBJECT_FORMAT`. +# Default value: "sha256" +#object_format = "sha256" + +[secrets] +# Can also be specified via environment variable `KNOT_SEALED_KEY_FILE`. +# Required! This value must be specified. +#sealed_key_file = + +# Can also be specified via environment variable `KNOT_MASTER_KEY_ENV`. +# Required! This value must be specified. +#master_key_env = + +[http] +# Can also be specified via environment variable `KNOT_HTTP_CONNECT_TIMEOUT_MS`. +# Default value: 5000 +#connect_timeout_ms = 5000 + +# Can also be specified via environment variable `KNOT_HTTP_READ_TIMEOUT_MS`. +# Default value: 30000 +#read_timeout_ms = 30000 + +# Can also be specified via environment variable `KNOT_HTTP_REQUEST_TIMEOUT_MS`. +# Default value: 60000 +#request_timeout_ms = 60000 + +# Can also be specified via environment variable `KNOT_HTTP_MAX_RESPONSE_BYTES`. +# Default value: 16777216 +#max_response_bytes = 16777216 + +# Can also be specified via environment variable `KNOT_HTTP_BLOCK_PRIVATE_ADDRESSES`. +# Default value: true +#block_private_addresses = true + +[atproto] +# Can also be specified via environment variable `KNOT_PLC_DIRECTORY`. +# Required! This value must be specified. +#plc_directory = + +# How many PLC documents the reconciler sweep checks against the directory concurrently. +# +# Can also be specified via environment variable `KNOT_PLC_SWEEP_WIDTH`. +# +# Default value: 1 +#plc_sweep_width = 1 + +# An archive of the per-repo signing keys, +# that gotta get squished into the canonical key store here +# at knot start time. +# If unset, the knot will attempt just `repo-signing-keys.json` +# next to `secrets.sealed_key_file` if that file exists too. +# +# Can also be specified via environment variable `KNOT_REPO_SIGNING_KEYS`. +#repo_signing_keys = + +# Relays that get a com.atproto.sync.requestCrawl for this knot on startup. +# +# Can also be specified via environment variable `KNOT_CRAWLERS`. +# +# Default value: ["https://index.tangled.network", "https://relay.fire.hose.cam", "https://relay3.fr.hose.cam", "https://bsky.network", "https://northamerica.firehose.network", "https://europe.firehose.network", "https://asia.firehose.network", "https://atproto.africa", "https://relay.upcloud.world"] +#crawlers = ["https://index.tangled.network", "https://relay.fire.hose.cam", "https://relay3.fr.hose.cam", "https://bsky.network", "https://northamerica.firehose.network", "https://europe.firehose.network", "https://asia.firehose.network", "https://atproto.africa", "https://relay.upcloud.world"] + +[xrpc] +# Can also be specified via environment variable `KNOT_XRPC_MAX_BODY_BYTES`. +# Default value: 65536 +#max_body_bytes = 65536 + +# Can also be specified via environment variable `KNOT_XRPC_MAX_RESPONSE_BYTES`. +# Default value: 5242880 +#max_response_bytes = 5242880 + +# Upper bound on bytes that a single archive spools, +# across all our surfaces: the org.tangled.git.getArchive query, +# `git archive --remote` over SSH, +# and the smart HTTP archive route. +# The knot will refuse writing smth that would blast an archive past this bound. +# +# Can also be specified via environment variable `KNOT_XRPC_MAX_ARCHIVE_BYTES`. +# +# Default value: 1073741824 +#max_archive_bytes = 1073741824 + +# Can also be specified via environment variable `KNOT_XRPC_TREE_LAST_COMMIT_BUDGET_MS`. +# Default value: 300 +#tree_last_commit_budget_ms = 300 + +# Can also be specified via environment variable `KNOT_XRPC_BLOB_LAST_COMMIT_BUDGET_MS`. +# Default value: 2000 +#blob_last_commit_budget_ms = 2000 + +# Can also be specified via environment variable `KNOT_XRPC_LANGUAGES_BUDGET_MS`. +# Default value: 1000 +#languages_budget_ms = 1000 + +# Body limit for the seedRepo procedure, whose record batches routinely +# exceed the general XRPC body limit. +# +# Can also be specified via environment variable `KNOT_XRPC_MAX_PATCH_BYTES`. +# +# Default value: 16777216 +#max_patch_bytes = 16777216 + +# Limit on the total decompressed size of a patch the merge procedures parse, +# bounding binary-delta inflation and hunk expansion apart from the +# compressed body limit above. +# +# Can also be specified via environment variable `KNOT_XRPC_MAX_PATCH_DECOMPRESSED_BYTES`. +# +# Default value: 134217728 +#max_patch_decompressed_bytes = 134217728 + +# Can also be specified via environment variable `KNOT_XRPC_PREAUTH_BURST`. +# Default value: 20 +#preauth_burst = 20 + +# Can also be specified via environment variable `KNOT_XRPC_PREAUTH_REFILL_MS`. +# Default value: 100 +#preauth_refill_ms = 100 + +# Can also be specified via environment variable `KNOT_XRPC_PER_PEER_INFLIGHT`. +# Default value: 8 +#per_peer_inflight = 8 + +# Can also be specified via environment variable `KNOT_XRPC_GLOBAL_INFLIGHT`. +# Default value: 64 +#global_inflight = 64 + +# Can also be specified via environment variable `KNOT_XRPC_MAX_PENDING_RESERVATIONS`. +# Default value: 256 +#max_pending_reservations = 256 + +# Per-account limit on reserved repository keys awaiting creation, so one +# account cannot consume the whole pending-reservation budget. +# +# Can also be specified via environment variable `KNOT_XRPC_PER_ACTOR_RESERVATIONS`. +# +# Default value: 32 +#per_actor_reservations = 32 + +# How long a reserved repository key is held before it lapses and its +# sealed key is reclaimed, in seconds. +# +# Can also be specified via environment variable `KNOT_XRPC_RESERVATION_TTL_SECS`. +# +# Default value: 3600 +#reservation_ttl_secs = 3600 + +# Can also be specified via environment variable `KNOT_XRPC_FORK_MAX_PACK_BYTES`. +# Default value: 1073741824 +#fork_max_pack_bytes = 1073741824 + +# Can also be specified via environment variable `KNOT_XRPC_IMPORT_MAX_PACK_BYTES`. +# Default value: 268435456 +#import_max_pack_bytes = 268435456 + +# Can also be specified via environment variable `KNOT_XRPC_FORK_FETCH_TIMEOUT_MS`. +# Default value: 600000 +#fork_fetch_timeout_ms = 600000 + +# When the knot runs behind a trusted reverse proxy that terminates TLS, +# set this to the header the proxy appends the client address to, +# for example x-forwarded-for. +# The knot will read the chain right -> left +# and take the first entry that `trusted_proxies` doesn't cover. +# Leave unset when the knot is directly exposed so the socket peer address is used. +# Only set this when a trusted proxy overwrites or appends the header, +# since a client can forge it otherwise. +# +# Can also be specified via environment variable `KNOT_XRPC_TRUSTED_PROXY_HEADER`. +#trusted_proxy_header = + +# Addresses whose `trusted_proxy_header` the knot honors, +# each a bare IP without a port or a CIDR block such as 173.245.48.0/20, +# for ex the loopback address of a reverse proxy on the same host. +# The knot will rate-limit a request from any other address +# by its own socket address and ignore the header. +# These same addresses are hops the knot will iterate over when it reads +# the header, so list every proxy you control in the path. +# A proxy that the knot doesn't know about becomes the entry it keys on, +# and everyone that proxy serves will then share one rate-limit bucket. +# The knot will read the last 32 entries of the chain, at most. +# When the list covers all 32, the knot +# will rate-limit by the address the request connected from. +# Leave empty to honor the header from every peer and take its rightmost +# entry, which is safe *only* while every route to this knot passes +# through the proxy. +# +# Can also be specified via environment variable `KNOT_XRPC_TRUSTED_PROXIES`. +# +# Default value: [] +#trusted_proxies = [] + +# Can also be specified via environment variable `KNOT_XRPC_EVENTS_REPLAY_BUFFER`. +# Default value: 4096 +#events_replay_buffer = 4096 + +# Can also be specified via environment variable `KNOT_XRPC_EVENTS_REPLAY_BYTES`. +# Default value: 67108864 +#events_replay_bytes = 67108864 + +# Can also be specified via environment variable `KNOT_XRPC_EVENTS_MAX_SUBSCRIBERS`. +# Default value: 256 +#events_max_subscribers = 256 + +# Can also be specified via environment variable `KNOT_XRPC_EVENTS_MAX_PER_PEER`. +# Default value: 8 +#events_max_per_peer = 8 + +[maintenance] +# Can also be specified via environment variable `KNOT_MAINTENANCE_ENABLED`. +# Default value: true +#enabled = true + +# Can also be specified via environment variable `KNOT_MAINTENANCE_COMMIT_GRAPH`. +# Default value: true +#commit_graph = true + +# Can also be specified via environment variable `KNOT_MAINTENANCE_MULTI_PACK_INDEX`. +# Default value: true +#multi_pack_index = true + +# Can also be specified via environment variable `KNOT_MAINTENANCE_BITMAP`. +# Default value: true +#bitmap = true + +# Can also be specified via environment variable `KNOT_MAINTENANCE_INTERVAL_SECS`. +# Default value: 21600 +#interval_secs = 21600 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_REPACK_MAX_OBJECTS`. +# Default value: 16000000 +#repack_max_objects = 16000000 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_REPACK_GEOMETRIC_FACTOR`. +# Default value: 2 +#repack_geometric_factor = 2 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_REPACK_LOOSE_THRESHOLD`. +# Default value: 6700 +#repack_loose_threshold = 6700 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_PRUNE_GRACE_SECS`. +# Default value: 1209600 +#prune_grace_secs = 1209600 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_FULL_REPACK_INTERVAL_SECS`. +# Default value: 432000 +#full_repack_interval_secs = 432000 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_REFLOG_EXPIRE_SECS`. +# Default value: 7776000 +#reflog_expire_secs = 7776000 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_LARGE_PUSH_BYTES`. +# Default value: 52428800 +#large_push_bytes = 52428800 + +[pack_cache] +# Can also be specified via environment variable `KNOT_PACK_CACHE_ENABLED`. +# Default value: true +#enabled = true + +# Can also be specified via environment variable `KNOT_PACK_CACHE_TTL_SECS`. +# Default value: 60 +#ttl_secs = 60 + +# Can also be specified via environment variable `KNOT_PACK_CACHE_MAX_ENTRY_BYTES`. +# Default value: 67108864 +#max_entry_bytes = 67108864 + +# Can also be specified via environment variable `KNOT_PACK_CACHE_MAX_TOTAL_BYTES`. +# Default value: 2147483648 +#max_total_bytes = 2147483648 + +[pack] +# Can also be specified via environment variable `KNOT_PACK_MAX_OBJECTS`. +# Default value: 16000000 +#max_objects = 16000000 + +# Can also be specified via environment variable `KNOT_PACK_MAX_TOTAL_BYTES`. +# Default value: 68719476736 +#max_total_bytes = 68719476736 + +# Can also be specified via environment variable `KNOT_PACK_SELECTION_MAX_OBJECTS`. +# Default value: 16000000 +#selection_max_objects = 16000000 + +# Can also be specified via environment variable `KNOT_PACK_SELECTION_TIME_BUDGET_SECS`. +# Default value: 600 +#selection_time_budget_secs = 600 + +[lfs] +# Can also be specified via environment variable `KNOT_LFS_STORE_PATH`. +# Required! This value must be specified. +#store_path = + +# Can also be specified via environment variable `KNOT_LFS_MAX_OBJECT_BYTES`. +# Default value: 5368709120 +#max_object_bytes = 5368709120 + +# Can also be specified via environment variable `KNOT_LFS_FREE_SPACE_FLOOR_BYTES`. +# Default value: 1073741824 +#free_space_floor_bytes = 1073741824 + +# Can also be specified via environment variable `KNOT_LFS_GC_GRACE_SECS`. +# Default value: 1209600 +#gc_grace_secs = 1209600 + +# Can also be specified via environment variable `KNOT_LFS_GC_INTERVAL_SECS`. +# Default value: 21600 +#gc_interval_secs = 21600 + +# Can also be specified via environment variable `KNOT_LFS_MAX_SSH_TRANSFERS`. +# Default value: 16 +#max_ssh_transfers = 16 + +# Can also be specified via environment variable `KNOT_LFS_MAX_HTTP_DOWNLOADS`. +# Default value: 64 +#max_http_downloads = 64 + +[social] +# Byte budget over the live record versions of one repository, summed from every atproto collection in the projection. +# An edit swaps the old version's bytes for the new one's, and a delete frees the record's bytes back out of the budget. +# A write past the budget 429s. +# +# Can also be specified via environment variable `KNOT_SOCIAL_REPO_BUDGET_BYTES`. +# +# Default value: 1073741824 +#repo_budget_bytes = 1073741824 + +# The "Foster Knot" from whom this knot will pull "shadowed" records from during a one-time upgrade, at boot time. +# Defaults to the "Tangled production foster knot", +# so there's not much use of it outside this one-time upgrade required +# on 2026-10-28. +# We are sorry for designing our distributed system +# in such a way that warrants what Lewis would call +# "phoning home" but the alternative is that each knot pings +# the entire atproto dataset, trying to hunt down the social records +# that it will be taking on as COBs starting 2026-10-28. +# We have this "Foster Knot" that has not only done this for you +# already, but also shadow-serves these social records on your knot's +# behalf until you upgrade, so there's no perceived incompatibility-time +# for reading on https://tangled.org . +# If you're uncomfortable with this enough that you want to unset this, +# please ping all of atproto to hunt down those social records +# regarding the git repos on this knot, and add them here. +# If this knot is being spun up for the first time and the current +# day is *after* 2026-10-28, please disregard. This +# setting will be gutted once all knots are upgraded. +# +# Can also be specified via environment variable `KNOT_SOCIAL_FOSTER_URL`. +# +# Default value: "https://knot2.mock.tangled.org" +#foster_url = "https://knot2.mock.tangled.org" + +# How many handovers that the upgrade of a knot2 at boot-time from a foster knot pull will run concurrently. +# +# Can also be specified via environment variable `KNOT_SOCIAL_FOSTER_BOOT_WIDTH`. +# +# Default value: 4 +#foster_boot_width = 4 + +# The following are for the "foster knot", you don't need to touch. +# +# Can also be specified via environment variable `KNOT_SOCIAL_FOSTER_ACK_SKIP_KNOT_SERVICE`. +# +# Default value: false +#foster_ack_skip_knot_service = false + +# Can also be specified via environment variable `KNOT_SOCIAL_FOSTER_ACK_SKIP_ORIGIN_CHECKS`. +# Default value: false +#foster_ack_skip_origin_checks = false + +# Can also be specified via environment variable `KNOT_SOCIAL_FOSTER_LIAR_HYDRANT_URL`. +# Default value: "" +#foster_liar_hydrant_url = "" + +[keyfill] +# Can also be specified via environment variable `KNOT_KEYFILL_KEY_BUDGET_MIB`. +# Default value: 64 +#key_budget_mib = 64 + +# Can also be specified via environment variable `KNOT_KEYFILL_TTL_SECS`. +# Default value: 3600 +#ttl_secs = 3600 + +# Can also be specified via environment variable `KNOT_KEYFILL_REPRIEVE_RETRY_SECS`. +# Default value: 300 +#reprieve_retry_secs = 300 + +# Can also be specified via environment variable `KNOT_KEYFILL_REPRIEVE_BUDGET_SECS`. +# Default value: 21600 +#reprieve_budget_secs = 21600 + +[resources] +# Can also be specified via environment variable `KNOT_MAX_THREADS`. +# Default value: 0 +#max_threads = 0 + +# Can also be specified via environment variable `KNOT_MAX_MEMORY_BYTES`. +# Default value: 0 +#max_memory_bytes = 0 + +[homepage] +# Can also be specified via environment variable `KNOT_HOMEPAGE_ENABLED`. +# Default value: true +#enabled = true + +# Can also be specified via environment variable `KNOT_HOMEPAGE_PATH`. +#path = + +[links] +# Can also be specified via environment variable `KNOT_LINKS_PRIVACY_POLICY`. +#privacy_policy = + +# Can also be specified via environment variable `KNOT_LINKS_TERMS_OF_SERVICE`. +#terms_of_service = + +[contact] +# Can also be specified via environment variable `KNOT_CONTACT_EMAIL`. +# Required! This value must be specified. +#email = + +[messages] +[messages.push] +# Can also be specified via environment variable `KNOT_MESSAGES_PUSH_ACK`. +# Default value: ["{knot} received {refs}."] +#ack = ["{knot} received {refs}."] + +# Can also be specified via environment variable `KNOT_MESSAGES_PUSH_PULL_REQUEST`. +# Default value: ["", "-> Open pull request for this branch:", " {url}", ""] +#pull_request = ["", "-> Open pull request for this branch:", " {url}", ""] + +# Can also be specified via environment variable `KNOT_MESSAGES_PUSH_PIPELINE_CLEAN`. +# Default value: ["pipeline compiled with no diagnostics"] +#pipeline_clean = ["pipeline compiled with no diagnostics"] + +# Can also be specified via environment variable `KNOT_MESSAGES_PUSH_PIPELINE_NONE`. +# Default value: ["no pipelines to compile"] +#pipeline_none = ["no pipelines to compile"] + +[messages.fetch] +# Can also be specified via environment variable `KNOT_MESSAGES_FETCH_MOTD`. +# Default value: ["Thanks for using {knot}!"] +#motd = ["Thanks for using {knot}!"] + +# Can also be specified via environment variable `KNOT_MESSAGES_FETCH_ENUMERATING`. +# Default value: ["Enumerating objects: {count}, done."] +#enumerating = ["Enumerating objects: {count}, done."] + +# Can also be specified via environment variable `KNOT_MESSAGES_FETCH_TOTAL`. +# Default value: ["Total {count}, done."] +#total = ["Total {count}, done."] + +# Can also be specified via environment variable `KNOT_MESSAGES_FETCH_FATAL`. +# Default value: "knot: {error}" +#fatal = "knot: {error}" + +[messages.reject] +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_RESERVED_REFS`. +# Default value: "refs/cobs/*, refs/cob-checkpoints/*, refs/hidden/*, refs/atproto/* and refs/tngl/* are reserved and can't be pushed" +#reserved_refs = "refs/cobs/*, refs/cob-checkpoints/*, refs/hidden/*, refs/atproto/* and refs/tngl/* are reserved and can't be pushed" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_COB_CREATE_ONLY`. +# Default value: "existing refs/cobs/* object cannot be modified or deleted over the wire" +#cob_create_only = "existing refs/cobs/* object cannot be modified or deleted over the wire" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_COB_DELETE`. +# Default value: "refs/cobs/* stores append-only collaborative objects and cannot be deleted" +#cob_delete = "refs/cobs/* stores append-only collaborative objects and cannot be deleted" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_HIDDEN_RESERVED`. +# Default value: "refs/hidden/* is reserved and can't be pushed" +#hidden_reserved = "refs/hidden/* is reserved and can't be pushed" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_ATPROTO_RESERVED`. +# Default value: "refs/atproto/* is reserved for the knot's atproto commit chain and can't be pushed" +#atproto_reserved = "refs/atproto/* is reserved for the knot's atproto commit chain and can't be pushed" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_TNGL_RESERVED`. +# Default value: "refs/tngl/* is reserved for knot-written keep refs and can't be pushed" +#tngl_reserved = "refs/tngl/* is reserved for knot-written keep refs and can't be pushed" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_CHECKPOINT_RESERVED`. +# Default value: "refs/cob-checkpoints/* is reserved for knot-written collaborative-object snapshots and can't be pushed" +#checkpoint_reserved = "refs/cob-checkpoints/* is reserved for knot-written collaborative-object snapshots and can't be pushed" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_COB_VERIFICATION`. +# Default value: "collaborative-object verification failed: {error}" +#cob_verification = "collaborative-object verification failed: {error}" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_REF_EXISTS`. +# Default value: "reference already exists" +#ref_exists = "reference already exists" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_STALE_OLD_VALUE`. +# Default value: "stale info: old value doesn't match" +#stale_old_value = "stale info: old value doesn't match" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_MISSING_OBJECTS`. +# Default value: "missing necessary objects" +#missing_objects = "missing necessary objects" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_MISSING_OBJECTS_FOR`. +# Default value: "missing necessary objects for {ref}" +#missing_objects_for = "missing necessary objects for {ref}" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_ATOMIC_FAILED`. +# Default value: "atomic transaction failed" +#atomic_failed = "atomic transaction failed" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_ATOMIC_ABORTED`. +# Default value: "atomic push aborted" +#atomic_aborted = "atomic push aborted" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_AUTHORIZATION_UNAVAILABLE`. +# Default value: "authorization unavailable" +#authorization_unavailable = "authorization unavailable" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_UNPACKER_ERROR`. +# Default value: "unpacker error" +#unpacker_error = "unpacker error" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_REF_SNAPSHOT_UNAVAILABLE`. +# Default value: "ref snapshot unavailable" +#ref_snapshot_unavailable = "ref snapshot unavailable" + +# Can also be specified via environment variable `KNOT_MESSAGES_REJECT_OBJECT_MIGRATION_FAILED`. +# Default value: "object migration failed" +#object_migration_failed = "object migration failed" + +[messages.ssh] +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_GREETING`. +# Default value: ["Hi {user}! You're authenticated to {knot} knot.", "This knot serves git over ssh, so there's no shell here. :P", "Clone repo with: git clone {knot}:"] +#greeting = ["Hi {user}! You're authenticated to {knot} knot.", "This knot serves git over ssh, so there's no shell here. :P", "Clone repo with: git clone {knot}:"] + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_GREETING_UNKNOWN`. +# Default value: ["Hi there! This is the {knot} knot.", "This knot serves git over ssh, so there's no shell here. :P", "Clone repo with: git clone {knot}:", "Publish your ssh key to your atproto account so this knot can identify your pushes.", "Put your handle in the url, as in yourhandle@{knot}:, so your ssh client can find your registered key on its own."] +#greeting_unknown = ["Hi there! This is the {knot} knot.", "This knot serves git over ssh, so there's no shell here. :P", "Clone repo with: git clone {knot}:", "Publish your ssh key to your atproto account so this knot can identify your pushes.", "Put your handle in the url, as in yourhandle@{knot}:, so your ssh client can find your registered key on its own."] + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_UNSUPPORTED_COMMAND`. +# Default value: "knot: unsupported command" +#unsupported_command = "knot: unsupported command" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_TOO_MANY_OPERATIONS`. +# Default value: "knot: too many concurrent operations from your address, try again shortly" +#too_many_operations = "knot: too many concurrent operations from your address, try again shortly" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_REPO_NOT_FOUND`. +# Default value: "knot: repository not found" +#repo_not_found = "knot: repository not found" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_INDEX_WARMING`. +# Default value: "knot: repository index is warming, retry shortly" +#index_warming = "knot: repository index is warming, retry shortly" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_KEY_NOT_REGISTERED`. +# Default value: "knot: this ssh key doesn't match any key published by the accounts that may push here. Authorized: {authorized}. If your agent offers several keys, add -o IdentitiesOnly=yes so it offers your registered key." +#key_not_registered = "knot: this ssh key doesn't match any key published by the accounts that may push here. Authorized: {authorized}. If your agent offers several keys, add -o IdentitiesOnly=yes so it offers your registered key." + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_IDENTITY_UNAVAILABLE`. +# Default value: "knot: couldn't read the account records needed to check your ssh key, retry shortly" +#identity_unavailable = "knot: couldn't read the account records needed to check your ssh key, retry shortly" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_PUSH_DENIED`. +# Default value: "knot: you aren't authorized to push to this repository." +#push_denied = "knot: you aren't authorized to push to this repository." + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_SHUTTING_DOWN`. +# Default value: "knot: server is shutting down" +#shutting_down = "knot: server is shutting down" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_ARCHIVE_MALFORMED`. +# Default value: "knot: malformed upload-archive request" +#archive_malformed = "knot: malformed upload-archive request" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_ARCHIVE_TIMEOUT`. +# Default value: "knot: upload-archive request timed out" +#archive_timeout = "knot: upload-archive request timed out" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_ARCHIVE_FAILED`. +# Default value: "knot: upload-archive failed" +#archive_failed = "knot: upload-archive failed" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_ADVERTISE_FAILED`. +# Default value: "knot: cannot advertise refs" +#advertise_failed = "knot: cannot advertise refs" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_PUSH_TOO_LARGE`. +# Default value: "knot: push exceeds configured size limit" +#push_too_large = "knot: push exceeds configured size limit" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_RECEIVE_DEADLINE`. +# Default value: "knot: receive exceeded its time budget" +#receive_deadline = "knot: receive exceeded its time budget" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_MALFORMED_PACK`. +# Default value: "knot: malformed pack stream" +#malformed_pack = "knot: malformed pack stream" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_RECEIVE_READ_ERROR`. +# Default value: "knot: receive read error" +#receive_read_error = "knot: receive read error" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_RECEIVE_ENDED_EARLY`. +# Default value: "knot: receive stream ended early" +#receive_ended_early = "knot: receive stream ended early" + +# Can also be specified via environment variable `KNOT_MESSAGES_SSH_RECEIVE_FAILED`. +# Default value: "knot: receive-pack failed" +#receive_failed = "knot: receive-pack failed" + +[messages.http] +# Can also be specified via environment variable `KNOT_MESSAGES_HTTP_PUSH_DENIED`. +# Default value: "you aren't authorized to push to this repository" +#push_denied = "you aren't authorized to push to this repository" + +# Can also be specified via environment variable `KNOT_MESSAGES_HTTP_REPO_NOT_FOUND`. +# Default value: "repository not found" +#repo_not_found = "repository not found" + +# Can also be specified via environment variable `KNOT_MESSAGES_HTTP_PUSH_TOO_LARGE`. +# Default value: "push exceeds the configured size limit" +#push_too_large = "push exceeds the configured size limit" + +# Can also be specified via environment variable `KNOT_MESSAGES_HTTP_MALFORMED_PACK`. +# Default value: "malformed pack stream: {error}" +#malformed_pack = "malformed pack stream: {error}" + +# Can also be specified via environment variable `KNOT_MESSAGES_HTTP_RECEIVE_ENDED_EARLY`. +# Default value: "receive stream ended early" +#receive_ended_early = "receive stream ended early" + +[messages.lfs] +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_INVALID_OID`. +# Default value: "invalid LFS oid {value}" +#invalid_oid = "invalid LFS oid {value}" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_HASH_MISMATCH`. +# Default value: "oid mismatch, declared {declared}, computed {computed}" +#hash_mismatch = "oid mismatch, declared {declared}, computed {computed}" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_SIZE_MISMATCH`. +# Default value: "size mismatch, declared {declared}, received {received}" +#size_mismatch = "size mismatch, declared {declared}, received {received}" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_SIZE_LIMIT_EXCEEDED`. +# Default value: "object size {declared} exceeds limit {limit}" +#size_limit_exceeded = "object size {declared} exceeds limit {limit}" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_FREE_SPACE_DENIED`. +# Default value: "free space {free} below floor {floor}" +#free_space_denied = "free space {free} below floor {floor}" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_NOT_FOUND`. +# Default value: "object {oid} not found" +#not_found = "object {oid} not found" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_FRAMING`. +# Default value: "protocol framing fault: {detail}" +#framing = "protocol framing fault: {detail}" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_TOO_MANY`. +# Default value: "too many {what} in one message, limit {limit}" +#too_many = "too many {what} in one message, limit {limit}" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_UNKNOWN_COMMAND`. +# Default value: "unknown command {command}" +#unknown_command = "unknown command {command}" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_UNSUPPORTED_VERSION`. +# Default value: "unsupported version {version}" +#unsupported_version = "unsupported version {version}" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_UNSUPPORTED_HASH`. +# Default value: "unsupported hash algorithm {algorithm}" +#unsupported_hash = "unsupported hash algorithm {algorithm}" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_PUT_ON_DOWNLOAD`. +# Default value: "put-object isn't allowed on a download channel" +#put_on_download = "put-object isn't allowed on a download channel" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_VERIFY_ON_DOWNLOAD`. +# Default value: "verify-object isn't allowed on a download channel" +#verify_on_download = "verify-object isn't allowed on a download channel" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_GET_ON_UPLOAD`. +# Default value: "get-object isn't allowed on an upload channel" +#get_on_upload = "get-object isn't allowed on an upload channel" + +# Can also be specified via environment variable `KNOT_MESSAGES_LFS_PUT_NO_BODY`. +# Default value: "put-object is missing its object body" +#put_no_body = "put-object is missing its object body" diff --git a/knot2/interop/atproto_firehose_test.go b/knot2/interop/atproto_firehose_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d8d4fa623a8488b4bd8f7b89f4a180a1d5a85043 --- /dev/null +++ b/knot2/interop/atproto_firehose_test.go @@ -0,0 +1,219 @@ +//go:build interop + +package interop + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "testing" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func readStreamHeader(r io.Reader) (string, error) { + maj, count, err := cbg.CborReadHeader(r) + if err != nil { + return "", err + } + if maj != cbg.MajMap { + return "", fmt.Errorf("expected a cbor map header, got major type %d", maj) + } + msgType := "" + for range count { + key, err := cbg.ReadString(r) + if err != nil { + return "", err + } + switch key { + case "op": + if _, _, err := cbg.CborReadHeader(r); err != nil { + return "", err + } + case "t": + if msgType, err = cbg.ReadString(r); err != nil { + return "", err + } + default: + return "", fmt.Errorf("unknown header field %q", key) + } + } + return msgType, nil +} + +type firehoseFrame struct { + MsgType string + Commit *comatproto.SyncSubscribeRepos_Commit + Sync *comatproto.SyncSubscribeRepos_Sync + Identity *comatproto.SyncSubscribeRepos_Identity + Account *comatproto.SyncSubscribeRepos_Account + Info *comatproto.SyncSubscribeRepos_Info +} + +func nextFrame(t *testing.T, conn *websocket.Conn) firehoseFrame { + t.Helper() + require.NoError(t, conn.SetReadDeadline(time.Now().Add(15*time.Second))) + messageType, data, err := conn.ReadMessage() + require.NoError(t, err) + require.Equal(t, websocket.BinaryMessage, messageType) + reader := bytes.NewReader(data) + msgType, err := readStreamHeader(reader) + require.NoError(t, err, "frame % x", data) + frame := firehoseFrame{MsgType: msgType} + switch msgType { + case "#commit": + var evt comatproto.SyncSubscribeRepos_Commit + require.NoError(t, evt.UnmarshalCBOR(reader)) + frame.Commit = &evt + case "#sync": + var evt comatproto.SyncSubscribeRepos_Sync + require.NoError(t, evt.UnmarshalCBOR(reader)) + frame.Sync = &evt + case "#identity": + var evt comatproto.SyncSubscribeRepos_Identity + require.NoError(t, evt.UnmarshalCBOR(reader)) + frame.Identity = &evt + case "#account": + var evt comatproto.SyncSubscribeRepos_Account + require.NoError(t, evt.UnmarshalCBOR(reader)) + frame.Account = &evt + case "#info": + var evt comatproto.SyncSubscribeRepos_Info + require.NoError(t, evt.UnmarshalCBOR(reader)) + frame.Info = &evt + default: + t.Fatalf("unknown frame type %q", msgType) + } + require.Zero(t, reader.Len(), "a frame holds exactly two objects") + return frame +} + +func authedPost(t *testing.T, path, token string, body any) map[string]any { + t.Helper() + encoded, err := json.Marshal(body) + require.NoError(t, err) + request, err := http.NewRequest( + http.MethodPost, + os.Getenv("KNOT_TEST_BASE_URL")+path, + bytes.NewReader(encoded), + ) + require.NoError(t, err) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Authorization", "Bearer "+token) + response, err := http.DefaultClient.Do(request) + require.NoError(t, err) + defer response.Body.Close() + raw, err := io.ReadAll(response.Body) + require.NoError(t, err) + require.Equal( + t, + http.StatusOK, + response.StatusCode, + "%s answered %s: %s", + path, + response.Status, + string(raw), + ) + decoded := map[string]any{} + if len(raw) > 0 { + require.NoError(t, json.Unmarshal(raw, &decoded)) + } + return decoded +} + +func TestKnotServesSubscribeRepos(t *testing.T) { + base := os.Getenv("KNOT_TEST_BASE_URL") + require.NotEmpty(t, base, "KNOT_TEST_BASE_URL is the knot under test") + createToken := os.Getenv("KNOT_TEST_CREATE_TOKEN") + require.NotEmpty(t, createToken, "KNOT_TEST_CREATE_TOKEN holds the create-scope token") + deleteToken := os.Getenv("KNOT_TEST_DELETE_TOKEN") + require.NotEmpty(t, deleteToken, "KNOT_TEST_DELETE_TOKEN holds the delete-scope token") + endpoint := "ws://" + strings.TrimPrefix(base, "http://") + "/xrpc/com.atproto.sync.subscribeRepos" + conn, _, err := websocket.DefaultDialer.DialContext(context.Background(), endpoint, nil) + require.NoError(t, err) + defer conn.Close() + + created := authedPost(t, "/xrpc/org.tangled.repo.createRepo", createToken, map[string]any{ + "name": "limpet", + }) + repoDid, _ := created["did"].(string) + require.NotEmpty(t, repoDid, "create named the repository it minted: %v", created) + + identity := nextFrame(t, conn) + require.NotNil(t, identity.Identity) + assert.Equal(t, repoDid, identity.Identity.Did) + assert.Greater(t, identity.Identity.Seq, int64(0)) + assert.Nil(t, identity.Identity.Handle, "the knot names no handle it doesn't answer for") + + account := nextFrame(t, conn) + require.NotNil(t, account.Account) + assert.Equal(t, repoDid, account.Account.Did) + assert.True(t, account.Account.Active) + assert.Nil(t, account.Account.Status) + + commit := nextFrame(t, conn) + require.NotNil(t, commit.Commit) + genesis := commit.Commit + assert.Equal(t, repoDid, genesis.Repo) + assert.Empty(t, genesis.Ops, "a fresh account's first commit is empty") + assert.False(t, genesis.Rebase) + assert.False(t, genesis.TooBig) + assert.Nil(t, genesis.Since, "a genesis commit names no predecessor") + assert.Nil(t, genesis.PrevData, "a genesis commit omits prevData rather than nulling it") + assert.Empty(t, genesis.Blobs) + rev, err := syntax.ParseTID(genesis.Rev) + require.NoError(t, err) + assert.Equal( + t, + rev.Time().UnixMicro(), + genesis.Seq, + "the seq is the microsecond tick of the commit's own rev", + ) + assert.NotEmpty(t, genesis.Blocks, "the commit frame includes its CAR") + assert.NotEmpty(t, genesis.Commit.String()) + + lastRev := genesis.Rev + lastSeq := genesis.Seq + sync := nextFrame(t, conn) + for sync.Sync == nil { + require.NotNil(t, sync.Commit, "creation only announces commits before the sync assertion") + assert.Equal(t, repoDid, sync.Commit.Repo) + assert.Less(t, lastSeq, sync.Commit.Seq, "creation's commits stay in seq order") + lastRev, lastSeq = sync.Commit.Rev, sync.Commit.Seq + sync = nextFrame(t, conn) + } + assert.Equal(t, repoDid, sync.Sync.Did) + assert.Equal(t, lastRev, sync.Sync.Rev, "the sync frame asserts the last commit creation announced") + assert.NotEmpty(t, sync.Sync.Blocks, "the sync frame includes the commit block") + + assert.Less(t, identity.Identity.Seq, account.Account.Seq, "identity precedes account") + assert.Less(t, account.Account.Seq, genesis.Seq, "account precedes the genesis commit") + assert.Less(t, genesis.Seq, sync.Sync.Seq, "the commit precedes its assertion") + + authedPost(t, "/xrpc/org.tangled.repo.deleteRepo", deleteToken, map[string]any{ + "repo": repoDid, + "force": true, + }) + for range 10 { + frame := nextFrame(t, conn) + if frame.Account == nil || frame.Account.Did != repoDid { + continue + } + assert.False(t, frame.Account.Active) + require.NotNil(t, frame.Account.Status) + assert.Equal(t, "deleted", *frame.Account.Status) + return + } + t.Fatal("no account frame closed the lifecycle") +} diff --git a/knot2/interop/atproto_plc_attestation_test.go b/knot2/interop/atproto_plc_attestation_test.go new file mode 100644 index 0000000000000000000000000000000000000000..fab1754598201ff217866ca1365bb3be1ad1a3f4 --- /dev/null +++ b/knot2/interop/atproto_plc_attestation_test.go @@ -0,0 +1,68 @@ +//go:build interop + +package interop + +import ( + "context" + "net/http" + "os" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const attestationCollection = "org.tangled.graph.attestation" +const attestationRkey = "self" + +func TestIndigoClientResolvesBobsIssueAndMatchesHisAttestation(t *testing.T) { + ctx := context.Background() + xc := knotClient(t) + did, err := syntax.ParseDID(os.Getenv("KNOT_TEST_REPO_DID")) + require.NoError(t, err, "KNOT_TEST_REPO_DID must be a did") + collection, err := syntax.ParseNSID(ticketCollection) + require.NoError(t, err, "ticket collection must parse as an nsid") + rkey, err := syntax.ParseRecordKey(os.Getenv("KNOT_TEST_TICKET_RKEY")) + require.NoError(t, err, "KNOT_TEST_TICKET_RKEY must be a record key") + editor := os.Getenv("KNOT_TEST_EDITOR") + bobDID := os.Getenv("KNOT_TEST_BOB_DID") + + dir := &identity.BaseDirectory{ + PLCURL: os.Getenv("KNOT_TEST_PLC_URL"), + PLCClient: &http.Client{Timeout: 10 * time.Second}, + } + + _, full, recordCID := provenRecord(t, ctx, xc, dir, did, collection, rkey) + + raw, _, err := full.GetRecordBytes(ctx, collection, rkey) + require.NoError(t, err, "signed tree must contain the ticket leaf") + stored, _ := leafFields(t, raw) + assert.Equal(t, editor, stored["x-tngl-editor"], "block must identify its editor") + + atURI := "at://" + did.String() + "/" + collection.String() + "/" + rkey.String() + + var attestation struct { + Value struct { + Subject struct { + Uri string `json:"uri"` + Cid string `json:"cid"` + } `json:"subject"` + } `json:"value"` + } + bob := &indigoxrpc.Client{ + Host: os.Getenv("KNOT_TEST_PDS_URL"), + Client: &http.Client{Timeout: 10 * time.Second}, + } + err = bob.LexDo(ctx, indigoxrpc.Query, "", "com.atproto.repo.getRecord", map[string]any{ + "repo": bobDID, + "collection": attestationCollection, + "rkey": attestationRkey, + }, nil, &attestation) + require.NoError(t, err, "getRecord must return the attestation") + assert.Equal(t, atURI, attestation.Value.Subject.Uri, "attestation subject must identify the verified ticket") + assert.Equal(t, recordCID.String(), attestation.Value.Subject.Cid, "attestation strongRef must point at the verified cid") +} diff --git a/knot2/interop/atproto_reads_test.go b/knot2/interop/atproto_reads_test.go new file mode 100644 index 0000000000000000000000000000000000000000..160d8afcb04c8e7d4716d0edeaf07e4ab91a5274 --- /dev/null +++ b/knot2/interop/atproto_reads_test.go @@ -0,0 +1,180 @@ +//go:build interop + +package interop + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "testing" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/repo" + "github.com/bluesky-social/indigo/atproto/syntax" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "github.com/ipfs/go-cid" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cbg "github.com/whyrusleeping/cbor-gen" +) + +type stringFields map[string]string + +func (fields stringFields) UnmarshalCBOR(r io.Reader) error { + maj, count, err := cbg.CborReadHeader(r) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("expected a cbor map, got major type %d", maj) + } + return fields.read(r, count) +} + +func (fields stringFields) read(r io.Reader, remaining uint64) error { + if remaining == 0 { + return nil + } + key, err := cbg.ReadString(r) + if err != nil { + return err + } + value, err := cbg.ReadString(r) + if err != nil { + return err + } + fields[key] = value + return fields.read(r, remaining-1) +} + +func knotClient(t *testing.T) *indigoxrpc.Client { + t.Helper() + return &indigoxrpc.Client{ + Host: os.Getenv("KNOT_TEST_BASE_URL"), + Client: &http.Client{Timeout: 10 * time.Second}, + } +} + +func directory(t *testing.T, didEnv string) identity.Directory { + t.Helper() + did, err := syntax.ParseDID(os.Getenv(didEnv)) + require.NoError(t, err, "%s must be a did", didEnv) + resp, err := http.Get(os.Getenv("KNOT_TEST_BASE_URL") + "/.well-known/did.json") + require.NoError(t, err, "did.json must be reachable over http") + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, "did.json fetch must return 200") + var doc identity.DIDDocument + require.NoError(t, json.NewDecoder(resp.Body).Decode(&doc), "did.json must parse as a did document") + require.Equal(t, did, doc.DID, "served did.json must match the knot did") + dir := identity.NewMockDirectory() + dir.Insert(identity.ParseIdentity(&doc)) + return dir +} + +func provenRecord(t *testing.T, ctx context.Context, xc *indigoxrpc.Client, dir identity.Directory, did syntax.DID, collection syntax.NSID, rkey syntax.RecordKey) (*repo.Commit, *repo.Repo, cid.Cid) { + t.Helper() + repoCar, err := comatproto.SyncGetRepo(ctx, xc, did.String(), "") + require.NoError(t, err, "sync.getRepo must succeed for an unmodified client, did %s", did) + + commit, err := repo.VerifyCommitSignatureFromCar(ctx, dir, repoCar) + require.NoError(t, err, "the signed commit verifies against the did document key") + assert.Equal(t, did.String(), commit.DID) + + _, full, err := repo.LoadRepoFromCAR(ctx, bytes.NewReader(repoCar)) + require.NoError(t, err, "the whole tree loads from the getRepo car") + recordCID, err := full.GetRecordCID(ctx, collection, rkey) + require.NoError(t, err, "the record's inclusion proof verifies the record against the mst under the signed commit") + require.NotNil(t, recordCID) + + proofCar, err := comatproto.SyncGetRecord(ctx, xc, collection.String(), did.String(), rkey.String()) + require.NoError(t, err, "sync.getRecord must succeed for an unmodified client") + proofCommit, _, err := repo.LoadCommitFromCAR(ctx, bytes.NewReader(proofCar)) + require.NoError(t, err, "the proof car roots at the signed commit") + assert.Equal(t, commit.Data.String(), proofCommit.Data.String()) + assert.Equal(t, commit.Sig, proofCommit.Sig, "the same commit rides both cars") + _, proofRepo, err := repo.LoadRepoFromCAR(ctx, bytes.NewReader(proofCar)) + require.NoError(t, err, "the inclusion proof has the blocks the walk needs") + proofCID, err := proofRepo.GetRecordCID(ctx, collection, rkey) + require.NoError(t, err) + assert.Equal(t, recordCID.String(), proofCID.String()) + return commit, full, *recordCID +} + +func TestKnotServesComAtprotoReads(t *testing.T) { + ctx := context.Background() + xc := knotClient(t) + did, err := syntax.ParseDID(os.Getenv("KNOT_TEST_DID")) + require.NoError(t, err, "KNOT_TEST_DID must be a did") + collection, err := syntax.ParseNSID(os.Getenv("KNOT_TEST_COLLECTION")) + require.NoError(t, err, "KNOT_TEST_COLLECTION must be an nsid") + rkey, err := syntax.ParseRecordKey(os.Getenv("KNOT_TEST_RKEY")) + require.NoError(t, err, "KNOT_TEST_RKEY must be a record key") + + latest, err := comatproto.SyncGetLatestCommit(ctx, xc, did.String()) + require.NoError(t, err, "sync.getLatestCommit must return a commit for an unmodified client") + assert.NotEmpty(t, latest.Rev) + + dir := directory(t, "KNOT_TEST_DID") + commit, full, recordCID := provenRecord(t, ctx, xc, dir, did, collection, rkey) + assert.Equal(t, latest.Rev, commit.Rev) + + blocksCar, err := comatproto.SyncGetBlocks(ctx, xc, []string{commit.Data.String()}, did.String()) + require.NoError(t, err, "sync.getBlocks must succeed for an unmodified client") + _, blocksRepo, err := repo.LoadRepoFromCAR(ctx, bytes.NewReader(blocksCar)) + require.NoError(t, err, "the mst root arrives by name") + blockCID, err := blocksRepo.GetRecordCID(ctx, collection, rkey) + require.NoError(t, err) + assert.Equal(t, recordCID.String(), blockCID.String()) + + listed, err := comatproto.SyncListRepos(ctx, xc, "", 100) + require.NoError(t, err, "sync.listRepos must succeed for an unmodified client") + assert.True(t, lo.ContainsBy(listed.Repos, func(row *comatproto.SyncListRepos_Repo) bool { + return row.Did == did.String() + })) + + var records struct { + Records []struct { + Uri string `json:"uri"` + Value struct { + CreatedAt string `json:"createdAt"` + } `json:"value"` + } `json:"records"` + } + err = xc.LexDo(ctx, indigoxrpc.Query, "", "com.atproto.repo.listRecords", map[string]any{ + "repo": did.String(), + "collection": collection.String(), + "limit": 50, + }, nil, &records) + require.NoError(t, err, "repo.listRecords must succeed for an unmodified client") + require.NotEmpty(t, records.Records) + assert.Equal(t, "at://"+did.String()+"/"+collection.String()+"/"+rkey.String(), records.Records[0].Uri) + assert.Equal(t, "1970-01-01T00:18:20Z", records.Records[0].Value.CreatedAt) + + raw, _, err := full.GetRecordBytes(ctx, collection, rkey) + require.NoError(t, err, "the stored record bytes come back out of the signed tree") + stored := stringFields{} + require.NoError(t, stored.UnmarshalCBOR(bytes.NewReader(raw))) + assert.Equal(t, "did:web:olaren.dev", stored["x-tngl-editor"], "the knot must serve the undeclared editor field") + assert.Equal(t, "1970-01-01T00:18:20Z", stored["createdAt"]) + + describe, err := comatproto.RepoDescribeRepo(ctx, xc, did.String()) + require.NoError(t, err, "repo.describeRepo must succeed for an unmodified client") + assert.Equal(t, did.String(), describe.Did) + assert.Contains(t, lo.Map(describe.Collections, func(row string, _ int) string { return row }), collection.String()) + + status, err := comatproto.SyncGetRepoStatus(ctx, xc, did.String()) + require.NoError(t, err, "sync.getRepoStatus must succeed for an unmodified client") + require.NotNil(t, status.Rev) + assert.Equal(t, latest.Rev, *status.Rev) + + resolved, err := comatproto.IdentityResolveHandle(ctx, xc, os.Getenv("KNOT_TEST_HANDLE")) + require.NoError(t, err, "identity.resolveHandle must succeed for an unmodified client") + assert.Equal(t, did.String(), resolved.Did) +} diff --git a/knot2/interop/atproto_ticket_test.go b/knot2/interop/atproto_ticket_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8699cef126b8fe36284fd01ec8efec9f68baeb71 --- /dev/null +++ b/knot2/interop/atproto_ticket_test.go @@ -0,0 +1,175 @@ +//go:build interop + +package interop + +import ( + "bufio" + "bytes" + "context" + "os" + "testing" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/repo" + "github.com/bluesky-social/indigo/atproto/syntax" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cbg "github.com/whyrusleeping/cbor-gen" +) + +const ticketCollection = "org.tangled.track.ticket" +const patchCollection = "org.tangled.review.patch" + +type ticketRecord struct { + Records []struct { + Uri string `json:"uri"` + Cid string `json:"cid"` + Value struct { + Type string `json:"$type"` + State string `json:"state"` + TargetBranch string `json:"targetBranch"` + Title struct { + Type string `json:"$type"` + Text string `json:"text"` + } `json:"title"` + CreatedAt string `json:"createdAt"` + Editor string `json:"x-tngl-editor"` + } `json:"value"` + } `json:"records"` +} + +type patchRecord struct { + Uri string `json:"uri"` + Cid string `json:"cid"` + Value struct { + Type string `json:"$type"` + Ticket string `json:"ticket"` + Base struct { + Oid string `json:"oid"` + } `json:"base"` + Head struct { + Oid string `json:"oid"` + } `json:"head"` + CreatedAt string `json:"createdAt"` + Editor string `json:"x-tngl-editor"` + } `json:"value"` +} + +func leafFields(t *testing.T, raw []byte) (stringFields, map[string]stringFields) { + t.Helper() + reader := bufio.NewReader(bytes.NewReader(raw)) + maj, pairs, err := cbg.CborReadHeader(reader) + require.NoError(t, err, "the ticket leaf opens as a cbor map") + require.Equal(t, byte(cbg.MajMap), maj, "the ticket leaf is a cbor map") + flat := stringFields{} + unions := map[string]stringFields{} + for i := uint64(0); i < pairs; i++ { + key, err := cbg.ReadString(reader) + require.NoError(t, err, "the leaf's keys are text strings") + head, err := reader.Peek(1) + require.NoError(t, err, "every leaf field has a value") + switch head[0] >> 5 { + case cbg.MajTextString: + value, err := cbg.ReadString(reader) + require.NoError(t, err, "the leaf's %s is a text string", key) + flat[key] = value + case cbg.MajMap: + union := stringFields{} + require.NoError(t, union.UnmarshalCBOR(reader), "the leaf's %s is a map of text fields", key) + unions[key] = union + default: + t.Fatalf("unexpected cbor major type %d for leaf field %s", head[0]>>5, key) + } + } + return flat, unions +} + +func TestKnotServesTicketsAndPatchesInTheOrgTangledShapes(t *testing.T) { + ctx := context.Background() + xc := knotClient(t) + did, err := syntax.ParseDID(os.Getenv("KNOT_TEST_REPO_DID")) + require.NoError(t, err, "KNOT_TEST_REPO_DID must be a did") + ticketRkey, err := syntax.ParseRecordKey(os.Getenv("KNOT_TEST_TICKET_RKEY")) + require.NoError(t, err, "KNOT_TEST_TICKET_RKEY must be a record key") + ticketCID := os.Getenv("KNOT_TEST_TICKET_CID") + require.NotEmpty(t, ticketCID, "KNOT_TEST_TICKET_CID must be the cid the write returned") + patchRkey, err := syntax.ParseRecordKey(os.Getenv("KNOT_TEST_PATCH_RKEY")) + require.NoError(t, err, "KNOT_TEST_PATCH_RKEY must be a record key") + patchCID := os.Getenv("KNOT_TEST_PATCH_CID") + require.NotEmpty(t, patchCID, "KNOT_TEST_PATCH_CID must be the cid the write returned") + title := os.Getenv("KNOT_TEST_TICKET_TITLE") + editor := os.Getenv("KNOT_TEST_EDITOR") + var tickets ticketRecord + err = xc.LexDo(ctx, indigoxrpc.Query, "", "com.atproto.repo.listRecords", map[string]any{ + "repo": did.String(), + "collection": ticketCollection, + "limit": 50, + }, nil, &tickets) + require.NoError(t, err, "repo.listRecords pages the ticket collection") + found := false + for _, row := range tickets.Records { + if row.Uri != "at://"+did.String()+"/"+ticketCollection+"/"+ticketRkey.String() { + continue + } + found = true + assert.Equal(t, ticketCID, row.Cid, "the listed ticket has the cid the write returned") + assert.Equal(t, ticketCollection, row.Value.Type) + assert.Equal(t, "org.tangled.markup.markdown#inline", row.Value.Title.Type, + "a ticket's title is the inline markdown union") + assert.Contains(t, []string{"open", "closed", "completed"}, row.Value.State, + "a ticket's state is one of the three track.defs tokens") + assert.Equal(t, title, row.Value.Title.Text, "the served ticket keeps the written title") + assert.NotEmpty(t, row.Value.CreatedAt) + assert.Equal(t, editor, row.Value.Editor, "the knot serves the undeclared editor field") + } + assert.True(t, found, "the wanted ticket must be among the listed records") + + var patch patchRecord + err = xc.LexDo(ctx, indigoxrpc.Query, "", "com.atproto.repo.getRecord", map[string]any{ + "repo": did.String(), + "collection": patchCollection, + "rkey": patchRkey.String(), + }, nil, &patch) + require.NoError(t, err, "repo.getRecord serves the patch's at-uri") + assert.Equal(t, patchCollection, patch.Value.Type) + assert.Equal(t, patchCID, patch.Cid, "the served patch has the cid the write returned") + assert.Equal(t, patchRkey.String(), patch.Cid, + "a patch's record key is the CID of its own record") + assert.NotEmpty(t, patch.Value.Ticket, "a patch cites its ticket's rkey") + for _, oid := range []string{patch.Value.Base.Oid, patch.Value.Head.Oid} { + length := len(oid) + assert.Contains(t, []int{40, 64}, length, "a patch's oids are hex object ids: %s", oid) + } + + collection, err := syntax.ParseNSID(ticketCollection) + require.NoError(t, err, "the ticket collection must parse as an nsid") + repoCar, err := comatproto.SyncGetRepo(ctx, xc, did.String(), "") + require.NoError(t, err, "sync.getRepo serves the ticket tree") + _, full, err := repo.LoadRepoFromCAR(ctx, bytes.NewReader(repoCar)) + require.NoError(t, err, "the whole tree loads from the getRepo car") + raw, leafCID, err := full.GetRecordBytes(ctx, collection, ticketRkey) + require.NoError(t, err, "getRepo serves the ticket leaf like any other block") + require.NotNil(t, leafCID) + assert.Equal(t, ticketCID, leafCID.String(), "the block walk resolves the cid the write returned") + stored, unions := leafFields(t, raw) + assert.Equal(t, ticketCollection, stored["$type"]) + assert.Contains(t, []string{"open", "closed", "completed"}, stored["state"], + "a ticket's state is one of the three track.defs tokens") + assert.Equal(t, editor, stored["x-tngl-editor"], "the block must identify who wrote it") + storedTitle, ok := unions["title"] + require.True(t, ok, "the leaf stores the title as the inline markup union") + assert.Equal(t, "org.tangled.markup.markdown#inline", storedTitle["$type"]) + assert.Equal(t, title, storedTitle["text"], "the block bytes keep the written title") + + blocksCar, err := comatproto.SyncGetBlocks(ctx, xc, []string{ticketCID}, did.String()) + require.NoError(t, err, "sync.getBlocks serves the ticket leaf by cid") + assert.Contains(t, string(blocksCar), "a forest", "the leaf must come back by cid") + + describe, err := comatproto.RepoDescribeRepo(ctx, xc, did.String()) + require.NoError(t, err, "repo.describeRepo serves the knot's repo") + collections := lo.Map(describe.Collections, func(row string, _ int) string { return row }) + assert.Contains(t, collections, ticketCollection, "describeRepo lists the ticket collection") + assert.Contains(t, collections, patchCollection, "describeRepo lists the patch collection") +} diff --git a/knot2/interop/foster_test.go b/knot2/interop/foster_test.go new file mode 100644 index 0000000000000000000000000000000000000000..504a0c8c1f29b740af4e2dfe200e49ef5866fa04 --- /dev/null +++ b/knot2/interop/foster_test.go @@ -0,0 +1,347 @@ +//go:build interop + +package interop + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "tangled.org/core/knotfeed" + "tangled.org/core/tapc" +) + +const seedRoute = "/xrpc/org.tangled.temp.server.seedRepo" +const fosterOwner = "did:plc:boltfulboy" +const fosterRkey = "the-feral-pearl-ledger" +const fosterCreatedAt = "1970-01-01T00:18:20Z" + +func seedPost(t *testing.T, user, password string, body any) (int, string) { + t.Helper() + encoded, err := json.Marshal(body) + require.NoError(t, err) + request, err := http.NewRequest( + http.MethodPost, + os.Getenv("KNOT_TEST_BASE_URL")+seedRoute, + bytes.NewReader(encoded), + ) + require.NoError(t, err) + request.Header.Set("Content-Type", "application/json") + if user != "" { + request.SetBasicAuth(user, password) + } + client := &http.Client{Timeout: 30 * time.Second} + response, err := client.Do(request) + require.NoError(t, err) + defer response.Body.Close() + raw, err := io.ReadAll(response.Body) + require.NoError(t, err) + return response.StatusCode, string(raw) +} + +func seedRequest(t *testing.T) (map[string]any, syntax.RecordKey) { + t.Helper() + rkey, err := syntax.ParseRecordKey(fosterRkey) + require.NoError(t, err, "fosterRkey must be a record key") + fosterRepoDid := os.Getenv("KNOT_TEST_FOSTER_REPO_DID") + require.NotEmpty( + t, + fosterRepoDid, + "KNOT_TEST_FOSTER_REPO_DID is the foreign repo DID the knot fosters", + ) + return map[string]any{ + "repoDid": fosterRepoDid, + "manifest": map[string]any{ + "$type": "org.tangled.repo.manifest", + "declaration": map[string]any{"owner": fosterOwner, "name": "boltfulboy"}, + }, + "records": []map[string]any{{ + "collection": ticketCollection, + "rkey": rkey.String(), + "author": fosterOwner, + "createdAt": 1100, + "record": map[string]any{ + "$type": ticketCollection, + "title": map[string]any{"$type": "org.tangled.markup.markdown#inline", "text": "a feral pearl"}, + "state": "open", + "createdAt": fosterCreatedAt, + "x-tngl-editor": fosterOwner, + }, + }}, + }, rkey +} + +func pushRepo(t *testing.T, source, addr, did, token string) { + t.Helper() + header := base64.StdEncoding.EncodeToString([]byte("x-tangled-token:" + token)) + command := exec.Command( + "git", "-C", source, + "-c", "http.extraHeader=Authorization: Basic "+header, + "push", "-q", "--tags", + fmt.Sprintf("http://%s/%s", addr, did), + "main", + ) + command.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0") + output, err := command.CombinedOutput() + require.NoError(t, err, "the receive-pack push must deliver the staged history: %s", output) +} + +func cloneRepo(t *testing.T, url string) string { + t.Helper() + clone := filepath.Join(t.TempDir(), "clone") + cloneCommand := exec.Command("git", "clone", "-q", url, clone) + cloneCommand.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0") + output, err := cloneCommand.CombinedOutput() + require.NoError(t, err, "git clone must fetch the fostered repo: %s", output) + return clone +} + +func gitIn(t *testing.T, dir string, args ...string) string { + t.Helper() + output, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput() + require.NoError(t, err, "git %v failed: %s", args, output) + return strings.TrimSpace(string(output)) +} + +func TestASeededKnotFostersAForeignRepoVerbatim(t *testing.T) { + base := os.Getenv("KNOT_TEST_BASE_URL") + require.NotEmpty(t, base, "KNOT_TEST_BASE_URL is the knot under test") + addr := os.Getenv("KNOT_TEST_ADDR") + require.NotEmpty(t, addr, "KNOT_TEST_ADDR is the knot's host:port") + secret := os.Getenv("KNOT_TEST_FOSTER_SECRET") + require.NotEmpty(t, secret, "KNOT_TEST_FOSTER_SECRET is the foster secret") + source := os.Getenv("KNOT_TEST_FOSTER_SOURCE") + require.NotEmpty(t, source, "KNOT_TEST_FOSTER_SOURCE is the staged source repo") + wantSha := os.Getenv("KNOT_TEST_SHA") + require.NotEmpty(t, wantSha, "KNOT_TEST_SHA is the staged source tip") + pushToken := os.Getenv("KNOT_TEST_PUSH_TOKEN") + require.NotEmpty(t, pushToken, "KNOT_TEST_PUSH_TOKEN authorizes the receive-pack push") + + ctx := context.Background() + xc := knotClient(t) + body, rkey := seedRequest(t) + + wrongPasswordStatus, wrongPasswordBody := seedPost(t, "admin", "not-the-secret", body) + wrongUserStatus, wrongUserBody := seedPost(t, "root", secret, body) + noHeaderStatus, noHeaderBody := seedPost(t, "", "", body) + for _, attempt := range []struct { + status int + name string + }{ + {wrongPasswordStatus, "a wrong password"}, + {wrongUserStatus, "a wrong user"}, + {noHeaderStatus, "no header"}, + } { + require.Equal(t, http.StatusUnauthorized, attempt.status, "%s is rejected", attempt.name) + } + assert.Equal(t, wrongPasswordBody, wrongUserBody, "a wrong password and a wrong user are indistinguishable") + assert.Equal(t, wrongPasswordBody, noHeaderBody, "a missing header is indistinguishable too") + + status, raw := seedPost(t, "admin", secret, body) + require.Equal(t, http.StatusOK, status, "the seed request must succeed: %s", raw) + var seeded struct { + Did string `json:"did"` + Key string `json:"key"` + Results []struct { + Collection string `json:"collection"` + Rkey string `json:"rkey"` + Error string `json:"error"` + } `json:"results"` + } + require.NoError(t, json.Unmarshal([]byte(raw), &seeded)) + require.Equal( + t, + os.Getenv("KNOT_TEST_FOSTER_REPO_DID"), + seeded.Did, + "the response echoes the supplied repo DID, never a minted one", + ) + require.NotEmpty(t, seeded.Key, "the response reports the repo's signing key") + require.Len(t, seeded.Results, 1, "one record was seeded") + assert.Equal(t, ticketCollection, seeded.Results[0].Collection) + assert.Equal(t, rkey.String(), seeded.Results[0].Rkey) + assert.Empty(t, seeded.Results[0].Error, "the verbatim record seeds without lexicon validation") + + did, err := syntax.ParseDID(seeded.Did) + require.NoError(t, err) + collection, err := syntax.ParseNSID(ticketCollection) + require.NoError(t, err) + + var listed struct { + Records []struct { + Uri string `json:"uri"` + Value struct { + State string `json:"state"` + CreatedAt string `json:"createdAt"` + Editor string `json:"x-tngl-editor"` + } `json:"value"` + } `json:"records"` + } + err = xc.LexDo(ctx, indigoxrpc.Query, "", "com.atproto.repo.listRecords", map[string]any{ + "repo": did.String(), + "collection": ticketCollection, + "limit": 50, + }, nil, &listed) + require.NoError(t, err, "repo.listRecords serves the fostered repo") + require.Len(t, listed.Records, 1, "exactly one ticket is listed") + assert.Equal(t, "at://"+did.String()+"/"+ticketCollection+"/"+rkey.String(), listed.Records[0].Uri) + assert.Equal(t, "feral", listed.Records[0].Value.State, "the lexicon-invalid state is served verbatim") + assert.Equal(t, fosterCreatedAt, listed.Records[0].Value.CreatedAt, "the 1970 createdAt is served verbatim") + assert.Equal(t, fosterOwner, listed.Records[0].Value.Editor, "the foreign editor is served verbatim") + + var fetched struct { + Uri string `json:"uri"` + Value struct { + Title struct { + Type string `json:"$type"` + Text string `json:"text"` + } `json:"title"` + } `json:"value"` + } + err = xc.LexDo(ctx, indigoxrpc.Query, "", "com.atproto.repo.getRecord", map[string]any{ + "repo": did.String(), + "collection": ticketCollection, + "rkey": rkey.String(), + }, nil, &fetched) + require.NoError(t, err, "repo.getRecord serves the seeded record") + assert.Equal(t, listed.Records[0].Uri, fetched.Uri) + assert.Equal(t, "org.tangled.markup.markdown#inline", fetched.Value.Title.Type, "the title union is served in the org.tangled shape") + assert.Equal(t, "a feral pearl", fetched.Value.Title.Text) + + document := fmt.Sprintf( + `{"id":%q,"verificationMethod":[{"id":%q,"type":"Multikey","controller":%q,"publicKeyMultibase":%q}]}`, + did.String(), did.String()+"#atproto", did.String(), seeded.Key, + ) + var doc identity.DIDDocument + require.NoError(t, json.Unmarshal([]byte(document), &doc), "the seed response includes the key material of a did document") + dir := identity.NewMockDirectory() + dir.Insert(identity.ParseIdentity(&doc)) + _, full, _ := provenRecord(t, ctx, xc, dir, did, collection, rkey) + + leaf, _, err := full.GetRecordBytes(ctx, collection, rkey) + require.NoError(t, err, "the stored record bytes come back out of the signed tree") + stored, unions := leafFields(t, leaf) + assert.Equal(t, ticketCollection, stored["$type"]) + assert.Equal(t, "feral", stored["state"], "the cbor leaf keeps the state the validated write path would refuse") + assert.Equal(t, fosterCreatedAt, stored["createdAt"], "the cbor leaf keeps the 1970 createdAt") + assert.Equal(t, fosterOwner, stored["x-tngl-editor"], "the cbor leaf keeps the foreign editor") + title, ok := unions["title"] + require.True(t, ok, "the leaf stores the title as the inline markup union") + assert.Equal(t, "org.tangled.markup.markdown#inline", title["$type"]) + assert.Equal(t, "a feral pearl", title["text"]) + + pushRepo(t, source, addr, did.String(), pushToken) + clone := cloneRepo(t, "http://"+addr+"/"+did.String()) + assert.Equal(t, wantSha, gitIn(t, clone, "rev-parse", "HEAD"), "the clone's HEAD is the pushed tip") + assert.Equal(t, gitIn(t, source, "symbolic-ref", "--short", "HEAD"), gitIn(t, clone, "symbolic-ref", "--short", "HEAD"), "the clone keeps the source's default branch") + assert.Equal(t, "v1.0.0", gitIn(t, clone, "tag", "--points-at", "HEAD"), "the annotated tag survives the round trip") +} + +func TestFosteredReposAnnounceOnTheFirehose(t *testing.T) { + base := os.Getenv("KNOT_TEST_BASE_URL") + require.NotEmpty(t, base, "KNOT_TEST_BASE_URL is the knot under test") + addr := os.Getenv("KNOT_TEST_ADDR") + require.NotEmpty(t, addr, "KNOT_TEST_ADDR is the knot's host:port") + secret := os.Getenv("KNOT_TEST_FOSTER_SECRET") + require.NotEmpty(t, secret, "KNOT_TEST_FOSTER_SECRET is the foster secret") + source := os.Getenv("KNOT_TEST_FOSTER_SOURCE") + require.NotEmpty(t, source, "KNOT_TEST_FOSTER_SOURCE is the staged source repo") + wantSha := os.Getenv("KNOT_TEST_SHA") + require.NotEmpty(t, wantSha, "KNOT_TEST_SHA is the staged source tip") + + body, rkey := seedRequest(t) + endpoint := "ws://" + strings.TrimPrefix(base, "http://") + "/xrpc/com.atproto.sync.subscribeRepos" + conn, _, err := websocket.DefaultDialer.DialContext(context.Background(), endpoint, nil) + require.NoError(t, err) + defer conn.Close() + + status, raw := seedPost(t, "admin", secret, body) + require.Equal(t, http.StatusOK, status, "the seed request must succeed: %s", raw) + var seeded struct { + Did string `json:"did"` + } + require.NoError(t, json.Unmarshal([]byte(raw), &seeded)) + did, err := syntax.ParseDID(seeded.Did) + require.NoError(t, err) + + wantPath := ticketCollection + "/" + rkey.String() + announced := false + scan := time.Now().Add(15 * time.Second) + for !announced && time.Now().Before(scan) { + frame := nextFrame(t, conn) + if frame.Commit == nil || frame.Commit.Repo != did.String() { + continue + } + for _, op := range frame.Commit.Ops { + if op.Path != wantPath || op.Action != string(tapc.RecordCreateAction) { + continue + } + announced = true + assert.NotNil(t, op.Cid, "the live op cites the record cid") + assert.NotEmpty(t, frame.Commit.Blocks, "the live commit frame includes its CAR") + } + } + require.True(t, announced, "a live commit frame announced the seeded record") + + pushToken := os.Getenv("KNOT_TEST_PUSH_TOKEN") + require.NotEmpty(t, pushToken, "KNOT_TEST_PUSH_TOKEN authorizes the receive-pack push") + pushRepo(t, source, addr, seeded.Did, pushToken) + records, cursor := knotfeedClient(t, addr, did) + sawRecord := false + sawRefs := false + deadline := time.After(60 * time.Second) + var settle <-chan time.Time +replay: + for { + if sawRecord && sawRefs && settle == nil { + settle = time.After(2 * time.Second) + } + select { + case op := <-records: + assert.NotEqual(t, tapc.RecordDeleteAction, op.Action, + "replay from zero mustn't serve a delete for %s/%s", op.Collection.String(), op.Rkey.String()) + switch { + case op.Collection.String() == ticketCollection && op.Rkey == rkey: + assert.Equal(t, tapc.RecordCreateAction, op.Action, "the replayed seed op is a create") + assert.NotEmpty(t, op.Bytes, "the replayed op includes the record bytes") + sawRecord = true + case op.Collection.String() == "org.tangled.git.refs": + assert.Equal(t, "self", op.Rkey.String(), "the refs snapshot is a single self record") + def, refs, err := knotfeed.DecodeRefsSnapshot(op.Bytes) + require.NoError(t, err, "the frozen record envelope decodes") + if def != "main" { + break + } + for _, line := range strings.Split(string(refs), "\n") { + line = strings.TrimSpace(line) + sha, name, ok := strings.Cut(line, " ") + if !ok || name != "refs/heads/main" || sha != wantSha { + continue + } + sawRefs = true + } + } + case <-settle: + break replay + case <-deadline: + t.Fatalf("replay served the seeded record %v and the refs snapshot %v", sawRecord, sawRefs) + } + } + require.Eventually(t, func() bool { return cursor.Load() >= int64(1) }, + 5*time.Second, 10*time.Millisecond, "the consumer advanced its cursor") +} diff --git a/knot2/interop/knotfeed_consumer_test.go b/knot2/interop/knotfeed_consumer_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0d2ef2d82ab28e63c31e11cd63a4da95be306c1a --- /dev/null +++ b/knot2/interop/knotfeed_consumer_test.go @@ -0,0 +1,104 @@ +//go:build interop + +package interop + +import ( + "context" + "log/slog" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/knotfeed" + "tangled.org/core/tapc" +) + +func knotfeedClient(t *testing.T, addr string, repoDid syntax.DID) (<-chan knotfeed.RecordOp, *atomic.Int64) { + t.Helper() + records := make(chan knotfeed.RecordOp, 16) + var cursor atomic.Int64 + consumer := &knotfeed.Consumer{ + Host: addr, + NoTLS: true, + ReplayFromStart: true, + Logger: slog.Default(), + LoadCursor: func(context.Context) (knotfeed.Cursor, error) { + return knotfeed.NewCursor(knotfeed.FeedAtproto, cursor.Load()), nil + }, + StoreCursor: func(_ context.Context, cur knotfeed.Cursor) error { + cursor.Store(cur.Seq()) + return nil + }, + Handle: func(_ context.Context, message knotfeed.Message) error { + if message.Type != knotfeed.TypeCommit || message.Commit == nil { + return nil + } + if message.Commit.Repo != repoDid { + return nil + } + for _, op := range message.Commit.Records { + select { + case records <- op: + default: + } + } + return nil + }, + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go consumer.Run(ctx) + return records, &cursor +} + +func TestKnotfeedConsumerReadsRefRecords(t *testing.T) { + addr := os.Getenv("KNOT_TEST_ADDR") + require.NotEmpty(t, addr, "KNOT_TEST_ADDR is the knot's host:port") + repoDid, err := syntax.ParseDID(os.Getenv("KNOT_TEST_REPO_DID")) + require.NoError(t, err, "KNOT_TEST_REPO_DID is the pushed repository") + wantSha := os.Getenv("KNOT_TEST_SHA") + require.NotEmpty(t, wantSha, "KNOT_TEST_SHA is the pushed tip") + + records, cursor := knotfeedClient(t, addr, repoDid) + deadline := time.After(60 * time.Second) + for { + select { + case op := <-records: + if op.Collection.String() != "org.tangled.git.refs" || op.Action != tapc.RecordCreateAction { + continue + } + assert.Equal(t, "self", op.Rkey.String(), "the refs snapshot is a single self record") + def, refs, err := knotfeed.DecodeRefsSnapshot(op.Bytes) + require.NoError(t, err, "the frozen record envelope decodes") + assert.Equal(t, "main", def, "the snapshot records the default branch") + var mainSha string + mainLines := 0 + for _, line := range strings.Split(string(refs), "\n") { + line = strings.TrimSpace(line) + if line == "" || line[0] == '#' || line[0] == '^' { + continue + } + sha, name, ok := strings.Cut(line, " ") + require.True(t, ok, "packed-refs lines pair an object id with a refname: %q", line) + if name != "refs/heads/main" { + continue + } + mainLines++ + mainSha = sha + } + require.Equal(t, 1, mainLines, "exactly one packed-refs line maps refs/heads/main") + assert.Equal(t, wantSha, mainSha, "refs/heads/main maps to the pushed tip") + require.Eventually(t, func() bool { return cursor.Load() >= int64(1) }, + 5*time.Second, 10*time.Millisecond, "the consumer advanced its cursor") + return + case <-deadline: + t.Fatal("no refs record arrived before the deadline") + } + } +} diff --git a/knot2/justfile b/knot2/justfile new file mode 100644 index 0000000000000000000000000000000000000000..a4b4da7e7ca86b1cd165600173441236360d11f2 --- /dev/null +++ b/knot2/justfile @@ -0,0 +1,169 @@ +set shell := ["bash", "-eu", "-o", "pipefail", "-c"] + +default: + @just --list + +gen-config: + cargo run -p knot-server -- config-template > example.toml + +fmt: + cargo fmt --all + +fmt-check: + cargo fmt --all --check + +clippy: + cargo clippy --locked -p 'knot-*' --all-targets -- -D warnings + +test: + cargo test --locked -p 'knot-*' + +fuzz crate='knot-pack' target='pack' time='60': + cd crates/{{crate}}/fuzz && RUSTUP_TOOLCHAIN=nightly cargo fuzz run {{target}} -- -max_total_time={{time}} + +fuzz-ci: (fuzz "knot-pack" "pkt" "30") (fuzz "knot-pack" "pack" "30") (fuzz "knot-pack" "receive_commands" "30") (fuzz "knot-pack" "upload_args" "30") (fuzz "knot-git" "patch" "30") (fuzz "knot-cobs" "cob_change" "30") (fuzz "knot-cobs" "cob_ref" "30") (fuzz "knot-atproto" "pubkey" "30") (fuzz "knot-atproto" "did_document" "30") (fuzz "knot-edge" "spki" "30") (fuzz "knot-edge" "spki_pin" "30") (fuzz "knot-lfs" "transfer" "30") (fuzz "knot-lfs" "batch" "30") (fuzz "knot-lfs" "pointer" "30") + +bench filter='': + cargo bench -p knot-bench --bench pack -- {{filter}} + cargo bench -p knot-bench --bench cob -- {{filter}} + cargo bench -p knot-bench --bench projection -- {{filter}} + +bench-scaling: + cargo bench -p knot-bench --bench coldstart + +instrument-tests: + cargo test --locked -p knot-bench --features instrument --tests + +differential: + cargo test -p knot-pack --test differential + +t55xx *tests: + internal_docs/t55xx/run.sh {{tests}} + +ci: fmt-check clippy test checks instrument-tests fuzz-ci + +checks: require-tools check-no-subprocess (check-no-banned-deps "no-sql" "an embedded database" "rusqlite|libsqlite3-sys|sqlx|sqlx-core|sled|fjall|redb") (check-no-banned-deps "no-native-git" "a native git or TLS shim" "git2|libgit2-sys|openssl-sys|zlib-ng|zlib-ng-sys") check-no-string-ids check-no-unguarded-receive check-fuzz-targets-enumerated check-workflow-toolchain + +require-tools: + #!/usr/bin/env bash + set -euo pipefail + missing="" + for tool in cargo comm cut find grep head just mktemp rm sed sort tr; do + command -v "$tool" >/dev/null 2>&1 || missing="$missing $tool" + done + if [ -n "$missing" ]; then + echo "require-tools failed, these aren't on PATH:$missing" >&2 + exit 1 + fi + echo "ok: every tool the checks below run is on PATH" + +check-no-subprocess: + #!/usr/bin/env bash + set -euo pipefail + hits=$(grep -rn --include="*.rs" --exclude-dir=_lex "process::Command" crates/*/src || [ $? = 1 ]) + if [ -n "$hits" ]; then + echo "no-subprocess check failed: server source spawns processes" >&2 + echo "$hits" >&2 + exit 1 + fi + echo "ok: no process spawning in server source" + +check-no-banned-deps name subject pattern: + #!/usr/bin/env bash + set -euo pipefail + tree=$(cargo tree --locked -p knot-server --edges normal,build --prefix none | sort -u) + hits=$(printf '%s\n' "$tree" | grep -iE '^({{pattern}}) v' || [ $? = 1 ]) + if [ -n "$hits" ]; then + echo "{{name}} check failed: {{subject}} is in the knot-server dependency tree" >&2 + echo "$hits" >&2 + exit 1 + fi + echo "ok: {{subject}} isn't in the knot-server dependency tree" + +check-no-string-ids: + #!/usr/bin/env bash + set -euo pipefail + declared=$(grep -nE 'pub fn .*(-> *String|: *String\b)' crates/knot-types/src/ids.rs || [ $? = 1 ]) + hits=$(printf '%s\n' "$declared" | grep -v 'fn to_hex' || [ $? = 1 ]) + if [ -n "$hits" ]; then + echo "no-string-ids check failed: a String-typed id crosses the knot-types boundary" >&2 + echo "$hits" >&2 + exit 1 + fi + echo "ok: no String-typed id crosses the knot-types boundary" + +check-no-unguarded-receive: + #!/usr/bin/env bash + set -euo pipefail + called=$(grep -rn --include="*.rs" 'receive_pack(\|receive_pack_with_limits(' crates/*/src || [ $? = 1 ]) + hits=$(printf '%s\n' "$called" | grep -v 'pub fn ' || [ $? = 1 ]) + if [ -n "$hits" ]; then + echo "no-unguarded-receive check failed: server source calls the unguarded receive path, use receive_pack_guarded" >&2 + echo "$hits" >&2 + exit 1 + fi + echo "ok: the unguarded receive path is reached only from tests" + +check-fuzz-targets-enumerated: + #!/usr/bin/env bash + set -euo pipefail + disk=$(mktemp) + recipe=$(mktemp) + triplet=$(mktemp) + trap 'rm -f "$disk" "$recipe" "$triplet"' EXIT + find crates -path '*/fuzz/fuzz_targets/*.rs' -not -path '*/target/*' | sed -E 's#crates/([^/]+)/fuzz/fuzz_targets/(.+)\.rs#\1 \2#' | sort -u > "$disk" + just --show fuzz-ci | grep -oE '\(fuzz "[^"]+" "[^"]+" "[^"]+"' | sed -E 's#\(fuzz "([^"]+)" "([^"]+)" "([^"]+)"#\1 \2 \3#' | sort -u > "$triplet" + cut -d' ' -f1,2 "$triplet" > "$recipe" + while read -r crate target secs; do + if ! [[ "$secs" =~ ^[1-9][0-9]*$ ]]; then + echo "fuzz-targets-enumerated check failed: target '$crate $target' runs for '$secs', not a positive number of seconds" >&2 + exit 1 + fi + done < "$triplet" + missing=$(comm -23 "$disk" "$recipe") + extra=$(comm -13 "$disk" "$recipe") + if [ -n "$missing" ] || [ -n "$extra" ]; then + echo "fuzz-targets-enumerated check failed: the fuzz-ci recipe and the targets on disk disagree" >&2 + if [ -n "$missing" ]; then + echo "on disk but absent from fuzz-ci:" >&2 + echo "$missing" >&2 + fi + if [ -n "$extra" ]; then + echo "in fuzz-ci but no matching target on disk:" >&2 + echo "$extra" >&2 + fi + exit 1 + fi + while read -r crate target; do + manifest="crates/$crate/fuzz/Cargo.toml" + if ! grep -qF "name = \"$target\"" "$manifest" || ! grep -qF "path = \"fuzz_targets/$target.rs\"" "$manifest"; then + echo "fuzz-targets-enumerated check failed: $manifest has no [[bin]] declaring target '$target'" >&2 + exit 1 + fi + entry=$(grep -oE 'knot_[a-z0-9_]+::fuzz::[a-z0-9_]+' "crates/$crate/fuzz/fuzz_targets/$target.rs" | head -1 | sed -E 's#.*::fuzz::##' || true) + smoke="crates/$crate/tests/fuzz_smoke.rs" + if [ -z "$entry" ] || ! grep -qE "fuzz::${entry}\(" "$smoke"; then + echo "fuzz-targets-enumerated check failed: target '$target' entry point $(echo "$crate" | tr - _)::fuzz::$entry has no smoke-test coverage in $smoke" >&2 + exit 1 + fi + done < "$disk" + echo "ok: every fuzz target is enumerated in fuzz-ci, declared in its fuzz manifest, and smoke-tested" + +check-workflow-toolchain: + #!/usr/bin/env bash + set -euo pipefail + locked=$(sed -n '/^ "fenix": {/,/^ }/p' ../flake.lock | sed -nE 's/.*"rev": "([0-9a-f]{40})".*/\1/p' | head -1) + if [ -z "$locked" ]; then + echo "workflow-toolchain check failed: no fenix node with a revision in flake.lock" >&2 + exit 1 + fi + used=$(grep -hoE 'github:nix-community/fenix/[0-9a-f]{40}' ../.tangled/workflows/*.yml | sed -E 's#.*/##' | sort -u || [ $? = 1 ]) + if [ -z "$used" ]; then + echo "workflow-toolchain check failed: no workflow in ../.tangled/workflows references a fenix revision" >&2 + exit 1 + fi + if [ "$used" != "$locked" ]; then + echo "workflow-toolchain check failed: flake.lock records fenix $locked and the workflows reference '$used'" >&2 + exit 1 + fi + echo "ok: every workflow builds with the fenix revision flake.lock records" diff --git a/knot2/third_party/gix-imara-diff/.cargo_vcs_info.json b/knot2/third_party/gix-imara-diff/.cargo_vcs_info.json new file mode 100644 index 0000000000000000000000000000000000000000..06d5dceb4d607ae9d7203f31a26bf320fa964376 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "10c58bb56597d9335611da121aac21f9b09b6e5b" + }, + "path_in_vcs": "gix-imara-diff" +} \ No newline at end of file diff --git a/knot2/third_party/gix-imara-diff/Cargo.toml b/knot2/third_party/gix-imara-diff/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..823f5d7ddd90ca2476264e02f6db9c3ab272e2c3 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/Cargo.toml @@ -0,0 +1,82 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.85" +name = "gix-imara-diff" +version = "0.2.2" +authors = [ + "pascalkuthe ", + "Sebastian Thiel ", +] +build = false +include = [ + "/src/**/*", + "/LICENSE", + "/README.md", + "!/src/tests.rs", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "A high performance library for computing diffs, maintained as a modified copy of upstream imara-diff for gitoxide." +readme = "README.md" +keywords = [ + "diff", + "difference", + "myers", + "compare", + "changes", +] +license = "Apache-2.0" +repository = "https://github.com/GitoxideLabs/gitoxide" +resolver = "2" + +[features] +default = [] +unified_diff = [] + +[lib] +name = "gix_imara_diff" +path = "src/lib.rs" + +[dependencies.bstr] +version = "1.12.0" +default-features = false + +[dependencies.hashbrown] +version = ">=0.15, <=0.17" +features = [ + "default-hasher", + "inline-more", +] +default-features = false + +[dev-dependencies.arbitrary] +version = "1.4.2" +features = ["derive"] + +[dev-dependencies.cov-mark] +version = "2.1.0" + +[dev-dependencies.expect-test] +version = "1.4.0" + +[dev-dependencies.gix-hash] +version = "^0.25.1" +features = ["sha1"] + +[dev-dependencies.gix-object] +version = "^0.61.0" +features = ["sha1"] diff --git a/knot2/third_party/gix-imara-diff/LICENSE b/knot2/third_party/gix-imara-diff/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..16fe87b06e802f094b3fbb0894b137bca2b16ef1 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/knot2/third_party/gix-imara-diff/README.md b/knot2/third_party/gix-imara-diff/README.md new file mode 100644 index 0000000000000000000000000000000000000000..37cb883bca781f0203ff47f35c2bab15b067e8d9 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/README.md @@ -0,0 +1,114 @@ +# imara-diff + +> Modified for gitoxide from the upstream imara-diff crate. +> Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:README.md + +[![crates.io](https://img.shields.io/crates/v/imara-diff?style=flat-square)](https://crates.io/crates/imara-diff) +[![crates.io](https://img.shields.io/docsrs/imara-diff?style=flat-square)](https://docs.rs/imara-diff/latest/imara_diff/) +![crates.io](https://img.shields.io/crates/l/imara-diff?style=flat-square) + +`imara-diff` is a solid (imara in swahili) diff library for rust. +Solid refers to the fact that imara-diff provides very good runtime performance even +in pathological cases so that your application never appears to freeze while waiting on a diff. +The performance improvements are achieved using battle tested heuristics used in gnu-diff and git +that are known to perform well while still providing good results. + +`imara-diff` is also designed to be flexible so that it can be used with arbitrary collections and +not just lists and strings and even allows reusing large parts of the computation when +comparing the same file to multiple different files. + +`imara-diff` provides two diff algorithms: + +* The linear-space variant of the well known [Myers algorithm](http://www.xmailserver.org/diff2.pdf) +* The **Histogram** algorithm which is a variant of the patience diff algorithm. + +Myers algorithm has been enhanced with preprocessing and multiple heuristics to ensure fast runtime in pathological +cases to avoid quadratic time complexity and closely matches the behavior of gnu-diff and git. +The histogram algorithm was originally ported from git but has been heavily optimized. +The **Histogram algorithm outperforms Myers algorithm** by 10% - 100% across a **wide variety of workloads**. + +## Limitations + +Even with the optimizations in this crate, performing a large diff without any tokenization (like character diff for a string) does not perform well. +To work around this problem a diff of the entire file with large tokens (like lines for a string) can be performed first. +The `Sink` implementation can then perform fine-grained diff on changed regions. +Note that this fine-grained diff should not be performed for pure insertions, pure deletions and very large changes. + +In an effort to improve performance, `imara-diff` makes heavy use of pointer compression. +That means that it can only support files with at most `2^31 - 2` tokens. +This should be rarely an issue in practice for textual diffs, because most (large) real-world files +have an average line-length of at least 8. +That means that this limitation only becomes a problem for files above 16GB while performing line-diffs. + +## Benchmarks + +The most used diffing libraries in the rust ecosystem are [similar](https://crates.io/crates/similar) and [dissimilar](https://crates.io/crates/dissimilar). +The fastest diff implementation both of these offer is a simple implementation of Myers algorithm +without preprocessing or additional heuristics. +As these implementations are very similar only `similar` was included in the benchmark. + +To provide a benchmark to reflects real-world workloads, the git history of different open source projects were used. +For each repo two (fairly different) tags were chosen. +A tree diff is performed with [gitoxide](https://github.com/Byron/gitoxide) and the pairs of files that should be saved are stored in memory. +The diffs collected using this method are often fairly large, because the repositories are compared over a large span of time. +Therefore, the tree diff of the last 30 commit before the tag (equivalent of `git diff TAG^ TAG`, `git diff TAG^^ TAG^^`) were also used to also include smaller diffs. + +The benchmark measures the runtime of performing a **line diff** between the collected files. +As a measure of complexity for each change `(M + N) D` was used where `M` and `N` are the lengths of the two compared files +and `D` is the length of the edit script required to transform these files into each other (determined with Myers algorithm). +This complexity measure is used to divide the changes into 10 badges. +The time to compute the line diffs in each badge was benchmarked. + +The plots below show the runtime for each **average** complexity (runtime is normalized by the number of diffs). +Note that these plots are shown in logarithmic scale due to the large runtime of `similar` for complex diffs. +Furthermore, to better highlight the performance of the Histogram algorithm, the speedup of the Histogram algorithm +compared to the Myers algorithm is shown separately. + +* [Linux](###Linux) +* [Rust](###Rust) +* [VSCode](###VSCode) +* [Helix](###Helix) + +### Linux + +The sourcecode of the linux kernel. + +- **Repo** - https://kernel.org +- **Tags** - `v5.7` and `v6.0` + +### Rust + +The sourcecode of the rust compiler, standard library and various related tooling. + +- **Repo** - https://github.com/rust-lang/rust +- **Tags** - `1.50.0` and `1.64.0` + +### VSCode + +The sourcecode of the vscode editor. + +- **Repo** - https://github.com/microsoft/vscode +- **Tags** - `1.41.0` and `1.72.2` + +### Helix + +The sourcecode of the helix editor. + +- **Repo** - https://github.com/helix-editor/helix +- **Tags** - `v0.5.0` and `22.08.1` + +## Testing + +`imara-diff` includes comprehensive fuzz testing using [cargo-fuzz](https://rust-fuzz.github.io/book/cargo-fuzz.html) to ensure robustness against arbitrary inputs. Fuzz tests cover all major algorithms (Myers, Histogram, MyersMinimal), postprocessing with different heuristics, and unified diff printing. See the [fuzz/README.md](fuzz/README.md) for more details on running fuzz tests locally. + +## Stability Policy + +`imara-diff` uses [Semantic Versioning (SemVer)](https://semver.org/). +All non-breaking changes to the public rust API will cause a minor `SemVer` bump. +All breaking changes to to the public rust API will cause a major `SemVer` bump. +Changes in the produced diffs are also considered breaking changes if the produced diff was valid. +If the produced diff was invalid the change will be considered a bugfix. + +Additionally all changes to the minimum stable rust version (MSRV) are also considered breaking changes. +The current **MSRV is 1.85**, matching the surrounding `gitoxide` workspace so hash-related dependencies +can use Rust 2024 crates such as `sha2` 0.11 and `hashbrown` 0.17. diff --git a/knot2/third_party/gix-imara-diff/src/histogram.rs b/knot2/third_party/gix-imara-diff/src/histogram.rs new file mode 100644 index 0000000000000000000000000000000000000000..6aba6486c5cb1343fda5eeb5c9b8aedaa0a96a4f --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/histogram.rs @@ -0,0 +1,109 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/histogram.rs + +use crate::histogram::lcs::find_lcs; +use crate::histogram::list_pool::{ListHandle, ListPool}; +use crate::intern::Token; +use crate::myers; + +mod lcs; +mod list_pool; + +/// Maximum number of occurrences tracked for a single token. +/// Tokens appearing more frequently fall back to Myers algorithm. +const MAX_CHAIN_LEN: u32 = 63; + +/// State for computing histogram-based diffs. +struct Histogram { + /// Tracks where each token appears in the "before" sequence. + token_occurrences: Vec, + /// Memory pool for efficiently storing occurrence lists. + pool: ListPool, +} + +/// Computes a diff using the histogram algorithm. +/// +/// # Parameters +/// +/// * `before` - The token sequence from the first file, before changes. +/// * `after` - The token sequence from the second file, after changes. +/// * `removed` - Output array marking removed tokens +/// * `added` - Output array marking added tokens +/// * `num_tokens` - The total number of distinct tokens +pub fn diff(before: &[Token], after: &[Token], removed: &mut [bool], added: &mut [bool], num_tokens: u32) { + let mut histogram = Histogram::new(num_tokens); + histogram.run(before, after, removed, added); +} + +impl Histogram { + fn new(num_buckets: u32) -> Histogram { + Histogram { + token_occurrences: vec![ListHandle::default(); num_buckets as usize], + pool: ListPool::new(2 * num_buckets), + } + } + + fn clear(&mut self) { + self.pool.clear(); + } + + fn token_occurrences(&self, token: Token) -> &[u32] { + self.token_occurrences[token.0 as usize].as_slice(&self.pool) + } + + fn num_token_occurrences(&self, token: Token) -> u32 { + self.token_occurrences[token.0 as usize].len(&self.pool) + } + + fn populate(&mut self, file: &[Token]) { + for (i, &token) in file.iter().enumerate() { + self.token_occurrences[token.0 as usize].push(i as u32, &mut self.pool); + } + } + + fn run(&mut self, mut before: &[Token], mut after: &[Token], mut removed: &mut [bool], mut added: &mut [bool]) { + loop { + if before.is_empty() { + added.fill(true); + return; + } else if after.is_empty() { + removed.fill(true); + return; + } + + self.populate(before); + match find_lcs(before, after, self) { + // no lcs was found, that means that file1 and file2 two have nothing in common + Some(lcs) if lcs.len == 0 => { + added.fill(true); + removed.fill(true); + return; + } + Some(lcs) => { + self.run( + &before[..lcs.before_start as usize], + &after[..lcs.after_start as usize], + &mut removed[..lcs.before_start as usize], + &mut added[..lcs.after_start as usize], + ); + + // this is equivalent to (tail) recursion but implement as a loop for efficiency reasons + let before_end = lcs.before_start + lcs.len; + before = &before[before_end as usize..]; + removed = &mut removed[before_end as usize..]; + + let after_end = lcs.after_start + lcs.len; + after = &after[after_end as usize..]; + added = &mut added[after_end as usize..]; + } + None => { + // we are diffing two extremely large repetitive files + // this is a worst case for histogram diff with O(N^2) performance + // fallback to myers to maintain linear time complexity + myers::diff((before, after), before, after, removed, added, false); + return; + } + } + } + } +} diff --git a/knot2/third_party/gix-imara-diff/src/histogram/lcs.rs b/knot2/third_party/gix-imara-diff/src/histogram/lcs.rs new file mode 100644 index 0000000000000000000000000000000000000000..01ecb3685895ad126b6a7985d0cbba968058dc58 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/histogram/lcs.rs @@ -0,0 +1,139 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/histogram/lcs.rs + +use crate::histogram::{Histogram, MAX_CHAIN_LEN}; +use crate::intern::Token; + +/// Finds the longest common subsequence (LCS) using a histogram-based approach. +/// +/// Returns `None` if the sequences are highly repetitive and should fall back to Myers. +pub(super) fn find_lcs(before: &[Token], after: &[Token], histogram: &mut Histogram) -> Option { + let mut search = LcsSearch { + lcs: Lcs::default(), + min_occurrences: MAX_CHAIN_LEN + 1, + found_cs: false, + }; + search.run(before, after, histogram); + if search.success() { Some(search.lcs) } else { None } +} + +/// Represents a longest common subsequence found by the histogram algorithm. +#[derive(Default, Debug)] +pub struct Lcs { + /// Starting position in the "before" sequence. + pub before_start: u32, + /// Starting position in the "after" sequence. + pub after_start: u32, + /// Length of the common subsequence. + pub len: u32, +} + +/// State for searching for the longest common subsequence. +pub struct LcsSearch { + /// The best LCS found so far. + lcs: Lcs, + /// The minimum occurrence count of tokens in the best LCS. + min_occurrences: u32, + /// Whether any common subsequence was found. + found_cs: bool, +} + +impl LcsSearch { + fn run(&mut self, before: &[Token], after: &[Token], histogram: &mut Histogram) { + let mut pos = 0; + while let Some(&token) = after.get(pos as usize) { + if histogram.num_token_occurrences(token) != 0 { + self.found_cs = true; + if histogram.num_token_occurrences(token) <= self.min_occurrences { + pos = self.update_lcs(pos, token, histogram, before, after); + continue; + } + } + + pos += 1; + } + + histogram.clear(); + } + + fn success(&mut self) -> bool { + !self.found_cs || self.min_occurrences <= MAX_CHAIN_LEN + } + + fn update_lcs( + &mut self, + after_pos: u32, + token: Token, + histogram: &Histogram, + before: &[Token], + after: &[Token], + ) -> u32 { + let mut next_token_idx2 = after_pos + 1; + let mut occurrences_iter = histogram.token_occurrences(token).iter().copied(); + let mut token_idx1 = occurrences_iter.next().unwrap(); + + 'occurrences_iter: loop { + let mut occurrences = histogram.num_token_occurrences(token); + let mut start1 = token_idx1; + let mut start2 = after_pos; + loop { + if start1 == 0 || start2 == 0 { + break; + } + let token1 = before.get(start1 as usize - 1); + let token2 = after.get(start2 as usize - 1); + if matches!((token1, token2), (Some(token1), Some(token2)) if token1 == token2) { + start1 -= 1; + start2 -= 1; + let new_occurrences = histogram.num_token_occurrences(before[start1 as usize]); + occurrences = occurrences.min(new_occurrences); + } else { + break; + } + } + + let mut end1 = token_idx1 + 1; + let mut end2 = after_pos + 1; + loop { + let token1 = before.get(end1 as usize); + let token2 = after.get(end2 as usize); + if matches!((token1, token2), (Some(token1), Some(token2)) if token1 == token2) { + let new_occurrences = histogram.num_token_occurrences(before[end1 as usize]); + occurrences = occurrences.min(new_occurrences); + end1 += 1; + end2 += 1; + } else { + break; + } + } + + if next_token_idx2 < end2 { + next_token_idx2 = end2; + } + + let len = end2 - start2; + debug_assert_eq!(len, end1 - start1); + if self.lcs.len < len || self.min_occurrences > occurrences { + self.min_occurrences = occurrences; + self.lcs = Lcs { + before_start: start1, + after_start: start2, + len, + }; + } + + loop { + if let Some(next_token_idx) = occurrences_iter.next() { + if next_token_idx > end2 { + token_idx1 = next_token_idx; + break; + } + } else { + break 'occurrences_iter; + } + } + } + + next_token_idx2 + } +} diff --git a/knot2/third_party/gix-imara-diff/src/histogram/list_pool.rs b/knot2/third_party/gix-imara-diff/src/histogram/list_pool.rs new file mode 100644 index 0000000000000000000000000000000000000000..063ff6d9bc5d7f3fd63020d385a8bf78561a3abf --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/histogram/list_pool.rs @@ -0,0 +1,259 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/histogram/list_pool.rs + +use crate::histogram::MAX_CHAIN_LEN; + +/// A small list of entity references allocated from a pool. +/// +/// An `ListHandle` type provides similar functionality to `Vec`, but with some important +/// differences in the implementation: +/// +/// 1. Memory is allocated from a `ListPool` instead of the global heap. +/// 2. The footprint of an entity list is 4 bytes, compared with the 24 bytes for `Vec`. +/// 3. An entity list doesn't implement `Drop`, leaving it to the pool to manage memory. +/// +/// The list pool is intended to be used as a LIFO allocator. After building up a larger data +/// structure with many list references, the whole thing can be discarded quickly by clearing the +/// pool. +/// +/// # Safety +/// +/// Entity lists are not as safe to use as `Vec`, but they never jeopardize Rust's memory safety +/// guarantees. These are the problems to be aware of: +/// +/// - If you lose track of an entity list, its memory won't be recycled until the pool is cleared. +/// This can cause the pool to grow very large with leaked lists. +/// - If entity lists are used after their pool is cleared, they may contain garbage data, and +/// modifying them may corrupt other lists in the pool. +/// - If an entity list is used with two different pool instances, both pools are likely to become +/// corrupted. +/// +/// Entity lists can be cloned, but that operation should only be used as part of cloning the whole +/// function they belong to. *Cloning an entity list does not allocate new memory for the clone*. +/// It creates an alias of the same memory. +/// +/// Entity lists cannot be hashed and compared for equality because it's not possible to compare the +/// contents of the list without the pool reference. +/// +/// # Implementation +/// +/// The `ListHandle` itself is designed to have the smallest possible footprint. This is important +/// because it is used inside very compact data structures like `InstructionData`. The list +/// contains only a 32-bit index into the pool's memory vector, pointing to the first element of +/// the list. +/// +/// The pool is just a single `Vec` containing all of the allocated lists. Each list is +/// represented as three contiguous parts: +/// +/// 1. The number of elements in the list. +/// 2. The list elements. +/// 3. Excess capacity elements. +/// +/// The total size of the three parts is always a power of two, and the excess capacity is always +/// as small as possible. This means that shrinking a list may cause the excess capacity to shrink +/// if a smaller power-of-two size becomes available. +/// +/// Both growing and shrinking a list may cause it to be reallocated in the pool vector. +/// +/// The index stored in an `ListHandle` points to part 2, the list elements. The value 0 is +/// reserved for the empty list which isn't allocated in the vector. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ListHandle { + index: u32, + generation: u32, + len: u32, +} + +/// Create an empty list. +impl Default for ListHandle { + fn default() -> Self { + Self { + index: 0, + generation: 0, + len: 0, + } + } +} + +const MAX_SIZE_CLASS: SizeClass = sclass_for_length(super::MAX_CHAIN_LEN - 1); +const NUM_SIZE_CLASS: usize = MAX_SIZE_CLASS as usize + 1; + +/// A memory pool for storing lists of `T`. +#[derive(Clone, Debug)] +pub struct ListPool { + // The main array containing the lists. + data: Vec, + + // Heads of the free lists, one for each size class. + free: [u32; NUM_SIZE_CLASS], + + generation: u32, +} + +/// Lists are allocated in sizes that are powers of two, starting from 4. +/// Each power of two is assigned a size class number, so the size is `4 << SizeClass`. +type SizeClass = u8; + +/// Get the size of a given size class. The size includes the length field, so the maximum list +/// length is one less than the class size. +#[inline] +const fn sclass_size(sclass: SizeClass) -> usize { + 4 << sclass +} + +/// Get the size class to use for a given list length. +/// This always leaves room for the length element in addition to the list elements. +#[inline] +const fn sclass_for_length(len: u32) -> SizeClass { + 30 - (len | 3).leading_zeros() as SizeClass +} + +/// Is `len` the minimum length in its size class? +#[inline] +fn is_sclass_max_length(len: u32) -> bool { + len > 3 && len.is_power_of_two() +} + +impl ListPool { + /// Create a new list pool. + pub fn new(capacity: u32) -> Self { + Self { + data: Vec::with_capacity(capacity as usize), + free: [u32::MAX; NUM_SIZE_CLASS], + generation: 1, + } + } + + /// Clear the pool, forgetting about all lists that use it. + /// + /// This invalidates any existing entity lists that used this pool to allocate memory. + /// + /// The pool's memory is not released to the operating system, but kept around for faster + /// allocation in the future. + pub fn clear(&mut self) { + self.data.clear(); + self.free.fill(u32::MAX); + self.generation += 1; + } + + /// Allocate a storage block with a size given by `sclass`. + /// + /// Returns the first index of an available segment of `self.data` containing + /// `sclass_size(sclass)` elements. The allocated memory is filled with reserved + /// values. + fn alloc(&mut self, sclass: SizeClass) -> usize { + let freelist_head = self.free[sclass as usize]; + // First try the free list for this size class. + if freelist_head == u32::MAX { + // Nothing on the free list. Allocate more memory. + let offset = self.data.len(); + self.data.resize(offset + sclass_size(sclass), u32::MAX); + offset + } else { + // take allocation of the free list (linked list) + self.free[sclass as usize] = self.data[freelist_head as usize]; + freelist_head as usize + } + } + + /// Free a storage block with a size given by `sclass`. + /// + /// This must be a block that was previously allocated by `alloc()` with the same size class. + fn free(&mut self, block: usize, sclass: SizeClass) { + let sclass = sclass as usize; + // Insert the block on the free list which is a single linked list. + self.data[block] = self.free[sclass]; + self.free[sclass] = block as u32 + } + + /// Returns two mutable slices representing the two requested blocks. + /// + /// The two returned slices can be longer than the blocks. Each block is located at the front + /// of the respective slice. + fn mut_slices(&mut self, block0: usize, block1: usize) -> (&mut [u32], &mut [u32]) { + if block0 < block1 { + let (s0, s1) = self.data.split_at_mut(block1); + (&mut s0[block0..], s1) + } else { + let (s1, s0) = self.data.split_at_mut(block0); + (s0, &mut s1[block1..]) + } + } + + /// Reallocate a block to a different size class. + /// + /// Copy `elems_to_copy` elements from the old to the new block. + fn realloc(&mut self, block: usize, from_sclass: SizeClass, to_sclass: SizeClass, elems_to_copy: usize) -> usize { + debug_assert!(elems_to_copy <= sclass_size(from_sclass)); + debug_assert!(elems_to_copy <= sclass_size(to_sclass)); + let new_block = self.alloc(to_sclass); + + let (old, new) = self.mut_slices(block, new_block); + new[0..elems_to_copy].copy_from_slice(&old[0..elems_to_copy]); + + self.free(block, from_sclass); + new_block + } +} + +impl ListHandle { + /// Get the number of elements in the list. + #[allow(clippy::len_without_is_empty)] + pub fn len(&self, pool: &ListPool) -> u32 { + if self.generation == pool.generation { + self.len + } else { + 0 + } + } + + /// Get the list as a slice. + pub fn as_slice<'a>(&'a self, pool: &'a ListPool) -> &'a [u32] { + let idx = self.index as usize; + match self.len(pool) { + 0 => &[], + 1 => std::slice::from_ref(&self.index), + len => &pool.data[idx..idx + len as usize], + } + } + + /// Appends an element to the back of the list. + /// Returns the index where the element was inserted. + pub fn push(&mut self, element: u32, pool: &mut ListPool) { + let len = self.len(pool); + match len { + 0 => { + self.generation = pool.generation; + self.index = element; + self.len = 1; + } + 1 => { + // This is an empty list. Allocate a block and set length=1. + let block = pool.alloc(0); + pool.data[block] = self.index; + pool.data[block + 1] = element; + self.index = block as u32; + self.len = 2; + } + 2..=MAX_CHAIN_LEN => { + // Do we need to reallocate? + let block; + let idx = self.index as usize; + if is_sclass_max_length(len) { + // Reallocate, preserving length + all old elements. + let sclass = sclass_for_length(len); + block = pool.realloc(idx, sclass - 1, sclass, len as usize); + self.index = block as u32; + } else { + block = idx; + } + pool.data[block + len as usize] = element; + self.len += 1; + } + + // ignore elements longer then MAX_CHAIN_LEN + // these are rarely relevant and if they are we fall back to myers + _ => (), + } + } +} diff --git a/knot2/third_party/gix-imara-diff/src/intern.rs b/knot2/third_party/gix-imara-diff/src/intern.rs new file mode 100644 index 0000000000000000000000000000000000000000..caf9665b58a44b437435824d09fb1a2b7ae55548 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/intern.rs @@ -0,0 +1,258 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/intern.rs + +use std::hash::{BuildHasher as _, Hash}; +use std::ops::Index; + +use hashbrown::DefaultHashBuilder as RandomState; +use hashbrown::hash_table::{Entry, HashTable}; + +/// A token represented as an interned integer. +/// +/// A token represents the smallest possible unit of change during a diff. +/// For text this is usually a line, a word or a single character. +/// All [algorithms](crate::Algorithm) operate on interned tokens instead +/// of using the token data directly. +/// This allows for much better performance by amortizing the cost of hashing/equality. +/// +/// While you can intern tokens yourself it is strongly recommended to use [`InternedInput`] module. +#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)] +#[repr(transparent)] +pub struct Token(pub u32); + +impl From for Token { + fn from(token: u32) -> Self { + Token(token) + } +} + +impl From for u32 { + fn from(token: Token) -> Self { + token.0 + } +} + +/// A trait for types that can be split into tokens for diffing. +/// +/// Implementing this trait allows a type to be used with [`InternedInput`] to create +/// interned token sequences for computing diffs. For example, `&str` implements this trait +/// by default to split text into lines. +pub trait TokenSource { + /// The type of token this source produces. Must be hashable and comparable for equality. + type Token: Hash + Eq; + /// An iterator that yields tokens from this source. + type Tokenizer: Iterator; + /// Creates an iterator that yields all tokens from this source. + fn tokenize(&self) -> Self::Tokenizer; + /// Provides an estimate of the number of tokens this source will produce. + /// + /// This is used to pre-allocate memory for better performance. The estimate + /// does not need to be exact. + fn estimate_tokens(&self) -> u32; +} + +/// Two lists of interned [tokens](Token) that a [`Diff`](crate::Diff) can be computed from. +/// +/// A token represents the smallest possible unit of change during a diff. +/// For text this is usually a line, a word or a single character. +/// All [algorithms](crate::Algorithm) operate on interned tokens instead +/// of using the token data directly. +/// This allows for much better performance by amortizing the cost of hashing/equality. +/// +/// While you can intern tokens yourself it is strongly recommended to use [`InternedInput`] module. +#[derive(Default)] +pub struct InternedInput { + /// The list of interned tokens from the first sequence (before changes). + pub before: Vec, + /// The list of interned tokens from the second sequence (after changes). + pub after: Vec, + /// The interner that stores the actual token data and maps tokens to their interned IDs. + pub interner: Interner, +} + +impl InternedInput { + /// Clears all token sequences and the interner. + /// + /// This removes all tokens from both the before and after sequences, as well as + /// clearing the interner's storage. + /// + /// Note that this will not free the allocated memory. + pub fn clear(&mut self) { + self.before.clear(); + self.after.clear(); + self.interner.clear(); + } +} + +impl InternedInput { + /// Creates a new `InternedInput` by tokenizing and interning two token sources. + /// + /// # Parameters + /// + /// * `before` - The token source for the first sequence + /// * `after` - The token source for the second sequence + /// + /// # Returns + /// + /// An `InternedInput` containing interned token sequences ready for diffing + pub fn new>(before: I, after: I) -> Self { + let token_estimate_before = before.estimate_tokens() as usize; + let token_estimate_after = after.estimate_tokens() as usize; + let mut res = Self { + before: Vec::with_capacity(token_estimate_before), + after: Vec::with_capacity(token_estimate_after), + interner: Interner::new(token_estimate_before + token_estimate_after), + }; + res.update_before(before.tokenize()); + res.update_after(after.tokenize()); + res + } + + /// Reserve capacity so that `before` and `after` would not need + /// to allocate if their [`estimate_tokens`](TokenSource::estimate_tokens) + /// would represent an exact match of their actual tokens. + /// + /// Useful for minimization of allocation before calls to + /// [`update_before`](InternedInput::update_before) and + /// [`update_after`](InternedInput::update_after). + pub fn reserve_for_token_source + ?Sized>(&mut self, before: &S, after: &S) { + self.reserve(before.estimate_tokens(), after.estimate_tokens()) + } + + /// Reserves capacity for the specified number of tokens in each sequence. + /// + /// # Parameters + /// + /// * `capacity_before` - The number of tokens to reserve for the "before" sequence + /// * `capacity_after` - The number of tokens to reserve for the "after" sequence + pub fn reserve(&mut self, capacity_before: u32, capacity_after: u32) { + self.before.reserve(capacity_before as usize); + self.after.reserve(capacity_after as usize); + self.interner + .reserve(capacity_before as usize + capacity_after as usize); + } + + /// replaces `self.before` with the interned Tokens yielded by `input` + /// Note that this does not erase any tokens from the interner and might therefore be considered + /// a memory leak. If this function is called often over a long-running process + /// consider clearing the interner with [`clear`](Interner::clear). + pub fn update_before(&mut self, input: impl Iterator) { + self.before.clear(); + self.before.extend(input.map(|token| self.interner.intern(token))); + } + + /// replaces `self.before` with the interned Tokens yielded by `input` + /// Note that this does not erase any tokens from the interner and might therefore be considered + /// a memory leak. If this function is called often over a long-running process + /// consider clearing the interner with [`clear`](Interner::clear) or + /// [`erase_tokens_after`](Interner::erase_tokens_after). + pub fn update_after(&mut self, input: impl Iterator) { + self.after.clear(); + self.after.extend(input.map(|token| self.interner.intern(token))); + } +} + +/// An interner that allows for fast access of tokens produced by a [`TokenSource`]. +#[derive(Default)] +pub struct Interner { + tokens: Vec, + table: HashTable, + hasher: RandomState, +} + +impl Interner { + /// Create an Interner with an initial capacity calculated by summing the results of calling + /// [`estimate_tokens`](TokenSource::estimate_tokens) methods of `before` and `after`. + pub fn new_for_token_source>(before: &S, after: &S) -> Self { + Self::new(before.estimate_tokens() as usize + after.estimate_tokens() as usize) + } + + /// Create an Interner with initial capacity `capacity`. + pub fn new(capacity: usize) -> Interner { + Interner { + tokens: Vec::with_capacity(capacity), + table: HashTable::with_capacity(capacity), + hasher: RandomState::default(), + } + } + + /// Remove all interned tokens. + pub fn clear(&mut self) { + self.table.clear(); + self.tokens.clear(); + } + + /// Returns to total number of **distinct** tokens currently interned. + pub fn num_tokens(&self) -> u32 { + self.tokens.len() as u32 + } +} + +impl Interner { + /// Create an Interner with an initial capacity calculated by calling + /// [`estimate_tokens`](TokenSource::estimate_tokens) methods of `before` and `after` + pub fn reserve_for_token_source>(&mut self, before: &S, after: &S) { + self.reserve(before.estimate_tokens() as usize + after.estimate_tokens() as usize) + } + + /// Reserves capacity for at least the specified number of additional tokens. + /// + /// # Parameters + /// + /// * `capacity` - The number of additional tokens to reserve space for + pub fn reserve(&mut self, capacity: usize) { + self.table + .reserve(capacity, |&token| self.hasher.hash_one(&self.tokens[token.0 as usize])); + self.tokens.reserve(capacity); + } + + /// Intern `token` and return the interned integer. + pub fn intern(&mut self, token: T) -> Token { + let hash = self.hasher.hash_one(&token); + match self.table.entry( + hash, + |&it| self.tokens[it.0 as usize] == token, + |&token| self.hasher.hash_one(&self.tokens[token.0 as usize]), + ) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => { + let interned = Token(self.tokens.len() as u32); + entry.insert(interned); + self.tokens.push(token); + interned + } + } + } + + /// Erases `first_erased_token` and any tokens interned afterward from the interner. + pub fn erase_tokens_after(&mut self, first_erased_token: Token) { + assert!(first_erased_token.0 <= self.tokens.len() as u32); + let retained = first_erased_token.0 as usize; + let erased = self.tokens.len() - retained; + if retained <= erased { + self.table.clear(); + for (i, token) in self.tokens[0..retained].iter().enumerate() { + let hash = self.hasher.hash_one(token); + self.table.insert_unique(hash, Token(i as u32), |&token| { + self.hasher.hash_one(&self.tokens[token.0 as usize]) + }); + } + } else { + for (i, token) in self.tokens[retained..].iter().enumerate() { + let hash = self.hasher.hash_one(token); + match self.table.find_entry(hash, |token| token.0 == (retained + i) as u32) { + Ok(occupied) => drop(occupied.remove()), + Err(_absent) => unreachable!(), + } + } + } + self.tokens.truncate(first_erased_token.0 as usize); + } +} + +impl Index for Interner { + type Output = T; + fn index(&self, index: Token) -> &Self::Output { + &self.tokens[index.0 as usize] + } +} diff --git a/knot2/third_party/gix-imara-diff/src/lib.rs b/knot2/third_party/gix-imara-diff/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..aee0c141ee813ad8b3ce865e88fd58ad74df7e7c --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/lib.rs @@ -0,0 +1,525 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/lib.rs + +#![deny(missing_docs)] +//! Imara-diff is a solid (imara in Swahili) diff library for Rust. +//! Solid refers to the fact that imara-diff provides very good runtime performance even +//! in pathological cases so that your application never appears to freeze while waiting on a diff. +//! The performance improvements are achieved using battle tested heuristics used in gnu-diff and git +//! that are known to yield fast runtime and performance. +//! +//! Imara-diff is also designed to be flexible so that it can be used with arbitrary collections and +//! not just lists and strings and even allows reusing large parts of the computation when +//! comparing the same file to multiple different files. +//! +//! Imara-diff provides two diff algorithms: +//! +//! * The linear-space variant of the well known [**Myers** algorithm](http://www.xmailserver.org/diff2.pdf) +//! * The **Histogram** algorithm which is a variant of the patience diff algorithm. +//! +//! Myers algorithm has been enhanced with preprocessing and multiple heuristics to ensure fast runtime in pathological +//! cases to avoid quadratic time complexity and closely matches the behavior of gnu-diff and git. +//! The Histogram algorithm was originally ported from git but has been heavily optimized. +//! The **Histogram algorithm outperforms Myers diff** by 10% - 100% across a **wide variety of workloads**. +//! +//! Imara-diffs algorithms have been benchmarked over a wide variety of real-world code. +//! For example, while comparing multiple different Linux kernel versions, it performs up to 30 times better than the `similar` crate. +//! +//! # API Overview +//! +//! ## Preparing the input +//! To compute a diff, an input sequence is required. `imara-diff` computes diffs on abstract +//! sequences represented as a slice of IDs/tokens: [`Token`]. To create +//! such a sequence from your input type (for example, text), the input needs to be interned. +//! For that `imara-diff` provides utilities in the form of the [`InternedInput`] struct and +//! the `TokenSource` trait to construct it. [`InternedInput`] contains the two sides of +//! the diff (used while computing the diff). As well as the interner that allows mapping +//! back tokens to their original data. +//! +//! The most common use case for diff is comparing text. `&str` implements `TokenSource` +//! by default to segment the text into lines. So creating an input for a text-based diff usually +//! looks something like the following: +//! +//! ``` +//! # use gix_imara_diff::InternedInput; +//! # +//! let before = "abc\ndef"; +//! let after = "abc\ndefg"; +//! let input = InternedInput::new(before, after); +//! assert_eq!(input.interner[input.before[0]], "abc\n"); +//! ``` +//! +//! Note that interning inputs is optional, and you could choose a different strategy +//! for creating a sequence of tokens. Instead of using the [`Diff::compute`] function, +//! [`Diff::compute_with`] can be used to provide a list of tokens directly, entirely +//! bypassing the interning step. +//! +//! ## Computing the Diff +//! +//! A diff of two sequences is represented by the [`Diff`] struct and computed by +//! [`Diff::compute`] / [`Diff::compute_with`]. An algorithm can also be chosen here. +//! In most situations, [`Algorithm::Histogram`] is a good choice; refer to the docs +//! of [`Algorithm`] for more details. +//! +//! After the initial computation, the diff can be *postprocessed*. If the diff is shown +//! to a human in some way (even indirectly), you always want to use this. +//! +//! However, when only counting the number of changed tokens quickly, this can be skipped. +//! The postprocessing allows you to provide your own +//! heuristic for selecting a slider position. An indentation-based heuristic is provided, +//! which is a good fit for all text-based line diffs. The internals of the heuristic are +//! public, so a tweaked heuristic can be built on top. +//! +//! ``` +//! # use gix_imara_diff::{InternedInput, Diff, Algorithm}; +//! # +//! let before = "abc\ndef"; +//! let after = "abc\ndefg"; +//! let input = InternedInput::new(before, after); +//! let mut diff = Diff::compute(Algorithm::Histogram, &input); +//! diff.postprocess_lines(&input); +//! assert!(!diff.is_removed(0) && !diff.is_added(0)); +//! assert!(diff.is_removed(1) && diff.is_added(1)); +//! ``` +//! +//! ## Accessing results +//! +//! [`Diff`] allows querying whether a particular position was removed/added on either +//! side of the diff with [`Diff::is_removed`] / [`Diff::is_added`]. The number +//! of additions/removals can be quickly counted with [`Diff::count_removals`] / +//! [`Diff::count_additions`]. The most powerful/useful interface is the hunk iterator +//! [`Diff::hunks`], which returns a list of additions/removals/modifications in the +//! order that they appear in the input. +//! +//! Finally, when built with the `unified_diff` feature, this crate also provides a +//! built-in unified diff/patch formatter similar to `git diff` or `diff -u`. +//! Note that while the formatter has a decent amount of flexibility, it is fairly +//! simplistic and not every formatting may be possible. It's meant to cover common +//! situations but not cover every advanced use case. Instead, if you need more advanced +//! printing, build your own printer on top of the [`Diff::hunks`] iterator; for that, you can +//! take inspiration from the built-in printer implementation. +//! +//! ``` +//! # use gix_imara_diff::{InternedInput, Diff, Algorithm, BasicLineDiffPrinter, UnifiedDiffConfig}; +//! # +//! +//! let before = r#"fn foo() -> Bar { +//! let mut foo = 2; +//! foo *= 50; +//! println!("hello world") +//! } +//! "#; +//! +//! let after = r#"// lorem ipsum +//! fn foo() -> Bar { +//! let mut foo = 2; +//! foo *= 50; +//! println!("hello world"); +//! println!("{foo}"); +//! } +//! // foo +//! "#; +//! let input = InternedInput::new(before, after); +//! let mut diff = Diff::compute(Algorithm::Histogram, &input); +//! diff.postprocess_lines(&input); +//! +//! assert_eq!( +//! diff.unified_diff( +//! &BasicLineDiffPrinter(&input.interner), +//! UnifiedDiffConfig::default(), +//! &input, +//! ) +//! .to_string(), +//! r#"@@ -1,5 +1,8 @@ +//! +// lorem ipsum +//! fn foo() -> Bar { +//! let mut foo = 2; +//! foo *= 50; +//! - println!("hello world") +//! + println!("hello world"); +//! + println!("{foo}"); +//! } +//! +// foo +//! "# +//! ); +//! ``` + +use std::ops::Range; +use std::slice; + +use crate::{ + sources::words, + util::{strip_common_postfix, strip_common_prefix}, +}; + +pub use crate::slider_heuristic::{IndentHeuristic, IndentLevel, NoSliderHeuristic, SliderHeuristic}; +pub use intern::{InternedInput, Interner, Token, TokenSource}; +#[cfg(feature = "unified_diff")] +pub use unified_diff::{BasicLineDiffPrinter, UnifiedDiff, UnifiedDiffConfig, UnifiedDiffPrinter}; + +mod histogram; +mod intern; +mod myers; +mod postprocess; +mod slider_heuristic; +pub mod sources; +#[cfg(test)] +mod tests; +#[cfg(feature = "unified_diff")] +mod unified_diff; +mod util; + +/// `imara-diff` supports multiple different algorithms +/// for computing an edit sequence. +/// These algorithms have different performance and all produce different output. +#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)] +pub enum Algorithm { + /// A variation of the [`patience` diff algorithm described by Bram Cohen's blog post](https://bramcohen.livejournal.com/73318.html) + /// that uses a histogram to find the least common LCS. + /// Just like the `patience` diff algorithm, this algorithm usually produces + /// more human-readable output than Myers algorithm. + /// However, compared to the `patience` diff algorithm (which is slower than Myers algorithm), + /// the Histogram algorithm performs much better. + /// + /// The implementation here was originally ported from `git` but has been significantly + /// modified to improve performance. + /// As a result, it consistently **performs better than Myers algorithm** (5%-100%) over + /// a wide variety of test data. + /// + /// For pathological subsequences that only contain highly repeating tokens (64+ occurrences) + /// the algorithm falls back on Myers algorithm (with heuristics) to avoid quadratic behavior. + /// + /// Compared to Myers algorithm, the Histogram diff algorithm is more focused on providing + /// human-readable diffs instead of minimal diffs. In practice, this means that the edit sequences + /// produced by the histogram diff are often longer than those produced by Myers algorithm. + /// + /// The heuristic used by the histogram diff does not work well for inputs with small (often repeated) + /// tokens. For example, **character diffs do not work well** as most (English) text is made up of + /// a fairly small set of characters. The `Histogram` algorithm will automatically detect these cases and + /// fall back to Myers algorithm. However, this detection has a nontrivial overhead, so + /// if it's known upfront that the sort of tokens is very small, `Myers` algorithm should + /// be used instead. + #[default] + Histogram, + /// An implementation of the linear space variant of + /// [Myers `O((N+M)D)` algorithm](http://www.xmailserver.org/diff2.pdf). + /// The algorithm is enhanced with preprocessing that removes + /// tokens that don't occur in the other file at all. + /// Furthermore, two heuristics for the middle snake search are implemented + /// that ensure reasonable runtime (mostly linear time complexity) even for large files. + /// + /// Due to the divide-and-conquer nature of the algorithm, + /// the edit sequences produced are still fairly small even when the middle snake + /// search is aborted by a heuristic. + /// However, the produced edit sequences are not guaranteed to be fully minimal. + /// If that property is vital to you, use the `MyersMinimal` algorithm instead. + /// + /// The implementation (including the preprocessing) is mostly + /// ported from `git` and `gnu-diff`, where Myers algorithm is used + /// as the default diff algorithm. + /// Therefore, the used heuristics have been heavily battle-tested and + /// are known to behave well over a large variety of inputs. + Myers, + /// Same as `Myers` but the early abort heuristics are disabled to guarantee + /// a minimal edit sequence. + /// This can mean significant slowdown in pathological cases. + MyersMinimal, +} + +/// Represents the difference between two sequences of tokens. +/// +/// A `Diff` stores which tokens were removed from the first sequence and which tokens were added to the second sequence. +#[derive(Default)] +pub struct Diff { + /// Tracks which tokens were removed from the first sequence (`before`), with + /// one entry for each one in the `before` sequence. + removed: Vec, + /// Tracks which tokens were added to the second sequence (`after`), with + /// one entry for each one in the `after` sequence. + added: Vec, +} + +impl std::fmt::Debug for Diff { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_list().entries(self.hunks()).finish() + } +} + +impl Diff { + /// Computes an edit-script that transforms `input.before` into `input.after` using + /// the specified `algorithm` + pub fn compute(algorithm: Algorithm, input: &InternedInput) -> Diff { + let mut diff = Diff::default(); + diff.compute_with(algorithm, &input.before, &input.after, input.interner.num_tokens()); + diff + } + + /// Computes an edit-script that transforms `before` into `after` using + /// the specified `algorithm`. + pub fn compute_with(&mut self, algorithm: Algorithm, mut before: &[Token], mut after: &[Token], num_tokens: u32) { + assert!( + before.len() < i32::MAX as usize, + "imara-diff only supports up to {} tokens", + i32::MAX + ); + assert!( + after.len() < i32::MAX as usize, + "imara-diff only supports up to {} tokens", + i32::MAX + ); + self.removed.clear(); + self.added.clear(); + self.removed.resize(before.len(), false); + self.added.resize(after.len(), false); + let whole = (before, after); + let common_prefix = strip_common_prefix(&mut before, &mut after) as usize; + let common_postfix = strip_common_postfix(&mut before, &mut after); + let range = common_prefix..self.removed.len() - common_postfix as usize; + let removed = &mut self.removed[range]; + let range = common_prefix..self.added.len() - common_postfix as usize; + let added = &mut self.added[range]; + match algorithm { + Algorithm::Histogram => histogram::diff(before, after, removed, added, num_tokens), + Algorithm::Myers => myers::diff(whole, before, after, removed, added, false), + Algorithm::MyersMinimal => myers::diff(whole, before, after, removed, added, true), + } + } + + /// Returns the total number of tokens that were added in the second sequence. + pub fn count_additions(&self) -> u32 { + self.added.iter().map(|&added| u32::from(added)).sum() + } + + /// Returns the total number of tokens that were removed from the first sequence (`before`). + pub fn count_removals(&self) -> u32 { + self.removed.iter().map(|&removed| u32::from(removed)).sum() + } + + /// Returns `true` if the token at the given index was removed from the first sequence (`before`). + /// + /// # Panics + /// + /// Panics if `token_idx` is out of bounds for the first sequence. + pub fn is_removed(&self, token_idx: u32) -> bool { + self.removed[token_idx as usize] + } + + /// Returns `true` if the token at the given index was added to the second sequence (`after`). + /// + /// # Panics + /// + /// Panics if `token_idx` is out of bounds for the second sequence (`after`). + pub fn is_added(&self, token_idx: u32) -> bool { + self.added[token_idx as usize] + } + + /// Postprocesses the diff to make it more human-readable. Certain hunks + /// have an ambiguous placement (even in a minimal diff) where they can move + /// downward or upward by removing a token (line) at the start and adding + /// one at the end (or the other way around). The postprocessing adjusts + /// these hunks according to a couple of rules: + /// + /// * Always merge multiple hunks if possible. + /// * Always try to create a single MODIFY hunk instead of multiple disjoint + /// ADDED/REMOVED hunks. + /// * Move sliders as far down as possible. + pub fn postprocess_no_heuristic(&mut self, input: &InternedInput) { + self.postprocess_with_heuristic(input, NoSliderHeuristic) + } + + /// Postprocesses the diff to make it more human-readable. Certain hunks + /// have an ambiguous placement (even in a minimal diff) where they can move + /// downward or upward by removing a token (line) at the start and adding + /// one at the end (or the other way around). The postprocessing adjusts + /// these hunks according to a couple of rules: + /// + /// * Always merge multiple hunks if possible. + /// * Always try to create a single MODIFY hunk instead of multiple disjoint + /// ADDED/REMOVED hunks. + /// * Based on a line's indentation level, heuristically compute the most + /// intuitive location to split lines. + /// * Move sliders as far down as possible. + pub fn postprocess_lines>(&mut self, input: &InternedInput) { + self.postprocess_with_heuristic( + input, + IndentHeuristic::new(|token| { + IndentLevel::for_ascii_line(input.interner[token].as_ref().iter().copied(), 8) + }), + ) + } + + /// Return an iterator that yields the changed hunks in this diff. + pub fn hunks(&self) -> HunkIter<'_> { + HunkIter { + removed: self.removed.iter(), + added: self.added.iter(), + pos_before: 0, + pos_after: 0, + } + } +} + +/// A single change in a `Diff` that represents a range of tokens (`before`) +/// in the first sequence that were replaced by a different range of tokens +/// in the second sequence (`after`). +/// +/// Each hunk identifies a contiguous region of change, where tokens from the `before` range +/// should be replaced with tokens from the `after` range. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] +pub struct Hunk { + /// The range of token indices in the first sequence (`before`) that were removed. + pub before: Range, + /// The range of token indices in the second sequence (`after`) that were added. + pub after: Range, +} + +impl Hunk { + /// Can be used instead of `Option::None` for better performance. + /// Because `imara-diff` does not support more than `i32::MAX` there is an unused bit pattern that can be used. + /// + /// It has some nice properties where it usually is not necessary to check for `None` separately: + /// Empty ranges fail contains checks and also fail smaller than checks. + pub const NONE: Hunk = Hunk { + before: u32::MAX..u32::MAX, + after: u32::MAX..u32::MAX, + }; + + /// Inverts a hunk so that it represents a change + /// that would undo this hunk. + pub fn invert(&self) -> Hunk { + Hunk { + before: self.after.clone(), + after: self.before.clone(), + } + } + + /// Returns whether tokens are only inserted and not removed in this hunk. + pub fn is_pure_insertion(&self) -> bool { + self.before.is_empty() + } + + /// Returns whether tokens are only removed and not inserted in this hunk. + pub fn is_pure_removal(&self) -> bool { + self.after.is_empty() + } + + /// Performs a word-diff on this hunk. + /// + /// This requires passing the original [`input`](InternedInput) in order to look up + /// the tokens of the current hunk, which typically are lines. + /// Each token is split into words using the built-in [`words`] tokenizer. + /// The resulting word tokens are stored in a second [`diff_input`](InternedInput), + /// and a [`diff`](Diff) is computed on them, with basic post-processing applied. + /// + /// For performance reasons, this second [`diff_input`](InternedInput) as well as + /// the computed [`diff`](Diff) need to be passed as parameters so that they can be + /// re-used when iterating over hunks. Note that word tokens are always + /// added but never removed from the interner. Consider clearing it if you expect + /// your input to have a large vocabulary. + /// + /// # Examples + /// + /// ``` + /// # use gix_imara_diff::{InternedInput, Diff, Algorithm}; + /// // Compute diff normally + /// let before = "before text"; + /// let after = "after text"; + /// let mut lines = InternedInput::new(before, after); + /// let mut diff = Diff::compute(Algorithm::Histogram, &lines); + /// diff.postprocess_lines(&lines); + /// + /// // Compute word-diff per hunk, reusing allocations across iterations + /// let mut hunk_diff_input = InternedInput::default(); + /// let mut hunk_diff = Diff::default(); + /// for hunk in diff.hunks() { + /// hunk.latin_word_diff(&lines, &mut hunk_diff_input, &mut hunk_diff); + /// let added = hunk_diff.count_additions(); + /// let removed = hunk_diff.count_removals(); + /// println!("word-diff of this hunk has {added} additions and {removed} removals"); + /// // optionally, clear the interner: + /// hunk_diff_input.clear(); + /// } + /// ``` + pub fn latin_word_diff<'a>( + &self, + input: &InternedInput<&'a str>, + word_tokens: &mut InternedInput<&'a str>, + diff: &mut Diff, + ) { + let Hunk { before, after } = self.clone(); + word_tokens.update_before( + before + .map(|index| input.before[index as usize]) + .map(|token| input.interner[token]) + .flat_map(|line| words(line)), + ); + word_tokens.update_after( + after + .map(|index| input.after[index as usize]) + .map(|token| input.interner[token]) + .flat_map(|line| words(line)), + ); + diff.removed.clear(); + diff.removed.resize(word_tokens.before.len(), false); + diff.added.clear(); + diff.added.resize(word_tokens.after.len(), false); + if self.is_pure_removal() { + diff.removed.fill(true); + } else if self.is_pure_insertion() { + diff.added.fill(true); + } else { + diff.compute_with( + Algorithm::Myers, + &word_tokens.before, + &word_tokens.after, + word_tokens.interner.num_tokens(), + ); + diff.postprocess_no_heuristic(word_tokens); + } + } +} + +/// Yields all [`Hunk`]s in a file in monotonically increasing order. +/// Monotonically increasing means here that the following holds for any two +/// consecutive [`Hunk`]s `x` and `y`: +/// +/// ``` no_compile +/// assert!(x.before.end < y.before.start); +/// assert!(x.after.end < y.after.start); +/// ``` +/// +pub struct HunkIter<'diff> { + removed: slice::Iter<'diff, bool>, + added: slice::Iter<'diff, bool>, + pos_before: u32, + pos_after: u32, +} + +impl Iterator for HunkIter<'_> { + type Item = Hunk; + + fn next(&mut self) -> Option { + loop { + let removed = (&mut self.removed).take_while(|&&removed| removed).count() as u32; + let added = (&mut self.added).take_while(|&&added| added).count() as u32; + if removed != 0 || added != 0 { + let start_before = self.pos_before; + let start_after = self.pos_after; + self.pos_before += removed; + self.pos_after += added; + let hunk = Hunk { + before: start_before..self.pos_before, + after: start_after..self.pos_after, + }; + self.pos_before += 1; + self.pos_after += 1; + return Some(hunk); + } else if self.removed.len() == 0 && self.added.len() == 0 { + return None; + } else { + self.pos_before += 1; + self.pos_after += 1; + } + } + } +} diff --git a/knot2/third_party/gix-imara-diff/src/myers.rs b/knot2/third_party/gix-imara-diff/src/myers.rs new file mode 100644 index 0000000000000000000000000000000000000000..6e8b858c2de1f23df5ffaa10bcbcf7bc81f35338 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/myers.rs @@ -0,0 +1,304 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/myers.rs + +use std::ptr::NonNull; + +use crate::intern::Token; +use crate::myers::middle_snake::{MiddleSnakeSearch, SearchResult}; +use crate::myers::slice::FileSlice; +use crate::util::sqrt; + +mod middle_snake; +mod preprocess; +mod slice; + +/// Computes a diff using the Myers algorithm. +/// +/// # Parameters +/// +/// * `whole` - The files to count occurrences in, before their common prefix and suffix were stripped. +/// Histogram's myers fallback passes its own sub-slices. +/// * `before` - The token sequence from the first file +/// * `after` - The token sequence from the second file +/// * `removed` - Output array marking removed tokens +/// * `added` - Output array marking added tokens +/// * `minimal` - If true, disables heuristics to guarantee a minimal diff +pub fn diff( + whole: (&[Token], &[Token]), + before: &[Token], + after: &[Token], + removed: &mut [bool], + added: &mut [bool], + minimal: bool, +) { + // Preprocess the files by removing parts of the file that are not contained in the other file at all. + // This process remaps the token indices so we have to account for that during the rest of the diff + let (before, after) = preprocess::preprocess(whole, before, after, removed, added); + + // Perform the actual diff + Myers::new(before.tokens.len(), after.tokens.len()).run( + FileSlice::new(&before, removed), + FileSlice::new(&after, added), + minimal, + ); +} + +/// Minimum edit cost before heuristics are applied to avoid quadratic behavior. +const HEUR_MIN_COST: u32 = 256; +/// Minimum value for the maximum cost threshold. +const MAX_COST_MIN: u32 = 256; + +/// The Myers diff algorithm implementation with linear space complexity. +/// +/// This structure maintains the internal state needed to compute diffs using +/// Myers' algorithm with forward and backward search. +pub struct Myers { + /// The backing memory for k-vectors. + kvec: NonNull<[i32]>, + /// Pointer to the forward search k-vector. + kforward: NonNull, + /// Pointer to the backward search k-vector. + kbackward: NonNull, + /// Maximum edit cost before applying heuristics. + max_cost: u32, +} + +impl Drop for Myers { + fn drop(&mut self) { + unsafe { drop(Box::from_raw(self.kvec.as_ptr())) } + } +} + +impl Myers { + fn new(len1: usize, len2: usize) -> Self { + let ndiags = len1 + len2 + 3; + let kvec: *mut [i32] = Box::into_raw(vec![0; 2 * ndiags + 2].into_boxed_slice()); + let (kforward, kbackward) = unsafe { + ( + NonNull::new_unchecked((kvec as *mut i32).add(len2 + 1)), + NonNull::new_unchecked((kvec as *mut i32).add(ndiags + len2 + 1)), + ) + }; + Self { + kvec: unsafe { NonNull::new_unchecked(kvec) }, + kforward, + kbackward, + max_cost: sqrt(ndiags).max(MAX_COST_MIN), + } + } + + fn run<'f>(&mut self, mut file1: FileSlice<'f>, mut file2: FileSlice<'f>, mut need_min: bool) { + loop { + file1.strip_common(&mut file2); + + if file1.is_empty() { + file2.mark_changed(); + return; + } else if file2.is_empty() { + file1.mark_changed(); + return; + } + + let split = self.split(&file1, &file2, need_min); + self.run( + file1.borrow().slice(..split.token_idx1 as u32), + file2.borrow().slice(..split.token_idx2 as u32), + split.minimized_lo, + ); + + file1 = file1.slice(split.token_idx1 as u32..); + file2 = file2.slice(split.token_idx2 as u32..); + need_min = split.minimized_hi + } + } + + /// See "An O(ND) Difference Algorithm and its Variations", by Eugene Myers. + /// Basically considers a "box" (off1, off2, lim1, lim2) and scan from both + /// the forward diagonal starting from (off1, off2) and the backward diagonal + /// starting from (lim1, lim2). If the K values on the same diagonal crosses + /// returns the furthest point of reach. We might encounter expensive edge cases + /// using this algorithm, so a little bit of heuristic is needed to cut the + /// search and to return a suboptimal point. + fn split(&mut self, file1: &FileSlice, file2: &FileSlice, need_min: bool) -> Split { + let mut forward_search = unsafe { MiddleSnakeSearch::::new(self.kforward, file1, file2) }; + let mut backwards_search = unsafe { MiddleSnakeSearch::::new(self.kbackward, file1, file2) }; + let is_odd = file2.len().wrapping_sub(file1.len()) & 1 != 0; + + let mut ec = 0; + + while ec <= self.max_cost { + let mut found_snake = false; + forward_search.next_d(); + if is_odd { + if let Some(res) = forward_search.run(file1, file2, |k, token_idx1| { + backwards_search.contains(k) && backwards_search.x_pos_at_diagonal(k) <= token_idx1 + }) { + #[cfg(test)] + cov_mark::hit!(ODD_SPLIT); + match res { + SearchResult::Snake => found_snake = true, + SearchResult::Found { token_idx1, token_idx2 } => { + return Split { + token_idx1, + token_idx2, + minimized_lo: true, + minimized_hi: true, + }; + } + } + } + } else { + found_snake |= forward_search.run(file1, file2, |_, _| false).is_some() + }; + + backwards_search.next_d(); + if !is_odd { + if let Some(res) = backwards_search.run(file1, file2, |k, token_idx1| { + forward_search.contains(k) && token_idx1 <= forward_search.x_pos_at_diagonal(k) + }) { + #[cfg(test)] + cov_mark::hit!(EVEN_SPLIT); + match res { + SearchResult::Snake => found_snake = true, + SearchResult::Found { token_idx1, token_idx2 } => { + return Split { + token_idx1, + token_idx2, + minimized_lo: true, + minimized_hi: true, + }; + } + } + } + } else { + found_snake |= backwards_search.run(file1, file2, |_, _| false).is_some() + }; + + if need_min { + ec += 1; + continue; + } + + // If the edit cost is above the heuristic trigger and if + // we got a good snake, we sample current diagonals to see + // if some of them have reached an "interesting" path. Our + // measure is a function of the distance from the diagonal + // corner (i1 + i2) penalized with the distance from the + // mid-diagonal itself. If this value is above the current + // edit cost times a magic factor (XDL_K_HEUR) we consider + // it interesting. + if found_snake && ec > HEUR_MIN_COST { + if let Some((token_idx1, token_idx2)) = forward_search.found_snake(ec, file1, file2) { + return Split { + token_idx1, + token_idx2, + minimized_lo: true, + minimized_hi: false, + }; + } + + if let Some((token_idx1, token_idx2)) = backwards_search.found_snake(ec, file1, file2) { + return Split { + token_idx1, + token_idx2, + minimized_lo: false, + minimized_hi: true, + }; + } + } + + ec += 1; + } + + let (distance_forward, token_idx1_forward) = forward_search.best_position(file1, file2); + let (distance_backwards, token_idx1_backwards) = backwards_search.best_position(file1, file2); + if distance_forward > file1.len() as isize + file2.len() as isize - distance_backwards { + Split { + token_idx1: token_idx1_forward, + token_idx2: (distance_forward - token_idx1_forward as isize) as i32, + minimized_lo: true, + minimized_hi: false, + } + } else { + Split { + token_idx1: token_idx1_backwards, + token_idx2: (distance_backwards - token_idx1_backwards as isize) as i32, + minimized_lo: false, + minimized_hi: true, + } + } + } +} + +/// Represents a split point in the divide-and-conquer approach. +/// +/// The split divides the problem into two subproblems at the given token positions. +#[derive(Debug)] +struct Split { + /// Token index in the first sequence where the split occurs. + token_idx1: i32, + /// Token index in the second sequence where the split occurs. + token_idx2: i32, + /// Whether the lower subproblem was minimized. + minimized_lo: bool, + /// Whether the upper subproblem was minimized. + minimized_hi: bool, +} + +// /// the mapping performed during preprocessing makes it impossible to directly call +// /// the `sink` during the diff itself. Instead, `file.changed` is set to true for all +// /// tokens that are changed +// /// below these arrays are used to call the sink function +// fn process_changes_with_sink( +// before: &PreprocessedFile, +// after: &PreprocessedFile, +// sink: &mut impl Sink, +// ) { +// let before_end = before.changed.len() as u32 + before.offset; +// let after_end = after.changed.len() as u32 + after.offset; + +// let mut before = before +// .changed +// .iter() +// .enumerate() +// .map(|(i, removed)| (i as u32 + before.offset, *removed)); + +// let mut after = after +// .changed +// .iter() +// .enumerate() +// .map(|(i, inserted)| (i as u32 + after.offset, *inserted)); + +// let mut next1 = before.next(); +// let mut next2 = after.next(); + +// while let (Some((before_pos, removed)), Some((after_pos, inserted))) = (next1, next2) { +// if !(removed | inserted) { +// next1 = before.next(); +// next2 = after.next(); +// continue; +// } + +// let mut hunk_before = before_pos..before_pos; +// let mut hunk_after = after_pos..after_pos; +// if removed { +// let end = before.find(|(_, changed)| !changed); +// next1 = end.map(|(end, _)| (end, false)); +// hunk_before.end = end.map_or(before_end, |(end, _)| end); +// }; + +// if inserted { +// let end = after.find(|(_, changed)| !changed); +// next2 = end.map(|(end, _)| (end, false)); +// hunk_after.end = end.map_or(after_end, |(end, _)| end); +// } + +// sink.process_change(hunk_before, hunk_after); +// } + +// if let Some((before_pos, _)) = next1 { +// sink.process_change(before_pos..before_end, after_end..after_end); +// } else if let Some((after_pos, _)) = next2 { +// sink.process_change(before_end..before_end, after_pos..after_end); +// } +// } diff --git a/knot2/third_party/gix-imara-diff/src/myers/middle_snake.rs b/knot2/third_party/gix-imara-diff/src/myers/middle_snake.rs new file mode 100644 index 0000000000000000000000000000000000000000..e475bc5910c43771be6d249663ee4818b82f0db7 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/myers/middle_snake.rs @@ -0,0 +1,278 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/myers/middle_snake.rs + +use std::ptr::NonNull; + +use crate::myers::slice::FileSlice; +use crate::util::{common_postfix, common_prefix}; + +/// Minimum snake length to be considered a "good" snake for heuristics. +const SNAKE_CNT: u32 = 20; +/// Heuristic multiplier used to evaluate snake quality. +const K_HEUR: u32 = 4; + +/// Performs forward or backward search for the middle snake in Myers' algorithm. +/// +/// The `BACK` const generic parameter determines the search direction: +/// `false` for forward search, `true` for backward search. +#[derive(Debug)] +pub struct MiddleSnakeSearch { + /// Pointer to the k-vector storage for this search direction. + kvec: NonNull, + /// Minimum k-diagonal currently being searched. + kmin: i32, + /// Maximum k-diagonal currently being searched. + kmax: i32, + /// Minimum possible k-diagonal value. + dmin: i32, + /// Maximum possible k-diagonal value. + dmax: i32, +} + +impl MiddleSnakeSearch { + /// # Safety + /// `data` must be valid for reads and writes between `-file2.len() - 1` and `file1.len() + 1` + pub unsafe fn new(data: NonNull, file1: &FileSlice, file2: &FileSlice) -> Self { + let dmin = -(file2.len() as i32); + let dmax = file1.len() as i32; + let kmid = if BACK { dmin + dmax } else { 0 }; + let mut res = Self { + kvec: data, + kmin: kmid, + kmax: kmid, + dmin, + dmax, + }; + let init = if BACK { file1.len() as i32 } else { 0 }; + res.write_xpos_at_diagonal(kmid, init); + res + } + + pub fn contains(&self, k: i32) -> bool { + (self.kmin..=self.kmax).contains(&k) + } + + pub fn bounds_check(&self, k: i32) { + debug_assert!((self.dmin - 1..=self.dmax + 1).contains(&k)); + } + + fn write_xpos_at_diagonal(&mut self, k: i32, token_idx1: i32) { + self.bounds_check(k); + unsafe { self.kvec.as_ptr().offset(k as isize).write(token_idx1) } + } + + pub fn x_pos_at_diagonal(&self, diagonal: i32) -> i32 { + self.bounds_check(diagonal); + unsafe { self.kvec.as_ptr().offset(diagonal as isize).read() } + } + + pub fn pos_at_diagonal(&self, diagonal: i32) -> (i32, i32) { + self.bounds_check(diagonal); + let token_idx1 = unsafe { self.kvec.as_ptr().offset(diagonal as isize).read() }; + let token_idx2 = token_idx1 - diagonal; + (token_idx1, token_idx2) + } + + /// We need to extend the diagonal "domain" by one. If the next + /// values exits the box boundaries we need to change it in the + /// opposite direction because (max - min) must be a power of + /// two. + /// + /// Also we initialize the external K value to -1 so that we can + /// avoid extra conditions in the check inside the core loop. + pub fn next_d(&mut self) { + let init_val = if BACK { + // value should always be larger then bounds + i32::MAX + } else { + // value should always be smaller then bounds + i32::MIN + }; + + if self.kmin > self.dmin { + self.kmin -= 1; + self.write_xpos_at_diagonal(self.kmin - 1, init_val); + } else { + self.kmin += 1; + } + + if self.kmax < self.dmax { + self.kmax += 1; + self.write_xpos_at_diagonal(self.kmax + 1, init_val); + } else { + self.kmax -= 1; + } + } + + pub fn run( + &mut self, + file1: &FileSlice, + file2: &FileSlice, + mut f: impl FnMut(i32, i32) -> bool, + ) -> Option { + let mut res = None; + let mut k = self.kmax; + while k >= self.kmin { + #[cfg(test)] + cov_mark::hit!(SPLIT_SEARCH_ITER); + let mut token_idx1 = if BACK { + if self.x_pos_at_diagonal(k - 1) < self.x_pos_at_diagonal(k + 1) { + self.x_pos_at_diagonal(k - 1) + } else { + self.x_pos_at_diagonal(k + 1) - 1 + } + } else if self.x_pos_at_diagonal(k - 1) >= self.x_pos_at_diagonal(k + 1) { + self.x_pos_at_diagonal(k - 1) + 1 + } else { + self.x_pos_at_diagonal(k + 1) + }; + + let mut token_idx2 = token_idx1 - k; + let off = if BACK { + if token_idx1 > 0 && token_idx2 > 0 { + let tokens1 = &file1.tokens[..token_idx1 as usize]; + let tokens2 = &file2.tokens[..token_idx2 as usize]; + common_postfix(tokens1, tokens2) + } else { + 0 + } + } else if token_idx1 < file1.len() as i32 && token_idx2 < file2.len() as i32 { + let tokens1 = &file1.tokens[token_idx1 as usize..]; + let tokens2 = &file2.tokens[token_idx2 as usize..]; + common_prefix(tokens1, tokens2) + } else { + 0 + }; + + if off > SNAKE_CNT { + res = Some(SearchResult::Snake) + } + + if BACK { + token_idx1 -= off as i32; + token_idx2 -= off as i32; + } else { + token_idx1 += off as i32; + token_idx2 += off as i32; + } + self.write_xpos_at_diagonal(k, token_idx1); + + if f(k, token_idx1) { + return Some(SearchResult::Found { token_idx1, token_idx2 }); + } + + k -= 2; + } + + res + } + + pub fn best_position(&self, file1: &FileSlice, file2: &FileSlice) -> (isize, i32) { + let mut best_distance: isize = if BACK { isize::MAX } else { -1 }; + let mut best_token_idx1 = if BACK { i32::MAX } else { -1 }; + let mut k = self.kmax; + while k >= self.kmin { + let mut token_idx1 = self.x_pos_at_diagonal(k); + if BACK { + token_idx1 = token_idx1.max(0); + } else { + token_idx1 = token_idx1.min(file1.len() as i32); + } + let mut token_idx2 = token_idx1 - k; + if BACK { + if token_idx2 < 0 { + token_idx1 = k; + token_idx2 = 0; + } + } else if token_idx2 > file2.len() as i32 { + token_idx1 = file2.len() as i32 + k; + token_idx2 = file2.len() as i32; + } + + let distance = token_idx1 as isize + token_idx2 as isize; + if BACK && distance < best_distance || !BACK && distance > best_distance { + best_distance = distance; + best_token_idx1 = token_idx1; + } + + k -= 2; + } + (best_distance, best_token_idx1) + } + + pub fn found_snake(&self, ec: u32, file1: &FileSlice, file2: &FileSlice) -> Option<(i32, i32)> { + let mut best_score = 0; + let mut best_token_idx1 = 0; + let mut best_token_idx2 = 0; + let mut k = self.kmax; + while k >= self.kmin { + let (token_idx1, token_idx2) = self.pos_at_diagonal(k); + if BACK { + if !(0..file1.len() as i32 - SNAKE_CNT as i32).contains(&token_idx1) { + k -= 2; + continue; + } + if !(0..file2.len() as i32 - SNAKE_CNT as i32).contains(&token_idx2) { + k -= 2; + continue; + } + } else { + if !(SNAKE_CNT as i32..file1.len() as i32).contains(&token_idx1) { + k -= 2; + continue; + } + if !(SNAKE_CNT as i32..file2.len() as i32).contains(&token_idx2) { + k -= 2; + continue; + } + } + + let main_diagonal_distance = k.unsigned_abs() as usize; + let distance = if BACK { + (file1.len() - token_idx1 as u32) + (file2.len() - token_idx2 as u32) + } else { + token_idx1 as u32 + token_idx2 as u32 + }; + let score = distance as usize + main_diagonal_distance; + if score > (K_HEUR * ec) as usize && score > best_score { + let is_snake = if BACK { + file1.tokens[token_idx1 as usize..] + .iter() + .zip(&file2.tokens[token_idx2 as usize..]) + .take(SNAKE_CNT as usize) + .all(|(token1, token2)| token1 == token2) + } else { + file1.tokens[..token_idx1 as usize] + .iter() + .zip(&file2.tokens[..token_idx2 as usize]) + .rev() + .take(SNAKE_CNT as usize) + .all(|(token1, token2)| token1 == token2) + }; + if is_snake { + best_token_idx1 = token_idx1; + best_token_idx2 = token_idx2; + best_score = score + } + } + + k -= 2; + } + + (best_score > 0).then_some((best_token_idx1, best_token_idx2)) + } +} + +/// The result of a middle snake search iteration. +#[derive(Debug)] +pub enum SearchResult { + /// A good snake was found but not necessarily the middle snake. + Snake, + /// The middle snake was found at the specified token positions. + Found { + /// Token index in the first sequence. + token_idx1: i32, + /// Token index in the second sequence. + token_idx2: i32, + }, +} diff --git a/knot2/third_party/gix-imara-diff/src/myers/preprocess.rs b/knot2/third_party/gix-imara-diff/src/myers/preprocess.rs new file mode 100644 index 0000000000000000000000000000000000000000..0edbd49eb51351108e2a3bcadfb46f51f869a74a --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/myers/preprocess.rs @@ -0,0 +1,231 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/myers/preprocess.rs + +use crate::intern::Token; +use crate::myers::sqrt; + +/// Preprocesses token sequences by removing tokens that don't appear in the other sequence. +/// +/// This optimization reduces the problem size for the Myers algorithm, improving performance +/// for files with many unique tokens. +pub fn preprocess<'a>( + whole: (&[Token], &[Token]), + before: &[Token], + after: &[Token], + removed: &'a mut [bool], + added: &'a mut [bool], +) -> (PreprocessedFile, PreprocessedFile) { + let (occurrences_before, occurrences_after) = token_occurrences(whole, before, after); + let file1 = PreprocessedFile::new(&occurrences_before, before, removed); + let file2 = PreprocessedFile::new(&occurrences_after, after, added); + (file1, file2) +} + +/// Only the middles `file1` and `file2` are classified, against counts and limits from `whole`. +fn token_occurrences( + whole: (&[Token], &[Token]), + file1: &[Token], + file2: &[Token], +) -> (Vec, Vec) { + const MAX_EQLIMIT: u32 = 1024; + + // compute the limit after which tokens are treated as `Occurrences::COMMON` + let eqlimit1 = sqrt(whole.0.len()).min(MAX_EQLIMIT); + let eqlimit2 = sqrt(whole.1.len()).min(MAX_EQLIMIT); + + // first collect how often each token occurs in a file + let count = |file: &[Token]| { + let mut occurrences = Vec::new(); + for token in file { + let bucket = token.0 as usize; + if bucket >= occurrences.len() { + occurrences.resize(bucket + 1, 0u32); + } + occurrences[bucket] += 1; + } + occurrences + }; + let (occurrences1, occurrences2) = (count(whole.0), count(whole.1)); + + let classify = |file: &[Token], other: &[u32], eqlimit: u32| -> Vec<_> { + file.iter() + .map(|token| { + let occurrences = *other.get(token.0 as usize).unwrap_or(&0); + Occurrences::from_occurrences(occurrences, eqlimit) + }) + .collect() + }; + ( + classify(file1, &occurrences2, eqlimit1), + classify(file2, &occurrences1, eqlimit2), + ) +} + +/// Categorizes how frequently a token appears in a file. +#[derive(Clone, Copy, Debug)] +enum Occurrences { + /// Token does not occur in the other file. + None, + /// Token occurs at least once in the other file. + Some, + /// Token occurs very frequently in the other file (exact threshold depends on file size). + /// Such tokens are usually empty lines or braces and are often not meaningful to a diff. + Common, +} + +impl Occurrences { + pub fn from_occurrences(occurrences: u32, eqlimit: u32) -> Occurrences { + if occurrences == 0 { + Occurrences::None + } else if occurrences >= eqlimit { + Occurrences::Common + } else { + Occurrences::Some + } + } +} + +/// A file after preprocessing has removed unmatched tokens. +#[derive(Debug)] +pub struct PreprocessedFile { + /// Maps from new token positions to original positions in the unpreprocessed file. + pub indices: Vec, + /// The tokens that remain after preprocessing. + pub tokens: Vec, +} + +impl PreprocessedFile { + fn new(token_occurrences: &[Occurrences], tokens: &[Token], changed: &mut [bool]) -> PreprocessedFile { + let (tokens, indices) = prune_unmatched_tokens(tokens, token_occurrences, changed); + PreprocessedFile { indices, tokens } + } +} + +fn prune_unmatched_tokens( + file: &[Token], + token_status: &[Occurrences], + changed: &mut [bool], +) -> (Vec, Vec) { + assert_eq!(token_status.len(), file.len()); + file.iter() + .zip(token_status) + .enumerate() + .filter_map(|(i, (&token, &status))| { + let prune = match status { + Occurrences::None => true, + Occurrences::Some => false, + Occurrences::Common => should_prune_common_line(token_status, i), + }; + if prune { + changed[i] = true; + None + } else { + Some((token, i as u32)) + } + }) + .unzip() +} + +// TODO do not unnecessarily rescan lines +fn should_prune_common_line(token_status: &[Occurrences], pos: usize) -> bool { + const WINDOW_SIZE: usize = 100; + + // like git's `xdl_clean_mmatch()`, both runs count the line itself as common + let mut unmatched_before = 0; + let mut common_before = 1; + + let start = pos.saturating_sub(WINDOW_SIZE); + for status in token_status[start..pos].iter().rev() { + match status { + Occurrences::None => { + unmatched_before += 1; + } + Occurrences::Common => { + common_before += 1; + } + Occurrences::Some => break, + } + } + + if unmatched_before == 0 { + return false; + } + + let end = token_status.len().min(pos + 1 + WINDOW_SIZE); + let mut unmatched_after = 0; + let mut common_after = 1; + for status in token_status[pos + 1..end].iter() { + match status { + Occurrences::None => { + unmatched_after += 1; + } + Occurrences::Common => { + common_after += 1; + } + Occurrences::Some => break, + } + } + + if unmatched_after == 0 { + return false; + } + + let common = common_before + common_after; + let unmatched = unmatched_before + unmatched_after; + + unmatched > 3 * common +} + +#[cfg(test)] +mod tests { + use super::{Occurrences, should_prune_common_line, token_occurrences}; + use crate::intern::Token; + + #[test] + fn common_line_pruning_ignores_distant_context() { + let mut token_status = vec![Occurrences::Some; 700]; + token_status[100..400].fill(Occurrences::None); + token_status[400..450].fill(Occurrences::None); + token_status[450..500].fill(Occurrences::Common); + token_status[500..550].fill(Occurrences::Common); + token_status[550..600].fill(Occurrences::None); + + assert!( + !should_prune_common_line(&token_status, 500), + "only the last 100 items before the current line should contribute to the backward scan" + ); + } + + #[test] + fn occurrences_count_the_whole_files() { + let (x, z) = (Token(0), Token(1)); + let whole1 = vec![x; 9]; + let mut whole2 = vec![x; 8]; + whole2.push(z); + let (occurrences1, _) = token_occurrences((&whole1, &whole2), &whole1[8..], &whole2[8..]); + assert!(matches!(occurrences1[0], Occurrences::Common)); + } + + #[test] + fn a_common_line_is_pruned_past_three_unmatched_per_common_counting_itself_in_both_runs() { + let (none, common) = (Occurrences::None, Occurrences::Common); + let status = [none, none, none, common, none, none, none]; + assert!(!should_prune_common_line(&status, 3)); + let status = [none, none, none, common, none, none, none, none]; + assert!(should_prune_common_line(&status, 3)); + } + + #[test] + fn the_run_after_a_common_line_covers_the_next_100_lines() { + let mut status = vec![Occurrences::Some; 240]; + let pos = 100; + status[pos - 35..pos - 32].fill(Occurrences::None); + status[pos - 32..=pos].fill(Occurrences::Common); + status[pos + 1..=pos + 100].fill(Occurrences::None); + assert!(should_prune_common_line(&status, pos)); + + status[pos - 35] = Occurrences::Some; + status[pos + 101] = Occurrences::None; + assert!(!should_prune_common_line(&status, pos)); + } +} diff --git a/knot2/third_party/gix-imara-diff/src/myers/slice.rs b/knot2/third_party/gix-imara-diff/src/myers/slice.rs new file mode 100644 index 0000000000000000000000000000000000000000..f266fa7e17f07eb4a8c6d61fa942607a4beca52b --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/myers/slice.rs @@ -0,0 +1,80 @@ +use std::mem::take; +use std::ops::RangeBounds; + +use crate::intern::Token; +use crate::myers::preprocess::PreprocessedFile; +use crate::util::common_edges; + +/// A slice of a preprocessed file used during the Myers algorithm's divide-and-conquer. +/// +/// This structure allows the algorithm to work on subproblems while maintaining +/// references to the original file's token indices and change tracking. +#[derive(Default)] +pub struct FileSlice<'a> { + /// The tokens in this slice. + pub tokens: &'a [Token], + /// Maps from slice positions back to original file positions. + indices: &'a [u32], + /// Tracks which tokens in the original file have changed. + changed: &'a mut [bool], +} + +impl<'a> FileSlice<'a> { + pub fn new(file: &'a PreprocessedFile, changed: &'a mut [bool]) -> Self { + Self { + tokens: &file.tokens, + indices: &file.indices, + changed, + } + } + + pub fn mark_changed(&mut self) { + for &i in self.indices { + self.changed[i as usize] = true; + } + } + + pub fn borrow(&mut self) -> FileSlice<'_> { + FileSlice { + tokens: self.tokens, + changed: self.changed, + indices: self.indices, + } + } + + pub fn slice>(self, range: R) -> Self { + let start = match range.start_bound() { + std::ops::Bound::Included(&start) => start, + std::ops::Bound::Excluded(&start) => start + 1, + std::ops::Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + std::ops::Bound::Included(&end) => end + 1, + std::ops::Bound::Excluded(&end) => end, + std::ops::Bound::Unbounded => self.len(), + }; + + Self { + tokens: &self.tokens[start as usize..end as usize], + changed: self.changed, + indices: &self.indices[start as usize..end as usize], + } + } + + pub fn strip_common(&mut self, other: &mut Self) { + let (start, common_postfix) = common_edges(self.tokens, other.tokens); + let end = self.len() - common_postfix; + *self = take(self).slice(start..end); + let end = other.len() - common_postfix; + *other = take(other).slice(start..end) + } + + pub fn len(&self) -> u32 { + self.tokens.len() as u32 + } + + pub fn is_empty(&self) -> bool { + self.tokens.is_empty() + } +} diff --git a/knot2/third_party/gix-imara-diff/src/postprocess.rs b/knot2/third_party/gix-imara-diff/src/postprocess.rs new file mode 100644 index 0000000000000000000000000000000000000000..16351c10ce54ee810db343b493fc4ac432f9a0f2 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/postprocess.rs @@ -0,0 +1,162 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/postprocess.rs + +use crate::intern::{InternedInput, Token}; +use crate::slider_heuristic::SliderHeuristic; +use crate::util::{find_hunk_end, find_hunk_start}; +use crate::{Diff, Hunk}; + +impl Diff { + /// Postprocesses the diff using explicit token sequences and a custom heuristic. + /// + /// This is a lower-level method that works directly with token sequences rather than + /// an `InternedInput`. Use [`postprocess_with_heuristic`](Self::postprocess_with_heuristic) + /// for a more convenient API. + /// + /// # Parameters + /// + /// * `before` - The token sequence from the first file, before changes + /// * `after` - The token sequence from the second file, after changes + /// * `heuristic` - The slider heuristic to use for positioning hunks + pub fn postprocess_with(&mut self, before: &[Token], after: &[Token], mut heuristic: impl SliderHeuristic) { + Postprocessor { + added: &mut self.added, + removed: &mut self.removed, + tokens: after, + hunk: Hunk { + before: 0..0, + after: 0..0, + }, + heuristic: &mut heuristic, + } + .run(); + Postprocessor { + added: &mut self.removed, + removed: &mut self.added, + tokens: before, + hunk: Hunk { + before: 0..0, + after: 0..0, + }, + heuristic: &mut heuristic, + } + .run() + } + + /// Postprocesses the diff using an `InternedInput` and a custom heuristic. + /// + /// This is a convenience wrapper around [`postprocess_with`](Self::postprocess_with) + /// that extracts the token sequences from the input automatically. + /// + /// # Parameters + /// + /// * `input` - The interned input containing the token sequences + /// * `heuristic` - The slider heuristic to use for positioning hunks + pub fn postprocess_with_heuristic(&mut self, input: &InternedInput, heuristic: impl SliderHeuristic) { + self.postprocess_with(&input.before, &input.after, heuristic); + } +} + +/// Internal state for postprocessing a diff to improve readability. +struct Postprocessor<'a, H> { + /// The mutable array tracking which tokens were added. + added: &'a mut [bool], + /// The immutable array tracking which tokens were removed. + removed: &'a [bool], + /// The token sequence being processed. + tokens: &'a [Token], + /// The current hunk being processed in the iteration. + hunk: Hunk, + /// The heuristic used to determine optimal hunk positions. + heuristic: &'a mut H, +} + +impl Postprocessor<'_, H> { + fn run(mut self) { + loop { + // find a hunk + if !self.hunk.next_hunk(self.removed, self.added) { + break; + } + + let mut earliest_end; + let mut is_modification; + loop { + // move hunk up as far as possible to possibly merge it with other hunks + // and discover if there are other possible positions + while self.slide_up() {} + earliest_end = self.hunk.after.end; + is_modification = self.hunk.before.start != self.hunk.before.end; + + let hunk_size_unexpanded = self.hunk.after.len(); + // move hunk down as far as possible (and merge with other hunks it if + // possible) sliding down is often the most preferred position + while self.slide_down() { + is_modification |= self.hunk.before.start != self.hunk.before.end; + } + // if this hunk was merged with another hunk while sliding down we might + // be able to slide up more otherwise we are done + if hunk_size_unexpanded == self.hunk.after.len() { + break; + } + } + + if self.hunk.after.end == earliest_end { + continue; + } + if is_modification { + // hunk can be moved and there is a removed hunk in the same region + // move the hunk so it align with the other hunk to produce a single + // MODIFIED hunk instead of two separate ADDED/REMOVED hunks + while self.hunk.before.start == self.hunk.before.end { + let success = self.slide_up(); + debug_assert!(success); + } + } else { + let slider_end = self + .heuristic + .best_slider_end(self.tokens, self.hunk.after.clone(), earliest_end); + for _ in slider_end..self.hunk.after.end { + let success = self.slide_up(); + debug_assert!(success); + } + } + } + } + + /// Slides a hunk down by one token/line, potentially merging it with a subsequent hunk. + fn slide_down(&mut self) -> bool { + let Some(&next_token) = self.tokens.get(self.hunk.after.end as usize) else { + return false; + }; + if self.tokens[self.hunk.after.start as usize] != next_token { + return false; + } + self.added[self.hunk.after.start as usize] = false; + self.added[self.hunk.after.end as usize] = true; + self.hunk.after.start += 1; + self.hunk.after.end = find_hunk_end(self.added, self.hunk.after.end); + // move the end of the remove range one down to keep the unchanged lines aligned + self.hunk.before.start = self.hunk.before.end + 1; + self.hunk.before.end = find_hunk_end(self.removed, self.hunk.before.start); + true + } + + /// Slides a hunk up by one token/line, potentially merging it with a previous hunk. + fn slide_up(&mut self) -> bool { + if self.hunk.after.start == 0 { + return false; + } + if self.tokens[self.hunk.after.start as usize - 1] != self.tokens[self.hunk.after.end as usize - 1] { + return false; + } + self.added[self.hunk.after.start as usize - 1] = true; + self.added[self.hunk.after.end as usize - 1] = false; + self.hunk.after.end -= 1; + self.hunk.after.start = find_hunk_start(self.added, self.hunk.after.start - 1); + // move the start of the remove range one up to keep the unchanged lines aligned + self.hunk.before.end = self.hunk.before.start - 1; + self.hunk.before.start = find_hunk_start(self.removed, self.hunk.before.start - 1); + true + } +} diff --git a/knot2/third_party/gix-imara-diff/src/slider_heuristic.rs b/knot2/third_party/gix-imara-diff/src/slider_heuristic.rs new file mode 100644 index 0000000000000000000000000000000000000000..7985b0cc8f46abb963085c827b5ef6f4d415e467 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/slider_heuristic.rs @@ -0,0 +1,396 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/slider_heuristic.rs + +use std::cmp::Ordering; +use std::hash::Hash; +use std::ops::{Add, Range}; + +use crate::intern::Token; + +/// A trait for heuristics that determine the best position for ambiguous diff hunks. +/// +/// During postprocessing, some hunks can be moved up or down without changing the +/// minimal nature of the diff. This trait allows customizing the logic for choosing +/// the optimal position for such hunks. +pub trait SliderHeuristic { + /// Determines the best ending position for a hunk that can be slid. + /// + /// # Parameters + /// + /// * `tokens` - The token sequence being diffed + /// * `hunk` - The range representing the current hunk position + /// * `earliest_end` - The earliest valid ending position for the hunk + /// + /// # Returns + /// + /// The preferred ending position for the hunk + fn best_slider_end(&mut self, tokens: &[Token], hunk: Range, earliest_end: u32) -> u32; +} + +impl SliderHeuristic for F +where + F: FnMut(&[Token], Range, u32) -> u32, +{ + fn best_slider_end(&mut self, tokens: &[Token], hunk: Range, earliest_end: u32) -> u32 { + self(tokens, hunk, earliest_end) + } +} + +/// A slider heuristic that doesn't adjust hunk positions. +/// +/// This heuristic always places hunks at their lowest possible position without +/// applying any additional logic. +pub struct NoSliderHeuristic; + +impl SliderHeuristic for NoSliderHeuristic { + fn best_slider_end(&mut self, _tokens: &[Token], hunk: Range, _earliest_end: u32) -> u32 { + hunk.end + } +} + +/// A slider heuristic that uses indentation levels to determine the best hunk position. +/// +/// This heuristic analyzes the indentation of lines surrounding potential hunk positions +/// and chooses the position that results in the most intuitive diff for human readers. +/// It's particularly effective for code and other indented text. +pub struct IndentHeuristic { + /// A function that computes the indentation level for a given token. + indent_of_token: IndentOfToken, +} + +impl IndentHeuristic { + /// Creates a new `IndentHeuristic` with the given indentation function. + /// + /// # Parameters + /// + /// * `indent_of_token` - A function that takes a token and returns its indentation level + pub fn new(indent_of_token: IndentOfToken) -> Self { + Self { indent_of_token } + } +} + +impl IndentLevel> SliderHeuristic for IndentHeuristic { + fn best_slider_end(&mut self, tokens: &[Token], hunk: Range, earliest_end: u32) -> u32 { + const MAX_SLIDING: u32 = 100; + // This is a pure insertion that can be moved freely up and down. + // To get more intuitive results, apply a heuristic. + let mut top_slider_end = earliest_end; + // TODO: why is this needed + if top_slider_end < hunk.start - 1 { + top_slider_end = hunk.start - 1; + } + if hunk.end > top_slider_end + MAX_SLIDING { + top_slider_end = hunk.end - MAX_SLIDING; + } + let group_size = hunk.end - hunk.start; + let mut best_score = Score::for_range( + top_slider_end - group_size..top_slider_end, + tokens, + &self.indent_of_token, + ); + let mut best_slider_end = top_slider_end; + for slider_end in (top_slider_end + 1)..=hunk.end { + let score = Score::for_range(slider_end - group_size..slider_end, tokens, &self.indent_of_token); + if score.is_improvement_over(best_score) { + best_score = score; + best_slider_end = slider_end; + } + } + best_slider_end + } +} + +/// Represents the indentation level of a line. +/// +/// Indentation is measured in spaces, with tabs expanded according to a configurable tab width. +/// Special values are used to represent blank lines and maximum indentation. +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd)] +pub struct IndentLevel(u8); + +impl IndentLevel { + /// Represents a line that is empty or contains only whitespace (or EOF). + const BLANK: IndentLevel = IndentLevel(u8::MAX); + /// The maximum trackable indentation level. + const MAX: IndentLevel = IndentLevel(200); + + /// Computes the indentation level for an ASCII line. + /// + /// # Parameters + /// + /// * `src` - An iterator over the bytes of the line + /// * `tab_width` - The number of spaces that a tab character represents (min is 1) + /// + /// # Returns + /// + /// The computed indentation level, or `BLANK` if the line contains only whitespace + pub fn for_ascii_line(src: impl IntoIterator, tab_width: u8) -> IndentLevel { + let mut indent_level = IndentLevel(0); + let tab_width = tab_width.max(1); + for c in src { + match c { + b' ' => indent_level = indent_level.increased_by(1), + b'\t' => indent_level = indent_level.increased_by(tab_width - indent_level.0 % tab_width), + b'\r' | b'\n' | b'\x0C' => (), + _ => return indent_level, + } + if indent_level >= Self::MAX { + return Self::MAX; + } + } + IndentLevel::BLANK + } + + /// Computes the indentation level for a Unicode line. + /// + /// # Parameters + /// + /// * `src` - An iterator over the characters of the line + /// * `tab_width` - The number of spaces that a tab character represents + /// + /// # Returns + /// + /// The computed indentation level, or `BLANK` if the line contains only whitespace + pub fn for_line(src: impl IntoIterator, tab_width: u8) -> IndentLevel { + let mut indent_level = IndentLevel(0); + let tab_width = tab_width.max(1); + for c in src { + match c { + ' ' => indent_level = indent_level.increased_by(1), + '\t' => indent_level = indent_level.increased_by(tab_width - indent_level.0 % tab_width), + '\r' | '\n' | '\x0C' => (), + _ => return indent_level, + } + if indent_level >= Self::MAX { + return Self::MAX; + } + } + IndentLevel::BLANK + } + + fn increased_by(self, amount: u8) -> Self { + IndentLevel(self.0.saturating_add(amount).min(Self::MAX.0)) + } + + fn map_or(self, default: T, f: impl FnOnce(u8) -> T) -> T { + if self == Self::BLANK { default } else { f(self.0) } + } + + fn or(self, default: Self) -> Self { + if self == Self::BLANK { default } else { self } + } +} + +/// Captures indentation information for a token and its surrounding context. +/// +/// This structure is used by the indent heuristic to evaluate different hunk positions. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Indents { + /// Indentation level of the current line/token. + indent: IndentLevel, + /// Indentation level of the previous non-blank line. + prev_indent: IndentLevel, + /// Indentation level of the next non-blank line. + next_indent: IndentLevel, + /// The number of consecutive blank lines above the current position. + leading_blanks: u8, + /// The number of blank lines after the line following the current position. + trailing_blanks: u8, +} + +/// Maximum number of consecutive blank lines to consider when computing indentation context. +const MAX_BLANKS: usize = 20; + +impl Indents { + fn at_token(tokens: &[Token], token_idx: usize, indent_of_token: impl Fn(Token) -> IndentLevel) -> Indents { + let (leading_blank_lines, indent_previous_line) = tokens[..token_idx] + .iter() + .rev() + .enumerate() + .find_map(|(i, &token)| { + if i == MAX_BLANKS { + Some((i, IndentLevel(0))) + } else { + let level = indent_of_token(token); + if level == IndentLevel::BLANK { + None + } else { + Some((i, level)) + } + } + }) + .unwrap_or((token_idx, IndentLevel::BLANK)); + let at_eof = token_idx == tokens.len(); + let (trailing_blank_lines, indent_next_line) = if at_eof { + (0, IndentLevel::BLANK) + } else { + tokens[token_idx + 1..] + .iter() + .enumerate() + .find_map(|(i, &token)| { + if i == MAX_BLANKS { + Some((i, IndentLevel(0))) + } else { + let level = indent_of_token(token); + if level == IndentLevel::BLANK { + None + } else { + Some((i, level)) + } + } + }) + .unwrap_or((token_idx, IndentLevel::BLANK)) + }; + let indent = tokens + .get(token_idx) + .map_or(IndentLevel::BLANK, |&token| indent_of_token(token)); + Indents { + indent, + prev_indent: indent_previous_line, + next_indent: indent_next_line, + leading_blanks: leading_blank_lines as u8, + trailing_blanks: trailing_blank_lines as u8, + } + } + + fn score(&self) -> Score { + let mut penalty = 0; + if self.prev_indent == IndentLevel::BLANK && self.leading_blanks == 0 { + penalty += START_OF_FILE_PENALTY; + } + if self.next_indent == IndentLevel::BLANK && self.trailing_blanks == 0 { + penalty += END_OF_FILE_PENALTY; + } + + let trailing_blank_lines = if self.indent == IndentLevel::BLANK { + self.trailing_blanks as i32 + 1 + } else { + 0 + }; + let total_blank_lines = trailing_blank_lines + self.leading_blanks as i32; + penalty += TOTAL_BLANK_LINE_WEIGHT * total_blank_lines + trailing_blank_lines * TRAILING_BLANK_LINES_WEIGHT; + let indent = self.indent.or(self.next_indent); + if indent != IndentLevel::BLANK && self.prev_indent != IndentLevel::BLANK { + match indent.0.cmp(&self.prev_indent.0) { + Ordering::Equal => {} + // self.next_indent != IndentLevel::BLANK follows for free here + // since indent != BLANK and therefore self.next_indent <= indent < BLANK + Ordering::Less if self.next_indent.0 <= indent.0 => { + penalty += if total_blank_lines != 0 { + RELATIVE_DEDENT_WITH_BLANK_PENALTY + } else { + RELATIVE_DEDENT_PENALTY + } + } + Ordering::Less => { + penalty += if total_blank_lines != 0 { + RELATIVE_OUTDENT_WITH_BLANK_PENALTY + } else { + RELATIVE_OUTDENT_PENALTY + } + } + Ordering::Greater => { + penalty += if total_blank_lines != 0 { + RELATIVE_INDENT_WITH_BLANK_PENALTY + } else { + RELATIVE_INDENT_PENALTY + } + } + } + } + Score { + indent: indent.map_or(-1, i32::from), + penalty, + } + } +} + +/// Penalty for placing a hunk at the start of a file. +const START_OF_FILE_PENALTY: i32 = 1; +/// Penalty for placing a hunk at the end of a file. +const END_OF_FILE_PENALTY: i32 = 21; +/// Weight applied to the total number of blank lines surrounding a hunk (negative means preferred). +const TOTAL_BLANK_LINE_WEIGHT: i32 = -30; +/// Additional weight for trailing blank lines. +const TRAILING_BLANK_LINES_WEIGHT: i32 = 6; + +/// Penalty for placing a hunk where indentation increases (negative means preferred). +const RELATIVE_INDENT_PENALTY: i32 = -4; +/// Penalty for placing a hunk where indentation increases with blank lines present. +const RELATIVE_INDENT_WITH_BLANK_PENALTY: i32 = 10; + +/// Penalty for placing a hunk where indentation decreases (outdent). +const RELATIVE_OUTDENT_PENALTY: i32 = 24; +/// Penalty for placing a hunk where indentation decreases with blank lines present. +const RELATIVE_OUTDENT_WITH_BLANK_PENALTY: i32 = 17; + +/// Penalty for placing a hunk where indentation decreases but stays aligned (dedent). +const RELATIVE_DEDENT_PENALTY: i32 = 23; +/// Penalty for placing a hunk where indentation decreases but stays aligned with blank lines present. +const RELATIVE_DEDENT_WITH_BLANK_PENALTY: i32 = 17; + +/// Weight factor for comparing indentation levels when scoring positions. +const INDENT_WEIGHT: i32 = 60; + +/// A score for evaluating the quality of a hunk position. +/// +/// Lower scores are better. The score considers both indentation level +/// and various penalties based on the surrounding context. +#[derive(PartialEq, Eq, Clone, Copy)] +struct Score { + /// The combined indentation level at the hunk boundaries. + indent: i32, + /// The total penalty from various heuristics. + penalty: i32, +} + +impl Score { + fn for_range(range: Range, tokens: &[Token], indent_of_token: impl Fn(Token) -> IndentLevel) -> Score { + Indents::at_token(tokens, range.start as usize, &indent_of_token).score() + + Indents::at_token(tokens, range.end as usize, &indent_of_token).score() + } +} + +impl Add for Score { + type Output = Score; + + fn add(self, rhs: Self) -> Self::Output { + Score { + indent: self.indent + rhs.indent, + penalty: self.penalty + rhs.penalty, + } + } +} + +impl Score { + fn is_improvement_over(self, prev_score: Self) -> bool { + // smaller indentation level is preferred (with a weight) + let indent_score = match prev_score.indent.cmp(&self.indent) { + Ordering::Less => INDENT_WEIGHT, + Ordering::Greater => -INDENT_WEIGHT, + Ordering::Equal => 0, + }; + (indent_score + self.penalty - prev_score.penalty) <= 0 + } +} + +#[cfg(test)] +mod tests { + use super::IndentLevel; + + #[test] + fn ascii_indent_clamps_before_overflow() { + assert_eq!( + IndentLevel::for_ascii_line(std::iter::repeat_n(b' ', 255), 1), + IndentLevel::MAX + ); + assert_eq!( + IndentLevel::for_ascii_line(std::iter::repeat_n(b'\t', 8), u8::MAX), + IndentLevel::MAX + ); + } + + #[test] + fn unicode_indent_treats_zero_tab_width_as_one() { + assert_eq!(IndentLevel::for_line(['\t', 'x'], 0), IndentLevel(1)); + } +} diff --git a/knot2/third_party/gix-imara-diff/src/sources.rs b/knot2/third_party/gix-imara-diff/src/sources.rs new file mode 100644 index 0000000000000000000000000000000000000000..9b2d5ea0d1d772c272598acaa47763cf12163851 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/sources.rs @@ -0,0 +1,232 @@ +//! Modified for gitoxide from the upstream imara-diff crate. +//! Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/sources.rs +//! +//! Utilities for creating token sources from common data types. +//! +//! This module provides implementations of [`TokenSource`] for +//! strings and byte slices, splitting them into lines by default. + +use std::str::from_utf8_unchecked; + +use bstr::ByteSlice; + +use crate::TokenSource; + +/// Returns a [`TokenSource`] that uses the lines in `data` as Tokens. The newline +/// separator (`\r\n` or `\n`) is included in the emitted tokens. This means that changing +/// the newline separator from `\r\n` to `\n` (or omitting it fully on the last line) is +/// detected by [`Diff`](crate::Diff). +pub fn lines(data: &str) -> Lines<'_> { + Lines(ByteLines(data.as_bytes())) +} + +/// Returns a [`TokenSource`] that uses the words in `data` as Tokens. A word is +/// a sequence of alphanumeric characters as determined by +/// `char::is_alphanumeric`, or a sequence of just the space character ' '. Any +/// other characters are their own word. +pub fn words(data: &str) -> Words<'_> { + Words(data) +} + +/// Returns a [`TokenSource`] that uses the lines in `data` as Tokens. The newline +/// separator (`\r\n` or `\n`) is included in the emitted tokens. This means that changing +/// the newline separator from `\r\n` to `\n` (or omitting it fully on the last line) is +/// detected when computing a [`Diff`](crate::Diff). +pub fn bstr_lines(data: &bstr::BStr) -> BStrLines<'_> { + BStrLines(data) +} + +/// Returns a [`TokenSource`] that uses the lines in `data` as Tokens. The newline +/// separator (`\r\n` or `\n`) is included in the emitted tokens. This means that changing +/// the newline separator from `\r\n` to `\n` (or omitting it fully on the last line) is +/// detected when computing a [`Diff`](crate::Diff). +pub fn byte_lines(data: &[u8]) -> ByteLines<'_> { + ByteLines(data) +} + +/// By default, a line diff is produced for a string +impl<'a> TokenSource for &'a str { + type Token = &'a str; + + type Tokenizer = Lines<'a>; + + fn tokenize(&self) -> Self::Tokenizer { + lines(self) + } + + fn estimate_tokens(&self) -> u32 { + lines(self).estimate_tokens() + } +} + +/// By default, a line diff is produced for a `BStr`. +impl<'a> TokenSource for &'a bstr::BStr { + type Token = Self; + type Tokenizer = BStrLines<'a>; + + fn tokenize(&self) -> Self::Tokenizer { + bstr_lines(self) + } + + fn estimate_tokens(&self) -> u32 { + bstr_lines(self).estimate_tokens() + } +} + +/// By default, a line diff is produced for a bytes +impl<'a> TokenSource for &'a [u8] { + type Token = Self; + type Tokenizer = ByteLines<'a>; + + fn tokenize(&self) -> Self::Tokenizer { + byte_lines(self) + } + + fn estimate_tokens(&self) -> u32 { + byte_lines(self).estimate_tokens() + } +} + +/// A [`TokenSource`] that returns the lines of a `str` as tokens. See [`lines`] for +/// details. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Lines<'a>(ByteLines<'a>); + +impl<'a> Iterator for Lines<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option { + // safety invariant: this struct may only contain valid utf8 + // dividing valid utf8 bytes by ascii characters always produces valid utf-8 + self.0.next().map(|it| unsafe { from_utf8_unchecked(it) }) + } +} + +/// By default, a line diff is produced for a string +impl<'a> TokenSource for Lines<'a> { + type Token = &'a str; + + type Tokenizer = Self; + + fn tokenize(&self) -> Self::Tokenizer { + *self + } + + fn estimate_tokens(&self) -> u32 { + self.0.estimate_tokens() + } +} + +/// A [`TokenSource`] that returns the words of a string as tokens. See +/// [`words`] for details. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Words<'a>(&'a str); + +impl<'a> Iterator for Words<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option { + if self.0.is_empty() { + return None; + } + + let initial = self.0.chars().next().unwrap(); + let word_len = if initial == ' ' { + self.0 + .char_indices() + .find(|(_, c)| *c != ' ') + .map_or(self.0.len(), |(index, _)| index) + } else if initial.is_alphanumeric() { + self.0 + .char_indices() + .find(|(_, c)| !c.is_alphanumeric() && *c != '_') + .map_or(self.0.len(), |(index, _)| index) + } else { + initial.len_utf8() + }; + + let (word, rem) = self.0.split_at(word_len); + self.0 = rem; + Some(word) + } +} +impl<'a> TokenSource for Words<'a> { + type Token = &'a str; + + type Tokenizer = Self; + + fn tokenize(&self) -> Self::Tokenizer { + *self + } + + fn estimate_tokens(&self) -> u32 { + (self.0.len() / 3) as u32 + } +} + +/// A [`TokenSource`] that returns the lines of a `BStr` as tokens. See [`bstr_lines`] for details. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct BStrLines<'a>(&'a bstr::BStr); + +impl<'a> Iterator for BStrLines<'a> { + type Item = &'a bstr::BStr; + + fn next(&mut self) -> Option { + if self.0.is_empty() { + return None; + } + let line_len = self.0.find_byte(b'\n').map_or(self.0.len(), |len| len + 1); + let (line, rem) = self.0.split_at(line_len); + self.0 = rem.into(); + Some(line.into()) + } +} + +impl<'a> TokenSource for BStrLines<'a> { + type Token = &'a bstr::BStr; + type Tokenizer = Self; + + fn tokenize(&self) -> Self::Tokenizer { + *self + } + + fn estimate_tokens(&self) -> u32 { + let len: usize = self.take(20).map(|line| line.len()).sum(); + (self.0.len() * 20).checked_div(len).unwrap_or(100) as u32 + } +} + +/// A [`TokenSource`] that returns the lines of a byte slice as tokens. See [`byte_lines`] +/// for details. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct ByteLines<'a>(&'a [u8]); + +impl<'a> Iterator for ByteLines<'a> { + type Item = &'a [u8]; + + fn next(&mut self) -> Option { + if self.0.is_empty() { + return None; + } + let line_len = self.0.find_byte(b'\n').map_or(self.0.len(), |len| len + 1); + let (line, rem) = self.0.split_at(line_len); + self.0 = rem; + Some(line) + } +} + +/// By default, a line diff is produced for a string +impl<'a> TokenSource for ByteLines<'a> { + type Token = &'a [u8]; + + type Tokenizer = Self; + + fn tokenize(&self) -> Self::Tokenizer { + *self + } + + fn estimate_tokens(&self) -> u32 { + let len: usize = self.take(20).map(|line| line.len()).sum(); + (self.0.len() * 20).checked_div(len).unwrap_or(100) as u32 + } +} diff --git a/knot2/third_party/gix-imara-diff/src/unified_diff.rs b/knot2/third_party/gix-imara-diff/src/unified_diff.rs new file mode 100644 index 0000000000000000000000000000000000000000..b33e0cba5d85677f65262b605dd38368d70942fa --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/unified_diff.rs @@ -0,0 +1,281 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/unified_diff.rs + +use std::fmt::{self, Display}; +use std::hash::Hash; + +use crate::Diff; +use crate::intern::{InternedInput, Interner, Token}; + +impl Diff { + /// Creates a unified diff output that can be formatted as a string. + /// + /// This is a convenience method that extracts the token sequences from the `input`. + /// + /// # Parameters + /// + /// * `printer` - A printer implementation that controls how tokens are displayed + /// * `config` - Configuration options for the unified diff format + /// * `input` - The interned input containing the token sequences + pub fn unified_diff<'a, P: UnifiedDiffPrinter, T: Hash + Eq>( + &'a self, + printer: &'a P, + config: UnifiedDiffConfig, + input: &'a InternedInput, + ) -> UnifiedDiff<'a, P> { + self.unified_diff_with(printer, config, &input.before, &input.after) + } + + /// Creates a unified diff output with explicit token sequences. + /// + /// # Parameters + /// + /// * `printer` - A printer implementation that controls how tokens are displayed + /// * `config` - Configuration options for the unified diff format + /// * `before` - The token sequence from the first file, before changes. + /// * `after` - The token sequence from the second file, after changes. + pub fn unified_diff_with<'a, P: UnifiedDiffPrinter>( + &'a self, + printer: &'a P, + config: UnifiedDiffConfig, + before: &'a [Token], + after: &'a [Token], + ) -> UnifiedDiff<'a, P> { + UnifiedDiff { + printer, + diff: self, + config, + before, + after, + } + } +} + +/// A trait for customizing the output format of unified diffs. +/// +/// Implementations of this trait control how different parts of a unified diff are displayed, +/// including headers, context lines, and changed hunks. +pub trait UnifiedDiffPrinter { + /// Displays the header for a hunk in the unified diff format. + /// + /// The header typically includes the line numbers and lengths for both files. + /// + /// # Parameters + /// + /// * `f` - The formatter to write to + /// * `start_before` - The starting line number in the first file (0-indexed) + /// * `start_after` - The starting line number in the second file (0-indexed) + /// * `len_before` - The number of lines from the first file in this hunk + /// * `len_after` - The number of lines from the second file in this hunk + fn display_header( + &self, + f: impl fmt::Write, + start_before: u32, + start_after: u32, + len_before: u32, + len_after: u32, + ) -> fmt::Result; + /// Displays a context token (an unchanged line) in the unified diff. + /// + /// # Parameters + /// + /// * `f` - The formatter to write to + /// * `token` - The token to display + fn display_context_token(&self, f: impl fmt::Write, token: Token) -> fmt::Result; + /// Displays a hunk showing the changes between before and after tokens. + /// + /// # Parameters + /// + /// * `f` - The formatter to write to + /// * `before` - The tokens from the first file that were removed + /// * `after` - The tokens from the second file that were added + fn display_hunk(&self, f: impl fmt::Write, before: &[Token], after: &[Token]) -> fmt::Result; +} + +/// Configuration options for unified diff output. +/// +/// Controls aspects of the unified diff format such as the number of context lines. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnifiedDiffConfig { + /// The number of unchanged lines to show around each hunk for context. + context_len: u32, +} + +impl Default for UnifiedDiffConfig { + fn default() -> Self { + UnifiedDiffConfig { context_len: 3 } + } +} + +impl UnifiedDiffConfig { + /// Sets the number of context lines to display around each hunk. + /// + /// # Parameters + /// + /// * `len` - The number of unchanged lines to show before and after each change + /// + /// # Returns + /// + /// A mutable reference to self for method chaining + pub fn context_len(&mut self, len: u32) -> &mut Self { + self.context_len = len; + self + } +} + +/// A helper trait for determining if a token ends with a newline. +/// +/// This is used by the unified diff printer to decide whether to add newlines +/// when displaying tokens. +pub trait EndsWithNewline { + /// Returns `true` if the token ends with a newline character. + fn ends_with_newline(&self) -> bool; +} + +impl + ?Sized> EndsWithNewline for T { + fn ends_with_newline(&self) -> bool { + self.as_ref().ends_with(b"\n") + } +} + +/// A basic implementation of [`UnifiedDiffPrinter`] for line-based diffs. +/// +/// This printer formats diffs in the standard unified diff format commonly used by +/// tools like `git diff` and `diff -u`. It displays removed lines with a `-` prefix +/// and added lines with a `+` prefix. +pub struct BasicLineDiffPrinter<'a, T: EndsWithNewline + ?Sized + Hash + Eq + Display>( + /// A reference to the interner containing the line data. + pub &'a Interner<&'a T>, +); + +impl UnifiedDiffPrinter for BasicLineDiffPrinter<'_, T> { + fn display_header( + &self, + mut f: impl fmt::Write, + start_before: u32, + start_after: u32, + len_before: u32, + len_after: u32, + ) -> fmt::Result { + writeln!( + f, + "@@ -{},{} +{},{} @@", + start_before + 1, + len_before, + start_after + 1, + len_after + ) + } + + fn display_context_token(&self, mut f: impl fmt::Write, token: Token) -> fmt::Result { + write!(f, " {}", &self.0[token])?; + if !&self.0[token].ends_with_newline() { + writeln!(f)?; + } + Ok(()) + } + + fn display_hunk(&self, mut f: impl fmt::Write, before: &[Token], after: &[Token]) -> fmt::Result { + if let Some(&last) = before.last() { + for &token in before { + let token = self.0[token]; + write!(f, "-{token}")?; + } + if !self.0[last].ends_with_newline() { + writeln!(f)?; + } + } + if let Some(&last) = after.last() { + for &token in after { + let token = self.0[token]; + write!(f, "+{token}")?; + } + if !self.0[last].ends_with_newline() { + writeln!(f)?; + } + } + Ok(()) + } +} + +/// A formatted unified diff that can be displayed as a string. +/// +/// This structure is created by [`Diff::unified_diff`] or [`Diff::unified_diff_with`] +/// and implements [`Display`] to produce standard unified diff output. +pub struct UnifiedDiff<'a, P: UnifiedDiffPrinter> { + /// The printer that controls output formatting. + printer: &'a P, + /// The computed diff to display. + diff: &'a Diff, + /// Configuration for the unified diff format. + config: UnifiedDiffConfig, + /// The token sequence from the first file, before changes. + before: &'a [Token], + /// The token sequence from the second file, after changes. + after: &'a [Token], +} + +impl Display for UnifiedDiff<'_, P> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let first_hunk = self.diff.hunks().next().unwrap_or_default(); + let context_len = self.config.context_len.min(1024 * 1024); + let mut pos = first_hunk.before.start.saturating_sub(context_len); + let mut before_context_start = pos; + let mut after_context_start = first_hunk.after.start.saturating_sub(context_len); + let mut before_context_len = 0; + let mut after_context_len = 0; + let mut buffer = String::new(); + for hunk in self.diff.hunks() { + if hunk.before.start - pos > 2 * context_len { + if !buffer.is_empty() { + let end = (pos + context_len).min(self.before.len() as u32); + self.printer.display_header( + &mut *f, + before_context_start, + after_context_start, + before_context_len + end - pos, + after_context_len + end - pos, + )?; + write!(f, "{buffer}")?; + for &token in &self.before[pos as usize..end as usize] { + self.printer.display_context_token(&mut *f, token)?; + } + buffer.clear(); + } + pos = hunk.before.start - context_len; + before_context_start = pos; + after_context_start = hunk.after.start - context_len; + before_context_len = 0; + after_context_len = 0; + } + for &token in &self.before[pos as usize..hunk.before.start as usize] { + self.printer.display_context_token(&mut buffer, token)?; + } + let context_len = hunk.before.start - pos; + before_context_len += hunk.before.len() as u32 + context_len; + after_context_len += hunk.after.len() as u32 + context_len; + self.printer.display_hunk( + &mut buffer, + &self.before[hunk.before.start as usize..hunk.before.end as usize], + &self.after[hunk.after.start as usize..hunk.after.end as usize], + )?; + pos = hunk.before.end; + } + if !buffer.is_empty() { + let end = (pos + context_len).min(self.before.len() as u32); + self.printer.display_header( + &mut *f, + before_context_start, + after_context_start, + before_context_len + end - pos, + after_context_len + end - pos, + )?; + write!(f, "{buffer}")?; + for &token in &self.before[pos as usize..end as usize] { + self.printer.display_context_token(&mut *f, token)?; + } + buffer.clear(); + } + Ok(()) + } +} diff --git a/knot2/third_party/gix-imara-diff/src/util.rs b/knot2/third_party/gix-imara-diff/src/util.rs new file mode 100644 index 0000000000000000000000000000000000000000..5f77225d6fe3e3e35525ebe56e6e490824c31a29 --- /dev/null +++ b/knot2/third_party/gix-imara-diff/src/util.rs @@ -0,0 +1,135 @@ +// Modified for gitoxide from the upstream imara-diff crate. +// Upstream source: git cat-file -p 32d1e45d3df061e6ccba6db7fdce92db29e345d8:src/util.rs + +use crate::Hunk; +use crate::intern::Token; + +/// Computes the number of common tokens at the start of two sequences. +pub fn common_prefix(file1: &[Token], file2: &[Token]) -> u32 { + let mut off = 0; + for (token1, token2) in file1.iter().zip(file2) { + if token1 != token2 { + break; + } + off += 1; + } + off +} + +/// Computes the number of common tokens at the end of two sequences. +pub fn common_postfix(file1: &[Token], file2: &[Token]) -> u32 { + let mut off = 0; + for (token1, token2) in file1.iter().rev().zip(file2.iter().rev()) { + if token1 != token2 { + break; + } + off += 1; + } + off +} + +/// Computes both the common prefix and postfix lengths of two sequences. +pub fn common_edges(file1: &[Token], file2: &[Token]) -> (u32, u32) { + let prefix = common_prefix(file1, file2); + let postfix = common_postfix(&file1[prefix as usize..], &file2[prefix as usize..]); + (prefix, postfix) +} + +/// Removes the common prefix from both sequences and returns its length. +pub fn strip_common_prefix(file1: &mut &[Token], file2: &mut &[Token]) -> u32 { + let off = common_prefix(file1, file2); + *file1 = &file1[off as usize..]; + *file2 = &file2[off as usize..]; + off +} + +/// Removes the common postfix from both sequences and returns its length. +pub fn strip_common_postfix(file1: &mut &[Token], file2: &mut &[Token]) -> u32 { + let off = common_postfix(file1, file2); + *file1 = &file1[..file1.len() - off as usize]; + *file2 = &file2[..file2.len() - off as usize]; + off +} + +/// Computes an approximation of the square root using bit operations, rounding up like git's `xdl_bogosqrt()`. +pub fn sqrt(val: usize) -> u32 { + let nbits = (usize::BITS - val.leading_zeros()).div_ceil(2); + 1 << nbits +} + +impl Hunk { + pub(crate) fn next_hunk(&mut self, removed: &[bool], added: &[bool]) -> bool { + let Some(off) = find_next_change(added, self.after.end) else { + return false; + }; + let mut off_before = 0; + loop { + debug_assert!( + removed.len() as u32 != self.before.end || off == 0, + "broken hunk alignment {self:?} " + ); + let unchanged_tokens = + find_next_change(removed, self.before.end).unwrap_or(removed.len() as u32 - self.before.end); + if off_before + unchanged_tokens > off { + self.before.start = self.before.end + (off - off_before); + self.before.end = self.before.start; + break; + } + off_before += unchanged_tokens; + self.before.start = self.before.end + unchanged_tokens; + self.before.end = find_hunk_end(removed, self.before.end + unchanged_tokens); + if off_before == off { + break; + } + } + self.after.start = self.after.end + off; + self.after.end = find_hunk_end(added, self.after.start); + true + } +} + +/// Finds the offset to the next changed token starting from the given position. +pub fn find_next_change(changes: &[bool], pos: u32) -> Option { + changes[pos as usize..] + .iter() + .position(|&changed| changed) + .map(|off| off as u32) +} + +/// Finds the end position of a hunk of changed tokens starting from the given position. +pub fn find_hunk_end(changes: &[bool], pos: u32) -> u32 { + pos + changes[pos as usize..].iter().take_while(|&&changed| changed).count() as u32 +} + +/// Finds the start position of a hunk of changed tokens ending at the given position. +pub fn find_hunk_start(changes: &[bool], pos: u32) -> u32 { + pos - changes[..pos as usize] + .iter() + .rev() + .take_while(|&&changed| changed) + .count() as u32 +} + +#[cfg(test)] +mod tests { + use super::sqrt; + + fn bogosqrt(mut n: usize) -> u32 { + let mut i = 1; + while n > 0 { + i <<= 1; + n >>= 2; + } + i + } + + #[test] + fn sqrt_rounds_up_like_xdl_bogosqrt() { + for (n, root) in [(0, 1), (1, 2), (3, 2), (4, 4), (15, 4), (16, 8), (63, 8), (64, 16)] { + assert_eq!(sqrt(n), root, "sqrt({n})"); + } + for n in 0..1 << 16 { + assert_eq!(sqrt(n), bogosqrt(n), "sqrt({n})"); + } + } +} diff --git a/knot2/third_party/gix-pack/.cargo_vcs_info.json b/knot2/third_party/gix-pack/.cargo_vcs_info.json new file mode 100644 index 0000000000000000000000000000000000000000..29449536d1269fb9aa5245cb2f22941a71ed3015 --- /dev/null +++ b/knot2/third_party/gix-pack/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "10c58bb56597d9335611da121aac21f9b09b6e5b" + }, + "path_in_vcs": "gix-pack" +} \ No newline at end of file diff --git a/knot2/third_party/gix-pack/Cargo.toml b/knot2/third_party/gix-pack/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..dd0ebff6b5345e3845103e0f5ccffc6280aabcd2 --- /dev/null +++ b/knot2/third_party/gix-pack/Cargo.toml @@ -0,0 +1,238 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.85" +name = "gix-pack" +version = "0.71.0" +authors = ["Sebastian Thiel "] +build = false +include = [ + "/src/**/*", + "/LICENSE-*", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Implements git packs and related data structures" +readme = false +license = "MIT OR Apache-2.0" +repository = "https://github.com/GitoxideLabs/gitoxide" +resolver = "2" + +[package.metadata.docs.rs] +all-features = true +features = [ + "sha1", + "sha256", + "document-features", + "pack-cache-lru-dynamic", + "object-cache-dynamic", + "serde", +] + +[features] +default = [ + "generate", + "streaming-input", +] +generate = [ + "dep:gix-traverse", + "dep:gix-diff", + "dep:parking_lot", + "dep:gix-hashtable", +] +object-cache-dynamic = [ + "dep:clru", + "dep:gix-hashtable", +] +pack-cache-lru-dynamic = ["dep:clru"] +pack-cache-lru-static = ["dep:uluru"] +parallel = ["gix-features/parallel"] +serde = [ + "dep:serde", + "gix-object/serde", +] +sha1 = ["gix-hash/sha1"] +sha256 = ["gix-hash/sha256"] +streaming-input = [ + "dep:parking_lot", + "dep:gix-tempfile", +] +wasm = ["gix-diff?/wasm"] + +[lib] +name = "gix_pack" +path = "src/lib.rs" +doctest = false + +[dependencies.clru] +version = "0.6.1" +optional = true + +[dependencies.document-features] +version = "0.2.0" +optional = true + +[dependencies.gix-chunk] +version = "^0.7.2" + +[dependencies.gix-diff] +version = "^0.64.0" +optional = true +default-features = false + +[dependencies.gix-error] +version = "^0.2.4" + +[dependencies.gix-features] +version = "^0.48.1" +features = [ + "crc32", + "progress", + "zlib", +] + +[dependencies.gix-hash] +version = "^0.25.1" + +[dependencies.gix-hashtable] +version = "^0.15.1" +optional = true + +[dependencies.gix-object] +version = "^0.61.0" + +[dependencies.gix-path] +version = "^0.12.1" + +[dependencies.gix-traverse] +version = "^0.58.0" +optional = true + +[dependencies.memmap2] +version = "0.9.10" + +[dependencies.parking_lot] +version = "0.12.4" +optional = true +default-features = false + +[dependencies.serde] +version = "1.0.114" +features = ["derive"] +optional = true +default-features = false + +[dependencies.smallvec] +version = "1.15.1" + +[dependencies.thiserror] +version = "2.0.18" + +[dependencies.uluru] +version = "3.0.0" +optional = true + +[dev-dependencies.bstr] +version = "1.12.0" +features = ["std"] +default-features = false + +[dev-dependencies.maplit] +version = "1.0.2" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.gix-tempfile] +version = "^23.0.0" +optional = true +default-features = false + +[lints.clippy] +bool_to_int_with_if = "allow" +borrow_as_ptr = "allow" +cast_lossless = "allow" +cast_possible_truncation = "allow" +cast_possible_wrap = "allow" +cast_precision_loss = "allow" +cast_sign_loss = "allow" +checked_conversions = "allow" +copy_iterator = "allow" +default_trait_access = "allow" +doc_markdown = "allow" +empty_docs = "allow" +enum_glob_use = "allow" +explicit_deref_methods = "allow" +explicit_into_iter_loop = "allow" +explicit_iter_loop = "allow" +filter_map_next = "allow" +fn_params_excessive_bools = "allow" +from_iter_instead_of_collect = "allow" +if_not_else = "allow" +ignored_unit_patterns = "allow" +implicit_clone = "allow" +inconsistent_struct_constructor = "allow" +inefficient_to_string = "allow" +inline_always = "allow" +items_after_statements = "allow" +iter_not_returning_iterator = "allow" +iter_without_into_iter = "allow" +large_enum_variant = "allow" +large_stack_arrays = "allow" +manual_assert = "allow" +manual_is_variant_and = "allow" +manual_let_else = "allow" +manual_string_new = "allow" +many_single_char_names = "allow" +match_bool = "allow" +match_same_arms = "allow" +match_wild_err_arm = "allow" +match_wildcard_for_single_variants = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +module_name_repetitions = "allow" +must_use_candidate = "allow" +mut_mut = "allow" +naive_bytecount = "allow" +needless_continue = "allow" +needless_for_each = "allow" +needless_pass_by_value = "allow" +needless_raw_string_hashes = "allow" +no_effect_underscore_binding = "allow" +option_option = "allow" +range_plus_one = "allow" +redundant_else = "allow" +result_large_err = "allow" +return_self_not_must_use = "allow" +should_panic_without_expect = "allow" +similar_names = "allow" +single_match_else = "allow" +stable_sort_primitive = "allow" +struct_excessive_bools = "allow" +struct_field_names = "allow" +too_long_first_doc_paragraph = "allow" +too_many_lines = "allow" +transmute_ptr_to_ptr = "allow" +trivially_copy_pass_by_ref = "allow" +unnecessary_join = "allow" +unnecessary_wraps = "allow" +unreadable_literal = "allow" +unused_self = "allow" +used_underscore_binding = "allow" +wildcard_imports = "allow" + +[lints.clippy.pedantic] +level = "warn" +priority = -1 + +[lints.rust] diff --git a/knot2/third_party/gix-pack/LICENSE-APACHE b/knot2/third_party/gix-pack/LICENSE-APACHE new file mode 100644 index 0000000000000000000000000000000000000000..f433b1a53f5b830a205fd2df78e2b34974656c7b --- /dev/null +++ b/knot2/third_party/gix-pack/LICENSE-APACHE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/knot2/third_party/gix-pack/LICENSE-MIT b/knot2/third_party/gix-pack/LICENSE-MIT new file mode 100644 index 0000000000000000000000000000000000000000..969d061e8ba2e38d69391910b1ef0d4869ff18d1 --- /dev/null +++ b/knot2/third_party/gix-pack/LICENSE-MIT @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/knot2/third_party/gix-pack/src/bundle/find.rs b/knot2/third_party/gix-pack/src/bundle/find.rs new file mode 100644 index 0000000000000000000000000000000000000000..3a4b413afffed6d1b03f8c041b3fe170d7c65f10 --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/find.rs @@ -0,0 +1,75 @@ +use gix_features::zlib; + +impl crate::Bundle { + /// Find an object with the given [`ObjectId`](gix_hash::ObjectId) and place its data into `out`. + /// `inflate` is used to decompress objects, and will be reset before first use, but not after the last use. + /// + /// [`cache`](crate::cache::DecodeEntry) is used to accelerate the lookup. + /// + /// **Note** that ref deltas are automatically resolved within this pack only, which makes this implementation unusable + /// for thin packs, which by now are expected to be resolved already. + pub fn find<'a>( + &self, + id: &gix_hash::oid, + out: &'a mut Vec, + inflate: &mut zlib::Inflate, + cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result< + Option<(gix_object::Data<'a>, crate::data::entry::Location)>, + crate::data::decode::Error, + > { + let idx = match self.index.lookup(id) { + Some(idx) => idx, + None => return Ok(None), + }; + self.get_object_by_index(idx, out, inflate, cache).map(Some) + } + + /// Special-use function to get an object given an index previously returned from + /// [index::File::](crate::index::File::lookup()). + /// `inflate` is used to decompress objects, and will be reset before first use, but not after the last use. + /// + /// # Panics + /// + /// If `index` is out of bounds. + pub fn get_object_by_index<'a>( + &self, + idx: u32, + out: &'a mut Vec, + inflate: &mut zlib::Inflate, + cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result<(gix_object::Data<'a>, crate::data::entry::Location), crate::data::decode::Error> + { + let ofs = self.index.pack_offset_at_index(idx); + let pack_entry = self.pack.entry(ofs)?; + let header_size = pack_entry.header_size(); + self.pack + .decode_entry( + pack_entry, + out, + inflate, + &|id, _out| { + let idx = self.index.lookup(id)?; + self.pack + .entry(self.index.pack_offset_at_index(idx)) + .ok() + .map(crate::data::decode::entry::ResolvedBase::InPack) + }, + cache, + ) + .map(move |r| { + ( + gix_object::Data { + kind: r.kind, + data: out.as_slice(), + object_hash: self.pack.object_hash(), + }, + crate::data::entry::Location { + pack_id: self.pack.id, + pack_offset: ofs, + entry_size: r.compressed_size + header_size, + }, + ) + }) + } +} diff --git a/knot2/third_party/gix-pack/src/bundle/init.rs b/knot2/third_party/gix-pack/src/bundle/init.rs new file mode 100644 index 0000000000000000000000000000000000000000..bde90bde13bd004b2f5ede07f97a86734cc7102f --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/init.rs @@ -0,0 +1,46 @@ +use std::path::{Path, PathBuf}; + +use crate::Bundle; + +/// Returned by [`Bundle::at()`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("An 'idx' extension is expected of an index file: '{0}'")] + InvalidPath(PathBuf), + #[error(transparent)] + Pack(#[from] crate::data::header::decode::Error), + #[error(transparent)] + Index(#[from] crate::index::init::Error), +} + +/// Initialization +impl Bundle { + /// Create a `Bundle` from `path`, which is either a pack file _(*.pack)_ or an index file _(*.idx)_. + /// + /// The corresponding complementary file is expected to be present. + /// + /// The `object_hash` is a way to read (and write) the same file format with different hashes, as the hash kind + /// isn't stored within the file format itself. + pub fn at(path: impl AsRef, object_hash: gix_hash::Kind) -> Result { + Self::at_inner(path.as_ref(), object_hash) + } + + fn at_inner(path: &Path, object_hash: gix_hash::Kind) -> Result { + let ext = path + .extension() + .and_then(std::ffi::OsStr::to_str) + .ok_or_else(|| Error::InvalidPath(path.to_owned()))?; + Ok(match ext { + "idx" => Self { + index: crate::index::File::at(path, object_hash)?, + pack: crate::data::File::at(path.with_extension("pack"), object_hash)?, + }, + "pack" => Self { + pack: crate::data::File::at(path, object_hash)?, + index: crate::index::File::at(path.with_extension("idx"), object_hash)?, + }, + _ => return Err(Error::InvalidPath(path.to_owned())), + }) + } +} diff --git a/knot2/third_party/gix-pack/src/bundle/mod.rs b/knot2/third_party/gix-pack/src/bundle/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..63628c64939dddf7ce51421b364d3b8429f8e6d9 --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/mod.rs @@ -0,0 +1,59 @@ +/// +pub mod init; + +mod find; +/// +#[cfg(all(not(feature = "wasm"), feature = "streaming-input"))] +pub mod write; + +/// +pub mod verify { + use std::sync::atomic::AtomicBool; + + use gix_features::progress::DynNestedProgress; + + /// + pub mod integrity { + /// Returned by [`Bundle::verify_integrity()`][crate::Bundle::verify_integrity()]. + pub struct Outcome { + /// The computed checksum of the index which matched the stored one. + pub actual_index_checksum: gix_hash::ObjectId, + /// The packs traversal outcome + pub pack_traverse_outcome: crate::index::traverse::Statistics, + } + } + + use crate::Bundle; + + impl Bundle { + /// Similar to [`crate::index::File::verify_integrity()`] but more convenient to call as the presence of the + /// pack file is a given. + pub fn verify_integrity( + &self, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + options: crate::index::verify::integrity::Options, + ) -> Result< + integrity::Outcome, + crate::index::traverse::Error, + > + where + C: crate::cache::DecodeEntry, + F: Fn() -> C + Send + Clone, + { + self.index + .verify_integrity( + Some(crate::index::verify::PackContext { + data: &self.pack, + options, + }), + progress, + should_interrupt, + ) + .map(|o| integrity::Outcome { + actual_index_checksum: o.actual_index_checksum, + pack_traverse_outcome: o.pack_traverse_statistics.expect("pack is set"), + }) + } + } +} diff --git a/knot2/third_party/gix-pack/src/bundle/write/error.rs b/knot2/third_party/gix-pack/src/bundle/write/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..883c34029ac1a3d5b4cde169a1289844e8c380e6 --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/write/error.rs @@ -0,0 +1,17 @@ +use std::io; + +use gix_tempfile::handle::Writable; + +/// The error returned by [`Bundle::write_to_directory()`][crate::Bundle::write_to_directory()] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("An IO error occurred when reading the pack or creating a temporary file")] + Io(#[from] io::Error), + #[error(transparent)] + PackIter(#[from] crate::data::input::Error), + #[error("Could not move a temporary file into its desired place")] + Persist(#[from] gix_tempfile::handle::persist::Error), + #[error(transparent)] + IndexWrite(#[from] crate::index::write::Error), +} diff --git a/knot2/third_party/gix-pack/src/bundle/write/mod.rs b/knot2/third_party/gix-pack/src/bundle/write/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..93eb80caa8c6905d84b258a6dfbf7fc905868f75 --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/write/mod.rs @@ -0,0 +1,444 @@ +use std::{ + io, + io::Write, + marker::PhantomData, + path::{Path, PathBuf}, + sync::{Arc, atomic::AtomicBool}, +}; + +use gix_features::{interrupt, progress, progress::Progress}; +use gix_tempfile::{AutoRemove, ContainingDirectory}; + +use crate::data; + +mod error; +pub use error::Error; +use gix_features::progress::prodash::DynNestedProgress; + +mod types; +use types::PassThrough; +pub use types::{Options, Outcome}; + +use crate::bundle::write::types::SharedTempFile; + +/// The progress ids used in [`write_to_directory()`][crate::Bundle::write_to_directory()]. +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// The amount of bytes read from the input pack data file. + ReadPackBytes, + /// A root progress counting logical steps towards an index file on disk. + /// + /// Underneath will be more progress information related to actually producing the index. + IndexingSteps(PhantomData), +} + +impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::ReadPackBytes => *b"BWRB", + ProgressId::IndexingSteps(_) => *b"BWCI", + } + } +} + +impl crate::Bundle { + /// Given a `pack` data stream, write it along with a generated index into the `directory` if `Some` or discard all output if `None`. + /// + /// In the latter case, the functionality provided here is more a kind of pack data stream validation. + /// + /// * `progress` provides detailed progress information which can be discarded with [`gix_features::progress::Discard`]. + /// * `should_interrupt` is checked regularly and when true, the whole operation will stop. + /// * `thin_pack_base_object_lookup` If set, we expect to see a thin-pack with objects that reference their base object by object id which is + /// expected to exist in the object database the bundle is contained within. + /// `options` further configure how the task is performed. + /// + /// # Note + /// + /// * the resulting pack may be empty, that is, contains zero objects in some situations. This is a valid reply by a server and should + /// be accounted for. + /// - Empty packs always have the same name and not handling this case will result in at most one superfluous pack. + pub fn write_to_directory( + pack: &mut dyn io::BufRead, + directory: Option<&Path>, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + thin_pack_base_object_lookup: Option, + options: Options, + ) -> Result { + let _span = gix_features::trace::coarse!("gix_pack::Bundle::write_to_directory()"); + let mut read_progress = + progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); + read_progress.init(None, progress::bytes()); + let pack = progress::Read { + inner: pack, + progress: progress::ThroughputOnDrop::new(read_progress), + }; + + let object_hash = options.object_hash; + let data_file = Arc::new(parking_lot::Mutex::new(io::BufWriter::with_capacity( + 64 * 1024, + match directory.as_ref() { + Some(directory) => { + gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)? + } + None => gix_tempfile::new( + std::env::temp_dir(), + ContainingDirectory::Exists, + AutoRemove::Tempfile, + )?, + }, + ))); + let pack = PassThrough { + reader: interrupt::Read { + inner: pack, + should_interrupt, + }, + writer: Some(data_file.clone()), + }; + // This buf-reader is required to assure we call 'read()' in order to fill the (extra) buffer. Otherwise all the counting + // we do with the wrapped pack reader doesn't work as it does not expect anyone to call BufRead functions directly. + // However, this is exactly what's happening in the ZipReader implementation that is eventually used. + // The performance impact of this is probably negligible, compared to all the other work that is done anyway :D. + let buffered_pack = io::BufReader::new(pack); + let pack_entries_iter = data::input::BytesToEntriesIter::new_from_header( + buffered_pack, + options.iteration_mode, + data::input::EntryDataMode::Crc32, + object_hash, + )?; + let pack_version = pack_entries_iter.version(); + let WriteOutcome { + outcome, + data_path, + index_path, + keep_path, + } = crate::Bundle::inner_write( + directory, + progress, + options, + data_file, + Box::new(pack_entries_iter), + should_interrupt, + pack_version, + thin_pack_base_object_lookup, + )?; + + Ok(Outcome { + index: outcome, + object_hash, + pack_version, + data_path, + index_path, + keep_path, + }) + } + + /// Equivalent to [`write_to_directory()`][crate::Bundle::write_to_directory()] but offloads reading of the pack into its own thread, hence the `Send + 'static'` bounds. + /// + /// # Note + /// + /// As it sends portions of the input to a thread it requires the 'static lifetime for the interrupt flags. This can only + /// be satisfied by a static `AtomicBool` which is only suitable for programs that only run one of these operations at a time + /// or don't mind that all of them abort when the flag is set. + pub fn write_to_directory_eagerly( + pack: Box, + pack_size: Option, + directory: Option>, + progress: &mut dyn DynNestedProgress, + should_interrupt: &'static AtomicBool, + thin_pack_base_object_lookup: Option, + options: Options, + ) -> Result { + let _span = gix_features::trace::coarse!("gix_pack::Bundle::write_to_directory_eagerly()"); + let mut read_progress = + progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); /* Bundle Write Read pack Bytes*/ + read_progress.init(pack_size.map(|s| s as usize), progress::bytes()); + let pack = progress::Read { + inner: pack, + progress: progress::ThroughputOnDrop::new(read_progress), + }; + + let data_file = Arc::new(parking_lot::Mutex::new(io::BufWriter::new( + match directory.as_ref() { + Some(directory) => { + gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)? + } + None => gix_tempfile::new( + std::env::temp_dir(), + ContainingDirectory::Exists, + AutoRemove::Tempfile, + )?, + }, + ))); + let object_hash = options.object_hash; + let eight_pages = 4096 * 8; + let pack = PassThrough { + reader: interrupt::Read { + inner: pack, + should_interrupt, + }, + writer: Some(data_file.clone()), + }; + let buffered_pack = io::BufReader::with_capacity(eight_pages, pack); + let pack_entries_iter = data::input::BytesToEntriesIter::new_from_header( + buffered_pack, + options.iteration_mode, + data::input::EntryDataMode::Crc32, + object_hash, + )?; + let pack_version = pack_entries_iter.version(); + let num_objects = pack_entries_iter.size_hint().0; + let pack_entries_iter = gix_features::parallel::EagerIterIf::new( + move || num_objects > 25_000, + pack_entries_iter, + 5_000, + 5, + ); + + let WriteOutcome { + outcome, + data_path, + index_path, + keep_path, + } = crate::Bundle::inner_write( + directory, + progress, + options, + data_file, + Box::new(pack_entries_iter), + should_interrupt, + pack_version, + thin_pack_base_object_lookup, + )?; + + Ok(Outcome { + index: outcome, + object_hash, + pack_version, + data_path, + index_path, + keep_path, + }) + } + + fn inner_write<'a>( + directory: Option>, + progress: &mut dyn DynNestedProgress, + Options { + thread_limit, + iteration_mode: _, + index_version: index_kind, + object_hash, + }: Options, + data_file: SharedTempFile, + mut pack_entries_iter: Box< + dyn Iterator> + 'a, + >, + should_interrupt: &AtomicBool, + pack_version: data::Version, + thin_pack_base_object_lookup: Option, + ) -> Result { + let mut complete_thin_pack = thin_pack_base_object_lookup.map(|find| { + let data_file = Arc::clone(&data_file); + let mut thin = None; + move |ids: &[gix_hash::ObjectId]| { + append_bases(&data_file, &find, ids, object_hash, &mut thin) + } + }); + let mut indexing_progress = progress.add_child_with_id( + "create index file".into(), + ProgressId::IndexingSteps(Default::default()).into(), + ); + Ok(match directory { + Some(directory) => { + let directory = directory.as_ref(); + let mut index_file = gix_tempfile::new( + directory, + ContainingDirectory::Exists, + AutoRemove::Tempfile, + )?; + + let outcome = crate::index::write_data_iter_to_stream( + index_kind, + { + let data_file = Arc::clone(&data_file); + move || new_pack_file_resolver(&data_file, object_hash) + }, + &mut pack_entries_iter, + thread_limit, + &mut indexing_progress, + &mut index_file, + should_interrupt, + object_hash, + pack_version, + complete_thin_pack + .as_mut() + .map(|f| f as crate::index::write::CompleteThinPack<'_>), + )?; + drop(pack_entries_iter); + drop(complete_thin_pack); + + if outcome.num_objects == 0 { + WriteOutcome { + outcome, + data_path: None, + index_path: None, + keep_path: None, + } + } else { + let data_path = + directory.join(format!("pack-{}.pack", outcome.data_hash.to_hex())); + let index_path = data_path.with_extension("idx"); + let keep_path = if data_path.is_file() { + // avoid trying to overwrite existing files, we know they have the same content + // and this is likely to fail on Windows as negotiation opened the pack. + None + } else { + let keep_path = data_path.with_extension("keep"); + + std::fs::write(&keep_path, b"")?; + Arc::try_unwrap(data_file) + .expect("only one handle left after pack was consumed") + .into_inner() + .into_inner() + .map_err(|err| Error::from(err.into_error()))? + .persist(&data_path)?; + Some(keep_path) + }; + if !index_path.is_file() { + index_file + .persist(&index_path) + .inspect_err(|_err| { + gix_features::trace::warn!("pack file at \"{}\" is retained despite failing to move the index file into place. You can use plumbing to make it usable.",data_path.display()); + })?; + } + WriteOutcome { + outcome, + data_path: Some(data_path), + index_path: Some(index_path), + keep_path, + } + } + } + None => WriteOutcome { + outcome: crate::index::write_data_iter_to_stream( + index_kind, + move || new_pack_file_resolver(&data_file, object_hash), + &mut pack_entries_iter, + thread_limit, + &mut indexing_progress, + &mut io::sink(), + should_interrupt, + object_hash, + pack_version, + complete_thin_pack + .as_mut() + .map(|f| f as crate::index::write::CompleteThinPack<'_>), + )?, + data_path: None, + index_path: None, + keep_path: None, + }, + }) + } +} + +fn append_bases( + data_file: &SharedTempFile, + find: &impl gix_object::Find, + ids: &[gix_hash::ObjectId], + object_hash: gix_hash::Kind, + thin: &mut Option<(u64, [u8; 12])>, +) -> Result> +{ + use std::os::unix::fs::FileExt; + + let mut guard = data_file.lock(); + guard.flush()?; + let file = guard.get_mut().with_mut(|f| f.as_file().try_clone())??; + let (thin_end, header) = match *thin { + Some(thin) => thin, + None => { + let end = file + .metadata()? + .len() + .checked_sub(object_hash.len_in_bytes() as u64) + .ok_or("pack is shorter than its trailer")?; + let mut header = [0u8; 12]; + file.read_exact_at(&mut header, 0)?; + *thin.insert((end, header)) + } + }; + let mut end = thin_end; + + let mut buf = Vec::new(); + let mut bytes = Vec::new(); + let mut entries = Vec::with_capacity(ids.len()); + for id in ids { + let Some(object) = find.try_find(id, &mut buf)? else { + continue; + }; + let entry = data::input::Entry::from_data_obj(&object, end)?; + bytes.clear(); + entry.header.write_to(entry.decompressed_size, &mut bytes)?; + bytes.extend_from_slice(entry.compressed.as_deref().expect("set for new entries")); + file.write_all_at(&bytes, end)?; + entries.push((*id, end, entry.crc32.expect("set for new entries"))); + end += bytes.len() as u64; + } + + let (version, num_objects) = data::header::decode(&header)?; + let num_objects = u32::try_from(entries.len()) + .ok() + .and_then(|appended| num_objects.checked_add(appended)) + .ok_or("too many objects for one pack")?; + file.write_all_at(&data::header::encode(version, num_objects), 0)?; + + let mut hasher = gix_hash::hasher(object_hash); + let mut chunk = vec![0; 64 * 1024]; + let mut pos = 0; + while pos < end { + let len = chunk.len().min((end - pos) as usize); + file.read_exact_at(&mut chunk[..len], pos)?; + hasher.update(&chunk[..len]); + pos += len as u64; + } + let pack_hash = hasher.try_finalize()?; + file.write_all_at(pack_hash.as_slice(), end)?; + file.set_len(end + pack_hash.as_slice().len() as u64)?; + + Ok(crate::index::write::AppendedBases { + entries, + pack_entries_end: end, + pack_hash, + }) +} + +fn resolve_entry(range: data::EntryRange, pack: &crate::data::File, buf: &mut Vec) -> bool { + pack.read_into(range, buf) +} + +#[allow(clippy::type_complexity)] // cannot typedef impl Fn +fn new_pack_file_resolver( + data_file: &SharedTempFile, + object_hash: gix_hash::Kind, +) -> io::Result<( + impl Fn(data::EntryRange, &crate::data::File, &mut Vec) -> bool + Send + Clone + use<>, + crate::data::File, +)> { + let mut guard = data_file.lock(); + guard.flush()?; + let path = guard.get_mut().with_mut(|f| f.path().to_owned())?; + let pack = crate::data::File::at(&path, object_hash) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; + Ok((resolve_entry, pack)) +} + +struct WriteOutcome { + outcome: crate::index::write::Outcome, + data_path: Option, + index_path: Option, + keep_path: Option, +} diff --git a/knot2/third_party/gix-pack/src/bundle/write/types.rs b/knot2/third_party/gix-pack/src/bundle/write/types.rs new file mode 100644 index 0000000000000000000000000000000000000000..526390da751a9c01a300b26aa9d5f1b84569ae1d --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/write/types.rs @@ -0,0 +1,97 @@ +use std::{hash::Hash, io, path::PathBuf, sync::Arc}; + +use gix_tempfile::handle::Writable; + +/// Configuration for [`write_to_directory`][crate::Bundle::write_to_directory()] or +/// [`write_to_directory_eagerly`][crate::Bundle::write_to_directory_eagerly()] +#[derive(Debug, Clone)] +pub struct Options { + /// The amount of threads to use at most when resolving the pack. If `None`, all logical cores are used. + pub thread_limit: Option, + /// Determine how much processing to spend on protecting against corruption or recovering from errors. + pub iteration_mode: crate::data::input::Mode, + /// The version of pack index to write, should be [`crate::index::Version::default()`] + pub index_version: crate::index::Version, + /// The kind of hash to use when writing the bundle. + pub object_hash: gix_hash::Kind, +} + +impl Default for Options { + /// Options which favor speed and correctness and write the most commonly supported index file. + fn default() -> Self { + Options { + thread_limit: None, + iteration_mode: crate::data::input::Mode::Verify, + index_version: Default::default(), + object_hash: Default::default(), + } + } +} + +/// Returned by [`write_to_directory`][crate::Bundle::write_to_directory()] or +/// [`write_to_directory_eagerly`][crate::Bundle::write_to_directory_eagerly()] +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// The successful result of the index write operation. + pub index: crate::index::write::Outcome, + /// The version of the pack. + pub pack_version: crate::data::Version, + /// The kind of hash stored within the pack and indices. + pub object_hash: gix_hash::Kind, + + /// The path to the pack index file. + pub index_path: Option, + /// The path to the pack data file. + pub data_path: Option, + /// The path to the `.keep` file to prevent collection of the newly written pack until refs are pointing to it. + /// It might be `None` if the file at `data_path` already existed, indicating that we have received a pack that + /// was already present locally. + /// + /// The file is created right before moving the pack data and index data into place (i.e. `data_path` and `index_path`) + /// and is expected to be removed by the caller when ready. + pub keep_path: Option, +} + +impl Outcome { + /// Instantiate a bundle from the newly written index and data file that are represented by this `Outcome` + pub fn to_bundle(&self) -> Option> { + self.index_path + .as_ref() + .map(|path| crate::Bundle::at(path, self.object_hash)) + } +} + +pub(crate) type SharedTempFile = + Arc>>>; + +pub(crate) struct PassThrough { + pub reader: R, + pub writer: Option, +} + +impl io::Read for PassThrough +where + R: io::Read, +{ + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let bytes_read = self.reader.read(buf)?; + if let Some(writer) = self.writer.as_mut() { + use std::io::Write; + writer.lock().write_all(&buf[..bytes_read])?; + } + Ok(bytes_read) + } +} +impl io::BufRead for PassThrough +where + R: io::BufRead, +{ + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.reader.fill_buf() + } + + fn consume(&mut self, amt: usize) { + self.reader.consume(amt); + } +} diff --git a/knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs b/knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs new file mode 100644 index 0000000000000000000000000000000000000000..1b7a4aa89b0725f1f1cb6132a5e0110a5632cdef --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs @@ -0,0 +1,169 @@ +use std::{ + fs, io, + io::{BufRead, Read, Seek, SeekFrom}, + sync::atomic::{AtomicBool, Ordering}, + time::Instant, +}; + +use gix_features::progress::{self, Progress}; + +use crate::{cache::delta::Tree, data}; + +/// Returned by [`Tree::from_offsets_in_pack()`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("{message}")] + Io { + source: io::Error, + message: &'static str, + }, + #[error(transparent)] + Header(#[from] crate::data::header::decode::Error), + #[error("Could find object with id {id} in this pack. Thin packs are not supported")] + UnresolvedRefDelta { id: gix_hash::ObjectId }, + #[error(transparent)] + Tree(#[from] crate::cache::delta::Error), + #[error("Interrupted")] + Interrupted, +} + +const PACK_HEADER_LEN: usize = 12; + +/// Generate tree from certain input +impl Tree { + /// Create a new `Tree` from any data sorted by offset, ascending as returned by the `data_sorted_by_offsets` iterator. + /// * `get_pack_offset(item: &T) -> data::Offset` is a function returning the pack offset of the given item, which can be used + /// for obtaining the objects entry within the pack. + /// * `pack_path` is the path to the pack file itself and from which to read the entry data, which is a pack file matching the offsets + /// returned by `get_pack_offset(…)`. + /// * `progress` is used to track progress when creating the tree. + /// * `resolve_in_pack_id(gix_hash::oid) -> Option` takes an object ID and tries to resolve it to an object within this pack if + /// possible. Failing to do so aborts the operation, and this function is not expected to be called in usual packs. It's a theoretical + /// possibility though as old packs might have referred to their objects using the 20 bytes hash, instead of their encoded offset from the base. + /// + /// Note that the sort order is ascending. The given pack file path must match the provided offsets. + pub fn from_offsets_in_pack( + pack_path: &std::path::Path, + data_sorted_by_offsets: impl Iterator, + get_pack_offset: &dyn Fn(&T) -> data::Offset, + resolve_in_pack_id: &dyn Fn(&gix_hash::oid) -> Option, + progress: &mut dyn Progress, + should_interrupt: &AtomicBool, + object_hash: gix_hash::Kind, + ) -> Result { + let mut r = io::BufReader::with_capacity( + 8192 * 8, // this value directly corresponds to performance, 8k (default) is about 4x slower than 64k + fs::File::open(pack_path).map_err(|err| Error::Io { + source: err, + message: "open pack path", + })?, + ); + + let anticipated_num_objects = data_sorted_by_offsets + .size_hint() + .1 + .inspect(|&num_objects| { + progress.init(Some(num_objects), progress::count("objects")); + }) + .unwrap_or_default(); + let mut tree = Tree::with_capacity(anticipated_num_objects)?; + + { + // safety check - assure ourselves it's a pack we can handle + let mut buf = [0u8; PACK_HEADER_LEN]; + r.read_exact(&mut buf).map_err(|err| Error::Io { + source: err, + message: "reading header buffer with at least 12 bytes failed - pack file truncated?", + })?; + crate::data::header::decode(&buf)?; + } + + let then = Instant::now(); + + let mut previous_cursor_position = None::; + + let hash_len = object_hash.len_in_bytes(); + for (idx, data) in data_sorted_by_offsets.enumerate() { + let pack_offset = get_pack_offset(&data); + if let Some(previous_offset) = previous_cursor_position { + Self::advance_cursor_to_pack_offset(&mut r, pack_offset, previous_offset)?; + } + let entry = + crate::data::Entry::from_read(&mut r, pack_offset, hash_len).map_err(|err| { + Error::Io { + source: err, + message: "EOF while parsing header", + } + })?; + previous_cursor_position = Some(pack_offset + entry.header_size() as u64); + + use crate::data::entry::Header::*; + match entry.header { + Tree | Blob | Commit | Tag => { + tree.add_root(pack_offset, data)?; + } + RefDelta { base_id } => { + resolve_in_pack_id(base_id.as_ref()) + .ok_or(Error::UnresolvedRefDelta { id: base_id }) + .and_then(|base_pack_offset| { + tree.add_child(base_pack_offset, pack_offset, data) + .map_err(Into::into) + })?; + } + OfsDelta { base_distance } => { + let base_pack_offset = pack_offset + .checked_sub(base_distance) + .expect("in bound distance for deltas"); + tree.add_child(base_pack_offset, pack_offset, data)?; + } + } + progress.inc(); + if idx % 10_000 == 0 && should_interrupt.load(Ordering::SeqCst) { + return Err(Error::Interrupted); + } + } + + progress.show_throughput(then); + Ok(tree) + } + + fn advance_cursor_to_pack_offset( + r: &mut io::BufReader, + pack_offset: u64, + previous_offset: u64, + ) -> Result<(), Error> { + let bytes_to_skip: u64 = pack_offset + .checked_sub(previous_offset) + .expect("continuously ascending pack offsets"); + if bytes_to_skip == 0 { + return Ok(()); + } + let buf = r.fill_buf().map_err(|err| Error::Io { + source: err, + message: "skip bytes", + })?; + if buf.is_empty() { + // This means we have reached the end of file and can't make progress anymore, before we have satisfied our need + // for more + return Err(Error::Io { + source: io::Error::new( + io::ErrorKind::UnexpectedEof, + "ran out of bytes before reading desired amount of bytes", + ), + message: "index file is damaged or corrupt", + }); + } + if bytes_to_skip <= u64::try_from(buf.len()).expect("sensible buffer size") { + // SAFETY: bytes_to_skip <= buf.len() <= usize::MAX + r.consume(bytes_to_skip as usize); + } else { + r.seek(SeekFrom::Start(pack_offset)) + .map_err(|err| Error::Io { + source: err, + message: "seek to next entry", + })?; + } + Ok(()) + } +} diff --git a/knot2/third_party/gix-pack/src/cache/delta/mod.rs b/knot2/third_party/gix-pack/src/cache/delta/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..afd02910abe67b442a4a845919267f08f17bab9a --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/delta/mod.rs @@ -0,0 +1,25 @@ +/// Returned when using various methods on a [`Tree`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error( + "Pack offsets must only increment. The previous pack offset was {last_pack_offset}, the current one is {pack_offset}" + )] + InvariantIncreasingPackOffset { + /// The last seen pack offset + last_pack_offset: crate::data::Offset, + /// The invariant violating offset + pack_offset: crate::data::Offset, + }, +} + +/// +pub mod traverse; + +/// +pub mod from_offsets; + +/// Tree datastructure +mod tree; + +pub use tree::{Item, Tree}; diff --git a/knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs b/knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..e5dc6756fdeddb759b94dd72a00a270ef23bc129 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs @@ -0,0 +1,295 @@ +use std::os::unix::fs::FileExt; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use gix_features::{ + parallel::in_parallel_with_slice, + progress::{self, DynNestedProgress, Progress}, + threading, + threading::{Mutable, OwnShared}, +}; + +use crate::{ + cache::delta::{Item, Tree}, + data::EntryRange, +}; + +mod resolve; + +#[derive(Clone, Copy)] +pub(crate) struct SpillRef { + offset: u64, + len: usize, +} + +pub struct BaseSpill { + budget: usize, + resident: AtomicUsize, + peak_resident: AtomicUsize, + spilled_bytes: AtomicU64, + file: std::fs::File, + write_cursor: Mutex, +} + +impl BaseSpill { + pub fn new(file: std::fs::File, budget: usize) -> Self { + Self { + budget, + resident: AtomicUsize::new(0), + peak_resident: AtomicUsize::new(0), + spilled_bytes: AtomicU64::new(0), + file, + write_cursor: Mutex::new(0), + } + } + + pub fn peak_resident(&self) -> usize { + self.peak_resident.load(Ordering::Relaxed) + } + + pub fn spilled_bytes(&self) -> u64 { + self.spilled_bytes.load(Ordering::Relaxed) + } + + fn account_push(&self, len: usize) { + let now = self.resident.fetch_add(len, Ordering::Relaxed) + len; + self.peak_resident.fetch_max(now, Ordering::Relaxed); + } + + fn account_pop_resident(&self, len: usize) { + self.resident.fetch_sub(len, Ordering::Relaxed); + } + + fn over_budget(&self) -> bool { + self.resident.load(Ordering::Relaxed) > self.budget + } + + fn spill(&self, bytes: &[u8]) -> std::io::Result { + let len = bytes.len(); + let offset = { + let mut cursor = self + .write_cursor + .lock() + .expect("base spill cursor poisoned"); + let offset = *cursor; + *cursor += len as u64; + offset + }; + self.file.write_all_at(bytes, offset)?; + self.resident.fetch_sub(len, Ordering::Relaxed); + self.spilled_bytes.fetch_add(len as u64, Ordering::Relaxed); + Ok(SpillRef { offset, len }) + } + + fn reload(&self, sref: SpillRef, out: &mut Vec) -> std::io::Result<()> { + out.resize(sref.len, 0); + self.file.read_exact_at(out, sref.offset) + } +} + +/// Returned by [`Tree::traverse()`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("{message}")] + ZlibInflate { + source: gix_features::zlib::inflate::Error, + message: &'static str, + }, + #[error("The resolver failed to obtain the pack entry bytes for the entry at {pack_offset}")] + ResolveFailed { pack_offset: u64 }, + #[error(transparent)] + EntryType(#[from] crate::data::entry::decode::Error), + #[error("One of the object inspectors failed")] + Inspect(#[from] Box), + #[error("Interrupted")] + Interrupted, + #[error( + "The base at {base_pack_offset} was referred to by a ref-delta, but it was never added to the tree as if the pack was still thin." + )] + OutOfPackRefDelta { + /// The base's offset which was from a resolved ref-delta that didn't actually get added to the tree + base_pack_offset: crate::data::Offset, + }, + #[error("Failed to spawn thread when switching to work-stealing mode")] + SpawnThread(#[from] std::io::Error), + #[error("Failed to page a delta base to the memory-budget overflow file")] + BaseSpill { source: std::io::Error }, + #[error( + "The base object at {pack_offset} decoded as a delta, but base objects cannot be deltas" + )] + UnexpectedRootDelta { pack_offset: crate::data::Offset }, + #[error(transparent)] + Delta(#[from] crate::data::delta::apply::Error), + #[error(transparent)] + Hash(#[from] gix_hash::hasher::Error), + #[error( + "The delta at {delta_pack_offset} declared a base of {declared} bytes but the resolved base is {actual} bytes" + )] + BaseSizeMismatch { + delta_pack_offset: crate::data::Offset, + declared: u64, + actual: u64, + }, + #[error( + "The delta at {delta_pack_offset} declared a reconstructed size of {declared} bytes, over the {limit}-byte object limit" + )] + DeltaResultTooLarge { + delta_pack_offset: crate::data::Offset, + declared: u64, + limit: u64, + }, +} + +/// Additional context passed to the `inspect_object(…)` function of the [`Tree::traverse()`] method. +#[allow(missing_docs)] +pub struct Context<'a> { + /// The pack entry describing the object + pub entry: &'a crate::data::Entry, + /// The offset at which `entry` ends in the pack, useful to learn about the exact range of `entry` within the pack. + pub entry_end: u64, + /// The decompressed object itself, ready to be decoded. + pub decompressed: &'a [u8], + /// The depth at which this object resides in the delta-tree. It represents the number of base objects, with 0 indicating + /// an 'undeltified' object, and higher values indicating delta objects with the given number of bases. + pub level: u16, + pub object_kind: gix_object::Kind, +} + +/// Options for [`Tree::traverse()`]. +pub struct Options<'a, 's> { + /// is a progress instance to track progress for each object in the traversal. + pub object_progress: Box, + /// is a progress instance to track the overall progress. + pub size_progress: &'s mut dyn Progress, + /// If `Some`, only use the given number of threads. Otherwise, the number of threads to use will be selected based on + /// the number of available logical cores. + pub thread_limit: Option, + /// Abort the operation if the value is `true`. + pub should_interrupt: &'a AtomicBool, + /// specifies what kind of hashes we expect to be stored in oid-delta entries, which is viable to decoding them + /// with the correct size. + pub object_hash: gix_hash::Kind, + pub base_spill: Option>, + pub collect_items: bool, + pub max_object_bytes: Option, +} + +/// The outcome of [`Tree::traverse()`] +#[allow(missing_docs)] +pub struct Outcome { + pub items: Vec>, + /// Ref-deltas whose base wasn't in the pack, with everything based on them, ready to be traversed again once + /// their bases were added with [`Tree::add_root()`]. + pub unresolved: Option>, +} + +impl Tree +where + T: Send + Sync + Clone, +{ + /// Traverse this tree of delta objects with a function `inspect_object` to process each object at will. + /// + /// * `should_run_in_parallel() -> bool` returns true if the underlying pack is big enough to warrant parallel traversal at all. + /// * `resolve(EntrySlice, &R, &mut Vec) -> bool` reads the raw pack bytes for the given `EntrySlice` into the + /// output vector, reusing its allocation. It returns `true` if the object existed in the pack, or `false` to indicate a + /// resolution error, which aborts the operation. + /// * `pack_entries_end` marks one-past-the-last byte of the last entry in the pack, as the last entries size would otherwise + /// be unknown as it's not part of the index file. + /// * `inspect_object(node_data: &mut T, progress: Progress, context: Context) -> Result<(), CustomError>` is a function + /// running for each thread receiving fully decoded objects along with contextual information, which either succeeds with `Ok(())` + /// or returns a `CustomError`. + /// Note that `node_data` can be modified to allow storing maintaining computation results on a per-object basis. It should contain + /// its own mutable per-thread data as required. + /// + /// This method returns a vector of all tree items, along with their potentially modified custom node data. + /// + /// _Note_ that this method consumed the Tree to assure safe parallel traversal with mutation support. + pub fn traverse( + self, + resolve: F, + resolve_data: &R, + pack_entries_end: u64, + inspect_object: MBFN, + Options { + thread_limit, + mut object_progress, + size_progress, + should_interrupt, + object_hash, + base_spill, + collect_items, + max_object_bytes, + }: Options<'_, '_>, + ) -> Result, Error> + where + F: Fn(EntryRange, &R, &mut Vec) -> bool + Send + Clone, + R: Send + Sync, + MBFN: FnMut(&mut T, &dyn Progress, Context<'_>) -> Result<(), E> + Send + Clone, + E: std::error::Error + Send + Sync + 'static, + { + let num_objects = self.num_items(); + let object_counter = { + let progress = &mut object_progress; + progress.init(Some(num_objects), progress::count("objects")); + progress.counter() + }; + size_progress.init(None, progress::bytes()); + let size_counter = size_progress.counter(); + let object_progress = OwnShared::new(Mutable::new(object_progress)); + + let start = std::time::Instant::now(); + let (owned_forest, mut roots) = self.into_forest(pack_entries_end, object_hash)?; + let forest = &owned_forest; + let base_spill = base_spill.as_deref(); + let harvested = in_parallel_with_slice( + &mut roots, + thread_limit, + { + let object_progress = object_progress.clone(); + move |thread_index| resolve::State { + delta_bytes: Vec::::with_capacity(4096), + fully_resolved_delta_bytes: Vec::::with_capacity(4096), + progress: Box::new( + threading::lock(&object_progress) + .add_child(format!("thread {thread_index}")), + ), + resolve: resolve.clone(), + modify_base: inspect_object.clone(), + out: Vec::new(), + } + }, + { + move |root_id: &mut u32, state, threads_left, should_interrupt| { + resolve::deltas( + object_counter.clone(), + size_counter.clone(), + *root_id, + forest, + state, + resolve_data, + object_hash.len_in_bytes(), + base_spill, + collect_items, + max_object_bytes, + threads_left, + should_interrupt, + ) + } + }, + || { + (!should_interrupt.load(Ordering::Relaxed)) + .then(|| std::time::Duration::from_millis(50)) + }, + |state| state.out, + )?; + + threading::lock(&object_progress).show_throughput(start); + size_progress.show_throughput(start); + + Ok(Outcome { + items: harvested.into_iter().flatten().collect(), + unresolved: owned_forest.into_unresolved(), + }) + } +} diff --git a/knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs b/knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs new file mode 100644 index 0000000000000000000000000000000000000000..eb8e2466314591b80a4492200e22935f5f577666 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs @@ -0,0 +1,601 @@ +use std::sync::atomic::{AtomicBool, AtomicIsize, Ordering}; + +use gix_features::{progress::Progress, threading, zlib}; + +use crate::{ + cache::delta::{ + Item, + traverse::{Context, Error}, + tree::Forest, + }, + data, + data::EntryRange, +}; + +struct Pending { + level: u16, + node_id: u32, + entry: data::Entry, + entry_end: u64, + base_bytes: Vec, + object_kind: gix_object::Kind, + ref_children: std::ops::Range, + spill_ref: Option, +} + +fn enforce_budget(spill: &super::BaseSpill, stack: &mut [Pending]) -> Result<(), Error> { + (0..stack.len()).try_for_each(|index| -> Result<(), Error> { + if spill.over_budget() + && stack[index].spill_ref.is_none() + && !stack[index].base_bytes.is_empty() + { + let bytes = std::mem::take(&mut stack[index].base_bytes); + let sref = spill + .spill(&bytes) + .map_err(|source| Error::BaseSpill { source })?; + stack[index].spill_ref = Some(sref); + } + Ok(()) + }) +} + +fn restore_base_bytes( + spill: Option<&super::BaseSpill>, + pending: &mut Pending, +) -> Result<(), Error> { + if let Some(spill) = spill { + match pending.spill_ref.take() { + Some(sref) => spill + .reload(sref, &mut pending.base_bytes) + .map_err(|source| Error::BaseSpill { source })?, + None => spill.account_pop_resident(pending.base_bytes.len()), + } + } + Ok(()) +} + +pub(super) struct State { + pub delta_bytes: Vec, + pub fully_resolved_delta_bytes: Vec, + pub progress: Box, + pub resolve: F, + pub modify_base: MBFN, + pub out: Vec>, +} + +pub(super) fn deltas( + objects: gix_features::progress::StepShared, + size: gix_features::progress::StepShared, + root_id: u32, + forest: &Forest, + State { + delta_bytes, + fully_resolved_delta_bytes, + progress, + resolve, + modify_base, + out, + }: &mut State, + resolve_data: &R, + hash_len: usize, + spill: Option<&super::BaseSpill>, + collect_items: bool, + max_object_bytes: Option, + threads_left: &AtomicIsize, + should_interrupt: &AtomicBool, +) -> Result<(), Error> +where + T: Send + Sync + Clone, + R: Send + Sync, + F: Fn(EntryRange, &R, &mut Vec) -> bool + Send + Clone, + MBFN: FnMut(&mut T, &dyn Progress, Context<'_>) -> Result<(), E> + Send + Clone, + E: std::error::Error + Send + Sync + 'static, +{ + let mut inflate = zlib::Inflate::default(); + let mut decompress = decompressor(&*resolve, resolve_data, hash_len, &mut inflate); + + let mut stack: Vec = Vec::new(); + + let mut root_bytes = Vec::new(); + let (root_entry, root_end) = decompress(forest.entry_slice(root_id), &mut root_bytes)?; + let object_kind = root_entry + .header + .as_kind() + .ok_or(Error::UnexpectedRootDelta { + pack_offset: forest.offset(root_id), + })?; + let mut root_data = forest.data[root_id as usize].clone(); + apply_base( + modify_base, + &mut root_data, + progress, + &root_entry, + root_end, + &root_bytes, + 0, + object_kind, + )?; + objects.fetch_add(1, Ordering::Relaxed); + size.fetch_add(root_bytes.len(), Ordering::Relaxed); + if collect_items { + out.push(forest.item(root_id, root_data)); + } + let root_ref_children = forest.claim_ref_children(object_kind, &root_bytes)?; + expand( + forest, + children(forest, root_id, root_ref_children), + &root_bytes, + object_kind, + 0, + &mut stack, + delta_bytes, + fully_resolved_delta_bytes, + modify_base, + &**progress, + out, + &mut decompress, + &objects, + &size, + spill, + collect_items, + max_object_bytes, + )?; + drop(root_bytes); + if let Some(spill) = spill { + if spill.over_budget() { + enforce_budget(spill, &mut stack)?; + } + } + + loop { + if stack.len() > 1 { + if let Ok(initial_threads) = + threads_left.try_update(Ordering::SeqCst, Ordering::SeqCst, |threads_available| { + (threads_available > 0).then_some(0) + }) + { + *delta_bytes = Vec::new(); + *fully_resolved_delta_bytes = Vec::new(); + return deltas_mt( + initial_threads, + stack, + objects, + size, + &**progress, + resolve.clone(), + resolve_data, + modify_base.clone(), + hash_len, + spill, + collect_items, + max_object_bytes, + forest, + out, + threads_left, + should_interrupt, + ); + } + } + + let Some(mut pending) = stack.pop() else { + break; + }; + restore_base_bytes(spill, &mut pending)?; + let Pending { + level, + node_id, + entry, + entry_end, + base_bytes, + object_kind, + ref_children, + .. + } = pending; + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + + let mut node_data = forest.data[node_id as usize].clone(); + apply_base( + modify_base, + &mut node_data, + progress, + &entry, + entry_end, + &base_bytes, + level, + object_kind, + )?; + objects.fetch_add(1, Ordering::Relaxed); + size.fetch_add(base_bytes.len(), Ordering::Relaxed); + if collect_items { + out.push(forest.item(node_id, node_data)); + } + expand( + forest, + children(forest, node_id, ref_children), + &base_bytes, + object_kind, + level, + &mut stack, + delta_bytes, + fully_resolved_delta_bytes, + modify_base, + &**progress, + out, + &mut decompress, + &objects, + &size, + spill, + collect_items, + max_object_bytes, + )?; + drop(base_bytes); + if let Some(spill) = spill { + if spill.over_budget() { + enforce_budget(spill, &mut stack)?; + } + } + } + + Ok(()) +} + +/// * `initial_threads` is the threads we may spawn, not accounting for our own thread which is still considered used by the parent +/// system. Since this thread will take a controlling function, we may spawn one more than that. In threaded mode, we will finish +/// all remaining work. +#[allow(clippy::too_many_arguments)] +fn deltas_mt( + mut threads_to_create: isize, + stack: Vec, + objects: gix_features::progress::StepShared, + size: gix_features::progress::StepShared, + progress: &dyn Progress, + resolve: F, + resolve_data: &R, + modify_base: MBFN, + hash_len: usize, + spill: Option<&super::BaseSpill>, + collect_items: bool, + max_object_bytes: Option, + forest: &Forest, + accumulator: &mut Vec>, + threads_left: &AtomicIsize, + should_interrupt: &AtomicBool, +) -> Result<(), Error> +where + T: Send + Sync + Clone, + R: Send + Sync, + F: Fn(EntryRange, &R, &mut Vec) -> bool + Send + Clone, + MBFN: FnMut(&mut T, &dyn Progress, Context<'_>) -> Result<(), E> + Send + Clone, + E: std::error::Error + Send + Sync + 'static, +{ + let stack = gix_features::threading::Mutable::new(stack); + threads_to_create += 1; // ourselves + let mut returned_ourselves = false; + + gix_features::parallel::threads(|s| -> Result<(), Error> { + let mut threads = Vec::new(); + let poll_interval = std::time::Duration::from_millis(100); + loop { + for tid in 0..threads_to_create { + let thread = gix_features::parallel::build_thread() + .name(format!("gix-pack.traverse_deltas.{tid}")) + .spawn_scoped(s, { + let stack = &stack; + let resolve = resolve.clone(); + let mut modify_base = modify_base.clone(); + let objects = &objects; + let size = &size; + + move || -> Result>, Error> { + let mut collected: Vec> = Vec::new(); + let mut delta_bytes = Vec::new(); + let mut fully_resolved_delta_bytes = Vec::new(); + let mut inflate = zlib::Inflate::default(); + let mut decompress = + decompressor(&resolve, resolve_data, hash_len, &mut inflate); + + loop { + let Some(mut pending) = threading::lock(stack).pop() else { + break; + }; + restore_base_bytes(spill, &mut pending)?; + let Pending { + level, + node_id, + entry, + entry_end, + base_bytes, + object_kind, + ref_children, + .. + } = pending; + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + + let mut node_data = forest.data[node_id as usize].clone(); + apply_base( + &mut modify_base, + &mut node_data, + progress, + &entry, + entry_end, + &base_bytes, + level, + object_kind, + )?; + objects.fetch_add(1, Ordering::Relaxed); + size.fetch_add(base_bytes.len(), Ordering::Relaxed); + if collect_items { + collected.push(forest.item(node_id, node_data)); + } + let mut produced: Vec = Vec::new(); + expand( + forest, + children(forest, node_id, ref_children), + &base_bytes, + object_kind, + level, + &mut produced, + &mut delta_bytes, + &mut fully_resolved_delta_bytes, + &mut modify_base, + progress, + &mut collected, + &mut decompress, + objects, + size, + spill, + collect_items, + max_object_bytes, + )?; + drop(base_bytes); + if !produced.is_empty() { + let mut guard = threading::lock(stack); + guard.append(&mut produced); + if let Some(spill) = spill { + if spill.over_budget() { + enforce_budget(spill, &mut guard[..])?; + } + } + } + } + Ok(collected) + } + })?; + threads.push(thread); + } + if threads_left + .try_update( + Ordering::SeqCst, + Ordering::SeqCst, + |threads_available: isize| { + (threads_available > 0).then(|| { + threads_to_create = + threads_available.min(threading::lock(&stack).len() as isize); + threads_available - threads_to_create + }) + }, + ) + .is_err() + { + threads_to_create = 0; + } + + // What we really want to do is either wait for one of our threads to go down + // or for another scheduled thread to become available. Unfortunately we can't do that, + // but may instead find a good way to set the polling interval instead of hard-coding it. + std::thread::sleep(poll_interval); + // Get out of threads are already starving or they would be starving soon as no work is left. + // + // Lint: ScopedJoinHandle is not the same depending on active features and is not exposed in some cases. + #[allow(clippy::redundant_closure_for_method_calls)] + if threads.iter().any(|t| t.is_finished()) { + let mut running_threads = Vec::new(); + for thread in threads.drain(..) { + if thread.is_finished() { + match thread.join() { + Ok(Err(err)) => return Err(err), + Ok(Ok(collected)) => { + accumulator.extend(collected); + if !returned_ourselves { + returned_ourselves = true; + } else { + threads_left.fetch_add(1, Ordering::SeqCst); + } + } + Err(err) => { + std::panic::resume_unwind(err); + } + } + } else { + running_threads.push(thread); + } + } + if running_threads.is_empty() && threading::lock(&stack).is_empty() { + break; + } + threads = running_threads; + } + } + + Ok(()) + }) +} + +fn decompressor<'a, F, R>( + resolve: &'a F, + resolve_data: &'a R, + hash_len: usize, + inflate: &'a mut zlib::Inflate, +) -> impl FnMut(EntryRange, &mut Vec) -> Result<(data::Entry, u64), Error> + 'a +where + F: Fn(EntryRange, &R, &mut Vec) -> bool, + R: Sync, +{ + let mut raw = Vec::new(); + move |slice: EntryRange, out: &mut Vec| -> Result<(data::Entry, u64), Error> { + if !resolve(slice.clone(), resolve_data, &mut raw) { + return Err(Error::ResolveFailed { + pack_offset: slice.start, + }); + } + let entry = data::Entry::from_bytes(&raw, slice.start, hash_len)?; + let compressed = &raw[entry.header_size()..]; + let decompressed_len = entry.decompressed_size as usize; + decompress_all_at_once_with(inflate, compressed, decompressed_len, out)?; + Ok((entry, slice.end)) + } +} + +fn apply_base( + modify_base: &mut MBFN, + data: &mut T, + progress: &dyn Progress, + entry: &data::Entry, + entry_end: u64, + decompressed: &[u8], + level: u16, + object_kind: gix_object::Kind, +) -> Result<(), Error> +where + MBFN: FnMut(&mut T, &dyn Progress, Context<'_>) -> Result<(), E>, + E: std::error::Error + Send + Sync + 'static, +{ + modify_base( + data, + progress, + Context { + entry, + entry_end, + decompressed, + level, + object_kind, + }, + ) + .map_err(|err| Box::new(err) as Box)?; + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn expand( + forest: &Forest, + children: impl Iterator, + base_bytes: &[u8], + object_kind: gix_object::Kind, + base_level: u16, + stack: &mut Vec, + delta_bytes: &mut Vec, + fully_resolved_delta_bytes: &mut Vec, + modify_base: &mut MBFN, + progress: &dyn Progress, + sink: &mut Vec>, + decompress: &mut F, + objects: &gix_features::progress::StepShared, + size: &gix_features::progress::StepShared, + spill: Option<&super::BaseSpill>, + collect_items: bool, + max_object_bytes: Option, +) -> Result<(), Error> +where + T: Clone, + F: FnMut(EntryRange, &mut Vec) -> Result<(data::Entry, u64), Error>, + MBFN: FnMut(&mut T, &dyn Progress, Context<'_>) -> Result<(), E>, + E: std::error::Error + Send + Sync + 'static, +{ + for child_id in children { + let (child_entry, entry_end) = decompress(forest.entry_slice(child_id), delta_bytes)?; + let (base_size, consumed) = data::delta::decode_header_size(delta_bytes)?; + let mut header_ofs = consumed; + if base_bytes.len() != base_size as usize { + return Err(Error::BaseSizeMismatch { + delta_pack_offset: forest.offset(child_id), + declared: base_size, + actual: base_bytes.len() as u64, + }); + } + let (result_size, consumed) = data::delta::decode_header_size(&delta_bytes[consumed..])?; + header_ofs += consumed; + + if let Some(limit) = max_object_bytes { + if result_size > limit { + return Err(Error::DeltaResultTooLarge { + delta_pack_offset: forest.offset(child_id), + declared: result_size, + limit, + }); + } + } + fully_resolved_delta_bytes.resize(result_size as usize, 0); + data::delta::apply( + base_bytes, + fully_resolved_delta_bytes, + &delta_bytes[header_ofs..], + )?; + + let ref_children = forest.claim_ref_children(object_kind, fully_resolved_delta_bytes)?; + if forest.children(child_id).is_empty() && ref_children.is_empty() { + let mut child_data = forest.data[child_id as usize].clone(); + apply_base( + modify_base, + &mut child_data, + progress, + &child_entry, + entry_end, + fully_resolved_delta_bytes, + base_level + 1, + object_kind, + )?; + objects.fetch_add(1, Ordering::Relaxed); + size.fetch_add(base_bytes.len(), Ordering::Relaxed); + if collect_items { + sink.push(forest.item(child_id, child_data)); + } + } else { + let base_bytes = std::mem::take(fully_resolved_delta_bytes); + if let Some(spill) = spill { + spill.account_push(base_bytes.len()); + } + stack.push(Pending { + level: base_level + 1, + node_id: child_id, + entry: child_entry, + entry_end, + base_bytes, + object_kind, + ref_children, + spill_ref: None, + }); + } + } + Ok(()) +} + +fn children( + forest: &Forest, + id: u32, + ref_children: std::ops::Range, +) -> impl Iterator + '_ { + forest + .children(id) + .iter() + .copied() + .chain(forest.ref_child_ids(ref_children)) +} + +fn decompress_all_at_once_with( + inflate: &mut zlib::Inflate, + b: &[u8], + decompressed_len: usize, + out: &mut Vec, +) -> Result<(), Error> { + out.resize(decompressed_len, 0); + inflate.reset(); + inflate.once(b, out).map_err(|err| Error::ZlibInflate { + source: err, + message: "Failed to decompress entry", + })?; + Ok(()) +} diff --git a/knot2/third_party/gix-pack/src/cache/delta/tree.rs b/knot2/third_party/gix-pack/src/cache/delta/tree.rs new file mode 100644 index 0000000000000000000000000000000000000000..79ece5b24b94f28c662e5391653ebbfebc8b0eb1 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/delta/tree.rs @@ -0,0 +1,352 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +use gix_hash::ObjectId; + +use super::{Error, traverse}; + +#[allow(missing_docs)] +pub struct Item { + pub offset: crate::data::Offset, + pub next_offset: crate::data::Offset, + pub data: T, +} + +impl Item { + pub(crate) fn new( + offset: crate::data::Offset, + next_offset: crate::data::Offset, + data: T, + ) -> Self { + Item { + offset, + next_offset, + data, + } + } +} + +#[derive(Clone)] +enum Parent { + Root, + Base(u32), + Pending(crate::data::Offset), + Ref(ObjectId), + Done, +} + +#[allow(missing_docs)] +#[derive(Clone)] +pub struct Tree { + offsets: Vec, + data: Vec, + parent: Vec, +} + +#[allow(missing_docs)] +impl Tree { + pub fn with_capacity(num_objects: usize) -> Result { + Ok(Tree { + offsets: Vec::with_capacity(num_objects), + data: Vec::with_capacity(num_objects), + parent: Vec::with_capacity(num_objects), + }) + } + + pub fn num_items(&self) -> usize { + self.parent + .iter() + .filter(|parent| !matches!(parent, Parent::Done)) + .count() + } + + fn assert_incrementing(&self, offset: crate::data::Offset) -> Result<(), Error> { + match self.offsets.last() { + Some(&last) if offset <= last => Err(Error::InvariantIncreasingPackOffset { + last_pack_offset: last, + pack_offset: offset, + }), + _ => Ok(()), + } + } + + pub fn add_root(&mut self, offset: crate::data::Offset, data: T) -> Result<(), Error> { + self.assert_incrementing(offset)?; + self.offsets.push(offset); + self.data.push(data); + self.parent.push(Parent::Root); + Ok(()) + } + + pub fn add_child( + &mut self, + base_offset: crate::data::Offset, + offset: crate::data::Offset, + data: T, + ) -> Result<(), Error> { + self.assert_incrementing(offset)?; + let parent = match self.offsets.binary_search(&base_offset) { + Ok(index) => Parent::Base(index as u32), + Err(_) => Parent::Pending(base_offset), + }; + self.offsets.push(offset); + self.data.push(data); + self.parent.push(parent); + Ok(()) + } + + /// Add a delta whose base is the object `base_id`, to be found anywhere in the pack or appended to it later. + pub fn add_ref_child( + &mut self, + base_id: ObjectId, + offset: crate::data::Offset, + data: T, + ) -> Result<(), Error> { + self.assert_incrementing(offset)?; + self.offsets.push(offset); + self.data.push(data); + self.parent.push(Parent::Ref(base_id)); + Ok(()) + } + + /// The ids of the bases that deltas added with [`Self::add_ref_child()`] still wait for, sorted and deduplicated. + pub fn ref_bases(&self) -> Vec { + let mut ids: Vec<_> = self + .parent + .iter() + .filter_map(|parent| match parent { + Parent::Ref(id) => Some(*id), + _ => None, + }) + .collect(); + ids.sort_unstable(); + ids.dedup(); + ids + } + + pub(super) fn into_forest( + self, + pack_entries_end: crate::data::Offset, + object_hash: gix_hash::Kind, + ) -> Result<(Forest, Vec), traverse::Error> { + let Tree { + offsets, + data, + mut parent, + } = self; + let num_nodes = offsets.len(); + + let mut child_start = vec![0u32; num_nodes + 1]; + let mut roots: Vec = Vec::new(); + let mut ref_children: Vec<(ObjectId, u32)> = Vec::new(); + for index in 0..num_nodes { + if let Parent::Pending(base_offset) = parent[index] { + let base = offsets.binary_search(&base_offset).map_err(|_| { + traverse::Error::OutOfPackRefDelta { + base_pack_offset: base_offset, + } + })?; + parent[index] = Parent::Base(base as u32); + } + match parent[index] { + Parent::Root => roots.push(index as u32), + Parent::Base(base) => child_start[base as usize + 1] += 1, + Parent::Ref(base_id) => ref_children.push((base_id, index as u32)), + Parent::Done => {} + Parent::Pending(_) => unreachable!("pending parents were resolved above"), + } + } + ref_children.sort_unstable(); + let claimed = ref_children + .iter() + .map(|_| AtomicBool::new(false)) + .collect(); + for index in 0..num_nodes { + child_start[index + 1] += child_start[index]; + } + let mut cursor = child_start.clone(); + let mut child_ids = vec![0u32; child_start[num_nodes] as usize]; + for index in 0..num_nodes { + if let Parent::Base(base) = parent[index] { + let slot = cursor[base as usize]; + child_ids[slot as usize] = index as u32; + cursor[base as usize] = slot + 1; + } + } + + Ok(( + Forest { + offsets, + data, + child_start, + child_ids, + ref_children, + claimed, + object_hash, + pack_entries_end, + }, + roots, + )) + } +} + +pub(super) struct Forest { + offsets: Vec, + pub(super) data: Vec, + child_start: Vec, + child_ids: Vec, + ref_children: Vec<(ObjectId, u32)>, + claimed: Vec, + object_hash: gix_hash::Kind, + pack_entries_end: crate::data::Offset, +} + +impl Forest { + pub(super) fn offset(&self, id: u32) -> crate::data::Offset { + self.offsets[id as usize] + } + + pub(super) fn next_offset(&self, id: u32) -> crate::data::Offset { + self.offsets + .get(id as usize + 1) + .copied() + .unwrap_or(self.pack_entries_end) + } + + pub(super) fn entry_slice(&self, id: u32) -> crate::data::EntryRange { + self.offset(id)..self.next_offset(id) + } + + pub(super) fn children(&self, id: u32) -> &[u32] { + let start = self.child_start[id as usize] as usize; + let end = self.child_start[id as usize + 1] as usize; + &self.child_ids[start..end] + } + + pub(super) fn item(&self, id: u32, data: T) -> Item { + Item::new(self.offset(id), self.next_offset(id), data) + } + + /// Hand the ref-deltas based on the object of `kind` with `decompressed` data to the caller, unless an object + /// with the same id already took them. + pub(super) fn claim_ref_children( + &self, + kind: gix_object::Kind, + decompressed: &[u8], + ) -> Result, gix_hash::hasher::Error> { + if self.ref_children.is_empty() { + return Ok(0..0); + } + let id = gix_object::compute_hash(self.object_hash, kind, decompressed)?; + let start = self.ref_children.partition_point(|(base, _)| *base < id); + let end = start + self.ref_children[start..].partition_point(|(base, _)| *base == id); + if start == end + || self.claimed[start] + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + return Ok(0..0); + } + Ok(start..end) + } + + pub(super) fn ref_child_ids( + &self, + claimed: std::ops::Range, + ) -> impl Iterator + '_ { + self.ref_children[claimed].iter().map(|(_, id)| *id) + } +} + +impl Forest { + /// The deltas whose base never showed up, along with everything based on them. All other nodes are kept as + /// done so entry boundaries stay intact. + pub(super) fn into_unresolved(self) -> Option> { + let mut waiting: Vec> = (0..self.offsets.len()).map(|_| None).collect(); + let mut stack = Vec::new(); + for (base_id, id) in &self.ref_children { + let group_start = self + .ref_children + .partition_point(|(base, _)| base < base_id); + if !self.claimed[group_start].load(Ordering::Relaxed) { + waiting[*id as usize] = Some(Parent::Ref(*base_id)); + stack.push(*id); + } + } + if stack.is_empty() { + return None; + } + while let Some(id) = stack.pop() { + for &child in self.children(id) { + waiting[child as usize] = Some(Parent::Pending(self.offset(id))); + stack.push(child); + } + } + Some(Tree { + parent: waiting + .into_iter() + .map(|parent| parent.unwrap_or(Parent::Done)) + .collect(), + offsets: self.offsets, + data: self.data, + }) + } +} + +#[cfg(test)] +mod tests { + mod from_offsets_in_pack { + use std::sync::atomic::AtomicBool; + + use crate as pack; + + const SMALL_PACK_INDEX: &str = + "objects/pack/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx"; + const SMALL_PACK: &str = "objects/pack/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack"; + + const INDEX_V1: &str = "objects/pack/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx"; + const PACK_FOR_INDEX_V1: &str = + "objects/pack/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack"; + + use gix_testtools::fixture_path; + + #[test] + fn v1() -> Result<(), Box> { + tree(INDEX_V1, PACK_FOR_INDEX_V1) + } + + #[test] + fn v2() -> Result<(), Box> { + tree(SMALL_PACK_INDEX, SMALL_PACK) + } + + fn tree(index_path: &str, pack_path: &str) -> Result<(), Box> { + let idx = pack::index::File::at(fixture_path(index_path), gix_hash::Kind::Sha1)?; + crate::cache::delta::Tree::from_offsets_in_pack( + &fixture_path(pack_path), + idx.sorted_offsets().into_iter(), + &|ofs| *ofs, + &|id| idx.lookup(id).map(|index| idx.pack_offset_at_index(index)), + &mut gix_features::progress::Discard, + &AtomicBool::new(false), + gix_hash::Kind::Sha1, + )?; + Ok(()) + } + } + + mod size { + use gix_testtools::size_ok; + + use super::super::Item; + + #[test] + fn size_of_pack_tree_item() { + let actual = std::mem::size_of::<[Item<()>; 7_500_000]>(); + let expected = 120_000_000; + assert!( + size_ok(actual, expected), + "we don't want these to grow unnoticed: {actual} <~ {expected}" + ); + } + } +} diff --git a/knot2/third_party/gix-pack/src/cache/lru.rs b/knot2/third_party/gix-pack/src/cache/lru.rs new file mode 100644 index 0000000000000000000000000000000000000000..9f46fe114a1e5baf3de8217d156a5f1dc1e738ab --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/lru.rs @@ -0,0 +1,292 @@ +use super::DecodeEntry; + +#[cfg(feature = "pack-cache-lru-dynamic")] +mod memory { + use std::num::NonZeroUsize; + + use clru::WeightScale; + + use super::DecodeEntry; + use crate::cache::set_vec_to_slice; + + struct Entry { + data: Vec, + kind: gix_object::Kind, + compressed_size: usize, + } + + type Key = (u32, u64); + struct CustomScale; + + impl WeightScale for CustomScale { + fn weight(&self, _key: &Key, value: &Entry) -> usize { + value.data.len() + } + } + + /// An LRU cache with hash map backing and an eviction rule based on the memory usage for object data in bytes. + pub struct MemoryCappedHashmap { + inner: clru::CLruCache, + free_list: Vec>, + debug: gix_features::cache::Debug, + } + + impl MemoryCappedHashmap { + /// Return a new instance which evicts least recently used items if it uses more than `memory_cap_in_bytes` + /// object data. + pub fn new(memory_cap_in_bytes: usize) -> MemoryCappedHashmap { + MemoryCappedHashmap { + inner: clru::CLruCache::with_config( + clru::CLruCacheConfig::new( + NonZeroUsize::new(memory_cap_in_bytes).expect("non zero"), + ) + .with_scale(CustomScale), + ), + free_list: Vec::new(), + debug: gix_features::cache::Debug::new(format!( + "MemoryCappedHashmap({memory_cap_in_bytes}B)" + )), + } + } + } + + impl DecodeEntry for MemoryCappedHashmap { + fn put( + &mut self, + pack_id: u32, + offset: u64, + data: &[u8], + kind: gix_object::Kind, + compressed_size: usize, + ) { + self.debug.put(); + let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) + else { + return; + }; + let res = self.inner.put_with_weight( + (pack_id, offset), + Entry { + data, + kind, + compressed_size, + }, + ); + match res { + Ok(Some(previous_entry)) => self.free_list.push(previous_entry.data), + Ok(None) => {} + Err((_key, value)) => self.free_list.push(value.data), + } + } + + fn get( + &mut self, + pack_id: u32, + offset: u64, + out: &mut Vec, + ) -> Option<(gix_object::Kind, usize)> { + let res = self.inner.get(&(pack_id, offset)).and_then(|e| { + set_vec_to_slice(out, &e.data)?; + Some((e.kind, e.compressed_size)) + }); + if res.is_some() { + self.debug.hit(); + } else { + self.debug.miss(); + } + res + } + } +} + +#[cfg(feature = "pack-cache-lru-dynamic")] +pub use memory::MemoryCappedHashmap; + +#[cfg(feature = "pack-cache-lru-static")] +mod _static { + use super::DecodeEntry; + use crate::cache::set_vec_to_slice; + struct Entry { + pack_id: u32, + offset: u64, + data: Vec, + kind: gix_object::Kind, + compressed_size: usize, + } + + /// A cache using a least-recently-used implementation capable of storing the `SIZE` most recent objects. + /// The cache must be small as the search is 'naive' and the underlying data structure is a linked list. + /// Values of 64 seem to improve performance. + pub struct StaticLinkedList { + inner: uluru::LRUCache, + last_evicted: Vec, + debug: gix_features::cache::Debug, + /// the amount of bytes we are currently holding, taking into account the capacities of all Vecs we keep. + mem_used: usize, + /// The total amount of memory we should be able to hold with all entries combined. + mem_limit: usize, + } + + impl StaticLinkedList { + /// Create a new list with a memory limit of `mem_limit` in bytes. If 0, there is no memory limit. + pub fn new(mem_limit: usize) -> Self { + StaticLinkedList { + inner: Default::default(), + last_evicted: Vec::new(), + debug: gix_features::cache::Debug::new(format!("StaticLinkedList<{SIZE}>")), + mem_used: 0, + mem_limit: if mem_limit == 0 { + usize::MAX + } else { + mem_limit + }, + } + } + } + + impl Default for StaticLinkedList { + fn default() -> Self { + Self::new(96 * 1024 * 1024) + } + } + + impl DecodeEntry for StaticLinkedList { + fn put( + &mut self, + pack_id: u32, + offset: u64, + data: &[u8], + kind: gix_object::Kind, + compressed_size: usize, + ) { + // We cannot possibly hold this much. + if data.len() > self.mem_limit { + return; + } + // If we could hold it but are at limit, all we can do is make space. + let mem_free = self.mem_limit - self.mem_used; + if data.len() > mem_free { + // prefer freeing free-lists instead of clearing our cache + let free_list_cap = self.last_evicted.len(); + self.last_evicted = Vec::new(); + // still not enough? clear everything + if data.len() > mem_free + free_list_cap { + self.inner.clear(); + self.mem_used = 0; + } else { + self.mem_used -= free_list_cap; + } + } + self.debug.put(); + let mut v = std::mem::take(&mut self.last_evicted); + self.mem_used -= v.capacity(); + if set_vec_to_slice(&mut v, data).is_none() { + return; + } + self.mem_used += v.capacity(); + if let Some(previous) = self.inner.insert(Entry { + offset, + pack_id, + data: v, + kind, + compressed_size, + }) { + // No need to adjust capacity as we already counted it. + self.last_evicted = previous.data; + } + } + + fn get( + &mut self, + pack_id: u32, + offset: u64, + out: &mut Vec, + ) -> Option<(gix_object::Kind, usize)> { + let res = self.inner.lookup(|e: &mut Entry| { + if e.pack_id == pack_id && e.offset == offset { + set_vec_to_slice(&mut *out, &e.data)?; + Some((e.kind, e.compressed_size)) + } else { + None + } + }); + if res.is_some() { + self.debug.hit(); + } else { + self.debug.miss(); + } + res + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn no_limit() { + let c = StaticLinkedList::<10>::new(0); + assert_eq!( + c.mem_limit, + usize::MAX, + "zero is automatically turned into a large limit that is equivalent to unlimited" + ); + } + + #[test] + fn journey() { + let mut c = StaticLinkedList::<10>::new(100); + assert_eq!(c.mem_limit, 100); + assert_eq!(c.mem_used, 0); + + // enough memory for normal operation + let mut last_mem_used = 0; + for _ in 0..10 { + c.put(0, 0, &[0], gix_object::Kind::Blob, 1); + assert!(c.mem_used > last_mem_used); + last_mem_used = c.mem_used; + } + assert_eq!(c.mem_used, 80, "there is a minimal vec size"); + assert_eq!(c.inner.len(), 10); + assert_eq!(c.last_evicted.len(), 0); + + c.put( + 0, + 0, + &(0..20).collect::>(), + gix_object::Kind::Blob, + 1, + ); + assert_eq!(c.inner.len(), 10); + assert_eq!(c.mem_used, 80 + 20); + assert_eq!(c.last_evicted.len(), 1); + + c.put( + 0, + 0, + &(0..50).collect::>(), + gix_object::Kind::Blob, + 1, + ); + assert_eq!(c.inner.len(), 1, "cache clearance wasn't necessary"); + assert_eq!(c.last_evicted.len(), 0, "the free list was cleared"); + assert_eq!(c.mem_used, 50); + + c.put( + 0, + 0, + &(0..101).collect::>(), + gix_object::Kind::Blob, + 1, + ); + assert_eq!( + c.inner.len(), + 1, + "objects that won't ever fit within the memory limit are ignored" + ); + } + } +} + +#[cfg(feature = "pack-cache-lru-static")] +pub use _static::StaticLinkedList; diff --git a/knot2/third_party/gix-pack/src/cache/mod.rs b/knot2/third_party/gix-pack/src/cache/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..5f3377245f41f289eeb55a8ff86a3b1af8d13826 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/mod.rs @@ -0,0 +1,97 @@ +use std::ops::DerefMut; + +use gix_object::Kind; + +/// A trait to model putting objects at a given pack `offset` into a cache, and fetching them. +/// +/// It is used to speed up [pack traversals][crate::index::File::traverse()]. +pub trait DecodeEntry { + /// Store a fully decoded object at `offset` of `kind` with `compressed_size` and `data` in the cache. + /// + /// It is up to the cache implementation whether that actually happens or not. + fn put( + &mut self, + pack_id: u32, + offset: u64, + data: &[u8], + kind: gix_object::Kind, + compressed_size: usize, + ); + /// Attempt to fetch the object at `offset` and store its decoded bytes in `out`, as previously stored with [`DecodeEntry::put()`], and return + /// its (object `kind`, `decompressed_size`) + fn get( + &mut self, + pack_id: u32, + offset: u64, + out: &mut Vec, + ) -> Option<(gix_object::Kind, usize)>; +} + +/// A cache that stores nothing and retrieves nothing, thus it _never_ caches. +#[derive(Default)] +pub struct Never; + +impl DecodeEntry for Never { + fn put( + &mut self, + _pack_id: u32, + _offset: u64, + _data: &[u8], + _kind: gix_object::Kind, + _compressed_size: usize, + ) { + } + fn get( + &mut self, + _pack_id: u32, + _offset: u64, + _out: &mut Vec, + ) -> Option<(gix_object::Kind, usize)> { + None + } +} + +impl DecodeEntry for Box { + fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: Kind, compressed_size: usize) { + self.deref_mut() + .put(pack_id, offset, data, kind, compressed_size); + } + + fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec) -> Option<(Kind, usize)> { + self.deref_mut().get(pack_id, offset, out) + } +} + +/// A way of storing and retrieving entire objects to and from a cache. +pub trait Object { + /// Put the object going by `id` of `kind` with `data` into the cache. + fn put(&mut self, id: gix_hash::ObjectId, kind: gix_object::Kind, data: &[u8]); + + /// Try to retrieve the object named `id` and place its data into `out` if available and return `Some(kind)` if found. + fn get(&mut self, id: &gix_hash::ObjectId, out: &mut Vec) -> Option; +} + +/// Various implementations of [`DecodeEntry`] using least-recently-used algorithms. +#[cfg(any(feature = "pack-cache-lru-dynamic", feature = "pack-cache-lru-static"))] +pub mod lru; + +pub mod object; + +/// +pub mod delta; + +/// Replaces content of the given `Vec` with the slice. The vec will have the same length +/// as the slice. The vec can be either `&mut Vec` or `Vec`. +/// Returns `None` if no memory could be allocated. +#[cfg(any( + feature = "pack-cache-lru-static", + feature = "pack-cache-lru-dynamic", + feature = "object-cache-dynamic" +))] +fn set_vec_to_slice>>(mut vec: V, source: &[u8]) -> Option { + let out = vec.borrow_mut(); + out.clear(); + out.try_reserve(source.len()).ok()?; + out.extend_from_slice(source); + Some(vec) +} diff --git a/knot2/third_party/gix-pack/src/cache/object.rs b/knot2/third_party/gix-pack/src/cache/object.rs new file mode 100644 index 0000000000000000000000000000000000000000..dc3c61235766e47af34c66685929bd69415f1687 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/object.rs @@ -0,0 +1,116 @@ +//! This module is a bit 'misplaced' if spelled out like '`gix_pack::cache::object::`*' but is best placed here for code reuse and +//! general usefulness. +use crate::cache; + +#[cfg(feature = "object-cache-dynamic")] +mod memory { + use std::num::NonZeroUsize; + + use clru::WeightScale; + + use crate::{cache, cache::set_vec_to_slice}; + + struct Entry { + data: Vec, + kind: gix_object::Kind, + } + + type Key = gix_hash::ObjectId; + + struct CustomScale; + + impl WeightScale for CustomScale { + fn weight(&self, key: &Key, value: &Entry) -> usize { + value.data.len() + std::mem::size_of::() + key.as_bytes().len() + } + } + + /// An LRU cache with hash map backing and an eviction rule based on the memory usage for object data in bytes. + pub struct MemoryCappedHashmap { + inner: clru::CLruCache, + free_list: Vec>, + debug: gix_features::cache::Debug, + } + + impl MemoryCappedHashmap { + /// The amount of bytes we can hold in total, or the value we saw in `new(…)`. + pub fn capacity(&self) -> usize { + self.inner.capacity() + } + /// Return a new instance which evicts least recently used items if it uses more than `memory_cap_in_bytes` + /// object data. + pub fn new(memory_cap_in_bytes: usize) -> MemoryCappedHashmap { + MemoryCappedHashmap { + inner: clru::CLruCache::with_config( + clru::CLruCacheConfig::new( + NonZeroUsize::new(memory_cap_in_bytes).expect("non zero"), + ) + .with_hasher(gix_hashtable::hash::Builder) + .with_scale(CustomScale), + ), + free_list: Vec::new(), + debug: gix_features::cache::Debug::new(format!( + "MemoryCappedObjectHashmap({memory_cap_in_bytes}B)" + )), + } + } + } + + impl cache::Object for MemoryCappedHashmap { + /// Put the object going by `id` of `kind` with `data` into the cache. + fn put(&mut self, id: gix_hash::ObjectId, kind: gix_object::Kind, data: &[u8]) { + self.debug.put(); + let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) + else { + return; + }; + let res = self.inner.put_with_weight(id, Entry { data, kind }); + match res { + Ok(Some(previous_entry)) => self.free_list.push(previous_entry.data), + Ok(None) => {} + Err((_key, value)) => self.free_list.push(value.data), + } + } + + /// Try to retrieve the object named `id` and place its data into `out` if available and return `Some(kind)` if found. + fn get(&mut self, id: &gix_hash::ObjectId, out: &mut Vec) -> Option { + let res = self.inner.get(id).and_then(|e| { + set_vec_to_slice(out, &e.data)?; + Some(e.kind) + }); + if res.is_some() { + self.debug.hit(); + } else { + self.debug.miss(); + } + res + } + } +} +#[cfg(feature = "object-cache-dynamic")] +pub use memory::MemoryCappedHashmap; + +/// A cache implementation that doesn't do any caching. +pub struct Never; + +impl cache::Object for Never { + /// Noop + fn put(&mut self, _id: gix_hash::ObjectId, _kind: gix_object::Kind, _data: &[u8]) {} + + /// Noop + fn get(&mut self, _id: &gix_hash::ObjectId, _out: &mut Vec) -> Option { + None + } +} + +impl cache::Object for Box { + fn put(&mut self, id: gix_hash::ObjectId, kind: gix_object::Kind, data: &[u8]) { + use std::ops::DerefMut; + self.deref_mut().put(id, kind, data); + } + + fn get(&mut self, id: &gix_hash::ObjectId, out: &mut Vec) -> Option { + use std::ops::DerefMut; + self.deref_mut().get(id, out) + } +} diff --git a/knot2/third_party/gix-pack/src/data/delta.rs b/knot2/third_party/gix-pack/src/data/delta.rs new file mode 100644 index 0000000000000000000000000000000000000000..9ff0944a4b05b0c3cec384c1d87792a97dcd6899 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/delta.rs @@ -0,0 +1,127 @@ +/// +pub mod apply { + /// Returned when failing to apply deltas. + #[derive(thiserror::Error, Debug)] + #[allow(missing_docs)] + pub enum Error { + #[error("Corrupt delta data: {message}")] + Corrupt { message: &'static str }, + #[error("Encountered unsupported command code: 0")] + UnsupportedCommandCode, + #[error("Delta copy from base: byte slices must match")] + DeltaCopyBaseSliceMismatch, + #[error("Delta copy data: byte slices must match")] + DeltaCopyDataSliceMismatch, + } +} + +/// Given the decompressed pack delta `d`, decode a size in bytes (either the base object size or the result object size) +/// Equivalent to [this canonical git function](https://github.com/git/git/blob/311531c9de557d25ac087c1637818bd2aad6eb3a/delta.h#L89) +pub(crate) fn decode_header_size(d: &[u8]) -> Result<(u64, usize), apply::Error> { + let mut shift = 0; + let mut size = 0u64; + let mut consumed = 0; + for cmd in d.iter() { + if shift >= u64::BITS { + return Err(apply::Error::Corrupt { + message: "delta header size uses more bits than fit into u64", + }); + } + consumed += 1; + size |= (u64::from(*cmd) & 0x7f) << shift; + shift += 7; + if *cmd & 0x80 == 0 { + return Ok((size, consumed)); + } + } + Err(apply::Error::Corrupt { + message: "delta header size is truncated", + }) +} + +pub(crate) fn apply(base: &[u8], mut target: &mut [u8], data: &[u8]) -> Result<(), apply::Error> { + fn next_byte(data: &[u8], i: &mut usize) -> Result { + let byte = *data.get(*i).ok_or(apply::Error::Corrupt { + message: "delta copy instruction is truncated", + })?; + *i += 1; + Ok(byte) + } + + let mut i = 0; + while let Some(cmd) = data.get(i) { + i += 1; + match cmd { + cmd if cmd & 0b1000_0000 != 0 => { + let (mut ofs, mut size): (u32, u32) = (0, 0); + if cmd & 0b0000_0001 != 0 { + ofs = u32::from(next_byte(data, &mut i)?); + } + if cmd & 0b0000_0010 != 0 { + ofs |= u32::from(next_byte(data, &mut i)?) << 8; + } + if cmd & 0b0000_0100 != 0 { + ofs |= u32::from(next_byte(data, &mut i)?) << 16; + } + if cmd & 0b0000_1000 != 0 { + ofs |= u32::from(next_byte(data, &mut i)?) << 24; + } + if cmd & 0b0001_0000 != 0 { + size = u32::from(next_byte(data, &mut i)?); + } + if cmd & 0b0010_0000 != 0 { + size |= u32::from(next_byte(data, &mut i)?) << 8; + } + if cmd & 0b0100_0000 != 0 { + size |= u32::from(next_byte(data, &mut i)?) << 16; + } + if size == 0 { + size = 0x10000; // 65536 + } + let ofs = ofs as usize; + let end = ofs + .checked_add(size as usize) + .ok_or(apply::Error::Corrupt { + message: "delta copy range overflows", + })?; + std::io::Write::write( + &mut target, + base.get(ofs..end).ok_or(apply::Error::Corrupt { + message: "delta copy range exceeds base object size", + })?, + ) + .map_err(|_e| apply::Error::DeltaCopyBaseSliceMismatch)?; + } + 0 => { + return Err(apply::Error::Corrupt { + message: "delta command 0 is reserved and invalid", + }); + } + size => { + let end = i.checked_add(*size as usize).ok_or(apply::Error::Corrupt { + message: "delta insert range overflows", + })?; + std::io::Write::write( + &mut target, + data.get(i..end).ok_or(apply::Error::Corrupt { + message: "delta insert data is truncated", + })?, + ) + .map_err(|_e| apply::Error::DeltaCopyDataSliceMismatch)?; + i = end; + } + } + } + debug_assert_eq!( + i, + data.len(), + "delta instructions were not consumed completely, should be impossible" + ); + if !target.is_empty() { + return Err(apply::Error::Corrupt { + message: "delta instructions produced fewer bytes than promised", + }); + } + + Ok(()) +} diff --git a/knot2/third_party/gix-pack/src/data/entry/decode.rs b/knot2/third_party/gix-pack/src/data/entry/decode.rs new file mode 100644 index 0000000000000000000000000000000000000000..9b04910b186e2b715c3bbaa776205a06a32a196d --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/entry/decode.rs @@ -0,0 +1,228 @@ +use std::io; + +use gix_features::decode::leb64_from_read; + +use super::{BLOB, COMMIT, OFS_DELTA, REF_DELTA, TAG, TREE}; +use crate::data; + +/// The error returned by [data::Entry::from_bytes()]. +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error("Object type {type_id} is unsupported")] + UnsupportedType { type_id: u8 }, + #[error("Pack entry is truncated: {message}")] + Corrupt { message: &'static str }, + #[error("Pack entry header value overflowed while decoding")] + Overflow, +} + +/// Decoding +impl data::Entry { + /// Decode an entry from the given entry data `d`, providing the `pack_offset` to allow tracking the start of the entry data section. + /// + /// # Panics + /// + /// If we cannot understand the header, garbage data is likely to trigger this. + pub fn from_bytes( + d: &[u8], + pack_offset: data::Offset, + hash_len: usize, + ) -> Result { + let (type_id, size, mut consumed) = parse_header_info(d)?; + + use crate::data::entry::Header::*; + let object = match type_id { + OFS_DELTA => { + let (distance, leb_bytes) = parse_leb64(&d[consumed..])?; + let delta = OfsDelta { + base_distance: distance, + }; + consumed += leb_bytes; + delta + } + REF_DELTA => { + let delta = RefDelta { + base_id: gix_hash::ObjectId::from_bytes_or_panic( + d.get(consumed..consumed + hash_len).ok_or(Error::Corrupt { + message: "ref-delta base object id", + })?, + ), + }; + consumed += hash_len; + delta + } + BLOB => Blob, + TREE => Tree, + COMMIT => Commit, + TAG => Tag, + other => return Err(Error::UnsupportedType { type_id: other }), + }; + Ok(data::Entry { + header: object, + decompressed_size: size, + data_offset: pack_offset + consumed as u64, + }) + } + + /// Instantiate an `Entry` from the reader `r`, providing the `pack_offset` to allow tracking the start of the entry data section. + pub fn from_read( + r: &mut dyn io::Read, + pack_offset: data::Offset, + hash_len: usize, + ) -> io::Result { + let (type_id, size, mut consumed) = streaming_parse_header_info(r)?; + + use crate::data::entry::Header::*; + let object = match type_id { + OFS_DELTA => { + let (distance, leb_bytes) = leb64_from_read(&mut *r)?; + let delta = OfsDelta { + base_distance: distance, + }; + consumed += leb_bytes; + delta + } + REF_DELTA => { + let mut buf = gix_hash::Kind::buf(); + let hash = &mut buf[..hash_len]; + r.read_exact(hash)?; + #[allow(clippy::redundant_slicing)] + let delta = RefDelta { + base_id: gix_hash::ObjectId::from_bytes_or_panic(&hash[..]), + }; + consumed += hash_len; + delta + } + BLOB => Blob, + TREE => Tree, + COMMIT => Commit, + TAG => Tag, + other => { + return Err(io::Error::other(format!( + "Object type {other} is unsupported" + ))); + } + }; + Ok(data::Entry { + header: object, + decompressed_size: size, + data_offset: pack_offset + consumed as u64, + }) + } +} + +#[inline] +fn streaming_parse_header_info(read: &mut dyn io::Read) -> Result<(u8, u64, usize), io::Error> { + let mut byte = [0u8; 1]; + read.read_exact(&mut byte)?; + let mut c = byte[0]; + let mut i = 1; + let type_id = (c >> 4) & 0b0000_0111; + let mut size = u64::from(c) & 0b0000_1111; + let mut shift = 4u32; + while c & 0b1000_0000 != 0 { + read.read_exact(&mut byte)?; + c = byte[0]; + i += 1; + let component = u64::from(c & 0b0111_1111) + .checked_shl(shift) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed") + })?; + size = size.checked_add(component).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed") + })?; + shift += 7; + } + if i != encoded_pack_entry_header_size(size) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "pack entry header uses a non-canonical size encoding", + )); + } + Ok((type_id, size, i)) +} + +/// Parses the header of a pack-entry, yielding object type id, decompressed object size, and consumed bytes +#[inline] +fn parse_header_info(data: &[u8]) -> Result<(u8, u64, usize), Error> { + let mut c = *data.first().ok_or(Error::Corrupt { + message: "need a pack entry header, got empty input", + })?; + let mut i = 1; + let type_id = (c >> 4) & 0b0000_0111; + let mut size = u64::from(c) & 0b0000_1111; + let mut shift = 4u32; + while c & 0b1000_0000 != 0 { + c = *data.get(i).ok_or(Error::Corrupt { + message: "pack entry header continuation byte", + })?; + i += 1; + let component = u64::from(c & 0b0111_1111) + .checked_shl(shift) + .ok_or(Error::Overflow)?; + size = size.checked_add(component).ok_or(Error::Overflow)?; + shift += 7; + } + if i != encoded_pack_entry_header_size(size) { + return Err(Error::Corrupt { + message: "pack entry header uses a non-canonical size encoding", + }); + } + Ok((type_id, size, i)) +} + +fn parse_leb64(data: &[u8]) -> Result<(u64, usize), Error> { + let mut i = 0; + let mut c = *data.first().ok_or(Error::Corrupt { + message: "an ofs-delta base distance", + })?; + i += 1; + let mut value = u64::from(c) & 0x7f; + while c & 0x80 != 0 { + c = *data.get(i).ok_or(Error::Corrupt { + message: "an ofs-delta base distance continuation byte", + })?; + i += 1; + value = value + .checked_add(1) + .and_then(|value| value.checked_shl(7)) + .and_then(|value| value.checked_add(u64::from(c) & 0x7f)) + .ok_or(Error::Overflow)?; + } + Ok((value, i)) +} + +/// Return the canonical byte length of a pack-entry size header for `size`. +/// +/// We use this to reject overlong size encodings during parsing. +/// That matters for our delta resolution implementation, which later reconstructs an entry's +/// pack offset from `data_offset - header_size()`. If we accepted non-canonical encodings here, +/// `header_size()` would compute the canonical length while `data_offset` would reflect the +/// actually consumed bytes, breaking that invariant and allowing malformed delta entries to point +/// back to themselves or otherwise walk the wrong base objects. +fn encoded_pack_entry_header_size(mut size: u64) -> usize { + let mut bytes = 1; + size >>= 4; + while size != 0 { + bytes += 1; + size >>= 7; + } + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_non_canonical_pack_entry_header_encoding() { + assert!(matches!( + data::Entry::from_bytes(&[0xed, 0x00], 0, gix_hash::Kind::Sha1.len_in_bytes()), + Err(Error::Corrupt { + message: "pack entry header uses a non-canonical size encoding" + }) + )); + } +} diff --git a/knot2/third_party/gix-pack/src/data/entry/header.rs b/knot2/third_party/gix-pack/src/data/entry/header.rs new file mode 100644 index 0000000000000000000000000000000000000000..7c419243503650ba5363118bb0f11af78bba55dc --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/entry/header.rs @@ -0,0 +1,164 @@ +use std::io; + +use super::{BLOB, COMMIT, OFS_DELTA, REF_DELTA, TAG, TREE}; +use crate::data; + +/// The header portion of a pack data entry, identifying the kind of stored object. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[allow(missing_docs)] +pub enum Header { + /// The object is a commit + Commit, + /// The object is a tree + Tree, + /// The object is a blob + Blob, + /// The object is a tag + Tag, + /// Describes a delta-object which needs to be applied to a base. The base object is identified by the `base_id` field + /// which is found within the parent repository. + /// Most commonly used for **thin-packs** when receiving pack files from the server to refer to objects that are not + /// part of the pack but expected to be present in the receivers repository. + /// + /// # Note + /// This could also be an object within this pack if the LSB encoded offset would be larger than 20 bytes, which is unlikely to + /// happen. + /// + /// **The naming** is exactly the same as the canonical implementation uses, namely **REF_DELTA**. + RefDelta { base_id: gix_hash::ObjectId }, + /// Describes a delta-object present in this pack which acts as base for this object. + /// The base object is measured as a distance from this objects + /// pack offset, so that `base_pack_offset = this_objects_pack_offset - base_distance` + /// + /// # Note + /// + /// **The naming** is exactly the same as the canonical implementation uses, namely **OFS_DELTA**. + OfsDelta { base_distance: u64 }, +} + +impl Header { + /// Subtract `distance` from `pack_offset` safely without the chance for overflow or no-ops if `distance` is 0. + pub fn verified_base_pack_offset( + pack_offset: data::Offset, + distance: u64, + ) -> Option { + if distance == 0 { + return None; + } + pack_offset.checked_sub(distance) + } + /// Convert the header's object kind into [`gix_object::Kind`] if possible + pub fn as_kind(&self) -> Option { + use gix_object::Kind::*; + Some(match self { + Header::Tree => Tree, + Header::Blob => Blob, + Header::Commit => Commit, + Header::Tag => Tag, + Header::RefDelta { .. } | Header::OfsDelta { .. } => return None, + }) + } + /// Convert this header's object kind into the packs internal representation + pub fn as_type_id(&self) -> u8 { + use Header::*; + match self { + Blob => BLOB, + Tree => TREE, + Commit => COMMIT, + Tag => TAG, + OfsDelta { .. } => OFS_DELTA, + RefDelta { .. } => REF_DELTA, + } + } + /// Return's true if this is a delta object, i.e. not a full object. + pub fn is_delta(&self) -> bool { + matches!(self, Header::OfsDelta { .. } | Header::RefDelta { .. }) + } + /// Return's true if this is a base object, i.e. not a delta object. + pub fn is_base(&self) -> bool { + !self.is_delta() + } +} + +impl Header { + /// Encode this header along the given `decompressed_size_in_bytes` into the `out` write stream for use within a data pack. + /// + /// Returns the amount of bytes written to `out`. + /// `decompressed_size_in_bytes` is the full size in bytes of the object that this header represents + pub fn write_to( + &self, + decompressed_size_in_bytes: u64, + out: &mut dyn io::Write, + ) -> io::Result { + let mut size = decompressed_size_in_bytes; + let mut written = 1; + let mut c: u8 = (self.as_type_id() << 4) | (size as u8 & 0b0000_1111); + size >>= 4; + while size != 0 { + out.write_all(&[c | 0b1000_0000])?; + written += 1; + c = size as u8 & 0b0111_1111; + size >>= 7; + } + out.write_all(&[c])?; + + use Header::*; + match self { + RefDelta { base_id: oid } => { + out.write_all(oid.as_slice())?; + written += oid.as_slice().len(); + } + OfsDelta { base_distance } => { + let mut buf = [0u8; 10]; + let buf = leb64_encode(*base_distance, &mut buf); + out.write_all(buf)?; + written += buf.len(); + } + Blob | Tree | Commit | Tag => {} + } + Ok(written) + } + + /// The size of the header in bytes when serialized + pub fn size(&self, decompressed_size: u64) -> usize { + self.write_to(decompressed_size, &mut io::sink()) + .expect("io::sink() to never fail") + } +} + +#[inline] +fn leb64_encode(mut n: u64, buf: &mut [u8; 10]) -> &[u8] { + let mut bytes_written = 1; + buf[buf.len() - 1] = n as u8 & 0b0111_1111; + for out in buf.iter_mut().rev().skip(1) { + n >>= 7; + if n == 0 { + break; + } + n -= 1; + *out = 0b1000_0000 | (n as u8 & 0b0111_1111); + bytes_written += 1; + } + debug_assert_eq!( + n, 0, + "BUG: buffer must be large enough to hold a 64 bit integer" + ); + &buf[buf.len() - bytes_written..] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn leb64_encode_max_int() { + let mut buf = [0u8; 10]; + let buf = leb64_encode(u64::MAX, &mut buf); + assert_eq!( + buf.len(), + 10, + "10 bytes should be used when 64bits are encoded" + ); + } +} diff --git a/knot2/third_party/gix-pack/src/data/entry/mod.rs b/knot2/third_party/gix-pack/src/data/entry/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..651cc06195b10aedce7167d4773b9a92ded26e85 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/entry/mod.rs @@ -0,0 +1,65 @@ +use crate::data::Entry; + +const _TYPE_EXT1: u8 = 0; +const COMMIT: u8 = 1; +const TREE: u8 = 2; +const BLOB: u8 = 3; +const TAG: u8 = 4; +const _TYPE_EXT2: u8 = 5; +const OFS_DELTA: u8 = 6; +const REF_DELTA: u8 = 7; + +/// A way to uniquely identify the location of an entry within a pack bundle +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Location { + /// The id of the pack containing the object. It's unique within its frame of reference which is the owning object database. + pub pack_id: u32, + /// The size of the entry of disk so that the range of bytes of the entry is `pack_offset..pack_offset + entry_size`. + pub entry_size: usize, + /// The start of the entry in the pack identified by `pack_id`. + pub pack_offset: data::Offset, +} + +impl Location { + /// Compute a range suitable for lookup in pack data using the [`entry_slice()`][crate::data::File::entry_slice()] method. + pub fn entry_range(&self, pack_offset: data::Offset) -> crate::data::EntryRange { + pack_offset..pack_offset + self.entry_size as u64 + } +} + +/// Access +impl Entry { + /// Compute the pack offset to the base entry of the object represented by this entry, or + /// return `None` if the distance would underflow or is invalid. + pub fn checked_base_pack_offset(&self, distance: u64) -> Option { + let pack_offset = self.data_offset - self.header_size() as u64; + Header::verified_base_pack_offset(pack_offset, distance) + } + + /// Compute the pack offset to the base entry of the object represented by this entry. + /// + /// # Panics + /// + /// Panics if the `distance` will cause an underflow or is invalid. + pub fn base_pack_offset(&self, distance: u64) -> data::Offset { + self.checked_base_pack_offset(distance) + .expect("in-bound distance of deltas") + } + /// The pack offset at which this entry starts + pub fn pack_offset(&self) -> data::Offset { + self.data_offset - self.header_size() as u64 + } + /// The amount of bytes used to describe this entry in the pack. The header starts at [`Self::pack_offset()`] + pub fn header_size(&self) -> usize { + self.header.size(self.decompressed_size) + } +} + +/// +pub mod decode; + +mod header; +pub use header::Header; + +use crate::data; diff --git a/knot2/third_party/gix-pack/src/data/file/decode/entry.rs b/knot2/third_party/gix-pack/src/data/file/decode/entry.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb79192e34394500bc47e8a2d79dc811b45a884f --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/decode/entry.rs @@ -0,0 +1,540 @@ +use std::ops::Range; + +use gix_features::zlib; +use smallvec::SmallVec; + +use crate::{ + cache, data, + data::{File, delta, file::decode::Error}, +}; + +/// A return value of a resolve function, which given an [`ObjectId`][gix_hash::ObjectId] determines where an object can be found. +#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum ResolvedBase { + /// Indicate an object is within this pack, at the given entry, and thus can be looked up locally. + InPack(data::Entry), + /// Indicates the object of `kind` was found outside of the pack, and its data was written into an output + /// vector which now has a length of `end`. + #[allow(missing_docs)] + OutOfPack { kind: gix_object::Kind, end: usize }, +} + +#[derive(Debug)] +struct Delta { + data: Range, + base_size: usize, + result_size: usize, + + decompressed_size: usize, + data_offset: data::Offset, +} + +/// Additional information and statistics about a successfully decoded object produced by [`File::decode_entry()`]. +/// +/// Useful to understand the effectiveness of the pack compression or the cost of decompression. +#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// The kind of resolved object. + pub kind: gix_object::Kind, + /// The amount of deltas in the chain of objects that had to be resolved beforehand. + /// + /// This number is affected by the [`Cache`][cache::DecodeEntry] implementation, with cache hits shortening the + /// delta chain accordingly + pub num_deltas: u32, + /// The total decompressed size of all pack entries in the delta chain + pub decompressed_size: u64, + /// The total compressed size of all pack entries in the delta chain + pub compressed_size: usize, + /// The total size of the decoded object. + pub object_size: u64, +} + +impl Outcome { + pub(crate) fn default_from_kind(kind: gix_object::Kind) -> Self { + Self { + kind, + num_deltas: 0, + decompressed_size: 0, + compressed_size: 0, + object_size: 0, + } + } + fn from_object_entry( + kind: gix_object::Kind, + entry: &data::Entry, + compressed_size: usize, + ) -> Self { + Self { + kind, + num_deltas: 0, + decompressed_size: entry.decompressed_size, + compressed_size, + object_size: entry.decompressed_size, + } + } +} + +/// Decompression of objects +impl File { + fn decoded_object_size(&self, size: u64) -> Result { + decoded_object_size(size, self.alloc_limit_bytes) + } + + /// Decompress the given `entry` into `out` and return the amount of bytes read from the pack data. + /// Note that `inflate` is not reset after usage, but will be reset before using it. + /// + /// _Note_ that this method does not resolve deltified objects, but merely decompresses their content + /// `out` is expected to be large enough to hold `entry.size` bytes. + pub fn decompress_entry( + &self, + entry: &data::Entry, + inflate: &mut zlib::Inflate, + out: &mut [u8], + ) -> Result { + let size: usize = entry + .decompressed_size + .try_into() + .map_err(|_| Error::OutOfMemory)?; + if out.len() < size { + return Err(Error::OutOfMemory); + } + self.decompress_entry_from_data_offset(entry.data_offset, inflate, &mut out[..size]) + } + + /// Obtain the [`Entry`][crate::data::Entry] at the given `offset` into the pack. + /// + /// The `offset` is typically obtained from the pack index file. + pub fn entry(&self, offset: data::Offset) -> Result { + let pack_offset: usize = offset.try_into().expect("offset representable by machine"); + if pack_offset > self.data_len() { + return Err(data::entry::decode::Error::Corrupt { + message: "an entry offset pointing beyond pack data", + }); + } + + let window = (self.data_len() - pack_offset).min(self.hash_len + 32); + let mut header = vec![0u8; window]; + self.read_exact_at(pack_offset, &mut header).map_err(|_| { + data::entry::decode::Error::Corrupt { + message: "failed to read entry header from pack data", + } + })?; + data::Entry::from_bytes(&header, offset, self.hash_len) + } + + /// Decompress the object expected at the given data offset, sans pack header. This information is only + /// known after the pack header was parsed. + /// Note that this method does not resolve deltified objects, but merely decompresses their content + /// `out` is expected to be large enough to hold `entry.size` bytes. + /// Returns the amount of packed bytes there read from the pack data file. + pub(crate) fn decompress_entry_from_data_offset( + &self, + data_offset: data::Offset, + inflate: &mut zlib::Inflate, + out: &mut [u8], + ) -> Result { + let (consumed_in, _consumed_out) = + self.decompress_complete_entry_from_data_offset(data_offset, inflate, out)?; + Ok(consumed_in) + } + + /// Like `decompress_entry_from_data_offset`, but returns `(consumed-input, consumed-output)`. + /// + /// The compressed stream must end exactly after producing `out.len()` bytes. Pack entry + /// headers are untrusted, so callers must not accept streams that stop early and + /// leave zero-filled slack in the destination buffer, nor streams that require more output + /// than the header promised. Both cases would make later delta parsing operate on bytes that + /// are not the entry payload described by the pack header. + pub(crate) fn decompress_complete_entry_from_data_offset( + &self, + data_offset: data::Offset, + inflate: &mut zlib::Inflate, + out: &mut [u8], + ) -> Result<(usize, usize), Error> { + let (status, consumed_in, consumed_out) = + self.decompress_entry_from_data_offset_unchecked(data_offset, inflate, out)?; + if status != zlib::Status::StreamEnd || consumed_out != out.len() { + return Err(data::entry::decode::Error::Corrupt { + message: "pack entry decompressed size does not match entry header", + } + .into()); + } + Ok((consumed_in, consumed_out)) + } + + /// Like [`Self::decompress_commplete_entry_from_data_offset()`], but allows callers to inspect incomplete streams. + /// + /// This is only for callers that intentionally decompress a prefix into a smaller buffer, such as + /// delta header probing. Full pack entry decoding should use [`Self::decompress_commplete_entry_from_data_offset()`]. + pub(crate) fn decompress_entry_from_data_offset_unchecked( + &self, + data_offset: data::Offset, + inflate: &mut zlib::Inflate, + out: &mut [u8], + ) -> Result<(zlib::Status, usize, usize), Error> { + let offset: usize = data_offset + .try_into() + .expect("offset representable by machine"); + if offset >= self.data_len() { + return Err(data::entry::decode::Error::Corrupt { + message: "an entry data offset pointing beyond pack data", + } + .into()); + } + + inflate.reset(); + let mut chunk = [0u8; 8192]; + let mut in_pos = offset; + let status = loop { + let avail = (self.data_len() - in_pos).min(chunk.len()); + self.read_exact_at(in_pos, &mut chunk[..avail]) + .map_err(|_| { + Error::from(data::entry::decode::Error::Corrupt { + message: "failed to read pack entry data", + }) + })?; + let out_pos = inflate.state.total_out() as usize; + let before_in = inflate.state.total_in(); + let status = inflate + .state + .decompress( + &chunk[..avail], + &mut out[out_pos..], + zlib::FlushDecompress::None, + ) + .map_err(|err| Error::from(zlib::inflate::Error::from(err)))?; + let advanced_in = inflate.state.total_in() != before_in; + let advanced_out = inflate.state.total_out() as usize != out_pos; + in_pos = offset + inflate.state.total_in() as usize; + match status { + zlib::Status::StreamEnd => break zlib::Status::StreamEnd, + zlib::Status::Ok | zlib::Status::BufError => { + if avail == 0 || (!advanced_in && !advanced_out) { + break status; + } + } + } + }; + Ok(( + status, + inflate.state.total_in() as usize, + inflate.state.total_out() as usize, + )) + } + + /// Decode an entry, resolving delta's as needed, while growing the `out` vector if there is not enough + /// space to hold the result object. + /// + /// The `entry` determines which object to decode, and is commonly obtained with the help of a pack index file or through pack iteration. + /// `inflate` will be used for decompressing entries, and will not be reset after usage, but before first using it. + /// + /// `resolve` is a function to lookup objects with the given [`ObjectId`][gix_hash::ObjectId], in case the full object id is used to refer to + /// a base object, instead of an in-pack offset. + /// + /// `delta_cache` is a mechanism to avoid looking up base objects multiple times when decompressing multiple objects in a row. + /// Use a [Noop-Cache][cache::Never] to disable caching all together at the cost of repeating work. + pub fn decode_entry( + &self, + entry: data::Entry, + out: &mut Vec, + inflate: &mut zlib::Inflate, + resolve: &dyn Fn(&gix_hash::oid, &mut Vec) -> Option, + delta_cache: &mut dyn cache::DecodeEntry, + ) -> Result { + use crate::data::entry::Header::*; + match entry.header { + Tree | Blob | Commit | Tag => { + let size = self.decoded_object_size(entry.decompressed_size)?; + if let Some(additional) = size.checked_sub(out.len()) { + out.try_reserve(additional)?; + } + out.resize(size, 0); + self.decompress_entry(&entry, inflate, out.as_mut_slice()) + .map(|consumed_input| { + Outcome::from_object_entry( + entry.header.as_kind().expect("a non-delta entry"), + &entry, + consumed_input, + ) + }) + } + OfsDelta { .. } | RefDelta { .. } => { + self.resolve_deltas(entry, resolve, inflate, out, delta_cache) + } + } + } + + /// resolve: technically, this shouldn't ever be required as stored local packs don't refer to objects by id + /// that are outside of the pack. Unless, of course, the ref refers to an object within this pack, which means + /// it's very, very large as 20bytes are smaller than the corresponding MSB encoded number + fn resolve_deltas( + &self, + last: data::Entry, + resolve: &dyn Fn(&gix_hash::oid, &mut Vec) -> Option, + inflate: &mut zlib::Inflate, + out: &mut Vec, + cache: &mut dyn cache::DecodeEntry, + ) -> Result { + // all deltas, from the one that produces the desired object (first) to the oldest at the end of the chain + let mut chain = SmallVec::<[Delta; 10]>::default(); + let first_entry = last.clone(); + let mut cursor = last; + let mut base_buffer_size: Option = None; + let mut object_kind: Option = None; + let mut consumed_input: Option = None; + + // Find the first full base, either an undeltified object in the pack or a reference to another object. + let mut total_delta_data_size: u64 = 0; + while cursor.header.is_delta() { + if let Some((kind, packed_size)) = cache.get(self.id, cursor.data_offset, out) { + base_buffer_size = Some(out.len()); + object_kind = Some(kind); + // If the input entry is a cache hit, keep the packed size as it must be returned. + // Otherwise, the packed size will be determined later when decompressing the input delta + if total_delta_data_size == 0 { + consumed_input = Some(packed_size); + } + break; + } + // This is a pessimistic guess, as worst possible compression should not be bigger than the data itself. + // TODO: is this assumption actually true? + total_delta_data_size = total_delta_data_size + .checked_add(cursor.decompressed_size) + .ok_or(Error::OutOfMemory)?; + let decompressed_size = self.decoded_object_size(cursor.decompressed_size)?; + chain.push(Delta { + data: Range { + start: 0, + end: decompressed_size, + }, + base_size: 0, + result_size: 0, + decompressed_size, + data_offset: cursor.data_offset, + }); + use crate::data::entry::Header; + cursor = match cursor.header { + Header::OfsDelta { base_distance } => { + self.entry(cursor.checked_base_pack_offset(base_distance).ok_or( + crate::data::entry::decode::Error::Corrupt { + message: "an ofs-delta base distance pointing before pack start", + }, + )?)? + } + Header::RefDelta { base_id } => match resolve(base_id.as_ref(), out) { + Some(ResolvedBase::InPack(entry)) => entry, + Some(ResolvedBase::OutOfPack { end, kind }) => { + base_buffer_size = Some(end); + object_kind = Some(kind); + break; + } + None => return Err(Error::DeltaBaseUnresolved(base_id)), + }, + _ => unreachable!("cursor.is_delta() only allows deltas here"), + }; + } + + // This can happen if the cache held the first entry itself + // We will just treat it as an object then, even though it's technically incorrect. + if chain.is_empty() { + return Ok(Outcome::from_object_entry( + object_kind.expect("object kind as set by cache"), + &first_entry, + consumed_input.expect("consumed bytes as set by cache"), + )); + } + + // First pass will decompress all delta data and keep it in our output buffer + // []... + // so that we can find the biggest result size. + let total_delta_data_size: usize = total_delta_data_size + .try_into() + .map_err(|_| Error::OutOfMemory)?; + + let chain_len = chain.len(); + let (first_buffer_end, second_buffer_end) = { + let delta_start = base_buffer_size.unwrap_or(0); + + let delta_range = Range { + start: delta_start, + end: delta_start + .checked_add(total_delta_data_size) + .ok_or(Error::OutOfMemory)?, + }; + out.try_reserve(delta_range.end.saturating_sub(out.len()))?; + out.resize(delta_range.end, 0); + + let mut instructions = &mut out[delta_range.clone()]; + let mut relative_delta_start = 0; + let mut biggest_result_size = 0; + for (delta_idx, delta) in chain.iter_mut().rev().enumerate() { + let (consumed_from_data_offset, consumed_out) = self + .decompress_complete_entry_from_data_offset( + delta.data_offset, + inflate, + &mut instructions[..delta.decompressed_size], + )?; + let is_last_delta_to_be_applied = delta_idx + 1 == chain_len; + if is_last_delta_to_be_applied { + consumed_input = Some(consumed_from_data_offset); + } + + let current_delta = &instructions[..consumed_out]; + let (base_size, offset) = delta::decode_header_size(current_delta)?; + let mut bytes_consumed_by_header = offset; + biggest_result_size = biggest_result_size.max(base_size); + delta.base_size = self.decoded_object_size(base_size)?; + + let (result_size, offset) = delta::decode_header_size(¤t_delta[offset..])?; + bytes_consumed_by_header += offset; + biggest_result_size = biggest_result_size.max(result_size); + delta.result_size = self.decoded_object_size(result_size)?; + + // the absolute location into the instructions buffer, so we keep track of the end point of the last + delta.data.start = relative_delta_start + bytes_consumed_by_header; + delta.data.end = relative_delta_start + consumed_out; + relative_delta_start += delta.decompressed_size; + + instructions = &mut instructions[delta.decompressed_size..]; + } + + // Now we can produce a buffer like this + // [] + // from []... + if base_buffer_size.is_none() { + biggest_result_size = biggest_result_size.max(cursor.decompressed_size); + } + let biggest_result_size = self.decoded_object_size(biggest_result_size)?; + let first_buffer_size = biggest_result_size; + let second_buffer_size = first_buffer_size; + let out_size = first_buffer_size + .checked_add(second_buffer_size) + .and_then(|size| size.checked_add(total_delta_data_size)) + .ok_or(Error::OutOfMemory)?; + out.try_reserve(out_size.saturating_sub(out.len()))?; + out.resize(out_size, 0); + + // Now 'rescue' the deltas, because in the next step we possibly overwrite that portion + // of memory with the base object (in the majority of cases) + let second_buffer_end = { + let end = first_buffer_size + .checked_add(second_buffer_size) + .ok_or(Error::OutOfMemory)?; + // Move the decompressed delta instructions behind the two work buffers so they remain intact + // while we repurpose the front of `out` for base-object materialization and delta application. + out.copy_within(delta_range, end); + end + }; + + // If we don't have a out-of-pack object already, fill the base-buffer by decompressing the full object + // at which the cursor is left after the iteration + if base_buffer_size.is_none() { + let base_entry = cursor; + debug_assert!(!base_entry.header.is_delta()); + object_kind = base_entry.header.as_kind(); + let base_size = self.decoded_object_size(base_entry.decompressed_size)?; + let out_base = &mut out[..base_size]; + self.decompress_entry_from_data_offset(base_entry.data_offset, inflate, out_base)?; + } + + (first_buffer_size, second_buffer_end) + }; + + // From oldest to most recent, apply all deltas, swapping the buffer back and forth + // TODO: once we have more tests, we could optimize this memory-intensive work to + // analyse the delta-chains to only copy data once - after all, with 'copy-from-base' deltas, + // all data originates from one base at some point. + // `out` is: [source-buffer][target-buffer][max-delta-instructions-buffer] + let (buffers, instructions) = out.split_at_mut(second_buffer_end); + let (mut source_buf, mut target_buf) = buffers.split_at_mut(first_buffer_end); + + let mut last_result_size = None; + for ( + delta_idx, + Delta { + data, + base_size, + result_size, + .. + }, + ) in chain.into_iter().rev().enumerate() + { + let data = &mut instructions[data]; + if delta_idx + 1 == chain_len { + last_result_size = Some(result_size); + } + delta::apply( + &source_buf[..base_size], + &mut target_buf[..result_size], + data, + )?; + // use the target as source for the next delta + std::mem::swap(&mut source_buf, &mut target_buf); + } + + let last_result_size = last_result_size.expect("at least one delta chain item"); + // uneven chains leave the target buffer after the source buffer + // FIXME(Performance) If delta-chains are uneven, we know we will have to copy bytes over here + // Instead we could use a different start buffer, to naturally end up with the result in the + // right one. + // However, this is a bit more complicated than just that - you have to deal with the base + // object, which should also be placed in the second buffer right away. You don't have that + // control/knowledge for out-of-pack bases, so this is a special case to deal with, too. + // Maybe these invariants can be represented in the type system though. + if chain_len % 2 == 1 { + // this seems inverted, but remember: we swapped the buffers on the last iteration + target_buf[..last_result_size].copy_from_slice(&source_buf[..last_result_size]); + } + debug_assert!(out.len() >= last_result_size); + out.truncate(last_result_size); + + let object_kind = object_kind + .expect("a base object as root of any delta chain that we are here to resolve"); + let consumed_input = consumed_input.expect("at least one decompressed delta object"); + cache.put( + self.id, + first_entry.data_offset, + out.as_slice(), + object_kind, + consumed_input, + ); + Ok(Outcome { + kind: object_kind, + // technically depending on the cache, the chain size is not correct as it might + // have been cut short by a cache hit. The caller must deactivate the cache to get + // actual results + num_deltas: chain_len as u32, + decompressed_size: first_entry.decompressed_size, + compressed_size: consumed_input, + object_size: last_result_size as u64, + }) + } +} + +/// Convert user-controlled sizes from pack data into allocation sizes while enforcing the configured allocation cap. +fn decoded_object_size(size: u64, alloc_limit_bytes: Option) -> Result { + let size: usize = size.try_into().map_err(|_| Error::OutOfMemory)?; + if alloc_limit_bytes.is_some_and(|limit| size > limit) { + return Err(Error::OutOfMemory); + } + Ok(size) +} + +#[cfg(test)] +mod tests { + use gix_testtools::size_ok; + + use super::*; + + #[test] + fn size_of_decode_entry_outcome() { + let actual = std::mem::size_of::(); + let expected = 32; + assert!( + size_ok(actual, expected), + "this shouldn't change without use noticing as it's returned a lot: {actual} <~ {expected}" + ); + } +} diff --git a/knot2/third_party/gix-pack/src/data/file/decode/header.rs b/knot2/third_party/gix-pack/src/data/file/decode/header.rs new file mode 100644 index 0000000000000000000000000000000000000000..580f1464558ee1c98ba88cf0f60302e8a4683968 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/decode/header.rs @@ -0,0 +1,164 @@ +use gix_features::zlib; + +use crate::{ + data, + data::{File, delta, file::decode::Error}, +}; + +/// A return value of a resolve function, which given an [`ObjectId`][gix_hash::ObjectId] determines where an object can be found. +#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum ResolvedBase { + /// Indicate an object is within this pack, at the given entry, and thus can be looked up locally. + InPack(data::Entry), + /// Indicates the object of `kind` was found outside of the pack. + OutOfPack { + /// The kind of object we found when reading the header of the out-of-pack base. + kind: gix_object::Kind, + /// The amount of deltas encountered if the object was packed as well. + num_deltas: Option, + }, +} + +/// Additional information and statistics about a successfully decoded object produced by [`File::decode_header()`]. +/// +/// Useful to understand the effectiveness of the pack compression or the cost of decompression. +#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// The kind of resolved object. + pub kind: gix_object::Kind, + /// The decompressed size of the object. + pub object_size: u64, + /// The amount of deltas in the chain of objects that had to be resolved beforehand. + pub num_deltas: u32, +} + +/// Obtain object information quickly. +impl File { + /// Resolve the object header information starting at `entry`, following the chain of entries as needed. + /// + /// The `entry` determines which object to decode, and is commonly obtained with the help of a pack index file or through pack iteration. + /// `inflate` will be used for (partially) decompressing entries, and will be reset before first use, but not after the last use. + /// + /// `resolve` is a function to lookup objects with the given [`ObjectId`][gix_hash::ObjectId], in case the full object id + /// is used to refer to a base object, instead of an in-pack offset. + /// + /// For delta entries, this only probes the initial delta header bytes to determine the result + /// object size. It can reject streams that end or overflow within that probe, but it does not + /// fully validate that the compressed stream produces exactly the decompressed size declared in + /// the pack entry header. Use [`File::decode_entry()`][crate::data::File::decode_entry()] when + /// callers need that full validation. + pub fn decode_header( + &self, + mut entry: data::Entry, + inflate: &mut zlib::Inflate, + resolve: &dyn Fn(&gix_hash::oid) -> Option, + ) -> Result { + use crate::data::entry::Header::*; + let mut num_deltas = 0; + let mut first_delta_decompressed_size = None::; + loop { + match entry.header { + Tree | Blob | Commit | Tag => { + return Ok(Outcome { + kind: entry.header.as_kind().expect("always valid for non-refs"), + object_size: first_delta_decompressed_size + .unwrap_or(entry.decompressed_size), + num_deltas, + }); + } + OfsDelta { base_distance } => { + num_deltas += 1; + if first_delta_decompressed_size.is_none() { + first_delta_decompressed_size = + Some(self.decode_delta_object_size(inflate, &entry)?); + } + entry = self.entry(entry.checked_base_pack_offset(base_distance).ok_or( + crate::data::entry::decode::Error::Corrupt { + message: "an ofs-delta base distance pointing before pack start", + }, + )?)?; + } + RefDelta { base_id } => { + num_deltas += 1; + if first_delta_decompressed_size.is_none() { + first_delta_decompressed_size = + Some(self.decode_delta_object_size(inflate, &entry)?); + } + match resolve(base_id.as_ref()) { + Some(ResolvedBase::InPack(base_entry)) => entry = base_entry, + Some(ResolvedBase::OutOfPack { + kind, + num_deltas: origin_num_deltas, + }) => { + return Ok(Outcome { + kind, + object_size: first_delta_decompressed_size + .unwrap_or(entry.decompressed_size), + num_deltas: origin_num_deltas.unwrap_or_default() + num_deltas, + }); + } + None => return Err(Error::DeltaBaseUnresolved(base_id)), + } + } + } + } + } + + /// Decode the result object size from the initial delta header bytes in `inflate`, using `entry` + /// for offsets. + /// + /// This intentionally mirrors Git's cheap header probe: only the first 20 decompressed bytes + /// are inspected, which is enough for the two `u64` varints that make up a valid delta header. + /// If the zlib stream ends within that probe, we can reject declared-size mismatches here. + /// Otherwise this result only proves that the delta header prefix is parseable; full + /// decompression through `decode_entry()` must still validate that the stream length matches + /// the pack entry header. + #[inline] + fn decode_delta_object_size( + &self, + inflate: &mut zlib::Inflate, + entry: &data::Entry, + ) -> Result { + let mut buf = [0_u8; 20]; + let max_size = entry.decompressed_size.min(buf.len() as u64) as usize; + let (status, _consumed_in, consumed_out) = self + .decompress_entry_from_data_offset_unchecked( + entry.data_offset, + inflate, + &mut buf[..max_size], + )?; + if status == zlib::Status::StreamEnd { + if consumed_out as u64 != entry.decompressed_size { + return Err(data::entry::decode::Error::Corrupt { + message: "pack entry decompressed to fewer bytes than declared in the entry header", + } + .into()); + } + } else if entry.decompressed_size == max_size as u64 { + return Err(data::entry::decode::Error::Corrupt { + message: "pack entry decompressed to more bytes than declared in the entry header", + } + .into()); + } + let buf = &buf[..consumed_out]; + let (_base_size, offset) = delta::decode_header_size(buf)?; + let (result_size, _offset) = delta::decode_header_size(&buf[offset..])?; + Ok(result_size) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn size_of_decode_entry_outcome() { + assert_eq!( + std::mem::size_of::(), + 16, + "this shouldn't change without use noticing as it's returned a lot" + ); + } +} diff --git a/knot2/third_party/gix-pack/src/data/file/decode/mod.rs b/knot2/third_party/gix-pack/src/data/file/decode/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..71bbf1595c4eb6b742f8b52c13aa6cefda6c847d --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/decode/mod.rs @@ -0,0 +1,31 @@ +use std::collections::TryReserveError; + +/// +pub mod entry; +/// +pub mod header; + +/// Returned by [`File::decode_header()`][crate::data::File::decode_header()], +/// [`File::decode_entry()`][crate::data::File::decode_entry()] and . +/// [`File::decompress_entry()`][crate::data::File::decompress_entry()] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("Failed to decompress pack entry")] + ZlibInflate(#[from] gix_features::zlib::inflate::Error), + #[error("A delta chain could not be followed as the ref base with id {0} could not be found")] + DeltaBaseUnresolved(gix_hash::ObjectId), + #[error(transparent)] + EntryType(#[from] crate::data::entry::decode::Error), + #[error("Entry too large to fit in memory")] + OutOfMemory, + #[error(transparent)] + Delta(#[from] crate::data::delta::apply::Error), +} + +impl From for Error { + #[cold] + fn from(_: TryReserveError) -> Self { + Self::OutOfMemory + } +} diff --git a/knot2/third_party/gix-pack/src/data/file/init.rs b/knot2/third_party/gix-pack/src/data/file/init.rs new file mode 100644 index 0000000000000000000000000000000000000000..8f2b2b940e1c15c7a35fad584256bdbe6ace1a40 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/init.rs @@ -0,0 +1,82 @@ +use std::path::Path; + +use crate::data; + +/// Instantiation +impl data::File { + /// Try opening a data file at the given `path`. + /// + /// The `object_hash` is a way to read (and write) the same file format with different hashes, as the hash kind + /// isn't stored within the file format itself. + /// + /// This constructor leaves allocation limiting disabled, allowing allocations of any size dictated by pack data. + /// Call [`File::with_alloc_limit_bytes()`][crate::data::File::with_alloc_limit_bytes()] before decoding entries from untrusted input. + pub fn at( + path: impl AsRef, + object_hash: gix_hash::Kind, + ) -> Result { + Self::at_inner(path.as_ref(), object_hash) + } + + fn at_inner( + path: &Path, + object_hash: gix_hash::Kind, + ) -> Result { + use std::os::unix::fs::FileExt; + + use crate::data::header::N32_SIZE; + let hash_len = object_hash.len_in_bytes(); + let file = std::fs::File::open(path).map_err(|e| data::header::decode::Error::Io { + source: e, + path: path.to_owned(), + })?; + let pack_len = file + .metadata() + .map_err(|e| data::header::decode::Error::Io { + source: e, + path: path.to_owned(), + })? + .len(); + let pack_len = usize::try_from(pack_len).map_err(|_| { + data::header::decode::Error::Corrupt(format!( + "Pack data of size {pack_len} is too large for this machine" + )) + })?; + if pack_len < N32_SIZE * 3 + hash_len { + return Err(data::header::decode::Error::Corrupt(format!( + "Pack data of size {pack_len} is too small for even an empty pack with shortest hash" + ))); + } + let mut header = [0u8; 12]; + file.read_exact_at(&mut header, 0) + .map_err(|e| data::header::decode::Error::Io { + source: e, + path: path.to_owned(), + })?; + let (version, num_objects) = data::header::decode(&header)?; + let id = gix_features::hash::crc32(path.as_os_str().to_string_lossy().as_bytes()); + Ok(Self { + file, + len: pack_len, + path: path.to_owned(), + id, + version, + num_objects, + hash_len, + object_hash, + alloc_limit_bytes: None, + }) + } + + /// Configure the maximum size of a single allocation caused by user-controlled on-disk pack data. + /// + /// Use `None` to disable the limit, which is also the default. + /// + /// This is currently enforced when decoding pack entries and resolving delta chains. + /// Callers that allocate from pack metadata directly should consult [`File::alloc_limit_bytes()`][crate::data::File::alloc_limit_bytes()] + /// and apply the same limit themselves. + pub fn with_alloc_limit_bytes(mut self, alloc_limit_bytes: Option) -> Self { + self.alloc_limit_bytes = alloc_limit_bytes; + self + } +} diff --git a/knot2/third_party/gix-pack/src/data/file/mod.rs b/knot2/third_party/gix-pack/src/data/file/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..6bfe0e2721b5cea99c0796b75a3106a31b6329d7 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/mod.rs @@ -0,0 +1,9 @@ +mod init; +/// +pub mod verify; + +/// +pub mod decode; + +/// The bytes used as header in a pack data file. +pub type Header = [u8; 12]; diff --git a/knot2/third_party/gix-pack/src/data/file/verify.rs b/knot2/third_party/gix-pack/src/data/file/verify.rs new file mode 100644 index 0000000000000000000000000000000000000000..e8e5197daa155c0062c5ec30930430405c24336f --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/verify.rs @@ -0,0 +1,62 @@ +use std::sync::atomic::AtomicBool; + +use gix_features::progress::Progress; + +use crate::data::File; + +/// +pub mod checksum { + /// Returned by [`data::File::verify_checksum()`][crate::data::File::verify_checksum()]. + pub type Error = crate::verify::checksum::Error; +} + +/// Checksums and verify checksums +impl File { + /// The checksum in the trailer of this pack data file + pub fn checksum(&self) -> gix_hash::ObjectId { + let trailer = self + .read_span( + (self.data_len() - self.object_hash.len_in_bytes()) as u64..self.data_len() as u64, + ) + .expect("pack trailer is within the pack data"); + gix_hash::ObjectId::from_bytes_or_panic(&trailer) + } + + /// Verifies that the checksum of the packfile over all bytes preceding it indeed matches the actual checksum, + /// returning the actual checksum equivalent to the return value of [`checksum()`][File::checksum()] if there + /// is no mismatch. + /// + /// Note that if no `progress` is desired, one can pass [`gix_features::progress::Discard`]. + /// + /// Have a look at [`index::File::verify_integrity(…)`][crate::index::File::verify_integrity()] for an + /// even more thorough integrity check. + pub fn verify_checksum( + &self, + progress: &mut dyn Progress, + should_interrupt: &AtomicBool, + ) -> Result { + let expected = self.checksum(); + let body_len = (self.data_len() - self.hash_len) as u64; + let actual = match gix_hash::bytes_of_file( + self.path(), + body_len, + self.object_hash, + progress, + should_interrupt, + ) { + Ok(id) => id, + Err(gix_hash::io::Error::Io(err)) if err.kind() == std::io::ErrorKind::Interrupted => { + return Err(checksum::Error::Interrupted); + } + Err(gix_hash::io::Error::Io(_)) => { + let data = self.materialized()?; + let mut hasher = gix_hash::hasher(self.object_hash); + hasher.update(&data[..body_len as usize]); + hasher.try_finalize()? + } + Err(gix_hash::io::Error::Hasher(err)) => return Err(checksum::Error::Hasher(err)), + }; + actual.verify(&expected)?; + Ok(actual) + } +} diff --git a/knot2/third_party/gix-pack/src/data/header.rs b/knot2/third_party/gix-pack/src/data/header.rs new file mode 100644 index 0000000000000000000000000000000000000000..14a90df25cee3e37856a948ebf157aaac3608f84 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/header.rs @@ -0,0 +1,57 @@ +use crate::data; + +pub(crate) const N32_SIZE: usize = std::mem::size_of::(); + +/// Parses the first 12 bytes of a pack file, returning the pack version as well as the number of objects contained in the pack. +pub fn decode(data: &[u8; 12]) -> Result<(data::Version, u32), decode::Error> { + let mut ofs = 0; + if &data[ofs..ofs + b"PACK".len()] != b"PACK" { + return Err(decode::Error::Corrupt( + "Pack data type not recognized".into(), + )); + } + ofs += N32_SIZE; + let kind = match crate::read_u32(&data[ofs..ofs + N32_SIZE]) { + 2 => data::Version::V2, + 3 => data::Version::V3, + v => return Err(decode::Error::UnsupportedVersion(v)), + }; + ofs += N32_SIZE; + let num_objects = crate::read_u32(&data[ofs..ofs + N32_SIZE]); + + Ok((kind, num_objects)) +} + +/// Write a pack data header at `version` with `num_objects` and return a buffer. +pub fn encode(version: data::Version, num_objects: u32) -> [u8; 12] { + use crate::data::Version::*; + let mut buf = [0u8; 12]; + buf[..4].copy_from_slice(b"PACK"); + buf[4..8].copy_from_slice( + &match version { + V2 => 2u32, + V3 => 3, + } + .to_be_bytes()[..], + ); + buf[8..].copy_from_slice(&num_objects.to_be_bytes()[..]); + buf +} + +/// +pub mod decode { + /// Returned by [`decode()`][super::decode()]. + #[derive(thiserror::Error, Debug)] + #[allow(missing_docs)] + pub enum Error { + #[error("Could not open pack file at '{path}'")] + Io { + source: std::io::Error, + path: std::path::PathBuf, + }, + #[error("{0}")] + Corrupt(String), + #[error("Unsupported pack version: {0}")] + UnsupportedVersion(u32), + } +} diff --git a/knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs b/knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs new file mode 100644 index 0000000000000000000000000000000000000000..a0f02337c6f809439c20dd5b12dbd40659d4f0a3 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs @@ -0,0 +1,332 @@ +use std::{fs, io}; + +use gix_features::zlib::Decompress; +use gix_hash::{Hasher, ObjectId}; + +use crate::data::input; + +/// An iterator over [`Entries`][input::Entry] in a byte stream. +/// +/// The iterator used as part of [`Bundle::write_to_directory(…)`][crate::Bundle::write_to_directory()]. +pub struct BytesToEntriesIter
    { + read: BR, + decompressor: Decompress, + offset: u64, + had_error: bool, + version: crate::data::Version, + objects_left: u32, + hash: Option, + mode: input::Mode, + compressed: input::EntryDataMode, + compressed_buf: Option>, + hash_len: usize, + object_hash: gix_hash::Kind, +} + +/// Access +impl
    BytesToEntriesIter
    { + /// The pack version currently being iterated + pub fn version(&self) -> crate::data::Version { + self.version + } + + /// The kind of iteration + pub fn mode(&self) -> input::Mode { + self.mode + } +} + +/// Initialization +impl
    BytesToEntriesIter
    +where + BR: io::BufRead, +{ + /// Obtain an iterator from a `read` stream to a pack data file and configure it using `mode` and `compressed`. + /// `object_hash` specifies which hash is used for objects in ref-delta entries. + /// + /// Note that `read` is expected at the beginning of a valid pack data file with a header, entries and a trailer. + pub fn new_from_header( + mut read: BR, + mode: input::Mode, + compressed: input::EntryDataMode, + object_hash: gix_hash::Kind, + ) -> Result, input::Error> { + let mut header_data = [0u8; 12]; + read.read_exact(&mut header_data) + .map_err(gix_hash::io::Error::from)?; + + let (version, num_objects) = crate::data::header::decode(&header_data)?; + match version { + crate::data::Version::V2 => {} + crate::data::Version::V3 => { + return Err(crate::data::header::decode::Error::UnsupportedVersion(3).into()); + } + } + Ok(BytesToEntriesIter { + read, + decompressor: Decompress::new(), + compressed, + offset: 12, + had_error: false, + version, + objects_left: num_objects, + hash: (mode != input::Mode::AsIs).then(|| { + let mut hash = gix_hash::hasher(object_hash); + hash.update(&header_data); + hash + }), + mode, + compressed_buf: None, + hash_len: object_hash.len_in_bytes(), + object_hash, + }) + } + + fn next_inner(&mut self) -> Result { + self.objects_left -= 1; // even an error counts as objects + + // Read header + let entry = match self.hash.as_mut() { + Some(hash) => { + let mut read = read_and_pass_to( + &mut self.read, + HashWrite { + inner: io::sink(), + hash, + }, + ); + crate::data::Entry::from_read(&mut read, self.offset, self.hash_len) + } + None => crate::data::Entry::from_read(&mut self.read, self.offset, self.hash_len), + } + .map_err(gix_hash::io::Error::from)?; + + // Decompress object to learn its compressed bytes + let compressed_buf = self + .compressed_buf + .take() + .unwrap_or_else(|| Vec::with_capacity(4096)); + self.decompressor.reset(); + let mut decompressed_reader = DecompressRead { + inner: read_and_pass_to( + &mut self.read, + if self.compressed.keep() { + Vec::with_capacity(entry.decompressed_size.min(65_536) as usize) + } else { + compressed_buf + }, + ), + decompressor: &mut self.decompressor, + }; + + let bytes_copied = io::copy(&mut decompressed_reader, &mut io::sink()) + .map_err(gix_hash::io::Error::from)?; + if bytes_copied != entry.decompressed_size { + return Err(input::Error::IncompletePack { + actual: bytes_copied, + expected: entry.decompressed_size, + }); + } + + let pack_offset = self.offset; + let compressed_size = decompressed_reader.decompressor.total_in(); + self.offset += entry.header_size() as u64 + compressed_size; + + let mut compressed = decompressed_reader.inner.write; + debug_assert_eq!( + compressed_size, + compressed.len() as u64, + "we must track exactly the same amount of bytes as read by the decompressor" + ); + if let Some(hash) = self.hash.as_mut() { + hash.update(&compressed); + } + + let crc32 = if self.compressed.crc32() { + let mut header_buf = [0u8; 12 + gix_hash::Kind::longest().len_in_bytes()]; + let header_len = entry + .header + .write_to(bytes_copied, &mut header_buf.as_mut()) + .map_err(gix_hash::io::Error::from)?; + let state = gix_features::hash::crc32_update(0, &header_buf[..header_len]); + Some(gix_features::hash::crc32_update(state, &compressed)) + } else { + None + }; + + let compressed = if self.compressed.keep() { + Some(compressed) + } else { + compressed.clear(); + self.compressed_buf = Some(compressed); + None + }; + + // Last objects gets trailer (which is potentially verified) + let trailer = self.try_read_trailer()?; + Ok(input::Entry { + header: entry.header, + header_size: entry.header_size() as u16, + compressed, + compressed_size, + crc32, + pack_offset, + decompressed_size: bytes_copied, + trailer, + }) + } + + fn try_read_trailer(&mut self) -> Result, input::Error> { + Ok(if self.objects_left == 0 { + let mut id = gix_hash::ObjectId::null(self.object_hash); + if let Err(err) = self.read.read_exact(id.as_mut_slice()) { + if self.mode != input::Mode::Restore { + return Err(input::Error::Io(err.into())); + } + } + + if let Some(hash) = self.hash.take() { + let actual_id = hash.try_finalize().map_err(gix_hash::io::Error::from)?; + if self.mode == input::Mode::Restore { + id = actual_id; + } else { + actual_id.verify(&id)?; + } + } + Some(id) + } else if self.mode == input::Mode::Restore { + let hash = self.hash.clone().expect("in restore mode a hash is set"); + Some(hash.try_finalize().map_err(gix_hash::io::Error::from)?) + } else { + None + }) + } +} + +fn read_and_pass_to(read: &mut R, to: W) -> PassThrough<&mut R, W> { + PassThrough { read, write: to } +} + +impl Iterator for BytesToEntriesIter +where + R: io::BufRead, +{ + type Item = Result; + + fn next(&mut self) -> Option { + if self.had_error || self.objects_left == 0 { + return None; + } + let result = self.next_inner(); + self.had_error = result.is_err(); + if self.had_error { + self.objects_left = 0; + } + if self.mode == input::Mode::Restore && self.had_error { + None + } else { + Some(result) + } + } + + fn size_hint(&self) -> (usize, Option) { + (self.objects_left as usize, Some(self.objects_left as usize)) + } +} + +impl std::iter::ExactSizeIterator for BytesToEntriesIter where R: io::BufRead {} + +struct PassThrough { + read: R, + write: W, +} + +impl io::BufRead for PassThrough +where + Self: io::Read, + R: io::BufRead, + W: io::Write, +{ + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.read.fill_buf() + } + + fn consume(&mut self, amt: usize) { + let buf = self + .read + .fill_buf() + .expect("never fail as we called fill-buf before and this does nothing"); + self.write + .write_all(&buf[..amt]) + .expect("a write to never fail - should be a memory buffer"); + self.read.consume(amt); + } +} + +impl io::Read for PassThrough +where + W: io::Write, + R: io::Read, +{ + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let bytes_read = self.read.read(buf)?; + self.write.write_all(&buf[..bytes_read])?; + Ok(bytes_read) + } +} + +impl crate::data::File { + /// Returns an iterator over [`Entries`][crate::data::input::Entry], without making use of the memory mapping. + pub fn streaming_iter(&self) -> Result, input::Error> { + let reader = io::BufReader::with_capacity( + 4096 * 8, + fs::File::open(&self.path).map_err(gix_hash::io::Error::from)?, + ); + BytesToEntriesIter::new_from_header( + reader, + input::Mode::Verify, + input::EntryDataMode::KeepAndCrc32, + self.object_hash, + ) + } +} + +/// The boxed variant is faster for what we do (moving the decompressor in and out a lot) +pub struct DecompressRead<'a, R> { + /// The reader from which bytes should be decompressed. + pub inner: R, + /// The decompressor doing all the work. + pub decompressor: &'a mut Decompress, +} + +impl io::Read for DecompressRead<'_, R> +where + R: io::BufRead, +{ + fn read(&mut self, into: &mut [u8]) -> io::Result { + gix_features::zlib::stream::inflate::read(&mut self.inner, self.decompressor, into) + } +} + +/// A utility to automatically generate a hash while writing into an inner writer. +pub struct HashWrite<'a, T> { + /// The hash implementation. + pub hash: &'a mut Hasher, + /// The inner writer. + pub inner: T, +} + +impl std::io::Write for HashWrite<'_, T> +where + T: std::io::Write, +{ + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.inner.write(buf)?; + self.hash.update(&buf[..written]); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} diff --git a/knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs b/knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs new file mode 100644 index 0000000000000000000000000000000000000000..8aa05edc1ffd7bc0faf28fab403e21d08f84fba2 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs @@ -0,0 +1,162 @@ +use std::iter::Peekable; + +use crate::data::input; + +/// An implementation of [`Iterator`] to write [encoded entries][input::Entry] to an inner implementation each time +/// `next()` is called. +/// +/// It is able to deal with an unknown amount of objects as it will rewrite the pack header once the entries iterator +/// is depleted and compute the hash in one go by re-reading the whole file. +pub struct EntriesToBytesIter { + /// An iterator for input [`input::Entry`] instances + pub input: Peekable, + /// A way of writing encoded bytes. + output: W, + /// Our trailing hash when done writing all input entries + trailer: Option, + /// The amount of objects in the iteration and the version of the packfile to be written. + /// Will be `None` to signal the header was written already. + data_version: crate::data::Version, + /// The amount of entries seen so far + num_entries: u32, + /// If we are done, no additional writes will occur + is_done: bool, + /// The kind of hash to use for the digest + object_hash: gix_hash::Kind, +} + +impl EntriesToBytesIter +where + I: Iterator>, + W: std::io::Read + std::io::Write + std::io::Seek, +{ + /// Create a new instance reading [entries][input::Entry] from an `input` iterator and write pack data bytes to + /// `output` writer, resembling a pack of `version`. The amount of entries will be dynamically determined and + /// the pack is completed once the last entry was written. + /// `object_hash` is the kind of hash to use for the pack checksum and maybe other places, depending on the version. + /// + /// # Panics + /// + /// Only [Version::V2](crate::data::Version::V2) is allowed for `version. + pub fn new( + input: I, + output: W, + version: crate::data::Version, + object_hash: gix_hash::Kind, + ) -> Self { + assert!( + matches!(version, crate::data::Version::V2), + "currently only pack version 2 can be written", + ); + EntriesToBytesIter { + input: input.peekable(), + output, + object_hash, + num_entries: 0, + trailer: None, + data_version: version, + is_done: false, + } + } + + /// Returns the trailing hash over all ~ entries once done. + /// It's `None` if we are not yet done writing. + pub fn digest(&self) -> Option { + self.trailer + } + + fn next_inner(&mut self, entry: input::Entry) -> Result { + if self.num_entries == 0 { + let header_bytes = crate::data::header::encode(self.data_version, 0); + self.output.write_all(&header_bytes[..])?; + } + self.num_entries += 1; + entry + .header + .write_to(entry.decompressed_size, &mut self.output)?; + self.output.write_all( + entry + .compressed + .as_deref() + .expect("caller must configure generator to keep compressed bytes"), + )?; + Ok(entry) + } + + fn write_header_and_digest( + &mut self, + last_entry: Option<&mut input::Entry>, + ) -> Result<(), gix_hash::io::Error> { + let header_bytes = crate::data::header::encode(self.data_version, self.num_entries); + let num_bytes_written = if last_entry.is_some() { + self.output.stream_position()? + } else { + header_bytes.len() as u64 + }; + self.output.rewind()?; + self.output.write_all(&header_bytes[..])?; + self.output.flush()?; + + self.output.rewind()?; + let interrupt_never = std::sync::atomic::AtomicBool::new(false); + let digest = gix_hash::bytes( + &mut self.output, + num_bytes_written, + self.object_hash, + &mut gix_features::progress::Discard, + &interrupt_never, + )?; + self.output.write_all(digest.as_slice())?; + self.output.flush()?; + + self.is_done = true; + if let Some(last_entry) = last_entry { + last_entry.trailer = Some(digest); + } + self.trailer = Some(digest); + Ok(()) + } +} + +impl Iterator for EntriesToBytesIter +where + I: Iterator>, + W: std::io::Read + std::io::Write + std::io::Seek, +{ + /// The amount of bytes written to `out` if `Ok` or the error `E` received from the input. + type Item = Result; + + fn next(&mut self) -> Option { + if self.is_done { + return None; + } + + match self.input.next() { + Some(res) => Some(match res { + Ok(entry) => self + .next_inner(entry) + .and_then(|mut entry| { + if self.input.peek().is_none() { + self.write_header_and_digest(Some(&mut entry)) + .map(|_| entry) + } else { + Ok(entry) + } + }) + .map_err(input::Error::from), + Err(err) => { + self.is_done = true; + Err(err) + } + }), + None => match self.write_header_and_digest(None) { + Ok(_) => None, + Err(err) => Some(Err(err.into())), + }, + } + } + + fn size_hint(&self) -> (usize, Option) { + self.input.size_hint() + } +} diff --git a/knot2/third_party/gix-pack/src/data/input/entry.rs b/knot2/third_party/gix-pack/src/data/input/entry.rs new file mode 100644 index 0000000000000000000000000000000000000000..f59aee334fd58ac71f450eb1b47e06f7969329c3 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/entry.rs @@ -0,0 +1,68 @@ +use std::io::Write; + +use crate::data::{entry::Header, input}; + +impl input::Entry { + /// Create a new input entry from a given data `obj` set to be placed at the given `pack_offset`. + /// + /// This method is useful when arbitrary base entries are created + pub fn from_data_obj( + obj: &gix_object::Data<'_>, + pack_offset: u64, + ) -> Result { + let header = to_header(obj.kind); + let compressed = compress_data(obj)?; + let compressed_size = compressed.len() as u64; + let mut entry = input::Entry { + header, + header_size: header.size(obj.data.len() as u64) as u16, + pack_offset, + compressed: Some(compressed), + compressed_size, + crc32: None, + decompressed_size: obj.data.len() as u64, + trailer: None, + }; + entry.crc32 = Some(entry.compute_crc32()); + Ok(entry) + } + /// The amount of bytes this entry may consume in a pack data file + pub fn bytes_in_pack(&self) -> u64 { + u64::from(self.header_size) + self.compressed_size + } + + /// Update our CRC value by recalculating it from our header and compressed data. + pub fn compute_crc32(&self) -> u32 { + let mut header_buf = [0u8; 12 + gix_hash::Kind::longest().len_in_bytes()]; + let header_len = self + .header + .write_to(self.decompressed_size, &mut header_buf.as_mut()) + .expect("write to memory will not fail"); + let state = gix_features::hash::crc32_update(0, &header_buf[..header_len]); + gix_features::hash::crc32_update(state, self.compressed.as_ref().expect("we always set it")) + } +} + +fn to_header(kind: gix_object::Kind) -> Header { + use gix_object::Kind::*; + match kind { + Tree => Header::Tree, + Blob => Header::Blob, + Commit => Header::Commit, + Tag => Header::Tag, + } +} + +fn compress_data(obj: &gix_object::Data<'_>) -> Result, input::Error> { + let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new()); + if let Err(err) = std::io::copy(&mut &*obj.data, &mut out) { + match err.kind() { + std::io::ErrorKind::Other => return Err(input::Error::Io(err.into())), + err => { + unreachable!("Should never see other errors than zlib, but got {:?}", err) + } + } + } + out.flush().expect("zlib flush should never fail"); + Ok(out.into_inner()) +} diff --git a/knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs b/knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs new file mode 100644 index 0000000000000000000000000000000000000000..2356d0aa36fc804268c22f24dd561dd44c134ced --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs @@ -0,0 +1,236 @@ +use gix_hash::ObjectId; + +use crate::data::{entry::Header, input}; + +/// An iterator to resolve thin packs on the fly. +pub struct LookupRefDeltaObjectsIter { + /// The inner iterator whose entries we will resolve. + pub inner: I, + lookup: Find, + /// The cached delta to provide next time we are called, it's the delta to go with the base we just resolved in its place. + next_delta: Option, + /// Fuse to stop iteration after first missing object. + error: bool, + /// The overall pack-offset we accumulated thus far. Each inserted entry offsets all following + /// objects by its length. We need to determine exactly where the object was inserted to see if its affected at all. + inserted_entry_length_at_offset: Vec, + /// The sum of all entries added so far, as a cache to avoid recomputation + inserted_entries_length_in_bytes: i64, + buf: Vec, +} + +impl LookupRefDeltaObjectsIter +where + I: Iterator>, + Find: gix_object::Find, +{ + /// Create a new instance wrapping `iter` and using `lookup` as function to retrieve objects that will serve as bases + /// for ref deltas seen while traversing `iter`. + pub fn new(iter: I, lookup: Find) -> Self { + LookupRefDeltaObjectsIter { + inner: iter, + lookup, + error: false, + inserted_entry_length_at_offset: Vec::new(), + inserted_entries_length_in_bytes: 0, + next_delta: None, + buf: Vec::new(), + } + } + + fn shifted_pack_offset(&self, pack_offset: u64) -> u64 { + let new_ofs = pack_offset as i64 + self.inserted_entries_length_in_bytes; + new_ofs + .try_into() + .expect("offset value is never becomes negative") + } + + /// positive `size_change` values mean an object grew or was more commonly, was inserted. Negative values + /// mean the object shrunk, usually because there header changed from ref-deltas to ofs deltas. + fn track_change( + &mut self, + shifted_pack_offset: u64, + pack_offset: u64, + size_change: i64, + oid: Option, + ) { + if size_change == 0 { + return; + } + self.inserted_entry_length_at_offset.push(Change { + shifted_pack_offset, + pack_offset, + size_change_in_bytes: size_change, + oid: oid.unwrap_or_else(|| + // NOTE: this value acts as sentinel and the actual hash kind doesn't matter. + gix_hash::Kind::shortest().null()), + }); + self.inserted_entries_length_in_bytes += size_change; + } + + fn shift_entry_and_point_to_base_by_offset( + &mut self, + entry: &mut input::Entry, + base_distance: u64, + ) { + let pack_offset = entry.pack_offset; + entry.pack_offset = self.shifted_pack_offset(pack_offset); + entry.header = Header::OfsDelta { base_distance }; + let previous_header_size = entry.header_size; + entry.header_size = entry.header.size(entry.decompressed_size) as u16; + + let change = i64::from(entry.header_size) - i64::from(previous_header_size); + entry.crc32 = Some(entry.compute_crc32()); + self.track_change(entry.pack_offset, pack_offset, change, None); + } +} + +impl Iterator for LookupRefDeltaObjectsIter +where + I: Iterator>, + Find: gix_object::Find, +{ + type Item = Result; + + fn next(&mut self) -> Option { + if self.error { + return None; + } + if let Some(delta) = self.next_delta.take() { + return Some(Ok(delta)); + } + match self.inner.next() { + Some(Ok(mut entry)) => match entry.header { + Header::RefDelta { base_id } => { + match self + .inserted_entry_length_at_offset + .iter() + .rfind(|e| e.oid == base_id) + { + None => { + let base_entry = + match self.lookup.try_find(&base_id, &mut self.buf).ok()? { + Some(obj) => { + let current_pack_offset = entry.pack_offset; + let mut entry = match input::Entry::from_data_obj(&obj, 0) { + Ok(e) => e, + Err(err) => return Some(Err(err)), + }; + entry.pack_offset = + self.shifted_pack_offset(current_pack_offset); + self.track_change( + entry.pack_offset, + current_pack_offset, + entry.bytes_in_pack() as i64, + Some(base_id), + ); + entry + } + None => { + self.error = true; + return Some(Err(input::Error::NotFound { + object_id: base_id, + })); + } + }; + + { + self.shift_entry_and_point_to_base_by_offset( + &mut entry, + base_entry.bytes_in_pack(), + ); + self.next_delta = Some(entry); + } + Some(Ok(base_entry)) + } + Some(base_entry) => { + let base_distance = self.shifted_pack_offset(entry.pack_offset) + - base_entry.shifted_pack_offset; + self.shift_entry_and_point_to_base_by_offset(&mut entry, base_distance); + Some(Ok(entry)) + } + } + } + _ => { + if self.inserted_entries_length_in_bytes != 0 { + if let Header::OfsDelta { base_distance } = entry.header { + // We have to find the new distance based on the previous distance to the base, using the absolute + // pack offset computed from it as stored in `base_pack_offset`. + let base_pack_offset = entry + .pack_offset + .checked_sub(base_distance) + .expect("distance to be in range of pack"); + match self + .inserted_entry_length_at_offset + .binary_search_by_key(&base_pack_offset, |c| c.pack_offset) + { + Ok(index) => { + let index = { + let maybe_index_of_actual_entry = index + 1; + self.inserted_entry_length_at_offset + .get(maybe_index_of_actual_entry) + .and_then(|c| { + (c.pack_offset == base_pack_offset) + .then_some(maybe_index_of_actual_entry) + }) + .unwrap_or(index) + }; + let new_distance = self + .shifted_pack_offset(entry.pack_offset) + .checked_sub( + self.inserted_entry_length_at_offset[index] + .shifted_pack_offset, + ) + .expect("a base that is behind us in the pack"); + self.shift_entry_and_point_to_base_by_offset( + &mut entry, + new_distance, + ); + } + Err(index) => { + let change_since_offset = self.inserted_entry_length_at_offset + [index..] + .iter() + .map(|c| c.size_change_in_bytes) + .sum::(); + let new_distance: u64 = { + (base_distance as i64 + change_since_offset) + .try_into() + .expect("it still points behind us") + }; + self.shift_entry_and_point_to_base_by_offset( + &mut entry, + new_distance, + ); + } + } + } else { + // Offset this entry by all changes (positive or negative) that we saw thus far. + entry.pack_offset = self.shifted_pack_offset(entry.pack_offset); + } + } + Some(Ok(entry)) + } + }, + other => other, + } + } + + fn size_hint(&self) -> (usize, Option) { + let (min, max) = self.inner.size_hint(); + max.map_or_else(|| (min * 2, None), |max| (min, Some(max * 2))) + } +} + +#[derive(Debug)] +struct Change { + /// The original pack offset as mentioned in the entry we saw. This is used to find this as base object if deltas refer to it by + /// old offset. + pack_offset: u64, + /// The new pack offset that is the shifted location of the pack entry in the pack. + shifted_pack_offset: u64, + /// The size change of the entry header, negative values denote shrinking, positive denote growing. + size_change_in_bytes: i64, + /// The object id of the entry responsible for the change, or null if it's an entry just for tracking an insertion. + oid: ObjectId, +} diff --git a/knot2/third_party/gix-pack/src/data/input/mod.rs b/knot2/third_party/gix-pack/src/data/input/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..a6476fd7a9a921ea7dd82845adac9ae483dec36c --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/mod.rs @@ -0,0 +1,41 @@ +/// An item of the iteration produced by [`BytesToEntriesIter`] +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Entry { + /// The header of a pack entry + pub header: crate::data::entry::Header, + /// The amount of bytes used to encode the `header`. `pack_offset + header_size` is the beginning of + /// the compressed data in the pack. + pub header_size: u16, + /// The first byte of the entry at which the `header` can be read. + pub pack_offset: u64, + /// The bytes consumed while producing `decompressed` + /// These do not contain the header, which makes it possible to easily replace a RefDelta with offset deltas + /// when resolving thin packs. + /// Depends on `CompressionMode` when the iterator is initialized. + pub compressed: Option>, + /// The amount of bytes the compressed portion of the entry takes, i.e. the portion behind the header. + pub compressed_size: u64, + /// The CRC32 over the complete entry, that is encoded header and compressed object data. + /// Depends on `CompressionMode` when the iterator is initialized + pub crc32: Option, + /// The amount of decompressed bytes of the entry. + pub decompressed_size: u64, + /// Set for the last object in the iteration, providing the hash over all bytes of the iteration + /// for use as trailer in a pack or to verify it matches the trailer. + pub trailer: Option, +} + +mod entry; + +mod types; +pub use types::{EntryDataMode, Error, Mode}; + +mod bytes_to_entries; +pub use bytes_to_entries::BytesToEntriesIter; + +mod lookup_ref_delta_objects; +pub use lookup_ref_delta_objects::LookupRefDeltaObjectsIter; + +mod entries_to_bytes; +pub use entries_to_bytes::EntriesToBytesIter; diff --git a/knot2/third_party/gix-pack/src/data/input/types.rs b/knot2/third_party/gix-pack/src/data/input/types.rs new file mode 100644 index 0000000000000000000000000000000000000000..1445e9f5ca1930314f34ad6fa95ca28ae38d13a7 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/types.rs @@ -0,0 +1,70 @@ +/// Returned by [`BytesToEntriesIter::new_from_header()`][crate::data::input::BytesToEntriesIter::new_from_header()] and as part +/// of `Item` of [`BytesToEntriesIter`][crate::data::input::BytesToEntriesIter]. +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("An IO operation failed while streaming an entry")] + Io(#[from] gix_hash::io::Error), + #[error(transparent)] + PackParse(#[from] crate::data::header::decode::Error), + #[error("Failed to verify pack checksum in trailer")] + Verify(#[from] gix_hash::verify::Error), + #[error( + "pack is incomplete: it was decompressed into {actual} bytes but {expected} bytes where expected." + )] + IncompletePack { actual: u64, expected: u64 }, + #[error("The object {object_id} could not be decoded or wasn't found")] + NotFound { object_id: gix_hash::ObjectId }, +} + +/// Iteration Mode +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Mode { + /// Provide the trailer as read from the pack + AsIs, + /// Generate an own hash and trigger an error on the last iterated object + /// if it does not match the hash provided with the pack. + /// + /// This way the one iterating the data cannot miss corruption as long as + /// the iteration is continued through to the end. + Verify, + /// Generate an own hash and if there was an error or the objects are depleted early + /// due to partial packs, return the last valid entry and with our own hash thus far. + /// Note that the existing pack hash, if present, will be ignored. + /// As we won't know which objects fails, every object will have the hash obtained thus far. + /// This also means that algorithms must know about this possibility, or else might wrongfully + /// assume the pack is finished. + Restore, +} + +/// Define what to do with the compressed bytes portion of a pack [`Entry`][super::Entry] +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum EntryDataMode { + /// Do nothing with the compressed bytes we read + Ignore, + /// Only create a CRC32 of the entry, otherwise similar to `Ignore` + Crc32, + /// Keep them and pass them along in a newly allocated buffer + Keep, + /// As above, but also compute a CRC32 + KeepAndCrc32, +} + +impl EntryDataMode { + /// Returns true if a crc32 should be computed + pub fn crc32(&self) -> bool { + match self { + EntryDataMode::KeepAndCrc32 | EntryDataMode::Crc32 => true, + EntryDataMode::Keep | EntryDataMode::Ignore => false, + } + } + /// Returns true if compressed bytes should be kept + pub fn keep(&self) -> bool { + match self { + EntryDataMode::Keep | EntryDataMode::KeepAndCrc32 => true, + EntryDataMode::Ignore | EntryDataMode::Crc32 => false, + } + } +} diff --git a/knot2/third_party/gix-pack/src/data/mod.rs b/knot2/third_party/gix-pack/src/data/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..3bbb37f0c4a77bcbdfb91943fa480bff3eebd98b --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/mod.rs @@ -0,0 +1,175 @@ +//! a pack data file +use std::path::Path; + +/// The offset to an entry into the pack data file, relative to its beginning. +pub type Offset = u64; + +/// An identifier to uniquely identify all packs loaded within a known context or namespace. +pub type Id = u32; + +/// An representing an full- or delta-object within a pack +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Entry { + /// The entry's header + pub header: entry::Header, + /// The decompressed size of the entry in bytes. + /// + /// Note that for non-delta entries this will be the size of the object itself. + pub decompressed_size: u64, + /// absolute offset to compressed object data in the pack, just behind the entry's header + pub data_offset: Offset, +} + +mod file; +pub use file::{Header, decode, verify}; +/// +pub mod header; + +/// +pub mod init { + pub use super::header::decode::Error; +} + +/// +pub mod entry; + +/// +#[cfg(feature = "streaming-input")] +pub mod input; + +/// Utilities to encode pack data entries and write them to a `Write` implementation to resemble a pack data file. +#[cfg(feature = "generate")] +pub mod output; + +/// A slice into a pack file denoting a pack entry. +/// +/// An entry can be decoded into an object. +pub type EntryRange = std::ops::Range; + +/// Supported versions of a pack data file +#[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Version { + /// The default pack data version. + /// + /// This is the version generated by Git and by `gix-pack` writers. + #[default] + V2, + /// A pack data version accepted by Git and recognized by `gix-pack` readers. + /// + /// Git does not generate this version, and `gix-pack` writers currently reject it. + /// Entries are decoded with the same layout as [`V2`](Version::V2); the difference + /// visible to this crate is the version number stored in the pack header. + V3, +} + +/// A pack data file, read from disk on demand rather than held in memory. +#[allow(missing_docs)] +pub struct File { + file: std::fs::File, + len: usize, + path: std::path::PathBuf, + pub id: Id, + version: Version, + num_objects: u32, + hash_len: usize, + object_hash: gix_hash::Kind, + alloc_limit_bytes: Option, +} + +/// Information about the pack data file itself +impl File { + /// The pack data version of this file + pub fn version(&self) -> Version { + self.version + } + /// The number of objects stored in this pack data file + pub fn num_objects(&self) -> u32 { + self.num_objects + } + /// The length of all pack data, including the pack header and the pack trailer + pub fn data_len(&self) -> usize { + self.len + } + /// The kind of hash we use internally. + pub fn object_hash(&self) -> gix_hash::Kind { + self.object_hash + } + /// The maximum size of a single allocation caused by user-controlled on-disk pack data. + /// + /// A value of `None` means no additional limit is enforced. + pub fn alloc_limit_bytes(&self) -> Option { + self.alloc_limit_bytes + } + /// The position of the byte one past the last pack entry, or in other terms, the first byte of the trailing hash. + pub fn pack_end(&self) -> usize { + self.len - self.hash_len + } + + /// The path to the pack data file on disk + pub fn path(&self) -> &Path { + &self.path + } + + pub(crate) fn read_exact_at(&self, offset: usize, buf: &mut [u8]) -> std::io::Result<()> { + use std::os::unix::fs::FileExt; + self.file.read_exact_at(buf, offset as u64) + } + + pub(crate) fn read_span(&self, slice: EntryRange) -> Option> { + let start = usize::try_from(slice.start).ok()?; + let end = usize::try_from(slice.end).ok()?; + if start > end || end > self.len { + return None; + } + let mut buf = vec![0u8; end - start]; + self.read_exact_at(start, &mut buf).ok()?; + Some(buf) + } + + #[allow(missing_docs)] + pub fn read_into(&self, slice: EntryRange, buf: &mut Vec) -> bool { + let (Ok(start), Ok(end)) = (usize::try_from(slice.start), usize::try_from(slice.end)) + else { + return false; + }; + if start > end || end > self.len { + return false; + } + buf.clear(); + buf.resize(end - start, 0); + self.read_exact_at(start, buf).is_ok() + } + + pub(crate) fn materialized(&self) -> std::io::Result { + // SAFETY: we have to take the risk of somebody changing the file underneath. Git never writes into the same file. + #[expect(unsafe_code)] + unsafe { + memmap2::MmapOptions::new().map_copy_read_only(&self.file) + } + } + + /// Returns the pack data at the given slice if its range is contained in the pack data. + pub fn entry_slice(&self, slice: EntryRange) -> Option> { + self.read_span(slice) + } + + /// Returns the CRC32 of the pack data indicated by `pack_offset` and the `size` of the data. + /// + /// _Note:_ finding the right size is only possible by decompressing + /// the pack entry beforehand, or by using the (to be sorted) offsets stored in an index file. + /// + /// # Panics + /// + /// If `pack_offset` or `size` are pointing to a range outside of the pack data. + pub fn entry_crc32(&self, pack_offset: Offset, size: usize) -> u32 { + let buf = self + .read_span(pack_offset..pack_offset + size as u64) + .expect("entry range within pack data"); + gix_features::hash::crc32(&buf) + } +} + +/// +pub mod delta; diff --git a/knot2/third_party/gix-pack/src/data/output/bytes.rs b/knot2/third_party/gix-pack/src/data/output/bytes.rs new file mode 100644 index 0000000000000000000000000000000000000000..0e39d01b9cbff8a876b72de09581602239475de5 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/bytes.rs @@ -0,0 +1,171 @@ +use std::io::Write; + +use crate::{data::output, exact_vec}; + +/// The error returned by `next()` in the [`FromEntriesIter`] iterator. +#[allow(missing_docs)] +#[derive(Debug, thiserror::Error)] +pub enum Error +where + E: std::error::Error + 'static, +{ + #[error(transparent)] + Io(#[from] gix_hash::io::Error), + #[error(transparent)] + Input(E), +} + +/// An implementation of [`Iterator`] to write [encoded entries][output::Entry] to an inner implementation each time +/// `next()` is called. +pub struct FromEntriesIter { + /// An iterator for input [`output::Entry`] instances + pub input: I, + /// A way of writing encoded bytes. + output: gix_hash::io::Write, + /// Our trailing hash when done writing all input entries + trailer: Option, + /// The amount of objects in the iteration and the version of the packfile to be written. + /// Will be `None` to signal the header was written already. + header_info: Option<(crate::data::Version, u32)>, + /// The pack data version with which pack entries should be written. + entry_version: crate::data::Version, + /// The amount of written bytes thus far + written: u64, + /// Required to quickly find offsets by object IDs, as future objects may refer to those in the past to become a delta offset base. + /// It stores the pack offsets at which objects begin. + /// Additionally we store if an object was invalid, and if so we will not write it nor will we allow delta objects to it. + pack_offsets_and_validity: Vec<(u64, bool)>, + /// If we are done, no additional writes will occur + is_done: bool, +} + +impl FromEntriesIter +where + I: Iterator, E>>, + W: std::io::Write, + E: std::error::Error + 'static, +{ + /// Create a new instance reading [entries][output::Entry] from an `input` iterator and write pack data bytes to + /// `output` writer, resembling a pack of `version` with exactly `num_entries` amount of objects contained in it. + /// `object_hash` is the kind of hash to use for the pack checksum and maybe other places, depending on the version. + /// + /// The input chunks are expected to be sorted already. You can use the [`InOrderIter`][gix_features::parallel::InOrderIter] to assure + /// this happens on the fly holding entire chunks in memory as long as needed for them to be dispensed in order. + /// + /// # Panics + /// + /// Not all combinations of `object_hash` and `version` are supported currently triggering assertion errors. + pub fn new( + input: I, + output: W, + num_entries: u32, + version: crate::data::Version, + object_hash: gix_hash::Kind, + ) -> Self { + assert!( + matches!(version, crate::data::Version::V2), + "currently only pack version 2 can be written", + ); + FromEntriesIter { + input, + output: gix_hash::io::Write::new(output, object_hash), + trailer: None, + entry_version: version, + pack_offsets_and_validity: exact_vec(num_entries as usize), + written: 0, + header_info: Some((version, num_entries)), + is_done: false, + } + } + + /// Consume this instance and return the `output` implementation. + /// + /// _Note_ that the `input` iterator can be moved out of this instance beforehand. + pub fn into_write(self) -> W { + self.output.inner + } + + /// Returns the trailing hash over all written entries once done. + /// It's `None` if we are not yet done writing. + pub fn digest(&self) -> Option { + self.trailer + } + + fn next_inner(&mut self) -> Result> { + let previous_written = self.written; + if let Some((version, num_entries)) = self.header_info.take() { + let header_bytes = crate::data::header::encode(version, num_entries); + self.output + .write_all(&header_bytes[..]) + .map_err(gix_hash::io::Error::from)?; + self.written += header_bytes.len() as u64; + } + match self.input.next() { + Some(entries) => { + for entry in entries.map_err(Error::Input)? { + if entry.is_invalid() { + self.pack_offsets_and_validity.push((0, false)); + continue; + } + self.pack_offsets_and_validity.push((self.written, true)); + let header = entry.to_entry_header(self.entry_version, |index| { + let (base_offset, is_valid_object) = self.pack_offsets_and_validity[index]; + if !is_valid_object { + unreachable!("if you see this the object database is correct as a delta refers to a non-existing object") + } + self.written - base_offset + }); + self.written += header + .write_to(entry.decompressed_size as u64, &mut self.output) + .map_err(gix_hash::io::Error::from)? + as u64; + self.written += std::io::copy(&mut &*entry.compressed_data, &mut self.output) + .map_err(gix_hash::io::Error::from)?; + } + } + None => { + let digest = self + .output + .hash + .clone() + .try_finalize() + .map_err(gix_hash::io::Error::from)?; + self.output + .inner + .write_all(digest.as_slice()) + .map_err(gix_hash::io::Error::from)?; + self.written += digest.as_slice().len() as u64; + self.output + .inner + .flush() + .map_err(gix_hash::io::Error::from)?; + self.is_done = true; + self.trailer = Some(digest); + } + } + Ok(self.written - previous_written) + } +} + +impl Iterator for FromEntriesIter +where + I: Iterator, E>>, + W: std::io::Write, + E: std::error::Error + 'static, +{ + /// The amount of bytes written to `out` if `Ok` or the error `E` received from the input. + type Item = Result>; + + fn next(&mut self) -> Option { + if self.is_done { + return None; + } + Some(match self.next_inner() { + Err(err) => { + self.is_done = true; + Err(err) + } + Ok(written) => Ok(written), + }) + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/mod.rs b/knot2/third_party/gix-pack/src/data/output/count/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..bc22343b8782df2c9c0e92a2b7e818341da6f268 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/mod.rs @@ -0,0 +1,57 @@ +use gix_hash::ObjectId; + +use crate::data::output::Count; + +/// Specifies how the pack location was handled during counting +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum PackLocation { + /// We did not lookup this object + NotLookedUp, + /// The object was looked up and there may be a location in a pack, along with entry information + LookedUp(Option), +} + +impl PackLocation { + /// Directly go through to `LookedUp` variant, panic otherwise + pub fn is_none(&self) -> bool { + match self { + PackLocation::LookedUp(opt) => opt.is_none(), + PackLocation::NotLookedUp => unreachable!("must have been resolved"), + } + } + /// Directly go through to `LookedUp` variant, panic otherwise + pub fn as_ref(&self) -> Option<&crate::data::entry::Location> { + match self { + PackLocation::LookedUp(opt) => opt.as_ref(), + PackLocation::NotLookedUp => unreachable!("must have been resolved"), + } + } +} + +impl Count { + /// Create a new instance from the given `oid` and its corresponding location. + pub fn from_data( + oid: impl Into, + location: Option, + ) -> Self { + Count { + id: oid.into(), + entry_pack_location: PackLocation::LookedUp(location), + } + } +} + +#[path = "objects/mod.rs"] +mod objects_impl; +pub use objects_impl::{objects, objects_unthreaded, reachable::reachable}; + +/// +pub mod objects { + pub use super::objects_impl::{Error, ObjectExpansion, Options, Outcome}; +} + +/// +pub mod reachable { + pub use super::objects_impl::reachable::{Error, Outcome}; +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..41a34b556f086f2460fd212ac0405d6426e61921 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs @@ -0,0 +1,501 @@ +use std::{cell::RefCell, sync::atomic::AtomicBool}; + +use gix_features::parallel; +use gix_hash::ObjectId; + +use crate::data::output; + +pub(in crate::data::output::count::objects_impl) mod reduce; +mod util; + +mod types; +pub use types::{Error, ObjectExpansion, Options, Outcome}; + +mod tree; + +pub(in crate::data::output::count) mod reachable; + +/// Generate [`Count`][output::Count]s from input `objects` with object expansion based on [`options`][Options] +/// to learn which objects would constitute a pack. This step is required to know exactly how many objects would +/// be in a pack while keeping data around to minimize database object access. +/// +/// A [`Count`][output::Count] object maintains enough state to greatly accelerate future access of packed objects. +/// +/// * `db` - the object store to use for accessing objects. +/// * `objects_ids` +/// * A list of objects IDs to add to the pack. Duplication checks are performed so no object is ever added to a pack twice. +/// * Objects may be expanded based on the provided [`options`][Options] +/// * `objects` +/// * count the amount of objects we encounter +/// * `should_interrupt` +/// * A flag that is set to true if the operation should stop +/// * `options` +/// * more configuration +pub fn objects( + db: Find, + objects_ids: Box< + dyn Iterator>> + + Send, + >, + objects: &dyn gix_features::progress::Count, + should_interrupt: &AtomicBool, + Options { + thread_limit, + input_object_expansion, + chunk_size, + }: Options, +) -> Result<(Vec, Outcome), Error> +where + Find: crate::Find + Send + Clone, +{ + let lower_bound = objects_ids.size_hint().0; + let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit( + chunk_size, + if lower_bound == 0 { + None + } else { + Some(lower_bound) + }, + thread_limit, + None, + ); + let chunks = gix_features::iter::Chunks { + inner: objects_ids, + size: chunk_size, + }; + let seen_objs = gix_hashtable::sync::ObjectIdMap::default(); + let objects = objects.counter(); + + parallel::in_parallel( + chunks, + thread_limit, + { + move |_| { + ( + Vec::new(), // object data buffer + Vec::new(), // object data buffer 2 to hold two objects at a time + objects.clone(), + ) + } + }, + { + let seen_objs = &seen_objs; + move |oids: Vec<_>, (buf1, buf2, objects)| { + expand::this( + &db, + input_object_expansion, + seen_objs, + &mut oids.into_iter(), + buf1, + buf2, + objects, + should_interrupt, + true, /*allow pack lookups*/ + ) + } + }, + reduce::Statistics::new(), + ) +} + +/// Like [`objects()`] but using a single thread only to mostly save on the otherwise required overhead. +pub fn objects_unthreaded( + db: &dyn crate::Find, + object_ids: &mut dyn Iterator< + Item = Result>, + >, + objects: &dyn gix_features::progress::Count, + should_interrupt: &AtomicBool, + input_object_expansion: ObjectExpansion, +) -> Result<(Vec, Outcome), Error> { + let seen_objs = RefCell::new(gix_hashtable::HashSet::default()); + + let (mut buf1, mut buf2) = (Vec::new(), Vec::new()); + expand::this( + db, + input_object_expansion, + &seen_objs, + object_ids, + &mut buf1, + &mut buf2, + &objects.counter(), + should_interrupt, + false, /*allow pack lookups*/ + ) +} + +mod expand { + use std::{ + cell::RefCell, + sync::atomic::{AtomicBool, Ordering}, + }; + + use gix_hash::{ObjectId, oid}; + use gix_object::{CommitRefIter, Data, TagRefIter}; + + use super::{ + tree, + types::{Error, ObjectExpansion, Outcome}, + util, + }; + use crate::{ + FindExt, + data::{output, output::count::PackLocation}, + }; + + #[allow(clippy::too_many_arguments)] + pub fn this( + db: &dyn crate::Find, + input_object_expansion: ObjectExpansion, + seen_objs: &impl util::InsertImmutable, + oids: &mut dyn Iterator< + Item = Result>, + >, + buf1: &mut Vec, + #[allow(clippy::ptr_arg)] buf2: &mut Vec, + objects: &gix_features::progress::AtomicStep, + should_interrupt: &AtomicBool, + allow_pack_lookups: bool, + ) -> Result<(Vec, Outcome), Error> { + use ObjectExpansion::*; + + let mut out = Vec::new(); + let mut tree_traversal_state = gix_traverse::tree::breadthfirst::State::default(); + let mut tree_diff_state = gix_diff::tree::State::default(); + let mut parent_commit_ids = Vec::new(); + let mut traverse_delegate = tree::traverse::AllUnseen::new(seen_objs); + let mut changes_delegate = tree::changes::AllNew::new(seen_objs); + let mut outcome = Outcome::default(); + + let stats = &mut outcome; + for id in oids { + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + + let id = id.map_err(Error::InputIteration)?; + let (obj, location) = db.find(&id, buf1)?; + stats.input_objects += 1; + match input_object_expansion { + TreeAdditionsComparedToAncestor => { + use gix_object::Kind::*; + let mut obj = obj; + let mut location = location; + let mut id = id.to_owned(); + + loop { + push_obj_count_unique( + &mut out, seen_objs, &id, location, objects, stats, false, + ); + match obj.kind { + Tree | Blob => break, + Tag => { + id = TagRefIter::from_bytes(obj.data, obj.object_hash) + .target_id() + .expect("every tag has a target"); + let tmp = db.find(&id, buf1)?; + + obj = tmp.0; + location = tmp.1; + + stats.expanded_objects += 1; + continue; + } + Commit => { + let current_tree_iter = { + let mut commit_iter = + CommitRefIter::from_bytes(obj.data, obj.object_hash); + let tree_id = + commit_iter.tree_id().expect("every commit has a tree"); + parent_commit_ids.clear(); + for token in commit_iter { + match token { + Ok(gix_object::commit::ref_iter::Token::Parent { + id, + }) => { + parent_commit_ids.push(id); + } + Ok(_) => break, + Err(err) => return Err(Error::CommitDecode(err)), + } + } + let (obj, location) = db.find(&tree_id, buf1)?; + push_obj_count_unique( + &mut out, seen_objs, &tree_id, location, objects, stats, + true, + ); + gix_object::TreeRefIter::from_bytes(obj.data, obj.object_hash) + }; + + let objects_ref = if parent_commit_ids.is_empty() { + traverse_delegate.clear(); + let objects = ExpandedCountingObjects::new(db, out, objects); + gix_traverse::tree::breadthfirst( + current_tree_iter, + &mut tree_traversal_state, + &objects, + &mut traverse_delegate, + ) + .map_err(Error::TreeTraverse)?; + out = objects.dissolve(stats); + &traverse_delegate.non_trees + } else { + for commit_id in &parent_commit_ids { + let parent_tree_id = { + let (parent_commit_obj, location) = + db.find(commit_id, buf2)?; + + push_obj_count_unique( + &mut out, seen_objs, commit_id, location, objects, + stats, true, + ); + CommitRefIter::from_bytes( + parent_commit_obj.data, + parent_commit_obj.object_hash, + ) + .tree_id() + .expect("every commit has a tree") + }; + let parent_tree = { + let (parent_tree_obj, location) = + db.find(&parent_tree_id, buf2)?; + push_obj_count_unique( + &mut out, + seen_objs, + &parent_tree_id, + location, + objects, + stats, + true, + ); + gix_object::TreeRefIter::from_bytes( + parent_tree_obj.data, + parent_tree_obj.object_hash, + ) + }; + + changes_delegate.clear(); + let objects = CountingObjects::new(db); + gix_diff::tree( + parent_tree, + current_tree_iter, + &mut tree_diff_state, + &objects, + &mut changes_delegate, + ) + .map_err(Error::TreeChanges)?; + stats.decoded_objects += objects.into_count(); + } + &changes_delegate.objects + }; + for id in objects_ref.iter() { + out.push(id_to_count( + db, + buf2, + id, + objects, + stats, + allow_pack_lookups, + )); + } + break; + } + } + } + } + TreeContents => { + use gix_object::Kind::*; + let mut id = id; + let mut obj = (obj, location); + loop { + push_obj_count_unique( + &mut out, + seen_objs, + &id, + obj.1.clone(), + objects, + stats, + false, + ); + match obj.0.kind { + Tree => { + traverse_delegate.clear(); + { + let objects = ExpandedCountingObjects::new(db, out, objects); + gix_traverse::tree::breadthfirst( + gix_object::TreeRefIter::from_bytes( + obj.0.data, + obj.0.object_hash, + ), + &mut tree_traversal_state, + &objects, + &mut traverse_delegate, + ) + .map_err(Error::TreeTraverse)?; + out = objects.dissolve(stats); + } + for id in &traverse_delegate.non_trees { + out.push(id_to_count( + db, + buf1, + id, + objects, + stats, + allow_pack_lookups, + )); + } + break; + } + Commit => { + id = CommitRefIter::from_bytes(obj.0.data, obj.0.object_hash) + .tree_id() + .expect("every commit has a tree"); + stats.expanded_objects += 1; + obj = db.find(&id, buf1)?; + continue; + } + Blob => break, + Tag => { + id = TagRefIter::from_bytes(obj.0.data, obj.0.object_hash) + .target_id() + .expect("every tag has a target"); + stats.expanded_objects += 1; + obj = db.find(&id, buf1)?; + continue; + } + } + } + } + AsIs => { + push_obj_count_unique(&mut out, seen_objs, &id, location, objects, stats, false) + } + } + } + outcome.total_objects = out.len(); + Ok((out, outcome)) + } + + #[inline] + fn push_obj_count_unique( + out: &mut Vec, + all_seen: &impl util::InsertImmutable, + id: &oid, + location: Option, + objects: &gix_features::progress::AtomicStep, + statistics: &mut Outcome, + count_expanded: bool, + ) { + let inserted = all_seen.insert(id.to_owned()); + if inserted { + objects.fetch_add(1, Ordering::Relaxed); + statistics.decoded_objects += 1; + if count_expanded { + statistics.expanded_objects += 1; + } + out.push(output::Count::from_data(id, location)); + } + } + + #[inline] + fn id_to_count( + db: &dyn crate::Find, + buf: &mut Vec, + id: &oid, + objects: &gix_features::progress::AtomicStep, + statistics: &mut Outcome, + allow_pack_lookups: bool, + ) -> output::Count { + objects.fetch_add(1, Ordering::Relaxed); + statistics.expanded_objects += 1; + output::Count { + id: id.to_owned(), + entry_pack_location: if allow_pack_lookups { + PackLocation::LookedUp(db.location_by_oid(id, buf)) + } else { + PackLocation::NotLookedUp + }, + } + } + + struct CountingObjects<'a> { + decoded_objects: std::cell::RefCell, + objects: &'a dyn crate::Find, + } + + impl<'a> CountingObjects<'a> { + fn new(objects: &'a dyn crate::Find) -> Self { + Self { + decoded_objects: Default::default(), + objects, + } + } + + fn into_count(self) -> usize { + self.decoded_objects.into_inner() + } + } + + impl gix_object::Find for CountingObjects<'_> { + fn try_find<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + ) -> Result>, gix_object::find::Error> { + let res = Ok(self.objects.try_find(id, buffer)?.map(|t| t.0)); + *self.decoded_objects.borrow_mut() += 1; + res + } + } + + struct ExpandedCountingObjects<'a> { + decoded_objects: std::cell::RefCell, + expanded_objects: std::cell::RefCell, + out: std::cell::RefCell>, + objects_count: &'a gix_features::progress::AtomicStep, + objects: &'a dyn crate::Find, + } + + impl<'a> ExpandedCountingObjects<'a> { + fn new( + objects: &'a dyn crate::Find, + out: Vec, + objects_count: &'a gix_features::progress::AtomicStep, + ) -> Self { + Self { + decoded_objects: Default::default(), + expanded_objects: Default::default(), + out: RefCell::new(out), + objects_count, + objects, + } + } + + fn dissolve(self, stats: &mut Outcome) -> Vec { + stats.decoded_objects += self.decoded_objects.into_inner(); + stats.expanded_objects += self.expanded_objects.into_inner(); + self.out.into_inner() + } + } + + impl gix_object::Find for ExpandedCountingObjects<'_> { + fn try_find<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + ) -> Result>, gix_object::find::Error> { + let maybe_obj = self.objects.try_find(id, buffer)?; + *self.decoded_objects.borrow_mut() += 1; + match maybe_obj { + None => Ok(None), + Some((obj, location)) => { + self.objects_count.fetch_add(1, Ordering::Relaxed); + *self.expanded_objects.borrow_mut() += 1; + self.out + .borrow_mut() + .push(output::Count::from_data(id, location)); + Ok(Some(obj)) + } + } + } + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/reachable.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/reachable.rs new file mode 100644 index 0000000000000000000000000000000000000000..a42fa5223f2235aa0d4bfb585a9623e3a8e96b26 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/reachable.rs @@ -0,0 +1,216 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +use gix_hash::{ObjectId, oid}; +use gix_hashtable::HashSet; +use gix_object::{CommitRefIter, Kind, TagRefIter, TreeRefIter, bstr::BStr, tree::EntryRef}; +use gix_traverse::tree::{Visit, breadthfirst, visit::Action}; + +use crate::{ + FindExt, + data::output::{self, count::PackLocation}, +}; + +/// The error returned by [`reachable()`]. +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error(transparent)] + FindExisting(#[from] gix_object::find::existing::Error), + #[error("want {id} could not be peeled to a commit")] + UnpeelableWant { id: ObjectId }, + #[error(transparent)] + Walk(#[from] gix_traverse::commit::simple::Error), + #[error(transparent)] + TreeTraverse(#[from] breadthfirst::Error), + #[error("Operation interrupted")] + Interrupted, +} + +/// Statistics of a [`reachable()`] run. +#[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +pub struct Outcome { + /// Commits reachable from the wants but not the haves. + pub commits: usize, + /// Commits the receiver has that the new ones have as parents. + pub boundary: usize, + /// The amount of counts returned. + pub objects: usize, + /// Haves that are missing from `db` or don't peel to a commit, and were ignored. + pub ignored_haves: usize, +} + +/// Count the objects reachable from `wants` that a receiver holding `haves` lacks. +pub fn reachable( + db: Find, + wants: impl IntoIterator, + haves: impl IntoIterator, + progress: &dyn gix_features::progress::Count, + should_interrupt: &AtomicBool, +) -> Result<(Vec, Outcome), Error> +where + Find: crate::Find + gix_object::Find, +{ + let progress = progress.counter(); + let mut outcome = Outcome::default(); + let mut buf = Vec::new(); + let mut out = Vec::new(); + + let mut want_commits = Vec::new(); + for id in wants { + want_commits.push( + peel_to_commit(&db, id, &mut buf, Some(&mut out)) + .ok_or(Error::UnpeelableWant { id })?, + ); + } + let mut have_commits = Vec::new(); + for id in haves { + match peel_to_commit(&db, id, &mut buf, None) { + Some(id) => have_commits.push(id), + None => outcome.ignored_haves += 1, + } + } + + let mut commits = Vec::new(); + let mut parents = Vec::new(); + for info in gix_traverse::commit::Simple::new(want_commits, &db).hide(have_commits)? { + let info = info?; + parents.extend(info.parent_ids); + commits.push(info.id); + } + outcome.commits = commits.len(); + + let mut seen: HashSet = commits.iter().copied().collect(); + seen.extend(out.iter().copied()); + let mut state = breadthfirst::State::default(); + for parent in parents { + if seen.insert(parent) { + outcome.boundary += 1; + add_tree( + &db, + &parent, + &mut state, + &mut buf, + &mut Collect { + seen: &mut seen, + out: None, + }, + )?; + } + } + for commit in commits { + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + out.push(commit); + let before = out.len(); + add_tree( + &db, + &commit, + &mut state, + &mut buf, + &mut Collect { + seen: &mut seen, + out: Some(&mut out), + }, + )?; + progress.fetch_add(1 + out.len() - before, Ordering::Relaxed); + } + + outcome.objects = out.len(); + let counts = out + .into_iter() + .map(|id| output::Count { + id, + entry_pack_location: PackLocation::NotLookedUp, + }) + .collect(); + Ok((counts, outcome)) +} + +fn peel_to_commit( + db: &impl crate::Find, + mut id: ObjectId, + buf: &mut Vec, + mut tags: Option<&mut Vec>, +) -> Option { + loop { + let (obj, _) = db.try_find(&id, buf).ok()??; + match obj.kind { + Kind::Commit => return Some(id), + Kind::Tag => { + if let Some(tags) = tags.as_deref_mut() { + tags.push(id); + } + id = TagRefIter::from_bytes(obj.data, obj.object_hash) + .target_id() + .expect("every tag has a target"); + } + _ => return None, + } + } +} + +fn add_tree( + db: &Find, + commit: &oid, + state: &mut breadthfirst::State, + buf: &mut Vec, + collect: &mut Collect<'_>, +) -> Result<(), Error> +where + Find: crate::Find + gix_object::Find, +{ + let (obj, _) = db.find(commit, buf)?; + let tree = CommitRefIter::from_bytes(obj.data, obj.object_hash) + .tree_id() + .expect("every commit has a tree"); + if !collect.add(&tree) { + return Ok(()); + } + let (obj, _) = db.find(&tree, buf)?; + breadthfirst( + TreeRefIter::from_bytes(obj.data, obj.object_hash), + state, + db, + collect, + )?; + Ok(()) +} + +struct Collect<'a> { + seen: &'a mut HashSet, + out: Option<&'a mut Vec>, +} + +impl Collect<'_> { + fn add(&mut self, id: &oid) -> bool { + let new = self.seen.insert(id.to_owned()); + if let (true, Some(out)) = (new, self.out.as_deref_mut()) { + out.push(id.to_owned()); + } + new + } +} + +impl Visit for Collect<'_> { + fn pop_back_tracked_path_and_set_current(&mut self) {} + + fn pop_front_tracked_path_and_set_current(&mut self) {} + + fn push_back_tracked_path_component(&mut self, _component: &BStr) {} + + fn push_path_component(&mut self, _component: &BStr) {} + + fn pop_path_component(&mut self) {} + + fn visit_tree(&mut self, entry: &EntryRef<'_>) -> Action { + std::ops::ControlFlow::Continue(self.add(entry.oid)) + } + + fn visit_nontree(&mut self, entry: &EntryRef<'_>) -> Action { + if !entry.mode.is_commit() { + self.add(entry.oid); + } + std::ops::ControlFlow::Continue(true) + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/reduce.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/reduce.rs new file mode 100644 index 0000000000000000000000000000000000000000..03144b60fd5d6dc52b79229455d957bb369fb390 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/reduce.rs @@ -0,0 +1,40 @@ +use std::marker::PhantomData; + +use gix_features::parallel; + +use super::Outcome; +use crate::data::output; + +pub struct Statistics { + total: Outcome, + counts: Vec, + _err: PhantomData, +} + +impl Statistics { + pub fn new() -> Self { + Statistics { + total: Default::default(), + counts: Default::default(), + _err: PhantomData, + } + } +} + +impl parallel::Reduce for Statistics { + type Input = Result<(Vec, Outcome), E>; + type FeedProduce = (); + type Output = (Vec, Outcome); + type Error = E; + + fn feed(&mut self, item: Self::Input) -> Result { + let (counts, stats) = item?; + self.total.aggregate(stats); + self.counts.extend(counts); + Ok(()) + } + + fn finalize(self) -> Result { + Ok((self.counts, self.total)) + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs new file mode 100644 index 0000000000000000000000000000000000000000..dda98993abc32000e21033be96c87a8ec3098699 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs @@ -0,0 +1,129 @@ +pub mod changes { + use gix_diff::tree::{ + Visit, + visit::{Action, Change}, + }; + use gix_hash::ObjectId; + use gix_object::bstr::BStr; + + use crate::data::output::count::objects_impl::util::InsertImmutable; + + pub struct AllNew<'a, H> { + pub objects: Vec, + all_seen: &'a H, + } + + impl<'a, H> AllNew<'a, H> + where + H: InsertImmutable, + { + pub fn new(all_seen: &'a H) -> Self { + AllNew { + objects: Default::default(), + all_seen, + } + } + pub fn clear(&mut self) { + self.objects.clear(); + } + } + + impl Visit for AllNew<'_, H> + where + H: InsertImmutable, + { + fn pop_front_tracked_path_and_set_current(&mut self) {} + + fn push_back_tracked_path_component(&mut self, _component: &BStr) {} + + fn push_path_component(&mut self, _component: &BStr) {} + + fn pop_path_component(&mut self) {} + + fn visit(&mut self, change: Change) -> Action { + match change { + Change::Addition { + oid, + entry_mode, + relation: _, + } + | Change::Modification { + oid, entry_mode, .. + } => { + if entry_mode.is_commit() { + return std::ops::ControlFlow::Continue(()); + } + let inserted = self.all_seen.insert(oid); + if inserted { + self.objects.push(oid); + } + } + Change::Deletion { .. } => {} + } + std::ops::ControlFlow::Continue(()) + } + } +} + +pub mod traverse { + use gix_hash::ObjectId; + use gix_object::{bstr::BStr, tree::EntryRef}; + use gix_traverse::tree::{Visit, visit::Action}; + + use crate::data::output::count::objects_impl::util::InsertImmutable; + + pub struct AllUnseen<'a, H> { + pub non_trees: Vec, + all_seen: &'a H, + } + + impl<'a, H> AllUnseen<'a, H> + where + H: InsertImmutable, + { + pub fn new(all_seen: &'a H) -> Self { + AllUnseen { + non_trees: Default::default(), + all_seen, + } + } + pub fn clear(&mut self) { + self.non_trees.clear(); + } + } + + impl Visit for AllUnseen<'_, H> + where + H: InsertImmutable, + { + fn pop_back_tracked_path_and_set_current(&mut self) {} + + fn pop_front_tracked_path_and_set_current(&mut self) {} + + fn push_back_tracked_path_component(&mut self, _component: &BStr) {} + + fn push_path_component(&mut self, _component: &BStr) {} + + fn pop_path_component(&mut self) {} + + fn visit_tree(&mut self, entry: &EntryRef<'_>) -> Action { + let inserted = self.all_seen.insert(entry.oid.to_owned()); + if inserted { + std::ops::ControlFlow::Continue(true) + } else { + std::ops::ControlFlow::Continue(false) + } + } + + fn visit_nontree(&mut self, entry: &EntryRef<'_>) -> Action { + if entry.mode.is_commit() { + return std::ops::ControlFlow::Continue(true); + } + let inserted = self.all_seen.insert(entry.oid.to_owned()); + if inserted { + self.non_trees.push(entry.oid.to_owned()); + } + std::ops::ControlFlow::Continue(true) + } + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/types.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/types.rs new file mode 100644 index 0000000000000000000000000000000000000000..4b0bad8d0d9254c890902694ba35ca56044a3c1d --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/types.rs @@ -0,0 +1,96 @@ +/// Information gathered during the run of [`iter_from_objects()`][super::objects()]. +#[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// The amount of objects provided to start the iteration. + pub input_objects: usize, + /// The amount of objects that have been expanded from the input source. + /// It's desirable to do that as expansion happens on multiple threads, allowing the amount of input objects to be small. + /// `expanded_objects - decoded_objects` is the 'cheap' object we found without decoding the object itself. + pub expanded_objects: usize, + /// The amount of fully decoded objects. These are the most expensive as they are fully decoded + pub decoded_objects: usize, + /// The total amount of encountered objects. Should be `expanded_objects + input_objects`. + pub total_objects: usize, +} + +impl Outcome { + pub(in crate::data::output::count) fn aggregate( + &mut self, + Outcome { + input_objects, + decoded_objects, + expanded_objects, + total_objects, + }: Self, + ) { + self.input_objects += input_objects; + self.decoded_objects += decoded_objects; + self.expanded_objects += expanded_objects; + self.total_objects += total_objects; + } +} + +/// The way input objects are handled +#[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum ObjectExpansion { + /// Don't do anything with the input objects except for transforming them into pack entries + #[default] + AsIs, + /// If the input object is a Commit then turn it into a pack entry. Additionally obtain its tree, turn it into a pack entry + /// along with all of its contents, that is nested trees, and any other objects reachable from it. + /// Otherwise, the same as [`AsIs`][ObjectExpansion::AsIs]. + /// + /// This mode is useful if all reachable objects should be added, as in cloning a repository. + TreeContents, + /// If the input is a commit, obtain its ancestors and turn them into pack entries. Obtain the ancestor trees along with the commits + /// tree and turn them into pack entries. Finally obtain the added/changed objects when comparing the ancestor trees with the + /// current tree and turn them into entries as well. + /// Otherwise, the same as [`AsIs`][ObjectExpansion::AsIs]. + /// + /// This mode is useful to build a pack containing only new objects compared to a previous state. + TreeAdditionsComparedToAncestor, +} + +/// Configuration options for the pack generation functions provided in [this module][crate::data::output]. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Options { + /// The amount of threads to use at most when resolving the pack. If `None`, all logical cores are used. + /// If more than one thread is used, the order of returned [counts][crate::data::output::Count] is not deterministic anymore + /// especially when tree traversal is involved. Thus deterministic ordering requires `Some(1)` to be set. + pub thread_limit: Option, + /// The amount of objects per chunk or unit of work to be sent to threads for processing + pub chunk_size: usize, + /// The way input objects are handled + pub input_object_expansion: ObjectExpansion, +} + +impl Default for Options { + fn default() -> Self { + Options { + thread_limit: None, + chunk_size: 10, + input_object_expansion: Default::default(), + } + } +} + +/// The error returned by the pack generation iterator [`bytes::FromEntriesIter`][crate::data::output::bytes::FromEntriesIter]. +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error(transparent)] + CommitDecode(gix_object::decode::Error), + #[error(transparent)] + FindExisting(#[from] gix_object::find::existing::Error), + #[error(transparent)] + InputIteration(Box), + #[error(transparent)] + TreeTraverse(gix_traverse::tree::breadthfirst::Error), + #[error(transparent)] + TreeChanges(gix_diff::tree::Error), + #[error("Operation interrupted")] + Interrupted, +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/util.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/util.rs new file mode 100644 index 0000000000000000000000000000000000000000..b1997e1c7229c551d214ba16c947402f7ddd467b --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/util.rs @@ -0,0 +1,24 @@ +pub trait InsertImmutable { + fn insert(&self, id: gix_hash::ObjectId) -> bool; +} + +mod trait_impls { + use std::cell::RefCell; + + use gix_hash::ObjectId; + use gix_hashtable::HashSet; + + use super::InsertImmutable; + + impl InsertImmutable for gix_hashtable::sync::ObjectIdMap<()> { + fn insert(&self, id: ObjectId) -> bool { + self.insert(id, ()).is_none() + } + } + + impl InsertImmutable for RefCell> { + fn insert(&self, item: ObjectId) -> bool { + self.borrow_mut().insert(item) + } + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs b/knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs new file mode 100644 index 0000000000000000000000000000000000000000..2de3376cc6c2de8482301e9fa3b622eaea050ded --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs @@ -0,0 +1,458 @@ +pub(crate) mod function { + use std::{cmp::Ordering, sync::Arc}; + + use gix_features::{ + parallel, + parallel::SequenceId, + progress::{ + Progress, + prodash::{Count, DynNestedProgress}, + }, + }; + + use super::{Error, Mode, Options, Outcome, ProgressId, reduce, util}; + use crate::data::output; + + /// Given a known list of object `counts`, calculate entries ready to be put into a data pack. + /// + /// This allows objects to be written quite soon without having to wait for the entire pack to be built in memory. + /// A chunk of objects is held in memory and compressed using DEFLATE, and serve the output of this iterator. + /// That way slow writers will naturally apply back pressure, and communicate to the implementation that more time can be + /// spent compressing objects. + /// + /// * `counts` + /// * A list of previously counted objects to add to the pack. Duplication checks are not performed, no object is expected to be duplicated. + /// * `progress` + /// * a way to obtain progress information + /// * `options` + /// * more configuration + /// + /// _Returns_ the checksum of the pack + /// + /// ## Discussion + /// + /// ### Advantages + /// + /// * Begins writing immediately and supports back-pressure. + /// * Abstract over object databases and how input is provided. + /// + /// ### Disadvantages + /// + /// * ~~currently there is no way to easily write the pack index, even though the state here is uniquely positioned to do + /// so with minimal overhead (especially compared to `gix index-from-pack`)~~ Probably works now by chaining Iterators + /// or keeping enough state to write a pack and then generate an index with recorded data. + /// + pub fn iter_from_counts( + mut counts: Vec, + db: Find, + mut progress: Box, + Options { + version, + mode, + allow_thin_pack, + thread_limit, + chunk_size, + }: Options, + ) -> impl Iterator), Error>> + + parallel::reduce::Finalize> + where + Find: crate::Find + Send + Clone + 'static, + { + assert!( + matches!(version, crate::data::Version::V2), + "currently we can only write version 2" + ); + let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit( + chunk_size, + Some(counts.len()), + thread_limit, + None, + ); + { + let progress = Arc::new(parking_lot::Mutex::new( + progress.add_child_with_id("resolving".into(), ProgressId::ResolveCounts.into()), + )); + progress + .lock() + .init(None, gix_features::progress::count("counts")); + let enough_counts_present = counts.len() > 4_000; + let start = std::time::Instant::now(); + parallel::in_parallel_if( + || enough_counts_present, + counts.chunks_mut(chunk_size), + thread_limit, + |_n| Vec::::new(), + { + let progress = Arc::clone(&progress); + let db = db.clone(); + move |chunk, buf| { + let chunk_size = chunk.len(); + for count in chunk { + use crate::data::output::count::PackLocation::*; + match count.entry_pack_location { + LookedUp(_) => continue, + NotLookedUp => { + count.entry_pack_location = + LookedUp(db.location_by_oid(&count.id, buf)) + } + } + } + progress.lock().inc_by(chunk_size); + Ok::<_, ()>(()) + } + }, + parallel::reduce::IdentityWithResult::<(), ()>::default(), + ) + .expect("infallible - we ignore none-existing objects"); + progress.lock().show_throughput(start); + } + let counts_range_by_pack_id = match mode { + Mode::PackCopyAndBaseObjects => { + let mut progress = + progress.add_child_with_id("sorting".into(), ProgressId::SortEntries.into()); + progress.init(Some(counts.len()), gix_features::progress::count("counts")); + let start = std::time::Instant::now(); + + use crate::data::output::count::PackLocation::*; + counts.sort_by(|lhs, rhs| { + match (&lhs.entry_pack_location, &rhs.entry_pack_location) { + (LookedUp(None), LookedUp(None)) => Ordering::Equal, + (LookedUp(Some(_)), LookedUp(None)) => Ordering::Greater, + (LookedUp(None), LookedUp(Some(_))) => Ordering::Less, + (LookedUp(Some(lhs)), LookedUp(Some(rhs))) => lhs + .pack_id + .cmp(&rhs.pack_id) + .then(lhs.pack_offset.cmp(&rhs.pack_offset)), + (_, _) => unreachable!("counts were resolved beforehand"), + } + }); + + let mut index: Vec<(u32, std::ops::Range)> = Vec::new(); + let mut chunks_pack_start = + counts.partition_point(|e| e.entry_pack_location.is_none()); + let mut slice = &counts[chunks_pack_start..]; + while !slice.is_empty() { + let current_pack_id = slice[0] + .entry_pack_location + .as_ref() + .expect("packed object") + .pack_id; + let pack_end = slice.partition_point(|e| { + e.entry_pack_location + .as_ref() + .expect("packed object") + .pack_id + == current_pack_id + }); + index.push(( + current_pack_id, + chunks_pack_start..chunks_pack_start + pack_end, + )); + slice = &slice[pack_end..]; + chunks_pack_start += pack_end; + } + + progress.set(counts.len()); + progress.show_throughput(start); + + index + } + }; + + let counts = Arc::new(counts); + let progress = Arc::new(parking_lot::Mutex::new(progress)); + let chunks = util::ChunkRanges::new(chunk_size, counts.len()); + + parallel::reduce::Stepwise::new( + chunks.enumerate(), + thread_limit, + { + let progress = Arc::clone(&progress); + move |n| { + ( + Vec::new(), // object data buffer + progress.lock().add_child_with_id( + format!("thread {n}"), + gix_features::progress::UNKNOWN, + ), + ) + } + }, + { + let counts = Arc::clone(&counts); + move |(chunk_id, chunk_range): (SequenceId, std::ops::Range), + (buf, progress)| { + let mut out = Vec::new(); + let chunk = &counts[chunk_range]; + let mut stats = Outcome::default(); + let mut pack_offsets_to_id = None; + progress.init(Some(chunk.len()), gix_features::progress::count("objects")); + + for count in chunk.iter() { + out.push(match count + .entry_pack_location + .as_ref() + .and_then(|l| db.entry_by_location(l).map(|pe| (l, pe))) + { + Some((location, pack_entry)) => { + if let Some((cached_pack_id, _)) = &pack_offsets_to_id { + if *cached_pack_id != location.pack_id { + pack_offsets_to_id = None; + } + } + let pack_range = counts_range_by_pack_id[counts_range_by_pack_id + .binary_search_by_key(&location.pack_id, |e| e.0) + .expect("pack-id always present")] + .1 + .clone(); + let base_index_offset = pack_range.start; + let counts_in_pack = &counts[pack_range]; + let entry = output::Entry::from_pack_entry( + pack_entry, + count, + counts_in_pack, + base_index_offset, + allow_thin_pack.then_some({ + |pack_id, base_offset| { + let (cached_pack_id, cache) = pack_offsets_to_id.get_or_insert_with(|| { + db.pack_offsets_and_oid(pack_id) + .map(|mut v| { + v.sort_by_key(|e| e.0); + (pack_id, v) + }) + .expect("pack used for counts is still available") + }); + debug_assert_eq!(*cached_pack_id, pack_id); + stats.ref_delta_objects += 1; + cache + .binary_search_by_key(&base_offset, |e| e.0) + .ok() + .map(|idx| cache[idx].1) + } + }), + version, + ); + match entry { + Some(entry) => { + stats.objects_copied_from_pack += 1; + entry + } + None => match db.try_find(&count.id, buf).map_err(Error::Find)? { + Some((obj, _location)) => { + stats.decoded_and_recompressed_objects += 1; + output::Entry::from_data(count, &obj) + } + None => { + stats.missing_objects += 1; + Ok(output::Entry::invalid()) + } + }, + } + } + None => match db.try_find(&count.id, buf).map_err(Error::Find)? { + Some((obj, _location)) => { + stats.decoded_and_recompressed_objects += 1; + output::Entry::from_data(count, &obj) + } + None => { + stats.missing_objects += 1; + Ok(output::Entry::invalid()) + } + }, + }?); + progress.inc(); + } + Ok((chunk_id, out, stats)) + } + }, + reduce::Statistics::default(), + ) + } +} + +mod util { + #[derive(Clone)] + pub struct ChunkRanges { + cursor: usize, + size: usize, + len: usize, + } + + impl ChunkRanges { + pub fn new(size: usize, total: usize) -> Self { + ChunkRanges { + cursor: 0, + size, + len: total, + } + } + } + + impl Iterator for ChunkRanges { + type Item = std::ops::Range; + + fn next(&mut self) -> Option { + if self.cursor >= self.len { + None + } else { + let upper = (self.cursor + self.size).min(self.len); + let range = self.cursor..upper; + self.cursor = upper; + Some(range) + } + } + } +} + +mod reduce { + use std::marker::PhantomData; + + use gix_features::{parallel, parallel::SequenceId}; + + use super::Outcome; + use crate::data::output; + + pub struct Statistics { + total: Outcome, + _err: PhantomData, + } + + impl Default for Statistics { + fn default() -> Self { + Statistics { + total: Default::default(), + _err: PhantomData, + } + } + } + + impl parallel::Reduce for Statistics { + type Input = Result<(SequenceId, Vec, Outcome), Error>; + type FeedProduce = (SequenceId, Vec); + type Output = Outcome; + type Error = Error; + + fn feed(&mut self, item: Self::Input) -> Result { + item.map(|(cid, entries, stats)| { + self.total.aggregate(stats); + (cid, entries) + }) + } + + fn finalize(self) -> Result { + Ok(self.total) + } + } +} + +mod types { + use crate::data::output::entry; + + /// Information gathered during the run of [`iter_from_counts()`][crate::data::output::entry::iter_from_counts()]. + #[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] + #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] + pub struct Outcome { + /// The amount of fully decoded objects. These are the most expensive as they are fully decoded. + pub decoded_and_recompressed_objects: usize, + /// The amount of objects that could not be located despite them being mentioned during iteration + pub missing_objects: usize, + /// The amount of base or delta objects that could be copied directly from the pack. These are cheapest as they + /// only cost a memory copy for the most part. + pub objects_copied_from_pack: usize, + /// The amount of objects that ref to their base as ref-delta, an indication for a thin back being created. + pub ref_delta_objects: usize, + } + + impl Outcome { + pub(in crate::data::output::entry) fn aggregate( + &mut self, + Outcome { + decoded_and_recompressed_objects: decoded_objects, + missing_objects, + objects_copied_from_pack, + ref_delta_objects, + }: Self, + ) { + self.decoded_and_recompressed_objects += decoded_objects; + self.missing_objects += missing_objects; + self.objects_copied_from_pack += objects_copied_from_pack; + self.ref_delta_objects += ref_delta_objects; + } + } + + /// The way the iterator operates. + #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] + #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] + pub enum Mode { + /// Copy base objects and deltas from packs, while non-packed objects will be treated as base objects + /// (i.e. without trying to delta compress them). This is a fast way of obtaining a back while benefiting + /// from existing pack compression and spending the smallest possible time on compressing unpacked objects at + /// the cost of bandwidth. + PackCopyAndBaseObjects, + } + + /// Configuration options for the pack generation functions provided in [`iter_from_counts()`][crate::data::output::entry::iter_from_counts()]. + #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] + #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] + pub struct Options { + /// The amount of threads to use at most when resolving the pack. If `None`, all logical cores are used. + pub thread_limit: Option, + /// The algorithm to produce a pack + pub mode: Mode, + /// If set, the resulting back can have deltas that refer to an object which is not in the pack. This can happen + /// if the initial counted objects do not contain an object that an existing packed delta refers to, for example, because + /// it wasn't part of the iteration, for instance when the iteration was performed on tree deltas or only a part of the + /// commit graph. Please note that thin packs are not valid packs at rest, thus they are only valid for packs in transit. + /// + /// If set to false, delta objects will be decompressed and recompressed as base objects. + pub allow_thin_pack: bool, + /// The amount of objects per chunk or unit of work to be sent to threads for processing + /// TODO: could this become the window size? + pub chunk_size: usize, + /// The pack data version to produce for each entry + pub version: crate::data::Version, + } + + impl Default for Options { + fn default() -> Self { + Options { + thread_limit: None, + mode: Mode::PackCopyAndBaseObjects, + allow_thin_pack: false, + chunk_size: 10, + version: Default::default(), + } + } + } + + /// The error returned by the pack generation function [`iter_from_counts()`][crate::data::output::entry::iter_from_counts()]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error(transparent)] + Find(gix_object::find::Error), + #[error(transparent)] + NewEntry(#[from] entry::Error), + } + + /// The progress ids used in [`write_to_directory()`][crate::Bundle::write_to_directory()]. + /// + /// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. + #[derive(Debug, Copy, Clone)] + pub enum ProgressId { + /// The amount of [`Count`][crate::data::output::Count] objects which are resolved to their pack location. + ResolveCounts, + /// Layout pack entries for placement into a pack (by pack-id and by offset). + SortEntries, + } + + impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::ResolveCounts => *b"ECRC", + ProgressId::SortEntries => *b"ECSE", + } + } + } +} +pub use types::{Error, Mode, Options, Outcome, ProgressId}; diff --git a/knot2/third_party/gix-pack/src/data/output/entry/mod.rs b/knot2/third_party/gix-pack/src/data/output/entry/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..03ebbdd475f5efcd56fe0342938cfbde88ebac05 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/entry/mod.rs @@ -0,0 +1,195 @@ +use std::io::Write; + +use gix_hash::ObjectId; + +use crate::{data, data::output, find}; + +/// +pub mod iter_from_counts; +pub use iter_from_counts::function::iter_from_counts; + +/// The kind of pack entry to be written +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Kind { + /// A complete base object, including its kind + Base(gix_object::Kind), + /// A delta against the object with the given index. It's always an index that was already encountered to refer only + /// to object we have written already. + DeltaRef { + /// The absolute index to the object to serve as base. It's up to the writer to maintain enough state to allow producing + /// a packed delta object from it. + object_index: usize, + }, + /// A delta against the given object as identified by its `ObjectId`. + /// This is the case for thin packs only, i.e. those that are sent over the wire. + /// Note that there is the option of the `ObjectId` being used to refer to an object within + /// the same pack, but it's a discontinued practice which won't be encountered here. + DeltaOid { + /// The object serving as base for this delta + id: ObjectId, + }, +} + +/// The error returned by [`output::Entry::from_data()`]. +#[allow(missing_docs)] +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("{0}")] + ZlibDeflate(#[from] std::io::Error), + #[error(transparent)] + EntryType(#[from] crate::data::entry::decode::Error), +} + +impl output::Entry { + /// An object which can be identified as invalid easily which happens if objects didn't exist even if they were referred to. + pub fn invalid() -> output::Entry { + output::Entry { + id: gix_hash::Kind::shortest().null(), // NOTE: the actual object hash used in the repo doesn't matter here, this is a sentinel value. + kind: Kind::Base(gix_object::Kind::Blob), + decompressed_size: 0, + compressed_data: vec![], + } + } + + /// Returns true if this object doesn't really exist but still has to be handled responsibly + /// + /// Note that this is true for tree entries that are commits/git submodules, or for objects which aren't present in our local clone + /// due to shallow clones. + pub fn is_invalid(&self) -> bool { + self.id.is_null() + } + + /// Create an Entry from a previously counted object which is located in a pack. It's `entry` is provided here. + /// The `version` specifies what kind of target `Entry` version the caller desires. + pub fn from_pack_entry( + mut entry: find::Entry, + count: &output::Count, + potential_bases: &[output::Count], + bases_index_offset: usize, + pack_offset_to_oid: Option Option>, + target_version: data::Version, + ) -> Option> { + if entry.version != target_version { + return None; + } + + let pack_offset_must_be_zero = 0; + let pack_entry = match data::Entry::from_bytes( + &entry.data, + pack_offset_must_be_zero, + count.id.as_slice().len(), + ) { + Ok(e) => e, + Err(err) => return Some(Err(err.into())), + }; + + use crate::data::entry::Header::*; + match pack_entry.header { + Commit => Some(output::entry::Kind::Base(gix_object::Kind::Commit)), + Tree => Some(output::entry::Kind::Base(gix_object::Kind::Tree)), + Blob => Some(output::entry::Kind::Base(gix_object::Kind::Blob)), + Tag => Some(output::entry::Kind::Base(gix_object::Kind::Tag)), + OfsDelta { base_distance } => { + let pack_location = count.entry_pack_location.as_ref().expect("packed"); + let base_offset = pack_location + .pack_offset + .checked_sub(base_distance) + .expect("pack-offset - distance is firmly within the pack"); + potential_bases + .binary_search_by(|e| { + e.entry_pack_location + .as_ref() + .expect("packed") + .pack_offset + .cmp(&base_offset) + }) + .ok() + .map(|idx| output::entry::Kind::DeltaRef { + object_index: idx + bases_index_offset, + }) + .or_else(|| { + pack_offset_to_oid + .and_then(|mut f| f(pack_location.pack_id, base_offset)) + .map(|id| output::entry::Kind::DeltaOid { id }) + }) + } + RefDelta { base_id: _ } => None, // ref deltas are for thin packs or legacy, repack them as base objects + } + .map(|kind| { + Ok(output::Entry { + id: count.id.to_owned(), + kind, + decompressed_size: pack_entry.decompressed_size as usize, + compressed_data: { + entry.data.copy_within(pack_entry.data_offset as usize.., 0); + entry.data.resize( + entry.data.len() + - usize::try_from(pack_entry.data_offset) + .expect("offset representable as usize"), + 0, + ); + entry.data + }, + }) + }) + } + + /// Create a new instance from the given `oid` and its corresponding git object data `obj`. + pub fn from_data(count: &output::Count, obj: &gix_object::Data<'_>) -> Result { + Ok(output::Entry { + id: count.id.to_owned(), + kind: Kind::Base(obj.kind), + decompressed_size: obj.data.len(), + compressed_data: { + let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new()); + if let Err(err) = std::io::copy(&mut &*obj.data, &mut out) { + match err.kind() { + std::io::ErrorKind::Other => return Err(Error::ZlibDeflate(err)), + err => unreachable!( + "Should never see other errors than zlib, but got {:?}", + err + ), + } + } + out.flush()?; + out.into_inner() + }, + }) + } + + /// Transform ourselves into pack entry header of `version` which can be written into a pack. + /// + /// `index_to_pack(object_index) -> pack_offset` is a function to convert the base object's index into + /// the input object array (if each object is numbered) to an offset into the pack. + /// This information is known to the one calling the method. + pub fn to_entry_header( + &self, + version: data::Version, + index_to_base_distance: impl FnOnce(usize) -> u64, + ) -> data::entry::Header { + assert!( + matches!(version, data::Version::V2), + "we can only write V2 pack entries for now" + ); + + use Kind::*; + match self.kind { + Base(kind) => { + use gix_object::Kind::*; + match kind { + Tree => data::entry::Header::Tree, + Blob => data::entry::Header::Blob, + Commit => data::entry::Header::Commit, + Tag => data::entry::Header::Tag, + } + } + DeltaOid { id } => data::entry::Header::RefDelta { + base_id: id.to_owned(), + }, + DeltaRef { object_index } => data::entry::Header::OfsDelta { + base_distance: index_to_base_distance(object_index), + }, + } + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/mod.rs b/knot2/third_party/gix-pack/src/data/output/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..338bb92b3eaa4f9ce99623d4afc8eb5df383c9be --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/mod.rs @@ -0,0 +1,41 @@ +use gix_hash::ObjectId; + +/// +pub mod count; + +/// An item representing a future Entry in the leanest way possible. +/// +/// One can expect to have one of these in memory when building big objects, so smaller is better here. +/// They should contain everything of importance to generate a pack as fast as possible. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Count { + /// The hash of the object to write + pub id: ObjectId, + /// A way to locate a pack entry in the object database, only available if the object is in a pack. + pub entry_pack_location: count::PackLocation, +} + +/// An entry to be written to a file. +/// +/// Some of these will be in-flight and in memory while waiting to be written. Memory requirements depend on the amount of compressed +/// data they hold. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Entry { + /// The hash of the object to write + pub id: ObjectId, + /// The kind of entry represented by `data`. It's used alongside with it to complete the pack entry + /// at rest or in transit. + pub kind: entry::Kind, + /// The size in bytes needed once `data` gets decompressed + pub decompressed_size: usize, + /// The compressed data right behind the header + pub compressed_data: Vec, +} + +/// +pub mod entry; + +/// +pub mod bytes; diff --git a/knot2/third_party/gix-pack/src/find.rs b/knot2/third_party/gix-pack/src/find.rs new file mode 100644 index 0000000000000000000000000000000000000000..efc9513edbbfd06fc108e82c0125ceecbbdba9f9 --- /dev/null +++ b/knot2/third_party/gix-pack/src/find.rs @@ -0,0 +1,12 @@ +/// An Entry in a pack providing access to its data. +/// +/// Its commonly retrieved by reading from a pack index file followed by a read from a pack data file. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[allow(missing_docs)] +pub struct Entry { + /// The pack-data encoded bytes of the pack data entry as present in the pack file, including the header followed by compressed data. + pub data: Vec, + /// The version of the pack file containing `data` + pub version: crate::data::Version, +} diff --git a/knot2/third_party/gix-pack/src/find_traits.rs b/knot2/third_party/gix-pack/src/find_traits.rs new file mode 100644 index 0000000000000000000000000000000000000000..007135202be98c1b132e64a86d979ccf0db0101b --- /dev/null +++ b/knot2/third_party/gix-pack/src/find_traits.rs @@ -0,0 +1,343 @@ +use crate::{data, find}; + +/// Describe how object can be located in an object store with built-in facilities to supports packs specifically. +/// +/// ## Notes +/// +/// Find effectively needs [generic associated types][issue] to allow a trait for the returned object type. +/// Until then, we will have to make due with explicit types and give them the potentially added features we want. +/// +/// Furthermore, despite this trait being in `gix-pack`, it leaks knowledge about objects potentially not being packed. +/// This is a necessary trade-off to allow this trait to live in `gix-pack` where it is used in functions to create a pack. +/// +/// [issue]: https://github.com/rust-lang/rust/issues/44265 +pub trait Find { + /// Returns true if the object exists in the database. + fn contains(&self, id: &gix_hash::oid) -> bool; + + /// Find an object matching `id` in the database while placing its raw, decoded data into `buffer`. + /// A `pack_cache` can be used to speed up subsequent lookups, set it to [`crate::cache::Never`] if the + /// workload isn't suitable for caching. + /// + /// Returns `Some((, ))` if it was present in the database, + /// or the error that occurred during lookup or object retrieval. + fn try_find<'a>( + &self, + id: &gix_hash::oid, + buffer: &'a mut Vec, + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + > { + self.try_find_cached(id, buffer, &mut crate::cache::Never) + } + + /// Like [`Find::try_find()`], but with support for controlling the pack cache. + /// A `pack_cache` can be used to speed up subsequent lookups, set it to [`crate::cache::Never`] if the + /// workload isn't suitable for caching. + /// + /// Returns `Some((, ))` if it was present in the database, + /// or the error that occurred during lookup or object retrieval. + fn try_find_cached<'a>( + &self, + id: &gix_hash::oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + >; + + /// Find the packs location where an object with `id` can be found in the database, or `None` if there is no pack + /// holding the object. + /// + /// _Note_ that this is always None if the object isn't packed even though it exists as loose object. + fn location_by_oid( + &self, + id: &gix_hash::oid, + buf: &mut Vec, + ) -> Option; + + /// Obtain a vector of all offsets, in index order, along with their object id. + fn pack_offsets_and_oid(&self, pack_id: u32) + -> Option>; + + /// Return the [`find::Entry`] for `location` if it is backed by a pack. + /// + /// Note that this is only in the interest of avoiding duplicate work during pack generation. + /// Pack locations can be obtained from [`Find::try_find()`]. + /// + /// # Notes + /// + /// Custom implementations might be interested in providing their own meta-data with `object`, + /// which currently isn't possible as the `Locate` trait requires GATs to work like that. + fn entry_by_location(&self, location: &data::entry::Location) -> Option; +} + +mod ext { + use gix_object::{ + BlobRef, CommitRef, CommitRefIter, Kind, ObjectRef, TagRef, TagRefIter, TreeRef, + TreeRefIter, + }; + + macro_rules! make_obj_lookup { + ($method:ident, $object_variant:path, $object_kind:path, $object_type:ty) => { + /// Like [`find(…)`][Self::find()], but flattens the `Result>` into a single `Result` making a non-existing object an error + /// while returning the desired object type. + fn $method<'a>( + &self, + id: &gix_hash::oid, + buffer: &'a mut Vec, + ) -> Result< + ($object_type, Option), + gix_object::find::existing_object::Error, + > { + let id = id.as_ref(); + self.try_find(id, buffer) + .map_err(gix_object::find::existing_object::Error::Find)? + .ok_or_else(|| gix_object::find::existing_object::Error::NotFound { + oid: id.as_ref().to_owned(), + }) + .and_then(|(o, l)| { + o.decode() + .map_err(|err| gix_object::find::existing_object::Error::Decode { + source: err, + oid: id.to_owned(), + }) + .map(|o| (o, l)) + }) + .and_then(|(o, l)| match o { + $object_variant(o) => return Ok((o, l)), + o => Err(gix_object::find::existing_object::Error::ObjectKind { + oid: id.to_owned(), + actual: o.kind(), + expected: $object_kind, + }), + }) + } + }; + } + + macro_rules! make_iter_lookup { + ($method:ident, $object_kind:path, $object_type:ty, $into_iter:tt) => { + /// Like [`find(…)`][Self::find()], but flattens the `Result>` into a single `Result` making a non-existing object an error + /// while returning the desired iterator type. + fn $method<'a>( + &self, + id: &gix_hash::oid, + buffer: &'a mut Vec, + ) -> Result< + ($object_type, Option), + gix_object::find::existing_iter::Error, + > { + let id = id.as_ref(); + self.try_find(id, buffer) + .map_err(gix_object::find::existing_iter::Error::Find)? + .ok_or_else(|| gix_object::find::existing_iter::Error::NotFound { + oid: id.as_ref().to_owned(), + }) + .and_then(|(o, l)| { + o.$into_iter() + .ok_or_else(|| gix_object::find::existing_iter::Error::ObjectKind { + oid: id.to_owned(), + actual: o.kind, + expected: $object_kind, + }) + .map(|i| (i, l)) + }) + } + }; + } + + /// An extension trait with convenience functions. + pub trait FindExt: super::Find { + /// Like [`try_find(…)`][super::Find::try_find()], but flattens the `Result>` into a single `Result` making a non-existing object an error. + fn find<'a>( + &self, + id: &gix_hash::oid, + buffer: &'a mut Vec, + ) -> Result< + (gix_object::Data<'a>, Option), + gix_object::find::existing::Error, + > { + self.try_find(id, buffer) + .map_err(gix_object::find::existing::Error::Find)? + .ok_or_else(|| gix_object::find::existing::Error::NotFound { + oid: id.as_ref().to_owned(), + }) + } + + make_obj_lookup!(find_commit, ObjectRef::Commit, Kind::Commit, CommitRef<'a>); + make_obj_lookup!(find_tree, ObjectRef::Tree, Kind::Tree, TreeRef<'a>); + make_obj_lookup!(find_tag, ObjectRef::Tag, Kind::Tag, TagRef<'a>); + make_obj_lookup!(find_blob, ObjectRef::Blob, Kind::Blob, BlobRef<'a>); + make_iter_lookup!( + find_commit_iter, + Kind::Blob, + CommitRefIter<'a>, + try_into_commit_iter + ); + make_iter_lookup!( + find_tree_iter, + Kind::Tree, + TreeRefIter<'a>, + try_into_tree_iter + ); + make_iter_lookup!(find_tag_iter, Kind::Tag, TagRefIter<'a>, try_into_tag_iter); + } + + impl FindExt for T {} +} +pub use ext::FindExt; + +mod find_impls { + use std::{ops::Deref, rc::Rc}; + + use gix_hash::oid; + + use crate::{data, find}; + + impl crate::Find for &T + where + T: crate::Find, + { + fn contains(&self, id: &oid) -> bool { + (*self).contains(id) + } + + fn try_find_cached<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + > { + (*self).try_find_cached(id, buffer, pack_cache) + } + + fn location_by_oid(&self, id: &oid, buf: &mut Vec) -> Option { + (*self).location_by_oid(id, buf) + } + + fn pack_offsets_and_oid( + &self, + pack_id: u32, + ) -> Option> { + (*self).pack_offsets_and_oid(pack_id) + } + + fn entry_by_location(&self, location: &data::entry::Location) -> Option { + (*self).entry_by_location(location) + } + } + + impl super::Find for std::sync::Arc + where + T: super::Find, + { + fn contains(&self, id: &oid) -> bool { + self.deref().contains(id) + } + + fn try_find_cached<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + > { + self.deref().try_find_cached(id, buffer, pack_cache) + } + + fn location_by_oid(&self, id: &oid, buf: &mut Vec) -> Option { + self.deref().location_by_oid(id, buf) + } + + fn pack_offsets_and_oid( + &self, + pack_id: u32, + ) -> Option> { + self.deref().pack_offsets_and_oid(pack_id) + } + + fn entry_by_location(&self, object: &data::entry::Location) -> Option { + self.deref().entry_by_location(object) + } + } + + impl super::Find for Rc + where + T: super::Find, + { + fn contains(&self, id: &oid) -> bool { + self.deref().contains(id) + } + + fn try_find_cached<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + > { + self.deref().try_find_cached(id, buffer, pack_cache) + } + + fn location_by_oid(&self, id: &oid, buf: &mut Vec) -> Option { + self.deref().location_by_oid(id, buf) + } + + fn pack_offsets_and_oid( + &self, + pack_id: u32, + ) -> Option> { + self.deref().pack_offsets_and_oid(pack_id) + } + + fn entry_by_location(&self, location: &data::entry::Location) -> Option { + self.deref().entry_by_location(location) + } + } + + impl super::Find for Box + where + T: super::Find, + { + fn contains(&self, id: &oid) -> bool { + self.deref().contains(id) + } + + fn try_find_cached<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result< + Option<(gix_object::Data<'a>, Option)>, + gix_object::find::Error, + > { + self.deref().try_find_cached(id, buffer, pack_cache) + } + + fn location_by_oid(&self, id: &oid, buf: &mut Vec) -> Option { + self.deref().location_by_oid(id, buf) + } + + fn pack_offsets_and_oid( + &self, + pack_id: u32, + ) -> Option> { + self.deref().pack_offsets_and_oid(pack_id) + } + + fn entry_by_location(&self, location: &data::entry::Location) -> Option { + self.deref().entry_by_location(location) + } + } +} diff --git a/knot2/third_party/gix-pack/src/index/access.rs b/knot2/third_party/gix-pack/src/index/access.rs new file mode 100644 index 0000000000000000000000000000000000000000..47f3573050d6161f337ab6486c80a7a13f8efe52 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/access.rs @@ -0,0 +1,312 @@ +use std::{mem::size_of, ops::Range}; + +use crate::{ + data, + index::{self, EntryIndex, FAN_LEN, PrefixLookupResult}, +}; + +const N32_SIZE: usize = size_of::(); +const N64_SIZE: usize = size_of::(); +const V1_HEADER_SIZE: usize = FAN_LEN * N32_SIZE; +const V2_HEADER_SIZE: usize = N32_SIZE * 2 + FAN_LEN * N32_SIZE; +const N32_HIGH_BIT: u32 = 1 << 31; + +/// Represents an entry within a pack index file, effectively mapping object [`IDs`][gix_hash::ObjectId] to pack data file locations. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Entry { + /// The ID of the object + pub oid: gix_hash::ObjectId, + /// The offset to the object's header in the pack data file + pub pack_offset: data::Offset, + /// The CRC32 hash over all bytes of the pack data entry. + /// + /// This can be useful for direct copies of pack data entries from one pack to another with insurance there was no bit rot. + /// _Note_: Only available in index version 2 or newer + pub crc32: Option, +} + +/// Iteration and access +impl index::File +where + T: crate::FileData, +{ + fn iter_v1(&self) -> impl Iterator + '_ { + match self.version { + index::Version::V1 => self.data[V1_HEADER_SIZE..] + .chunks_exact(N32_SIZE + self.hash_len) + .take(self.num_objects as usize) + .map(|c| { + let (ofs, oid) = c.split_at(N32_SIZE); + Entry { + oid: gix_hash::ObjectId::from_bytes_or_panic(oid), + pack_offset: u64::from(crate::read_u32(ofs)), + crc32: None, + } + }), + _ => panic!("Cannot use iter_v1() on index of type {:?}", self.version), + } + } + + fn iter_v2(&self) -> impl Iterator + '_ { + let pack64_offset = self.offset_pack_offset64_v2(); + let oids = self.data[V2_HEADER_SIZE..] + .chunks_exact(self.hash_len) + .take(self.num_objects as usize); + let crcs = self.data[self.offset_crc32_v2()..] + .chunks_exact(N32_SIZE) + .take(self.num_objects as usize); + let offsets = self.data[self.offset_pack_offset_v2()..] + .chunks_exact(N32_SIZE) + .take(self.num_objects as usize); + assert_eq!(oids.len(), crcs.len()); + assert_eq!(crcs.len(), offsets.len()); + match self.version { + index::Version::V2 => { + izip!(oids, crcs, offsets).map(move |(oid, crc32, ofs32)| Entry { + oid: gix_hash::ObjectId::from_bytes_or_panic(oid), + pack_offset: self.pack_offset_from_offset_v2(ofs32, pack64_offset), + crc32: Some(crate::read_u32(crc32)), + }) + } + _ => panic!("Cannot use iter_v2() on index of type {:?}", self.version), + } + } + + /// Returns the object hash at the given index in our list of (sorted) sha1 hashes. + /// The index ranges from 0 to `self.num_objects()` + /// + /// # Panics + /// + /// If `index` is out of bounds. + pub fn oid_at_index(&self, index: EntryIndex) -> &gix_hash::oid { + let index = index as usize; + let start = match self.version { + index::Version::V2 => V2_HEADER_SIZE + index * self.hash_len, + index::Version::V1 => V1_HEADER_SIZE + index * (N32_SIZE + self.hash_len) + N32_SIZE, + }; + gix_hash::oid::from_bytes_unchecked(&self.data[start..][..self.hash_len]) + } + + /// Returns the offset into our pack data file at which to start reading the object at `index`. + /// + /// # Panics + /// + /// If `index` is out of bounds. + pub fn pack_offset_at_index(&self, index: EntryIndex) -> data::Offset { + let index = index as usize; + match self.version { + index::Version::V2 => { + let start = self.offset_pack_offset_v2() + index * N32_SIZE; + self.pack_offset_from_offset_v2( + &self.data[start..][..N32_SIZE], + self.offset_pack_offset64_v2(), + ) + } + index::Version::V1 => { + let start = V1_HEADER_SIZE + index * (N32_SIZE + self.hash_len); + u64::from(crate::read_u32(&self.data[start..][..N32_SIZE])) + } + } + } + + /// Returns the CRC32 of the object at the given `index`. + /// + /// _Note_: These are always present for index version 2 or higher. + /// # Panics + /// + /// If `index` is out of bounds. + pub fn crc32_at_index(&self, index: EntryIndex) -> Option { + let index = index as usize; + match self.version { + index::Version::V2 => { + let start = self.offset_crc32_v2() + index * N32_SIZE; + Some(crate::read_u32(&self.data[start..start + N32_SIZE])) + } + index::Version::V1 => None, + } + } + + /// Returns the `index` of the given hash for use with the [`oid_at_index()`][index::File::oid_at_index()], + /// [`pack_offset_at_index()`][index::File::pack_offset_at_index()] or [`crc32_at_index()`][index::File::crc32_at_index()]. + // NOTE: pretty much the same things as in `multi_index::File::lookup`, change things there + // as well. + pub fn lookup(&self, id: impl AsRef) -> Option { + lookup(id.as_ref(), &self.fan, &|idx| self.oid_at_index(idx)) + } + + /// Given a `prefix`, find an object that matches it uniquely within this index and return `Some(Ok(entry_index))`. + /// If there is more than one object matching the object `Some(Err(())` is returned. + /// + /// Finally, if no object matches the index, the return value is `None`. + /// + /// Pass `candidates` to obtain the set of entry-indices matching `prefix`, with the same return value as + /// one would have received if it remained `None`. It will be empty if no object matched the `prefix`. + /// + // NOTE: pretty much the same things as in `index::File::lookup`, change things there + // as well. + pub fn lookup_prefix( + &self, + prefix: gix_hash::Prefix, + candidates: Option<&mut Range>, + ) -> Option { + lookup_prefix( + prefix, + candidates, + &self.fan, + &|idx| self.oid_at_index(idx), + self.num_objects, + ) + } + + /// An iterator over all [`Entries`][Entry] of this index file. + pub fn iter<'a>(&'a self) -> Box + 'a> { + match self.version { + index::Version::V2 => Box::new(self.iter_v2()), + index::Version::V1 => Box::new(self.iter_v1()), + } + } + + /// Return a vector of ascending offsets into our respective pack data file. + /// + /// Useful to control an iteration over all pack entries in a cache-friendly way. + pub fn sorted_offsets(&self) -> Vec { + let mut ofs: Vec<_> = match self.version { + index::Version::V1 => self.iter().map(|e| e.pack_offset).collect(), + index::Version::V2 => { + let offset32_start = &self.data[self.offset_pack_offset_v2()..]; + let offsets32 = offset32_start + .chunks_exact(N32_SIZE) + .take(self.num_objects as usize); + assert_eq!(self.num_objects as usize, offsets32.len()); + let pack_offset_64_start = self.offset_pack_offset64_v2(); + offsets32 + .map(|offset| self.pack_offset_from_offset_v2(offset, pack_offset_64_start)) + .collect() + } + }; + ofs.sort_unstable(); + ofs + } + + #[inline] + fn offset_crc32_v2(&self) -> usize { + V2_HEADER_SIZE + self.num_objects as usize * self.hash_len + } + + #[inline] + fn offset_pack_offset_v2(&self) -> usize { + self.offset_crc32_v2() + self.num_objects as usize * N32_SIZE + } + + #[inline] + fn offset_pack_offset64_v2(&self) -> usize { + self.offset_pack_offset_v2() + self.num_objects as usize * N32_SIZE + } + + #[inline] + fn pack_offset_from_offset_v2(&self, offset: &[u8], pack64_offset: usize) -> data::Offset { + debug_assert_eq!(self.version, index::Version::V2); + let ofs32 = crate::read_u32(offset); + if (ofs32 & N32_HIGH_BIT) == N32_HIGH_BIT { + let from = pack64_offset + (ofs32 ^ N32_HIGH_BIT) as usize * N64_SIZE; + crate::read_u64(&self.data[from..][..N64_SIZE]) + } else { + u64::from(ofs32) + } + } +} + +pub(crate) fn lookup_prefix<'a>( + prefix: gix_hash::Prefix, + candidates: Option<&mut Range>, + fan: &[u32; FAN_LEN], + oid_at_index: &dyn Fn(EntryIndex) -> &'a gix_hash::oid, + num_objects: u32, +) -> Option { + let first_byte = prefix.as_oid().first_byte() as usize; + let mut upper_bound = fan[first_byte]; + let mut lower_bound = if first_byte != 0 { + fan[first_byte - 1] + } else { + 0 + }; + + // Bisect using indices + while lower_bound < upper_bound { + let mid = u32::midpoint(lower_bound, upper_bound); + let mid_sha = oid_at_index(mid); + + use std::cmp::Ordering::*; + match prefix.cmp_oid(mid_sha) { + Less => upper_bound = mid, + Equal => match candidates { + Some(candidates) => { + let first_past_entry = ((0..mid).rev()) + .take_while(|prev| prefix.cmp_oid(oid_at_index(*prev)) == Equal) + .last(); + + let last_future_entry = ((mid + 1)..num_objects) + .take_while(|next| prefix.cmp_oid(oid_at_index(*next)) == Equal) + .last(); + + *candidates = match (first_past_entry, last_future_entry) { + (Some(first), Some(last)) => first..last + 1, + (Some(first), None) => first..mid + 1, + (None, Some(last)) => mid..last + 1, + (None, None) => mid..mid + 1, + }; + + return if candidates.len() > 1 { + Some(Err(())) + } else { + Some(Ok(mid)) + }; + } + None => { + let next = mid + 1; + if next < num_objects && prefix.cmp_oid(oid_at_index(next)) == Equal { + return Some(Err(())); + } + if mid != 0 && prefix.cmp_oid(oid_at_index(mid - 1)) == Equal { + return Some(Err(())); + } + return Some(Ok(mid)); + } + }, + Greater => lower_bound = mid + 1, + } + } + + if let Some(candidates) = candidates { + *candidates = 0..0; + } + None +} + +pub(crate) fn lookup<'a>( + id: &gix_hash::oid, + fan: &[u32; FAN_LEN], + oid_at_index: &dyn Fn(EntryIndex) -> &'a gix_hash::oid, +) -> Option { + let first_byte = id.first_byte() as usize; + let mut upper_bound = fan[first_byte]; + let mut lower_bound = if first_byte != 0 { + fan[first_byte - 1] + } else { + 0 + }; + + while lower_bound < upper_bound { + let mid = u32::midpoint(lower_bound, upper_bound); + let mid_sha = oid_at_index(mid); + + use std::cmp::Ordering::*; + match id.cmp(mid_sha) { + Less => upper_bound = mid, + Equal => return Some(mid), + Greater => lower_bound = mid + 1, + } + } + None +} diff --git a/knot2/third_party/gix-pack/src/index/encode.rs b/knot2/third_party/gix-pack/src/index/encode.rs new file mode 100644 index 0000000000000000000000000000000000000000..98e3be722798f03e4cd90418874b1be0840557c5 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/encode.rs @@ -0,0 +1,170 @@ +use std::cmp::Ordering; + +pub(crate) const LARGE_OFFSET_THRESHOLD: u64 = 0x7fff_ffff; +pub(crate) const HIGH_BIT: u32 = 0x8000_0000; + +pub(crate) fn fanout(iter: &mut dyn ExactSizeIterator) -> [u32; 256] { + let mut fan_out = [0u32; 256]; + let entries_len = iter.len() as u32; + let mut iter = iter.enumerate(); + let mut idx_and_entry = iter.next(); + let mut upper_bound = 0; + + for (offset_be, byte) in fan_out.iter_mut().zip(0u8..=255) { + *offset_be = match idx_and_entry.as_ref() { + Some((_idx, first_byte)) => match first_byte.cmp(&byte) { + Ordering::Less => { + unreachable!("ids should be ordered, and we make sure to keep ahead with them") + } + Ordering::Greater => upper_bound, + Ordering::Equal => { + if byte == 255 { + entries_len + } else { + idx_and_entry = iter.find(|(_, first_byte)| *first_byte != byte); + upper_bound = idx_and_entry + .as_ref() + .map_or(entries_len, |(idx, _)| *idx as u32); + upper_bound + } + } + }, + None => entries_len, + }; + } + + fan_out +} + +#[cfg(feature = "streaming-input")] +mod function { + use std::io; + + use gix_features::progress::{self, DynNestedProgress}; + + use super::{HIGH_BIT, LARGE_OFFSET_THRESHOLD, fanout}; + use crate::index::V2_SIGNATURE; + + struct Count { + bytes: u64, + inner: W, + } + + impl Count { + fn new(inner: W) -> Self { + Count { bytes: 0, inner } + } + } + + impl io::Write for Count + where + W: io::Write, + { + fn write(&mut self, buf: &[u8]) -> io::Result { + let written = self.inner.write(buf)?; + self.bytes += written as u64; + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } + } + + pub(crate) fn write_to( + out: &mut dyn io::Write, + entries_sorted_by_oid: Vec>, + pack_hash: &gix_hash::ObjectId, + kind: crate::index::Version, + object_hash: gix_hash::Kind, + progress: &mut dyn DynNestedProgress, + ) -> Result { + use io::Write; + assert_eq!( + kind, + crate::index::Version::V2, + "Can only write V2 packs right now" + ); + assert!( + entries_sorted_by_oid.len() <= u32::MAX as usize, + "a pack cannot have more than u32::MAX objects" + ); + + // Write header + let mut out = Count::new(std::io::BufWriter::with_capacity( + 8 * 4096, + gix_hash::io::Write::new(out, object_hash), + )); + out.write_all(V2_SIGNATURE)?; + out.write_all(&(kind as u32).to_be_bytes())?; + + progress.init(Some(4), progress::steps()); + let start = std::time::Instant::now(); + let _info = progress.add_child_with_id( + "writing fan-out table".into(), + gix_features::progress::UNKNOWN, + ); + let fan_out = fanout(&mut entries_sorted_by_oid.iter().map(|e| e.data.id.first_byte())); + + for value in fan_out.iter() { + out.write_all(&value.to_be_bytes())?; + } + + progress.inc(); + let _info = + progress.add_child_with_id("writing ids".into(), gix_features::progress::UNKNOWN); + for entry in &entries_sorted_by_oid { + out.write_all(entry.data.id.as_slice())?; + } + + progress.inc(); + let _info = + progress.add_child_with_id("writing crc32".into(), gix_features::progress::UNKNOWN); + for entry in &entries_sorted_by_oid { + out.write_all(&entry.data.crc32.to_be_bytes())?; + } + + progress.inc(); + let _info = + progress.add_child_with_id("writing offsets".into(), gix_features::progress::UNKNOWN); + { + let mut offsets64 = Vec::::new(); + for entry in &entries_sorted_by_oid { + let offset: u32 = if entry.offset > LARGE_OFFSET_THRESHOLD { + assert!( + offsets64.len() < LARGE_OFFSET_THRESHOLD as usize, + "Encoding breakdown - way too many 64bit offsets" + ); + offsets64.push(entry.offset); + ((offsets64.len() - 1) as u32) | HIGH_BIT + } else { + entry.offset as u32 + }; + out.write_all(&offset.to_be_bytes())?; + } + for value in offsets64 { + out.write_all(&value.to_be_bytes())?; + } + } + + out.write_all(pack_hash.as_slice())?; + + let bytes_written_without_trailer = out.bytes; + let out = out.inner.into_inner().map_err(io::Error::from)?; + let index_hash = out.hash.try_finalize()?; + out.inner.write_all(index_hash.as_slice())?; + out.inner.flush()?; + + progress.inc(); + progress.show_throughput_with( + start, + (bytes_written_without_trailer + object_hash.len_in_bytes() as u64) as usize, + progress::bytes().expect("unit always set"), + progress::MessageLevel::Success, + ); + + Ok(index_hash) + } +} +#[cfg(feature = "streaming-input")] +pub(crate) use function::write_to; diff --git a/knot2/third_party/gix-pack/src/index/init.rs b/knot2/third_party/gix-pack/src/index/init.rs new file mode 100644 index 0000000000000000000000000000000000000000..36ed53ee6f67d82e44152b15d5100e52f5e30d60 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/init.rs @@ -0,0 +1,211 @@ +use std::{ + mem::size_of, + path::{Path, PathBuf}, +}; + +use crate::index::{self, FAN_LEN, V2_SIGNATURE, Version}; + +/// Returned by [`index::File::at()`]. +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("Could not open pack index file at '{path}'")] + Io { + source: std::io::Error, + path: std::path::PathBuf, + }, + #[error("{message}")] + Corrupt { message: String }, + #[error("Unsupported index version: {version})")] + UnsupportedVersion { version: u32 }, +} + +const N32_SIZE: usize = size_of::(); + +/// Instantiation +impl index::File { + /// Open the pack index file at the given `path`. + /// + /// The `object_hash` is a way to read (and write) the same file format with different hashes, as the hash kind + /// isn't stored within the file format itself. + pub fn at(path: impl AsRef, object_hash: gix_hash::Kind) -> Result { + Self::at_inner(path.as_ref(), object_hash) + } + + fn at_inner(path: &Path, object_hash: gix_hash::Kind) -> Result { + let data = crate::mmap::read_only(path).map_err(|source| Error::Io { + source, + path: path.to_owned(), + })?; + Self::from_data(data, path.to_owned(), object_hash) + } +} + +impl index::File +where + T: crate::FileData, +{ + /// Instantiate an index file from `data` as assumed to be read or memory-mapped from `path`. + pub fn from_data(data: T, path: PathBuf, object_hash: gix_hash::Kind) -> Result { + let idx_len = data.len(); + let hash_len = object_hash.len_in_bytes(); + + let footer_size = hash_len * 2; + if idx_len < FAN_LEN * N32_SIZE + footer_size { + return Err(Error::Corrupt { + message: format!( + "Pack index of size {idx_len} is too small for even an empty index" + ), + }); + } + let (kind, fan, num_objects) = { + let (kind, d) = { + let (sig, d) = data.split_at(V2_SIGNATURE.len()); + if sig == V2_SIGNATURE { + (Version::V2, d) + } else { + (Version::V1, &data[..]) + } + }; + let d = { + if let Version::V2 = kind { + let (vd, dr) = d.split_at(N32_SIZE); + let version = crate::read_u32(vd); + if version != Version::V2 as u32 { + return Err(Error::UnsupportedVersion { version }); + } + dr + } else { + d + } + }; + let (fan, bytes_read) = read_fan(d); + let (_, _d) = d.split_at(bytes_read); + let num_objects = fan[FAN_LEN - 1]; + + (kind, fan, num_objects) + }; + validate_fan(&fan)?; + validate_size(&data, kind, num_objects, hash_len)?; + Ok(Self { + data, + path, + version: kind, + num_objects, + fan, + hash_len, + object_hash, + }) + } +} + +fn read_fan(d: &[u8]) -> ([u32; FAN_LEN], usize) { + assert!(d.len() >= FAN_LEN * N32_SIZE); + + let mut fan = [0; FAN_LEN]; + for (c, f) in d.chunks_exact(N32_SIZE).zip(fan.iter_mut()) { + *f = crate::read_u32(c); + } + (fan, FAN_LEN * N32_SIZE) +} + +fn validate_fan(fan: &[u32; FAN_LEN]) -> Result<(), Error> { + if !crate::fan_is_monotonically_increasing(fan) { + return Err(Error::Corrupt { + message: "Pack index fan-out table must be monotonically increasing".into(), + }); + } + Ok(()) +} + +fn validate_size( + data: &[u8], + kind: Version, + num_objects: u32, + hash_len: usize, +) -> Result<(), Error> { + let num_objects = num_objects as usize; + let footer_size = hash_len * 2; + let expected_size = match kind { + Version::V1 => FAN_LEN + .checked_mul(N32_SIZE) + .and_then(|size| size.checked_add(num_objects.checked_mul(N32_SIZE + hash_len)?)) + .and_then(|size| size.checked_add(footer_size)) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while validating version 1 layout".into(), + })?, + Version::V2 => { + let v2_header_size = V2_SIGNATURE.len() + N32_SIZE + FAN_LEN * N32_SIZE; + let oid_bytes = num_objects + .checked_mul(hash_len) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while validating object ids".into(), + })?; + let table_bytes = num_objects + .checked_mul(N32_SIZE) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while validating 32-bit tables".into(), + })?; + let offset32_start = v2_header_size + .checked_add(oid_bytes) + .and_then(|size| size.checked_add(table_bytes)) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while locating 32-bit offsets".into(), + })?; + let offset32_end = + offset32_start + .checked_add(table_bytes) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while locating 32-bit offsets".into(), + })?; + if offset32_end > data.len() { + return Err(Error::Corrupt { + message: format!( + "Pack index of size {} is too small for {} objects in version 2", + data.len(), + num_objects + ), + }); + } + let (large_offsets, max_large_offset_index) = data[offset32_start..offset32_end] + .chunks_exact(N32_SIZE) + .filter_map(|offset| { + let offset = crate::read_u32(offset); + (offset & (1 << 31) != 0).then_some((offset ^ (1 << 31)) as usize) + }) + .fold((0usize, 0usize), |(count, max_index), index| { + (count + 1, max_index.max(index)) + }); + v2_header_size + .checked_add(oid_bytes) + .and_then(|size| size.checked_add(table_bytes)) + .and_then(|size| size.checked_add(table_bytes)) + .and_then(|size| size.checked_add(large_offsets.checked_mul(size_of::())?)) + .and_then(|size| size.checked_add(footer_size)) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while validating version 2 layout".into(), + }) + .and_then(|expected_size| { + if large_offsets > 0 && max_large_offset_index >= large_offsets { + return Err(Error::Corrupt { + message: format!( + "Pack index references large offset {max_large_offset_index}, but only {large_offsets} large offsets are present" + ), + }); + } + Ok(expected_size) + })? + } + }; + if data.len() != expected_size { + // Aborting here is needed for protection against malformed inputs, or the offset access done later can panic + // as it's done without explicit error handling. + return Err(Error::Corrupt { + message: format!( + "Pack index size is incorrect, expected {expected_size} bytes for {num_objects} objects in version {kind:?}, but got {} bytes", + data.len() + ), + }); + } + Ok(()) +} diff --git a/knot2/third_party/gix-pack/src/index/mod.rs b/knot2/third_party/gix-pack/src/index/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..e2f78e19e2ad388e1704c6945308664d55134d97 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/mod.rs @@ -0,0 +1,163 @@ +//! an index into the pack file + +/// From itertools +/// Create an iterator running multiple iterators in lockstep. +/// +/// The `izip!` iterator yields elements until any subiterator +/// returns `None`. +/// +/// This is a version of the standard ``.zip()`` that's supporting more than +/// two iterators. The iterator element type is a tuple with one element +/// from each of the input iterators. Just like ``.zip()``, the iteration stops +/// when the shortest of the inputs reaches its end. +/// +/// **Note:** The result of this macro is in the general case an iterator +/// composed of repeated `.zip()` and a `.map()`; it has an anonymous type. +/// The special cases of one and two arguments produce the equivalent of +/// `$a.into_iter()` and `$a.into_iter().zip($b)` respectively. +/// +/// Prefer this macro `izip!()` over [`multizip`] for the performance benefits +/// of using the standard library `.zip()`. +/// +/// [`multizip`]: fn.multizip.html +/// +/// ```ignore +/// # use itertools::izip; +/// # +/// # fn main() { +/// +/// // iterate over three sequences side-by-side +/// let mut results = [0, 0, 0, 0]; +/// let inputs = [3, 7, 9, 6]; +/// +/// for (r, index, input) in izip!(&mut results, 0..10, &inputs) { +/// *r = index * 10 + input; +/// } +/// +/// assert_eq!(results, [0 + 3, 10 + 7, 29, 36]); +/// # } +/// ``` +/// +/// (The above is vendored from [itertools](https://github.com/rust-itertools/itertools), +/// including the original doctest, though it has been marked `ignore` here.) +macro_rules! izip { + // @closure creates a tuple-flattening closure for .map() call. usage: + // @closure partial_pattern => partial_tuple , rest , of , iterators + // eg. izip!( @closure ((a, b), c) => (a, b, c) , dd , ee ) + ( @closure $p:pat => $tup:expr ) => { + |$p| $tup + }; + + // The "b" identifier is a different identifier on each recursion level thanks to hygiene. + ( @closure $p:pat => ( $($tup:tt)* ) , $_iter:expr $( , $tail:expr )* ) => { + izip!(@closure ($p, b) => ( $($tup)*, b ) $( , $tail )*) + }; + + // unary + ($first:expr $(,)*) => { + std::iter::IntoIterator::into_iter($first) + }; + + // binary + ($first:expr, $second:expr $(,)*) => { + izip!($first) + .zip($second) + }; + + // n-ary where n > 2 + ( $first:expr $( , $rest:expr )* $(,)* ) => { + izip!($first) + $( + .zip($rest) + )* + .map( + izip!(@closure a => (a) $( , $rest )*) + ) + }; +} + +use crate::MMap; + +/// The version of an index file +#[derive(Default, PartialEq, Eq, Ord, PartialOrd, Debug, Hash, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[allow(missing_docs)] +pub enum Version { + V1 = 1, + #[default] + V2 = 2, +} + +impl Version { + /// The kind of hash to produce to be compatible to this kind of index + pub fn hash(&self) -> gix_hash::Kind { + #[cfg(not(feature = "sha1"))] + unreachable!("pack index versions V1 and V2 require SHA1 support"); + #[cfg(feature = "sha1")] + match self { + Version::V1 | Version::V2 => gix_hash::Kind::Sha1, + } + } +} + +/// A way to indicate if a lookup, despite successful, was ambiguous or yielded exactly +/// one result in the particular index. +pub type PrefixLookupResult = Result; + +/// The type for referring to indices of an entry within the index file. +pub type EntryIndex = u32; + +const FAN_LEN: usize = 256; + +/// A representation of a pack index file +pub struct File { + data: T, + path: std::path::PathBuf, + version: Version, + num_objects: u32, + fan: [u32; FAN_LEN], + hash_len: usize, + object_hash: gix_hash::Kind, +} + +/// Basic file information +impl File +where + T: crate::FileData, +{ + /// The version of the pack index + pub fn version(&self) -> Version { + self.version + } + /// The path of the opened index file + pub fn path(&self) -> &std::path::Path { + &self.path + } + /// The amount of objects stored in the pack and index, as one past the highest entry index. + pub fn num_objects(&self) -> EntryIndex { + self.num_objects + } + /// The kind of hash we assume + pub fn object_hash(&self) -> gix_hash::Kind { + self.object_hash + } +} + +const V2_SIGNATURE: &[u8] = b"\xfftOc"; +/// +pub mod init; + +pub(crate) mod access; +pub use access::Entry; + +pub(crate) mod encode; +/// +pub mod traverse; +mod util; +/// +pub mod verify; +/// +#[cfg(feature = "streaming-input")] +pub mod write; +#[cfg(feature = "streaming-input")] +pub use write::function::write_data_iter_to_stream; diff --git a/knot2/third_party/gix-pack/src/index/traverse/error.rs b/knot2/third_party/gix-pack/src/index/traverse/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..a77b011af271b243458cdc888ad4bb71a6f391b7 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/error.rs @@ -0,0 +1,44 @@ +use crate::index; + +/// Returned by [`index::File::traverse_with_index()`] and [`index::File::traverse_with_lookup`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("One of the traversal processors failed")] + Processor(#[source] E), + #[error("Failed to verify index file checksum")] + IndexVerify(#[source] index::verify::checksum::Error), + #[error("The pack delta tree index could not be built")] + Tree(#[from] crate::cache::delta::from_offsets::Error), + #[error("The tree traversal failed")] + TreeTraversal(#[from] crate::cache::delta::traverse::Error), + #[error(transparent)] + EntryType(#[from] crate::data::entry::decode::Error), + #[error("Object {id} at offset {offset} could not be decoded")] + PackDecode { + id: gix_hash::ObjectId, + offset: u64, + source: crate::data::decode::Error, + }, + #[error("The packfiles checksum didn't match the index file checksum")] + PackMismatch(#[source] gix_hash::verify::Error), + #[error("Failed to verify pack file checksum")] + PackVerify(#[source] crate::verify::checksum::Error), + #[error("Error verifying object at offset {offset} against checksum in the index file")] + PackObjectVerify { + offset: u64, + #[source] + source: gix_object::data::verify::Error, + }, + #[error( + "The CRC32 of {kind} object at offset {offset} didn't match the checksum in the index file: expected {expected}, got {actual}" + )] + Crc32Mismatch { + expected: u32, + actual: u32, + offset: u64, + kind: gix_object::Kind, + }, + #[error("Interrupted")] + Interrupted, +} diff --git a/knot2/third_party/gix-pack/src/index/traverse/mod.rs b/knot2/third_party/gix-pack/src/index/traverse/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..aa865b3902d3c9e685d478d69e9b7e38949cd1f9 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/mod.rs @@ -0,0 +1,242 @@ +use std::sync::atomic::AtomicBool; + +use gix_features::{parallel, progress::Progress, zlib}; + +use crate::index; + +mod reduce; +/// +pub mod with_index; +/// +pub mod with_lookup; +use reduce::Reducer; + +mod error; +pub use error::Error; +use gix_features::progress::DynNestedProgress; + +mod types; +pub use types::{Algorithm, ProgressId, SafetyCheck, Statistics}; + +/// Traversal options for [`index::File::traverse()`]. +#[derive(Debug, Clone)] +pub struct Options { + /// The algorithm to employ. + pub traversal: Algorithm, + /// If `Some`, only use the given number of threads. Otherwise, the number of threads to use will be selected based on + /// the number of available logical cores. + pub thread_limit: Option, + /// The kinds of safety checks to perform. + pub check: SafetyCheck, + /// A function to create a pack cache + pub make_pack_lookup_cache: F, +} + +impl Default for Options crate::cache::Never> { + fn default() -> Self { + Options { + check: Default::default(), + traversal: Default::default(), + thread_limit: None, + make_pack_lookup_cache: || crate::cache::Never, + } + } +} + +/// The outcome of the [`traverse()`][index::File::traverse()] method. +pub struct Outcome { + /// The checksum obtained when hashing the file, which matched the checksum contained within the file. + pub actual_index_checksum: gix_hash::ObjectId, + /// The statistics obtained during traversal. + pub statistics: Statistics, +} + +/// Traversal of pack data files using an index file +impl index::File +where + T: crate::FileData + Sync, +{ + /// Iterate through all _decoded objects_ in the given `pack` and handle them with a `Processor`. + /// The return value is (pack-checksum, [`Outcome`], `progress`), thus the pack traversal will always verify + /// the whole packs checksum to assure it was correct. In case of bit-rod, the operation will abort early without + /// verifying all objects using the [interrupt mechanism][gix_features::interrupt] mechanism. + /// + /// # Algorithms + /// + /// Using the [`Options::traversal`] field one can chose between two algorithms providing different tradeoffs. Both invoke + /// `new_processor()` to create functions receiving decoded objects, their object kind, index entry and a progress instance to provide + /// progress information. + /// + /// * [`Algorithm::DeltaTreeLookup`] builds an index to avoid any unnecessary computation while resolving objects, avoiding + /// the need for a cache entirely, rendering `new_cache()` unused. + /// One could also call [`traverse_with_index()`][index::File::traverse_with_index()] directly. + /// * [`Algorithm::Lookup`] uses a cache created by `new_cache()` to avoid having to re-compute all bases of a delta-chain while + /// decoding objects. + /// One could also call [`traverse_with_lookup()`][index::File::traverse_with_lookup()] directly. + /// + /// Use [`thread_limit`][Options::thread_limit] to further control parallelism and [`check`][SafetyCheck] to define how much the passed + /// objects shall be verified beforehand. + pub fn traverse( + &self, + pack: &crate::data::File, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + processor: Processor, + Options { + traversal, + thread_limit, + check, + make_pack_lookup_cache, + }: Options, + ) -> Result> + where + C: crate::cache::DecodeEntry, + E: std::error::Error + Send + Sync + 'static, + Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> + + Send + + Clone, + F: Fn() -> C + Send + Clone, + { + match traversal { + Algorithm::Lookup => self.traverse_with_lookup( + processor, + pack, + progress, + should_interrupt, + with_lookup::Options { + thread_limit, + check, + make_pack_lookup_cache, + }, + ), + Algorithm::DeltaTreeLookup => self.traverse_with_index( + pack, + processor, + progress, + should_interrupt, + with_index::Options { + check, + thread_limit, + }, + ), + } + } + + fn possibly_verify( + &self, + pack: &crate::data::File, + check: SafetyCheck, + pack_progress: &mut dyn Progress, + index_progress: &mut dyn Progress, + should_interrupt: &AtomicBool, + ) -> Result> + where + E: std::error::Error + Send + Sync + 'static, + { + Ok(if check.file_checksum() { + pack.checksum() + .verify(&self.pack_checksum()) + .map_err(Error::PackMismatch)?; + let (pack_res, id) = parallel::join( + move || pack.verify_checksum(pack_progress, should_interrupt), + move || self.verify_checksum(index_progress, should_interrupt), + ); + pack_res.map_err(Error::PackVerify)?; + id.map_err(Error::IndexVerify)? + } else { + self.index_checksum() + }) + } + + #[allow(clippy::too_many_arguments)] + fn decode_and_process_entry( + &self, + check: SafetyCheck, + pack: &crate::data::File, + cache: &mut C, + buf: &mut Vec, + inflate: &mut zlib::Inflate, + progress: &mut dyn Progress, + index_entry: &index::Entry, + processor: &mut impl FnMut( + gix_object::Kind, + &[u8], + &index::Entry, + &dyn Progress, + ) -> Result<(), E>, + ) -> Result> + where + C: crate::cache::DecodeEntry, + E: std::error::Error + Send + Sync + 'static, + { + let pack_entry = pack.entry(index_entry.pack_offset)?; + let pack_entry_data_offset = pack_entry.data_offset; + let entry_stats = pack + .decode_entry( + pack_entry, + buf, + inflate, + &|id, _| { + let index = self.lookup(id)?; + pack.entry(self.pack_offset_at_index(index)) + .ok() + .map(crate::data::decode::entry::ResolvedBase::InPack) + }, + cache, + ) + .map_err(|e| Error::PackDecode { + source: e, + id: index_entry.oid, + offset: index_entry.pack_offset, + })?; + let object_kind = entry_stats.kind; + let header_size = (pack_entry_data_offset - index_entry.pack_offset) as usize; + let entry_len = header_size + entry_stats.compressed_size; + + process_entry( + check, + object_kind, + buf, + index_entry, + || pack.entry_crc32(index_entry.pack_offset, entry_len), + progress, + processor, + )?; + Ok(entry_stats) + } +} + +#[allow(clippy::too_many_arguments)] +fn process_entry( + check: SafetyCheck, + object_kind: gix_object::Kind, + decompressed: &[u8], + index_entry: &index::Entry, + pack_entry_crc32: impl FnOnce() -> u32, + progress: &dyn Progress, + processor: &mut impl FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E>, +) -> Result<(), Error> +where + E: std::error::Error + Send + Sync + 'static, +{ + if check.object_checksum() { + gix_object::Data::new(decompressed, object_kind, index_entry.oid.kind()) + .verify_checksum(&index_entry.oid) + .map_err(|source| Error::PackObjectVerify { + offset: index_entry.pack_offset, + source, + })?; + if let Some(desired_crc32) = index_entry.crc32 { + let actual_crc32 = pack_entry_crc32(); + if actual_crc32 != desired_crc32 { + return Err(Error::Crc32Mismatch { + actual: actual_crc32, + expected: desired_crc32, + offset: index_entry.pack_offset, + kind: object_kind, + }); + } + } + } + processor(object_kind, decompressed, index_entry, progress).map_err(Error::Processor) +} diff --git a/knot2/third_party/gix-pack/src/index/traverse/reduce.rs b/knot2/third_party/gix-pack/src/index/traverse/reduce.rs new file mode 100644 index 0000000000000000000000000000000000000000..d636231e5220e2b691ea0c1eb814d87bb56a2953 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/reduce.rs @@ -0,0 +1,135 @@ +use std::{ + sync::atomic::{AtomicBool, Ordering}, + time::Instant, +}; + +use gix_features::{ + parallel, + progress::Progress, + threading::{Mutable, OwnShared, lock}, +}; + +use crate::{data, index::traverse}; + +fn add_decode_result(lhs: &mut data::decode::entry::Outcome, rhs: data::decode::entry::Outcome) { + lhs.num_deltas += rhs.num_deltas; + lhs.decompressed_size += rhs.decompressed_size; + lhs.compressed_size += rhs.compressed_size; + lhs.object_size += rhs.object_size; +} + +fn div_decode_result(lhs: &mut data::decode::entry::Outcome, div: usize) { + if div != 0 { + lhs.num_deltas = (lhs.num_deltas as f32 / div as f32) as u32; + lhs.decompressed_size /= div as u64; + lhs.compressed_size /= div; + lhs.object_size /= div as u64; + } +} + +pub struct Reducer<'a, P, E> { + progress: OwnShared>, + check: traverse::SafetyCheck, + then: Instant, + entries_seen: usize, + stats: traverse::Statistics, + should_interrupt: &'a AtomicBool, + _error: std::marker::PhantomData, +} + +impl<'a, P, E> Reducer<'a, P, E> +where + P: Progress, +{ + pub fn from_progress( + progress: OwnShared>, + pack_data_len_in_bytes: usize, + check: traverse::SafetyCheck, + should_interrupt: &'a AtomicBool, + ) -> Self { + let stats = traverse::Statistics { + pack_size: pack_data_len_in_bytes as u64, + ..Default::default() + }; + Reducer { + progress, + check, + then: Instant::now(), + entries_seen: 0, + should_interrupt, + stats, + _error: Default::default(), + } + } +} + +impl parallel::Reduce for Reducer<'_, P, E> +where + P: Progress, + E: std::error::Error + Send + Sync + 'static, +{ + type Input = Result, traverse::Error>; + type FeedProduce = (); + type Output = traverse::Statistics; + type Error = traverse::Error; + + fn feed(&mut self, input: Self::Input) -> Result<(), Self::Error> { + let chunk_stats: Vec<_> = match input { + Err(err @ traverse::Error::PackDecode { .. }) if !self.check.fatal_decode_error() => { + lock(&self.progress).info(format!("Ignoring decode error: {err}")); + return Ok(()); + } + res => res, + }?; + self.entries_seen += chunk_stats.len(); + + let chunk_total = chunk_stats.into_iter().fold( + data::decode::entry::Outcome::default_from_kind(gix_object::Kind::Tree), + |mut total, stats| { + *self + .stats + .objects_per_chain_length + .entry(stats.num_deltas) + .or_insert(0) += 1; + self.stats.total_decompressed_entries_size += stats.decompressed_size; + self.stats.total_compressed_entries_size += stats.compressed_size as u64; + self.stats.total_object_size += stats.object_size; + use gix_object::Kind::*; + match stats.kind { + Commit => self.stats.num_commits += 1, + Tree => self.stats.num_trees += 1, + Blob => self.stats.num_blobs += 1, + Tag => self.stats.num_tags += 1, + } + add_decode_result(&mut total, stats); + total + }, + ); + + add_decode_result(&mut self.stats.average, chunk_total); + lock(&self.progress).set(self.entries_seen); + + if self.should_interrupt.load(Ordering::SeqCst) { + return Err(Self::Error::Interrupted); + } + Ok(()) + } + + fn finalize(mut self) -> Result { + div_decode_result(&mut self.stats.average, self.entries_seen); + + let elapsed_s = self.then.elapsed().as_secs_f32(); + let objects_per_second = (self.entries_seen as f32 / elapsed_s) as u32; + + lock(&self.progress).info(format!( + "of {} objects done in {:.2}s ({} objects/s, ~{}/s)", + self.entries_seen, + elapsed_s, + objects_per_second, + gix_features::progress::bytesize::ByteSize( + self.stats.average.object_size * u64::from(objects_per_second) + ) + )); + Ok(self.stats) + } +} diff --git a/knot2/third_party/gix-pack/src/index/traverse/types.rs b/knot2/third_party/gix-pack/src/index/traverse/types.rs new file mode 100644 index 0000000000000000000000000000000000000000..10571f53e64856430dc742e15a974ad1b9898554 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/types.rs @@ -0,0 +1,116 @@ +use std::{collections::BTreeMap, marker::PhantomData}; + +/// Statistics regarding object encountered during execution of the [`traverse()`][crate::index::File::traverse()] method. +#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Statistics { + /// The average over all decoded objects + pub average: crate::data::decode::entry::Outcome, + /// A mapping of the length of the chain to the amount of objects at that length. + /// + /// A length of 0 indicates full objects, and everything more than that uses the given number + /// of delta objects. + pub objects_per_chain_length: BTreeMap, + /// The amount of bytes in all compressed streams, one per entry + pub total_compressed_entries_size: u64, + /// The amount of bytes in all decompressed streams, one per entry + pub total_decompressed_entries_size: u64, + /// The amount of bytes occupied by all undeltified, decompressed objects + pub total_object_size: u64, + /// The amount of bytes occupied by the pack itself, in bytes + pub pack_size: u64, + /// The amount of objects encountered that where commits + pub num_commits: u32, + /// The amount of objects encountered that where trees + pub num_trees: u32, + /// The amount of objects encountered that where tags + pub num_tags: u32, + /// The amount of objects encountered that where blobs + pub num_blobs: u32, +} + +impl Default for Statistics { + fn default() -> Self { + Statistics { + average: crate::data::decode::entry::Outcome::default_from_kind(gix_object::Kind::Tree), + objects_per_chain_length: Default::default(), + total_compressed_entries_size: 0, + total_decompressed_entries_size: 0, + total_object_size: 0, + pack_size: 0, + num_blobs: 0, + num_commits: 0, + num_trees: 0, + num_tags: 0, + } + } +} + +/// The ways to validate decoded objects before passing them to the processor. +#[derive(Default, Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum SafetyCheck { + /// Don't verify the validity of the checksums stored in the index and pack file + SkipFileChecksumVerification, + + /// All of the above, and also don't perform any object checksum verification + SkipFileAndObjectChecksumVerification, + + /// All of the above, and only log object decode errors. + /// + /// Useful if there is a damaged pack and you would like to traverse as many objects as possible. + SkipFileAndObjectChecksumVerificationAndNoAbortOnDecodeError, + + /// Perform all available safety checks before operating on the pack and + /// abort if any of them fails + #[default] + All, +} + +impl SafetyCheck { + pub(crate) fn file_checksum(&self) -> bool { + matches!(self, SafetyCheck::All) + } + pub(crate) fn object_checksum(&self) -> bool { + matches!( + self, + SafetyCheck::All | SafetyCheck::SkipFileChecksumVerification + ) + } + pub(crate) fn fatal_decode_error(&self) -> bool { + match self { + SafetyCheck::All + | SafetyCheck::SkipFileChecksumVerification + | SafetyCheck::SkipFileAndObjectChecksumVerification => true, + SafetyCheck::SkipFileAndObjectChecksumVerificationAndNoAbortOnDecodeError => false, + } + } +} + +/// The way we verify the pack +#[derive(Default, Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Algorithm { + /// Build an index to allow decoding each delta and base exactly once, saving a lot of computational + /// resource at the expense of resident memory, as we will use an additional `DeltaTree` to accelerate + /// delta chain resolution. + #[default] + DeltaTreeLookup, + /// We lookup each object similarly to what would happen during normal repository use. + /// Uses more compute resources as it will resolve delta chains from back to front, but start right away + /// without indexing or investing any memory in indices. + /// + /// This option may be well suited for big packs in memory-starved system that support memory mapping. + Lookup, +} + +/// The progress ids used in [`traverse()`][crate::index::File::traverse()] . +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// A root progress which isn't actually used, but links to the `ProgressId` of the lookup version of the algorithm. + WithLookup(PhantomData), + /// A root progress which isn't actually used, but links to the `ProgressId` of the indexed version of the algorithm. + WithIndex(PhantomData), +} diff --git a/knot2/third_party/gix-pack/src/index/traverse/with_index.rs b/knot2/third_party/gix-pack/src/index/traverse/with_index.rs new file mode 100644 index 0000000000000000000000000000000000000000..4d256842f23ed4ec104275bd85d7465720850f5b --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/with_index.rs @@ -0,0 +1,258 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +use gix_features::{parallel, progress::DynNestedProgress}; + +use super::Error; +use crate::{ + cache::delta::traverse, + index::{self, traverse::Outcome, util::index_entries_sorted_by_offset_ascending}, +}; + +/// Traversal options for [`traverse_with_index()`][index::File::traverse_with_index()] +#[derive(Default)] +pub struct Options { + /// If `Some`, only use the given number of threads. Otherwise, the number of threads to use will be selected based on + /// the number of available logical cores. + pub thread_limit: Option, + /// The kinds of safety checks to perform. + pub check: crate::index::traverse::SafetyCheck, +} + +/// The progress ids used in [`index::File::traverse_with_index()`]. +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// The amount of bytes currently processed to generate a checksum of the *pack data file*. + HashPackDataBytes, + /// The amount of bytes currently processed to generate a checksum of the *pack index file*. + HashPackIndexBytes, + /// Collect all object hashes into a vector and sort it by their pack offset. + CollectSortedIndexEntries, + /// Count the objects processed when building a cache tree from all objects in a pack index. + TreeFromOffsetsObjects, + /// The amount of objects which were decoded. + DecodedObjects, + /// The amount of bytes that were decoded in total, as the sum of all bytes to represent all decoded objects. + DecodedBytes, +} + +impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::HashPackDataBytes => *b"PTHP", + ProgressId::HashPackIndexBytes => *b"PTHI", + ProgressId::CollectSortedIndexEntries => *b"PTCE", + ProgressId::TreeFromOffsetsObjects => *b"PTDI", + ProgressId::DecodedObjects => *b"PTRO", + ProgressId::DecodedBytes => *b"PTDB", + } + } +} + +/// Traversal with index +impl index::File +where + T: crate::FileData + Sync, +{ + /// Iterate through all _decoded objects_ in the given `pack` and handle them with a `Processor`, using an index to reduce waste + /// at the cost of memory. + /// + /// For more details, see the documentation on the [`traverse()`][index::File::traverse()] method. + pub fn traverse_with_index( + &self, + pack: &crate::data::File, + mut processor: Processor, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + Options { + check, + thread_limit, + }: Options, + ) -> Result> + where + Processor: FnMut( + gix_object::Kind, + &[u8], + &index::Entry, + &dyn gix_features::progress::Progress, + ) -> Result<(), E> + + Send + + Clone, + E: std::error::Error + Send + Sync + 'static, + { + let (verify_result, traversal_result) = parallel::join( + { + let mut pack_progress = progress.add_child_with_id( + format!("Hash of pack '{}'", crate::source_name(pack.path())), + ProgressId::HashPackDataBytes.into(), + ); + let mut index_progress = progress.add_child_with_id( + format!("Hash of index '{}'", crate::source_name(&self.path)), + ProgressId::HashPackIndexBytes.into(), + ); + move || { + let res = self.possibly_verify( + pack, + check, + &mut pack_progress, + &mut index_progress, + should_interrupt, + ); + if res.is_err() { + should_interrupt.store(true, Ordering::SeqCst); + } + res + } + }, + || -> Result<_, Error<_>> { + let sorted_entries = index_entries_sorted_by_offset_ascending( + self, + &mut progress.add_child_with_id( + "collecting sorted index".into(), + ProgressId::CollectSortedIndexEntries.into(), + ), + ); /* Pack Traverse Collect sorted Entries */ + let tree = crate::cache::delta::Tree::from_offsets_in_pack( + pack.path(), + sorted_entries.into_iter().map(Entry::from), + &|e| e.index_entry.pack_offset, + &|id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)), + &mut progress.add_child_with_id( + "indexing".into(), + ProgressId::TreeFromOffsetsObjects.into(), + ), + should_interrupt, + self.object_hash, + )?; + let mut outcome = digest_statistics(tree.traverse( + |slice: crate::data::EntryRange, + source: &crate::data::File, + buf: &mut Vec| { source.read_into(slice, buf) }, + pack, + pack.pack_end() as u64, + move |data, + progress, + traverse::Context { + entry: pack_entry, + entry_end, + decompressed: bytes, + level, + object_kind, + }| { + data.level = level; + data.decompressed_size = pack_entry.decompressed_size; + data.object_kind = object_kind; + data.compressed_size = entry_end - pack_entry.data_offset; + data.object_size = bytes.len() as u64; + let result = index::traverse::process_entry( + check, + object_kind, + bytes, + &data.index_entry, + || { + // TODO: Fix this - we overwrite the header of 'data' which also changes the computed entry size, + // causing index and pack to seemingly mismatch. This is surprising, and should be done differently. + // debug_assert_eq!(&data.index_entry.pack_offset, &pack_entry.pack_offset()); + pack.entry_crc32( + data.index_entry.pack_offset, + (entry_end - data.index_entry.pack_offset) as usize, + ) + }, + progress, + &mut processor, + ); + match result { + Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => { + progress.info(format!("Ignoring decode error: {err}")); + Ok(()) + } + res => res, + } + }, + traverse::Options { + object_progress: Box::new(progress.add_child_with_id( + "Resolving".into(), + ProgressId::DecodedObjects.into(), + )), + size_progress: + &mut progress.add_child_with_id( + "Decoding".into(), + ProgressId::DecodedBytes.into(), + ), + thread_limit, + should_interrupt, + object_hash: self.object_hash, + base_spill: None, + collect_items: true, + max_object_bytes: None, + }, + )?); + outcome.pack_size = pack.data_len() as u64; + Ok(outcome) + }, + ); + Ok(Outcome { + actual_index_checksum: verify_result?, + statistics: traversal_result?, + }) + } +} + +#[derive(Clone)] +struct Entry { + index_entry: crate::index::Entry, + object_kind: gix_object::Kind, + object_size: u64, + decompressed_size: u64, + compressed_size: u64, + level: u16, +} + +impl From for Entry { + fn from(index_entry: crate::index::Entry) -> Self { + Entry { + index_entry, + level: 0, + object_kind: gix_object::Kind::Tree, + object_size: 0, + decompressed_size: 0, + compressed_size: 0, + } + } +} + +fn digest_statistics( + traverse::Outcome { items, .. }: traverse::Outcome, +) -> index::traverse::Statistics { + let mut res = index::traverse::Statistics::default(); + let average = &mut res.average; + for item in items.iter() { + res.total_compressed_entries_size += item.data.compressed_size; + res.total_decompressed_entries_size += item.data.decompressed_size; + res.total_object_size += item.data.object_size; + *res.objects_per_chain_length + .entry(u32::from(item.data.level)) + .or_insert(0) += 1; + + average.decompressed_size += item.data.decompressed_size; + average.compressed_size += item.data.compressed_size as usize; + average.object_size += item.data.object_size; + average.num_deltas += u32::from(item.data.level); + use gix_object::Kind::*; + match item.data.object_kind { + Blob => res.num_blobs += 1, + Tree => res.num_trees += 1, + Tag => res.num_tags += 1, + Commit => res.num_commits += 1, + } + } + + let num_nodes = items.len(); + average.decompressed_size /= num_nodes as u64; + average.compressed_size /= num_nodes; + average.object_size /= num_nodes as u64; + average.num_deltas /= num_nodes as u32; + + res +} diff --git a/knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs b/knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs new file mode 100644 index 0000000000000000000000000000000000000000..55f46684291364075054b0c133b2571af4688246 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs @@ -0,0 +1,208 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +use gix_features::{ + parallel::{self, in_parallel_if}, + progress::{self, Count, DynNestedProgress, Progress}, + threading::{Mutable, OwnShared, lock}, + zlib, +}; + +use super::{Error, Reducer}; +use crate::{ + data, exact_vec, index, + index::{traverse::Outcome, util}, +}; + +/// Traversal options for [`index::File::traverse_with_lookup()`] +pub struct Options { + /// If `Some`, only use the given number of threads. Otherwise, the number of threads to use will be selected based on + /// the number of available logical cores. + pub thread_limit: Option, + /// The kinds of safety checks to perform. + pub check: index::traverse::SafetyCheck, + /// A function to create a pack cache + pub make_pack_lookup_cache: F, +} + +impl Default for Options crate::cache::Never> { + fn default() -> Self { + Options { + check: Default::default(), + thread_limit: None, + make_pack_lookup_cache: || crate::cache::Never, + } + } +} + +/// The progress ids used in [`index::File::traverse_with_lookup()`]. +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// The amount of bytes currently processed to generate a checksum of the *pack data file*. + HashPackDataBytes, + /// The amount of bytes currently processed to generate a checksum of the *pack index file*. + HashPackIndexBytes, + /// Collect all object hashes into a vector and sort it by their pack offset. + CollectSortedIndexEntries, + /// The amount of objects which were decoded by brute-force. + DecodedObjects, +} + +impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::HashPackDataBytes => *b"PTHP", + ProgressId::HashPackIndexBytes => *b"PTHI", + ProgressId::CollectSortedIndexEntries => *b"PTCE", + ProgressId::DecodedObjects => *b"PTRO", + } + } +} + +/// Verify and validate the content of the index file +impl index::File +where + T: crate::FileData + Sync, +{ + /// Iterate through all _decoded objects_ in the given `pack` and handle them with a `Processor` using a cache to reduce the amount of + /// waste while decoding objects. + /// + /// For more details, see the documentation on the [`traverse()`][index::File::traverse()] method. + pub fn traverse_with_lookup( + &self, + mut processor: Processor, + pack: &data::File, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + Options { + thread_limit, + check, + make_pack_lookup_cache, + }: Options, + ) -> Result> + where + C: crate::cache::DecodeEntry, + E: std::error::Error + Send + Sync + 'static, + Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> + + Send + + Clone, + F: Fn() -> C + Send + Clone, + { + let (verify_result, traversal_result) = parallel::join( + { + let mut pack_progress = progress.add_child_with_id( + format!("Hash of pack '{}'", crate::source_name(pack.path())), + ProgressId::HashPackDataBytes.into(), + ); + let mut index_progress = progress.add_child_with_id( + format!("Hash of index '{}'", crate::source_name(&self.path)), + ProgressId::HashPackIndexBytes.into(), + ); + move || { + let res = self.possibly_verify( + pack, + check, + &mut pack_progress, + &mut index_progress, + should_interrupt, + ); + if res.is_err() { + should_interrupt.store(true, Ordering::SeqCst); + } + res + } + }, + || { + let index_entries = util::index_entries_sorted_by_offset_ascending( + self, + &mut progress.add_child_with_id( + "collecting sorted index".into(), + ProgressId::CollectSortedIndexEntries.into(), + ), + ); + + let (chunk_size, thread_limit, available_cores) = + parallel::optimize_chunk_size_and_thread_limit( + 1000, + Some(index_entries.len()), + thread_limit, + None, + ); + let there_are_enough_entries_to_process = + || index_entries.len() > chunk_size * available_cores; + let input_chunks = index_entries.chunks(chunk_size); + let reduce_progress = OwnShared::new(Mutable::new({ + let mut p = progress + .add_child_with_id("Traversing".into(), ProgressId::DecodedObjects.into()); + p.init( + Some(self.num_objects() as usize), + progress::count("objects"), + ); + p + })); + let state_per_thread = { + let reduce_progress = reduce_progress.clone(); + move |index| { + ( + make_pack_lookup_cache(), + Vec::with_capacity(2048), // decode buffer + zlib::Inflate::default(), + lock(&reduce_progress).add_child_with_id( + format!("thread {index}"), + gix_features::progress::UNKNOWN, + ), // per thread progress + ) + } + }; + + in_parallel_if( + there_are_enough_entries_to_process, + input_chunks, + thread_limit, + state_per_thread, + move |entries: &[index::Entry], + (cache, buf, inflate, progress)| + -> Result, Error<_>> { + progress.init( + Some(entries.len()), + gix_features::progress::count_with_decimals("objects", 2), + ); + let mut stats = exact_vec(entries.len()); + progress.set(0); + for index_entry in entries.iter() { + let result = self.decode_and_process_entry( + check, + pack, + cache, + buf, + inflate, + progress, + index_entry, + &mut processor, + ); + progress.inc(); + let stat = match result { + Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => { + progress.info(format!("Ignoring decode error: {err}")); + continue; + } + res => res, + }?; + stats.push(stat); + if should_interrupt.load(Ordering::Relaxed) { + break; + } + } + Ok(stats) + }, + Reducer::from_progress(reduce_progress, pack.data_len(), check, should_interrupt), + ) + }, + ); + Ok(Outcome { + actual_index_checksum: verify_result?, + statistics: traversal_result?, + }) + } +} diff --git a/knot2/third_party/gix-pack/src/index/util.rs b/knot2/third_party/gix-pack/src/index/util.rs new file mode 100644 index 0000000000000000000000000000000000000000..42eb3474c8584eee33c68a3a266a752a73f8d749 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/util.rs @@ -0,0 +1,23 @@ +use std::time::Instant; + +use gix_features::progress::{self, Progress}; + +use crate::exact_vec; + +pub(crate) fn index_entries_sorted_by_offset_ascending( + idx: &crate::index::File, + progress: &mut dyn Progress, +) -> Vec { + progress.init(Some(idx.num_objects as usize), progress::count("entries")); + let start = Instant::now(); + + let mut v = exact_vec(idx.num_objects as usize); + for entry in idx.iter() { + v.push(entry); + progress.inc(); + } + v.sort_by_key(|e| e.pack_offset); + + progress.show_throughput(start); + v +} diff --git a/knot2/third_party/gix-pack/src/index/verify.rs b/knot2/third_party/gix-pack/src/index/verify.rs new file mode 100644 index 0000000000000000000000000000000000000000..6b601a17473154a388d0838cc94e1abe8844861c --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/verify.rs @@ -0,0 +1,285 @@ +use std::sync::atomic::AtomicBool; + +use gix_features::progress::{DynNestedProgress, Progress}; +use gix_object::WriteTo; + +use crate::index; + +/// +pub mod integrity { + use std::marker::PhantomData; + + use gix_object::bstr::BString; + + /// Returned by [`index::File::verify_integrity()`][crate::index::File::verify_integrity()]. + #[derive(thiserror::Error, Debug)] + #[allow(missing_docs)] + pub enum Error { + #[error("Reserialization of an object failed")] + Io(#[from] std::io::Error), + #[error( + "The fan at index {index} is out of order as it's larger then the following value." + )] + Fan { index: usize }, + #[error("{kind} object {id} could not be decoded")] + ObjectDecode { + source: gix_object::decode::Error, + kind: gix_object::Kind, + id: gix_hash::ObjectId, + }, + #[error( + "{kind} object {id} wasn't re-encoded without change, wanted\n{expected}\n\nGOT\n\n{actual}" + )] + ObjectEncodeMismatch { + kind: gix_object::Kind, + id: gix_hash::ObjectId, + expected: BString, + actual: BString, + }, + } + + /// Returned by [`index::File::verify_integrity()`][crate::index::File::verify_integrity()]. + pub struct Outcome { + /// The computed checksum of the index which matched the stored one. + pub actual_index_checksum: gix_hash::ObjectId, + /// The packs traversal outcome, if one was provided + pub pack_traverse_statistics: Option, + } + + /// Additional options to define how the integrity should be verified. + #[derive(Clone)] + pub struct Options { + /// The thoroughness of the verification + pub verify_mode: crate::index::verify::Mode, + /// The way to traverse packs + pub traversal: crate::index::traverse::Algorithm, + /// The amount of threads to use of `Some(N)`, with `None|Some(0)` using all available cores are used. + pub thread_limit: Option, + /// A function to create a pack cache + pub make_pack_lookup_cache: F, + } + + impl Default for Options crate::cache::Never> { + fn default() -> Self { + Options { + verify_mode: Default::default(), + traversal: Default::default(), + thread_limit: None, + make_pack_lookup_cache: || crate::cache::Never, + } + } + } + + /// The progress ids used in [`index::File::verify_integrity()`][crate::index::File::verify_integrity()]. + /// + /// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. + #[derive(Debug, Copy, Clone)] + pub enum ProgressId { + /// The amount of bytes read to verify the index checksum. + ChecksumBytes, + /// A root progress for traversal which isn't actually used directly, but here to link to the respective `ProgressId` types. + Traverse(PhantomData), + } + + impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::ChecksumBytes => *b"PTHI", + ProgressId::Traverse(_) => gix_features::progress::UNKNOWN, + } + } + } +} + +/// +pub mod checksum { + /// Returned by [`index::File::verify_checksum()`][crate::index::File::verify_checksum()]. + pub type Error = crate::verify::checksum::Error; +} + +/// Various ways in which a pack and index can be verified +#[derive(Default, Debug, Eq, PartialEq, Hash, Clone, Copy)] +pub enum Mode { + /// Validate the object hash and CRC32 + HashCrc32, + /// Validate hash and CRC32, and decode each non-Blob object. + /// Each object should be valid, i.e. be decodable. + HashCrc32Decode, + /// Validate hash and CRC32, and decode and encode each non-Blob object. + /// Each object should yield exactly the same hash when re-encoded. + #[default] + HashCrc32DecodeEncode, +} + +/// Information to allow verifying the integrity of an index with the help of its corresponding pack. +pub struct PackContext<'a, F> { + /// The pack data file itself. + pub data: &'a crate::data::File, + /// The options further configuring the pack traversal and verification + pub options: integrity::Options, +} + +/// Verify and validate the content of the index file +impl index::File +where + T: crate::FileData + Sync, +{ + /// Returns the trailing hash stored at the end of this index file. + /// + /// It's a hash over all bytes of the index. + pub fn index_checksum(&self) -> gix_hash::ObjectId { + gix_hash::ObjectId::from_bytes_or_panic(&self.data[self.data.len() - self.hash_len..]) + } + + /// Returns the hash of the pack data file that this index file corresponds to. + /// + /// It should [`crate::data::File::checksum()`] of the corresponding pack data file. + pub fn pack_checksum(&self) -> gix_hash::ObjectId { + let from = self.data.len() - self.hash_len * 2; + gix_hash::ObjectId::from_bytes_or_panic(&self.data[from..][..self.hash_len]) + } + + /// Validate that our [`index_checksum()`][index::File::index_checksum()] matches the actual contents + /// of this index file, and return it if it does. + pub fn verify_checksum( + &self, + progress: &mut dyn Progress, + should_interrupt: &AtomicBool, + ) -> Result { + crate::verify::checksum_on_disk_or_mmap( + self.path(), + &self.data, + self.index_checksum(), + self.object_hash, + progress, + should_interrupt, + ) + } + + /// The most thorough validation of integrity of both index file and the corresponding pack data file, if provided. + /// Returns the checksum of the index file, the traversal outcome and the given progress if the integrity check is successful. + /// + /// If `pack` is provided, it is expected (and validated to be) the pack belonging to this index. + /// It will be used to validate internal integrity of the pack before checking each objects integrity + /// is indeed as advertised via its SHA1 as stored in this index, as well as the CRC32 hash. + /// The last member of the Option is a function returning an implementation of [`crate::cache::DecodeEntry`] to be used if + /// the [`index::traverse::Algorithm`] is `Lookup`. + /// To set this to `None`, use `None::<(_, _, _, fn() -> crate::cache::Never)>`. + /// + /// The `thread_limit` optionally specifies the amount of threads to be used for the [pack traversal][index::File::traverse()]. + /// `make_cache` is only used in case a `pack` is specified, use existing implementations in the [`crate::cache`] module. + /// + /// # Tradeoffs + /// + /// The given `progress` is inevitably consumed if there is an error, which is a tradeoff chosen to easily allow using `?` in the + /// error case. + pub fn verify_integrity( + &self, + pack: Option>, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + ) -> Result> + where + C: crate::cache::DecodeEntry, + F: Fn() -> C + Send + Clone, + { + if let Some(first_invalid) = crate::verify::fan(&self.fan) { + return Err(index::traverse::Error::Processor(integrity::Error::Fan { + index: first_invalid, + })); + } + + match pack { + Some(PackContext { + data: pack, + options: + integrity::Options { + verify_mode, + traversal, + thread_limit, + make_pack_lookup_cache, + }, + }) => self + .traverse( + pack, + progress, + should_interrupt, + { + let mut encode_buf = Vec::with_capacity(2048); + move |kind, data, index_entry, progress| { + Self::verify_entry( + verify_mode, + &mut encode_buf, + kind, + data, + index_entry, + progress, + ) + } + }, + index::traverse::Options { + traversal, + thread_limit, + check: index::traverse::SafetyCheck::All, + make_pack_lookup_cache, + }, + ) + .map(|o| integrity::Outcome { + actual_index_checksum: o.actual_index_checksum, + pack_traverse_statistics: Some(o.statistics), + }), + None => self + .verify_checksum( + &mut progress.add_child_with_id( + "Sha1 of index".into(), + integrity::ProgressId::ChecksumBytes.into(), + ), + should_interrupt, + ) + .map_err(index::traverse::Error::IndexVerify) + .map(|id| integrity::Outcome { + actual_index_checksum: id, + pack_traverse_statistics: None, + }), + } + } + + #[allow(clippy::too_many_arguments)] + fn verify_entry( + verify_mode: Mode, + encode_buf: &mut Vec, + object_kind: gix_object::Kind, + buf: &[u8], + index_entry: &index::Entry, + _progress: &dyn gix_features::progress::Progress, + ) -> Result<(), integrity::Error> { + if let Mode::HashCrc32Decode | Mode::HashCrc32DecodeEncode = verify_mode { + use gix_object::Kind::*; + match object_kind { + Tree | Commit | Tag => { + let object = + gix_object::ObjectRef::from_bytes(buf, object_kind, index_entry.oid.kind()) + .map_err(|err| integrity::Error::ObjectDecode { + source: err, + kind: object_kind, + id: index_entry.oid, + })?; + if let Mode::HashCrc32DecodeEncode = verify_mode { + encode_buf.clear(); + object.write_to(&mut *encode_buf)?; + if encode_buf.as_slice() != buf { + return Err(integrity::Error::ObjectEncodeMismatch { + kind: object_kind, + id: index_entry.oid, + expected: buf.into(), + actual: encode_buf.clone().into(), + }); + } + } + } + Blob => {} + } + } + Ok(()) + } +} diff --git a/knot2/third_party/gix-pack/src/index/write/error.rs b/knot2/third_party/gix-pack/src/index/write/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..4a44d739bfbfd2a138990d31f044137027e131ad --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/write/error.rs @@ -0,0 +1,27 @@ +/// Returned by [`crate::index::write_data_iter_to_stream()`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("An error occurred when writing the pack index file")] + Io(#[from] gix_hash::io::Error), + #[error("A pack entry could not be extracted")] + PackEntryDecode(#[from] crate::data::input::Error), + #[error("Indices of type {} cannot be written, only {} are supported", *.0 as usize, crate::index::Version::default() as usize)] + Unsupported(crate::index::Version), + #[error("pack has {0} unresolved deltas")] + UnresolvedRefDeltas(usize), + #[error("Could not append the missing bases of a thin pack")] + CompleteThinPack(#[source] Box), + #[error( + "The iterator failed to set a trailing hash over all prior pack entries in the last provided entry" + )] + IteratorInvariantTrailer, + #[error("Only u32::MAX objects can be stored in a pack, found {0}")] + IteratorInvariantTooManyObjects(usize), + #[error("{pack_offset} is not a valid offset for pack offset {distance}")] + IteratorInvariantBaseOffset { pack_offset: u64, distance: u64 }, + #[error(transparent)] + Tree(#[from] crate::cache::delta::Error), + #[error(transparent)] + TreeTraversal(#[from] crate::cache::delta::traverse::Error), +} diff --git a/knot2/third_party/gix-pack/src/index/write/mod.rs b/knot2/third_party/gix-pack/src/index/write/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..f91def6f179c7ad8aa0d8a81f38858a4348ca96d --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/write/mod.rs @@ -0,0 +1,385 @@ +pub use error::Error; + +mod error; + +#[derive(Clone)] +pub(crate) struct TreeEntry { + pub id: gix_hash::ObjectId, + pub crc32: u32, +} + +/// Information gathered while executing [`write_data_iter_to_stream()`][crate::index::write_data_iter_to_stream] +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// The version of the verified index + pub index_version: crate::index::Version, + /// The verified checksum of the verified index + pub index_hash: gix_hash::ObjectId, + + /// The hash of the '.pack' file, also found in its trailing bytes + pub data_hash: gix_hash::ObjectId, + /// The amount of objects that were verified, always the amount of objects in the pack. + pub num_objects: u32, +} + +/// The bases a [thin pack completion](crate::index::write_data_iter_to_stream()) appended to the pack. +pub struct AppendedBases { + /// The id, pack offset and CRC32 of each appended base. Requested bases that couldn't be found are left out. + pub entries: Vec<(gix_hash::ObjectId, crate::data::Offset, u32)>, + /// The offset one past the last appended entry. + pub pack_entries_end: u64, + /// The trailing hash of the rewritten pack. + pub pack_hash: gix_hash::ObjectId, +} + +/// Appends the given base objects to the pack and rewrites its header and trailer, as `git index-pack --fix-thin` does. +/// Each call replaces the bases appended by the previous one. +pub type CompleteThinPack<'a> = + &'a mut dyn FnMut( + &[gix_hash::ObjectId], + ) + -> Result>; + +/// The progress ids used in [`write_data_iter_to_stream()`][crate::index::write_data_iter_to_stream()]. +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// Counts the amount of objects that were index thus far. + IndexObjects, + /// The amount of bytes that were decompressed while decoding pack entries. + /// + /// This is done to determine entry boundaries. + DecompressedBytes, + /// The amount of objects whose hashes were computed. + /// + /// This is done by decoding them, which typically involves decoding delta objects. + ResolveObjects, + /// The amount of bytes that were decoded in total, as the sum of all bytes to represent all resolved objects. + DecodedBytes, + /// The amount of bytes written to the index file. + IndexBytesWritten, +} + +impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::IndexObjects => *b"IWIO", + ProgressId::DecompressedBytes => *b"IWDB", + ProgressId::ResolveObjects => *b"IWRO", + ProgressId::DecodedBytes => *b"IWDB", + ProgressId::IndexBytesWritten => *b"IWBW", + } + } +} + +pub(super) mod function { + use std::{io, sync::atomic::AtomicBool}; + + use gix_features::progress::{self, Count, Progress, prodash::DynNestedProgress}; + + use crate::cache::delta::{Item, Tree, traverse}; + + use super::{ + AppendedBases, CompleteThinPack, Error, Outcome, ProgressId, TreeEntry, modify_base, + }; + + /// Write information about `entries` as obtained from a pack data file into a pack index file via the `out` stream. + /// The resolver produced by `make_resolver` must resolve pack entries from the same pack data file that produced the + /// `entries` iterator. + /// + /// * `kind` is the version of pack index to produce, use [`crate::index::Version::default()`] if in doubt. + /// * `tread_limit` is used for a parallel tree traversal for obtaining object hashes with optimal performance. + /// * `root_progress` is the top-level progress to stay informed about the progress of this potentially long-running + /// computation. + /// * `object_hash` defines what kind of object hash we write into the index file. + /// * `pack_version` is the version of the underlying pack for which `entries` are read. It's used in case none of these objects are provided + /// to compute a pack-hash. + /// * `complete_thin_pack` is called with the ids of the bases of ref-deltas that aren't in the pack. Without it, such + /// ref-deltas fail the write. + /// + /// # Remarks + /// + /// * `make_resolver()` will only be called after the iterator stopped returning elements and produces a function that + /// provides all bytes belonging to a pack entry writing them to the given mutable output `Vec`. + /// It should return `None` if the entry cannot be resolved from the pack that produced the `entries` iterator, causing + /// the write operation to fail. It's called again after `complete_thin_pack` changed the pack. + #[allow(clippy::too_many_arguments)] + pub fn write_data_iter_to_stream( + version: crate::index::Version, + make_resolver: F, + entries: &mut dyn Iterator< + Item = Result, + >, + thread_limit: Option, + root_progress: &mut dyn DynNestedProgress, + out: &mut dyn io::Write, + should_interrupt: &AtomicBool, + object_hash: gix_hash::Kind, + pack_version: crate::data::Version, + complete_thin_pack: Option>, + ) -> Result + where + F: Fn() -> io::Result<(F2, R)>, + R: Send + Sync, + F2: Fn(crate::data::EntryRange, &R, &mut Vec) -> bool + Send + Clone, + { + if version != crate::index::Version::default() { + return Err(Error::Unsupported(version)); + } + let mut num_objects: usize = 0; + let mut last_seen_trailer = None; + let (anticipated_num_objects, upper_bound) = entries.size_hint(); + let worst_case_num_objects_after_thin_pack_resolution = + upper_bound.unwrap_or(anticipated_num_objects); + let mut tree = Tree::with_capacity(worst_case_num_objects_after_thin_pack_resolution)?; + let indexing_start = std::time::Instant::now(); + + root_progress.init(Some(4), progress::steps()); + let mut objects_progress = + root_progress.add_child_with_id("indexing".into(), ProgressId::IndexObjects.into()); + objects_progress.init(Some(anticipated_num_objects), progress::count("objects")); + let mut decompressed_progress = root_progress + .add_child_with_id("decompressing".into(), ProgressId::DecompressedBytes.into()); + decompressed_progress.init(None, progress::bytes()); + let mut pack_entries_end: u64 = 0; + + for entry in entries { + let crate::data::input::Entry { + header, + pack_offset, + crc32, + header_size, + compressed: _, + compressed_size, + decompressed_size, + trailer, + } = entry?; + + decompressed_progress.inc_by(decompressed_size as usize); + + let entry_len = u64::from(header_size) + compressed_size; + pack_entries_end = pack_offset + entry_len; + + let crc32 = crc32.expect( + "crc32 to be computed by the iterator. Caller assures correct configuration.", + ); + + use crate::data::entry::Header::*; + match header { + Tree | Blob | Commit | Tag => { + tree.add_root( + pack_offset, + TreeEntry { + id: object_hash.null(), + crc32, + }, + )?; + } + RefDelta { base_id } => { + tree.add_ref_child( + base_id, + pack_offset, + TreeEntry { + id: object_hash.null(), + crc32, + }, + )?; + } + OfsDelta { base_distance } => { + let base_pack_offset = crate::data::entry::Header::verified_base_pack_offset( + pack_offset, + base_distance, + ) + .ok_or(Error::IteratorInvariantBaseOffset { + pack_offset, + distance: base_distance, + })?; + tree.add_child( + base_pack_offset, + pack_offset, + TreeEntry { + id: object_hash.null(), + crc32, + }, + )?; + } + } + last_seen_trailer = trailer; + num_objects += 1; + objects_progress.inc(); + } + let mut num_objects: u32 = num_objects + .try_into() + .map_err(|_| Error::IteratorInvariantTooManyObjects(num_objects))?; + + objects_progress.show_throughput(indexing_start); + decompressed_progress.show_throughput(indexing_start); + drop(objects_progress); + drop(decompressed_progress); + + root_progress.inc(); + + let mut traverse = + |tree: Tree, pack_entries_end: u64| { + let (resolver, pack) = make_resolver().map_err(gix_hash::io::Error::from)?; + tree.traverse( + resolver, + &pack, + pack_entries_end, + |data, + _progress, + traverse::Context { + decompressed: bytes, + object_kind, + .. + }| { modify_base(data, bytes, object_kind, object_hash) }, + traverse::Options { + object_progress: Box::new(root_progress.add_child_with_id( + "Resolving".into(), + ProgressId::ResolveObjects.into(), + )), + size_progress: &mut root_progress + .add_child_with_id("Decoding".into(), ProgressId::DecodedBytes.into()), + thread_limit, + should_interrupt, + object_hash, + base_spill: None, + collect_items: true, + max_object_bytes: None, + }, + ) + .map_err(Error::from) + }; + let sorted_pack_offsets_by_oid = { + let traverse::Outcome { + mut items, + unresolved, + } = traverse(tree, pack_entries_end)?; + if let Some(unresolved) = unresolved { + let appended = resolve_thin_pack( + unresolved, + &mut items, + complete_thin_pack, + pack_entries_end, + object_hash, + traverse, + )?; + num_objects = u32::try_from(appended.entries.len()) + .ok() + .and_then(|appended| num_objects.checked_add(appended)) + .ok_or(Error::IteratorInvariantTooManyObjects( + num_objects as usize + appended.entries.len(), + ))?; + last_seen_trailer = Some(appended.pack_hash); + } + root_progress.inc(); + + { + let _progress = root_progress + .add_child_with_id("sorting by id".into(), gix_features::progress::UNKNOWN); + items.sort_by_key(|e| e.data.id); + } + + root_progress.inc(); + items + }; + + let pack_hash = match last_seen_trailer { + Some(ph) => ph, + None if num_objects == 0 => { + let header = crate::data::header::encode(pack_version, 0); + let mut hasher = gix_hash::hasher(object_hash); + hasher.update(&header); + hasher.try_finalize().map_err(gix_hash::io::Error::from)? + } + None => return Err(Error::IteratorInvariantTrailer), + }; + let index_hash = crate::index::encode::write_to( + out, + sorted_pack_offsets_by_oid, + &pack_hash, + version, + object_hash, + &mut root_progress.add_child_with_id( + "writing index file".into(), + ProgressId::IndexBytesWritten.into(), + ), + )?; + root_progress.show_throughput_with( + indexing_start, + num_objects as usize, + progress::count("objects").expect("unit always set"), + progress::MessageLevel::Success, + ); + Ok(Outcome { + index_version: version, + index_hash, + data_hash: pack_hash, + num_objects, + }) + } + + fn resolve_thin_pack( + unresolved: Tree, + items: &mut Vec>, + complete_thin_pack: Option>, + pack_entries_end: u64, + object_hash: gix_hash::Kind, + mut traverse: impl FnMut(Tree, u64) -> Result, Error>, + ) -> Result { + let Some(complete_thin_pack) = complete_thin_pack else { + return Err(Error::UnresolvedRefDeltas(unresolved.num_items())); + }; + let mut with_bases = |ids: &[gix_hash::ObjectId]| { + let appended = complete_thin_pack(ids).map_err(Error::CompleteThinPack)?; + let mut tree = unresolved.clone(); + for &(_, offset, crc32) in &appended.entries { + tree.add_root( + offset, + TreeEntry { + id: object_hash.null(), + crc32, + }, + )?; + } + Ok::<_, Error>((tree, appended)) + }; + let (tree, mut appended) = with_bases(&unresolved.ref_bases())?; + let mut outcome = traverse(tree, appended.pack_entries_end)?; + let in_pack: std::collections::HashSet<_> = outcome + .items + .iter() + .filter(|item| item.offset < pack_entries_end) + .map(|item| item.data.id) + .collect(); + if appended.entries.iter().any(|(id, ..)| in_pack.contains(id)) { + let missing: Vec<_> = appended + .entries + .iter() + .map(|(id, ..)| *id) + .filter(|id| !in_pack.contains(id)) + .collect(); + let tree; + (tree, appended) = with_bases(&missing)?; + outcome = traverse(tree, appended.pack_entries_end)?; + } + if let Some(unresolved) = outcome.unresolved { + return Err(Error::UnresolvedRefDeltas(unresolved.num_items())); + } + items.extend(outcome.items); + Ok(appended) + } +} + +fn modify_base( + entry: &mut TreeEntry, + decompressed: &[u8], + object_kind: gix_object::Kind, + hash: gix_hash::Kind, +) -> Result<(), gix_hash::hasher::Error> { + let id = gix_object::compute_hash(hash, object_kind, decompressed)?; + entry.id = id; + Ok(()) +} diff --git a/knot2/third_party/gix-pack/src/lib.rs b/knot2/third_party/gix-pack/src/lib.rs new file mode 100755 index 0000000000000000000000000000000000000000..9b7f4a7c86c3536f66310dc6873f62d7ab39cc94 --- /dev/null +++ b/knot2/third_party/gix-pack/src/lib.rs @@ -0,0 +1,104 @@ +//! Git stores all of its data as _Objects_, which are data along with a hash over all data. Storing objects efficiently +//! is what git packs are concerned about. +//! +//! Packs consist of [data files][data::File] and [index files][index::File]. The latter can be generated from a data file +//! and make accessing objects within a pack feasible. +//! +//! A [Bundle] conveniently combines a data pack alongside its index to allow [finding][Find] objects or verifying the pack. +//! Objects returned by `.find(…)` are [objects][gix_object::Data] which know their pack location in order to speed up +//! various common operations like creating new packs from existing ones. +//! +//! When traversing all objects in a pack, a _delta tree acceleration structure_ can be built from pack data or an index +//! in order to decompress packs in parallel and without any waste. +//! ## Feature Flags +#![cfg_attr( + all(doc, feature = "document-features"), + doc = ::document_features::document_features!() +)] +#![cfg_attr(all(doc, feature = "document-features"), feature(doc_cfg))] +#![deny(unsafe_code)] + +use std::{borrow::Cow, ops::Deref, path::Path}; + +pub use memmap2::Mmap as MMap; + +/// A byte-oriented backing store for pack data and indices. +pub trait FileData: Deref {} + +impl FileData for T where T: Deref {} + +/// +pub mod bundle; +/// A bundle of pack data and the corresponding pack index +pub struct Bundle { + /// The pack file corresponding to `index` + pub pack: data::File, + /// The index file corresponding to `pack` + pub index: index::File, +} + +/// +pub mod find; + +/// +pub mod cache; +/// +pub mod data; + +mod find_traits; +pub use find_traits::{Find, FindExt}; + +/// +pub mod index; +/// +pub mod multi_index; + +/// +pub mod verify; + +mod mmap { + use std::path::Path; + + pub fn read_only(path: &Path) -> std::io::Result { + let file = std::fs::File::open(path)?; + // SAFETY: we have to take the risk of somebody changing the file underneath. Git never writes into the same file. + #[expect(unsafe_code)] + unsafe { + memmap2::MmapOptions::new().map_copy_read_only(&file) + } + } +} + +/// Return a display-friendly name for pack- or index-related progress messages. +/// +/// Prefer the file name, but fall back to the full path for paths without a terminal component. +fn source_name(path: &Path) -> Cow<'_, str> { + if path.as_os_str().is_empty() { + Cow::Borrowed("") + } else if let Some(name) = path.file_name() { + name.to_string_lossy() + } else { + path.as_os_str().to_string_lossy() + } +} + +#[inline] +fn read_u32(b: &[u8]) -> u32 { + u32::from_be_bytes(b.try_into().unwrap()) +} + +#[inline] +fn read_u64(b: &[u8]) -> u64 { + u64::from_be_bytes(b.try_into().unwrap()) +} + +fn exact_vec(capacity: usize) -> Vec { + let mut v = Vec::new(); + v.reserve_exact(capacity); + v +} + +#[inline] +fn fan_is_monotonically_increasing(fan: &[u32]) -> bool { + !fan.windows(2).any(|window| window[0] > window[1]) +} diff --git a/knot2/third_party/gix-pack/src/multi_index/access.rs b/knot2/third_party/gix-pack/src/multi_index/access.rs new file mode 100644 index 0000000000000000000000000000000000000000..2f797535e9fd90b4a6d3e47fb36d3cb4b1266849 --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/access.rs @@ -0,0 +1,149 @@ +use std::{ + ops::Range, + path::{Path, PathBuf}, +}; + +use crate::{ + data, + index::PrefixLookupResult, + multi_index::{EntryIndex, File, PackIndex, Version}, +}; + +/// Represents an entry within a multi index file, effectively mapping object [`IDs`][gix_hash::ObjectId] to pack data +/// files and the offset within. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Entry { + /// The ID of the object. + pub oid: gix_hash::ObjectId, + /// The offset to the object's header in the pack data file. + pub pack_offset: data::Offset, + /// The index of the pack matching our [`File::index_names()`] slice. + pub pack_index: PackIndex, +} + +/// Access methods +impl File +where + T: crate::FileData, +{ + /// Returns the version of the multi-index file. + pub fn version(&self) -> Version { + self.version + } + /// Returns the path from which the multi-index file was loaded. + /// + /// Note that it might have changed in the mean time, or might have been removed as well. + pub fn path(&self) -> &Path { + &self.path + } + /// Returns the amount of indices stored in this multi-index file. It's the same as [File::index_names().len()][File::index_names()], + /// and returned as one past the highest known index. + pub fn num_indices(&self) -> PackIndex { + self.num_indices + } + /// Returns the total amount of objects available for lookup, and returned as one past the highest known entry index + pub fn num_objects(&self) -> EntryIndex { + self.num_objects + } + /// Returns the kind of hash function used for object ids available in this index. + pub fn object_hash(&self) -> gix_hash::Kind { + self.object_hash + } + /// Returns the checksum over the entire content of the file (excluding the checksum itself). + /// + /// It can be used to validate it didn't change after creation. + pub fn checksum(&self) -> gix_hash::ObjectId { + gix_hash::ObjectId::from_bytes_or_panic(&self.data[self.data.len() - self.hash_len..]) + } + /// Return all names of index files (`*.idx`) whose objects we contain. + /// + /// The corresponding pack can be found by replacing the `.idx` extension with `.pack`. + pub fn index_names(&self) -> &[PathBuf] { + &self.index_names + } +} + +impl File +where + T: crate::FileData, +{ + /// Return the object id at the given `index`, which ranges from 0 to [File::num_objects()]. + pub fn oid_at_index(&self, index: EntryIndex) -> &gix_hash::oid { + debug_assert!(index < self.num_objects, "index out of bounds"); + let index: usize = index as usize; + let start = self.lookup_ofs + index * self.hash_len; + gix_hash::oid::from_bytes_unchecked(&self.data[start..][..self.hash_len]) + } + + /// Given a `prefix`, find an object that matches it uniquely within this index and return `Some(Ok(entry_index))`. + /// If there is more than one object matching the object `Some(Err(())` is returned. + /// + /// Finally, if no object matches the index, the return value is `None`. + /// + /// Pass `candidates` to obtain the set of entry-indices matching `prefix`, with the same return value as + /// one would have received if it remained `None`. It will be empty if no object matched the `prefix`. + /// + // NOTE: pretty much the same things as in `index::File::lookup`, change things there + // as well. + pub fn lookup_prefix( + &self, + prefix: gix_hash::Prefix, + candidates: Option<&mut Range>, + ) -> Option { + crate::index::access::lookup_prefix( + prefix, + candidates, + &self.fan, + &|idx| self.oid_at_index(idx), + self.num_objects, + ) + } + + /// Find the index ranging from 0 to [File::num_objects()] that belongs to data associated with `id`, or `None` if it wasn't found. + /// + /// Use this index for finding additional information via [`File::pack_id_and_pack_offset_at_index()`]. + pub fn lookup(&self, id: impl AsRef) -> Option { + crate::index::access::lookup(id.as_ref(), &self.fan, &|idx| self.oid_at_index(idx)) + } + + /// Given the `index` ranging from 0 to [File::num_objects()], return the pack index and its absolute offset into the pack. + /// + /// The pack-index refers to an entry in the [`index_names`][File::index_names()] list, from which the pack can be derived. + pub fn pack_id_and_pack_offset_at_index(&self, index: EntryIndex) -> (PackIndex, data::Offset) { + const OFFSET_ENTRY_SIZE: usize = 4 + 4; + let index = index as usize; + let start = self.offsets_ofs + index * OFFSET_ENTRY_SIZE; + + const HIGH_BIT: u32 = 1 << 31; + + let pack_index = crate::read_u32(&self.data[start..][..4]); + let offset = &self.data[start + 4..][..4]; + let ofs32 = crate::read_u32(offset); + let pack_offset = if (ofs32 & HIGH_BIT) == HIGH_BIT { + // We determine if large offsets are actually larger than 4GB and if not, we don't use the high-bit to signal anything + // but allow the presence of the large-offset chunk to signal what's happening. + if let Some(offsets_64) = self.large_offsets_ofs { + let from = offsets_64 + (ofs32 ^ HIGH_BIT) as usize * 8; + crate::read_u64(&self.data[from..][..8]) + } else { + u64::from(ofs32) + } + } else { + u64::from(ofs32) + }; + (pack_index, pack_offset) + } + + /// Return an iterator over all entries within this file. + pub fn iter(&self) -> impl Iterator + '_ { + (0..self.num_objects).map(move |idx| { + let (pack_index, pack_offset) = self.pack_id_and_pack_offset_at_index(idx); + Entry { + oid: self.oid_at_index(idx).to_owned(), + pack_offset, + pack_index, + } + }) + } +} diff --git a/knot2/third_party/gix-pack/src/multi_index/chunk.rs b/knot2/third_party/gix-pack/src/multi_index/chunk.rs new file mode 100644 index 0000000000000000000000000000000000000000..a3c688ae530a3c3362d7e850a51526ec980bb093 --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/chunk.rs @@ -0,0 +1,318 @@ +/// Information for the chunk about index names +pub mod index_names { + use std::path::{Path, PathBuf}; + + use gix_object::bstr::{BString, ByteSlice}; + + /// The ID used for the index-names chunk. + pub const ID: gix_chunk::Id = *b"PNAM"; + + /// + pub mod decode { + use std::collections::TryReserveError; + + use gix_object::bstr::BString; + + /// The error returned by [`from_bytes()`][super::from_bytes()]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error("The pack names were not ordered alphabetically.")] + NotOrderedAlphabetically, + #[error("Each pack path name must be terminated with a null byte")] + MissingNullByte, + #[error("Entry too large to fit in memory")] + OutOfMemory, + #[error("Couldn't turn path '{path}' into OS path due to encoding issues")] + PathEncoding { path: BString }, + #[error("non-padding bytes found after all paths were read.")] + UnknownTrailerBytes, + } + + impl From for Error { + #[cold] + fn from(_: TryReserveError) -> Self { + Self::OutOfMemory + } + } + } + + /// Parse null-separated index names from the given `chunk` of bytes. + /// + /// `chunk` + ///: Contains `num_packs` null-terminated index names, optionally followed by padding bytes which are typically `\0`. + /// + /// `num_packs` + ///: The number of index names expected to be present in `chunk`. + /// + /// `alloc_limit_bytes` + ///: Limits allocations caused by attacker-controlled on-disk multi-index data. + /// It is used to reject reserving the output `Vec` if its capacity estimate exceeds the limit, + /// and to reject any single path entry whose byte length exceeds the limit before turning it into a `PathBuf`. + /// Use `None` to disable this limit. + pub fn from_bytes( + mut chunk: &[u8], + num_packs: u32, + alloc_limit_bytes: Option, + ) -> Result, decode::Error> { + let mut out = Vec::new(); + let num_packs = usize::try_from(num_packs).map_err(|_| decode::Error::OutOfMemory)?; + let vec_allocation = num_packs + .checked_mul(std::mem::size_of::()) + .ok_or(decode::Error::OutOfMemory)?; + if alloc_limit_bytes.is_some_and(|limit| vec_allocation > limit) { + return Err(decode::Error::OutOfMemory); + } + out.try_reserve(num_packs)?; + + for _ in 0..num_packs { + let null_byte_pos = chunk + .find_byte(b'\0') + .ok_or(decode::Error::MissingNullByte)?; + + let path = &chunk[..null_byte_pos]; + if alloc_limit_bytes.is_some_and(|limit| path.len() > limit) { + return Err(decode::Error::OutOfMemory); + } + let path = gix_path::try_from_byte_slice(path) + .map_err(|_| decode::Error::PathEncoding { + path: BString::from(path), + })? + .to_owned(); + + if let Some(previous) = out.last() { + if previous >= &path { + return Err(decode::Error::NotOrderedAlphabetically); + } + } + out.push(path); + + chunk = &chunk[null_byte_pos + 1..]; + } + + if !chunk.is_empty() && !chunk.iter().all(|b| *b == 0) { + return Err(decode::Error::UnknownTrailerBytes); + } + // NOTE: git writes garbage into this chunk, usually extra \0 bytes, which we simply ignore. If we were strict + // about it we couldn't read this chunk data at all. + Ok(out) + } + + /// Calculate the size on disk for our chunk with the given index paths. Note that these are expected to have been processed already + /// to actually be file names. + pub fn storage_size(paths: impl IntoIterator>) -> u64 { + let mut count = 0u64; + for path in paths { + let path = path.as_ref(); + let ascii_path = path.to_str().expect("UTF-8 compatible paths"); + assert!( + ascii_path.is_ascii(), + "must use ascii bytes for correct size computation" + ); + count += (ascii_path.len() + 1/* null byte */) as u64; + } + + let needed_alignment = CHUNK_ALIGNMENT - (count % CHUNK_ALIGNMENT); + if needed_alignment < CHUNK_ALIGNMENT { + count += needed_alignment; + } + count + } + + /// Write all `paths` in order to `out`, including padding. + pub fn write( + paths: impl IntoIterator>, + out: &mut dyn std::io::Write, + ) -> std::io::Result<()> { + let mut written_bytes = 0; + for path in paths { + let path = path.as_ref().to_str().expect("UTF-8 path"); + out.write_all(path.as_bytes())?; + out.write_all(&[0])?; + written_bytes += path.len() as u64 + 1; + } + + let needed_alignment = CHUNK_ALIGNMENT - (written_bytes % CHUNK_ALIGNMENT); + if needed_alignment < CHUNK_ALIGNMENT { + let padding = [0u8; CHUNK_ALIGNMENT as usize]; + out.write_all(&padding[..needed_alignment as usize])?; + } + Ok(()) + } + + const CHUNK_ALIGNMENT: u64 = 4; +} + +/// Information for the chunk with the fanout table +pub mod fanout { + use crate::multi_index; + + /// The size of the fanout table + pub const SIZE: usize = 4 * 256; + + /// The id uniquely identifying the fanout table. + pub const ID: gix_chunk::Id = *b"OIDF"; + + /// Decode the fanout table contained in `chunk`, or return `None` if it didn't have the expected size. + pub fn from_bytes(chunk: &[u8]) -> Option<[u32; 256]> { + if chunk.len() != SIZE { + return None; + } + let mut out = [0; 256]; + for (c, f) in chunk.chunks_exact(4).zip(out.iter_mut()) { + *f = u32::from_be_bytes(c.try_into().unwrap()); + } + out.into() + } + + /// Write the fanout for the given entries, which must be sorted by oid + pub(crate) fn write( + sorted_entries: &[multi_index::write::Entry], + out: &mut dyn std::io::Write, + ) -> std::io::Result<()> { + let fanout = + crate::index::encode::fanout(&mut sorted_entries.iter().map(|e| e.id.first_byte())); + + for value in fanout.iter() { + out.write_all(&value.to_be_bytes())?; + } + Ok(()) + } +} + +/// Information about the oid lookup table. +pub mod lookup { + use std::ops::Range; + + use crate::multi_index; + + /// The id uniquely identifying the oid lookup table. + pub const ID: gix_chunk::Id = *b"OIDL"; + + /// Return the number of bytes needed to store the data on disk for the given amount of `entries` + pub fn storage_size(entries: usize, object_hash: gix_hash::Kind) -> u64 { + (entries * object_hash.len_in_bytes()) as u64 + } + + pub(crate) fn write( + sorted_entries: &[multi_index::write::Entry], + out: &mut dyn std::io::Write, + ) -> std::io::Result<()> { + for entry in sorted_entries { + out.write_all(entry.id.as_slice())?; + } + Ok(()) + } + + /// Return true if the size of the `offset` range seems to match for a `hash` of the given kind and the amount of objects. + pub fn is_valid(offset: &Range, hash: gix_hash::Kind, num_objects: u32) -> bool { + (offset.end - offset.start) == (num_objects as usize).saturating_mul(hash.len_in_bytes()) + } +} + +/// Information about the offsets table. +pub mod offsets { + use std::ops::Range; + + use crate::multi_index; + + /// The id uniquely identifying the offsets table. + pub const ID: gix_chunk::Id = *b"OOFF"; + + /// Return the amount of bytes needed to offset data for `entries`. + pub fn storage_size(entries: usize) -> u64 { + (entries * (4 /*pack-id*/ + 4/* pack offset */)) as u64 + } + + pub(crate) fn write( + sorted_entries: &[multi_index::write::Entry], + large_offsets_needed: bool, + out: &mut dyn std::io::Write, + ) -> std::io::Result<()> { + use crate::index::encode::{HIGH_BIT, LARGE_OFFSET_THRESHOLD}; + let mut num_large_offsets = 0u32; + + for entry in sorted_entries { + out.write_all(&entry.pack_index.to_be_bytes())?; + + let offset: u32 = if large_offsets_needed { + if entry.pack_offset > LARGE_OFFSET_THRESHOLD { + let res = num_large_offsets | HIGH_BIT; + num_large_offsets += 1; + res + } else { + entry.pack_offset as u32 + } + } else { + entry + .pack_offset + .try_into() + .expect("without large offsets, pack-offset fits u32") + }; + out.write_all(&offset.to_be_bytes())?; + } + Ok(()) + } + + /// Returns true if the `offset` range seems to match the size required for the untrusted `num_objects`. + pub fn is_valid(offset: &Range, num_objects: u32) -> bool { + let entry_size = 4 /* pack-id */ + 4 /* pack-offset */; + (offset.end - offset.start) == (num_objects as usize).saturating_mul(entry_size) + } +} + +/// Information about the large offsets table. +pub mod large_offsets { + use std::ops::Range; + + use crate::{index::encode::LARGE_OFFSET_THRESHOLD, multi_index}; + + /// The id uniquely identifying the large offsets table (with 64 bit offsets) + pub const ID: gix_chunk::Id = *b"LOFF"; + + /// Returns Some(num-large-offset) if there are offsets larger than u32. + pub(crate) fn num_large_offsets(entries: &[multi_index::write::Entry]) -> Option { + let mut num_large_offsets = 0; + let mut needs_large_offsets = false; + for entry in entries { + if entry.pack_offset > LARGE_OFFSET_THRESHOLD { + num_large_offsets += 1; + } + if entry.pack_offset > crate::data::Offset::from(u32::MAX) { + needs_large_offsets = true; + } + } + + needs_large_offsets.then_some(num_large_offsets) + } + /// Returns true if the `offsets` range seems to be properly aligned for the data we expect. + pub fn is_valid(offset: &Range) -> bool { + (offset.end - offset.start) % 8 == 0 + } + + pub(crate) fn write( + sorted_entries: &[multi_index::write::Entry], + mut num_large_offsets: usize, + out: &mut dyn std::io::Write, + ) -> std::io::Result<()> { + for offset in sorted_entries + .iter() + .filter_map(|e| (e.pack_offset > LARGE_OFFSET_THRESHOLD).then_some(e.pack_offset)) + { + out.write_all(&offset.to_be_bytes())?; + num_large_offsets = num_large_offsets + .checked_sub(1) + .expect("BUG: wrote more offsets the previously found"); + } + assert_eq!( + num_large_offsets, 0, + "BUG: wrote less offsets than initially counted" + ); + Ok(()) + } + + /// Return the number of bytes needed to store the given amount of `large_offsets` + pub(crate) fn storage_size(large_offsets: usize) -> u64 { + 8 * large_offsets as u64 + } +} diff --git a/knot2/third_party/gix-pack/src/multi_index/init.rs b/knot2/third_party/gix-pack/src/multi_index/init.rs new file mode 100644 index 0000000000000000000000000000000000000000..c0845e98b96148e8ba84207cc657750929e8440a --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/init.rs @@ -0,0 +1,189 @@ +use std::path::{Path, PathBuf}; + +use crate::multi_index::{File, Version, chunk}; + +mod error { + use crate::multi_index::chunk; + + /// The error returned by [File::at()][super::File::at()]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error("Could not open multi-index file at '{path}'")] + Io { + source: std::io::Error, + path: std::path::PathBuf, + }, + #[error("{message}")] + Corrupt { message: &'static str }, + #[error("Unsupported multi-index version: {version})")] + UnsupportedVersion { version: u8 }, + #[error("Unsupported hash kind: {kind})")] + UnsupportedObjectHash { kind: u8 }, + #[error(transparent)] + ChunkFileQuery(#[from] gix_error::Message), + #[error(transparent)] + ChunkFileDecode(#[from] gix_error::ValidationError), + #[error("The multi-pack fan doesn't have the correct size of 256 * 4 bytes")] + MultiPackFanSize, + #[error(transparent)] + PackNames(#[from] chunk::index_names::decode::Error), + #[error("multi-index chunk {:?} has invalid size: {message}", String::from_utf8_lossy(.id))] + InvalidChunkSize { + id: gix_chunk::Id, + message: &'static str, + }, + } +} + +pub use error::Error; + +/// Initialization +impl File { + /// Open the multi-index file at the given `path`. + /// + /// `alloc_limit_bytes` bounds each allocation caused by user-controlled on-disk data, useful for untrusted input. + /// Use `None` to disable the limit. + pub fn at(path: impl AsRef, alloc_limit_bytes: Option) -> Result { + Self::at_inner(path.as_ref(), alloc_limit_bytes) + } + + fn at_inner(path: &Path, alloc_limit_bytes: Option) -> Result { + let data = crate::mmap::read_only(path).map_err(|source| Error::Io { + source, + path: path.to_owned(), + })?; + Self::from_data(data, path.to_owned(), alloc_limit_bytes) + } +} + +impl File +where + T: crate::FileData, +{ + /// Instantiate a multi-index file from `data` as assumed to be read or memory-mapped from `path`. + /// + /// `alloc_limit_bytes` bounds each allocation caused by untrusted on-disk multi-index data. + /// Use `None` to disable the limit. + /// + /// It is used to reject reserving the output `Vec` if its capacity estimate exceeds the limit, + /// and to reject any single path entry whose byte length exceeds the limit before turning it into a `PathBuf`. + pub fn from_data( + data: T, + path: PathBuf, + alloc_limit_bytes: Option, + ) -> Result { + const TRAILER_LEN: usize = gix_hash::Kind::shortest().len_in_bytes(); /* trailing hash */ + if data.len() + < Self::HEADER_LEN + + gix_chunk::file::Index::size_for_entries( + 4, /*index names, fan, offsets, oids*/ + ) + + chunk::fanout::SIZE + + TRAILER_LEN + { + return Err(Error::Corrupt { + message: "multi-index file is truncated and too short", + }); + } + + let (version, object_hash, num_chunks, num_indices) = { + let (signature, data) = data.split_at(4); + if signature != Self::SIGNATURE { + return Err(Error::Corrupt { + message: "Invalid signature", + }); + } + let (version, data) = data.split_at(1); + let version = match version[0] { + 1 => Version::V1, + version => return Err(Error::UnsupportedVersion { version }), + }; + + let (object_hash, data) = data.split_at(1); + let object_hash = gix_hash::Kind::try_from(object_hash[0]) + .map_err(|unknown| Error::UnsupportedObjectHash { kind: unknown })?; + let (num_chunks, data) = data.split_at(1); + let num_chunks = num_chunks[0]; + + let (_num_base_files, data) = data.split_at(1); // TODO: handle base files once it's clear what this does + + let (num_indices, _) = data.split_at(4); + let num_indices = crate::read_u32(num_indices); + + (version, object_hash, num_chunks, num_indices) + }; + + let chunks = + gix_chunk::file::Index::from_bytes(&data, Self::HEADER_LEN, u32::from(num_chunks))?; + + let index_names = chunks.data_by_id(&data, chunk::index_names::ID)?; + let index_names = + chunk::index_names::from_bytes(index_names, num_indices, alloc_limit_bytes)?; + + let fan = chunks.data_by_id(&data, chunk::fanout::ID)?; + let fan = chunk::fanout::from_bytes(fan).ok_or(Error::MultiPackFanSize)?; + let num_objects = fan[255]; + validate_fan(&fan)?; + + let lookup = chunks.validated_usize_offset_by_id(chunk::lookup::ID, |offset| { + chunk::lookup::is_valid(&offset, object_hash, num_objects) + .then_some(offset) + .ok_or(Error::InvalidChunkSize { + id: chunk::lookup::ID, + message: "The chunk with alphabetically ordered object ids doesn't have the correct size", + }) + })??; + let offsets = chunks.validated_usize_offset_by_id(chunk::offsets::ID, |offset| { + chunk::offsets::is_valid(&offset, num_objects) + .then_some(offset) + .ok_or(Error::InvalidChunkSize { + id: chunk::offsets::ID, + message: "The chunk with offsets into the pack doesn't have the correct size", + }) + })??; + let large_offsets = chunks + .validated_usize_offset_by_id(chunk::large_offsets::ID, |offset| { + chunk::large_offsets::is_valid(&offset) + .then_some(offset) + .ok_or(Error::InvalidChunkSize { + id: chunk::large_offsets::ID, + message: "The chunk with large offsets into the pack doesn't have the correct size", + }) + }) + .ok() + .transpose()?; + + let checksum_offset = chunks.highest_offset() as usize; + let trailer = &data[checksum_offset..]; + if trailer.len() != object_hash.len_in_bytes() { + return Err(Error::Corrupt { + message: "Trailing checksum didn't have the expected size or there were unknown bytes after the checksum.", + }); + } + + Ok(File { + data, + path, + version, + hash_len: object_hash.len_in_bytes(), + object_hash, + fan, + index_names, + lookup_ofs: lookup.start, + offsets_ofs: offsets.start, + large_offsets_ofs: large_offsets.map(|r| r.start), + num_objects, + num_indices, + }) + } +} + +fn validate_fan(fan: &[u32; 256]) -> Result<(), Error> { + if !crate::fan_is_monotonically_increasing(fan) { + return Err(Error::Corrupt { + message: "multi-index fan-out table must be monotonically increasing", + }); + } + Ok(()) +} diff --git a/knot2/third_party/gix-pack/src/multi_index/mod.rs b/knot2/third_party/gix-pack/src/multi_index/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..5c978ccc3819bcfd71ff3cb8b43a33335d83a268 --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/mod.rs @@ -0,0 +1,53 @@ +use std::path::PathBuf; + +use crate::MMap; + +/// Known multi-index file versions +#[derive(Default, PartialEq, Eq, Ord, PartialOrd, Debug, Hash, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[allow(missing_docs)] +pub enum Version { + #[default] + V1 = 1, +} + +/// An index into our [`File::index_names()`] array yielding the name of the index and by implication, its pack file. +pub type PackIndex = u32; + +/// The type for referring to indices of an entry within the index file. +pub type EntryIndex = u32; + +/// A representation of an index file for multiple packs at the same time, typically stored in a file +/// named 'multi-pack-index'. +pub struct File { + data: T, + path: std::path::PathBuf, + version: Version, + hash_len: usize, + object_hash: gix_hash::Kind, + /// The amount of pack files contained within + num_indices: u32, + num_objects: u32, + + fan: [u32; 256], + index_names: Vec, + lookup_ofs: usize, + offsets_ofs: usize, + large_offsets_ofs: Option, +} + +/// +pub mod write; +pub use write::function::write_from_index_paths; + +/// +mod access; + +/// +pub mod verify; + +/// +pub mod chunk; + +/// +pub mod init; diff --git a/knot2/third_party/gix-pack/src/multi_index/verify.rs b/knot2/third_party/gix-pack/src/multi_index/verify.rs new file mode 100644 index 0000000000000000000000000000000000000000..3194b6951d2ff87b65768cd30145a235535e4b2b --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/verify.rs @@ -0,0 +1,343 @@ +use std::{cmp::Ordering, sync::atomic::AtomicBool, time::Instant}; + +use gix_features::progress::{Count, DynNestedProgress, Progress}; + +use crate::{exact_vec, index, multi_index::File}; + +/// +pub mod integrity { + use crate::multi_index::EntryIndex; + + /// Returned by [`multi_index::File::verify_integrity()`][crate::multi_index::File::verify_integrity()]. + #[derive(thiserror::Error, Debug)] + #[allow(missing_docs)] + pub enum Error { + #[error( + "Object {id} should be at pack-offset {expected_pack_offset} but was found at {actual_pack_offset}" + )] + PackOffsetMismatch { + id: gix_hash::ObjectId, + expected_pack_offset: u64, + actual_pack_offset: u64, + }, + #[error(transparent)] + MultiIndexChecksum(#[from] crate::multi_index::verify::checksum::Error), + #[error(transparent)] + IndexIntegrity(#[from] crate::index::verify::integrity::Error), + #[error(transparent)] + BundleInit(#[from] crate::bundle::init::Error), + #[error("Counted {actual} objects, but expected {expected} as per multi-index")] + UnexpectedObjectCount { actual: usize, expected: usize }, + #[error("{id} wasn't found in the index referenced in the multi-pack index")] + OidNotFound { id: gix_hash::ObjectId }, + #[error("The object id at multi-index entry {index} wasn't in order")] + OutOfOrder { index: EntryIndex }, + #[error( + "The fan at index {index} is out of order as it's larger then the following value." + )] + Fan { index: usize }, + #[error("The multi-index claims to have no objects")] + Empty, + #[error("The multi-index path '{path}' has no parent directory")] + InvalidPath { path: std::path::PathBuf }, + #[error("Interrupted")] + Interrupted, + } + + /// Returned by [`multi_index::File::verify_integrity()`][crate::multi_index::File::verify_integrity()]. + pub struct Outcome { + /// The computed checksum of the multi-index which matched the stored one. + pub actual_index_checksum: gix_hash::ObjectId, + /// The for each entry in [`index_names()`][super::File::index_names()] provide the corresponding pack traversal outcome. + pub pack_traverse_statistics: Vec, + } + + /// The progress ids used in [`multi_index::File::verify_integrity()`][crate::multi_index::File::verify_integrity()]. + /// + /// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. + #[derive(Debug, Copy, Clone)] + pub enum ProgressId { + /// The amount of bytes read to verify the multi-index checksum. + ChecksumBytes, + /// The amount of objects whose offset has been checked. + ObjectOffsets, + } + + impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::ChecksumBytes => *b"MVCK", + ProgressId::ObjectOffsets => *b"MVOF", + } + } + } +} + +/// +pub mod checksum { + /// Returned by [`multi_index::File::verify_checksum()`][crate::multi_index::File::verify_checksum()]. + pub type Error = crate::verify::checksum::Error; +} + +impl File +where + T: crate::FileData, +{ + /// Validate that our [`checksum()`][File::checksum()] matches the actual contents + /// of this index file, and return it if it does. + pub fn verify_checksum( + &self, + progress: &mut dyn Progress, + should_interrupt: &AtomicBool, + ) -> Result { + crate::verify::checksum_on_disk_or_mmap( + self.path(), + &self.data, + self.checksum(), + self.object_hash, + progress, + should_interrupt, + ) + } + + /// Similar to [`verify_integrity()`][File::verify_integrity()] but without any deep inspection of objects. + /// + /// Instead we only validate the contents of the multi-index itself. + pub fn verify_integrity_fast( + &self, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + ) -> Result { + self.verify_integrity_inner( + progress, + should_interrupt, + false, + index::verify::integrity::Options::default(), + ) + .map_err(|err| match err { + index::traverse::Error::Processor(err) => err, + _ => unreachable!("BUG: no other error type is possible"), + }) + .map(|o| o.actual_index_checksum) + } + + /// Similar to [`crate::Bundle::verify_integrity()`] but checks all contained indices and their packs. + /// + /// Note that it's considered a failure if an index doesn't have a corresponding pack. + pub fn verify_integrity( + &self, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + options: index::verify::integrity::Options, + ) -> Result> + where + C: crate::cache::DecodeEntry, + F: Fn() -> C + Send + Clone, + { + self.verify_integrity_inner(progress, should_interrupt, true, options) + } + + fn verify_integrity_inner( + &self, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + deep_check: bool, + options: index::verify::integrity::Options, + ) -> Result> + where + C: crate::cache::DecodeEntry, + F: Fn() -> C + Send + Clone, + { + let parent = self.path.parent().ok_or_else(|| { + index::traverse::Error::Processor(integrity::Error::InvalidPath { + path: self.path.clone(), + }) + })?; + + let actual_index_checksum = self + .verify_checksum( + &mut progress.add_child_with_id( + format!("{}: checksum", self.path.display()), + integrity::ProgressId::ChecksumBytes.into(), + ), + should_interrupt, + ) + .map_err(integrity::Error::from) + .map_err(index::traverse::Error::Processor)?; + + if let Some(first_invalid) = crate::verify::fan(&self.fan) { + return Err(index::traverse::Error::Processor(integrity::Error::Fan { + index: first_invalid, + })); + } + + if self.num_objects == 0 { + return Err(index::traverse::Error::Processor(integrity::Error::Empty)); + } + + let mut pack_traverse_statistics = Vec::new(); + + let operation_start = Instant::now(); + let mut total_objects_checked = 0; + let mut pack_ids_and_offsets = exact_vec(self.num_objects as usize); + { + let order_start = Instant::now(); + let mut progress = progress + .add_child_with_id("checking oid order".into(), gix_features::progress::UNKNOWN); + progress.init( + Some(self.num_objects as usize), + gix_features::progress::count("objects"), + ); + + for entry_index in 0..(self.num_objects - 1) { + let lhs = self.oid_at_index(entry_index); + let rhs = self.oid_at_index(entry_index + 1); + + if rhs.cmp(lhs) != Ordering::Greater { + return Err(index::traverse::Error::Processor( + integrity::Error::OutOfOrder { index: entry_index }, + )); + } + let (pack_id, _) = self.pack_id_and_pack_offset_at_index(entry_index); + pack_ids_and_offsets.push((pack_id, entry_index)); + progress.inc(); + } + { + let entry_index = self.num_objects - 1; + let (pack_id, _) = self.pack_id_and_pack_offset_at_index(entry_index); + pack_ids_and_offsets.push((pack_id, entry_index)); + } + // sort by pack-id to allow handling all indices matching a pack while its open. + pack_ids_and_offsets.sort_by_key(|l| l.0); + progress.show_throughput(order_start); + }; + + progress.init( + Some(self.num_indices as usize), + gix_features::progress::count("indices"), + ); + + let mut pack_ids_slice = pack_ids_and_offsets.as_slice(); + + for (pack_id, index_file_name) in self.index_names.iter().enumerate() { + progress.set_name(index_file_name.display().to_string()); + progress.inc(); + + let mut bundle = None; + let index; + let index_path = parent.join(index_file_name); + let index = if deep_check { + bundle = crate::Bundle::at(index_path, self.object_hash) + .map_err(integrity::Error::from) + .map_err(index::traverse::Error::Processor)? + .into(); + bundle.as_ref().map(|b| &b.index).expect("just set") + } else { + index = Some( + index::File::at(index_path, self.object_hash) + .map_err(|err| { + integrity::Error::BundleInit(crate::bundle::init::Error::Index(err)) + }) + .map_err(index::traverse::Error::Processor)?, + ); + index.as_ref().expect("just set") + }; + + let slice_end = pack_ids_slice.partition_point(|e| e.0 == pack_id as crate::data::Id); + let multi_index_entries_to_check = &pack_ids_slice[..slice_end]; + { + let offset_start = Instant::now(); + let mut offsets_progress = progress.add_child_with_id( + "verify object offsets".into(), + integrity::ProgressId::ObjectOffsets.into(), + ); + offsets_progress.init( + Some(pack_ids_and_offsets.len()), + gix_features::progress::count("objects"), + ); + pack_ids_slice = &pack_ids_slice[slice_end..]; + + for entry_id in multi_index_entries_to_check.iter().map(|e| e.1) { + let oid = self.oid_at_index(entry_id); + let (_, expected_pack_offset) = self.pack_id_and_pack_offset_at_index(entry_id); + let entry_in_bundle_index = index.lookup(oid).ok_or_else(|| { + index::traverse::Error::Processor(integrity::Error::OidNotFound { + id: oid.to_owned(), + }) + })?; + let actual_pack_offset = index.pack_offset_at_index(entry_in_bundle_index); + if actual_pack_offset != expected_pack_offset { + return Err(index::traverse::Error::Processor( + integrity::Error::PackOffsetMismatch { + id: oid.to_owned(), + expected_pack_offset, + actual_pack_offset, + }, + )); + } + offsets_progress.inc(); + } + + if should_interrupt.load(std::sync::atomic::Ordering::Relaxed) { + return Err(index::traverse::Error::Processor( + integrity::Error::Interrupted, + )); + } + offsets_progress.show_throughput(offset_start); + } + + total_objects_checked += multi_index_entries_to_check.len(); + + if let Some(bundle) = bundle { + progress.set_name(format!("Validating {}", index_file_name.display())); + let crate::bundle::verify::integrity::Outcome { + actual_index_checksum: _, + pack_traverse_outcome, + } = bundle + .verify_integrity(progress, should_interrupt, options.clone()) + .map_err(|err| { + use index::traverse::Error::*; + match err { + Processor(err) => Processor(integrity::Error::IndexIntegrity(err)), + IndexVerify(err) => IndexVerify(err), + Tree(err) => Tree(err), + TreeTraversal(err) => TreeTraversal(err), + PackVerify(err) => PackVerify(err), + PackDecode { id, offset, source } => PackDecode { id, offset, source }, + PackMismatch(err) => PackMismatch(err), + EntryType(err) => EntryType(err), + PackObjectVerify { offset, source } => { + PackObjectVerify { offset, source } + } + Crc32Mismatch { + expected, + actual, + offset, + kind, + } => Crc32Mismatch { + expected, + actual, + offset, + kind, + }, + Interrupted => Interrupted, + } + })?; + pack_traverse_statistics.push(pack_traverse_outcome); + } + } + + assert_eq!( + self.num_objects as usize, total_objects_checked, + "BUG: our slicing should allow to visit all objects" + ); + + progress.set_name("Validating multi-pack".into()); + progress.show_throughput(operation_start); + + Ok(integrity::Outcome { + actual_index_checksum, + pack_traverse_statistics, + }) + } +} diff --git a/knot2/third_party/gix-pack/src/multi_index/write.rs b/knot2/third_party/gix-pack/src/multi_index/write.rs new file mode 100644 index 0000000000000000000000000000000000000000..489d08971cdf45cd6376cc504e0530ba4c1d2378 --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/write.rs @@ -0,0 +1,289 @@ +use std::time::SystemTime; + +use crate::multi_index; + +mod error { + /// The error returned by [`crate::multi_index::write_from_index_paths()`]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error(transparent)] + Io(#[from] gix_hash::io::Error), + #[error("Interrupted")] + Interrupted, + #[error(transparent)] + OpenIndex(#[from] crate::index::init::Error), + } +} +pub use error::Error; + +/// An entry suitable for sorting and writing +pub(crate) struct Entry { + pub(crate) id: gix_hash::ObjectId, + pub(crate) pack_index: u32, + pub(crate) pack_offset: crate::data::Offset, + /// Used for sorting in case of duplicates + index_mtime: SystemTime, +} + +/// Options for use in [`multi_index::write_from_index_paths()`]. +pub struct Options { + /// The kind of hash to use for objects and to expect in the input files. + pub object_hash: gix_hash::Kind, +} + +/// The result of [`multi_index::write_from_index_paths()`]. +pub struct Outcome { + /// The calculated multi-index checksum of the file at `multi_index_path`. + pub multi_index_checksum: gix_hash::ObjectId, +} + +/// The progress ids used in [`crate::multi_index::write_from_index_paths()`]. +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// Counts each path in the input set whose entries we enumerate and write into the multi-index + FromPathsCollectingEntries, + /// The amount of bytes written as part of the multi-index. + BytesWritten, +} + +impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::FromPathsCollectingEntries => *b"MPCE", + ProgressId::BytesWritten => *b"MPBW", + } + } +} + +impl multi_index::File { + pub(crate) const SIGNATURE: &'static [u8] = b"MIDX"; + pub(crate) const HEADER_LEN: usize = 4 /*signature*/ + + 1 /*version*/ + + 1 /*object id version*/ + + 1 /*num chunks */ + + 1 /*num base files */ + + 4 /*num pack files*/; +} + +pub(super) mod function { + use std::{ + path::PathBuf, + sync::atomic::{AtomicBool, Ordering}, + time::{Instant, SystemTime}, + }; + + use gix_features::progress::{Count, DynNestedProgress, Progress}; + + use crate::{MMap, multi_index}; + + use super::{Entry, Error, Options, Outcome, ProgressId}; + + /// Create a new multi-index file for writing to `out` from the pack index files at `index_paths`. + /// + /// Progress is sent to `progress` and interruptions checked via `should_interrupt`. + pub fn write_from_index_paths( + mut index_paths: Vec, + out: &mut dyn std::io::Write, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + Options { object_hash }: Options, + ) -> Result { + let out = gix_hash::io::Write::new(out, object_hash); + let (index_paths_sorted, index_filenames_sorted) = { + index_paths.sort(); + let file_names = index_paths + .iter() + .map(|p| PathBuf::from(p.file_name().expect("file name present"))) + .collect::>(); + (index_paths, file_names) + }; + + let entries = { + let mut entries = Vec::new(); + let start = Instant::now(); + let mut progress = progress.add_child_with_id( + "Collecting entries".into(), + ProgressId::FromPathsCollectingEntries.into(), + ); + progress.init( + Some(index_paths_sorted.len()), + gix_features::progress::count("indices"), + ); + + // This could be parallelized… but it's probably not worth it unless you have 500mio objects. + for (index_id, index) in index_paths_sorted.iter().enumerate() { + let mtime = index + .metadata() + .and_then(|m| m.modified()) + .unwrap_or(SystemTime::UNIX_EPOCH); + let index = crate::index::File::at(index, object_hash)?; + + entries.reserve(index.num_objects() as usize); + entries.extend(index.iter().map(|e| Entry { + id: e.oid, + pack_index: index_id as u32, + pack_offset: e.pack_offset, + index_mtime: mtime, + })); + progress.inc(); + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + } + progress.show_throughput(start); + + let start = Instant::now(); + progress.set_name("Deduplicate".into()); + progress.init( + Some(entries.len()), + gix_features::progress::count("entries"), + ); + entries.sort_by(|l, r| { + l.id.cmp(&r.id) + .then_with(|| l.index_mtime.cmp(&r.index_mtime).reverse()) + .then_with(|| l.pack_index.cmp(&r.pack_index)) + }); + entries.dedup_by_key(|e| e.id); + progress.inc_by(entries.len()); + progress.show_throughput(start); + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + entries + }; + + let mut cf = gix_chunk::file::Index::for_writing(); + cf.plan_chunk( + multi_index::chunk::index_names::ID, + multi_index::chunk::index_names::storage_size(&index_filenames_sorted), + ); + cf.plan_chunk( + multi_index::chunk::fanout::ID, + multi_index::chunk::fanout::SIZE as u64, + ); + cf.plan_chunk( + multi_index::chunk::lookup::ID, + multi_index::chunk::lookup::storage_size(entries.len(), object_hash), + ); + cf.plan_chunk( + multi_index::chunk::offsets::ID, + multi_index::chunk::offsets::storage_size(entries.len()), + ); + + let num_large_offsets = multi_index::chunk::large_offsets::num_large_offsets(&entries); + if let Some(num_large_offsets) = num_large_offsets { + cf.plan_chunk( + multi_index::chunk::large_offsets::ID, + multi_index::chunk::large_offsets::storage_size(num_large_offsets), + ); + } + + let mut write_progress = progress.add_child_with_id( + "Writing multi-index".into(), + ProgressId::BytesWritten.into(), + ); + let write_start = Instant::now(); + write_progress.init( + Some(cf.planned_storage_size() as usize + multi_index::File::::HEADER_LEN), + gix_features::progress::bytes(), + ); + let mut out = gix_features::progress::Write { + inner: out, + progress: write_progress, + }; + + let bytes_written = multi_index::File::::write_header( + &mut out, + cf.num_chunks() + .try_into() + .expect("BUG: wrote more than 256 chunks"), + index_paths_sorted.len() as u32, + object_hash, + ) + .map_err(gix_hash::io::Error::from)?; + + { + progress.set_name("Writing chunks".into()); + progress.init( + Some(cf.num_chunks()), + gix_features::progress::count("chunks"), + ); + + let mut chunk_write = cf + .into_write(&mut out, bytes_written) + .map_err(gix_hash::io::Error::from)?; + while let Some(chunk_to_write) = chunk_write.next_chunk() { + match chunk_to_write { + multi_index::chunk::index_names::ID => multi_index::chunk::index_names::write( + &index_filenames_sorted, + &mut chunk_write, + ), + multi_index::chunk::fanout::ID => { + multi_index::chunk::fanout::write(&entries, &mut chunk_write) + } + multi_index::chunk::lookup::ID => { + multi_index::chunk::lookup::write(&entries, &mut chunk_write) + } + multi_index::chunk::offsets::ID => multi_index::chunk::offsets::write( + &entries, + num_large_offsets.is_some(), + &mut chunk_write, + ), + multi_index::chunk::large_offsets::ID => { + multi_index::chunk::large_offsets::write( + &entries, + num_large_offsets.expect("available if planned"), + &mut chunk_write, + ) + } + unknown => unreachable!( + "BUG: forgot to implement chunk {:?}", + std::str::from_utf8(&unknown) + ), + } + .map_err(gix_hash::io::Error::from)?; + progress.inc(); + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + } + } + + // write trailing checksum + let multi_index_checksum = out + .inner + .hash + .try_finalize() + .map_err(gix_hash::io::Error::from)?; + out.inner + .inner + .write_all(multi_index_checksum.as_slice()) + .map_err(gix_hash::io::Error::from)?; + out.progress.show_throughput(write_start); + + Ok(Outcome { + multi_index_checksum, + }) + } +} + +impl multi_index::File { + fn write_header( + out: &mut dyn std::io::Write, + num_chunks: u8, + num_indices: u32, + object_hash: gix_hash::Kind, + ) -> std::io::Result { + out.write_all(Self::SIGNATURE)?; + out.write_all(&[crate::multi_index::Version::V1 as u8])?; + out.write_all(&[object_hash as u8])?; + out.write_all(&[num_chunks])?; + out.write_all(&[0])?; /* unused number of base files */ + out.write_all(&num_indices.to_be_bytes())?; + + Ok(Self::HEADER_LEN) + } +} diff --git a/knot2/third_party/gix-pack/src/verify.rs b/knot2/third_party/gix-pack/src/verify.rs new file mode 100644 index 0000000000000000000000000000000000000000..e17b73652b19822fa7dd2689e7e24fc45587573f --- /dev/null +++ b/knot2/third_party/gix-pack/src/verify.rs @@ -0,0 +1,65 @@ +use std::{path::Path, sync::atomic::AtomicBool}; + +use gix_features::progress::Progress; + +/// +pub mod checksum { + /// Returned by various methods to verify the checksum of a memory mapped file that might also exist on disk. + #[derive(thiserror::Error, Debug)] + #[allow(missing_docs)] + pub enum Error { + #[error("Interrupted by user")] + Interrupted, + #[error("Failed to hash data")] + Hasher(#[from] gix_hash::hasher::Error), + #[error(transparent)] + Verify(#[from] gix_hash::verify::Error), + #[error("Failed to read pack data for checksum verification")] + Io(#[from] std::io::Error), + } +} + +/// Returns the `index` at which the following `index + 1` value is not an increment over the value at `index`. +pub fn fan(data: &[u32]) -> Option { + data.windows(2) + .enumerate() + .find_map(|(win_index, v)| (v[0] > v[1]).then_some(win_index)) +} + +/// Calculate the hash of the given kind by trying to read the file from disk at `data_path` or falling back on the mapped content in `data`. +/// `Ok(expected)` or [`checksum::Error::Verify`] is returned if the hash matches or mismatches. +/// If the [`checksum::Error::Interrupted`] is returned, the operation was interrupted. +pub fn checksum_on_disk_or_mmap( + data_path: &Path, + data: &[u8], + expected: gix_hash::ObjectId, + object_hash: gix_hash::Kind, + progress: &mut dyn Progress, + should_interrupt: &AtomicBool, +) -> Result { + let data_len_without_trailer = data.len() - object_hash.len_in_bytes(); + let actual = match gix_hash::bytes_of_file( + data_path, + data_len_without_trailer as u64, + object_hash, + progress, + should_interrupt, + ) { + Ok(id) => id, + Err(gix_hash::io::Error::Io(err)) if err.kind() == std::io::ErrorKind::Interrupted => { + return Err(checksum::Error::Interrupted); + } + Err(gix_hash::io::Error::Io(_io_err)) => { + let start = std::time::Instant::now(); + let mut hasher = gix_hash::hasher(object_hash); + hasher.update(&data[..data_len_without_trailer]); + progress.inc_by(data_len_without_trailer); + progress.show_throughput(start); + hasher.try_finalize()? + } + Err(gix_hash::io::Error::Hasher(err)) => return Err(checksum::Error::Hasher(err)), + }; + + actual.verify(&expected)?; + Ok(actual) +} diff --git a/knotfeed/frame.go b/knotfeed/frame.go new file mode 100644 index 0000000000000000000000000000000000000000..2102e213012128ce8e22018399c8598714330879 --- /dev/null +++ b/knotfeed/frame.go @@ -0,0 +1,330 @@ +package knotfeed + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "strings" + "unicode" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + atprotorepo "github.com/bluesky-social/indigo/atproto/repo" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/samber/lo" + cbg "github.com/whyrusleeping/cbor-gen" + "tangled.org/core/tapc" +) + +const ( + TypeCommit = "#commit" + TypeSync = "#sync" + TypeIdentity = "#identity" + TypeAccount = "#account" + TypeInfo = "#info" + TypeError = "#error" +) + +type Message struct { + Type string + Feed Feed + Commit *Commit + InfoName string + Error string + Detail string +} + +type RecordOp struct { + Action tapc.RecordAction + Collection syntax.NSID + Rkey syntax.RecordKey + Bytes []byte + Prior PriorSha +} + +func (op RecordOp) Deleted() bool { + return op.Action == tapc.RecordDeleteAction +} + +const ( + sha1HexLen = 40 + sha256HexLen = 64 +) + +type ObjectID struct { + hex string +} + +func ParseObjectID(reported string) (ObjectID, bool) { + sized := len(reported) == sha1HexLen || len(reported) == sha256HexLen + if !sized || strings.ContainsFunc(reported, notHexDigit) { + return ObjectID{}, false + } + if strings.Trim(reported, "0") == "" { + return ObjectID{}, true + } + return ObjectID{hex: reported}, true +} + +func (o ObjectID) String() string { return o.hex } + +func (o ObjectID) IsZero() bool { return o.hex == "" } + +func notHexDigit(r rune) bool { return !strings.ContainsRune(hexDigits, unicode.ToLower(r)) } + +type PriorSha struct { + sha ObjectID + known bool +} + +func ParsePriorSha(reported string) PriorSha { + sha, ok := ParseObjectID(reported) + if !ok { + return PriorSha{} + } + return PriorSha{sha: sha, known: true} +} + +func (p PriorSha) Sha() (ObjectID, bool) { return p.sha, p.known } + +type Commit struct { + Repo syntax.DID + Seq int64 + Rev string + Records []RecordOp +} + +func Decode(data []byte, log *slog.Logger) (Message, error) { + r := bytes.NewReader(data) + head, err := readHeader(r) + if err != nil { + return Message{}, fmt.Errorf("frame header: %w", err) + } + if head.isError { + return decodeError(r, head) + } + if head.msgType != "" { + return decodeTyped(head.msgType, r, log) + } + return Message{}, fmt.Errorf("frame header names neither a type or an error") +} + +func decodeError(r io.Reader, head frameHeader) (Message, error) { + msg := Message{Type: TypeError, Error: head.errName, Detail: head.errDetail} + maj, count, err := cbg.CborReadHeader(r) + if errors.Is(err, io.EOF) { + return msg, nil + } + if err != nil { + return Message{}, fmt.Errorf("error frame body: %w", err) + } + if maj != cbg.MajMap { + return Message{}, fmt.Errorf("error frame body is major type %d, not a map", maj) + } + for range count { + key, err := cbg.ReadString(r) + if err != nil { + return Message{}, fmt.Errorf("error frame body: %w", err) + } + switch key { + case "error": + if msg.Error, err = cbg.ReadString(r); err != nil { + return Message{}, fmt.Errorf("error frame body: %w", err) + } + case "message": + if msg.Detail, err = cbg.ReadString(r); err != nil { + return Message{}, fmt.Errorf("error frame body: %w", err) + } + default: + if err := skipValue(r, 0); err != nil { + return Message{}, err + } + } + } + return msg, nil +} + +type frameHeader struct { + msgType string + isError bool + errName string + errDetail string +} + +func readHeader(r io.Reader) (frameHeader, error) { + maj, count, err := cbg.CborReadHeader(r) + if err != nil { + return frameHeader{}, err + } + if maj != cbg.MajMap { + return frameHeader{}, fmt.Errorf("expected a cbor map header, got major type %d", maj) + } + head := frameHeader{} + for range count { + key, err := cbg.ReadString(r) + if err != nil { + return frameHeader{}, err + } + switch key { + case "t": + if head.msgType, err = cbg.ReadString(r); err != nil { + return frameHeader{}, err + } + case "op": + maj, val, err := cbg.CborReadHeader(r) + if err != nil { + return frameHeader{}, err + } + head.isError = maj == cbg.MajNegativeInt && val == 0 + case "error": + if head.errName, err = cbg.ReadString(r); err != nil { + return frameHeader{}, err + } + case "message": + if head.errDetail, err = cbg.ReadString(r); err != nil { + return frameHeader{}, err + } + default: + if err := skipValue(r, 0); err != nil { + return frameHeader{}, err + } + } + } + return head, nil +} + +func decodeTyped(msgType string, r io.Reader, log *slog.Logger) (Message, error) { + switch msgType { + case TypeCommit: + var evt comatproto.SyncSubscribeRepos_Commit + if err := evt.UnmarshalCBOR(r); err != nil { + return Message{}, fmt.Errorf("commit frame: %w", err) + } + commit, err := resolveCommit(&evt, log) + if err != nil { + return Message{}, err + } + return Message{Type: TypeCommit, Commit: commit}, nil + case TypeSync, TypeIdentity, TypeAccount: + return Message{Type: msgType}, nil + case TypeInfo: + var evt comatproto.SyncSubscribeRepos_Info + if err := evt.UnmarshalCBOR(r); err != nil { + return Message{}, fmt.Errorf("info frame: %w", err) + } + return Message{Type: TypeInfo, InfoName: evt.Name}, nil + default: + if err := skipValue(r, 0); err != nil { + return Message{}, err + } + return Message{Type: msgType}, nil + } +} + +func resolveCommit(evt *comatproto.SyncSubscribeRepos_Commit, log *slog.Logger) (*Commit, error) { + repoDid, err := syntax.ParseDID(evt.Repo) + if err != nil { + return nil, fmt.Errorf("commit frame names repo %q, which isn't a did: %w", evt.Repo, err) + } + commit := &Commit{ + Repo: repoDid, + Seq: evt.Seq, + Rev: evt.Rev, + } + if len(evt.Ops) == 0 { + return commit, nil + } + ctx := context.Background() + _, repo, err := atprotorepo.LoadRepoFromCAR(ctx, bytes.NewReader(evt.Blocks)) + if err != nil { + log.Error("commit frame blocks didn't decode as a car, dropping the commit", "repo", repoDid, "seq", evt.Seq, "err", err) + return commit, nil + } + commit.Records = lo.FilterMap(evt.Ops, func(op *comatproto.SyncSubscribeRepos_RepoOp, _ int) (RecordOp, bool) { + if op == nil { + return RecordOp{}, false + } + record, err := resolveOp(ctx, op, repo) + if err != nil { + log.Warn("dropping a commit op", "repo", repoDid, "path", op.Path, "err", err) + return RecordOp{}, false + } + return record, true + }) + return commit, nil +} + +var recordActions = map[string]tapc.RecordAction{ + string(tapc.RecordCreateAction): tapc.RecordCreateAction, + string(tapc.RecordUpdateAction): tapc.RecordUpdateAction, + string(tapc.RecordDeleteAction): tapc.RecordDeleteAction, +} + +func resolveOp(ctx context.Context, op *comatproto.SyncSubscribeRepos_RepoOp, repo *atprotorepo.Repo) (RecordOp, error) { + action, known := recordActions[op.Action] + if !known { + return RecordOp{}, fmt.Errorf("op action %q is none of create, update or delete", op.Action) + } + rawCollection, rawRkey, found := strings.Cut(op.Path, "/") + if !found { + return RecordOp{}, errors.New("op path has no rkey") + } + collection, err := syntax.ParseNSID(rawCollection) + if err != nil { + return RecordOp{}, fmt.Errorf("op collection: %w", err) + } + rkey, err := syntax.ParseRecordKey(rawRkey) + if err != nil { + return RecordOp{}, fmt.Errorf("op rkey: %w", err) + } + record := RecordOp{Action: action, Collection: collection, Rkey: rkey} + if record.Deleted() || repo == nil { + return record, nil + } + payload, _, err := repo.GetRecordBytes(ctx, collection, rkey) + if err != nil { + return RecordOp{}, fmt.Errorf("record bytes missing from the frame car: %w", err) + } + record.Bytes = payload + return record, nil +} + +const maxCborDepth = 64 + +func skipValue(r io.Reader, depth int) error { + if depth > maxCborDepth { + return fmt.Errorf("cbor value nests past %d levels", maxCborDepth) + } + maj, val, err := cbg.CborReadHeader(r) + if err != nil { + return err + } + switch maj { + case cbg.MajByteString, cbg.MajTextString: + _, err = io.CopyN(io.Discard, r, int64(val)) + return err + case cbg.MajArray: + for range val { + if err := skipValue(r, depth+1); err != nil { + return err + } + } + return nil + case cbg.MajMap: + for range val { + if err := skipValue(r, depth+1); err != nil { + return err + } + if err := skipValue(r, depth+1); err != nil { + return err + } + } + return nil + case cbg.MajTag: + return skipValue(r, depth+1) + default: + return nil + } +} diff --git a/knotfeed/frame_test.go b/knotfeed/frame_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ff1c360228e8d9e7ed6f4b568be2b181e741d25f --- /dev/null +++ b/knotfeed/frame_test.go @@ -0,0 +1,236 @@ +package knotfeed + +import ( + "bytes" + "log/slog" + "testing" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func writeText(w *bytes.Buffer, s string) { + if err := cbg.CborWriteHeader(w, cbg.MajTextString, uint64(len(s))); err != nil { + w.Reset() + } + w.WriteString(s) +} + +func headerFrame(t *testing.T, pairs ...any) []byte { + t.Helper() + if len(pairs)%2 != 0 { + t.Fatal("header pairs must alternate key and value") + } + var out bytes.Buffer + if err := cbg.CborWriteHeader(&out, cbg.MajMap, uint64(len(pairs)/2)); err != nil { + t.Fatalf("map header: %v", err) + } + for i := 0; i < len(pairs); i += 2 { + key, ok := pairs[i].(string) + if !ok { + t.Fatal("header keys must be strings") + } + writeText(&out, key) + switch value := pairs[i+1].(type) { + case string: + writeText(&out, value) + case int: + if value < 0 { + if err := cbg.CborWriteHeader(&out, cbg.MajNegativeInt, uint64(-1-value)); err != nil { + t.Fatalf("value for %q: %v", key, err) + } + } else { + if err := cbg.CborWriteHeader(&out, cbg.MajUnsignedInt, uint64(value)); err != nil { + t.Fatalf("value for %q: %v", key, err) + } + } + case []string: + if err := cbg.CborWriteHeader(&out, cbg.MajArray, uint64(len(value))); err != nil { + t.Fatalf("value for %q: %v", key, err) + } + for _, s := range value { + writeText(&out, s) + } + case []byte: + out.Write(value) + default: + t.Fatalf("unsupported header value for %q", key) + } + } + return out.Bytes() +} + +func testCid(t *testing.T) cid.Cid { + t.Helper() + sum, err := multihash.Sum([]byte("knotfeed"), multihash.SHA2_256, -1) + if err != nil { + t.Fatalf("multihash: %v", err) + } + return cid.NewCidV1(cid.DagCBOR, sum) +} + +func TestDecodeReadsACommitFrameWithoutBlocks(t *testing.T) { + var payload bytes.Buffer + evt := comatproto.SyncSubscribeRepos_Commit{ + Repo: "did:plc:scallop", + Seq: 42, + Rev: "3lb2xkw2qrs2j", + Commit: lexutil.LexLink(testCid(t)), + } + if err := evt.MarshalCBOR(&payload); err != nil { + t.Fatalf("MarshalCBOR: %v", err) + } + frame := append(headerFrame(t, "t", "#commit"), payload.Bytes()...) + + message, err := Decode(frame, slog.Default()) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if message.Type != TypeCommit { + t.Fatalf("Type = %q, want %q", message.Type, TypeCommit) + } + if message.Commit == nil { + t.Fatal("Decode answered no commit") + } + if message.Commit.Repo != "did:plc:scallop" || message.Commit.Seq != 42 || message.Commit.Rev != "3lb2xkw2qrs2j" { + t.Fatalf("Commit = %+v", message.Commit) + } + if message.Commit.Records != nil { + t.Fatalf("a frame with no ops answered records: %+v", message.Commit.Records) + } +} + +func TestDecodeDropsUnresolvableOpsAndKeepsFrame(t *testing.T) { + var payload bytes.Buffer + evt := comatproto.SyncSubscribeRepos_Commit{ + Repo: "did:plc:scallop", + Seq: 43, + Commit: lexutil.LexLink(testCid(t)), + Ops: []*comatproto.SyncSubscribeRepos_RepoOp{ + {Action: "create", Path: GitRefCollection.String() + "/refs~2fheads~2fmain"}, + }, + } + if err := evt.MarshalCBOR(&payload); err != nil { + t.Fatalf("MarshalCBOR: %v", err) + } + frame := append(headerFrame(t, "t", "#commit"), payload.Bytes()...) + + message, err := Decode(frame, slog.Default()) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if message.Commit == nil { + t.Fatal("Decode answered no commit") + } + if len(message.Commit.Records) != 0 { + t.Fatalf("a frame without a car answered records: %+v", message.Commit.Records) + } + if message.Commit.Seq != 43 { + t.Fatalf("Seq = %d, want 43", message.Commit.Seq) + } +} + +func errorFrameBody(t *testing.T, name, detail string) []byte { + t.Helper() + var body bytes.Buffer + if err := cbg.CborWriteHeader(&body, cbg.MajMap, 2); err != nil { + t.Fatalf("map header: %v", err) + } + writeText(&body, "error") + writeText(&body, name) + writeText(&body, "message") + writeText(&body, detail) + return body.Bytes() +} + +func errorFrame(t *testing.T, name, detail string) []byte { + t.Helper() + return append(headerFrame(t, "op", -1), errorFrameBody(t, name, detail)...) +} + +func TestDecodeReadsErrorFrames(t *testing.T) { + for _, tt := range []struct { + name string + frame []byte + wantError string + wantDetail string + }{ + {"two-item frame", errorFrame(t, "FutureCursor", "the cursor is ahead of the knot"), "FutureCursor", "the cursor is ahead of the knot"}, + {"single-map frame", headerFrame(t, "op", -1, "t", "#error", "error", "FutureCursor", "message", "the cursor is ahead of the knot"), "FutureCursor", "the cursor is ahead of the knot"}, + {"body wins over the header", append( + headerFrame(t, "op", -1, "error", "StaleHeader", "message", "from the header"), + errorFrameBody(t, "FutureCursor", "from the body")..., + ), "FutureCursor", "from the body"}, + } { + t.Run(tt.name, func(t *testing.T) { + message, err := Decode(tt.frame, slog.Default()) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if message.Type != TypeError || message.Error != tt.wantError || message.Detail != tt.wantDetail { + t.Fatalf("Message = %+v", message) + } + }) + } +} + +func TestDecodeRefusesDeeplyNestedUnknownFields(t *testing.T) { + var nested bytes.Buffer + for range maxCborDepth + 10 { + if err := cbg.CborWriteHeader(&nested, cbg.MajArray, 1); err != nil { + t.Fatalf("array header: %v", err) + } + } + if err := cbg.CborWriteHeader(&nested, cbg.MajUnsignedInt, 0); err != nil { + t.Fatalf("leaf: %v", err) + } + if _, err := Decode(headerFrame(t, "t", "#account", "extra", nested.Bytes()), slog.Default()); err == nil { + t.Fatal("Decode accepted a frame nesting unknown fields past the depth budget") + } +} + +func TestDecodeReadsAnInfoFrame(t *testing.T) { + var payload bytes.Buffer + evt := comatproto.SyncSubscribeRepos_Info{ + Name: "OutdatedCursor", + } + if err := evt.MarshalCBOR(&payload); err != nil { + t.Fatalf("MarshalCBOR: %v", err) + } + frame := append(headerFrame(t, "t", "#info"), payload.Bytes()...) + + message, err := Decode(frame, slog.Default()) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if message.Type != TypeInfo || message.InfoName != "OutdatedCursor" { + t.Fatalf("Message = %+v", message) + } +} + +func TestDecodeSkipsUnknownHeaderFields(t *testing.T) { + var payload bytes.Buffer + evt := comatproto.SyncSubscribeRepos_Account{Did: "did:plc:scallop", Active: true} + if err := evt.MarshalCBOR(&payload); err != nil { + t.Fatalf("MarshalCBOR: %v", err) + } + frame := append(headerFrame(t, "t", "#account", "extra", []string{"a", "b"}), payload.Bytes()...) + + message, err := Decode(frame, slog.Default()) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if message.Type != TypeAccount { + t.Fatalf("Type = %q, want %q", message.Type, TypeAccount) + } +} + +func TestDecodeRefusesFrameNamingNeitherTypeOrError(t *testing.T) { + frame := headerFrame(t, "ops", []string{"nothing"}) + if _, err := Decode(frame, slog.Default()); err == nil { + t.Fatal("Decode accepted a frame naming neither a type or an error") + } +} diff --git a/knotfeed/legacy.go b/knotfeed/legacy.go new file mode 100644 index 0000000000000000000000000000000000000000..207922c43f1d1fd8eeb1ca6ad0465cf0c021172a --- /dev/null +++ b/knotfeed/legacy.go @@ -0,0 +1,140 @@ +package knotfeed + +import ( + "cmp" + "encoding/json" + "fmt" + "log/slog" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/samber/lo" + "tangled.org/core/tapc" +) + +const ( + legacyRefUpdateNSID syntax.NSID = "sh.tangled.git.refUpdate" + legacyCollaboratorUpdateNSID syntax.NSID = "sh.tangled.repo.collaboratorUpdate" + + collaboratorInviteCollection syntax.NSID = "sh.tangled.repo.collaboratorInvite" +) + +type legacyAclOp string + +const ( + legacyAclOpAdd legacyAclOp = "add" + legacyAclOpRemove legacyAclOp = "remove" +) + +func (op legacyAclOp) action() tapc.RecordAction { + return lo.Ternary(strings.EqualFold(string(op), string(legacyAclOpRemove)), + tapc.RecordDeleteAction, tapc.RecordCreateAction) +} + +type legacyFrame struct { + Nsid syntax.NSID `json:"nsid"` + Created int64 `json:"created"` + Event json.RawMessage `json:"event"` +} + +type legacyRefUpdate struct { + Repo syntax.DID `json:"repo"` + LegacyRepo syntax.DID `json:"repoDid"` // knots older than 1.14 name the repo this way + Ref string `json:"ref"` + OldSha string `json:"oldSha"` + NewSha string `json:"newSha"` + CommitterDid string `json:"committerDid"` + PushOptions []string `json:"pushOptions"` +} + +type legacyAclUpdate struct { + Op legacyAclOp `json:"op"` + Repo syntax.DID `json:"repo"` + Subject syntax.DID `json:"subject"` +} + +var legacyOps = map[syntax.NSID]func(json.RawMessage, *Commit) error{ + legacyRefUpdateNSID: legacyRefOp, + legacyCollaboratorUpdateNSID: legacyCollaboratorOp, +} + +func decodeLegacy(data []byte, _ *slog.Logger) (Message, error) { + var frame legacyFrame + if err := json.Unmarshal(data, &frame); err != nil { + return Message{}, fmt.Errorf("legacy frame: %w", err) + } + if frame.Created <= 0 { + return Message{}, fmt.Errorf("legacy %s frame lacks its created stamp: %d", frame.Nsid, frame.Created) + } + + commit := &Commit{Seq: frame.Created} + if op, known := legacyOps[frame.Nsid]; known { + if err := op(frame.Event, commit); err != nil { + return Message{}, err + } + } + return Message{Type: TypeCommit, Commit: commit}, nil +} + +func legacyRefOp(payload json.RawMessage, commit *Commit) error { + var event legacyRefUpdate + if err := json.Unmarshal(payload, &event); err != nil { + return fmt.Errorf("legacy ref update: %w", err) + } + rkey, short := EscapeRefname(event.Ref) + newSha, isObject := ParseObjectID(event.NewSha) + // pre-1.13 knots put the owner's did in repoDid, so this can miss on lookup; + // the event still has to flow through so the host's cursor keeps moving + repo := cmp.Or(event.Repo, event.LegacyRepo) + if event.Ref == "" || repo == "" || !short || !isObject { + return nil + } + op := RecordOp{ + Action: tapc.RecordDeleteAction, + Collection: GitRefCollection, + Rkey: rkey, + Prior: ParsePriorSha(event.OldSha), + } + if !newSha.IsZero() { + // old knots leave committerDid empty; attribute those pushes to the repo itself + editor, err := syntax.ParseDID(event.CommitterDid) + if err != nil { + editor = repo + } + record, err := encodeRefRecord(RefRecord{ + Sha: newSha, + Editor: editor, + PushOptions: event.PushOptions, + }) + if err != nil { + return fmt.Errorf("re-encoding legacy ref update: %w", err) + } + op.Action, op.Bytes = tapc.RecordCreateAction, record + } + + commit.Repo = repo + commit.Records = []RecordOp{op} + return nil +} + +func legacyCollaboratorOp(payload json.RawMessage, commit *Commit) error { + var event legacyAclUpdate + if err := json.Unmarshal(payload, &event); err != nil { + return fmt.Errorf("legacy collaborator update: %w", err) + } + if event.Repo == "" || event.Subject == "" { + return nil + } + + rkey, err := syntax.ParseRecordKey(event.Subject.String()) + if err != nil { + return fmt.Errorf("legacy collaborator update names subject %q: %w", event.Subject, err) + } + commit.Repo = event.Repo + commit.Records = []RecordOp{{ + Action: event.Op.action(), + Collection: collaboratorInviteCollection, + Rkey: rkey, + }} + return nil +} diff --git a/knotfeed/legacy_test.go b/knotfeed/legacy_test.go new file mode 100644 index 0000000000000000000000000000000000000000..60cafa7ae96e41e0483056e8b0124cfcb9bb6393 --- /dev/null +++ b/knotfeed/legacy_test.go @@ -0,0 +1,273 @@ +package knotfeed + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "tangled.org/core/tapc" +) + +const ( + legacyNewSha = "bcfa6359b037bc411e9626260cc3dd9a28408d2c" + legacyOldSha = "9eab6c4f29bc9e609ec52888cd6a4ef716b567c9" + zeroSha1 = "0000000000000000000000000000000000000000" +) + +const legacyRefFrame = `{ + "rkey": "3mug", + "nsid": "sh.tangled.git.refUpdate", + "created": 1788245839553422000, + "event": { + "$type": "sh.tangled.git.refUpdate", + "ref": "refs/heads/iso16283-low-frequency-corner", + "oldSha": "9eab6c4f29bc9e609ec52888cd6a4ef716b567c9", + "newSha": "bcfa6359b037bc411e9626260cc3dd9a28408d2c", + "committerDid": "did:plc:jder4wkjnwh6mzhrx4n5ebti", + "ownerDid": "did:plc:jder4wkjnwh6mzhrx4n5ebti", + "repo": "did:plc:5i6o2hrw2viwpxgicwad7f2h", + "pushOptions": ["skip-ci"], + "changedFiles": ["tests/building/measurement/test_low_frequency.py"], + "meta": {"isDefaultRef": false, "commitCount": {"byEmail": [{"email": "mail@jmrp.io", "count": 1}]}} + } +}` + +func refFrame(ref, repo, oldSha, newSha string) string { + return fmt.Sprintf( + `{"nsid":"sh.tangled.git.refUpdate","created":11,`+ + `"event":{"ref":%q,"repo":%q,"oldSha":%q,"newSha":%q}}`, + ref, repo, oldSha, newSha) +} + +func sha(t *testing.T, hex string) ObjectID { + t.Helper() + id, ok := ParseObjectID(hex) + if !ok { + t.Fatalf("ParseObjectID(%q)", hex) + } + return id +} + +func refOp(t *testing.T, refname, oldSha string, rec RefRecord) RecordOp { + t.Helper() + rkey, ok := EscapeRefname(refname) + if !ok { + t.Fatalf("EscapeRefname(%q)", refname) + } + op := RecordOp{ + Action: tapc.RecordDeleteAction, + Collection: GitRefCollection, + Rkey: rkey, + Prior: ParsePriorSha(oldSha), + } + if rec.Sha.IsZero() { + return op + } + record, err := encodeRefRecord(rec) + if err != nil { + t.Fatalf("encodeRefRecord: %v", err) + } + op.Action, op.Bytes = tapc.RecordCreateAction, record + return op +} + +func TestLegacyFrameReadsAsCommit(t *testing.T) { + for _, tt := range []struct { + name string + frame string + want Commit + }{ + { + name: "push", + frame: legacyRefFrame, + want: Commit{ + Repo: "did:plc:5i6o2hrw2viwpxgicwad7f2h", + Seq: 1788245839553422000, + Records: []RecordOp{refOp(t, "refs/heads/iso16283-low-frequency-corner", legacyOldSha, + RefRecord{ + Sha: sha(t, legacyNewSha), + Editor: "did:plc:jder4wkjnwh6mzhrx4n5ebti", + PushOptions: []string{"skip-ci"}, + })}, + }, + }, + { + name: "deleted branch", + frame: refFrame("refs/heads/gone", "did:plc:scallop", legacyOldSha, zeroSha1), + want: Commit{ + Repo: "did:plc:scallop", + Seq: 11, + Records: []RecordOp{refOp(t, "refs/heads/gone", legacyOldSha, RefRecord{})}, + }, + }, + { + name: "deleted branch in a sha256 repo", + frame: refFrame("refs/heads/gone", "did:plc:scallop", + strings.Repeat("ab", 32), strings.Repeat("0", 64)), + want: Commit{ + Repo: "did:plc:scallop", + Seq: 11, + Records: []RecordOp{refOp(t, "refs/heads/gone", strings.Repeat("ab", 32), RefRecord{})}, + }, + }, + { + name: "brand new branch", + frame: refFrame("refs/heads/new", "did:plc:scallop", zeroSha1, legacyNewSha), + want: Commit{ + Repo: "did:plc:scallop", + Seq: 11, + Records: []RecordOp{refOp(t, "refs/heads/new", zeroSha1, + RefRecord{Sha: sha(t, legacyNewSha), Editor: "did:plc:scallop"})}, + }, + }, + { + name: "push that left oldSha empty", + frame: refFrame("refs/heads/new", "did:plc:scallop", "", legacyNewSha), + want: Commit{ + Repo: "did:plc:scallop", + Seq: 11, + Records: []RecordOp{refOp(t, "refs/heads/new", "", + RefRecord{Sha: sha(t, legacyNewSha), Editor: "did:plc:scallop"})}, + }, + }, + { + name: "push from a pre-1.14 knot, which names the repo repoDid", + frame: `{"nsid":"sh.tangled.git.refUpdate","created":11,"event":{` + + `"ref":"refs/heads/main","ownerDid":"did:plc:clam","repoDid":"did:plc:scallop",` + + `"repoName":"shell","committerDid":"did:plc:clam",` + + `"oldSha":"` + legacyOldSha + `","newSha":"` + legacyNewSha + `"}}`, + want: Commit{ + Repo: "did:plc:scallop", + Seq: 11, + Records: []RecordOp{refOp(t, "refs/heads/main", legacyOldSha, + RefRecord{Sha: sha(t, legacyNewSha), Editor: "did:plc:clam"})}, + }, + }, + { + name: "push with an empty committerDid", + frame: `{"nsid":"sh.tangled.git.refUpdate","created":11,` + + `"event":{"ref":"refs/heads/main","repo":"did:plc:scallop","committerDid":"",` + + `"oldSha":"` + legacyOldSha + `","newSha":"` + legacyNewSha + `"}}`, + want: Commit{ + Repo: "did:plc:scallop", + Seq: 11, + Records: []RecordOp{refOp(t, "refs/heads/main", legacyOldSha, + RefRecord{Sha: sha(t, legacyNewSha), Editor: "did:plc:scallop"})}, + }, + }, + { + name: "collaborator removal", + frame: `{"nsid":"sh.tangled.repo.collaboratorUpdate","created":9, + "event":{"op":"remove","subject":"did:plc:olaren","repo":"did:plc:scallop"}}`, + want: Commit{ + Repo: "did:plc:scallop", + Seq: 9, + Records: []RecordOp{{ + Action: tapc.RecordDeleteAction, + Collection: collaboratorInviteCollection, + Rkey: "did:plc:olaren", + }}, + }, + }, + { + name: "knot member update", + frame: `{"nsid":"sh.tangled.knot.memberUpdate","created":11, + "event":{"op":"add","subject":"did:plc:limpet"}}`, + want: Commit{Seq: 11}, + }, + { + name: "ref update with an empty sha", + frame: refFrame("refs/heads/main", "did:plc:scallop", legacyOldSha, ""), + want: Commit{Seq: 11}, + }, + { + name: "ref update with an empty refname", + frame: refFrame("", "did:plc:scallop", legacyOldSha, legacyNewSha), + want: Commit{Seq: 11}, + }, + { + name: "new sha with shell junk in it", + frame: refFrame("refs/heads/main", "did:plc:scallop", legacyOldSha, "HEAD~1; rm -rf /"), + want: Commit{Seq: 11}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + msg, err := decodeLegacy([]byte(tt.frame), nil) + if err != nil { + t.Fatalf("decodeLegacy: %v", err) + } + if msg.Type != TypeCommit || msg.Commit == nil { + t.Fatalf("message type = %q, want a commit", msg.Type) + } + if !reflect.DeepEqual(*msg.Commit, tt.want) { + t.Errorf("commit = %+v, want %+v", *msg.Commit, tt.want) + } + }) + } +} + +func TestLegacyFrameWithoutCursorErrorsOut(t *testing.T) { + if _, err := decodeLegacy([]byte(`{"nsid":"sh.tangled.git.refUpdate"}`), nil); err == nil { + t.Error("frame without a created stamp decoded anyway") + } +} + +func TestOnlyObjectIdReadsAsPriorSha(t *testing.T) { + for _, reported := range []string{"", "def456", "HEAD", strings.Repeat("zz", 20), zeroSha1 + "0"} { + if id, known := ParsePriorSha(reported).Sha(); known { + t.Errorf("ParsePriorSha(%q) = %q, want an unknown prior", reported, id) + } + } + if _, known := (RecordOp{}).Prior.Sha(); known { + t.Error("zero-value RecordOp came back with a known prior sha") + } + + hex := strings.Repeat("ab", 32) + if got, known := ParsePriorSha(hex).Sha(); !known || got.String() != hex { + t.Errorf("ParsePriorSha(%q) = (%q, %v), want the sha back", hex, got, known) + } + if got, known := ParsePriorSha(zeroSha1).Sha(); !known || !got.IsZero() { + t.Errorf("ParsePriorSha(%q) = (%q, %v), want a known absence", zeroSha1, got, known) + } +} + +func TestParseObjectIDTakesOnlyRealObjectIds(t *testing.T) { + for _, hex := range []string{ + "", "def456", strings.Repeat("ab", 21), strings.Repeat("zz", 20), "HEAD~1; rm -rf /", + } { + if id, ok := ParseObjectID(hex); ok { + t.Errorf("ParseObjectID(%q) = %q, want a refusal", hex, id) + } + } + for _, hex := range []string{zeroSha1, strings.Repeat("0", 64)} { + id, ok := ParseObjectID(hex) + if !ok || !id.IsZero() { + t.Errorf("ParseObjectID(%q) = (%q, %v), want the null object id", hex, id, ok) + } + } + for _, hex := range []string{legacyNewSha, strings.Repeat("AB", 20), strings.Repeat("cD", 32)} { + id, ok := ParseObjectID(hex) + if !ok || id.String() != hex { + t.Errorf("ParseObjectID(%q) = (%q, %v), want the sha back", hex, id, ok) + } + } +} + +func TestLegacyFrameWithUnusableFieldsIsRefused(t *testing.T) { + for _, tt := range []struct { + name string + frame string + }{ + {"nsid that isn't an nsid", `{"nsid":"not an nsid","created":11}`}, + {"repo that isn't a did", refFrame("refs/heads/main", "scallop", legacyOldSha, legacyNewSha)}, + {"collaborator subject that isn't a did", `{"nsid":"sh.tangled.repo.collaboratorUpdate","created":9, + "event":{"op":"remove","subject":"olaren","repo":"did:plc:scallop"}}`}, + } { + t.Run(tt.name, func(t *testing.T) { + if _, err := decodeLegacy([]byte(tt.frame), nil); err == nil { + t.Error("frame decoded anyway") + } + }) + } +} diff --git a/knotfeed/record.go b/knotfeed/record.go new file mode 100644 index 0000000000000000000000000000000000000000..7ebb1f76ec77c05a2945f23133f6d0c39381b9cd --- /dev/null +++ b/knotfeed/record.go @@ -0,0 +1,288 @@ +package knotfeed + +import ( + "bytes" + "fmt" + "io" + "slices" + "strings" + "unicode/utf8" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/samber/lo" + cbg "github.com/whyrusleeping/cbor-gen" +) + +const maxPushOptions = 64 +const GitRefCollection syntax.NSID = "sh.tangled.git.ref" + +const mstKeyBudget = 256 + +const MaxRefRkeyBytes = mstKeyBudget - len(GitRefCollection) - 1 + +const hexDigits = "0123456789abcdef" + +func plainByte(b byte) bool { + switch { + case b >= 'A' && b <= 'Z', b >= 'a' && b <= 'z', b >= '0' && b <= '9': + return true + case b == '-', b == '_': + return true + default: + return false + } +} + +func allPlain(s string) bool { + for i := range len(s) { + if !plainByte(s[i]) { + return false + } + } + return true +} + +func EscapeRefname(refname string) (syntax.RecordKey, bool) { + out := make([]byte, 0, len(refname)) + for i := range len(refname) { + b := refname[i] + if plainByte(b) { + out = append(out, b) + continue + } + out = append(out, '~', hexDigits[b>>4], hexDigits[b&0x0f]) + } + if len(out) > MaxRefRkeyBytes { + return "", false + } + return syntax.RecordKey(out), true +} + +func UnescapeRkey(rkey syntax.RecordKey) (string, bool) { + escaped := rkey.String() + if !strings.Contains(escaped, "~") { + return escaped, allPlain(escaped) + } + var out []byte + rest := escaped + for { + idx := strings.IndexByte(rest, '~') + if idx < 0 { + if !allPlain(rest) { + return "", false + } + if !utf8.Valid(out) { + return "", false + } + return string(append(out, rest...)), true + } + head, chunk := rest[:idx], rest[idx+1:] + if !allPlain(head) || len(chunk) < 2 { + return "", false + } + hi, ok := nibble(chunk[0]) + if !ok { + return "", false + } + lo, ok := nibble(chunk[1]) + if !ok { + return "", false + } + out = append(out, head...) + out = append(out, hi<<4|lo) + rest = chunk[2:] + } +} + +func nibble(b byte) (byte, bool) { + switch { + case b >= '0' && b <= '9': + return b - '0', true + case b >= 'a' && b <= 'f': + return b - 'a' + 10, true + default: + return 0, false + } +} + +type RefRecord struct { + Sha ObjectID + Editor syntax.DID + PushOptions []string +} + +func DecodeRefRecord(data []byte) (RefRecord, error) { + var rec RefRecord + var rawSha, rawEditor string + r := bytes.NewReader(data) + maj, count, err := cbg.CborReadHeader(r) + if err != nil { + return rec, err + } + if maj != cbg.MajMap { + return rec, fmt.Errorf("expected a cbor map, got major type %d", maj) + } + for range count { + key, err := cbg.ReadString(r) + if err != nil { + return rec, err + } + switch key { + case "sha": + if rawSha, err = cbg.ReadString(r); err != nil { + return rec, err + } + case "x-tngl-editor": + if rawEditor, err = cbg.ReadString(r); err != nil { + return rec, err + } + case "x-tngl-push-options": + if rec.PushOptions, err = readStringArray(r); err != nil { + return rec, err + } + default: + if err := skipValue(r, 0); err != nil { + return rec, err + } + } + } + if r.Len() != 0 { + return rec, fmt.Errorf("ref record has %d trailing bytes after its map", r.Len()) + } + sha, ok := ParseObjectID(rawSha) + if !ok || sha.IsZero() { + return rec, fmt.Errorf("sha %q isn't 40 or 64 hex digits naming an object", rawSha) + } + rec.Sha = sha + if rawEditor == "" { + return rec, nil + } + editor, err := syntax.ParseDID(rawEditor) + if err != nil { + return rec, fmt.Errorf("ref record editor: %w", err) + } + rec.Editor = editor + return rec, nil +} + +func readStringArray(r *bytes.Reader) ([]string, error) { + maj, count, err := cbg.CborReadHeader(r) + if err != nil { + return nil, err + } + if maj != cbg.MajArray { + return nil, fmt.Errorf("expected a cbor array, got major type %d", maj) + } + if count > maxPushOptions { + return nil, fmt.Errorf("push options array claims %d entries", count) + } + out := make([]string, 0, count) + for range count { + s, err := cbg.ReadString(r) + if err != nil { + return nil, err + } + out = append(out, s) + } + return out, nil +} + +func DecodeRefsSnapshot(data []byte) (defaultBranch string, packedRefs []byte, err error) { + r := bytes.NewReader(data) + maj, count, err := cbg.CborReadHeader(r) + if err != nil { + return "", nil, err + } + if maj != cbg.MajMap { + return "", nil, fmt.Errorf("expected a cbor map, got major type %d", maj) + } + var sawBranch, sawRefs bool + for range count { + key, err := cbg.ReadString(r) + if err != nil { + return "", nil, err + } + switch key { + case "defaultBranch": + if defaultBranch, err = cbg.ReadString(r); err != nil { + return "", nil, err + } + sawBranch = true + case "refs": + maj, n, err := cbg.CborReadHeader(r) + if err != nil { + return "", nil, err + } + if maj != cbg.MajByteString { + return "", nil, fmt.Errorf("refs should be bytes, got major type %d", maj) + } + packedRefs = make([]byte, n) + if _, err := io.ReadFull(r, packedRefs); err != nil { + return "", nil, err + } + sawRefs = true + default: + var skipped cbg.Deferred + if err := skipped.UnmarshalCBOR(r); err != nil { + return "", nil, err + } + } + } + if !sawBranch { + return "", nil, fmt.Errorf("refs snapshot is missing defaultBranch") + } + if !sawRefs { + return "", nil, fmt.Errorf("refs snapshot is missing refs") + } + return defaultBranch, packedRefs, nil +} + +type cborBuf struct{ bytes.Buffer } + +func (b *cborBuf) header(major byte, count uint64) { + _ = cbg.CborWriteHeader(&b.Buffer, major, count) +} + +func (b *cborBuf) str(s string) { + b.header(cbg.MajTextString, uint64(len(s))) + b.WriteString(s) +} + +func (b *cborBuf) pair(key, value string) { + b.str(key) + b.str(value) +} + +func encodeRefRecord(rec RefRecord) ([]byte, error) { + past := func(s string) bool { return len(s) > cbg.MaxLength } + switch { + case rec.Sha.IsZero(): + return nil, fmt.Errorf("won't encode a ref record without a sha") + case len(rec.PushOptions) > maxPushOptions: + return nil, fmt.Errorf("won't encode %d push options, past the %d limit", len(rec.PushOptions), maxPushOptions) + case past(rec.Editor.String()) || slices.ContainsFunc(rec.PushOptions, past): + return nil, fmt.Errorf("ref record holds a string past the %d byte cbor limit", cbg.MaxLength) + } + + pairs := uint64(2) + if rec.Editor != "" { + pairs++ + } + if len(rec.PushOptions) > 0 { + pairs++ + } + + var out cborBuf + out.header(cbg.MajMap, pairs) + out.pair("sha", rec.Sha.String()) + out.pair("$type", GitRefCollection.String()) + if rec.Editor != "" { + out.pair("x-tngl-editor", rec.Editor.String()) + } + if len(rec.PushOptions) > 0 { + out.str("x-tngl-push-options") + out.header(cbg.MajArray, uint64(len(rec.PushOptions))) + lo.ForEach(rec.PushOptions, func(option string, _ int) { out.str(option) }) + } + return out.Bytes(), nil +} diff --git a/knotfeed/record_test.go b/knotfeed/record_test.go new file mode 100644 index 0000000000000000000000000000000000000000..430f0d9013d5ed054ca24d798faa559e7b49f307 --- /dev/null +++ b/knotfeed/record_test.go @@ -0,0 +1,171 @@ +package knotfeed + +import ( + "bytes" + "encoding/hex" + "slices" + "strings" + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" + cbg "github.com/whyrusleeping/cbor-gen" +) + +const frozenPushedHex = "a4637368617828616261626162616261626162616261626162616261626162616261626162616261626162616261626524747970657273682e74616e676c65642e6769742e7265666d782d746e676c2d656469746f726e6469643a706c633a6c696d70657473782d746e676c2d707573682d6f7074696f6e738167736b69702d6369" + +const frozenBareHex = "a2637368617840636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646524747970657273682e74616e676c65642e6769742e726566" + +func TestFrozenRecordBytesDecodeAndEncodeBackAsThemselves(t *testing.T) { + for _, tt := range []struct { + name string + hex string + want RefRecord + }{ + {"pushed", frozenPushedHex, RefRecord{ + Sha: sha(t, strings.Repeat("ab", 20)), Editor: "did:plc:limpet", PushOptions: []string{"skip-ci"}, + }}, + {"catch-up", frozenBareHex, RefRecord{Sha: sha(t, strings.Repeat("cd", 32))}}, + } { + t.Run(tt.name, func(t *testing.T) { + data, err := hex.DecodeString(tt.hex) + if err != nil { + t.Fatalf("frozen hex: %v", err) + } + rec, err := DecodeRefRecord(data) + if err != nil { + t.Fatalf("DecodeRefRecord: %v", err) + } + if rec.Sha != tt.want.Sha || rec.Editor != tt.want.Editor || + !slices.Equal(rec.PushOptions, tt.want.PushOptions) { + t.Fatalf("record = %+v, want %+v", rec, tt.want) + } + again, err := encodeRefRecord(tt.want) + if err != nil { + t.Fatalf("encodeRefRecord: %v", err) + } + if !bytes.Equal(again, data) { + t.Errorf("re-encoded record\n got %x\nwant %x", again, data) + } + }) + } +} + +func shaRecord(t *testing.T, reported string) []byte { + t.Helper() + var out bytes.Buffer + if err := cbg.CborWriteHeader(&out, cbg.MajMap, 1); err != nil { + t.Fatalf("map header: %v", err) + } + writeText(&out, "sha") + writeText(&out, reported) + return out.Bytes() +} + +func TestDecodeRefRecordRefusesMalformedRecords(t *testing.T) { + const overlongShaRun = "a2637368617840636463646364636463646364636463646364636463646364636463" + + "64636463646364636463646364636463646364636463646364636463646364636463646364636463" + + "646524747970657273682e74616e676c65642e6769742e726566" + + overlong, err := hex.DecodeString(overlongShaRun) + if err != nil { + t.Fatalf("decoding overlong record: %v", err) + } + var hugeArray bytes.Buffer + if err := cbg.CborWriteHeader(&hugeArray, cbg.MajMap, 1); err != nil { + t.Fatalf("map header: %v", err) + } + writeText(&hugeArray, "x-tngl-push-options") + if err := cbg.CborWriteHeader(&hugeArray, cbg.MajArray, 1<<32); err != nil { + t.Fatalf("array header: %v", err) + } + + refused := [][]byte{{0xa1, 0x63, 'a', 'b', 0x00}, hugeArray.Bytes(), overlong} + for _, reported := range []string{"", "def456", strings.Repeat("ab", 21), strings.Repeat("zz", 20)} { + refused = append(refused, shaRecord(t, reported)) + } + for _, data := range refused { + if _, err := DecodeRefRecord(data); err == nil { + t.Errorf("DecodeRefRecord accepted % x", data) + } + } + + for _, reported := range []string{strings.Repeat("ab", 20), strings.Repeat("AB", 32)} { + if _, err := DecodeRefRecord(shaRecord(t, reported)); err != nil { + t.Errorf("DecodeRefRecord(%q): %v", reported, err) + } + } +} + +func TestEncodeRefRecordRefusesWhatItsOwnDecoderWouldReject(t *testing.T) { + present := sha(t, strings.Repeat("ab", 20)) + huge := strings.Repeat("x", cbg.MaxLength+1) + for _, tt := range []struct { + name string + rec RefRecord + }{ + {"no sha at all", RefRecord{}}, + {"editor past the cbor limit", RefRecord{Sha: present, Editor: syntax.DID(huge)}}, + {"push option past the cbor limit", RefRecord{Sha: present, PushOptions: []string{huge}}}, + } { + if _, err := encodeRefRecord(tt.rec); err == nil { + t.Errorf("%s encoded anyway, and the record would fail its own decode", tt.name) + } + } +} + +func TestEscapeRefnameRulesTheRkey(t *testing.T) { + for _, tt := range []struct { + refname string + want syntax.RecordKey + wantOK bool + }{ + {"refs/heads/main", "refs~2fheads~2fmain", true}, + {"refs/heads/~x", "refs~2fheads~2f~7ex", true}, + {strings.Repeat("/", 79), syntax.RecordKey(strings.Repeat("~2f", 79)), true}, + {strings.Repeat("/", 80), "", false}, + } { + rkey, ok := EscapeRefname(tt.refname) + if ok != tt.wantOK || (ok && rkey != tt.want) { + t.Fatalf("EscapeRefname(%q) = %q, %v", tt.refname, rkey, ok) + } + } +} + +func TestUnescapeRkeyRoundTripsPlainAndEscapedRefnames(t *testing.T) { + for _, refname := range []string{ + "refs/heads/main", + "main", + "refs/tags/v1.0.0+build", + "refs/heads/feature/foo_bar-squid.conch", + "refs/heads/~tilde", + "refs/heads/ branch with spaces ", + "refs/heads/日本語ブランチ", + "refs/heads/emoji-🍀", + } { + rkey, ok := EscapeRefname(refname) + if !ok { + t.Fatalf("EscapeRefname refused %q", refname) + } + decoded, ok := UnescapeRkey(rkey) + if !ok { + t.Fatalf("UnescapeRkey refused %q", rkey) + } + if decoded != refname { + t.Fatalf("round trip of %q answered %q", refname, decoded) + } + } +} + +func TestUnescapeRkeyRefusesMalformedKeys(t *testing.T) { + for _, rkey := range []syntax.RecordKey{ + "refs~2fheads~", + "refs~2fheads~2", + "refs~2fheads~zz", + "refs~2fhea ds", + "refs~ff", + } { + if decoded, ok := UnescapeRkey(rkey); ok { + t.Fatalf("UnescapeRkey(%q) = %q, want refusal", rkey, decoded) + } + } +} diff --git a/knotfeed/subscribe.go b/knotfeed/subscribe.go new file mode 100644 index 0000000000000000000000000000000000000000..75f5e44cafcdc9383c202775205884e8694db538 --- /dev/null +++ b/knotfeed/subscribe.go @@ -0,0 +1,397 @@ +package knotfeed + +import ( + "cmp" + "context" + "errors" + "fmt" + "log/slog" + "net/url" + "slices" + "strconv" + "strings" + "time" + + "github.com/gorilla/websocket" +) + +const ( + livenessTimeout = 90 * time.Second + pongWriteTimeout = 5 * time.Second + initialBackoff = time.Second + maxBackoff = 60 * time.Second + maxFrameBytes = 8 << 20 + subscribeReposNS = "com.atproto.sync.subscribeRepos" + eventsPath = "/events" + + defaultMaxHandlerAttempts = 3 + defaultPoisonGrace = 10 * time.Minute +) + +type Feed struct { + kind uint8 +} + +const ( + atprotoKind uint8 = iota + legacyKind +) + +var ( + FeedAtproto = Feed{kind: atprotoKind} + FeedLegacy = Feed{kind: legacyKind} +) + +type feedSpec struct { + token string + name string + path string + msgKind int + other Feed + decode func([]byte, *slog.Logger) (Message, error) +} + +var feedSpecs = [...]feedSpec{ + atprotoKind: { + token: "atproto", name: "atproto firehose", path: "/xrpc/" + subscribeReposNS, + msgKind: websocket.BinaryMessage, other: FeedLegacy, decode: Decode, + }, + legacyKind: { + token: "legacy", name: "knot event stream", path: eventsPath, + msgKind: websocket.TextMessage, other: FeedAtproto, decode: decodeLegacy, + }, +} + +func (f Feed) spec() feedSpec { return feedSpecs[f.kind] } + +func (f Feed) Token() string { return f.spec().token } + +func (f Feed) String() string { return f.spec().name } + +func (f Feed) other() Feed { return f.spec().other } + +func (f Feed) Live(now time.Time) Cursor { + if f == FeedLegacy { + return NewCursor(f, now.UnixNano()) + } + return NewCursor(f, 0) +} + +var ErrUnrecognizedFeed = errors.New("unrecognized feed token") + +func ParseFeed(token string) (Feed, error) { + kind := slices.IndexFunc(feedSpecs[:], func(spec feedSpec) bool { return spec.token == token }) + if kind < 0 { + return FeedAtproto, fmt.Errorf("%w: %q", ErrUnrecognizedFeed, token) + } + return Feed{kind: uint8(kind)}, nil +} + +type Cursor struct { + feed Feed + seq int64 +} + +func NewCursor(feed Feed, seq int64) Cursor { + return Cursor{feed: feed, seq: seq} +} + +func (c Cursor) Feed() Feed { return c.feed } + +func (c Cursor) Seq() int64 { return c.seq } + +func (c Cursor) Encode() string { + return c.feed.Token() + ":" + strconv.FormatInt(c.seq, 10) +} + +func ParseCursor(raw string) (Cursor, error) { + token, rest, named := strings.Cut(raw, ":") + if !named { + token, rest = FeedAtproto.Token(), raw + } + feed, err := ParseFeed(token) + if err != nil { + return Cursor{}, err + } + seq, err := strconv.ParseInt(rest, 10, 64) + if err != nil { + return Cursor{}, fmt.Errorf("cursor %q: %w", raw, err) + } + return NewCursor(feed, seq), nil +} + +type Consumer struct { + Host string + NoTLS bool + Dialer *websocket.Dialer + Logger *slog.Logger + + LoadCursor func(context.Context) (Cursor, error) + StoreCursor func(context.Context, Cursor) error + Handle func(context.Context, Message) error + OutdatedReplay func(context.Context, Feed) Cursor + OnConnectError func(error) + + ReplayFromStart bool + + MaxHandlerAttempts int + PoisonGrace time.Duration + + now func() time.Time + + current Feed + adopted bool + proven bool + poisonSeq int64 + poisonAttempts int + poisonSince time.Time +} + +func (c *Consumer) Run(ctx context.Context) error { + c.fill() + backoff := initialBackoff + for { + if ctx.Err() != nil { + return nil + } + advanced, resync, err := c.session(ctx) + switch { + case err != nil: + c.reactTo(err) + case resync: + advanced = true + } + if advanced { + backoff = initialBackoff + } else { + backoff = min(backoff*2, maxBackoff) + } + if resync { + continue + } + timer := time.NewTimer(backoff) + select { + case <-ctx.Done(): + timer.Stop() + return nil + case <-timer.C: + } + } +} + +func (c *Consumer) reactTo(err error) { + var mismatch feedMismatch + switch { + case errors.As(err, &mismatch): + case isConnectError(err): + if c.OnConnectError != nil { + c.OnConnectError(err) + } + if c.proven { + c.Logger.Error("knot turned us away on a feed it had already served", + "host", c.Host, "feed", c.current, "err", err) + return + } + default: + c.Logger.Error("session died for its own reasons", + "host", c.Host, "feed", c.current, "err", err) + return + } + c.Logger.Warn("trying the other feed, since this knot won't serve the one we asked for", + "host", c.Host, "tried", c.current, "next", c.current.other(), "err", err) + c.current, c.proven = c.current.other(), false +} + +type connectError struct{ err error } + +func (e connectError) Error() string { return e.err.Error() } + +func (e connectError) Unwrap() error { return e.err } + +func isConnectError(err error) bool { + var ce connectError + return errors.As(err, &ce) +} + +type feedMismatch struct{ feed Feed } + +func (e feedMismatch) Error() string { + return fmt.Sprintf("%s sent the %s's frames", e.feed, e.feed.other()) +} + +func (c *Consumer) session(ctx context.Context) (advanced bool, resync bool, err error) { + stored, err := c.LoadCursor(ctx) + if err != nil { + return false, false, fmt.Errorf("loading cursor: %w", err) + } + if !c.adopted { + c.adopted = true + if stored.Seq() > 0 { + c.current = stored.Feed() + } + } + cursor := c.sessionCursor(stored) + conn, _, err := c.Dialer.DialContext(ctx, c.url(cursor), nil) + if err != nil { + return false, false, connectError{err} + } + c.proven = true + defer conn.Close() + watcher := context.AfterFunc(ctx, func() { conn.Close() }) + defer watcher() + conn.SetReadLimit(maxFrameBytes) + c.Logger.Info("subscribed to knot feed", "host", c.Host, "feed", c.current, "cursor", cursor) + + conn.SetPingHandler(func(payload string) error { + conn.SetReadDeadline(time.Now().Add(livenessTimeout)) + return conn.WriteControl(websocket.PongMessage, []byte(payload), time.Now().Add(pongWriteTimeout)) + }) + conn.SetReadDeadline(time.Now().Add(livenessTimeout)) + want := c.current.spec().msgKind + for { + kind, data, err := conn.ReadMessage() + if err != nil { + return advanced, false, err + } + conn.SetReadDeadline(time.Now().Add(livenessTimeout)) + if kind != want { + return advanced, false, feedMismatch{feed: c.current} + } + message, err := c.decode(data) + if err != nil { + c.Logger.Error("undecodable knot feed frame", "host", c.Host, "feed", c.current, "err", err) + continue + } + resync, err := c.dispatch(ctx, message, &cursor) + if resync { + return advanced, true, nil + } + if err != nil { + return false, false, err + } + if message.Type == TypeCommit { + advanced = true + } + } +} + +func (c *Consumer) decode(data []byte) (Message, error) { + message, err := c.current.spec().decode(data, c.Logger) + message.Feed = c.current + return message, err +} + +func (c *Consumer) dispatch(ctx context.Context, message Message, cursor *Cursor) (bool, error) { + if message.Type == TypeCommit { + return c.commit(ctx, message, cursor) + } + if c.unreachable(message) { + c.Logger.Warn("firehose can't serve our cursor, resuming live", "host", c.Host, "error", c.resumeReason(message)) + *cursor = c.replayCursor(ctx) + if err := c.store(ctx, *cursor); err != nil { + return false, err + } + return true, nil + } + if c.Handle != nil { + if err := c.Handle(ctx, message); err != nil { + c.Logger.Error("firehose handler failed", "host", c.Host, "frame", message.Type, "err", err) + } + } + return false, nil +} + +func (c *Consumer) replayCursor(ctx context.Context) Cursor { + live := c.current.Live(c.now()) + if c.OutdatedReplay == nil { + return live + } + replay := c.OutdatedReplay(ctx, c.current) + if replay.Feed() != c.current { + c.Logger.Warn("replay hook named the other feed, so resuming live instead", + "host", c.Host, "feed", c.current, "replay", replay.Feed()) + return live + } + return replay +} + +func (c *Consumer) commit(ctx context.Context, message Message, cursor *Cursor) (bool, error) { + if c.Handle != nil { + if err := c.Handle(ctx, message); err != nil { + if !c.poison(message.Commit.Seq) { + return false, fmt.Errorf("frame seq %d: %w", message.Commit.Seq, err) + } + c.Logger.Error("handler keeps failing at one frame, skipping it", + "host", c.Host, "seq", message.Commit.Seq, "attempts", c.poisonAttempts, "err", err) + } + } + *cursor = NewCursor(c.current, message.Commit.Seq) + if err := c.store(ctx, *cursor); err != nil { + return false, err + } + return false, nil +} + +func (c *Consumer) store(ctx context.Context, cursor Cursor) error { + if err := c.StoreCursor(ctx, cursor); err != nil { + return fmt.Errorf("storing cursor: %w", err) + } + return nil +} + +func (c *Consumer) poison(seq int64) bool { + if seq != c.poisonSeq { + c.poisonSeq = seq + c.poisonAttempts = 0 + c.poisonSince = c.now() + } + c.poisonAttempts++ + return c.poisonAttempts >= c.MaxHandlerAttempts && c.now().Sub(c.poisonSince) >= c.PoisonGrace +} + +func (c *Consumer) unreachable(message Message) bool { + if message.Type == TypeError { + return message.Error == "FutureCursor" + } + return message.Type == TypeInfo && message.InfoName == "OutdatedCursor" +} + +func (c *Consumer) resumeReason(message Message) string { + if message.Type == TypeError { + return message.Error + } + return message.InfoName +} + +func (c *Consumer) url(cursor Cursor) string { + scheme := "wss" + if c.NoTLS { + scheme = "ws" + } + endpoint := url.URL{Scheme: scheme, Host: c.Host, Path: c.current.spec().path} + if seq := cursor.Seq(); seq > 0 || (seq == 0 && c.ReplayFromStart) { + endpoint.RawQuery = "cursor=" + strconv.FormatInt(seq, 10) + } + return endpoint.String() +} + +func (c *Consumer) sessionCursor(stored Cursor) Cursor { + switch { + case stored.Feed() == c.current && stored.Seq() > 0: + return stored + case c.ReplayFromStart: + return NewCursor(c.current, 0) + default: + return c.current.Live(c.now()) + } +} + +func (c *Consumer) fill() { + c.MaxHandlerAttempts = cmp.Or(c.MaxHandlerAttempts, defaultMaxHandlerAttempts) + c.PoisonGrace = cmp.Or(c.PoisonGrace, defaultPoisonGrace) + c.Dialer = cmp.Or(c.Dialer, websocket.DefaultDialer) + c.Logger = cmp.Or(c.Logger, slog.Default()) + if c.now == nil { + c.now = time.Now + } +} diff --git a/knotfeed/subscribe_test.go b/knotfeed/subscribe_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0080b43c498ed8f94b4dd970231a3c57301dfbea --- /dev/null +++ b/knotfeed/subscribe_test.go @@ -0,0 +1,376 @@ +package knotfeed + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "slices" + "sync/atomic" + "testing" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" + "github.com/gorilla/websocket" +) + +func poisonTestLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +func commitFrameForSeq(t *testing.T, seq int64) []byte { + t.Helper() + var payload bytes.Buffer + evt := comatproto.SyncSubscribeRepos_Commit{ + Repo: "did:plc:scallop", + Seq: seq, + Rev: "3lb2xkw2qrs2j", + Commit: lexutil.LexLink(testCid(t)), + } + if err := evt.MarshalCBOR(&payload); err != nil { + t.Fatalf("MarshalCBOR: %v", err) + } + return append(headerFrame(t, "t", "#commit"), payload.Bytes()...) +} + +func frameOnceServer(t *testing.T, frame []byte) *httptest.Server { + t.Helper() + upgrader := websocket.Upgrader{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + if err := conn.WriteMessage(websocket.BinaryMessage, frame); err != nil { + return + } + _ = conn.WriteControl(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) + conn.Close() + })) + t.Cleanup(srv.Close) + return srv +} + +func poisonConsumer(srv *httptest.Server, cursor *atomic.Int64, stored *atomic.Int64, now *atomic.Int64, grace time.Duration) *Consumer { + return &Consumer{ + Host: srv.Listener.Addr().String(), + NoTLS: true, + Logger: poisonTestLogger(), + + LoadCursor: func(context.Context) (Cursor, error) { + return NewCursor(FeedAtproto, cursor.Load()), nil + }, + StoreCursor: func(_ context.Context, cur Cursor) error { + cursor.Store(cur.Seq()) + stored.Store(cur.Seq()) + return nil + }, + Handle: func(context.Context, Message) error { + return errors.New("boom") + }, + + MaxHandlerAttempts: 3, + PoisonGrace: grace, + now: func() time.Time { + return time.Unix(0, now.Load()) + }, + } +} + +func poisonSequence(t *testing.T, c *Consumer, now, stored *atomic.Int64) []int64 { + t.Helper() + var seen []int64 + for attempt := range 3 { + now.Store(int64(attempt+1) * 6 * int64(time.Minute)) + _, _, _ = c.session(context.Background()) + seen = append(seen, stored.Load()) + } + return seen +} + +func TestPoisonFrameSkipsAfterThreeFailuresAcrossTheGrace(t *testing.T) { + srv := frameOnceServer(t, commitFrameForSeq(t, 100)) + + var cursor, stored, now atomic.Int64 + c := poisonConsumer(srv, &cursor, &stored, &now, defaultPoisonGrace) + + if seen := poisonSequence(t, c, &now, &stored); !slices.Equal(seen, []int64{0, 0, 100}) { + t.Fatalf("cursor stores per attempt = %v, want [0 0 100] once the poison lands", seen) + } +} + +func TestPoisonFrameHoldsWhileFailuresStayInsideTheGrace(t *testing.T) { + srv := frameOnceServer(t, commitFrameForSeq(t, 100)) + + var cursor, stored, now atomic.Int64 + c := poisonConsumer(srv, &cursor, &stored, &now, time.Hour) + + if seen := poisonSequence(t, c, &now, &stored); !slices.Equal(seen, []int64{0, 0, 0}) { + t.Fatalf("cursor stores per attempt = %v, want all zero while inside the grace", seen) + } +} + +type pathLog struct{ paths chan string } + +func (p *pathLog) add(path string) { + select { + case p.paths <- path: + default: + } +} + +func (p *pathLog) take(t *testing.T, n int) []string { + t.Helper() + var seen []string + for range n { + select { + case path := <-p.paths: + seen = append(seen, path) + case <-time.After(10 * time.Second): + t.Fatalf("the consumer stopped after %v", seen) + } + } + return seen +} + +func wsServer(t *testing.T, refuse string, serve func(conn *websocket.Conn, connect int64)) (*httptest.Server, *pathLog) { + t.Helper() + upgrader := websocket.Upgrader{} + log := &pathLog{paths: make(chan string, 8)} + var connects atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.add(r.URL.Path) + if r.URL.Path == refuse { + w.WriteHeader(http.StatusNotFound) + return + } + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + serve(conn, connects.Add(1)) + _ = conn.WriteControl(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) + })) + t.Cleanup(srv.Close) + return srv, log +} + +func feedConsumer(t *testing.T, srv *httptest.Server, stored Cursor) *Consumer { + t.Helper() + c := &Consumer{ + Host: srv.Listener.Addr().String(), + NoTLS: true, + Logger: poisonTestLogger(), + LoadCursor: func(context.Context) (Cursor, error) { return stored, nil }, + StoreCursor: func(context.Context, Cursor) error { return nil }, + Handle: func(context.Context, Message) error { return nil }, + } + c.fill() + return c +} + +func runConsumer(t *testing.T, c *Consumer) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = c.Run(ctx) + }() + t.Cleanup(func() { + cancel() + <-done + }) +} + +func TestWrongKindFrameEndsSessionAsMismatch(t *testing.T) { + srv, _ := wsServer(t, "", func(conn *websocket.Conn, _ int64) { + _ = conn.WriteMessage(websocket.TextMessage, []byte(`{"nsid":"x","created":1}`)) + }) + c := feedConsumer(t, srv, Cursor{}) + + _, _, err := c.session(context.Background()) + + var mismatch feedMismatch + if !errors.As(err, &mismatch) { + t.Fatalf("session error = %v, want a feed mismatch", err) + } + if mismatch.feed != FeedAtproto { + t.Errorf("mismatch = %v, want the firehose we asked for", mismatch.feed) + } +} + +func TestMismatchTextSpellsOutBothFeeds(t *testing.T) { + got := feedMismatch{feed: FeedAtproto}.Error() + if got != "atproto firehose sent the knot event stream's frames" { + t.Errorf("mismatch reads %q", got) + } +} + +func TestConsumerLeavesFeedOnlyWhenKnotSaysTo(t *testing.T) { + for _, tt := range []struct { + name string + proven bool + err error + want Feed + }{ + {"mismatch on proven feed", true, feedMismatch{feed: FeedAtproto}, FeedLegacy}, + {"refused connection on unproven feed", false, connectError{errors.New("no route")}, FeedLegacy}, + {"refused connection on proven feed", true, connectError{errors.New("connection reset")}, FeedAtproto}, + {"session that failed for its own reasons", true, errors.New("boom"), FeedAtproto}, + } { + t.Run(tt.name, func(t *testing.T) { + c := &Consumer{Logger: poisonTestLogger()} + c.fill() + c.proven = tt.proven + + c.reactTo(tt.err) + + if c.current != tt.want { + t.Errorf("feed = %v, want %v", c.current, tt.want) + } + if switched := tt.want != FeedAtproto; switched && c.proven { + t.Error("new feed counts as proven before delivering anything") + } + }) + } +} + +func TestQuietKnotStillProvesFeedItAccepted(t *testing.T) { + srv, _ := wsServer(t, "", func(*websocket.Conn, int64) {}) + c := feedConsumer(t, srv, NewCursor(FeedLegacy, 11)) + + advanced, _, err := c.session(context.Background()) + + if advanced { + t.Error("empty session reported progress") + } + if err == nil { + t.Fatal("closed connection didn't error out") + } + if !c.proven { + t.Fatal("knot accepted the upgrade, and the feed still reads unproven") + } + + c.reactTo(connectError{errors.New("connection refused")}) + if c.current != FeedLegacy { + t.Errorf("feed = %v; we left a path the knot had already served after one refusal", c.current) + } +} + +func TestSwitchStaysUnprovenUntilNewFeedDelivers(t *testing.T) { + srv, paths := wsServer(t, eventsPath, func(conn *websocket.Conn, _ int64) { + _ = conn.WriteMessage(websocket.BinaryMessage, commitFrameForSeq(t, 500)) + _ = conn.WriteMessage(websocket.TextMessage, []byte(`{"nsid":"x","created":1}`)) + time.Sleep(time.Second) + }) + runConsumer(t, feedConsumer(t, srv, Cursor{})) + + tried := paths.take(t, 3) + + if tried[1] != eventsPath { + t.Fatalf("stray text frame sent us to %q, want the legacy path", tried[1]) + } + if tried[2] == eventsPath { + t.Error("we retried the refused legacy path, so the switch counted as proven too early") + } +} + +func TestStoredFeedIsAdoptedOnceAndNeverUndoesSwitch(t *testing.T) { + c := &Consumer{ + Host: "127.0.0.1:1", + NoTLS: true, + Logger: poisonTestLogger(), + LoadCursor: func(context.Context) (Cursor, error) { + return NewCursor(FeedAtproto, 500), nil + }, + StoreCursor: func(context.Context, Cursor) error { return nil }, + } + c.fill() + c.current = FeedLegacy + + _, _, err := c.session(context.Background()) + if c.current != FeedAtproto { + t.Fatalf("feed = %v, want the stored cursor's feed on the first session", c.current) + } + c.reactTo(err) + if c.current != FeedLegacy { + t.Fatalf("feed = %v, want a switch after the refused connection", c.current) + } + + for attempt := range 3 { + if _, _, _ = c.session(context.Background()); c.current != FeedLegacy { + t.Fatalf("session %d re-adopted the stored feed %v", attempt+2, c.current) + } + } +} + +func TestSubscriptionAsksForItsOwnFeedAndPosition(t *testing.T) { + frozen := time.Unix(0, 1788257313450261754) + const legacySeq = 1788245839553422000 + live := fmt.Sprint(frozen.UnixNano()) + for _, tt := range []struct { + name string + noTLS bool + feed Feed + replay bool + stored Cursor + want string + }{ + {"firehose position on the firehose", true, FeedAtproto, false, NewCursor(FeedAtproto, 42), + "ws://oyster.cafe/xrpc/com.atproto.sync.subscribeRepos?cursor=42"}, + {"nothing stored on the firehose", true, FeedAtproto, false, Cursor{}, + "ws://oyster.cafe/xrpc/com.atproto.sync.subscribeRepos"}, + {"legacy position on the legacy feed", false, FeedLegacy, false, NewCursor(FeedLegacy, legacySeq), + "wss://oyster.cafe/events?cursor=" + fmt.Sprint(legacySeq)}, + {"firehose position on the legacy feed", false, FeedLegacy, false, NewCursor(FeedAtproto, 42), + "wss://oyster.cafe/events?cursor=" + live}, + {"legacy position on the firehose", false, FeedAtproto, false, NewCursor(FeedLegacy, legacySeq), + "wss://oyster.cafe/xrpc/com.atproto.sync.subscribeRepos"}, + {"nothing stored on the legacy feed", false, FeedLegacy, false, Cursor{}, + "wss://oyster.cafe/events?cursor=" + live}, + {"nothing stored while replaying from the start", false, FeedLegacy, true, Cursor{}, + "wss://oyster.cafe/events?cursor=0"}, + } { + c := &Consumer{Host: "oyster.cafe", NoTLS: tt.noTLS, ReplayFromStart: tt.replay} + c.fill() + c.current = tt.feed + c.now = func() time.Time { return frozen } + if got := c.url(c.sessionCursor(tt.stored)); got != tt.want { + t.Errorf("%s asks for %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestEveryFeedAndCursorRoundTripsItsText(t *testing.T) { + for _, feed := range []Feed{FeedAtproto, FeedLegacy} { + if parsed, err := ParseFeed(feed.Token()); err != nil || parsed != feed { + t.Errorf("ParseFeed(%q) = (%v, %v), want %v", feed.Token(), parsed, err, feed) + } + } + for _, want := range []Cursor{NewCursor(FeedAtproto, 500), NewCursor(FeedLegacy, 1788245839553422000), {}} { + got, err := ParseCursor(want.Encode()) + if err != nil || got != want { + t.Errorf("ParseCursor(%q) = (%v, %v), want %v", want.Encode(), got, err, want) + } + } + if got, err := ParseCursor("500"); err != nil || got != NewCursor(FeedAtproto, 500) { + t.Errorf("bare sequence parsed as (%v, %v), want the firehose at 500", got, err) + } + for _, raw := range []string{"carrier pigeon:7", "legacy:soon"} { + if _, err := ParseCursor(raw); err == nil { + t.Errorf("ParseCursor(%q) parsed anyway", raw) + } + } + if _, err := ParseFeed("carrier pigeon"); !errors.Is(err, ErrUnrecognizedFeed) { + t.Errorf("unrecognized token gave %v, want something callers can classify", err) + } +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-27286d65426e3f215744767b774bede293bb1e45b7a6a1aec0af40a729cf89d0.json b/knotmirror/crates/knotmirror-db/.sqlx/query-27286d65426e3f215744767b774bede293bb1e45b7a6a1aec0af40a729cf89d0.json new file mode 100644 index 0000000000000000000000000000000000000000..934db02e215537e58c60c4baa12f97f7c012ab8f --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-27286d65426e3f215744767b774bede293bb1e45b7a6a1aec0af40a729cf89d0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "insert into hydrant_cursor (id, last_event_id)\n values (0, $1)\n on conflict (id) do update set last_event_id = excluded.last_event_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "27286d65426e3f215744767b774bede293bb1e45b7a6a1aec0af40a729cf89d0" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-27487b1b800decd26cb5a1500711c474657fa4a1f7899135a690b98cc59a6f97.json b/knotmirror/crates/knotmirror-db/.sqlx/query-27487b1b800decd26cb5a1500711c474657fa4a1f7899135a690b98cc59a6f97.json new file mode 100644 index 0000000000000000000000000000000000000000..3067de0bcc44f754ce05b352a2cefed4ae51f26d --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-27487b1b800decd26cb5a1500711c474657fa4a1f7899135a690b98cc59a6f97.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "insert into repos (did, refs_cid, default_branch, refs, deleted_at)\n values ($1, $2, $3, $4, null)\n on conflict (did) do update set\n refs_cid = excluded.refs_cid,\n default_branch = excluded.default_branch,\n refs = excluded.refs,\n deleted_at = null,\n state = 'pending',\n error_msg = null,\n retry_count = 0,\n retry_after_us = null,\n updated_at = now()\n where repos.refs_cid is distinct from excluded.refs_cid\n or repos.deleted_at is not null", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Bytea" + ] + }, + "nullable": [] + }, + "hash": "27487b1b800decd26cb5a1500711c474657fa4a1f7899135a690b98cc59a6f97" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-4639c2d4a3d29f03f9145d7352eb9d9751d1ffec1101bf42196c52fff6047a5a.json b/knotmirror/crates/knotmirror-db/.sqlx/query-4639c2d4a3d29f03f9145d7352eb9d9751d1ffec1101bf42196c52fff6047a5a.json new file mode 100644 index 0000000000000000000000000000000000000000..73e39111b5b38f326b87323b551b8735bc184128 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-4639c2d4a3d29f03f9145d7352eb9d9751d1ffec1101bf42196c52fff6047a5a.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "update repos set retry_after_us = $3, updated_at = now()\n where did = $1 and refs_cid = $2 and atproto_rev is not distinct from $4\n and deleted_at is null", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4639c2d4a3d29f03f9145d7352eb9d9751d1ffec1101bf42196c52fff6047a5a" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-48bc0c0d369583d5cce9f45760338feedf8c3f714eabe06dc1e2aba3dddb0c62.json b/knotmirror/crates/knotmirror-db/.sqlx/query-48bc0c0d369583d5cce9f45760338feedf8c3f714eabe06dc1e2aba3dddb0c62.json new file mode 100644 index 0000000000000000000000000000000000000000..8063524f59ccf646189145c1f4ae8b898913f972 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-48bc0c0d369583d5cce9f45760338feedf8c3f714eabe06dc1e2aba3dddb0c62.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "update repos\n set state = 'error', error_msg = $3, retry_count = retry_count + 1,\n retry_after_us = $4, updated_at = now()\n where did = $1 and refs_cid = $2 and atproto_rev is not distinct from $5\n and deleted_at is null", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "48bc0c0d369583d5cce9f45760338feedf8c3f714eabe06dc1e2aba3dddb0c62" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-593e82adb20973a2ddd9e853e731f5c01b8576aa4b8782b97c20aeed78009328.json b/knotmirror/crates/knotmirror-db/.sqlx/query-593e82adb20973a2ddd9e853e731f5c01b8576aa4b8782b97c20aeed78009328.json new file mode 100644 index 0000000000000000000000000000000000000000..a0bee7b1097f2318bff0bad8fb4a372f2b3f0cbb --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-593e82adb20973a2ddd9e853e731f5c01b8576aa4b8782b97c20aeed78009328.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "select did from repos where deleted_at is null and state = 'active' order by did", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "did", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "did" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "593e82adb20973a2ddd9e853e731f5c01b8576aa4b8782b97c20aeed78009328" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-63518fc08ee86cdaa6a881183578d5665152b7431776b565fa9ed9324090a78d.json b/knotmirror/crates/knotmirror-db/.sqlx/query-63518fc08ee86cdaa6a881183578d5665152b7431776b565fa9ed9324090a78d.json new file mode 100644 index 0000000000000000000000000000000000000000..4bb75e2e895d83056cffa842d264d06dece49608 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-63518fc08ee86cdaa6a881183578d5665152b7431776b565fa9ed9324090a78d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "insert into repo_languages (did, language, size)\n select $1, * from unnest($2::text[], $3::bigint[])", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "63518fc08ee86cdaa6a881183578d5665152b7431776b565fa9ed9324090a78d" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-66df0c0bb00794d4fabfc89d3ca20a0367e03cd610a742b878b416eeb9a3f561.json b/knotmirror/crates/knotmirror-db/.sqlx/query-66df0c0bb00794d4fabfc89d3ca20a0367e03cd610a742b878b416eeb9a3f561.json new file mode 100644 index 0000000000000000000000000000000000000000..1eba488e16ff30897da395700d943bb1fef4b5c6 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-66df0c0bb00794d4fabfc89d3ca20a0367e03cd610a742b878b416eeb9a3f561.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "select did, default_branch, default_branch_commit, index_retry_count\n from repos\n where deleted_at is null\n and state = 'active'\n and default_branch_commit is distinct from indexed_commit\n and (index_retry_after_us is null or index_retry_after_us <= $1)\n order by index_retry_after_us nulls first, updated_at\n limit $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "did", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "did" + } + } + }, + { + "ordinal": 1, + "name": "default_branch", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "default_branch" + } + } + }, + { + "ordinal": 2, + "name": "default_branch_commit", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "default_branch_commit" + } + } + }, + { + "ordinal": 3, + "name": "index_retry_count", + "type_info": "Int4", + "origin": { + "Table": { + "table": "repos", + "name": "index_retry_count" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false + ] + }, + "hash": "66df0c0bb00794d4fabfc89d3ca20a0367e03cd610a742b878b416eeb9a3f561" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-6a8ebbb538b3254515d113c27135f20cb2a40f8a2f3c89b0a07115caf920c876.json b/knotmirror/crates/knotmirror-db/.sqlx/query-6a8ebbb538b3254515d113c27135f20cb2a40f8a2f3c89b0a07115caf920c876.json new file mode 100644 index 0000000000000000000000000000000000000000..67e34aab3390707c78abac7f0844f942ddb9e95f --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-6a8ebbb538b3254515d113c27135f20cb2a40f8a2f3c89b0a07115caf920c876.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "select case when state = 'error' and retry_count >= $2 then 'error'\n when state = 'error' then 'backoff'\n else state end as \"state!\",\n (state <> 'active' and (retry_after_us is null or retry_after_us <= $1)) as \"due!\",\n count(*) as \"count!\"\n from repos\n where deleted_at is null\n group by 1, 2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "state!", + "type_info": "Text", + "origin": "Expression" + }, + { + "ordinal": 1, + "name": "due!", + "type_info": "Bool", + "origin": "Expression" + }, + { + "ordinal": 2, + "name": "count!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int4" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "6a8ebbb538b3254515d113c27135f20cb2a40f8a2f3c89b0a07115caf920c876" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-77c15f179c796a3a8d1d6e3049797238098885401886cc6aab8d386405c1370c.json b/knotmirror/crates/knotmirror-db/.sqlx/query-77c15f179c796a3a8d1d6e3049797238098885401886cc6aab8d386405c1370c.json new file mode 100644 index 0000000000000000000000000000000000000000..f399f70b044c35a9c1be8826531a26902d9ce09b --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-77c15f179c796a3a8d1d6e3049797238098885401886cc6aab8d386405c1370c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "delete from commit_counts where month < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Date" + ] + }, + "nullable": [] + }, + "hash": "77c15f179c796a3a8d1d6e3049797238098885401886cc6aab8d386405c1370c" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-7ff739bd0a1b556db369e2948df9e61a7c74c77ae3418ea43a9bc929d38f324e.json b/knotmirror/crates/knotmirror-db/.sqlx/query-7ff739bd0a1b556db369e2948df9e61a7c74c77ae3418ea43a9bc929d38f324e.json new file mode 100644 index 0000000000000000000000000000000000000000..35c8da8b3942bd9b4c1325dd32094ddb495036ec --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-7ff739bd0a1b556db369e2948df9e61a7c74c77ae3418ea43a9bc929d38f324e.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "delete from repo_languages where did = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "7ff739bd0a1b556db369e2948df9e61a7c74c77ae3418ea43a9bc929d38f324e" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-886c1448ba85e0aba97c0017c24120962bc21fe3a3303e88bcb7d071d817c253.json b/knotmirror/crates/knotmirror-db/.sqlx/query-886c1448ba85e0aba97c0017c24120962bc21fe3a3303e88bcb7d071d817c253.json new file mode 100644 index 0000000000000000000000000000000000000000..66f4ff7756285226b63ec1a95c9e1ef8988aa7d6 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-886c1448ba85e0aba97c0017c24120962bc21fe3a3303e88bcb7d071d817c253.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "update repos\n set activity_synced_at = $2, activity_tips = $3, activity_error = null,\n activity_retry_count = 0, activity_retry_after_us = null\n where did = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Timestamptz", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "886c1448ba85e0aba97c0017c24120962bc21fe3a3303e88bcb7d071d817c253" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-93000db396c7146e57bd7d7e60eba285f3a15b97646b005ef28a2975217ba20a.json b/knotmirror/crates/knotmirror-db/.sqlx/query-93000db396c7146e57bd7d7e60eba285f3a15b97646b005ef28a2975217ba20a.json new file mode 100644 index 0000000000000000000000000000000000000000..49aa78b31873db07f8ddd9b26f5d37bd7b5b717e --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-93000db396c7146e57bd7d7e60eba285f3a15b97646b005ef28a2975217ba20a.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "select 1 from repos where did = $1 and indexed_commit = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "93000db396c7146e57bd7d7e60eba285f3a15b97646b005ef28a2975217ba20a" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-ab795867f17041560dff32d76690dac22678ede2b8949cd557b6ce704d79b0b7.json b/knotmirror/crates/knotmirror-db/.sqlx/query-ab795867f17041560dff32d76690dac22678ede2b8949cd557b6ce704d79b0b7.json new file mode 100644 index 0000000000000000000000000000000000000000..daa6559a24321b027a8da588fa262f1aa547504d --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-ab795867f17041560dff32d76690dac22678ede2b8949cd557b6ce704d79b0b7.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "select last_event_id from hydrant_cursor where id = 0", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_event_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "hydrant_cursor", + "name": "last_event_id" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "ab795867f17041560dff32d76690dac22678ede2b8949cd557b6ce704d79b0b7" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-b2e6317ee0c6d9ef282e9d02c5b6b42d7b656676e2465608ff0618b702735270.json b/knotmirror/crates/knotmirror-db/.sqlx/query-b2e6317ee0c6d9ef282e9d02c5b6b42d7b656676e2465608ff0618b702735270.json new file mode 100644 index 0000000000000000000000000000000000000000..a976214ba151806e332c4d4069dc3bba4155058d --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-b2e6317ee0c6d9ef282e9d02c5b6b42d7b656676e2465608ff0618b702735270.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "update repos\n set state = 'active', error_msg = null, retry_count = 0, retry_after_us = null,\n default_branch_commit = $3, updated_at = now()\n where did = $1 and refs_cid = $2 and atproto_rev is not distinct from $4\n and deleted_at is null", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b2e6317ee0c6d9ef282e9d02c5b6b42d7b656676e2465608ff0618b702735270" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-bdcad83ba694785812ced85914595227d765603eb5299393e1ba39af09222fa7.json b/knotmirror/crates/knotmirror-db/.sqlx/query-bdcad83ba694785812ced85914595227d765603eb5299393e1ba39af09222fa7.json new file mode 100644 index 0000000000000000000000000000000000000000..e3be41c088784930014834d549a0c761b2ba895f --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-bdcad83ba694785812ced85914595227d765603eb5299393e1ba39af09222fa7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "update repos\n set atproto_rev = $2, state = 'pending', error_msg = null,\n retry_count = 0, retry_after_us = null, updated_at = now()\n where did = $1 and deleted_at is null\n and (atproto_rev is null or atproto_rev < $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bdcad83ba694785812ced85914595227d765603eb5299393e1ba39af09222fa7" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-bebe7c55b9010d9e1f1aa5daef9e291b13885004da7d0c98201c98e98e805649.json b/knotmirror/crates/knotmirror-db/.sqlx/query-bebe7c55b9010d9e1f1aa5daef9e291b13885004da7d0c98201c98e98e805649.json new file mode 100644 index 0000000000000000000000000000000000000000..99ea12c382344a573227cc7dcf97beef841f1edf --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-bebe7c55b9010d9e1f1aa5daef9e291b13885004da7d0c98201c98e98e805649.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "update repos set deleted_at = now(), updated_at = now()\n where did = $1 and deleted_at is null", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "bebe7c55b9010d9e1f1aa5daef9e291b13885004da7d0c98201c98e98e805649" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-c0899b71414aaa2fb7885289cb34f120f79ba4de8b635035ea61883ee1ff4595.json b/knotmirror/crates/knotmirror-db/.sqlx/query-c0899b71414aaa2fb7885289cb34f120f79ba4de8b635035ea61883ee1ff4595.json new file mode 100644 index 0000000000000000000000000000000000000000..e1cb984f7dc9da1129f9c0f887f446d85d2cb978 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-c0899b71414aaa2fb7885289cb34f120f79ba4de8b635035ea61883ee1ff4595.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "update repos\n set activity_error = $2, activity_retry_count = activity_retry_count + 1,\n activity_retry_after_us = $3\n where did = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "c0899b71414aaa2fb7885289cb34f120f79ba4de8b635035ea61883ee1ff4595" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-c15b02adc2ecf9a6f65e30af9ac43c3d663f73924e887886e15aa971c79f5053.json b/knotmirror/crates/knotmirror-db/.sqlx/query-c15b02adc2ecf9a6f65e30af9ac43c3d663f73924e887886e15aa971c79f5053.json new file mode 100644 index 0000000000000000000000000000000000000000..3c7f124fff21261d558a85672f0f27e30ed6c5d7 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-c15b02adc2ecf9a6f65e30af9ac43c3d663f73924e887886e15aa971c79f5053.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "with input as (\n select distinct on (oid) oid, committer, month\n from unnest($1::bytea[], $2::text[], $3::date[]) as t(oid, committer, month)\n ), inserted as (\n insert into seen_commits (oid, month)\n select oid, month from input\n on conflict (oid) do nothing\n returning oid\n ), counted as (\n insert into commit_counts (committer, month, commits)\n select committer, month, count(*)\n from input join inserted using (oid)\n group by committer, month\n on conflict (committer, month) do update\n set commits = commit_counts.commits + excluded.commits\n )\n select count(*) as \"count!\" from inserted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "ByteaArray", + "TextArray", + "DateArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c15b02adc2ecf9a6f65e30af9ac43c3d663f73924e887886e15aa971c79f5053" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-c467065187d676ef4fb8e9d3bdbb6b0a7a33ecf56143fa25e76b2e27df7de3c2.json b/knotmirror/crates/knotmirror-db/.sqlx/query-c467065187d676ef4fb8e9d3bdbb6b0a7a33ecf56143fa25e76b2e27df7de3c2.json new file mode 100644 index 0000000000000000000000000000000000000000..9d09d4262f7e2a749c2de4a8ba9cddff0d25b859 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-c467065187d676ef4fb8e9d3bdbb6b0a7a33ecf56143fa25e76b2e27df7de3c2.json @@ -0,0 +1,89 @@ +{ + "db_name": "PostgreSQL", + "query": "select did, refs_cid, default_branch, refs, atproto_rev, retry_count\n from repos\n where deleted_at is null\n and state <> 'active'\n and (retry_after_us is null or retry_after_us <= $1)\n order by retry_after_us nulls first, updated_at\n limit $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "did", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "did" + } + } + }, + { + "ordinal": 1, + "name": "refs_cid", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "refs_cid" + } + } + }, + { + "ordinal": 2, + "name": "default_branch", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "default_branch" + } + } + }, + { + "ordinal": 3, + "name": "refs", + "type_info": "Bytea", + "origin": { + "Table": { + "table": "repos", + "name": "refs" + } + } + }, + { + "ordinal": 4, + "name": "atproto_rev", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "atproto_rev" + } + } + }, + { + "ordinal": 5, + "name": "retry_count", + "type_info": "Int4", + "origin": { + "Table": { + "table": "repos", + "name": "retry_count" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + false + ] + }, + "hash": "c467065187d676ef4fb8e9d3bdbb6b0a7a33ecf56143fa25e76b2e27df7de3c2" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-c95a6e06127c35f49b322512d19e6af8677050b0984519f7150e3b755511768f.json b/knotmirror/crates/knotmirror-db/.sqlx/query-c95a6e06127c35f49b322512d19e6af8677050b0984519f7150e3b755511768f.json new file mode 100644 index 0000000000000000000000000000000000000000..98f77c67166ada7f7a6cfc61917d41528fe1ac06 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-c95a6e06127c35f49b322512d19e6af8677050b0984519f7150e3b755511768f.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "select language, size from repo_languages where did = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "language", + "type_info": "Text", + "origin": { + "Table": { + "table": "repo_languages", + "name": "language" + } + } + }, + { + "ordinal": 1, + "name": "size", + "type_info": "Int8", + "origin": { + "Table": { + "table": "repo_languages", + "name": "size" + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "c95a6e06127c35f49b322512d19e6af8677050b0984519f7150e3b755511768f" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-d2272d8970ee58876b35266ab2ba3997759e6abe4478ea7020bbe9cf133168bc.json b/knotmirror/crates/knotmirror-db/.sqlx/query-d2272d8970ee58876b35266ab2ba3997759e6abe4478ea7020bbe9cf133168bc.json new file mode 100644 index 0000000000000000000000000000000000000000..adaca06571d634f3965e01db35bf2196783de7c2 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-d2272d8970ee58876b35266ab2ba3997759e6abe4478ea7020bbe9cf133168bc.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "select did, updated_at, activity_tips, activity_retry_count\n from repos\n where deleted_at is null\n and state = 'active'\n and activity_synced_at is distinct from updated_at\n and (activity_retry_after_us is null or activity_retry_after_us <= $1)\n order by activity_retry_after_us nulls first, updated_at\n limit $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "did", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "did" + } + } + }, + { + "ordinal": 1, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "repos", + "name": "updated_at" + } + } + }, + { + "ordinal": 2, + "name": "activity_tips", + "type_info": "TextArray", + "origin": { + "Table": { + "table": "repos", + "name": "activity_tips" + } + } + }, + { + "ordinal": 3, + "name": "activity_retry_count", + "type_info": "Int4", + "origin": { + "Table": { + "table": "repos", + "name": "activity_retry_count" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "d2272d8970ee58876b35266ab2ba3997759e6abe4478ea7020bbe9cf133168bc" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-d947bb488a5ec5f7f30a17d4faf90746f85e8fc0d88ed641ec9f901ad3fcacfa.json b/knotmirror/crates/knotmirror-db/.sqlx/query-d947bb488a5ec5f7f30a17d4faf90746f85e8fc0d88ed641ec9f901ad3fcacfa.json new file mode 100644 index 0000000000000000000000000000000000000000..86d4bf4fbb0a7b51473be576b6545b0ef064c4e2 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-d947bb488a5ec5f7f30a17d4faf90746f85e8fc0d88ed641ec9f901ad3fcacfa.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "delete from seen_commits where month < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Date" + ] + }, + "nullable": [] + }, + "hash": "d947bb488a5ec5f7f30a17d4faf90746f85e8fc0d88ed641ec9f901ad3fcacfa" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-e680cd589b5830abdbf1eaad8be5f0ea4a5d3323784cd4f261efb99aa02679fc.json b/knotmirror/crates/knotmirror-db/.sqlx/query-e680cd589b5830abdbf1eaad8be5f0ea4a5d3323784cd4f261efb99aa02679fc.json new file mode 100644 index 0000000000000000000000000000000000000000..20d1ae9da57579c767cc32e48f81afcca8701dd6 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-e680cd589b5830abdbf1eaad8be5f0ea4a5d3323784cd4f261efb99aa02679fc.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "select month, sum(commits)::bigint as \"commits!\"\n from commit_counts\n where committer = any($1) and month >= $2 and month < $3\n group by month\n order by month desc\n limit $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "month", + "type_info": "Date", + "origin": { + "Table": { + "table": "commit_counts", + "name": "month" + } + } + }, + { + "ordinal": 1, + "name": "commits!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Date", + "Date", + "Int8" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "e680cd589b5830abdbf1eaad8be5f0ea4a5d3323784cd4f261efb99aa02679fc" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-ee3e927e80d0f3051ed3d2de99c0891856550ef9da693a41d18cd62189aad307.json b/knotmirror/crates/knotmirror-db/.sqlx/query-ee3e927e80d0f3051ed3d2de99c0891856550ef9da693a41d18cd62189aad307.json new file mode 100644 index 0000000000000000000000000000000000000000..8a12b47e2c03a0f83b5e4c35ce4550ace93c249e --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-ee3e927e80d0f3051ed3d2de99c0891856550ef9da693a41d18cd62189aad307.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "update repos\n set indexed_commit = $2, index_error = null, index_retry_count = 0,\n index_retry_after_us = null\n where did = $1 and default_branch_commit is not distinct from $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ee3e927e80d0f3051ed3d2de99c0891856550ef9da693a41d18cd62189aad307" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-ef69831ad84fc5dc6989cd661cfab230c726baf267f68f625aa1cdbcba5bd312.json b/knotmirror/crates/knotmirror-db/.sqlx/query-ef69831ad84fc5dc6989cd661cfab230c726baf267f68f625aa1cdbcba5bd312.json new file mode 100644 index 0000000000000000000000000000000000000000..b5803eae0fdc0a235d2bb4c95a8fcbab4aaed290 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-ef69831ad84fc5dc6989cd661cfab230c726baf267f68f625aa1cdbcba5bd312.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "update repos\n set index_error = $3, index_retry_count = index_retry_count + 1,\n index_retry_after_us = $4\n where did = $1 and default_branch_commit is not distinct from $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "ef69831ad84fc5dc6989cd661cfab230c726baf267f68f625aa1cdbcba5bd312" +} diff --git a/knotmirror/crates/knotmirror-db/.sqlx/query-fba3815aee0953d64337931438e6f9266ffce82420c0e8b00c8e8b1e863aa938.json b/knotmirror/crates/knotmirror-db/.sqlx/query-fba3815aee0953d64337931438e6f9266ffce82420c0e8b00c8e8b1e863aa938.json new file mode 100644 index 0000000000000000000000000000000000000000..1fef2aea219d58e3fd8cf705f471eb3602a2d078 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/.sqlx/query-fba3815aee0953d64337931438e6f9266ffce82420c0e8b00c8e8b1e863aa938.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "select r.did,\n r.indexed_commit as \"commit!\",\n coalesce(array_agg(l.language order by l.size desc, l.language)\n filter (where l.language is not null), '{}') as \"names!\",\n coalesce(array_agg(l.size order by l.size desc, l.language)\n filter (where l.language is not null), '{}') as \"sizes!\"\n from repos r\n left join repo_languages l on l.did = r.did\n where r.deleted_at is null\n and r.indexed_commit is not null\n and ($1::text is null or r.did > $1)\n group by r.did\n order by r.did\n limit $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "did", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "did" + } + } + }, + { + "ordinal": 1, + "name": "commit!", + "type_info": "Text", + "origin": { + "Table": { + "table": "repos", + "name": "indexed_commit" + } + } + }, + { + "ordinal": 2, + "name": "names!", + "type_info": "TextArray", + "origin": "Expression" + }, + { + "ordinal": 3, + "name": "sizes!", + "type_info": "Int8Array", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + false, + true, + null, + null + ] + }, + "hash": "fba3815aee0953d64337931438e6f9266ffce82420c0e8b00c8e8b1e863aa938" +} diff --git a/knotmirror/crates/knotmirror-db/Cargo.toml b/knotmirror/crates/knotmirror-db/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..59f0a185c1df09167eae55b2a57146a72a4403a6 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "knotmirror-db" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +chrono = { workspace = true } +jacquard-common = { workspace = true } +sqlx = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +url = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } + +[lints] +workspace = true diff --git a/knotmirror/crates/knotmirror-db/migrations/20260916000000_initial.sql b/knotmirror/crates/knotmirror-db/migrations/20260916000000_initial.sql new file mode 100644 index 0000000000000000000000000000000000000000..32d72929429b2b3be7a1ec0ebb7cb23de5518a57 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/migrations/20260916000000_initial.sql @@ -0,0 +1,29 @@ +create table if not exists hydrant_cursor ( + id smallint primary key check (id = 0), + last_event_id bigint not null +); + +create table if not exists repos ( + -- at:///org.tangled.git.refs/self + did text primary key, + refs_cid text not null, + default_branch text not null, + refs bytea not null, + + -- git sync state + state text not null default 'pending', + error_msg text, + retry_count integer not null default 0, + retry_after_us bigint, + + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + -- set when the record is deleted or the account goes inactive + deleted_at timestamptz +); + +create index if not exists idx_repos_updated_at on repos (updated_at desc); + +create index if not exists idx_repos_sync_queue + on repos (retry_after_us nulls first, updated_at) + where deleted_at is null and state <> 'active'; diff --git a/knotmirror/crates/knotmirror-db/migrations/20260926000000_index.sql b/knotmirror/crates/knotmirror-db/migrations/20260926000000_index.sql new file mode 100644 index 0000000000000000000000000000000000000000..bcbaccedfe59731a4633f4e073a2265e19cce36f --- /dev/null +++ b/knotmirror/crates/knotmirror-db/migrations/20260926000000_index.sql @@ -0,0 +1,18 @@ +alter table repos + add column default_branch_commit text, + add column indexed_commit text, + add column index_error text, + add column index_retry_count integer not null default 0, + add column index_retry_after_us bigint; + +create index if not exists idx_repos_index_queue + on repos (index_retry_after_us nulls first, updated_at) + where deleted_at is null and state = 'active' + and default_branch_commit is distinct from indexed_commit; + +create table if not exists repo_languages ( + did text not null references repos (did), + language text not null, + size bigint not null, + primary key (did, language) +); diff --git a/knotmirror/crates/knotmirror-db/migrations/20261019000000_atproto_rev.sql b/knotmirror/crates/knotmirror-db/migrations/20261019000000_atproto_rev.sql new file mode 100644 index 0000000000000000000000000000000000000000..ffd6c335a71414b405a04a7d22bfad7736e43fb9 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/migrations/20261019000000_atproto_rev.sql @@ -0,0 +1 @@ +alter table repos add column atproto_rev text; diff --git a/knotmirror/crates/knotmirror-db/migrations/20261029000000_commit_activity.sql b/knotmirror/crates/knotmirror-db/migrations/20261029000000_commit_activity.sql new file mode 100644 index 0000000000000000000000000000000000000000..1b5dc5b60772bb50f099111776953a9f47284c77 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/migrations/20261029000000_commit_activity.sql @@ -0,0 +1,27 @@ +alter table repos + add column activity_synced_at timestamptz, + add column activity_tips text[] not null default '{}', + add column activity_error text, + add column activity_retry_count integer not null default 0, + add column activity_retry_after_us bigint; + +-- only mark_synced moves updated_at on an active repo, and a null activity_synced_at is the backfill +create index if not exists idx_repos_activity_queue + on repos (activity_retry_after_us nulls first, updated_at) + where deleted_at is null and state = 'active' + and activity_synced_at is distinct from updated_at; + +-- one row per oid across every repo, which keeps forks from counting a commit twice +create table if not exists seen_commits ( + oid bytea primary key, + month date not null +); + +create index if not exists idx_seen_commits_month on seen_commits (month); + +create table if not exists commit_counts ( + committer text not null, + month date not null, + commits bigint not null check (commits >= 0), + primary key (committer, month) +); diff --git a/knotmirror/crates/knotmirror-db/migrations/20261030000000_commit_details.sql b/knotmirror/crates/knotmirror-db/migrations/20261030000000_commit_details.sql new file mode 100644 index 0000000000000000000000000000000000000000..c6454209776e85ce142b8955a24da813d4082bba --- /dev/null +++ b/knotmirror/crates/knotmirror-db/migrations/20261030000000_commit_details.sql @@ -0,0 +1,13 @@ +alter table seen_commits + add column repo_did text, + add column committer text, + add column committed_at timestamptz, + add column subject text; + +create index if not exists idx_seen_commits_committer + on seen_commits (committer, month, committed_at desc) + where committer is not null; + +update repos +set activity_synced_at = null, activity_tips = '{}' +where deleted_at is null; diff --git a/knotmirror/crates/knotmirror-db/src/lib.rs b/knotmirror/crates/knotmirror-db/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..88e18681c39ed039a4ecedd57300066724fa2632 --- /dev/null +++ b/knotmirror/crates/knotmirror-db/src/lib.rs @@ -0,0 +1,736 @@ +use std::borrow::Cow; + +use chrono::{DateTime, Datelike as _, Months, NaiveDate, Utc}; +use jacquard_common::deps::bytes::Bytes; +use jacquard_common::deps::smol_str::SmolStr; +use jacquard_common::types::did::Did; +use sqlx::postgres::PgPoolOptions; +use std::time::Duration; +use thiserror::Error; +use tracing::{debug, warn}; + +#[derive(Debug, Error)] +pub enum DbError { + #[error("query failed: {0}")] + Query(String), + #[error("row not found")] + NotFound, + #[error("constraint violation: {0}")] + Constraint(String), + #[error("connection failed: {0}")] + Connection(String), + #[error("migration failed: {0}")] + Migrate(String), + #[error("{0}")] + Other(String), +} + +impl From for DbError { + fn from(e: sqlx::Error) -> Self { + match e { + sqlx::Error::RowNotFound => Self::NotFound, + sqlx::Error::Database(db_err) => { + let msg = db_err.message().to_owned(); + if db_err.is_unique_violation() || db_err.is_foreign_key_violation() { + Self::Constraint(msg) + } else { + Self::Query(msg) + } + } + sqlx::Error::PoolTimedOut => Self::Connection("pool timed out".to_owned()), + sqlx::Error::Io(e) => Self::Connection(e.to_string()), + other => Self::Other(other.to_string()), + } + } +} + +impl From for DbError { + fn from(e: sqlx::migrate::MigrateError) -> Self { + Self::Migrate(e.to_string()) + } +} + +#[derive(Clone)] +pub struct Db(sqlx::PgPool); + +/// The body of one `org.tangled.git.refs` record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GitRefs { + pub default_branch: SmolStr, + /// Git refs in `git show-ref` format. + pub refs: Bytes, +} + +#[derive(Debug)] +pub struct RefsUpsert<'a> { + pub did: &'a Did, + pub refs_cid: &'a str, + pub refs: &'a GitRefs, +} + +#[derive(Debug)] +pub struct IndexRow { + pub did: Did, + pub default_branch: SmolStr, + pub default_branch_commit: Option, + pub index_retry_count: i32, +} + +#[derive(Debug)] +pub struct IndexedLanguages { + pub did: Did, + pub commit: String, + pub languages: Vec<(String, i64)>, +} + +#[derive(Debug)] +pub struct IndexedLanguagesPage { + pub repos: Vec, + pub next: Option, +} + +/// also the most months `getCommitStats` accepts +pub const ACTIVITY_MONTHS: u32 = 24; + +/// utc months, `before` excluded +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MonthWindow { + pub since: NaiveDate, + pub before: NaiveDate, +} + +impl MonthWindow { + /// the last `months` months, counting the one `now_us` is in + pub fn ending_at(now_us: i64, months: u32) -> Self { + let current = month_start( + DateTime::from_timestamp_micros(now_us) + .unwrap_or_default() + .date_naive(), + ); + Self { + since: current - Months::new(months.saturating_sub(1)), + before: current + Months::new(1), + } + } + + pub fn month_of(&self, seconds: i64) -> Option { + let date = DateTime::from_timestamp(seconds, 0)?.date_naive(); + (self.since..self.before) + .contains(&date) + .then(|| month_start(date)) + } +} + +fn month_start(date: NaiveDate) -> NaiveDate { + date.with_day(1).expect("every month has a first day") +} + +#[derive(Debug)] +pub struct ActivityRow { + pub did: Did, + pub synced_at: DateTime, + /// branch tips already counted + pub tips: Vec, + pub retry_count: i32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommitActivity { + pub oid: Vec, + pub committer: String, + pub month: NaiveDate, +} + +#[derive(Debug)] +pub struct RepoRow { + pub did: Did, + pub refs_cid: String, + pub refs: GitRefs, + pub atproto_rev: Option, + pub retry_count: i32, +} + +impl Db { + pub async fn connect(url: &str, max_connections: u32) -> Result { + let pool = PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(Duration::from_secs(10)) + .idle_timeout(Duration::from_secs(300)) + .max_lifetime(Duration::from_secs(1800)) + .connect(url) + .await?; + Ok(Self(pool)) + } + + pub fn from_pool(pool: sqlx::PgPool) -> Self { + Self(pool) + } + + pub fn pool(&self) -> &sqlx::PgPool { + &self.0 + } + + pub async fn migrate(&self) -> Result<(), DbError> { + sqlx::migrate!("./migrations").run(&self.0).await?; + Ok(()) + } + + pub async fn cursor(&self) -> Result, DbError> { + let row = sqlx::query!("select last_event_id from hydrant_cursor where id = 0") + .fetch_optional(&self.0) + .await?; + Ok(row.map(|r| r.last_event_id as u64)) + } + + /// Checkpoint on its own, for stretches of events that touched no row. + pub async fn set_cursor(&self, cursor: u64) -> Result<(), DbError> { + sqlx::query!( + "insert into hydrant_cursor (id, last_event_id) + values (0, $1) + on conflict (id) do update set last_event_id = excluded.last_event_id", + i64::try_from(cursor).unwrap_or(i64::MAX), + ) + .execute(&self.0) + .await?; + Ok(()) + } + + /// Stores the record, returning whether the row actually changed. + pub async fn upsert_refs(&self, r: RefsUpsert<'_>) -> Result { + let done = sqlx::query!( + "insert into repos (did, refs_cid, default_branch, refs, deleted_at) + values ($1, $2, $3, $4, null) + on conflict (did) do update set + refs_cid = excluded.refs_cid, + default_branch = excluded.default_branch, + refs = excluded.refs, + deleted_at = null, + state = 'pending', + error_msg = null, + retry_count = 0, + retry_after_us = null, + updated_at = now() + where repos.refs_cid is distinct from excluded.refs_cid + or repos.deleted_at is not null", + r.did.as_str(), + r.refs_cid, + r.refs.default_branch.as_str(), + &r.refs.refs[..], + ) + .execute(&self.0) + .await?; + Ok(done.rows_affected() > 0) + } + + /// Records a newer atproto commit rev for the account, queueing a resync. Returns whether + /// the rev moved forward; repos with no refs record yet are left alone. + pub async fn bump_rev(&self, did: &Did, rev: &str) -> Result { + let done = sqlx::query!( + "update repos + set atproto_rev = $2, state = 'pending', error_msg = null, + retry_count = 0, retry_after_us = null, updated_at = now() + where did = $1 and deleted_at is null + and (atproto_rev is null or atproto_rev < $2)", + did.as_str(), + rev, + ) + .execute(&self.0) + .await?; + Ok(done.rows_affected() > 0) + } + + pub async fn mark_deleted(&self, did: &Did) -> Result<(), DbError> { + sqlx::query!( + "update repos set deleted_at = now(), updated_at = now() + where did = $1 and deleted_at is null", + did.as_str(), + ) + .execute(&self.0) + .await?; + Ok(()) + } + + /// Repos due for a git resync, longest-waiting first. + pub async fn pending_repos(&self, now_us: i64, limit: i64) -> Result, DbError> { + let rows = sqlx::query!( + "select did, refs_cid, default_branch, refs, atproto_rev, retry_count + from repos + where deleted_at is null + and state <> 'active' + and (retry_after_us is null or retry_after_us <= $1) + order by retry_after_us nulls first, updated_at + limit $2", + now_us, + limit, + ) + .fetch_all(&self.0) + .await?; + Ok(rows + .into_iter() + .filter_map(|r| { + let did = Did::new_owned(&r.did) + .inspect_err(|e| warn!(did = %r.did, error = %e, "unparseable did in repos")) + .ok()?; + Some(RepoRow { + did, + refs_cid: r.refs_cid, + refs: GitRefs { + default_branch: r.default_branch.into(), + refs: Bytes::from(r.refs), + }, + atproto_rev: r.atproto_rev, + retry_count: r.retry_count, + }) + }) + .collect()) + } + + /// Live repo counts by `(state, due)`, with `error` split into `backoff` / `error` at + /// `max_retry`. + pub async fn repo_state_counts( + &self, + now_us: i64, + max_retry: i32, + ) -> Result, DbError> { + let rows = sqlx::query!( + r#"select case when state = 'error' and retry_count >= $2 then 'error' + when state = 'error' then 'backoff' + else state end as "state!", + (state <> 'active' and (retry_after_us is null or retry_after_us <= $1)) as "due!", + count(*) as "count!" + from repos + where deleted_at is null + group by 1, 2"#, + now_us, + max_retry, + ) + .fetch_all(&self.0) + .await?; + Ok(rows + .into_iter() + .map(|r| (r.state, r.due, r.count)) + .collect()) + } + + pub async fn active_repos(&self) -> Result, DbError> { + let rows = sqlx::query!( + "select did from repos where deleted_at is null and state = 'active' order by did" + ) + .fetch_all(&self.0) + .await?; + Ok(rows + .into_iter() + .filter_map(|r| { + Did::new_owned(&r.did) + .inspect_err(|e| warn!(did = %r.did, error = %e, "unparseable did in repos")) + .ok() + }) + .collect()) + } + + pub async fn mark_synced( + &self, + row: &RepoRow, + default_branch_commit: Option<&str>, + ) -> Result<(), DbError> { + let (did, refs_cid) = (&row.did, &row.refs_cid); + let done = sqlx::query!( + "update repos + set state = 'active', error_msg = null, retry_count = 0, retry_after_us = null, + default_branch_commit = $3, updated_at = now() + where did = $1 and refs_cid = $2 and atproto_rev is not distinct from $4 + and deleted_at is null", + did.as_str(), + refs_cid, + default_branch_commit, + row.atproto_rev, + ) + .execute(&self.0) + .await?; + if done.rows_affected() > 0 { + debug!(%did, refs_cid, "synced"); + } else { + debug!(%did, refs_cid, "refs or rev changed mid-sync, dropped the stale success"); + } + Ok(()) + } + + pub async fn mark_deferred(&self, row: &RepoRow, retry_after_us: i64) -> Result<(), DbError> { + let (did, refs_cid) = (&row.did, &row.refs_cid); + let done = sqlx::query!( + "update repos set retry_after_us = $3, updated_at = now() + where did = $1 and refs_cid = $2 and atproto_rev is not distinct from $4 + and deleted_at is null", + did.as_str(), + refs_cid, + retry_after_us, + row.atproto_rev, + ) + .execute(&self.0) + .await?; + if done.rows_affected() > 0 { + debug!(%did, refs_cid, retry_after_us, "knot busy, deferred"); + } else { + debug!(%did, refs_cid, "refs or rev changed mid-sync, dropped the stale defer"); + } + Ok(()) + } + + pub async fn mark_sync_failed( + &self, + row: &RepoRow, + error: &str, + retry_after_us: i64, + ) -> Result<(), DbError> { + let (did, refs_cid) = (&row.did, &row.refs_cid); + // postgres `text` rejects NUL + let error = if error.contains('\0') { + Cow::Owned(error.replace('\0', "")) + } else { + Cow::Borrowed(error) + }; + let done = sqlx::query!( + "update repos + set state = 'error', error_msg = $3, retry_count = retry_count + 1, + retry_after_us = $4, updated_at = now() + where did = $1 and refs_cid = $2 and atproto_rev is not distinct from $5 + and deleted_at is null", + did.as_str(), + refs_cid, + &*error, + retry_after_us, + row.atproto_rev, + ) + .execute(&self.0) + .await?; + if done.rows_affected() > 0 { + warn!(%did, refs_cid, retry_after_us, error = %error, "sync failed, backing off"); + } else { + debug!(%did, refs_cid, "refs or rev changed mid-sync, dropped the stale failure"); + } + Ok(()) + } + + /// Repos whose default branch commit moved since the last index, longest-waiting first. + pub async fn pending_index(&self, now_us: i64, limit: i64) -> Result, DbError> { + let rows = sqlx::query!( + "select did, default_branch, default_branch_commit, index_retry_count + from repos + where deleted_at is null + and state = 'active' + and default_branch_commit is distinct from indexed_commit + and (index_retry_after_us is null or index_retry_after_us <= $1) + order by index_retry_after_us nulls first, updated_at + limit $2", + now_us, + limit, + ) + .fetch_all(&self.0) + .await?; + Ok(rows + .into_iter() + .filter_map(|r| { + let did = Did::new_owned(&r.did) + .inspect_err(|e| warn!(did = %r.did, error = %e, "unparseable did in repos")) + .ok()?; + Some(IndexRow { + did, + default_branch: r.default_branch.into(), + default_branch_commit: r.default_branch_commit, + index_retry_count: r.index_retry_count, + }) + }) + .collect()) + } + + pub async fn set_languages( + &self, + did: &Did, + commit: Option<&str>, + langs: &[(String, i64)], + ) -> Result<(), DbError> { + let mut tx = self.0.begin().await?; + let done = sqlx::query!( + "update repos + set indexed_commit = $2, index_error = null, index_retry_count = 0, + index_retry_after_us = null + where did = $1 and default_branch_commit is not distinct from $2", + did.as_str(), + commit, + ) + .execute(&mut *tx) + .await?; + if done.rows_affected() == 0 { + debug!(%did, ?commit, "default branch moved mid-index, dropped the stale languages"); + return Ok(()); + } + sqlx::query!("delete from repo_languages where did = $1", did.as_str()) + .execute(&mut *tx) + .await?; + let (names, sizes): (Vec<&str>, Vec) = + langs.iter().map(|(n, s)| (n.as_str(), *s)).unzip(); + sqlx::query!( + "insert into repo_languages (did, language, size) + select $1, * from unnest($2::text[], $3::bigint[])", + did.as_str(), + &names as &[&str], + &sizes, + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + debug!(%did, ?commit, langs = langs.len(), "indexed languages"); + Ok(()) + } + + pub async fn mark_index_failed( + &self, + did: &Did, + commit: Option<&str>, + error: &str, + retry_after_us: i64, + ) -> Result<(), DbError> { + let error = if error.contains('\0') { + Cow::Owned(error.replace('\0', "")) + } else { + Cow::Borrowed(error) + }; + let done = sqlx::query!( + "update repos + set index_error = $3, index_retry_count = index_retry_count + 1, + index_retry_after_us = $4 + where did = $1 and default_branch_commit is not distinct from $2", + did.as_str(), + commit, + &*error, + retry_after_us, + ) + .execute(&self.0) + .await?; + if done.rows_affected() > 0 { + warn!(%did, ?commit, retry_after_us, error = %error, "index failed, backing off"); + } + Ok(()) + } + + pub async fn indexed_languages_page( + &self, + after: Option<&str>, + limit: i64, + ) -> Result { + let rows = sqlx::query!( + r#"select r.did, + r.indexed_commit as "commit!", + coalesce(array_agg(l.language order by l.size desc, l.language) + filter (where l.language is not null), '{}') as "names!", + coalesce(array_agg(l.size order by l.size desc, l.language) + filter (where l.language is not null), '{}') as "sizes!" + from repos r + left join repo_languages l on l.did = r.did + where r.deleted_at is null + and r.indexed_commit is not null + and ($1::text is null or r.did > $1) + group by r.did + order by r.did + limit $2"#, + after, + limit, + ) + .fetch_all(&self.0) + .await?; + let next = (rows.len() as i64 == limit) + .then(|| rows.last().map(|r| r.did.clone())) + .flatten(); + let repos = rows + .into_iter() + .filter_map(|r| { + let did = Did::new_owned(&r.did) + .inspect_err(|e| warn!(did = %r.did, error = %e, "unparseable did in repos")) + .ok()?; + Some(IndexedLanguages { + did, + commit: r.commit, + languages: r.names.into_iter().zip(r.sizes).collect(), + }) + }) + .collect(); + Ok(IndexedLanguagesPage { repos, next }) + } + + /// Stored languages, only when `commit` is the one they were indexed from. + pub async fn indexed_languages( + &self, + did: &Did, + commit: &str, + ) -> Result>, DbError> { + let indexed = sqlx::query_scalar!( + "select 1 from repos where did = $1 and indexed_commit = $2", + did.as_str(), + commit, + ) + .fetch_optional(&self.0) + .await?; + if indexed.is_none() { + return Ok(None); + } + let rows = sqlx::query!( + "select language, size from repo_languages where did = $1", + did.as_str(), + ) + .fetch_all(&self.0) + .await?; + Ok(Some( + rows.into_iter().map(|r| (r.language, r.size)).collect(), + )) + } + + pub async fn pending_activity( + &self, + now_us: i64, + limit: i64, + ) -> Result, DbError> { + let rows = sqlx::query!( + "select did, updated_at, activity_tips, activity_retry_count + from repos + where deleted_at is null + and state = 'active' + and activity_synced_at is distinct from updated_at + and (activity_retry_after_us is null or activity_retry_after_us <= $1) + order by activity_retry_after_us nulls first, updated_at + limit $2", + now_us, + limit, + ) + .fetch_all(&self.0) + .await?; + Ok(rows + .into_iter() + .filter_map(|r| { + let did = Did::new_owned(&r.did) + .inspect_err(|e| warn!(did = %r.did, error = %e, "unparseable did in repos")) + .ok()?; + Some(ActivityRow { + did, + synced_at: r.updated_at, + tips: r.activity_tips, + retry_count: r.activity_retry_count, + }) + }) + .collect()) + } + + /// returns how many commits no repo had shown before + pub async fn save_commit_activity(&self, commits: &[CommitActivity]) -> Result { + let (oids, committers, months): (Vec<&[u8]>, Vec<&str>, Vec) = commits + .iter() + .map(|c| (c.oid.as_slice(), c.committer.as_str(), c.month)) + .collect(); + let inserted = sqlx::query_scalar!( + r#"with input as ( + select distinct on (oid) oid, committer, month + from unnest($1::bytea[], $2::text[], $3::date[]) as t(oid, committer, month) + ), inserted as ( + insert into seen_commits (oid, month) + select oid, month from input + on conflict (oid) do nothing + returning oid + ), counted as ( + insert into commit_counts (committer, month, commits) + select committer, month, count(*) + from input join inserted using (oid) + group by committer, month + on conflict (committer, month) do update + set commits = commit_counts.commits + excluded.commits + ) + select count(*) as "count!" from inserted"#, + &oids as &[&[u8]], + &committers as &[&str], + &months, + ) + .fetch_one(&self.0) + .await?; + Ok(inserted as u64) + } + + pub async fn finish_activity( + &self, + did: &Did, + synced_at: DateTime, + tips: &[String], + ) -> Result<(), DbError> { + sqlx::query!( + "update repos + set activity_synced_at = $2, activity_tips = $3, activity_error = null, + activity_retry_count = 0, activity_retry_after_us = null + where did = $1", + did.as_str(), + synced_at, + tips, + ) + .execute(&self.0) + .await?; + Ok(()) + } + + pub async fn mark_activity_failed( + &self, + did: &Did, + error: &str, + retry_after_us: i64, + ) -> Result<(), DbError> { + let error = if error.contains('\0') { + Cow::Owned(error.replace('\0', "")) + } else { + Cow::Borrowed(error) + }; + sqlx::query!( + "update repos + set activity_error = $2, activity_retry_count = activity_retry_count + 1, + activity_retry_after_us = $3 + where did = $1", + did.as_str(), + &*error, + retry_after_us, + ) + .execute(&self.0) + .await?; + warn!(%did, retry_after_us, error = %error, "counting commits failed, backing off"); + Ok(()) + } + + pub async fn prune_commit_activity(&self, before: NaiveDate) -> Result<(), DbError> { + let mut tx = self.0.begin().await?; + sqlx::query!("delete from seen_commits where month < $1", before) + .execute(&mut *tx) + .await?; + sqlx::query!("delete from commit_counts where month < $1", before) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + pub async fn commit_counts( + &self, + committers: &[String], + since: NaiveDate, + before: NaiveDate, + limit: i64, + ) -> Result, DbError> { + let rows = sqlx::query!( + r#"select month, sum(commits)::bigint as "commits!" + from commit_counts + where committer = any($1) and month >= $2 and month < $3 + group by month + order by month desc + limit $4"#, + committers, + since, + before, + limit, + ) + .fetch_all(&self.0) + .await?; + Ok(rows.into_iter().map(|r| (r.month, r.commits)).collect()) + } +} diff --git a/knotmirror/crates/knotmirror-db/tests/commit_activity.rs b/knotmirror/crates/knotmirror-db/tests/commit_activity.rs new file mode 100644 index 0000000000000000000000000000000000000000..ce4518af520eaa1679326709488a0bdd02e713bc --- /dev/null +++ b/knotmirror/crates/knotmirror-db/tests/commit_activity.rs @@ -0,0 +1,182 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use chrono::{Months, NaiveDate}; +use jacquard_common::deps::bytes::Bytes; +use jacquard_common::types::did::Did; +use knotmirror_db::{CommitActivity, Db, GitRefs, MonthWindow, RefsUpsert}; + +const ACTOR: &str = "did:plc:actor"; + +async fn test_db() -> Option { + let Ok(url) = std::env::var("KNOTMIRROR_TEST_DATABASE_URL") else { + return None; + }; + static NEXT: AtomicUsize = AtomicUsize::new(0); + let name = format!( + "commit_activity_{}_{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + let admin = Db::connect(&url, 1).await.unwrap(); + sqlx::query(sqlx::AssertSqlSafe(format!("create database {name}"))) + .execute(admin.pool()) + .await + .unwrap(); + let mut url = url::Url::parse(&url).unwrap(); + url.set_path(&name); + let db = Db::connect(url.as_str(), 4).await.unwrap(); + db.migrate().await.unwrap(); + Some(db) +} + +fn commit(oid: u8, committer: &str, month: NaiveDate) -> CommitActivity { + CommitActivity { + oid: vec![oid; 20], + committer: committer.to_owned(), + month, + } +} + +async fn synced_repo(db: &Db, did: &Did) { + db.upsert_refs(RefsUpsert { + did, + refs_cid: "bafyrefs", + refs: &GitRefs { + default_branch: "main".into(), + refs: Bytes::new(), + }, + }) + .await + .unwrap(); + sync_pending(db, did).await; +} + +async fn sync_pending(db: &Db, did: &Did) { + let row = db + .pending_repos(i64::MAX, 10) + .await + .unwrap() + .into_iter() + .find(|row| &row.did == did) + .unwrap(); + db.mark_synced(&row, None).await.unwrap(); +} + +#[tokio::test] +async fn a_commit_counts_once_however_often_it_is_seen() { + let Some(db) = test_db().await else { return }; + let month = NaiveDate::from_ymd_opt(2026, 9, 1).unwrap(); + let commits = [ + commit(0xaa, ACTOR, month), + commit(0xbb, "person@example.com", month), + commit(0xcc, "person@example.com", month), + ]; + assert_eq!(db.save_commit_activity(&commits).await.unwrap(), 3); + assert_eq!(db.save_commit_activity(&commits).await.unwrap(), 0); + + let committers = [ACTOR.to_owned(), "person@example.com".to_owned()]; + let counts = db + .commit_counts(&committers, month, month + Months::new(1), 24) + .await + .unwrap(); + assert_eq!(counts, [(month, 3)]); + let counts = db + .commit_counts(&committers[1..], month, month + Months::new(1), 24) + .await + .unwrap(); + assert_eq!(counts, [(month, 2)]); +} + +#[tokio::test] +async fn a_synced_repo_is_counted_until_its_next_sync() { + let Some(db) = test_db().await else { return }; + let did = Did::new_owned("did:plc:repo").unwrap(); + synced_repo(&db, &did).await; + + let pending = db.pending_activity(i64::MAX, 10).await.unwrap(); + assert_eq!(pending.len(), 1, "an already mirrored repo is backfilled"); + assert!(pending[0].tips.is_empty()); + + let tips = ["d".repeat(40)]; + db.finish_activity(&did, pending[0].synced_at, &tips) + .await + .unwrap(); + assert!(db.pending_activity(i64::MAX, 10).await.unwrap().is_empty()); + + db.bump_rev(&did, "3lrev").await.unwrap(); + sync_pending(&db, &did).await; + let pending = db.pending_activity(i64::MAX, 10).await.unwrap(); + assert_eq!(pending.len(), 1, "a new sync queues the repo again"); + assert_eq!(pending[0].tips, tips); +} + +#[tokio::test] +async fn a_failed_count_waits_out_its_backoff() { + let Some(db) = test_db().await else { return }; + let did = Did::new_owned("did:plc:repo").unwrap(); + synced_repo(&db, &did).await; + db.mark_activity_failed(&did, "boom", 1_000).await.unwrap(); + assert!(db.pending_activity(999, 10).await.unwrap().is_empty()); + let pending = db.pending_activity(1_000, 10).await.unwrap(); + assert_eq!(pending[0].retry_count, 1); +} + +#[tokio::test] +async fn pruning_drops_months_before_the_window() { + let Some(db) = test_db().await else { return }; + let current = NaiveDate::from_ymd_opt(2026, 9, 1).unwrap(); + let old = current - Months::new(24); + db.save_commit_activity(&[commit(0xee, ACTOR, old), commit(0xff, ACTOR, current)]) + .await + .unwrap(); + db.prune_commit_activity(current - Months::new(23)) + .await + .unwrap(); + + for query in [ + "select count(*) from seen_commits", + "select count(*) from commit_counts", + ] { + let rows: i64 = sqlx::query_scalar(query) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(rows, 1, "{query}"); + } +} + +#[test] +fn the_window_ends_with_the_current_utc_month() { + let at = |s: &str| { + chrono::DateTime::parse_from_rfc3339(s) + .unwrap() + .timestamp_micros() + }; + let day = |y, m, d| NaiveDate::from_ymd_opt(y, m, d).unwrap(); + let window = MonthWindow::ending_at(at("2026-03-31T23:30:00-05:00"), 24); + assert_eq!(window.since, day(2024, 5, 1)); + assert_eq!(window.before, day(2026, 5, 1)); + assert_eq!( + MonthWindow::ending_at(at("2026-03-01T00:00:00Z"), 1), + MonthWindow { + since: day(2026, 3, 1), + before: day(2026, 4, 1) + } + ); + + let seconds = |s: &str| at(s) / 1_000_000; + assert_eq!( + window.month_of(seconds("2026-04-30T23:59:59Z")), + Some(day(2026, 4, 1)) + ); + assert_eq!(window.month_of(seconds("2026-05-01T00:00:00Z")), None); + assert_eq!( + window.month_of(seconds("2024-05-01T00:00:00Z")), + Some(day(2024, 5, 1)) + ); + assert_eq!(window.month_of(seconds("2024-04-30T23:59:59Z")), None); +} diff --git a/knotmirror/crates/knotmirror-git/Cargo.toml b/knotmirror/crates/knotmirror-git/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..ba098ac31b177f869049ebafd8944b822eb055b7 --- /dev/null +++ b/knotmirror/crates/knotmirror-git/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "knotmirror-git" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +gix-rebase = { workspace = true } + +gix = { workspace = true } +tempfile = { workspace = true } +thiserror = { workspace = true } +itertools = { workspace = true } +jacquard-common = { workspace = true } + +[lints] +workspace = true diff --git a/knotmirror/crates/knotmirror-git/src/layout.rs b/knotmirror/crates/knotmirror-git/src/layout.rs new file mode 100644 index 0000000000000000000000000000000000000000..1d06a295cfa4adde18a8c3d2c62cca5fb0aa8b6c --- /dev/null +++ b/knotmirror/crates/knotmirror-git/src/layout.rs @@ -0,0 +1,73 @@ +use std::path::PathBuf; + +use itertools::Itertools as _; +use jacquard_common::types::did::Did; + +use crate::scratch::TempRepository; + +#[derive(Debug, Clone)] +pub struct Layout { + scan_path: PathBuf, +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("repo not found")] + RepoNotFound, + #[error("open_scratch requires at least one repo")] + NotEnoughRepos, + #[error("scratch repo: {0}")] + Io(#[from] std::io::Error), + #[error("init scratch repo: {0}")] + Init(#[from] gix::init::Error), + #[error("open scratch repo: {0}")] + Open(#[from] gix::open::Error), +} + +impl Layout { + pub fn new(scan_path: PathBuf) -> Self { + Self { scan_path } + } + + pub fn repo_path(&self, repo: &Did) -> PathBuf { + // TODO: normalize DID like knot_git::Layout does. + self.scan_path.join(repo.as_str()) + } + + pub fn open(&self, repo: &Did) -> Result { + gix::open(self.repo_path(repo)) + } + + pub fn open_scratch(&self, dids: &[&Did]) -> Result { + if dids.is_empty() { + return Err(Error::NotEnoughRepos); + } + let alternates = dids + .iter() + .copied() + .unique() + .map(|did| { + std::fs::canonicalize(self.repo_path(did).join("objects")) + .map(|dir| format!("{}\n", dir.display())) + .map_err(|_| Error::RepoNotFound) + }) + .collect::>()?; + + let object_hash = self.open(dids[0])?.object_hash(); + let scratch = tempfile::tempdir()?; + gix::ThreadSafeRepository::init( + scratch.path(), + gix::create::Kind::Bare, + gix::create::Options { + object_hash: Some(object_hash), + ..Default::default() + }, + )?; + + let info_dir = scratch.path().join("objects").join("info"); + std::fs::create_dir_all(&info_dir)?; + std::fs::write(info_dir.join("alternates"), alternates)?; + + Ok(TempRepository::open(scratch)?) + } +} diff --git a/knotmirror/crates/knotmirror-git/src/lib.rs b/knotmirror/crates/knotmirror-git/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..888c7cd0c3577b52d58baee57d993e781ccf6553 --- /dev/null +++ b/knotmirror/crates/knotmirror-git/src/lib.rs @@ -0,0 +1,8 @@ +#![allow( + clippy::result_large_err, + reason = "gix's own error types are this large" +)] + +pub mod layout; +pub mod repo_ext; +pub mod scratch; diff --git a/knotmirror/crates/knotmirror-git/src/repo_ext.rs b/knotmirror/crates/knotmirror-git/src/repo_ext.rs new file mode 100644 index 0000000000000000000000000000000000000000..2deaf41e8b35fbc8d41856e726774629e0232c41 --- /dev/null +++ b/knotmirror/crates/knotmirror-git/src/repo_ext.rs @@ -0,0 +1,113 @@ +//! Extended [`gix::Repository`]. + +use std::collections::{HashMap, HashSet}; + +#[derive(Debug, thiserror::Error)] +pub enum MergeError { + /// Not a failure so much as an answer: these paths couldn't be reconciled, and nothing + /// reachable was written. + #[error("the merge conflicted in {}", .0.join(", "))] + Conflicts(Vec), + #[error("could not walk the commits to replay")] + Walk(#[from] gix::revision::walk::Error), + #[error("could not read a commit to replay")] + ReadCommit(#[from] gix::revision::walk::iter::Error), + #[error("could not set up the blob merge")] + BlobMerge(#[from] Box), + #[error("could not set up the tree diff")] + TreeDiff(#[from] Box), + #[error("could not replay the commits")] + Replay(#[from] gix_rebase::replay::Error), +} + +pub trait RepositoryExt { + /// Replay `source` and the ancestors of it that `onto` doesn't already have on top of `onto`. + fn rebase( + &self, + source: gix::ObjectId, + onto: gix::ObjectId, + committer: gix::actor::Signature, + ) -> Result; +} + +impl RepositoryExt for gix::Repository { + fn rebase( + &self, + source: gix::ObjectId, + onto: gix::ObjectId, + committer: gix::actor::Signature, + ) -> Result { + let picks = replay_list(self, source, onto)?; + + let options: gix_rebase::replay::Options = Default::default(); + let treat_as_unresolved = options.treat_as_unresolved; + + let mut blob_merge = self + .merge_resource_cache(Default::default()) + .map_err(Box::new)?; + let mut diff_cache = self.diff_resource_cache_for_tree_diff().map_err(Box::new)?; + let mut diff_state = gix::diff::tree::State::default(); + + let outcome = gix_rebase::replay( + &onto, + &picks, + &committer, + Default::default(), + self, + &mut diff_state, + &mut diff_cache, + &mut blob_merge, + options, + )?; + + match outcome { + gix_rebase::replay::Outcome::Complete(done) => Ok(done.tip), + gix_rebase::replay::Outcome::Conflict(conflict) => Err(MergeError::Conflicts( + conflict + .unresolved_paths(treat_as_unresolved) + .map(|path| path.to_string()) + .collect(), + )), + } + } +} + +/// The commits [`RepositoryExt::rebase`] replays, oldest first: everything reachable from `source` +/// that `onto` doesn't have, without the merge commits — the set `git rebase` puts on its todo list. +fn replay_list( + repo: &gix::Repository, + source: gix::ObjectId, + onto: gix::ObjectId, +) -> Result, MergeError> { + let mut parents_of = HashMap::new(); + for info in repo.rev_walk([source]).with_hidden([onto]).all()? { + let info = info?; + parents_of.insert(info.id, info.parent_ids); + } + + // `rev_walk` offers no topological sorting through the gix façade, and replaying a commit + // before its own parent would rebase it onto the wrong tree, so lay the picks out with an + // explicit post-order walk. Iterative, because a branch can be far deeper than the stack. + let mut order = Vec::with_capacity(parents_of.len()); + let mut seen = HashSet::new(); + let mut stack = vec![(source, false)]; + while let Some((id, parents_done)) = stack.pop() { + let Some(parents) = parents_of.get(&id) else { + continue; + }; + if parents_done { + // Merge commits are left out, exactly as `git rebase` does without + // `--rebase-merges`; the commits they brought in are replayed on their own. + if parents.len() < 2 { + order.push(id); + } + continue; + } + if !seen.insert(id) { + continue; + } + stack.push((id, true)); + stack.extend(parents.iter().map(|parent| (*parent, false))); + } + Ok(order) +} diff --git a/knotmirror/crates/knotmirror-git/src/scratch.rs b/knotmirror/crates/knotmirror-git/src/scratch.rs new file mode 100644 index 0000000000000000000000000000000000000000..56c0aff8b1945cc1f54b674d9d8f83ddb812e528 --- /dev/null +++ b/knotmirror/crates/knotmirror-git/src/scratch.rs @@ -0,0 +1,30 @@ +use tempfile::TempDir; + +/// A [`gix::Repository`] backed by a temporary directory that is removed on drop. Derefs to +/// `gix::Repository`, so it is used just like one; keeping it alive keeps the scratch dir alive. +pub struct TempRepository { + repo: gix::Repository, + // Declared AFTER `repo` so `repo` drops first: any file handles into the scratch dir close + // before the dir itself is removed (Rust drops struct fields in declaration order). + _dir: TempDir, +} + +impl TempRepository { + pub fn open(dir: TempDir) -> Result { + let repo = gix::open(dir.path())?; + Ok(Self { repo, _dir: dir }) + } + + #[allow(dead_code)] + pub fn with_object_memory(mut self) -> Self { + self.repo.objects.enable_object_memory(); + self + } +} + +impl std::ops::Deref for TempRepository { + type Target = gix::Repository; + fn deref(&self) -> &gix::Repository { + &self.repo + } +} diff --git a/knotmirror/crates/knotmirror-index/Cargo.toml b/knotmirror/crates/knotmirror-index/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..f6efed523fd4a1c4a82af9fff38402ae525930ae --- /dev/null +++ b/knotmirror/crates/knotmirror-index/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "knotmirror-index" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-runtime = { workspace = true } +knotmirror-db = { workspace = true } +knotmirror-git = { workspace = true } +knotmirror-types = { workspace = true } +knot-git = { workspace = true } +knot-langs = { workspace = true } +knot-types = { workspace = true } + +gix = { workspace = true } +jacquard-common = { workspace = true } +reqwest = { workspace = true } +scc = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +chrono = { workspace = true } +tempfile = { workspace = true } + +[lints] +workspace = true diff --git a/knotmirror/crates/knotmirror-index/src/activity.rs b/knotmirror/crates/knotmirror-index/src/activity.rs new file mode 100644 index 0000000000000000000000000000000000000000..bf6756ea5291e5914ff888810a6d23d31305162a --- /dev/null +++ b/knotmirror/crates/knotmirror-index/src/activity.rs @@ -0,0 +1,332 @@ +use std::sync::Arc; +use std::time::Duration; + +use bobbin_runtime::Clock; +use gix::ObjectId; +use jacquard_common::types::did::Did; +use knotmirror_db::{ACTIVITY_MONTHS, ActivityRow, CommitActivity, Db, DbError, MonthWindow}; +use knotmirror_git::layout::Layout; +use knotmirror_types::{backoff_us, now_us}; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info}; + +use crate::IndexError; + +const BATCH_SIZE: i64 = 64; +const SAVE_CHUNK: usize = 1_000; +const MAX_COMMITTER_LEN: usize = 2_048; +const PRUNE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60); + +pub struct ActivityRuntime { + pub db: Db, + pub clock: Arc, + pub cancel: CancellationToken, + pub layout: Layout, +} + +pub async fn run(poll_interval: Duration, rt: ActivityRuntime) -> Result<(), IndexError> { + info!("commit activity worker started"); + let mut next_prune = rt.clock.now_instant(); + + while !rt.cancel.is_cancelled() { + if rt.clock.now_instant() >= next_prune { + let window = MonthWindow::ending_at(now_us(&*rt.clock), ACTIVITY_MONTHS); + rt.db.prune_commit_activity(window.since).await?; + next_prune = rt.clock.now_instant() + PRUNE_INTERVAL; + } + + let rows = rt + .db + .pending_activity(now_us(&*rt.clock), BATCH_SIZE) + .await?; + if rows.is_empty() { + tokio::select! { + () = rt.cancel.cancelled() => break, + () = rt.clock.sleep(poll_interval) => {} + } + } + for row in rows { + if rt.cancel.is_cancelled() { + break; + } + count_repo(&rt, row).await; + } + } + + info!("commit activity worker stopped"); + Ok(()) +} + +async fn count_repo(rt: &ActivityRuntime, row: ActivityRow) { + let path = rt.layout.repo_path(&row.did); + let window = MonthWindow::ending_at(now_us(&*rt.clock), ACTIVITY_MONTHS); + let counted = row.tips.clone(); + let scanned = tokio::task::spawn_blocking(move || { + let repo = gix::open(path).map_err(|e| e.to_string())?; + scan(&repo, &counted, window) + }) + .await + .unwrap_or_else(|_| Err("commit scan panicked".to_owned())); + + let write = match scanned { + Ok(scan) => save(&rt.db, &row, scan).await, + Err(e) => { + rt.db + .mark_activity_failed( + &row.did, + &e, + now_us(&*rt.clock) + backoff_us(row.retry_count), + ) + .await + } + }; + if let Err(e) = write { + error!(did = %row.did, error = %e, "recording commit activity failed"); + } +} + +async fn save(db: &Db, row: &ActivityRow, scan: Scan) -> Result<(), DbError> { + let mut new = 0; + for chunk in scan.commits.chunks(SAVE_CHUNK) { + new += db.save_commit_activity(chunk).await?; + } + db.finish_activity(&row.did, row.synced_at, &scan.tips) + .await?; + debug!(did = %row.did, commits = scan.commits.len(), new, "counted commits"); + Ok(()) +} + +#[derive(Debug)] +pub struct Scan { + pub tips: Vec, + pub commits: Vec, +} + +/// a counted tip that no longer exists is ignored, which walks its history again +pub fn scan( + repo: &gix::Repository, + counted: &[String], + window: MonthWindow, +) -> Result { + let mut tips = Vec::new(); + for reference in repo + .references() + .map_err(|e| e.to_string())? + .prefixed("refs/heads/") + .map_err(|e| e.to_string())? + { + tips.push( + reference + .map_err(|e| e.to_string())? + .into_fully_peeled_id() + .map_err(|e| e.to_string())? + .detach(), + ); + } + tips.sort_unstable(); + tips.dedup(); + + let hidden = counted + .iter() + .filter_map(|tip| ObjectId::from_hex(tip.as_bytes()).ok()) + .filter(|tip| repo.has_object(tip)); + let mut commits = Vec::new(); + if !tips.is_empty() { + let walk = repo + .rev_walk(tips.iter().copied()) + .with_hidden(hidden) + .all() + .map_err(|e| e.to_string())?; + for info in walk { + let info = info.map_err(|e| e.to_string())?; + let commit = info.object().map_err(|e| e.to_string())?; + let committer = commit.committer().map_err(|e| e.to_string())?; + let Some(month) = committer + .time() + .ok() + .and_then(|time| window.month_of(time.seconds)) + else { + continue; + }; + let Some(committer) = normalize_committer(&committer.email.to_string()) else { + continue; + }; + commits.push(CommitActivity { + oid: info.id.as_bytes().to_vec(), + committer, + month, + }); + } + } + + Ok(Scan { + tips: tips.iter().map(ToString::to_string).collect(), + commits, + }) +} + +/// dids keep their case and emails get lowercased +fn normalize_committer(email: &str) -> Option { + let email = email.trim(); + if email.is_empty() || email.len() > MAX_COMMITTER_LEN { + return None; + } + Some(match Did::new(email) { + Ok(did) => did.as_str().to_owned(), + Err(_) => email.to_lowercase(), + }) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::process::Command; + + use chrono::{Days, Months, NaiveDate, Utc}; + + use super::*; + + fn git(dir: &Path, env: &[(&str, &str)], args: &[&str]) { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .envs(env.iter().copied()) + .output() + .expect("run git"); + assert!(out.status.success(), "git {args:?}: {out:?}"); + } + + fn commit(dir: &Path, file: &str, committer: &str, at: NaiveDate) { + std::fs::write(dir.join(file), file).unwrap(); + git(dir, &[], &["add", file]); + let date = at.and_hms_opt(12, 0, 0).unwrap().and_utc().to_rfc3339(); + git( + dir, + &[ + ("GIT_AUTHOR_NAME", "author"), + ("GIT_AUTHOR_EMAIL", "not-the-committer@example.com"), + ("GIT_AUTHOR_DATE", &date), + ("GIT_COMMITTER_NAME", "committer"), + ("GIT_COMMITTER_EMAIL", committer), + ("GIT_COMMITTER_DATE", &date), + ], + &["commit", "-q", "-m", file], + ); + } + + struct Fixture { + dir: tempfile::TempDir, + current: NaiveDate, + window: MonthWindow, + } + + impl Fixture { + fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + let now = Utc::now(); + let window = MonthWindow::ending_at(now.timestamp_micros(), ACTIVITY_MONTHS); + let current = window.before - Months::new(1); + let day = |months| current - Months::new(months) + Days::new(1); + let path = dir.path(); + git(path, &[], &["init", "-q", "-b", "main"]); + commit(path, "base", "did:plc:alice", day(2)); + git(path, &[], &["branch", "feature"]); + commit(path, "main", "did:plc:alice", day(0)); + git(path, &[], &["checkout", "-q", "feature"]); + commit(path, "feature", "did:plc:bob", day(1)); + git(path, &[], &["checkout", "-q", "main"]); + Self { + dir, + current, + window, + } + } + + fn scan(&self, counted: &[String]) -> Scan { + scan(&gix::open(self.dir.path()).unwrap(), counted, self.window).unwrap() + } + + fn commit(&self, file: &str, committer: &str) { + commit( + self.dir.path(), + file, + committer, + self.current + Days::new(2), + ); + } + } + + fn counted(scan: &Scan) -> Vec<(String, NaiveDate)> { + let mut counted: Vec<_> = scan + .commits + .iter() + .map(|c| (c.committer.clone(), c.month)) + .collect(); + counted.sort(); + counted + } + + #[test] + fn every_branch_counts_under_its_committer_and_utc_month() { + let fixture = Fixture::new(); + let scan = fixture.scan(&[]); + let month = |months| fixture.current - Months::new(months); + assert_eq!( + counted(&scan), + [ + ("did:plc:alice".to_owned(), month(2)), + ("did:plc:alice".to_owned(), month(0)), + ("did:plc:bob".to_owned(), month(1)), + ] + ); + assert_eq!(scan.tips.len(), 2); + } + + #[test] + fn counted_tips_hide_their_history() { + let fixture = Fixture::new(); + let before = fixture.scan(&[]); + fixture.commit("new", "did:plc:carol"); + let scan = fixture.scan(&before.tips); + assert_eq!( + counted(&scan), + [("did:plc:carol".to_owned(), fixture.current)] + ); + } + + #[test] + fn a_vanished_counted_tip_walks_everything_again() { + let fixture = Fixture::new(); + let scan = fixture.scan(&["f".repeat(40), "not hex".to_owned()]); + assert_eq!(scan.commits.len(), 3); + } + + #[test] + fn a_branch_sharing_a_tip_is_walked_once() { + let fixture = Fixture::new(); + git(fixture.dir.path(), &[], &["branch", "same-as-main", "main"]); + let scan = fixture.scan(&[]); + assert_eq!(scan.commits.len(), 3); + assert_eq!(scan.tips.len(), 2); + } + + #[test] + fn emails_are_lowercased_and_old_commits_skipped() { + let fixture = Fixture::new(); + fixture.commit("upper", "Person@Example.COM"); + commit( + fixture.dir.path(), + "ancient", + "person@example.com", + fixture.window.since - Days::new(1), + ); + let scan = fixture.scan(&[]); + assert!( + counted(&scan).contains(&("person@example.com".to_owned(), fixture.current)), + "{scan:?}" + ); + assert_eq!(scan.commits.len(), 4); + } +} diff --git a/knotmirror/crates/knotmirror-index/src/lib.rs b/knotmirror/crates/knotmirror-index/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..8c3f9f13234b995491a2417350025c9a51cc4b8f --- /dev/null +++ b/knotmirror/crates/knotmirror-index/src/lib.rs @@ -0,0 +1,204 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use bobbin_runtime::Clock; +use jacquard_common::types::did::Did; +use knotmirror_db::{Db, DbError, IndexRow}; +use knotmirror_git::layout::Layout; +use knotmirror_types::{OpsEndpoint, backoff_us, now_us}; +use thiserror::Error; +use tokio::sync::{Notify, Semaphore}; +use tokio_util::sync::CancellationToken; +use tracing::{Instrument as _, debug, error, info, warn}; + +pub mod activity; + +const BATCH_SIZE: i64 = 64; + +#[derive(Clone, Debug)] +pub struct IndexConfig { + pub concurrency: usize, + pub poll_interval: Duration, + pub langs_timeout: Duration, + pub zoekt_url: Option, +} + +pub struct IndexRuntime { + pub db: Db, + pub clock: Arc, + pub cancel: CancellationToken, + pub http: reqwest::Client, + pub layout: Layout, + pub notify: Arc, +} + +#[derive(Debug, Error)] +pub enum IndexError { + #[error("database: {0}")] + Db(#[from] DbError), +} + +struct Job { + db: Db, + clock: Arc, + http: reqwest::Client, + layout: Layout, + langs_timeout: Duration, + zoekt_url: Option, +} + +struct InFlightGuard { + did: Did, + set: Arc>, +} + +impl Drop for InFlightGuard { + fn drop(&mut self) { + let _ = self.set.remove_sync(&self.did); + } +} + +pub async fn run(config: IndexConfig, rt: IndexRuntime) -> Result<(), IndexError> { + info!("index worker started"); + + let job = Arc::new(Job { + db: rt.db.clone(), + clock: rt.clock.clone(), + http: rt.http.clone(), + layout: rt.layout.clone(), + langs_timeout: config.langs_timeout, + zoekt_url: config.zoekt_url.clone(), + }); + let semaphore = Arc::new(Semaphore::new(config.concurrency)); + let in_flight: Arc> = Arc::new(scc::HashSet::new()); + let drained = Arc::new(Notify::new()); + + loop { + if rt.cancel.is_cancelled() { + break; + } + + let mut spawned = 0; + for row in rt.db.pending_index(now_us(&*rt.clock), BATCH_SIZE).await? { + if in_flight.contains_sync(&row.did) { + continue; + } + let Ok(permit) = semaphore.clone().try_acquire_owned() else { + break; + }; + if in_flight.insert_sync(row.did.clone()).is_err() { + continue; + } + let guard = InFlightGuard { + did: row.did.clone(), + set: in_flight.clone(), + }; + + let job = job.clone(); + let drained = drained.clone(); + let span = tracing::info_span!("index", did = %row.did); + tokio::spawn( + async move { + let _guard = guard; + let _permit = permit; + job.run(row).await; + drained.notify_one(); + } + .instrument(span), + ); + spawned += 1; + } + + if spawned == 0 { + tokio::select! { + () = rt.cancel.cancelled() => break, + () = drained.notified() => {} + () = rt.notify.notified() => {} + () = rt.clock.sleep(config.poll_interval) => {} + } + } + } + + info!("index worker stopped"); + Ok(()) +} + +impl Job { + async fn run(&self, row: IndexRow) { + let commit = row.default_branch_commit.as_deref(); + + // trigger zoekt code index + // TODO(post-1.0): zoekt can sync on its own, not relying on here. + if let (Some(url), Some(commit)) = (&self.zoekt_url, commit) { + tokio::spawn( + request_zoekt( + self.http.clone(), + url.admin("enqueueIndex"), + serde_json::json!({ + "repo": row.did.as_str(), + "branches": [{ "name": row.default_branch.as_str(), "version": commit }], + }), + ) + .in_current_span(), + ); + } + + let langs = match commit { + Some(commit) => { + index_languages(self.layout.repo_path(&row.did), commit, self.langs_timeout).await + } + None => Ok(Vec::new()), + }; + let write = match langs { + Ok(langs) => self.db.set_languages(&row.did, commit, &langs).await, + Err(e) => { + self.db + .mark_index_failed( + &row.did, + commit, + &e, + now_us(&*self.clock) + backoff_us(row.index_retry_count), + ) + .await + } + }; + if let Err(e) = write { + error!(error = %e, "recording index outcome failed"); + } + } +} + +async fn index_languages( + path: PathBuf, + commit: &str, + timeout: Duration, +) -> Result, String> { + let commit = knot_types::Oid::from_hex(commit).map_err(|e| e.to_string())?; + tokio::task::spawn_blocking(move || { + let repo = knot_git::Repo::open(path).map_err(|e| e.to_string())?; + let started = Instant::now(); + let sizes = knot_langs::analyze(&repo, commit, Some(started + timeout)) + .map_err(|e| e.to_string())?; + debug!(elapsed = ?started.elapsed(), langs = sizes.len(), "analyzed languages"); + Ok(sizes + .into_iter() + .map(|(name, bytes)| { + ( + name.as_str().to_owned(), + i64::try_from(bytes.get()).unwrap_or(i64::MAX), + ) + }) + .collect()) + }) + .await + .map_err(|_| "language index task panicked".to_owned())? +} + +async fn request_zoekt(http: reqwest::Client, url: String, body: serde_json::Value) { + match http.post(&url).json(&body).send().await { + Ok(resp) if resp.status().is_success() => debug!("requested zoekt index"), + Ok(resp) => warn!(status = %resp.status(), "zoekt index request rejected"), + Err(e) => warn!(error = %e, "zoekt index request failed"), + } +} diff --git a/knotmirror/crates/knotmirror-ingest/Cargo.toml b/knotmirror/crates/knotmirror-ingest/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..2d50e46c0f5105f42989dec6ae49dec103c5350a --- /dev/null +++ b/knotmirror/crates/knotmirror-ingest/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "knotmirror-ingest" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-runtime = { workspace = true } +knotmirror-db = { workspace = true } +lexicons = { workspace = true } + +jacquard-common = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[lints] +workspace = true diff --git a/knotmirror/crates/knotmirror-ingest/src/lib.rs b/knotmirror/crates/knotmirror-ingest/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..7921a1a3dc30f520d4ee792fcbac45693d1dd01f --- /dev/null +++ b/knotmirror/crates/knotmirror-ingest/src/lib.rs @@ -0,0 +1,328 @@ +//! Mirror `org.tangled.git.refs` records into postgres. + +use std::sync::Arc; +use std::time::Duration; + +use bobbin_runtime::{Clock, WsConn, WsMessage, WsTransport}; +use jacquard_common::types::collection::Collection as _; +use jacquard_common::types::did::Did; +use jacquard_common::types::tid::Tid; +use knotmirror_db::{Db, DbError, GitRefs, RefsUpsert}; +use lexicons::org_tangled::git::refs::{Refs, RefsRecord}; +use serde::Deserialize; +use serde_json::value::RawValue; +use thiserror::Error; +use tokio::sync::Notify; +use tokio_util::sync::CancellationToken; +use tracing::{debug, info, warn}; +use url::Url; + +const REFS_COLLECTION: &str = RefsRecord::NSID; +const REFS_RKEY: &str = "self"; +const TANGLED_PREFIX: &str = "org.tangled."; + +pub const DEFAULT_CHECKPOINT_INTERVAL: Duration = Duration::from_secs(10); + +const RECONNECT_TIMEOUT: Duration = Duration::from_secs(15); +const RECONNECT_DELAY: Duration = Duration::from_secs(1); +const IDLE_TIMEOUT: Duration = Duration::from_secs(90); + +#[derive(Debug, Error)] +pub enum IngestError { + #[error("database: {0}")] + Db(#[from] DbError), + #[error("invalid hydrant url: {0}")] + Url(String), +} + +pub struct IngestConfig { + /// Base URL of hydrant; `http(s)://` is rewritten to `ws(s)://`. + pub hydrant_url: Url, + /// How often a stretch of events that touched no row still advances the stored cursor. + pub checkpoint_interval: Duration, +} + +impl IngestConfig { + pub fn new(hydrant_url: Url) -> Self { + Self { + hydrant_url, + checkpoint_interval: DEFAULT_CHECKPOINT_INTERVAL, + } + } +} + +pub struct IngestRuntime { + pub db: Db, + pub clock: Arc, + pub ws: Arc, + pub cancel: CancellationToken, + pub sync_notify: Arc, +} + +#[derive(Debug, Deserialize)] +struct Frame { + /// Absent on error frames. + #[serde(default)] + id: u64, + #[serde(rename = "type")] + kind: String, + #[serde(default)] + record: Option, + #[serde(default)] + account: Option, + /// Set on `type: "error"` frames, sent right before hydrant closes the stream. + #[serde(default)] + error: Option, + #[serde(default)] + message: Option, +} + +#[derive(Debug, Deserialize)] +struct RecordFrame { + did: Did, + collection: String, + rkey: String, + action: String, + #[serde(default)] + record: Option>, + #[serde(default)] + cid: Option, + /// The account's atproto commit rev. Kept as a string so a malformed one skips this + /// record instead of failing the whole frame. + #[serde(default)] + rev: Option, +} + +#[derive(Debug, Deserialize)] +struct AccountFrame { + did: Did, + active: bool, +} + +fn stream_url(base: &Url, cursor: u64) -> Result { + let mut url = base.clone(); + let scheme = match url.scheme() { + "http" => "ws", + "https" => "wss", + s @ ("ws" | "wss") => s, + other => return Err(IngestError::Url(format!("unsupported scheme {other}"))), + } + .to_owned(); + url.set_scheme(&scheme) + .map_err(|()| IngestError::Url(format!("cannot set scheme {scheme}")))?; + url.set_path("/stream"); + url.query_pairs_mut() + .clear() + .append_pair("cursor", &cursor.to_string()); + Ok(url) +} + +pub async fn run(config: IngestConfig, rt: IngestRuntime) -> Result<(), IngestError> { + // fail fast on a bad url + stream_url(&config.hydrant_url, 0)?; + loop { + let last = rt.db.cursor().await?; + let resume = last.map_or(0, |c| c + 1); + let url = stream_url(&config.hydrant_url, resume)?; + info!(%url, last, "connecting to hydrant /stream"); + + let connect = tokio::select! { + () = rt.cancel.cancelled() => break, + r = tokio::time::timeout(RECONNECT_TIMEOUT, rt.ws.connect(url)) => r, + }; + match connect { + Ok(Ok(conn)) => consume(conn, &rt, config.checkpoint_interval).await?, + Ok(Err(e)) => warn!(error = %e, "hydrant connect failed"), + Err(_) => warn!(timeout = ?RECONNECT_TIMEOUT, "hydrant connect timed out"), + } + + if rt.cancel.is_cancelled() { + break; + } + rt.clock.sleep(RECONNECT_DELAY).await; + } + info!("knotmirror-ingest stopped after shutdown signal"); + Ok(()) +} + +/// Drains one websocket session. Returns `Ok` when the session ends for any transport reason; +/// only database errors are fatal. +async fn consume( + conn: WsConn, + rt: &IngestRuntime, + checkpoint_interval: Duration, +) -> Result<(), IngestError> { + let WsConn { + mut sink, + mut stream, + } = conn; + let mut unflushed: Option = None; + let mut next_flush = rt.clock.now_instant() + checkpoint_interval; + let mut idle_deadline = rt.clock.now_instant() + IDLE_TIMEOUT; + + let result = loop { + tokio::select! { + biased; + () = rt.cancel.cancelled() => { + let _ = sink + .send(WsMessage::Close { code: 1000, reason: "knotmirror shutdown".to_owned() }) + .await; + break Ok(()); + } + () = rt.clock.sleep_until(next_flush) => { + flush(rt, &mut unflushed).await?; + next_flush = rt.clock.now_instant() + checkpoint_interval; + } + () = rt.clock.sleep_until(idle_deadline) => { + warn!(timeout = ?IDLE_TIMEOUT, "hydrant stream idle, reconnecting"); + break Ok(()); + } + msg = stream.next() => { + idle_deadline = rt.clock.now_instant() + IDLE_TIMEOUT; + match msg { + None => { + warn!("hydrant stream ended"); + break Ok(()); + } + Some(Err(e)) => { + warn!(error = %e, "hydrant stream failed"); + break Ok(()); + } + Some(Ok(WsMessage::Text(text))) => { + let frame: Frame = match serde_json::from_str(&text) { + Ok(frame) => frame, + Err(e) => { + warn!(error = %e, "undecodable hydrant frame, reconnecting"); + break Ok(()); + } + }; + if frame.kind == "error" { + warn!( + error = frame.error.as_deref().unwrap_or_default(), + message = frame.message.as_deref().unwrap_or_default(), + "hydrant closed stream with error" + ); + break Ok(()); + } + let cursor = frame.id; + if let Err(e) = handle(frame, rt).await { + break Err(e); + } + unflushed = Some(cursor); + } + Some(Ok(WsMessage::Ping(payload))) => { + if let Err(e) = sink.send(WsMessage::Pong(payload)).await { + warn!(error = %e, "hydrant pong failed"); + break Ok(()); + } + } + Some(Ok(WsMessage::Close { code, reason })) => { + debug!(code, %reason, "hydrant closed stream"); + break Ok(()); + } + Some(Ok(WsMessage::Pong(_) | WsMessage::Binary(_))) => {} + } + } + } + }; + flush(rt, &mut unflushed).await?; + result +} + +async fn flush(rt: &IngestRuntime, unflushed: &mut Option) -> Result<(), IngestError> { + if let Some(cursor) = unflushed.take() { + rt.db.set_cursor(cursor).await?; + debug!(cursor, "checkpointed hydrant cursor"); + } + Ok(()) +} + +async fn handle(frame: Frame, rt: &IngestRuntime) -> Result<(), IngestError> { + let cursor = frame.id; + + if let Some(record) = frame.record { + if !record.collection.starts_with(TANGLED_PREFIX) { + return Ok(()); + } + if record.collection == REFS_COLLECTION && record.rkey == REFS_RKEY { + handle_refs(&record, cursor, rt).await?; + } + let did = &record.did; + let Some(rev) = record.rev.as_deref() else { + return Ok(()); + }; + if Tid::new(rev).is_err() { + warn!(%did, cursor, rev, "malformed rev, skipping"); + return Ok(()); + } + if rt.db.bump_rev(did, rev).await? { + debug!(%did, cursor, rev, "atproto rev moved, queued resync"); + rt.sync_notify.notify_one(); + } + Ok(()) + } else if let Some(account) = frame.account + && !account.active + { + let did = &account.did; + rt.db.mark_deleted(did).await?; + info!(%did, cursor, "account inactive, tombstoned row"); + Ok(()) + } else { + Ok(()) + } +} + +async fn handle_refs( + record: &RecordFrame, + cursor: u64, + rt: &IngestRuntime, +) -> Result<(), IngestError> { + let did = &record.did; + match record.action.as_str() { + "create" | "update" => { + let Some(raw) = record.record.as_ref() else { + warn!(%did, cursor, "refs create/update with no body, skipping"); + return Ok(()); + }; + let Some(cid) = record.cid.as_ref() else { + warn!(%did, cursor, "refs create/update with no cid, skipping"); + return Ok(()); + }; + let refs: Refs = match serde_json::from_str(raw.get()) { + Ok(refs) => refs, + Err(e) => { + warn!(%did, cursor, error = %e, "undecodable refs record, skipping"); + return Ok(()); + } + }; + let git_refs = GitRefs { + default_branch: (&*refs.default_branch).into(), + refs: refs.refs, + }; + let changed = rt + .db + .upsert_refs(RefsUpsert { + did, + refs_cid: cid, + refs: &git_refs, + }) + .await?; + if changed { + debug!(%did, cursor, "stored refs"); + rt.sync_notify.notify_one(); + } else { + debug!(%did, cursor, "refs unchanged, skipped write"); + } + Ok(()) + } + "delete" => { + rt.db.mark_deleted(did).await?; + info!(%did, cursor, "refs record deleted, tombstoned row"); + Ok(()) + } + other => { + debug!(%did, cursor, action = other, "ignoring unknown record action"); + Ok(()) + } + } +} diff --git a/knotmirror/crates/knotmirror-maintenance/Cargo.toml b/knotmirror/crates/knotmirror-maintenance/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..734bae20ee721cb16ff1c6dbad72a943fea0778a --- /dev/null +++ b/knotmirror/crates/knotmirror-maintenance/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "knotmirror-maintenance" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-runtime = { workspace = true } +knotmirror-db = { workspace = true } +knotmirror-git = { workspace = true } +knot-git = { workspace = true } +knot-maintenance = { workspace = true } +knot-types = { workspace = true } + +jacquard-common = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/knotmirror/crates/knotmirror-maintenance/src/lib.rs b/knotmirror/crates/knotmirror-maintenance/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..c253ee4ccd529aa5a5015257d44ebf8ed86bc0b0 --- /dev/null +++ b/knotmirror/crates/knotmirror-maintenance/src/lib.rs @@ -0,0 +1,118 @@ +//! periodic repository maintenance (repack, prune, commit-graph, midx) + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use bobbin_runtime::Clock; +use jacquard_common::types::did::Did; +use knot_maintenance::{RepackStatus, Report}; +use knot_types::UnixSeconds; +use knotmirror_db::Db; +use knotmirror_git::layout::Layout; +use tokio_util::sync::CancellationToken; +use tracing::{error, info, warn}; + +pub struct MaintenanceConfig { + pub interval: Duration, + pub options: knot_maintenance::Options, +} + +pub struct MaintenanceRuntime { + pub db: Db, + pub clock: Arc, + pub cancel: CancellationToken, + pub layout: Layout, +} + +pub struct MaintenanceWorker { + rt: MaintenanceRuntime, + config: MaintenanceConfig, +} + +impl MaintenanceWorker { + pub fn new(rt: MaintenanceRuntime, config: MaintenanceConfig) -> Self { + Self { rt, config } + } + + pub async fn run(self) { + info!("maintenance worker started"); + loop { + tokio::select! { + () = self.rt.cancel.cancelled() => break, + () = self.rt.clock.sleep(self.config.interval) => {} + } + let repos = match self.rt.db.active_repos().await { + Ok(repos) => repos, + Err(e) => { + error!(error = %e, "listing repos for maintenance failed"); + continue; + } + }; + let started = Instant::now(); + let count = repos.len(); + info!(repos = count, "maintenance pass started"); + for did in repos { + if self.rt.cancel.is_cancelled() { + break; + } + self.maintain_one(did).await; + } + info!( + repos = count, + duration_ms = started.elapsed().as_millis() as u64, + "maintenance pass finished" + ); + } + info!("maintenance worker stopped"); + } + + async fn maintain_one(&self, did: Did) { + maintain(&self.rt.layout, &*self.rt.clock, did, self.config.options).await; + } +} + +pub async fn maintain( + layout: &Layout, + clock: &dyn Clock, + did: Did, + options: knot_maintenance::Options, +) -> Option { + let now = UnixSeconds::new((clock.now_unix_micros().raw() / 1_000_000) as i64); + let path = layout.repo_path(&did); + let outcome = tokio::task::spawn_blocking(move || { + knot_git::Repo::open(&path) + .map_err(knot_maintenance::MaintError::from) + .and_then(|repo| knot_maintenance::run_repo(&repo, now, &options)) + }) + .await; + match outcome { + Ok(Ok(report)) => { + report_skips(&did, &report); + Some(report) + } + Ok(Err(error)) => { + error!(repo = %did, %error, "maintenance run failed"); + None + } + Err(join) => { + error!(repo = %did, %join, "maintenance task panicked"); + None + } + } +} + +fn report_skips(did: &Did, report: &Report) { + match report.repack.status { + RepackStatus::SkippedTooLarge => warn!( + repo = %did, + reason = "reachable set exceeds repack_max_objects", + "skipped repack" + ), + RepackStatus::ClosureFailed => warn!( + repo = %did, + reason = "couldn't compute reachable set", + "skipped repack and prune" + ), + _ => {} + } +} diff --git a/knotmirror/crates/knotmirror-sync/Cargo.toml b/knotmirror/crates/knotmirror-sync/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..41fe692fc13814f741649c10cb40b6c9432b5e40 --- /dev/null +++ b/knotmirror/crates/knotmirror-sync/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "knotmirror-sync" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-runtime = { workspace = true } +knotmirror-db = { workspace = true } +knotmirror-git = { workspace = true } +knotmirror-types = { workspace = true } +knot-git = { workspace = true } +knot-resource = { workspace = true } + +gix = { workspace = true } +gix-hash = { workspace = true } +gix-negotiate = { workspace = true } +gix-object = { workspace = true } +gix-pack = { workspace = true } +gix-protocol = { workspace = true } +gix-transport = { workspace = true, features = ["http-client-reqwest-rust-tls"] } +jacquard-common = { workspace = true } +jacquard-identity = { workspace = true } +metrics = { workspace = true } +reqwest = { workspace = true } +scc = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +sqlx = { workspace = true } +tempfile = { workspace = true } + +[lints] +workspace = true diff --git a/knotmirror/crates/knotmirror-sync/src/admission.rs b/knotmirror/crates/knotmirror-sync/src/admission.rs new file mode 100644 index 0000000000000000000000000000000000000000..6595e678566569e43a0f945b0e80566456454ead --- /dev/null +++ b/knotmirror/crates/knotmirror-sync/src/admission.rs @@ -0,0 +1,35 @@ +#![allow(dead_code)] + +use std::sync::Arc; + +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use url::Url; + +#[derive(Clone)] +pub(crate) struct BackfillAdmission { + gates: Arc>>, + per_knot_limit: usize, +} + +impl BackfillAdmission { + pub(crate) fn new(global_limit: usize, per_knot_limit: usize) -> Self { + // leave room for another knot whenever the global limit allows it + let fair_limit = global_limit.saturating_sub(1).max(1); + Self { + gates: Arc::new(scc::HashMap::new()), + per_knot_limit: per_knot_limit.min(fair_limit).max(1), + } + } + + pub(crate) async fn try_acquire(&self, knot: &Url) -> Option { + let limit = self.per_knot_limit; + let gate = self + .gates + .entry_async(knot.clone()) + .await + .or_insert_with(|| Arc::new(Semaphore::new(limit))) + .get() + .clone(); + gate.try_acquire_owned().ok() + } +} diff --git a/knotmirror/crates/knotmirror-sync/src/lib.rs b/knotmirror/crates/knotmirror-sync/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..205bdfd1dd42303bc4e252f434abe220065e8580 --- /dev/null +++ b/knotmirror/crates/knotmirror-sync/src/lib.rs @@ -0,0 +1,328 @@ +//! sync git repository objects (packfile) +#![allow( + clippy::result_large_err, + reason = "gix's own error types are this large" +)] + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use bobbin_runtime::Clock; + +use crate::admission::BackfillAdmission; +use jacquard_common::types::did::Did; +use knotmirror_db::{Db, DbError}; +use knotmirror_git::layout::Layout; +use knotmirror_types::{KnotPolicy, backoff_us, now_us}; +use metrics::{gauge, histogram}; +use thiserror::Error; +use tokio::sync::{Notify, Semaphore}; +use tokio_util::sync::CancellationToken; +use tracing::{Instrument as _, error, info, warn}; + +mod admission; +mod task; + +/// The identity resolver, same concrete type the XRPC side uses. +pub type Directory = jacquard_identity::JacquardResolver; + +#[derive(Debug, Error)] +pub enum SyncError { + #[error("database: {0}")] + Db(#[from] DbError), +} + +/// Rows pulled per queue scan. Not a config knob: it only bounds one pass's memory, and the +/// semaphore is what actually limits work in flight. +const BATCH_SIZE: i64 = 64; + +/// How long a repo is parked when the knot was busy rather than broken. Short, because this is +/// not a backoff: it only has to be long enough to get the row out of the scan window. +const DEFER_HOLD_US: i64 = 5 * 1_000_000; + +/// Metric-only split between `backoff` and `error`; retries continue past it. +const MAX_RETRY: i32 = 12; + +const STATE_METRICS_INTERVAL: Duration = Duration::from_secs(30); + +pub const DEFAULT_CONCURRENCY: usize = 8; +pub const DEFAULT_PER_KNOT_CONCURRENCY: usize = 4; +pub const DEFAULT_POLL_INTERVAL: Duration = Duration::from_secs(5); + +pub struct SyncConfig { + pub concurrency: usize, + pub per_knot_concurrency: usize, + pub poll_interval: Duration, +} + +impl Default for SyncConfig { + fn default() -> Self { + Self { + concurrency: DEFAULT_CONCURRENCY, + per_knot_concurrency: DEFAULT_PER_KNOT_CONCURRENCY, + poll_interval: DEFAULT_POLL_INTERVAL, + } + } +} + +pub struct SyncRuntime { + pub db: Db, + pub clock: Arc, + pub cancel: CancellationToken, + pub http: reqwest::Client, + pub directory: Arc, + pub layout: Layout, + pub knot_policy: KnotPolicy, + pub index_notify: Arc, + /// ingest wakes this when it queues a repo, so a sync doesn't wait out `poll_interval` + pub notify: Arc, +} + +// NOTE: see hydrant/src/backfill/worker.rs as reference + +pub struct BackfillWorker { + rt: SyncRuntime, + poll_interval: Duration, + semaphore: Arc, + admission: BackfillAdmission, + in_flight: Arc>, + /// Woken by a finishing task so a pass that hit the concurrency limit picks up the next repo + /// immediately instead of waiting out `poll_interval`. + drained: Arc, +} + +impl BackfillWorker { + pub fn new(rt: SyncRuntime, config: SyncConfig) -> Self { + Self { + rt, + poll_interval: config.poll_interval, + semaphore: Arc::new(Semaphore::new(config.concurrency)), + admission: BackfillAdmission::new(config.concurrency, config.per_knot_concurrency), + in_flight: Arc::new(scc::HashSet::new()), + drained: Arc::new(Notify::new()), + } + } +} + +struct InFlightGuard { + did: Did, + set: Arc>, +} + +impl Drop for InFlightGuard { + fn drop(&mut self) { + let _ = self.set.remove_sync(&self.did); + gauge!("gitmirror_sync_in_flight").decrement(1); + } +} + +impl BackfillWorker { + pub async fn run(self) -> Result<(), SyncError> { + info!("backfill worker started"); + + loop { + if self.rt.cancel.is_cancelled() { + break; + } + + let mut spawned = 0; + for row in self + .rt + .db + .pending_repos(now_us(&*self.rt.clock), BATCH_SIZE) + .await? + { + if self.in_flight.contains_sync(&row.did) { + continue; + } + + let permit = match self.semaphore.clone().try_acquire_owned() { + Ok(p) => p, + Err(_) => break, + }; + + // only mark as in flight if we can acquire a permit + if self.in_flight.insert_sync(row.did.clone()).is_err() { + // a task is already running, weh + // so we don't need this one anymore... + break; + } + + gauge!("gitmirror_sync_in_flight").increment(1); + let guard = InFlightGuard { + did: row.did.clone(), + set: self.in_flight.clone(), + }; + + let db = self.rt.db.clone(); + let clock = self.rt.clock.clone(); + let http = self.rt.http.clone(); + let directory = self.rt.directory.clone(); + let layout = self.rt.layout.clone(); + let knot_policy = self.rt.knot_policy; + let admission = self.admission.clone(); + let drained = self.drained.clone(); + let index_notify = self.rt.index_notify.clone(); + + let span = tracing::info_span!( + "backfill", + did = %row.did, + knot = tracing::field::Empty + ); + tokio::spawn( + async move { + let _guard = guard; + let started = Instant::now(); + let res = task::did_task( + &http, + &directory, + &layout, + knot_policy, + &row, + &admission, + permit, + ) + .await; + + let duration = started.elapsed().as_secs_f64(); + let outcome = match &res { + Ok(task::TaskDisposition::Finished { .. }) => "finished", + Ok(task::TaskDisposition::Deferred) => "deferred", + Err(_) => "failed", + }; + histogram!("gitmirror_sync_task_duration_seconds", "outcome" => outcome) + .record(duration); + info!( + outcome, + duration_ms = duration * 1e3, + retry_count = row.retry_count, + error = res.as_ref().err().map(tracing::field::display), + "sync finished" + ); + + let write = match &res { + Ok(task::TaskDisposition::Finished { + default_branch_commit, + }) => { + let commit = default_branch_commit.map(|oid| oid.to_string()); + let res = db.mark_synced(&row, commit.as_deref()).await; + if res.is_ok() { + index_notify.notify_one(); + } + res + } + Ok(task::TaskDisposition::Deferred) => { + db.mark_deferred(&row, now_us(&*clock) + DEFER_HOLD_US) + .await + } + Err(e) => { + db.mark_sync_failed( + &row, + &e.to_string(), + now_us(&*clock) + backoff_us(row.retry_count), + ) + .await + } + }; + + if let Err(e) = write { + error!(error = %e, "recording sync outcome failed"); + } + + // wake worker to pick up more (in case we were sleeping at limit) + drained.notify_one(); + } + .instrument(span), + ); + + spawned += 1; + } + + if spawned == 0 + && !woken( + &self.rt.cancel, + &self.drained, + &self.rt.notify, + &*self.rt.clock, + self.poll_interval, + ) + .await + { + break; + } + // loop immediately since we might have more tasks + } + + info!("backfill worker stopped"); + Ok(()) + } +} + +async fn woken( + cancel: &CancellationToken, + drained: &Notify, + queued: &Notify, + clock: &dyn Clock, + poll_interval: Duration, +) -> bool { + tokio::select! { + () = cancel.cancelled() => false, + () = drained.notified() => true, + () = queued.notified() => true, + () = clock.sleep(poll_interval) => true, + } +} + +/// Publishes `knotmirror_sync_repos` from the DB until cancelled. +pub async fn report_repo_states(db: Db, clock: Arc, cancel: CancellationToken) { + loop { + match db.repo_state_counts(now_us(&*clock), MAX_RETRY).await { + Ok(rows) => { + for state in ["pending", "active", "backoff", "error"] { + for due in [true, false] { + let count = rows + .iter() + .find(|(s, d, _)| s == state && *d == due) + .map_or(0, |(_, _, c)| *c); + gauge!("gitmirror_sync_repos", "state" => state, "due" => due.to_string()) + .set(count as f64); + } + } + } + Err(e) => warn!(error = %e, "counting repo states failed"), + } + + tokio::select! { + () = cancel.cancelled() => break, + () = clock.sleep(STATE_METRICS_INTERVAL) => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bobbin_runtime::SystemClock; + + #[tokio::test] + async fn a_queued_repo_ends_the_idle_wait_before_the_poll() { + let (cancel, drained, queued) = (CancellationToken::new(), Notify::new(), Notify::new()); + let clock = SystemClock::new(); + let waiting = woken( + &cancel, + &drained, + &queued, + &clock, + Duration::from_secs(3600), + ); + let queueing = async { + tokio::task::yield_now().await; + queued.notify_one(); + }; + let woke = tokio::time::timeout(Duration::from_secs(1), async { + tokio::join!(waiting, queueing).0 + }) + .await; + assert_eq!(woke, Ok(true)); + } +} diff --git a/knotmirror/crates/knotmirror-sync/src/task.rs b/knotmirror/crates/knotmirror-sync/src/task.rs new file mode 100644 index 0000000000000000000000000000000000000000..992997e84a99e24cff44faf9c16fcb0074e60f8c --- /dev/null +++ b/knotmirror/crates/knotmirror-sync/src/task.rs @@ -0,0 +1,433 @@ +use std::path::Path; +use std::sync::atomic::AtomicBool; + +use gix::refs::FullName; +use gix_hash::ObjectId; +use jacquard_common::types::did::Did; +use jacquard_identity::resolver::IdentityResolver as _; +use knotmirror_db::RepoRow; +use knotmirror_git::layout::Layout; +use knotmirror_types::jacquard_ext::DidDocumentExt as _; +use knotmirror_types::{KnotHost, KnotPolicy, KnotRepoUrl as _}; +use thiserror::Error; +use tracing::debug; + +use crate::Directory; +use crate::admission::BackfillAdmission; + +const AGENT: &str = concat!("knotmirror/", env!("CARGO_PKG_VERSION")); +const LISTED_PREFIXES: [&str; 4] = ["refs/heads/", "refs/tags/", "refs/cobs/", "refs/tngl/keep/"]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TaskDisposition { + Finished { + default_branch_commit: Option, + }, + Deferred, +} + +#[derive(Debug, Error)] +pub enum BackfillError { + #[error("resolve {0}: {1}")] + Resolve(Did, String), + #[error("{0} declares no knot service")] + NoKnotEndpoint(Did), + #[error("{0} declares an unusable knot endpoint: {1}")] + UnsafeKnotEndpoint(Did, &'static str), + #[error("malformed refs record: {0}")] + MalformedRefs(String), + #[error("git: {0}")] + Git(#[from] BackfillGitError), + #[error("fetch task panicked")] + Panicked, +} + +#[derive(Debug, Error)] +pub enum BackfillGitError { + #[error("init repo: {0}")] + Init(#[from] gix::init::Error), + #[error("repo: {0}")] + Repo(#[from] knot_git::GitError), + #[error("write: {0}")] + Write(#[from] knot_resource::FsError), + #[error("peel: {0}")] + Peel(#[from] gix::object::find::existing::Error), + #[error("decode tag: {0}")] + DecodeTag(#[from] gix_object::decode::Error), + #[error("connect host: {0}")] + Connect(#[from] gix_transport::client::blocking_io::connect::Error), + #[error("handshake host: {0}")] + Handshake(#[from] gix_protocol::handshake::Error), + #[error("list cob refs: {0}")] + LsRefs(#[from] gix_protocol::ls_refs::Error), + #[error("fetch repo: {0}")] + Fetch(#[from] gix_protocol::fetch::Error), + #[error("unsupported object format: {0}")] + ObjectFormat(String), + #[error("object format mismatch: local {local}, knot {remote}")] + ObjectFormatMismatch { + local: gix_hash::Kind, + remote: gix_hash::Kind, + }, +} + +/// syncs git repository +pub async fn did_task( + _http: &reqwest::Client, + directory: &Directory, + layout: &Layout, + policy: KnotPolicy, + repo: &RepoRow, + admission: &BackfillAdmission, + _permit: tokio::sync::OwnedSemaphorePermit, +) -> Result { + // git-fetch is usually done in three steps: + // 1. handshake (host capability check) + // 2. advertisement (ls-refs) + // 3. git-upload-pack + // + // The handshake result will be cached per-host, and we already know the refs list (in db). + // So we can go straight to git-upload-pack call. + + let knot = knot_host(directory, &repo.did, policy).await?; + let knot_url = knot.url(); + tracing::Span::current().record("knot", knot_url.as_str()); + + let Some(_knot_permit) = admission.try_acquire(knot_url).await else { + return Ok(TaskDisposition::Deferred); + }; + + let default_branch = FullName::try_from(format!("refs/heads/{}", repo.refs.default_branch)) + .map_err(|e| BackfillError::MalformedRefs(e.to_string()))?; + + let url = knot.repo_url(repo.did.as_str()); + let path = layout.repo_path(&repo.did); + let refs = repo.refs.refs.clone(); + let default_branch_commit = + tokio::task::spawn_blocking(move || fetch_into(&url, &path, &refs, default_branch)) + .await + .map_err(|_| BackfillError::Panicked)??; + + Ok(TaskDisposition::Finished { + default_branch_commit, + }) +} + +/// The repo's own knot, from its DID document. Re-resolved every task: a repo can migrate. +async fn knot_host( + directory: &Directory, + did: &Did, + policy: KnotPolicy, +) -> Result { + let document = directory + .resolve_did_doc_owned(did) + .await + .map_err(|e| BackfillError::Resolve(did.clone(), e.to_string()))?; + + let endpoint = document + .knot_endpoint() + // TODO(post-1.0): remove legacy `#atproto_pds` fallback + .or_else(|| document.pds_endpoint()) + .ok_or_else(|| BackfillError::NoKnotEndpoint(did.clone()))?; + + let host = policy + .safe_knot_host(endpoint.as_ref()) + .map_err(|_reason| { + BackfillError::UnsafeKnotEndpoint(did.clone(), "is invalid knot service endpoint") + })?; + + Ok(host) +} + +fn parse_refs( + blob: &[u8], + kind: gix_hash::Kind, +) -> Result, BackfillError> { + let malformed = |e: &dyn std::fmt::Display| BackfillError::MalformedRefs(e.to_string()); + let mut refs = gix::refs::packed::Iter::new(blob, kind) + .map_err(|e| malformed(&e))? + .map(|entry| entry.map_err(|e| malformed(&e))) + // only branches and tags; e.g. nixpkgs' 400k+ `refs/pull/*` overflow the fetch request + .filter(|entry| { + entry.as_ref().map_or(true, |entry| { + let name = entry.name.as_bstr(); + name.starts_with(b"refs/heads/") || name.starts_with(b"refs/tags/") + }) + }) + .map(|entry| entry.map(|entry| (entry.target(), entry.name.to_owned()))) + .collect::, BackfillError>>()?; + refs.sort_by(|(_, a), (_, b)| a.cmp(b)); + if let Some(pair) = refs.windows(2).find(|pair| pair[0].1 == pair[1].1) { + return Err(malformed(&format_args!("duplicate ref {}", pair[0].1))); + } + Ok(refs) +} + +fn fetch_into( + url: &str, + path: &Path, + refs: &[u8], + default_branch: FullName, +) -> Result, BackfillError> { + let mut transport = gix_transport::client::blocking_io::connect::connect( + url, + gix_transport::client::blocking_io::connect::Options { + version: gix_transport::Protocol::V2, + ssh: Default::default(), + trace: false, + }, + ) + .map_err(BackfillGitError::from)?; + + let mut progress = gix::progress::Discard; + let should_interrupt = AtomicBool::new(false); + + // TODO: cache per knot. We ignore `handshake.refs` which will be None for v2 protocol anyhow + let mut handshake = gix_protocol::handshake( + &mut transport, + gix_transport::Service::UploadPack, + // no auth + |_| Ok(None), + Default::default(), + &mut progress, + ) + .map_err(BackfillGitError::from)?; + + let object_hash = match handshake + .capabilities + .capability("object-format") + .and_then(|c| c.value()) + { + Some(format) => format + .to_string() + .parse() + .map_err(BackfillGitError::ObjectFormat)?, + None => gix_hash::Kind::Sha1, + }; + + // TODO(pre-1.0): restore this just before the release + let _refs = parse_refs(refs, object_hash)?; + let mut refs: Vec<(ObjectId, FullName)> = Vec::new(); + // cob and keep refs aren't in the refs record; the knot lists them only on an explicit ref-prefix + let mut prefixes = gix_protocol::ls_refs::RefPrefixes::new(); + prefixes.extend(LISTED_PREFIXES.map(Into::into)); + let ls_refs = gix_protocol::LsRefsCommand::new( + Some(prefixes), + &handshake.capabilities, + ("agent", Some(AGENT.into())), + ) + .invoke_blocking(&mut transport, &mut progress, false) + .map_err(BackfillGitError::from)?; + for remote_ref in &ls_refs { + let (name, Some(oid), _) = remote_ref.unpack() else { + continue; + }; + if !LISTED_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix.as_bytes())) + { + continue; + } + let name = FullName::try_from(name.to_owned()) + .map_err(|e| BackfillError::MalformedRefs(e.to_string()))?; + refs.push((oid.to_owned(), name)); + } + refs.sort_by(|(_, a), (_, b)| a.cmp(b)); + let default_branch_commit = refs + .iter() + .find(|(_, name)| *name == default_branch) + .map(|(oid, _)| *oid); + + if !path.join("HEAD").exists() { + gix::ThreadSafeRepository::init( + path, + gix::create::Kind::Bare, + gix::create::Options { + object_hash: Some(object_hash), + ..Default::default() + }, + ) + .map_err(BackfillGitError::from)?; + } + let live = knot_git::Repo::open(path).map_err(BackfillGitError::from)?; + if live.git().object_hash() != object_hash { + return Err(BackfillGitError::ObjectFormatMismatch { + local: live.git().object_hash(), + remote: object_hash, + } + .into()); + } + let staging = knot_git::Staging::new(&live).map_err(BackfillGitError::from)?; + let repo = staging.repo().git(); + + let pack_dir = staging.repo().objects_dir().join("pack"); + let odb = repo.objects.clone(); + + let cache = live.git().commit_graph_if_enabled().ok().flatten(); + let mut negotiate = StoredRefs { + repo, + live: live.git(), + wants: &refs, + negotiator: gix_negotiate::Algorithm::Consecutive.into_negotiator(), + graph: repo.revision_graph(cache.as_ref()), + }; + + gix_protocol::fetch( + &mut negotiate, + |pack, progress, should_interrupt| { + gix_pack::Bundle::write_to_directory( + pack, + Some(&pack_dir), + progress, + should_interrupt, + Some(odb.clone()), + gix_pack::bundle::write::Options { + object_hash, + ..Default::default() + }, + ) + .map(|_| true) + }, + &mut progress, + &should_interrupt, + gix_protocol::fetch::Context { + handshake: &mut handshake, + transport: &mut transport, + user_agent: ("agent", Some(AGENT.into())), + trace_packetlines: false, + }, + gix_protocol::fetch::Options { + shallow_file: path.join("shallow"), + shallow: &gix_protocol::fetch::Shallow::NoChange, + tags: gix_protocol::fetch::Tags::None, + reject_shallow_remote: true, + }, + ) + .map_err(BackfillGitError::from)?; + + let packed_refs = packed_refs(repo, &refs)?; + let head_file = format!("ref: {}\n", default_branch.as_bstr()); + live.with_ref_lock(|| { + staging.migrate_into(&live)?; + let git_dir = live.git().git_dir(); + let mode = knot_resource::FileMode::Inherited; + knot_resource::atomic_write_bytes(&git_dir.join("packed-refs"), &packed_refs, mode)?; + knot_resource::atomic_write_bytes(&git_dir.join("HEAD"), head_file.as_bytes(), mode)?; + Ok::<_, BackfillGitError>(()) + })?; + + debug!(refs = refs.len(), "fetched and updated refs"); + Ok(default_branch_commit) +} + +fn packed_refs( + repo: &gix::Repository, + refs: &[(ObjectId, FullName)], +) -> Result, BackfillGitError> { + let mut out = b"# pack-refs with: peeled fully-peeled sorted \n".to_vec(); + for (oid, name) in refs { + out.extend_from_slice(format!("{oid} {}\n", name.as_bstr()).as_bytes()); + if repo.find_header(*oid)?.kind() != gix::object::Kind::Tag { + continue; + } + let mut object = repo.find_object(*oid)?; + while object.kind == gix::object::Kind::Tag { + let target = object.to_tag_ref_iter().target_id()?; + object = repo.find_object(target)?; + } + out.extend_from_slice(format!("^{}\n", object.id).as_bytes()); + } + Ok(out) +} + +struct StoredRefs<'a, 'cache> { + repo: &'a gix::Repository, + live: &'a gix::Repository, + wants: &'a [(ObjectId, FullName)], + negotiator: Box, + graph: gix_negotiate::Graph<'a, 'cache>, +} + +impl gix_protocol::fetch::Negotiate for StoredRefs<'_, '_> { + fn mark_complete_and_common_ref( + &mut self, + ) -> Result { + use gix::refs::file::ReferenceExt as _; + use gix_object::Exists as _; + use gix_protocol::fetch::negotiate::Action; + + let (repo, wants) = (self.repo, self.wants); + if wants.is_empty() { + return Ok(Action::NoChange); + } + + let known: Vec = wants + .iter() + .map(|(oid, _)| repo.objects.exists(oid)) + .collect(); + if known.iter().all(|k| *k) { + return Ok(Action::SkipToRefUpdate); + } + + for (oid, _) in wants.iter().zip(&known).filter_map(|(w, k)| k.then_some(w)) { + // TODO(boltless): load known commit to find max commit_time (used for cutoff_date) + self.negotiator.known_common(*oid, &mut self.graph)?; + } + + // TODO(boltless): walk parents back like + // [`gix_protocol::fetch::negotiate::mark_recent_complete_commits`] does. + let store = &self.live.refs; + let packed = store.cached_packed_buffer()?; + let mut tips = Vec::new(); + for local_ref in store.iter()?.all()? { + let mut local_ref = local_ref?; + let id = local_ref.peel_to_id_packed( + store, + &repo.objects, + packed.as_ref().map(|b| &***b), + )?; + tips.push(id); + } + for tip in tips { + self.negotiator.add_tip(tip, &mut self.graph)?; + } + + Ok(Action::MustNegotiate { + remote_ref_target_known: known, + }) + } + + fn add_wants( + &mut self, + arguments: &mut gix_protocol::fetch::Arguments, + remote_ref_target_known: &[bool], + ) -> bool { + let mut has_want = false; + for (oid, _) in self + .wants + .iter() + .zip(remote_ref_target_known) + .filter_map(|(want, known)| (!known).then_some(want)) + { + arguments.want(oid); + has_want = true; + } + has_want + } + + fn one_round( + &mut self, + state: &mut gix_protocol::fetch::negotiate::one_round::State, + arguments: &mut gix_protocol::fetch::Arguments, + previous_response: Option<&gix_protocol::fetch::Response>, + ) -> Result<(gix_protocol::fetch::negotiate::Round, bool), gix_protocol::fetch::negotiate::Error> + { + gix_protocol::fetch::negotiate::one_round( + &mut *self.negotiator, + &mut self.graph, + state, + arguments, + previous_response, + ) + } +} diff --git a/knotmirror/crates/knotmirror-types/Cargo.toml b/knotmirror/crates/knotmirror-types/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..4b83e519d3e5c4e3413b35289c452a689985c4bb --- /dev/null +++ b/knotmirror/crates/knotmirror-types/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "knotmirror-types" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-runtime = { workspace = true } +jacquard-common = { workspace = true } +bobbin-knot-proxy = { workspace = true } +url = { workspace = true } +serde = { workspace = true } + +[lints] +workspace = true diff --git a/knotmirror/crates/knotmirror-types/src/jacquard_ext.rs b/knotmirror/crates/knotmirror-types/src/jacquard_ext.rs new file mode 100644 index 0000000000000000000000000000000000000000..c0b8b7f7c48045e63482602d07f2efadcd64ced1 --- /dev/null +++ b/knotmirror/crates/knotmirror-types/src/jacquard_ext.rs @@ -0,0 +1,29 @@ +use jacquard_common::Bos; +use jacquard_common::deps::fluent_uri::Uri; +use jacquard_common::types::did_doc::DidDocument; +use jacquard_common::types::string::AtprotoStr; +use jacquard_common::types::value::Data; + +const KNOT_SERVICE_TYPE: &str = "TangledKnot"; + +pub trait DidDocumentExt { + /// The `tangled_knot` service endpoint, if the document declares one. + fn knot_endpoint(&self) -> Option>; +} + +impl + AsRef + Clone> DidDocumentExt for DidDocument { + fn knot_endpoint(&self) -> Option> { + self.service.as_ref().and_then(|services| { + services.iter().find_map(|service| { + if service.r#type.as_ref() != KNOT_SERVICE_TYPE { + return None; + } + match &service.service_endpoint { + Some(Data::String(AtprotoStr::Uri(uri))) => Uri::parse(uri.as_ref()).ok(), + Some(Data::String(AtprotoStr::String(raw))) => Uri::parse(raw.as_ref()).ok(), + _ => None, + } + }) + }) + } +} diff --git a/knotmirror/crates/knotmirror-types/src/lib.rs b/knotmirror/crates/knotmirror-types/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..1cf4a2ab8eee688bc11705753273842f49bfd810 --- /dev/null +++ b/knotmirror/crates/knotmirror-types/src/lib.rs @@ -0,0 +1,135 @@ +pub mod jacquard_ext; + +pub use bobbin_knot_proxy::KnotHost; + +use bobbin_runtime::Clock; + +/// Ceiling on the exponential retry backoff. +const MAX_BACKOFF_SECS: i64 = 60 * 60; + +/// The clock reading `retry_after_us` is written from and compared against. +pub fn now_us(clock: &dyn Clock) -> i64 { + i64::try_from(clock.now_unix_micros().raw()).unwrap_or(i64::MAX) +} + +/// 2^n seconds in micros, capped. +pub fn backoff_us(retry_count: i32) -> i64 { + let shift = retry_count.clamp(0, 62) as u32; + // cap before scaling; the shift alone would overflow the multiply + (1i64 << shift).min(MAX_BACKOFF_SECS) * 1_000_000 +} + +#[derive(Debug, Clone, Copy)] +pub struct KnotPolicy { + pub allow_private: bool, + pub require_https: bool, +} + +impl KnotPolicy { + pub fn safe_knot_host(&self, endpoint: &str) -> Result { + let host = KnotHost::parse(endpoint).map_err(|_| "is not a usable base URL")?; + if self.require_https && host.url().scheme() != "https" { + return Err("is not https"); + } + if !self.allow_private && host.private_literal_reason().is_some() { + return Err("is a private address"); + } + Ok(host) + } +} + +pub trait KnotRepoUrl { + fn repo_url(&self, did: &str) -> String; +} + +impl KnotRepoUrl for KnotHost { + fn repo_url(&self, did: &str) -> String { + format!("{}{}", self.url(), did) + } +} + +#[derive(Debug, Clone)] +pub struct OpsEndpoint { + url: url::Url, +} + +impl OpsEndpoint { + pub fn parse(input: &str) -> Result { + let url = + url::Url::parse(input).map_err(|e| format!("{input:?} isn't a valid base URL: {e}"))?; + if !matches!(url.scheme(), "http" | "https") { + return Err(format!("{input:?} must use http/https")); + } + if url.host_str().is_none() { + return Err(format!("{input:?} hasn't got a host")); + } + if !url.username().is_empty() || url.password().is_some() { + return Err(format!("{input:?} mustn't embed userinfo")); + } + if url.query().is_some() || url.fragment().is_some() { + return Err(format!("{input:?} must be bareback base URL")); + } + let mut url = url; + let path = url.path().trim_end_matches('/').to_owned(); + url.set_path(&path); + Ok(Self { url }) + } + + fn joined(&self, suffix: &str) -> String { + let mut url = self.url.clone(); + url.set_path(&format!( + "{}/{}", + self.url.path().trim_end_matches('/'), + suffix + )); + url.to_string() + } + + pub fn xrpc(&self, nsid: &str) -> String { + self.joined(&format!("xrpc/{nsid}")) + } + + pub fn admin(&self, path: &str) -> String { + self.joined(&format!("admin/{path}")) + } +} + +impl<'de> serde::Deserialize<'de> for OpsEndpoint { + fn deserialize>(deserializer: D) -> Result { + let raw: String = serde::Deserialize::deserialize(deserializer)?; + OpsEndpoint::parse(&raw).map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +mod ops_endpoint_tests { + use super::OpsEndpoint; + + #[test] + fn ops_endpoint_joins_paths_onto_good_base() { + assert_eq!( + OpsEndpoint::parse("https://ops.example") + .unwrap() + .xrpc("a.b.c"), + "https://ops.example/xrpc/a.b.c" + ); + assert_eq!( + OpsEndpoint::parse("http://ops.example/") + .unwrap() + .admin("enqueueIndex"), + "http://ops.example/admin/enqueueIndex" + ); + } + + #[test] + fn anything_else_isnt_classed_as_ops_endpoints() { + for raw in [ + "ops.example", + "ftp://ops.example", + "https://user:pw@ops.example", + "https://ops.example/?q=1", + ] { + assert!(OpsEndpoint::parse(raw).is_err(), "{raw} parsed"); + } + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/Cargo.toml b/knotmirror/crates/knotmirror-xrpc/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..87f506396a141c3cc5b689b81d5378107033204a --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/Cargo.toml @@ -0,0 +1,63 @@ +[package] +name = "knotmirror-xrpc" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-knot-proxy = { workspace = true } +bobbin-runtime = { workspace = true } +knotmirror-db = { workspace = true } +knotmirror-git = { workspace = true } +knotmirror-types = { workspace = true } +gix-receive-pack = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-git = { workspace = true } +knot-langs = { workspace = true } +knot-types = { workspace = true } +lexicons = { workspace = true } +tangled-axum = { workspace = true } + +anyhow = { workspace = true } +axum = { workspace = true } +base64 = { workspace = true } +bs58 = { workspace = true } +chrono = { workspace = true } +deadpool-redis = { workspace = true } +jacquard-axum = { workspace = true } +jacquard-common = { workspace = true } +jacquard-identity = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["net", "rt-multi-thread", "sync", "time"] } +thiserror = { workspace = true } +tracing = "0.1" +line-numbers = "0.4.0" +rustc-hash = "2.1.2" +fixedbitset = "0.5" +itertools = { workspace = true } +k256 = { workspace = true } +moka = { workspace = true } +gix = { version = "0.84", features = ["parallel", "blob-diff", "blame", "merge", "sha1", "sha256", "revision", "tree-editor", "worktree-stream"] } +gix-archive = { workspace = true } +gix-pack = { workspace = true } +gix-transport = { workspace = true, features = ["async-client"] } +quick_cache = { workspace = true } +futures-lite = { workspace = true } +reqwest = { workspace = true } +sha2 = { workspace = true } +ssh-key = { version = "0.7.0-rc.10", default-features = false, features = ["std", "ed25519", "p256", "p384", "p521"] } +tempfile = "3" + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "io-util"] } +knot-record = { workspace = true } +knot-runtime = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +sqlx = { workspace = true } + +[lints] +workspace = true diff --git a/knotmirror/crates/knotmirror-xrpc/clippy.toml b/knotmirror/crates/knotmirror-xrpc/clippy.toml new file mode 100644 index 0000000000000000000000000000000000000000..b2754b898969ffc14d34ae79a52877c13fc5f060 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/clippy.toml @@ -0,0 +1,3 @@ +disallowed-methods = [ + { path = "tokio::task::spawn_blocking", reason = "use crate::metrics::blocking so the pool gauges stay accurate" }, +] diff --git a/knotmirror/crates/knotmirror-xrpc/src/archive.rs b/knotmirror/crates/knotmirror-xrpc/src/archive.rs new file mode 100644 index 0000000000000000000000000000000000000000..1f46cf96fc6b155102863b82d4051a6303ca51d7 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/archive.rs @@ -0,0 +1,170 @@ +//! ported from `knot2/crates/knot-git/src/archive.rs` + +use std::io::{Seek, SeekFrom, Write}; + +use gix::ObjectId; +use gix::bstr::BString; +use gix::error::ResultExt as _; +use gix::objs::tree::EntryKind; + +use crate::error::XrpcError; + +const TAR_BLOCK: u64 = 512; +const MAX_TREE_DEPTH: usize = 1024; +const MAX_PREFIX_BYTES: usize = 255; + +pub(crate) fn validate_prefix(value: &str) -> Result<(), XrpcError> { + let safe = !value.is_empty() + && value.len() <= MAX_PREFIX_BYTES + && !value.contains('\0') + && !value.starts_with(['/', '\\']) + && value.split(['/', '\\']).all(|component| component != ".."); + match safe { + true => Ok(()), + false => Err(XrpcError::InvalidRequest(format!( + "invalid archive prefix {value:?}" + ))), + } +} + +pub(crate) fn bound_source( + repo: &gix::Repository, + tree: ObjectId, + limit: u64, +) -> Result<(), XrpcError> { + bound_tree(repo, tree, limit, MAX_TREE_DEPTH, &mut 0) +} + +fn bound_tree( + repo: &gix::Repository, + tree: ObjectId, + limit: u64, + nesting: usize, + spooled: &mut u64, +) -> Result<(), XrpcError> { + if nesting == 0 { + return Err(XrpcError::InvalidRequest( + "tree nesting is too deep to archive".to_owned(), + )); + } + if tree == ObjectId::empty_tree(repo.object_hash()) { + return Ok(()); + } + let internal = |e: &dyn std::fmt::Display| XrpcError::Internal(e.to_string()); + let object = repo.find_tree(tree).map_err(|e| internal(&e))?; + let decoded = object.decode().map_err(|e| internal(&e))?; + decoded.entries.iter().try_for_each(|entry| { + let oid = entry.oid.to_owned(); + *spooled = spooled.saturating_add(TAR_BLOCK); + match entry.mode.kind() { + _ if *spooled > limit => Err(XrpcError::ArchiveTooLarge { limit }), + EntryKind::Commit => Ok(()), + EntryKind::Tree => bound_tree(repo, oid, limit, nesting - 1, spooled), + _ => { + let content = repo.find_header(oid).map_err(|e| internal(&e))?.size(); + *spooled = spooled.saturating_add(content.next_multiple_of(TAR_BLOCK)); + match *spooled > limit { + true => Err(XrpcError::ArchiveTooLarge { limit }), + false => Ok(()), + } + } + } + }) +} + +pub(crate) fn write_tar_gz( + repo: &gix::Repository, + tree: ObjectId, + prefix: &str, + modification_time: i64, + limit: u64, + out: impl Write, +) -> anyhow::Result<()> { + let (mut stream, _index) = repo.worktree_stream(tree)?; + let out = Bounded { + inner: out, + written: 0, + limit, + }; + gix_archive::write_stream( + &mut stream, + |stream| stream.next_entry().or_erased(), + out, + gix_archive::Options { + format: gix_archive::Format::TarGz { + compression_level: None, + }, + tree_prefix: Some(BString::from(format!("{prefix}/"))), + modification_time, + }, + ) + .map_err(|e| anyhow::anyhow!("{e}"))?; + Ok(()) +} + +pub(crate) fn write_zip( + repo: &gix::Repository, + tree: ObjectId, + prefix: &str, + modification_time: i64, + limit: u64, + mut out: impl Write, +) -> anyhow::Result<()> { + let (mut stream, _index) = repo.worktree_stream(tree)?; + let mut spool = Bounded { + inner: tempfile::tempfile()?, + written: 0, + limit, + }; + gix_archive::write_stream_seek( + &mut stream, + |stream| stream.next_entry().or_erased(), + &mut spool, + gix_archive::Options { + format: gix_archive::Format::Zip { + compression_level: None, + }, + tree_prefix: Some(BString::from(format!("{prefix}/"))), + modification_time, + }, + ) + .map_err(|e| anyhow::anyhow!("{e}"))?; + let mut file = spool.inner; + file.rewind()?; + std::io::copy(&mut file, &mut out)?; + out.flush()?; + Ok(()) +} + +struct Bounded { + inner: W, + written: u64, + limit: u64, +} + +impl Write for Bounded { + fn write(&mut self, data: &[u8]) -> std::io::Result { + let remaining = self.limit.saturating_sub(self.written); + if data.len() as u64 > remaining { + return Err(std::io::Error::other(format!( + "archive exceeds {} bytes", + self.limit + ))); + } + let written = self.inner.write(data)?; + self.written = self.written.saturating_add(written as u64); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +impl Seek for Bounded { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + let position = self.inner.seek(pos)?; + self.written = position; + Ok(position) + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/committers.rs b/knotmirror/crates/knotmirror-xrpc/src/committers.rs new file mode 100644 index 0000000000000000000000000000000000000000..9f7dfed4986928910b0acce78a4cba78526d759e --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/committers.rs @@ -0,0 +1,113 @@ +use std::sync::Arc; +use std::time::Duration; + +use bobbin_runtime::Clock; +use jacquard_common::types::did::Did; +use lexicons::sh_tangled::identity::resolve_committers::ResolveCommittersOutput; +use serde_json::json; +use tracing::warn; + +use crate::signer::ServiceSigner; + +const RESOLVE_COMMITTERS: &str = "sh.tangled.identity.resolveCommitters"; +const TIMEOUT: Duration = Duration::from_secs(1); +const MAX_RESPONSE_BYTES: usize = 64 * 1024; +const MAX_COMMITTERS: usize = 251; +const MAX_COMMITTER_LEN: usize = 320; +const CACHE_CAPACITY: u64 = 4_096; +const CACHE_FRESHNESS: Duration = Duration::from_secs(30); + +pub struct Deliberi { + pub url: knotmirror_types::OpsEndpoint, + pub did: Did, +} + +/// falls back to only the actor's did when deliberi is unset or fails +pub(crate) struct Committers { + deliberi: Option<(Deliberi, Arc)>, + cache: moka::future::Cache>, +} + +impl Committers { + pub(crate) fn new(deliberi: Option<(Deliberi, Arc)>) -> Self { + Self { + deliberi, + cache: moka::future::Cache::builder() + .max_capacity(CACHE_CAPACITY) + .time_to_live(CACHE_FRESHNESS) + .build(), + } + } + + pub(crate) async fn resolve( + &self, + http: &reqwest::Client, + clock: &dyn Clock, + actor: &Did, + ) -> Arc<[String]> { + let Some((deliberi, signer)) = &self.deliberi else { + return Arc::from([actor.as_str().to_owned()]); + }; + let now_secs = clock.now_unix_micros().raw() / 1_000_000; + let fetch = fetch(http, deliberi, signer, actor, now_secs); + match self.cache.try_get_with(actor.clone(), fetch).await { + Ok(committers) => committers, + Err(e) => { + warn!(%actor, error = %e, "resolving verified committer emails failed"); + Arc::from([actor.as_str().to_owned()]) + } + } + } +} + +async fn fetch( + http: &reqwest::Client, + deliberi: &Deliberi, + signer: &ServiceSigner, + actor: &Did, + now_secs: u64, +) -> Result, String> { + let mut response = http + .post(deliberi.url.xrpc(RESOLVE_COMMITTERS)) + .bearer_auth(signer.token(deliberi.did.as_str(), RESOLVE_COMMITTERS, now_secs)) + .json(&json!({ "actor": actor.as_str() })) + .timeout(TIMEOUT) + .send() + .await + .map_err(|e| e.to_string())?; + if !response.status().is_success() { + return Err(format!("deliberi returned {}", response.status())); + } + let mut body = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(|e| e.to_string())? { + if body.len() + chunk.len() > MAX_RESPONSE_BYTES { + return Err("deliberi response is too large".to_owned()); + } + body.extend_from_slice(&chunk); + } + let output: ResolveCommittersOutput = + serde_json::from_slice(&body).map_err(|e| e.to_string())?; + Ok(committers(actor, &output.committers).into()) +} + +/// lowercased because the activity index stores emails lowercased +fn committers(actor: &Did, resolved: &[impl AsRef]) -> Vec { + let mut out = vec![actor.as_str().to_owned()]; + for value in resolved { + let value = value.as_ref().trim().to_lowercase(); + if value.is_empty() || value.len() > MAX_COMMITTER_LEN || value.contains('\0') { + continue; + } + let usable = match Did::new(value.as_str()) { + Ok(did) => did.as_str() == actor.as_str(), + Err(_) => value.contains('@'), + }; + if usable && !out.contains(&value) { + out.push(value); + if out.len() == MAX_COMMITTERS { + break; + } + } + } + out +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/did_ext.rs b/knotmirror/crates/knotmirror-xrpc/src/did_ext.rs new file mode 100644 index 0000000000000000000000000000000000000000..a8204a226b4305404253299e3b18c36e3c3b0577 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/did_ext.rs @@ -0,0 +1,29 @@ +use jacquard_common::Bos; +use jacquard_common::deps::fluent_uri::Uri; +use jacquard_common::types::did_doc::DidDocument; +use jacquard_common::types::string::AtprotoStr; +use jacquard_common::types::value::Data; + +const KNOT_SERVICE_TYPE: &str = "TangledKnot"; + +pub(crate) trait DidDocumentExt { + /// The `tangled_knot` service endpoint, if the document declares one. + fn knot_endpoint(&self) -> Option>; +} + +impl + AsRef + Clone> DidDocumentExt for DidDocument { + fn knot_endpoint(&self) -> Option> { + self.service.as_ref().and_then(|services| { + services.iter().find_map(|service| { + if service.r#type.as_ref() != KNOT_SERVICE_TYPE { + return None; + } + match &service.service_endpoint { + Some(Data::String(AtprotoStr::Uri(uri))) => Uri::parse(uri.as_ref()).ok(), + Some(Data::String(AtprotoStr::String(raw))) => Uri::parse(raw.as_ref()).ok(), + _ => None, + } + }) + }) + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/diff.rs b/knotmirror/crates/knotmirror-xrpc/src/diff.rs new file mode 100644 index 0000000000000000000000000000000000000000..a3d7b0338c03e78398cef22901bc9ec6cb225994 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/diff.rs @@ -0,0 +1,418 @@ +use gix::ObjectId; +use gix::bstr::{BStr, ByteSlice as _}; +use gix::object::tree::diff::ChangeDetached; +use gix::objs::tree::{EntryKind, EntryMode}; +use line_numbers::{LineNumber, SingleLineSpan}; +use rustc_hash::FxHashSet; +use tracing::info; + +/// Blobs larger than this are treated as binary. +const LARGE_FILE_THRESHOLD_BYTES: u64 = 1024 * 1024; +const STAT_CACHE_CAPACITY: usize = 65_536; + +#[derive(Debug)] +pub struct FileContent { + pub(crate) path: String, + pub(crate) oid: String, + pub(crate) size: usize, + pub(crate) is_binary: bool, + pub(crate) is_submodule: bool, + /// Raw file bytes, inlined when the `oid` is not fetchable by the client (interdiff's + /// synthetic rebased tree). `None` for ordinary diffs and for binary/oversized blobs. + pub(crate) content: Option>, +} + +#[derive(Debug, Clone)] +pub struct Hunk { + pub(crate) novel_lhs: FxHashSet, + pub(crate) novel_rhs: FxHashSet, + pub(crate) lines: Vec<(Option, Option)>, +} + +/// A matched token (an atom, a delimiter, or a comment word). +#[derive(PartialEq, Eq, Debug, Clone)] +pub enum MatchKind { + // TBD +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MatchedPos { + pub(crate) kind: MatchKind, + pub(crate) pos: SingleLineSpan, +} + +#[derive(Debug)] +pub struct Diff { + pub(crate) lhs_src: FileContent, + pub(crate) rhs_src: FileContent, + pub(crate) hunks: Vec, + + #[allow(unused)] + pub(crate) lhs_positions: Vec, + #[allow(unused)] + pub(crate) rhs_positions: Vec, + + /// If the two files do not have exactly the same bytes, the + /// number of bytes in each file. + pub(crate) has_byte_changes: Option<(usize, usize)>, + pub(crate) has_syntactic_changes: bool, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct Stat { + pub insertions: u64, + pub deletions: u64, + pub files_changed: u64, +} + +pub struct StatCache { + pub(crate) commits: quick_cache::sync::Cache, +} + +impl Default for StatCache { + fn default() -> Self { + Self { + commits: quick_cache::sync::Cache::new(STAT_CACHE_CAPACITY), + } + } +} + +impl StatCache { + pub fn commit(&self, repo: &gix::Repository, commit: &gix::Commit<'_>) -> anyhow::Result { + self.commits + .get_or_insert_with(&commit.id, || commit_stat(repo, commit)) + } +} + +fn parent_tree<'repo>( + repo: &'repo gix::Repository, + commit: &gix::Commit<'_>, +) -> anyhow::Result> { + Ok(match commit.parent_ids().next() { + Some(parent) => repo.find_commit(parent.detach())?.tree()?, + None => repo.empty_tree(), + }) +} + +pub fn commit_stat(repo: &gix::Repository, commit: &gix::Commit<'_>) -> anyhow::Result { + stat(repo, &parent_tree(repo, commit)?, &commit.tree()?) +} + +pub fn stat( + repo: &gix::Repository, + old_tree: &gix::Tree<'_>, + new_tree: &gix::Tree<'_>, +) -> anyhow::Result { + changes(repo, old_tree, new_tree, build_stat)?.try_fold(Stat::default(), |acc, s| { + let s = s?; + Ok(Stat { + insertions: acc.insertions + s.insertions, + deletions: acc.deletions + s.deletions, + files_changed: acc.files_changed + s.files_changed, + }) + }) +} + +/// Diff two trees, yielding one [`Diff`] per changed file (line-level [`Hunk`]s included). +/// +/// Returns a lazy iterator of `anyhow::Result`. +pub fn diff<'repo>( + repo: &'repo gix::Repository, + old_tree: &gix::Tree<'_>, + new_tree: &gix::Tree<'_>, + embed_content: bool, +) -> anyhow::Result> + 'repo> { + changes( + repo, + old_tree, + new_tree, + move |repo, change, old, new, cache| { + // Submodules (gitlinks) can't go through the blob pipeline — their commit lives in + // another repo. Emit the commit oids directly for the caller to compare. + if is_kind(old, EntryKind::Commit) || is_kind(new, EntryKind::Commit) { + return Ok(submodule_diff(repo, change, old, new)); + } + build_diff(repo, change, cache, embed_content) + }, + ) +} + +/// Diff two trees and hand every changed file to `build`, yielding its results lazily. +fn changes<'repo, T, F>( + repo: &'repo gix::Repository, + old_tree: &gix::Tree<'_>, + new_tree: &gix::Tree<'_>, + build: F, +) -> anyhow::Result> +where + F: FnMut( + &gix::Repository, + &ChangeDetached, + Option>, + Option>, + &mut gix::diff::blob::Platform, + ) -> anyhow::Result, +{ + let changes = repo.diff_tree_to_tree(Some(old_tree), Some(new_tree), None)?; + let mut cache = repo.diff_resource_cache_for_tree_diff()?; + cache.filter.options.large_file_threshold_bytes = LARGE_FILE_THRESHOLD_BYTES; + Ok(ChangeIter { + repo, + changes: changes.into_iter(), + cache, + build, + }) +} + +/// Lazy iterator over per-file results produced by a builder callback. +struct ChangeIter<'repo, F> { + repo: &'repo gix::Repository, + changes: std::vec::IntoIter, + cache: gix::diff::blob::Platform, + build: F, +} + +impl Iterator for ChangeIter<'_, F> +where + F: FnMut( + &gix::Repository, + &ChangeDetached, + Option>, + Option>, + &mut gix::diff::blob::Platform, + ) -> anyhow::Result, +{ + type Item = anyhow::Result; + + fn next(&mut self) -> Option { + loop { + let change = self.changes.next()?; + let (old, new) = sides(&change); + // Trees are recursed into by gix and never diffed directly; skip defensively. + if is_kind(old, EntryKind::Tree) || is_kind(new, EntryKind::Tree) { + continue; + } + let result = (self.build)(self.repo, &change, old, new, &mut self.cache); + self.cache.clear_resource_cache_keep_allocation(); + return Some(result); + } + } +} + +#[derive(Clone, Copy)] +pub(crate) struct Side<'a> { + pub(crate) path: &'a BStr, + pub(crate) mode: EntryMode, + pub(crate) id: ObjectId, +} + +pub(crate) fn sides(change: &ChangeDetached) -> (Option>, Option>) { + let side = |path, (mode, id): (EntryMode, &gix::oid)| Side { + path, + mode, + id: id.to_owned(), + }; + let old = side(change.source_location(), change.source_entry_mode_and_id()); + let new = side(change.location(), change.entry_mode_and_id()); + match change { + ChangeDetached::Addition { .. } => (None, Some(new)), + ChangeDetached::Deletion { .. } => (Some(old), None), + ChangeDetached::Modification { .. } | ChangeDetached::Rewrite { .. } => { + (Some(old), Some(new)) + } + } +} + +fn is_kind(side: Option>, kind: EntryKind) -> bool { + side.is_some_and(|side| side.mode.kind() == kind) +} + +/// Build a [`Diff`] for a submodule (gitlink) change: no hunks, both sides carry their commit +/// oid with `is_submodule` set. The caller compares the oids as submodule pointers. +fn submodule_diff( + repo: &gix::Repository, + change: &ChangeDetached, + old: Option>, + new: Option>, +) -> Diff { + let path = change.location().to_string(); + let side = |s: Option>| FileContent { + path: path.clone(), + oid: s + .map_or_else(|| ObjectId::null(repo.object_hash()), |side| side.id) + .to_string(), + size: 0, + is_binary: false, + is_submodule: true, + content: None, + }; + Diff { + lhs_src: side(old), + rhs_src: side(new), + hunks: Vec::new(), + lhs_positions: Vec::new(), + rhs_positions: Vec::new(), + has_byte_changes: None, + has_syntactic_changes: false, + } +} + +/// Count the changed lines of a single file; submodules count as a changed file with no lines. +fn build_stat( + repo: &gix::Repository, + change: &ChangeDetached, + old: Option>, + new: Option>, + cache: &mut gix::diff::blob::Platform, +) -> anyhow::Result { + use gix::diff::blob::platform::prepare_diff::Operation; + use gix::diff::blob::{Diff as BlobDiff, InternedInput}; + use gix::prelude::TreeDiffChangeExt as _; + + let mut stat = Stat { + files_changed: 1, + ..Stat::default() + }; + if is_kind(old, EntryKind::Commit) || is_kind(new, EntryKind::Commit) { + return Ok(stat); + } + let blob = change.attach(repo, repo).diff(cache)?; + let out = blob.resource_cache.prepare_diff()?; + if let Operation::InternalDiff { algorithm } = out.operation { + let input = InternedInput::new(out.old.intern_source(), out.new.intern_source()); + for hunk in BlobDiff::compute(algorithm, &input).hunks() { + stat.deletions += hunk.before.len() as u64; + stat.insertions += hunk.after.len() as u64; + } + } + Ok(stat) +} + +/// Build the [`Diff`] for a single changed file. +fn build_diff( + repo: &gix::Repository, + change: &gix::object::tree::diff::ChangeDetached, + cache: &mut gix::diff::blob::Platform, + embed_content: bool, +) -> anyhow::Result { + use gix::diff::blob::platform::prepare_diff::Operation; + use gix::diff::blob::platform::resource::Data; + use gix::diff::blob::{Diff as BlobDiff, InternedInput}; + use gix::prelude::TreeDiffChangeExt as _; + + let change = change.attach(repo, repo); + let blob = change.diff(cache)?; + let out = blob.resource_cache.prepare_diff()?; + + // Per-side file metadata, derived from the prepared resources. + let side = + |res: &gix::diff::blob::platform::Resource<'_>, embed_content: bool| -> FileContent { + let (size, is_binary, content) = match res.data { + // Only textual buffers carry inlinable bytes; binary/missing sides fall back to oid. + Data::Buffer { buf, .. } => (buf.len(), false, embed_content.then(|| buf.to_vec())), + Data::Binary { size } => (size as usize, true, None), + Data::Missing => (0, false, None), + }; + FileContent { + path: res.rela_path.to_string(), + oid: res.id.to_string(), + size, + is_binary, + is_submodule: false, + content, + } + }; + let mut lhs_src = side(&out.old, embed_content); + let mut rhs_src = side(&out.new, false); + + let has_byte_changes = if lhs_src.size == rhs_src.size && out.old.id == out.new.id { + None + } else { + Some((lhs_src.size, rhs_src.size)) + }; + + let hunks = match out.operation { + Operation::InternalDiff { algorithm } => { + let input = InternedInput::new(out.old.intern_source(), out.new.intern_source()); + let mut d = BlobDiff::compute(algorithm, &input); + d.postprocess_lines(&input); + d.hunks() + .map(|h| { + let before: Vec = h.before.clone().map(LineNumber).collect(); + let after: Vec = h.after.clone().map(LineNumber).collect(); + let n = before.len().max(after.len()); + let lines = (0..n) + .map(|i| (before.get(i).copied(), after.get(i).copied())) + .collect(); + Hunk { + novel_lhs: before.iter().copied().collect(), + novel_rhs: after.iter().copied().collect(), + lines, + } + }) + .collect() + } + Operation::SourceOrDestinationIsBinary => { + lhs_src.is_binary = true; + rhs_src.is_binary = true; + Vec::new() + } + Operation::ExternalCommand { .. } => unreachable!("we disabled that"), + }; + + Ok(Diff { + lhs_src, + rhs_src, + hunks, + lhs_positions: Vec::new(), + rhs_positions: Vec::new(), + has_byte_changes, + has_syntactic_changes: false, + }) +} + +/// Compute the two trees whose diff is the interdiff between an old patch +/// version (`from_base..from_head`) and a new one (`to_base..to_head`). +/// +/// Rebase `from_base..from_head` onto `to_base` and return rebased tree so +/// caller can diff between `rebased_tree` and `to_head.tree` +pub fn prepare_interdiff<'repo>( + repo: &'repo gix::Repository, + (from_base_id, from_head_id): (ObjectId, ObjectId), + to_base_id: ObjectId, +) -> anyhow::Result> { + let from_head = repo.find_commit(from_head_id)?; + + let from_base = repo.find_commit(from_base_id)?; + let to_base = repo.find_commit(to_base_id)?; + + // Endpoint trees for the 3-way merge. + let from_base_tree = from_base.tree_id()?.detach(); + let from_head_tree = from_head.tree_id()?.detach(); + let to_base_tree = to_base.tree_id()?.detach(); + + let options = repo.tree_merge_options()?; + let labels = gix::merge::blob::builtin_driver::text::Labels { + ancestor: Some("from.base".as_bytes().as_bstr()), + current: Some("to.base".as_bytes().as_bstr()), + other: Some("from.head".as_bytes().as_bstr()), + }; + let mut outcome = repo.merge_trees( + from_base_tree, + to_base_tree, + from_head_tree, + labels, + options, + )?; + + if !outcome.conflicts.is_empty() { + info!( + conflicts = outcome.conflicts.len(), + "interdiff rebase produced conflicts; tree contains conflict markers" + ); + } + + let rebased_id = outcome.tree.write()?.detach(); + let rebased_tree = repo.find_tree(rebased_id)?; + Ok(rebased_tree) +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/error.rs b/knotmirror/crates/knotmirror-xrpc/src/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..c09c8f0aaf583d1a1d4c084c41ab11c1ecc707fc --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/error.rs @@ -0,0 +1,174 @@ +use axum::{ + Json, + response::{IntoResponse, Response}, +}; +use reqwest::StatusCode; +use serde_json::json; +use tracing::error; + +#[derive(Clone, Copy)] +pub(crate) struct ErrorKind(pub(crate) &'static str); + +#[derive(Debug)] +pub(crate) enum XrpcError { + InvalidRequest(String), + NotFound(String), + RepoNotFound { detail: String }, + RefNotFound { rev: String, detail: String }, + RevisionNotFound { rev: String }, + NotIndexed { commit: String }, + EntryNotFound { path: String }, + PathNotFound { path: String }, + NoMergeBase { base: String, head: String }, + CompareError(String), + FileNotFound { path: String }, + BlameTooLarge, + BlobTooLarge { limit: u64 }, + ArchiveTooLarge { limit: u64 }, + PatchTooLarge { limit: usize }, + DiffTooLarge { limit: usize }, + BlameTimeout, + DiffTimeout, + MergeConflict(Vec), + PushRejected(String), + Unauthorized(String), + Internal(String), +} + +impl IntoResponse for XrpcError { + fn into_response(self) -> Response { + let (status, error, message) = match self { + Self::InvalidRequest(m) => (StatusCode::BAD_REQUEST, "InvalidRequest", m), + Self::NotFound(m) => (StatusCode::NOT_FOUND, "NotFound", m), + Self::RepoNotFound { detail } => { + error!(error = %detail, "repo not found"); + ( + StatusCode::NOT_FOUND, + "RepoNotFound", + "repository not found".to_owned(), + ) + } + Self::RefNotFound { rev, detail } => { + error!(error = %detail, rev = %rev, "revision not found"); + ( + StatusCode::NOT_FOUND, + "RefNotFound", + format!("revision not found: {rev}"), + ) + } + Self::RevisionNotFound { rev } => ( + StatusCode::NOT_FOUND, + "RevisionNotFound", + format!("commit not found: {rev}"), + ), + Self::NotIndexed { commit } => ( + StatusCode::NOT_FOUND, + "NotIndexed", + format!("languages not indexed for {commit}"), + ), + Self::EntryNotFound { path } => ( + StatusCode::NOT_FOUND, + "EntryNotFound", + format!("entry not found: {path}"), + ), + Self::PathNotFound { path } => ( + StatusCode::NOT_FOUND, + "PathNotFound", + format!("path not found: {path}"), + ), + Self::NoMergeBase { base, head } => ( + StatusCode::NOT_FOUND, + "NoMergeBase", + format!("no common ancestor between {base} and {head}"), + ), + Self::CompareError(m) => { + error!(error = %m, "compare failed"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + "CompareError", + "failed to compare revisions".to_owned(), + ) + } + Self::FileNotFound { path } => ( + StatusCode::NOT_FOUND, + "FileNotFound", + format!("file not found: {path}"), + ), + Self::BlobTooLarge { limit } => ( + StatusCode::PAYLOAD_TOO_LARGE, + "BlobTooLarge", + format!("file is too large to serve, limit is {limit} bytes"), + ), + Self::ArchiveTooLarge { limit } => ( + StatusCode::PAYLOAD_TOO_LARGE, + "ArchiveTooLarge", + format!("archive is too large to serve, limit is {limit} bytes"), + ), + Self::PatchTooLarge { limit } => ( + StatusCode::PAYLOAD_TOO_LARGE, + "PatchTooLarge", + format!("patch is too large to serve, limit is {limit} commits"), + ), + Self::DiffTooLarge { limit } => ( + StatusCode::PAYLOAD_TOO_LARGE, + "DiffTooLarge", + format!("diff has more than {limit} files, too many to show"), + ), + Self::BlameTooLarge => ( + StatusCode::PAYLOAD_TOO_LARGE, + "BlameTooLarge", + "blame exceeds resource limits".to_owned(), + ), + Self::BlameTimeout => ( + StatusCode::GATEWAY_TIMEOUT, + "BlameTimeout", + "blame computation timed out".to_owned(), + ), + Self::DiffTimeout => ( + StatusCode::GATEWAY_TIMEOUT, + "DiffTimeout", + "diff took too long to compute".to_owned(), + ), + Self::MergeConflict(paths) => ( + StatusCode::CONFLICT, + "MergeConflict", + format!("merge produced conflicts in {}", paths.join(", ")), + ), + Self::PushRejected(detail) => { + error!(error = %detail, "push rejected"); + ( + StatusCode::CONFLICT, + "PushRejected", + "knot rejected the push".to_owned(), + ) + } + Self::Unauthorized(detail) => { + error!(error = %detail, "authentication failed"); + (StatusCode::UNAUTHORIZED, "AuthenticationRequired", detail) + } + Self::Internal(m) => { + error!(error = %m, "xrpc request failed"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + "InternalServerError", + "internal error".to_owned(), + ) + } + }; + let mut response = + (status, Json(json!({ "error": error, "message": message }))).into_response(); + response.extensions_mut().insert(ErrorKind(error)); + response + } +} + +impl From for XrpcError { + fn from(e: knotmirror_git::layout::Error) -> Self { + match e { + knotmirror_git::layout::Error::RepoNotFound => Self::RepoNotFound { + detail: e.to_string(), + }, + _ => Self::Internal(e.to_string()), + } + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/format_patch.rs b/knotmirror/crates/knotmirror-xrpc/src/format_patch.rs new file mode 100644 index 0000000000000000000000000000000000000000..cc7c935edde752b311b90acc0f6a55bd4d21bfa4 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/format_patch.rs @@ -0,0 +1,2461 @@ +use std::io::{self, Write}; + +use anyhow::Context as _; +use fixedbitset::FixedBitSet; +use gix::ObjectId; +use gix::bstr::{BStr, ByteSlice as _}; +use gix::diff::blob::sources::byte_lines; +use gix::diff::blob::{Algorithm, Hunk, InternedInput, diff_with_slider_heuristics}; +use gix::object::tree::diff::ChangeDetached; +use gix::objs::TreeRefIter; +use gix::objs::tree::EntryKind; +use gix::prelude::ObjectIdExt as _; +use gix::revision::walk::Sorting; + +use crate::diff::{Side, sides}; +use crate::error::XrpcError; + +const CONTEXT: u32 = 3; +const STAT_WIDTH: usize = 72; +const HEADER_WIDTH: usize = 78; +const ENCODED_WIDTH: usize = 76; +const FUNCNAME_BYTES: usize = 80; +/// bigger files show as binary, diffing 2m churned lines takes ~14s +const MAX_DIFF_LINES: usize = 2_000_000; + +pub(crate) const MAX_PATCH_COMMITS: usize = 100; + +/// merges are left out +pub(crate) fn series( + repo: &gix::Repository, + from: ObjectId, + to: ObjectId, + limit: usize, +) -> Result, XrpcError> { + let internal = |e: &dyn std::fmt::Display| XrpcError::Internal(e.to_string()); + let walk = repo + .rev_walk([to]) + .sorting(Sorting::ByCommitTime(Default::default())) + .with_hidden([from]) + .all() + .map_err(|e| internal(&e))?; + let mut commits = Vec::new(); + for info in walk { + let info = info.map_err(|e| internal(&e))?; + if info.parent_ids.len() > 1 { + continue; + } + if commits.len() == limit { + return Err(XrpcError::PatchTooLarge { limit }); + } + commits.push(info.id); + } + commits.reverse(); + Ok(commits) +} + +pub(crate) fn write_series( + repo: &gix::Repository, + commits: &[ObjectId], + signature: &str, + max_blob_bytes: u64, + out: &mut impl Write, +) -> anyhow::Result<()> { + let ctx = Ctx { + repo, + max_blob_bytes, + }; + // gix builds this from the whole HEAD tree in a bare repo, so once per series + let mut cache = repo.diff_resource_cache_for_tree_diff()?; + for (index, &commit) in commits.iter().enumerate() { + if index > 0 { + out.write_all(b"\n")?; + } + let number = (commits.len() > 1).then_some((index + 1, commits.len())); + write_patch(&ctx, &mut cache, commit, number, out)?; + write!(out, "-- \n{signature}\n\n")?; + } + Ok(()) +} + +fn write_patch( + ctx: &Ctx<'_>, + cache: &mut gix::diff::blob::Platform, + id: ObjectId, + number: Option<(usize, usize)>, + out: &mut impl Write, +) -> anyhow::Result<()> { + let repo = ctx.repo; + let commit = repo.find_commit(id)?; + let decoded = commit.decode()?; + let author = decoded.author()?; + + let mut head = Vec::new(); + writeln!(head, "From {id} Mon Sep 17 00:00:00 2001").expect("vec write"); + write_from(&mut head, author.name, author.email); + writeln!(head, "Date: {}", rfc2822_date(author.time)?).expect("vec write"); + write_message(&mut head, decoded.message, number); + out.write_all(&head)?; + + let old_tree = match decoded.parents().next() { + Some(parent) => repo.find_commit(parent)?.tree()?, + None => repo.empty_tree(), + }; + let (rewrites, configured) = gix::diff::new_rewrites(&repo.config_snapshot(), true)?; + let rewrites = if configured { + rewrites + } else { + Some(Default::default()) + }; + // gix skips empty files, git pairs them as exact renames + // git gives up on inexact renames only past the limit squared in candidate pairs + let rewrites = rewrites.map(|rewrites| gix::diff::Rewrites { + track_empty: true, + limit: rewrites.limit.saturating_mul(rewrites.limit), + ..rewrites + }); + let options = gix::diff::Options::default().with_rewrites(rewrites); + let new_tree = commit.tree()?; + let mut changes: Vec = Vec::new(); + gix::diff::tree_with_rewrites( + TreeRefIter::from_bytes(&old_tree.data, old_tree.id.kind()), + TreeRefIter::from_bytes(&new_tree.data, new_tree.id.kind()), + cache, + &mut Default::default(), + &repo.objects, + |change| -> Result<_, std::convert::Infallible> { + changes.push(change.into_owned()); + Ok(std::ops::ControlFlow::Continue(())) + }, + options.into(), + )?; + cache.clear_resource_cache_keep_allocation(); + changes.retain(|change| { + let (old, new) = sides(change); + !old.is_some_and(|side| side.mode.is_tree()) && !new.is_some_and(|side| side.mode.is_tree()) + }); + if changes.is_empty() { + return Ok(()); + } + changes.sort_by(|a, b| a.location().cmp(b.location())); + + out.write_all(b"---\n")?; + let (stats, lines): (Vec<_>, Vec<_>) = changes + .iter() + .map(|change| file_stat(ctx, change)) + .collect::>>()? + .into_iter() + .unzip(); + write_stat(out, &stats)?; + for change in &changes { + write_summary(out, change)?; + } + out.write_all(b"\n")?; + for (change, lines) in changes.iter().zip(&lines) { + write_file_patch(ctx, change, lines.as_ref(), out)?; + } + Ok(()) +} + +fn write_from(buf: &mut Vec, name: &BStr, email: &BStr) { + buf.extend_from_slice(b"From: "); + let mut width = HEADER_WIDTH; + if needs_rfc2047(name) { + add_rfc2047(buf, name, true); + width = ENCODED_WIDTH; + } else if name.iter().copied().any(is_rfc822_special) { + let mut quoted = vec![b'"']; + for &byte in name.iter() { + if byte == b'"' || byte == b'\\' { + quoted.push(b'\\'); + } + quoted.push(byte); + } + quoted.push(b'"'); + add_wrapped(buf, "ed, -6, 1, HEADER_WIDTH); + } else { + add_wrapped(buf, name, -6, 1, HEADER_WIDTH); + } + if width < last_line_len(buf) + email.len() + 3 { + buf.push(b'\n'); + } + buf.extend_from_slice(b" <"); + buf.extend_from_slice(email); + buf.extend_from_slice(b">\n"); +} + +/// like git, a broken zone like `+5785265` still shifts the time, past an int it is 0, minutes*60 wraps +fn rfc2822_date(raw: &str) -> anyhow::Result { + let mut fields = raw.split_ascii_whitespace(); + let seconds: i64 = fields + .next() + .and_then(|field| field.parse().ok()) + .unwrap_or(0); + let zone = fields.next().map_or(0, leading_number); + let zone = i32::try_from(zone) + .ok() + .filter(|zone| *zone != i32::MIN && *zone != i32::MAX) + .unwrap_or(0); + let minutes = zone.signum() * (zone.abs() / 100 * 60 + zone.abs() % 100); + let local = seconds + .checked_add(minutes.wrapping_mul(60).into()) + .and_then(|local| chrono::DateTime::from_timestamp(local, 0)) + .context("commit timestamp out of range")? + .naive_utc(); + Ok(format!( + "{} {zone:+05}", + local.format("%a, %-d %b %Y %H:%M:%S") + )) +} + +fn leading_number(text: &str) -> i64 { + let digits = text.strip_prefix(['+', '-']).unwrap_or(text); + let end = digits + .bytes() + .position(|byte| !byte.is_ascii_digit()) + .unwrap_or(digits.len()); + let value: i64 = digits[..end].parse().unwrap_or(0); + if text.starts_with('-') { -value } else { value } +} + +fn write_message(buf: &mut Vec, message: &BStr, number: Option<(usize, usize)>) { + let mut lines = message.lines_with_terminator().peekable(); + while lines.next_if(|line| trim_end(line).is_empty()).is_some() {} + let mut title = Vec::new(); + for line in lines.by_ref() { + let line = trim_end(line); + if line.is_empty() { + break; + } + if !title.is_empty() { + title.push(b' '); + } + title.extend_from_slice(line); + } + + match number { + Some((nr, total)) => { + let digits = decimal_width(total as u64); + write!(buf, "Subject: [PATCH {nr:0digits$}/{total}] ").expect("vec write") + } + None => buf.extend_from_slice(b"Subject: [PATCH] "), + } + if needs_rfc2047(&title) { + add_rfc2047(buf, &title, false); + } else { + let indent = -(last_line_len(buf) as isize); + add_wrapped(buf, &title, indent, 1, HEADER_WIDTH); + } + buf.push(b'\n'); + if message.iter().any(|&byte| non_ascii(byte)) { + buf.extend_from_slice( + b"MIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\n", + ); + } + buf.push(b'\n'); + + let body = buf.len(); + let mut first = true; + for line in lines { + let line = trim_end(line); + if line.is_empty() && first { + continue; + } + first = false; + buf.extend_from_slice(line); + buf.push(b'\n'); + } + buf.truncate(trim_end(buf).len()); + buf.push(b'\n'); + if buf.len() <= body { + buf.push(b'\n'); + } +} + +fn non_ascii(byte: u8) -> bool { + !byte.is_ascii() || byte == 0x1b +} + +fn needs_rfc2047(text: &[u8]) -> bool { + text.iter().any(|&byte| non_ascii(byte) || byte == b'\n') || text.windows(2).any(|w| w == b"=?") +} + +fn is_rfc822_special(byte: u8) -> bool { + b"()<>[]:;@,.\"\\".contains(&byte) +} + +fn is_rfc2047_special(byte: u8, address: bool) -> bool { + if non_ascii(byte) || !(byte.is_ascii_graphic() || byte == b' ') { + return true; + } + if byte.is_ascii_whitespace() || matches!(byte, b'=' | b'?' | b'_') { + return true; + } + address && !(byte.is_ascii_alphanumeric() || matches!(byte, b'!' | b'*' | b'+' | b'-' | b'/')) +} + +/// a broken sequence steps one byte, like git's `mbs_chrlen()` +fn utf8_char_len(text: &[u8]) -> usize { + match gix::bstr::decode_utf8(text) { + (Some(_), len) => len, + _ => 1, + } +} + +fn add_rfc2047(buf: &mut Vec, text: &[u8], address: bool) { + const PREFIX: &[u8] = b"=?UTF-8?q?"; + let mut line_len = last_line_len(buf) + PREFIX.len(); + buf.extend_from_slice(PREFIX); + let mut rest = text; + while !rest.is_empty() { + let len = utf8_char_len(rest); + let (char, tail) = rest.split_at(len); + rest = tail; + let special = len > 1 || is_rfc2047_special(char[0], address); + let encoded_len = if special { 3 * len } else { 1 }; + if line_len + encoded_len + 2 > ENCODED_WIDTH { + buf.extend_from_slice(b"?=\n "); + buf.extend_from_slice(PREFIX); + line_len = PREFIX.len() + 1; + } + for &byte in char { + if special { + write!(buf, "={byte:02X}").expect("vec write"); + } else { + buf.push(byte); + } + } + line_len += encoded_len; + } + buf.extend_from_slice(b"?="); +} + +fn last_line_len(buf: &[u8]) -> usize { + buf.len() + - buf + .iter() + .rposition(|&byte| byte == b'\n') + .map_or(0, |pos| pos + 1) +} + +fn is_space(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\n' | b'\r') +} + +fn trim_end(text: &[u8]) -> &[u8] { + let end = text + .iter() + .rposition(|&byte| !is_space(byte)) + .map_or(0, |pos| pos + 1); + &text[..end] +} + +/// negative `indent1` means that many columns of the first line are already used +fn add_wrapped(buf: &mut Vec, text: &[u8], indent1: isize, indent2: isize, width: usize) { + let width = width as isize; + let (mut bol, mut pos) = (0, 0); + let mut indent = indent1; + let mut w = indent1; + let mut space = None; + if indent1 < 0 { + w = -indent1; + space = Some(0); + } + loop { + let c = text.get(pos).copied(); + if c.is_none_or(is_space) { + if w <= width || space.is_none() { + if c.is_none() && pos == bol { + return; + } + let start = match space { + Some(space) => space, + None => { + buf.extend(std::iter::repeat_n(b' ', indent.max(0) as usize)); + bol + } + }; + buf.extend_from_slice(&text[start..pos]); + let Some(c) = c else { return }; + space = Some(pos); + if c == b'\t' { + w |= 0x07; + } + w += 1; + pos += 1; + } else { + buf.push(b'\n'); + let at = space.expect("checked above"); + pos = at + usize::from(text.get(at).copied().is_some_and(is_space)); + bol = pos; + space = None; + indent = indent2; + w = indent2; + } + continue; + } + w += if c.is_some_and(|c| c < 0x20 || c == 0x7f) { + -1 + } else { + 1 + }; + pos += 1; + } +} + +fn needs_quote(byte: u8) -> bool { + !(0x20..0x7f).contains(&byte) || byte == b'"' || byte == b'\\' +} + +fn quote_inner(out: &mut Vec, path: &[u8]) { + for &byte in path { + let escape = match byte { + 0x07 => b'a', + 0x08 => b'b', + b'\t' => b't', + b'\n' => b'n', + 0x0b => b'v', + 0x0c => b'f', + b'\r' => b'r', + b'"' | b'\\' => byte, + _ if needs_quote(byte) => { + write!(out, "\\{byte:03o}").expect("vec write"); + continue; + } + _ => { + out.push(byte); + continue; + } + }; + out.extend_from_slice(&[b'\\', escape]); + } +} + +fn quote(path: &[u8]) -> Vec { + quote_prefixed(b"", path) +} + +fn quote_prefixed(prefix: &[u8], path: &[u8]) -> Vec { + let mut out = Vec::with_capacity(prefix.len() + path.len() + 2); + if prefix.iter().chain(path).copied().any(needs_quote) { + out.push(b'"'); + quote_inner(&mut out, prefix); + quote_inner(&mut out, path); + out.push(b'"'); + } else { + out.extend_from_slice(prefix); + out.extend_from_slice(path); + } + out +} + +/// shared prefix and suffix go outside the braces, `pfx{a => b}sfx` +fn rename_name(a: &[u8], b: &[u8]) -> Vec { + let mut name = Vec::new(); + if a.iter().chain(b).copied().any(needs_quote) { + name.extend(quote(a)); + name.extend_from_slice(b" => "); + name.extend(quote(b)); + return name; + } + let mut pfx = 0; + for (i, (x, y)) in a.iter().zip(b).enumerate() { + if x != y { + break; + } + if *x == b'/' { + pfx = i + 1; + } + } + let adjust = usize::from(pfx > 0); + let mut sfx = 0; + let (mut i, mut j) = (a.len() as isize, b.len() as isize); + let floor = (pfx - adjust) as isize; + while floor <= i && floor <= j { + let x = a.get(i as usize).copied().unwrap_or(0); + let y = b.get(j as usize).copied().unwrap_or(0); + if x != y { + break; + } + if x == b'/' { + sfx = a.len() - i as usize; + } + i -= 1; + j -= 1; + } + let a_mid = a.len().saturating_sub(pfx + sfx); + let b_mid = b.len().saturating_sub(pfx + sfx); + let braces = pfx + sfx > 0; + if braces { + name.extend_from_slice(&a[..pfx]); + name.push(b'{'); + } + name.extend_from_slice(&a[pfx..pfx + a_mid]); + name.extend_from_slice(b" => "); + name.extend_from_slice(&b[pfx..pfx + b_mid]); + if braces { + name.push(b'}'); + name.extend_from_slice(&a[a.len() - sfx..]); + } + name +} + +impl Side<'_> { + fn octal(&self) -> u32 { + u32::from(self.mode.value()) + } + + fn file_type(&self) -> u32 { + self.octal() & 0o170000 + } +} + +/// (similarity percent, is a copy) +fn rewrite(change: &ChangeDetached) -> Option<(u32, bool)> { + match change { + ChangeDetached::Rewrite { diff, copy, .. } => { + Some((diff.map_or(100, |stats| percent(stats.similarity)), *copy)) + } + _ => None, + } +} + +/// 53/100 as f32 times 100 floors to 52, so a percent that maps back to the same f32 is exact +fn percent(similarity: f32) -> u32 { + let rounded = (similarity * 100.0).round(); + if rounded / 100.0 == similarity { + rounded as u32 + } else { + (f64::from(similarity) * 100.0) as u32 + } +} + +enum Content { + Text(Vec), + /// None when the blob is too big to diff + Binary { + data: Option>, + size: u64, + }, +} + +impl Content { + fn is_binary(&self) -> bool { + matches!(self, Content::Binary { .. }) + } + + fn size(&self) -> u64 { + match self { + Content::Text(data) => data.len() as u64, + Content::Binary { size, .. } => *size, + } + } + + fn data(&self) -> Option<&[u8]> { + match self { + Content::Text(data) => Some(data), + Content::Binary { data, .. } => data.as_deref(), + } + } +} + +struct Ctx<'a> { + repo: &'a gix::Repository, + max_blob_bytes: u64, +} + +impl Ctx<'_> { + fn load(&self, side: Option>) -> anyhow::Result { + let Some(side) = side else { + return Ok(Content::Text(Vec::new())); + }; + if side.mode.kind() == EntryKind::Commit { + return Ok(Content::Text(knot_git::subproject_line(side.id.into()))); + } + let size = self.repo.find_header(side.id)?.size(); + if size > self.max_blob_bytes { + return Ok(Content::Binary { data: None, size }); + } + let data = self.repo.find_blob(side.id)?.detach().data; + if knot_git::is_binary(&data) { + Ok(Content::Binary { + data: Some(data), + size, + }) + } else if byte_lines(&data).nth(MAX_DIFF_LINES).is_some() { + Ok(Content::Binary { data: None, size }) + } else { + Ok(Content::Text(data)) + } + } +} + +struct FileStat { + name: Vec, + added: u64, + deleted: u64, + binary: bool, +} + +/// one bit per line, so the diff can wait for the stat without keeping the text +struct ChangedLines { + removed: FixedBitSet, + added: FixedBitSet, +} + +impl ChangedLines { + fn diff(old: &[u8], new: &[u8]) -> Self { + let input = InternedInput::new(old, new); + let diff = diff_with_slider_heuristics(Algorithm::Myers, &input); + let mut lines = ChangedLines { + removed: FixedBitSet::with_capacity(input.before.len()), + added: FixedBitSet::with_capacity(input.after.len()), + }; + let range = |range: std::ops::Range| range.start as usize..range.end as usize; + for hunk in diff.hunks() { + lines.removed.insert_range(range(hunk.before)); + lines.added.insert_range(range(hunk.after)); + } + lines + } + + fn hunks(&self) -> Vec { + let (n1, n2) = (self.removed.len(), self.added.len()); + let (mut i, mut j) = (0, 0); + let mut hunks = Vec::new(); + loop { + let start = (i, j); + while i < n1 && self.removed[i] { + i += 1; + } + while j < n2 && self.added[j] { + j += 1; + } + if (i, j) != start { + hunks.push(Hunk { + before: start.0 as u32..i as u32, + after: start.1 as u32..j as u32, + }); + } + if i >= n1 && j >= n2 { + return hunks; + } + i += 1; + j += 1; + } + } +} + +fn file_stat( + ctx: &Ctx<'_>, + change: &ChangeDetached, +) -> anyhow::Result<(FileStat, Option)> { + let (old, new) = sides(change); + let name = match (old, new, rewrite(change)) { + (Some(old), Some(new), Some(_)) => rename_name(old.path, new.path), + _ => quote(new.or(old).expect("a change has a side").path), + }; + let same = old.zip(new).is_some_and(|(old, new)| old.id == new.id); + let (old_content, new_content) = (ctx.load(old)?, ctx.load(new)?); + let (added, deleted, lines) = match (&old_content, &new_content) { + _ if same => (0, 0, None), + (Content::Text(old), Content::Text(new)) => { + let lines = ChangedLines::diff(old, new); + ( + lines.added.count_ones(..) as u64, + lines.removed.count_ones(..) as u64, + Some(lines), + ) + } + _ => (new_content.size(), old_content.size(), None), + }; + let stat = FileStat { + name, + added, + deleted, + binary: old_content.is_binary() || new_content.is_binary(), + }; + Ok((stat, lines)) +} + +fn decimal_width(n: u64) -> usize { + n.checked_ilog10().map_or(1, |digits| digits as usize + 1) +} + +fn write_stat(out: &mut impl Write, stats: &[FileStat]) -> io::Result<()> { + let mut max_change = 0; + let mut max_len = 0; + let mut number_width = 0; + let mut bin_width = 0; + for stat in stats { + max_len = max_len.max(stat.name.len()); + if stat.binary { + bin_width = bin_width.max(14 + decimal_width(stat.added) + decimal_width(stat.deleted)); + number_width = 3; + } else { + max_change = max_change.max(stat.added + stat.deleted); + } + } + number_width = number_width.max(decimal_width(max_change)); + let width = STAT_WIDTH.max(16 + 6 + number_width); + let max_change_width = max_change as usize; + let mut graph_width = if max_change_width + 4 > bin_width { + max_change_width + } else { + bin_width - 4 + }; + let mut name_width = max_len; + if name_width + number_width + 6 + graph_width > width { + let fair = (width * 3 / 8).saturating_sub(number_width + 6); + if graph_width > fair { + graph_width = fair.max(6); + } + if name_width > width.saturating_sub(number_width + 6 + graph_width) { + name_width = width.saturating_sub(number_width + 6 + graph_width); + } else { + graph_width = width - number_width - 6 - name_width; + } + } + + let (mut adds, mut dels) = (0, 0); + for stat in stats { + let mut name: &[u8] = &stat.name; + let mut prefix: &[u8] = b""; + let mut len = name_width; + if name_width < name.len() { + prefix = b"..."; + len = len.saturating_sub(3); + name = &name[name.len() - len..]; + if let Some(slash) = name.iter().position(|&byte| byte == b'/') { + name = &name[slash..]; + } + } + let padding = len.saturating_sub(name.len()); + out.write_all(b" ")?; + out.write_all(prefix)?; + out.write_all(name)?; + write!(out, "{:padding$} | ", "")?; + if stat.binary { + write!(out, "{:>number_width$}", "Bin")?; + if stat.added != 0 || stat.deleted != 0 { + write!(out, " {} -> {} bytes", stat.deleted, stat.added)?; + } + out.write_all(b"\n")?; + continue; + } + adds += stat.added; + dels += stat.deleted; + let (mut add, mut del) = (stat.added, stat.deleted); + if graph_width as u64 <= max_change { + let scale = |it: u64| { + if it == 0 { + 0 + } else { + 1 + it * (graph_width as u64 - 1) / max_change + } + }; + let mut total = scale(add + del); + if total < 2 && add != 0 && del != 0 { + total = 2; + } + if add < del { + add = scale(add); + del = total - add; + } else { + del = scale(del); + add = total - del; + } + } + let changed = stat.added + stat.deleted; + write!(out, "{changed:>number_width$}")?; + if changed != 0 { + out.write_all(b" ")?; + } + out.write_all(&b"+".repeat(add as usize))?; + out.write_all(&b"-".repeat(del as usize))?; + out.write_all(b"\n")?; + } + + let files = stats.len() as u64; + write!(out, " {files} file{} changed", plural(files))?; + if adds != 0 || dels == 0 { + write!(out, ", {adds} insertion{}(+)", plural(adds))?; + } + if dels != 0 || adds == 0 { + write!(out, ", {dels} deletion{}(-)", plural(dels))?; + } + out.write_all(b"\n") +} + +fn plural(count: u64) -> &'static str { + if count == 1 { "" } else { "s" } +} + +fn write_summary(out: &mut impl Write, change: &ChangeDetached) -> io::Result<()> { + let (old, new) = sides(change); + match (old, new, rewrite(change)) { + (None, Some(side), _) | (Some(side), None, _) => { + let what = if old.is_none() { "create" } else { "delete" }; + write!(out, " {what} mode {:06o} ", side.octal())?; + out.write_all("e(side.path))?; + out.write_all(b"\n") + } + (Some(old), Some(new), Some((similarity, copy))) => { + let what = if copy { "copy" } else { "rename" }; + write!(out, " {what} ")?; + out.write_all(&rename_name(old.path, new.path))?; + writeln!(out, " ({similarity}%)")?; + if old.octal() != new.octal() { + writeln!( + out, + " mode change {:06o} => {:06o}", + old.octal(), + new.octal() + )?; + } + Ok(()) + } + (Some(old), Some(new), None) if old.octal() != new.octal() => { + write!( + out, + " mode change {:06o} => {:06o} ", + old.octal(), + new.octal() + )?; + out.write_all("e(new.path))?; + out.write_all(b"\n") + } + _ => Ok(()), + } +} + +fn write_file_patch( + ctx: &Ctx<'_>, + change: &ChangeDetached, + lines: Option<&ChangedLines>, + out: &mut impl Write, +) -> anyhow::Result<()> { + let (old, new) = sides(change); + if let (Some(old), Some(new)) = (old, new) + && old.file_type() != new.file_type() + { + // git shows a type change as a delete plus an add + write_pair(ctx, Some(old), None, None, None, out)?; + return write_pair(ctx, None, Some(new), None, None, out); + } + write_pair(ctx, old, new, rewrite(change), lines, out) +} + +fn abbrev(repo: &gix::Repository, id: ObjectId, full: bool) -> String { + if full { + return id.to_string(); + } + match id.attach(repo).shorten() { + Ok(prefix) => prefix.to_string(), + Err(_) => missing_abbrev(repo, id), + } +} + +/// a submodule commit isn't stored, so it gets the repo's length, longer while a stored id shares it +fn missing_abbrev(repo: &gix::Repository, id: ObjectId) -> String { + let objects = repo.objects.packed_object_count().unwrap_or(0); + let auto = (u64::BITS - objects.leading_zeros()).div_ceil(2).max(7) as usize; + let configured = repo.config_snapshot().integer("core.abbrev"); + let full = id.kind().len_in_hex(); + let mut len = configured.map_or(auto, |len| (len.max(4) as usize).min(full)); + while len < full { + let prefix = gix::hash::Prefix::new(&id, len).expect("length is in range"); + match repo.objects.lookup_prefix(prefix, None) { + Ok(None) => break, + _ => len += 1, + } + } + id.to_hex_with_len(len).to_string() +} + +fn write_pair( + ctx: &Ctx<'_>, + old: Option>, + new: Option>, + rewrite: Option<(u32, bool)>, + lines: Option<&ChangedLines>, + out: &mut impl Write, +) -> anyhow::Result<()> { + let either = new.or(old).expect("a pair has a side"); + let a = quote_prefixed(b"a/", old.unwrap_or(either).path); + let b = quote_prefixed(b"b/", new.unwrap_or(either).path); + let (label_a, label_b) = (label(old, &a), label(new, &b)); + let (mut header, must_show_header) = pair_header(&a, &b, old, new, rewrite); + let mut hunks = Vec::new(); + let ids = (old.map(|side| side.id), new.map(|side| side.id)); + if ids.0 != ids.1 { + let (old_content, new_content) = (ctx.load(old)?, ctx.load(new)?); + let binary = old_content.is_binary() || new_content.is_binary(); + let short = |id: Option| id.map(|id| abbrev(ctx.repo, id, binary)); + let (short_a, short_b) = (short(ids.0), short(ids.1)); + let len = short_a.as_ref().or(short_b.as_ref()).map_or(0, String::len); + let null = || { + ObjectId::null(ctx.repo.object_hash()) + .to_hex_with_len(len) + .to_string() + }; + write!( + header, + "index {}..{}", + short_a.unwrap_or_else(null), + short_b.unwrap_or_else(null) + ) + .expect("vec write"); + match (old, new) { + (Some(old), Some(new)) if old.octal() == new.octal() => { + writeln!(header, " {:06o}", old.octal()).expect("vec write") + } + _ => header.push(b'\n'), + } + + let (Content::Text(old_text), Content::Text(new_text)) = (&old_content, &new_content) + else { + out.write_all(&header)?; + return write_binary(out, &old_content, &new_content, label_a, label_b); + }; + let diffed; + let lines = match lines { + Some(lines) => lines, + // split halves get no precomputed lines + None => { + diffed = ChangedLines::diff(old_text, new_text); + &diffed + } + }; + write_hunks(&mut hunks, old_text, new_text, lines); + } + if hunks.is_empty() { + if must_show_header { + out.write_all(&header)?; + } + return Ok(()); + } + out.write_all(&header)?; + for (marker, label) in [(&b"--- "[..], label_a), (&b"+++ "[..], label_b)] { + out.write_all(marker)?; + out.write_all(label)?; + if label.contains(&b' ') { + out.write_all(b"\t")?; + } + out.write_all(b"\n")?; + } + out.write_all(&hunks)?; + Ok(()) +} + +fn label<'a>(side: Option>, quoted: &'a [u8]) -> &'a [u8] { + match side { + Some(_) => quoted, + None => b"/dev/null", + } +} + +/// the bool says if git prints it for a pair with no hunks +fn pair_header( + a: &[u8], + b: &[u8], + old: Option>, + new: Option>, + rewrite: Option<(u32, bool)>, +) -> (Vec, bool) { + let mut header = Vec::new(); + header.extend_from_slice(b"diff --git "); + header.extend_from_slice(a); + header.push(b' '); + header.extend_from_slice(b); + header.push(b'\n'); + let mut must_show_header = true; + match (old, new) { + (None, Some(new)) => { + writeln!(header, "new file mode {:06o}", new.octal()).expect("vec write") + } + (Some(old), None) => { + writeln!(header, "deleted file mode {:06o}", old.octal()).expect("vec write") + } + (Some(old), Some(new)) => { + must_show_header = rewrite.is_some() || old.octal() != new.octal(); + if old.octal() != new.octal() { + writeln!( + header, + "old mode {:06o}\nnew mode {:06o}", + old.octal(), + new.octal() + ) + .expect("vec write"); + } + } + (None, None) => unreachable!("a pair has a side"), + } + if let (Some(old), Some(new), Some((similarity, copy))) = (old, new, rewrite) { + let what = if copy { "copy" } else { "rename" }; + writeln!(header, "similarity index {similarity}%").expect("vec write"); + write!(header, "{what} from ").expect("vec write"); + header.extend(quote(old.path)); + write!(header, "\n{what} to ").expect("vec write"); + header.extend(quote(new.path)); + header.push(b'\n'); + } + (header, must_show_header) +} + +fn write_binary( + out: &mut impl Write, + old: &Content, + new: &Content, + label_a: &[u8], + label_b: &[u8], +) -> anyhow::Result<()> { + if let (Some(old), Some(new)) = (old.data(), new.data()) { + let mut literals = String::new(); + knot_git::literal_block(new, &mut literals)?; + knot_git::literal_block(old, &mut literals)?; + out.write_all(b"GIT binary patch\n")?; + out.write_all(literals.as_bytes())?; + } else { + out.write_all(b"Binary files ")?; + out.write_all(label_a)?; + out.write_all(b" and ")?; + out.write_all(label_b)?; + out.write_all(b" differ\n")?; + } + Ok(()) +} + +fn funcname(line: &[u8]) -> Option<&[u8]> { + let first = *line.first()?; + if !(first.is_ascii_alphabetic() || first == b'_' || first == b'$') { + return None; + } + Some(trim_end(&line[..line.len().min(FUNCNAME_BYTES)])) +} + +fn write_hunks(out: &mut Vec, old: &[u8], new: &[u8], lines: &ChangedLines) { + let before: Vec<&[u8]> = byte_lines(old).collect(); + let after: Vec<&[u8]> = byte_lines(new).collect(); + let hunks = lines.hunks(); + let (n1, n2) = (before.len() as u32, after.len() as u32); + + let mut func: &[u8] = b""; + let mut func_searched_to: i64 = -1; + let mut i = 0; + while i < hunks.len() { + let mut j = i; + while j + 1 < hunks.len() && hunks[j + 1].before.start - hunks[j].before.end <= 2 * CONTEXT + { + j += 1; + } + let (first, last) = (&hunks[i], &hunks[j]); + let s1 = first.before.start.saturating_sub(CONTEXT); + let s2 = first.after.start.saturating_sub(CONTEXT); + let tail = CONTEXT.min(n1 - last.before.end).min(n2 - last.after.end); + let (e1, e2) = (last.before.end + tail, last.after.end + tail); + + let mut line = i64::from(s1) - 1; + while line != func_searched_to && line >= 0 { + if let Some(found) = funcname(before[line as usize]) { + func = found; + break; + } + line -= 1; + } + func_searched_to = i64::from(s1) - 1; + + let mut header = Vec::with_capacity(128); + header.extend_from_slice(b"@@ -"); + write_range(&mut header, s1, e1 - s1); + header.extend_from_slice(b" +"); + write_range(&mut header, s2, e2 - s2); + header.extend_from_slice(b" @@"); + if !func.is_empty() { + header.push(b' '); + let room = 127 - header.len(); + header.extend_from_slice(&func[..func.len().min(room)]); + } + header.push(b'\n'); + out.extend_from_slice(&header); + + let mut context_from = s2; + for hunk in &hunks[i..=j] { + for line in &after[context_from as usize..hunk.after.start as usize] { + write_line(out, b' ', line); + } + for line in &before[hunk.before.start as usize..hunk.before.end as usize] { + write_line(out, b'-', line); + } + for line in &after[hunk.after.start as usize..hunk.after.end as usize] { + write_line(out, b'+', line); + } + context_from = hunk.after.end; + } + for line in &after[context_from as usize..e2 as usize] { + write_line(out, b' ', line); + } + i = j + 1; + } +} + +fn write_range(out: &mut Vec, start: u32, len: u32) { + write!(out, "{}", if len == 0 { start } else { start + 1 }).expect("vec write"); + if len != 1 { + write!(out, ",{len}").expect("vec write"); + } +} + +fn write_line(out: &mut Vec, marker: u8, line: &[u8]) { + out.push(marker); + out.extend_from_slice(line); + if !line.ends_with(b"\n") { + out.extend_from_slice(b"\n\\ No newline at end of file\n"); + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::process::{Command, Stdio}; + + use super::*; + + const SIGNATURE: &str = "test-signature"; + + fn git(dir: &Path, args: &[&str]) -> Vec { + git_stdin(dir, args, b"") + } + + fn git_stdin(dir: &Path, args: &[&str], input: &[u8]) -> Vec { + let mut child = Command::new("git") + .current_dir(dir) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_AUTHOR_NAME", "Ada Lövelace") + .env("GIT_AUTHOR_EMAIL", "ada@example.com") + .env("GIT_AUTHOR_DATE", "2024-01-02T03:04:05+0530") + .env("GIT_COMMITTER_NAME", "c") + .env("GIT_COMMITTER_EMAIL", "c@example.com") + .env("GIT_COMMITTER_DATE", "2024-01-02T03:04:05+0530") + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("run git"); + child.stdin.take().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + out.stdout + } + + fn init() -> tempfile::TempDir { + init_as("sha1") + } + + fn init_as(object_format: &str) -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + let format = format!("--object-format={object_format}"); + git(dir.path(), &["init", "-q", "-b", "main", &format]); + dir + } + + fn head(dir: &Path) -> ObjectId { + let out = git(dir, &["rev-parse", "HEAD"]); + ObjectId::from_hex(out.trim_ascii()).unwrap() + } + + fn commit(dir: &Path, message: &str) -> ObjectId { + git(dir, &["add", "-A"]); + commit_index(dir, message) + } + + fn commit_index(dir: &Path, message: &str) -> ObjectId { + git(dir, &["commit", "-q", "--allow-empty", "-m", message]); + head(dir) + } + + fn write(dir: &Path, path: &str, contents: impl AsRef<[u8]>) { + let path = dir.join(path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, contents).unwrap(); + } + + fn numbered(count: usize) -> String { + (0..count).map(|i| format!("line {i}\n")).collect() + } + + fn history(object_format: &str) -> (tempfile::TempDir, ObjectId, ObjectId) { + let dir = init_as(object_format); + let root = dir.path(); + write( + root, + "src/lib.rs", + "fn main() {\n one();\n two();\n three();\n four();\n five();\n six();\n seven();\n}\n\nfn other() {\n a();\n b();\n c();\n d();\n e();\n}\n", + ); + write(root, "a.txt", numbered(40)); + write(root, "gone.txt", "bye\n"); + write(root, "dir/moved.txt", numbered(10)); + write(root, "empty", ""); + let base = commit(root, "root"); + + write( + root, + "src/lib.rs", + "fn main() {\n one();\n two();\n THREE();\n four();\n five();\n six();\n seven();\n}\n\nfn other() {\n a();\n b();\n c();\n D();\n e();\n}\n", + ); + write( + root, + "a.txt", + numbered(40) + .replace("line 5\n", "line five\n") + .replace("line 30\n", ""), + ); + write(root, "no newline.txt", "no newline"); + commit( + root, + "Ünïcode subject that is fairly long and goes on past the fold limit of seventy-six\n\nbody line \n\n\nsecond paragraph\n", + ); + + std::fs::remove_file(root.join("gone.txt")).unwrap(); + std::fs::create_dir_all(root.join("dir/sub")).unwrap(); + git(root, &["mv", "dir/moved.txt", "dir/sub/moved.txt"]); + git(root, &["mv", "empty", "dir/empty"]); + std::fs::set_permissions( + root.join("a.txt"), + std::os::unix::fs::PermissionsExt::from_mode(0o755), + ) + .unwrap(); + write(root, "dir/é \"quoted\".txt", "hi\n"); + commit( + root, + "a plain ascii subject that is long enough that git has to wrap it onto the next header line\nwith a second title line", + ); + + git(root, &["checkout", "-q", "-b", "side"]); + write(root, "side.txt", "side\n"); + commit(root, "side branch"); + git(root, &["checkout", "-q", "main"]); + write( + root, + "a-very-long-directory-name/with-a-nested-path/and-a-long-file-name.txt", + numbered(120), + ); + git(root, &["mv", "dir/sub/moved.txt", "renamed.txt"]); + write( + root, + "renamed.txt", + numbered(10).replace("line 4\n", "line four\n"), + ); + commit(root, "big file with a long name"); + git( + root, + &["merge", "-q", "--no-ff", "side", "-m", "merge side"], + ); + + std::fs::remove_file(root.join("src/lib.rs")).unwrap(); + std::os::unix::fs::symlink("target", root.join("src/lib.rs")).unwrap(); + git(root, &["add", "-A"]); + let gitlink = head(root).to_string(); + git( + root, + &[ + "update-index", + "--add", + "--cacheinfo", + &format!("160000,{gitlink},vendor"), + ], + ); + commit_index(root, "turn a file into a symlink and add a submodule"); + + commit_index(root, "empty"); + let tip = head(root); + (dir, base, tip) + } + + fn ours(dir: &Path, from: ObjectId, to: ObjectId) -> Vec { + let repo = gix::open(dir).unwrap(); + let commits = series(&repo, from, to, usize::MAX).unwrap(); + let mut out = Vec::new(); + write_series(&repo, &commits, SIGNATURE, u64::MAX, &mut out).unwrap(); + out + } + + fn theirs(dir: &Path, from: ObjectId, to: ObjectId) -> Vec { + let signature = format!("--signature={SIGNATURE}"); + git( + dir, + &[ + "format-patch", + "--stdout", + &signature, + &format!("{from}..{to}"), + ], + ) + } + + fn assert_reads_like_git(dir: &Path, from: ObjectId, to: ObjectId) -> Vec { + let ours = ours(dir, from, to); + assert_eq!(ours.to_str_lossy(), theirs(dir, from, to).to_str_lossy()); + ours + } + + #[test] + fn a_range_reads_byte_for_byte_like_git_format_patch() { + let (dir, base, tip) = history("sha1"); + assert_reads_like_git(dir.path(), base, tip); + } + + #[test] + fn a_sha256_range_reads_like_git_too() { + let (dir, base, tip) = history("sha256"); + assert_eq!(tip.kind(), gix::hash::Kind::Sha256); + assert_reads_like_git(dir.path(), base, tip); + } + + #[test] + fn a_single_commit_is_an_unnumbered_patch() { + let (dir, base, _) = history("sha1"); + let repo = gix::open(dir.path()).unwrap(); + let first = repo.rev_parse_single("HEAD~5").unwrap().detach(); + let parent = repo.rev_parse_single("HEAD~6").unwrap().detach(); + assert_eq!(parent, base); + let ours = assert_reads_like_git(dir.path(), parent, first); + assert!(ours.starts_with(format!("From {first} ").as_bytes())); + assert!(ours.contains_str("Subject: [PATCH] =?UTF-8?q?")); + } + + #[test] + fn an_empty_or_reversed_range_has_no_patches() { + let (dir, base, tip) = history("sha1"); + let repo = gix::open(dir.path()).unwrap(); + assert!(series(&repo, tip, tip, 0).unwrap().is_empty()); + assert!(series(&repo, tip, base, 0).unwrap().is_empty()); + assert!(ours(dir.path(), tip, base).is_empty()); + } + + #[test] + fn merges_are_left_out_of_the_series() { + let (dir, base, tip) = history("sha1"); + let repo = gix::open(dir.path()).unwrap(); + let subjects: Vec = series(&repo, base, tip, usize::MAX) + .unwrap() + .into_iter() + .map(|id| { + let commit = repo.find_commit(id).unwrap(); + commit.message().unwrap().summary().to_string() + }) + .collect(); + assert_eq!(subjects.len(), 6); + assert!(!subjects.iter().any(|subject| subject.starts_with("merge"))); + assert!(subjects.contains(&"side branch".to_owned())); + } + + #[test] + fn a_range_past_the_commit_limit_is_refused_before_any_patch() { + let (dir, base, tip) = history("sha1"); + let repo = gix::open(dir.path()).unwrap(); + assert_eq!(series(&repo, base, tip, 6).unwrap().len(), 6); + + let refused = series(&repo, base, tip, 5).unwrap_err(); + assert!(matches!(refused, XrpcError::PatchTooLarge { limit: 5 })); + let response = axum::response::IntoResponse::into_response(refused); + assert_eq!(response.status(), axum::http::StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(response.headers()["content-type"], "application/json"); + } + + #[test] + fn binary_changes_apply_with_git_am() { + let dir = init(); + let root = dir.path(); + let blob: Vec = (0..=255u8).cycle().take(4096).collect(); + write(root, "bin.dat", &blob); + write(root, "gone.bin", [0u8, 1, 2]); + let base = commit(root, "root"); + write(root, "bin.dat", &blob[..3000]); + write(root, "new.bin", [0u8; 100]); + std::fs::remove_file(root.join("gone.bin")).unwrap(); + let tip = commit(root, "binary edits"); + + let patch = ours(root, base, tip); + let stat = "bin.dat | Bin 4096 -> 3000 bytes"; + assert!(patch.contains_str(stat), "{}", patch.to_str_lossy()); + assert!(patch.contains_str("GIT binary patch\nliteral 3000\n")); + + let clone = tempfile::tempdir().unwrap(); + git(clone.path(), &["clone", "-q", root.to_str().unwrap(), "."]); + git(clone.path(), &["checkout", "-q", &base.to_string()]); + git_stdin(clone.path(), &["am", "-q"], &patch); + let tree = |dir: &Path, rev: &str| git(dir, &["rev-parse", &format!("{rev}^{{tree}}")]); + assert_eq!(tree(clone.path(), "HEAD"), tree(root, &tip.to_string())); + } + + #[test] + fn blobs_past_the_limit_are_named_but_not_inlined() { + let dir = init(); + let root = dir.path(); + write(root, "big.txt", numbered(10)); + commit(root, "root"); + write(root, "big.txt", numbered(20)); + let tip = commit(root, "grow"); + + let repo = gix::open(root).unwrap(); + let mut out = Vec::new(); + write_series(&repo, &[tip], SIGNATURE, 16, &mut out).unwrap(); + assert!( + out.contains_str(" big.txt | Bin 70 -> 150 bytes\n"), + "{}", + out.to_str_lossy() + ); + assert!(out.contains_str("Binary files a/big.txt and b/big.txt differ\n")); + } + + #[test] + fn text_past_the_line_limit_is_named_but_not_diffed() { + let dir = init(); + let root = dir.path(); + write(root, "long.txt", "\n".repeat(MAX_DIFF_LINES)); + write(root, "longer.txt", "\n".repeat(MAX_DIFF_LINES + 1)); + let base = commit(root, "root"); + write(root, "long.txt", "\n".repeat(MAX_DIFF_LINES - 1) + "end\n"); + write(root, "longer.txt", "\n".repeat(MAX_DIFF_LINES) + "end\n"); + let tip = commit(root, "grow"); + + let out = ours(root, base, tip); + let bytes = MAX_DIFF_LINES + 1; + assert!( + out.contains_str(format!( + " longer.txt | Bin {bytes} -> {} bytes\n", + bytes + 3 + )), + "{}", + out.to_str_lossy() + ); + assert!(out.contains_str("Binary files a/longer.txt and b/longer.txt differ\n")); + assert!(out.contains_str("--- a/long.txt\n+++ b/long.txt\n")); + } + + #[test] + fn a_missing_submodule_commit_is_abbreviated_like_stored_objects() { + let dir = init(); + let root = dir.path(); + git(root, &["config", "core.abbrev", "12"]); + write(root, "a.txt", "a\n"); + let base = commit(root, "root"); + let missing = "0123456789abcdef0123456789abcdef01234567"; + git( + root, + &[ + "update-index", + "--add", + "--cacheinfo", + &format!("160000,{missing},vendor"), + ], + ); + let tip = commit_index(root, "add a submodule"); + let ours = assert_reads_like_git(root, base, tip); + assert!(ours.contains_str("index 000000000000..0123456789ab\n")); + } + + #[test] + fn renames_are_found_up_to_the_limit_squared_in_pairs_like_git() { + let dir = init(); + let root = dir.path(); + let contents = |file: usize| -> String { + (0..20) + .map(|line| format!("file {file} line {line}\n")) + .collect() + }; + for file in 0..33 { + write(root, &format!("old/{file}.txt"), contents(file)); + } + let base = commit(root, "root"); + std::fs::remove_dir_all(root.join("old")).unwrap(); + for file in 0..33 { + let edited = contents(file).replace("line 3\n", "line three\n"); + write(root, &format!("new/{file}.txt"), edited); + } + let tip = commit(root, "move and edit 33 files"); + let ours = ours(root, base, tip); + assert!( + ours.contains_str(" rename {old => new}/0.txt ("), + "{}", + ours.to_str_lossy() + ); + let theirs = theirs(root, base, tip); + assert_eq!( + without_similarity(&ours).to_str_lossy(), + without_similarity(&theirs).to_str_lossy() + ); + } + + /// git won't write a broken zone itself, so the commit is hashed by hand + fn zoned(zone: &str) -> (tempfile::TempDir, ObjectId, ObjectId) { + let dir = init(); + let root = dir.path(); + write(root, "a.txt", "a\n"); + let base = commit(root, "root"); + write(root, "a.txt", "b\n"); + git(root, &["add", "-A"]); + let tree = git(root, &["write-tree"]); + let raw = format!( + "tree {}\nparent {base}\nauthor a 2212270408 {zone}\n\ + committer a 2212270408 {zone}\n\nzone\n", + tree.trim_ascii().to_str_lossy() + ); + let hash = [ + "hash-object", + "-t", + "commit", + "-w", + "--literally", + "--stdin", + ]; + let out = git_stdin(root, &hash, raw.as_bytes()); + let tip = ObjectId::from_hex(out.trim_ascii()).unwrap(); + (dir, base, tip) + } + + #[test] + fn broken_time_zones_shift_the_date_like_git() { + for (zone, date) in [ + ("+5785265", "Fri, 14 Sep 2046 12:38:28 +5785265"), + ("+2147483646", "Tue, 7 Feb 2040 23:50:40 +2147483646"), + ("+300000000000000000", "Tue, 7 Feb 2040 23:33:28 +0000"), + ] { + let (dir, base, tip) = zoned(zone); + let ours = assert_reads_like_git(dir.path(), base, tip); + assert!(ours.contains_str(format!("Date: {date}\n"))); + } + } + + fn edit_reads_like_git(old: &str, new: &str) { + let dir = init(); + let root = dir.path(); + write(root, "file.c", old); + let base = commit(root, "old"); + write(root, "file.c", new); + let tip = commit(root, "new"); + assert_reads_like_git(root, base, tip); + } + + #[test] + fn lines_are_counted_over_the_whole_file_like_git() { + edit_reads_like_git( + "\n *\n\n info_class->info_type = GI_INFO_TYPE_UNRESOLVED;\n}\n", + " * version 2 of the License, or (at your option) any later version.\n *\n *\n */\n\n info_class->info_type = GI_INFO_TYPE_UNRESOLVED;\n}\n", + ); + } + + #[test] + fn the_too_common_limit_rounds_up_like_git() { + edit_reads_like_git( + "\n\n\n\n", + "\tdef __init__ (self) :\n\t\tself.xc = None\n\n\tdef callback (self) :\n\t\tid = self.get_uint()\n\t\tself.xc.medialib_get_info(id, MediainfoRes)\n\nxc = xmmsclient.XMMS ()\nxc.connect()\n\t\nxc.signal_playback_playtime(PlaytimeRes)\n#res = xc.broadcast_playback_current_id(CurrentIDRes)\n#res.xc = xc\n\nxc.python_loop()\n\n", + ); + } + + /// git counts the common line itself in both runs + #[test] + fn a_common_line_needs_more_unmatched_neighbours_to_be_dropped_like_git() { + edit_reads_like_git( + "{ return NULL; }\n\nvoid cleanup_shm (xmmsc_vis_unixshm_t *t) {}\nint read_start_shm (xmmsc_vis_unixshm_t *t, int blocking, xmmsc_vischunk_t **dest) {}\nvoid read_finish_shm (xmmsc_vis_unixshm_t *t, xmmsc_vischunk_t *dest) {}\n", + "\n\n\n\n", + ); + } + + #[test] + fn similarity_floors_to_a_percent_like_git() { + for max in 1..=2000u32 { + for copied in max / 2..=max { + let git = copied * 60000 / max * 100 / 60000; + let similarity = copied as f32 / max as f32; + assert_eq!(percent(similarity), git, "{copied}/{max}"); + } + } + } + + #[test] + fn subjects_wrap_like_git() { + let mut buf = b"Subject: [PATCH 1/2] ".to_vec(); + let indent = -(last_line_len(&buf) as isize); + add_wrapped( + &mut buf, + &b"word ".repeat(20)[..99], + indent, + 1, + HEADER_WIDTH, + ); + let lines: Vec<&[u8]> = buf.lines().collect(); + assert_eq!(lines.len(), 2); + assert!(lines.iter().all(|line| line.len() <= HEADER_WIDTH)); + assert!(lines[1].starts_with(b" word")); + } + + #[test] + fn renames_pull_out_the_shared_directories() { + let name = |a: &str, b: &str| { + rename_name(a.as_bytes(), b.as_bytes()) + .to_str_lossy() + .into_owned() + }; + assert_eq!(name("a.txt", "b.txt"), "a.txt => b.txt"); + assert_eq!(name("dir/a.txt", "dir/b.txt"), "dir/{a.txt => b.txt}"); + assert_eq!(name("dir/x/f", "dir/y/f"), "dir/{x => y}/f"); + assert_eq!(name("dir/f", "dir/sub/f"), "dir/{ => sub}/f"); + assert_eq!(name("é", "a"), "\"\\303\\251\" => a"); + } + + /// xorshift so inputs are the same every run + struct Rng(u64); + + impl Rng { + fn seeded(seed: u64) -> Self { + Rng(seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1) + } + + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + } + + /// run with `cargo test --release -p knotmirror-xrpc large_diff -- --ignored --nocapture` + fn large_diff(edit: impl Fn(usize, usize) -> String) { + for lines in [SMALL, LARGE] { + let old: String = (0..lines).map(padded_line).collect(); + let new: String = (0..lines).map(|i| edit(i, lines)).collect(); + time_diff(old.as_bytes(), new.as_bytes()); + } + } + + #[allow(clippy::print_stderr)] + fn time_diff(old: &[u8], new: &[u8]) { + let start = std::time::Instant::now(); + let lines = ChangedLines::diff(old, new); + let mut out = Vec::new(); + write_hunks(&mut out, old, new, &lines); + eprintln!( + "{} KiB, {} lines: {:?}, {} bytes out", + old.len() >> 10, + lines.removed.len(), + start.elapsed(), + out.len() + ); + } + + fn padded(text: String) -> String { + format!("{text:<47}\n") + } + + fn padded_line(i: usize) -> String { + padded(format!("line {i}")) + } + + const SMALL: usize = (4 << 20) / 48; + const LARGE: usize = (20 << 20) / 48; + + #[test] + #[ignore] + fn large_diff_few_edits() { + large_diff(|i, _| match i % 50_000 { + 7 => padded(format!("edited {i}")), + _ => padded_line(i), + }); + } + + #[test] + #[ignore] + fn large_diff_spread_edits() { + large_diff(|i, _| match i % 10 { + 3 => padded(format!("edited {i}")), + _ => padded_line(i), + }); + } + + #[test] + #[ignore] + fn large_diff_all_different() { + large_diff(|i, _| padded(format!("different {i}"))); + } + + #[test] + #[ignore] + fn large_diff_reversed() { + large_diff(|i, lines| padded_line(lines - 1 - i)); + } + + /// short lines from two values, the slowest input for myers we found + #[test] + #[ignore] + fn large_diff_churn() { + let mut rng = Rng(7); + let mut churn = |lines| -> Vec { + (0..lines) + .flat_map(|_| [b"ab"[rng.below(2)], b'\n']) + .collect() + }; + for lines in [50_000, 100_000, 200_000, 400_000, 1_000_000, MAX_DIFF_LINES] { + time_diff(&churn(lines), &churn(lines)); + } + } + + const DIRS: &[&str] = &[ + "src", + "dir with space", + "é", + "日本", + "q\"uote", + "back\\slash", + "a-rather-long-directory-name-that-pushes-the-stat-past-its-width", + ]; + const LEAVES: &[&str] = &[ + "a.txt", + "b.rs", + "name with spaces.md", + "ünïcödé.txt", + "tab\there", + "new\nline", + "\"quoted\"", + "x", + "a-very-long-file-name-that-goes-on-and-on-well-past-the-stat-width.txt", + ]; + const NAMES: &[&str] = &[ + "Ada Lövelace", + "plain name", + "Quote \"Q\" Name", + "dots. and, commas", + "a very long author name that goes on well past the seventy-eight column header limit", + "日本 語の名前", + "=?not encoded?=", + ]; + const WORDS: &[&str] = &[ + "fix", + "the", + "über", + "parser", + "=?", + "a", + "[tag]", + "long-hyphenated-word", + "日本", + "", + "tab\tin", + "x", + ]; + + fn pick<'a>(rng: &mut Rng, from: &[&'a str]) -> &'a str { + from[rng.below(from.len())] + } + + fn random_path(rng: &mut Rng) -> String { + let mut path = String::new(); + for _ in 0..rng.below(3) { + path.push_str(pick(rng, DIRS)); + path.push('/'); + } + path + pick(rng, LEAVES) + } + + fn random_line(rng: &mut Rng) -> String { + match rng.below(6) { + 0 => format!("fn item_{}() {{", rng.below(50)), + 1 => " }".to_owned(), + 2 => String::new(), + _ => format!(" value {} {}", rng.below(1000), pick(rng, WORDS)), + } + } + + fn random_contents(rng: &mut Rng) -> Vec { + let ending = if rng.below(5) == 0 { "\r\n" } else { "\n" }; + let mut text = Vec::new(); + for _ in 0..rng.below(40) { + text.extend_from_slice(random_line(rng).as_bytes()); + text.extend_from_slice(ending.as_bytes()); + } + match rng.below(8) { + 0 => text.clear(), + 1 => text.extend_from_slice(b"no newline"), + 2 => text.splice(0..0, [0, 1, 2, 3]).for_each(drop), + _ => {} + } + text + } + + fn edited(rng: &mut Rng, contents: &[u8]) -> Vec { + let mut lines: Vec> = contents + .lines_with_terminator() + .map(<[u8]>::to_vec) + .collect(); + for _ in 0..1 + rng.below(3) { + let at = rng.below(lines.len() + 1); + match rng.below(3) { + 0 if at < lines.len() => { + lines.remove(at); + } + 1 if at < lines.len() => lines[at] = (random_line(rng) + "\n").into_bytes(), + _ => lines.insert(at, (random_line(rng) + "\n").into_bytes()), + } + } + lines.concat() + } + + fn random_words(rng: &mut Rng, min: usize, max: usize) -> String { + let count = min + rng.below(max - min); + (0..count) + .map(|_| pick(rng, WORDS)) + .collect::>() + .join(" ") + } + + fn random_message(rng: &mut Rng) -> String { + let mut message = random_words(rng, 1, 25); + if rng.below(3) == 0 { + message.push('\n'); + message.push_str(&random_words(rng, 1, 8)); + } + if rng.below(2) == 0 { + message.push_str("\n\n"); + message.push_str(&random_words(rng, 0, 30)); + } + message + } + + /// a `git fast-import` stream and how many commits it has past the root + fn random_stream(rng: &mut Rng) -> (Vec, usize) { + let mut files: Vec<(String, Vec, &str)> = Vec::new(); + let mut stream = Vec::new(); + let commits = 1 + rng.below(4); + for index in 0..=commits { + let (author, message) = match index { + 0 => ("root", "root".to_owned()), + _ => (pick(rng, NAMES), random_message(rng)), + }; + write!( + stream, + "commit refs/heads/main\nauthor {author} 1704144845 +0530\n\ + committer c 1704144845 +0530\ndata {}\n{message}\n", + message.len() + ) + .unwrap(); + let mut touched = Vec::new(); + let ops = if index == 0 { 0 } else { 1 + rng.below(4) }; + for _ in 0..ops { + let op = if files.is_empty() { 0 } else { rng.below(6) }; + let at = rng.below(files.len().max(1)); + match op { + 0 => { + let Some(path) = unused_path(rng, &files) else { + continue; + }; + files.push((path.clone(), random_contents(rng), "100644")); + touched.push(path); + } + 3 => { + let Some(path) = unused_path(rng, &files) else { + continue; + }; + let (from, contents, mode) = files.remove(at); + delete(&mut stream, &from); + let contents = match rng.below(2) { + 0 => edited(rng, &contents), + _ => contents, + }; + files.push((path.clone(), contents, mode)); + touched.push(path); + } + 1 | 2 => { + files[at].1 = edited(rng, &files[at].1); + touched.push(files[at].0.clone()); + } + 4 => { + let modes = ["100644", "100755", "120000"]; + let mut others = modes.into_iter().filter(|&mode| mode != files[at].2); + files[at].2 = others.nth(rng.below(2)).unwrap(); + touched.push(files[at].0.clone()); + } + _ => { + let (path, ..) = files.remove(at); + delete(&mut stream, &path); + } + } + } + for (path, contents, mode) in &files { + if touched.contains(path) { + write!(stream, "M {mode} inline ").unwrap(); + stream.extend(quote(path.as_bytes())); + write!(stream, "\ndata {}\n", contents.len()).unwrap(); + stream.extend_from_slice(contents); + stream.push(b'\n'); + } + } + stream.push(b'\n'); + } + (stream, commits) + } + + fn unused_path(rng: &mut Rng, files: &[(String, Vec, &str)]) -> Option { + let path = random_path(rng); + (!files.iter().any(|(file, ..)| file == &path)).then_some(path) + } + + fn delete(stream: &mut Vec, path: &str) { + stream.extend_from_slice(b"D "); + stream.extend(quote(path.as_bytes())); + stream.push(b'\n'); + } + + /// built with one `git fast-import` so a case costs few processes + fn random_history(seed: u64) -> (tempfile::TempDir, ObjectId, ObjectId) { + let (stream, commits) = random_stream(&mut Rng::seeded(seed)); + let dir = init(); + let root = dir.path(); + git_stdin(root, &["fast-import", "--quiet"], &stream); + let ids = git(root, &["rev-parse", &format!("main~{commits}"), "main"]); + let mut ids = ids.lines().map(|id| ObjectId::from_hex(id).unwrap()); + (dir, ids.next().unwrap(), ids.next().unwrap()) + } + + /// binary bodies differ from git's, `binary_changes_apply_with_git_am` checks they apply + fn without_binary_bodies(patch: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut blanks_left = 0; + for line in patch.lines_with_terminator() { + if blanks_left > 0 { + blanks_left -= usize::from(line == b"\n"); + continue; + } + if line == b"GIT binary patch\n" { + blanks_left = 2; + } + out.extend_from_slice(line); + } + out + } + + /// gix scores renames by lines, git by bytes + fn without_similarity(patch: &[u8]) -> Vec { + let mut out = Vec::new(); + for line in patch.lines_with_terminator() { + let scored = line.starts_with(b"similarity index ") + || ((line.starts_with(b" rename ") || line.starts_with(b" copy ")) + && line.ends_with(b"%)\n")); + match line.iter().rposition(u8::is_ascii_whitespace) { + Some(end) if scored => { + let start = line[..end].rfind_byte(b' ').map_or(0, |space| space + 1); + out.extend_from_slice(&line[..start]); + out.extend_from_slice(b"N%"); + out.extend_from_slice(&line[end..]); + } + _ => out.extend_from_slice(line), + } + } + out + } + + fn normalized(patch: &[u8]) -> Vec { + without_similarity(&without_binary_bodies(patch)) + } + + /// keeps input order, workers pull one case at a time so a slow one doesn't stall a chunk + fn in_parallel(items: &[T], f: impl Fn(&T) -> R + Sync) -> Vec { + let next = std::sync::atomic::AtomicUsize::new(0); + let threads = std::thread::available_parallelism().map_or(4, usize::from); + let mut done: Vec<(usize, R)> = std::thread::scope(|scope| { + let workers: Vec<_> = (0..threads) + .map(|_| { + scope.spawn(|| { + let mut done = Vec::new(); + loop { + let index = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let Some(item) = items.get(index) else { + return done; + }; + done.push((index, f(item))); + } + }) + }) + .collect(); + workers + .into_iter() + .flat_map(|worker| worker.join().unwrap()) + .collect() + }); + done.sort_unstable_by_key(|(index, _)| *index); + done.into_iter().map(|(_, result)| result).collect() + } + + /// `keep` leaves each diverging repo and both patches on disk, named on stderr + #[allow(clippy::print_stderr)] + fn diverging(seeds: std::ops::Range, keep: bool) -> Vec { + let check = |&seed: &u64| { + let (dir, base, tip) = random_history(seed); + let ours = normalized(&ours(dir.path(), base, tip)); + let theirs = normalized(&theirs(dir.path(), base, tip)); + let diverged = ours != theirs; + if diverged && keep { + let dir = dir.keep(); + keep_patches(&dir, "", &ours, &theirs); + eprintln!("seed {seed} diverged: {}", dir.display()); + } + diverged + }; + let seeds: Vec = seeds.collect(); + let diverged = in_parallel(&seeds, check); + seeds + .into_iter() + .zip(diverged) + .filter_map(|(seed, diverged)| diverged.then_some(seed)) + .collect() + } + + /// seed 3 is a binary rename with an edit, git pairs those by bytes and gix doesn't + #[test] + fn random_histories_read_like_git() { + assert_eq!(diverging(0..100, false), [3]); + } + + /// known drift: binary renames with edits, renames near 50%, blank lines imara-diff slides elsewhere + #[test] + #[ignore] + #[allow(clippy::print_stderr)] + fn many_random_histories_read_like_git() { + eprintln!("{:?}", diverging(0..1000, true)); + } + + #[derive(Clone, Copy, PartialEq)] + enum Section { + Series, + Author, + Date, + Subject, + Mime, + Message, + Diffstat, + DiffstatTotal, + Summary, + DiffHeader, + HunkHeader, + HunkBody, + Binary, + Signature, + } + + impl std::fmt::Display for Section { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Section::Series => "series", + Section::Author => "author", + Section::Date => "date", + Section::Subject => "subject", + Section::Mime => "mime", + Section::Message => "message", + Section::Diffstat => "diffstat", + Section::DiffstatTotal => "diffstat total", + Section::Summary => "summary", + Section::DiffHeader => "diff header", + Section::HunkHeader => "hunk header", + Section::HunkBody => "hunk body", + Section::Binary => "binary", + Section::Signature => "signature", + }) + } + } + + #[derive(Clone, Copy)] + enum State { + Headers, + Message, + Stat, + Diff, + Hunk, + Binary, + Signature, + } + + struct Sections { + state: State, + header: Section, + } + + impl Sections { + fn new() -> Self { + Sections { + state: State::Headers, + header: Section::Author, + } + } + + fn of(&mut self, line: &[u8]) -> Section { + let starts = |prefix: &[u8]| line.starts_with(prefix); + if starts(b"From ") && line.ends_with(b" Mon Sep 17 00:00:00 2001\n") { + self.state = State::Headers; + return Section::Series; + } + match self.state { + State::Headers => { + if line == b"\n" { + self.state = State::Message; + return Section::Message; + } + if !starts(b" ") { + self.header = if starts(b"From: ") { + Section::Author + } else if starts(b"Date: ") { + Section::Date + } else if starts(b"Subject: ") { + Section::Subject + } else { + Section::Mime + }; + } + self.header + } + State::Message => { + if line == b"---\n" { + self.state = State::Stat; + } + Section::Message + } + State::Stat => { + if line == b"\n" { + self.state = State::Diff; + Section::Summary + } else if line.contains_str(" | ") { + Section::Diffstat + } else if line.contains_str(" changed") && !starts(b" mode change") { + Section::DiffstatTotal + } else { + Section::Summary + } + } + _ if line == b"-- \n" => { + self.state = State::Signature; + Section::Signature + } + State::Diff | State::Hunk | State::Binary if starts(b"diff --git ") => { + self.state = State::Diff; + Section::DiffHeader + } + State::Diff | State::Hunk if starts(b"@@ ") => { + self.state = State::Hunk; + Section::HunkHeader + } + State::Diff if starts(b"GIT binary patch") || starts(b"Binary files ") => { + self.state = State::Binary; + Section::Binary + } + State::Diff => Section::DiffHeader, + State::Hunk => Section::HunkBody, + State::Binary => Section::Binary, + State::Signature => Section::Signature, + } + } + } + + /// drops hunks and the stat, so what is left differs only when it isn't the line diff + fn skeleton(patch: &[u8]) -> Vec { + let mut sections = Sections::new(); + let mut out = Vec::new(); + for line in patch.lines_with_terminator() { + let section = sections.of(line); + if !matches!( + section, + Section::HunkHeader + | Section::HunkBody + | Section::Diffstat + | Section::DiffstatTotal + ) { + out.extend_from_slice(line); + } + } + out + } + + fn first_difference(ours: &[u8], theirs: &[u8]) -> Section { + let ours: Vec<&[u8]> = ours.lines_with_terminator().collect(); + let theirs: Vec<&[u8]> = theirs.lines_with_terminator().collect(); + let mut sections = Sections::new(); + for index in 0..ours.len().max(theirs.len()) { + let line = theirs.get(index).or(ours.get(index)).expect("in range"); + let section = sections.of(line); + if ours.get(index) != theirs.get(index) { + return section; + } + } + unreachable!("only called on patches that differ") + } + + fn env_knob(name: &str, default: usize) -> usize { + std::env::var(name).map_or(default, |value| value.parse().expect(name)) + } + + fn rev_list(dir: &Path, args: &[&str]) -> Vec> { + let out = git(dir, &[&["rev-list"], args].concat()); + out.lines() + .map(|line| { + let ids = line + .split_str(" ") + .map(|id| ObjectId::from_hex(id).unwrap()); + ids.collect() + }) + .collect() + } + + fn sample(rng: &mut Rng, items: &[T], count: usize) -> Vec { + let mut items = items.to_vec(); + let count = count.min(items.len()); + for i in 0..count { + let j = i + rng.below(items.len() - i); + items.swap(i, j); + } + items.truncate(count); + items + } + + #[derive(Clone, Copy, PartialEq)] + enum Class { + Same, + LineDiffOnly, + OverCap, + Error, + Panic, + Differs(Section), + } + + impl std::fmt::Display for Class { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Class::Same => f.write_str("same"), + Class::LineDiffOnly => f.write_str("line diff only"), + Class::OverCap => f.write_str("over the commit cap"), + Class::Error => f.write_str("error"), + Class::Panic => f.write_str("panic"), + Class::Differs(section) => section.fmt(f), + } + } + } + + struct Checked { + class: Class, + ours: Vec, + theirs: Vec, + ours_time: std::time::Duration, + git_time: std::time::Duration, + } + + fn panic_message(panic: &(dyn std::any::Any + Send)) -> String { + panic + .downcast_ref::() + .cloned() + .or_else(|| panic.downcast_ref::<&str>().map(|text| (*text).to_owned())) + .unwrap_or_default() + } + + fn check_real( + dir: &Path, + repo: &gix::ThreadSafeRepository, + from: ObjectId, + to: ObjectId, + ) -> Checked { + let repo = repo.to_thread_local(); + let start = std::time::Instant::now(); + let ours = match series(&repo, from, to, MAX_PATCH_COMMITS) { + Err(XrpcError::PatchTooLarge { .. }) => Err((Class::OverCap, String::new())), + Err(error) => Err((Class::Error, format!("{error:?}"))), + Ok(commits) => { + let written = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut out = Vec::new(); + write_series(&repo, &commits, SIGNATURE, u64::MAX, &mut out).map(|()| out) + })); + match written { + Ok(Ok(out)) => Ok(out), + Ok(Err(error)) => Err((Class::Error, format!("{error:#}"))), + Err(panic) => Err((Class::Panic, panic_message(&*panic))), + } + } + }; + let ours_time = start.elapsed(); + let start = std::time::Instant::now(); + let theirs = theirs(dir, from, to); + let git_time = start.elapsed(); + let (class, ours) = match ours { + Ok(ours) => { + let (normal_ours, normal_theirs) = (normalized(&ours), normalized(&theirs)); + let (bare_ours, bare_theirs) = (skeleton(&normal_ours), skeleton(&normal_theirs)); + let class = if normal_ours == normal_theirs { + Class::Same + } else if bare_ours == bare_theirs { + Class::LineDiffOnly + } else { + Class::Differs(first_difference(&bare_ours, &bare_theirs)) + }; + (class, ours) + } + Err((class, message)) => (class, message.into_bytes()), + }; + Checked { + class, + ours, + theirs, + ours_time, + git_time, + } + } + + /// non-merge commits against their parent, then first-parent ranges of 2 to 20 commits + fn sample_cases(dir: &Path, seed: u64) -> (Vec<(ObjectId, ObjectId)>, usize) { + let mut rng = Rng::seeded(seed); + + // roots are left out, the api has no way to ask for a range that ends at nothing + let singles = rev_list( + dir, + &["--all", "--no-merges", "--min-parents=1", "--parents"], + ); + let mut cases: Vec<(ObjectId, ObjectId)> = + sample(&mut rng, &singles, env_knob("KNOTMIRROR_FUZZ_COMMITS", 500)) + .into_iter() + .map(|ids| (ids[1], ids[0])) + .collect(); + let singles_len = cases.len(); + let repo = gix::open(dir).unwrap(); + let tips = rev_list(dir, &["--all"]); + let ranges = env_knob("KNOTMIRROR_FUZZ_RANGES", 100); + for _ in 0..ranges * 4 { + if cases.len() == singles_len + ranges { + break; + } + let tip = tips[rng.below(tips.len())][0]; + let mut base = Some(tip); + for _ in 0..2 + rng.below(19) { + base = base.and_then(|id| { + let commit = repo.find_commit(id).unwrap(); + commit.parent_ids().next().map(gix::Id::detach) + }); + } + if let Some(base) = base { + cases.push((base, tip)); + } + } + (cases, singles_len) + } + + fn keep_patches(dir: &Path, prefix: &str, ours: &[u8], theirs: &[u8]) { + std::fs::write(dir.join(format!("{prefix}ours.patch")), ours).unwrap(); + std::fs::write(dir.join(format!("{prefix}theirs.patch")), theirs).unwrap(); + } + + /// (cases, how many are single commits, label for the kept dir) + fn real_cases(dir: &Path) -> (Vec<(ObjectId, ObjectId)>, usize, String) { + let Ok(ranges) = std::env::var("KNOTMIRROR_FUZZ_CASES") else { + let seed = env_knob("KNOTMIRROR_FUZZ_SEED", 1) as u64; + let (cases, singles_len) = sample_cases(dir, seed); + return (cases, singles_len, seed.to_string()); + }; + let cases: Vec<_> = ranges + .split_whitespace() + .map(|range| { + let (from, to) = range.split_once("..").expect("from..to"); + ( + ObjectId::from_hex(from.as_bytes()).unwrap(), + ObjectId::from_hex(to.as_bytes()).unwrap(), + ) + }) + .collect(); + let len = cases.len(); + (cases, len, "cases".to_owned()) + } + + #[allow(clippy::print_stderr)] + fn report( + dir: &Path, + cases: &[(ObjectId, ObjectId)], + singles_len: usize, + results: &[Checked], + kept: &Path, + wall: std::time::Duration, + ) { + let name = dir.file_name().unwrap().to_string_lossy(); + let mut index = format!("repo {}\n", dir.display()); + let mut classes: std::collections::BTreeMap> = Default::default(); + for (number, (&(from, to), result)) in cases.iter().zip(results).enumerate() { + let range = format!("{from}..{to}"); + let kind = if number < singles_len { + "commit" + } else { + "range" + }; + let class = result.class.to_string(); + classes + .entry(class.clone()) + .or_default() + .push(range.clone()); + if !matches!(result.class, Class::Same | Class::OverCap) { + keep_patches(kept, &format!("{number}."), &result.ours, &result.theirs); + index.push_str(&format!("{number} {kind} {range} {class}\n")); + } + } + std::fs::write(kept.join("cases.txt"), index).unwrap(); + + eprintln!( + "{name}: {singles_len} commits and {} ranges in {wall:?}, divergences in {}", + cases.len() - singles_len, + kept.display() + ); + for (class, ranges) in &classes { + eprintln!(" {:>5} {class}, e.g. {}", ranges.len(), ranges[0]); + } + let ours_total: std::time::Duration = results.iter().map(|result| result.ours_time).sum(); + let git_total: std::time::Duration = results.iter().map(|result| result.git_time).sum(); + eprintln!(" ours {ours_total:?}, git {git_total:?}, slowest for us:"); + let mut slowest: Vec<_> = cases.iter().zip(results).collect(); + slowest.sort_by_key(|(_, result)| std::cmp::Reverse(result.ours_time)); + for ((from, to), result) in slowest.into_iter().take(10) { + eprintln!( + " {from}..{to} ours {:?} git {:?} {}", + result.ours_time, result.git_time, result.class + ); + } + } + + /// `KNOTMIRROR_FUZZ_REPO= cargo test --release -p knotmirror-xrpc real_history -- --ignored --nocapture` + /// known drift: hunks imara-diff slides or splits, renames git pairs by bytes and best score first, + /// messages git re-encodes from `encoding` + #[test] + #[ignore] + #[allow(clippy::print_stderr)] + fn real_history_reads_like_git() { + let Some(path) = std::env::var_os("KNOTMIRROR_FUZZ_REPO") else { + eprintln!("set KNOTMIRROR_FUZZ_REPO to a clone to compare against git"); + return; + }; + let dir = Path::new(&path); + let (cases, singles_len, label) = real_cases(dir); + + let shared = gix::open(dir).unwrap().into_sync(); + let started = std::time::Instant::now(); + let results = in_parallel(&cases, |&(from, to)| check_real(dir, &shared, from, to)); + let wall = started.elapsed(); + + let name = dir.file_name().unwrap().to_string_lossy(); + let kept = std::env::temp_dir().join(format!("knotmirror-realfuzz-{name}-{label}")); + std::fs::create_dir_all(&kept).unwrap(); + report(dir, &cases, singles_len, &results, &kept, wall); + let diverged = results + .iter() + .filter(|result| !matches!(result.class, Class::Same | Class::OverCap)) + .count(); + assert_eq!(diverged, 0, "see {}", kept.display()); + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/git_transport.rs b/knotmirror/crates/knotmirror-xrpc/src/git_transport.rs new file mode 100644 index 0000000000000000000000000000000000000000..7e4edf83b1599f507a2da4e13f8ac1a6118c57b3 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/git_transport.rs @@ -0,0 +1,446 @@ +//! `gix_transport`'s async [`Transport`] over knotmirror's own `reqwest` client, which is what +//! [`gix_receive_pack::Session`] talks through. +//! +//! Smart HTTP, both halves of it: +//! +//! - `handshake` — `GET /info/refs?service=git-receive-pack`, whose body is fed to a +//! packetline reader after the `application/x-git-receive-pack-advertisement` content-type check +//! and the `# service=` line, exactly as `gix_transport::client::blocking_io::http::Transport` +//! does; [`Handshake::from_lines_with_version_detection`] takes it from there. +//! - `request` — `POST /git-receive-pack` with `application/x-git-receive-pack-request`, and a +//! [`RequestWriter`] over the body it carries. +//! +//! `gix-transport`'s own blocking HTTP transport is the reference implementation; its async +//! counterpart doesn't exist, which is why this is here. +//! +//! Neither body is streamed: `request` is not `async`, so the POST cannot be awaited where it is +//! built, and both bodies are small or already in memory — the pack `gix-receive-pack` writes is a +//! `&[u8]` to begin with, and the reply is one line per ref. So the request body is collected until +//! the response is first read, and the response is read in full before its first byte is handed +//! out. The one thing lost is live side-band progress, which arrives with the report instead; the +//! caller passes `gix::progress::Discard` anyway. + +use std::any::Any; +use std::borrow::Cow; +use std::cell::RefCell; +use std::future::Future; +use std::io; +use std::pin::Pin; +use std::rc::Rc; +use std::task::{Context, Poll, ready}; + +use futures_lite::AsyncReadExt as _; +use gix_transport::async_trait::async_trait; +use gix_transport::bstr::BStr; +use gix_transport::client::async_io::{self, HandleProgress, RequestWriter, SetServiceResponse}; +use gix_transport::client::capabilities::async_recv::Handshake; +use gix_transport::client::{self, MessageKind, TransportWithoutIO, WriteMode}; +use gix_transport::futures_io::{AsyncRead, AsyncWrite}; +use gix_transport::packetline::PacketLineRef; +use gix_transport::packetline::async_io::StreamingPeekableIter; +use gix_transport::{Protocol, Service}; + +/// The `User-Agent` header, which is not the `agent=` capability — that one is the caller's. +const USER_AGENT: &str = concat!("knotmirror/", env!("CARGO_PKG_VERSION")); + +/// A v1 advertisement and a v1 report both end at a flush. +const DELIMITERS: &[PacketLineRef<'static>] = &[PacketLineRef::Flush]; + +pub(crate) struct HttpTransport { + http: reqwest::Client, + /// The caller's service-auth JWT, sent on both halves: a knot authorizes the + /// `git-receive-pack` advertisement, not just the push that follows it. + token: String, + /// The repository's base URL, without the `/info/refs` or `/git-receive-pack` suffix. + url: String, + /// The service `handshake` was performed for; `None` until then. + service: Option, + /// The reader over the response in flight. Created by `handshake` and reused by `request`, both + /// for its line buffers and because the POST is only legal after the advertisement. + lines: Option>, +} + +impl HttpTransport { + pub(crate) fn new(http: reqwest::Client, url: String, token: String) -> Self { + Self { + http, + token, + url, + service: None, + lines: None, + } + } +} + +impl TransportWithoutIO for HttpTransport { + fn to_url(&self) -> Cow<'_, BStr> { + Cow::Borrowed(BStr::new(&self.url)) + } + + /// v0 only: `git-receive-pack` has no v2 command set, and `V1` is the only value that sends no + /// `Git-Protocol` header at all. + fn supported_protocol_versions(&self) -> &[Protocol] { + &[Protocol::V1] + } + + /// Stateless RPC: every turn is its own request. + fn connection_persists_across_multiple_requests(&self) -> bool { + false + } + + fn configure( + &mut self, + _config: &dyn Any, + ) -> Result<(), Box> { + Ok(()) + } +} + +#[async_trait(?Send)] +impl async_io::Transport for HttpTransport { + /// `extra_parameters` is ignored: it is only ever sent in the `Git-Protocol` header, which v1 + /// does not carry, and this transport speaks nothing else. + async fn handshake<'a>( + &mut self, + service: Service, + _extra_parameters: &'a [(&'a str, Option<&'a str>)], + ) -> Result, client::Error> { + let url = append_url( + &self.url, + &format!("info/refs?service={}", service.as_str()), + ); + let response = self + .http + .get(url) + .header(reqwest::header::USER_AGENT, USER_AGENT) + .bearer_auth(&self.token) + .send() + .await + .and_then(reqwest::Response::error_for_status) + .map_err(io::Error::other)?; + check_content_type(&response, service, "advertisement")?; + let body = response.bytes().await.map_err(io::Error::other)?; + + let line_reader = self.lines.insert(StreamingPeekableIter::new( + ResponseBody::ready(body.to_vec()), + DELIMITERS, + false, + )); + + // The service announcement is only sent by some servers; eat it when it is there, and check + // that it names the service we asked for. Reading to the flush that follows is what lets + // the advertisement itself be read next: `WithSidebands` resets the iterator when dropped. + let announced_service = { + let line_ = line_reader + .peek_line() + .await + .ok_or(client::Error::ExpectedLine( + "capabilities, version or service", + ))???; + let line = line_.as_text().ok_or(client::Error::ExpectedLine("text"))?; + line.as_bstr() + .strip_prefix(b"# service=") + .map(<[u8]>::to_vec) + }; + if let Some(announced_service) = announced_service { + if announced_service != service.as_str().as_bytes() { + return Err(io::Error::other(format!( + "expected service {:?}, but the remote announced {:?}", + service.as_str(), + String::from_utf8_lossy(&announced_service) + )) + .into()); + } + + line_reader.as_read().read_to_end(&mut Vec::new()).await?; + } + + let Handshake { + capabilities, + refs, + protocol: actual_protocol, + } = Handshake::from_lines_with_version_detection(line_reader).await?; + self.service = Some(service); + Ok(SetServiceResponse { + actual_protocol, + capabilities, + refs, + }) + } + + fn request( + &mut self, + write_mode: WriteMode, + on_into_read: MessageKind, + trace: bool, + ) -> Result, client::Error> { + let service = self.service.ok_or(client::Error::MissingHandshake)?; + let url = append_url(&self.url, service.as_str()); + let http = self.http.clone(); + let token = self.token.clone(); + + // Written into by the `RequestWriter` below, taken by the POST when the response is first + // read — by then the caller has written everything and dropped the writer. + let body = Rc::new(RefCell::new(Vec::new())); + let post = { + let body = Rc::clone(&body); + async move { + let body = std::mem::take(&mut *body.borrow_mut()); + let response = http + .post(url) + .header(reqwest::header::USER_AGENT, USER_AGENT) + .header( + reqwest::header::CONTENT_TYPE, + content_type(service, "request"), + ) + .header(reqwest::header::ACCEPT, content_type(service, "result")) + .bearer_auth(&token) + .body(body) + .send() + .await + .and_then(reqwest::Response::error_for_status) + .map_err(io::Error::other)?; + check_content_type(&response, service, "result")?; + Ok(response.bytes().await.map_err(io::Error::other)?.to_vec()) + } + }; + + let lines = self.lines.as_mut().ok_or(client::Error::MissingHandshake)?; + lines.replace(ResponseBody::pending(post)); + Ok(RequestWriter::new_from_bufread( + BodyWriter(body), + Box::new(lines.as_read_without_sidebands::>()), + write_mode, + on_into_read, + trace, + )) + } +} + +fn append_url(base: &str, suffix: &str) -> String { + format!("{}/{suffix}", base.trim_end_matches('/')) +} + +/// What a smart-HTTP body of the given `kind` is announced as, like +/// `application/x-git-receive-pack-request`. +fn content_type(service: Service, kind: &str) -> String { + format!("application/x-{}-{kind}", service.as_str()) +} + +/// Reject anything but the smart protocol, which is the only one either side speaks. +fn check_content_type( + response: &reqwest::Response, + service: Service, + kind: &str, +) -> io::Result<()> { + let wanted = content_type(service, kind); + let found = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default(); + if found.trim() != wanted { + return Err(io::Error::other(format!( + "expected a {wanted:?} response to indicate the 'smart' protocol, but got {found:?}" + ))); + } + Ok(()) +} + +/// Collects the request body for [`ResponseBody`] to send. +struct BodyWriter(Rc>>); + +impl AsyncWrite for BodyWriter { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.0.borrow_mut().extend_from_slice(buf); + Poll::Ready(Ok(buf.len())) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + +/// The POST that a [`ResponseBody`] is still waiting on. +type PendingResponse = Pin>>>>; + +/// A response body, either already in hand (the advertisement) or still to be asked for (the POST, +/// which `request` had no `await` to send it in). +struct ResponseBody { + /// The request that will produce `body`, until it has answered. + pending: Option, + body: io::Cursor>, +} + +impl ResponseBody { + fn ready(body: Vec) -> Self { + Self { + pending: None, + body: io::Cursor::new(body), + } + } + + fn pending(request: impl Future>> + 'static) -> Self { + Self { + pending: Some(Box::pin(request)), + body: io::Cursor::new(Vec::new()), + } + } +} + +impl AsyncRead for ResponseBody { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let this = self.get_mut(); + if let Some(pending) = this.pending.as_mut() { + let body = ready!(pending.as_mut().poll(cx)); + // Cleared first: a failed request must not be polled again on the next read. + this.pending = None; + this.body = io::Cursor::new(body?); + } + Poll::Ready(io::Read::read(&mut this.body, buf)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures_lite::{AsyncBufReadExt as _, AsyncWriteExt as _}; + use gix_transport::client::async_io::ReadlineBufRead as _; + use gix_transport::client::async_io::Transport as _; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + fn pkt(line: &str) -> String { + format!("{:04x}{line}", line.len() + 4) + } + + /// One canned reply per request, in order, on a throwaway port. Returns the base URL and, per + /// request, the head the client sent (lowercased) and its body. + async fn serve( + replies: Vec, + ) -> (String, tokio::task::JoinHandle)>>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/repo", listener.local_addr().unwrap()); + let handle = tokio::spawn(async move { + let mut requests = Vec::new(); + for reply in replies { + let (mut socket, _) = listener.accept().await.unwrap(); + // Read the head, then exactly as many body bytes as were announced. + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + assert_eq!( + socket.read(&mut byte).await.unwrap(), + 1, + "request cut short" + ); + request.push(byte[0]); + } + let head = String::from_utf8(request).unwrap().to_lowercase(); + let len: usize = head + .split("content-length: ") + .nth(1) + .map_or(0, |v| v.split("\r\n").next().unwrap().parse().unwrap()); + let mut body = vec![0; len]; + socket.read_exact(&mut body).await.unwrap(); + requests.push((head.clone(), body)); + + let kind = if head.starts_with("get ") { + "advertisement" + } else { + "result" + }; + socket + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/x-git-receive-pack-{kind}\r\n\ + Content-Length: {}\r\n\r\n{reply}", + reply.len() + ) + .as_bytes(), + ) + .await + .unwrap(); + socket.shutdown().await.unwrap(); + } + requests + }); + (url, handle) + } + + /// The whole exchange against a canned knot: the advertisement is parsed, and the POST body is + /// only sent once the reply is read. + #[tokio::test] + async fn a_handshake_and_a_request() { + let tip = "0123456789012345678901234567890123456789"; + let (url, server) = serve(vec![ + format!( + "{}0000{}0000", + pkt("# service=git-receive-pack\n"), + pkt(&format!( + "{tip} refs/heads/main\0report-status side-band-64k\n" + )), + ), + format!("{}{}0000", pkt("unpack ok\n"), pkt("ok refs/heads/main\n")), + ]) + .await; + + let mut transport = + HttpTransport::new(reqwest::Client::new(), url, "a.service.jwt".to_owned()); + // Scoped: the response borrows the transport, which the request below needs back. + { + let response = transport + .handshake(Service::ReceivePack, &[]) + .await + .unwrap(); + assert_eq!(response.actual_protocol, Protocol::V1); + assert!(response.capabilities.contains("report-status")); + let mut refs = response.refs.expect("v1 advertises refs"); + let mut advertised = String::new(); + refs.readline_str(&mut advertised).await.unwrap(); + assert_eq!(advertised, format!("{tip} refs/heads/main\n")); + } + + let (mut writer, mut reader) = transport + .request(WriteMode::Binary, MessageKind::Flush, false) + .unwrap() + .into_parts(); + writer.write_all(b"the pack").await.unwrap(); + writer.flush().await.unwrap(); + drop(writer); + + let mut report = String::new(); + loop { + let line = reader.fill_buf().await.unwrap(); + if line.is_empty() { + break; + } + let consumed = line.len(); + report.push_str(std::str::from_utf8(line).unwrap()); + reader.consume(consumed); + } + assert_eq!(report, "unpack ok\nok refs/heads/main\n"); + + let requests = server.await.unwrap(); + assert_eq!(requests[0].1, b"", "the advertisement is a GET"); + assert_eq!(requests[1].1, b"the pack"); + // Both halves are authorized by the knot, so both carry the credential. + for (head, _) in &requests { + assert!( + head.contains("authorization: bearer a.service.jwt\r\n"), + "missing credential in {head:?}" + ); + } + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/last_commit.rs b/knotmirror/crates/knotmirror-xrpc/src/last_commit.rs new file mode 100644 index 0000000000000000000000000000000000000000..b392864869cd3406847cf64762c99bfe36750228 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/last_commit.rs @@ -0,0 +1,556 @@ +use std::collections::hash_map::Entry; +use std::collections::{BinaryHeap, HashMap}; +use std::time::{Duration, Instant}; + +use fixedbitset::FixedBitSet; +use gix::ObjectId; +use gix::bstr::{BStr, BString, ByteSlice as _}; +use gix::commitgraph::bloom::BloomKey; +use gix::commitgraph::file::Commit as GraphCommit; +use gix::commitgraph::{BloomFilterSettings, Graph}; +use gix::objs::tree::EntryRef; +use itertools::Itertools as _; +use metrics::{counter, gauge, histogram}; +use tracing::warn; + +pub(crate) const BUDGET: Duration = Duration::from_secs(1); + +pub(crate) fn join(dir: impl AsRef, name: impl AsRef) -> BString { + let (dir, name) = (dir.as_ref(), name.as_ref()); + let mut out = dir.to_owned(); + if !dir.is_empty() && !name.is_empty() { + out.push(b'/'); + } + out.extend_from_slice(name); + out +} + +fn is_dir_prefix(path: &BStr, prefix: &BStr) -> bool { + path.strip_prefix(prefix.as_bytes()) + .is_some_and(|rest| rest.is_empty() || rest.starts_with_str("/")) +} + +fn within_depth(name: &BStr, depth: usize, max_depth: usize) -> bool { + depth + name.find_iter("/").count() <= max_depth +} + +struct Node { + id: ObjectId, + generation: u32, + time: i64, + tree: ObjectId, + parents: Vec, +} + +impl Node { + /// Newest generation first, then newest commit time. + fn queue_key(&self) -> (u32, i64, ObjectId) { + (self.generation, self.time, self.id) + } +} + +fn node(repo: &gix::Repository, graph: Option<&Graph>, id: ObjectId) -> Option { + if let Some(graph) = graph + && let Some(commit) = graph.commit_by_id(id) + { + return Some(Node { + id, + generation: commit.generation(), + time: commit.committer_timestamp() as i64, + tree: commit.root_tree_id().to_owned(), + parents: commit + .iter_parents() + .map(|parent| parent.map(|pos| graph.id_at(pos).to_owned())) + .collect::>() + .ok()?, + }); + } + let commit = repo.find_commit(id).ok()?; + Some(Node { + id, + generation: u32::MAX, + time: commit.time().ok()?.seconds, + tree: commit.tree_id().ok()?.detach(), + parents: commit.parent_ids().map(|id| id.detach()).collect(), + }) +} + +/// A queued commit and the paths it may still be credited with. +struct Active { + node: Node, + pending: FixedBitSet, +} + +struct TreeDiff<'a> { + repo: &'a gix::Repository, + pathspec: Option<&'a BStr>, + /// `None` recurses without limit. + max_depth: Option, + /// Also emit the trees that are recursed into. + show_trees: bool, +} + +impl TreeDiff<'_> { + /// `path` is inside the pathspec, or a directory leading to it. + fn interesting(&self, path: &BStr, is_dir: bool) -> bool { + self.pathspec + .is_none_or(|ps| is_dir_prefix(path, ps) || (is_dir && is_dir_prefix(ps, path))) + } + + fn should_recurse(&self, path: &BStr) -> bool { + let Some(max_depth) = self.max_depth else { + return true; + }; + match self.pathspec { + None => within_depth(path, 1, max_depth), + Some(ps) if path.len() >= ps.len() => { + is_dir_prefix(path, ps) && within_depth(path[ps.len()..].as_bstr(), 1, max_depth) + } + Some(ps) => is_dir_prefix(ps, path), + } + } + + /// The paths that differ between the trees `old` and `new`; `None` is the empty tree. + fn changed_paths(&self, old: Option, new: Option) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![(BString::default(), old, new)]; + while let Some((base, old, new)) = stack.pop() { + let tree = |id: Option| id.and_then(|id| self.repo.find_tree(id).ok()); + let (old, new) = (tree(old), tree(new)); + // Entries are sorted by `EntryRef`'s order, where a tree and a blob of the same + // name differ. + for pair in entries(old.as_ref()).merge_join_by(entries(new.as_ref()), Ord::cmp) { + let entry = pair.clone().reduce(|_, new| new); + let (old, new) = pair.left_and_right(); + let path = join(&base, entry.filename); + if old == new || !self.interesting(path.as_bstr(), entry.mode.is_tree()) { + continue; + } + if entry.mode.is_tree() && self.should_recurse(path.as_bstr()) { + if self.show_trees { + out.push(path.clone()); + } + let id = |side: Option| side.map(|e| e.oid.to_owned()); + stack.push((path, id(old), id(new))); + } else { + out.push(path); + } + } + } + out + } +} + +/// `None` and unreadable trees have no entries. +fn entries<'a>(tree: Option<&'a gix::Tree<'_>>) -> impl Iterator> { + tree.and_then(|tree| tree.decode().ok()) + .into_iter() + .flat_map(|tree| tree.entries) +} + +/// The paths to resolve; a path's index is its bit in every bitset. +struct Paths { + paths: Vec, + index: HashMap, + /// `None` without Bloom filters. + keys: Option>, +} + +impl Paths { + fn new(diff: &TreeDiff, head_tree: ObjectId, bloom: Option) -> Self { + let paths = diff.changed_paths(None, Some(head_tree)); + Self { + index: paths.iter().cloned().zip(0..).collect(), + keys: bloom.map(|settings| { + paths + .iter() + .map(|path| BloomKey::from_path(path.as_bstr(), settings)) + .collect() + }), + paths, + } + } + + /// `false` only if the Bloom filter rules out every active path. + fn maybe_changed(&self, commit: Option>, active: &FixedBitSet) -> bool { + let (Some(keys), Some(commit)) = (&self.keys, commit) else { + return true; + }; + // `maybe_contains_all_keys` treats an empty filter as a miss. + if commit.maybe_contains_path(BStr::new("")).is_none() { + return true; + } + active + .ones() + .any(|k| commit.maybe_contains_all_keys(std::slice::from_ref(&keys[k])) != Some(false)) + } +} + +/// Moves the paths unchanged from `parent_tree` out of `pending` and returns them. +fn process_parent( + diff: &TreeDiff, + paths: &Paths, + graph: Option<&Graph>, + commit: &Node, + pending: &mut FixedBitSet, + parent_i: usize, + parent_tree: ObjectId, +) -> FixedBitSet { + let mut changed = FixedBitSet::with_capacity(paths.paths.len()); + let graph_commit = || graph.and_then(|graph| graph.commit_by_id(commit.id)); + if parent_i > 0 || paths.maybe_changed(graph_commit(), pending) { + changed.extend( + diff.changed_paths(Some(parent_tree), Some(commit.tree)) + .iter() + .filter_map(|path| paths.index.get(path).copied()), + ); + } + let mut same = pending.clone(); + same.difference_with(&changed); + pending.difference_with(&same); + same +} + +struct WalkGuard { + outcome: &'static str, + commits: usize, +} + +impl WalkGuard { + fn new() -> Self { + gauge!("gitmirror_last_commit_walks_running").increment(1); + Self { + outcome: "error", + commits: 0, + } + } +} + +impl Drop for WalkGuard { + fn drop(&mut self) { + gauge!("gitmirror_last_commit_walks_running").decrement(1); + counter!("gitmirror_last_commit_walks_total", "outcome" => self.outcome).increment(1); + histogram!("gitmirror_last_commit_commits_walked", "outcome" => self.outcome) + .record(self.commits as f64); + } +} + +/// The last commit to change each path under `pathspec` ("" for the whole tree). +pub(crate) fn walk( + repo: &gix::Repository, + head: ObjectId, + pathspec: &str, + max_depth: Option, + show_trees: bool, + deadline: Instant, +) -> HashMap { + let mut out = HashMap::new(); + let mut stats = WalkGuard::new(); + + let graph = repo.commit_graph_if_enabled().ok().flatten(); + let graph = graph.as_ref(); + let Some(head_node) = node(repo, graph, head) else { + warn!(%head, "last-commit walk could not start"); + return out; + }; + + let pathspec = pathspec.trim_matches('/'); + let diff = TreeDiff { + repo, + pathspec: (!pathspec.is_empty()).then(|| pathspec.into()), + max_depth, + show_trees, + }; + let paths = Paths::new( + &diff, + head_node.tree, + graph.and_then(Graph::bloom_filter_settings), + ); + + let mut all = FixedBitSet::with_capacity(paths.paths.len()); + all.insert_range(..); + let mut queue = BinaryHeap::from([head_node.queue_key()]); + let mut active = HashMap::from([( + head, + Active { + node: head_node, + pending: all, + }, + )]); + + while let Some((_, _, id)) = queue.pop() { + if Instant::now() >= deadline { + stats.outcome = "deadline"; + break; + } + stats.commits += 1; + let Active { + node: commit, + mut pending, + } = active.remove(&id).expect("queued commits are active"); + + for (i, &parent) in commit.parents.iter().enumerate() { + if pending.is_clear() { + break; + } + let Some(parent) = node(repo, graph, parent) else { + warn!(%id, %parent, "last-commit parent unavailable"); + continue; + }; + + let same = process_parent(&diff, &paths, graph, &commit, &mut pending, i, parent.tree); + if same.is_clear() { + continue; + } + + match active.entry(parent.id) { + Entry::Occupied(mut entry) => entry.get_mut().pending.union_with(&same), + Entry::Vacant(entry) => { + queue.push(parent.queue_key()); + entry.insert(Active { + node: parent, + pending: same, + }); + } + } + } + + for k in pending.ones() { + out.insert(paths.paths[k].clone(), id); + } + } + + if out.len() == paths.paths.len() { + stats.outcome = "resolved"; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use gix::bstr::B; + use std::path::Path; + use std::process::Command; + + fn git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "t") + .env("GIT_AUTHOR_EMAIL", "t@t") + .env("GIT_COMMITTER_NAME", "t") + .env("GIT_COMMITTER_EMAIL", "t@t") + .output() + .expect("run git"); + assert!(out.status.success(), "git {args:?} failed"); + String::from_utf8(out.stdout).unwrap().trim().to_owned() + } + + fn commit(dir: &Path, file: &str, content: &str) -> ObjectId { + let path = dir.join(file); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, content).unwrap(); + git(dir, &["add", "-A"]); + git(dir, &["commit", "-q", "-m", file]); + head(dir) + } + + fn head(dir: &Path) -> ObjectId { + ObjectId::from_hex(git(dir, &["rev-parse", "HEAD"]).as_bytes()).unwrap() + } + + /// `(pathspec, max_depth, show_trees, expected)` + type Case<'a, K> = (&'a str, Option, bool, Vec<(K, ObjectId)>); + + fn assert_walks>(dir: &Path, cases: &[Case]) { + let head = head(dir); + let deadline = Instant::now() + BUDGET; + let check = |repo: &gix::Repository| { + for (pathspec, max_depth, show_trees, expected) in cases { + let expected: HashMap = expected + .iter() + .map(|(path, id)| (path.as_ref().into(), *id)) + .collect(); + assert_eq!( + walk(repo, head, pathspec, *max_depth, *show_trees, deadline), + expected, + "{pathspec:?} {max_depth:?} {show_trees}" + ); + } + }; + + check(&gix::open(dir).unwrap()); + + git( + dir, + &[ + "-c", + "commitGraph.changedPathsVersion=2", + "commit-graph", + "write", + "--reachable", + "--changed-paths", + ], + ); + let repo = gix::open(dir).unwrap(); + assert!( + repo.commit_graph() + .unwrap() + .bloom_filter_settings() + .is_some() + ); + check(&repo); + } + + #[test] + fn walk_linear_history() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path(); + git(p, &["init", "-q", "-b", "main"]); + let a = commit(p, "src/a", "1"); + commit(p, "docs/x", "1"); + let b = commit(p, "src/lib/b", "1"); + let x2 = commit(p, "docs/x", "2"); + let c = commit(p, "src/c", "1"); + let head = commit(p, "docs/y", "1"); + + let files: [(&str, _); 5] = [ + ("src/a", a), + ("src/lib/b", b), + ("src/c", c), + ("docs/x", x2), + ("docs/y", head), + ]; + let trees = [("src", c), ("src/lib", b), ("docs", head)]; + assert_walks( + p, + &[ + ( + "src", + Some(1), + true, + vec![("src", c), ("src/a", a), ("src/lib", b), ("src/c", c)], + ), + ("", Some(0), false, vec![("src", c), ("docs", head)]), + ("src", Some(0), false, vec![("src", c)]), + ("src/a", Some(0), false, vec![("src/a", a)]), + ( + "src/lib", + Some(1), + true, + vec![("src", c), ("src/lib", b), ("src/lib/b", b)], + ), + ( + "", + Some(1), + false, + vec![ + ("src/a", a), + ("src/lib", b), + ("src/c", c), + ("docs/x", x2), + ("docs/y", head), + ], + ), + ("", None, false, files.to_vec()), + ("", None, true, [&files[..], &trees[..]].concat()), + ("nope", None, true, vec![]), + ], + ); + } + + #[test] + fn walk_credits_merges_only_when_they_differ_from_every_parent() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path(); + git(p, &["init", "-q", "-b", "main"]); + let a = commit(p, "src/a", "1"); + git(p, &["switch", "-q", "-c", "side"]); + let s = commit(p, "src/s", "1"); + let d = commit(p, "docs/x", "1"); + git(p, &["switch", "-q", "main"]); + let m = commit(p, "src/m", "1"); + git(p, &["merge", "-q", "--no-ff", "-m", "merge", "side"]); + let merge = head(p); + + assert_walks( + p, + &[ + ( + "src", + Some(1), + true, + vec![("src", merge), ("src/a", a), ("src/s", s), ("src/m", m)], + ), + ("", Some(0), false, vec![("src", merge), ("docs", d)]), + ], + ); + } + + #[test] + fn walk_non_utf8_filename() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path(); + git(p, &["init", "-q", "-b", "main"]); + let first = commit(p, "a", "1"); + + // Add a `\xff` entry with plumbing; the filesystem may reject the name. + let blob = git(p, &["hash-object", "-w", "--stdin"]); + let mut listing = git(p, &["ls-tree", "HEAD"]).into_bytes(); + listing.extend_from_slice(format!("\n100644 blob {blob}\t").as_bytes()); + listing.extend_from_slice(b"\xff\n"); + let mut mktree = Command::new("git") + .arg("mktree") + .current_dir(p) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + std::io::Write::write_all(&mut mktree.stdin.take().unwrap(), &listing).unwrap(); + let tree = String::from_utf8(mktree.wait_with_output().unwrap().stdout).unwrap(); + let tree = tree.trim(); + let added = git( + p, + &["commit-tree", tree, "-p", &first.to_string(), "-m", "ff"], + ); + git(p, &["update-ref", "HEAD", &added]); + git(p, &["reset", "-q"]); + let added = ObjectId::from_hex(added.as_bytes()).unwrap(); + + std::fs::write(p.join("a"), "2").unwrap(); + git(p, &["add", "a"]); + git(p, &["commit", "-q", "-m", "a"]); + let last = head(p); + + assert_walks( + p, + &[( + "", + Some(0), + false, + vec![(B("a"), last), (B(b"\xff"), added)], + )], + ); + } + + #[test] + fn walk_merge_undoes_changes() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path(); + git(p, &["init", "-q", "-b", "main"]); + commit(p, "file", "A"); + let b2 = commit(p, "file", "B"); + commit(p, "file", "C"); + commit(p, "file", "D"); + git(p, &["switch", "-q", "-c", "side", &b2.to_string()]); + let b5 = commit(p, "file2", "2"); + git(p, &["switch", "-q", "main"]); + git(p, &["merge", "-q", "--no-commit", "--no-ff", "side"]); + std::fs::write(p.join("file"), "B").unwrap(); + git(p, &["add", "file"]); + git(p, &["commit", "-q", "-m", "merge"]); + + assert_walks(p, &[("", None, false, vec![("file2", b5), ("file", b2)])]); + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/lib.rs b/knotmirror/crates/knotmirror-xrpc/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..8428caa6ee2ba29f04c1a03aa724734c77a0fd45 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/lib.rs @@ -0,0 +1,141 @@ +mod archive; +mod committers; +mod did_ext; +mod diff; +mod error; +mod format_patch; +mod git_transport; +mod last_commit; +mod merge; +pub mod metrics; +mod routes; +mod signer; +mod sniff; + +use std::{path::PathBuf, sync::Arc}; + +use axum::{ + Router, + routing::{get, post}, +}; +use bobbin_runtime::{Clock, ReqwestHttp}; +use jacquard_axum::service_auth::{self, ServiceAuthError}; +use jacquard_common::service_auth::ServiceAuthClaims; +use jacquard_identity::JacquardResolver; +use knotmirror_git::layout::Layout; +use tangled_axum::atproto::{AtprotoService, ClaimsPolicy, Verified}; + +use crate::committers::Committers; +pub use crate::committers::Deliberi; +pub use crate::signer::ServiceSigner; + +pub(crate) type Directory = JacquardResolver; + +#[derive(Clone)] +pub struct AppState { + pub(crate) db: knotmirror_db::Db, + pub(crate) layout: Arc, + pub(crate) http: reqwest::Client, + pub(crate) service_auth: service_auth::ServiceAuthConfig>, + pub(crate) knot_policy: KnotPolicy, + pub(crate) clock: Arc, + pub(crate) stats: Arc, + #[allow(unused)] + pub(crate) redis: Option, + pub(crate) max_blob_bytes: u64, + pub(crate) max_archive_bytes: u64, + pub(crate) signer: Option>, + pub(crate) committers: Arc, +} + +impl AppState { + #[allow(clippy::too_many_arguments)] + pub fn new( + db: knotmirror_db::Db, + repo_base: Arc, + http: reqwest::Client, + service_auth: service_auth::ServiceAuthConfig>, + knot_policy: KnotPolicy, + clock: Arc, + redis: Option, + max_blob_bytes: u64, + max_archive_bytes: u64, + ) -> Self { + let layout = Arc::new(Layout::new((*repo_base).clone())); + Self { + db, + layout, + http, + service_auth, + knot_policy, + clock, + stats: Arc::default(), + redis, + max_blob_bytes, + max_archive_bytes, + signer: None, + committers: Arc::new(Committers::new(None)), + } + } + + pub fn with_signer(mut self, signer: ServiceSigner, deliberi: Option) -> Self { + let signer = Arc::new(signer); + self.committers = Arc::new(Committers::new( + deliberi.map(|deliberi| (deliberi, signer.clone())), + )); + self.signer = Some(signer); + self + } +} + +impl AtprotoService for AppState { + type Resolver = Directory; + + fn resolver(&self) -> &Self::Resolver { + self.service_auth.resolver() + } + + async fn verify_claims( + &self, + claims: &ServiceAuthClaims, + ) -> Result { + ClaimsPolicy::from(&self.service_auth) + .skip_audience() + .check(claims, self.clock.now_unix_micros()) + .await + } +} + +#[rustfmt::skip] +pub fn router(state: AppState) -> Router { + use routes::*; + Router::new() + .route("/.well-known/did.json", get(activity::did_document)) + .route("/xrpc/sh.tangled.git.temp2.getBlame", get(git::get_blame)) + .route("/xrpc/sh.tangled.git.temp2.getCommitStats", get(activity::get_commit_stats)) + .route("/xrpc/sh.tangled.git.temp2.getDiff", get(git::get_diff)) + .route("/xrpc/sh.tangled.git.temp2.getInterdiff", get(git::get_interdiff)) + .route("/xrpc/sh.tangled.git.temp2.mergeCheck", get(git::merge_check)) + .route("/xrpc/sh.tangled.git.mergeCommit", post(git::merge_commit)) + .route("/xrpc/sh.tangled.git.temp.listIndexedLanguages", get(reads::list_indexed_languages)) + .route("/xrpc/sh.tangled.git.temp.listLanguages", get(reads::list_languages)) + .route("/xrpc/sh.tangled.repo.listCobVersions", get(cob::list_cob_versions)) + .route("/xrpc/org.tangled.temp.repo.listRecordOrigins", get(cob::list_record_origins)) + .route("/xrpc/org.tangled.temp.git.deleteBranch", post(git::delete_branch)) + .route("/xrpc/org.tangled.temp.git.formatPatch", get(reads::format_patch)) + .route("/xrpc/org.tangled.temp.git.getArchive", get(reads::get_archive)) + .route("/xrpc/org.tangled.temp.git.getBlob", get(reads::get_blob)) + .route("/xrpc/org.tangled.temp.git.getEntry", get(reads::get_entry)) + .route("/xrpc/org.tangled.temp.git.getMergeBase", get(reads::get_merge_base)) + .route("/xrpc/org.tangled.temp.git.getRef", get(reads::get_ref)) + .route("/xrpc/org.tangled.temp.git.getTree", get(reads::get_tree)) + .route("/xrpc/org.tangled.temp.git.listBranches", get(reads::list_branches)) + .route("/xrpc/org.tangled.temp.git.listCommits", get(git::list_commits)) + .route("/xrpc/org.tangled.temp.git.listCommitsByRef", get(git::list_commits_by_ref)) + .route("/xrpc/org.tangled.temp.git.listTags", get(reads::list_tags)) + .route("/xrpc/org.tangled.temp.git.syncFork", post(git::sync_fork)) + .layer(axum::middleware::from_fn(metrics::metrics_middleware)) + .with_state(state) +} + +pub use knotmirror_types::KnotPolicy; diff --git a/knotmirror/crates/knotmirror-xrpc/src/merge.rs b/knotmirror/crates/knotmirror-xrpc/src/merge.rs new file mode 100644 index 0000000000000000000000000000000000000000..a4b58861c430bbc140b2ee6f5f17974fe8790089 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/merge.rs @@ -0,0 +1,293 @@ +use gix::bstr::ByteSlice as _; +use gix::diff::tree_with_rewrites::Change; +use gix::merge::tree::{Conflict, TreatAsUnresolved}; + +use lexicons::sh_tangled::git::temp2::merge_check::{Conflict as MergeConflict, MergeCheckOutput}; + +/// Perform a 3-way merge check between `target` (ours) and `source` (theirs), using their +/// merge base as the common ancestor. +/// +/// This is read-only: it inspects only the conflicts of the in-memory merge and never persists +/// the merged tree. The caller must have opened `repo` with [`gix::Repository::with_object_memory`] +/// so that any objects `merge_trees` writes while merging land in memory rather than on disk — +/// the mirrored repositories (reached read-only via alternates) are never mutated. +pub fn merge_check( + repo: &gix::Repository, + target_oid: gix::ObjectId, + source_oid: gix::ObjectId, +) -> anyhow::Result { + let our_tree = repo.find_commit(target_oid)?.tree_id()?.detach(); + let their_tree = repo.find_commit(source_oid)?.tree_id()?.detach(); + + // Common ancestor for the 3-way merge. When the histories are unrelated (no merge base), + // fall back to the empty tree — the same behaviour as `git merge --allow-unrelated-histories`, + // which surfaces add/add conflicts with real filenames instead of erroring out. + let (ancestor_tree, unrelated) = match repo.merge_base(target_oid, source_oid) { + Ok(base) => (repo.find_commit(base.detach())?.tree_id()?.detach(), false), + Err(_) => (gix::ObjectId::empty_tree(repo.object_hash()), true), + }; + + let options = repo.tree_merge_options()?; + let labels = gix::merge::blob::builtin_driver::text::Labels { + ancestor: Some("ancestor".as_bytes().as_bstr()), + current: Some("target".as_bytes().as_bstr()), + other: Some("source".as_bytes().as_bstr()), + }; + + let outcome = repo.merge_trees(ancestor_tree, our_tree, their_tree, labels, options)?; + + // Report only genuinely-unresolved conflicts (matching git's notion), so gix's + // auto-resolutions don't show up as false positives. The merged tree is intentionally + // never written. + let mut conflicts = Vec::new(); + for conflict in &outcome.conflicts { + if !conflict.is_unresolved(TreatAsUnresolved::git()) { + continue; + } + conflicts.push(MergeConflict { + filename: conflict.ours.location().to_string().into(), + reason: conflict_reason(conflict).into(), + extra_data: Default::default(), + }); + } + + let is_conflicted = !conflicts.is_empty(); + let message = match (is_conflicted, unrelated) { + (true, true) => Some(format!( + "{} conflicting file(s); unrelated histories (no common ancestor)", + conflicts.len() + )), + (true, false) => Some(format!("{} conflicting file(s)", conflicts.len())), + (false, true) => Some("no common ancestor (unrelated histories)".to_string()), + (false, false) => None, + }; + + Ok(MergeCheckOutput { + is_conflicted, + conflicts, + message: message.map(Into::into), + extra_data: Default::default(), + }) +} + +/// Classify a conflict into a short, human-readable reason from the shape of the two changes. +/// Derived from the change variants rather than the resolution failure so it stays stable +/// across gix versions. +fn conflict_reason(conflict: &Conflict) -> &'static str { + match (&conflict.ours, &conflict.theirs) { + (Change::Deletion { .. }, _) | (_, Change::Deletion { .. }) => "modify/delete conflict", + (Change::Addition { .. }, Change::Addition { .. }) => "add/add conflict", + (Change::Rewrite { .. }, _) | (_, Change::Rewrite { .. }) => "rename conflict", + _ => "content conflict", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + use std::process::Command; + + fn git(dir: &Path, args: &[&str]) { + let status = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "t") + .env("GIT_AUTHOR_EMAIL", "t@t") + .env("GIT_COMMITTER_NAME", "t") + .env("GIT_COMMITTER_EMAIL", "t@t") + .status() + .expect("run git"); + assert!(status.success(), "git {args:?} failed"); + } + + fn write(dir: &Path, name: &str, content: &str) { + std::fs::write(dir.join(name), content).unwrap(); + } + + fn rev_parse(dir: &Path, rev: &str) -> gix::ObjectId { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(dir) + .output() + .expect("rev-parse"); + let hex = String::from_utf8(out.stdout).unwrap(); + gix::ObjectId::from_hex(hex.trim().as_bytes()).unwrap() + } + + /// Count loose object files under `/.git/objects`, ignoring `info` and `pack`. + fn count_loose_objects(git_dir: &Path) -> usize { + let objects = git_dir.join(".git").join("objects"); + let mut n = 0; + for shard in std::fs::read_dir(&objects).unwrap() { + let shard = shard.unwrap(); + let name = shard.file_name(); + if name == "info" || name == "pack" || !shard.file_type().unwrap().is_dir() { + continue; + } + n += std::fs::read_dir(shard.path()).unwrap().count(); + } + n + } + + /// Build a repo with a shared base commit, a `main` (target/ours) tip, and a `source` + /// (theirs) tip built by the two closures. + fn fixture( + target: impl FnOnce(&Path), + source: impl FnOnce(&Path), + ) -> (tempfile::TempDir, gix::ObjectId, gix::ObjectId) { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path(); + git(p, &["init", "-q", "-b", "main"]); + write(p, "a.txt", "line1\nline2\nline3\n"); + write(p, "b.txt", "keep\n"); + git(p, &["add", "."]); + git(p, &["commit", "-q", "-m", "base"]); + + git(p, &["branch", "source"]); + // target edits on main + target(p); + git(p, &["add", "-A"]); + git(p, &["commit", "-q", "-m", "target"]); + let target_oid = rev_parse(p, "main"); + + git(p, &["checkout", "-q", "source"]); + source(p); + git(p, &["add", "-A"]); + git(p, &["commit", "-q", "-m", "source"]); + let source_oid = rev_parse(p, "source"); + + (dir, target_oid, source_oid) + } + + fn open_in_memory(dir: &Path) -> gix::Repository { + gix::open(dir).unwrap().with_object_memory() + } + + /// Mirror of `crate::git::open_scratch`: a throwaway repo whose alternates point at the + /// given object dirs, opened in-memory. + fn scratch_with_alternates( + object_dirs: &[std::path::PathBuf], + ) -> (tempfile::TempDir, gix::Repository) { + let scratch = tempfile::tempdir().unwrap(); + gix::init_bare(scratch.path()).unwrap(); + let info = scratch.path().join("objects").join("info"); + std::fs::create_dir_all(&info).unwrap(); + let mut alternates = String::new(); + for d in object_dirs { + alternates.push_str(&std::fs::canonicalize(d).unwrap().display().to_string()); + alternates.push('\n'); + } + std::fs::write(info.join("alternates"), alternates).unwrap(); + let repo = gix::open(scratch.path()).unwrap().with_object_memory(); + (scratch, repo) + } + + #[test] + fn cross_repo_via_alternates_never_touches_mirrors() { + // Two separate repos sharing a base commit: `a` holds the target tip, `b` the source + // tip. This is the real knotmirror shape (target and source live in different mirrors). + let a = tempfile::tempdir().unwrap(); + let ap = a.path(); + git(ap, &["init", "-q", "-b", "main"]); + write(ap, "a.txt", "line1\nline2\nline3\n"); + git(ap, &["add", "."]); + git(ap, &["commit", "-q", "-m", "base"]); + // target edit in repo `a` + write(ap, "a.txt", "line1\nOURS\nline3\n"); + git(ap, &["add", "-A"]); + git(ap, &["commit", "-q", "-m", "target"]); + let target = rev_parse(ap, "main"); + + // Clone `a` → `b`, reset to the shared base, and make a conflicting source edit. + let b = tempfile::tempdir().unwrap(); + let bp = b.path(); + git(bp, &["clone", "-q", ap.to_str().unwrap(), "."]); + git(bp, &["checkout", "-q", "-B", "source", &target.to_string()]); + git(bp, &["reset", "-q", "--hard", "HEAD~1"]); // back to base + write(bp, "a.txt", "line1\nTHEIRS\nline3\n"); + git(bp, &["add", "-A"]); + git(bp, &["commit", "-q", "-m", "source"]); + let source = rev_parse(bp, "source"); + + let a_objs = ap.join(".git").join("objects"); + let b_objs = bp.join(".git").join("objects"); + let before_a = count_loose_objects(ap); + let before_b = count_loose_objects(bp); + + let (_scratch, repo) = scratch_with_alternates(&[a_objs, b_objs]); + let out = merge_check(&repo, target, source).unwrap(); + + assert!(out.is_conflicted); + assert_eq!(out.conflicts.len(), 1); + assert_eq!(out.conflicts[0].filename, "a.txt"); + // Neither mirror gained loose objects — the merge was read-only against both. + assert_eq!(count_loose_objects(ap), before_a, "target mirror mutated"); + assert_eq!(count_loose_objects(bp), before_b, "source mirror mutated"); + } + + #[test] + fn non_overlapping_edits_are_clean() { + let (dir, t, s) = fixture( + |p| write(p, "a.txt", "line1\nCHANGED\nline3\n"), + |p| write(p, "b.txt", "keep\nappended\n"), + ); + let repo = open_in_memory(dir.path()); + let before = count_loose_objects(dir.path()); + let out = merge_check(&repo, t, s).unwrap(); + assert!(!out.is_conflicted, "expected clean, got {out:?}"); + assert!(out.conflicts.is_empty()); + // never-touch-the-mirror: the in-memory merge writes no loose objects to disk. + assert_eq!(count_loose_objects(dir.path()), before); + } + + #[test] + fn same_line_edits_conflict() { + let (dir, t, s) = fixture( + |p| write(p, "a.txt", "line1\nOURS\nline3\n"), + |p| write(p, "a.txt", "line1\nTHEIRS\nline3\n"), + ); + let repo = open_in_memory(dir.path()); + let before = count_loose_objects(dir.path()); + let out = merge_check(&repo, t, s).unwrap(); + assert!(out.is_conflicted); + assert_eq!(out.conflicts.len(), 1); + assert_eq!(out.conflicts[0].filename, "a.txt"); + assert_eq!(out.conflicts[0].reason, "content conflict"); + assert_eq!(count_loose_objects(dir.path()), before); + } + + #[test] + fn modify_delete_conflict() { + let (dir, t, s) = fixture( + |p| write(p, "a.txt", "line1\nOURS\nline3\n"), + |p| std::fs::remove_file(p.join("a.txt")).unwrap(), + ); + let repo = open_in_memory(dir.path()); + let out = merge_check(&repo, t, s).unwrap(); + assert!(out.is_conflicted); + assert_eq!(out.conflicts[0].filename, "a.txt"); + assert_eq!(out.conflicts[0].reason, "modify/delete conflict"); + } + + #[test] + fn source_is_ancestor_is_clean() { + // Source is an ancestor of target (i.e. already merged / up to date) → no conflicts. + let dir = tempfile::tempdir().unwrap(); + let p = dir.path(); + git(p, &["init", "-q", "-b", "main"]); + write(p, "a.txt", "line1\nline2\n"); + git(p, &["add", "."]); + git(p, &["commit", "-q", "-m", "base"]); + let base = rev_parse(p, "main"); + write(p, "a.txt", "line1\nline2\nline3\n"); + git(p, &["add", "-A"]); + git(p, &["commit", "-q", "-m", "target"]); + let target = rev_parse(p, "main"); + + let repo = open_in_memory(p); + let out = merge_check(&repo, target, base).unwrap(); + assert!(!out.is_conflicted, "expected clean, got {out:?}"); + assert!(out.conflicts.is_empty()); + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/metrics.rs b/knotmirror/crates/knotmirror-xrpc/src/metrics.rs new file mode 100644 index 0000000000000000000000000000000000000000..0891554baf1681e5a8e3358fa90ae5e516c19687 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/metrics.rs @@ -0,0 +1,352 @@ +use axum::{ + Router, + extract::MatchedPath, + http::{Method, Request, StatusCode}, + middleware::Next, + response::{IntoResponse, Response}, + routing::get, +}; +use metrics::{counter, gauge, histogram}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; +use std::sync::OnceLock; +use std::time::Instant; +use tracing::Instrument as _; + +use crate::error::ErrorKind; + +static PROMETHEUS_HANDLE: OnceLock = OnceLock::new(); + +const XRPC_DURATION_BUCKETS: &[f64] = &[ + 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + 30.0, 60.0, +]; + +const BLOCKING_WAIT_BUCKETS: &[f64] = &[ + 0.00001, 0.000025, 0.00005, 0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.05, 0.25, + 1.0, +]; + +const COUNT_BUCKETS: &[f64] = &[1.0, 10.0, 100.0, 1e3, 1e4, 1e5, 1e6, 1e7]; + +const SYNC_TASK_DURATION_BUCKETS: &[f64] = &[ + 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1800.0, +]; + +pub fn init_metrics() -> PrometheusHandle { + let builder = PrometheusBuilder::new() + .set_buckets_for_metric( + Matcher::Full("gitmirror_xrpc_duration_seconds".to_owned()), + XRPC_DURATION_BUCKETS, + ) + .expect("xrpc duration buckets should not be empty") + .set_buckets_for_metric( + Matcher::Full("gitmirror_blocking_queue_wait_seconds".to_owned()), + BLOCKING_WAIT_BUCKETS, + ) + .expect("blocking wait buckets should not be empty") + .set_buckets_for_metric( + Matcher::Full("gitmirror_sync_task_duration_seconds".to_owned()), + SYNC_TASK_DURATION_BUCKETS, + ) + .expect("sync task duration buckets should not be empty") + .set_buckets_for_metric( + Matcher::Prefix("gitmirror_last_commit_".to_owned()), + COUNT_BUCKETS, + ) + .expect("count buckets should not be empty"); + let handle = builder + .install_recorder() + .expect("failed to install Prometheus recorder"); + + PROMETHEUS_HANDLE.set(handle.clone()).ok(); + describe_metrics(); + + handle +} + +fn describe_metrics() { + metrics::describe_counter!( + "gitmirror_xrpc_requests_total", + "Total number of XRPC requests" + ); + metrics::describe_histogram!( + "gitmirror_xrpc_duration_seconds", + "XRPC request duration in seconds" + ); + metrics::describe_gauge!( + "gitmirror_xrpc_in_flight", + "Number of XRPC requests currently being handled" + ); + metrics::describe_gauge!( + "gitmirror_blocking_queued", + "Tasks waiting for a thread on the blocking pool" + ); + metrics::describe_gauge!( + "gitmirror_blocking_running", + "Tasks currently occupying a thread on the blocking pool" + ); + metrics::describe_histogram!( + "gitmirror_blocking_queue_wait_seconds", + "Time a task spent waiting for a blocking-pool thread" + ); + metrics::describe_gauge!( + "gitmirror_sync_repos", + "Repos by sync state and whether they are due for a sync" + ); + metrics::describe_gauge!("gitmirror_sync_in_flight", "Sync tasks currently running"); + metrics::describe_histogram!( + "gitmirror_sync_task_duration_seconds", + "Sync task duration in seconds" + ); + metrics::describe_gauge!( + "gitmirror_process_memory_bytes", + "Resident memory of this process by kind" + ); + metrics::describe_gauge!( + "gitmirror_cgroup_memory_bytes", + "Memory usage of this process's cgroup by kind" + ); + metrics::describe_gauge!( + "gitmirror_cgroup_memory_max_bytes", + "Memory limit of this process's cgroup" + ); + metrics::describe_gauge!( + "gitmirror_last_commit_walks_running", + "Last-commit walks currently running" + ); + metrics::describe_counter!( + "gitmirror_last_commit_walks_total", + "Finished last-commit walks by outcome" + ); + metrics::describe_histogram!( + "gitmirror_last_commit_commits_walked", + "Commits visited per last-commit walk" + ); +} + +fn field(text: &str, key: &str) -> Option { + text.lines().find_map(|line| { + let mut words = line.split_whitespace(); + (words.next()? == key).then(|| words.next()?.parse().ok())? + }) +} + +fn record_memory() { + if let Ok(status) = std::fs::read_to_string("/proc/self/status") { + for (key, kind) in [ + ("RssAnon:", "anon"), + ("RssFile:", "file"), + ("RssShmem:", "shmem"), + ] { + if let Some(kb) = field(&status, key) { + gauge!("gitmirror_process_memory_bytes", "kind" => kind).set(kb * 1024.0); + } + } + } + + let Some(dir) = std::fs::read_to_string("/proc/self/cgroup") + .ok() + .and_then(|cgroup| { + let path = cgroup.lines().find_map(|line| line.strip_prefix("0::"))?; + Some(format!("/sys/fs/cgroup{}", path.trim_end_matches('/'))) + }) + else { + return; + }; + let read = |file: &str| std::fs::read_to_string(format!("{dir}/{file}")).ok(); + + if let Some(current) = read("memory.current").and_then(|v| v.trim().parse::().ok()) { + gauge!("gitmirror_cgroup_memory_bytes", "kind" => "current").set(current); + } + if let Some(max) = read("memory.max").and_then(|v| v.trim().parse::().ok()) { + gauge!("gitmirror_cgroup_memory_max_bytes").set(max); + } + if let Some(stat) = read("memory.stat") { + for kind in [ + "anon", + "file", + "file_mapped", + "active_file", + "inactive_file", + ] { + if let Some(value) = field(&stat, kind) { + gauge!("gitmirror_cgroup_memory_bytes", "kind" => kind).set(value); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::field; + + #[test] + fn field_matches_whole_key() { + let stat = "anon_thp 7\nanon 4096\nfile 8\n"; + assert_eq!(field(stat, "anon"), Some(4096.0)); + assert_eq!(field("RssAnon:\t 123 kB\n", "RssAnon:"), Some(123.0)); + assert_eq!(field(stat, "shmem"), None); + } +} + +#[derive(Clone, Copy)] +pub(crate) enum BlockingKind { + Git, + Transport, +} + +impl BlockingKind { + fn as_str(self) -> &'static str { + match self { + Self::Git => "git", + Self::Transport => "transport", + } + } +} + +struct RunningGuard(&'static str); + +impl Drop for RunningGuard { + fn drop(&mut self) { + gauge!("gitmirror_blocking_running", "kind" => self.0).decrement(1); + } +} + +pub(crate) async fn blocking( + kind: BlockingKind, + f: impl FnOnce() -> T + Send + 'static, +) -> Result +where + T: Send + 'static, +{ + let kind = kind.as_str(); + let queued_at = Instant::now(); + gauge!("gitmirror_blocking_queued", "kind" => kind).increment(1); + + #[allow(clippy::disallowed_methods)] + tokio::task::spawn_blocking(move || { + gauge!("gitmirror_blocking_queued", "kind" => kind).decrement(1); + histogram!("gitmirror_blocking_queue_wait_seconds", "kind" => kind) + .record(queued_at.elapsed().as_secs_f64()); + gauge!("gitmirror_blocking_running", "kind" => kind).increment(1); + let _running = RunningGuard(kind); + f() + }) + .await +} + +pub fn router() -> Router { + Router::new().route("/metrics", get(metrics_handler)) +} + +pub async fn metrics_handler() -> impl IntoResponse { + match PROMETHEUS_HANDLE.get() { + Some(handle) => { + record_memory(); + handle.run_upkeep(); + ( + StatusCode::OK, + [("content-type", "text/plain; version=0.0.4")], + handle.render(), + ) + } + None => ( + StatusCode::INTERNAL_SERVER_ERROR, + [("content-type", "text/plain")], + "Metrics not initialized".to_string(), + ), + } +} + +struct InFlightGuard { + method: &'static str, + path: String, +} + +impl Drop for InFlightGuard { + fn drop(&mut self) { + gauge!( + "gitmirror_xrpc_in_flight", + "method" => self.method, + "path" => self.path.clone() + ) + .decrement(1); + } +} + +pub async fn metrics_middleware(request: Request, next: Next) -> Response { + let start = Instant::now(); + let method = normalize_method(request.method()); + let path = normalize_path(&request); + + let span = tracing::info_span!( + "xrpc", + method = %method, + path = %path, + repo = tracing::field::Empty, + base_repo = tracing::field::Empty, + status = tracing::field::Empty, + error = tracing::field::Empty, + duration_ms = tracing::field::Empty, + ); + + gauge!( + "gitmirror_xrpc_in_flight", + "method" => method, + "path" => path.clone() + ) + .increment(1); + let _in_flight = InFlightGuard { + method, + path: path.clone(), + }; + + let response = next.run(request).instrument(span.clone()).await; + + let duration = start.elapsed().as_secs_f64(); + let status = response.status().as_u16().to_string(); + let error = response + .extensions() + .get::() + .map_or("none", |kind| kind.0); + + counter!( + "gitmirror_xrpc_requests_total", + "method" => method, + "path" => path.clone(), + "status" => status.clone(), + "error" => error + ) + .increment(1); + + histogram!( + "gitmirror_xrpc_duration_seconds", + "method" => method, + "path" => path + ) + .record(duration); + + span.record("status", status.as_str()); + span.record("error", error); + span.record("duration_ms", duration * 1e3); + tracing::info!(parent: &span, "xrpc served"); + + response +} + +fn normalize_method(method: &Method) -> &'static str { + match *method { + Method::GET => "GET", + Method::HEAD => "HEAD", + Method::POST => "POST", + _ => "other", + } +} + +fn normalize_path(request: &Request) -> String { + request + .extensions() + .get::() + .map(|matched| matched.as_str().to_string()) + .unwrap_or_else(|| "unknown".to_string()) +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/routes/activity.rs b/knotmirror/crates/knotmirror-xrpc/src/routes/activity.rs new file mode 100644 index 0000000000000000000000000000000000000000..f97b3ee48e7d0d5e379d3c3cfbc46b87879ffc89 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/routes/activity.rs @@ -0,0 +1,338 @@ +use axum::Json; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::{IntoResponse as _, Response}; +use jacquard_axum::{ExtractXrpc, XrpcResponse}; +use jacquard_common::types::string::Datetime; +use knotmirror_db::{ACTIVITY_MONTHS, MonthWindow}; +use knotmirror_types::now_us; +use lexicons::sh_tangled::git::temp2::get_commit_stats; + +use crate::{AppState, error::XrpcError}; + +const DEFAULT_MONTHS: i64 = 7; + +pub(crate) async fn get_commit_stats( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + let months = args.months.unwrap_or(DEFAULT_MONTHS); + let Some(months) = u32::try_from(months) + .ok() + .filter(|months| (1..=ACTIVITY_MONTHS).contains(months)) + else { + return Err(XrpcError::InvalidRequest(format!( + "months must be between 1 and {ACTIVITY_MONTHS}" + ))); + }; + let window = MonthWindow::ending_at(now_us(&*state.clock), months); + let committers = state + .committers + .resolve(&state.http, &*state.clock, &args.actor) + .await; + let counts = state + .db + .commit_counts(&committers, window.since, window.before, i64::from(months)) + .await + .map_err(|e| XrpcError::Internal(e.to_string()))?; + Ok(XrpcResponse(get_commit_stats::GetCommitStatsOutput { + months: counts + .into_iter() + .map(|(month, commits)| get_commit_stats::Month { + commits, + start: Datetime::new(month.and_time(Default::default()).and_utc().fixed_offset()), + extra_data: Default::default(), + }) + .collect(), + extra_data: Default::default(), + })) +} + +pub(crate) async fn did_document(State(state): State) -> Response { + match &state.signer { + Some(signer) => Json(signer.did_document()).into_response(), + None => StatusCode::NOT_FOUND.into_response(), + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + use axum::extract::State; + use axum::http::{HeaderMap, StatusCode}; + use axum::routing::post; + use bobbin_runtime::{ReqwestHttp, SystemClock}; + use chrono::{Months, Utc}; + use jacquard_axum::service_auth::ServiceAuthConfig; + use jacquard_common::service_auth::{PublicKey, parse_jwt, verify_signature}; + use jacquard_common::types::did::Did; + use jacquard_identity::JacquardResolver; + use knotmirror_db::{ACTIVITY_MONTHS, CommitActivity, Db, DbError, MonthWindow}; + use serde_json::{Value, json}; + + use crate::{AppState, Deliberi, KnotPolicy, ServiceSigner}; + + const ACTOR: &str = "did:plc:profileactor"; + const MIRROR: &str = "did:web:mirror.example"; + const DELIBERI: &str = "did:web:deliberi.example"; + + fn signer() -> ServiceSigner { + let mut key = vec![0x81, 0x26]; + key.extend_from_slice(&[7; 32]); + let encoded = format!("z{}", bs58::encode(key).into_string()); + ServiceSigner::new(Did::new_owned(MIRROR).unwrap(), &encoded).unwrap() + } + + async fn test_db() -> Option { + let Ok(url) = std::env::var("KNOTMIRROR_TEST_DATABASE_URL") else { + return None; + }; + static NEXT: AtomicUsize = AtomicUsize::new(0); + let name = format!( + "commit_stats_{}_{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + let admin = connect_with_retry(&url, 1).await; + sqlx::query(sqlx::AssertSqlSafe(format!("create database {name}"))) + .execute(admin.pool()) + .await + .unwrap(); + let mut url = reqwest::Url::parse(&url).unwrap(); + url.set_path(&name); + let db = connect_with_retry(url.as_str(), 4).await; + db.migrate().await.unwrap(); + Some(db) + } + + async fn connect_with_retry(url: &str, max_connections: u32) -> Db { + let mut backoff = Duration::from_millis(250); + for _ in 0..5 { + match Db::connect(url, max_connections).await { + Ok(db) => return db, + Err(DbError::Connection(err)) => { + tracing::warn!("connect to {url} timed out ({err}), retrying"); + tokio::time::sleep(backoff).await; + } + Err(err) => panic!("connect to {url}: {err}"), + } + backoff *= 2; + } + Db::connect(url, max_connections).await.unwrap() + } + + #[derive(Clone)] + struct FakeDeliberi { + public_key: String, + calls: Arc, + status: StatusCode, + } + + async fn resolve_committers( + State(fake): State, + headers: HeaderMap, + body: axum::Json, + ) -> (StatusCode, axum::Json) { + fake.calls.fetch_add(1, Ordering::SeqCst); + let token = headers["authorization"] + .to_str() + .unwrap() + .strip_prefix("Bearer ") + .unwrap(); + let jwt = parse_jwt(token).unwrap(); + let multikey = bs58::decode(fake.public_key.strip_prefix('z').unwrap()) + .into_vec() + .unwrap(); + let key = + PublicKey::from_k256_bytes(multikey.strip_prefix(&[0xe7, 0x01]).unwrap()).unwrap(); + verify_signature(&jwt, &key).unwrap(); + let claims = jwt.claims(); + assert_eq!(claims.iss.as_str(), MIRROR); + assert_eq!(claims.aud.as_str(), DELIBERI); + assert_eq!( + claims.lxm.as_ref().map(|lxm| lxm.as_str()), + Some("sh.tangled.identity.resolveCommitters") + ); + assert_eq!(body["actor"], ACTOR); + let committers = json!({"committers": [ + ACTOR, + "Person@Example.com", + "person@example.com", + "not-an-email", + "did:plc:someoneelse", + ]}); + (fake.status, axum::Json(committers)) + } + + async fn serve(router: axum::Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, router).await }); + url + } + + struct Mirror { + url: String, + deliberi_calls: Arc, + _repos: tempfile::TempDir, + } + + async fn mirror(db: Db, deliberi_status: StatusCode) -> Mirror { + let signer = signer(); + let fake = FakeDeliberi { + public_key: signer.did_document()["verificationMethod"][0]["publicKeyMultibase"] + .as_str() + .unwrap() + .to_owned(), + calls: Arc::default(), + status: deliberi_status, + }; + let deliberi_calls = fake.calls.clone(); + let deliberi = serve( + axum::Router::new() + .route( + "/xrpc/sh.tangled.identity.resolveCommitters", + post(resolve_committers), + ) + .with_state(fake), + ) + .await; + + let repos = tempfile::tempdir().unwrap(); + let http = reqwest::Client::new(); + let directory = Arc::new(JacquardResolver::new( + ReqwestHttp::new(http.clone()), + Default::default(), + )); + let state = AppState::new( + db, + Arc::new(repos.path().to_owned()), + http, + ServiceAuthConfig::new(Did::raw(MIRROR.into()), directory), + KnotPolicy { + allow_private: false, + require_https: true, + }, + Arc::new(SystemClock::new()), + None, + 1 << 20, + 1 << 20, + ) + .with_signer( + signer, + Some(Deliberi { + url: knotmirror_types::OpsEndpoint::parse(&deliberi).unwrap(), + did: Did::new_owned(DELIBERI).unwrap(), + }), + ); + Mirror { + url: serve(crate::router(state)).await, + deliberi_calls, + _repos: repos, + } + } + + impl Mirror { + async fn stats(&self, params: &[(&str, &str)]) -> (StatusCode, Value) { + let url = reqwest::Url::parse_with_params( + &format!("{}/xrpc/sh.tangled.git.temp2.getCommitStats", self.url), + params, + ) + .unwrap(); + let response = reqwest::get(url).await.unwrap(); + let status = response.status(); + (status, response.json().await.unwrap_or(Value::Null)) + } + } + + async fn seed(db: &Db) -> [String; 2] { + let window = MonthWindow::ending_at(Utc::now().timestamp_micros(), ACTIVITY_MONTHS); + let current = window.before - Months::new(1); + let commit = |oid: u8, committer: &str, months_back| CommitActivity { + oid: vec![oid; 20], + committer: committer.to_owned(), + month: current - Months::new(months_back), + }; + db.save_commit_activity(&[ + commit(1, ACTOR, 0), + commit(2, "person@example.com", 0), + commit(3, "person@example.com", 1), + commit(4, "person@example.com", 3), + commit(5, "someone@else.example", 0), + ]) + .await + .unwrap(); + let start = + |months_back| format!("{}T00:00:00.000000Z", current - Months::new(months_back)); + [start(0), start(1)] + } + + #[tokio::test] + async fn stats_count_the_did_and_verified_emails() { + let Some(db) = test_db().await else { return }; + let [this_month, last_month] = seed(&db).await; + let mirror = mirror(db, StatusCode::OK).await; + + let (status, body) = mirror.stats(&[("actor", ACTOR), ("months", "2")]).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + body, + json!({"months": [ + {"start": this_month, "commits": 2}, + {"start": last_month, "commits": 1}, + ]}) + ); + + let (status, body) = mirror.stats(&[("actor", ACTOR)]).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["months"].as_array().unwrap().len(), 3); + assert_eq!(mirror.deliberi_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn stats_fall_back_to_the_did_when_deliberi_fails() { + let Some(db) = test_db().await else { return }; + let [this_month, _] = seed(&db).await; + let mirror = mirror(db, StatusCode::INTERNAL_SERVER_ERROR).await; + let (status, body) = mirror.stats(&[("actor", ACTOR), ("months", "2")]).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + body, + json!({"months": [{"start": this_month, "commits": 1}]}) + ); + } + + #[tokio::test] + async fn stats_reject_bad_parameters() { + let Some(db) = test_db().await else { return }; + let mirror = mirror(db, StatusCode::OK).await; + for params in [ + [("actor", ACTOR), ("months", "0")], + [("actor", ACTOR), ("months", "25")], + [("actor", "not-a-did"), ("months", "7")], + ] { + let (status, body) = mirror.stats(¶ms).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{params:?}: {body}"); + } + } + + #[tokio::test] + async fn the_did_document_publishes_the_signing_key() { + let Some(db) = test_db().await else { return }; + let mirror = mirror(db, StatusCode::OK).await; + let body: Value = reqwest::get(format!("{}/.well-known/did.json", mirror.url)) + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(body, signer().did_document()); + assert_eq!(body["id"], MIRROR); + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/routes/cob.rs b/knotmirror/crates/knotmirror-xrpc/src/routes/cob.rs new file mode 100644 index 0000000000000000000000000000000000000000..a66bff9d97fdfa05b027659e612e8b11301bac03 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/routes/cob.rs @@ -0,0 +1,581 @@ +use std::collections::BTreeMap; +use std::collections::btree_map::Entry; +use std::path::Path; + +use axum::extract::State; +use gix::bstr::ByteSlice as _; +use jacquard_axum::{ExtractXrpc, XrpcResponse}; +use jacquard_common::ToSmolStr as _; +use jacquard_common::types::ident::AtIdentifier; +use jacquard_common::types::string::{Cid, Datetime, Did, RecordKey, Rkey}; +use knot_cob::{CobStore, HistoryModel}; +use knot_cobs::{Social as _, SocialChange}; +use knot_types::{ChangeId, CobId, Oid, SubjectNumber, TypeName, UnixSeconds}; +use lexicons::org_tangled::temp::repo::list_record_origins; +use lexicons::sh_tangled::repo::list_cob_versions; + +use crate::{AppState, error::XrpcError, metrics::BlockingKind}; + +const DEFAULT_LIMIT: i64 = 100; +const MAX_LIMIT: i64 = 500; +const DEFAULT_ORIGINS_LIMIT: i64 = 500; +const MAX_ORIGINS_LIMIT: i64 = 1000; + +fn internal(e: impl std::fmt::Display) -> XrpcError { + XrpcError::Internal(e.to_string()) +} + +pub(crate) async fn list_cob_versions( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + let uri = args.uri; + let invalid = |m: &str| XrpcError::InvalidRequest(m.to_owned()); + let AtIdentifier::Did(did) = uri.authority() else { + return Err(invalid("the URI's authority must be a DID")); + }; + let did = Did::new_owned(did.as_str()).map_err(|_| invalid("invalid DID"))?; + tracing::Span::current().record("repo", did.as_str()); + let (Some(collection), Some(rkey)) = (uri.collection(), uri.rkey()) else { + return Err(invalid("the URI must name a record")); + }; + let type_name = versioned_type(collection.as_str())?; + let rkey = rkey.as_str().to_owned(); + let newest_first = match args.order.as_deref() { + None | Some("" | "asc") => false, + Some("desc") => true, + Some(_) => return Err(invalid("order must be 'asc' or 'desc'")), + }; + let limit = args.limit.unwrap_or(DEFAULT_LIMIT); + if !(1..=MAX_LIMIT).contains(&limit) { + return Err(XrpcError::InvalidRequest(format!( + "limit must be between 1 and {MAX_LIMIT}" + ))); + } + let limit = limit as usize; + let offset = args + .cursor + .as_deref() + .map(|cursor| { + cursor + .parse::() + .map_err(|_| XrpcError::InvalidRequest(format!("invalid cursor {cursor:?}"))) + }) + .transpose()? + .unwrap_or(0); + let path = state.layout.repo_path(&did); + if !path.exists() { + return Err(XrpcError::RepoNotFound { + detail: format!("no mirror at {}", path.display()), + }); + } + + crate::metrics::blocking(BlockingKind::Git, move || { + let repo = knot_git::Repo::open(&path).map_err(|e| XrpcError::RepoNotFound { + detail: e.to_string(), + })?; + let object = object_at(&repo, &type_name, &rkey)? + .ok_or_else(|| XrpcError::NotFound(format!("no record at {}", uri.as_str())))?; + let changes = CobStore::new(&repo) + .changes_of(&type_name, object, HistoryModel::Convergent, None) + .map_err(internal)? + .changes; + let total = changes.len(); + let end = offset.saturating_add(limit).min(total); + let versions = (offset.min(end)..end) + .map(|at| &changes[if newest_first { total - 1 - at } else { at }]) + .map(|change| { + let payload = knot_cobs::decode_social(change.payload()).map_err(internal)?; + let emission = payload.emission(); + Ok(list_cob_versions::Version { + change_id: change.id.to_smolstr(), + cid: emission.produces().map(|cid| Cid::ipld(cid.cid())), + created_at: Datetime::new( + chrono::DateTime::from_timestamp(emission.at().get(), 0) + .unwrap_or_default() + .fixed_offset(), + ), + editor: Did::new_owned(emission.author().as_str()).map_err(internal)?, + extra_data: Default::default(), + }) + }) + .collect::, XrpcError>>()?; + Ok(list_cob_versions::ListCobVersionsOutput { + cursor: (end < total).then(|| end.to_smolstr()), + total: Some(total as i64), + uri, + versions, + extra_data: Default::default(), + }) + }) + .await + .map_err(internal)? + .map(XrpcResponse) +} + +pub(crate) async fn list_record_origins( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + let did = Did::new_owned(args.repo.as_str()) + .map_err(|_| XrpcError::InvalidRequest("invalid DID".to_owned()))?; + tracing::Span::current().record("repo", did.as_str()); + let type_name = versioned_type(args.collection.as_str())?; + let limit = args.limit.unwrap_or(DEFAULT_ORIGINS_LIMIT); + if !(1..=MAX_ORIGINS_LIMIT).contains(&limit) { + return Err(XrpcError::InvalidRequest(format!( + "limit must be between 1 and {MAX_ORIGINS_LIMIT}" + ))); + } + let after = args + .cursor + .as_deref() + .map(|cursor| { + RecordKey::::any_owned(cursor) + .map(|_| cursor.to_owned()) + .map_err(|_| XrpcError::InvalidRequest(format!("invalid cursor {cursor:?}"))) + }) + .transpose()?; + let path = state.layout.repo_path(&did); + + crate::metrics::blocking(BlockingKind::Git, move || { + let (page, more) = origins_page(&path, &type_name, limit as usize, after.as_deref())?; + let cursor = more + .then(|| page.last().map(|(rkey, _)| rkey.to_smolstr())) + .flatten(); + let origins = page + .into_iter() + .map(|(rkey, root)| { + let at = root.creation.created_at.get(); + Ok(list_record_origins::Origin { + rkey: RecordKey::any_owned(&rkey).map_err(internal)?, + author: Did::new_owned(root.creation.author.as_str()).map_err(internal)?, + change_id: ChangeId::new(root.object.oid()).to_smolstr(), + created_at: Datetime::new( + chrono::DateTime::from_timestamp(at, 0) + .unwrap_or_default() + .fixed_offset(), + ), + extra_data: Default::default(), + }) + }) + .collect::, XrpcError>>()?; + Ok(list_record_origins::ListRecordOriginsOutput { + collection: args.collection, + cursor, + origins, + repo: args.repo, + extra_data: Default::default(), + }) + }) + .await + .map_err(internal)? + .map(XrpcResponse) +} + +fn versioned_type(collection: &str) -> Result { + if !knot_cobs::social_collections() + .iter() + .any(|social| social.as_str() == collection) + { + return Err(XrpcError::InvalidRequest(format!( + "{collection} isn't a versioned collection" + ))); + } + TypeName::new(collection).map_err(internal) +} + +fn origins_page( + path: &Path, + type_name: &TypeName, + limit: usize, + after: Option<&str>, +) -> Result<(Vec<(String, SettledRoot)>, bool), XrpcError> { + if !path.exists() { + return Err(XrpcError::RepoNotFound { + detail: format!("no mirror at {}", path.display()), + }); + } + let repo = knot_git::Repo::open(path).map_err(|e| XrpcError::RepoNotFound { + detail: e.to_string(), + })?; + let mut page: Vec<_> = settled_roots(&repo, type_name, None, after)? + .into_iter() + .take(limit + 1) + .collect(); + let more = page.len() > limit; + page.truncate(limit); + Ok((page, more)) +} + +// picks the same root as the listing, so fetching one ticket and listing them agree +fn object_at( + repo: &knot_git::Repo, + type_name: &TypeName, + rkey: &str, +) -> Result, XrpcError> { + Ok(settled_roots(repo, type_name, Some(rkey), None)? + .remove(rkey) + .map(|root| root.object)) +} + +struct SettledRoot { + object: CobId, + creation: knot_cobs::Creation, +} + +impl SettledRoot { + fn rank(&self) -> (SubjectNumber, UnixSeconds, CobId) { + (self.creation.number, self.creation.created_at, self.object) + } +} + +// racing creates can leave two roots for one rkey. the lowest (number, created at, id) wins, same as knot-index +fn settled_roots( + repo: &knot_git::Repo, + type_name: &TypeName, + only: Option<&str>, + after: Option<&str>, +) -> Result, XrpcError> { + let git = repo.git(); + let prefix = format!("refs/cobs/{}/", type_name.as_str()); + let refs = git.references().map_err(internal)?; + let mut settled: BTreeMap = BTreeMap::new(); + for reference in refs.prefixed(prefix.as_str()).map_err(internal)? { + let reference = reference.map_err(internal)?; + let name = reference.name().as_bstr(); + let Some(object) = name + .strip_prefix(prefix.as_bytes()) + .and_then(|hex| Oid::from_hex(hex.to_str().ok()?).ok()) + .map(CobId::new) + else { + continue; + }; + let Some(SocialChange::Open { creation, .. }) = root_payload(git, object)? else { + continue; + }; + let rkey = creation.address.rkey().as_str().to_owned(); + if only.is_some_and(|only| only != rkey) + || after.is_some_and(|after| rkey.as_str() <= after) + { + continue; + } + let candidate = SettledRoot { object, creation }; + match settled.entry(rkey) { + Entry::Vacant(slot) => { + slot.insert(candidate); + } + Entry::Occupied(mut slot) if candidate.rank() < slot.get().rank() => { + slot.insert(candidate); + } + Entry::Occupied(_) => {} + } + } + Ok(settled) +} + +/// The root change's payload; the object's id is its root commit. +fn root_payload( + git: &gix::Repository, + object: CobId, +) -> Result, XrpcError> { + let Some(commit) = git + .find_object(object.oid().object_id()) + .ok() + .and_then(|found| found.try_into_commit().ok()) + else { + return Ok(None); + }; + let tree = commit.tree().map_err(internal)?; + let Some(entry) = tree.find_entry("payload") else { + return Ok(None); + }; + let blob = entry.object().map_err(internal)?; + Ok(knot_cobs::decode_social(&blob.data).ok()) +} + +#[cfg(test)] +mod tests { + use super::*; + use knot_cob::{Approved, CobHome, Materialized, Record, WriteContext}; + use knot_cobs::{Creation, Editing, TicketChange, TicketCob}; + use knot_record::Prose; + use knot_record::ticket::{TicketRecord, TicketState, TicketTitle, ticket_collection}; + use knot_runtime::{K256Signer, SeededEntropy}; + use knot_types::{ + AccountDid, ContributionPermission, ContributionPolicy, RecordAddress, RecordBody, + RecordCid, RecordRkey, RepoDid, RepoRole, SubjectNumber, Tid, UnixSeconds, + }; + + const T0: i64 = 1_700_000_000; + + struct Fixture { + _dir: tempfile::TempDir, + layout: knot_git::Layout, + repo: RepoDid, + signer: K256Signer, + } + + fn account(name: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{name}")).unwrap() + } + + fn rkey(index: u64) -> RecordRkey { + RecordRkey::minted(Tid::from_time(1_700_000_000_000_000 + index, 0)) + } + + fn body(owner: &AccountDid, revision: u32) -> RecordBody { + let prose = Prose::new( + TicketTitle::new(format!("revision {revision}")).unwrap(), + None, + ); + TicketRecord::new( + prose, + TicketState::Open, + None, + None, + None, + None, + UnixSeconds::new(T0), + owner.clone(), + ) + .unwrap() + .body() + .unwrap() + } + + fn ticket(state: TicketState) -> knot_cobs::Ticket { + knot_cobs::Ticket { + state, + tangled_labels: None, + assignees: None, + } + } + + impl Fixture { + fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + let layout = knot_git::Layout::new(dir.path().join("repos")); + let repo = RepoDid::new("did:plc:repo").unwrap(); + layout.create(&repo).unwrap(); + Self { + _dir: dir, + layout, + repo, + signer: K256Signer::generate(&SeededEntropy::new(9)), + } + } + + fn path(&self) -> std::path::PathBuf { + self.layout.repo_path(&self.repo).unwrap() + } + + fn context<'a>(&'a self, actor: &'a AccountDid) -> WriteContext<'a> { + WriteContext::new( + actor, + &self.repo, + ContributionPermission::new( + knot_types::Blocked::No, + RepoRole::Owner, + ContributionPolicy::Collaborators, + ), + ) + } + + fn open(&self, author: &str, index: u64, number: u32, at: i64) -> CobId { + let author = account(author); + let record = Record::version(body(&author, 0)); + let repo = self.layout.open(&self.repo).unwrap(); + let cx = self.context(&author); + let change = TicketChange::Open { + creation: Creation { + address: RecordAddress::new(ticket_collection(), rkey(index)), + author: author.clone(), + number: SubjectNumber::new(number).unwrap(), + parent: None, + created_at: UnixSeconds::new(at), + produces: body(&author, 0).cid(), + }, + body: ticket(TicketState::Open), + }; + let approved = Approved::signed_by(&cx, Materialized { change, record }).unwrap(); + CobStore::new(&repo) + .create_approved::( + &CobHome::from(&self.repo), + &approved, + &self.signer, + UnixSeconds::new(at), + ) + .unwrap() + .object + } + + fn edit(&self, object: CobId, editor: &str, opener: &str, revision: u32) { + let (editor, opener) = (account(editor), account(opener)); + let repo = self.layout.open(&self.repo).unwrap(); + let cx = self.context(&editor); + let at = T0 + i64::from(revision) * 100; + let from: RecordCid = body(&opener, revision - 1).cid(); + let record = Record::version(body(&opener, revision)); + let change = TicketChange::Edit { + editing: Editing { + author: editor.clone(), + from, + produces: body(&opener, revision).cid(), + edited_at: UnixSeconds::new(at), + }, + body: ticket(TicketState::Closed), + }; + let approved = Approved::signed_by(&cx, Materialized { change, record }).unwrap(); + CobStore::new(&repo) + .append_approved::( + &CobHome::from(&self.repo), + object, + &approved, + &self.signer, + UnixSeconds::new(at), + ) + .unwrap(); + } + + fn origins(&self, limit: usize, after: Option<&str>) -> (Vec<(String, SettledRoot)>, bool) { + origins_page(&self.path(), &ticket_type(), limit, after).unwrap() + } + } + + fn ticket_type() -> TypeName { + versioned_type(ticket_collection().as_str()).unwrap() + } + + fn rkeys(page: &[(String, SettledRoot)]) -> Vec<&str> { + page.iter().map(|(rkey, _)| rkey.as_str()).collect() + } + + #[test] + fn lists_each_opener_sorted_by_rkey() { + let fixture = Fixture::new(); + fixture.open("zed", 3, 3, T0 + 3); + fixture.open("amy", 1, 1, T0 + 1); + fixture.open("bob", 2, 2, T0 + 2); + + let (page, more) = fixture.origins(10, None); + assert!(!more); + let expected: Vec = [1, 2, 3].map(|i| rkey(i).as_str().to_owned()).into(); + assert_eq!(rkeys(&page), expected); + let authors: Vec<&str> = page + .iter() + .map(|(_, root)| root.creation.author.as_str()) + .collect(); + assert_eq!(authors, ["did:plc:amy", "did:plc:bob", "did:plc:zed"]); + } + + #[test] + fn later_editors_never_replace_the_opener() { + let fixture = Fixture::new(); + let object = fixture.open("amy", 1, 1, T0); + fixture.edit(object, "maintainer", "amy", 1); + fixture.edit(object, "other", "amy", 2); + + let (page, _) = fixture.origins(10, None); + assert_eq!(page.len(), 1); + let (_, root) = &page[0]; + assert_eq!(root.creation.author.as_str(), "did:plc:amy"); + assert_eq!(root.object, object); + let changes = CobStore::new(&fixture.layout.open(&fixture.repo).unwrap()) + .changes_of(&ticket_type(), object, HistoryModel::Convergent, None) + .unwrap() + .changes; + assert_eq!(changes.len(), 3); + } + + #[test] + fn lowest_root_wins_an_address() { + let fixture = Fixture::new(); + let late = fixture.open("late", 1, 2, T0); + let early = fixture.open("early", 1, 1, T0 + 50); + let (page, _) = fixture.origins(10, None); + assert_eq!(page.len(), 1); + assert_eq!(page[0].1.object, early); + assert_ne!(early, late); + let key = rkey(1); + assert_eq!( + object_at( + &knot_git::Repo::open(fixture.path()).unwrap(), + &ticket_type(), + key.as_str() + ) + .unwrap(), + Some(early) + ); + } + + #[test] + fn pages_resume_after_the_cursor() { + let fixture = Fixture::new(); + (1..=5).for_each(|i| { + fixture.open("amy", i, i as u32, T0 + i as i64); + }); + let expected: Vec = (1..=5).map(|i| rkey(i).as_str().to_owned()).collect(); + + let (first, more) = fixture.origins(2, None); + assert!(more); + assert_eq!(rkeys(&first), expected[..2]); + let (second, more) = fixture.origins(2, Some(&first[1].0)); + assert!(more); + assert_eq!(rkeys(&second), expected[2..4]); + let (last, more) = fixture.origins(2, Some(&second[1].0)); + assert!(!more); + assert_eq!(rkeys(&last), expected[4..]); + let (exact, more) = fixture.origins(5, None); + assert!(!more); + assert_eq!(exact.len(), 5); + } + + #[test] + fn agrees_with_the_first_listed_version() { + let fixture = Fixture::new(); + let first = fixture.open("amy", 1, 1, T0 + 1); + fixture.open("bob", 2, 2, T0 + 2); + fixture.open("late", 2, 3, T0 + 3); + fixture.edit(first, "maintainer", "amy", 1); + + let repo = knot_git::Repo::open(fixture.path()).unwrap(); + let (page, _) = fixture.origins(10, None); + assert_eq!(page.len(), 2); + for (rkey, root) in &page { + let object = object_at(&repo, &ticket_type(), rkey).unwrap().unwrap(); + let versions = CobStore::new(&repo) + .changes_of(&ticket_type(), object, HistoryModel::Convergent, None) + .unwrap() + .changes; + let version = &versions[0]; + let payload = knot_cobs::decode_social(version.payload()).unwrap(); + let emission = payload.emission(); + assert_eq!(object, root.object); + assert_eq!( + version.id.to_smolstr(), + ChangeId::new(root.object.oid()).to_smolstr() + ); + assert_eq!(emission.author(), &root.creation.author); + assert_eq!(emission.at(), root.creation.created_at); + } + } + + #[test] + fn unknown_collection_is_invalid() { + assert!(matches!( + versioned_type("app.bsky.feed.post"), + Err(XrpcError::InvalidRequest(_)) + )); + } + + #[test] + fn unmirrored_repo_is_not_found() { + let fixture = Fixture::new(); + let missing = fixture.path().with_file_name("absent"); + assert!(matches!( + origins_page(&missing, &ticket_type(), 10, None), + Err(XrpcError::RepoNotFound { .. }) + )); + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/routes/git.rs b/knotmirror/crates/knotmirror-xrpc/src/routes/git.rs new file mode 100644 index 0000000000000000000000000000000000000000..ab461a360fbc6b2e3db02542af5c7298583e0298 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/routes/git.rs @@ -0,0 +1,2288 @@ +use std::collections::BTreeSet; +use std::path::Path; +use std::time::{Duration, Instant}; + +use axum::extract::State; +use bobbin_knot_proxy::KnotHost; +use gix::ObjectId; +use gix::bstr::ByteSlice as _; +use gix::revision::plumbing::Spec as RevSpec; +use gix::revision::walk::Sorting; +use jacquard_axum::{ExtractXrpc, XrpcResponse}; +use jacquard_common::ToSmolStr; +use jacquard_common::deps::smol_str::SmolStr; +use jacquard_common::service_auth::ServiceAuthClaims; +use jacquard_common::types::did::Did; +use jacquard_common::types::string::Datetime; +use jacquard_identity::resolver::IdentityResolver as _; +use knotmirror_git::repo_ext::{MergeError, RepositoryExt as _}; +use lexicons::org_tangled::temp::git::{ + delete_branch, list_commits, list_commits_by_ref, sync_fork, +}; +use lexicons::sh_tangled::git::{ + merge_commit, + temp2::{get_blame, get_diff, get_interdiff, merge_check}, +}; +use lexicons::{org_tangled, sh_tangled}; +use tangled_axum::atproto::{AtprotoService as _, ExtractAnyServiceAuth}; +use tokio::time::timeout; +use tracing::{info, warn}; + +use crate::did_ext::DidDocumentExt as _; +use crate::routes::reads::open_repos; +use crate::{AppState, diff, error::XrpcError, metrics::BlockingKind}; +use knotmirror_types::KnotRepoUrl as _; + +const DEFAULT_LIMIT: u32 = 50; +const MAX_LIMIT: u32 = 100; + +const AGENT: &str = concat!("knotmirror/", env!("CARGO_PKG_VERSION")); + +const COMMITTER_NAME: &str = "Tangled"; +const COMMITTER_EMAIL: &str = "noreply@tangled.sh"; +const PUSH_NSID: &str = "org.tangled.git.receivePack"; + +const MAX_BLAME_OBJECT_BYTES: u64 = 1024 * 1024; +const MAX_BLAME_RUNS: usize = 50_000; +const BLAME_TIMEOUT: Duration = Duration::from_secs(5); + +// a compare across years of a big repo can touch tens of thousands of files and run for minutes +const MAX_DIFF_FILES: usize = 3_000; +const DIFF_TIMEOUT: Duration = Duration::from_secs(10); + +#[derive(Clone, Copy)] +struct DiffLimits { + files: usize, + deadline: Instant, +} + +impl DiffLimits { + fn starting_now() -> Self { + Self { + files: MAX_DIFF_FILES, + deadline: Instant::now() + DIFF_TIMEOUT, + } + } +} + +fn collect_diffs( + diffs: impl Iterator>, + limits: DiffLimits, +) -> Result, XrpcError> { + let mut out = Vec::new(); + for d in diffs { + if out.len() == limits.files { + return Err(XrpcError::DiffTooLarge { + limit: limits.files, + }); + } + if Instant::now() > limits.deadline { + return Err(XrpcError::DiffTimeout); + } + out.push( + d.map_err(|e| XrpcError::CompareError(e.to_string()))? + .into(), + ); + } + Ok(out) +} + +enum MergeStyle { + Rebase, + // TODO(post-1.0): support more merge strategies + // Merge, + // RebaseMerge, + // SquashRebase, + // SquashRebaseMerge, + // FastForwardOnly, +} + +impl MergeStyle { + fn parse(style: &str) -> Option { + match style { + "rebase" => Some(Self::Rebase), + _ => None, + } + } +} + +pub(crate) struct GixSignature<'a>(pub(crate) gix::actor::SignatureRef<'a>); + +impl TryFrom> for sh_tangled::git::Signature { + type Error = anyhow::Error; + + fn try_from(GixSignature(sig): GixSignature) -> Result { + let time = sig.time()?; + let offset = chrono::FixedOffset::east_opt(time.offset).ok_or_else(|| { + anyhow::anyhow!("commit timezone offset out of range: {}", time.offset) + })?; + let when = chrono::DateTime::from_timestamp(time.seconds, 0) + .ok_or_else(|| anyhow::anyhow!("commit timestamp out of range: {}", time.seconds))? + .with_timezone(&offset); + Ok(Self { + name: sig.name.to_smolstr(), + email: sig.email.to_smolstr(), + when: Datetime::new(when), + extra_data: Default::default(), + }) + } +} + +impl TryFrom> for org_tangled::git::Signature { + type Error = anyhow::Error; + + fn try_from(sig: GixSignature) -> Result { + let sh_tangled::git::Signature { + name, email, when, .. + } = sig.try_into()?; + Ok(Self { + name, + email, + when, + extra_data: Default::default(), + }) + } +} + +struct GixCommit<'a>(&'a gix::Commit<'a>); + +impl TryFrom> for list_commits::Commit { + type Error = anyhow::Error; + + fn try_from(GixCommit(commit): GixCommit<'_>) -> Result { + let decoded = commit.decode()?; + Ok(Self { + oid: commit.id.to_smolstr(), + parents: decoded + .parents + .iter() + .map(|parent| parent.to_smolstr()) + .collect(), + tree: decoded.tree.to_smolstr(), + author: GixSignature(decoded.author()?).try_into()?, + committer: GixSignature(decoded.committer()?).try_into()?, + extra_headers: decoded + .extra_headers + .iter() + .map(|(key, value)| list_commits::Header { + key: key.to_smolstr(), + value: value.to_smolstr(), + extra_data: Default::default(), + }) + .collect(), + message: decoded.message.to_smolstr(), + stat: None, + signature: ssh_signature(commit), + extra_data: Default::default(), + }) + } +} + +/// verifies the commit's ssh signature against the key embedded in it, so it only proves +/// the commit is intact. +fn ssh_signature(commit: &gix::Commit<'_>) -> Option { + ssh_signature_of(&commit.data, commit.id.kind()) +} + +fn ssh_signature_of(raw: &[u8], kind: gix::hash::Kind) -> Option { + let (signature, payload) = split_signature(raw, kind)?; + let signature = ssh_key::SshSig::from_pem(signature).ok()?; + let key = ssh_key::PublicKey::from(signature.public_key().clone()); + key.verify("git", &payload, &signature).ok()?; + Some(list_commits::SshSignature { + fingerprint: key.fingerprint(ssh_key::HashAlg::Sha256).to_smolstr(), + // HACK: jacquard doesn't encode `$type` in single-variant union + extra_data: Some( + [( + "$type".into(), + "org.tangled.temp.git.listCommits#sshSignature".into(), + )] + .into(), + ), + }) +} + +// HACK: `Commit::signature` doesn't handle `gpgsig-sha256` case +fn split_signature(raw: &[u8], kind: gix::hash::Kind) -> Option<(Vec, Vec)> { + let ours: &[u8] = match kind { + gix::hash::Kind::Sha1 => b"gpgsig", + gix::hash::Kind::Sha256 => b"gpgsig-sha256", + _ => return None, + }; + let end = raw.find(b"\n\n")? + 1; + let mut signature: Option> = None; + let mut payload = Vec::with_capacity(raw.len()); + // while inside a signature header: whether it is the one we verify + let mut inside: Option = None; + for line in raw[..end].split_inclusive(|&b| b == b'\n') { + if let (Some(is_ours), Some(rest)) = (inside, line.strip_prefix(b" ")) { + if is_ours { + signature.as_mut()?.extend_from_slice(rest); + } + continue; + } + inside = None; + let name = line.split(|&b| b == b' ').next().unwrap_or_default(); + if name == b"gpgsig" || name == b"gpgsig-sha256" { + let is_ours = name == ours; + if is_ours { + signature = Some(line[name.len() + 1..].to_vec()); + } + inside = Some(is_ours); + continue; + } + payload.extend_from_slice(line); + } + payload.extend_from_slice(&raw[end..]); + Some((signature?, payload)) +} + +impl From for org_tangled::git::DiffStat { + fn from(stat: diff::Stat) -> Self { + Self { + insertions: stat.insertions as i64, + deletions: stat.deletions as i64, + files_changed: stat.files_changed as i64, + extra_data: Default::default(), + } + } +} + +impl From for sh_tangled::git::DiffSrc { + fn from(f: crate::diff::FileContent) -> Self { + Self { + path: f.path.into(), + oid: f.oid.into(), + size: f.size as i64, + is_binary: f.is_binary, + is_submodule: f.is_submodule, + content: f + .content + .map(|b| String::from_utf8_lossy(&b).into_owned().into()), + extra_data: Default::default(), + } + } +} + +impl From for sh_tangled::git::DiffHunk { + fn from(h: crate::diff::Hunk) -> Self { + // The novel sets are hash sets; sort them so the output is stable across runs. + let sorted = |set: rustc_hash::FxHashSet| -> Vec { + let mut v: Vec = set.into_iter().map(|n| i64::from(n.0)).collect(); + v.sort_unstable(); + v + }; + Self { + novel_lhs: sorted(h.novel_lhs), + novel_rhs: sorted(h.novel_rhs), + lines: h + .lines + .into_iter() + .map(|(lhs, rhs)| sh_tangled::git::LinePair { + lhs: lhs.map(|n| i64::from(n.0)), + rhs: rhs.map(|n| i64::from(n.0)), + extra_data: Default::default(), + }) + .collect(), + extra_data: Default::default(), + } + } +} + +impl From for sh_tangled::git::FileDiff { + fn from(d: crate::diff::Diff) -> Self { + Self { + lhs_src: d.lhs_src.into(), + rhs_src: d.rhs_src.into(), + hunks: d.hunks.into_iter().map(Into::into).collect(), + has_byte_changes: d + .has_byte_changes + .map(|(lhs, rhs)| sh_tangled::git::ByteChanges { + lhs: lhs as i64, + rhs: rhs as i64, + extra_data: Default::default(), + }), + has_syntactic_changes: d.has_syntactic_changes, + extra_data: Default::default(), + } + } +} + +/// Resolve a full hex oid to a commit that actually exists in `repo`. +pub(crate) fn find_commit(repo: &gix::Repository, sha: &str) -> Result { + let oid = gix::ObjectId::from_hex(sha.as_bytes()) + .map_err(|e| XrpcError::InvalidRequest(format!("bad commit sha {sha:?}: {e}")))?; + repo.find_commit(oid) + .map_err(|_| XrpcError::RevisionNotFound { + rev: sha.to_owned(), + })?; + Ok(oid) +} + +pub(crate) fn find_diff_base( + repo: &gix::Repository, + sha: &str, +) -> Result, XrpcError> { + let oid = gix::ObjectId::from_hex(sha.as_bytes()) + .map_err(|e| XrpcError::InvalidRequest(format!("bad commit sha {sha:?}: {e}")))?; + if oid == gix::ObjectId::empty_tree(repo.object_hash()) { + return Ok(None); + } + find_commit(repo, sha).map(Some) +} + +// ── blame ────────────────────────────────────────────────────────────── + +#[derive(Debug, thiserror::Error)] +#[error("blame computation deadline exceeded")] +struct BlameDeadline { + source: BlameDeadlineMarker, +} + +#[derive(Debug, thiserror::Error)] +#[error("blame deadline marker")] +struct BlameDeadlineMarker; + +struct DeadlineObjects<'a> { + git: &'a gix::Repository, + deadline: Instant, +} + +impl DeadlineObjects<'_> { + fn check(&self) -> Result<(), gix::objs::find::Error> { + if Instant::now() >= self.deadline { + return Err(Box::new(BlameDeadline { + source: BlameDeadlineMarker, + })); + } + Ok(()) + } +} + +impl gix::objs::FindHeader for DeadlineObjects<'_> { + fn try_header( + &self, + id: &gix::hash::oid, + ) -> Result, gix::objs::find::Error> { + self.check()?; + gix::objs::FindHeader::try_header(self.git, id) + } +} + +impl gix::objs::Find for DeadlineObjects<'_> { + fn try_find<'a>( + &self, + id: &gix::hash::oid, + buffer: &'a mut Vec, + ) -> Result>, gix::objs::find::Error> { + self.check()?; + gix::objs::Find::try_find(self.git, id, buffer) + } +} + +fn caused_by( + error: &(dyn std::error::Error + 'static), + predicate: fn(&T) -> bool, +) -> bool { + error.downcast_ref::().is_some_and(predicate) + || error + .source() + .is_some_and(|source| caused_by(source, predicate)) +} + +pub(crate) fn allocation_limited(error: &(dyn std::error::Error + 'static)) -> bool { + caused_by::(error, |error| { + matches!(error, gix::odb::loose::find::Error::OutOfMemory { .. }) + }) || caused_by::(error, |error| { + matches!(error, gix::odb::pack::data::decode::Error::OutOfMemory) + }) +} + +fn map_blame_error(error: &(dyn std::error::Error + 'static)) -> XrpcError { + if caused_by::(error, |_| true) { + XrpcError::BlameTimeout + } else if allocation_limited(error) { + XrpcError::BlameTooLarge + } else { + XrpcError::Internal(error.to_string()) + } +} + +fn get_blame_inner( + repo_path: &Path, + refspec: &str, + path: &str, + deadline: Instant, +) -> Result { + if path.is_empty() { + return Err(XrpcError::InvalidRequest( + "missing path parameter".to_owned(), + )); + } + if refspec.is_empty() || refspec.contains('\0') || refspec.starts_with('-') { + return Err(XrpcError::InvalidRequest(format!( + "invalid ref spec {refspec:?}" + ))); + } + + let repo = gix::open_opts( + repo_path, + gix::open::Options::default().config_overrides([format!( + "gitoxide.objects.allocLimit={MAX_BLAME_OBJECT_BYTES}" + )]), + ) + .map_err(|e| XrpcError::RepoNotFound { + detail: e.to_string(), + })?; + + let id = match repo.rev_parse_single(refspec.as_bytes()) { + Ok(id) => id, + Err(gix::revision::spec::parse::single::Error::RangedRev { spec }) => { + return Err(XrpcError::InvalidRequest(format!( + "ref must resolve to a single revision, got a range: {spec}" + ))); + } + Err(error) => { + return Err(XrpcError::RefNotFound { + rev: refspec.to_owned(), + detail: error.to_string(), + }); + } + }; + let peeled = id + .object() + .and_then(|object| object.peel_tags_to_end()) + .map_err(|e| XrpcError::RefNotFound { + rev: refspec.to_owned(), + detail: e.to_string(), + })?; + let commit_oid = peeled.id; + let commit = peeled + .try_into_commit() + .map_err(|_| XrpcError::RefNotFound { + rev: refspec.to_owned(), + detail: "revision does not point at a commit".to_owned(), + })?; + + let tree = commit + .tree() + .map_err(|e| XrpcError::Internal(e.to_string()))?; + let entry = tree + .lookup_entry_by_path(path) + .map_err(|e| XrpcError::Internal(e.to_string()))? + .filter(|entry| { + matches!( + entry.mode().kind(), + gix::objs::tree::EntryKind::Blob + | gix::objs::tree::EntryKind::BlobExecutable + | gix::objs::tree::EntryKind::Link + ) + }) + .ok_or_else(|| XrpcError::FileNotFound { + path: path.to_owned(), + })?; + + let header = repo + .find_header(entry.object_id()) + .map_err(|e| XrpcError::Internal(e.to_string()))?; + if header.size() > MAX_BLAME_OBJECT_BYTES { + return Err(XrpcError::BlameTooLarge); + } + + let file_path = path.as_bytes().as_bstr(); + let cache = repo + .commit_graph_if_enabled() + .map_err(|e| XrpcError::Internal(e.to_string()))?; + let mut resource_cache = repo + .diff_resource_cache_for_tree_diff() + .map_err(|e| XrpcError::Internal(e.to_string()))?; + let options = gix::blame::Options { + diff_algorithm: repo + .diff_algorithm() + .map_err(|e| XrpcError::Internal(e.to_string()))?, + rewrites: Some(gix::diff::Rewrites::default()), + ..Default::default() + }; + let outcome = gix::blame::file( + DeadlineObjects { + git: &repo, + deadline, + }, + commit_oid, + cache, + &mut resource_cache, + file_path, + options, + ) + .map_err(|error| match error { + gix::blame::Error::FileMissing { file_path, .. } => XrpcError::FileNotFound { + path: file_path.to_string(), + }, + error => map_blame_error(&error), + })?; + if outcome.entries.len() > MAX_BLAME_RUNS { + return Err(XrpcError::BlameTooLarge); + } + + let commit_oids: BTreeSet = outcome + .entries + .iter() + .map(|entry| entry.commit_id) + .collect(); + let commits = commit_oids + .into_iter() + .map(|oid| -> Result { + if Instant::now() >= deadline { + return Err(XrpcError::BlameTimeout); + } + let commit = repo + .find_commit(oid) + .map_err(|e| XrpcError::Internal(e.to_string()))?; + let decoded = commit + .decode() + .map_err(|e| XrpcError::Internal(e.to_string()))?; + Ok(get_blame::Commit { + oid: oid.to_smolstr(), + message: decoded.message.to_smolstr(), + author: GixSignature( + decoded + .author() + .map_err(|e| XrpcError::Internal(e.to_string()))?, + ) + .try_into() + .map_err(|e: anyhow::Error| XrpcError::Internal(e.to_string()))?, + extra_data: Default::default(), + }) + }) + .collect::, _>>()?; + + let runs = outcome + .entries + .into_iter() + .map(|entry| get_blame::Run { + start: i64::from(entry.start_in_blamed_file) + 1, + count: i64::from(entry.len.get()), + commit: entry.commit_id.to_smolstr(), + extra_data: Default::default(), + }) + .collect(); + + Ok(get_blame::GetBlameOutput { + commit: commit_oid.to_smolstr(), + commits, + runs, + extra_data: Default::default(), + }) +} + +pub(crate) async fn get_blame( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + tracing::Span::current().record("repo", args.repo.as_str()); + let repo_path = state.layout.repo_path(&args.repo); + let deadline = Instant::now() + BLAME_TIMEOUT; + let task = crate::metrics::blocking(BlockingKind::Git, move || { + get_blame_inner( + &repo_path, + args.r#ref.as_ref(), + args.path.as_ref(), + deadline, + ) + }); + timeout(BLAME_TIMEOUT, task) + .await + .map_err(|_| XrpcError::BlameTimeout)? + .map_err(|e| XrpcError::Internal(e.to_string()))? + .map(XrpcResponse) +} + +fn get_diff_inner( + repo: &gix::Repository, + base: Option, + head: gix::ObjectId, + limits: DiffLimits, +) -> Result, XrpcError> { + let trees = || -> anyhow::Result<_> { + let old = match base { + Some(base) => { + let merge_base = repo.merge_base(base, head)?.detach(); + repo.find_tree(repo.find_commit(merge_base)?.tree_id()?)? + } + None => repo.empty_tree(), + }; + let new = repo.find_tree(repo.find_commit(head)?.tree_id()?)?; + Ok((old, new)) + }; + let (old, new) = trees().map_err(|e| XrpcError::CompareError(e.to_string()))?; + let diffs = + diff::diff(repo, &old, &new, false).map_err(|e| XrpcError::CompareError(e.to_string()))?; + collect_diffs(diffs, limits) +} + +#[axum::debug_handler] +pub(crate) async fn get_diff( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + let span = tracing::Span::current(); + span.record("repo", args.head_repo.as_str()); + span.record("base_repo", args.base_repo.as_str()); + let scratch = state + .layout + .open_scratch(&[&args.head_repo, &args.base_repo])?; + let base = find_diff_base(&scratch, args.base_commit.as_ref())?; + let head = find_commit(&scratch, args.head_commit.as_ref())?; + + let limits = DiffLimits::starting_now(); + let task = crate::metrics::blocking(BlockingKind::Git, move || { + get_diff_inner(&scratch, base, head, limits) + }); + timeout(DIFF_TIMEOUT, task) + .await + .map_err(|_| XrpcError::DiffTimeout)? + .map_err(|e| XrpcError::Internal(e.to_string()))? + .map(|diffs| { + XrpcResponse(get_diff::GetDiffOutput { + diffs, + extra_data: Default::default(), + }) + }) +} + +fn get_interdiff_inner( + repo: &gix::Repository, + (from_base_id, from_head_id): (ObjectId, ObjectId), + (to_base_id, to_head_id): (ObjectId, ObjectId), + limits: DiffLimits, +) -> Result, XrpcError> { + let to_head = repo + .find_commit(to_head_id) + .map_err(|e| XrpcError::Internal(e.to_string()))?; + let to_head_tree = to_head + .tree() + .map_err(|e| XrpcError::Internal(e.to_string()))?; + let rebased_tree = diff::prepare_interdiff(repo, (from_base_id, from_head_id), to_base_id) + .map_err(|e| XrpcError::Internal(e.to_string()))?; + let diffs = diff::diff(repo, &rebased_tree, &to_head_tree, true) + .map_err(|e| XrpcError::CompareError(e.to_string()))?; + collect_diffs(diffs, limits) +} + +#[axum::debug_handler] +pub(crate) async fn get_interdiff( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + let span = tracing::Span::current(); + span.record("repo", args.head_repo.as_str()); + span.record("base_repo", args.base_repo.as_str()); + let from_base_id = ObjectId::from_hex(args.base_commit1.as_bytes()) + .map_err(|e| XrpcError::InvalidRequest(e.to_string()))?; + let from_head_id = ObjectId::from_hex(args.base_commit2.as_bytes()) + .map_err(|e| XrpcError::InvalidRequest(e.to_string()))?; + let to_base_id = ObjectId::from_hex(args.head_commit1.as_bytes()) + .map_err(|e| XrpcError::InvalidRequest(e.to_string()))?; + let to_head_id = ObjectId::from_hex(args.head_commit2.as_bytes()) + .map_err(|e| XrpcError::InvalidRequest(e.to_string()))?; + + let scratch = state + .layout + .open_scratch(&[&args.head_repo, &args.base_repo])?; + + let limits = DiffLimits::starting_now(); + let task = crate::metrics::blocking(BlockingKind::Git, move || { + get_interdiff_inner( + &scratch, + (from_base_id, from_head_id), + (to_base_id, to_head_id), + limits, + ) + }); + timeout(DIFF_TIMEOUT, task) + .await + .map_err(|_| XrpcError::DiffTimeout)? + .map_err(|e| XrpcError::Internal(e.to_string()))? + .map(|diffs| { + XrpcResponse(get_interdiff::GetInterdiffOutput { + diffs, + extra_data: Default::default(), + }) + }) +} + +fn commit_log_walk<'repo>( + repo: &'repo gix::Repository, + ranges: &[SmolStr], +) -> Result, XrpcError> { + let mut tips = Vec::new(); + let mut hidden = Vec::new(); + for range in ranges { + match range.split_once("..") { + Some((from, to)) => { + hidden.push(find_commit(repo, from)?); + tips.push(find_commit(repo, to)?); + } + None => tips.push(find_commit(repo, range)?), + } + } + + repo.rev_walk(tips) + .sorting(Sorting::ByCommitTime(Default::default())) + .with_hidden(hidden) + .all() + .map_err(|e| XrpcError::Internal(e.to_string())) +} + +fn list_commits_inner( + repo: &gix::Repository, + stats: &diff::StatCache, + args: list_commits::ListCommits, +) -> Result, XrpcError> { + let walk = commit_log_walk(repo, args.ranges.as_deref().unwrap_or_default())?; + + let limit = args + .limit + .map(|limit| limit as u32) + .unwrap_or(DEFAULT_LIMIT); + if limit == 0 || limit > MAX_LIMIT { + return Err(XrpcError::InvalidRequest(format!( + "limit must be between 1 and {MAX_LIMIT}" + ))); + } + + let mut commits: Vec = Vec::with_capacity(limit as usize); + let mut skipped = 0usize; + + for info in walk { + let info = info.map_err(|e| XrpcError::Internal(e.to_string()))?; + + if (skipped as i64) < args.skip.unwrap_or(0) { + skipped += 1; + continue; + } + if (commits.len() as i64) == args.limit.unwrap_or(50) { + break; + } + + let commit = info + .object() + .map_err(|e| XrpcError::Internal(e.to_string()))?; + let mut row = list_commits::Commit::try_from(GixCommit(&commit)) + .map_err(|e| XrpcError::Internal(e.to_string()))?; + if args.include_stats.unwrap_or(false) { + let stat = stats.commit(repo, &commit); + row.stat = stat + .inspect_err(|error| warn!(oid = %commit.id, %error, "commit stat failed")) + .ok() + .map(Into::into); + } + commits.push(row); + } + + Ok(commits) +} + +#[axum::debug_handler] +pub(crate) async fn list_commits( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + let repo = open_repos(&state, &args.repos)?; + let stats = state.stats.clone(); + crate::metrics::blocking(BlockingKind::Git, move || { + list_commits_inner(&repo, &stats, args) + }) + .await + .map_err(|e| XrpcError::Internal(e.to_string()))? + .map(|commits| { + XrpcResponse(list_commits::ListCommitsOutput { + commits, + extra_data: Default::default(), + }) + }) +} + +fn resolve_ref_ranges( + repo: &gix::Repository, + ranges: &[SmolStr], +) -> Result, XrpcError> { + let peel = |id| -> anyhow::Result { Ok(repo.find_object(id)?.peel_to_commit()?.id) }; + ranges + .iter() + .map(|range| { + let resolve = || -> anyhow::Result { + Ok(match repo.rev_parse(range.as_bytes().as_bstr())?.detach() { + RevSpec::Include(id) => peel(id)?.to_smolstr(), + RevSpec::Range { from, to } => format!("{}..{}", peel(from)?, peel(to)?).into(), + spec => anyhow::bail!("The spec isn't currently supported: {spec:?}"), + }) + }; + resolve().map_err(|e| XrpcError::RefNotFound { + rev: range.to_string(), + detail: e.to_string(), + }) + }) + .collect() +} + +#[axum::debug_handler] +pub(crate) async fn list_commits_by_ref( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + let repo = open_repos(&state, std::slice::from_ref(&args.repo))?; + let stats = state.stats.clone(); + crate::metrics::blocking(BlockingKind::Git, move || { + let ranges = resolve_ref_ranges(&repo, args.ranges.as_deref().unwrap_or_default())?; + list_commits_inner( + &repo, + &stats, + list_commits::ListCommits { + repos: vec![args.repo], + ranges: Some(ranges), + limit: args.limit, + skip: args.skip, + include_stats: args.include_stats, + }, + ) + }) + .await + .map_err(|e| XrpcError::Internal(e.to_string()))? + .map(|commits| { + XrpcResponse(list_commits_by_ref::ListCommitsByRefOutput { + commits, + extra_data: Default::default(), + }) + }) +} + +#[axum::debug_handler] +pub(crate) async fn merge_check( + State(state): State, + ExtractXrpc(params): ExtractXrpc, +) -> Result, XrpcError> { + let span = tracing::Span::current(); + span.record("repo", params.target_repo.as_str()); + span.record("base_repo", params.source_repo.as_str()); + let scratch = state + .layout + .open_scratch(&[¶ms.target_repo, ¶ms.source_repo])?; + let target = find_commit(&scratch, params.target_commit.as_ref())?; + let source = find_commit(&scratch, params.source_commit.as_ref())?; + + crate::metrics::blocking(BlockingKind::Git, move || { + crate::merge::merge_check(&scratch, target, source) + }) + .await + .map_err(|e| XrpcError::Internal(e.to_string()))? + .map(XrpcResponse) + .map_err(|e| XrpcError::Internal(e.to_string())) +} + +#[axum::debug_handler] +pub(crate) async fn merge_commit( + State(state): State, + ExtractAnyServiceAuth(auth, token): ExtractAnyServiceAuth, + ExtractXrpc(input): ExtractXrpc, +) -> Result, XrpcError> { + let span = tracing::Span::current(); + span.record("repo", input.target.repo.as_str()); + span.record("base_repo", input.source.repo.as_str()); + require_push_token(&auth)?; + + // 1. validate inputs + let merge_style = MergeStyle::parse(input.style.as_str()).ok_or_else(|| { + XrpcError::InvalidRequest(format!("unknown merge style: {:?}", input.style.as_str())) + })?; + let target_branch = head_ref(input.target.branch.as_str())?; + + // 2. load target & source repos as scratch repo. + let scratch = state + .layout + .open_scratch(&[&input.target.repo, &input.source.repo])?; + let source_commit_id = find_commit(&scratch, input.source.commit.as_str())?; + let target_commit_id = find_commit(&scratch, input.target.commit.as_str())?; + + // TODO(boltless): check knot reachability here + + let mut hidden = vec![target_commit_id]; + if input.source.repo == input.target.repo { + hidden.push(source_commit_id); + } + + let remote = push_remote(&state, &input.target.repo, &auth).await?; + + let http = state.http.clone(); + let committer = gix::actor::Signature { + name: COMMITTER_NAME.into(), + email: COMMITTER_EMAIL.into(), + time: gix::date::Time::new(chrono::Utc::now().timestamp(), 0), + }; + // NOTE: gix's async traits doesn't implement `Send`, so we should basically do a blocking IO + // while pushing. Not using `gix_transport::blocking_io` because that one is for one-shot and + // loose connection between handshake & git-receive-pack calls. + // TODO(boltless): implement `gix_transport::async_io_send` or something like that. + let runtime = tokio::runtime::Handle::current(); + let (merged, statuses) = crate::metrics::blocking(BlockingKind::Transport, move || { + // 3. merge + let merged = match merge_style { + MergeStyle::Rebase => scratch.rebase(source_commit_id, target_commit_id, committer), + // TODO(post-1.0): support more merge strategies + } + .map_err(|e| match e { + MergeError::Conflicts(paths) => XrpcError::MergeConflict(paths), + _ => XrpcError::Internal(e.to_string()), + })?; + + // 4. push the merged tip to the target repository + let mut transport = crate::git_transport::HttpTransport::new(http, remote.clone(), token); + let mut session = runtime + .block_on(gix_receive_pack::Session::handshake( + &mut transport, + Some(AGENT), + )) + .map_err(|e| XrpcError::Internal(format!("{remote}: {e}")))?; + + let pack = build_pack(&scratch, session.advertised(), merged, hidden) + .map_err(|e| XrpcError::Internal(format!("{e:#}")))?; + + let statuses = runtime + .block_on(session.push( + false, + &[gix_receive_pack::Update { + name: target_branch, + expected_old: target_commit_id, + new: merged, + }], + pack.as_deref(), + gix_receive_pack::Options { + atomic: true, + push_options: &[], + quiet: false, + }, + &mut gix::progress::Discard, + &gix::interrupt::IS_INTERRUPTED, + )) + .map_err(|e| XrpcError::Internal(format!("{remote}: {e}")))?; + Ok::<_, XrpcError>((merged, statuses)) + }) + .await + .map_err(|e| XrpcError::Internal(e.to_string()))??; + + if let Some(reason) = statuses.iter().find_map(|s| s.status.as_ref().err()) { + return Err(XrpcError::PushRejected(reason.to_string())); + } + + info!( + pusher = %auth.iss.as_str(), + repo = %input.target.repo.as_str(), + branch = %input.target.branch.as_str(), + commit = %merged, + "merged" + ); + Ok(XrpcResponse(())) +} + +#[axum::debug_handler] +pub(crate) async fn delete_branch( + State(state): State, + ExtractAnyServiceAuth(auth, token): ExtractAnyServiceAuth, + ExtractXrpc(input): ExtractXrpc, +) -> Result, XrpcError> { + tracing::Span::current().record("repo", input.repo.as_str()); + require_push_token(&auth)?; + let refname = head_ref(input.branch.as_str())?; + + let expected_old = { + let repo = state + .layout + .open(&input.repo) + .map_err(|e| XrpcError::RepoNotFound { + detail: e.to_string(), + })?; + let reference = repo + .try_find_reference(refname.as_ref().as_partial_name()) + .map_err(|e| XrpcError::Internal(e.to_string()))? + .ok_or_else(|| XrpcError::RefNotFound { + rev: input.branch.to_string(), + detail: "the mirror holds no such branch".to_owned(), + })?; + // Not peeled: `expected_old` is the value the knot holds for the ref itself, and peeling an + // annotated tag would name something the knot never wrote there. + reference + .target() + .try_id() + .ok_or_else(|| { + XrpcError::InvalidRequest(format!( + "{:?} is a symbolic ref, not a branch", + input.branch.as_str() + )) + })? + .to_owned() + }; + + let remote = push_remote(&state, &input.repo, &auth).await?; + let http = state.http.clone(); + // NOTE: see note from `merge_commit()`. + // TODO(boltless): implement `gix_transport::async_io_send` or something like that. + let runtime = tokio::runtime::Handle::current(); + let statuses = crate::metrics::blocking(BlockingKind::Transport, move || { + let mut transport = crate::git_transport::HttpTransport::new(http, remote.clone(), token); + let mut session = runtime + .block_on(gix_receive_pack::Session::handshake( + &mut transport, + Some(AGENT), + )) + .map_err(|e| XrpcError::Internal(format!("{remote}: {e}")))?; + + runtime + .block_on(session.push( + false, + &[gix_receive_pack::Update { + name: refname, + expected_old, + new: gix::ObjectId::null(expected_old.kind()), + }], + None, + gix_receive_pack::Options { + atomic: true, + push_options: &[], + quiet: false, + }, + &mut gix::progress::Discard, + &gix::interrupt::IS_INTERRUPTED, + )) + .map_err(|e| XrpcError::Internal(format!("{remote}: {e}"))) + }) + .await + .map_err(|e| XrpcError::Internal(e.to_string()))??; + + if let Some(reason) = statuses.iter().find_map(|s| s.status.as_ref().err()) { + return Err(XrpcError::PushRejected(reason.to_string())); + } + + info!( + pusher = %auth.iss.as_str(), + repo = %input.repo.as_str(), + branch = %input.branch.as_str(), + "deleted branch" + ); + Ok(XrpcResponse(())) +} + +#[axum::debug_handler] +pub(crate) async fn sync_fork( + State(state): State, + ExtractAnyServiceAuth(auth, token): ExtractAnyServiceAuth, + ExtractXrpc(input): ExtractXrpc, +) -> Result, XrpcError> { + let span = tracing::Span::current(); + span.record("repo", input.repo.as_str()); + span.record("base_repo", input.source.as_str()); + require_push_token(&auth)?; + let refname = head_ref(input.branch.as_str())?; + + let tip = { + let source = state + .layout + .open(&input.source) + .map_err(|e| XrpcError::RepoNotFound { + detail: e.to_string(), + })?; + source + .try_find_reference(refname.as_ref().as_partial_name()) + .map_err(|e| XrpcError::Internal(e.to_string()))? + .ok_or_else(|| XrpcError::RefNotFound { + rev: input.branch.to_string(), + detail: "the mirror holds no such branch on the upstream".to_owned(), + })? + .peel_to_commit() + .map_err(|e| XrpcError::Internal(e.to_string()))? + .id + }; + let scratch = state.layout.open_scratch(&[&input.repo, &input.source])?; + + let remote = push_remote(&state, &input.repo, &auth).await?; + let http = state.http.clone(); + // NOTE: see note from `merge_commit()`. + // TODO(boltless): implement `gix_transport::async_io_send` or something like that. + let runtime = tokio::runtime::Handle::current(); + let statuses = crate::metrics::blocking(BlockingKind::Transport, move || { + let mut transport = crate::git_transport::HttpTransport::new(http, remote.clone(), token); + let mut session = runtime + .block_on(gix_receive_pack::Session::handshake( + &mut transport, + Some(AGENT), + )) + .map_err(|e| XrpcError::Internal(format!("{remote}: {e}")))?; + + // Compare against what the knot holds, not the mirror, which may lag behind it. + let expected_old = session + .advertised() + .remote_refs + .get(&refname) + .copied() + .unwrap_or_else(|| gix::ObjectId::null(tip.kind())); + if expected_old == tip { + return Ok(Vec::new()); + } + + let pack = build_pack(&scratch, session.advertised(), tip, Vec::new()) + .map_err(|e| XrpcError::Internal(format!("{e:#}")))?; + + runtime + .block_on(session.push( + false, + &[gix_receive_pack::Update { + name: refname, + expected_old, + new: tip, + }], + pack.as_deref(), + gix_receive_pack::Options { + atomic: true, + push_options: &[], + quiet: false, + }, + &mut gix::progress::Discard, + &gix::interrupt::IS_INTERRUPTED, + )) + .map_err(|e| XrpcError::Internal(format!("{remote}: {e}"))) + }) + .await + .map_err(|e| XrpcError::Internal(e.to_string()))??; + + if let Some(reason) = statuses.iter().find_map(|s| s.status.as_ref().err()) { + return Err(XrpcError::PushRejected(reason.to_string())); + } + + info!( + pusher = %auth.iss.as_str(), + repo = %input.repo.as_str(), + source = %input.source.as_str(), + branch = %input.branch.as_str(), + commit = %tip, + "synced fork" + ); + Ok(XrpcResponse(())) +} + +fn require_push_token(auth: &ServiceAuthClaims) -> Result<(), XrpcError> { + if auth + .lxm + .as_ref() + .is_none_or(|lxm| lxm.as_str() != PUSH_NSID) + { + return Err(XrpcError::Unauthorized(format!( + "the token is bound to {:?}, which is not the push method", + auth.lxm.as_ref().map(|lxm| lxm.as_str()) + ))); + } + Ok(()) +} + +fn head_ref(branch: &str) -> Result { + gix::refs::FullName::try_from(format!("refs/heads/{branch}")) + .map_err(|e| XrpcError::InvalidRequest(format!("bad branch name {branch:?}: {e}"))) +} + +async fn push_remote( + state: &AppState, + repo: &Did, + auth: &ServiceAuthClaims, +) -> Result { + let did = repo.as_str(); + let document = state + .resolver() + .resolve_did_doc_owned(repo) + .await + .map_err(|e| XrpcError::Internal(format!("could not resolve {did}: {e}")))?; + let endpoint = document + .knot_endpoint() + // TODO(post-1.0): remove legacy `#atproto_pds` fallback + .or_else(|| document.pds_endpoint()) + .ok_or_else(|| XrpcError::InvalidRequest(format!("{did} declares no knot service")))?; + let knot = state + .knot_policy + .safe_knot_host(endpoint.as_str()) + .map_err(|_reason| { + XrpcError::InvalidRequest(format!("{did} declares invalid knot service endpoint")) + })?; + + let audience = knot_service_did(&knot); + if auth.aud.audience().as_str() != audience { + return Err(XrpcError::Unauthorized(format!( + "the token is audienced to {}, but {did} is hosted on {audience}", + auth.aud.as_str() + ))); + } + + Ok(knot.repo_url(did)) +} + +fn knot_service_did(knot: &KnotHost) -> String { + let url = knot.url(); + match url.port() { + Some(port) => format!("did:web:{}%3A{port}", url.host_str().unwrap_or_default()), + None => format!("did:web:{}", url.host_str().unwrap_or_default()), + } +} + +fn build_pack( + scratch: &knotmirror_git::scratch::TempRepository, + advertised: &gix_receive_pack::ReceivePackAdvertisement, + new_commit_id: gix::ObjectId, + hidden: Vec, +) -> anyhow::Result>> { + use gix::parallel::{InOrderIter, reduce::Finalize as _}; + use gix::progress::Discard; + use gix_pack::data::{Version, output}; + + const CHUNK_SIZE: usize = 50; + + anyhow::ensure!( + advertised.capabilities.contains("ofs-delta"), + "the knot does not read offset deltas" + ); + let object_hash = advertised.object_hash()?; + anyhow::ensure!( + object_hash == scratch.object_hash(), + "the knot speaks {object_hash} but the mirror is {}", + scratch.object_hash() + ); + + let mut objects = scratch.objects.clone().into_inner(); + objects.prevent_pack_unload(); + + let haves = hidden + .into_iter() + // TODO: chain synced ref tips in local mirror + .chain(advertised.remote_refs.values().copied()) + .chain(advertised.advertised_haves.iter().copied()); + let (counts, outcome) = output::count::reachable( + objects.clone(), + [new_commit_id], + haves, + &Discard, + &gix::interrupt::IS_INTERRUPTED, + )?; + info!( + commits = outcome.commits, + boundary = outcome.boundary, + objects = outcome.objects, + ignored_haves = outcome.ignored_haves, + "counted pack" + ); + if counts.is_empty() { + return Ok(None); + } + + let num_entries = u32::try_from(counts.len()) + .map_err(|_| anyhow::anyhow!("{} objects do not fit into a pack", counts.len()))?; + let counted = output::entry::iter_from_counts( + counts, + objects, + Box::new(Discard), + output::entry::iter_from_counts::Options { + // TODO(boltless): get thread limit like knot does + thread_limit: Some(1), + mode: output::entry::iter_from_counts::Mode::PackCopyAndBaseObjects, + allow_thin_pack: false, + chunk_size: CHUNK_SIZE, + version: Version::V2, + }, + ); + + let mut in_order = InOrderIter::from(counted); + let mut pack = Vec::new(); + let mut writer = output::bytes::FromEntriesIter::new( + in_order.by_ref(), + &mut pack, + num_entries, + Version::V2, + object_hash, + ); + writer.try_fold((), |(), written| written.map(|_| ()))?; + + let stats = in_order.inner.finalize()?; + anyhow::ensure!( + stats.missing_objects == 0, + "{} object(s) could not be found, which would have produced a corrupt pack", + stats.missing_objects + ); + + Ok(Some(pack)) +} + +#[cfg(test)] +mod diff_tests { + use super::*; + use jacquard_common::types::did::Did; + use knotmirror_git::layout::Layout; + use std::path::Path; + use std::process::Command; + + const BASE_DID: &str = "did:plc:upstream"; + const HEAD_DID: &str = "did:plc:fork"; + + fn git(dir: &Path, args: &[&str]) { + let status = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "t") + .env("GIT_AUTHOR_EMAIL", "t@t") + .env("GIT_COMMITTER_NAME", "t") + .env("GIT_COMMITTER_EMAIL", "t@t") + .status() + .expect("run git"); + assert!(status.success(), "git {args:?} failed"); + } + + fn rev_parse(dir: &Path, rev: &str) -> String { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(dir) + .output() + .expect("rev-parse"); + String::from_utf8(out.stdout).unwrap().trim().to_owned() + } + + /// The real fork shape: the base tip lives only in the upstream mirror and the head tip only + /// in the fork's, with a shared ancestor. Returns `(repo_base, base_commit, head_commit)`. + fn fork_fixture() -> (tempfile::TempDir, String, String) { + let root = tempfile::tempdir().unwrap(); + let repo_base = root.path().join("repos"); + std::fs::create_dir_all(&repo_base).unwrap(); + + let upstream = root.path().join("upstream"); + std::fs::create_dir_all(&upstream).unwrap(); + git(&upstream, &["init", "-q", "-b", "main"]); + std::fs::write(upstream.join("a.txt"), "line1\nline2\nline3\n").unwrap(); + std::fs::write(upstream.join("b.txt"), "keep\n").unwrap(); + git(&upstream, &["add", "."]); + git(&upstream, &["commit", "-q", "-m", "shared ancestor"]); + + // Fork before upstream moves on, so neither tip is reachable from the other. + let fork = root.path().join("fork"); + git( + root.path(), + &[ + "clone", + "-q", + upstream.to_str().unwrap(), + fork.to_str().unwrap(), + ], + ); + + std::fs::write(upstream.join("upstream.txt"), "theirs\n").unwrap(); + git(&upstream, &["add", "."]); + git(&upstream, &["commit", "-q", "-m", "upstream only"]); + let base_commit = rev_parse(&upstream, "HEAD"); + + std::fs::write(fork.join("a.txt"), "line1\nCHANGED\nline3\n").unwrap(); + git(&fork, &["mv", "b.txt", "c.txt"]); + git(&fork, &["add", "-A"]); + git(&fork, &["commit", "-q", "-m", "fork only"]); + let head_commit = rev_parse(&fork, "HEAD"); + + for (did, src) in [(BASE_DID, &upstream), (HEAD_DID, &fork)] { + git( + root.path(), + &[ + "clone", + "-q", + "--bare", + src.to_str().unwrap(), + repo_base.join(did).to_str().unwrap(), + ], + ); + } + + (root, base_commit, head_commit) + } + + fn diff_fixture( + root: &Path, + base_commit: &str, + head_commit: &str, + ) -> Result, XrpcError> { + diff_fixture_within(root, base_commit, head_commit, DiffLimits::starting_now()) + } + + fn diff_fixture_within( + root: &Path, + base_commit: &str, + head_commit: &str, + limits: DiffLimits, + ) -> Result, XrpcError> { + let repo_base = root.join("repos"); + let scratch = Layout::new(repo_base) + .open_scratch(&[&Did::raw(HEAD_DID.into()), &Did::raw(BASE_DID.into())])?; + let base = find_diff_base(&scratch, base_commit)?; + let head = find_commit(&scratch, head_commit)?; + get_diff_inner(&scratch, base, head, limits) + } + + fn scratch(root: &Path, dids: &[&str]) -> knotmirror_git::scratch::TempRepository { + let dids: Vec = dids.iter().map(|did| Did::raw((*did).into())).collect(); + Layout::new(root.join("repos")) + .open_scratch(&dids.iter().collect::>()) + .unwrap() + } + + fn list(repo: &gix::Repository, range: &str) -> Result, XrpcError> { + list_commits_inner( + repo, + &diff::StatCache::default(), + list_commits::ListCommits { + repos: vec![Did::raw(HEAD_DID.into()), Did::raw(BASE_DID.into())], + ranges: Some(vec![range.into()]), + include_stats: None, + limit: None, + skip: None, + }, + ) + } + + #[test] + fn a_cross_repo_range_lists_only_the_fork_commits() { + let (root, base, head) = fork_fixture(); + let repo = scratch(root.path(), &[HEAD_DID, BASE_DID]); + + let commits = list(&repo, &format!("{base}..{head}")).unwrap(); + + assert_eq!(commits.len(), 1); + assert_eq!(commits[0].oid, head); + } + + #[test] + fn a_range_needs_both_commits_in_the_given_repos() { + let (root, base, head) = fork_fixture(); + let repo = scratch(root.path(), &[HEAD_DID]); + + assert!(matches!( + list(&repo, &format!("{base}..{head}")), + Err(XrpcError::RevisionNotFound { .. }) + )); + } + + #[test] + fn ref_names_are_not_resolved() { + let (root, _base, _head) = fork_fixture(); + let repo = scratch(root.path(), &[HEAD_DID, BASE_DID]); + + assert!(matches!( + list(&repo, "main..main"), + Err(XrpcError::InvalidRequest(_)) + )); + } + + #[test] + fn a_cross_repo_merge_base_is_the_shared_ancestor() { + let (root, base, head) = fork_fixture(); + let repo = scratch(root.path(), &[HEAD_DID, BASE_DID]); + let shared = rev_parse(&root.path().join("upstream"), "HEAD~1"); + + let commit = crate::routes::reads::merge_base_inner(&repo, &base, &head).unwrap(); + + assert_eq!(commit, shared); + } + + fn empty_tree() -> String { + gix::ObjectId::empty_tree(gix::hash::Kind::Sha1).to_string() + } + + fn paths(diffs: &[sh_tangled::git::FileDiff]) -> Vec { + let mut paths: Vec = diffs.iter().map(|d| d.rhs_src.path.to_string()).collect(); + paths.sort(); + paths + } + + #[test] + fn a_root_commit_diffs_against_the_empty_tree() { + let (root, _base, _head) = fork_fixture(); + let root_commit = rev_parse(&root.path().join("upstream"), "HEAD~1"); + + let diffs = diff_fixture(root.path(), &empty_tree(), &root_commit).unwrap(); + + assert_eq!(paths(&diffs), ["a.txt", "b.txt"]); + let null = gix::ObjectId::null(gix::hash::Kind::Sha1).to_string(); + for diff in &diffs { + assert_eq!(diff.lhs_src.oid.to_string(), null); + } + + let a = diffs.iter().find(|d| d.rhs_src.path == "a.txt").unwrap(); + assert_eq!(a.hunks.len(), 1); + assert!(a.hunks[0].novel_lhs.is_empty()); + assert_eq!(a.hunks[0].novel_rhs, [0, 1, 2]); + } + + #[test] + fn the_empty_tree_base_skips_the_merge_base_and_reports_the_whole_head() { + let (root, base, _head) = fork_fixture(); + + let diffs = diff_fixture(root.path(), &empty_tree(), &base).unwrap(); + + assert_eq!(paths(&diffs), ["a.txt", "b.txt", "upstream.txt"]); + } + + #[test] + fn a_cross_repo_diff_reports_only_what_the_head_side_changed() { + let (root, base, head) = fork_fixture(); + let diffs = diff_fixture(root.path(), &base, &head).unwrap(); + + let mut paths: Vec = diffs + .iter() + .flat_map(|d| [d.lhs_src.path.to_string(), d.rhs_src.path.to_string()]) + .collect(); + paths.sort(); + paths.dedup(); + // `upstream.txt` is on the base side only: three-dot semantics exclude it. + assert_eq!(paths, ["a.txt", "b.txt", "c.txt"]); + + let a = diffs + .iter() + .find(|d| d.rhs_src.path == "a.txt") + .expect("a.txt is in the diff"); + // Only line 2 (0-based: 1) changed. + assert_eq!(a.hunks.len(), 1); + assert_eq!(a.hunks[0].novel_lhs, [1]); + assert_eq!(a.hunks[0].novel_rhs, [1]); + assert_eq!(a.hunks[0].lines, { + vec![sh_tangled::git::LinePair { + lhs: Some(1), + rhs: Some(1), + extra_data: Default::default(), + }] + }); + // "line2\n" -> "CHANGED\n" is two bytes longer. + assert_eq!( + a.has_byte_changes, + Some(sh_tangled::git::ByteChanges { + lhs: 18, + rhs: 20, + extra_data: Default::default(), + }) + ); + } + + #[test] + fn a_diff_with_more_files_than_the_limit_is_refused() { + let (root, base, _head) = fork_fixture(); + let within = |files| DiffLimits { + files, + ..DiffLimits::starting_now() + }; + + assert!(matches!( + diff_fixture_within(root.path(), &empty_tree(), &base, within(2)), + Err(XrpcError::DiffTooLarge { limit: 2 }) + )); + assert_eq!( + diff_fixture_within(root.path(), &empty_tree(), &base, within(3)) + .unwrap() + .len(), + 3 + ); + } + + #[test] + fn a_diff_still_running_at_its_deadline_is_refused() { + let (root, base, _head) = fork_fixture(); + let late = DiffLimits { + deadline: Instant::now() - Duration::from_secs(1), + ..DiffLimits::starting_now() + }; + + assert!(matches!( + diff_fixture_within(root.path(), &empty_tree(), &base, late), + Err(XrpcError::DiffTimeout) + )); + } + + #[test] + fn a_malformed_sha_is_a_client_error_and_an_absent_one_is_a_miss() { + let (root, base, head) = fork_fixture(); + + assert!(matches!( + diff_fixture(root.path(), &base, "not-a-sha"), + Err(XrpcError::InvalidRequest(_)) + )); + assert!(matches!( + diff_fixture(root.path(), &base, &"0".repeat(head.len())), + Err(XrpcError::RevisionNotFound { .. }) + )); + } +} + +#[cfg(test)] +mod stat_tests { + use super::*; + use std::io::Write as _; + use std::path::Path; + use std::process::{Command, Stdio}; + + fn git(dir: &Path, args: &[&str]) -> String { + git_stdin(dir, args, b"") + } + + fn git_stdin(dir: &Path, args: &[&str], stdin: &[u8]) -> String { + let mut child = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "t") + .env("GIT_AUTHOR_EMAIL", "t@t") + .env("GIT_COMMITTER_NAME", "t") + .env("GIT_COMMITTER_EMAIL", "t@t") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(stdin).unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git {args:?} failed"); + String::from_utf8(out.stdout).unwrap().trim().to_owned() + } + + fn fixture() -> (tempfile::TempDir, gix::Repository) { + let root = tempfile::tempdir().unwrap(); + let work = root.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + git(&work, &["init", "-q", "-b", "main"]); + std::fs::write(work.join("a.txt"), "line1\nline2\nline3\n").unwrap(); + std::fs::write(work.join("b.txt"), "keep\n").unwrap(); + git(&work, &["add", "."]); + git(&work, &["commit", "-q", "-m", "root"]); + std::fs::write(work.join("a.txt"), "line1\nCHANGED\nline3\n").unwrap(); + git(&work, &["commit", "-q", "-am", "edit"]); + git(&work, &["mv", "b.txt", "c.txt"]); + std::fs::write(work.join("c.txt"), "keep\nmore\n").unwrap(); + git(&work, &["add", "-A"]); + git(&work, &["commit", "-q", "-m", "rename"]); + let repo = gix::open(&work).unwrap(); + (root, repo) + } + + fn list( + repo: &gix::Repository, + stats: &diff::StatCache, + range: &str, + include_stats: bool, + ) -> Vec { + list_commits_inner( + repo, + stats, + list_commits::ListCommits { + repos: vec![Did::raw("did:plc:log".into())], + ranges: Some(resolve_ref_ranges(repo, &[range.into()]).unwrap()), + include_stats: Some(include_stats), + limit: None, + skip: None, + }, + ) + .unwrap() + } + + fn counts(commits: &[list_commits::Commit]) -> Vec> { + commits + .iter() + .map(|commit| { + commit + .stat + .as_ref() + .map(|s| (s.insertions, s.deletions, s.files_changed)) + }) + .collect() + } + + #[test] + fn ref_ranges_resolve_to_commit_ids() { + let (_root, repo) = fixture(); + let head = repo.head_id().unwrap().detach(); + let parent = repo.rev_parse_single("HEAD~1").unwrap().detach(); + + assert_eq!( + resolve_ref_ranges(&repo, &["main".into(), "HEAD~1..main".into()]).unwrap(), + [head.to_smolstr(), format!("{parent}..{head}").into()] + ); + assert!(matches!( + resolve_ref_ranges(&repo, &["nope".into()]), + Err(XrpcError::RefNotFound { .. }) + )); + } + + #[test] + fn stats_are_absent_unless_asked_for() { + let (_root, repo) = fixture(); + let stats = diff::StatCache::default(); + assert_eq!( + counts(&list(&repo, &stats, "HEAD", false)), + [None, None, None] + ); + } + + #[test] + fn each_commit_is_measured_against_its_first_parent() { + let (_root, repo) = fixture(); + let stats = diff::StatCache::default(); + assert_eq!( + counts(&list(&repo, &stats, "HEAD", true)), + [Some((1, 0, 1)), Some((1, 1, 1)), Some((4, 0, 2))] + ); + } + + #[test] + fn a_measured_commit_is_served_from_the_cache() { + let (_root, repo) = fixture(); + let stats = diff::StatCache::default(); + let head = repo.head_commit().unwrap(); + stats.commits.insert( + head.id, + diff::Stat { + insertions: 7, + deletions: 7, + files_changed: 7, + }, + ); + assert_eq!( + counts(&list(&repo, &stats, "HEAD", true))[0], + Some((7, 7, 7)) + ); + } +} + +#[cfg(test)] +mod merge_commit_tests { + use super::*; + use crate::KnotPolicy; + use jacquard_common::types::did::Did; + use std::io::Write as _; + use std::path::{Path, PathBuf}; + use std::process::{Command, Stdio}; + + use knotmirror_git::layout::Layout; + + const TARGET_DID: &str = "did:plc:upstream"; + const SOURCE_DID: &str = "did:plc:fork"; + const CHANGE_ID: &str = "zzzzkzkzkzkzkzkzkzkzkzkzkzkz"; + + fn committer() -> gix::actor::Signature { + gix::actor::Signature { + name: COMMITTER_NAME.into(), + email: COMMITTER_EMAIL.into(), + time: gix::date::Time::new(chrono::Utc::now().timestamp(), 0), + } + } + + fn git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "t") + .env("GIT_AUTHOR_EMAIL", "t@t") + .env("GIT_COMMITTER_NAME", "t") + .env("GIT_COMMITTER_EMAIL", "t@t") + .output() + .expect("run git"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap().trim().to_owned() + } + + /// `git hash-object` and friends need the object on stdin. + fn git_stdin(dir: &Path, args: &[&str], stdin: &[u8]) -> String { + let mut child = Command::new("git") + .args(args) + .current_dir(dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.take().unwrap().write_all(stdin).unwrap(); + let out = child.wait_with_output().expect("run git"); + assert!(out.status.success(), "git {args:?} failed"); + String::from_utf8(out.stdout).unwrap().trim().to_owned() + } + + /// Loose objects under a bare repo's `objects` dir, ignoring `info` and `pack`. + fn count_loose_objects(objects: &Path) -> usize { + let mut n = 0; + for shard in std::fs::read_dir(objects).unwrap() { + let shard = shard.unwrap(); + let name = shard.file_name(); + if name == "info" || name == "pack" || !shard.file_type().unwrap().is_dir() { + continue; + } + n += std::fs::read_dir(shard.path()).unwrap().count(); + } + n + } + + struct Fixture { + root: tempfile::TempDir, + layout: Layout, + /// The target branch tip, living only in the target mirror. + onto: gix::ObjectId, + /// The source tip, living only in the source mirror. + source: gix::ObjectId, + /// The tree real `git rebase` produces for the same replay. + expected_tree: gix::ObjectId, + } + + impl Fixture { + fn objects(&self, did: &str) -> PathBuf { + self.layout.repo_path(&Did::raw(did.into())).join("objects") + } + } + + /// Two mirrors sharing a base commit, the real knotmirror shape: the target tip is only in the + /// target's mirror and the source tip only in the source's. The source's first commit carries a + /// hand-written `change-id`, which `git commit` cannot produce. + fn two_mirrors(target_a: &str, source_a: &str) -> Fixture { + let root = tempfile::tempdir().unwrap(); + let repo_base = root.path().join("repos"); + std::fs::create_dir_all(&repo_base).unwrap(); + + let target = root.path().join("target"); + std::fs::create_dir_all(&target).unwrap(); + git(&target, &["init", "-q", "-b", "main"]); + std::fs::write(target.join("a.txt"), "line1\nline2\nline3\n").unwrap(); + git(&target, &["add", "."]); + git(&target, &["commit", "-q", "-m", "shared ancestor"]); + + // Fork before the target moves on, so neither tip is reachable from the other. + let source = root.path().join("source"); + git( + root.path(), + &[ + "clone", + "-q", + target.to_str().unwrap(), + source.to_str().unwrap(), + ], + ); + + std::fs::write(target.join("a.txt"), target_a).unwrap(); + git(&target, &["add", "-A"]); + git(&target, &["commit", "-q", "-m", "target moves on"]); + + std::fs::write(source.join("a.txt"), source_a).unwrap(); + git(&source, &["add", "-A"]); + let tree = git(&source, &["write-tree"]); + let parent = git(&source, &["rev-parse", "HEAD"]); + let raw = format!( + "tree {tree}\nparent {parent}\n\ + author A 1700000060 +0000\n\ + committer C 1700000060 +0000\n\ + change-id {CHANGE_ID}\n\n\ + source edit with a change-id\n" + ); + let id = git_stdin( + &source, + &["hash-object", "-t", "commit", "-w", "--stdin"], + raw.as_bytes(), + ); + git(&source, &["update-ref", "refs/heads/main", &id]); + git(&source, &["reset", "-q", "--hard", "main"]); + std::fs::write(source.join("source.txt"), "mine\n").unwrap(); + git(&source, &["add", "-A"]); + git(&source, &["commit", "-q", "-m", "plain follow-up"]); + + let onto = git(&target, &["rev-parse", "HEAD"]); + let source_tip = git(&source, &["rev-parse", "HEAD"]); + + for (did, src) in [(TARGET_DID, &target), (SOURCE_DID, &source)] { + git( + root.path(), + &[ + "clone", + "-q", + "--bare", + src.to_str().unwrap(), + repo_base.join(did).to_str().unwrap(), + ], + ); + } + + // The baseline: what real `git rebase` makes of the same replay. Run after the mirrors are + // cloned, so it can't influence what they hold. + git(&source, &["remote", "add", "up", target.to_str().unwrap()]); + git(&source, &["fetch", "-q", "up"]); + let rebase = Command::new("git") + .args(["rebase", "-q", &onto]) + .current_dir(&source) + .env("GIT_AUTHOR_NAME", "t") + .env("GIT_AUTHOR_EMAIL", "t@t") + .env("GIT_COMMITTER_NAME", "t") + .env("GIT_COMMITTER_EMAIL", "t@t") + .output() + .expect("run git rebase"); + let expected_tree = if rebase.status.success() { + git(&source, &["rev-parse", "HEAD^{tree}"]) + } else { + // A conflicting fixture has no baseline tree; the test asserting the conflict ignores it. + onto.clone() + }; + + Fixture { + layout: Layout::new(repo_base), + onto: gix::ObjectId::from_hex(onto.as_bytes()).unwrap(), + source: gix::ObjectId::from_hex(source_tip.as_bytes()).unwrap(), + expected_tree: gix::ObjectId::from_hex(expected_tree.as_bytes()).unwrap(), + root, + } + } + + fn scratch(fixture: &Fixture) -> knotmirror_git::scratch::TempRepository { + fixture + .layout + .open_scratch(&[&Did::raw(TARGET_DID.into()), &Did::raw(SOURCE_DID.into())]) + .unwrap() + } + + #[test] + fn a_cross_mirror_rebase_matches_git_and_keeps_the_change_id() { + // The target edits the last line, the source the first: a clean three-way merge. + let fixture = two_mirrors("line1\nline2\nTARGET\n", "SOURCE\nline2\nline3\n"); + let before: Vec = [TARGET_DID, SOURCE_DID] + .iter() + .map(|did| count_loose_objects(&fixture.objects(did))) + .collect(); + + let repo = scratch(&fixture); + let tip = repo + .rebase(fixture.source, fixture.onto, committer()) + .expect("a clean rebase"); + assert_eq!( + repo.find_commit(tip).unwrap().tree_id().unwrap().detach(), + fixture.expected_tree, + "tree differs from what `git rebase` produces" + ); + + // The `change-id` rides along on the rewritten commit, which `git rebase` itself drops. + let replayed = repo.find_commit(tip).unwrap(); + let first = replayed.decode().unwrap().parents().next().unwrap(); + let first = repo.find_commit(first).unwrap(); + let decoded = first.decode().unwrap(); + assert_eq!( + decoded + .extra_headers() + .find("change-id") + .map(|v| v.to_string()) + .as_deref(), + Some(CHANGE_ID), + ); + // Author preserved, committer replaced by ours. + assert_eq!(decoded.author().unwrap().email, "a@example.com"); + assert_eq!(decoded.committer().unwrap().email, COMMITTER_EMAIL); + + // Nothing was written to either mirror: the replay lives in the scratch repo's memory. + for (did, before) in [TARGET_DID, SOURCE_DID].iter().zip(before) { + assert_eq!( + count_loose_objects(&fixture.objects(did)), + before, + "{did} mirror was mutated" + ); + } + drop(fixture.root); + } + + /// What a knot advertises for a target sitting at `onto`, plus a ref the mirror has never seen + /// — a branch pushed to the knot after the mirror last fetched. + fn advertisement(onto: gix::ObjectId) -> gix_receive_pack::ReceivePackAdvertisement { + let (capabilities, _) = gix_transport::client::Capabilities::from_bytes( + b"\0report-status delete-refs atomic ofs-delta side-band-64k", + ) + .unwrap(); + gix_receive_pack::ReceivePackAdvertisement { + server_protocol_version: gix_transport::Protocol::V1, + capabilities, + remote_refs: [ + ( + gix::refs::FullName::try_from("refs/heads/main").unwrap(), + onto, + ), + ( + gix::refs::FullName::try_from("refs/heads/elsewhere").unwrap(), + gix::ObjectId::from_hex(b"1111111111111111111111111111111111111111").unwrap(), + ), + ] + .into_iter() + .collect(), + advertised_haves: Vec::new(), + } + } + + #[test] + fn the_pack_carries_everything_the_knot_is_missing() { + let fixture = two_mirrors("line1\nline2\nTARGET\n", "SOURCE\nline2\nline3\n"); + let repo = scratch(&fixture); + let merged = repo + .rebase(fixture.source, fixture.onto, committer()) + .expect("a clean rebase"); + + let pack = build_pack( + &repo, + &advertisement(fixture.onto), + merged, + vec![fixture.onto], + ) + .expect("the pack is built despite an advertised ref the mirror lacks") + .expect("a rebase writes new commits, so there is a pack"); + + // A stand-in for the knot: it holds exactly what the target mirror holds, which is what its + // advertisement said. Anything the merged tip needs beyond that has to be in the pack. + let knot = fixture.root.path().join("knot"); + git( + fixture.root.path(), + &[ + "clone", + "-q", + "--bare", + fixture + .layout + .repo_path(&Did::raw(TARGET_DID.into())) + .to_str() + .unwrap(), + knot.to_str().unwrap(), + ], + ); + git_stdin(&knot, &["unpack-objects", "-q"], &pack); + git( + &knot, + &["update-ref", "refs/heads/main", &merged.to_string()], + ); + // Walks every object the branch reaches and fails on one that is not there. + git(&knot, &["fsck", "--connectivity-only", "--no-dangling"]); + assert_eq!( + git(&knot, &["rev-parse", "main^{tree}"]), + fixture.expected_tree.to_string(), + "the knot ends up with the tree `git rebase` produces" + ); + drop(fixture.root); + } + + #[test] + fn a_tip_the_knot_already_has_needs_no_pack() { + // What a fast-forward within one repository will look like: the update names a commit the + // knot advertised, so the walk comes up empty and there is nothing to send. + let fixture = two_mirrors("line1\nline2\nTARGET\n", "SOURCE\nline2\nline3\n"); + let repo = scratch(&fixture); + let pack = build_pack( + &repo, + &advertisement(fixture.onto), + fixture.onto, + vec![fixture.onto], + ) + .expect("the walk runs"); + assert!(pack.is_none(), "no objects, so no pack"); + drop(fixture.root); + } + + #[test] + fn merging_what_the_knot_has_sends_only_the_merge() { + let fixture = two_mirrors("line1\nline2\nTARGET\n", "SOURCE\nline2\nline3\n"); + let repo = scratch(&fixture); + let rebased = repo + .rebase(fixture.source, fixture.onto, committer()) + .expect("a clean rebase"); + let tree = repo.find_commit(rebased).unwrap().tree_id().unwrap(); + let sig = committer(); + let (mut committed, mut authored) = ( + gix::date::parse::TimeBuf::default(), + gix::date::parse::TimeBuf::default(), + ); + let merge = repo + .new_commit_as( + sig.to_ref(&mut committed), + sig.to_ref(&mut authored), + "merge", + tree, + [fixture.onto, fixture.source], + ) + .unwrap() + .id; + + let pack = build_pack( + &repo, + &advertisement(fixture.onto), + merge, + vec![fixture.onto, fixture.source], + ) + .unwrap() + .unwrap(); + // The merge commit, its root tree and the merged `a.txt`; `source.txt` is in a parent. + let entries = u32::from_be_bytes(pack[8..12].try_into().unwrap()); + assert_eq!(entries, 3); + drop(fixture.root); + } + + #[test] + fn a_knot_endpoint_becomes_a_base_url_and_only_a_public_https_one() { + let deployed = KnotPolicy { + allow_private: false, + require_https: true, + }; + let base = |endpoint| { + deployed + .safe_knot_host(endpoint) + .map(|host| host.url().to_string()) + }; + assert_eq!( + base("https://knot.example/repo/abc").ok().as_deref(), + Some("https://knot.example/"), + "the tagged repo path is not part of the knot's base" + ); + assert_eq!(base("http://knot.example"), Err("is not https")); + assert_eq!( + base("https://127.0.0.1/repo/abc"), + Err("is a private address") + ); + assert_eq!( + base("https://localhost/repo/abc"), + Err("is a private address") + ); + assert_eq!(base("not a url"), Err("is not a usable base URL")); + + // The local-development hatch: the same endpoints, minus both guards. + let dev = KnotPolicy { + allow_private: true, + require_https: false, + }; + let base = |endpoint| { + dev.safe_knot_host(endpoint) + .map(|host| host.url().to_string()) + }; + assert_eq!( + base("http://127.0.0.1:5555/repo/abc").ok().as_deref(), + Some("http://127.0.0.1:5555/") + ); + assert_eq!( + base("http://localhost:5555").ok().as_deref(), + Some("http://localhost:5555/") + ); + // Not a hatch for nonsense: parsing still has to succeed. + assert_eq!(base("not a url"), Err("is not a usable base URL")); + } + + /// The audience a push token has to name to be forwarded to this knot. The handler compares + /// `claims.aud.audience()` against exactly this string, so a token minted for one knot cannot + /// be spent at another. + #[test] + fn a_knot_is_addressed_by_the_did_web_of_its_authority() { + let dev = KnotPolicy { + allow_private: true, + require_https: false, + }; + let did = |endpoint| knot_service_did(&dev.safe_knot_host(endpoint).unwrap()); + + assert_eq!(did("https://knot.example/repo/abc"), "did:web:knot.example"); + // The port separator is percent-encoded, or it would read as a `did:web` path segment. + assert_eq!(did("http://localhost:5555"), "did:web:localhost%3A5555"); + // A default port is not part of the authority the appview mints from either. + assert_eq!(did("https://knot.example:443"), "did:web:knot.example"); + } + + #[test] + fn a_conflicting_rebase_names_the_path_and_writes_no_commit() { + // Both sides rewrite the same line. + let fixture = two_mirrors("line1\nTARGET\nline3\n", "line1\nSOURCE\nline3\n"); + let repo = scratch(&fixture); + let err = repo + .rebase(fixture.source, fixture.onto, committer()) + .expect_err("a conflict"); + assert!( + matches!(err, MergeError::Conflicts(ref paths) if *paths == ["a.txt"]), + "{err:?}" + ); + drop(fixture.root); + } +} + +#[cfg(test)] +mod ssh_signature_tests { + use super::*; + use gix::hash::Kind; + + const TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + + fn key() -> ssh_key::PrivateKey { + ssh_key::private::Ed25519Keypair::from_seed(&[7; 32]).into() + } + + /// `git commit -S` with `gpg.format=ssh`: sign everything but the header, then splice the + /// armored signature in after `committer` as a continued `header` + fn signed_commit(payload: &str, signed_as: &str, namespace: &str, header: &str) -> String { + let sig = ssh_key::SshSig::sign( + &key(), + namespace, + ssh_key::HashAlg::Sha512, + signed_as.as_bytes(), + ) + .unwrap() + .to_pem(ssh_key::LineEnding::LF) + .unwrap(); + let header = format!("{header} {}\n", sig.trim_end().replace('\n', "\n ")); + let (head, message) = payload.split_once("\n\n").unwrap(); + format!("{head}\n{header}\n{message}") + } + + fn payload(message: &str) -> String { + format!( + "tree {TREE}\nauthor a 1700000000 +0000\ncommitter a 1700000000 +0000\n\n{message}\n" + ) + } + + #[test] + fn verifies_against_the_embedded_key() { + let unsigned = payload("signed"); + for (kind, header) in [(Kind::Sha1, "gpgsig"), (Kind::Sha256, "gpgsig-sha256")] { + let raw = signed_commit(&unsigned, &unsigned, "git", header); + let got = ssh_signature_of(raw.as_bytes(), kind).unwrap(); + assert_eq!( + serde_json::to_value(&got).unwrap()["$type"], + "org.tangled.temp.git.listCommits#sshSignature" + ); + assert_eq!( + got.fingerprint, + key() + .public_key() + .fingerprint(ssh_key::HashAlg::Sha256) + .to_smolstr() + ); + } + } + + #[test] + fn rejects_tampered_wrong_namespace_wrong_header_and_unsigned() { + let unsigned = payload("signed"); + let check = |raw: String, kind| ssh_signature_of(raw.as_bytes(), kind); + assert!( + check( + signed_commit(&payload("tampered"), &unsigned, "git", "gpgsig"), + Kind::Sha1 + ) + .is_none() + ); + assert!( + check( + signed_commit(&unsigned, &unsigned, "file", "gpgsig"), + Kind::Sha1 + ) + .is_none() + ); + // a sha256 commit is signed under its own header, `gpgsig` alone doesn't count + assert!( + check( + signed_commit(&unsigned, &unsigned, "git", "gpgsig"), + Kind::Sha256 + ) + .is_none() + ); + assert!(check(unsigned, Kind::Sha1).is_none()); + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/routes/mod.rs b/knotmirror/crates/knotmirror-xrpc/src/routes/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..7d03da7fa80953b8738f2fd4a80c5c46c3793ef9 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/routes/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod activity; +pub(crate) mod cob; +pub mod git; +pub(crate) mod reads; diff --git a/knotmirror/crates/knotmirror-xrpc/src/routes/reads.rs b/knotmirror/crates/knotmirror-xrpc/src/routes/reads.rs new file mode 100644 index 0000000000000000000000000000000000000000..f0d76e19c6bd9bac8a0cd9f1d2330891942c1a5d --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/routes/reads.rs @@ -0,0 +1,1467 @@ +use std::collections::HashMap; +use std::io::{BufWriter, Write}; +use std::time::{Duration, Instant}; + +use axum::body::Body; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode, header}; +use axum::response::{AppendHeaders, IntoResponse as _, Response}; +use gix::ObjectId; +use gix::bstr::{BString, ByteSlice as _}; +use gix::objs::tree::EntryKind; +use jacquard_axum::{ExtractXrpc, XrpcResponse}; +use jacquard_common::ToSmolStr; +use jacquard_common::deps::smol_str::SmolStr; +use jacquard_common::types::did::Did; +use lexicons::org_tangled; +use lexicons::org_tangled::temp::git::{ + format_patch, get_archive, get_blob, get_entry, get_merge_base, get_ref, get_tree, + list_branches, list_tags, +}; +use lexicons::sh_tangled::git::temp::{list_indexed_languages, list_languages}; +use sha2::{Digest as _, Sha256}; +use tracing::{error, warn}; + +use crate::routes::git::{GixSignature, allocation_limited, find_commit}; +use crate::{AppState, archive, error::XrpcError, last_commit, metrics::BlockingKind, sniff}; + +const IMMUTABLE: &str = "public, max-age=31536000, immutable"; +const MUTABLE: &str = "private, no-store"; + +const BLOB_ETAG_DOMAIN: &str = "knotmirror.blob.v1"; +const LARGE_BLOB: u64 = 1024 * 1024; + +const ARCHIVE_ETAG_DOMAIN: &str = "gitutil.archive.v1"; +const STREAM_CHUNK: usize = 64 * 1024; + +const PATCH_SIGNATURE: &str = concat!("knotmirror ", env!("CARGO_PKG_VERSION")); + +const LANGUAGES_BUDGET: Duration = Duration::from_secs(1); +const TOTAL_COMMITS_BUDGET: Duration = Duration::from_secs(1); +const TOTAL_COMMITS_OVER_BUDGET: i64 = 999; + +const DEFAULT_REFS_LIMIT: i64 = 50; +const MAX_REFS_LIMIT: i64 = 100; + +const DEFAULT_INDEXED_LANGUAGES_LIMIT: i64 = 500; +const MAX_INDEXED_LANGUAGES_LIMIT: i64 = 1000; +const SANDBOX_CSP: &str = "default-src 'none'; style-src 'unsafe-inline'; sandbox"; + +fn sandboxed_blob_headers(cache: &str, content_type: String, content_length: usize) -> HeaderMap { + let mut headers = HeaderMap::new(); + [ + (header::CACHE_CONTROL, cache.to_owned()), + (header::CONTENT_TYPE, content_type), + (header::CONTENT_LENGTH, content_length.to_string()), + (header::X_CONTENT_TYPE_OPTIONS, "nosniff".to_owned()), + (header::CONTENT_SECURITY_POLICY, SANDBOX_CSP.to_owned()), + ] + .into_iter() + .for_each(|(name, value)| { + headers.insert( + name, + value.parse().expect("blob header values we mint are ascii"), + ); + }); + headers +} + +fn pinned(refs: &[&str]) -> bool { + !refs.is_empty() + && refs + .iter() + .all(|r| ObjectId::from_hex(r.as_bytes()).is_ok()) +} + +fn cache_control(refs: &[&str]) -> &'static str { + if pinned(refs) { IMMUTABLE } else { MUTABLE } +} + +fn cache_header(value: &'static str) -> [(header::HeaderName, &'static str); 1] { + [(header::CACHE_CONTROL, value)] +} + +fn open(state: &AppState, repo: &Did) -> Result { + state + .layout + .open(repo) + .map_err(|e| XrpcError::RepoNotFound { + detail: e.to_string(), + }) +} + +pub(crate) enum Repo { + Plain(gix::Repository), + Scratch(knotmirror_git::scratch::TempRepository), +} + +impl std::ops::Deref for Repo { + type Target = gix::Repository; + fn deref(&self) -> &gix::Repository { + match self { + Self::Plain(repo) => repo, + Self::Scratch(repo) => repo, + } + } +} + +pub(crate) fn open_repos(state: &AppState, repos: &[Did]) -> Result { + let span = tracing::Span::current(); + match repos { + [repo] => { + span.record("repo", repo.as_str()); + open(state, repo).map(Repo::Plain) + } + [repo, other] => { + span.record("repo", repo.as_str()); + span.record("base_repo", other.as_str()); + Ok(Repo::Scratch(state.layout.open_scratch(&[repo, other])?)) + } + _ => Err(XrpcError::InvalidRequest( + "repos must name one or two repositories".to_owned(), + )), + } +} + +fn open_capped(state: &AppState, repo: &Did, limit: u64) -> Result { + gix::open_opts( + state.layout.repo_path(repo), + gix::open::Options::default() + .config_overrides([format!("gitoxide.objects.allocLimit={limit}")]), + ) + .map_err(|e| XrpcError::RepoNotFound { + detail: e.to_string(), + }) +} + +fn resolve_commit(repo: &gix::Repository, refspec: &str) -> Result { + if refspec.is_empty() || refspec.contains('\0') || refspec.starts_with('-') { + return Err(XrpcError::InvalidRequest(format!( + "invalid ref spec {refspec:?}" + ))); + } + let not_found = |detail: String| XrpcError::RefNotFound { + rev: refspec.to_owned(), + detail, + }; + let id = match repo.rev_parse_single(refspec.as_bytes()) { + Ok(id) => id, + Err(gix::revision::spec::parse::single::Error::RangedRev { spec }) => { + return Err(XrpcError::InvalidRequest(format!( + "ref must resolve to a single revision, got a range: {spec}" + ))); + } + Err(error) => return Err(not_found(error.to_string())), + }; + id.object() + .and_then(|object| object.peel_tags_to_end()) + .map_err(|e| not_found(e.to_string()))? + .try_into_commit() + .map(|commit| commit.id) + .map_err(|_| not_found("revision does not point at a commit".to_owned())) +} + +fn mode_of(kind: EntryKind) -> String { + format!("{:07o}", kind as u16) +} + +fn blob_size(repo: &gix::Repository, oid: ObjectId) -> Result { + repo.find_header(oid) + .map(|header| header.size()) + .map_err(|e| XrpcError::Internal(e.to_string())) +} + +fn hydrate(repo: &gix::Repository, oid: ObjectId) -> Result { + let internal = |e: &dyn std::fmt::Display| XrpcError::Internal(e.to_string()); + let commit = repo.find_commit(oid).map_err(|e| internal(&e))?; + let decoded = commit.decode().map_err(|e| internal(&e))?; + Ok(org_tangled::git::Commit { + hash: oid.to_smolstr(), + tree: decoded.tree.to_smolstr(), + author: GixSignature(decoded.author().map_err(|e| internal(&e))?) + .try_into() + .map_err(|e: anyhow::Error| internal(&e))?, + committer: GixSignature(decoded.committer().map_err(|e| internal(&e))?) + .try_into() + .map_err(|e: anyhow::Error| internal(&e))?, + message: decoded.message.to_smolstr(), + extra_data: Default::default(), + }) +} + +fn hydrate_all( + repo: &gix::Repository, + found: &HashMap, +) -> HashMap { + let mut commits: HashMap = HashMap::new(); + found + .iter() + .filter_map(|(name, oid)| { + let commit = match commits.get(oid) { + Some(commit) => commit.clone(), + None => { + let commit = hydrate(repo, *oid) + .inspect_err(|error| warn!(?error, "could not read last commit")) + .ok()?; + commits.insert(*oid, commit.clone()); + commit + } + }; + Some((name.clone(), commit)) + }) + .collect() +} + +async fn last_commits( + repo: gix::Repository, + head: ObjectId, + pathspec: String, + max_depth: usize, + show_trees: bool, +) -> Result, XrpcError> { + let deadline = Instant::now() + last_commit::BUDGET; + crate::metrics::blocking(BlockingKind::Git, move || { + let found = last_commit::walk( + &repo, + head, + &pathspec, + Some(max_depth), + show_trees, + deadline, + ); + hydrate_all(&repo, &found) + }) + .await + .map_err(|e| XrpcError::Internal(e.to_string())) +} +pub(crate) async fn get_merge_base( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result< + ( + [(header::HeaderName, &'static str); 1], + XrpcResponse, + ), + XrpcError, +> { + let repo = open_repos(&state, &args.repos)?; + let (base, head) = (args.base.to_string(), args.head.to_string()); + let commit = crate::metrics::blocking(BlockingKind::Git, move || { + merge_base_inner(&repo, &base, &head) + }) + .await + .map_err(|e| XrpcError::Internal(e.to_string()))??; + + Ok(( + cache_header(IMMUTABLE), + XrpcResponse(get_merge_base::GetMergeBaseOutput { + commit, + extra_data: Default::default(), + }), + )) +} + +pub(super) fn merge_base_inner( + repo: &gix::Repository, + base: &str, + head: &str, +) -> Result { + let base_id = find_commit(repo, base)?; + let head_id = find_commit(repo, head)?; + match repo.merge_base(head_id, base_id) { + Ok(id) => Ok(id.detach().to_smolstr()), + Err(gix::repository::merge_base::Error::NotFound { .. }) => Err(XrpcError::NoMergeBase { + base: base.to_owned(), + head: head.to_owned(), + }), + Err(error) => Err(XrpcError::Internal(error.to_string())), + } +} + +fn split_path(path: &str) -> (String, String) { + match path.rsplit_once('/') { + Some((dir, name)) => (dir.to_owned(), name.to_owned()), + None => (String::new(), path.to_owned()), + } +} + +fn submodule_at( + repo: &gix::Repository, + commit: ObjectId, + path: &str, +) -> Result, XrpcError> { + let Some(entry) = repo + .find_commit(commit) + .ok() + .and_then(|commit| commit.tree().ok()) + .and_then(|tree| tree.lookup_entry_by_path(".gitmodules").ok().flatten()) + else { + return Ok(None); + }; + if !entry.mode().is_blob() { + return Ok(None); + } + let object = entry + .object() + .map_err(|e| XrpcError::Internal(e.to_string()))?; + let modules = gix::submodule::File::from_bytes(&object.data, None, &Default::default()) + .map_err(|e| XrpcError::Internal(e.to_string()))?; + + let names: Vec<_> = modules.names().map(ToOwned::to_owned).collect(); + for name in names { + let name = name.as_bstr(); + let matches = modules + .path(name) + .ok() + .is_some_and(|value| value.as_ref() == path.as_bytes()); + if !matches { + continue; + } + let url = modules + .url(name) + .map_err(|e| XrpcError::Internal(e.to_string()))?; + let branch = match modules.branch(name).ok().flatten() { + Some(gix::submodule::config::Branch::Name(name)) => { + Some(name.to_str_lossy().as_ref().into()) + } + Some(gix::submodule::config::Branch::CurrentInSuperproject) => None, + None => None, + }; + return Ok(Some(org_tangled::git::Submodule { + name: name.to_str_lossy().as_ref().into(), + url: url.to_bstring().to_str_lossy().as_ref().into(), + branch, + extra_data: Default::default(), + })); + } + Ok(None) +} + +pub(crate) async fn get_entry( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result< + ( + [(header::HeaderName, &'static str); 1], + XrpcResponse, + ), + XrpcError, +> { + tracing::Span::current().record("repo", args.repo.as_str()); + let repo = open(&state, &args.repo)?; + let refspec = args.r#ref.as_deref().unwrap_or("HEAD").to_owned(); + let cache = cache_control(&[&refspec]); + let path = args.path.to_string(); + if path.is_empty() { + return Err(XrpcError::InvalidRequest( + "missing path parameter".to_owned(), + )); + } + + let (repo, head, mut output) = crate::metrics::blocking(BlockingKind::Git, move || { + let head = resolve_commit(&repo, &refspec)?; + let (kind, oid) = { + let entry = repo + .find_commit(head) + .map_err(|e| XrpcError::Internal(e.to_string()))? + .tree() + .map_err(|e| XrpcError::Internal(e.to_string()))? + .lookup_entry_by_path(&path) + .map_err(|e| XrpcError::Internal(e.to_string()))? + .ok_or_else(|| XrpcError::EntryNotFound { path: path.clone() })?; + (entry.mode().kind(), entry.object_id()) + }; + let size = match kind { + EntryKind::Tree | EntryKind::Commit => 0, + _ => blob_size(&repo, oid)? as i64, + }; + let submodule = match kind { + EntryKind::Commit => submodule_at(&repo, head, &path)?, + _ => None, + }; + + Ok::<_, XrpcError>(( + repo, + head, + get_entry::GetEntryOutput { + name: split_path(&path).1.as_str().into(), + mode: mode_of(kind).as_str().into(), + oid: oid.to_smolstr(), + size, + last_commit: None, + submodule, + extra_data: Default::default(), + }, + )) + }) + .await + .map_err(|e| XrpcError::Internal(e.to_string()))??; + + let path = args.path.trim_matches('/').to_owned(); + output.last_commit = last_commits(repo, head, path.clone(), 0, false) + .await? + .remove(path.as_bytes().as_bstr()); + + Ok((cache_header(cache), XrpcResponse(output))) +} + +pub(crate) async fn get_tree( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result< + ( + [(header::HeaderName, &'static str); 1], + XrpcResponse, + ), + XrpcError, +> { + tracing::Span::current().record("repo", args.repo.as_str()); + let repo = open(&state, &args.repo)?; + let refspec = args.r#ref.as_deref().unwrap_or("HEAD").to_owned(); + let cache = cache_control(&[&refspec]); + let tree_path = args + .path + .as_deref() + .unwrap_or("") + .trim_matches('/') + .to_owned(); + + let path = tree_path.clone(); + let (repo, head, entries) = crate::metrics::blocking(BlockingKind::Git, move || { + let head = resolve_commit(&repo, &refspec)?; + let entries = { + let root = repo + .find_commit(head) + .map_err(|e| XrpcError::Internal(e.to_string()))? + .tree() + .map_err(|e| XrpcError::Internal(e.to_string()))?; + let subtree = if path.is_empty() { + root + } else { + let entry = root + .lookup_entry_by_path(&path) + .map_err(|e| XrpcError::Internal(e.to_string()))? + .filter(|entry| entry.mode().is_tree()) + .ok_or_else(|| XrpcError::PathNotFound { path: path.clone() })?; + repo.find_tree(entry.object_id()) + .map_err(|_| XrpcError::PathNotFound { path: path.clone() })? + }; + + let mut entries = Vec::new(); + for entry in subtree.iter() { + let entry = entry.map_err(|e| XrpcError::Internal(e.to_string()))?; + let kind = entry.mode().kind(); + let size = match kind { + EntryKind::Tree | EntryKind::Commit => 0, + _ => blob_size(&repo, entry.object_id())? as i64, + }; + let name = entry.filename().to_owned(); + entries.push(( + get_tree::TreeEntry { + name: name.to_str_lossy().as_ref().into(), + mode: mode_of(kind).as_str().into(), + size, + last_commit: None, + extra_data: Default::default(), + }, + name, + )); + } + entries + }; + Ok::<_, XrpcError>((repo, head, entries)) + }) + .await + .map_err(|e| XrpcError::Internal(e.to_string()))??; + + // `git last-modified -t --max-depth=1 -- `; without a pathspec git counts depth from the + // top-level entries, so the root is `--max-depth=0` and reports no commit for itself. + let max_depth = if tree_path.is_empty() { 0 } else { 1 }; + let mut found = last_commits(repo, head, tree_path.clone(), max_depth, true).await?; + + let entries = entries + .into_iter() + .map(|(mut entry, name)| { + entry.last_commit = found.remove(&last_commit::join(tree_path.as_str(), name)); + entry + }) + .collect(); + + Ok(( + cache_header(cache), + XrpcResponse(get_tree::GetTreeOutput { + entries, + last_commit: found.remove(tree_path.as_bytes().as_bstr()), + extra_data: Default::default(), + }), + )) +} + +fn blob_etag(contents: &[u8]) -> String { + let mut digest = Sha256::new(); + digest.update(BLOB_ETAG_DOMAIN.as_bytes()); + digest.update(b"\0"); + digest.update(contents); + let hex: String = digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + format!("\"{hex}\"") +} + +fn etag_matches(headers: &HeaderMap, etag: &str) -> bool { + headers + .get(header::IF_NONE_MATCH) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value == "*" + || value + .split(',') + .map(str::trim) + .any(|candidate| candidate.trim_start_matches("W/") == etag) + }) +} + +pub(crate) async fn get_blob( + State(state): State, + headers: HeaderMap, + ExtractXrpc(args): ExtractXrpc, +) -> Result { + tracing::Span::current().record("repo", args.repo.as_str()); + let limit = state.max_blob_bytes; + let repo = open_capped(&state, &args.repo, limit)?; + let refspec = args.r#ref.as_deref().unwrap_or("HEAD").to_owned(); + let cache = cache_control(&[&refspec]); + let path = args.path.to_string(); + if path.is_empty() { + return Err(XrpcError::InvalidRequest( + "missing path parameter".to_owned(), + )); + } + + let disposition = args + .download + .unwrap_or(false) + .then(|| content_disposition(path.rsplit('/').next().unwrap_or(&path))); + let lookup = path.clone(); + let contents = crate::metrics::blocking(BlockingKind::Git, move || { + let head = resolve_commit(&repo, &refspec)?; + let entry = repo + .find_commit(head) + .map_err(|e| XrpcError::Internal(e.to_string()))? + .tree() + .map_err(|e| XrpcError::Internal(e.to_string()))? + .lookup_entry_by_path(&lookup) + .map_err(|e| XrpcError::Internal(e.to_string()))? + .filter(|entry| entry.mode().is_blob_or_symlink()) + .ok_or_else(|| XrpcError::FileNotFound { + path: lookup.clone(), + })?; + + let oid = entry.object_id(); + if blob_size(&repo, oid)? > limit { + return Err(XrpcError::BlobTooLarge { limit }); + } + repo.find_object(oid) + .map(|object| object.data.clone()) + .map_err(|error| { + if allocation_limited(&error) { + XrpcError::BlobTooLarge { limit } + } else { + XrpcError::Internal(error.to_string()) + } + }) + }) + .await + .map_err(|e| XrpcError::Internal(e.to_string()))??; + + let disposition = disposition.map(|value| (header::CONTENT_DISPOSITION, value)); + if contents.len() as u64 > LARGE_BLOB { + return Ok(( + StatusCode::OK, + sandboxed_blob_headers(cache, "application/octet-stream".to_owned(), contents.len()), + AppendHeaders(disposition), + contents, + ) + .into_response()); + } + + let etag = blob_etag(&contents); + if etag_matches(&headers, &etag) { + return Ok((StatusCode::NOT_MODIFIED, cache_header(cache)).into_response()); + } + + let detected = sniff::override_by_extension(&path, sniff::detect_content_type(&contents)); + let content_type = if detected.starts_with("image/") || detected.starts_with("video/") { + detected.to_owned() + } else if sniff::is_textual_mime(detected) { + "text/plain; charset=utf-8".to_owned() + } else { + "application/octet-stream".to_owned() + }; + + let mut headers = sandboxed_blob_headers(cache, content_type, contents.len()); + headers.insert( + header::ETAG, + etag.parse().expect("the etag we minted is ascii"), + ); + + Ok(( + StatusCode::OK, + headers, + AppendHeaders(disposition), + contents, + ) + .into_response()) +} + +fn internal(e: impl std::fmt::Display) -> XrpcError { + XrpcError::Internal(e.to_string()) +} + +fn page(limit: Option, cursor: Option<&str>) -> Result<(usize, usize), XrpcError> { + let limit = limit.unwrap_or(DEFAULT_REFS_LIMIT); + if !(1..=MAX_REFS_LIMIT).contains(&limit) { + return Err(XrpcError::InvalidRequest(format!( + "limit must be between 1 and {MAX_REFS_LIMIT}" + ))); + } + let offset = cursor + .map(|cursor| { + cursor + .parse::() + .map_err(|_| XrpcError::InvalidRequest(format!("invalid cursor {cursor:?}"))) + }) + .transpose()? + .unwrap_or(0); + Ok((offset, limit as usize)) +} + +fn paginate( + mut rows: Vec<(i64, String, T)>, + offset: usize, + limit: usize, +) -> (Vec, Option) { + rows.sort_by(|a, b| b.0.cmp(&a.0).then_with(|| a.1.cmp(&b.1))); + let next = offset.saturating_add(limit); + let cursor = (next < rows.len()).then(|| next.to_smolstr()); + let rows = rows + .into_iter() + .skip(offset) + .take(limit) + .map(|(_, _, row)| row) + .collect(); + (rows, cursor) +} + +fn summarize( + commit: &gix::Commit<'_>, +) -> Result<(ObjectId, SmolStr, org_tangled::git::Signature), XrpcError> { + let decoded = commit.decode().map_err(internal)?; + let author = GixSignature(decoded.author().map_err(internal)?) + .try_into() + .map_err(|e: anyhow::Error| internal(e))?; + Ok((commit.id, decoded.message.to_smolstr(), author)) +} + +pub(crate) async fn list_branches( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + tracing::Span::current().record("repo", args.repo.as_str()); + let repo = open(&state, &args.repo)?; + let (offset, limit) = page(args.limit, args.cursor.as_deref())?; + + crate::metrics::blocking(BlockingKind::Git, move || { + let platform = repo.references().map_err(internal)?; + let mut rows = Vec::new(); + let mut total = 0; + for reference in platform.local_branches().map_err(internal)? { + let mut reference = reference.map_err(internal)?; + total += 1; + let Ok(commit) = reference.peel_to_commit() else { + continue; + }; + let time = commit.time().map_err(internal)?.seconds; + let name = reference.name().shorten().to_str_lossy().into_owned(); + rows.push((time, name.clone(), (name, commit.id))); + } + let default_branch = repo.head_name().map_err(internal)?.and_then(|name| { + match name.category_and_short_name() { + Some((gix::refs::Category::LocalBranch, short)) => { + Some(short.to_str_lossy().into()) + } + _ => None, + } + }); + let (rows, cursor) = paginate(rows, offset, limit); + let branches = rows + .into_iter() + .map(|(name, oid)| { + let commit = repo.find_commit(oid).map_err(internal)?; + let (oid, message, author) = summarize(&commit)?; + let committer = GixSignature(commit.committer().map_err(internal)?) + .try_into() + .map_err(|e: anyhow::Error| internal(e))?; + Ok(list_branches::BranchInfo { + name: name.as_str().into(), + oid: oid.to_smolstr(), + message, + author, + committer, + extra_data: Default::default(), + }) + }) + .collect::, XrpcError>>()?; + Ok(list_branches::ListBranchesOutput { + branches, + cursor, + total, + default_branch, + extra_data: Default::default(), + }) + }) + .await + .map_err(internal)? + .map(XrpcResponse) +} + +fn tag_info( + repo: &gix::Repository, + reference: &gix::Reference<'_>, +) -> Result, XrpcError> { + let Some(target) = reference.try_id() else { + return Ok(None); + }; + let object = repo.find_object(target).map_err(internal)?; + let (annotation, tagged_at) = match object.kind { + gix::object::Kind::Tag => { + let tag = object.clone().into_tag(); + let decoded = tag.decode().map_err(internal)?; + let tagger = decoded.tagger().map_err(internal)?; + let tagged_at = tagger + .map(|tagger| tagger.time().map(|time| time.seconds)) + .transpose() + .map_err(internal)?; + let tagger = tagger + .map(|tagger| GixSignature(tagger).try_into()) + .transpose() + .map_err(|e: anyhow::Error| internal(e))?; + let annotation = list_tags::Annotation { + message: decoded.message.to_smolstr(), + oid: Some(target.to_smolstr()), + tagger, + extra_data: Default::default(), + }; + (Some(annotation), tagged_at) + } + _ => (None, None), + }; + let Ok(commit) = object + .peel_tags_to_end() + .map_err(internal)? + .try_into_commit() + else { + return Ok(None); + }; + let time = match tagged_at { + Some(time) => time, + None => commit.time().map_err(internal)?.seconds, + }; + let (oid, message, author) = summarize(&commit)?; + Ok(Some(( + time, + list_tags::TagInfo { + name: reference.name().shorten().to_str_lossy().as_ref().into(), + oid: oid.to_smolstr(), + message, + author, + annotation, + artifacts: None, + extra_data: Default::default(), + }, + ))) +} + +pub(crate) async fn list_tags( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + tracing::Span::current().record("repo", args.repo.as_str()); + let repo = open(&state, &args.repo)?; + let (offset, limit) = page(args.limit, args.cursor.as_deref())?; + + crate::metrics::blocking(BlockingKind::Git, move || { + let platform = repo.references().map_err(internal)?; + let mut rows = Vec::new(); + let mut total = 0; + for reference in platform.tags().map_err(internal)? { + let reference = reference.map_err(internal)?; + total += 1; + if let Some((time, tag)) = tag_info(&repo, &reference)? { + rows.push((time, tag.name.to_string(), tag)); + } + } + let (tags, cursor) = paginate(rows, offset, limit); + Ok(list_tags::ListTagsOutput { + tags, + cursor, + total, + extra_data: Default::default(), + }) + }) + .await + .map_err(internal)? + .map(XrpcResponse) +} + +#[axum::debug_handler] +pub(crate) async fn get_ref( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + tracing::Span::current().record("repo", args.repo.as_str()); + let repo = open(&state, &args.repo)?; + let path = state.layout.repo_path(&args.repo); + let name = args.r#ref.to_string(); + + crate::metrics::blocking(BlockingKind::Git, move || { + let not_found = |detail: &str| XrpcError::RefNotFound { + rev: name.clone(), + detail: detail.to_owned(), + }; + let mut reference = repo + .try_find_reference(name.as_str()) + .map_err(|e| not_found(&e.to_string()))? + .ok_or_else(|| not_found("the mirror holds no such reference"))?; + let (_, info) = tag_info(&repo, &reference)? + .ok_or_else(|| not_found("reference does not point at a commit"))?; + let tip = reference.peel_to_commit().map_err(internal)?.id; + let counted = knot_git::count_commits( + &knot_git::Repo::open(path).map_err(internal)?, + knot_types::Oid::from(tip), + Instant::now() + TOTAL_COMMITS_BUDGET, + ) + .map_err(internal)?; + let total_commits = counted.map_or_else( + || { + error!( + %tip, + budget = ?TOTAL_COMMITS_BUDGET, + "commit count exceeded budget; reporting a placeholder", + ); + TOTAL_COMMITS_OVER_BUDGET + }, + |count| i64::try_from(count).unwrap_or(i64::MAX), + ); + Ok(get_ref::GetRefOutput { + name: reference.name().as_bstr().to_str_lossy().as_ref().into(), + oid: info.oid, + message: info.message, + author: info.author, + annotation: info.annotation, + total_commits, + extra_data: Default::default(), + }) + }) + .await + .map_err(internal)? + .map(XrpcResponse) +} + +fn archive_etag(repo: &str, commit: ObjectId, format: &str, prefix: &str) -> String { + let digest = Sha256::digest( + [ + ARCHIVE_ETAG_DOMAIN, + repo, + &commit.to_string(), + format, + prefix, + ] + .join("\0"), + ); + let hex: String = digest.iter().map(|byte| format!("{byte:02x}")).collect(); + format!("\"{hex}\"") +} + +fn content_disposition(filename: &str) -> String { + let plain = |c: char| (' '..='~').contains(&c) && c != '"' && c != '\\'; + if filename.chars().all(plain) { + return format!("attachment; filename=\"{filename}\""); + } + let fallback: String = filename + .chars() + .map(|c| if plain(c) { c } else { '_' }) + .collect(); + let encoded: String = filename + .bytes() + .map(|byte| match byte { + b'A'..=b'Z' + | b'a'..=b'z' + | b'0'..=b'9' + | b'!' + | b'#' + | b'$' + | b'&' + | b'+' + | b'-' + | b'.' + | b'^' + | b'_' + | b'`' + | b'|' + | b'~' => (byte as char).to_string(), + _ => format!("%{byte:02X}"), + }) + .collect(); + format!("attachment; filename=\"{fallback}\"; filename*=utf-8''{encoded}") +} + +struct ChannelWriter(tokio::sync::mpsc::Sender>>); + +impl Write for ChannelWriter { + fn write(&mut self, data: &[u8]) -> std::io::Result { + self.0 + .blocking_send(Ok(data.to_vec())) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::BrokenPipe))?; + Ok(data.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// a failure part way cuts the body short +fn stream_body( + what: &'static str, + write: impl FnOnce(BufWriter) -> anyhow::Result<()> + Send + 'static, +) -> Body { + let (tx, rx) = tokio::sync::mpsc::channel(8); + let errors = tx.clone(); + tokio::spawn(async move { + let written = crate::metrics::blocking(BlockingKind::Git, move || { + write(BufWriter::with_capacity(STREAM_CHUNK, ChannelWriter(tx))) + }) + .await; + let error = match written { + Ok(Ok(())) => return, + Ok(Err(error)) => error.to_string(), + Err(error) => error.to_string(), + }; + warn!(%error, what, "response stream failed"); + let _ = errors.send(Err(std::io::Error::other(error))).await; + }); + Body::from_stream(futures_lite::stream::unfold(rx, |mut rx| async move { + rx.recv().await.map(|chunk| (chunk, rx)) + })) +} + +pub(crate) async fn get_archive( + State(state): State, + headers: HeaderMap, + ExtractXrpc(args): ExtractXrpc, +) -> Result { + tracing::Span::current().record("repo", args.repo.as_str()); + let repo = open(&state, &args.repo)?; + let format = args.format.as_deref().unwrap_or("tar.gz").to_owned(); + if format != "tar.gz" && format != "zip" { + return Err(XrpcError::InvalidRequest(format!( + "unsupported archive format {format:?}" + ))); + } + let prefix = args.prefix.to_string(); + archive::validate_prefix(&prefix)?; + let refspec = args.r#ref.to_string(); + // a commit never changes. a branch revalidates against the etag + let cache = if pinned(&[&refspec]) { + IMMUTABLE + } else { + "no-cache" + }; + let limit = state.max_archive_bytes; + + let (repo, commit, tree, time) = crate::metrics::blocking(BlockingKind::Git, move || { + let commit = resolve_commit(&repo, &refspec)?; + let (tree, time) = { + let found = repo.find_commit(commit).map_err(internal)?; + let tree = found.tree_id().map_err(internal)?.detach(); + (tree, found.time().map_err(internal)?.seconds) + }; + archive::bound_source(&repo, tree, limit)?; + Ok::<_, XrpcError>((repo, commit, tree, time)) + }) + .await + .map_err(internal)??; + + let etag = archive_etag(args.repo.as_str(), commit, &format, &prefix); + if etag_matches(&headers, &etag) { + return Ok(( + StatusCode::NOT_MODIFIED, + [ + (header::ETAG, etag), + (header::CACHE_CONTROL, cache.to_owned()), + ], + ) + .into_response()); + } + + let filename = format!("{}.{format}", prefix.replace(['/', '\\'], "-")); + let content_type = match format.as_str() { + "zip" => "application/zip", + _ => "application/gzip", + }; + // TODO(boltless): serve a spooled file with `tower_http::services::ServeFile` for resumable archives + let tree_prefix = prefix.clone(); + let body = stream_body("archive", move |out| match format.as_str() { + "zip" => archive::write_zip(&repo, tree, &tree_prefix, time, limit, out), + _ => archive::write_tar_gz(&repo, tree, &tree_prefix, time, limit, out), + }); + + Ok(( + StatusCode::OK, + [ + (header::CONTENT_TYPE, content_type.to_owned()), + (header::CONTENT_DISPOSITION, content_disposition(&filename)), + (header::ETAG, etag), + (header::CACHE_CONTROL, cache.to_owned()), + (header::X_CONTENT_TYPE_OPTIONS, "nosniff".to_owned()), + (header::CONTENT_SECURITY_POLICY, SANDBOX_CSP.to_owned()), + ], + body, + ) + .into_response()) +} + +pub(crate) async fn format_patch( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result { + tracing::Span::current().record("repo", args.repo.as_str()); + let repo = open(&state, &args.repo)?; + let (from, to) = (args.from.to_string(), args.to.to_string()); + let cache = cache_control(&[&from, &to]); + let max_blob_bytes = state.max_blob_bytes; + + let (repo, commits) = crate::metrics::blocking(BlockingKind::Git, move || { + let revision = |rev: &str| { + resolve_commit(&repo, rev).map_err(|error| match error { + XrpcError::RefNotFound { rev, .. } => XrpcError::RevisionNotFound { rev }, + error => error, + }) + }; + let (from, to) = (revision(&from)?, revision(&to)?); + let commits = + crate::format_patch::series(&repo, from, to, crate::format_patch::MAX_PATCH_COMMITS)?; + Ok::<_, XrpcError>((repo, commits)) + }) + .await + .map_err(internal)??; + + let body = stream_body("patch", move |mut out| { + crate::format_patch::write_series( + &repo, + &commits, + PATCH_SIGNATURE, + max_blob_bytes, + &mut out, + )?; + Ok(out.flush()?) + }); + Ok(( + StatusCode::OK, + [ + (header::CONTENT_TYPE, "text/plain; charset=utf-8"), + (header::CACHE_CONTROL, cache), + ], + body, + ) + .into_response()) +} + +pub(crate) async fn list_languages( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + tracing::Span::current().record("repo", args.repo.as_str()); + let repo = open(&state, &args.repo)?; + let refspec = args.r#ref.as_deref().unwrap_or("HEAD").to_owned(); + + let spec = refspec.clone(); + let commit = crate::metrics::blocking(BlockingKind::Git, move || resolve_commit(&repo, &spec)) + .await + .map_err(internal)??; + + let did = args.repo; + let commit_hex = commit.to_string(); + let stored = state + .db + .indexed_languages(&did, &commit_hex) + .await + .map_err(internal)?; + let sizes = match stored { + Some(sizes) => sizes, + None if args.indexed_only.unwrap_or(false) => { + return Err(XrpcError::NotIndexed { commit: commit_hex }); + } + None => { + let path = state.layout.repo_path(&did); + crate::metrics::blocking(BlockingKind::Git, move || { + let repo = knot_git::Repo::open(path).map_err(internal)?; + let sizes = knot_langs::analyze( + &repo, + knot_types::Oid::from(commit), + Some(Instant::now() + LANGUAGES_BUDGET), + ) + .map_err(internal)?; + Ok::<_, XrpcError>( + sizes + .into_iter() + .map(|(name, size)| { + ( + name.as_str().to_owned(), + i64::try_from(size.get()).unwrap_or(i64::MAX), + ) + }) + .collect::>(), + ) + }) + .await + .map_err(internal)?? + } + }; + + let languages = language_list(sizes); + let total = languages.iter().map(|l| l.size).sum(); + Ok(XrpcResponse(list_languages::ListLanguagesOutput { + languages, + r#ref: refspec.into(), + total, + extra_data: Default::default(), + })) +} + +pub(crate) async fn list_indexed_languages( + State(state): State, + ExtractXrpc(args): ExtractXrpc, +) -> Result, XrpcError> { + let limit = args.limit.unwrap_or(DEFAULT_INDEXED_LANGUAGES_LIMIT); + if !(1..=MAX_INDEXED_LANGUAGES_LIMIT).contains(&limit) { + return Err(XrpcError::InvalidRequest(format!( + "limit must be between 1 and {MAX_INDEXED_LANGUAGES_LIMIT}" + ))); + } + let page = state + .db + .indexed_languages_page(args.cursor.as_deref(), limit) + .await + .map_err(internal)?; + Ok(XrpcResponse( + list_indexed_languages::ListIndexedLanguagesOutput { + repos: page + .repos + .into_iter() + .map(|repo| list_indexed_languages::RepoLanguages { + repo: repo.did, + commit: repo.commit.into(), + languages: language_list(repo.languages), + extra_data: Default::default(), + }) + .collect(), + cursor: page.next.map(Into::into), + extra_data: Default::default(), + }, + )) +} + +fn language_list(mut sizes: Vec<(String, i64)>) -> Vec { + sizes.retain(|(_, size)| *size > 0); + sizes.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + sizes + .into_iter() + .map(|(name, size)| list_languages::Language { + name: name.into(), + size, + extra_data: Default::default(), + }) + .collect() +} + +#[cfg(test)] +mod format_patch_tests { + use std::path::Path; + use std::process::Command; + use std::sync::Arc; + + use bobbin_runtime::{ReqwestHttp, SystemClock}; + use jacquard_axum::service_auth::ServiceAuthConfig; + use jacquard_identity::JacquardResolver; + + use crate::{AppState, KnotPolicy}; + + const REPO: &str = "did:plc:formatpatch"; + + const SCRIPTED_SVG: &str = + ""; + + fn git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_AUTHOR_NAME", "t") + .env("GIT_AUTHOR_EMAIL", "t@t") + .env("GIT_COMMITTER_NAME", "t") + .env("GIT_COMMITTER_EMAIL", "t@t") + .output() + .expect("run git"); + assert!(out.status.success(), "git {args:?} failed"); + String::from_utf8(out.stdout).unwrap().trim().to_owned() + } + + struct Mirror { + _root: tempfile::TempDir, + url: String, + commits: Vec, + } + + async fn mirror() -> Mirror { + let root = tempfile::tempdir().unwrap(); + let work = root.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + git(&work, &["init", "-q", "-b", "main"]); + std::fs::write(work.join("logo.svg"), SCRIPTED_SVG).unwrap(); + let mut commits = Vec::new(); + for (n, subject) in ["first", "second change", "third change"] + .iter() + .enumerate() + { + std::fs::write(work.join("file.txt"), format!("{n}\n")).unwrap(); + std::fs::create_dir_all(work.join("dir")).unwrap(); + std::fs::write(work.join("dir/odd \"na\\me\u{e9}.txt"), "odd\n").unwrap(); + git(&work, &["add", "."]); + git(&work, &["commit", "-q", "-m", subject]); + commits.push(git(&work, &["rev-parse", "HEAD"])); + } + let repos = root.path().join("repos"); + let bare = repos.join(REPO); + git( + root.path(), + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + ); + + let http = reqwest::Client::new(); + let directory = Arc::new(JacquardResolver::new( + ReqwestHttp::new(http.clone()), + Default::default(), + )); + let pool = sqlx::postgres::PgPoolOptions::new() + .connect_lazy("postgres://localhost/unused") + .unwrap(); + let state = AppState::new( + knotmirror_db::Db::from_pool(pool), + Arc::new(repos), + http, + ServiceAuthConfig::new( + jacquard_common::types::did::Did::raw("did:web:mirror.test".into()), + directory, + ), + KnotPolicy { + allow_private: false, + require_https: true, + }, + Arc::new(SystemClock::new()), + None, + 1 << 20, + 1 << 20, + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, crate::router(state)).await }); + Mirror { + _root: root, + url, + commits, + } + } + + impl Mirror { + async fn call(&self, nsid: &str, params: &[(&str, &str)]) -> reqwest::Response { + let url = reqwest::Url::parse_with_params(&format!("{}/xrpc/{nsid}", self.url), params) + .unwrap(); + reqwest::Client::new().get(url).send().await.unwrap() + } + + async fn get(&self, params: &[(&str, &str)]) -> reqwest::Response { + self.call("org.tangled.temp.git.formatPatch", params).await + } + + async fn archive(&self, rev: &str) -> reqwest::Response { + let params = [("repo", REPO), ("ref", rev), ("prefix", "src")]; + self.call("org.tangled.temp.git.getArchive", ¶ms).await + } + + async fn blob(&self, path: &str, download: Option<&str>) -> reqwest::Response { + let mut params = vec![("repo", REPO), ("ref", "main"), ("path", path)]; + params.extend(download.map(|value| ("download", value))); + self.call("org.tangled.temp.git.getBlob", ¶ms).await + } + + async fn patch(&self, from: &str, to: &str) -> reqwest::Response { + self.get(&[("repo", REPO), ("from", from), ("to", to)]) + .await + } + } + + async fn error_name(response: reqwest::Response) -> String { + let body: serde_json::Value = response.json().await.unwrap(); + body["error"].as_str().unwrap().to_owned() + } + + #[tokio::test] + async fn a_range_streams_its_commits_as_a_patch_series() { + let mirror = mirror().await; + let (first, third) = (&mirror.commits[0], &mirror.commits[2]); + let response = mirror.patch(first, third).await; + assert_eq!(response.status(), 200); + let headers = response.headers(); + assert_eq!(headers["content-type"], "text/plain; charset=utf-8"); + assert_eq!( + headers["cache-control"], + "public, max-age=31536000, immutable" + ); + let body = response.text().await.unwrap(); + assert!(body.starts_with(&format!( + "From {} Mon Sep 17 00:00:00 2001\n", + mirror.commits[1] + ))); + assert!(body.contains("Subject: [PATCH 1/2] second change\n")); + assert!(body.contains("Subject: [PATCH 2/2] third change\n")); + assert!(body.contains("-1\n+2\n")); + } + + #[tokio::test] + async fn a_parent_range_is_one_unnumbered_patch() { + let mirror = mirror().await; + let third = &mirror.commits[2]; + let response = mirror.patch(&format!("{third}^"), third).await; + assert_eq!(response.status(), 200); + assert_eq!(response.headers()["cache-control"], "private, no-store"); + let body = response.text().await.unwrap(); + assert!(body.starts_with(&format!("From {third} "))); + assert!(body.contains("Subject: [PATCH] third change\n")); + assert_eq!(body.matches("\nFrom ").count(), 0); + } + + #[tokio::test] + async fn an_empty_or_reversed_range_is_an_empty_patch() { + let mirror = mirror().await; + let (first, third) = (&mirror.commits[0], &mirror.commits[2]); + for (from, to) in [(third, third), (third, first)] { + let response = mirror.patch(from, to).await; + assert_eq!(response.status(), 200); + assert_eq!(response.text().await.unwrap(), ""); + } + } + + #[tokio::test] + async fn an_unknown_repo_or_revision_is_not_found() { + let mirror = mirror().await; + let first = &mirror.commits[0]; + let response = mirror + .get(&[("repo", "did:plc:nobody"), ("from", first), ("to", first)]) + .await; + assert_eq!(response.status(), 404); + assert_eq!(error_name(response).await, "RepoNotFound"); + + for (from, to) in [ + (first.as_str(), "nope"), + ("0000000000000000000000000000000000000000", first), + ] { + let response = mirror.patch(from, to).await; + assert_eq!(response.status(), 404); + assert_eq!(error_name(response).await, "RevisionNotFound"); + } + } + + #[tokio::test] + async fn an_archive_of_a_commit_is_immutable_and_of_a_branch_revalidates() { + let mirror = mirror().await; + for (rev, cache) in [ + ( + mirror.commits[2].as_str(), + "public, max-age=31536000, immutable", + ), + ("main", "no-cache"), + ] { + let response = mirror.archive(rev).await; + assert_eq!(response.status(), 200); + assert_eq!(response.headers()["cache-control"], cache, "{rev}"); + assert_eq!(response.headers()["content-type"], "application/gzip"); + } + } + + #[tokio::test] + async fn a_blob_is_only_named_for_saving_when_asked_to_download() { + let mirror = mirror().await; + let response = mirror.blob("file.txt", None).await; + assert_eq!(response.status(), 200); + assert!(response.headers().get("content-disposition").is_none()); + + let response = mirror.blob("file.txt", Some("true")).await; + assert_eq!(response.status(), 200); + assert_eq!( + response.headers()["content-disposition"], + "attachment; filename=\"file.txt\"" + ); + assert_eq!(response.text().await.unwrap(), "2\n"); + + let response = mirror.blob("file.txt", Some("false")).await; + assert!(response.headers().get("content-disposition").is_none()); + } + + #[tokio::test] + async fn svg_blobs_should_be_behind_sandbox() { + let mirror = mirror().await; + let response = mirror.blob("logo.svg", None).await; + assert_eq!(response.status(), 200); + assert_eq!(response.headers()["content-type"], "image/svg+xml"); + assert_eq!( + response.headers()["content-security-policy"], + super::SANDBOX_CSP + ); + assert_eq!(response.headers()["x-content-type-options"], "nosniff"); + } + + #[tokio::test] + async fn a_downloaded_blob_is_named_by_its_basename_with_an_encoded_form() { + let mirror = mirror().await; + let response = mirror + .blob("dir/odd \"na\\me\u{e9}.txt", Some("true")) + .await; + assert_eq!(response.status(), 200); + assert_eq!( + response.headers()["content-disposition"], + "attachment; filename=\"odd _na_me_.txt\"; filename*=utf-8''odd%20%22na%5Cme%C3%A9.txt" + ); + } + + #[tokio::test] + async fn a_flag_like_or_missing_revision_is_rejected() { + let mirror = mirror().await; + let first = &mirror.commits[0]; + let response = mirror.patch("--output=/tmp/x", first).await; + assert_eq!(response.status(), 400); + assert_eq!(error_name(response).await, "InvalidRequest"); + let response = mirror.get(&[("repo", REPO), ("from", first)]).await; + assert_eq!(response.status(), 400); + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/signer.rs b/knotmirror/crates/knotmirror-xrpc/src/signer.rs new file mode 100644 index 0000000000000000000000000000000000000000..9ba3d9c79fe7737dfcadc204644de616b871d561 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/signer.rs @@ -0,0 +1,70 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use jacquard_common::types::did::Did; +use k256::ecdsa::{Signature, SigningKey, signature::Signer as _}; +use serde_json::{Value, json}; + +const SECP256K1_PRIVATE: [u8; 2] = [0x81, 0x26]; +const SECP256K1_PUBLIC: [u8; 2] = [0xe7, 0x01]; +const TOKEN_TTL_SECS: u64 = 60; + +pub struct ServiceSigner { + did: Did, + key: SigningKey, + issued: AtomicU64, +} + +impl ServiceSigner { + /// `encoded` is `z` plus base58 of 0x8126 and the 32 key bytes, the atproto multibase form + pub fn new(did: Did, encoded: &str) -> Result { + let bytes = encoded + .strip_prefix('z') + .and_then(|b58| bs58::decode(b58).into_vec().ok()) + .ok_or("service private key is not multibase base58btc")?; + let raw = bytes + .strip_prefix(&SECP256K1_PRIVATE) + .ok_or("service private key is not a secp256k1 key")?; + let key = SigningKey::from_slice(raw).map_err(|e| format!("service private key: {e}"))?; + Ok(Self { + did, + key, + issued: AtomicU64::new(0), + }) + } + + pub(crate) fn token(&self, audience: &str, lxm: &str, now_secs: u64) -> String { + let jti = format!("{now_secs}-{}", self.issued.fetch_add(1, Ordering::Relaxed)); + let claims = json!({ + "iss": self.did.as_str(), + "aud": audience, + "lxm": lxm, + "iat": now_secs, + "exp": now_secs + TOKEN_TTL_SECS, + "jti": jti, + }); + let input = format!( + "{}.{}", + URL_SAFE_NO_PAD.encode(br#"{"alg":"ES256K","typ":"JWT"}"#), + URL_SAFE_NO_PAD.encode(claims.to_string()), + ); + let signature: Signature = self.key.sign(input.as_bytes()); + format!("{input}.{}", URL_SAFE_NO_PAD.encode(signature.to_bytes())) + } + + pub(crate) fn did_document(&self) -> Value { + let did = self.did.as_str(); + let mut key = SECP256K1_PUBLIC.to_vec(); + key.extend_from_slice(self.key.verifying_key().to_encoded_point(true).as_bytes()); + json!({ + "@context": ["https://www.w3.org/ns/did/v1"], + "id": did, + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": format!("z{}", bs58::encode(key).into_string()), + }], + }) + } +} diff --git a/knotmirror/crates/knotmirror-xrpc/src/sniff.rs b/knotmirror/crates/knotmirror-xrpc/src/sniff.rs new file mode 100644 index 0000000000000000000000000000000000000000..f653573cf2d26c2296eee654c8227f22622c84f6 --- /dev/null +++ b/knotmirror/crates/knotmirror-xrpc/src/sniff.rs @@ -0,0 +1,267 @@ +//! ported from `knot2/crates/knot-xrpc/src/sniff.rs` +// TODO: there can be only one + +const SNIFF_LIMIT: usize = 512; + +fn is_ws(byte: u8) -> bool { + matches!(byte, b'\t' | b'\n' | 0x0c | b'\r' | b' ') +} + +fn is_tt(byte: u8) -> bool { + matches!(byte, b' ' | b'>') +} + +enum Sig { + Exact(&'static [u8], &'static str), + Masked { + mask: &'static [u8], + pat: &'static [u8], + skip_ws: bool, + ct: &'static str, + }, + Html(&'static [u8]), + Mp4, + Text, +} + +impl Sig { + fn detect(&self, data: &[u8], first_non_ws: usize) -> Option<&'static str> { + match self { + Sig::Exact(sig, ct) => data.starts_with(sig).then_some(*ct), + Sig::Masked { + mask, + pat, + skip_ws, + ct, + } => { + let data = if *skip_ws { + &data[first_non_ws..] + } else { + data + }; + (mask.len() == pat.len() + && data.len() >= pat.len() + && pat + .iter() + .zip(mask.iter()) + .enumerate() + .all(|(index, (byte, mask))| data[index] & mask == *byte)) + .then_some(*ct) + } + Sig::Html(tag) => { + let data = &data[first_non_ws..]; + (data.len() > tag.len() + && tag.iter().enumerate().all(|(index, byte)| { + let candidate = data[index]; + let candidate = match byte.is_ascii_uppercase() { + true => candidate & 0xDF, + false => candidate, + }; + *byte == candidate + }) + && is_tt(data[tag.len()])) + .then_some("text/html; charset=utf-8") + } + Sig::Mp4 => mp4(data), + Sig::Text => text(data, first_non_ws), + } + } +} + +fn mp4(data: &[u8]) -> Option<&'static str> { + if data.len() < 12 { + return None; + } + let box_size = u32::from_be_bytes([data[0], data[1], data[2], data[3]]) as usize; + if data.len() < box_size || !box_size.is_multiple_of(4) || &data[4..8] != b"ftyp" { + return None; + } + (8..box_size) + .step_by(4) + .filter(|start| *start != 12) + .any(|start| &data[start..start + 3] == b"mp4") + .then_some("video/mp4") +} + +fn text(data: &[u8], first_non_ws: usize) -> Option<&'static str> { + data[first_non_ws..] + .iter() + .all(|byte| !matches!(byte, 0x00..=0x08 | 0x0b | 0x0e..=0x1a | 0x1c..=0x1f)) + .then_some("text/plain; charset=utf-8") +} + +const SIGNATURES: &[Sig] = &[ + Sig::Html(b" &'static str { + let data = &content[..content.len().min(SNIFF_LIMIT)]; + let first_non_ws = data + .iter() + .position(|byte| !is_ws(*byte)) + .unwrap_or(data.len()); + SIGNATURES + .iter() + .find_map(|sig| sig.detect(data, first_non_ws)) + .unwrap_or("application/octet-stream") +} + +pub(crate) fn override_by_extension(path: &str, detected: &'static str) -> &'static str { + let extension = path.rsplit_once('.').map(|(_, ext)| ext).unwrap_or(""); + match extension.to_ascii_lowercase().as_str() { + "svg" => "image/svg+xml", + "avif" => "image/avif", + "jxl" => "image/jxl", + "heic" | "heif" => "image/heif", + _ => detected, + } +} + +pub(crate) fn is_textual_mime(mime: &str) -> bool { + mime.starts_with("text/") + || matches!( + mime, + "application/json" + | "application/xml" + | "application/yaml" + | "application/x-yaml" + | "application/toml" + | "application/javascript" + | "application/ecmascript" + ) +} + +#[cfg(test)] +mod tests { + use super::detect_content_type; + + #[test] + fn detects_common_content_signatures() { + let cases: &[(&[u8], &str)] = &[ + (b" \n", "text/html; charset=utf-8"), + (b"", "text/html; charset=utf-8"), + (b"", "text/html; charset=utf-8"), + (b"\n\t", "text/xml; charset=utf-8"), + (b"\xfe\xff\x00h", "text/plain; charset=utf-16be"), + (b"\xef\xbb\xbfhello", "text/plain; charset=utf-8"), + (b"\x89PNG\x0d\x0a\x1a\x0a", "image/png"), + (b"GIF89a", "image/gif"), + (b"fn main() {}\n", "text/plain; charset=utf-8"), + (b"\x00\x01\x02\x03", "application/octet-stream"), + (b"", "text/plain; charset=utf-8"), + ]; + cases.iter().for_each(|(input, expected)| { + assert_eq!(detect_content_type(input), *expected); + }); + } +} diff --git a/knotmirror/crates/knotmirror/Cargo.toml b/knotmirror/crates/knotmirror/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..1e2dba9b6ea9c8d78e2b57dcb39205b7433d03eb --- /dev/null +++ b/knotmirror/crates/knotmirror/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "knotmirror" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[[bin]] +name = "knotmirror" +path = "src/main.rs" + +[dependencies] +bobbin-runtime = { workspace = true } +knotmirror-db = { workspace = true } +knotmirror-git = { workspace = true } +knotmirror-index = { workspace = true } +knotmirror-ingest = { workspace = true } +knotmirror-maintenance = { workspace = true } +knotmirror-sync = { workspace = true } +knotmirror-types = { workspace = true } +knotmirror-xrpc = { workspace = true } +knot-maintenance = { workspace = true } +knot-runtime = { workspace = true } +knot-pack = { workspace = true } +knot-types = { workspace = true } + +anyhow = { workspace = true } +deadpool-redis = { workspace = true } +axum = { workspace = true } +clap = { workspace = true } +confique = { workspace = true } +futures = { workspace = true } +jacquard-axum = { workspace = true } +jacquard-common = { workspace = true } +jacquard-identity = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry-appender-tracing = { workspace = true } +opentelemetry-otlp = { workspace = true } +opentelemetry_sdk = { workspace = true } +reqwest = { workspace = true } +rustls = { workspace = true } +socket2 = "0.6" +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +url = { workspace = true } + +[lints] +workspace = true diff --git a/knotmirror/crates/knotmirror/src/config.rs b/knotmirror/crates/knotmirror/src/config.rs new file mode 100644 index 0000000000000000000000000000000000000000..0273a2ebbaf4202837e43d8d624f0f50c0b93943 --- /dev/null +++ b/knotmirror/crates/knotmirror/src/config.rs @@ -0,0 +1,272 @@ +use std::{net::SocketAddr, path::PathBuf, str::FromStr as _}; + +use anyhow::Context as _; +use confique::Config; +use jacquard_common::types::did::Did; + +const SYSTEM_CONFIG_PATH: &str = "/etc/knotmirror/config.toml"; + +#[derive(Debug, Config)] +pub struct MirrorConfig { + #[config(nested)] + pub server: ServerConfig, + #[config(nested)] + pub repo: RepoConfig, + + #[config(nested)] + pub service_auth: ServiceAuthConfig, + #[config(nested)] + pub identity: IdentityConfig, + #[config(nested)] + pub deliberi: DeliberiConfig, + + #[config(nested)] + pub redis: RedisConfig, + #[config(nested)] + pub knot: KnotConfig, + #[config(nested)] + pub log: LogConfig, + + #[config(nested)] + pub database: DatabaseConfig, + #[config(nested)] + pub ingest: IngestConfig, + #[config(nested)] + pub sync: SyncConfig, + #[config(nested)] + pub index: IndexConfig, + #[config(nested)] + pub maintenance: MaintenanceConfig, +} + +#[derive(Debug, Config)] +pub struct ServerConfig { + /// Addresses the XRPC server listens on. When using as an env var, comma-separated. + #[config( + env = "KNOTMIRROR_BIND", + parse_env = parse_binds, + default = ["127.0.0.1:9001", "[::1]:9001"] + )] + pub binds: Vec, + + /// Addresses the Prometheus metrics endpoint listens on. When using as an env var, + /// comma-separated; set it empty to disable the endpoint entirely. + #[config( + env = "KNOTMIRROR_METRICS_BIND", + parse_env = parse_optional_binds, + default = ["127.0.0.1:9091"] + )] + pub metrics_binds: Vec, + + /// Seconds to let the remaining task drain after the server or ingest stops before aborting it. + #[config(env = "KNOTMIRROR_SHUTDOWN_GRACE_SECS", default = 30)] + pub shutdown_grace_secs: u64, +} + +// TODO: share codes with bobbin + +#[derive(Debug, thiserror::Error)] +pub enum BindParseError { + #[error("KNOTMIRROR_BIND must list at least one address")] + Empty, + #[error("invalid bind entry `{0}`: {1}")] + Invalid(String, std::net::AddrParseError), +} + +fn parse_optional_binds(raw: &str) -> Result, BindParseError> { + raw.split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|s| SocketAddr::from_str(s).map_err(|e| BindParseError::Invalid(s.to_owned(), e))) + .collect() +} + +fn parse_binds(raw: &str) -> Result, BindParseError> { + let addrs = parse_optional_binds(raw)?; + if addrs.is_empty() { + return Err(BindParseError::Empty); + } + Ok(addrs) +} + +#[derive(Debug, Config)] +pub struct RepoConfig { + #[config(env = "KNOTMIRROR_SCAN_PATH")] + pub scan_path: PathBuf, + + /// Largest blob getBlob will serve. gix cannot stream objects, so this bounds how much + /// a single request reads into memory. + #[config(env = "KNOTMIRROR_MAX_BLOB_BYTES", default = 26_214_400)] + pub max_blob_bytes: u64, + + #[config(env = "KNOTMIRROR_MAX_ARCHIVE_BYTES", default = 1_073_741_824)] + pub max_archive_bytes: u64, +} + +#[derive(Debug, Config)] +pub struct ServiceAuthConfig { + /// This service's own DID, e.g. `did:web:mirror.tangled.network`. + #[config(env = "KNOTMIRROR_SERVICE_DID")] + pub did: Did, + + /// multibase secp256k1 private key (`z...`) for outgoing service auth and `/.well-known/did.json`. + /// needed to call deliberi. + #[config(env = "KNOTMIRROR_SERVICE_PRIVATE_KEY")] + pub private_key: Option, +} + +#[derive(Debug, Config)] +pub struct DeliberiConfig { + /// deliberi base url for verified emails. unset counts only commits whose committer email is the did. + #[config(env = "KNOTMIRROR_DELIBERI_URL")] + pub url: Option, + + /// audience of the service auth sent to deliberi. + #[config(env = "KNOTMIRROR_DELIBERI_DID")] + pub did: Option, +} + +#[derive(Debug, Config)] +pub struct IdentityConfig { + #[config(env = "KNOTMIRROR_PLC_URL", default = "https://plc.directory")] + pub plc_url: String, +} + +#[derive(Debug, Config)] +pub struct RedisConfig { + #[config(env = "KNOTMIRROR_REDIS_URL")] + pub url: Option, +} + +#[derive(Debug, Config)] +pub struct KnotConfig { + /// Allow knot endpoints on private or loopback addresses. + #[config(env = "KNOTMIRROR_KNOT_ALLOW_PRIVATE", default = false)] + pub allow_private: bool, + + /// Require https on knot endpoints. + #[config(env = "KNOTMIRROR_KNOT_REQUIRE_HTTPS", default = true)] + pub require_https: bool, +} + +#[derive(Debug, Config)] +pub struct LogConfig { + /// `tracing-subscriber` env-filter directive, e.g. `knotmirror_xrpc=debug,info`. + #[config(env = "KNOTMIRROR_LOG", default = "info")] + pub filter: String, + + /// Full OTLP/HTTP endpoint logs are exported to, e.g. + /// `http://prometheus:3100/otlp/v1/logs`. Empty disables log export. + #[config(env = "KNOTMIRROR_LOG_ENDPOINT", default = "")] + pub endpoint: String, + + /// `service.name` carried by exported logs + #[config(env = "KNOTMIRROR_LOG_SERVICE_NAME", default = "gitmirror")] + pub service_name: String, +} + +#[derive(Debug, Config)] +pub struct DatabaseConfig { + /// Postgres connection string, e.g. `postgres://user:pass@host/gitmirror`. + #[config(env = "KNOTMIRROR_DATABASE_URL")] + pub url: String, + + #[config(env = "KNOTMIRROR_DATABASE_MAX_CONNECTIONS", default = 16)] + pub max_connections: u32, +} + +#[derive(Debug, Config)] +pub struct IngestConfig { + /// Base URL of the hydrant instance - the cursor-replayable /stream lives + /// under this. Use `ws://` or `wss://` - `http://` and `https://` + /// are rewritten to the corresponding ws scheme at connection time! + #[config(env = "KNOTMIRROR_HYDRANT_URL", default = "http://127.0.0.1:3000")] + pub hydrant_url: url::Url, + + /// Seconds between cursor checkpoints for stretches of events that touched no row. + #[config(env = "KNOTMIRROR_CHECKPOINT_INTERVAL_SECS", default = 10)] + pub checkpoint_interval_secs: u64, +} + +#[derive(Debug, Config)] +pub struct SyncConfig { + /// How many repositories may be fetched at once. + #[config(env = "KNOTMIRROR_SYNC_CONCURRENCY", default = 8)] + pub concurrency: usize, + + /// How many of those may target the same knot, so one slow host cannot take every slot. + #[config(env = "KNOTMIRROR_SYNC_PER_KNOT_CONCURRENCY", default = 4)] + pub per_knot_concurrency: usize, + + /// Seconds the sync worker waits before rescanning the queue when it found nothing to do. + /// Also the resolution at which a repo's `retry_after` comes due. + #[config(env = "KNOTMIRROR_SYNC_POLL_INTERVAL_SECS", default = 5)] + pub poll_interval_secs: u64, +} + +#[derive(Debug, Config)] +pub struct IndexConfig { + /// How many repositories may be indexed at once. + #[config(env = "KNOTMIRROR_INDEX_CONCURRENCY", default = 4)] + pub concurrency: usize, + + /// Seconds the index worker waits before rescanning the queue when it found nothing to do. + #[config(env = "KNOTMIRROR_INDEX_POLL_INTERVAL_SECS", default = 5)] + pub poll_interval_secs: u64, + + /// Seconds a language analysis may run before it stops and keeps what it counted so far. + #[config(env = "KNOTMIRROR_INDEX_LANGS_TIMEOUT_SECS", default = 60)] + pub langs_timeout_secs: u64, + + /// Base URL of the zoekt index server. Code search indexing is skipped when unset. + #[config(env = "KNOTMIRROR_ZOEKT_URL")] + pub zoekt_url: Option, +} + +#[derive(Debug, Config)] +pub struct MaintenanceConfig { + #[config(env = "KNOTMIRROR_MAINTENANCE_ENABLED", default = true)] + pub enabled: bool, + + /// Seconds between maintenance sweeps. The first sweep runs one interval after startup. + #[config(env = "KNOTMIRROR_MAINTENANCE_INTERVAL_SECS", default = 21600)] + pub interval_secs: u64, + + /// Repos whose reachable set exceeds this many objects are not repacked. + #[config( + env = "KNOTMIRROR_MAINTENANCE_REPACK_MAX_OBJECTS", + default = 16_000_000 + )] + pub repack_max_objects: u64, + + #[config(env = "KNOTMIRROR_MAINTENANCE_REPACK_GEOMETRIC_FACTOR", default = 2)] + pub repack_geometric_factor: u64, + + /// Seconds an unreachable object is kept before it is pruned. + #[config(env = "KNOTMIRROR_MAINTENANCE_PRUNE_GRACE_SECS", default = 1_209_600)] + pub prune_grace_secs: u64, + + #[config(env = "KNOTMIRROR_MAINTENANCE_COMMIT_GRAPH", default = true)] + pub commit_graph: bool, + + #[config(env = "KNOTMIRROR_MAINTENANCE_MULTI_PACK_INDEX", default = true)] + pub multi_pack_index: bool, + + #[config(env = "KNOTMIRROR_MAINTENANCE_BITMAP", default = false)] + pub bitmap: bool, +} + +pub fn load(path: Option<&PathBuf>) -> anyhow::Result { + let mut builder = MirrorConfig::builder().env(); + if let Some(p) = path { + builder = builder.file(p); + } + builder + .file(SYSTEM_CONFIG_PATH) + .load() + .context("load configuration") +} + +pub fn template() -> String { + confique::toml::template::(confique::toml::FormatOptions::default()) +} diff --git a/knotmirror/crates/knotmirror/src/main.rs b/knotmirror/crates/knotmirror/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..7a85bc9e5b283b78d867fbdd31fff7637d2e3d67 --- /dev/null +++ b/knotmirror/crates/knotmirror/src/main.rs @@ -0,0 +1,528 @@ +use std::{net::SocketAddr, path::PathBuf, process::ExitCode, sync::Arc, time::Duration}; + +use anyhow::{Context as _, anyhow}; +use bobbin_runtime::{Clock, ReqwestHttp, SystemClock, TungsteniteWs, WsTls}; +use clap::{Parser, Subcommand}; +use jacquard_axum::service_auth::ServiceAuthConfig; +use jacquard_common::deps::fluent_uri::Uri; +use jacquard_identity::{JacquardResolver, resolver::PlcSource}; +use opentelemetry_appender_tracing::layer::{OpenTelemetryTracingBridge, TracingSpanAttributes}; +use opentelemetry_otlp::{LogExporter, Protocol, WithExportConfig as _}; +use opentelemetry_sdk::{Resource, logs::SdkLoggerProvider}; +use tokio::{ + signal::unix::{SignalKind, signal}, + task::JoinSet, +}; +use tokio_util::sync::CancellationToken; +use tracing::level_filters::LevelFilter; +use tracing_subscriber::{ + EnvFilter, Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _, +}; + +mod config; + +use config::MirrorConfig; +use knotmirror_git::layout::Layout; +use knotmirror_ingest::{IngestRuntime, run as run_ingest}; +use knotmirror_sync::{BackfillWorker, Directory, SyncRuntime}; +use knotmirror_xrpc::{AppState, Deliberi, KnotPolicy, ServiceSigner, router}; + +#[derive(Parser)] +#[command(name = "knotmirror", about = "KnotMirror service")] +struct Cli { + /// Path to a TOML config file. Environment variables override file values + /// for any `KNOTMIRROR_*` setting; `/etc/knotmirror/config.toml` is consulted as a + /// final fallback so distro packaging can drop a default in place. + #[arg(short, long, value_name = "FILE", env = "KNOTMIRROR_CONFIG")] + config: Option, + + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Print a fully-commented TOML template to stdout. Use this to seed + /// `config.toml` for a fresh deploy. + ConfigTemplate, + #[command(flatten)] + Configured(ConfiguredCommand), +} + +#[derive(Subcommand)] +enum ConfiguredCommand { + /// Load and validate the configuration without starting the server. + Validate, + /// Run the XRPC server. + Serve, +} + +#[tokio::main] +#[allow( + clippy::print_stdout, + clippy::print_stderr, + reason = "the cli prints its results and any error from before tracing starts" +)] +async fn main() -> ExitCode { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let cli = Cli::parse(); + + let command = match cli.command { + Command::ConfigTemplate => { + print!("{}", config::template()); + return ExitCode::SUCCESS; + } + Command::Configured(command) => command, + }; + + let cfg = match config::load(cli.config.as_ref()) { + Ok(c) => c, + Err(e) => { + eprintln!("failed to load configuration: {e:#}"); + return ExitCode::FAILURE; + } + }; + + // TODO: embed inside serve(). + let logs = match init_tracing(&cfg) { + Ok(logs) => logs, + Err(e) => { + eprintln!("failed to install tracing subscriber: {e}"); + return ExitCode::FAILURE; + } + }; + + let code = match command { + ConfiguredCommand::Validate => { + println!("configuration is valid"); + ExitCode::SUCCESS + } + ConfiguredCommand::Serve => match serve(cfg).await { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + tracing::error!(error = ?e, "fatal"); + ExitCode::FAILURE + } + }, + }; + + if let Some(provider) = logs + && let Err(e) = provider.shutdown() + { + eprintln!("failed to flush exported logs: {e}"); + } + + code +} + +fn init_tracing(cfg: &MirrorConfig) -> Result, String> { + let combined = format!("{},{}", LevelFilter::INFO, cfg.log.filter); + let filter = || { + EnvFilter::try_new(&combined) + .map_err(|e| format!("invalid log filter `{}`: {e}", cfg.log.filter)) + }; + + let provider = if cfg.log.endpoint.is_empty() { + None + } else { + let exporter = LogExporter::builder() + .with_http() + .with_protocol(Protocol::HttpBinary) + .with_endpoint(&cfg.log.endpoint) + .build() + .map_err(|e| format!("OTLP log exporter for `{}`: {e}", cfg.log.endpoint))?; + Some( + SdkLoggerProvider::builder() + .with_batch_exporter(exporter) + .with_resource( + Resource::builder() + .with_service_name(cfg.log.service_name.clone()) + .build(), + ) + .build(), + ) + }; + + let export_layer = provider + .as_ref() + .map(|provider| { + let filter = filter()? + .add_directive("hyper=off".parse().expect("static directive")) + .add_directive("h2=off".parse().expect("static directive")) + .add_directive("reqwest=off".parse().expect("static directive")); + Ok::<_, String>( + OpenTelemetryTracingBridge::builder(provider) + .with_tracing_span_attributes(TracingSpanAttributes::all()) + .build() + .with_filter(filter), + ) + }) + .transpose()?; + + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_filter(filter()?)) + .with(export_layer) + .try_init() + .map_err(|e| e.to_string())?; + + Ok(provider) +} + +fn http_client(allow_private: bool) -> reqwest::Result { + let builder = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_mins(5)) + .redirect(reqwest::redirect::Policy::none()); + if allow_private { + builder.build() + } else { + builder + .dns_resolver(std::sync::Arc::new(knot_runtime::GuardedResolver)) + .build() + } +} + +fn ops_client() -> reqwest::Result { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_mins(5)) + .build() +} + +fn directory(http: &reqwest::Client, plc_url: &str) -> anyhow::Result> { + Ok(Arc::new( + JacquardResolver::new(ReqwestHttp::new(http.clone()), Default::default()) + .with_plc_source(plc_source(plc_url)?), + )) +} + +async fn serve(cfg: MirrorConfig) -> anyhow::Result<()> { + let db = knotmirror_db::Db::connect(&cfg.database.url, cfg.database.max_connections) + .await + .context("connect to postgres")?; + db.migrate().await.context("run migrations")?; + std::fs::create_dir_all(&cfg.repo.scan_path) + .with_context(|| format!("create {}", cfg.repo.scan_path.display()))?; + let swept = knot_pack::sweep_incoming(&cfg.repo.scan_path); + tracing::info!(swept, "swept stale staging dirs"); + + let http = http_client(cfg.knot.allow_private)?; + let ops_http = ops_client()?; + let directory = directory(&http, &cfg.identity.plc_url)?; + let service_auth = ServiceAuthConfig::new(cfg.service_auth.did.clone(), directory.clone()) + .disable_replay_protection(); + let signer = cfg + .service_auth + .private_key + .as_deref() + .map(|key| ServiceSigner::new(cfg.service_auth.did.clone(), key)) + .transpose() + .map_err(|e| anyhow!(e))?; + let deliberi = match (cfg.deliberi.url, cfg.deliberi.did) { + (Some(url), Some(did)) => Some(Deliberi { url, did }), + (None, None) => None, + _ => return Err(anyhow!("deliberi url and did must be set together")), + }; + if signer.is_none() && deliberi.is_some() { + return Err(anyhow!("calling deliberi needs a service private key")); + } + let redis = cfg + .redis + .url + .as_deref() + .map(|url| { + deadpool_redis::Config::from_url(url) + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .with_context(|| format!("build redis pool for {url}")) + }) + .transpose()?; + + let clock: Arc = Arc::new(SystemClock::new()); + let cancel = CancellationToken::new(); + let index_notify = Arc::new(tokio::sync::Notify::new()); + let sync_notify = Arc::new(tokio::sync::Notify::new()); + let layout = Layout::new(cfg.repo.scan_path.clone()); + + let worker = BackfillWorker::new( + SyncRuntime { + db: db.clone(), + clock: clock.clone(), + cancel: cancel.clone(), + directory, + http: http.clone(), + layout: layout.clone(), + knot_policy: knotmirror_types::KnotPolicy { + allow_private: cfg.knot.allow_private, + require_https: cfg.knot.require_https, + }, + index_notify: index_notify.clone(), + notify: sync_notify.clone(), + }, + knotmirror_sync::SyncConfig { + concurrency: cfg.sync.concurrency, + per_knot_concurrency: cfg.sync.per_knot_concurrency, + poll_interval: Duration::from_secs(cfg.sync.poll_interval_secs), + }, + ); + let ingest_cfg = knotmirror_ingest::IngestConfig { + hydrant_url: cfg.ingest.hydrant_url.clone(), + checkpoint_interval: Duration::from_secs(cfg.ingest.checkpoint_interval_secs), + }; + let index_cfg = knotmirror_index::IndexConfig { + concurrency: cfg.index.concurrency, + poll_interval: Duration::from_secs(cfg.index.poll_interval_secs), + langs_timeout: Duration::from_secs(cfg.index.langs_timeout_secs), + zoekt_url: cfg.index.zoekt_url.clone(), + }; + let index_runtime = knotmirror_index::IndexRuntime { + db: db.clone(), + clock: clock.clone(), + cancel: cancel.clone(), + http: ops_http.clone(), + layout: layout.clone(), + notify: index_notify, + }; + let activity_runtime = knotmirror_index::activity::ActivityRuntime { + db: db.clone(), + clock: clock.clone(), + cancel: cancel.clone(), + layout, + }; + let maintenance = cfg.maintenance.enabled.then(|| { + knotmirror_maintenance::MaintenanceWorker::new( + knotmirror_maintenance::MaintenanceRuntime { + db: db.clone(), + clock: clock.clone(), + cancel: cancel.clone(), + layout: Layout::new(cfg.repo.scan_path.clone()), + }, + knotmirror_maintenance::MaintenanceConfig { + interval: Duration::from_secs(cfg.maintenance.interval_secs), + options: knot_maintenance::Options { + repack_max_objects: knot_types::ObjectCount::new( + cfg.maintenance.repack_max_objects as usize, + ), + geometric_factor: knot_maintenance::GeometricFactor::new( + cfg.maintenance.repack_geometric_factor, + ), + loose_threshold: knot_types::ObjectCount::new(6700), + prune_grace: knot_maintenance::PruneGrace::from_secs( + cfg.maintenance.prune_grace_secs, + ), + full_repack_interval: knot_maintenance::FullRepackInterval::from_secs(432_000), + reflog_floor: knot_maintenance::ReflogRetention::from_secs(90 * 24 * 60 * 60), + commit_graph: cfg.maintenance.commit_graph, + multi_pack_index: cfg.maintenance.multi_pack_index, + bitmap: cfg.maintenance.bitmap, + }, + }, + ) + }); + let ingest_runtime = IngestRuntime { + db: db.clone(), + clock: clock.clone(), + ws: TungsteniteWs::shared(WsTls::from_native_roots()?), + cancel: cancel.clone(), + sync_notify, + }; + let mut tasks = JoinSet::new(); + tasks.spawn(async move { + let res = run_ingest(ingest_cfg, ingest_runtime).await; + ("ingest", res.map_err(anyhow::Error::from)) + }); + tasks.spawn(async move { ("sync", worker.run().await.map_err(anyhow::Error::from)) }); + let activity_poll = index_cfg.poll_interval; + tasks.spawn(async move { + let res = knotmirror_index::run(index_cfg, index_runtime).await; + ("index", res.map_err(anyhow::Error::from)) + }); + tasks.spawn(async move { + let res = knotmirror_index::activity::run(activity_poll, activity_runtime).await; + ("activity", res.map_err(anyhow::Error::from)) + }); + if let Some(maintenance) = maintenance { + tasks.spawn(async move { + maintenance.run().await; + ("maintenance", Ok(())) + }); + } + + let max_blob_bytes = cfg.repo.max_blob_bytes; + let max_archive_bytes = cfg.repo.max_archive_bytes; + let mut state = AppState::new( + db.clone(), + Arc::new(cfg.repo.scan_path), + http, + service_auth, + KnotPolicy { + allow_private: cfg.knot.allow_private, + require_https: cfg.knot.require_https, + }, + clock.clone(), + redis, + max_blob_bytes, + max_archive_bytes, + ); + if let Some(signer) = signer { + state = state.with_signer(signer, deliberi); + } + let app = router(state); + + // TODO: add debug routes + + let binds = cfg.server.binds.clone(); + tracing::info!(binds = %bind_display(&binds), "knotmirror listening"); + let metrics_binds = cfg.server.metrics_binds.clone(); + if metrics_binds.is_empty() { + tracing::info!("metrics endpoint disabled"); + } else { + knotmirror_xrpc::metrics::init_metrics(); + let (clock, cancel) = (clock.clone(), cancel.clone()); + tasks.spawn(async move { + knotmirror_sync::report_repo_states(db, clock, cancel).await; + ("sync-metrics", Ok(())) + }); + tracing::info!(binds = %bind_display(&metrics_binds), "knotmirror metrics listening"); + } + + let mut targets: Vec<(SocketAddr, axum::Router)> = + binds.into_iter().map(|addr| (addr, app.clone())).collect(); + targets.extend( + metrics_binds + .into_iter() + .map(|addr| (addr, knotmirror_xrpc::metrics::router())), + ); + + let server_cancel = cancel.clone(); + tasks.spawn(async move { ("server", serve_all(targets, server_cancel).await) }); + + let first = tasks.join_next().await.expect("tasks is not empty"); + let was_cancelled = cancel.is_cancelled(); + cancel.cancel(); + + let grace = Duration::from_secs(cfg.server.shutdown_grace_secs); + tracing::info!(grace_secs = grace.as_secs(), "draining tasks"); + tokio::select! { + () = async { while tasks.join_next().await.is_some() {} } => {} + () = clock.sleep(grace) => { + tracing::warn!( + grace_secs = grace.as_secs(), + remaining = tasks.len(), + "tasks did not stop within grace, aborting" + ); + tasks.abort_all(); + } + } + + match first { + Ok((_, Ok(()))) if was_cancelled => Ok(()), + Ok((name, Ok(()))) => Err(anyhow!("{name} exited unexpectedly")), + Ok((name, Err(e))) => Err(e.context(format!("{name} failed"))), + Err(join) => Err(anyhow!("task panicked: {join}")), + } +} + +fn bind_display(binds: &[SocketAddr]) -> String { + binds + .iter() + .map(|bind| bind.to_string()) + .collect::>() + .join(",") +} + +async fn serve_all( + targets: Vec<(SocketAddr, axum::Router)>, + cancel: CancellationToken, +) -> anyhow::Result<()> { + let listeners = targets + .into_iter() + .map(|(addr, app)| { + let listener = bind_listener(addr).with_context(|| format!("bind {addr}"))?; + tracing::info!(%addr, "knotmirror listener bound"); + anyhow::Ok((listener, app)) + }) + .collect::>>()?; + + let trigger = cancel.clone(); + let signal_task = tokio::spawn(async move { + wait_for_shutdown().await; + tracing::info!("shutdown signal received, draining server"); + trigger.cancel(); + }); + + let result = futures::future::try_join_all(listeners.into_iter().map(|(listener, app)| { + let cancel = cancel.clone(); + async move { + axum::serve(listener, app) + .with_graceful_shutdown(async move { cancel.cancelled().await }) + .await + } + })) + .await; + signal_task.abort(); + result.map(|_| ()).context("axum server failed") +} + +fn bind_listener(addr: SocketAddr) -> std::io::Result { + let domain = match addr { + SocketAddr::V4(_) => socket2::Domain::IPV4, + SocketAddr::V6(_) => socket2::Domain::IPV6, + }; + let socket = socket2::Socket::new(domain, socket2::Type::STREAM, Some(socket2::Protocol::TCP))?; + // Otherwise a dual-stack `[::]` bind swallows the port and the sibling `0.0.0.0` bind fails. + if matches!(addr, SocketAddr::V6(_)) { + socket.set_only_v6(true)?; + } + socket.set_reuse_address(true)?; + socket.set_nonblocking(true)?; + socket.bind(&addr.into())?; + socket.listen(1024)?; + tokio::net::TcpListener::from_std(socket.into()) +} + +async fn wait_for_shutdown() { + let ctrl_c = tokio::signal::ctrl_c(); + let mut sigterm = match signal(SignalKind::terminate()) { + Ok(s) => s, + Err(e) => { + tracing::warn!( + ?e, + "could not install SIGTERM handler, shutdown will only honor ctrl-c" + ); + ctrl_c.await.ok(); + return; + } + }; + tokio::select! { + _ = ctrl_c => {} + _ = sigterm.recv() => {} + } +} + +/// `jacquard-identity`'s `DidStep::PlcHttp` joins the base and the DID by concatenation, so a base +/// missing its trailing slash would ask for `https://plc.exampledid:plc:...`. +fn plc_source(url: &str) -> anyhow::Result { + let base = format!("{}/", url.trim_end_matches('/')); + Ok(PlcSource::PlcDirectory { + base: Uri::parse(base.as_str()) + .map_err(|e| anyhow!("{url:?} is not a usable PLC directory: {e}"))? + .to_owned(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_plc_base_carries_the_separator_the_resolver_leaves_out() { + let base = |url| match plc_source(url).unwrap() { + PlcSource::PlcDirectory { base } => base.to_string(), + other => panic!("{other:?}"), + }; + assert_eq!(base("https://plc.example"), "https://plc.example/"); + assert_eq!(base("https://plc.example/"), "https://plc.example/"); + assert!(plc_source("not a url").is_err()); + } +} diff --git a/lexicon-build-config.json b/lexicon-build-config.json new file mode 100644 index 0000000000000000000000000000000000000000..8af0ae39fb8587d1b3e264c256509b1983bab288 --- /dev/null +++ b/lexicon-build-config.json @@ -0,0 +1,22 @@ +[ + { + "package": "atproto", + "prefix": "com.atproto", + "outdir": "/tmp/atproto", + "import": "github.com/bluesky-social/indigo/api/atproto" + }, + { + "package": "tangled", + "prefix": "sh.tangled", + "outdir": "api/tangled", + "import": "tangled.org/core/api/tangled", + "gen-server": true + }, + { + "package": "org_tangled", + "prefix": "org.tangled", + "outdir": "api/org_tangled", + "import": "tangled.org/core/api/org_tangled", + "gen-server": true + } +] diff --git a/lexicons/com/atproto/label/defs.json b/lexicons/com/atproto/label/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..0de199444da16b31125b6a6e9fd032f37dfc25d4 --- /dev/null +++ b/lexicons/com/atproto/label/defs.json @@ -0,0 +1,152 @@ +{ + "lexicon": 1, + "id": "com.atproto.label.defs", + "defs": { + "label": { + "type": "object", + "description": "Metadata tag on an atproto resource (eg, repo or record).", + "required": ["src", "uri", "val", "cts"], + "properties": { + "ver": { + "type": "integer", + "description": "The AT Protocol version of the label object." + }, + "src": { + "type": "string", + "format": "did", + "description": "DID of the actor who created this label." + }, + "uri": { + "type": "string", + "format": "uri", + "description": "AT URI of the record, repository (account), or other resource that this label applies to." + }, + "cid": { + "type": "string", + "description": "Optionally, CID specifying the specific version of 'uri' resource this label applies to. Upstream marks this `format: cid`; dropped here because jacquard decodes that format as a dag-cbor cid-link (tag 42), while labels carry it as a plain string." + }, + "val": { + "type": "string", + "maxLength": 128, + "description": "The short string name of the value or type of this label." + }, + "neg": { + "type": "boolean", + "description": "If true, this is a negation label, overwriting a previous label." + }, + "cts": { + "type": "string", + "format": "datetime", + "description": "Timestamp when this label was created." + }, + "exp": { + "type": "string", + "format": "datetime", + "description": "Timestamp at which this label expires (no longer applies)." + }, + "sig": { + "type": "bytes", + "description": "Signature of dag-cbor encoded label." + } + } + }, + "selfLabels": { + "type": "object", + "description": "Metadata tags on an atproto record, published by the author within the record.", + "required": ["values"], + "properties": { + "values": { + "type": "array", + "items": { "type": "ref", "ref": "#selfLabel" }, + "maxLength": 10 + } + } + }, + "selfLabel": { + "type": "object", + "description": "Metadata tag on an atproto record, published by the author within the record. Note that schemas should use #selfLabels, not #selfLabel.", + "required": ["val"], + "properties": { + "val": { + "type": "string", + "maxLength": 128, + "description": "The short string name of the value or type of this label." + } + } + }, + "labelValueDefinition": { + "type": "object", + "description": "Declares a label value and its expected interpretations and behaviors.", + "required": ["identifier", "severity", "blurs", "locales"], + "properties": { + "identifier": { + "type": "string", + "description": "The value of the label being defined. Must only include lowercase ascii and the '-' character ([a-z-]+).", + "maxLength": 100, + "maxGraphemes": 100 + }, + "severity": { + "type": "string", + "description": "How should a client visually convey this label? 'inform' means neutral and informational; 'alert' means negative and warning; 'none' means show nothing.", + "knownValues": ["inform", "alert", "none"] + }, + "blurs": { + "type": "string", + "description": "What should this label hide in the UI, if applied? 'content' hides all of the target; 'media' hides the images/video/audio; 'none' hides nothing.", + "knownValues": ["content", "media", "none"] + }, + "defaultSetting": { + "type": "string", + "description": "The default setting for this label.", + "knownValues": ["ignore", "warn", "hide"], + "default": "warn" + }, + "adultOnly": { + "type": "boolean", + "description": "Does the user need to have adult content enabled in order to configure this label?" + }, + "locales": { + "type": "array", + "items": { "type": "ref", "ref": "#labelValueDefinitionStrings" } + } + } + }, + "labelValueDefinitionStrings": { + "type": "object", + "description": "Strings which describe the label in the UI, localized into a specific language.", + "required": ["lang", "name", "description"], + "properties": { + "lang": { + "type": "string", + "description": "The code of the language these strings are written in.", + "format": "language" + }, + "name": { + "type": "string", + "description": "A short human-readable name for the label.", + "maxGraphemes": 64, + "maxLength": 640 + }, + "description": { + "type": "string", + "description": "A longer description of what the label means and why it might be applied.", + "maxGraphemes": 10000, + "maxLength": 100000 + } + } + }, + "labelValue": { + "type": "string", + "knownValues": [ + "!hide", + "!warn", + "!no-unauthenticated", + "porn", + "sexual", + "nudity", + "graphic-media", + "bot" + ] + } + } +} diff --git a/lexicons/com/atproto/label/subscribeLabels.json b/lexicons/com/atproto/label/subscribeLabels.json new file mode 100644 index 0000000000000000000000000000000000000000..5fb1a852d3d5484b8b1b2ac97396e844cdad2ccc --- /dev/null +++ b/lexicons/com/atproto/label/subscribeLabels.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "com.atproto.label.subscribeLabels", + "defs": { + "main": { + "type": "subscription", + "description": "Subscribe to stream of labels (and negations). Public endpoint implemented by mod services. Uses same sequencing scheme as repo event stream.", + "parameters": { + "type": "params", + "properties": { + "cursor": { + "type": "integer", + "description": "The last known event seq number to backfill from." + } + } + }, + "message": { + "schema": { + "type": "union", + "refs": ["#labels", "#info"] + } + }, + "errors": [{ "name": "FutureCursor" }] + }, + "labels": { + "type": "object", + "required": ["seq", "labels"], + "properties": { + "seq": { "type": "integer" }, + "labels": { + "type": "array", + "items": { "type": "ref", "ref": "com.atproto.label.defs#label" } + } + } + }, + "info": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string", + "knownValues": ["OutdatedCursor"] + }, + "message": { + "type": "string" + } + } + } + } +} diff --git a/lexicons/com/atproto/repo/strongRef.json b/lexicons/com/atproto/repo/strongRef.json new file mode 100644 index 0000000000000000000000000000000000000000..cb79625165c7ac6ad8b6654e9716d55b78f83511 --- /dev/null +++ b/lexicons/com/atproto/repo/strongRef.json @@ -0,0 +1,15 @@ +{ + "lexicon": 1, + "id": "com.atproto.repo.strongRef", + "description": "A URI with a content-hash fingerprint.", + "defs": { + "main": { + "type": "object", + "required": ["uri", "cid"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" } + } + } + } +} diff --git a/lexicons/com/atproto/sync/subscribeRepos.json b/lexicons/com/atproto/sync/subscribeRepos.json new file mode 100644 index 0000000000000000000000000000000000000000..0edec94ba59df92f17c62559e9c96b1e8740f523 --- /dev/null +++ b/lexicons/com/atproto/sync/subscribeRepos.json @@ -0,0 +1,222 @@ +{ + "lexicon": 1, + "id": "com.atproto.sync.subscribeRepos", + "defs": { + "main": { + "type": "subscription", + "description": "Repository event stream, aka Firehose endpoint. Outputs repo commits with diff data, and identity update events, for all repositories on the current server. See the atproto specifications for details around stream sequencing, repo versioning, CAR diff format, and more. Public and does not require auth; implemented by PDS and Relay.", + "parameters": { + "type": "params", + "properties": { + "cursor": { + "type": "integer", + "description": "The last known event seq number to backfill from." + } + } + }, + "message": { + "schema": { + "type": "union", + "refs": [ + "#commit", + "#sync", + "#identity", + "#account", + "#info", + "org.tangled.git.pushEvent" + ] + } + }, + "errors": [ + { "name": "FutureCursor" }, + { + "name": "ConsumerTooSlow", + "description": "If the consumer of the stream can not keep up with events, and a backlog gets too large, the server will drop the connection." + } + ] + }, + "commit": { + "type": "object", + "description": "Represents an update of repository state. Note that empty commits are allowed, which include no repo data changes, but an update to rev and signature.", + "required": [ + "seq", + "rebase", + "tooBig", + "repo", + "commit", + "rev", + "since", + "blocks", + "ops", + "blobs", + "time" + ], + "nullable": ["since"], + "properties": { + "seq": { + "type": "integer", + "description": "The stream sequence number of this message." + }, + "rebase": { "type": "boolean", "description": "DEPRECATED -- unused" }, + "tooBig": { + "type": "boolean", + "description": "DEPRECATED -- replaced by #sync event and data limits. Indicates that this commit contained too many ops, or data size was too large. Consumers will need to make a separate request to get missing data." + }, + "repo": { + "type": "string", + "format": "did", + "description": "The repo this event comes from. Note that all other message types name this field 'did'." + }, + "commit": { + "type": "cid-link", + "description": "Repo commit object CID." + }, + "rev": { + "type": "string", + "format": "tid", + "description": "The rev of the emitted commit. Note that this information is also in the commit object included in blocks, unless this is a tooBig event." + }, + "since": { + "type": "string", + "format": "tid", + "description": "The rev of the last emitted commit from this repo (if any)." + }, + "blocks": { + "type": "bytes", + "description": "CAR file containing relevant blocks, as a diff since the previous repo state. The commit must be included as a block, and the commit block CID must be the first entry in the CAR header 'roots' list.", + "maxLength": 2000000 + }, + "ops": { + "type": "array", + "items": { + "type": "ref", + "ref": "#repoOp", + "description": "List of repo mutation operations in this commit (eg, records created, updated, or deleted)." + }, + "maxLength": 200 + }, + "blobs": { + "type": "array", + "items": { + "type": "cid-link", + "description": "DEPRECATED -- will soon always be empty. List of new blobs (by CID) referenced by records in this commit." + } + }, + "prevData": { + "type": "cid-link", + "description": "The root CID of the MST tree for the previous commit from this repo (indicated by the 'since' revision field in this message). Corresponds to the 'data' field in the repo commit object. NOTE: this field is effectively required for the 'inductive' version of firehose." + }, + "time": { + "type": "string", + "format": "datetime", + "description": "Timestamp of when this message was originally broadcast." + } + } + }, + "sync": { + "type": "object", + "description": "Updates the repo to a new state, without necessarily including that state on the firehose. Used to recover from broken commit streams, data loss incidents, or in situations where upstream host does not know recent state of the repository.", + "required": ["seq", "did", "blocks", "rev", "time"], + "properties": { + "seq": { + "type": "integer", + "description": "The stream sequence number of this message." + }, + "did": { + "type": "string", + "format": "did", + "description": "The account this repo event corresponds to. Must match that in the commit object." + }, + "blocks": { + "type": "bytes", + "description": "CAR file containing the commit, as a block. The CAR header must include the commit block CID as the first 'root'.", + "maxLength": 10000 + }, + "rev": { + "type": "string", + "description": "The rev of the commit. This value must match that in the commit object." + }, + "time": { + "type": "string", + "format": "datetime", + "description": "Timestamp of when this message was originally broadcast." + } + } + }, + "identity": { + "type": "object", + "description": "Represents a change to an account's identity. Could be an updated handle, signing key, or pds hosting endpoint. Serves as a prod to all downstream services to refresh their identity cache.", + "required": ["seq", "did", "time"], + "properties": { + "seq": { "type": "integer" }, + "did": { "type": "string", "format": "did" }, + "time": { "type": "string", "format": "datetime" }, + "handle": { + "type": "string", + "format": "handle", + "description": "The current handle for the account, or 'handle.invalid' if validation fails. This field is optional, might have been validated or passed-through from an upstream source. Semantics and behaviors for PDS vs Relay may evolve in the future; see atproto specs for more details." + } + } + }, + "account": { + "type": "object", + "description": "Represents a change to an account's status on a host (eg, PDS or Relay). The semantics of this event are that the status is at the host which emitted the event, not necessarily that at the currently active PDS. Eg, a Relay takedown would emit a takedown with active=false, even if the PDS is still active.", + "required": ["seq", "did", "time", "active"], + "properties": { + "seq": { "type": "integer" }, + "did": { "type": "string", "format": "did" }, + "time": { "type": "string", "format": "datetime" }, + "active": { + "type": "boolean", + "description": "Indicates that the account has a repository which can be fetched from the host that emitted this event." + }, + "status": { + "type": "string", + "description": "If active=false, this optional field indicates a reason for why the account is not active.", + "knownValues": [ + "takendown", + "suspended", + "deleted", + "deactivated", + "desynchronized", + "throttled" + ] + } + } + }, + "info": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string", + "knownValues": ["OutdatedCursor"] + }, + "message": { + "type": "string" + } + } + }, + "repoOp": { + "type": "object", + "description": "A repo operation, ie a mutation of a single record.", + "required": ["action", "path", "cid"], + "nullable": ["cid"], + "properties": { + "action": { + "type": "string", + "knownValues": ["create", "update", "delete"] + }, + "path": { "type": "string" }, + "cid": { + "type": "cid-link", + "description": "For creates and updates, the new record CID. For deletions, null." + }, + "prev": { + "type": "cid-link", + "description": "For updates and deletes, the previous record CID (required for inductive firehose). For creations, field should not be defined." + } + } + } + } +} diff --git a/lexicons/com/bad-example/pocket/defs.json b/lexicons/com/bad-example/pocket/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..852a3e4e892ec8977082127ee137642a495a5272 --- /dev/null +++ b/lexicons/com/bad-example/pocket/defs.json @@ -0,0 +1,18 @@ +{ + "lexicon": 1, + "id": "com.bad-example.pocket.defs", + "defs": { + "preferences": { + "type": "array", + "items": { + "type": "union", + "refs": [ + "org.tangled.temp.actor.defs#savedKnotPref", + "org.tangled.temp.actor.defs#savedSpindlePref", + "org.tangled.temp.actor.defs#onboardingPref", + "org.tangled.temp.actor.defs#recentsPref" + ] + } + } + } +} diff --git a/lexicons/com/bad-example/pocket/getPreferences.json b/lexicons/com/bad-example/pocket/getPreferences.json new file mode 100644 index 0000000000000000000000000000000000000000..7c4d46f8de90fbb7fefc1678bfa1df8a3c824faf --- /dev/null +++ b/lexicons/com/bad-example/pocket/getPreferences.json @@ -0,0 +1,30 @@ +{ + "lexicon": 1, + "id": "com.bad-example.pocket.getPreferences", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "properties": {} + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["preferences"], + "properties": { + "preferences": { + "type": "ref", + "ref": "com.bad-example.pocket.defs#preferences" + }, + "rev": { + "type": "string", + "description": "opaque revision of the stored preferences, pass as swapRev. pockets without compare-and-swap leave it out" + } + } + } + } + } + } +} diff --git a/lexicons/com/bad-example/pocket/putPreferences.json b/lexicons/com/bad-example/pocket/putPreferences.json new file mode 100644 index 0000000000000000000000000000000000000000..5a9f39a49b63ec4dcbfeb0e8c57a5125e2cf7541 --- /dev/null +++ b/lexicons/com/bad-example/pocket/putPreferences.json @@ -0,0 +1,32 @@ +{ + "lexicon": 1, + "id": "com.bad-example.pocket.putPreferences", + "defs": { + "main": { + "type": "procedure", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["preferences"], + "nullable": ["swapRev"], + "properties": { + "preferences": { + "type": "ref", + "ref": "com.bad-example.pocket.defs#preferences" + }, + "swapRev": { + "type": "string", + "description": "write only if the stored rev is still this one, or only if nothing is stored yet when null. absent means a blind write" + } + } + } + }, + "errors": [ + { + "name": "InvalidSwap" + } + ] + } + } +} diff --git a/lexicons/git/defs.json b/lexicons/git/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..0df3d1a1bcb5c0aa874eb3ac5908ddc977b06cca --- /dev/null +++ b/lexicons/git/defs.json @@ -0,0 +1,98 @@ +{ + "lexicon": 1, + "id": "sh.tangled.git.defs", + "defs": { + "signature": { + "type": "object", + "required": ["name", "email", "when"], + "properties": { + "name": { + "type": "string", + "description": "Person name" + }, + "email": { + "type": "string", + "description": "Person email" + }, + "when": { + "type": "string", + "format": "datetime", + "description": "Timestamp of the signature" + } + } + }, + "fileDiff": { + "type": "object", + "required": ["lhsSrc", "rhsSrc", "hunks", "hasSyntacticChanges"], + "properties": { + "lhsSrc": { "type": "ref", "ref": "#diffSrc" }, + "rhsSrc": { "type": "ref", "ref": "#diffSrc" }, + "hunks": { "type": "array", "items": { "type": "ref", "ref": "#diffHunk" } }, + "hasByteChanges": { + "type": "ref", + "ref": "#byteChanges", + "description": "Byte count per side, present only when the two sides differ" + }, + "hasSyntacticChanges": { "type": "boolean" } + } + }, + "diffSrc": { + "type": "object", + "description": "One side of a file diff.", + "required": ["path", "oid", "size", "isBinary", "isSubmodule"], + "properties": { + "path": { "type": "string" }, + "oid": { "type": "string", "description": "Blob oid, or the commit oid for a submodule" }, + "size": { "type": "integer", "minimum": 0 }, + "isBinary": { "type": "boolean" }, + "isSubmodule": { "type": "boolean" }, + "content": { + "type": "string", + "description": "Inlined content. Used on interdiff's LHS." + } + } + }, + "diffHunk": { + "type": "object", + "required": ["novelLhs", "novelRhs", "lines"], + "properties": { + "novelLhs": { + "type": "array", + "description": "0-based line numbers holding novel content on the left side", + "items": { "type": "integer", "minimum": 0 } + }, + "novelRhs": { + "type": "array", + "description": "0-based line numbers holding novel content on the right side", + "items": { "type": "integer", "minimum": 0 } + }, + "lines": { "type": "array", "items": { "type": "ref", "ref": "#linePair" } } + } + }, + "linePair": { + "type": "object", + "description": "One aligned line pair; a side is absent when that side has no line.", + "properties": { + "lhs": { "type": "integer", "minimum": 0 }, + "rhs": { "type": "integer", "minimum": 0 } + } + }, + "byteChanges": { + "type": "object", + "required": ["lhs", "rhs"], + "properties": { + "lhs": { "type": "integer", "minimum": 0 }, + "rhs": { "type": "integer", "minimum": 0 } + } + }, + "diffStat": { + "type": "object", + "required": ["insertions", "deletions", "filesChanged"], + "properties": { + "insertions": { "type": "integer", "minimum": 0 }, + "deletions": { "type": "integer", "minimum": 0 }, + "filesChanged": { "type": "integer", "minimum": 0 } + } + } + } +} diff --git a/lexicons/git/mergeCommit.json b/lexicons/git/mergeCommit.json new file mode 100644 index 0000000000000000000000000000000000000000..2cdb5614289cbe76bdc98f511237c05bb159fa16 --- /dev/null +++ b/lexicons/git/mergeCommit.json @@ -0,0 +1,85 @@ +{ + "lexicon": 1, + "id": "sh.tangled.git.mergeCommit", + "defs": { + "main": { + "type": "procedure", + "description": "DEPRECATED: use `org.tangled.temp.git.mergeCommit`", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["target", "source", "style"], + "properties": { + "target": { + "type": "ref", + "ref": "#target" + }, + "source": { + "type": "ref", + "ref": "#source" + }, + "style": { + "type": "string", + "enum": [ + "rebase", + "merge", + "rebase-merge", + "squash-rebase", + "squash-rebase-merge", + "fast-forward-only" + ] + }, + "mergeCommit": { + "type": "ref", + "ref": "#mergeCommitContent", + "description": "Merge commit content. Used when merge commit is created." + } + } + } + } + }, + "target": { + "type": "object", + "required": ["repo", "branch", "commit"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of target git repository" + }, + "branch": { + "type": "string", + "description": "Target branch to merge into" + }, + "commit": { + "type": "string", + "description": "Head commit ID of target branch" + } + } + }, + "source": { + "type": "object", + "required": ["repo", "commit"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of source git repository" + }, + "commit": { + "type": "string", + "description": "Head commit ID to merge" + } + } + }, + "mergeCommitContent": { + "type": "object", + "description": "Merge commit content. Used when merge commit is created.", + "properties": { + "author": { "type": "ref", "ref": "sh.tangled.git.defs#signature" }, + "message": { "type": "string", "maxLength": 1000000 } + } + } + } +} diff --git a/lexicons/git/temp/listIndexedLanguages.json b/lexicons/git/temp/listIndexedLanguages.json new file mode 100644 index 0000000000000000000000000000000000000000..ddaf7ba093db7b080ad8757b2cfd1af1f2da04c8 --- /dev/null +++ b/lexicons/git/temp/listIndexedLanguages.json @@ -0,0 +1,74 @@ +{ + "lexicon": 1, + "id": "sh.tangled.git.temp.listIndexedLanguages", + "defs": { + "main": { + "type": "query", + "description": "Page through the stored default-branch language breakdown of every indexed repository.", + "parameters": { + "type": "params", + "properties": { + "limit": { + "type": "integer", + "description": "Maximum number of repositories to return", + "minimum": 1, + "maximum": 1000, + "default": 500 + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repos"], + "properties": { + "repos": { + "type": "array", + "items": { + "type": "ref", + "ref": "#repoLanguages" + } + }, + "cursor": { + "type": "string" + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "repoLanguages": { + "type": "object", + "required": ["repo", "commit", "languages"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "commit": { + "type": "string", + "description": "Default-branch commit the languages were indexed from" + }, + "languages": { + "type": "array", + "description": "Languages ordered by size, largest first", + "items": { + "type": "ref", + "ref": "sh.tangled.git.temp.listLanguages#language" + } + } + } + } + } +} diff --git a/lexicons/git/temp/listLanguages.json b/lexicons/git/temp/listLanguages.json new file mode 100644 index 0000000000000000000000000000000000000000..349665469b12d909511186d1cd71300af2eeb683 --- /dev/null +++ b/lexicons/git/temp/listLanguages.json @@ -0,0 +1,86 @@ +{ + "lexicon": 1, + "id": "sh.tangled.git.temp.listLanguages", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git reference (branch, tag, or commit SHA)", + "default": "HEAD" + }, + "indexedOnly": { + "type": "boolean", + "description": "Only serve stored languages, failing with NotIndexed rather than analyzing the commit on demand", + "default": false + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["ref", "languages", "total"], + "properties": { + "ref": { + "type": "string", + "description": "The git reference used" + }, + "languages": { + "type": "array", + "items": { + "type": "ref", + "ref": "#language" + } + }, + "total": { + "type": "integer", + "description": "Total size of all analyzed files in bytes" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "NotIndexed", + "description": "No stored languages for this commit yet, only with indexedOnly" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "language": { + "type": "object", + "required": ["name", "size"], + "properties": { + "name": { + "type": "string", + "description": "Programming language name" + }, + "size": { + "type": "integer", + "description": "Total size of files in this language (bytes)" + } + } + } + } +} diff --git a/lexicons/git/temp2/getBlame.json b/lexicons/git/temp2/getBlame.json new file mode 100644 index 0000000000000000000000000000000000000000..f28d676d85805cf172aba69a88b16a6c0ad37103 --- /dev/null +++ b/lexicons/git/temp2/getBlame.json @@ -0,0 +1,136 @@ +{ + "lexicon": 1, + "id": "sh.tangled.git.temp2.getBlame", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo", + "ref", + "path" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Revspec to blame at: a branch, tag, or commit sha" + }, + "path": { + "type": "string", + "description": "Path to the file within the repository" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "commit", + "commits", + "runs" + ], + "properties": { + "commit": { + "type": "string", + "description": "The resolved git commit object id" + }, + "commits": { + "type": "array", + "description": "Unique commits referenced by blame runs", + "items": { + "type": "ref", + "ref": "#commit" + } + }, + "runs": { + "type": "array", + "description": "Complete, gap-free one-based line ranges sorted by start", + "items": { + "type": "ref", + "ref": "#run" + } + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "FileNotFound", + "description": "File not found at the specified path" + }, + { + "name": "BlameTooLarge", + "description": "Blame exceeds file byte, line, or run limits" + }, + { + "name": "BlameTimeout", + "description": "Blame exceeded its computation time budget" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "commit": { + "type": "object", + "required": [ + "oid", + "message", + "author" + ], + "properties": { + "oid": { + "type": "string" + }, + "message": { + "type": "string", + "description": "Full commit message" + }, + "author": { + "type": "ref", + "ref": "sh.tangled.git.defs#signature" + } + } + }, + "run": { + "type": "object", + "required": [ + "start", + "count", + "commit" + ], + "properties": { + "start": { + "type": "integer", + "minimum": 1, + "description": "First one-based line in this run" + }, + "count": { + "type": "integer", + "minimum": 1, + "description": "Number of consecutive lines in this run" + }, + "commit": { + "type": "string", + "description": "oid of a commit in the output commits array" + } + } + } + } +} diff --git a/lexicons/git/temp2/getCommitStats.json b/lexicons/git/temp2/getCommitStats.json new file mode 100644 index 0000000000000000000000000000000000000000..132e072b3b122de33c7a6d1931b809c275cfd3e1 --- /dev/null +++ b/lexicons/git/temp2/getCommitStats.json @@ -0,0 +1,73 @@ +{ + "lexicon": 1, + "id": "sh.tangled.git.temp2.getCommitStats", + "defs": { + "main": { + "type": "query", + "description": "Get an actor's monthly commit counts.", + "parameters": { + "type": "params", + "required": [ + "actor" + ], + "properties": { + "actor": { + "type": "string", + "format": "did", + "description": "Actor whose commits to count." + }, + "months": { + "type": "integer", + "minimum": 1, + "maximum": 24, + "default": 7 + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "months" + ], + "properties": { + "months": { + "type": "array", + "maxLength": 24, + "items": { + "type": "ref", + "ref": "#month" + } + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "month": { + "type": "object", + "required": [ + "start", + "commits" + ], + "properties": { + "start": { + "type": "string", + "format": "datetime", + "description": "Start of this UTC calendar month." + }, + "commits": { + "type": "integer", + "minimum": 0, + "description": "Commits attributed to the actor during this month." + } + } + } + } +} diff --git a/lexicons/git/temp2/getDiff.json b/lexicons/git/temp2/getDiff.json new file mode 100644 index 0000000000000000000000000000000000000000..bc0e6daa6e1c5ecf6fc61a77f5ec541c6d83d5e4 --- /dev/null +++ b/lexicons/git/temp2/getDiff.json @@ -0,0 +1,51 @@ +{ + "lexicon": 1, + "id": "sh.tangled.git.temp2.getDiff", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["baseRepo", "baseCommit", "headRepo", "headCommit"], + "properties": { + "baseRepo": { "type": "string", "format": "did" }, + "baseCommit": { "type": "string" }, + "headRepo": { "type": "string", "format": "did" }, + "headCommit": { "type": "string" } + } + }, + "output": { + "encoding": "application/json", + "description": "TBD", + "schema": { + "type": "object", + "required": ["diffs"], + "properties": { + "diffs": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.git.defs#fileDiff" } + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RevisionNotFound", + "description": "One or both revisions not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "CompareError", + "description": "Failed to compare revisions" + } + ] + } + } +} diff --git a/lexicons/git/temp2/getInterdiff.json b/lexicons/git/temp2/getInterdiff.json new file mode 100644 index 0000000000000000000000000000000000000000..08c6b20549c3d7282fb0bfb7a99633da5b608837 --- /dev/null +++ b/lexicons/git/temp2/getInterdiff.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "sh.tangled.git.temp2.getInterdiff", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "baseRepo", + "baseCommit1", + "baseCommit2", + "headRepo", + "headCommit1", + "headCommit2" + ], + "properties": { + "baseRepo": { "type": "string", "format": "did" }, + "baseCommit1": { "type": "string" }, + "baseCommit2": { "type": "string" }, + "headRepo": { "type": "string", "format": "did" }, + "headCommit1": { "type": "string" }, + "headCommit2": { "type": "string" } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["diffs"], + "properties": { + "diffs": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.git.defs#fileDiff" } + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RevisionNotFound", + "description": "One or both revisions not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "CompareError", + "description": "Failed to compare revisions" + } + ] + } + } +} diff --git a/lexicons/git/temp2/mergeCheck.json b/lexicons/git/temp2/mergeCheck.json new file mode 100644 index 0000000000000000000000000000000000000000..84cd70e4f793445ca2758d63d7e5a6f91c839ce0 --- /dev/null +++ b/lexicons/git/temp2/mergeCheck.json @@ -0,0 +1,75 @@ +{ + "lexicon": 1, + "id": "sh.tangled.git.temp2.mergeCheck", + "defs": { + "main": { + "type": "query", + "description": "3-way merge check between a target branch tip (ours) and a source commit (theirs). Read-only: the merged tree is never persisted.", + "parameters": { + "type": "params", + "required": ["targetRepo", "targetCommit", "sourceRepo", "sourceCommit"], + "properties": { + "targetRepo": { "type": "string", "format": "did" }, + "targetCommit": { + "type": "string", + "description": "Target branch tip to merge into (ours)" + }, + "sourceRepo": { "type": "string", "format": "did" }, + "sourceCommit": { + "type": "string", + "description": "Source commit to merge (theirs)" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["isConflicted", "conflicts"], + "properties": { + "isConflicted": { + "type": "boolean", + "description": "Whether the merge has unresolved conflicts" + }, + "conflicts": { + "type": "array", + "items": { "type": "ref", "ref": "#conflict" } + }, + "message": { + "type": "string", + "description": "Additional detail, e.g. unrelated histories" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RevisionNotFound", + "description": "One or both commits not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "conflict": { + "type": "object", + "required": ["filename", "reason"], + "properties": { + "filename": { + "type": "string", + "description": "Path of the conflicted file" + }, + "reason": { + "type": "string", + "description": "Short classification of the conflict" + } + } + } + } +} diff --git a/lexicons/identity/resolveCommitters.json b/lexicons/identity/resolveCommitters.json new file mode 100644 index 0000000000000000000000000000000000000000..aa7de70ebfb393dd4b5acceb8c94b487f3f5be80 --- /dev/null +++ b/lexicons/identity/resolveCommitters.json @@ -0,0 +1,53 @@ +{ + "lexicon": 1, + "id": "sh.tangled.identity.resolveCommitters", + "defs": { + "main": { + "type": "procedure", + "description": "Resolve an actor DID to its verified Git committer identities.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "actor" + ], + "properties": { + "actor": { + "type": "string", + "format": "did" + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "committers" + ], + "properties": { + "committers": { + "type": "array", + "maxLength": 251, + "items": { + "type": "string" + } + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "The actor is not a valid DID" + }, + { + "name": "Forbidden", + "description": "The calling service is not authorized" + } + ] + } + } +} diff --git a/lexicons/org/tangled/actor/defs.json b/lexicons/org/tangled/actor/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..818f4bc7cad6cd8110ba26a03500b3d8008be3a7 --- /dev/null +++ b/lexicons/org/tangled/actor/defs.json @@ -0,0 +1,65 @@ +{ + "lexicon": 1, + "id": "org.tangled.actor.defs", + "defs": { + "profileViewBasic": { + "type": "object", + "required": ["did", "handle"], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string", + "format": "handle" + }, + "avatar": { + "type": "string", + "format": "uri" + }, + "avatarCid": { + "type": "string", + "format": "cid", + "description": "The cid of the avatar blob in the account's profile record. Absent when the record has no avatar or bobbin couldn't read it." + }, + "viewer": { + "type": "ref", + "ref": "#viewerState" + } + } + }, + "profileViewDetailed": { + "type": "object", + "description": "profileViewBasic + follow counts", + "required": ["did", "handle"], + "properties": { + "did": { "type": "string", "format": "did" }, + "handle": { "type": "string", "format": "handle" }, + "avatar": { "type": "string", "format": "uri" }, + "avatarCid": { + "type": "string", + "format": "cid", + "description": "The cid of the avatar blob in the account's profile record. Absent when the record has no avatar or bobbin couldn't read it." + }, + "followersCount": { "type": "integer" }, + "followsCount": { "type": "integer" }, + "viewer": { "type": "ref", "ref": "#viewerState" } + } + }, + "viewerState": { + "type": "object", + "description": "Metadata about the requesting account's relationship with the subject account. Only has meaningful content for authed requests.", + "properties": { + "following": { + "type": "string", + "format": "at-uri" + }, + "followedBy": { + "type": "string", + "format": "at-uri" + } + } + } + } +} diff --git a/lexicons/org/tangled/actor/getProfile.json b/lexicons/org/tangled/actor/getProfile.json new file mode 100644 index 0000000000000000000000000000000000000000..30ec3c76d58dd52df46fbb9cbda08d01ec74af62 --- /dev/null +++ b/lexicons/org/tangled/actor/getProfile.json @@ -0,0 +1,41 @@ +{ + "lexicon": 1, + "id": "org.tangled.actor.getProfile", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["actor"], + "properties": { + "actor": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the sh.tangled.actor.profile record to fetch." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.actor.profile record." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/actor/getProfileActivity.json b/lexicons/org/tangled/actor/getProfileActivity.json new file mode 100644 index 0000000000000000000000000000000000000000..7b46af13851a7f8ddf87b28d3bee61d706430094 --- /dev/null +++ b/lexicons/org/tangled/actor/getProfileActivity.json @@ -0,0 +1,205 @@ +{ + "lexicon": 1, + "id": "org.tangled.actor.getProfileActivity", + "defs": { + "main": { + "type": "query", + "description": "Get recent Tangled activity for an actor, grouped by calendar month.", + "parameters": { + "type": "params", + "required": [ + "actor" + ], + "properties": { + "actor": { + "type": "string", + "format": "did", + "description": "Actor whose activity to return." + }, + "months": { + "type": "integer", + "minimum": 1, + "maximum": 12, + "default": 7, + "description": "Number of UTC calendar months to include, including the current month." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "months", + "truncated" + ], + "properties": { + "months": { + "type": "array", + "items": { + "type": "ref", + "ref": "#activityMonth" + } + }, + "truncated": { + "type": "boolean", + "description": "True when the result reached the server activity-item safety limit." + } + } + } + } + }, + "activityMonth": { + "type": "object", + "required": [ + "start", + "commits", + "repos", + "issues", + "pulls" + ], + "properties": { + "start": { + "type": "string", + "format": "datetime", + "description": "Start of this UTC calendar month." + }, + "commits": { + "type": "integer", + "minimum": 0 + }, + "repos": { + "type": "array", + "items": { + "type": "ref", + "ref": "#repoActivity" + } + }, + "issues": { + "type": "array", + "items": { + "type": "ref", + "ref": "#issueActivity" + } + }, + "pulls": { + "type": "array", + "items": { + "type": "ref", + "ref": "#pullActivity" + } + } + } + }, + "repoRef": { + "type": "object", + "required": [ + "uri", + "ownerDid", + "ownerHandle", + "name" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "did": { + "type": "string", + "format": "did" + }, + "ownerDid": { + "type": "string", + "format": "did" + }, + "ownerHandle": { + "type": "string", + "format": "handle" + }, + "name": { + "type": "string" + } + } + }, + "repoActivity": { + "type": "object", + "required": [ + "repo", + "kind" + ], + "properties": { + "repo": { + "type": "ref", + "ref": "#repoRef" + }, + "kind": { + "type": "string", + "knownValues": [ + "created", + "forked" + ] + } + } + }, + "issueActivity": { + "type": "object", + "required": [ + "uri", + "title", + "state", + "repo" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "title": { + "type": "string" + }, + "state": { + "type": "string", + "enum": [ + "open", + "closed", + "completed" + ] + }, + "repo": { + "type": "ref", + "ref": "#repoRef" + } + } + }, + "pullActivity": { + "type": "object", + "required": [ + "uri", + "title", + "state", + "repo" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "title": { + "type": "string" + }, + "state": { + "type": "string", + "enum": [ + "open", + "closed", + "completed" + ] + }, + "repo": { + "type": "ref", + "ref": "#repoRef" + } + } + } + } +} diff --git a/lexicons/org/tangled/actor/getProfiles.json b/lexicons/org/tangled/actor/getProfiles.json new file mode 100644 index 0000000000000000000000000000000000000000..2038201367014390751e6f7f3b5c56f00b5dc57c --- /dev/null +++ b/lexicons/org/tangled/actor/getProfiles.json @@ -0,0 +1,58 @@ +{ + "lexicon": 1, + "id": "org.tangled.actor.getProfiles", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["actors"], + "properties": { + "actors": { + "type": "array", + "items": { + "type": "string", + "format": "at-uri" + }, + "maxLength": 50, + "description": "AT-URIs of the sh.tangled.actor.profile records to fetch. At most 50 per request." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "#recordView" + } + } + } + } + } + }, + "recordView": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.actor.profile record." + } + } + } + } +} diff --git a/lexicons/org/tangled/actor/getTrending.json b/lexicons/org/tangled/actor/getTrending.json new file mode 100644 index 0000000000000000000000000000000000000000..310b153d04935e40034a9003997fc6b012feef8e --- /dev/null +++ b/lexicons/org/tangled/actor/getTrending.json @@ -0,0 +1,35 @@ +{ + "lexicon": 1, + "id": "org.tangled.actor.getTrending", + "defs": { + "main": { + "type": "query", + "description": "Get trending people and repositories.", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["people", "repos"], + "properties": { + "people": { + "type": "array", + "maxLength": 5, + "items": { + "type": "ref", + "ref": "org.tangled.actor.defs#profileViewDetailed" + } + }, + "repos": { + "type": "array", + "maxLength": 4, + "items": { + "type": "ref", + "ref": "org.tangled.repo.defs#repoViewBasic" + } + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/actor/profile.json b/lexicons/org/tangled/actor/profile.json new file mode 100644 index 0000000000000000000000000000000000000000..4a9e6c23f1b2c3186f30a143ff4a1f7ba715c53f --- /dev/null +++ b/lexicons/org/tangled/actor/profile.json @@ -0,0 +1,86 @@ +{ + "lexicon": 1, + "id": "org.tangled.actor.profile", + "defs": { + "main": { + "type": "record", + "description": "A declaration of a Tangled account profile.", + "key": "literal:self", + "record": { + "type": "object", + "required": [], + "properties": { + "avatar": { + "type": "blob", + "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'", + "maxSize": 1000000 + }, + "description": { + "type": "string", + "description": "Free-form profile description text.", + "maxGraphemes": 256, + "maxLength": 2560 + }, + "links": { + "type": "array", + "maxLength": 5, + "items": { + "type": "string", + "description": "Any URI, intended for social profiles or websites, can be used to link DIDs/AT-URIs too.", + "format": "uri" + } + }, + "isOrganization": { "type": "boolean" }, + "stats": { + "type": "array", + "maxLength": 2, + "items": { + "type": "string", + "description": "Vanity stats.", + "knownValues": [ + "merged-pull-request-count", + "closed-pull-request-count", + "open-pull-request-count", + "open-issue-count", + "closed-issue-count", + "repository-count", + "star-count" + ] + } + }, + "location": { + "type": "string", + "description": "Free-form location text.", + "maxGraphemes": 40, + "maxLength": 400 + }, + "pinnedRepositories": { + "type": "array", + "description": "Pinned repositories. Values are repo DIDs for repos that have them, or AT-URIs for legacy repos.", + "maxLength": 6, + "items": { + "type": "string", + "format": "did" + } + }, + "pronouns": { + "type": "string", + "description": "Preferred gender pronouns.", + "maxGraphemes": 30, + "maxLength": 200 + }, + "preferredHandle": { + "type": "string", + "description": "A handle the user prefers to be displayed as.", + "format": "handle" + }, + "labels": { + "type": "ref", + "description": "Self-label values for this record. Most common use-case is 'bot'.", + "ref": "com.atproto.label.defs#selfLabels" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/actor/searchActorsTypeahead.json b/lexicons/org/tangled/actor/searchActorsTypeahead.json new file mode 100644 index 0000000000000000000000000000000000000000..5174b8a2d234d19bfaf0e36bbfc75c7c061fb54d --- /dev/null +++ b/lexicons/org/tangled/actor/searchActorsTypeahead.json @@ -0,0 +1,63 @@ +{ + "lexicon": 1, + "id": "org.tangled.actor.searchActorsTypeahead", + "defs": { + "main": { + "type": "query", + "description": "Find actors by their verified handle.", + "parameters": { + "type": "params", + "required": ["q"], + "properties": { + "q": { + "type": "string", + "minLength": 1, + "maxLength": 254, + "description": "The beginning of a handle. A leading @ is optional." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 10 + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["actors"], + "properties": { + "actors": { + "type": "array", + "items": { + "type": "ref", + "ref": "#actor" + } + } + } + } + } + }, + "actor": { + "type": "object", + "required": ["uri", "did", "handle"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "AT-URI for the actor's Tangled profile." + }, + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string", + "format": "handle" + } + } + } + } +} diff --git a/lexicons/org/tangled/bobbin/awaitRecord.json b/lexicons/org/tangled/bobbin/awaitRecord.json new file mode 100644 index 0000000000000000000000000000000000000000..eb0ce858643aab600e93252898371ce63a2790b5 --- /dev/null +++ b/lexicons/org/tangled/bobbin/awaitRecord.json @@ -0,0 +1,71 @@ +{ + "lexicon": 1, + "id": "org.tangled.bobbin.awaitRecord", + "defs": { + "main": { + "type": "query", + "description": "Wait for a record written to a PDS to reach this bobbin instance, then report what happened to it.", + "parameters": { + "type": "params", + "required": ["uri"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the record that was written." + }, + "cid": { + "type": "string", + "format": "cid", + "description": "CID of the revision that was written, as returned by createRecord, putRecord or applyWrites. Omit it only when awaiting a delete, which has no CID. any other responds with whatever revision of the record settles first." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["status"], + "properties": { + "status": { + "type": "string", + "knownValues": ["indexed", "deleted", "rejected"], + "description": "What bobbin did with the record." + }, + "cid": { + "type": "string", + "format": "cid", + "description": "CID of the revision this result is about." + }, + "reason": { + "type": "string", + "knownValues": [ + "unknownCollection", + "undecodable", + "invalid", + "missingBody", + "notIndexed" + ], + "description": "Why a rejected record will not appear." + }, + "detail": { + "type": "string", + "maxLength": 200, + "description": "Opaque error message which can be more detailed than the reason." + } + } + } + }, + "errors": [ + { + "name": "NotSettled", + "description": "The record had not reached the appview before the wait ran out" + }, + { + "name": "Overloaded", + "description": "Too many callers are already waiting, retry later or poll the record instead" + } + ] + } + } +} diff --git a/lexicons/org/tangled/ci/approvePipeline.json b/lexicons/org/tangled/ci/approvePipeline.json new file mode 100644 index 0000000000000000000000000000000000000000..987544bf7f494ffe6941e4c1ada35975874463c7 --- /dev/null +++ b/lexicons/org/tangled/ci/approvePipeline.json @@ -0,0 +1,43 @@ +{ + "lexicon": 1, + "id": "org.tangled.ci.approvePipeline", + "defs": { + "main": { + "type": "procedure", + "description": "Approve blocked pipeline.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "pipeline" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "git repository DID" + }, + "pipeline": { + "type": "string", + "format": "record-key", + "description": "pipeline id" + } + } + } + }, + "errors": [ + { + "name": "PipelineNotFound", + "description": "Pipeline not found" + }, + { + "name": "PipelineNotBlocked", + "description": "Pipeline has no workflow waiting for approval" + } + ] + } + } +} + diff --git a/lexicons/org/tangled/ci/cancelPipeline.json b/lexicons/org/tangled/ci/cancelPipeline.json new file mode 100644 index 0000000000000000000000000000000000000000..5987b196cb5fc93ffb24a99fef99075201cd10dc --- /dev/null +++ b/lexicons/org/tangled/ci/cancelPipeline.json @@ -0,0 +1,53 @@ +{ + "lexicon": 1, + "id": "org.tangled.ci.cancelPipeline", + "defs": { + "main": { + "type": "procedure", + "description": "Cancel running pipeline or specific workflows", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "pipeline" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "git repository DID" + }, + "pipeline": { + "type": "string", + "description": "Spindle-local pipeline id" + }, + "workflows": { + "type": "array", + "items": { + "type": "string", + "description": "workflow name" + }, + "description": "Workflow names to filter. When not provided, entire pipeline will be canceled." + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "RepoNotFound", + "description": "Repository not found" + }, + { + "name": "PipelineNotFound", + "description": "Pipeline not found in this repository" + } + ] + } + } +} diff --git a/lexicons/org/tangled/ci/defs.json b/lexicons/org/tangled/ci/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..968008b2b4d96ad54fac0c150943c83775be6f0a --- /dev/null +++ b/lexicons/org/tangled/ci/defs.json @@ -0,0 +1,219 @@ +{ + "lexicon": 1, + "id": "org.tangled.ci.defs", + "defs": { + "pipeline": { + "type": "object", + "description": "A CI pipeline. Record-like, but owned by the spindle rather than a PDS.", + "required": [ + "id", + "repo", + "trigger", + "commit", + "workflows" + ], + "properties": { + "id": { + "type": "string", + "format": "record-key", + "description": "Spindle-local pipeline id" + }, + "repo": { + "type": "string", + "format": "did", + "description": "Repository DID" + }, + "trigger": { + "type": "union", + "closed": false, + "refs": [ + "org.tangled.event.push", + "org.tangled.event.pullRequest", + "org.tangled.ci.trigger.manual", + "org.tangled.ci.trigger.schedule" + ], + "description": "Trigger event that triggered this pipeline." + }, + "commit": { + "type": "string", + "description": "Commit Id this pipeline is running on" + }, + "sourceRepo": { + "type": "string", + "format": "did", + "description": "Repository DID that the commit was checked out from, if different from repo (e.g. a fork for a fork-based pull request)" + }, + "createdAt": { + "type": "string", + "format": "datetime" + }, + "workflows": { + "type": "array", + "description": "Triggered workflows", + "minLength": 1, + "maxLength": 50, + "items": { + "type": "ref", + "ref": "#workflowRun" + } + } + } + }, + "workflowRun": { + "type": "object", + "required": [ + "definition", + "id", + "status" + ], + "properties": { + "id": { + "type": "string", + "description": "Pipeline-local workflow id, usually same as name.", + "minLength": 1, + "maxLength": 128 + }, + "name": { + "type": "string", + "description": "Human-readable name of the workflow", + "maxLength": 128 + }, + "status": { + "type": "string", + "description": "Workflow status", + "enum": [ + "pending", + "blocked", + "running", + "failed", + "timeout", + "cancelled", + "success" + ] + }, + "startedAt": { + "type": "string", + "format": "datetime" + }, + "finishedAt": { + "type": "string", + "format": "datetime" + }, + "error": { + "type": "string" + }, + "definition": { + "type": "ref", + "ref": "#workflow", + "description": "Workflow definition where this workflow run is originated from." + } + } + }, + "workflow": { + "type": "object", + "description": "A workflow definition", + "required": [ + "id", + "source", + "triggers" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "hash": { + "type": "string", + "description": "Fingerprint of the workflow definition as resolved by the spindle." + }, + "name": { + "type": "string", + "description": "Human-readable name of the workflow", + "minLength": 1, + "maxLength": 128 + }, + "source": { + "type": "union", + "refs": [ + "#fileSource", + "#externalSource" + ] + }, + "triggers": { + "type": "array", + "description": "Event types that triggers this workflow.", + "items": { + "type": "string", + "knownValues": [ + "org.tangled.event.push", + "org.tangled.event.pullRequest", + "org.tangled.ci.trigger.manual", + "org.tangled.ci.trigger.schedule" + ] + } + } + } + }, + "fileSource": { + "type": "object", + "required": [ + "repo", + "commit", + "path" + ], + "properties": { + "repo": { + "type": "string", + "format": "did" + }, + "commit": { + "type": "string", + "minLength": 40, + "maxLength": 128 + }, + "path": { + "type": "string", + "minLength": 1 + }, + "lines": { + "type": "ref", + "ref": "#lineRange" + } + } + }, + "lineRange": { + "type": "object", + "required": [ + "start", + "end" + ], + "properties": { + "start": { + "type": "integer", + "minimum": 1 + }, + "end": { + "type": "integer", + "minimum": 1 + } + } + }, + "externalSource": { + "type": "object", + "required": [ + "link" + ], + "properties": { + "name": { + "type": "string", + "description": "human-readable name of external source" + }, + "link": { + "type": "string", + "format": "uri" + } + } + } + } +} diff --git a/lexicons/org/tangled/ci/getPipeline.json b/lexicons/org/tangled/ci/getPipeline.json new file mode 100644 index 0000000000000000000000000000000000000000..13378c8905ac71049dd3b25f3d4187010abbb601 --- /dev/null +++ b/lexicons/org/tangled/ci/getPipeline.json @@ -0,0 +1,44 @@ +{ + "lexicon": 1, + "id": "org.tangled.ci.getPipeline", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo", + "pipeline" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "Repository DID" + }, + "pipeline": { + "type": "string", + "description": "Spindle-local pipeline id" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "ref", + "ref": "org.tangled.ci.defs#pipeline" + } + }, + "errors": [ + { + "name": "PipelineNotFound", + "description": "Pipeline not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/ci/getWorkflowDefinition.json b/lexicons/org/tangled/ci/getWorkflowDefinition.json new file mode 100644 index 0000000000000000000000000000000000000000..7a7731423d1561b34d787fee0b97046bfdf49de1 --- /dev/null +++ b/lexicons/org/tangled/ci/getWorkflowDefinition.json @@ -0,0 +1,56 @@ +{ + "lexicon": 1, + "id": "org.tangled.ci.getWorkflowDefinition", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo", + "workflow" + ], + "properties": { + "repo": { + "type": "string", + "format": "did" + }, + "workflow": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "workflow" + ], + "properties": { + "workflow": { + "type": "ref", + "ref": "org.tangled.ci.defs#workflow" + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "RepoNotFound", + "description": "Repository not found" + }, + { + "name": "WorkflowNotFound", + "description": "Workflow not found" + } + ] + } + } +} diff --git a/lexicons/org/tangled/ci/listWorkflowDefinitions.json b/lexicons/org/tangled/ci/listWorkflowDefinitions.json new file mode 100644 index 0000000000000000000000000000000000000000..6d5c9e40abdf62a4f3f8a38ddcdbfcaf6bf83b9e --- /dev/null +++ b/lexicons/org/tangled/ci/listWorkflowDefinitions.json @@ -0,0 +1,58 @@ +{ + "lexicon": 1, + "id": "org.tangled.ci.listWorkflowDefinitions", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo" + ], + "properties": { + "repo": { + "type": "string", + "format": "did" + }, + "ref": { + "type": "string", + "default": "HEAD", + "maxLength": 256 + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "workflows" + ], + "properties": { + "workflows": { + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.ci.defs#workflow" + } + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "RepoNotFound", + "description": "Repository not found" + }, + { + "name": "RefNotFound", + "description": "No branch or tag of the repository has this name" + } + ] + } + } +} diff --git a/lexicons/org/tangled/ci/queryPipelines.json b/lexicons/org/tangled/ci/queryPipelines.json new file mode 100644 index 0000000000000000000000000000000000000000..180d7fdbd016d85df4d8e94e10ca328a0b39ee4d --- /dev/null +++ b/lexicons/org/tangled/ci/queryPipelines.json @@ -0,0 +1,92 @@ +{ + "lexicon": 1, + "id": "org.tangled.ci.queryPipelines", + "defs": { + "main": { + "type": "query", + "description": "Query pipelines in a git repository", + "parameters": { + "type": "params", + "required": [ + "repo" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "commits": { + "type": "array", + "items": { + "type": "string", + "maxLength": 64, + "description": "commit id (SHA-1 or SHA-256)" + }, + "description": "Filter pipelines by commit ids." + }, + "triggers": { + "type": "array", + "items": { + "type": "string", + "knownValues": [ + "org.tangled.event.push", + "org.tangled.event.pullRequest", + "org.tangled.ci.trigger.manual", + "org.tangled.ci.trigger.schedule" + ] + }, + "description": "Filter pipelines by trigger kind. When provided, pipelines matching any listed kind are returned; when omitted, every kind is returned." + }, + "limit": { + "type": "integer", + "description": "Maximum number of pipelines to return", + "minimum": 1, + "maximum": 250, + "default": 50 + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "total", + "pipelines" + ], + "properties": { + "cursor": { + "type": "string" + }, + "total": { + "type": "integer", + "description": "Number of matching pipelines" + }, + "pipelines": { + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.ci.defs#pipeline" + } + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "RepoNotFound", + "description": "Repository not found" + } + ] + } + } +} diff --git a/lexicons/org/tangled/ci/retryPipeline.json b/lexicons/org/tangled/ci/retryPipeline.json new file mode 100644 index 0000000000000000000000000000000000000000..703f1606f08b031e9be08abbd825562e9db04b93 --- /dev/null +++ b/lexicons/org/tangled/ci/retryPipeline.json @@ -0,0 +1,64 @@ +{ + "lexicon": 1, + "id": "org.tangled.ci.retryPipeline", + "defs": { + "main": { + "type": "procedure", + "description": "Duplicate existing pipeline for rerun.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "pipeline" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "git repository DID" + }, + "pipeline": { + "type": "string", + "format": "record-key", + "description": "pipeline id" + }, + "workflows": { + "type": "array", + "items": { + "type": "string", + "description": "workflow name" + }, + "description": "Workflow names to filter. When not provided, entire pipeline will be duplicated." + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["pipeline"], + "properties": { + "pipeline": { + "type": "string", + "format": "record-key", + "description": "Scheduled pipeline id" + } + } + } + }, + "errors": [ + { + "name": "PipelineNotFound", + "description": "Pipeline not found" + }, + { + "name": "PipelineNotRetryable", + "description": "Pipeline is still blocked or has no stored spec to rerun" + } + ] + } + } +} diff --git a/lexicons/org/tangled/ci/subscribePipelineLogs.json b/lexicons/org/tangled/ci/subscribePipelineLogs.json new file mode 100644 index 0000000000000000000000000000000000000000..40425710048c6386dafcef21fec27433af1f87eb --- /dev/null +++ b/lexicons/org/tangled/ci/subscribePipelineLogs.json @@ -0,0 +1,147 @@ +{ + "lexicon": 1, + "id": "org.tangled.ci.subscribePipelineLogs", + "defs": { + "main": { + "type": "subscription", + "description": "Pipeline logs stream", + "parameters": { + "type": "params", + "required": [ + "repo", + "pipeline" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "Repository DID" + }, + "pipeline": { + "type": "string", + "description": "Pipeline ID" + }, + "workflows": { + "type": "array", + "items": { + "type": "string", + "description": "Workflow name" + }, + "description": "filter logs by specific workflows" + } + } + }, + "message": { + "schema": { + "type": "union", + "refs": [ + "#control", + "#data" + ] + } + }, + "errors": [ + { + "name": "PipelineNotFound", + "description": "Pipeline not found" + }, + { + "name": "WorkflowNotFound", + "description": "Workflow not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "control": { + "type": "object", + "required": [ + "time", + "workflow", + "step", + "content" + ], + "properties": { + "time": { + "type": "string", + "format": "datetime" + }, + "workflow": { + "type": "string", + "description": "workflow name" + }, + "step": { + "type": "integer", + "description": "Step ID" + }, + "content": { + "type": "string" + }, + "command": { + "type": "string", + "description": "Step command" + }, + "status": { + "type": "string", + "enum": [ + "start", + "end" + ], + "description": "Step status" + }, + "kind": { + "type": "string", + "enum": [ + "system", + "user" + ], + "description": "Step kind" + }, + "outcome": { + "type": "string", + "enum": [ + "success", + "failed" + ], + "description": "How the step ended, only set on end frames for steps that can fail" + } + } + }, + "data": { + "type": "object", + "required": [ + "time", + "workflow", + "step", + "content", + "stream" + ], + "properties": { + "time": { + "type": "string", + "format": "datetime" + }, + "workflow": { + "type": "string", + "description": "workflow name" + }, + "step": { + "type": "integer", + "description": "Step ID" + }, + "content": { + "type": "string" + }, + "stream": { + "type": "string", + "enum": [ + "stdout", + "stderr" + ] + } + } + } + } +} diff --git a/lexicons/org/tangled/ci/trigger/manual.json b/lexicons/org/tangled/ci/trigger/manual.json new file mode 100644 index 0000000000000000000000000000000000000000..71932c8a29638d81c192b44b205deeec0db3fa81 --- /dev/null +++ b/lexicons/org/tangled/ci/trigger/manual.json @@ -0,0 +1,55 @@ +{ + "lexicon": 1, + "id": "org.tangled.ci.trigger.manual", + "needsCbor": true, + "needsType": true, + "defs": { + "main": { + "type": "object", + "required": ["repo", "commit"], + "properties": { + "repo": { + "type": "string", + "format": "did" + }, + "commit": { + "type": "string", + "description": "Commit id the pipeline will run on.", + "minLength": 40, + "maxLength": 64 + }, + "ref": { + "type": "string", + "description": "Optional ref the commit was resolved from." + }, + "workflows": { + "type": "array", + "items": { + "type": "string", + "description": "workflow name" + }, + "description": "Workflow names to run. When not provided, every dispatchable workflow is run." + }, + "inputs": { + "type": "array", + "items": { + "type": "ref", + "ref": "#pair" + } + } + } + }, + "pair": { + "type": "object", + "required": ["key", "value"], + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + } +} diff --git a/lexicons/org/tangled/ci/trigger/schedule.json b/lexicons/org/tangled/ci/trigger/schedule.json new file mode 100644 index 0000000000000000000000000000000000000000..7339fb06bc2b8d355a77fd29add45abd580e5f82 --- /dev/null +++ b/lexicons/org/tangled/ci/trigger/schedule.json @@ -0,0 +1,34 @@ +{ + "lexicon": 1, + "id": "org.tangled.ci.trigger.schedule", + "needsCbor": true, + "needsType": true, + "defs": { + "main": { + "type": "object", + "description": "A pipeline the spindle started on a workflow's cron schedule.", + "required": ["repo", "commit", "ref", "scheduledAt"], + "properties": { + "repo": { + "type": "string", + "format": "did" + }, + "commit": { + "type": "string", + "description": "Commit id the pipeline ran on, the head of ref at dispatch.", + "minLength": 40, + "maxLength": 64 + }, + "ref": { + "type": "string", + "description": "Branch the commit was resolved from, the repository's default branch." + }, + "scheduledAt": { + "type": "string", + "format": "datetime", + "description": "The schedule tick this run is for, to the minute." + } + } + } + } +} diff --git a/lexicons/org/tangled/embed/commit.json b/lexicons/org/tangled/embed/commit.json new file mode 100644 index 0000000000000000000000000000000000000000..1f952ee9ff3571ee1392d542f3837ad6680d028a --- /dev/null +++ b/lexicons/org/tangled/embed/commit.json @@ -0,0 +1,29 @@ +{ + "lexicon": 1, + "id": "org.tangled.embed.commit", + "defs": { + "main": { + "type": "object", + "required": [ + "repo", + "commit" + ], + "properties": { + "repo": { "type": "string", "format": "did" }, + "commit": { "type": "ref", "ref": "org.tangled.git.oid" } + } + }, + "view": { + "type": "object", + "required": [ + "repo", + "commit" + ], + "properties": { + "repo": { "type": "string", "format": "did" }, + "commit": { "type": "string", "maxLength": 64 }, + "change": { "type": "string", "maxLength": 32 } + } + } + } +} diff --git a/lexicons/org/tangled/embed/unspecced/file.json b/lexicons/org/tangled/embed/unspecced/file.json new file mode 100644 index 0000000000000000000000000000000000000000..897385e8ea71d70af93b29f7bcddba587e224967 --- /dev/null +++ b/lexicons/org/tangled/embed/unspecced/file.json @@ -0,0 +1,19 @@ +{ + "lexicon": 1, + "id": "org.tangled.embed.unspecced.file", + "defs": { + "main": { + "type": "object", + "required": [ + "repo", + "commit", + "path" + ], + "properties": { + "repo": { "type": "string", "format": "did" }, + "commit": { "type": "ref", "ref": "org.tangled.git.oid" }, + "path": { "type": "string" } + } + } + } +} diff --git a/lexicons/org/tangled/event/notifyEvent.json b/lexicons/org/tangled/event/notifyEvent.json new file mode 100644 index 0000000000000000000000000000000000000000..0b5fba03c06276b9799e35ed80b4033d401c292d --- /dev/null +++ b/lexicons/org/tangled/event/notifyEvent.json @@ -0,0 +1,64 @@ +{ + "lexicon": 1, + "id": "org.tangled.event.notifyEvent", + "defs": { + "main": { + "type": "procedure", + "description": "notify an event from external source", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repo", "event"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "Repository DID" + }, + "event": { + "type": "union", + "refs": [ + "org.tangled.ci.trigger.manual", + "org.tangled.event.push" + ], + "description": "Tangled event object" + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [], + "properties": { + "ci": { "type": "ref", "ref": "#ciOutput" } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "ciOutput": { + "type": "object", + "required": [], + "properties": { + "pipelines": { + "type": "array", + "items": { "type": "string", "format": "record-key" }, + "description": "IDs of the scheduled pipelines" + }, + "message": { + "type": "string", + "maxLength": 1000, + "description": "Text returned by the server for this push, like how to follow a started CI." + } + } + } + } +} diff --git a/lexicons/org/tangled/event/pullRequest.json b/lexicons/org/tangled/event/pullRequest.json new file mode 100644 index 0000000000000000000000000000000000000000..ceef166f51ba49e932182775d8804a302c08e620 --- /dev/null +++ b/lexicons/org/tangled/event/pullRequest.json @@ -0,0 +1,47 @@ +{ + "lexicon": 1, + "id": "org.tangled.event.pullRequest", + "needsCbor": true, + "needsType": true, + "defs": { + "main": { + "type": "object", + "description": "When pull request is created or resubmitted.", + "properties": { + "action": { + "type": "string", + "description": "The pull request lifecycle action that produced this trigger.", + "enum": [ + "opened", + "reopened", + "closed", + "merged", + "synchronize" + ] + }, + "pull": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the pull request record this run belongs to." + }, + "sourceRepo": { + "type": "string", + "format": "did", + "description": "Repository DID to check out code and workflow definitions from, if different from the target repo." + }, + "sourceBranch": { + "type": "string" + }, + "targetBranch": { + "type": "string" + }, + "sourceSha": { + "type": "string", + "description": "Commit id of the pull request head.", + "minLength": 40, + "maxLength": 64 + } + } + } + } +} diff --git a/lexicons/org/tangled/event/push.json b/lexicons/org/tangled/event/push.json new file mode 100644 index 0000000000000000000000000000000000000000..49ece8c7e16533b9f8822773d4b3aa3000a7eb0b --- /dev/null +++ b/lexicons/org/tangled/event/push.json @@ -0,0 +1,46 @@ +{ + "lexicon": 1, + "id": "org.tangled.event.push", + "needsCbor": true, + "needsType": true, + "defs": { + "main": { + "type": "object", + "description": "When a repository ref is updated.", + "required": [ + "repo", + "ref", + "oldSha", + "newSha" + ], + "properties": { + "repo": { + "type": "string", + "format": "did" + }, + "ref": { + "type": "string" + }, + "oldSha": { + "type": "string" + }, + "newSha": { + "type": "string" + }, + "actor": { + "type": "string", + "format": "did", + "description": "The account that pushed." + }, + "pushOptions": { + "type": "array", + "items": { + "type": "string" + }, + "maxLength": 50, + "description": "The push options sent with the push, omitted when there were none." + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/comment.json b/lexicons/org/tangled/feed/comment.json new file mode 100644 index 0000000000000000000000000000000000000000..f7eae1691ab583400790b749e36b7ae00a1ffe8b --- /dev/null +++ b/lexicons/org/tangled/feed/comment.json @@ -0,0 +1,42 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.comment", + "defs": { + "main": { + "type": "record", + "key": "tid", + "record": { + "type": "object", + "required": [ + "subject", + "body", + "createdAt" + ], + "properties": { + "subject": { + "type": "union", + "refs": ["com.atproto.repo.strongRef"] + }, + "replyTo": { + "type": "ref", + "ref": "com.atproto.repo.strongRef" + }, + "body": { + "type": "union", + "refs": ["org.tangled.markup.markdown"] + }, + "embed": { + "type": "union", + "refs": [ + "org.tangled.embed.commit" + ] + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/countComments.json b/lexicons/org/tangled/feed/countComments.json new file mode 100644 index 0000000000000000000000000000000000000000..627de71ceab30afca1c17ad670243feee1e7a9a7 --- /dev/null +++ b/lexicons/org/tangled/feed/countComments.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.countComments", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "at-uri", + "description": "Record AT-URI the comments attach to." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/countCommentsBy.json b/lexicons/org/tangled/feed/countCommentsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..dbb1d1ed79b458d138335019107546f484e3e65a --- /dev/null +++ b/lexicons/org/tangled/feed/countCommentsBy.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.countCommentsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose comment authorings to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/countReactions.json b/lexicons/org/tangled/feed/countReactions.json new file mode 100644 index 0000000000000000000000000000000000000000..8efeb62059181ae6948d5a9b619f6c949b866988 --- /dev/null +++ b/lexicons/org/tangled/feed/countReactions.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.countReactions", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "at-uri", + "description": "Record AT-URI the reactions target." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/countReactionsBy.json b/lexicons/org/tangled/feed/countReactionsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..d5aac22f47b1e6a53dc7703ae91b5056eff2b842 --- /dev/null +++ b/lexicons/org/tangled/feed/countReactionsBy.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.countReactionsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose reaction authorings to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/countStars.json b/lexicons/org/tangled/feed/countStars.json new file mode 100644 index 0000000000000000000000000000000000000000..33a6dbe65c4bc67bcfb7638031105cf0441f72f4 --- /dev/null +++ b/lexicons/org/tangled/feed/countStars.json @@ -0,0 +1,38 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.countStars", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Repo DID to list star edges for." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/countStarsBy.json b/lexicons/org/tangled/feed/countStarsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..4bbf6a5f0baea297eaa0f2b07b4104e912f33c93 --- /dev/null +++ b/lexicons/org/tangled/feed/countStarsBy.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.countStarsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose star authorings to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/defs.json b/lexicons/org/tangled/feed/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..88930b7ca50934148039367237e3cf3f706a195c --- /dev/null +++ b/lexicons/org/tangled/feed/defs.json @@ -0,0 +1,37 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.defs", + "needsCbor": true, + "needsType": true, + "defs": { + "commentView": { + "type": "object", + "required": [ + "uri", + "cid", + "subject", + "body", + "createdAt" + ], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "subject": { "type": "ref", "ref": "com.atproto.repo.strongRef" }, + "body": { "type": "union", "refs": ["org.tangled.markup.markdown"] }, + "createdAt": { "type": "string", "format": "datetime" }, + "replyTo": { "type": "ref", "ref": "com.atproto.repo.strongRef" }, + "editor": { + "type": "string", + "format": "did", + "description": "Who wrote this version (`x-tngl-editor`). A knot-hosted comment's uri names the repo, not the writer." + }, + "embed": { + "type": "union", + "refs": [ + "org.tangled.embed.commit#view" + ] + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/getStar.json b/lexicons/org/tangled/feed/getStar.json new file mode 100644 index 0000000000000000000000000000000000000000..0c8ba3387670134f8262cd319ab78628a71c0d79 --- /dev/null +++ b/lexicons/org/tangled/feed/getStar.json @@ -0,0 +1,38 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.getStar", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["actor", "subject"], + "properties": { + "actor": { + "type": "string", + "format": "did", + "description": "DID of the potential stargazer." + }, + "subject": { + "type": "string", + "description": "Repo DID to check." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "Uri of actor's star record for subject." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/getTimeline.json b/lexicons/org/tangled/feed/getTimeline.json new file mode 100644 index 0000000000000000000000000000000000000000..9d8b9e9e09e7aea771ba5b110ada6e5e5583cc30 --- /dev/null +++ b/lexicons/org/tangled/feed/getTimeline.json @@ -0,0 +1,98 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.getTimeline", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "properties": { + "viewer": { + "type": "string", + "format": "did", + "description": "TODO: deprecate this and use auth instead." + }, + "followingOnly": { "type": "boolean" }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 50 + }, + "cursor": { "type": "string" } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["feed"], + "properties": { + "feed": { + "type": "array", + "items": { "type": "ref", "ref": "#timelineItem" } + }, + "cursor": { "type": "string" } + } + } + }, + "errors": [] + }, + "timelineItem": { + "type": "object", + "required": ["eventAt", "event"], + "properties": { + "eventAt": { "type": "string", "format": "datetime" }, + "event": { + "type": "union", + "refs": ["#repoEvent", "#starEvent", "#followEvent"] + } + } + }, + "repoEvent": { + "type": "object", + "description": "A repository was created (or forked). The actor is repo.owner.", + "required": ["uri", "cid", "repo"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "repo": { "type": "ref", "ref": "org.tangled.repo.defs#repoViewBasic" }, + "source": { + "type": "ref", + "ref": "org.tangled.repo.defs#repoViewBasic", + "description": "Fork parent, present only if this repo is a fork." + } + } + }, + "starEvent": { + "type": "object", + "description": "A repository was starred.", + "required": ["uri", "cid", "actor", "repo"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "actor": { + "type": "ref", + "ref": "org.tangled.actor.defs#profileViewBasic", + "description": "Who starred." + }, + "repo": { + "type": "ref", + "ref": "org.tangled.repo.defs#repoViewBasic", + "description": "The starred repo." + } + } + }, + "followEvent": { + "type": "object", + "description": "A user followed another user.", + "required": ["uri", "cid", "actor", "subject"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "actor": { "type": "ref", "ref": "org.tangled.actor.defs#profileViewBasic" }, + "subject": { "type": "ref", "ref": "org.tangled.actor.defs#profileViewDetailed" } + } + } + } +} diff --git a/lexicons/org/tangled/feed/listComments.json b/lexicons/org/tangled/feed/listComments.json new file mode 100644 index 0000000000000000000000000000000000000000..ab1077fd711a4554e1549deab643621fff205d5b --- /dev/null +++ b/lexicons/org/tangled/feed/listComments.json @@ -0,0 +1,72 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.listComments", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "at-uri", + "description": "Record AT-URI the comments attach to." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "org.tangled.feed.defs#commentView" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "description": "DEPRECATED: only org.tangled.feed.listCommentsBy still answers with this shape. Use #commentView.", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded org.tangled.feed.comment record" + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/listCommentsBy.json b/lexicons/org/tangled/feed/listCommentsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..1b30838bbe0d24e2a8bf958bbe5add865eac874a --- /dev/null +++ b/lexicons/org/tangled/feed/listCommentsBy.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.listCommentsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose comment authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "org.tangled.feed.listComments#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/listReactions.json b/lexicons/org/tangled/feed/listReactions.json new file mode 100644 index 0000000000000000000000000000000000000000..586162d665e6da6979a0a55f2b9146757fe77d20 --- /dev/null +++ b/lexicons/org/tangled/feed/listReactions.json @@ -0,0 +1,71 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.listReactions", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "at-uri", + "description": "Record AT-URI the reactions target." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded org.tangled.feed.reaction record" + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/listReactionsBy.json b/lexicons/org/tangled/feed/listReactionsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..ac9db475e5a7b1bc9210f715faccae9ebd7a44fd --- /dev/null +++ b/lexicons/org/tangled/feed/listReactionsBy.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.listReactionsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose reaction authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "org.tangled.feed.listReactions#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/listStars.json b/lexicons/org/tangled/feed/listStars.json new file mode 100644 index 0000000000000000000000000000000000000000..3cbd53697071bba5eb6812aed3ac0953b2a6d57c --- /dev/null +++ b/lexicons/org/tangled/feed/listStars.json @@ -0,0 +1,70 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.listStars", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Repo DID to list star edges for." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.feed.star record" + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/listStarsBy.json b/lexicons/org/tangled/feed/listStarsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..cc32f1fbdc10fbad4c5e3f0bb94d3b4d45a03b52 --- /dev/null +++ b/lexicons/org/tangled/feed/listStarsBy.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.listStarsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose star authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "org.tangled.feed.listStars#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/reaction.json b/lexicons/org/tangled/feed/reaction.json new file mode 100644 index 0000000000000000000000000000000000000000..858d21440892d6526dcf5196c03f5ca9703fc01f --- /dev/null +++ b/lexicons/org/tangled/feed/reaction.json @@ -0,0 +1,33 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.reaction", + "defs": { + "main": { + "type": "record", + "key": "tid", + "record": { + "type": "object", + "required": [ + "subject", + "reaction", + "createdAt" + ], + "properties": { + "subject": { + "type": "ref", + "ref": "com.atproto.repo.strongRef" + }, + "reaction": { + "type": "string", + "maxGraphemes": 1, + "enum": ["👍", "👎", "😆", "🎉", "🫤", "❤️", "🚀", "👀"] + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/feed/star.json b/lexicons/org/tangled/feed/star.json new file mode 100644 index 0000000000000000000000000000000000000000..32dfb8bddb6588697a8c15179085ef6cdca688c6 --- /dev/null +++ b/lexicons/org/tangled/feed/star.json @@ -0,0 +1,42 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.star", + "defs": { + "main": { + "type": "record", + "key": "tid", + "record": { + "type": "object", + "required": [ + "subject", + "createdAt" + ], + "properties": { + "subject": { + "type": "union", + "refs": ["#repo", "#record"], + "closed": true + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + }, + "repo": { + "type": "object", + "required": ["identity"], + "properties": { + "identity": { "type": "string", "format": "did" } + } + }, + "record": { + "type": "object", + "required": ["record"], + "properties": { + "record": { "type": "ref", "ref": "com.atproto.repo.strongRef" } + } + } + } +} diff --git a/lexicons/org/tangled/feed/subscription.json b/lexicons/org/tangled/feed/subscription.json new file mode 100644 index 0000000000000000000000000000000000000000..b94727b61113c0632533dfa0f51c5d95984ed174 --- /dev/null +++ b/lexicons/org/tangled/feed/subscription.json @@ -0,0 +1,46 @@ +{ + "lexicon": 1, + "id": "org.tangled.feed.subscription", + "defs": { + "main": { + "type": "record", + "key": "tid", + "record": { + "type": "object", + "required": [ + "subject", + "createdAt" + ], + "properties": { + "subject": { + "type": "union", + "refs": ["#record", "#repo"], + "closed": true + }, + "ignore": { + "type": "boolean", + "description": "mute this thread" + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + }, + "record": { + "type": "object", + "required": ["uri"], + "properties": { + "uri": { "type": "string", "format": "at-uri" } + } + }, + "repo": { + "type": "object", + "required": ["did"], + "properties": { + "did": { "type": "string", "format": "did" } + } + } + } +} diff --git a/lexicons/org/tangled/git/defs.json b/lexicons/org/tangled/git/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..8f38c754b756f684cfc43731d15aad7674ae4049 --- /dev/null +++ b/lexicons/org/tangled/git/defs.json @@ -0,0 +1,235 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.defs", + "defs": { + "signature": { + "type": "object", + "required": [ + "name", + "email", + "when" + ], + "properties": { + "name": { + "type": "string", + "description": "Person name" + }, + "email": { + "type": "string", + "description": "Person email" + }, + "when": { + "type": "string", + "format": "datetime", + "description": "Timestamp of the signature" + } + } + }, + "commit": { + "type": "object", + "required": [ + "hash", + "author", + "committer", + "message", + "tree" + ], + "properties": { + "hash": { + "type": "string", + "description": "Commit Object ID" + }, + "author": { + "type": "ref", + "ref": "#signature" + }, + "committer": { + "type": "ref", + "ref": "#signature" + }, + "message": { + "type": "string" + }, + "tree": { + "type": "string", + "description": "Tree Object ID" + } + } + }, + "submodule": { + "type": "object", + "required": [ + "name", + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Submodule name" + }, + "url": { + "type": "string", + "description": "Submodule repository URL" + }, + "branch": { + "type": "string", + "description": "Branch to track in the submodule" + } + } + }, + "fileDiff": { + "type": "object", + "required": [ + "lhsSrc", + "rhsSrc", + "hunks", + "hasSyntacticChanges" + ], + "properties": { + "lhsSrc": { + "type": "ref", + "ref": "#diffSrc" + }, + "rhsSrc": { + "type": "ref", + "ref": "#diffSrc" + }, + "hunks": { + "type": "array", + "items": { + "type": "ref", + "ref": "#diffHunk" + } + }, + "hasByteChanges": { + "type": "ref", + "ref": "#byteChanges", + "description": "Byte count per side, present only when the two sides differ" + }, + "hasSyntacticChanges": { + "type": "boolean" + } + } + }, + "diffSrc": { + "type": "object", + "description": "One side of a file diff.", + "required": [ + "path", + "oid", + "size", + "isBinary", + "isSubmodule" + ], + "properties": { + "path": { + "type": "string" + }, + "oid": { + "type": "string", + "description": "Blob oid, or the commit oid for a submodule" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "isBinary": { + "type": "boolean" + }, + "isSubmodule": { + "type": "boolean" + }, + "content": { + "type": "string", + "description": "Inlined content. Used on interdiff's LHS." + } + } + }, + "diffHunk": { + "type": "object", + "required": [ + "novelLhs", + "novelRhs", + "lines" + ], + "properties": { + "novelLhs": { + "type": "array", + "description": "0-based line numbers holding novel content on the left side", + "items": { + "type": "integer", + "minimum": 0 + } + }, + "novelRhs": { + "type": "array", + "description": "0-based line numbers holding novel content on the right side", + "items": { + "type": "integer", + "minimum": 0 + } + }, + "lines": { + "type": "array", + "items": { + "type": "ref", + "ref": "#linePair" + } + } + } + }, + "linePair": { + "type": "object", + "description": "One aligned line pair; a side is absent when that side has no line.", + "properties": { + "lhs": { + "type": "integer", + "minimum": 0 + }, + "rhs": { + "type": "integer", + "minimum": 0 + } + } + }, + "byteChanges": { + "type": "object", + "required": [ + "lhs", + "rhs" + ], + "properties": { + "lhs": { + "type": "integer", + "minimum": 0 + }, + "rhs": { + "type": "integer", + "minimum": 0 + } + } + }, + "diffStat": { + "type": "object", + "required": [ + "insertions", + "deletions", + "filesChanged" + ], + "properties": { + "insertions": { + "type": "integer", + "minimum": 0 + }, + "deletions": { + "type": "integer", + "minimum": 0 + }, + "filesChanged": { + "type": "integer", + "minimum": 0 + } + } + } + } +} diff --git a/lexicons/org/tangled/git/deleteBranch.json b/lexicons/org/tangled/git/deleteBranch.json new file mode 100644 index 0000000000000000000000000000000000000000..d4e6d84beb8609695c5af8254b634cea344ffcb4 --- /dev/null +++ b/lexicons/org/tangled/git/deleteBranch.json @@ -0,0 +1,30 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.deleteBranch", + "defs": { + "main": { + "type": "procedure", + "description": "Delete a branch on this repository", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "branch" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "branch": { + "type": "string" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/git/getArchive.json b/lexicons/org/tangled/git/getArchive.json new file mode 100644 index 0000000000000000000000000000000000000000..3254606ef121063aa172ace632b9c9d66d45c5d3 --- /dev/null +++ b/lexicons/org/tangled/git/getArchive.json @@ -0,0 +1,63 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.getArchive", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo", + "ref", + "prefix" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git reference (branch, tag, or commit SHA)" + }, + "format": { + "type": "string", + "description": "Archive format", + "enum": [ + "tar.gz", + "zip" + ], + "default": "tar.gz" + }, + "prefix": { + "type": "string", + "description": "Prefix for files in the archive" + } + } + }, + "output": { + "encoding": "*/*", + "description": "Binary archive data" + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "ArchiveTooLarge", + "description": "Archive exceeds the size limit" + } + ] + } + } +} diff --git a/lexicons/org/tangled/git/getBlame.json b/lexicons/org/tangled/git/getBlame.json new file mode 100644 index 0000000000000000000000000000000000000000..9f7462f62271da00f0ab0d513ff6ebab1e09aab9 --- /dev/null +++ b/lexicons/org/tangled/git/getBlame.json @@ -0,0 +1,139 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.getBlame", + "defs": { + "main": { + "type": "query", + "description": "Line-by-line attribution of a file at a ref.", + "parameters": { + "type": "params", + "required": ["repo", "ref", "path"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git revision (branch, tag, or commit Object ID)" + }, + "path": { + "type": "string", + "description": "Path of the file to blame" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["commit", "commits", "runs"], + "properties": { + "commit": { + "type": "string", + "description": "The resolved commit Object ID" + }, + "commits": { + "type": "array", + "description": "Unique commits referenced by blame runs", + "items": { + "type": "ref", + "ref": "#commit" + } + }, + "runs": { + "type": "array", + "description": "Complete, gap-free one-based line ranges sorted by start", + "items": { + "type": "ref", + "ref": "#run" + } + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repo is not registered on this knot" + }, + { + "name": "RefNotFound", + "description": "The given ref doesn't resolve in this repo" + }, + { + "name": "FileNotFound", + "description": "The given path doesn't exist at the given ref" + }, + { + "name": "BlameTooLarge", + "description": "The file is too large to blame" + }, + { + "name": "BlameTimeout", + "description": "Blame computation timed out" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "commit": { + "type": "object", + "required": ["sha", "subject", "authoredAt", "author"], + "properties": { + "sha": { + "type": "string", + "description": "Commit Object ID" + }, + "subject": { + "type": "string", + "description": "First line of the commit message" + }, + "authoredAt": { + "type": "string", + "format": "datetime", + "description": "When the change was authored" + }, + "author": { + "type": "ref", + "ref": "#author" + } + } + }, + "author": { + "type": "object", + "required": ["name", "email"], + "properties": { + "name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + }, + "run": { + "type": "object", + "required": ["start", "count", "commit"], + "properties": { + "start": { + "type": "integer", + "minimum": 1, + "description": "First line of the run, one-based" + }, + "count": { + "type": "integer", + "minimum": 1, + "description": "Number of lines in the run" + }, + "commit": { + "type": "string", + "description": "The commit Object ID of the entry in commits this run is attributed to" + } + } + } + } +} diff --git a/lexicons/org/tangled/git/getBlob.json b/lexicons/org/tangled/git/getBlob.json new file mode 100644 index 0000000000000000000000000000000000000000..b8ddd745e07bf1a07097283fb9cfc3a930a9ccfe --- /dev/null +++ b/lexicons/org/tangled/git/getBlob.json @@ -0,0 +1,36 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.getBlob", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo", + "path" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git reference (branch, tag, or commit SHA)", + "default": "HEAD" + }, + "path": { + "type": "string", + "description": "Path within the repository tree" + } + } + }, + "output": { + "encoding": "*/*", + "description": "raw blob served in octet-stream" + } + } + } +} diff --git a/lexicons/org/tangled/git/getBranch.json b/lexicons/org/tangled/git/getBranch.json new file mode 100644 index 0000000000000000000000000000000000000000..aec0e87d4815eb094c3f26fb2af4ef285d71b650 --- /dev/null +++ b/lexicons/org/tangled/git/getBranch.json @@ -0,0 +1,76 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.getBranch", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo", + "name" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "name": { + "type": "string", + "description": "Branch name to get information for" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "name", + "hash", + "message", + "author", + "totalCommits" + ], + "properties": { + "name": { + "type": "string", + "description": "Branch name" + }, + "hash": { + "type": "string", + "description": "Latest commit hash on this branch" + }, + "message": { + "type": "string", + "description": "Latest commit message" + }, + "author": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + }, + "totalCommits": { + "type": "integer", + "description": "Number of commits reachable from this branch" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "BranchNotFound", + "description": "Branch not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/git/getDiff.json b/lexicons/org/tangled/git/getDiff.json new file mode 100644 index 0000000000000000000000000000000000000000..1e9777c819bc6cce8308bf1711345a17c9e7388f --- /dev/null +++ b/lexicons/org/tangled/git/getDiff.json @@ -0,0 +1,67 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.getDiff", + "defs": { + "main": { + "type": "query", + "description": "Full diff between two revisions of a repository.", + "parameters": { + "type": "params", + "required": ["repo", "base", "head"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "base": { + "type": "string", + "description": "Base revision (branch, tag, or commit Object ID)" + }, + "head": { + "type": "string", + "description": "Head revision (branch, tag, or commit Object ID)" + }, + "paths": { + "type": "string", + "description": "Only include diffs touching these paths, comma-separated" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["diffs"], + "properties": { + "diffs": { + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.git.defs#fileDiff" + } + }, + "stats": { + "type": "ref", + "ref": "org.tangled.git.defs#diffStat" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repo is not registered on this knot" + }, + { + "name": "RefNotFound", + "description": "A given revision doesn't resolve in this repo" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/git/getEntry.json b/lexicons/org/tangled/git/getEntry.json new file mode 100644 index 0000000000000000000000000000000000000000..4a9350c8598fc071e3130f6d5710bf0d2b19ce9f --- /dev/null +++ b/lexicons/org/tangled/git/getEntry.json @@ -0,0 +1,95 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.getEntry", + "defs": { + "main": { + "type": "query", + "description": "get metadata of blob by ref and path", + "parameters": { + "type": "params", + "required": [ + "repo", + "path" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git revision (branch, tag, or commit Object ID)", + "default": "HEAD" + }, + "path": { + "type": "string", + "description": "path of the entity" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "name", + "mode", + "oid", + "size" + ], + "properties": { + "name": { + "type": "string", + "description": "The file name" + }, + "mode": { + "type": "string", + "enum": [ + "0040000", + "0100644", + "0100755", + "0120000", + "0160000" + ] + }, + "oid": { + "type": "string" + }, + "size": { + "type": "integer", + "description": "Blob size" + }, + "lastCommit": { + "type": "ref", + "ref": "org.tangled.git.defs#commit" + }, + "submodule": { + "type": "ref", + "ref": "org.tangled.git.defs#submodule", + "description": "Submodule information if path is a submodule" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "EntryNotFound", + "description": "Entry not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/git/getTag.json b/lexicons/org/tangled/git/getTag.json new file mode 100644 index 0000000000000000000000000000000000000000..e14b1feab06044ce0d0cf1823eb6c3e8a3a735cc --- /dev/null +++ b/lexicons/org/tangled/git/getTag.json @@ -0,0 +1,76 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.getTag", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo", + "tag" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "tag": { + "type": "string", + "description": "Name of tag, such as v1.3.0" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "name", + "oid", + "message", + "author" + ], + "properties": { + "name": { + "type": "string", + "description": "Tag name" + }, + "oid": { + "type": "string", + "description": "Latest commit oid on this tag" + }, + "message": { + "type": "string", + "description": "Latest commit message" + }, + "author": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + }, + "annotation": { + "type": "ref", + "ref": "org.tangled.git.listTags#annotation", + "description": "Present only for annotated tags" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/git/getTree.json b/lexicons/org/tangled/git/getTree.json new file mode 100644 index 0000000000000000000000000000000000000000..16f2f3e3705f3b2d17934a51f27c759552ecf6c8 --- /dev/null +++ b/lexicons/org/tangled/git/getTree.json @@ -0,0 +1,105 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.getTree", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git revision (branch, tag, or commit Object ID)", + "default": "HEAD" + }, + "path": { + "type": "string", + "description": "Path within the repository tree", + "default": "" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "entries" + ], + "properties": { + "entries": { + "type": "array", + "items": { + "type": "ref", + "ref": "#treeEntry" + } + }, + "lastCommit": { + "type": "ref", + "ref": "org.tangled.git.defs#commit" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "PathNotFound", + "description": "Path not found in repository tree" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "treeEntry": { + "type": "object", + "required": [ + "name", + "mode", + "size" + ], + "properties": { + "name": { + "type": "string", + "description": "Relative file or directory name" + }, + "mode": { + "type": "string", + "description": "File mode", + "enum": [ + "0040000", + "0100644", + "0100755", + "0120000", + "0160000" + ] + }, + "size": { + "type": "integer", + "description": "File size in bytes" + }, + "lastCommit": { + "type": "ref", + "ref": "org.tangled.git.defs#commit" + } + } + } + } +} diff --git a/lexicons/org/tangled/git/listBranches.json b/lexicons/org/tangled/git/listBranches.json new file mode 100644 index 0000000000000000000000000000000000000000..257688731d4ea98a3f7716f08e92d7cb44ea3669 --- /dev/null +++ b/lexicons/org/tangled/git/listBranches.json @@ -0,0 +1,105 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.listBranches", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "limit": { + "type": "integer", + "description": "Maximum number of branches to return", + "minimum": 1, + "maximum": 100, + "default": 50 + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "branches", + "total" + ], + "properties": { + "branches": { + "type": "array", + "items": { + "type": "ref", + "ref": "#branchInfo" + } + }, + "cursor": { + "type": "string" + }, + "total": { + "type": "integer", + "description": "Total number of branches in the repository" + }, + "defaultBranch": { + "type": "string", + "description": "Name of the default branch" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "branchInfo": { + "type": "object", + "required": [ + "name", + "oid", + "message", + "author", + "committer" + ], + "properties": { + "name": { + "type": "string", + "description": "Branch name" + }, + "oid": { + "type": "string", + "description": "Latest commit oid on this branch" + }, + "message": { + "type": "string", + "description": "Latest commit message" + }, + "author": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + }, + "committer": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + } + } + } + } +} diff --git a/lexicons/org/tangled/git/listCommits.json b/lexicons/org/tangled/git/listCommits.json new file mode 100644 index 0000000000000000000000000000000000000000..fce2f29ca560e335ee53bdf886300aa3d0b41c7a --- /dev/null +++ b/lexicons/org/tangled/git/listCommits.json @@ -0,0 +1,79 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.listCommits", + "defs": { + "main": { + "type": "query", + "description": "List commit history reachable from a ref, newest first. The ref may be any full ref name, including refs/cobs/*.", + "parameters": { + "type": "params", + "required": ["repo", "ref"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git revision (branch, tag, commit Object ID, or full ref name)" + }, + "path": { + "type": "string", + "description": "Only list commits touching this path", + "default": "" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 50, + "description": "Maximum number of commits to return in this page" + }, + "cursor": { + "type": "string", + "description": "Commit Object ID of the last commit in the previous page" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["commits"], + "properties": { + "commits": { + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.git.defs#commit" + } + }, + "cursor": { + "type": "string", + "description": "Cursor for the next page, absent when the last page is reached" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repo is not registered on this knot" + }, + { + "name": "RefNotFound", + "description": "The given ref doesn't resolve in this repo" + }, + { + "name": "PathNotFound", + "description": "The given path doesn't exist at the given ref" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/git/listLanguages.json b/lexicons/org/tangled/git/listLanguages.json new file mode 100644 index 0000000000000000000000000000000000000000..07ca67c37a6c3b693287be80ac7e76aadfe832bc --- /dev/null +++ b/lexicons/org/tangled/git/listLanguages.json @@ -0,0 +1,107 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.listLanguages", + "defs": { + "main": { + "type": "query", + "description": "Language breakdown of a repository's tree at a ref.", + "parameters": { + "type": "params", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git revision (branch, tag, or commit Object ID)", + "default": "HEAD" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["ref", "languages"], + "properties": { + "ref": { + "type": "string", + "description": "The ref the breakdown was computed at" + }, + "languages": { + "type": "array", + "items": { + "type": "ref", + "ref": "#language" + } + }, + "totalSize": { + "type": "integer", + "minimum": 0, + "description": "Total bytes across counted files" + }, + "totalFiles": { + "type": "integer", + "minimum": 0, + "description": "Total counted files" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repo is not registered on this knot" + }, + { + "name": "RefNotFound", + "description": "The given ref doesn't resolve in this repo" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "language": { + "type": "object", + "required": ["name", "size", "percentage"], + "properties": { + "name": { + "type": "string", + "description": "Language name" + }, + "size": { + "type": "integer", + "minimum": 0, + "description": "Bytes counted for this language" + }, + "percentage": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "description": "Share of total size, 0-100" + }, + "fileCount": { + "type": "integer", + "minimum": 0, + "description": "Files counted for this language" + }, + "color": { + "type": "string", + "description": "Render color for this language" + }, + "extensions": { + "type": "array", + "items": { + "type": "string" + }, + "description": "File extensions recognized as this language" + } + } + } + } +} diff --git a/lexicons/org/tangled/git/listRefs.json b/lexicons/org/tangled/git/listRefs.json new file mode 100644 index 0000000000000000000000000000000000000000..4dcb7b03df16377af265449298e262cad0733316 --- /dev/null +++ b/lexicons/org/tangled/git/listRefs.json @@ -0,0 +1,100 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.listRefs", + "defs": { + "main": { + "type": "query", + "description": "List every ref & its commit SHA for a git repo, equivalent to git ls-remote. Gives the full ref state of a repo.", + "parameters": { + "type": "params", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the git repo as minted by the knot" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 100, + "description": "Maximum number of refs to return in this page" + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["refs"], + "properties": { + "refs": { + "type": "array", + "items": { "type": "ref", "ref": "#ref" }, + "maxLength": 1000 + }, + "cursor": { + "type": "string", + "description": "Cursor for the next page, absent when the last page is reached" + }, + "defaultBranch": { + "type": "ref", + "ref": "#defaultBranch" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repo is not registered on this knot" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "ref": { + "type": "object", + "required": ["ref", "sha"], + "properties": { + "ref": { + "type": "string", + "description": "Full ref name, eg. refs/heads/main or refs/tags/v1.0", + "maxGraphemes": 256, + "maxLength": 2560 + }, + "sha": { + "type": "string", + "description": "Object SHA the ref points at. Width depends on the repo's git object-format.", + "minLength": 40, + "maxLength": 128 + } + } + }, + "defaultBranch": { + "type": "object", + "required": ["ref"], + "properties": { + "ref": { + "type": "string", + "description": "Default branch ref name that HEAD points at, eg. refs/heads/main.", + "maxGraphemes": 256, + "maxLength": 2560 + }, + "head": { + "type": "string", + "description": "Commit SHA at the tip of the default branch, for reconciling against a last-known state. Width depends on the repo's git object-format.", + "minLength": 40, + "maxLength": 128 + } + } + } + } +} diff --git a/lexicons/org/tangled/git/listTags.json b/lexicons/org/tangled/git/listTags.json new file mode 100644 index 0000000000000000000000000000000000000000..142e4bbb9eee9ceebc9894fc3ffc2f11f1f213f4 --- /dev/null +++ b/lexicons/org/tangled/git/listTags.json @@ -0,0 +1,117 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.listTags", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "limit": { + "type": "integer", + "description": "Maximum number of tags to return", + "minimum": 1, + "maximum": 100, + "default": 50 + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "tags", + "total" + ], + "properties": { + "tags": { + "type": "array", + "items": { + "type": "ref", + "ref": "#tagInfo" + } + }, + "cursor": { + "type": "string" + }, + "total": { + "type": "integer", + "description": "Total number of tags in the repository" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "tagInfo": { + "type": "object", + "required": [ + "name", + "oid", + "message", + "author" + ], + "properties": { + "name": { + "type": "string", + "description": "Tag name" + }, + "oid": { + "type": "string", + "description": "Latest commit oid on this tag" + }, + "message": { + "type": "string", + "description": "Latest commit message" + }, + "author": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + }, + "annotation": { + "type": "ref", + "ref": "#annotation", + "description": "Present only for annotated tags" + } + } + }, + "annotation": { + "type": "object", + "required": [ + "message" + ], + "properties": { + "tagger": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + }, + "message": { + "type": "string", + "description": "Tag message" + } + } + } + } +} diff --git a/lexicons/org/tangled/git/oid.json b/lexicons/org/tangled/git/oid.json new file mode 100644 index 0000000000000000000000000000000000000000..d131b65616aa288ed93f18f47617fd5ee523aa5f --- /dev/null +++ b/lexicons/org/tangled/git/oid.json @@ -0,0 +1,18 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.oid", + "description": "Git object-id similar to concrete `cid-link` type.", + "defs": { + "main": { + "type": "object", + "required": ["oid"], + "properties": { + "oid": { + "type": "string", + "minLength": 40, + "maxLength": 64 + } + } + } + } +} diff --git a/lexicons/org/tangled/git/pushEvent.json b/lexicons/org/tangled/git/pushEvent.json new file mode 100644 index 0000000000000000000000000000000000000000..5f6f04dd4e3fa9de9c2552f662f04a889b96e04b --- /dev/null +++ b/lexicons/org/tangled/git/pushEvent.json @@ -0,0 +1,73 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.pushEvent", + "defs": { + "main": { + "type": "object", + "description": "Represents git-push operation to target repository.", + "required": [ + "seq", + "repo", + "pusher", + "edits", + "time", + "sig" + ], + "properties": { + "seq": { + "type": "integer", + "description": "The stream sequence number of this message." + }, + "repo": { + "type": "string", + "format": "did", + "description": "The repo this event comes from." + }, + "pusher": { + "type": "string", + "format": "did", + "description": "The account that performed this push." + }, + "edits": { + "type": "array", + "items": { + "type": "ref", + "ref": "#refEdit", + "description": "List of git reference update operations in this push." + }, + "maxLength": 200 + }, + "time": { + "type": "string", + "format": "datetime", + "description": "Timestamp of when this message was originally broadcast." + }, + "options": { + "type": "array", + "description": "Git push options.", + "items": { + "type": "string", + "maxLength": 1024, + "description": "Git push option." + }, + "maxLength": 10 + }, + "sig": { + "type": "bytes", + "description": "Signature over the DAG-CBOR of this message with the seq and sig keys absent (repo/pusher DID, edits, time, options), by the repo's atproto signing key. seq is excluded so a relay can reassign it without invalidating the signature." + } + } + }, + "refEdit": { + "type": "object", + "description": "A git ref edit.", + "required": ["name", "new", "old"], + "nullable": ["new", "old"], + "properties": { + "name": { "type": "string", "description": "Full git reference name, ie refs/heads/main." }, + "new": { "type": "string", "description": "Full hex object id the ref now points at." }, + "old": { "type": "string", "description": "Full hex object id the ref pointed at before." } + } + } + } +} diff --git a/lexicons/org/tangled/git/receivePack.json b/lexicons/org/tangled/git/receivePack.json new file mode 100644 index 0000000000000000000000000000000000000000..e072834e1c8b272222cbf28da40802d53ce849ee --- /dev/null +++ b/lexicons/org/tangled/git/receivePack.json @@ -0,0 +1,10 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.receivePack", + "defs": { + "main": { + "type": "procedure", + "description": "No-op xrpc endpoint for `git-push` permission scope." + } + } +} diff --git a/lexicons/org/tangled/git/refs.json b/lexicons/org/tangled/git/refs.json new file mode 100644 index 0000000000000000000000000000000000000000..278f9abb1110fd11b76fd6eb7d06511a9e8a217a --- /dev/null +++ b/lexicons/org/tangled/git/refs.json @@ -0,0 +1,30 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.refs", + "defs": { + "main": { + "type": "record", + "key": "literal:self", + "record": { + "type": "object", + "required": [ + "defaultBranch", + "refs" + ], + "properties": { + "defaultBranch": { + "type": "string", + "description": "Default branch name without refs/heads/ prefix.", + "examples": ["main", "master"], + "maxLength": 244 + }, + "refs": { + "type": "bytes", + "description": "List of git refs in `git show-ref` format.", + "maxLength": 1000000 + } + } + } + } + } +} diff --git a/lexicons/org/tangled/git/setDefaultBranch.json b/lexicons/org/tangled/git/setDefaultBranch.json new file mode 100644 index 0000000000000000000000000000000000000000..e228daa0c064a546d240d3dca6ba93a9a6eef4aa --- /dev/null +++ b/lexicons/org/tangled/git/setDefaultBranch.json @@ -0,0 +1,51 @@ +{ + "lexicon": 1, + "id": "org.tangled.git.setDefaultBranch", + "defs": { + "main": { + "type": "procedure", + "description": "Set the default branch of this repository", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "branch" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "branch": { + "type": "string", + "description": "Branch name without the refs/heads/ prefix", + "maxLength": 244 + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "cid"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "The repository's org.tangled.git.refs record" + }, + "cid": { + "type": "string", + "format": "cid", + "description": "Version of the refs record after the move" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/graph/attestation.json b/lexicons/org/tangled/graph/attestation.json new file mode 100644 index 0000000000000000000000000000000000000000..10fe90d5cecc6566bd72b040c5589651fbe37460 --- /dev/null +++ b/lexicons/org/tangled/graph/attestation.json @@ -0,0 +1,29 @@ +{ + "lexicon": 1, + "id": "org.tangled.graph.attestation", + "defs": { + "main": { + "type": "record", + "description": "Attesting that I the user claim a given record over on a knot! A knot will claim that I wrote something, but it's only really tied to me if I too say that I wrote it. The 'change' field is literally the Git Object ID of the Collaborative Object (COB) change that made the knot-side record.", + "key": "tid", + "record": { + "type": "object", + "required": ["subject", "createdAt"], + "properties": { + "subject": { + "type": "ref", + "ref": "com.atproto.repo.strongRef" + }, + "change": { + "type": "string", + "description": "The Git Object ID of the COB change that made the knot-side record." + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/graph/collaborator.json b/lexicons/org/tangled/graph/collaborator.json new file mode 100644 index 0000000000000000000000000000000000000000..f0cfe28266e2758a061057ebe907ff0ff5ec7d27 --- /dev/null +++ b/lexicons/org/tangled/graph/collaborator.json @@ -0,0 +1,23 @@ +{ + "lexicon": 1, + "id": "org.tangled.graph.collaborator", + "defs": { + "main": { + "type": "record", + "description": "Offer of collaboration on this repo, w/ normalized invitee-DID as rkey.", + "key": "any", + "record": { + "type": "object", + "required": [ + "createdAt" + ], + "properties": { + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/graph/countFollows.json b/lexicons/org/tangled/graph/countFollows.json new file mode 100644 index 0000000000000000000000000000000000000000..2dd2a97cdbe8af66952d3068e5158fbef5851a8e --- /dev/null +++ b/lexicons/org/tangled/graph/countFollows.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.graph.countFollows", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Followee DID whose inbound follows to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/graph/countFollowsBy.json b/lexicons/org/tangled/graph/countFollowsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..786a2d7f10e3c888652b940eed44b90c78d62b82 --- /dev/null +++ b/lexicons/org/tangled/graph/countFollowsBy.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.graph.countFollowsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose follow authorings to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/graph/follow.json b/lexicons/org/tangled/graph/follow.json new file mode 100644 index 0000000000000000000000000000000000000000..2a03275623ce682836d9956e0f79a26a245b0012 --- /dev/null +++ b/lexicons/org/tangled/graph/follow.json @@ -0,0 +1,23 @@ +{ + "lexicon": 1, + "id": "org.tangled.graph.follow", + "defs": { + "main": { + "type": "record", + "description": "Record declaring a social 'follow' relationship of another account, w/ normalized subject-DID as rkey.", + "key": "any", + "record": { + "type": "object", + "required": [ + "createdAt" + ], + "properties": { + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/graph/getFollow.json b/lexicons/org/tangled/graph/getFollow.json new file mode 100644 index 0000000000000000000000000000000000000000..f6072ca3a8222966f12631ef4a6e8fec094dbc3f --- /dev/null +++ b/lexicons/org/tangled/graph/getFollow.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.graph.getFollow", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["actor", "subject"], + "properties": { + "actor": { + "type": "string", + "format": "did", + "description": "DID of the potential follower." + }, + "subject": { + "type": "string", + "format": "did", + "description": "Followee DID to check." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "Uri of actor's follow record for subject." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/graph/listFollows.json b/lexicons/org/tangled/graph/listFollows.json new file mode 100644 index 0000000000000000000000000000000000000000..03c4d76b93b87d2dd326f54a89b5fb0a79a90eaf --- /dev/null +++ b/lexicons/org/tangled/graph/listFollows.json @@ -0,0 +1,76 @@ +{ + "lexicon": 1, + "id": "org.tangled.graph.listFollows", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Followee DID whose inbound follows to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "subject", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "subject": { + "type": "string", + "format": "did", + "description": "The followed account, which the record only names by its rkey" + }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.graph.follow record" + } + } + } + } +} diff --git a/lexicons/org/tangled/graph/listFollowsBy.json b/lexicons/org/tangled/graph/listFollowsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..0e701fe76f15ec00e137d7ffac2e8a6fa8861804 --- /dev/null +++ b/lexicons/org/tangled/graph/listFollowsBy.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "org.tangled.graph.listFollowsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose follow authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "org.tangled.graph.listFollows#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/graph/listNetworkVouches.json b/lexicons/org/tangled/graph/listNetworkVouches.json new file mode 100644 index 0000000000000000000000000000000000000000..9e9ef2c5e23885f56e8c1a1e7ca41ce5fadc0891 --- /dev/null +++ b/lexicons/org/tangled/graph/listNetworkVouches.json @@ -0,0 +1,68 @@ +{ + "lexicon": 1, + "id": "org.tangled.graph.listNetworkVouches", + "defs": { + "main": { + "type": "query", + "description": "List vouches involving one or more actors as seen through a viewer's vouch network: vouches the viewer's network placed on an actor, and vouches an actor placed on the viewer's network. Requires authentication.", + "parameters": { + "type": "params", + "required": ["actors"], + "properties": { + "actors": { + "type": "array", + "items": { + "type": "string", + "format": "did" + }, + "minLength": 1, + "maxLength": 50, + "description": "DIDs of the profiles whose network vouches to list. At most 50 per request." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor. Only accepted when a single actor is requested." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 50, + "description": "Maximum number of vouches per actor." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["actor", "uri", "value"], + "properties": { + "actor": { + "type": "string", + "format": "did", + "description": "The requested actor this vouch involves." + }, + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.graph.vouch record" + } + } + } + } +} diff --git a/lexicons/org/tangled/graph/vouch.json b/lexicons/org/tangled/graph/vouch.json new file mode 100644 index 0000000000000000000000000000000000000000..7e41ce6b1dc8d6f21c1f6ec0200d078348024741 --- /dev/null +++ b/lexicons/org/tangled/graph/vouch.json @@ -0,0 +1,45 @@ +{ + "lexicon": 1, + "id": "org.tangled.graph.vouch", + "defs": { + "main": { + "type": "record", + "description": "normalized subject-DID as rkey.", + "key": "any", + "record": { + "type": "object", + "required": [ + "kind", + "createdAt" + ], + "properties": { + "kind": { + "type": "string", + "description": "Whether this user is being vouched for or denounced", + "enum": ["vouch", "denounce"], + "default": "vouch" + }, + "reason": { + "type": "string", + "description": "The reason for this vouch/denouncement", + "maxGraphemes": 256, + "maxLength": 2560 + }, + "evidences": { + "type": "array", + "description": "Optional list of ATURIs serving as evidence for this vouch (ex. issues, PRs)", + "maxLength": 10, + "items": { + "type": "ref", + "ref": "com.atproto.repo.strongRef" + } + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/key/sshKey.json b/lexicons/org/tangled/key/sshKey.json new file mode 100644 index 0000000000000000000000000000000000000000..7d3564e5d72dd37fe4157f4a1e0638ad110326d2 --- /dev/null +++ b/lexicons/org/tangled/key/sshKey.json @@ -0,0 +1,44 @@ +{ + "lexicon": 1, + "id": "org.tangled.key.sshKey", + "defs": { + "main": { + "type": "record", + "description": "ssh public key", + "key": "tid", + "record": { + "type": "object", + "required": [ + "name", + "publicKey", + "createdAt" + ], + "properties": { + "name": { + "type": "string", + "maxLength": 255, + "description": "human-readable name for this key" + }, + "publicKey": { + "type": "string", + "maxLength": 4096, + "description": "public key in OpenSSH authorized_keys format" + }, + "usages": { + "type": "array", + "description": "default to `[\"auth\", \"sign\"]` when omitted.", + "minLength": 1, + "items": { + "type": "string", + "knownValues": ["auth", "sign"] + } + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/key/unspecced/gpgKey.json b/lexicons/org/tangled/key/unspecced/gpgKey.json new file mode 100644 index 0000000000000000000000000000000000000000..fbbcaceafca22da0606e429494e8c8a103e37e23 --- /dev/null +++ b/lexicons/org/tangled/key/unspecced/gpgKey.json @@ -0,0 +1,35 @@ +{ + "lexicon": 1, + "id": "org.tangled.key.unspecced.gpgKey", + "defs": { + "main": { + "type": "record", + "description": "gpg public key", + "key": "tid", + "record": { + "type": "object", + "required": [ + "name", + "publicKey", + "createdAt" + ], + "properties": { + "name": { + "type": "string", + "maxLength": 255, + "description": "human-readable name for this key" + }, + "publicKey": { + "type": "string", + "maxLength": 8192, + "description": "ASCII-armored public key block" + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/knot/block.json b/lexicons/org/tangled/knot/block.json new file mode 100644 index 0000000000000000000000000000000000000000..6c8795911ecb8ab1392665d2fae83646700bbd91 --- /dev/null +++ b/lexicons/org/tangled/knot/block.json @@ -0,0 +1,23 @@ +{ + "lexicon": 1, + "id": "org.tangled.knot.block", + "defs": { + "main": { + "type": "record", + "description": "Ban of an account from this knot, w/ the banned DID as rkey.", + "key": "any", + "record": { + "type": "object", + "required": [ + "createdAt" + ], + "properties": { + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/knot/describeKnot.json b/lexicons/org/tangled/knot/describeKnot.json new file mode 100644 index 0000000000000000000000000000000000000000..ffbf455f7575a5c91d82b8e0fb9af082c6514cbe --- /dev/null +++ b/lexicons/org/tangled/knot/describeKnot.json @@ -0,0 +1,61 @@ +{ + "lexicon": 1, + "id": "org.tangled.knot.describeKnot", + "defs": { + "main": { + "type": "query", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "did", + "version" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "version": { + "type": "string" + }, + "capabilities": { + "type": "array", + "items": { + "type": "string", + "knownValues": [ + "tangled1.0" + ] + }, + "description": "Protocol capability tokens this knot implements. Knots that omit this field are treated as legacy." + }, + "links": { + "type": "ref", + "description": "URLs of service policy documents.", + "ref": "#links" + }, + "contact": { + "type": "ref", + "description": "Contact information", + "ref": "#contact" + } + } + } + } + }, + "links": { + "type": "object", + "properties": { + "privacyPolicy": { "type": "string", "format": "uri" }, + "termsOfService": { "type": "string", "format": "uri" } + } + }, + "contact": { + "type": "object", + "properties": { + "email": { "type": "string" } + } + } + } +} diff --git a/lexicons/org/tangled/knot/member.json b/lexicons/org/tangled/knot/member.json new file mode 100644 index 0000000000000000000000000000000000000000..207ca96fd42334b0ffe7142ca59248790ddc8aca --- /dev/null +++ b/lexicons/org/tangled/knot/member.json @@ -0,0 +1,23 @@ +{ + "lexicon": 1, + "id": "org.tangled.knot.member", + "defs": { + "main": { + "type": "record", + "description": "Offer of membership on this knot, w/ member-DID as rkey.", + "key": "any", + "record": { + "type": "object", + "required": [ + "createdAt" + ], + "properties": { + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/label/countDefinitions.json b/lexicons/org/tangled/label/countDefinitions.json new file mode 100644 index 0000000000000000000000000000000000000000..fa80e9c74360c6df7980b8b492dbd2076584927a --- /dev/null +++ b/lexicons/org/tangled/label/countDefinitions.json @@ -0,0 +1,38 @@ +{ + "lexicon": 1, + "id": "org.tangled.label.countDefinitions", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "DID of the repo whose label definitions to count." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/label/definition.json b/lexicons/org/tangled/label/definition.json new file mode 100644 index 0000000000000000000000000000000000000000..0c47d084f6907aac731ddf757de35b3c8fe28e7e --- /dev/null +++ b/lexicons/org/tangled/label/definition.json @@ -0,0 +1,30 @@ +{ + "lexicon": 1, + "id": "org.tangled.label.definition", + "defs": { + "main": { + "type": "record", + "key": "any", + "record": { + "type": "object", + "required": ["name", "createdAt"], + "properties": { + "name": { + "type": "string", + "description": "The display name of this label.", + "maxLength": 640, + "maxGraphemes": 64 + }, + "color": { + "type": "string", + "description": "The hex value for the background color for the label. Appviews may choose to respect this." + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/label/defs.json b/lexicons/org/tangled/label/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..7a797765d85b152f39e4d420c37cb1b8e305518a --- /dev/null +++ b/lexicons/org/tangled/label/defs.json @@ -0,0 +1,17 @@ +{ + "lexicon": 1, + "id": "org.tangled.label.defs", + "defs": { + "labelToken": { + "type": "object", + "required": ["key"], + "properties": { + "key": { + "type": "string", + "description": "Record-key of org.tangled.label.definition owned by this repository.", + "format": "record-key" + } + } + } + } +} diff --git a/lexicons/org/tangled/label/listDefinitions.json b/lexicons/org/tangled/label/listDefinitions.json new file mode 100644 index 0000000000000000000000000000000000000000..5dad8848232f97bd6b9f013fa42c426a58605ddb --- /dev/null +++ b/lexicons/org/tangled/label/listDefinitions.json @@ -0,0 +1,70 @@ +{ + "lexicon": 1, + "id": "org.tangled.label.listDefinitions", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "DID of the repo whose label definitions to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded org.tangled.label.definition record" + } + } + } + } +} diff --git a/lexicons/org/tangled/markup/embed/image.json b/lexicons/org/tangled/markup/embed/image.json new file mode 100644 index 0000000000000000000000000000000000000000..49ebcd3df8ecc6fa56ec361ad7aaac126f1573c2 --- /dev/null +++ b/lexicons/org/tangled/markup/embed/image.json @@ -0,0 +1,17 @@ +{ + "lexicon": 1, + "id": "org.tangled.markup.embed.image", + "defs": { + "main": { + "type": "object", + "required": ["image"], + "properties": { + "image": { + "type": "blob", + "accept": ["image/*"], + "maxSize": 1000000 + } + } + } + } +} diff --git a/lexicons/org/tangled/markup/embed/video.json b/lexicons/org/tangled/markup/embed/video.json new file mode 100644 index 0000000000000000000000000000000000000000..09f85f1fe9fa5625a2a1e964190c20d32855d248 --- /dev/null +++ b/lexicons/org/tangled/markup/embed/video.json @@ -0,0 +1,17 @@ +{ + "lexicon": 1, + "id": "org.tangled.markup.embed.video", + "defs": { + "main": { + "type": "object", + "required": ["video"], + "properties": { + "video": { + "type": "blob", + "accept": ["video/mp4"], + "maxSize": 5000000 + } + } + } + } +} diff --git a/lexicons/org/tangled/markup/markdown.json b/lexicons/org/tangled/markup/markdown.json new file mode 100644 index 0000000000000000000000000000000000000000..a7b34a253af9c7e9766ab29d10beb615aa30a2d6 --- /dev/null +++ b/lexicons/org/tangled/markup/markdown.json @@ -0,0 +1,43 @@ +{ + "lexicon": 1, + "id": "org.tangled.markup.markdown", + "defs": { + "main": { + "type": "object", + "required": ["text"], + "description": "Tangled Flavored Markdown format text", + "properties": { + "text": { + "type": "string", + "maxGraphemes": 65536, + "maxLength": 262144, + "description": "Final post-processed markdown content that will be rendered" + }, + "blobs": { + "type": "array", + "items": { + "type": "union", + "refs": [ + "org.tangled.markup.embed.image", + "org.tangled.markup.embed.video" + ] + }, + "description": "list of blobs referenced in markdown" + } + } + }, + "inline": { + "type": "object", + "required": ["text"], + "description": "Tangled Flavored Markdown format text (inline)", + "properties": { + "text": { + "type": "string", + "maxGraphemes": 256, + "maxLength": 1024, + "description": "Final post-processed markdown content that will be rendered" + } + } + } + } +} diff --git a/lexicons/org/tangled/markup/plaintext.json b/lexicons/org/tangled/markup/plaintext.json new file mode 100644 index 0000000000000000000000000000000000000000..f84f69e520cb461db417a1b7d740f6f8d0e70192 --- /dev/null +++ b/lexicons/org/tangled/markup/plaintext.json @@ -0,0 +1,16 @@ +{ + "lexicon": 1, + "id": "org.tangled.markup.plaintext", + "defs": { + "main": { + "type": "object", + "required": ["text"], + "properties": { + "text": { + "type": "string", + "maxLength": 262144 + } + } + } + } +} diff --git a/lexicons/org/tangled/publicKey/countKeys.json b/lexicons/org/tangled/publicKey/countKeys.json new file mode 100644 index 0000000000000000000000000000000000000000..1cde25a977fa9ce627a80b8d84924b6475324af5 --- /dev/null +++ b/lexicons/org/tangled/publicKey/countKeys.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.publicKey.countKeys", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Owner DID whose public-key records to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/publicKey/getPublicKey.json b/lexicons/org/tangled/publicKey/getPublicKey.json new file mode 100644 index 0000000000000000000000000000000000000000..bd8a0ee51669bfd2c2f075ef658071c8161aaafb --- /dev/null +++ b/lexicons/org/tangled/publicKey/getPublicKey.json @@ -0,0 +1,41 @@ +{ + "lexicon": 1, + "id": "org.tangled.publicKey.getPublicKey", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["publicKey"], + "properties": { + "publicKey": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the sh.tangled.publicKey record to fetch." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.publicKey record." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/publicKey/listKeys.json b/lexicons/org/tangled/publicKey/listKeys.json new file mode 100644 index 0000000000000000000000000000000000000000..de446e54e3f30131450d29408d3d56c472ca0ed3 --- /dev/null +++ b/lexicons/org/tangled/publicKey/listKeys.json @@ -0,0 +1,71 @@ +{ + "lexicon": 1, + "id": "org.tangled.publicKey.listKeys", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Owner DID whose public-key records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.publicKey record" + } + } + } + } +} diff --git a/lexicons/org/tangled/query/enrichResponse.json b/lexicons/org/tangled/query/enrichResponse.json new file mode 100644 index 0000000000000000000000000000000000000000..9ff01597ce1c3618c6280b2a226a3d649f9e8c74 --- /dev/null +++ b/lexicons/org/tangled/query/enrichResponse.json @@ -0,0 +1,156 @@ +{ + "lexicon": 1, + "id": "org.tangled.query.enrichResponse", + "defs": { + "main": { + "type": "procedure", + "description": "Run an XRPC query and hydrate its response with a data sidecar: typed payloads for every did / at-uri / strong-ref found in the response.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["xrpc", "enrich"], + "properties": { + "xrpc": { + "type": "string", + "format": "nsid", + "description": "NSID of the inner query to run, e.g. org.tangled.repo.listRepos." + }, + "params": { + "type": "unknown", + "description": "Parameters for the inner query, exactly as it declares them." + }, + "enrich": { + "type": "array", + "items": { + "type": "ref", + "ref": "#linkDescriptor" + }, + "description": "Payloads to compute for each found reference." + }, + "viewer": { + "type": "string", + "format": "did", + "description": "DID whose own relation to each reference is looked up for viewer payloads and views. Required when any descriptor requests a viewer payload." + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["output", "data"], + "properties": { + "output": { + "type": "unknown", + "description": "The inner query's response, unmodified." + }, + "data": { + "type": "unknown", + "description": "Reference (did or at-uri) -> link source -> payload type nsid -> payload. References no descriptor applies to are absent." + } + } + } + }, + "errors": [ + { + "name": "UnknownQuery", + "description": "The xrpc parameter does not name a query this server can execute" + }, + { + "name": "InvalidTargetPath", + "description": "A targets entry is not a valid record path" + }, + { + "name": "InvalidLinkDescriptor", + "description": "An enrich entry names an unknown payload type, collection, or path" + } + ] + }, + "linkDescriptor": { + "type": "object", + "required": ["source", "type"], + "description": "One sidecar payload, computed from the records at a link source whose reference equals the found reference.", + "properties": { + "source": { + "type": "string", + "description": "Collection whose records are linked, a colon, then an index-backed path (subject or .repo), or . for the record the reference names, which hydrates into that collection's view." + }, + "type": { + "type": "string", + "knownValues": [ + "org.tangled.query.enrichResponse#count", + "org.tangled.query.enrichResponse#distinctAuthors", + "org.tangled.query.enrichResponse#viewer", + "blue.microcosm.identity.resolveMiniDoc", + "org.tangled.repo.defs#repoViewBasic", + "org.tangled.query.enrichResponse#labels", + "org.tangled.query.enrichResponse#reactions" + ], + "description": "NSID of the payload type to produce, which also defines the payload's shape. Results land at data[ref][source][type]." + }, + "targets": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Record paths into the inner response selecting references for this payload. A path can end in an envelope: path:.repo for a ref's author, path:.rkey for the did its rkey names, path:. for the ref itself. All references are considered when omitted." + } + } + }, + "labels": { + "type": "object", + "description": "Moderation labels the configured labeler applied. Only !hide and !warn are served, takedowns leave subjects out instead. Use collection:. for labels on the record, collection:.repo for labels on its author's account. Absent when there are none.", + "required": ["labels"], + "properties": { + "labels": { + "type": "array", + "items": { "type": "ref", "ref": "com.atproto.label.defs#label" } + } + } + }, + "reactions": { + "type": "object", + "description": "Reaction counts per emoji on a subject, each with its newest reactors and the viewer's own reaction. Use org.tangled.feed.reaction:subject. Reactors' profiles come back as blue.microcosm.identity.resolveMiniDoc payloads under their dids when cached. Emojis keep a fixed order, not by count. Absent when nobody reacted.", + "required": ["reactions"], + "properties": { + "reactions": { + "type": "array", + "items": { "type": "ref", "ref": "#reactionSummary" } + } + } + }, + "reactionSummary": { + "type": "object", + "required": ["reaction", "count", "reactors"], + "properties": { + "reaction": { "type": "string", "maxGraphemes": 1 }, + "count": { "type": "integer", "minimum": 1 }, + "reactors": { + "type": "array", + "maxLength": 10, + "items": { "type": "string", "format": "did" }, + "description": "The newest few who reacted with this emoji, at most 10." + }, + "viewer": { + "type": "ref", + "ref": "#viewerReaction", + "description": "The viewer's own reaction with this emoji, when a viewer was given and has one." + } + } + }, + "viewerReaction": { + "type": "object", + "required": ["uri"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { + "type": "string", + "format": "cid", + "description": "Absent when the record couldn't be fetched." + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/artifact.json b/lexicons/org/tangled/repo/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..e9b8e52389dca6d183ab184c9d4f2ea26c835c10 --- /dev/null +++ b/lexicons/org/tangled/repo/artifact.json @@ -0,0 +1,43 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.artifact", + "defs": { + "main": { + "type": "record", + "key": "tid", + "record": { + "type": "object", + "required": [ + "subject", + "name", + "createdAt", + "artifact" + ], + "properties": { + "subject": { + "type": "union", + "refs": ["org.tangled.git.oid"] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "name of the artifact" + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "time of creation of this artifact" + }, + "artifact": { + "type": "blob", + "description": "the artifact", + "accept": [ + "*/*" + ], + "maxSize": 52428800 + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/countArtifacts.json b/lexicons/org/tangled/repo/countArtifacts.json new file mode 100644 index 0000000000000000000000000000000000000000..e7f350600636db15cff3ad7b2b17d89f1e0bc6df --- /dev/null +++ b/lexicons/org/tangled/repo/countArtifacts.json @@ -0,0 +1,38 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.countArtifacts", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Repo or release identifier whose artifacts to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/countArtifactsBy.json b/lexicons/org/tangled/repo/countArtifactsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..9bf555d829c5929254abe4af3aad5feca26721da --- /dev/null +++ b/lexicons/org/tangled/repo/countArtifactsBy.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.countArtifactsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose artifact authorings to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/countCollaborators.json b/lexicons/org/tangled/repo/countCollaborators.json new file mode 100644 index 0000000000000000000000000000000000000000..4c1b2cd7903523b54b33220c32383312411f7997 --- /dev/null +++ b/lexicons/org/tangled/repo/countCollaborators.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.countCollaborators", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID whose collaborator records to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/countCollaboratorsBy.json b/lexicons/org/tangled/repo/countCollaboratorsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..3d83af8f5d84f37f59dd44f593a1d9ced4c4954a --- /dev/null +++ b/lexicons/org/tangled/repo/countCollaboratorsBy.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.countCollaboratorsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose collaborator authorings to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/countForks.json b/lexicons/org/tangled/repo/countForks.json new file mode 100644 index 0000000000000000000000000000000000000000..a4f9b4a0b0dea50d94fc2d7ef179c30efb88b989 --- /dev/null +++ b/lexicons/org/tangled/repo/countForks.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.countForks", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID to count forks of." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/countIssues.json b/lexicons/org/tangled/repo/countIssues.json new file mode 100644 index 0000000000000000000000000000000000000000..ca2dd602141a00672dda9820b6de1dfa5430a47d --- /dev/null +++ b/lexicons/org/tangled/repo/countIssues.json @@ -0,0 +1,49 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.countIssues", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID to list issues for" + }, + "author": { + "type": "string", + "format": "did", + "description": "Restrict to issues authored by this user DID." + }, + "state": { + "type": "string", + "knownValues": ["open", "closed"], + "description": "Restrict to issues whose latest derived state matches." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/countPulls.json b/lexicons/org/tangled/repo/countPulls.json new file mode 100644 index 0000000000000000000000000000000000000000..05411eec7c07031bc1c74ab15a4e3ccfc155afbc --- /dev/null +++ b/lexicons/org/tangled/repo/countPulls.json @@ -0,0 +1,49 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.countPulls", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID to list pulls for" + }, + "author": { + "type": "string", + "format": "did", + "description": "Restrict to pulls authored by this user DID." + }, + "status": { + "type": "string", + "knownValues": ["open", "closed", "merged"], + "description": "Restrict to pulls whose latest derived status matches." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/countRepos.json b/lexicons/org/tangled/repo/countRepos.json new file mode 100644 index 0000000000000000000000000000000000000000..3892c63180c3c4baca1b0e0b61ea3e3c0f73667a --- /dev/null +++ b/lexicons/org/tangled/repo/countRepos.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.countRepos", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Owner DID whose repo records to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/createRepo.json b/lexicons/org/tangled/repo/createRepo.json new file mode 100644 index 0000000000000000000000000000000000000000..de4a2524c341c76bcdfc143f5dcd0b1deb840a40 --- /dev/null +++ b/lexicons/org/tangled/repo/createRepo.json @@ -0,0 +1,84 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.createRepo", + "defs": { + "main": { + "type": "procedure", + "description": "Create a new repository", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string", + "format": "record-key", + "description": "Name of the repository." + }, + "did": { + "type": "string", + "format": "did", + "description": "Optional user-provided DID to use as the repo identity instead of minting one." + }, + "defaultBranch": { + "type": "string", + "description": "Default branch of the repository." + }, + "spindle": { + "type": "string", + "format": "did" + }, + "description": { + "type": "string", + "minGraphemes": 1, + "maxGraphemes": 140 + }, + "website": { + "type": "string", + "format": "uri", + "description": "Any URI related to the repo" + }, + "topics": { + "type": "array", + "description": "Topics related to the repo", + "items": { + "type": "string", + "maxLength": 50 + }, + "maxLength": 50 + }, + "source": { + "type": "union", + "refs": [ + "org.tangled.repo.defs#sourceTangled", + "org.tangled.repo.defs#sourceExternal" + ], + "description": "A source to clone from, populate this when forking or importing a repository." + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "key": { + "type": "string", + "description": "Multibase-encoded public signing key the knot holds for this repository" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/declaration.json b/lexicons/org/tangled/repo/declaration.json new file mode 100644 index 0000000000000000000000000000000000000000..9e1d739d0388c82af647e118d7dc0337f62fd0b4 --- /dev/null +++ b/lexicons/org/tangled/repo/declaration.json @@ -0,0 +1,26 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.declaration", + "defs": { + "main": { + "type": "record", + "key": "any", + "description": "Git repository declaration. Record is owned by the repository owner and record-key is slug of the git repository.", + "record": { + "type": "object", + "required": ["repo", "createdAt"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repo" + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/defs.json b/lexicons/org/tangled/repo/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..453d1a1769047d1b1844a7c6e201a0082c312cda --- /dev/null +++ b/lexicons/org/tangled/repo/defs.json @@ -0,0 +1,54 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.defs", + "defs": { + "sourceTangled": { + "type": "object", + "required": ["did"], + "properties": { + "did": { "type": "string", "format": "did", "description": "DID of the repo identity" } + } + }, + "sourceExternal": { + "type": "object", + "required": ["uri"], + "properties": { + "uri": { "type": "string", "format": "uri", "description": "git-clone uri of the repo" } + } + }, + "repoViewBasic": { + "type": "object", + "required": ["did", "owner", "slug", "createdAt"], + "properties": { + "did": { "type": "string", "format": "did" }, + "owner": { "type": "ref", "ref": "org.tangled.actor.defs#profileViewBasic" }, + "slug": { "type": "string" }, + "description": { "type": "string" }, + "createdAt": { "type": "string", "format": "datetime" }, + "forkCount": { "type": "integer" }, + "starCount": { "type": "integer" }, + "openIssueCount": { "type": "integer" }, + "openPullCount": { "type": "integer" }, + "primaryLanguage": { "type": "string", "description": "Largest language on the default branch, by bytes." }, + "viewer": { "type": "ref", "ref": "#viewerState" }, + "isStarred": { "type": "boolean" } + } + }, + "language": { + "type": "object", + "required": ["name", "size"], + "properties": { + "name": { "type": "string" }, + "size": { "type": "integer", "description": "Total size of files in this language, in bytes." } + } + }, + "viewerState": { + "type": "object", + "description": "Metadata about the requesting account's relationship with the subject content. Only has meaningful content for authed requests.", + "properties": { + "fork": { "type": "string", "format": "at-uri" }, + "star": { "type": "string", "format": "at-uri" } + } + } + } +} diff --git a/lexicons/org/tangled/repo/deleteRepo.json b/lexicons/org/tangled/repo/deleteRepo.json new file mode 100644 index 0000000000000000000000000000000000000000..681865fab00105400bfbc9a3b9917cd6c8bf632b --- /dev/null +++ b/lexicons/org/tangled/repo/deleteRepo.json @@ -0,0 +1,27 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.deleteRepo", + "defs": { + "main": { + "type": "procedure", + "description": "Delete a repository", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "did" + }, + "force": { + "type": "boolean", + "description": "Skip the check for a declaration record on the owner's PDS. Knot admins only." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/forkSync.json b/lexicons/org/tangled/repo/forkSync.json new file mode 100644 index 0000000000000000000000000000000000000000..96dd82aa93488bed5cc8e790456bb71040f47a3e --- /dev/null +++ b/lexicons/org/tangled/repo/forkSync.json @@ -0,0 +1,28 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.forkSync", + "defs": { + "main": { + "type": "procedure", + "description": "Fetch the upstream of a fork into this repository, updating the remote-tracking ref for the given branch.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repo", "branch"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the fork repository" + }, + "branch": { + "type": "string", + "description": "Branch to fetch from the upstream" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/getImportStatus.json b/lexicons/org/tangled/repo/getImportStatus.json new file mode 100644 index 0000000000000000000000000000000000000000..f388e38730bf2f8042de888844fdc80b3437a53e --- /dev/null +++ b/lexicons/org/tangled/repo/getImportStatus.json @@ -0,0 +1,48 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.getImportStatus", + "defs": { + "main": { + "type": "query", + "description": "How far the knot got importing a repository from its source. Only the repository owner and knot admins see the real reason; everyone else gets a generic one.", + "parameters": { + "type": "params", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "status": { + "type": "string", + "knownValues": ["fetching", "present", "partial", "failed"], + "description": "Where the import from the source is. Omitted when the repository wasn't created from a source." + }, + "reason": { + "type": "string", + "description": "Why a partial or failed import ended that way" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repo is not registered on this knot" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/repo/getRepo.json b/lexicons/org/tangled/repo/getRepo.json new file mode 100644 index 0000000000000000000000000000000000000000..766bd4646f6c5359d30503b90409a1c1145d08c5 --- /dev/null +++ b/lexicons/org/tangled/repo/getRepo.json @@ -0,0 +1,45 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.getRepo", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the sh.tangled.repo record to fetch." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo record." + }, + "knot": { + "type": "string", + "description": "Host of the knot named by the #tangled_knot service in the repo's DID document. Absent when it names none." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/getRepoByName.json b/lexicons/org/tangled/repo/getRepoByName.json new file mode 100644 index 0000000000000000000000000000000000000000..1b454d00668c772a491d7be0d754cd84da772102 --- /dev/null +++ b/lexicons/org/tangled/repo/getRepoByName.json @@ -0,0 +1,49 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.getRepoByName", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["owner", "name"], + "properties": { + "owner": { + "type": "string", + "format": "did", + "description": "DID of the account that owns the repo." + }, + "name": { + "type": "string", + "description": "Name of the repo as it appears in its url." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo record." + }, + "knot": { + "type": "string", + "description": "Host of the knot named by the #tangled_knot service in the repo's DID document. Absent when it names none." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/getRepoByRepoDid.json b/lexicons/org/tangled/repo/getRepoByRepoDid.json new file mode 100644 index 0000000000000000000000000000000000000000..a0cda195305580d9ce744d48090a6c21c9f36a49 --- /dev/null +++ b/lexicons/org/tangled/repo/getRepoByRepoDid.json @@ -0,0 +1,45 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.getRepoByRepoDid", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repoDid"], + "properties": { + "repoDid": { + "type": "string", + "format": "did", + "description": "Repo DID whose sh.tangled.repo record to fetch." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo record." + }, + "knot": { + "type": "string", + "description": "Host of the knot named by the #tangled_knot service in the repo's DID document. Absent when it names none." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/getRepos.json b/lexicons/org/tangled/repo/getRepos.json new file mode 100644 index 0000000000000000000000000000000000000000..423756ec4cebd2589ba9fd93a50138281298b964 --- /dev/null +++ b/lexicons/org/tangled/repo/getRepos.json @@ -0,0 +1,58 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.getRepos", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repos"], + "properties": { + "repos": { + "type": "array", + "items": { + "type": "string", + "format": "at-uri" + }, + "maxLength": 50, + "description": "AT-URIs of the sh.tangled.repo records to fetch. At most 50 per request." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "#recordView" + } + } + } + } + } + }, + "recordView": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo record." + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/getReposByRepoDids.json b/lexicons/org/tangled/repo/getReposByRepoDids.json new file mode 100644 index 0000000000000000000000000000000000000000..a517c7525557c9f7940ca142ff4047d7e3db1e05 --- /dev/null +++ b/lexicons/org/tangled/repo/getReposByRepoDids.json @@ -0,0 +1,46 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.getReposByRepoDids", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["dids"], + "properties": { + "dids": { + "type": "array", + "items": { "type": "string", "format": "did" }, + "maxLength": 50, + "description": "Repo DIDs to fetch. DIDs that resolve to none are omitted from the response." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "sh.tangled.repo record" + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listArtifacts.json b/lexicons/org/tangled/repo/listArtifacts.json new file mode 100644 index 0000000000000000000000000000000000000000..8bd9eec255f647c8a1895b749bb1d0f8d8715e72 --- /dev/null +++ b/lexicons/org/tangled/repo/listArtifacts.json @@ -0,0 +1,70 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listArtifacts", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Repo or release identifier whose artifacts to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded org.tangled.repo.artifact record" + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listArtifactsBy.json b/lexicons/org/tangled/repo/listArtifactsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..036db2233e8baf693d6130aaf23dcf21dcea53b3 --- /dev/null +++ b/lexicons/org/tangled/repo/listArtifactsBy.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listArtifactsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose artifact authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "org.tangled.repo.listArtifacts#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listCollaboratorInvitesBy.json b/lexicons/org/tangled/repo/listCollaboratorInvitesBy.json new file mode 100644 index 0000000000000000000000000000000000000000..c18904f53b44b32df68c1243020aa36796bdba4e --- /dev/null +++ b/lexicons/org/tangled/repo/listCollaboratorInvitesBy.json @@ -0,0 +1,75 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listCollaboratorInvitesBy", + "defs": { + "main": { + "type": "query", + "description": "Collaboration offers an actor DID has waiting for them. The inverse of sh.tangled.repo.listCollaboratorInvites, which knots serve only for their own repo.", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID offered collaboration." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items", "pending", "truncated"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#inviteItem" } + }, + "pending": { + "type": "array", + "description": "Knots that this index hasn't read yet", + "items": { "type": "string", "format": "did" } + }, + "truncated": { + "type": "boolean", + "description": "Per-subject dropped offers from items. If truncated is true, missing offers won't say if they were withdrawn, same with pending knots." + } + } + } + } + }, + "inviteItem": { + "type": "object", + "description": "An offer awaiting the subject's acceptance.", + "required": ["uri", "knot", "repo", "addedBy", "createdAt"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "Invite record on the repository." + }, + "knot": { + "type": "string", + "format": "did", + "description": "did:web of knot hosting this repo." + }, + "repo": { + "type": "string", + "format": "did", + "description": "DID of repo that made the offer." + }, + "addedBy": { + "type": "string", + "format": "did", + "description": "DID that made the offer." + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "When the repository made the offer. This listing sorts it by newest first." + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listCollaborators.json b/lexicons/org/tangled/repo/listCollaborators.json new file mode 100644 index 0000000000000000000000000000000000000000..7490c3f7adf46b7e4859ca1e8a36241766556f93 --- /dev/null +++ b/lexicons/org/tangled/repo/listCollaborators.json @@ -0,0 +1,100 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listCollaborators", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID whose collaborator records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by effectiveSince." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["subject", "addedBy", "createdAt", "effectiveSince"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "DID of the collaborator" + }, + "addedBy": { + "type": "string", + "format": "did", + "description": "DID that added this collaborator" + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "When the repository offered collaboration" + }, + "verifiedAt": { + "type": "string", + "format": "datetime", + "description": "When the knot resolved the collaborator's own acceptance record. Absent on a grant that predates the consent model." + }, + "effectiveSince": { + "type": "string", + "format": "datetime", + "description": "When the collaboration took effect. This listing sorts and pages on it." + }, + "uri": { + "type": "string", + "format": "at-uri", + "description": "Optional record AT-URI for record-backed indexers" + }, + "cid": { + "type": "string", + "format": "cid", + "description": "Optional record CID for record-backed indexers" + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listCollaboratorsBy.json b/lexicons/org/tangled/repo/listCollaboratorsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..e3b4e73b039592811959b609c1820c0e8aa95971 --- /dev/null +++ b/lexicons/org/tangled/repo/listCollaboratorsBy.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listCollaboratorsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose collaborator authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "org.tangled.repo.listCollaborators#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listForks.json b/lexicons/org/tangled/repo/listForks.json new file mode 100644 index 0000000000000000000000000000000000000000..1e23676841f56646ead9522b7e48d06580716692 --- /dev/null +++ b/lexicons/org/tangled/repo/listForks.json @@ -0,0 +1,71 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listForks", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID to list forks of." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded org.tangled.repo.manifest record of the fork" + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listIssues.json b/lexicons/org/tangled/repo/listIssues.json new file mode 100644 index 0000000000000000000000000000000000000000..fa4e1ded33b8e83e1965d1608f9623544e90391f --- /dev/null +++ b/lexicons/org/tangled/repo/listIssues.json @@ -0,0 +1,112 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listIssues", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID to list issues for" + }, + "author": { + "type": "string", + "format": "did", + "description": "Restrict to issues authored by this user DID." + }, + "state": { + "type": "string", + "knownValues": ["open", "closed"], + "description": "Restrict to issues whose latest derived state matches." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "#issueListItem" + } + }, + "cursor": { + "type": "string" + }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "issueListItem": { + "type": "object", + "required": ["uri", "value", "state", "commentCount"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo.issue record" + }, + "author": { + "type": "string", + "format": "did", + "description": "The account that opened the record, the editor of its first version. Left out when the author can't be told." + }, + "state": { + "type": "string", + "knownValues": ["open", "closed"], + "description": "Latest derived state." + }, + "stateUpdatedAt": { + "type": "string", + "format": "datetime", + "description": "TID-derived timestamp of the latest state record." + }, + "commentCount": { + "type": "integer", + "minimum": 0, + "description": "Count of sh.tangled.repo.issue.comment records targeting this issue." + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listIssuesBy.json b/lexicons/org/tangled/repo/listIssuesBy.json new file mode 100644 index 0000000000000000000000000000000000000000..7c88f8d811899c2373c12bdfbfd4691ca863f39b --- /dev/null +++ b/lexicons/org/tangled/repo/listIssuesBy.json @@ -0,0 +1,69 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listIssuesBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose issue authorings to list" + }, + "state": { + "type": "string", + "knownValues": ["open", "closed"], + "description": "Restrict to issues whose latest derived state matches." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.repo.listIssues#issueListItem" + } + }, + "cursor": { + "type": "string" + }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listPulls.json b/lexicons/org/tangled/repo/listPulls.json new file mode 100644 index 0000000000000000000000000000000000000000..df3fe0e2353503958b5e0823aef87057acb32cb0 --- /dev/null +++ b/lexicons/org/tangled/repo/listPulls.json @@ -0,0 +1,112 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listPulls", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID to list pulls for" + }, + "author": { + "type": "string", + "format": "did", + "description": "Restrict to pulls authored by this user DID." + }, + "status": { + "type": "string", + "knownValues": ["open", "closed", "merged"], + "description": "Restrict to pulls whose latest derived status matches." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "#pullListItem" + } + }, + "cursor": { + "type": "string" + }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "pullListItem": { + "type": "object", + "required": ["uri", "value", "state", "commentCount"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo.pull record" + }, + "author": { + "type": "string", + "format": "did", + "description": "The account that opened the record, the editor of its first version. Left out when the author can't be told." + }, + "state": { + "type": "string", + "knownValues": ["open", "closed", "merged"], + "description": "Latest derived state." + }, + "stateUpdatedAt": { + "type": "string", + "format": "datetime", + "description": "TID-derived timestamp of the latest pull status record." + }, + "commentCount": { + "type": "integer", + "minimum": 0, + "description": "Count of sh.tangled.repo.pull.comment records targeting this pull." + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listPullsBy.json b/lexicons/org/tangled/repo/listPullsBy.json new file mode 100644 index 0000000000000000000000000000000000000000..d0f7a2255ac3cd472ab706944b94b43a83064705 --- /dev/null +++ b/lexicons/org/tangled/repo/listPullsBy.json @@ -0,0 +1,69 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listPullsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose pull authorings to list" + }, + "status": { + "type": "string", + "knownValues": ["open", "closed", "merged"], + "description": "Restrict to pulls whose latest derived status matches." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.repo.listPulls#pullListItem" + } + }, + "cursor": { + "type": "string" + }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listRecordVersions.json b/lexicons/org/tangled/repo/listRecordVersions.json new file mode 100644 index 0000000000000000000000000000000000000000..07acb729bd46b2641c2fe529d2baf0ca6452e912 --- /dev/null +++ b/lexicons/org/tangled/repo/listRecordVersions.json @@ -0,0 +1,100 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listRecordVersions", + "defs": { + "main": { + "type": "query", + "description": "Every version of a record this knot hosts, oldest first unless order says otherwise. A version's bytes are fetched by content id through com.atproto.sync.getBlocks.", + "parameters": { + "type": "params", + "required": ["uri"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "Record in a repo this knot hosts" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 500, + "default": 100 + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "asc", + "description": "asc lists the oldest version first, desc the newest." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "versions"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "versions": { + "type": "array", + "items": { + "type": "ref", + "ref": "#version" + } + }, + "cursor": { + "type": "string", + "description": "Cursor for the next page, left out when the page already reaches the newest version" + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "The uri isn't an at-uri, or the collection isn't versioned" + }, + { + "name": "NotFound", + "description": "No such record on this knot" + }, + { + "name": "ProjectionWarming", + "description": "The record's change history is still being folded; retry shortly" + } + ] + }, + "version": { + "type": "object", + "required": ["changeId", "editor", "createdAt"], + "properties": { + "changeId": { + "type": "string", + "description": "Object ID of the git change that produced this version" + }, + "editor": { + "type": "string", + "format": "did", + "description": "The account that authored the change" + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "When this knot wrote the commit this version rides in. Not the record's own `createdAt`." + }, + "cid": { + "type": "string", + "format": "cid", + "description": "Content id of this version. Missing when the version was removed or the change is a deletion." + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/listRepos.json b/lexicons/org/tangled/repo/listRepos.json new file mode 100644 index 0000000000000000000000000000000000000000..7f32b3b67ad0b1e73b806cf9e617e31cafcf1559 --- /dev/null +++ b/lexicons/org/tangled/repo/listRepos.json @@ -0,0 +1,71 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.listRepos", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Owner DID whose repo records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo record" + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/manifest.json b/lexicons/org/tangled/repo/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..6c2a9c5098e9959db6b3c730a926b436cdaf72d5 --- /dev/null +++ b/lexicons/org/tangled/repo/manifest.json @@ -0,0 +1,76 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.manifest", + "defs": { + "main": { + "type": "record", + "description": "Git repository manifest record. This record is used for git repository discovery like how `org.tangled.actor.profile` is used to find users.", + "key": "literal:self", + "record": { + "type": "object", + "required": [ + "declaration", + "createdAt" + ], + "properties": { + "declaration": { + "type": "ref", + "ref": "#declaration" + }, + "spindle": { + "type": "string", + "format": "did" + }, + "description": { + "type": "string", + "minGraphemes": 1, + "maxGraphemes": 140 + }, + "website": { + "type": "string", + "format": "uri", + "description": "Any URI related to the repo" + }, + "topics": { + "type": "array", + "description": "Topics related to the repo", + "items": { + "type": "string", + "maxLength": 50 + }, + "maxLength": 50 + }, + "source": { + "type": "union", + "description": "source of the repo", + "refs": [ + "org.tangled.repo.defs#sourceTangled", + "org.tangled.repo.defs#sourceExternal" + ] + }, + "contributionPolicy": { + "type": "string", + "knownValues": ["anyone", "members", "collaborators"], + "description": "Who may open tickets and patches on this repo. anyone = any DID, members = the knot's member roll, collaborators = the owner plus repo collaborators. Absent means the knot-wide default." + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + }, + "declaration": { + "type": "object", + "required": ["owner", "name"], + "properties": { + "owner": { "type": "string", "format": "did" }, + "name": { + "type": "string", + "format": "record-key", + "description": "repository name; the slug (rkey of org.tangled.repo.declaration) is lowercase(name)" + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/mintToken.json b/lexicons/org/tangled/repo/mintToken.json new file mode 100644 index 0000000000000000000000000000000000000000..1c2c29d9930bd7ee5bc17f628f070b6d00ff7d4e --- /dev/null +++ b/lexicons/org/tangled/repo/mintToken.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.mintToken", + "defs": { + "main": { + "type": "procedure", + "description": "Mint a service-auth JWT whose issuer is this repo's DID, signed by the knot that owns the repo's signing key. When calling this, you'll need the same perms as git push, ie. being repo owner/collaborator/knot-admin.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "aud", + "lxm", + "exp" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repo that becomes the token's issuer. Must be the audience of a given calling session." + }, + "aud": { + "type": "string", + "format": "did", + "description": "Audience of the minted token, a DID service ref for the service, that the token is addressed to." + }, + "lxm": { + "type": "string", + "format": "nsid", + "description": "Lexicon method to which minted token binds." + }, + "exp": { + "type": "integer", + "format": "int64", + "description": "Unix seconds. Must be in the future, and at most 60s after mint time." + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "token" + ], + "properties": { + "token": { + "type": "string", + "description": "The signed service-auth JWT." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/repo/reserveKey.json b/lexicons/org/tangled/repo/reserveKey.json new file mode 100644 index 0000000000000000000000000000000000000000..8af516cbeb0217904ba7dd071a6dfb8e2ac8f63e --- /dev/null +++ b/lexicons/org/tangled/repo/reserveKey.json @@ -0,0 +1,41 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.reserveKey", + "defs": { + "main": { + "type": "procedure", + "description": "Reserve a repo identity under a DID the caller brings. The knot seals and holds a signing key for the repo DID; the caller publishes the matching public key in that DID's document before calling org.tangled.repo.createRepo with the same DID.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "The caller-minted DID to reserve" + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repo", "key"], + "properties": { + "repo": { + "type": "string", + "format": "did" + }, + "key": { + "type": "string", + "description": "Multibase-encoded public signing key the knot holds for this repo identity" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/review/patch.json b/lexicons/org/tangled/review/patch.json new file mode 100644 index 0000000000000000000000000000000000000000..8b425c73a640620a6f1806d6db9d697ec5d8ca55 --- /dev/null +++ b/lexicons/org/tangled/review/patch.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.review.patch", + "defs": { + "main": { + "type": "record", + "key": "any", + "description": "Immutable patch attached to the ticket. Record-key must be CID of this record.", + "record": { + "type": "object", + "required": [ + "ticket", + "base", + "head", + "createdAt" + ], + "properties": { + "ticket": { + "type": "string", + "format": "record-key", + "description": "ticket-id" + }, + "base": { "type": "ref", "ref": "org.tangled.git.oid" }, + "head": { "type": "ref", "ref": "org.tangled.git.oid" }, + "source": { "type": "ref", "ref": "#source" }, + "createdAt": { "type": "string", "format": "datetime" } + } + } + }, + "source": { + "type": "object", + "required": ["repo", "branch"], + "properties": { + "repo": { "type": "string", "format": "did" }, + "branch": { "type": "string", "maxLength": 244 } + } + } + } +} diff --git a/lexicons/org/tangled/review/submitPatch.json b/lexicons/org/tangled/review/submitPatch.json new file mode 100644 index 0000000000000000000000000000000000000000..1e125f30dec99f1220c8e4d6e6ede7533ff8a2b8 --- /dev/null +++ b/lexicons/org/tangled/review/submitPatch.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "org.tangled.review.submitPatch", + "defs": { + "main": { + "type": "procedure", + "description": "Create a patch record, fetching its head from the source repository first. The only way to write org.tangled.review.patch records.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repo", "patch"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "patch": { "type": "ref", "ref": "#patch" } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "cid"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of created org.tangled.review.patch record." + }, + "cid": { + "type": "string", + "format": "cid", + "description": "CID of created org.tangled.review.patch record." + } + } + } + } + }, + "patch": { + "type": "object", + "required": ["ticket", "base", "head", "sourceRepo"], + "properties": { + "ticket": { "type": "string", "format": "record-key", "description": "ticket-id" }, + "base": { "type": "ref", "ref": "org.tangled.git.oid" }, + "head": { "type": "ref", "ref": "org.tangled.git.oid" }, + "sourceRepo": { + "type": "string", + "format": "did", + "description": "DID of the repository to fetch `head` from." + }, + "sourceBranch": { "type": "string", "maxLength": 244 } + } + } + } +} diff --git a/lexicons/org/tangled/review/unspecced/defs.json b/lexicons/org/tangled/review/unspecced/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..e54c7cca426c74349eba9ea77981d20859dfc4f3 --- /dev/null +++ b/lexicons/org/tangled/review/unspecced/defs.json @@ -0,0 +1,15 @@ +{ + "lexicon": 1, + "id": "org.tangled.review.unspecced.defs", + "defs": { + "reviewState": { + "type": "string", + "knownValues": [ + "commented", + "approved", + "changes-required", + "dismissed" + ] + } + } +} diff --git a/lexicons/org/tangled/review/unspecced/round.json b/lexicons/org/tangled/review/unspecced/round.json new file mode 100644 index 0000000000000000000000000000000000000000..ad8255d9e3cae262b534dbca8466daf632a2f605 --- /dev/null +++ b/lexicons/org/tangled/review/unspecced/round.json @@ -0,0 +1,43 @@ +{ + "lexicon": 1, + "id": "org.tangled.repo.review.unspecced.round", + "defs": { + "main": { + "type": "record", + "key": "tid", + "description": "Review round created by a reviewer.", + "record": { + "type": "object", + "required": [ + "subject", + "commit", + "state", + "createdAt" + ], + "properties": { + "subject": { + "type": "string", + "format": "record-key", + "description": "ticket-id" + }, + "commit": { + "type": "ref", + "ref": "org.tangled.git.oid", + "description": "ID of commit that is getting reviewed." + }, + "state": { + "type": "ref", + "ref": "org.tangled.review.unspecced.defs#reviewState", + "description": "The reviewed state of that commit. This is one and only meaningful content of this record." + }, + "labels": { + "type": "ref", + "description": "Self-label values for this record. Most common use-case is 'bot'.", + "ref": "com.atproto.label.defs#selfLabels" + }, + "createdAt": { "type": "string", "format": "datetime" } + } + } + } + } +} diff --git a/lexicons/org/tangled/search/query.json b/lexicons/org/tangled/search/query.json new file mode 100644 index 0000000000000000000000000000000000000000..2274dd0fe48d3943143f8ffd0ec9d917c9bc9b4a --- /dev/null +++ b/lexicons/org/tangled/search/query.json @@ -0,0 +1,175 @@ +{ + "lexicon": 1, + "id": "org.tangled.search.query", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [], + "properties": { + "q": { + "type": "string", + "maxLength": 1024, + "description": "Search query: free text plus key:value qualifiers. See the docs for more information." + }, + "nsid": { + "type": "string", + "format": "nsid", + "description": "Restrict to records of this collection." + }, + "author": { + "type": "string", + "format": "did", + "description": "Restrict to records authored by this DID." + }, + "repo": { + "type": "string", + "format": "did", + "description": "Restrict to records under this repo DID." + }, + "since": { + "type": "string", + "format": "datetime", + "description": "Restrict to records created at or after this time." + }, + "until": { + "type": "string", + "format": "datetime", + "description": "Restrict to records created at or before this time." + }, + "kind": { + "type": "string", + "knownValues": ["issue", "pull"], + "description": "Restrict tickets to issues or pull requests." + }, + "label": { + "type": "array", + "items": { + "type": "string", + "maxLength": 256 + }, + "maxLength": 16, + "description": "Restrict tickets to those with all of these label names. Requires repo." + }, + "sort": { + "type": "string", + "knownValues": ["relevance", "created-desc", "created-asc"], + "description": "Result order." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for numbered pages. Mutually exclusive with cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["hits"], + "properties": { + "hits": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hit" + } + }, + "cursor": { + "type": "string" + }, + "total": { + "type": "integer", + "minimum": 0, + "description": "Number of matches across all pages." + }, + "stateCounts": { + "type": "ref", + "ref": "#stateCounts", + "description": "Matches per ticket state, without the state filters." + } + } + } + }, + "description": "Full-text and faceted search over indexed records.", + "errors": [ + { + "name": "InvalidQuery", + "description": "The query failed to parse." + } + ] + }, + "hit": { + "type": "object", + "required": ["uri", "nsid", "score", "value"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "nsid": { + "type": "string", + "format": "nsid", + "description": "Collection of the matched record." + }, + "score": { + "type": "unknown", + "description": "Relevance score of the hit, a floating-point number where higher ranks first." + }, + "value": { + "type": "unknown", + "description": "Embedded matched record." + }, + "author": { + "type": "string", + "format": "did", + "description": "Present for tickets. The account that opened the ticket, the editor of its first version." + }, + "state": { + "type": "ref", + "ref": "org.tangled.track.defs#ticketState", + "description": "Present for tickets." + }, + "commentCount": { + "type": "integer", + "minimum": 0, + "description": "Present for tickets." + } + } + }, + "stateCounts": { + "type": "object", + "required": ["open", "closed", "completed"], + "properties": { + "open": { + "type": "integer", + "minimum": 0 + }, + "closed": { + "type": "integer", + "minimum": 0 + }, + "completed": { + "type": "integer", + "minimum": 0 + } + } + } + } +} diff --git a/lexicons/org/tangled/secret/addSecret.json b/lexicons/org/tangled/secret/addSecret.json new file mode 100644 index 0000000000000000000000000000000000000000..f135269abf39475082d89b4c58633399c7c99942 --- /dev/null +++ b/lexicons/org/tangled/secret/addSecret.json @@ -0,0 +1,51 @@ +{ + "lexicon": 1, + "id": "org.tangled.secret.addSecret", + "defs": { + "main": { + "type": "procedure", + "description": "Add a CI secret", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "key", + "value" + ], + "properties": { + "repo": { + "type": "string", + "format": "did" + }, + "key": { + "type": "string", + "maxLength": 50, + "minLength": 1 + }, + "value": { + "type": "string", + "maxLength": 4096, + "minLength": 1 + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "RepoNotFound", + "description": "Repository not found" + }, + { + "name": "SecretAlreadyExists", + "description": "A secret with this key already exists." + } + ] + } + } +} diff --git a/lexicons/org/tangled/secret/listSecrets.json b/lexicons/org/tangled/secret/listSecrets.json new file mode 100644 index 0000000000000000000000000000000000000000..be20b5e4999aa2f9820d9117d9a922e71ffd9e6c --- /dev/null +++ b/lexicons/org/tangled/secret/listSecrets.json @@ -0,0 +1,72 @@ +{ + "lexicon": 1, + "id": "org.tangled.secret.listSecrets", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo" + ], + "properties": { + "repo": { + "type": "string", + "format": "did" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "secrets" + ], + "properties": { + "secrets": { + "type": "array", + "items": { + "type": "ref", + "ref": "#secret" + } + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "RepoNotFound", + "description": "Repository not found" + } + ] + }, + "secret": { + "type": "object", + "required": [ + "key", + "createdAt", + "createdBy" + ], + "properties": { + "key": { + "type": "string", + "maxLength": 50, + "minLength": 1 + }, + "createdAt": { + "type": "string", + "format": "datetime" + }, + "createdBy": { + "type": "string", + "format": "did" + } + } + } + } +} diff --git a/lexicons/org/tangled/secret/removeSecret.json b/lexicons/org/tangled/secret/removeSecret.json new file mode 100644 index 0000000000000000000000000000000000000000..c177a515a5d486f0e3606519832b804a6cf82e49 --- /dev/null +++ b/lexicons/org/tangled/secret/removeSecret.json @@ -0,0 +1,45 @@ +{ + "lexicon": 1, + "id": "org.tangled.secret.removeSecret", + "defs": { + "main": { + "type": "procedure", + "description": "Remove a CI secret", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "key" + ], + "properties": { + "repo": { + "type": "string", + "format": "did" + }, + "key": { + "type": "string", + "maxLength": 50, + "minLength": 1 + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "RepoNotFound", + "description": "Repository not found" + }, + { + "name": "SecretNotFound", + "description": "The requested secret does not exist." + } + ] + } + } +} diff --git a/lexicons/org/tangled/snippet/string.json b/lexicons/org/tangled/snippet/string.json new file mode 100644 index 0000000000000000000000000000000000000000..c7cbd4f8aba59ad46e0c226c6d5079a2b67ac5b5 --- /dev/null +++ b/lexicons/org/tangled/snippet/string.json @@ -0,0 +1,44 @@ +{ + "lexicon": 1, + "id": "org.tangled.snippet.string", + "defs": { + "main": { + "type": "record", + "description": "TID is preferred for record-key, but not enforced.", + "key": "any", + "record": { + "type": "object", + "required": ["snippet", "createdAt"], + "properties": { + "description": { + "type": "string", + "maxGraphemes": 256 + }, + "snippet": { + "type": "union", + "refs": ["#textFile"] + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + }, + "textFile": { + "type": "object", + "required": ["name", "content"], + "properties": { + "name": { + "type": "string", + "description": "file name", + "maxGraphemes": 140 + }, + "content": { + "type": "blob", + "maxLength": 1000000 + } + } + } + } +} diff --git a/lexicons/org/tangled/spindle/describeSpindle.json b/lexicons/org/tangled/spindle/describeSpindle.json new file mode 100644 index 0000000000000000000000000000000000000000..2a508893bc9a17a6d2dd077d4e8d67626552d378 --- /dev/null +++ b/lexicons/org/tangled/spindle/describeSpindle.json @@ -0,0 +1,61 @@ +{ + "lexicon": 1, + "id": "org.tangled.spindle.describeSpindle", + "defs": { + "main": { + "type": "query", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "did", + "version" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "version": { + "type": "string" + }, + "capabilities": { + "type": "array", + "items": { + "type": "string", + "knownValues": [ + "tangled1.0" + ] + }, + "description": "Protocol capability tokens this spindle implements. Spindle that omit this field are treated as legacy." + }, + "links": { + "type": "ref", + "description": "URLs of service policy documents.", + "ref": "#links" + }, + "contact": { + "type": "ref", + "description": "Contact information", + "ref": "#contact" + } + } + } + } + }, + "links": { + "type": "object", + "properties": { + "privacyPolicy": { "type": "string", "format": "uri" }, + "termsOfService": { "type": "string", "format": "uri" } + } + }, + "contact": { + "type": "object", + "properties": { + "email": { "type": "string" } + } + } + } +} diff --git a/lexicons/org/tangled/string/countStrings.json b/lexicons/org/tangled/string/countStrings.json new file mode 100644 index 0000000000000000000000000000000000000000..1e230a8a4183c2abaf7b7b148f5f1dc7207c4f6f --- /dev/null +++ b/lexicons/org/tangled/string/countStrings.json @@ -0,0 +1,38 @@ +{ + "lexicon": 1, + "id": "org.tangled.string.countStrings", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Scope identifier whose string records to list." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count", "distinctAuthors"], + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "description": "Total number of matching records." + }, + "distinctAuthors": { + "type": "integer", + "minimum": 0, + "description": "Number of distinct authors among the matching records." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/string/getString.json b/lexicons/org/tangled/string/getString.json new file mode 100644 index 0000000000000000000000000000000000000000..e9ca47b4963ed10a649c9a03170c0a0e8b3f0009 --- /dev/null +++ b/lexicons/org/tangled/string/getString.json @@ -0,0 +1,41 @@ +{ + "lexicon": 1, + "id": "org.tangled.string.getString", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["uri"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the sh.tangled.string record to fetch." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.string record." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/string/listStrings.json b/lexicons/org/tangled/string/listStrings.json new file mode 100644 index 0000000000000000000000000000000000000000..2fa8c0a4a949219fb26ad2ae2e744bc6c7c7f4e4 --- /dev/null +++ b/lexicons/org/tangled/string/listStrings.json @@ -0,0 +1,70 @@ +{ + "lexicon": 1, + "id": "org.tangled.string.listStrings", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Scope identifier whose string records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.string record" + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/actor/defs.json b/lexicons/org/tangled/temp/actor/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..547cfee40b9081c86d2725f8cf21c478c4ea008f --- /dev/null +++ b/lexicons/org/tangled/temp/actor/defs.json @@ -0,0 +1,171 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.actor.defs", + "defs": { + "savedService": { + "type": "object", + "required": ["id", "value"], + "properties": { + "id": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "onboardingPref": { + "type": "object", + "required": ["startedAt"], + "properties": { + "startedAt": { + "type": "string", + "format": "datetime" + }, + "completedAt": { + "type": "string", + "format": "datetime" + }, + "nudgeDismissedAt": { + "type": "string", + "format": "datetime" + } + } + }, + "savedKnotPref": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "#savedService" + } + } + } + }, + "savedSpindlePref": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "#savedService" + } + } + } + }, + "recentsPref": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "maxLength": 5, + "items": { + "type": "union", + "refs": ["#recentRepo", "#recentIssue", "#recentPull"] + } + } + } + }, + "recentRepo": { + "type": "object", + "required": ["ownerHandle", "name", "visited"], + "properties": { + "ownerHandle": { + "type": "string", + "maxLength": 253 + }, + "ownerDid": { + "type": "string", + "format": "did" + }, + "name": { + "type": "string", + "maxLength": 100 + }, + "visited": { + "type": "string", + "format": "datetime" + } + } + }, + "recentIssue": { + "type": "object", + "required": [ + "ownerHandle", + "repoName", + "title", + "rkey", + "open", + "visited" + ], + "properties": { + "ownerHandle": { + "type": "string", + "maxLength": 253 + }, + "repoName": { + "type": "string", + "maxLength": 100 + }, + "title": { + "type": "string", + "maxLength": 400 + }, + "rkey": { + "type": "string", + "format": "record-key" + }, + "open": { + "type": "boolean" + }, + "visited": { + "type": "string", + "format": "datetime" + } + } + }, + "recentPull": { + "type": "object", + "required": [ + "ownerHandle", + "repoName", + "title", + "rkey", + "state", + "visited" + ], + "properties": { + "ownerHandle": { + "type": "string", + "maxLength": 253 + }, + "repoName": { + "type": "string", + "maxLength": 100 + }, + "title": { + "type": "string", + "maxLength": 400 + }, + "rkey": { + "type": "string", + "format": "record-key" + }, + "state": { + "type": "string", + "knownValues": ["open", "closed", "completed"] + }, + "visited": { + "type": "string", + "format": "datetime" + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/git/deleteBranch.json b/lexicons/org/tangled/temp/git/deleteBranch.json new file mode 100644 index 0000000000000000000000000000000000000000..958909906be0a532b8c47bc66253dcd3d44699ac --- /dev/null +++ b/lexicons/org/tangled/temp/git/deleteBranch.json @@ -0,0 +1,30 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.deleteBranch", + "defs": { + "main": { + "type": "procedure", + "description": "Delete a branch on this repository", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "branch" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "branch": { + "type": "string" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/git/formatPatch.json b/lexicons/org/tangled/temp/git/formatPatch.json new file mode 100644 index 0000000000000000000000000000000000000000..f6c6ac411fb4290e6c877c52c2d51fb5a0e4f72c --- /dev/null +++ b/lexicons/org/tangled/temp/git/formatPatch.json @@ -0,0 +1,51 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.formatPatch", + "defs": { + "main": { + "type": "query", + "description": "Serve the commits reachable from 'to' but not from 'from' as a git format-patch mailbox, oldest first.", + "parameters": { + "type": "params", + "required": ["repo", "from", "to"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "from": { + "type": "string", + "description": "Revision the series starts after (exclusive)" + }, + "to": { + "type": "string", + "description": "Revision the series ends at (inclusive)" + } + } + }, + "output": { + "encoding": "*/*", + "description": "Patch series as text/plain" + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RevisionNotFound", + "description": "One or both revisions not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "PatchTooLarge", + "description": "Range has more commits than the limit" + } + ] + } + } +} diff --git a/lexicons/org/tangled/temp/git/getArchive.json b/lexicons/org/tangled/temp/git/getArchive.json new file mode 100644 index 0000000000000000000000000000000000000000..433a2e348341b056e03d31300e2187bad5cca7dd --- /dev/null +++ b/lexicons/org/tangled/temp/git/getArchive.json @@ -0,0 +1,56 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.getArchive", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repo", "ref", "prefix"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git reference (branch, tag, or commit SHA)" + }, + "format": { + "type": "string", + "description": "Archive format", + "enum": ["tar.gz", "zip"], + "default": "tar.gz" + }, + "prefix": { + "type": "string", + "description": "Prefix for files in the archive" + } + } + }, + "output": { + "encoding": "*/*", + "description": "Binary archive data" + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + }, + { + "name": "ArchiveTooLarge", + "description": "Archive exceeds the size limit" + } + ] + } + } +} diff --git a/lexicons/org/tangled/temp/git/getBlob.json b/lexicons/org/tangled/temp/git/getBlob.json new file mode 100644 index 0000000000000000000000000000000000000000..4f99ab7d117995249f1e93994229c081dd3fce3a --- /dev/null +++ b/lexicons/org/tangled/temp/git/getBlob.json @@ -0,0 +1,38 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.getBlob", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repo", "path"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git reference (branch, tag, or commit SHA)", + "default": "HEAD" + }, + "path": { + "type": "string", + "description": "Path within the repository tree" + }, + "download": { + "type": "boolean", + "description": "Serve the blob as an attachment named after the file", + "default": false + } + } + }, + "output": { + "encoding": "*/*", + "description": "raw blob served in octet-stream" + } + } + } +} diff --git a/lexicons/org/tangled/temp/git/getBranch.json b/lexicons/org/tangled/temp/git/getBranch.json new file mode 100644 index 0000000000000000000000000000000000000000..6cf872bc3c70039674b0f5a8d8d7aac99667fc18 --- /dev/null +++ b/lexicons/org/tangled/temp/git/getBranch.json @@ -0,0 +1,67 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.getBranch", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repo", "name"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "name": { + "type": "string", + "description": "Branch name to get information for" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["name", "hash", "message", "author", "totalCommits"], + "properties": { + "name": { + "type": "string", + "description": "Branch name" + }, + "hash": { + "type": "string", + "description": "Latest commit hash on this branch" + }, + "message": { + "type": "string", + "description": "Latest commit message" + }, + "author": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + }, + "totalCommits": { + "type": "integer", + "description": "Number of commits reachable from this branch" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "BranchNotFound", + "description": "Branch not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/temp/git/getEntry.json b/lexicons/org/tangled/temp/git/getEntry.json new file mode 100644 index 0000000000000000000000000000000000000000..83f474fa645afd991405a2bb446199ca436ac18d --- /dev/null +++ b/lexicons/org/tangled/temp/git/getEntry.json @@ -0,0 +1,81 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.getEntry", + "defs": { + "main": { + "type": "query", + "description": "get metadata of blob by ref and path", + "parameters": { + "type": "params", + "required": ["repo", "path"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git revision (branch, tag, or commit id)", + "default": "HEAD" + }, + "path": { + "type": "string", + "description": "path of the entity" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["name", "mode", "oid", "size"], + "properties": { + "name": { + "type": "string", + "description": "The file name" + }, + "mode": { + "type": "string", + "enum": ["0040000", "0100644", "0100664", "0100755", "0120000", "0160000"] + }, + "oid": { + "type": "string" + }, + "size": { + "type": "integer", + "description": "Blob size" + }, + "lastCommit": { + "type": "ref", + "ref": "org.tangled.git.defs#commit" + }, + "submodule": { + "type": "ref", + "ref": "org.tangled.git.defs#submodule", + "description": "Submodule information if path is a submodule" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "EntryNotFound", + "description": "Entry not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/temp/git/getMergeBase.json b/lexicons/org/tangled/temp/git/getMergeBase.json new file mode 100644 index 0000000000000000000000000000000000000000..00ef991ea30c5a5f1e7f1ef5b38903feab67c1c5 --- /dev/null +++ b/lexicons/org/tangled/temp/git/getMergeBase.json @@ -0,0 +1,60 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.getMergeBase", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repos", "base", "head"], + "properties": { + "repos": { + "type": "array", + "items": { "type": "string", "format": "did" }, + "minLength": 1, + "maxLength": 2, + "description": "DIDs of the repositories to read objects from" + }, + "base": { + "type": "string", + "description": "Commit used to find the common base." + }, + "head": { + "type": "string", + "description": "Commit to list history for." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["commit"], + "properties": { + "commit": { + "type": "string" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RevisionNotFound", + "description": "One or both commits not found" + }, + { + "name": "NoMergeBase", + "description": "Two commits share no common ancestor" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/temp/git/getRef.json b/lexicons/org/tangled/temp/git/getRef.json new file mode 100644 index 0000000000000000000000000000000000000000..305e10be5a47f7495e0283853fcb0b4b290dac30 --- /dev/null +++ b/lexicons/org/tangled/temp/git/getRef.json @@ -0,0 +1,75 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.getRef", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo", + "ref" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Reference name. Such as master, refs/heads/master, or v1.16.0-alpha." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["name", "oid", "message", "author", "totalCommits"], + "properties": { + "name": { + "type": "string", + "description": "Full reference name. Example: refs/heads/master" + }, + "oid": { + "type": "string", + "description": "Latest commit oid on this reference." + }, + "message": { + "type": "string", + "description": "Latest commit message" + }, + "author": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + }, + "annotation": { + "type": "ref", + "ref": "org.tangled.temp.git.listTags#annotation", + "description": "Present only for annotated tags" + }, + "totalCommits": { + "type": "integer", + "description": "Number of commits reachable from this tip" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/temp/git/getTag.json b/lexicons/org/tangled/temp/git/getTag.json new file mode 100644 index 0000000000000000000000000000000000000000..543f11483389018ac232b1af285f30cd7c7ece80 --- /dev/null +++ b/lexicons/org/tangled/temp/git/getTag.json @@ -0,0 +1,71 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.getTag", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "repo", + "tag" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "tag": { + "type": "string", + "description": "Name of tag, such as v1.3.0" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["name", "oid", "message", "author"], + "properties": { + "name": { + "type": "string", + "description": "Tag name" + }, + "oid": { + "type": "string", + "description": "Latest commit oid on this tag" + }, + "message": { + "type": "string", + "description": "Latest commit message" + }, + "author": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + }, + "annotation": { + "type": "ref", + "ref": "org.tangled.temp.git.listTags#annotation", + "description": "Present only for annotated tags" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/temp/git/getTree.json b/lexicons/org/tangled/temp/git/getTree.json new file mode 100644 index 0000000000000000000000000000000000000000..8b55e483024ccbed099a4b1f03b390fdfc33f432 --- /dev/null +++ b/lexicons/org/tangled/temp/git/getTree.json @@ -0,0 +1,97 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.getTree", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ref": { + "type": "string", + "description": "Git revision (branch, tag, or commit id)", + "default": "HEAD" + }, + "path": { + "type": "string", + "description": "Path within the repository tree", + "default": "" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "entries" + ], + "properties": { + "entries": { + "type": "array", + "items": { + "type": "ref", + "ref": "#treeEntry" + } + }, + "lastCommit": { + "type": "ref", + "ref": "org.tangled.git.defs#commit" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "PathNotFound", + "description": "Path not found in repository tree" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "treeEntry": { + "type": "object", + "required": [ + "name", + "mode", + "size" + ], + "properties": { + "name": { + "type": "string", + "description": "Relative file or directory name" + }, + "mode": { + "type": "string", + "description": "File mode", + "enum": ["0040000", "0100644", "0100664", "0100755", "0120000", "0160000"] + }, + "size": { + "type": "integer", + "description": "File size in bytes" + }, + "lastCommit": { + "type": "ref", + "ref": "org.tangled.git.defs#commit" + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/git/listBranches.json b/lexicons/org/tangled/temp/git/listBranches.json new file mode 100644 index 0000000000000000000000000000000000000000..cf0b1226a6aeeab9adec8b2f6c3ce7e8846a615a --- /dev/null +++ b/lexicons/org/tangled/temp/git/listBranches.json @@ -0,0 +1,92 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.listBranches", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "limit": { + "type": "integer", + "description": "Maximum number of branches to return", + "minimum": 1, + "maximum": 100, + "default": 50 + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["branches", "total"], + "properties": { + "branches": { + "type": "array", + "items": { + "type": "ref", + "ref": "#branchInfo" + } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total number of branches in the repository" + }, + "defaultBranch": { + "type": "string", + "description": "Name of the default branch" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "branchInfo": { + "type": "object", + "required": ["name", "oid", "message", "author", "committer"], + "properties": { + "name": { + "type": "string", + "description": "Branch name" + }, + "oid": { + "type": "string", + "description": "Latest commit oid on this branch" + }, + "message": { + "type": "string", + "description": "Latest commit message" + }, + "author": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + }, + "committer": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/git/listCommits.json b/lexicons/org/tangled/temp/git/listCommits.json new file mode 100644 index 0000000000000000000000000000000000000000..f24219f26186fd3e1b0a183247f8e8693feda97b --- /dev/null +++ b/lexicons/org/tangled/temp/git/listCommits.json @@ -0,0 +1,120 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.listCommits", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repos"], + "properties": { + "repos": { + "type": "array", + "items": { "type": "string", "format": "did" }, + "minLength": 1, + "maxLength": 2, + "description": "DIDs of the repositories to read objects from" + }, + "ranges": { + "type": "array", + "items": { "type": "string" }, + "description": "Commit ids or `..` ranges" + }, + "limit": { + "type": "integer", + "description": "Maximum number of commits to return", + "minimum": 1, + "maximum": 100, + "default": 50 + }, + "skip": { + "type": "integer", + "description": "Number of matching commits to skip. A poor man's pagination.", + "minimum": 0, + "default": 0 + }, + "includeStats": { + "type": "boolean", + "description": "Diff each commit against its first parent and include a line stat", + "default": false + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["commits"], + "properties": { + "commits": { + "type": "array", + "items": { + "type": "ref", + "ref": "#commit" + } + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RevisionNotFound", + "description": "Commit not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "commit": { + "type": "object", + "required": [ + "oid", + "parents", + "tree", + "author", + "committer", + "message", + "extraHeaders" + ], + "properties": { + "oid": { "type": "string" }, + "parents": { "type": "array", "items": { "type": "string" } }, + "tree": { "type": "string" }, + "author": { "type": "ref", "ref": "org.tangled.git.defs#signature" }, + "committer": { "type": "ref", "ref": "org.tangled.git.defs#signature" }, + "message": { "type": "string" }, + "extraHeaders": { + "type": "array", + "items": { "type": "ref", "ref": "#header" } + }, + "stat": { "type": "ref", "ref": "org.tangled.git.defs#diffStat" }, + "signature": { + "type": "union", + "refs": ["#sshSignature"], + "description": "Set only when the commit's signature verifies against the key embedded in it. Says nothing about who owns that key." + } + } + }, + "sshSignature": { + "type": "object", + "required": ["fingerprint"], + "properties": { + "fingerprint": { "type": "string", "description": "SHA256 fingerprint of the signing key, `SHA256:...`" } + } + }, + "header": { + "type": "object", + "required": ["key", "value"], + "properties": { + "key": { "type": "string" }, + "value": { "type": "string" } + } + } + } +} diff --git a/lexicons/org/tangled/temp/git/listCommitsByRef.json b/lexicons/org/tangled/temp/git/listCommitsByRef.json new file mode 100644 index 0000000000000000000000000000000000000000..04251da5aeb2710e4d242efadf751089a5be08d2 --- /dev/null +++ b/lexicons/org/tangled/temp/git/listCommitsByRef.json @@ -0,0 +1,69 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.listCommitsByRef", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "ranges": { "type": "array", "items": { "type": "string" } }, + "limit": { + "type": "integer", + "description": "Maximum number of commits to return", + "minimum": 1, + "maximum": 100, + "default": 50 + }, + "skip": { + "type": "integer", + "description": "Number of matching commits to skip. A poor man's pagination.", + "minimum": 0, + "default": 0 + }, + "includeStats": { + "type": "boolean", + "description": "Diff each commit against its first parent and include a line stat", + "default": false + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["commits"], + "properties": { + "commits": { + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.temp.git.listCommits#commit" + } + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "RefNotFound", + "description": "Git reference not found" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/org/tangled/temp/git/listTags.json b/lexicons/org/tangled/temp/git/listTags.json new file mode 100644 index 0000000000000000000000000000000000000000..bcc4ad865182196c31571a603ea781aeee55492a --- /dev/null +++ b/lexicons/org/tangled/temp/git/listTags.json @@ -0,0 +1,112 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.listTags", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository" + }, + "limit": { + "type": "integer", + "description": "Maximum number of tags to return", + "minimum": 1, + "maximum": 100, + "default": 50 + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["tags", "total"], + "properties": { + "tags": { + "type": "array", + "items": { + "type": "ref", + "ref": "#tagInfo" + } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total number of tags in the repository" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repository not found or access denied" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + }, + "tagInfo": { + "type": "object", + "required": ["name", "oid", "message", "author"], + "properties": { + "name": { + "type": "string", + "description": "Tag name" + }, + "oid": { + "type": "string", + "description": "Latest commit oid on this tag" + }, + "message": { + "type": "string", + "description": "Latest commit message" + }, + "author": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + }, + "annotation": { + "type": "ref", + "ref": "#annotation", + "description": "Present only for annotated tags" + }, + "artifacts": { + "type": "array", + "description": "org.tangled.repo.artifact records whose subject is this tag's oid", + "items": { "type": "ref", "ref": "org.tangled.repo.listArtifacts#listItem" } + } + } + }, + "annotation": { + "type": "object", + "required": ["message"], + "properties": { + "tagger": { + "type": "ref", + "ref": "org.tangled.git.defs#signature" + }, + "message": { + "type": "string", + "description": "Tag message" + }, + "oid": { + "type": "string", + "description": "Tag object oid" + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/git/mergeCommit.json b/lexicons/org/tangled/temp/git/mergeCommit.json new file mode 100644 index 0000000000000000000000000000000000000000..f8c2dc5309fff1fca1444dcb9f426c0937d663e2 --- /dev/null +++ b/lexicons/org/tangled/temp/git/mergeCommit.json @@ -0,0 +1,85 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.mergeCommit", + "defs": { + "main": { + "type": "procedure", + "description": "Merge a commit into repository's branch", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["target", "source", "style"], + "properties": { + "target": { + "type": "ref", + "ref": "#target" + }, + "source": { + "type": "ref", + "ref": "#source" + }, + "style": { + "type": "string", + "enum": [ + "rebase", + "merge", + "rebase-merge", + "squash-rebase", + "squash-rebase-merge", + "fast-forward-only" + ] + }, + "mergeCommit": { + "type": "ref", + "ref": "#mergeCommitContent", + "description": "Merge commit content. Used when merge commit is created." + } + } + } + } + }, + "target": { + "type": "object", + "required": ["repo", "branch", "commit"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of target git repository" + }, + "branch": { + "type": "string", + "description": "Target branch to merge into" + }, + "commit": { + "type": "string", + "description": "Head commit ID of target branch" + } + } + }, + "source": { + "type": "object", + "required": ["repo", "commit"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of source git repository" + }, + "commit": { + "type": "string", + "description": "Head commit ID to merge" + } + } + }, + "mergeCommitContent": { + "type": "object", + "description": "Merge commit content. Used when merge commit is created.", + "properties": { + "author": { "type": "ref", "ref": "org.tangled.git.defs#signature" }, + "message": { "type": "string", "maxLength": 1000000 } + } + } + } +} diff --git a/lexicons/org/tangled/temp/git/syncFork.json b/lexicons/org/tangled/temp/git/syncFork.json new file mode 100644 index 0000000000000000000000000000000000000000..3307e9da48460744de2671b7a3ff461192976c8c --- /dev/null +++ b/lexicons/org/tangled/temp/git/syncFork.json @@ -0,0 +1,36 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.git.syncFork", + "defs": { + "main": { + "type": "procedure", + "description": "Update a branch on a fork to match the same branch on its upstream", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "source", + "branch" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the fork repository" + }, + "source": { + "type": "string", + "format": "did", + "description": "DID of the upstream repository" + }, + "branch": { + "type": "string" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/repo/getRepoByName.json b/lexicons/org/tangled/temp/repo/getRepoByName.json new file mode 100644 index 0000000000000000000000000000000000000000..737ff081a112f4bf373a4d2af2b4e16bb85cee8f --- /dev/null +++ b/lexicons/org/tangled/temp/repo/getRepoByName.json @@ -0,0 +1,75 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.repo.getRepoByName", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["owner", "name"], + "properties": { + "owner": { + "type": "string", + "format": "did", + "description": "DID of the account that owns the repo." + }, + "name": { + "type": "string", + "description": "Name of the repo as it appears in its url." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded org.tangled.repo.manifest record." + }, + "knot": { + "type": "string", + "description": "Host of the knot named by the #tangled_knot service in the repo's DID document. Absent when it names none." + }, + "git": { + "type": "ref", + "ref": "#gitInfo" + } + } + } + } + }, + "gitInfo": { + "type": "object", + "required": ["defaultBranch", "isEmpty"], + "properties": { + "defaultBranch": { + "type": "string", + "description": "Default branch name from `org.tangled.git.refs/self` record." + }, + "isEmpty": { + "type": "boolean", + "description": "True if `refs` in `org.tangled.git.refs/self` record is empty." + }, + "languages": { + "type": "array", + "description": "Languages on the default branch ordered by size, largest first. Absent until the mirror has indexed the repo.", + "items": { + "type": "ref", + "ref": "org.tangled.repo.defs#language" + } + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/repo/listRecordOrigins.json b/lexicons/org/tangled/temp/repo/listRecordOrigins.json new file mode 100644 index 0000000000000000000000000000000000000000..ede46265fa22c686dc2a56e92bb22360091456c2 --- /dev/null +++ b/lexicons/org/tangled/temp/repo/listRecordOrigins.json @@ -0,0 +1,110 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.repo.listRecordOrigins", + "defs": { + "main": { + "type": "query", + "description": "Who opened each COB record of one collection in a repository, and when. Gives the same author as the first version in `org.tangled.repo.listRecordVersions`, for every record in one walk, sorted by record key.", + "parameters": { + "type": "params", + "required": [ + "repo", + "collection" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository." + }, + "collection": { + "type": "string", + "format": "nsid", + "description": "A versioned collection, such as `org.tangled.track.ticket`." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 500 + }, + "cursor": { + "type": "string", + "description": "Pass the cursor from one page into the next query to keep walking records." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "collection", + "origins" + ], + "properties": { + "repo": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "cursor": { + "type": "string", + "description": "Left out when the page already reaches the last record." + }, + "origins": { + "type": "array", + "items": { + "type": "ref", + "ref": "#origin" + } + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "The collection isn't a versioned one, or a parameter is malformed." + }, + { + "name": "RepoNotFound", + "description": "This mirror doesn't have the repository." + } + ] + }, + "origin": { + "type": "object", + "required": [ + "rkey", + "author", + "changeId", + "createdAt" + ], + "properties": { + "rkey": { + "type": "string", + "format": "record-key" + }, + "author": { + "type": "string", + "format": "did", + "description": "The DID of the account that opened the record, the editor of its first version." + }, + "changeId": { + "type": "string", + "description": "Git Object ID of the change that opened the record." + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "When the knot wrote the opening change. Not the record's own `createdAt`." + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/server/ackHandover.json b/lexicons/org/tangled/temp/server/ackHandover.json new file mode 100644 index 0000000000000000000000000000000000000000..7e700bba9f81cf9ce6fc5a08602ad322d4ef3dd5 --- /dev/null +++ b/lexicons/org/tangled/temp/server/ackHandover.json @@ -0,0 +1,33 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.server.ackHandover", + "defs": { + "main": { + "type": "procedure", + "description": "Upgrading knot sends this once it is ready to serve *every* fostered record of a repo - at exact CIDs the foster knot was giving out at ingestion time. The signature thingy is this 3rd party requesting-knot's key over dag-cbor of the same fields using the context org.tangled.temp.handover.ack.v1, such that the foster knot can verify it against the requester's did:web doc.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repoDid", "records", "ackedAt", "signature"], + "properties": { + "repoDid": { + "type": "string", + "format": "did" + }, + "records": { + "type": "integer" + }, + "ackedAt": { + "type": "integer", + "description": "Ack time in Unix seconds." + }, + "signature": { + "type": "string" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/server/cancelHandover.json b/lexicons/org/tangled/temp/server/cancelHandover.json new file mode 100644 index 0000000000000000000000000000000000000000..777d00bcd7b893c7afca721c0e4e4210e7105efc --- /dev/null +++ b/lexicons/org/tangled/temp/server/cancelHandover.json @@ -0,0 +1,36 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.server.cancelHandover", + "defs": { + "main": { + "type": "procedure", + "description": "Abandoning a repo's handover job mid-way; sidecar is removed, boot replay will stop retrying sidecarring. Only a knot admin should/could ever call this.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repoDid"], + "properties": { + "repoDid": { + "type": "string", + "format": "did" + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repoDid"], + "properties": { + "repoDid": { + "type": "string", + "format": "did" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/server/createInviteCode.json b/lexicons/org/tangled/temp/server/createInviteCode.json new file mode 100644 index 0000000000000000000000000000000000000000..5153280615df3303c67aa03a3624166ce740042f --- /dev/null +++ b/lexicons/org/tangled/temp/server/createInviteCode.json @@ -0,0 +1,31 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.server.createInviteCode", + "defs": { + "main": { + "type": "procedure", + "description": "Create an invite code. Similar to `com.atproto.server.createInviteCode` but used for Knot/Spindle membership.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["useCount"], + "properties": { + "useCount": { "type": "integer" }, + "forAccount": { "type": "string", "format": "did" } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["code"], + "properties": { + "code": { "type": "string" } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/server/handoverRepo.json b/lexicons/org/tangled/temp/server/handoverRepo.json new file mode 100644 index 0000000000000000000000000000000000000000..343fdc1b49f682c1e6a060dfcf53fbfc78dfce06 --- /dev/null +++ b/lexicons/org/tangled/temp/server/handoverRepo.json @@ -0,0 +1,44 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.server.handoverRepo", + "defs": { + "main": { + "type": "procedure", + "description": "Pull a hosted repo's shadowed COB records back from a given foster knot, rebuild the record chain under this knot's own authority, confirm record CIDs against foster knot's state, finally ack the handover. Since it may take a bit, the endpoint will already give a response as soon as the job is staged up with state \"running\". Then the next steps run in background finish into the sidecar + logs. Only a knot admin can call this!", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repoDid", "fosterUrl"], + "properties": { + "repoDid": { + "type": "string", + "format": "did" + }, + "fosterUrl": { + "type": "string", + "format": "uri" + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repoDid", "state"], + "properties": { + "repoDid": { + "type": "string", + "format": "did" + }, + "state": { + "type": "string", + "knownValues": ["running"] + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/server/redeemInviteCode.json b/lexicons/org/tangled/temp/server/redeemInviteCode.json new file mode 100644 index 0000000000000000000000000000000000000000..6f42e6b3b5d995a4ee804565deb5186b7b58cf1f --- /dev/null +++ b/lexicons/org/tangled/temp/server/redeemInviteCode.json @@ -0,0 +1,20 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.server.redeemInviteCode", + "defs": { + "main": { + "type": "procedure", + "description": "Clarifies the membership in server via invite code.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["code"], + "properties": { + "code": { "type": "string" } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/server/seedRepo.json b/lexicons/org/tangled/temp/server/seedRepo.json new file mode 100644 index 0000000000000000000000000000000000000000..e4577d26e641ca51eceb3f672a6c0021511f9973 --- /dev/null +++ b/lexicons/org/tangled/temp/server/seedRepo.json @@ -0,0 +1,106 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.server.seedRepo", + "defs": { + "main": { + "type": "procedure", + "description": "Foster an empty git repo TEMPORARILY for an owner from outside the knot, and write the given manifest and records on this knot without lex validation. This endpoint is possible only when legacy admin secret is in use.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repoDid", "manifest", "records"], + "properties": { + "repoDid": { + "type": "string", + "format": "did", + "description": "Existing repo DID which our COB records belong to. The foster knot will serve this DID as a shadow, and will give the record (authority) back to the principal knot once that given knot that *actually* owns the repo DID upgrades!" + }, + "manifest": { + "type": "unknown", + "description": "Raw `org.tangled.repo.manifest` record body" + }, + "records": { + "type": "array", + "maxLength": 200, + "description": "Records to write besides the manifest. Max 200 per req.", + "items": { + "type": "ref", + "ref": "#seedRecord" + } + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["did", "key", "results"], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "key": { + "type": "string", + "description": "Multibase-encoded public signing key that the knot has for this repo." + }, + "results": { + "type": "array", + "items": { + "type": "ref", + "ref": "#recordResult" + } + } + } + } + } + }, + "seedRecord": { + "type": "object", + "required": ["collection", "rkey", "author", "createdAt", "record"], + "properties": { + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string", + "format": "record-key" + }, + "author": { + "type": "string", + "format": "did", + "description": "DID which will be attributed to the record as the author." + }, + "createdAt": { + "type": "integer", + "description": "Creation time in Unix seconds." + }, + "record": { + "type": "unknown", + "description": "Record body, DAG-CBOR w/o lex validation. Max 1,000,000 bytes post-encode." + } + } + }, + "recordResult": { + "type": "object", + "required": ["collection", "rkey"], + "properties": { + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string", + "format": "record-key" + }, + "error": { + "type": "string", + "description": "Reason that a record wasn't written." + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/server/uploadBlob.json b/lexicons/org/tangled/temp/server/uploadBlob.json new file mode 100644 index 0000000000000000000000000000000000000000..6af94ca18f94ebae1b0eb1d7736766d644d66679 --- /dev/null +++ b/lexicons/org/tangled/temp/server/uploadBlob.json @@ -0,0 +1,36 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.server.uploadBlob", + "defs": { + "main": { + "type": "procedure", + "description": "Upload a blob into a fostered repo on this knot. This endpoint is possible only when legacy admin secret is in use.", + "parameters": { + "type": "params", + "required": ["repoDid"], + "properties": { + "repoDid": { + "type": "string", + "format": "did", + "description": "Repo DID hosted on this knot that the blob belongs to." + } + } + }, + "input": { + "encoding": "*/*" + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["blob"], + "properties": { + "blob": { + "type": "blob" + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/spindle/allowList/add.json b/lexicons/org/tangled/temp/spindle/allowList/add.json new file mode 100644 index 0000000000000000000000000000000000000000..9aba1762ad1a1320f0834a007b534d5768cca3ea --- /dev/null +++ b/lexicons/org/tangled/temp/spindle/allowList/add.json @@ -0,0 +1,24 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.allowList.add", + "defs": { + "main": { + "type": "procedure", + "description": "Add a DID to this spindle's allow-list, so the spindle runs CI for its repos while admission is closed. Only callable by a spindle admin.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["did"], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID of account that should be allowed to enroll repos on this here spindle." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/spindle/allowList/list.json b/lexicons/org/tangled/temp/spindle/allowList/list.json new file mode 100644 index 0000000000000000000000000000000000000000..17a1e4cb9a3f2c1a54ca1283d7b354348d46c05a --- /dev/null +++ b/lexicons/org/tangled/temp/spindle/allowList/list.json @@ -0,0 +1,24 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.allowList.list", + "defs": { + "main": { + "type": "query", + "description": "Read this spindle's allow-list. Only callable by a spindle admin.", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["dids"], + "properties": { + "dids": { + "type": "array", + "description": "Allow-list entries.", + "items": { "type": "string", "format": "did" } + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/temp/spindle/allowList/remove.json b/lexicons/org/tangled/temp/spindle/allowList/remove.json new file mode 100644 index 0000000000000000000000000000000000000000..18d5fc5ee2f38a31bd972afcc5848f3390cd9c13 --- /dev/null +++ b/lexicons/org/tangled/temp/spindle/allowList/remove.json @@ -0,0 +1,24 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.allowList.remove", + "defs": { + "main": { + "type": "procedure", + "description": "Remove a DID from this spindle's allow-list. Only callable by a spindle admin.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["did"], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID to remove from allow-list." + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/track/defs.json b/lexicons/org/tangled/track/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..188d8ca65b3d30f7ae57bd26f9300aa6077b853c --- /dev/null +++ b/lexicons/org/tangled/track/defs.json @@ -0,0 +1,80 @@ +{ + "lexicon": 1, + "id": "org.tangled.track.defs", + "defs": { + "ticketState": { + "type": "string", + "enum": [ + "open", + "closed", + "completed" + ] + }, + "ticketViewDetailed": { + "type": "object", + "required": [ + "uri", + "cid", + "author", + "repo", + "title", + "state", + "createdAt" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "repo": { + "type": "ref", + "ref": "org.tangled.repo.defs#repoViewBasic" + }, + "author": { + "type": "ref", + "description": "The account that opened the ticket, the editor of its first version. Not the account that last edited or closed it.", + "ref": "org.tangled.actor.defs#profileViewBasic" + }, + "title": { + "type": "union", + "refs": [ + "org.tangled.markup.markdown#inline" + ] + }, + "body": { + "type": "union", + "refs": [ + "org.tangled.markup.markdown" + ] + }, + "state": { + "type": "ref", + "ref": "#ticketState" + }, + "tangledLabels": { + "type": "array", + "items": { + "type": "union", + "refs": ["org.tangled.label.defs#labelToken"] + } + }, + "assignees": { + "type": "array", + "items": { "type": "string", "format": "did" } + }, + "targetBranch": { + "type": "string", + "description": "Present when the ticket is a pull request." + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } +} diff --git a/lexicons/org/tangled/track/getTicketView.json b/lexicons/org/tangled/track/getTicketView.json new file mode 100644 index 0000000000000000000000000000000000000000..52570a26d944ccee39f5ce376d290e88a6b21d96 --- /dev/null +++ b/lexicons/org/tangled/track/getTicketView.json @@ -0,0 +1,26 @@ +{ + "lexicon": 1, + "id": "org.tangled.track.getTicketView", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["uri"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "ref", + "ref": "org.tangled.track.defs#ticketViewDetailed" + } + } + } + } +} diff --git a/lexicons/org/tangled/track/ticket.json b/lexicons/org/tangled/track/ticket.json new file mode 100644 index 0000000000000000000000000000000000000000..4e441f0880d7bdc4bb87583e32dd179c5e018c29 --- /dev/null +++ b/lexicons/org/tangled/track/ticket.json @@ -0,0 +1,58 @@ +{ + "lexicon": 1, + "id": "org.tangled.track.ticket", + "defs": { + "main": { + "type": "record", + "key": "any", + "record": { + "type": "object", + "required": [ + "title", + "state", + "createdAt" + ], + "properties": { + "title": { + "type": "union", + "refs": [ + "org.tangled.markup.markdown#inline" + ] + }, + "body": { + "type": "union", + "refs": [ + "org.tangled.markup.markdown" + ] + }, + "state": { + "type": "ref", + "ref": "org.tangled.track.defs#ticketState" + }, + "labels": { + "type": "ref", + "description": "Self-label values for this record.", + "ref": "com.atproto.label.defs#selfLabels" + }, + "tangledLabels": { + "type": "array", + "items": { + "type": "union", + "refs": ["org.tangled.label.defs#labelToken"] + } + }, + "assignees": { + "type": "array", + "items": { "type": "string", "format": "did" } + }, + "createdAt": { "type": "string", "format": "datetime" }, + "targetBranch": { + "type": "string", + "description": "Target branch name. Must not include the `refs/heads/` prefix.", + "maxLength": 244 + } + } + } + } + } +} diff --git a/lexicons/org/tangled/track/unspecced/milestone.json b/lexicons/org/tangled/track/unspecced/milestone.json new file mode 100644 index 0000000000000000000000000000000000000000..b7dcdd3db0a784e811ead548e720a2032439a30c --- /dev/null +++ b/lexicons/org/tangled/track/unspecced/milestone.json @@ -0,0 +1,15 @@ +{ + "lexicon": 1, + "id": "org.tangled.track.unspecced.milestone", + "defs": { + "main": { + "type": "record", + "key": "tid", + "description": "TBD", + "record": { + "type": "object", + "properties": {} + } + } + } +} diff --git a/lexicons/org/tangled/webhook/createWebhook.json b/lexicons/org/tangled/webhook/createWebhook.json new file mode 100644 index 0000000000000000000000000000000000000000..b074a49cbfdbe5f57a317c0b1f4ff37ca4ddb576 --- /dev/null +++ b/lexicons/org/tangled/webhook/createWebhook.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "org.tangled.webhook.createWebhook", + "defs": { + "main": { + "type": "procedure", + "description": "Create a new webhook for a repository.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repo", "url", "events"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository as minted by the knot." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Endpoint URL that will receive webhook payloads." + }, + "active": { + "type": "boolean", + "default": true, + "description": "Whether the webhook should be active immediately." + }, + "events": { + "type": "array", + "description": "Event types to subscribe to.", + "items": { + "type": "ref", + "ref": "org.tangled.webhook.defs#eventType" + } + }, + "secret": { + "type": "string", + "description": "Optional HMAC secret used to sign payloads. If omitted, payloads are not signed." + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["id"], + "properties": { + "id": { + "type": "string", + "description": "ID of the newly created webhook." + } + } + } + }, + "errors": [ + { "name": "NoEventsSelected", "description": "At least one event type must be specified." } + ] + } + } +} diff --git a/lexicons/org/tangled/webhook/defs.json b/lexicons/org/tangled/webhook/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..33d2eb5747f6dfa51658b3854cea6b221fb892a1 --- /dev/null +++ b/lexicons/org/tangled/webhook/defs.json @@ -0,0 +1,73 @@ +{ + "lexicon": 1, + "id": "org.tangled.webhook.defs", + "defs": { + "eventType": { + "type": "string", + "knownValues": [ + "push", + "repository:renamed", + "pull_request:created", + "pull_request:resubmitted", + "pull_request:merged", + "pull_request:closed", + "pull_request:reopened", + "org.tangled.event.pullRequest" + ] + }, + "webhook": { + "type": "object", + "required": ["id", "url", "active", "events", "createdAt"], + "properties": { + "id": { + "type": "string", + "description": "Webhook identifier." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Endpoint URL that receives webhook payloads." + }, + "active": { + "type": "boolean", + "description": "Whether the webhook is currently enabled." + }, + "events": { + "type": "array", + "description": "Event types this webhook is subscribed to.", + "items": { + "type": "ref", + "ref": "#eventType" + } + }, + "createdAt": { "type": "string", "format": "datetime" }, + "updatedAt": { "type": "string", "format": "datetime" } + } + }, + "delivery": { + "type": "object", + "required": ["id", "webhook", "event", "url", "success", "createdAt"], + "properties": { + "id": { + "type": "string", + "description": "Unique ID for tracking this delivery attempt." + }, + "webhook": { + "type": "string", + "description": "ID of the webhook this delivery belongs to." + }, + "event": { + "type": "ref", + "ref": "#eventType", + "description": "Event type that triggered the delivery." + }, + "url": { "type": "string", "format": "uri" }, + "requestBody": { "type": "bytes" }, + "responseCode": { "type": "integer" }, + "responseBody": { "type": "bytes" }, + "success": { "type": "boolean" }, + "createdAt": { "type": "string", "format": "datetime" } + } + } + } +} diff --git a/lexicons/org/tangled/webhook/deleteWebhook.json b/lexicons/org/tangled/webhook/deleteWebhook.json new file mode 100644 index 0000000000000000000000000000000000000000..ddfa59ea147b50a98e3967226ec15aa35f9bbba0 --- /dev/null +++ b/lexicons/org/tangled/webhook/deleteWebhook.json @@ -0,0 +1,31 @@ +{ + "lexicon": 1, + "id": "org.tangled.webhook.deleteWebhook", + "defs": { + "main": { + "type": "procedure", + "description": "Delete a webhook from a repository.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repo", "webhook"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository as minted by the knot." + }, + "webhook": { + "type": "string", + "description": "Webhook ID to delete." + } + } + } + }, + "errors": [ + { "name": "WebhookNotFound" } + ] + } + } +} diff --git a/lexicons/org/tangled/webhook/listWebhookDeliveries.json b/lexicons/org/tangled/webhook/listWebhookDeliveries.json new file mode 100644 index 0000000000000000000000000000000000000000..3dc1de5bc0249c2aa75c5bac1ccba5ec4ef20f19 --- /dev/null +++ b/lexicons/org/tangled/webhook/listWebhookDeliveries.json @@ -0,0 +1,48 @@ +{ + "lexicon": 1, + "id": "org.tangled.webhook.listWebhookDeliveries", + "defs": { + "main": { + "type": "query", + "description": "List recent delivery attempts for a webhook.", + "parameters": { + "type": "params", + "required": ["repo", "webhooks"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository as minted by the knot." + }, + "webhooks": { + "type": "array", + "items": { "type": "string", "description": "Webhook ID." }, + "maxLength": 50 + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 50 + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["deliveries"], + "properties": { + "deliveries": { + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.webhook.defs#delivery" + } + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/webhook/listWebhooks.json b/lexicons/org/tangled/webhook/listWebhooks.json new file mode 100644 index 0000000000000000000000000000000000000000..419642205188f6eca3f098b3df6026a3cafb6b9f --- /dev/null +++ b/lexicons/org/tangled/webhook/listWebhooks.json @@ -0,0 +1,37 @@ +{ + "lexicon": 1, + "id": "org.tangled.webhook.listWebhooks", + "defs": { + "main": { + "type": "query", + "description": "List webhooks configured for a repository.", + "parameters": { + "type": "params", + "required": ["repo"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository as minted by the knot." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["webhooks"], + "properties": { + "webhooks": { + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.webhook.defs#webhook" + } + } + } + } + } + } + } +} diff --git a/lexicons/org/tangled/webhook/retryWebhookDelivery.json b/lexicons/org/tangled/webhook/retryWebhookDelivery.json new file mode 100644 index 0000000000000000000000000000000000000000..960080afe63392a4ef1a4e11b58426952a4290c5 --- /dev/null +++ b/lexicons/org/tangled/webhook/retryWebhookDelivery.json @@ -0,0 +1,36 @@ +{ + "lexicon": 1, + "id": "org.tangled.webhook.retryWebhookDelivery", + "defs": { + "main": { + "type": "procedure", + "description": "Re-send a specific webhook delivery.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repo", "webhook", "delivery"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository as minted by the knot." + }, + "webhook": { + "type": "string", + "description": "Webhook ID that owns the delivery." + }, + "delivery": { + "type": "string", + "description": "Delivery ID to retry." + } + } + } + }, + "errors": [ + { "name": "WebhookNotFound" }, + { "name": "DeliveryNotFound" } + ] + } + } +} diff --git a/lexicons/org/tangled/webhook/updateWebhook.json b/lexicons/org/tangled/webhook/updateWebhook.json new file mode 100644 index 0000000000000000000000000000000000000000..f881ed3a7775946b180f15a2fab86bdd470b70d6 --- /dev/null +++ b/lexicons/org/tangled/webhook/updateWebhook.json @@ -0,0 +1,48 @@ +{ + "lexicon": 1, + "id": "org.tangled.webhook.updateWebhook", + "defs": { + "main": { + "type": "procedure", + "description": "Update an existing webhook. Only supplied fields are changed.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repo", "webhook"], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository as minted by the knot." + }, + "webhook": { + "type": "string", + "description": "Webhook ID to update." + }, + "url": { + "type": "string", + "format": "uri" + }, + "secret": { + "type": "string" + }, + "active": { + "type": "boolean" + }, + "events": { + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.webhook.defs#eventType" + } + } + } + } + }, + "errors": [ + { "name": "WebhookNotFound" } + ] + } + } +} diff --git a/lexicons/repo/listCobVersions.json b/lexicons/repo/listCobVersions.json new file mode 100644 index 0000000000000000000000000000000000000000..08cbe3be1bbdc2d1287f98ca16b07bbffa39b9d4 --- /dev/null +++ b/lexicons/repo/listCobVersions.json @@ -0,0 +1,112 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.listCobVersions", + "defs": { + "main": { + "type": "query", + "description": "Every version of a COB record! Give a record's URI and get back every saved version of that record, oldest to newest unless `order` says otherwise. Fetch a version's CID through `com.atproto.sync.getBlocks` for as long as the knot still has it.", + "parameters": { + "type": "params", + "required": [ + "uri" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of a record in a repository this knot hosts, its repository authority a DID." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 500, + "default": 100 + }, + "cursor": { + "type": "string", + "description": "Pass the cursor from one page into the next query to keep walking versions." + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "asc", + "description": "Which end the walk starts from. `desc` puts the newest version first, which is where the version you just wrote is." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "uri", + "versions" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cursor": { + "type": "string", + "description": "Left out when the page already reaches the last version in the requested order." + }, + "total": { + "type": "integer", + "description": "How many versions the record has, whatever the page." + }, + "versions": { + "type": "array", + "items": { + "type": "ref", + "ref": "sh.tangled.repo.listCobVersions#version" + } + } + } + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "The URI isn't an AT-URI, or its collection isn't a versioned one." + }, + { + "name": "NotFound", + "description": "The knot doesn't have a record at that URI." + }, + { + "name": "ProjectionWarming" + } + ] + }, + "version": { + "type": "object", + "required": [ + "changeId", + "editor", + "createdAt" + ], + "properties": { + "cid": { + "type": "string", + "format": "cid", + "description": "Missing once the version is removed, or when the change is a deletion." + }, + "changeId": { + "type": "string", + "description": "Git Object ID of the change associated with this version." + }, + "editor": { + "type": "string", + "format": "did", + "description": "The DID of the account that made this version." + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "When this knot wrote the commit this version rides in. Not the record's own `createdAt`." + } + } + } + } +} diff --git a/lexicons/temp/account/addEmail.json b/lexicons/temp/account/addEmail.json new file mode 100644 index 0000000000000000000000000000000000000000..5b06850aaf848abc205c0a24e4ffccfc887bb53f --- /dev/null +++ b/lexicons/temp/account/addEmail.json @@ -0,0 +1,26 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.addEmail", + "defs": { + "main": { + "type": "procedure", + "description": "Add an email address to the authenticated account. Sends a verification code; the address remains unverified until the code is submitted via verifyEmail. Calling again for an already-added unverified address resends a fresh code.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["email"], + "properties": { + "email": { + "type": "string", + "description": "Email address to add." + } + } + } + }, + "errors": [ + { "name": "EmailAlreadyRegistered", "description": "An account already exists for this email address." } + ] + } + } +} diff --git a/lexicons/temp/account/beginSignup.json b/lexicons/temp/account/beginSignup.json new file mode 100644 index 0000000000000000000000000000000000000000..5dad3fbda2427ca7b18f7a62505eb8b1391b8179 --- /dev/null +++ b/lexicons/temp/account/beginSignup.json @@ -0,0 +1,34 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.beginSignup", + "defs": { + "main": { + "type": "procedure", + "description": "Begin signup: validate the Turnstile token and email a verification code to the address.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["email", "turnstileToken"], + "properties": { + "email": { + "type": "string", + "description": "Email address for the new account." + }, + "turnstileToken": { + "type": "string", + "description": "Cloudflare Turnstile challenge response token." + } + } + } + }, + "errors": [ + { "name": "SignupDisabled", "description": "Signup is not currently enabled on this service." }, + { "name": "InvalidTurnstileToken", "description": "The Turnstile captcha response could not be verified." }, + { "name": "InvalidEmail", "description": "The email address is not valid." }, + { "name": "EmailAlreadyRegistered", "description": "An account already exists for this email address." }, + { "name": "TooManyRequests", "description": "A signup email was sent for this address very recently; wait before asking for another." } + ] + } + } +} diff --git a/lexicons/temp/account/completeSignup.json b/lexicons/temp/account/completeSignup.json new file mode 100644 index 0000000000000000000000000000000000000000..c63cea4410429044207691a98ae614cafb1f1fa7 --- /dev/null +++ b/lexicons/temp/account/completeSignup.json @@ -0,0 +1,76 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.completeSignup", + "defs": { + "main": { + "type": "procedure", + "description": "Complete signup with the link token, choosing a username and password. Deliberi provisions the PDS account internally using a pre-minted invite code.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["token", "username", "password"], + "properties": { + "token": { + "type": "string", + "description": "Verification token from the emailed link." + }, + "username": { + "type": "string", + "description": "Desired username; the handle is .." + }, + "password": { + "type": "string", + "description": "Password for the new account." + }, + "requestUri": { + "type": "string", + "description": "request_uri of the browser's pushed OAuth authorization request at the PDS. When the PDS supports it, the new account authorizes that request without the PDS sign-in and consent screens." + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["did", "handle"], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID of the newly provisioned account." + }, + "handle": { + "type": "string", + "description": "The assigned handle, .." + }, + "authorization": { + "type": "ref", + "ref": "#authorization", + "description": "Present when the PDS authorized requestUri for the new account. Without it the browser signs in through the PDS." + } + } + } + }, + "errors": [ + { "name": "SignupDisabled", "description": "Signup is not currently enabled on this service." }, + { "name": "InvalidCode", "description": "The verification link is invalid or expired." }, + { "name": "InvalidUsername", "description": "The username is invalid or not allowed." }, + { "name": "UsernameNotAvailable", "description": "The username is already taken." }, + { "name": "InvalidPassword", "description": "The password is not allowed." }, + { "name": "EmailAlreadyRegistered", "description": "An account already exists for this email." } + ] + }, + "authorization": { + "type": "object", + "description": "The OAuth authorization response for requestUri, as the PDS would have put it in the redirect.", + "required": ["code", "state", "iss"], + "properties": { + "code": { "type": "string" }, + "state": { "type": "string" }, + "iss": { "type": "string" } + } + } + } +} diff --git a/lexicons/temp/account/deleteEmail.json b/lexicons/temp/account/deleteEmail.json new file mode 100644 index 0000000000000000000000000000000000000000..934dfc67ab783bfd7a9f28781713addb745b6b9b --- /dev/null +++ b/lexicons/temp/account/deleteEmail.json @@ -0,0 +1,26 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.deleteEmail", + "defs": { + "main": { + "type": "procedure", + "description": "Remove an email address from the authenticated account. Cannot remove the primary address.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["email"], + "properties": { + "email": { + "type": "string", + "description": "Email address to remove." + } + } + } + }, + "errors": [ + { "name": "CannotDeletePrimary", "description": "The primary email address cannot be deleted. Set another address as primary first." } + ] + } + } +} diff --git a/lexicons/temp/account/dismissNewsletter.json b/lexicons/temp/account/dismissNewsletter.json new file mode 100644 index 0000000000000000000000000000000000000000..38622c4701fe40f4385ba29c5835b6f02d3f4a93 --- /dev/null +++ b/lexicons/temp/account/dismissNewsletter.json @@ -0,0 +1,10 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.dismissNewsletter", + "defs": { + "main": { + "type": "procedure", + "description": "Dismiss the newsletter sign-up prompt without subscribing." + } + } +} diff --git a/lexicons/temp/account/getNewsletterPreference.json b/lexicons/temp/account/getNewsletterPreference.json new file mode 100644 index 0000000000000000000000000000000000000000..0988e448edda73f3313140775979454d9093e143 --- /dev/null +++ b/lexicons/temp/account/getNewsletterPreference.json @@ -0,0 +1,22 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.getNewsletterPreference", + "defs": { + "main": { + "type": "query", + "description": "Get the authenticated account's newsletter choice. status is absent when the account has neither subscribed nor dismissed the prompt.", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "status": { + "type": "string", + "knownValues": ["subscribed", "dismissed"] + } + } + } + } + } + } +} diff --git a/lexicons/temp/account/getSignup.json b/lexicons/temp/account/getSignup.json new file mode 100644 index 0000000000000000000000000000000000000000..c3a13af11c8ee2042434d67b5b37ce15d7a6db7d --- /dev/null +++ b/lexicons/temp/account/getSignup.json @@ -0,0 +1,42 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.getSignup", + "defs": { + "main": { + "type": "query", + "description": "Look up a pending signup by its link token, so the completion form can reject a dead link before the user fills it in.", + "parameters": { + "type": "params", + "required": ["token"], + "properties": { + "token": { + "type": "string", + "description": "Verification token from the emailed link." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["email", "pds"], + "properties": { + "email": { + "type": "string", + "description": "Email address the signup was started with." + }, + "pds": { + "type": "string", + "format": "uri", + "description": "PDS the account will be created on, where the browser starts the OAuth request it passes to completeSignup." + } + } + } + }, + "errors": [ + { "name": "SignupDisabled", "description": "Signup is not currently enabled on this service." }, + { "name": "InvalidCode", "description": "The verification link is invalid or expired." } + ] + } + } +} diff --git a/lexicons/temp/account/listEmails.json b/lexicons/temp/account/listEmails.json new file mode 100644 index 0000000000000000000000000000000000000000..ca66e1ffdfe0b60523496af925c12b8d95a02a60 --- /dev/null +++ b/lexicons/temp/account/listEmails.json @@ -0,0 +1,48 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.listEmails", + "defs": { + "main": { + "type": "query", + "description": "List all email addresses associated with the authenticated account.", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["emails"], + "properties": { + "emails": { + "type": "array", + "items": { + "type": "ref", + "ref": "#email" + } + } + } + } + } + }, + "email": { + "type": "object", + "required": ["address", "verified", "primary", "createdAt"], + "properties": { + "address": { + "type": "string", + "description": "Email address." + }, + "verified": { + "type": "boolean", + "description": "Whether the address has been verified." + }, + "primary": { + "type": "boolean", + "description": "Whether this is the primary email address." + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } +} diff --git a/lexicons/temp/account/resendSignup.json b/lexicons/temp/account/resendSignup.json new file mode 100644 index 0000000000000000000000000000000000000000..62fdd324857dab38c204d3dc4f0fdf71710a153b --- /dev/null +++ b/lexicons/temp/account/resendSignup.json @@ -0,0 +1,29 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.resendSignup", + "defs": { + "main": { + "type": "procedure", + "description": "Re-send the signup verification link for a pending signup. Re-mails the same live link (same token, no new invite code) for a signup begun but not yet completed; fails when nothing is in flight for the email or the link has expired.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["email"], + "properties": { + "email": { + "type": "string", + "description": "Email address used to begin the signup." + } + } + } + }, + "errors": [ + { "name": "SignupDisabled", "description": "Signup is not currently enabled on this service." }, + { "name": "InvalidEmail", "description": "The email address is not valid." }, + { "name": "NoPendingSignup", "description": "There is no pending (unexpired) signup for this email; start again from the signup page." }, + { "name": "TooManyRequests", "description": "A signup email was sent for this address very recently; wait before asking for another." } + ] + } + } +} diff --git a/lexicons/temp/account/setPrimaryEmail.json b/lexicons/temp/account/setPrimaryEmail.json new file mode 100644 index 0000000000000000000000000000000000000000..e13d49ae9efc1727a2c495fc67739f61062abd1e --- /dev/null +++ b/lexicons/temp/account/setPrimaryEmail.json @@ -0,0 +1,27 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.setPrimaryEmail", + "defs": { + "main": { + "type": "procedure", + "description": "Set an email address as the primary address for the authenticated account. The address must already be verified.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["email"], + "properties": { + "email": { + "type": "string", + "description": "Email address to promote to primary." + } + } + } + }, + "errors": [ + { "name": "EmailNotVerified", "description": "The email address must be verified before it can be made primary." }, + { "name": "EmailNotFound", "description": "The email address is not associated with this account." } + ] + } + } +} diff --git a/lexicons/temp/account/subscribeNewsletter.json b/lexicons/temp/account/subscribeNewsletter.json new file mode 100644 index 0000000000000000000000000000000000000000..6ec6c73d2924732fcfdd3d91db2224a7231506ff --- /dev/null +++ b/lexicons/temp/account/subscribeNewsletter.json @@ -0,0 +1,10 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.subscribeNewsletter", + "defs": { + "main": { + "type": "procedure", + "description": "Opt the authenticated account into newsletter / announcement emails." + } + } +} diff --git a/lexicons/temp/account/verifyEmail.json b/lexicons/temp/account/verifyEmail.json new file mode 100644 index 0000000000000000000000000000000000000000..298e738c0130cb56413207ca52211af20e87bf10 --- /dev/null +++ b/lexicons/temp/account/verifyEmail.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.account.verifyEmail", + "defs": { + "main": { + "type": "procedure", + "description": "Verify an email address using the token from the verification link emailed by addEmail. The address becomes verified, and is made primary if it is the account's first verified address. Tokens are single-use.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["token"], + "properties": { + "token": { + "type": "string", + "description": "Verification token from the emailed link." + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["email"], + "properties": { + "email": { + "type": "string", + "description": "The address that was verified." + } + } + } + }, + "errors": [ + { "name": "InvalidCode", "description": "The verification link is invalid or expired." } + ] + } + } +} diff --git a/lexicons/temp/focus/beginSession.json b/lexicons/temp/focus/beginSession.json new file mode 100644 index 0000000000000000000000000000000000000000..ec13b0b52b7c868f3c23b16682e664e7a294b0f1 --- /dev/null +++ b/lexicons/temp/focus/beginSession.json @@ -0,0 +1,43 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.focus.beginSession", + "defs": { + "main": { + "type": "procedure", + "description": "Activate focus mode. Returns the oldest unread work notification so the client can navigate to it.", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "notificationId": { + "type": "integer", + "description": "ID of the first notification to address. Absent if the queue is empty." + }, + "repoDid": { + "type": "string", + "format": "did", + "description": "DID of the repository the item belongs to. Absent if the queue is empty." + }, + "issueAt": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the issue to navigate to, if the item is an issue." + }, + "pullAt": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the pull to navigate to, if the item is a pull." + } + } + } + }, + "errors": [ + { + "name": "NoFocusItems", + "description": "There are no unread work notifications to focus on." + } + ] + } + } +} diff --git a/lexicons/temp/focus/endSession.json b/lexicons/temp/focus/endSession.json new file mode 100644 index 0000000000000000000000000000000000000000..ab115e4948823e2254301023f431cffa3bacf63b --- /dev/null +++ b/lexicons/temp/focus/endSession.json @@ -0,0 +1,10 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.focus.endSession", + "defs": { + "main": { + "type": "procedure", + "description": "Deactivate focus mode." + } + } +} diff --git a/lexicons/temp/focus/nextItem.json b/lexicons/temp/focus/nextItem.json new file mode 100644 index 0000000000000000000000000000000000000000..e051d93941116d4a6e0d3dd75380e09bb816d71b --- /dev/null +++ b/lexicons/temp/focus/nextItem.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.focus.nextItem", + "defs": { + "main": { + "type": "procedure", + "description": "Mark the current focus notification as read and return the next one. Ends focus mode and returns no item when the queue is exhausted.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["currentId"], + "properties": { + "currentId": { + "type": "integer", + "description": "ID of the notification just addressed, to be marked read." + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "notificationId": { + "type": "integer", + "description": "ID of the next notification to address. Absent when focus mode has ended." + }, + "repoDid": { + "type": "string", + "format": "did", + "description": "DID of the repository the next item belongs to. Absent when focus mode has ended." + }, + "issueAt": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the issue to navigate to, if the item is an issue." + }, + "pullAt": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the pull to navigate to, if the item is a pull." + } + } + } + } + } + } +} diff --git a/lexicons/temp/migrator/createTask.json b/lexicons/temp/migrator/createTask.json new file mode 100644 index 0000000000000000000000000000000000000000..bbc0812230a583fb81f4af80fd76b463171b927d --- /dev/null +++ b/lexicons/temp/migrator/createTask.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.migrator.createTask", + "defs": { + "main": { + "type": "procedure", + "description": "Create a task to migrate repositories into knots.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "requestId", + "jobs" + ], + "properties": { + "requestId": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "Idempotency key for this task." + }, + "githubToken": { + "type": "string", + "description": "GitHub token for reading private source repositories." + }, + "jobs": { + "type": "array", + "minLength": 1, + "maxLength": 50, + "items": { + "type": "ref", + "ref": "org.tangled.temp.migrator.defs#newJob" + } + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "ref", + "ref": "org.tangled.temp.migrator.defs#task" + } + }, + "errors": [ + { + "name": "InvalidRequest", + "description": "Malformed jobs or missing githubToken for a private repository." + }, + { + "name": "TaskConflict", + "description": "requestId was previously used with different jobs." + }, + { + "name": "GrantRequired", + "description": "Migrator is not authorized. Body contains the authorization URL." + } + ] + } + } +} diff --git a/lexicons/temp/migrator/defs.json b/lexicons/temp/migrator/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..bad8ddfa658777cc8911b9566ae7b97e30eaaa0e --- /dev/null +++ b/lexicons/temp/migrator/defs.json @@ -0,0 +1,137 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.migrator.defs", + "defs": { + "newJob": { + "type": "object", + "description": "A repository to migrate.", + "required": [ + "name", + "knotDid", + "sourceUrl" + ], + "properties": { + "name": { + "type": "string", + "maxLength": 100, + "description": "Repository name reserved on the destination knot." + }, + "knotDid": { + "type": "string", + "format": "did", + "description": "DID of the destination knot." + }, + "sourceUrl": { + "type": "string", + "format": "uri", + "description": "HTTPS URL of the source repository." + }, + "description": { + "type": "string", + "maxGraphemes": 140, + "description": "Repository description." + }, + "private": { + "type": "boolean", + "default": false, + "description": "Whether the source repository is private." + } + } + }, + "job": { + "type": "object", + "description": "Migration status of a repository.", + "required": [ + "id", + "name", + "knotDid", + "sourceUrl", + "private", + "status", + "attempts" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique job ID." + }, + "name": { + "type": "string", + "maxLength": 100 + }, + "repoDid": { + "type": "string", + "format": "did", + "description": "DID of the created repository, once created on the knot." + }, + "knotDid": { + "type": "string", + "format": "did" + }, + "sourceUrl": { + "type": "string", + "format": "uri" + }, + "private": { + "type": "boolean" + }, + "status": { + "type": "string", + "knownValues": [ + "queued", + "cloning", + "importing", + "completed", + "failed", + "authorization_required" + ], + "description": "Current status of the migration job." + }, + "attempts": { + "type": "integer", + "minimum": 0, + "description": "Number of migration attempts." + }, + "error": { + "type": "string", + "description": "Error message from the last failed attempt." + } + } + }, + "task": { + "type": "object", + "description": "A migration task containing repository jobs.", + "required": [ + "id", + "ownerDid", + "createdAt", + "jobs" + ], + "properties": { + "id": { + "type": "string" + }, + "ownerDid": { + "type": "string", + "format": "did", + "description": "DID of the user who created the task." + }, + "createdAt": { + "type": "string", + "format": "datetime" + }, + "updatedAt": { + "type": "string", + "format": "datetime" + }, + "jobs": { + "type": "array", + "items": { + "type": "ref", + "ref": "#job" + } + } + } + } + } +} diff --git a/lexicons/temp/migrator/describeSource.json b/lexicons/temp/migrator/describeSource.json new file mode 100644 index 0000000000000000000000000000000000000000..b6f1c18bf6853f58ce6d38e6099db5d2d8b5a7bc --- /dev/null +++ b/lexicons/temp/migrator/describeSource.json @@ -0,0 +1,48 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.migrator.describeSource", + "defs": { + "main": { + "type": "query", + "description": "Describe the git repository at a source URL before migrating it.", + "parameters": { + "type": "params", + "required": ["sourceUrl"], + "properties": { + "sourceUrl": { + "type": "string", + "description": "HTTPS clone URL of the source repository." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["name", "defaultBranch"], + "properties": { + "name": { + "type": "string", + "maxLength": 100, + "description": "Repository name derived from the URL." + }, + "defaultBranch": { + "type": "string", + "description": "Branch HEAD points at on the remote." + } + } + } + }, + "errors": [ + { + "name": "InvalidSource", + "description": "The source URL is one the migrator refuses." + }, + { + "name": "UnreachableSource", + "description": "The repository could not be reached." + } + ] + } + } +} diff --git a/lexicons/temp/migrator/getTask.json b/lexicons/temp/migrator/getTask.json new file mode 100644 index 0000000000000000000000000000000000000000..df7c41d55e25cd38b641c15900cd6dc92d2b6361 --- /dev/null +++ b/lexicons/temp/migrator/getTask.json @@ -0,0 +1,35 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.migrator.getTask", + "defs": { + "main": { + "type": "query", + "description": "Fetch a migration task and its jobs.", + "parameters": { + "type": "params", + "required": [ + "taskId" + ], + "properties": { + "taskId": { + "type": "string", + "description": "ID of the task to fetch." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "ref", + "ref": "org.tangled.temp.migrator.defs#task" + } + }, + "errors": [ + { + "name": "TaskNotFound", + "description": "Migration task not found." + } + ] + } + } +} diff --git a/lexicons/temp/migrator/listTasks.json b/lexicons/temp/migrator/listTasks.json new file mode 100644 index 0000000000000000000000000000000000000000..453391f5db9a8188ac82e66e4e1a744cddbf69bb --- /dev/null +++ b/lexicons/temp/migrator/listTasks.json @@ -0,0 +1,40 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.migrator.listTasks", + "defs": { + "main": { + "type": "query", + "description": "List migration tasks for the caller.", + "parameters": { + "type": "params", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 20, + "description": "Maximum number of tasks to return." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "tasks" + ], + "properties": { + "tasks": { + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.temp.migrator.defs#task" + } + } + } + } + } + } + } +} diff --git a/lexicons/temp/migrator/retryJob.json b/lexicons/temp/migrator/retryJob.json new file mode 100644 index 0000000000000000000000000000000000000000..8c2a05760f720ba14fe6e6007389a186a81936a8 --- /dev/null +++ b/lexicons/temp/migrator/retryJob.json @@ -0,0 +1,51 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.migrator.retryJob", + "defs": { + "main": { + "type": "procedure", + "description": "Retry a failed or unauthorized migration job.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "taskId", + "jobId" + ], + "properties": { + "taskId": { + "type": "string", + "description": "ID of the task the job belongs to." + }, + "jobId": { + "type": "string", + "description": "ID of the job to retry." + }, + "githubToken": { + "type": "string", + "description": "GitHub token for retrying a private repository." + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "ref", + "ref": "org.tangled.temp.migrator.defs#task" + } + }, + "errors": [ + { + "name": "TaskNotFound", + "description": "Migration task not found." + }, + { + "name": "JobNotRetryable", + "description": "Job is completed or currently running." + } + ] + } + } +} diff --git a/lexicons/temp/newsletter/signup.json b/lexicons/temp/newsletter/signup.json new file mode 100644 index 0000000000000000000000000000000000000000..734653b99b12d4b0b5f47127b7d121b9d3aaa064 --- /dev/null +++ b/lexicons/temp/newsletter/signup.json @@ -0,0 +1,26 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.newsletter.signup", + "defs": { + "main": { + "type": "procedure", + "description": "Add an email address to the newsletter. Unauthenticated; signed-in accounts should use org.tangled.temp.account.subscribeNewsletter so the choice is remembered.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["email"], + "properties": { + "email": { + "type": "string", + "description": "Email address to subscribe." + } + } + } + }, + "errors": [ + { "name": "InvalidEmail", "description": "The email address is not valid." } + ] + } + } +} diff --git a/lexicons/temp/notification/deleteNotification.json b/lexicons/temp/notification/deleteNotification.json new file mode 100644 index 0000000000000000000000000000000000000000..89376f02b1562fb1c063ca6b72b7bcabd7f2cebc --- /dev/null +++ b/lexicons/temp/notification/deleteNotification.json @@ -0,0 +1,23 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.notification.deleteNotification", + "defs": { + "main": { + "type": "procedure", + "description": "Delete a notification.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri"], + "properties": { + "uri": { + "type": "string", + "description": "notification key as returned by listNotifications; pass it back unchanged." + } + } + } + } + } + } +} diff --git a/lexicons/temp/notification/getPreferences.json b/lexicons/temp/notification/getPreferences.json new file mode 100644 index 0000000000000000000000000000000000000000..7b5f80c9e11bcb7a8ddddebb1a3601d00929b072 --- /dev/null +++ b/lexicons/temp/notification/getPreferences.json @@ -0,0 +1,44 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.notification.getPreferences", + "defs": { + "main": { + "type": "query", + "description": "Get notification preferences for the authenticated user.", + "output": { + "encoding": "application/json", + "schema": { + "type": "ref", + "ref": "#preferences" + } + } + }, + "preferences": { + "type": "object", + "required": [ + "repoStarred", + "issueCreated", + "issueCommented", + "issueClosed", + "pullCreated", + "pullCommented", + "pullMerged", + "followed", + "userMentioned", + "emailNotifications" + ], + "properties": { + "repoStarred": { "type": "boolean" }, + "issueCreated": { "type": "boolean" }, + "issueCommented": { "type": "boolean" }, + "issueClosed": { "type": "boolean" }, + "pullCreated": { "type": "boolean" }, + "pullCommented": { "type": "boolean" }, + "pullMerged": { "type": "boolean" }, + "followed": { "type": "boolean" }, + "userMentioned": { "type": "boolean" }, + "emailNotifications": { "type": "boolean" } + } + } + } +} diff --git a/lexicons/temp/notification/getUnreadCount.json b/lexicons/temp/notification/getUnreadCount.json new file mode 100644 index 0000000000000000000000000000000000000000..74e842d30b38b7b4ea0c3b02a06533cf62e1130f --- /dev/null +++ b/lexicons/temp/notification/getUnreadCount.json @@ -0,0 +1,22 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.notification.getUnreadCount", + "defs": { + "main": { + "type": "query", + "description": "Get the total number of unread notifications for the authenticated user. Used to drive the bell badge.", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["count"], + "properties": { + "count": { + "type": "integer" + } + } + } + } + } + } +} diff --git a/lexicons/temp/notification/listNotifications.json b/lexicons/temp/notification/listNotifications.json new file mode 100644 index 0000000000000000000000000000000000000000..dfdbca00c6bc9e5ebb9ef7fed903856db35f9f83 --- /dev/null +++ b/lexicons/temp/notification/listNotifications.json @@ -0,0 +1,102 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.notification.listNotifications", + "defs": { + "main": { + "type": "query", + "description": "List notifications for the authenticated user, split by category.", + "parameters": { + "type": "params", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "description": "Max notifications per category to return." + }, + "cursor": { + "type": "string" + }, + "read": { + "type": "string", + "description": "Filter by read state: 'all' or 'unread'." + }, + "category": { + "type": "string", + "description": "Filter by category: 'all', 'social', or 'work'." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["notifications", "workUnreadCount", "socialUnreadCount"], + "properties": { + "notifications": { + "type": "array", + "items": { + "type": "ref", + "ref": "#notification" + } + }, + "workUnreadCount": { + "type": "integer" + }, + "socialUnreadCount": { + "type": "integer" + }, + "cursor": { + "type": "string" + } + } + } + } + }, + "notification": { + "type": "object", + "required": ["uri", "type", "category", "actorDid", "read", "createdAt"], + "properties": { + "uri": { + "type": "string", + "description": "notification key; pass it back to updateSeen and deleteNotification." + }, + "type": { + "type": "string", + "description": "Notification type: repo_starred, issue_created, issue_commented, issue_closed, issue_reopen, issue_assigned, issue_unassigned, pull_created, pull_commented, pull_merged, pull_closed, pull_reopen, pull_assigned, pull_unassigned, followed, user_mentioned, knot_invited, collaborator_invited." + }, + "category": { + "type": "string", + "description": "Broad category: 'social' or 'work'." + }, + "actorDid": { + "type": "string", + "format": "did", + "description": "DID of the user who triggered this notification." + }, + "read": { + "type": "boolean" + }, + "createdAt": { + "type": "string", + "format": "datetime" + }, + "repoDid": { + "type": "string", + "format": "did", + "description": "DID of the related repository, if applicable." + }, + "issueAt": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the related org.tangled.issue.issue record, if applicable." + }, + "pullAt": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the related org.tangled.pulls.pull record, if applicable." + } + } + } + } +} diff --git a/lexicons/temp/notification/listRecipients.json b/lexicons/temp/notification/listRecipients.json new file mode 100644 index 0000000000000000000000000000000000000000..0390af942c783c06612ffe207f7faa23ffe289bc --- /dev/null +++ b/lexicons/temp/notification/listRecipients.json @@ -0,0 +1,38 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.notification.listRecipients", + "defs": { + "main": { + "type": "query", + "description": "Service-internal. Hosted by bobbin. Given an entity at-uri or repo DID, returns the dids subscribed to it. deliberi calls this while ingesting the firehose to fan a notification out to subscribers.", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "at-uri of the entity (for entity-level subscribers) or a repo DID (for repo-level subscribers)." + }, + "collection": { + "type": "string", + "format": "nsid", + "description": "Optional collection NSID to filter subscribers by subscription scope. Only subscribers with no collection filter or a filter containing this NSID are returned." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["dids"], + "properties": { + "dids": { + "type": "array", + "items": { "type": "string", "format": "did" } + } + } + } + } + } + } +} diff --git a/lexicons/temp/notification/markAllRead.json b/lexicons/temp/notification/markAllRead.json new file mode 100644 index 0000000000000000000000000000000000000000..f9deeca4ce2a78b83171053b2e508a009e03e48b --- /dev/null +++ b/lexicons/temp/notification/markAllRead.json @@ -0,0 +1,10 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.notification.markAllRead", + "defs": { + "main": { + "type": "procedure", + "description": "Mark all of the authenticated user's notifications as read." + } + } +} diff --git a/lexicons/temp/notification/markEntityRead.json b/lexicons/temp/notification/markEntityRead.json new file mode 100644 index 0000000000000000000000000000000000000000..46dc52942bf63ea274d2bc22df5852a2e89e7e0f --- /dev/null +++ b/lexicons/temp/notification/markEntityRead.json @@ -0,0 +1,24 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.notification.markEntityRead", + "defs": { + "main": { + "type": "procedure", + "description": "Mark all of the caller's notifications for a given issue or pull as read. Used when the user visits the entity directly, rather than through a notification.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["entityAt"], + "properties": { + "entityAt": { + "type": "string", + "format": "at-uri", + "description": "at-uri of the issue or pull whose notifications should be marked read." + } + } + } + } + } + } +} diff --git a/lexicons/temp/notification/updatePreferences.json b/lexicons/temp/notification/updatePreferences.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0ff22dc3b09a8d8b37cdfa367892d78e78f77b --- /dev/null +++ b/lexicons/temp/notification/updatePreferences.json @@ -0,0 +1,28 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.notification.updatePreferences", + "defs": { + "main": { + "type": "procedure", + "description": "Update notification preferences for the authenticated user. Only provided fields are updated.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "repoStarred": { "type": "boolean" }, + "issueCreated": { "type": "boolean" }, + "issueCommented": { "type": "boolean" }, + "issueClosed": { "type": "boolean" }, + "pullCreated": { "type": "boolean" }, + "pullCommented": { "type": "boolean" }, + "pullMerged": { "type": "boolean" }, + "followed": { "type": "boolean" }, + "userMentioned": { "type": "boolean" }, + "emailNotifications": { "type": "boolean" } + } + } + } + } + } +} diff --git a/lexicons/temp/notification/updateSeen.json b/lexicons/temp/notification/updateSeen.json new file mode 100644 index 0000000000000000000000000000000000000000..adc93b4153296ba4287ef476701d4ae97d8d3d45 --- /dev/null +++ b/lexicons/temp/notification/updateSeen.json @@ -0,0 +1,26 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.notification.updateSeen", + "defs": { + "main": { + "type": "procedure", + "description": "Mark a single notification as read or unread.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "read"], + "properties": { + "uri": { + "type": "string", + "description": "notification key as returned by listNotifications; pass it back unchanged." + }, + "read": { + "type": "boolean" + } + } + } + } + } + } +} diff --git a/lexicons/temp/repo/deploySite.json b/lexicons/temp/repo/deploySite.json new file mode 100644 index 0000000000000000000000000000000000000000..9abce3972cac74364a53d0c4ad1bb344d4b956e8 --- /dev/null +++ b/lexicons/temp/repo/deploySite.json @@ -0,0 +1,33 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.repo.deploySite", + "defs": { + "main": { + "type": "procedure", + "description": "Manually deploy a repository's site now. Authenticated with the caller's serviceauth token (audience = the sites worker's DID); only the repository owner may trigger. Queues the deploy; the outcome is recorded in the repository's deploy history.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repoDid"], + "properties": { + "repoDid": { + "type": "string", + "format": "did", + "description": "DID of the repository to deploy." + } + } + } + }, + "errors": [ + { "name": "AuthError", "description": "Serviceauth verification failed." }, + { "name": "NotOwner", "description": "Only the repository owner can deploy." }, + { "name": "RepoNotFound", "description": "No repository record publishes this repo DID." }, + { "name": "OwnershipUnverifiable", "description": "Repository ownership could not be verified." }, + { "name": "SiteConfigDisputed", "description": "The stored site config names a different owner and must be re-saved by the repository owner." }, + { "name": "SiteNotFound", "description": "The repository has no site configuration or the owner has no active claim." }, + { "name": "WorkerUnavailable", "description": "The sites Worker could not be reached." } + ] + } + } +} \ No newline at end of file diff --git a/lexicons/temp/repo/disableSite.json b/lexicons/temp/repo/disableSite.json new file mode 100644 index 0000000000000000000000000000000000000000..dcb4dd089c6ee60b64c614ef9be3b3a32ca8fd1b --- /dev/null +++ b/lexicons/temp/repo/disableSite.json @@ -0,0 +1,31 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.repo.disableSite", + "defs": { + "main": { + "type": "procedure", + "description": "Remove the site configuration for a repository, deleting its deployed files.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repoDid"], + "properties": { + "repoDid": { + "type": "string", + "format": "did", + "description": "DID of the repository as minted by the knot." + } + } + } + }, + "errors": [ + { "name": "SiteNotFound", "description": "No site configuration exists for this repository." }, + { "name": "NotOwner", "description": "The account does not own this repository." }, + { "name": "RepoNotFound", "description": "No repository record publishes this repo DID." }, + { "name": "OwnershipUnverifiable", "description": "Repository ownership could not be verified." }, + { "name": "SiteConfigDisputed", "description": "The stored site config names a different owner and must be re-saved by the repository owner." } + ] + } + } +} diff --git a/lexicons/temp/repo/getDeployHistory.json b/lexicons/temp/repo/getDeployHistory.json new file mode 100644 index 0000000000000000000000000000000000000000..03ced65191218425181caaa72fe8cf758d2ff950 --- /dev/null +++ b/lexicons/temp/repo/getDeployHistory.json @@ -0,0 +1,80 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.repo.getDeployHistory", + "defs": { + "main": { + "type": "query", + "description": "Get the recent deploy history for a repository, newest first. Authenticated with the caller's serviceauth token (audience = the sites worker's DID); only the repository owner may read it.", + "parameters": { + "type": "params", + "required": ["repoDid"], + "properties": { + "repoDid": { + "type": "string", + "format": "did", + "description": "DID of the repository as minted by the knot." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["deploys"], + "properties": { + "deploys": { + "type": "array", + "maxLength": 20, + "items": { "type": "ref", "ref": "#deploy" }, + "description": "The 20 most recent deploys, newest first. Empty when the repository has never deployed." + } + } + } + }, + "errors": [ + { "name": "AuthError", "description": "Serviceauth verification failed." }, + { "name": "NotOwner", "description": "The account does not own this repository." }, + { "name": "RepoNotFound", "description": "No repository record publishes this repo DID." }, + { "name": "OwnershipUnverifiable", "description": "Repository ownership could not be verified." }, + { "name": "SiteNotFound", "description": "The repository has no site configuration or the owner has no active claim." } + ] + }, + "deploy": { + "type": "object", + "required": ["status", "trigger", "branch", "dir", "commitSha", "error", "createdAt"], + "properties": { + "status": { + "type": "string", + "knownValues": ["success", "failure"], + "description": "Outcome of the deploy." + }, + "trigger": { + "type": "string", + "knownValues": ["config_change", "push", "manual"], + "description": "What asked for the deploy: a site config change, a push to the deployed branch, or a manual redeploy." + }, + "branch": { + "type": "string", + "description": "Branch the deploy built." + }, + "dir": { + "type": "string", + "description": "Directory within the repository the deploy built (e.g. '/' or '/docs')." + }, + "commitSha": { + "type": "string", + "description": "Commit the deploy built. Empty for a config-change deploy, which builds the branch tip at deploy time." + }, + "error": { + "type": "string", + "description": "Why the deploy failed. Empty when it succeeded." + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "When the deploy finished." + } + } + } + } +} diff --git a/lexicons/temp/repo/getSiteConfig.json b/lexicons/temp/repo/getSiteConfig.json new file mode 100644 index 0000000000000000000000000000000000000000..fab28cb946430c3b48ad33325c35b14c3ec3fd87 --- /dev/null +++ b/lexicons/temp/repo/getSiteConfig.json @@ -0,0 +1,57 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.repo.getSiteConfig", + "defs": { + "main": { + "type": "query", + "description": "Get the site configuration for a repository, if one exists.", + "parameters": { + "type": "params", + "required": ["repoDid"], + "properties": { + "repoDid": { + "type": "string", + "format": "did", + "description": "DID of the repository as minted by the knot." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "config": { + "type": "ref", + "ref": "#siteConfig", + "description": "Site config for the repository. Absent if no site is configured." + } + } + } + }, + "errors": [ + { "name": "NotOwner", "description": "The account does not own this repository." }, + { "name": "RepoNotFound", "description": "No repository record publishes this repo DID." }, + { "name": "OwnershipUnverifiable", "description": "Repository ownership could not be verified." } + ] + }, + "siteConfig": { + "type": "object", + "required": ["branch", "dir", "isIndex"], + "properties": { + "branch": { + "type": "string", + "description": "Branch to deploy from." + }, + "dir": { + "type": "string", + "description": "Directory within the repository to deploy (e.g. '/' or '/docs')." + }, + "isIndex": { + "type": "boolean", + "description": "Whether this repo serves as the index (root) for the domain." + } + } + } + } +} diff --git a/lexicons/temp/repo/updateSiteConfig.json b/lexicons/temp/repo/updateSiteConfig.json new file mode 100644 index 0000000000000000000000000000000000000000..4306982102b696add584fd48a614b58023b13717 --- /dev/null +++ b/lexicons/temp/repo/updateSiteConfig.json @@ -0,0 +1,61 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.repo.updateSiteConfig", + "defs": { + "main": { + "type": "procedure", + "description": "Create or update the site configuration for a repository owned by the authenticated user. Writing the config schedules a deploy attempt; the deploy outcome is recorded in history. If the deploy service is disabled, the config is saved but WorkerUnavailable is returned.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repoDid", "name", "rkey", "knot", "branch", "dir"], + "properties": { + "repoDid": { + "type": "string", + "format": "did", + "description": "DID of the repository as minted by the knot." + }, + "name": { + "type": "string", + "description": "Repository name as its knot reports it (the first path segment its sub-path site is served under)." + }, + "rkey": { + "type": "string", + "description": "Atproto record key of the repository; identifies the {did}/{rkey}/ prefix in R2." + }, + "knot": { + "type": "string", + "description": "Knot host that hosts the repository." + }, + "branch": { + "type": "string", + "description": "Branch to deploy from." + }, + "dir": { + "type": "string", + "description": "Directory within the repository to deploy (e.g. '/' or '/docs')." + }, + "isIndex": { + "type": "boolean", + "description": "Whether this repo should serve as the index (root) for the claimed domain." + } + } + } + }, + "errors": [ + { "name": "NotOwner", "description": "The account is not the publisher of this repository's record." }, + { "name": "RepoNotFound", "description": "No repository record publishes this repo DID." }, + { "name": "OwnershipUnverifiable", "description": "Repository ownership could not be verified; the config was not saved." }, + { "name": "RepoIdentityMismatch", "description": "The request's repository name, key or knot does not match its record." }, + { "name": "NoDomainClaim", "description": "The account does not have an active domain claim." }, + { "name": "HandleNotOwned", "description": "The handle authority identifies a different account." }, + { "name": "HandleUnknown", "description": "The handle authority does not know this handle." }, + { "name": "ClaimUnverifiable", "description": "The handle authority could not be reached to verify this claim." }, + { "name": "InvalidDirectory", "description": "The deploy directory path is invalid." }, + { "name": "IndexSiteTaken", "description": "Another repository is already the index site of the account's domain." }, + { "name": "WorkerUnavailable", "description": "The config was saved but the deploy service is disabled." } + ] + } + } +} \ No newline at end of file diff --git a/lexicons/temp/review/listPatches.json b/lexicons/temp/review/listPatches.json new file mode 100644 index 0000000000000000000000000000000000000000..17296ee307816631b2eaedb288a89fb976aa6bd3 --- /dev/null +++ b/lexicons/temp/review/listPatches.json @@ -0,0 +1,71 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.review.listPatches", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["ticket"], + "properties": { + "ticket": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the org.tangled.track.ticket the patches belong to." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "offset": { + "type": "integer", + "minimum": 0, + "description": "Absolute offset for random-access pagination. Mutually exclusive with cursor; offsets drift under concurrent writes, so follow up with the returned cursor." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" }, + "total": { + "type": "integer", + "description": "Total items in the full list; omitted for filtered or merged views" + } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded org.tangled.review.patch record" + } + } + } + } +} diff --git a/lexicons/temp/search/searchCode.json b/lexicons/temp/search/searchCode.json new file mode 100644 index 0000000000000000000000000000000000000000..41e7fb3f0e736381217ac732a0df2b9e44170e1f --- /dev/null +++ b/lexicons/temp/search/searchCode.json @@ -0,0 +1,127 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.search.searchCode", + "defs": { + "main": { + "type": "query", + "description": "Full-text code search across indexed repositories, backed by Zoekt.", + "parameters": { + "type": "params", + "required": ["q"], + "properties": { + "q": { + "type": "string", + "description": "Search query string." + }, + "repo": { + "type": "string", + "format": "did", + "description": "Restrict search to a specific repository, by its DID." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + "cursor": { + "type": "string" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["results"], + "properties": { + "results": { + "type": "array", + "items": { + "type": "ref", + "ref": "#fileResult" + } + }, + "cursor": { + "type": "string" + } + } + } + } + }, + "fileResult": { + "type": "object", + "required": ["repo", "path", "commit"], + "properties": { + "repo": { + "type": "ref", + "ref": "org.tangled.repo.defs#repoViewBasic" + }, + "path": { + "type": "string", + "description": "File path relative to repository root." + }, + "branches": { + "type": "array", + "items": { "type": "string" } + }, + "commit": { "type": "string" }, + "language": { + "type": "string", + "description": "Detected programming language." + }, + "ranges": { + "type": "array", + "description": "set for a filename match", + "items": { + "type": "ref", + "ref": "#highlight" + } + }, + "chunks": { + "type": "array", + "description": "set for content matches", + "items": { + "type": "ref", + "ref": "#chunk" + } + } + } + }, + "chunk": { + "type": "object", + "required": ["content", "lineStart"], + "properties": { + "content": { + "type": "string", + "description": "Source lines for this match chunk." + }, + "lineStart": { + "type": "integer", + "description": "1-based line number of the first line in content." + }, + "highlights": { + "type": "array", + "description": "Byte-offset ranges within content that match the query.", + "items": { + "type": "ref", + "ref": "#highlight" + } + } + } + }, + "highlight": { + "type": "object", + "required": ["start", "end"], + "properties": { + "start": { + "type": "integer", + "description": "Start byte offset within the chunk content string." + }, + "end": { + "type": "integer", + "description": "End byte offset (exclusive) within the chunk content string." + } + } + } + } +} diff --git a/lexicons/temp/site/claimDomain.json b/lexicons/temp/site/claimDomain.json new file mode 100644 index 0000000000000000000000000000000000000000..36a736469f17e29c121aee05dc8b9a67061ab15e --- /dev/null +++ b/lexicons/temp/site/claimDomain.json @@ -0,0 +1,31 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.site.claimDomain", + "defs": { + "main": { + "type": "procedure", + "description": "Claim a full sites domain (e.g. alice.tngl.io) for the authenticated user. A claim is one per account. tngl.sh handles require confirmation from the handle authority and cannot be released.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["domain"], + "properties": { + "domain": { + "type": "string", + "description": "Full domain to claim, e.g. alice.tngl.io. Label rules: 1-63 lowercase letters, digits and hyphens, no leading/trailing hyphen." + } + } + } + }, + "errors": [ + { "name": "DomainTaken", "description": "That domain is already claimed by another user." }, + { "name": "DomainCooldown", "description": "That domain was recently released and is in a 30-day cooldown period." }, + { "name": "AlreadyClaimed", "description": "You already have a domain claimed. Release it before claiming a new one." }, + { "name": "InvalidSubdomain", "description": "The domain label or TLD is invalid." }, + { "name": "HandleNotOwned", "description": "The handle authority identifies a different account." }, + { "name": "HandleUnknown", "description": "The handle authority does not know this handle." } + ] + } + } +} \ No newline at end of file diff --git a/lexicons/temp/site/getDomainClaim.json b/lexicons/temp/site/getDomainClaim.json new file mode 100644 index 0000000000000000000000000000000000000000..1457781e479bc80aebf2b51f88a39c7694e74c61 --- /dev/null +++ b/lexicons/temp/site/getDomainClaim.json @@ -0,0 +1,25 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.site.getDomainClaim", + "defs": { + "main": { + "type": "query", + "description": "Get the active sites domain or a verified signup handle for the authenticated user, if any. A signup handle may not have a D1 claim row until the first site write.", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "description": "the site domain (e.g. 'alice.tngl.io') or a verified signup handle, absent if neither exists" + } + } + } + }, + "errors": [ + { "name": "ClaimUnverifiable", "description": "The handle authority could not be reached to verify this claim." } + ] + } + } +} diff --git a/lexicons/temp/site/releaseDomain.json b/lexicons/temp/site/releaseDomain.json new file mode 100644 index 0000000000000000000000000000000000000000..81429c04077eb9c9cabfd8250c86a4c07200165a --- /dev/null +++ b/lexicons/temp/site/releaseDomain.json @@ -0,0 +1,27 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.site.releaseDomain", + "defs": { + "main": { + "type": "procedure", + "description": "Release the authenticated user's active domain claim, removing all associated site data.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["domain"], + "properties": { + "domain": { + "type": "string", + "description": "Full domain to release (must match the user's active claim)." + } + } + } + }, + "errors": [ + { "name": "DomainNotFound", "description": "No active claim found for the given domain under this account." }, + { "name": "DomainPermanent", "description": "The claimed domain is a permanent tngl.sh handle domain and cannot be released." } + ] + } + } +} diff --git a/lexicons/temp/spindle/moderation/ban.json b/lexicons/temp/spindle/moderation/ban.json new file mode 100644 index 0000000000000000000000000000000000000000..326d67bd4f5bc7186bc5a052bfef1b717675a3b8 --- /dev/null +++ b/lexicons/temp/spindle/moderation/ban.json @@ -0,0 +1,31 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.moderation.ban", + "defs": { + "main": { + "type": "procedure", + "description": "ban a repository or account owner and immediately wipe matching spindle state", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID of the repository or account owner to ban" + } + } + } + }, + "errors": [ + { + "name": "InvalidDid" + } + ] + } + } +} diff --git a/lexicons/temp/spindle/moderation/defs.json b/lexicons/temp/spindle/moderation/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..66952395ff3d69f14ca2a68391eb9c962f004e9d --- /dev/null +++ b/lexicons/temp/spindle/moderation/defs.json @@ -0,0 +1,26 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.moderation.defs", + "defs": { + "ban": { + "description": "an active ban for a repository or account owner", + "type": "object", + "required": [ + "did", + "createdAt" + ], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID of the banned subject, either a repository or its owner" + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "time at which the ban was created" + } + } + } + } +} diff --git a/lexicons/temp/spindle/moderation/getBan.json b/lexicons/temp/spindle/moderation/getBan.json new file mode 100644 index 0000000000000000000000000000000000000000..06e3321d973a7ac358653843bae6145739e07384 --- /dev/null +++ b/lexicons/temp/spindle/moderation/getBan.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.moderation.getBan", + "defs": { + "main": { + "type": "query", + "description": "return the active ban for a repository or account owner DID", + "parameters": { + "type": "params", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID of the repository or account owner to look up" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "ban" + ], + "properties": { + "ban": { + "description": "active ban matching the requested DID", + "type": "ref", + "ref": "org.tangled.temp.spindle.moderation.defs#ban" + } + } + } + } + } + } +} diff --git a/lexicons/temp/spindle/moderation/listBans.json b/lexicons/temp/spindle/moderation/listBans.json new file mode 100644 index 0000000000000000000000000000000000000000..8963932ab2c19e1a0600df402572b453cd61acbc --- /dev/null +++ b/lexicons/temp/spindle/moderation/listBans.json @@ -0,0 +1,29 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.moderation.listBans", + "defs": { + "main": { + "type": "query", + "description": "list all active repository and account-owner bans", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "bans" + ], + "properties": { + "bans": { + "description": "active bans ordered by subject DID", + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.temp.spindle.moderation.defs#ban" + } + } + } + } + } + } + } +} diff --git a/lexicons/temp/spindle/moderation/unban.json b/lexicons/temp/spindle/moderation/unban.json new file mode 100644 index 0000000000000000000000000000000000000000..39d683e887917077a9709c5a5f6b693719191ccf --- /dev/null +++ b/lexicons/temp/spindle/moderation/unban.json @@ -0,0 +1,34 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.moderation.unban", + "defs": { + "main": { + "type": "procedure", + "description": "remove an existing ban for a repository or account owner", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID of the repository or account owner to unban" + } + } + } + }, + "errors": [ + { + "name": "InvalidDid" + }, + { + "name": "BanNotFound" + } + ] + } + } +} diff --git a/lexicons/temp/spindle/quota/defs.json b/lexicons/temp/spindle/quota/defs.json new file mode 100644 index 0000000000000000000000000000000000000000..2b2352719e6cd9e119960871f3f6199ff1812966 --- /dev/null +++ b/lexicons/temp/spindle/quota/defs.json @@ -0,0 +1,63 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.quota.defs", + "defs": { + "limit": { + "description": "a per-subject override for one quota resource", + "type": "object", + "required": [ + "did", + "resource", + "limit" + ], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID whose quota is overridden, either a repository or an account owner" + }, + "resource": { + "type": "string", + "description": "quota resource name" + }, + "limit": { + "type": "integer", + "description": "maximum amount in the resource's native units; -1 means unlimited" + } + } + }, + "usage": { + "description": "current usage aggregated for one subject and resource", + "type": "object", + "required": [ + "scope", + "did", + "resource", + "used" + ], + "properties": { + "scope": { + "type": "string", + "knownValues": [ + "user", + "repo" + ], + "description": "aggregation axis for this row: user or repo" + }, + "did": { + "type": "string", + "format": "did", + "description": "DID for the user or repository scope" + }, + "resource": { + "type": "string", + "description": "quota resource name" + }, + "used": { + "type": "integer", + "description": "amount currently allocated or reserved in native resource units" + } + } + } + } +} diff --git a/lexicons/temp/spindle/quota/get.json b/lexicons/temp/spindle/quota/get.json new file mode 100644 index 0000000000000000000000000000000000000000..11e8b1ed0ac8a332946e163aa3c1cb6d71a8c8d6 --- /dev/null +++ b/lexicons/temp/spindle/quota/get.json @@ -0,0 +1,44 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.quota.get", + "defs": { + "main": { + "type": "query", + "description": "return a per-subject quota override for one resource", + "parameters": { + "type": "params", + "required": [ + "did", + "resource" + ], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID of the repository or account owner" + }, + "resource": { + "type": "string", + "description": "quota resource to look up" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "limit" + ], + "properties": { + "limit": { + "description": "quota override for the requested DID and resource", + "type": "ref", + "ref": "org.tangled.temp.spindle.quota.defs#limit" + } + } + } + } + } + } +} diff --git a/lexicons/temp/spindle/quota/list.json b/lexicons/temp/spindle/quota/list.json new file mode 100644 index 0000000000000000000000000000000000000000..836d0d7a90cf602289d66867c13f088a87eb4a3e --- /dev/null +++ b/lexicons/temp/spindle/quota/list.json @@ -0,0 +1,29 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.quota.list", + "defs": { + "main": { + "type": "query", + "description": "list all per-subject quota overrides", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "limits" + ], + "properties": { + "limits": { + "description": "quota overrides ordered by DID and resource", + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.temp.spindle.quota.defs#limit" + } + } + } + } + } + } + } +} diff --git a/lexicons/temp/spindle/quota/releaseNars.json b/lexicons/temp/spindle/quota/releaseNars.json new file mode 100644 index 0000000000000000000000000000000000000000..bea91f050cead01efb9dc3be1bacdbdcac25c6a8 --- /dev/null +++ b/lexicons/temp/spindle/quota/releaseNars.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.quota.releaseNars", + "defs": { + "main": { + "type": "procedure", + "description": "give back the cache storage quota of nars an operator removed from the nix cache. admin only and idempotent, unknown nars are reported and not an error", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "nars" + ], + "properties": { + "nars": { + "type": "array", + "description": "nars to release, as cache keys like nar/.nar.xz (a bare file name works too) or as nix store paths like /nix/store/-name", + "minLength": 1, + "maxLength": 1000, + "items": { + "type": "string", + "maxLength": 512 + } + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "released", + "bytes", + "unmatched" + ], + "properties": { + "released": { + "type": "integer", + "description": "how many allocations were deleted" + }, + "bytes": { + "type": "integer", + "description": "cache storage bytes given back" + }, + "unmatched": { + "type": "array", + "description": "the inputs that matched no allocation", + "items": { + "type": "string" + } + } + } + } + } + } + } +} diff --git a/lexicons/temp/spindle/quota/set.json b/lexicons/temp/spindle/quota/set.json new file mode 100644 index 0000000000000000000000000000000000000000..5411ecf7cf9a0e916e551cff54d705c1a7598a37 --- /dev/null +++ b/lexicons/temp/spindle/quota/set.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.quota.set", + "defs": { + "main": { + "type": "procedure", + "description": "set a quota override for a repository or account owner; values use the resource's native units", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "did", + "resource" + ], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID of the repository or account owner" + }, + "resource": { + "type": "string", + "description": "quota resource name, such as workflows, vcpus, memory_mib, disk_mib, or cache_storage_bytes" + }, + "limit": { + "type": "integer", + "description": "positive maximum in the resource's native units; cannot be combined with unlimited" + }, + "unlimited": { + "type": "boolean", + "description": "store an unlimited override; cannot be combined with limit" + } + } + } + } + } + } +} diff --git a/lexicons/temp/spindle/quota/unset.json b/lexicons/temp/spindle/quota/unset.json new file mode 100644 index 0000000000000000000000000000000000000000..31002941bcd533ac3a115f8f0c6bbe081c79bd30 --- /dev/null +++ b/lexicons/temp/spindle/quota/unset.json @@ -0,0 +1,31 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.quota.unset", + "defs": { + "main": { + "type": "procedure", + "description": "remove the quota override for a repository or account owner and resource", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "did", + "resource" + ], + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "DID of the repository or account owner" + }, + "resource": { + "type": "string", + "description": "quota resource whose override to remove" + } + } + } + } + } + } +} diff --git a/lexicons/temp/spindle/quota/usage.json b/lexicons/temp/spindle/quota/usage.json new file mode 100644 index 0000000000000000000000000000000000000000..0b18b6d83de331d376b854411192445a0102fba9 --- /dev/null +++ b/lexicons/temp/spindle/quota/usage.json @@ -0,0 +1,47 @@ +{ + "lexicon": 1, + "id": "org.tangled.temp.spindle.quota.usage", + "defs": { + "main": { + "type": "query", + "description": "list non-zero quota usage aggregated by repository and account owner", + "parameters": { + "type": "params", + "properties": { + "scope": { + "type": "string", + "knownValues": [ + "user", + "repo" + ], + "description": "limit results to one aggregation axis: user or repo" + }, + "did": { + "type": "string", + "format": "did", + "description": "limit results to rows for this subject DID" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "usages" + ], + "properties": { + "usages": { + "description": "non-zero usage rows, including active reservations and committed allocations", + "type": "array", + "items": { + "type": "ref", + "ref": "org.tangled.temp.spindle.quota.defs#usage" + } + } + } + } + } + } + } +} diff --git a/lexutil/client.go b/lexutil/client.go new file mode 100644 index 0000000000000000000000000000000000000000..131e33fbf90d9fbbb47811c455337605db47b1e8 --- /dev/null +++ b/lexutil/client.go @@ -0,0 +1,243 @@ +package lexutil + +import ( + "cmp" + "context" + "fmt" + "log/slog" + "net/http" + "net/url" + "time" + + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "github.com/carlmjohnson/versioninfo" + "github.com/gorilla/websocket" + cbg "github.com/whyrusleeping/cbor-gen" +) + +const minHealthyConn = 30 * time.Second + +type Client struct { + indigoxrpc.Client + Dialer websocket.Dialer + Logger *slog.Logger +} + +var _ LexClient = (*Client)(nil) + +func makeParams(p map[string]any) url.Values { + params := url.Values{} + for k, v := range p { + if s, ok := v.([]string); ok { + for _, v := range s { + params.Add(k, v) + } + } else { + params.Add(k, fmt.Sprint(v)) + } + } + return params +} + +type processFn func(ctx context.Context, cr *cbg.CborReader) error + +func (c *Client) LexDo(ctx context.Context, method string, inputEncoding string, endpoint string, params map[string]any, bodyData any, out any) error { + switch method { + case Subscription: + if process, ok := out.(func(context.Context, *cbg.CborReader) error); ok { + return c.LexSubscribe(ctx, endpoint, params, process) + } else if process, ok := out.(processFn); ok { + return c.LexSubscribe(ctx, endpoint, params, process) + } else if redialer, ok := out.(Redialer); ok { + return c.LexSubscribeWithRedialer(ctx, endpoint, params, redialer) + } else { + return fmt.Errorf("unknown output type: %T", out) + } + default: + return c.Client.LexDo(ctx, method, inputEncoding, endpoint, params, bodyData, out) + } +} + +func (c *Client) getHeader() http.Header { + header := http.Header{} + if c.UserAgent != nil { + header.Set("User-Agent", *c.UserAgent) + } else { + header.Set("User-Agent", "extlexutil/"+versioninfo.Short()) + } + if c.Headers != nil { + for k, v := range c.Headers { + header.Set(k, v) + } + } + return header +} + +func (c *Client) LexSubscribe(ctx context.Context, endpoint string, params map[string]any, process func(ctx context.Context, cr *cbg.CborReader) error) error { + logger := cmp.Or(c.Logger, slog.Default().With("system", "events")) + rurl, err := url.Parse(c.Host) + if err != nil { + return err + } + if rurl.Scheme == "http" { + rurl.Scheme = "ws" + } else { + rurl.Scheme = "wss" + } + surl := rurl.JoinPath("/xrpc", endpoint) + surl.RawQuery = makeParams(params).Encode() + + header := c.getHeader() + + u := surl.String() + conn, resp, err := c.Dialer.DialContext(ctx, u, header) + if err != nil { + return fmt.Errorf("%w: %w", ErrDialFailure, err) + } + + logger.Debug("event subscription response", "code", resp.StatusCode, "url", u) + + return c.handleConn(ctx, conn, process) +} + +func (c *Client) LexSubscribeWithRedialer(ctx context.Context, endpoint string, params map[string]any, redialer Redialer) error { + logger := cmp.Or(c.Logger, slog.Default().With("system", "events")) + rurl, err := url.Parse(c.Host) + if err != nil { + return err + } + if rurl.Scheme == "http" { + rurl.Scheme = "ws" + } else { + rurl.Scheme = "wss" + } + surl := rurl.JoinPath("/xrpc", endpoint) + + header := c.getHeader() + + var backoff int + // returns false if the retry budget is exhausted + sleepBackoff := func() bool { + select { + case <-ctx.Done(): + case <-time.After(time.Duration(5+backoff) * time.Second): + } + backoff++ + return backoff <= 15 + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + surl.RawQuery = makeParams(params).Encode() + + u := surl.String() + conn, resp, err := c.Dialer.DialContext(ctx, u, header) + if err != nil { + logger.Warn("dialing failed", "err", err, "backoff", backoff) + if !sleepBackoff() { + return fmt.Errorf("%w: %w", ErrDialFailure, err) + } + continue + } + + logger.Debug("event subscription response", "code", resp.StatusCode, "url", u) + + connectedAt := time.Now() + connErr := c.handleConn(ctx, conn, redialer.Process) + if connErr != nil { + logger.Warn("host connection failed", "err", connErr, "backoff", backoff) + } + + // updates cursor + updated := redialer.UpdateParams(ctx, params) + + // a connection that drops immediately shouldnt reset backoff + // this to avoid reconnect storms + if updated || time.Since(connectedAt) >= minHealthyConn { + backoff = 0 + continue + } + if !sleepBackoff() { + return fmt.Errorf("%w: %w", ErrConnFailure, connErr) + } + } +} + +func (c *Client) handleConn(ctx context.Context, conn *websocket.Conn, process func(ctx context.Context, cr *cbg.CborReader) error) error { + logger := cmp.Or(c.Logger, slog.Default().With("system", "events")) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + t := time.NewTicker(time.Second * 30) + defer t.Stop() + failcount := 0 + + for { + + select { + case <-t.C: + if err := conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(time.Second*10)); err != nil { + logger.Warn("failed to ping", "err", err) + failcount++ + if failcount >= 4 { + logger.Error("too many ping fails", "count", failcount) + conn.Close() + return + } + } else { + failcount = 0 // ok ping + } + case <-ctx.Done(): + conn.Close() + return + } + } + }() + + conn.SetPingHandler(func(message string) error { + err := conn.WriteControl(websocket.PongMessage, []byte(message), time.Now().Add(time.Second*60)) + if err == websocket.ErrCloseSent { + return nil + } + return err + }) + + conn.SetPongHandler(func(_ string) error { + if err := conn.SetReadDeadline(time.Now().Add(time.Minute)); err != nil { + logger.Error("failed to set read deadline", "err", err) + } + + return nil + }) + + cr := new(cbg.CborReader) + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + mt, rawReader, err := conn.NextReader() + if err != nil { + return fmt.Errorf("conn err at read: %w", err) + } + + if mt != websocket.BinaryMessage { + return fmt.Errorf("expected binary message from subscription endpoint") + } + + cr.SetReader(rawReader) + + if err := process(ctx, cr); err != nil { + return err + } + } +} diff --git a/lexutil/client_test.go b/lexutil/client_test.go new file mode 100644 index 0000000000000000000000000000000000000000..b33840a5e820d3813bf0198d537ec617f42de654 --- /dev/null +++ b/lexutil/client_test.go @@ -0,0 +1,700 @@ +package lexutil + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/bluesky-social/indigo/events" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "github.com/gorilla/websocket" + cid "github.com/ipfs/go-cid" + "github.com/stretchr/testify/assert" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +type SubscribeExample_Foo struct { + Seq int64 + Foo string +} + +type SubscribeExample_Bar struct { + Seq int64 + Bar string +} + +type SubscribeExample_Event struct { + Error *events.ErrorFrame + Foo *SubscribeExample_Foo + Bar *SubscribeExample_Bar +} + +func (e *SubscribeExample_Event) seq() int64 { + switch { + case e.Foo != nil: + return e.Foo.Seq + case e.Bar != nil: + return e.Bar.Seq + default: + return 0 + } +} + +func (e *SubscribeExample_Event) Serialize(w io.Writer) error { + cw := cbg.NewCborWriter(w) + header := events.EventHeader{Op: events.EvtKindMessage} + + switch { + case e.Error != nil: + header.Op = events.EvtKindErrorFrame + if err := header.MarshalCBOR(cw); err != nil { + return err + } + return e.Error.MarshalCBOR(cw) + case e.Foo != nil: + header.MsgType = "#foo" + if err := header.MarshalCBOR(cw); err != nil { + return err + } + return e.Foo.MarshalCBOR(cw) + case e.Bar != nil: + header.MsgType = "#bar" + if err := header.MarshalCBOR(cw); err != nil { + return err + } + return e.Bar.MarshalCBOR(cw) + default: + return fmt.Errorf("unrecognized event kind") + } +} + +func (e *SubscribeExample_Event) Deserialize(r io.Reader) error { + var header events.EventHeader + if err := header.UnmarshalCBOR(r); err != nil { + return fmt.Errorf("reading header: %w", err) + } + switch header.Op { + case events.EvtKindMessage: + switch header.MsgType { + case "#foo": + var evt SubscribeExample_Foo + if err := evt.UnmarshalCBOR(r); err != nil { + return err + } + e.Foo = &evt + case "#bar": + var evt SubscribeExample_Bar + if err := evt.UnmarshalCBOR(r); err != nil { + return err + } + e.Bar = &evt + default: + return fmt.Errorf("unknown message type: %s", header.MsgType) + } + case events.EvtKindErrorFrame: + var errframe events.ErrorFrame + if err := errframe.UnmarshalCBOR(r); err != nil { + return err + } + e.Error = &errframe + default: + return fmt.Errorf("unrecognized event stream type: %d", header.Op) + } + return nil +} + +func (t *SubscribeExample_Foo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Foo (string) (string) + if len("foo") > 1000000 { + return xerrors.Errorf("Value in field \"foo\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("foo"))); err != nil { + return err + } + if _, err := cw.WriteString(string("foo")); err != nil { + return err + } + + if len(t.Foo) > 1000000 { + return xerrors.Errorf("Value in field t.Foo was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Foo))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Foo)); err != nil { + return err + } + + // t.Seq (int64) (int64) + if len("seq") > 1000000 { + return xerrors.Errorf("Value in field \"seq\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("seq"))); err != nil { + return err + } + if _, err := cw.WriteString(string("seq")); err != nil { + return err + } + + if t.Seq >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Seq)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Seq-1)); err != nil { + return err + } + } + return nil +} + +func (t *SubscribeExample_Foo) UnmarshalCBOR(r io.Reader) (err error) { + *t = SubscribeExample_Foo{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SubscribeExample_Foo: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 8) + for range n { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Foo (string) (string) + case "foo": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Foo = string(sval) + } + // t.Seq (int64) (int64) + case "seq": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Seq = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} + +func (t *SubscribeExample_Bar) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Bar (string) (string) + if len("bar") > 1000000 { + return xerrors.Errorf("Value in field \"bar\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("bar"))); err != nil { + return err + } + if _, err := cw.WriteString(string("bar")); err != nil { + return err + } + + if len(t.Bar) > 1000000 { + return xerrors.Errorf("Value in field t.Bar was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Bar))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Bar)); err != nil { + return err + } + + // t.Seq (int64) (int64) + if len("seq") > 1000000 { + return xerrors.Errorf("Value in field \"seq\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("seq"))); err != nil { + return err + } + if _, err := cw.WriteString(string("seq")); err != nil { + return err + } + + if t.Seq >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Seq)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Seq-1)); err != nil { + return err + } + } + return nil +} + +func (t *SubscribeExample_Bar) UnmarshalCBOR(r io.Reader) (err error) { + *t = SubscribeExample_Bar{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SubscribeExample_Bar: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 8) + for range n { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Bar (string) (string) + case "bar": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Bar = string(sval) + } + // t.Seq (int64) (int64) + case "seq": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Seq = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} + +type testScheduler struct { + ch chan *SubscribeExample_Event + lastSeq atomic.Int64 +} + +var _ SeqScheduler[SubscribeExample_Event] = (*testScheduler)(nil) + +func newTestScheduler() *testScheduler { + return &testScheduler{ch: make(chan *SubscribeExample_Event, 8)} +} + +func (s *testScheduler) AddWork(ctx context.Context, namespace string, val *SubscribeExample_Event) error { + s.lastSeq.Store(val.seq()) + select { + case s.ch <- val: + case <-ctx.Done(): + } + return nil +} + +func (s *testScheduler) Shutdown() {} + +func (s *testScheduler) LastSeq() int64 { return s.lastSeq.Load() } + +// recv returns the next scheduled event. The timeout is generous because a +// redial during the offline window backs off >=5s. +func (s *testScheduler) recv(t *testing.T) *SubscribeExample_Event { + t.Helper() + select { + case e := <-s.ch: + return e + case <-time.After(12 * time.Second): + t.Fatal("timed out waiting for an event") + return nil + } +} + +type testRedialer struct { + sched *testScheduler +} + +var _ Redialer = (*testRedialer)(nil) + +func (r *testRedialer) Process(ctx context.Context, cr *cbg.CborReader) error { + var evt SubscribeExample_Event + if err := evt.Deserialize(cr); err != nil { + return err + } + return r.sched.AddWork(ctx, "", &evt) +} + +func (r *testRedialer) UpdateParams(ctx context.Context, params map[string]any) bool { + last := r.sched.LastSeq() + if last == 0 { + return false + } + params["cursor"] = last + return true +} + +const testEndpoint = "com.example.subscribeExample" + +// testServer live-tails an append-only event log over a websocket: each +// connection streams events whose seq exceeds the requested cursor, including +// ones added after it opened. The httptest listener stays bound the whole time +// (so the port can't be taken over); Close/Start toggle offline by having the +// handler answer 404, which fails the client's websocket handshake and drives +// it into its redial/backoff loop. +type testServer struct { + URL string + + mu sync.Mutex + events []SubscribeExample_Event + conns map[*websocket.Conn]struct{} + serving bool +} + +func newTestServer(t *testing.T) *testServer { + t.Helper() + ts := &testServer{ + conns: make(map[*websocket.Conn]struct{}), + serving: true, + } + srv := httptest.NewServer(ts.handler()) + ts.URL = srv.URL + t.Cleanup(func() { + ts.Close() // drop hijacked conns before srv.Close so it won't block + srv.Close() + }) + return ts +} + +func (ts *testServer) handler() http.Handler { + up := websocket.Upgrader{} + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/"+testEndpoint, func(w http.ResponseWriter, r *http.Request) { + if !ts.isServing() { + http.Error(w, "offline", http.StatusNotFound) + return + } + + var cursor int64 + if c := r.URL.Query().Get("cursor"); c != "" { + cursor, _ = strconv.ParseInt(c, 10, 64) + } + + conn, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + + ts.mu.Lock() + ts.conns[conn] = struct{}{} + ts.mu.Unlock() + defer func() { + ts.mu.Lock() + delete(ts.conns, conn) + ts.mu.Unlock() + conn.Close() + }() + + sent := cursor + for { + ts.mu.Lock() + var batch []SubscribeExample_Event + for _, e := range ts.events { + if e.seq() > sent { + batch = append(batch, e) + } + } + ts.mu.Unlock() + + for i := range batch { + if err := func(conn *websocket.Conn, e *SubscribeExample_Event) error { + var buf bytes.Buffer + if err := e.Serialize(&buf); err != nil { + return err + } + return conn.WriteMessage(websocket.BinaryMessage, buf.Bytes()) + }(conn, &batch[i]); err != nil { + return + } + sent = batch[i].seq() + } + time.Sleep(5 * time.Millisecond) + } + }) + return mux +} + +func (ts *testServer) isServing() bool { + ts.mu.Lock() + defer ts.mu.Unlock() + return ts.serving +} + +// Start brings the server back online. +func (ts *testServer) Start() { + ts.mu.Lock() + ts.serving = true + ts.mu.Unlock() +} + +// Close takes the server offline: new handshakes get 404 and active websocket +// connections are dropped, forcing the subscription to redial. +func (ts *testServer) Close() { + ts.mu.Lock() + ts.serving = false + conns := make([]*websocket.Conn, 0, len(ts.conns)) + for c := range ts.conns { + conns = append(conns, c) + } + ts.mu.Unlock() + + for _, c := range conns { + c.Close() + } +} + +func (ts *testServer) AddEvent(e SubscribeExample_Event) { + ts.mu.Lock() + ts.events = append(ts.events, e) + ts.mu.Unlock() +} + +func waitForReturn(t *testing.T, done <-chan error) { + t.Helper() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("subscription did not return after context cancel") + } +} + +func TestLexSubscribe_ConsumesAndSchedules(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + c := &Client{Client: indigoxrpc.Client{Host: srv.URL}} + sched := newTestScheduler() + + process := func(ctx context.Context, cr *cbg.CborReader) error { + var evt SubscribeExample_Event + if err := evt.Deserialize(cr); err != nil { + return err + } + return sched.AddWork(ctx, "", &evt) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- c.LexSubscribe(ctx, testEndpoint, map[string]any{"cursor": int64(0)}, process) + }() + + srv.AddEvent(SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 1, Foo: "foo-1"}}) + srv.AddEvent(SubscribeExample_Event{Bar: &SubscribeExample_Bar{Seq: 2, Bar: "bar-2"}}) + assert.Equal(t, &SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 1, Foo: "foo-1"}}, sched.recv(t)) + assert.Equal(t, &SubscribeExample_Event{Bar: &SubscribeExample_Bar{Seq: 2, Bar: "bar-2"}}, sched.recv(t)) + + srv.AddEvent(SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 3, Foo: "foo-3"}}) + assert.Equal(t, &SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 3, Foo: "foo-3"}}, sched.recv(t)) + + cancel() + waitForReturn(t, done) +} + +func TestLexSubscribeWithRedialer_ConsumesAndSchedules(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + c := &Client{Client: indigoxrpc.Client{Host: srv.URL}} + sched := newTestScheduler() + redialer := &testRedialer{sched: sched} + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- c.LexSubscribeWithRedialer(ctx, testEndpoint, map[string]any{"cursor": int64(0)}, redialer) + }() + + srv.AddEvent(SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 1, Foo: "foo-1"}}) + srv.AddEvent(SubscribeExample_Event{Bar: &SubscribeExample_Bar{Seq: 2, Bar: "bar-2"}}) + assert.Equal(t, &SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 1, Foo: "foo-1"}}, sched.recv(t)) + assert.Equal(t, &SubscribeExample_Event{Bar: &SubscribeExample_Bar{Seq: 2, Bar: "bar-2"}}, sched.recv(t)) + + srv.AddEvent(SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 3, Foo: "foo-3"}}) + srv.AddEvent(SubscribeExample_Event{Bar: &SubscribeExample_Bar{Seq: 4, Bar: "bar-4"}}) + assert.Equal(t, &SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 3, Foo: "foo-3"}}, sched.recv(t)) + assert.Equal(t, &SubscribeExample_Event{Bar: &SubscribeExample_Bar{Seq: 4, Bar: "bar-4"}}, sched.recv(t)) + + cancel() + waitForReturn(t, done) +} + +func TestLexSubscribeWithRedialer_HandlesDowntime(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + c := &Client{Client: indigoxrpc.Client{Host: srv.URL}} + sched := newTestScheduler() + redialer := &testRedialer{sched: sched} + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- c.LexSubscribeWithRedialer(ctx, testEndpoint, map[string]any{"cursor": int64(0)}, redialer) + }() + + srv.AddEvent(SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 1, Foo: "foo-1"}}) + srv.AddEvent(SubscribeExample_Event{Bar: &SubscribeExample_Bar{Seq: 2, Bar: "bar-2"}}) + assert.Equal(t, &SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 1, Foo: "foo-1"}}, sched.recv(t)) + assert.Equal(t, &SubscribeExample_Event{Bar: &SubscribeExample_Bar{Seq: 2, Bar: "bar-2"}}, sched.recv(t)) + + // offline, add events, back online: the subscription redials and resumes + srv.Close() + + srv.AddEvent(SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 3, Foo: "foo-3"}}) + srv.AddEvent(SubscribeExample_Event{Bar: &SubscribeExample_Bar{Seq: 4, Bar: "bar-4"}}) + + srv.Start() + + assert.Equal(t, &SubscribeExample_Event{Foo: &SubscribeExample_Foo{Seq: 3, Foo: "foo-3"}}, sched.recv(t)) + assert.Equal(t, &SubscribeExample_Event{Bar: &SubscribeExample_Bar{Seq: 4, Bar: "bar-4"}}, sched.recv(t)) + + cancel() + waitForReturn(t, done) +} diff --git a/lexutil/lexutil.go b/lexutil/lexutil.go new file mode 100644 index 0000000000000000000000000000000000000000..dae74c2e78619a2c4f705aa623d58777741833f8 --- /dev/null +++ b/lexutil/lexutil.go @@ -0,0 +1,49 @@ +// extended version of indigo/lex/util package before upstreaming it +package lexutil + +import ( + "context" + "errors" + "io" + + lexutil "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" +) + +type LexClient interface { + lexutil.LexClient + // LexSubscribe is basic event subscriber without redialing logic + // + // golang doesnt allow generics in method so we have to pass raw processFn here instead of Scheduler[T] + LexSubscribe(ctx context.Context, endpoint string, params map[string]any, process func(ctx context.Context, cr *cbg.CborReader) error) error +} + +const Subscription = "subscription" + +var ( + ErrDialFailure = errors.New("dialing failed") + ErrConnFailure = errors.New("connection failed") +) + +type EventStreamMessage interface { + Serialize(wc io.Writer) error + Deserialize(r io.Reader) error +} + +type Scheduler[T any] interface { + AddWork(ctx context.Context, namespace string, val *T) error + Shutdown() +} + +type SeqScheduler[T any] interface { + Scheduler[T] + LastSeq() int64 +} + +type Redialer interface { + // Process decodes the raw message and schedule it + Process(ctx context.Context, cr *cbg.CborReader) error + + // UpdateParams increments the cursor parameter based on LastSeq stored in internal scheduler + UpdateParams(ctx context.Context, params map[string]any) (updated bool) +} diff --git a/license b/license new file mode 100644 index 0000000000000000000000000000000000000000..df942ccb21f935abe166d8408b8015dab854ab5c --- /dev/null +++ b/license @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2025 Anirudh Oppiliappan, Akshay Oppiliappan and +contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/localinfra/cmd/deliberi-bootstrap/main.go b/localinfra/cmd/deliberi-bootstrap/main.go new file mode 100644 index 0000000000000000000000000000000000000000..3a35a4d5e7b81e3b585a1d7a457e16aab1915d58 --- /dev/null +++ b/localinfra/cmd/deliberi-bootstrap/main.go @@ -0,0 +1,128 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net/mail" + "os" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" + deldb "tangled.org/core/deliberi/db" +) + +const maxVerifiedEmailFixtures = 1000 + +type verifiedEmailFixture struct { + DID string `json:"did"` + Email string `json:"email"` +} + +func main() { + dbPath := flag.String("db-path", os.Getenv("DELIBERI_DB_PATH"), "path to the Deliberi SQLite database") + fixturePath := flag.String("file", "", "path to the verified email JSONL fixture") + flag.Parse() + if *dbPath == "" || *fixturePath == "" { + fmt.Fprintln(os.Stderr, "both --db-path and --file are required") + os.Exit(2) + } + if err := bootstrapVerifiedEmails(context.Background(), *dbPath, *fixturePath); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func parseVerifiedEmailFixtures(r io.Reader) ([]verifiedEmailFixture, error) { + scanner := bufio.NewScanner(r) + fixtures := make([]verifiedEmailFixture, 0) + seenEmails := make(map[string]string) + for lineNumber := 1; scanner.Scan(); lineNumber++ { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + if len(fixtures) >= maxVerifiedEmailFixtures { + return nil, fmt.Errorf("verified email fixture contains more than %d entries", maxVerifiedEmailFixtures) + } + + var fixture verifiedEmailFixture + if err := json.Unmarshal([]byte(line), &fixture); err != nil { + return nil, fmt.Errorf("verified email fixture line %d: %w", lineNumber, err) + } + did, err := syntax.ParseDID(fixture.DID) + if err != nil { + return nil, fmt.Errorf("verified email fixture line %d has invalid DID: %w", lineNumber, err) + } + fixture.DID = did.String() + fixture.Email = strings.ToLower(strings.TrimSpace(fixture.Email)) + address, err := mail.ParseAddress(fixture.Email) + if err != nil || address.Name != "" || address.Address != fixture.Email { + return nil, fmt.Errorf("verified email fixture line %d has invalid email address", lineNumber) + } + if owner, ok := seenEmails[fixture.Email]; ok { + if owner != fixture.DID { + return nil, fmt.Errorf("verified email fixture assigns %s to both %s and %s", fixture.Email, owner, fixture.DID) + } + continue + } + seenEmails[fixture.Email] = fixture.DID + fixtures = append(fixtures, fixture) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("reading verified email fixture: %w", err) + } + if len(fixtures) == 0 { + return nil, fmt.Errorf("verified email fixture contains no entries") + } + return fixtures, nil +} + +func bootstrapVerifiedEmails(ctx context.Context, dbPath, fixturePath string) error { + file, err := os.Open(fixturePath) + if err != nil { + return fmt.Errorf("opening verified email fixture: %w", err) + } + defer file.Close() + fixtures, err := parseVerifiedEmailFixtures(file) + if err != nil { + return err + } + + database, err := deldb.Make(ctx, dbPath) + if err != nil { + return fmt.Errorf("initializing db: %w", err) + } + defer database.Close() + tx, err := database.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("starting verified email fixture transaction: %w", err) + } + defer tx.Rollback() + + for _, fixture := range fixtures { + if _, err := tx.Exec(` + insert into emails (did, email, verified, is_primary, verification_code) + values (?, ?, true, false, '') + on conflict(did, email) do update set verified = true, verification_code = ''`, + fixture.DID, fixture.Email); err != nil { + return fmt.Errorf("upserting %s for %s: %w", fixture.Email, fixture.DID, err) + } + var hasPrimary bool + if err := tx.QueryRow(`select exists(select 1 from emails where did = ? and verified = true and is_primary = true)`, fixture.DID).Scan(&hasPrimary); err != nil { + return fmt.Errorf("checking primary email for %s: %w", fixture.DID, err) + } + if !hasPrimary { + if err := deldb.MakeEmailPrimary(tx, fixture.DID, fixture.Email); err != nil { + return fmt.Errorf("making %s primary for %s: %w", fixture.Email, fixture.DID, err) + } + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("committing verified email fixtures: %w", err) + } + return nil +} diff --git a/localinfra/cmd/deliberi-bootstrap/main_test.go b/localinfra/cmd/deliberi-bootstrap/main_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6455e282fca538dd0468ca19c8eda59c1ec00f70 --- /dev/null +++ b/localinfra/cmd/deliberi-bootstrap/main_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "os" + "strings" + "testing" + + deldb "tangled.org/core/deliberi/db" +) + +func TestParseVerifiedEmailFixtures(t *testing.T) { + fixtures, err := parseVerifiedEmailFixtures(strings.NewReader(` +{"did":"did:plc:alicefixture","email":" ALICE@Example.Test "} +{"did":"did:plc:bobfixture","email":"bob@example.test"} +`)) + if err != nil { + t.Fatal(err) + } + if len(fixtures) != 2 { + t.Fatalf("len(fixtures) = %d, want 2", len(fixtures)) + } + if got, want := fixtures[0].Email, "alice@example.test"; got != want { + t.Fatalf("email = %q, want %q", got, want) + } +} + +func TestParseVerifiedEmailFixturesRejectsConflictingOwners(t *testing.T) { + _, err := parseVerifiedEmailFixtures(strings.NewReader(` +{"did":"did:plc:alicefixture","email":"shared@example.test"} +{"did":"did:plc:bobfixture","email":"shared@example.test"} +`)) + if err == nil || !strings.Contains(err.Error(), "assigns shared@example.test to both") { + t.Fatalf("error = %v, want conflicting owner error", err) + } +} + +func TestBootstrapVerifiedEmailsIsIdempotent(t *testing.T) { + ctx := context.Background() + dbPath := t.TempDir() + "/deliberi.db" + fixturePath := t.TempDir() + "/verified-emails.jsonl" + fixture := `{"did":"did:plc:alicefixture","email":"alice@example.test"} +{"did":"did:plc:bobfixture","email":"bob@example.test"} +` + if err := writeFixture(fixturePath, fixture); err != nil { + t.Fatal(err) + } + + for range 2 { + if err := bootstrapVerifiedEmails(ctx, dbPath, fixturePath); err != nil { + t.Fatal(err) + } + } + + database, err := deldb.Make(ctx, dbPath) + if err != nil { + t.Fatal(err) + } + defer database.Close() + for _, did := range []string{"did:plc:alicefixture", "did:plc:bobfixture"} { + addresses, err := deldb.GetVerifiedEmailAddresses(database, did) + if err != nil { + t.Fatal(err) + } + if len(addresses) != 1 { + t.Fatalf("verified addresses for %s = %v, want one", did, addresses) + } + } + var count int + if err := database.QueryRow(`select count(*) from emails`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatalf("email count = %d, want 2", count) + } +} + +func TestBootstrapVerifiedEmailsMarksExistingAddressVerified(t *testing.T) { + ctx := context.Background() + dbPath := t.TempDir() + "/deliberi.db" + database, err := deldb.Make(ctx, dbPath) + if err != nil { + t.Fatal(err) + } + if err := deldb.InsertUnverifiedEmail(database, "did:plc:alicefixture", "other@example.test", "other-token"); err != nil { + t.Fatal(err) + } + if err := deldb.InsertUnverifiedEmail(database, "did:plc:alicefixture", "alice@example.test", "fixture-token"); err != nil { + t.Fatal(err) + } + if err := database.Close(); err != nil { + t.Fatal(err) + } + + fixturePath := t.TempDir() + "/verified-emails.jsonl" + if err := writeFixture(fixturePath, `{"did":"did:plc:alicefixture","email":"alice@example.test"}`); err != nil { + t.Fatal(err) + } + if err := bootstrapVerifiedEmails(ctx, dbPath, fixturePath); err != nil { + t.Fatal(err) + } + + database, err = deldb.Make(ctx, dbPath) + if err != nil { + t.Fatal(err) + } + defer database.Close() + email, err := deldb.GetEmail(database, "did:plc:alicefixture", "alice@example.test") + if err != nil { + t.Fatal(err) + } + if !email.Verified { + t.Fatal("existing fixture email is not verified") + } + if !email.Primary { + t.Fatal("first verified fixture email is not primary") + } + if email.VerificationCode != "" { + t.Fatalf("verification code = %q, want empty", email.VerificationCode) + } +} + +func writeFixture(path, contents string) error { + return os.WriteFile(path, []byte(contents), 0o600) +} diff --git a/localinfra/cmd/seed/main.go b/localinfra/cmd/seed/main.go new file mode 100644 index 0000000000000000000000000000000000000000..b9c1254b7973d346cfbb57a03471d6bad7669098 --- /dev/null +++ b/localinfra/cmd/seed/main.go @@ -0,0 +1,443 @@ +package main + +import ( + "bytes" + "context" + "crypto" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "net/http" + "net/url" + "os" + "slices" + "time" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/auth" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/did-method-plc/go-didplc/didplc" + "gitlab.com/yawning/secp256k1-voi/secec" +) + +const usage = `usage: + seed account + seed repo-genesis + seed repo-handover + seed repo-lend + seed repo-return + seed labeler ` + +type env struct { + pdsURL string + pdsHostname string + adminPassword string + password string + knotURL string + plc *didplc.Client + signer *secec.PrivateKey + atKey atcrypto.PrivateKey + didKey string +} + +func main() { + if len(os.Args) != 3 && len(os.Args) != 4 { + fmt.Fprintln(os.Stderr, usage) + os.Exit(2) + } + e, err := loadEnv() + if err == nil { + ctx := context.Background() + a, b := os.Args[2], "" + if len(os.Args) == 4 { + b = os.Args[3] + } + switch os.Args[1] { + case "account": + err = e.account(ctx, a, b) + case "repo-genesis": + err = e.repoGenesis(ctx, a, b) + case "repo-handover": + err = e.repoHandover(ctx, a, b, true) + case "repo-lend": + err = e.repoHandover(ctx, a, b, false) + case "repo-return": + err = e.repoReturn(ctx, a) + case "labeler": + err = e.labeler(ctx, a, b) + default: + fmt.Fprintln(os.Stderr, usage) + os.Exit(2) + } + } + if err != nil { + fmt.Fprintln(os.Stderr, "seed:", err) + os.Exit(1) + } +} + +func loadEnv() (*env, error) { + vars := map[string]string{} + for _, name := range []string{"PDS_URL", "PDS_HOSTNAME", "PDS_ADMIN_PASSWORD", "PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX", "PLC_URL", "KNOT_HOSTNAME"} { + vars[name] = os.Getenv(name) + if vars[name] == "" { + return nil, fmt.Errorf("%s must be set", name) + } + } + raw, err := hex.DecodeString(vars["PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX"]) + if err != nil { + return nil, fmt.Errorf("rotation key: %w", err) + } + signer, err := secec.NewPrivateKey(raw) + if err != nil { + return nil, fmt.Errorf("rotation key: %w", err) + } + atKey, err := atcrypto.ParsePrivateBytesK256(raw) + if err != nil { + return nil, fmt.Errorf("rotation key: %w", err) + } + pub, err := atKey.PublicKey() + if err != nil { + return nil, err + } + password := os.Getenv("PASSWORD") + if password == "" { + password = "password" + } + return &env{ + pdsURL: vars["PDS_URL"], + pdsHostname: vars["PDS_HOSTNAME"], + adminPassword: vars["PDS_ADMIN_PASSWORD"], + password: password, + knotURL: "https://" + vars["KNOT_HOSTNAME"], + plc: &didplc.Client{DirectoryURL: vars["PLC_URL"]}, + signer: signer, + atKey: atKey, + didKey: pub.DIDKey(), + }, nil +} + +func (e *env) account(ctx context.Context, user, expected string) error { + handle := user + "." + e.pdsHostname + genesis := didplc.RegularOp{ + Type: "plc_operation", + RotationKeys: []string{e.didKey}, + VerificationMethods: map[string]string{"atproto": e.didKey}, + AlsoKnownAs: []string{"at://" + handle}, + Services: map[string]didplc.OpService{ + "atproto_pds": {Type: "AtprotoPersonalDataServer", Endpoint: "https://" + e.pdsHostname}, + }, + } + did, err := e.signGenesis(&genesis, expected) + if err != nil { + return err + } + + var resolved struct { + DID string `json:"did"` + } + err = e.xrpc(ctx, http.MethodGet, "com.atproto.identity.resolveHandle?handle="+url.QueryEscape(handle), "", nil, &resolved) + var status *statusError + switch { + case err == nil && resolved.DID != did: + return fmt.Errorf("%s already resolves to %s, expected %s", handle, resolved.DID, did) + case err == nil: + fmt.Fprintf(os.Stderr, "[skip] %s = %s\n", handle, did) + fmt.Printf("%s\t%s\n", user, did) + return nil + case !errors.As(err, &status) || status.code != http.StatusBadRequest: + return err + } + + if err := e.submitGenesis(ctx, did, &genesis); err != nil { + return err + } + + var invite struct { + Code string `json:"code"` + } + admin := "Basic " + base64.StdEncoding.EncodeToString([]byte("admin:"+e.adminPassword)) + if err := e.xrpc(ctx, http.MethodPost, "com.atproto.server.createInviteCode", admin, map[string]any{"useCount": 1}, &invite); err != nil { + return err + } + + lxm := syntax.NSID("com.atproto.server.createAccount") + token, err := auth.SignServiceAuth(syntax.DID(did), "did:web:"+e.pdsHostname, time.Minute, &lxm, e.atKey) + if err != nil { + return err + } + var session struct { + AccessJwt string `json:"accessJwt"` + } + if err := e.xrpc(ctx, http.MethodPost, "com.atproto.server.createAccount", "Bearer "+token, map[string]string{ + "did": did, + "handle": handle, + "email": user + "@" + e.pdsHostname, + "password": e.password, + "inviteCode": invite.Code, + }, &session); err != nil { + return err + } + bearer := "Bearer " + session.AccessJwt + + var creds struct { + RotationKeys []string `json:"rotationKeys"` + AlsoKnownAs []string `json:"alsoKnownAs"` + VerificationMethods map[string]string `json:"verificationMethods"` + Services map[string]didplc.OpService `json:"services"` + } + if err := e.xrpc(ctx, http.MethodGet, "com.atproto.identity.getRecommendedDidCredentials", bearer, nil, &creds); err != nil { + return err + } + if err := e.submitUpdate(ctx, did, func(last *didplc.RegularOp) didplc.RegularOp { + return didplc.RegularOp{ + Type: "plc_operation", + RotationKeys: creds.RotationKeys, + VerificationMethods: creds.VerificationMethods, + AlsoKnownAs: creds.AlsoKnownAs, + Services: creds.Services, + } + }); err != nil { + return err + } + + if err := e.xrpc(ctx, http.MethodPost, "com.atproto.server.activateAccount", bearer, nil, nil); err != nil { + return err + } + fmt.Fprintf(os.Stderr, "[create] %s = %s (password: %s)\n", handle, did, e.password) + fmt.Printf("%s\t%s\n", user, did) + return nil +} + +func (e *env) repoGenesis(ctx context.Context, tag, expected string) error { + genesis := didplc.RegularOp{ + Type: "plc_operation", + RotationKeys: []string{e.didKey}, + VerificationMethods: map[string]string{}, + AlsoKnownAs: []string{}, + Services: map[string]didplc.OpService{ + "tangled_knot": {Type: "TangledKnot", Endpoint: e.knotURL + "/repo/" + tag}, + }, + } + did, err := e.signGenesis(&genesis, expected) + if err != nil { + return err + } + if err := e.submitGenesis(ctx, did, &genesis); err != nil { + return err + } + fmt.Println(did) + return nil +} + +// repoHandover points #atproto and #atproto_pds at the knot, which is what org.tangled.repo.createRepo +// checks. With rotate it also hands the knot the rotation key for good; without (repo-lend) the +// seed keeps it, so repo-return can take the account back once the knot has created the repo. +func (e *env) repoHandover(ctx context.Context, did, knotKey string, rotate bool) error { + knotDIDKey := "did:key:" + knotKey + return e.submitUpdate(ctx, did, func(last *didplc.RegularOp) didplc.RegularOp { + rotation := last.RotationKeys + if rotate { + rotation = []string{knotDIDKey} + } + if slices.Equal(last.RotationKeys, rotation) && last.VerificationMethods["atproto"] == knotDIDKey && + last.Services["atproto_pds"].Endpoint == e.knotURL { + return didplc.RegularOp{} + } + methods := maps.Clone(last.VerificationMethods) + methods["atproto"] = knotDIDKey + services := maps.Clone(last.Services) + services["atproto_pds"] = didplc.OpService{Type: "AtprotoPersonalDataServer", Endpoint: e.knotURL} + return didplc.RegularOp{ + Type: "plc_operation", + RotationKeys: rotation, + VerificationMethods: methods, + AlsoKnownAs: last.AlsoKnownAs, + Services: services, + } + }) +} + +// repoReturn hands a lent repo DID back to its pds account: records stay on the pds, and +// #tangled_knot tells git clients where the repository lives. The knot keeps no rotation key, +// so its PLC sweeper gives up on the DID instead of pulling #atproto_pds back. +func (e *env) repoReturn(ctx context.Context, did string) error { + var session struct { + AccessJwt string `json:"accessJwt"` + } + if err := e.xrpc(ctx, http.MethodPost, "com.atproto.server.createSession", "", map[string]string{ + "identifier": did, + "password": e.password, + }, &session); err != nil { + return err + } + var creds struct { + RotationKeys []string `json:"rotationKeys"` + AlsoKnownAs []string `json:"alsoKnownAs"` + VerificationMethods map[string]string `json:"verificationMethods"` + Services map[string]didplc.OpService `json:"services"` + } + if err := e.xrpc(ctx, http.MethodGet, "com.atproto.identity.getRecommendedDidCredentials", "Bearer "+session.AccessJwt, nil, &creds); err != nil { + return err + } + creds.Services["tangled_knot"] = didplc.OpService{Type: "TangledKnot", Endpoint: e.knotURL} + return e.submitUpdate(ctx, did, func(last *didplc.RegularOp) didplc.RegularOp { + if slices.Equal(last.RotationKeys, creds.RotationKeys) && + maps.Equal(last.VerificationMethods, creds.VerificationMethods) && + maps.Equal(last.Services, creds.Services) { + return didplc.RegularOp{} + } + return didplc.RegularOp{ + Type: "plc_operation", + RotationKeys: creds.RotationKeys, + VerificationMethods: creds.VerificationMethods, + AlsoKnownAs: last.AlsoKnownAs, + Services: creds.Services, + } + }) +} + +// labeler makes did an ozone labeler: #atproto_label is the key ozone signs labels with +// (OZONE_SIGNING_KEY_HEX), #atproto_labeler is where its subscribeLabels stream lives. +// The account's rotation key is the pds one the seed signs with, so no email token is needed. +func (e *env) labeler(ctx context.Context, did, endpoint string) error { + raw, err := hex.DecodeString(os.Getenv("OZONE_SIGNING_KEY_HEX")) + if err != nil || len(raw) == 0 { + return fmt.Errorf("OZONE_SIGNING_KEY_HEX must be a hex k256 private key") + } + key, err := atcrypto.ParsePrivateBytesK256(raw) + if err != nil { + return fmt.Errorf("OZONE_SIGNING_KEY_HEX: %w", err) + } + pub, err := key.PublicKey() + if err != nil { + return err + } + labelKey := pub.DIDKey() + if err := e.submitUpdate(ctx, did, func(last *didplc.RegularOp) didplc.RegularOp { + if last.VerificationMethods["atproto_label"] == labelKey && + last.Services["atproto_labeler"].Endpoint == endpoint { + return didplc.RegularOp{} + } + methods := maps.Clone(last.VerificationMethods) + methods["atproto_label"] = labelKey + services := maps.Clone(last.Services) + services["atproto_labeler"] = didplc.OpService{Type: "AtprotoLabeler", Endpoint: endpoint} + return didplc.RegularOp{ + Type: "plc_operation", + RotationKeys: last.RotationKeys, + VerificationMethods: methods, + AlsoKnownAs: last.AlsoKnownAs, + Services: services, + } + }); err != nil { + return err + } + fmt.Fprintf(os.Stderr, "[labeler] %s → %s (label key %s)\n", did, endpoint, labelKey) + return nil +} + +func (e *env) sign(op *didplc.RegularOp) error { + hash := sha256.Sum256(op.UnsignedCBORBytes()) + sig, err := e.signer.Sign(secec.RFC6979SHA256(), hash[:], &secec.ECDSAOptions{Hash: crypto.SHA256, Encoding: secec.EncodingCompact}) + if err != nil { + return err + } + encoded := base64.RawURLEncoding.EncodeToString(sig) + op.Sig = &encoded + return nil +} + +func (e *env) signGenesis(op *didplc.RegularOp, expected string) (string, error) { + if err := e.sign(op); err != nil { + return "", err + } + did, err := op.DID() + if err != nil { + return "", err + } + if did != expected { + return "", fmt.Errorf("genesis DID is %s, expected %s", did, expected) + } + return did, nil +} + +func (e *env) submitGenesis(ctx context.Context, did string, op *didplc.RegularOp) error { + _, err := e.plc.Resolve(ctx, did) + if errors.Is(err, didplc.ErrDIDNotFound) { + return e.plc.Submit(ctx, did, op) + } + return err +} + +func (e *env) submitUpdate(ctx context.Context, did string, next func(last *didplc.RegularOp) didplc.RegularOp) error { + log, err := e.plc.OpLog(ctx, did) + if err != nil { + return err + } + last := log[len(log)-1].Regular + if last == nil { + return fmt.Errorf("%s: last PLC operation isn't a plc_operation", did) + } + op := next(last) + if op.Type == "" { + return nil + } + prev := last.CID().String() + op.Prev = &prev + if err := e.sign(&op); err != nil { + return err + } + return e.plc.Submit(ctx, did, &op) +} + +type statusError struct { + code int + body string +} + +func (s *statusError) Error() string { + return fmt.Sprintf("HTTP %d: %s", s.code, s.body) +} + +func (e *env) xrpc(ctx context.Context, method, path, authorization string, in, out any) error { + var body io.Reader + if in != nil { + encoded, err := json.Marshal(in) + if err != nil { + return err + } + body = bytes.NewReader(encoded) + } + req, err := http.NewRequestWithContext(ctx, method, e.pdsURL+"/xrpc/"+path, body) + if err != nil { + return err + } + if in != nil { + req.Header.Set("Content-Type", "application/json") + } + if authorization != "" { + req.Header.Set("Authorization", authorization) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s: %w", path, &statusError{code: resp.StatusCode, body: string(respBody)}) + } + if out == nil { + return nil + } + return json.Unmarshal(respBody, out) +} diff --git a/localinfra/fixtures/blobs/did:plc:igzuueb63dytyy3tpaweofuu/bafkreieify5bix4cmrj4iuxiceb6srr2boltkbbf7yizwtgwurqv42sltm b/localinfra/fixtures/blobs/did:plc:igzuueb63dytyy3tpaweofuu/bafkreieify5bix4cmrj4iuxiceb6srr2boltkbbf7yizwtgwurqv42sltm new file mode 100644 index 0000000000000000000000000000000000000000..2b031556185e90324ed29acc37aeb79eed5b572a --- /dev/null +++ b/localinfra/fixtures/blobs/did:plc:igzuueb63dytyy3tpaweofuu/bafkreieify5bix4cmrj4iuxiceb6srr2boltkbbf7yizwtgwurqv42sltm @@ -0,0 +1,16 @@ +# Local stack notes + +Things that bit me, in order: + +1. the dev CA has to be in the **system** trust store, not just the browser +2. `TANGLED_APPVIEW_HOST` must be a loopback IP, not `localhost` +3. the knot wants `/shared/owner-did` before it will start + +## Handy + +```sh +docker compose logs -f appview | grep ingest +``` + +Still unclear: how [spindle](https://tangled.org/tangled.org/core) picks up +`.tangled/workflows`. \ No newline at end of file diff --git a/localinfra/fixtures/blobs/did:plc:igzuueb63dytyy3tpaweofuu/bafkreifqrrwn3ioqkrquhibb3csu6i3kogd76uk6ktptc7s5exim6rfu6u b/localinfra/fixtures/blobs/did:plc:igzuueb63dytyy3tpaweofuu/bafkreifqrrwn3ioqkrquhibb3csu6i3kogd76uk6ktptc7s5exim6rfu6u new file mode 100644 index 0000000000000000000000000000000000000000..6082eefc42e3daf34e4e77dcf9e91402331c4d67 --- /dev/null +++ b/localinfra/fixtures/blobs/did:plc:igzuueb63dytyy3tpaweofuu/bafkreifqrrwn3ioqkrquhibb3csu6i3kogd76uk6ktptc7s5exim6rfu6u @@ -0,0 +1,22 @@ +package main + +import "fmt" + +func sieve(n int) []int { + composite := make([]bool, n+1) + var primes []int + for i := 2; i <= n; i++ { + if composite[i] { + continue + } + primes = append(primes, i) + for j := i * i; j <= n; j += i { + composite[j] = true + } + } + return primes +} + +func main() { + fmt.Println(sieve(50)) +} \ No newline at end of file diff --git a/localinfra/fixtures/blobs/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafkreiaovmar5rtz7dw35cfylzmb3htqg7kpksf76pvascxlv3s56rb7o4 b/localinfra/fixtures/blobs/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafkreiaovmar5rtz7dw35cfylzmb3htqg7kpksf76pvascxlv3s56rb7o4 new file mode 100644 index 0000000000000000000000000000000000000000..fc54f4343c68fb3f02b5946311f24ff5714ed411 --- /dev/null +++ b/localinfra/fixtures/blobs/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafkreiaovmar5rtz7dw35cfylzmb3htqg7kpksf76pvascxlv3s56rb7o4 @@ -0,0 +1,4 @@ +- [ ] figure out why the first ssh push races +- [x] get the local knot talking to the mirror +- [ ] write up the bootstrap order somewhere findable +- [ ] ask about symlinks in the tree view \ No newline at end of file diff --git a/localinfra/fixtures/records/did:plc:77uzwanh7ixsvttf6cvkoszj/org.tangled.git.refs/self/bafyreig6c2mmv2wctety2ngvjho6rx7dbvmlrbuzfybf5ygdxxqhpjdef4.json b/localinfra/fixtures/records/did:plc:77uzwanh7ixsvttf6cvkoszj/org.tangled.git.refs/self/bafyreig6c2mmv2wctety2ngvjho6rx7dbvmlrbuzfybf5ygdxxqhpjdef4.json new file mode 100644 index 0000000000000000000000000000000000000000..0f35c9002f2dc32757a741ea3e3c049623828b69 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:77uzwanh7ixsvttf6cvkoszj/org.tangled.git.refs/self/bafyreig6c2mmv2wctety2ngvjho6rx7dbvmlrbuzfybf5ygdxxqhpjdef4.json @@ -0,0 +1,7 @@ +{ + "$type": "org.tangled.git.refs", + "defaultBranch": "master", + "refs": { + "$bytes": "" + } +} diff --git a/localinfra/fixtures/records/did:plc:77uzwanh7ixsvttf6cvkoszj/org.tangled.repo.manifest/self/bafyreid3zlshhjzryrysr3egr3yutd26fsmq2kfdaibiccx7sq57qwvuty.json b/localinfra/fixtures/records/did:plc:77uzwanh7ixsvttf6cvkoszj/org.tangled.repo.manifest/self/bafyreid3zlshhjzryrysr3egr3yutd26fsmq2kfdaibiccx7sq57qwvuty.json new file mode 100644 index 0000000000000000000000000000000000000000..bc8c9cc537e19c5b6b880706aa8a2ade9a15fcfb --- /dev/null +++ b/localinfra/fixtures/records/did:plc:77uzwanh7ixsvttf6cvkoszj/org.tangled.repo.manifest/self/bafyreid3zlshhjzryrysr3egr3yutd26fsmq2kfdaibiccx7sq57qwvuty.json @@ -0,0 +1,9 @@ +{ + "$type": "org.tangled.repo.manifest", + "createdAt": "2025-09-22T11:14:35+01:00", + "declaration": { + "owner": "did:plc:xbserj4jfuhzkbl3mszkvvrb", + "slug": "empty-repo" + }, + "spindle": "did:web:spindle.tngl.boltless.dev" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3lzg6rlrui222/bafyreietwvvhovqaxq2llr35tg7fslxikhjfffulmovin3q6ngfbsyqkle.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3lzg6rlrui222/bafyreietwvvhovqaxq2llr35tg7fslxikhjfffulmovin3q6ngfbsyqkle.json new file mode 100644 index 0000000000000000000000000000000000000000..cc15325e0eba5ced23d5a4a3b788ecde9d3eca64 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3lzg6rlrui222/bafyreietwvvhovqaxq2llr35tg7fslxikhjfffulmovin3q6ngfbsyqkle.json @@ -0,0 +1,13 @@ +{ + "$type": "org.tangled.feed.comment", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "Worth calling out the ordering explicitly — I lost an afternoon to the knot\nexiting because `/shared/owner-did` wasn't there yet." + }, + "createdAt": "2025-09-22T10:21:35Z", + "subject": { + "cid": "bafyreig33fgtw2h2lwz7uex4yepqgq5kez64vm56trqlhsnrup37am6r2q", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/2" + }, + "x-tngl-editor": "did:plc:xbserj4jfuhzkbl3mszkvvrb" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3lzg6teywa222/bafyreigsfex4ezvor5tig2u2fvh7dwfxanrpu2t5a6iut2pf2h6tov5z7i.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3lzg6teywa222/bafyreigsfex4ezvor5tig2u2fvh7dwfxanrpu2t5a6iut2pf2h6tov5z7i.json new file mode 100644 index 0000000000000000000000000000000000000000..cf4f9637b1938e44e8a11aab8c185d44907dd14c --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3lzg6teywa222/bafyreigsfex4ezvor5tig2u2fvh7dwfxanrpu2t5a6iut2pf2h6tov5z7i.json @@ -0,0 +1,17 @@ +{ + "$type": "org.tangled.feed.comment", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "Agreed. The compose `depends_on` encodes it already, just nowhere a human would look." + }, + "createdAt": "2025-09-22T10:22:35Z", + "replyTo": { + "cid": "bafyreietwvvhovqaxq2llr35tg7fslxikhjfffulmovin3q6ngfbsyqkle", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3lzg6rlrui222" + }, + "subject": { + "cid": "bafyreig33fgtw2h2lwz7uex4yepqgq5kez64vm56trqlhsnrup37am6r2q", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/2" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqk2a4c22/bafyreibxtmildrcsuhjvijs2kbcotijs6qhu2rsbm3rleskancgxb5mk6y.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqk2a4c22/bafyreibxtmildrcsuhjvijs2kbcotijs6qhu2rsbm3rleskancgxb5mk6y.json new file mode 100644 index 0000000000000000000000000000000000000000..7a2e43e920c74fb634a848169e4b34d24f9aa686 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqk2a4c22/bafyreibxtmildrcsuhjvijs2kbcotijs6qhu2rsbm3rleskancgxb5mk6y.json @@ -0,0 +1,20 @@ +{ + "$type": "org.tangled.feed.comment", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "Splitting the user-owned and repo-owned records this early feels right, but `repo.declaration` naming the repo by DID means the slug lives in the rkey. Deliberate?" + }, + "createdAt": "2026-08-15T09:12:00.000Z", + "embed": { + "$type": "org.tangled.embed.commit", + "commit": { + "oid": "0e96abbc399191e105951f0fc3338b7936451ca5" + }, + "repo": "did:plc:i3cuazypmjja73m3ahwrsh2z" + }, + "subject": { + "cid": "bafyreidv3qgzq3mm53gfcxfep5ztun3tjfkhpm54odte3gjh2j6oiwko4y", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/8" + }, + "x-tngl-editor": "did:plc:xbserj4jfuhzkbl3mszkvvrb" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqm7b6d22/bafyreih3mcgo3rtzqxmf2inhabpmw4i4n7bev7wadlaoahfwnivogujupy.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqm7b6d22/bafyreih3mcgo3rtzqxmf2inhabpmw4i4n7bev7wadlaoahfwnivogujupy.json new file mode 100644 index 0000000000000000000000000000000000000000..a29ddc48122497eefebec32565cf5a2f55577b80 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqm7b6d22/bafyreih3mcgo3rtzqxmf2inhabpmw4i4n7bev7wadlaoahfwnivogujupy.json @@ -0,0 +1,24 @@ +{ + "$type": "org.tangled.feed.comment", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "Deliberate. The rkey is the slug so a rename is a record move, not a field edit." + }, + "createdAt": "2026-08-15T10:03:00.000Z", + "embed": { + "$type": "org.tangled.embed.commit", + "commit": { + "oid": "0e96abbc399191e105951f0fc3338b7936451ca5" + }, + "repo": "did:plc:i3cuazypmjja73m3ahwrsh2z" + }, + "replyTo": { + "cid": "bafyreibxtmildrcsuhjvijs2kbcotijs6qhu2rsbm3rleskancgxb5mk6y", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqk2a4c22" + }, + "subject": { + "cid": "bafyreidv3qgzq3mm53gfcxfep5ztun3tjfkhpm54odte3gjh2j6oiwko4y", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/8" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqp3c5e22/bafyreicbbu3wdenayqq4ubqflygph5msxghmuw7buyph3dbq6btlkwoi3y.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqp3c5e22/bafyreicbbu3wdenayqq4ubqflygph5msxghmuw7buyph3dbq6btlkwoi3y.json new file mode 100644 index 0000000000000000000000000000000000000000..3c5167bca43a41bc90ade71256ce4f4530d24e5e --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqp3c5e22/bafyreicbbu3wdenayqq4ubqflygph5msxghmuw7buyph3dbq6btlkwoi3y.json @@ -0,0 +1,20 @@ +{ + "$type": "org.tangled.feed.comment", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "The spindle secret lexicons landed in this round — worth a line in the description saying they are xrpc-only and never hit a repo." + }, + "createdAt": "2026-08-20T08:45:00.000Z", + "embed": { + "$type": "org.tangled.embed.commit", + "commit": { + "oid": "778fe6786af294b13c74ee45973cf6a8791dd0bb" + }, + "repo": "did:plc:i3cuazypmjja73m3ahwrsh2z" + }, + "subject": { + "cid": "bafyreidv3qgzq3mm53gfcxfep5ztun3tjfkhpm54odte3gjh2j6oiwko4y", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/8" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqr5d2f22/bafyreic2qpppocpszrr7nlysndsvhum5x3d57wllrxbelmrhgvp6lfvnli.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqr5d2f22/bafyreic2qpppocpszrr7nlysndsvhum5x3d57wllrxbelmrhgvp6lfvnli.json new file mode 100644 index 0000000000000000000000000000000000000000..72bf6c72be31823e0bba2588c2e51b6c6d751391 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mszlqr5d2f22/bafyreic2qpppocpszrr7nlysndsvhum5x3d57wllrxbelmrhgvp6lfvnli.json @@ -0,0 +1,20 @@ +{ + "$type": "org.tangled.feed.comment", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "Rebase looks clean. Only thing still open from my side is whether the knot xrpc lexicons block this or ride a follow-up." + }, + "createdAt": "2026-09-05T11:20:00.000Z", + "embed": { + "$type": "org.tangled.embed.commit", + "commit": { + "oid": "a30623cbe5ea1e0e945cc94d8e9529320fe25577" + }, + "repo": "did:plc:i3cuazypmjja73m3ahwrsh2z" + }, + "subject": { + "cid": "bafyreidv3qgzq3mm53gfcxfep5ztun3tjfkhpm54odte3gjh2j6oiwko4y", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/8" + }, + "x-tngl-editor": "did:plc:xbserj4jfuhzkbl3mszkvvrb" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mwkb2qu3gr22/bafyreiagzsjco3xpkp7wrek4sltctxu4ub4v6dtglfjwdlyfvnjc6ulmpi.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mwkb2qu3gr22/bafyreiagzsjco3xpkp7wrek4sltctxu4ub4v6dtglfjwdlyfvnjc6ulmpi.json new file mode 100644 index 0000000000000000000000000000000000000000..8c4aeb784178eb374b7a1ad326a081bc2a67ad42 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3mwkb2qu3gr22/bafyreiagzsjco3xpkp7wrek4sltctxu4ub4v6dtglfjwdlyfvnjc6ulmpi.json @@ -0,0 +1,13 @@ +{ + "$type": "org.tangled.feed.comment", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "wouldn't it be better to have the root be backed by disk instead? RAM is already constrained in CI" + }, + "createdAt": "2025-09-22T10:41:35Z", + "subject": { + "cid": "bafyreiedbneftlnpu47xjsqkxtncjxoe324kpklmdd4qnttvuptadfjh2y", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/4" + }, + "x-tngl-editor": "did:plc:xbserj4jfuhzkbl3mszkvvrb" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg6yqo3i222/bafyreibmpb4deiyux2bm2m5j7ybz2ufvgnszzgnwtd66d3qtoenla7ohma.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg6yqo3i222/bafyreibmpb4deiyux2bm2m5j7ybz2ufvgnszzgnwtd66d3qtoenla7ohma.json new file mode 100644 index 0000000000000000000000000000000000000000..4fb92aeadaca9d7ed62f652ae3639d5091af262a --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg6yqo3i222/bafyreibmpb4deiyux2bm2m5j7ybz2ufvgnszzgnwtd66d3qtoenla7ohma.json @@ -0,0 +1,10 @@ +{ + "$type": "org.tangled.feed.reaction", + "createdAt": "2025-09-22T10:25:35Z", + "reaction": "🎉", + "subject": { + "cid": "bafyreig33fgtw2h2lwz7uex4yepqgq5kez64vm56trqlhsnrup37am6r2q", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/2" + }, + "x-tngl-editor": "did:plc:xbserj4jfuhzkbl3mszkvvrb" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg72jv5a222/bafyreib3rqmzxclxnqnkdwrzp7wcnndbp46g7acfhhque5i2y53ndek4rm.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg72jv5a222/bafyreib3rqmzxclxnqnkdwrzp7wcnndbp46g7acfhhque5i2y53ndek4rm.json new file mode 100644 index 0000000000000000000000000000000000000000..5deff1e26254890400646f4d2a4c62c1a2410142 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg72jv5a222/bafyreib3rqmzxclxnqnkdwrzp7wcnndbp46g7acfhhque5i2y53ndek4rm.json @@ -0,0 +1,10 @@ +{ + "$type": "org.tangled.feed.reaction", + "createdAt": "2025-09-22T10:26:35Z", + "reaction": "👀", + "subject": { + "cid": "bafyreig33fgtw2h2lwz7uex4yepqgq5kez64vm56trqlhsnrup37am6r2q", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/2" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg74d46y222/bafyreiau7g54kz5ftdos3r2kldwcjutvsvmdh7hptnx7bmwhprvoeb75fi.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg74d46y222/bafyreiau7g54kz5ftdos3r2kldwcjutvsvmdh7hptnx7bmwhprvoeb75fi.json new file mode 100644 index 0000000000000000000000000000000000000000..2f6c80253e6538a61a5ad95d56468670d277e1d5 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.reaction/3lzg74d46y222/bafyreiau7g54kz5ftdos3r2kldwcjutvsvmdh7hptnx7bmwhprvoeb75fi.json @@ -0,0 +1,10 @@ +{ + "$type": "org.tangled.feed.reaction", + "createdAt": "2025-09-22T10:27:35Z", + "reaction": "👍", + "subject": { + "cid": "bafyreietwvvhovqaxq2llr35tg7fslxikhjfffulmovin3q6ngfbsyqkle", + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.feed.comment/3lzg6rlrui222" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.git.refs/self/bafyreigkwkkfyqmuwvl5zx552ivpes3epirw6dpsvzo546kc4agy7r4gtm.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.git.refs/self/bafyreigkwkkfyqmuwvl5zx552ivpes3epirw6dpsvzo546kc4agy7r4gtm.json new file mode 100644 index 0000000000000000000000000000000000000000..2b083a97c9127cf2c903bbda7eec54847825e121 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.git.refs/self/bafyreigkwkkfyqmuwvl5zx552ivpes3epirw6dpsvzo546kc4agy7r4gtm.json @@ -0,0 +1,7 @@ +{ + "$type": "org.tangled.git.refs", + "defaultBranch": "master", + "refs": { + "$bytes": "MDExMTdmZGRkMzUwMmM1N2JjMjBhNjVhNzVhNjZhNDkwMGQ1ZDY3ZCByZWZzL2hlYWRzL21hc3Rlcgo5YTkyNWVmZWY2YTBlNmRmY2QyZDQzMTdiNGExZWVlODc1MjkyOGI4IHJlZnMvaGVhZHMvc2wvcmVmLWJhc2VkLXByCg" + } +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/documentation/bafyreif5mbys4b7mn4rquma3gqqnr3ldjgpspdkfqfu3feolsrftnxznne.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/documentation/bafyreif5mbys4b7mn4rquma3gqqnr3ldjgpspdkfqfu3feolsrftnxznne.json new file mode 100644 index 0000000000000000000000000000000000000000..31c94f8844c686a67f27a2a9a9d1cd0910884717 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/documentation/bafyreif5mbys4b7mn4rquma3gqqnr3ldjgpspdkfqfu3feolsrftnxznne.json @@ -0,0 +1,6 @@ +{ + "$type": "org.tangled.label.definition", + "color": "#06b6d4", + "createdAt": "2025-09-22T11:14:35+01:00", + "name": "documentation" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/duplicate/bafyreie224tuyp3mrrp5cosezrcmdxgutjctoresjsv3r4pfbwvlnqyre4.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/duplicate/bafyreie224tuyp3mrrp5cosezrcmdxgutjctoresjsv3r4pfbwvlnqyre4.json new file mode 100644 index 0000000000000000000000000000000000000000..ff77dc45f4d25f3e11f46ad98aa18a0132864931 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/duplicate/bafyreie224tuyp3mrrp5cosezrcmdxgutjctoresjsv3r4pfbwvlnqyre4.json @@ -0,0 +1,6 @@ +{ + "$type": "org.tangled.label.definition", + "color": "#ef4444", + "createdAt": "2025-09-22T11:14:35+01:00", + "name": "duplicate" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/good-first-issue/bafyreiguf6otpa3jlctujcnvfhqt6thqx76ol5sqmanzhkiru24kdgple4.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/good-first-issue/bafyreiguf6otpa3jlctujcnvfhqt6thqx76ol5sqmanzhkiru24kdgple4.json new file mode 100644 index 0000000000000000000000000000000000000000..f0be187cc0cdd04c177f62a3cf263fba9ab62a02 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/good-first-issue/bafyreiguf6otpa3jlctujcnvfhqt6thqx76ol5sqmanzhkiru24kdgple4.json @@ -0,0 +1,6 @@ +{ + "$type": "org.tangled.label.definition", + "color": "#8B5CF6", + "createdAt": "2025-09-22T11:14:35+01:00", + "name": "good-first-issue" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/wontfix/bafyreihhxjm2coiw242usm6hnid3i7ius6kdkkyph7afjc7jjp2fb7bz6e.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/wontfix/bafyreihhxjm2coiw242usm6hnid3i7ius6kdkkyph7afjc7jjp2fb7bz6e.json new file mode 100644 index 0000000000000000000000000000000000000000..84db58d732f534fed9bb369de18639f3ddb37755 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.label.definition/wontfix/bafyreihhxjm2coiw242usm6hnid3i7ius6kdkkyph7afjc7jjp2fb7bz6e.json @@ -0,0 +1,6 @@ +{ + "$type": "org.tangled.label.definition", + "color": "#64748b", + "createdAt": "2025-09-22T11:14:35+01:00", + "name": "wontfix" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.repo.manifest/self/bafyreie6djkwfaeyrpulc373tkf4hoimhfc5t2jdrmsgqkblalrcl52y7a.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.repo.manifest/self/bafyreie6djkwfaeyrpulc373tkf4hoimhfc5t2jdrmsgqkblalrcl52y7a.json new file mode 100644 index 0000000000000000000000000000000000000000..c589ba87d9cca96139db9f377401c1f9b93ab1dd --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.repo.manifest/self/bafyreie6djkwfaeyrpulc373tkf4hoimhfc5t2jdrmsgqkblalrcl52y7a.json @@ -0,0 +1,13 @@ +{ + "$type": "org.tangled.repo.manifest", + "createdAt": "2025-09-22T11:14:35+01:00", + "declaration": { + "owner": "did:plc:igzuueb63dytyy3tpaweofuu", + "slug": "core" + }, + "source": { + "$type": "org.tangled.repo.defs#sourceExternal", + "uri": "https://knot1.tangled.sh/did:plc:ioighzh4jnrhybayw65gfbia" + }, + "spindle": "did:web:spindle.tngl.boltless.dev" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreiarut2cqseyjfwefvssd4pnbyqjtzl5n4nc5gqbgpbsdekdf3mfwu/bafyreiarut2cqseyjfwefvssd4pnbyqjtzl5n4nc5gqbgpbsdekdf3mfwu.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreiarut2cqseyjfwefvssd4pnbyqjtzl5n4nc5gqbgpbsdekdf3mfwu/bafyreiarut2cqseyjfwefvssd4pnbyqjtzl5n4nc5gqbgpbsdekdf3mfwu.json new file mode 100644 index 0000000000000000000000000000000000000000..972cfdad842a7bd34d84ae14a506085d752bf383 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreiarut2cqseyjfwefvssd4pnbyqjtzl5n4nc5gqbgpbsdekdf3mfwu/bafyreiarut2cqseyjfwefvssd4pnbyqjtzl5n4nc5gqbgpbsdekdf3mfwu.json @@ -0,0 +1,12 @@ +{ + "$type": "org.tangled.review.patch", + "base": { + "oid": "01117fddd3502c57bc20a65a75a66a4900d5d67d" + }, + "createdAt": "2025-09-22T10:42:35Z", + "head": { + "oid": "f3c356fc8da46818a8ab0c24f22ee61854fe4b5f" + }, + "ticket": "5", + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreicbcuncwl73zhwlwxtgqfj3o72mlgdp3ncfvvolfwce535kitqwni/bafyreicbcuncwl73zhwlwxtgqfj3o72mlgdp3ncfvvolfwce535kitqwni.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreicbcuncwl73zhwlwxtgqfj3o72mlgdp3ncfvvolfwce535kitqwni/bafyreicbcuncwl73zhwlwxtgqfj3o72mlgdp3ncfvvolfwce535kitqwni.json new file mode 100644 index 0000000000000000000000000000000000000000..0a57b685c225e759c73fa4320087c3a986de9136 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreicbcuncwl73zhwlwxtgqfj3o72mlgdp3ncfvvolfwce535kitqwni/bafyreicbcuncwl73zhwlwxtgqfj3o72mlgdp3ncfvvolfwce535kitqwni.json @@ -0,0 +1,12 @@ +{ + "$type": "org.tangled.review.patch", + "base": { + "oid": "01117fddd3502c57bc20a65a75a66a4900d5d67d" + }, + "createdAt": "2025-09-22T10:43:35Z", + "head": { + "oid": "9a925efef6a0e6dfcd2d4317b4a1eee8752928b8" + }, + "ticket": "5", + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreidyg53s7fyjbh63abeog2h2g7vg5g67shjkk2hukto76jo4gjgb3u/bafyreidyg53s7fyjbh63abeog2h2g7vg5g67shjkk2hukto76jo4gjgb3u.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreidyg53s7fyjbh63abeog2h2g7vg5g67shjkk2hukto76jo4gjgb3u/bafyreidyg53s7fyjbh63abeog2h2g7vg5g67shjkk2hukto76jo4gjgb3u.json new file mode 100644 index 0000000000000000000000000000000000000000..4546c4c569443cf04af7fc8b32f8993583df2d60 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreidyg53s7fyjbh63abeog2h2g7vg5g67shjkk2hukto76jo4gjgb3u/bafyreidyg53s7fyjbh63abeog2h2g7vg5g67shjkk2hukto76jo4gjgb3u.json @@ -0,0 +1,12 @@ +{ + "$type": "org.tangled.review.patch", + "base": { + "oid": "a30e171f6297ac08777ce78bfea3fc9de27be8be" + }, + "createdAt": "2025-09-22T10:44:35Z", + "head": { + "oid": "7a486c07f881800c5dd5eec98dfb5fb10d96bddc" + }, + "ticket": "6", + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreifhetuhjxfajktrkc5wc4a3ikmg5ytard3b6g5plli3t3p5rkbuf4/bafyreifhetuhjxfajktrkc5wc4a3ikmg5ytard3b6g5plli3t3p5rkbuf4.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreifhetuhjxfajktrkc5wc4a3ikmg5ytard3b6g5plli3t3p5rkbuf4/bafyreifhetuhjxfajktrkc5wc4a3ikmg5ytard3b6g5plli3t3p5rkbuf4.json new file mode 100644 index 0000000000000000000000000000000000000000..d1b01a968f1d7725a9b833ae850cce3a991af607 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreifhetuhjxfajktrkc5wc4a3ikmg5ytard3b6g5plli3t3p5rkbuf4/bafyreifhetuhjxfajktrkc5wc4a3ikmg5ytard3b6g5plli3t3p5rkbuf4.json @@ -0,0 +1,12 @@ +{ + "$type": "org.tangled.review.patch", + "base": { + "oid": "912ecc1546e8a3698d7d68a9120ddc78fbf934d9" + }, + "createdAt": "2026-08-14T07:04:02.394Z", + "head": { + "oid": "0e96abbc399191e105951f0fc3338b7936451ca5" + }, + "ticket": "8", + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreig4rj2sbb3jjo6l2djoqzlim6a32vc7jfpa7pxub5ctv2c7eveavm/bafyreig4rj2sbb3jjo6l2djoqzlim6a32vc7jfpa7pxub5ctv2c7eveavm.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreig4rj2sbb3jjo6l2djoqzlim6a32vc7jfpa7pxub5ctv2c7eveavm/bafyreig4rj2sbb3jjo6l2djoqzlim6a32vc7jfpa7pxub5ctv2c7eveavm.json new file mode 100644 index 0000000000000000000000000000000000000000..038c798d13314d32503075d99e8c0a1cb4e358d1 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreig4rj2sbb3jjo6l2djoqzlim6a32vc7jfpa7pxub5ctv2c7eveavm/bafyreig4rj2sbb3jjo6l2djoqzlim6a32vc7jfpa7pxub5ctv2c7eveavm.json @@ -0,0 +1,12 @@ +{ + "$type": "org.tangled.review.patch", + "base": { + "oid": "928e8aca42bbbb8514cd666a780ac12f340fff96" + }, + "createdAt": "2026-09-04T19:06:18.222Z", + "head": { + "oid": "a30623cbe5ea1e0e945cc94d8e9529320fe25577" + }, + "ticket": "8", + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreiglruqe4czsrzewu6yy2evebgr6yv5ajgdztxoodaihsoddfn3g4u/bafyreiglruqe4czsrzewu6yy2evebgr6yv5ajgdztxoodaihsoddfn3g4u.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreiglruqe4czsrzewu6yy2evebgr6yv5ajgdztxoodaihsoddfn3g4u/bafyreiglruqe4czsrzewu6yy2evebgr6yv5ajgdztxoodaihsoddfn3g4u.json new file mode 100644 index 0000000000000000000000000000000000000000..1d8dd48d17c2130f9b96e43fb6c80c1851f5f55e --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreiglruqe4czsrzewu6yy2evebgr6yv5ajgdztxoodaihsoddfn3g4u/bafyreiglruqe4czsrzewu6yy2evebgr6yv5ajgdztxoodaihsoddfn3g4u.json @@ -0,0 +1,12 @@ +{ + "$type": "org.tangled.review.patch", + "base": { + "oid": "244eca1ec71c6c236118634ca3a6194a39f42246" + }, + "createdAt": "2026-09-14T07:09:10.682Z", + "head": { + "oid": "5d79aca95ea16e4f5e46bb3e5e00e7775252ba8e" + }, + "ticket": "8", + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreignfuruuizasc4cinen3s7cdmdyczm6t37y6eof36aoobmeclv2tq/bafyreignfuruuizasc4cinen3s7cdmdyczm6t37y6eof36aoobmeclv2tq.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreignfuruuizasc4cinen3s7cdmdyczm6t37y6eof36aoobmeclv2tq/bafyreignfuruuizasc4cinen3s7cdmdyczm6t37y6eof36aoobmeclv2tq.json new file mode 100644 index 0000000000000000000000000000000000000000..88b7d047f9d68a1245cbc391f797e3d9afe5eec0 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreignfuruuizasc4cinen3s7cdmdyczm6t37y6eof36aoobmeclv2tq/bafyreignfuruuizasc4cinen3s7cdmdyczm6t37y6eof36aoobmeclv2tq.json @@ -0,0 +1,12 @@ +{ + "$type": "org.tangled.review.patch", + "base": { + "oid": "8aec7d7c0e1eedf7dbbbe89f6d4f6048e8eb4eef" + }, + "createdAt": "2026-08-19T13:00:41.843Z", + "head": { + "oid": "778fe6786af294b13c74ee45973cf6a8791dd0bb" + }, + "ticket": "8", + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreigsqm243ma2cjtfwcmulqkyvcdrrbakd5amj2vswxcoukgsl7bo3a/bafyreigsqm243ma2cjtfwcmulqkyvcdrrbakd5amj2vswxcoukgsl7bo3a.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreigsqm243ma2cjtfwcmulqkyvcdrrbakd5amj2vswxcoukgsl7bo3a/bafyreigsqm243ma2cjtfwcmulqkyvcdrrbakd5amj2vswxcoukgsl7bo3a.json new file mode 100644 index 0000000000000000000000000000000000000000..eca03d14c21635e1258fa4fad04a9b86f3857190 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.review.patch/bafyreigsqm243ma2cjtfwcmulqkyvcdrrbakd5amj2vswxcoukgsl7bo3a/bafyreigsqm243ma2cjtfwcmulqkyvcdrrbakd5amj2vswxcoukgsl7bo3a.json @@ -0,0 +1,12 @@ +{ + "$type": "org.tangled.review.patch", + "base": { + "oid": "01117fddd3502c57bc20a65a75a66a4900d5d67d" + }, + "createdAt": "2025-09-22T10:45:35Z", + "head": { + "oid": "9a925efef6a0e6dfcd2d4317b4a1eee8752928b8" + }, + "ticket": "7", + "x-tngl-editor": "did:plc:xbserj4jfuhzkbl3mszkvvrb" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/1/bafyreidbxu22o4wwg2io5yd26ju4uadsdq3ikpff2hofr5cgnah356mnhu.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/1/bafyreidbxu22o4wwg2io5yd26ju4uadsdq3ikpff2hofr5cgnah356mnhu.json new file mode 100644 index 0000000000000000000000000000000000000000..4515f7253d3cfaceb0afee597c097f3338a32978 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/1/bafyreidbxu22o4wwg2io5yd26ju4uadsdq3ikpff2hofr5cgnah356mnhu.json @@ -0,0 +1,20 @@ +{ + "$type": "org.tangled.track.ticket", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "Symlinked entries in the file tree show up with a blank size and a link that\n404s. Expected them to render with the target path, the way a plain file does.\n\nReproduced on the default branch with `contrib/` present." + }, + "createdAt": "2025-09-22T10:14:35Z", + "state": "closed", + "title": { + "$type": "org.tangled.markup.markdown#inline", + "text": "Repo tree does not render symlinks" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu", + "tangledLabels": [ + { + "$type": "org.tangled.label.defs#labelToken", + "key": "duplicate" + } + ] +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/2/bafyreig33fgtw2h2lwz7uex4yepqgq5kez64vm56trqlhsnrup37am6r2q.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/2/bafyreig33fgtw2h2lwz7uex4yepqgq5kez64vm56trqlhsnrup37am6r2q.json new file mode 100644 index 0000000000000000000000000000000000000000..5792bd55ae8cb61e801f67a33c378877977a67b0 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/2/bafyreig33fgtw2h2lwz7uex4yepqgq5kez64vm56trqlhsnrup37am6r2q.json @@ -0,0 +1,14 @@ +{ + "$type": "org.tangled.track.ticket", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "## What's missing\n\n\u003e blah blah blah\n\n```\nI won't let claude to yap some nonsense here.\n```\n\n---\n\nsomething _something_" + }, + "createdAt": "2025-09-22T10:15:35Z", + "state": "open", + "title": { + "$type": "org.tangled.markup.markdown#inline", + "text": "Document the knot bootstrap flow" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/3/bafyreie3gbwyh7d5ctgicimydbxyvini2xvmuux4ambboqyrv24nnuvcee.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/3/bafyreie3gbwyh7d5ctgicimydbxyvini2xvmuux4ambboqyrv24nnuvcee.json new file mode 100644 index 0000000000000000000000000000000000000000..cad039a6575659e187ceb5765897fbc260c25779 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/3/bafyreie3gbwyh7d5ctgicimydbxyvini2xvmuux4ambboqyrv24nnuvcee.json @@ -0,0 +1,20 @@ +{ + "$type": "org.tangled.track.ticket", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "Fresh clone, first push to a branch nobody has pushed before, and the remote\nhangs up:\n\n```\n$ git push origin sl/my-branch\nEnumerating objects: 12, done.\nremote: error: cannot lock ref 'refs/heads/sl/my-branch'\nTo git@knot.tngl.boltless.dev:alice.pds.tngl.boltless.dev/core\n ! [remote rejected] sl/my-branch -> sl/my-branch (failed to update ref)\n```\n\nSecond attempt succeeds, so it looks like a race in the hook rather than a\npermissions problem." + }, + "createdAt": "2025-09-22T10:16:35Z", + "state": "open", + "title": { + "$type": "org.tangled.markup.markdown#inline", + "text": "Clone over ssh fails on first push" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu", + "tangledLabels": [ + { + "$type": "org.tangled.label.defs#labelToken", + "key": "good-first-issue" + } + ] +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/4/bafyreiedbneftlnpu47xjsqkxtncjxoe324kpklmdd4qnttvuptadfjh2y.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/4/bafyreiedbneftlnpu47xjsqkxtncjxoe324kpklmdd4qnttvuptadfjh2y.json new file mode 100644 index 0000000000000000000000000000000000000000..bd15276d8e256d1faf87dacfe6b2404fbc6b298b --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/4/bafyreiedbneftlnpu47xjsqkxtncjxoe324kpklmdd4qnttvuptadfjh2y.json @@ -0,0 +1,15 @@ +{ + "$type": "org.tangled.track.ticket", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "systemd-volatile-root uses only 25% of RAM for the rootfs, which is\nbelow the tmpfs default of 50% and makes it hard to do anything useful.\nThere is no way to change this, so instead, we use a simple custom\nscript to set up the rootfs before the initrd exits, with a better size\nused." + }, + "createdAt": "2025-09-22T10:40:35Z", + "state": "open", + "targetBranch": "master", + "title": { + "$type": "org.tangled.markup.markdown#inline", + "text": "nix/spindle: fix almalinux10 images having an undersized rootfs" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/5/bafyreidjm7iv7dstbkm5q5gt4haea6qxekd4lbu2ant2c3m5yqucqyz2wi.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/5/bafyreidjm7iv7dstbkm5q5gt4haea6qxekd4lbu2ant2c3m5yqucqyz2wi.json new file mode 100644 index 0000000000000000000000000000000000000000..c6e35dec89277bbd45c853cc78f8116c9fc7cb39 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/5/bafyreidjm7iv7dstbkm5q5gt4haea6qxekd4lbu2ant2c3m5yqucqyz2wi.json @@ -0,0 +1,21 @@ +{ + "$type": "org.tangled.track.ticket", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "something something markdown" + }, + "createdAt": "2025-09-22T10:42:35Z", + "state": "open", + "targetBranch": "master", + "title": { + "$type": "org.tangled.markup.markdown#inline", + "text": "ref based PR" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu", + "tangledLabels": [ + { + "$type": "org.tangled.label.defs#labelToken", + "key": "wontfix" + } + ] +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/6/bafyreieopiuug56i4hcivpqrlgt4govlivo6iy6id3vshljqgt4awnof4u.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/6/bafyreieopiuug56i4hcivpqrlgt4govlivo6iy6id3vshljqgt4awnof4u.json new file mode 100644 index 0000000000000000000000000000000000000000..4b1d787962429edac123f11270ad1e12b83c9a8e --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/6/bafyreieopiuug56i4hcivpqrlgt4govlivo6iy6id3vshljqgt4awnof4u.json @@ -0,0 +1,25 @@ +{ + "$type": "org.tangled.track.ticket", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "Fixture for a PR whose merge-base went stale because the *target* branch moved,\nnot the source:\n\n1. `sv-fe` branches off `master`\n2. `dwn/profile-activity` branches off `sv-fe` — merge-base recorded as `a30e171f`\n3. `sv-fe` is rebased wholesale onto a newer `master`\n4. `a30e171f` is no longer an ancestor of `sv-fe`, so the recorded base is stale\n\nThis is why the diff says 62 files changed while merge-check says 1155 conflicted files" + }, + "createdAt": "2025-09-22T10:44:35Z", + "state": "open", + "targetBranch": "sv-fe", + "title": { + "$type": "org.tangled.markup.markdown#inline", + "text": "Add profile activity feed" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu", + "tangledLabels": [ + { + "$type": "org.tangled.label.defs#labelToken", + "key": "documentation" + }, + { + "$type": "org.tangled.label.defs#labelToken", + "key": "good-first-issue" + } + ] +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/7/bafyreign52t3km5rd6l2x6aulj6hixoojfmifrdjbic2coo7musy6gutla.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/7/bafyreign52t3km5rd6l2x6aulj6hixoojfmifrdjbic2coo7musy6gutla.json new file mode 100644 index 0000000000000000000000000000000000000000..132713ba044ab473f2d7115ae8113007ca28f6fe --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/7/bafyreign52t3km5rd6l2x6aulj6hixoojfmifrdjbic2coo7musy6gutla.json @@ -0,0 +1,15 @@ +{ + "$type": "org.tangled.track.ticket", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "opened from core-fork, so source and target are different repos" + }, + "createdAt": "2025-09-22T10:45:35Z", + "state": "open", + "targetBranch": "master", + "title": { + "$type": "org.tangled.markup.markdown#inline", + "text": "ref based PR from a fork" + }, + "x-tngl-editor": "did:plc:xbserj4jfuhzkbl3mszkvvrb" +} diff --git a/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/8/bafyreidv3qgzq3mm53gfcxfep5ztun3tjfkhpm54odte3gjh2j6oiwko4y.json b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/8/bafyreidv3qgzq3mm53gfcxfep5ztun3tjfkhpm54odte3gjh2j6oiwko4y.json new file mode 100644 index 0000000000000000000000000000000000000000..cd6f0f749988fc3a112480edf80d124ab1ac0c88 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/8/bafyreidv3qgzq3mm53gfcxfep5ztun3tjfkhpm54odte3gjh2j6oiwko4y.json @@ -0,0 +1,15 @@ +{ + "$type": "org.tangled.track.ticket", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "I made few opinionated questionable choices here. See the commit message for each changes.\nIt might be helpful for ask AI agent to summarize all changes.\n\nIncluded:\n- User-owned record lexicons\n- Repo-owned record lexicons\n- Spindle xrpc lexicons (ci, webhook, secret)\n\nKnot xrpc lexicons are not included. We can finalize them after playing with COBs." + }, + "createdAt": "2026-08-14T07:04:02.394Z", + "state": "open", + "targetBranch": "sv-fe", + "title": { + "$type": "org.tangled.markup.markdown#inline", + "text": "lexicons: Tangled 1.0" + }, + "x-tngl-editor": "did:plc:igzuueb63dytyy3tpaweofuu" +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.feed.comment/3lzg6wxgzq222/bafyreihkqylhqxdmt2je5qiw6kfsgov7rh4h4fdzx4xqhk5q3wosuk7gbi.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.feed.comment/3lzg6wxgzq222/bafyreihkqylhqxdmt2je5qiw6kfsgov7rh4h4fdzx4xqhk5q3wosuk7gbi.json new file mode 100644 index 0000000000000000000000000000000000000000..d94889fbd8fdd52f80936d079a1b836bd4b7fcf0 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.feed.comment/3lzg6wxgzq222/bafyreihkqylhqxdmt2je5qiw6kfsgov7rh4h4fdzx4xqhk5q3wosuk7gbi.json @@ -0,0 +1,16 @@ +{ + "$type": "org.tangled.feed.comment", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "Only correct because the outer loop skips composites — otherwise it would miss factors." + }, + "createdAt": "2025-09-22T10:24:35Z", + "replyTo": { + "cid": "bafyreic35k7ygx5jg5mxqcu2x56ptgvynt4eb3g535fuf5azyfranlxok4", + "uri": "at://did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.feed.comment/3lzg6v67xy222" + }, + "subject": { + "cid": "bafyreiaskfifni2inl42xzbei2thp7wdxepgzeitnmv2aj7ww7sc5wq3ne", + "uri": "at://did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.snippet.string/3lzg6ma4pa222" + } +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.feed.reaction/3lzg764daq222/bafyreifgmnafntxfnfdx4gmhhdxqdolkwxycugm3hhppjh5d3ht4ww3umi.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.feed.reaction/3lzg764daq222/bafyreifgmnafntxfnfdx4gmhhdxqdolkwxycugm3hhppjh5d3ht4ww3umi.json new file mode 100644 index 0000000000000000000000000000000000000000..bf75023ef693c17b791b9939a15bded09f93bc49 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.feed.reaction/3lzg764daq222/bafyreifgmnafntxfnfdx4gmhhdxqdolkwxycugm3hhppjh5d3ht4ww3umi.json @@ -0,0 +1,9 @@ +{ + "$type": "org.tangled.feed.reaction", + "createdAt": "2025-09-22T10:28:35Z", + "reaction": "🚀", + "subject": { + "cid": "bafyreic35k7ygx5jg5mxqcu2x56ptgvynt4eb3g535fuf5azyfranlxok4", + "uri": "at://did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.feed.comment/3lzg6v67xy222" + } +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7splszs22/bafyreib76tnkofhqjhrcgox4etvcptekjj7aa5d5jmpniljh3hfqe5zt24.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7splszs22/bafyreib76tnkofhqjhrcgox4etvcptekjj7aa5d5jmpniljh3hfqe5zt24.json new file mode 100644 index 0000000000000000000000000000000000000000..988b7a2d41e46b55949abce40c9f411a7b22594d --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7splszs22/bafyreib76tnkofhqjhrcgox4etvcptekjj7aa5d5jmpniljh3hfqe5zt24.json @@ -0,0 +1,8 @@ +{ + "$type": "org.tangled.graph.attestation", + "subject": { + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/1", + "cid": "bafyreidbxu22o4wwg2io5yd26ju4uadsdq3ikpff2hofr5cgnah356mnhu" + }, + "createdAt": "2025-09-22T12:14:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7spnnms22/bafyreichv6dkuk3fmgpcwuevdxhapoginh2snmu4vuyuwzkvjvnmfnckm4.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7spnnms22/bafyreichv6dkuk3fmgpcwuevdxhapoginh2snmu4vuyuwzkvjvnmfnckm4.json new file mode 100644 index 0000000000000000000000000000000000000000..a3ab03749a757211fbe524826e7e01bce3efdd91 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7spnnms22/bafyreichv6dkuk3fmgpcwuevdxhapoginh2snmu4vuyuwzkvjvnmfnckm4.json @@ -0,0 +1,8 @@ +{ + "$type": "org.tangled.graph.attestation", + "subject": { + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/2", + "cid": "bafyreig33fgtw2h2lwz7uex4yepqgq5kez64vm56trqlhsnrup37am6r2q" + }, + "createdAt": "2025-09-22T12:15:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sppi7s22/bafyreihz5nhffleuard4bunztrrxqhr3qkbvqwsscrvnukpjqedszx7tnq.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sppi7s22/bafyreihz5nhffleuard4bunztrrxqhr3qkbvqwsscrvnukpjqedszx7tnq.json new file mode 100644 index 0000000000000000000000000000000000000000..7b9ddcf226a4f118a0ec0e49a12947c2fb164cd5 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sppi7s22/bafyreihz5nhffleuard4bunztrrxqhr3qkbvqwsscrvnukpjqedszx7tnq.json @@ -0,0 +1,8 @@ +{ + "$type": "org.tangled.graph.attestation", + "subject": { + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/3", + "cid": "bafyreie3gbwyh7d5ctgicimydbxyvini2xvmuux4ambboqyrv24nnuvcee" + }, + "createdAt": "2025-09-22T12:16:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sr3ghs22/bafyreiaencwsxfzy3o7uxput3prbhbkyyz7r4pf3r6ejwu6nqoj4u5ehmq.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sr3ghs22/bafyreiaencwsxfzy3o7uxput3prbhbkyyz7r4pf3r6ejwu6nqoj4u5ehmq.json new file mode 100644 index 0000000000000000000000000000000000000000..29ce065b15cc735590918b12606d102ecc9cdf02 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sr3ghs22/bafyreiaencwsxfzy3o7uxput3prbhbkyyz7r4pf3r6ejwu6nqoj4u5ehmq.json @@ -0,0 +1,8 @@ +{ + "$type": "org.tangled.graph.attestation", + "subject": { + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/4", + "cid": "bafyreiedbneftlnpu47xjsqkxtncjxoe324kpklmdd4qnttvuptadfjh2y" + }, + "createdAt": "2025-09-22T12:40:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sr73ns22/bafyreifbgyq24ub4mphl3pmt4toeiq4u46fbqtdhos54dmvcuqckjqkhkq.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sr73ns22/bafyreifbgyq24ub4mphl3pmt4toeiq4u46fbqtdhos54dmvcuqckjqkhkq.json new file mode 100644 index 0000000000000000000000000000000000000000..805ddfe56f75a915727b0c586f092ba8cf833222 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7sr73ns22/bafyreifbgyq24ub4mphl3pmt4toeiq4u46fbqtdhos54dmvcuqckjqkhkq.json @@ -0,0 +1,8 @@ +{ + "$type": "org.tangled.graph.attestation", + "subject": { + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/5", + "cid": "bafyreidjm7iv7dstbkm5q5gt4haea6qxekd4lbu2ant2c3m5yqucqyz2wi" + }, + "createdAt": "2025-09-22T12:42:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7srcqts22/bafyreif7u22hndb4el5xkgohfym2fp7oanx4cbvkjubatnxrdkpl7n62zm.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7srcqts22/bafyreif7u22hndb4el5xkgohfym2fp7oanx4cbvkjubatnxrdkpl7n62zm.json new file mode 100644 index 0000000000000000000000000000000000000000..511421bdd06ecfcd80d1a6107f76e0ec90846709 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223n7srcqts22/bafyreif7u22hndb4el5xkgohfym2fp7oanx4cbvkjubatnxrdkpl7n62zm.json @@ -0,0 +1,8 @@ +{ + "$type": "org.tangled.graph.attestation", + "subject": { + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/6", + "cid": "bafyreieopiuug56i4hcivpqrlgt4govlivo6iy6id3vshljqgt4awnof4u" + }, + "createdAt": "2025-09-22T12:44:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223nzzsbaeu22/bafyreigtl4u36d3uy5ymeycyxy3eomtcgheevpsnd6vevshtihb65cwqwe.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223nzzsbaeu22/bafyreigtl4u36d3uy5ymeycyxy3eomtcgheevpsnd6vevshtihb65cwqwe.json new file mode 100644 index 0000000000000000000000000000000000000000..6d6b2608d9dbe1654832c08f109aef4f8e1e2236 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.attestation/223nzzsbaeu22/bafyreigtl4u36d3uy5ymeycyxy3eomtcgheevpsnd6vevshtihb65cwqwe.json @@ -0,0 +1,8 @@ +{ + "$type": "org.tangled.graph.attestation", + "subject": { + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/8", + "cid": "bafyreidv3qgzq3mm53gfcxfep5ztun3tjfkhpm54odte3gjh2j6oiwko4y" + }, + "createdAt": "2026-08-14T09:04:02Z" +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.follow/did:plc:pgkgoz6rpvmgynswviohltpx/bafyreibzfpdma3syj27j42juei6wcmh2fpjxqp23urpivj4kp6xnccl6pa.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.follow/did:plc:pgkgoz6rpvmgynswviohltpx/bafyreibzfpdma3syj27j42juei6wcmh2fpjxqp23urpivj4kp6xnccl6pa.json new file mode 100644 index 0000000000000000000000000000000000000000..014a755f3636cf36e8637ca30cdced9618dfd754 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.follow/did:plc:pgkgoz6rpvmgynswviohltpx/bafyreibzfpdma3syj27j42juei6wcmh2fpjxqp23urpivj4kp6xnccl6pa.json @@ -0,0 +1,4 @@ +{ + "$type": "org.tangled.graph.follow", + "createdAt": "2025-09-22T10:30:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.follow/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreifl7dsgyvu7fvkl4l5umvrrewoatpia5hgjuckybf3tfk4y5bnm3i.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.follow/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreifl7dsgyvu7fvkl4l5umvrrewoatpia5hgjuckybf3tfk4y5bnm3i.json new file mode 100644 index 0000000000000000000000000000000000000000..14d8c5be6f5459f09505bfb6fd08abfe156f9791 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.follow/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreifl7dsgyvu7fvkl4l5umvrrewoatpia5hgjuckybf3tfk4y5bnm3i.json @@ -0,0 +1,4 @@ +{ + "$type": "org.tangled.graph.follow", + "createdAt": "2025-09-22T10:29:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.vouch/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreifm5ul6hhiesuuhukw45gol7z3w67ktf27dlvluu2h4a6m54fx3ga.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.vouch/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreifm5ul6hhiesuuhukw45gol7z3w67ktf27dlvluu2h4a6m54fx3ga.json new file mode 100644 index 0000000000000000000000000000000000000000..2de701c6b1c03dcbc04d503bb25addba3ad3b6b6 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.graph.vouch/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreifm5ul6hhiesuuhukw45gol7z3w67ktf27dlvluu2h4a6m54fx3ga.json @@ -0,0 +1,12 @@ +{ + "$type": "org.tangled.graph.vouch", + "createdAt": "2025-09-22T10:35:35Z", + "evidences": [ + { + "cid": "bafyreietwvvhovqaxq2llr35tg7fslxikhjfffulmovin3q6ngfbsyqkle", + "uri": "at://did:plc:2pt74ankjflp6tryyp7aloqe/org.tangled.feed.comment/3lzg6rlrui222" + } + ], + "kind": "vouch", + "reason": "Reviewed his bootstrap-ordering work on core, knows the stack." +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.repo.declaration/core/bafyreigpl5grnco2z7nkadnag57vntcoc7ye2oq22fgsqirkjgwbgv6gx4.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.repo.declaration/core/bafyreigpl5grnco2z7nkadnag57vntcoc7ye2oq22fgsqirkjgwbgv6gx4.json new file mode 100644 index 0000000000000000000000000000000000000000..701583ad72286019e1e3a0ec8dff1eb56f74c86a --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.repo.declaration/core/bafyreigpl5grnco2z7nkadnag57vntcoc7ye2oq22fgsqirkjgwbgv6gx4.json @@ -0,0 +1,5 @@ +{ + "$type": "org.tangled.repo.declaration", + "createdAt": "2025-09-22T11:14:35+01:00", + "repo": "did:plc:2pt74ankjflp6tryyp7aloqe" +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.snippet.string/3lzg6ma4pa222/bafyreiaskfifni2inl42xzbei2thp7wdxepgzeitnmv2aj7ww7sc5wq3ne.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.snippet.string/3lzg6ma4pa222/bafyreiaskfifni2inl42xzbei2thp7wdxepgzeitnmv2aj7ww7sc5wq3ne.json new file mode 100644 index 0000000000000000000000000000000000000000..a0f3b9c7b21553c7a3a12100bdf85595e8bc5ee9 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.snippet.string/3lzg6ma4pa222/bafyreiaskfifni2inl42xzbei2thp7wdxepgzeitnmv2aj7ww7sc5wq3ne.json @@ -0,0 +1,17 @@ +{ + "$type": "org.tangled.snippet.string", + "createdAt": "2025-09-22T10:18:35Z", + "description": "primes below n, no allocations past the sieve", + "snippet": { + "$type": "org.tangled.snippet.string#textFile", + "content": { + "$type": "blob", + "mimeType": "text/plain", + "ref": { + "$link": "bafkreifqrrwn3ioqkrquhibb3csu6i3kogd76uk6ktptc7s5exim6rfu6u" + }, + "size": 318 + }, + "name": "sieve.go" + } +} diff --git a/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.snippet.string/3lzg6nzdqy222/bafyreifweqc5cgslk56emxrcqisv6ctlza4mfkhxdcs5iuba3t6kfagh3y.json b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.snippet.string/3lzg6nzdqy222/bafyreifweqc5cgslk56emxrcqisv6ctlza4mfkhxdcs5iuba3t6kfagh3y.json new file mode 100644 index 0000000000000000000000000000000000000000..5764e8a999f1b6752f773259a171432d7701674a --- /dev/null +++ b/localinfra/fixtures/records/did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.snippet.string/3lzg6nzdqy222/bafyreifweqc5cgslk56emxrcqisv6ctlza4mfkhxdcs5iuba3t6kfagh3y.json @@ -0,0 +1,17 @@ +{ + "$type": "org.tangled.snippet.string", + "createdAt": "2025-09-22T10:19:35Z", + "description": "scratch notes from bringing up the local stack", + "snippet": { + "$type": "org.tangled.snippet.string#textFile", + "content": { + "$type": "blob", + "mimeType": "text/plain", + "ref": { + "$link": "bafkreieify5bix4cmrj4iuxiceb6srr2boltkbbf7yizwtgwurqv42sltm" + }, + "size": 416 + }, + "name": "notes.md" + } +} diff --git a/localinfra/fixtures/records/did:plc:pgkgoz6rpvmgynswviohltpx/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreifuy2yp7w7c6z3frzi2t7rxlvovw3cz6ivk5l2d7omuptcfje7a6i.json b/localinfra/fixtures/records/did:plc:pgkgoz6rpvmgynswviohltpx/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreifuy2yp7w7c6z3frzi2t7rxlvovw3cz6ivk5l2d7omuptcfje7a6i.json new file mode 100644 index 0000000000000000000000000000000000000000..d771a00d9014120d1e2bb4890d508552c0572968 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:pgkgoz6rpvmgynswviohltpx/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreifuy2yp7w7c6z3frzi2t7rxlvovw3cz6ivk5l2d7omuptcfje7a6i.json @@ -0,0 +1,4 @@ +{ + "$type": "org.tangled.graph.follow", + "createdAt": "2025-09-22T10:32:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:pgkgoz6rpvmgynswviohltpx/org.tangled.graph.vouch/did:plc:ryxq2u2b6sadhhjflpwiefg4/bafyreig2c6t2yhkdxuupyhouevcgx7kbay4b4nj2og5qlpfjh6sugrdux4.json b/localinfra/fixtures/records/did:plc:pgkgoz6rpvmgynswviohltpx/org.tangled.graph.vouch/did:plc:ryxq2u2b6sadhhjflpwiefg4/bafyreig2c6t2yhkdxuupyhouevcgx7kbay4b4nj2og5qlpfjh6sugrdux4.json new file mode 100644 index 0000000000000000000000000000000000000000..8cb909b1f35ac678fd4ef103205c68d7251cd69c --- /dev/null +++ b/localinfra/fixtures/records/did:plc:pgkgoz6rpvmgynswviohltpx/org.tangled.graph.vouch/did:plc:ryxq2u2b6sadhhjflpwiefg4/bafyreig2c6t2yhkdxuupyhouevcgx7kbay4b4nj2og5qlpfjh6sugrdux4.json @@ -0,0 +1,6 @@ +{ + "$type": "org.tangled.graph.vouch", + "createdAt": "2025-09-22T10:37:35Z", + "kind": "vouch", + "reason": "Long-time collaborator, reviews carefully." +} diff --git a/localinfra/fixtures/records/did:plc:qbqdycrgcna6n7w5lskwod6y/org.tangled.git.refs/self/bafyreigkwkkfyqmuwvl5zx552ivpes3epirw6dpsvzo546kc4agy7r4gtm.json b/localinfra/fixtures/records/did:plc:qbqdycrgcna6n7w5lskwod6y/org.tangled.git.refs/self/bafyreigkwkkfyqmuwvl5zx552ivpes3epirw6dpsvzo546kc4agy7r4gtm.json new file mode 100644 index 0000000000000000000000000000000000000000..2b083a97c9127cf2c903bbda7eec54847825e121 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:qbqdycrgcna6n7w5lskwod6y/org.tangled.git.refs/self/bafyreigkwkkfyqmuwvl5zx552ivpes3epirw6dpsvzo546kc4agy7r4gtm.json @@ -0,0 +1,7 @@ +{ + "$type": "org.tangled.git.refs", + "defaultBranch": "master", + "refs": { + "$bytes": "MDExMTdmZGRkMzUwMmM1N2JjMjBhNjVhNzVhNjZhNDkwMGQ1ZDY3ZCByZWZzL2hlYWRzL21hc3Rlcgo5YTkyNWVmZWY2YTBlNmRmY2QyZDQzMTdiNGExZWVlODc1MjkyOGI4IHJlZnMvaGVhZHMvc2wvcmVmLWJhc2VkLXByCg" + } +} diff --git a/localinfra/fixtures/records/did:plc:qbqdycrgcna6n7w5lskwod6y/org.tangled.repo.manifest/self/bafyreic6d75ygy5fdaoonunl4v5ki6nyazawttswpas3giafc7dgd7uuza.json b/localinfra/fixtures/records/did:plc:qbqdycrgcna6n7w5lskwod6y/org.tangled.repo.manifest/self/bafyreic6d75ygy5fdaoonunl4v5ki6nyazawttswpas3giafc7dgd7uuza.json new file mode 100644 index 0000000000000000000000000000000000000000..43135c753bf505cfc1e561007b4dd72d538d7150 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:qbqdycrgcna6n7w5lskwod6y/org.tangled.repo.manifest/self/bafyreic6d75ygy5fdaoonunl4v5ki6nyazawttswpas3giafc7dgd7uuza.json @@ -0,0 +1,13 @@ +{ + "$type": "org.tangled.repo.manifest", + "createdAt": "2025-09-22T11:14:35+01:00", + "declaration": { + "owner": "did:plc:xbserj4jfuhzkbl3mszkvvrb", + "slug": "core-fork" + }, + "source": { + "$type": "org.tangled.repo.defs#sourceTangled", + "did": "did:plc:i3cuazypmjja73m3ahwrsh2z" + }, + "spindle": "did:web:spindle.tngl.boltless.dev" +} diff --git a/localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreicpsiuzccktcxgcp4cl6mshfgdicv45ple4hrxrrgd7b3t4masruy.json b/localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreicpsiuzccktcxgcp4cl6mshfgdicv45ple4hrxrrgd7b3t4masruy.json new file mode 100644 index 0000000000000000000000000000000000000000..42e0f807e5eeaf41ffaa9d0edae8d2de9bd1099a --- /dev/null +++ b/localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreicpsiuzccktcxgcp4cl6mshfgdicv45ple4hrxrrgd7b3t4masruy.json @@ -0,0 +1,4 @@ +{ + "$type": "org.tangled.graph.follow", + "createdAt": "2025-09-22T10:33:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.follow/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreiewemtnmq2h2nugt2sbm4xavv7jqzvec6hbwqypinudwonkoukvti.json b/localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.follow/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreiewemtnmq2h2nugt2sbm4xavv7jqzvec6hbwqypinudwonkoukvti.json new file mode 100644 index 0000000000000000000000000000000000000000..781644c7f308c82ab12cc10d549a8f8c810f66b2 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.follow/did:plc:xbserj4jfuhzkbl3mszkvvrb/bafyreiewemtnmq2h2nugt2sbm4xavv7jqzvec6hbwqypinudwonkoukvti.json @@ -0,0 +1,4 @@ +{ + "$type": "org.tangled.graph.follow", + "createdAt": "2025-09-22T10:34:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.vouch/did:plc:igzuueb63dytyy3tpaweofuu/bafyreiemubhyfd7wu4louq2och6fomjpne7kleyfvt75s5si3px76gdcoy.json b/localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.vouch/did:plc:igzuueb63dytyy3tpaweofuu/bafyreiemubhyfd7wu4louq2och6fomjpne7kleyfvt75s5si3px76gdcoy.json new file mode 100644 index 0000000000000000000000000000000000000000..34888189caa14e10b4867686ae757dd868c3e9e9 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:ryxq2u2b6sadhhjflpwiefg4/org.tangled.graph.vouch/did:plc:igzuueb63dytyy3tpaweofuu/bafyreiemubhyfd7wu4louq2och6fomjpne7kleyfvt75s5si3px76gdcoy.json @@ -0,0 +1,6 @@ +{ + "$type": "org.tangled.graph.vouch", + "createdAt": "2025-09-22T10:38:35Z", + "kind": "vouch", + "reason": "Runs the knot, has never lost a repo." +} diff --git a/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.feed.comment/3lzg6v67xy222/bafyreic35k7ygx5jg5mxqcu2x56ptgvynt4eb3g535fuf5azyfranlxok4.json b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.feed.comment/3lzg6v67xy222/bafyreic35k7ygx5jg5mxqcu2x56ptgvynt4eb3g535fuf5azyfranlxok4.json new file mode 100644 index 0000000000000000000000000000000000000000..bf1d72995012fb7cfb601ce022ab5bd8a9b14a81 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.feed.comment/3lzg6v67xy222/bafyreic35k7ygx5jg5mxqcu2x56ptgvynt4eb3g535fuf5azyfranlxok4.json @@ -0,0 +1,12 @@ +{ + "$type": "org.tangled.feed.comment", + "body": { + "$type": "org.tangled.markup.markdown", + "text": "Starting the inner loop at `i*i` is the bit everyone forgets." + }, + "createdAt": "2025-09-22T10:23:35Z", + "subject": { + "cid": "bafyreiaskfifni2inl42xzbei2thp7wdxepgzeitnmv2aj7ww7sc5wq3ne", + "uri": "at://did:plc:igzuueb63dytyy3tpaweofuu/org.tangled.snippet.string/3lzg6ma4pa222" + } +} diff --git a/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.attestation/223n7srelgs22/bafyreiegqxgcgfs3vlusn6cky7p7kjgl5eb3xxwye3kk6ajbtvb2lpxc5m.json b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.attestation/223n7srelgs22/bafyreiegqxgcgfs3vlusn6cky7p7kjgl5eb3xxwye3kk6ajbtvb2lpxc5m.json new file mode 100644 index 0000000000000000000000000000000000000000..0925fb45bfd11f1c9b8ca28fbd98759858c60caa --- /dev/null +++ b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.attestation/223n7srelgs22/bafyreiegqxgcgfs3vlusn6cky7p7kjgl5eb3xxwye3kk6ajbtvb2lpxc5m.json @@ -0,0 +1,8 @@ +{ + "$type": "org.tangled.graph.attestation", + "subject": { + "uri": "at://did:plc:i3cuazypmjja73m3ahwrsh2z/org.tangled.track.ticket/7", + "cid": "bafyreign52t3km5rd6l2x6aulj6hixoojfmifrdjbic2coo7musy6gutla" + }, + "createdAt": "2025-09-22T12:45:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreiec5mggcrtwhdlkpymftts44zumyz5m6qwccqrpt4gihmnu652qae.json b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreiec5mggcrtwhdlkpymftts44zumyz5m6qwccqrpt4gihmnu652qae.json new file mode 100644 index 0000000000000000000000000000000000000000..2c280d361af72a63fa5b67cd80c0b586c4155c3c --- /dev/null +++ b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.follow/did:plc:igzuueb63dytyy3tpaweofuu/bafyreiec5mggcrtwhdlkpymftts44zumyz5m6qwccqrpt4gihmnu652qae.json @@ -0,0 +1,4 @@ +{ + "$type": "org.tangled.graph.follow", + "createdAt": "2025-09-22T10:31:35Z" +} diff --git a/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.vouch/did:plc:pgkgoz6rpvmgynswviohltpx/bafyreif7tywomyuzgtcawmppnrnmicfzcty5d42kjz24igyktw6ekxfhpa.json b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.vouch/did:plc:pgkgoz6rpvmgynswviohltpx/bafyreif7tywomyuzgtcawmppnrnmicfzcty5d42kjz24igyktw6ekxfhpa.json new file mode 100644 index 0000000000000000000000000000000000000000..293d1b592062bc73299ffd28d06951e232d2e6d9 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.vouch/did:plc:pgkgoz6rpvmgynswviohltpx/bafyreif7tywomyuzgtcawmppnrnmicfzcty5d42kjz24igyktw6ekxfhpa.json @@ -0,0 +1,6 @@ +{ + "$type": "org.tangled.graph.vouch", + "createdAt": "2025-09-22T10:36:35Z", + "kind": "vouch", + "reason": "Worked together on the knot ACL rewrite." +} diff --git a/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.vouch/did:plc:ryxq2u2b6sadhhjflpwiefg4/bafyreic7neuxz42d4373k5xdda7b66ycxnt6ct6w5gyfzb6mlozgbqxlg4.json b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.vouch/did:plc:ryxq2u2b6sadhhjflpwiefg4/bafyreic7neuxz42d4373k5xdda7b66ycxnt6ct6w5gyfzb6mlozgbqxlg4.json new file mode 100644 index 0000000000000000000000000000000000000000..27ef703470a964868f7876fbd72ca9a0368bc737 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.graph.vouch/did:plc:ryxq2u2b6sadhhjflpwiefg4/bafyreic7neuxz42d4373k5xdda7b66ycxnt6ct6w5gyfzb6mlozgbqxlg4.json @@ -0,0 +1,6 @@ +{ + "$type": "org.tangled.graph.vouch", + "createdAt": "2025-09-22T10:39:35Z", + "kind": "denounce", + "reason": "Force-pushed over master twice in one week." +} diff --git a/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.repo.declaration/core-fork/bafyreiacqdhafvv7x4aefu4zyheqhyk24dbxtlcqxowgfpz5g2wpeclkgm.json b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.repo.declaration/core-fork/bafyreiacqdhafvv7x4aefu4zyheqhyk24dbxtlcqxowgfpz5g2wpeclkgm.json new file mode 100644 index 0000000000000000000000000000000000000000..f7b60ba5323ccab41ab54240bd8c50f68f0906e0 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.repo.declaration/core-fork/bafyreiacqdhafvv7x4aefu4zyheqhyk24dbxtlcqxowgfpz5g2wpeclkgm.json @@ -0,0 +1,5 @@ +{ + "$type": "org.tangled.repo.declaration", + "createdAt": "2025-09-22T11:14:35+01:00", + "repo": "did:plc:5a5citbzkm5lnzxpoxbhvpae" +} diff --git a/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.repo.declaration/empty-repo/bafyreic2ngxwckzpfk7tlzxofcfagfld3dg2qslubaewelbpkoz5qsepcy.json b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.repo.declaration/empty-repo/bafyreic2ngxwckzpfk7tlzxofcfagfld3dg2qslubaewelbpkoz5qsepcy.json new file mode 100644 index 0000000000000000000000000000000000000000..93ba3ce6f7b825c3f0f1e005dbaa975202aae10f --- /dev/null +++ b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.repo.declaration/empty-repo/bafyreic2ngxwckzpfk7tlzxofcfagfld3dg2qslubaewelbpkoz5qsepcy.json @@ -0,0 +1,5 @@ +{ + "$type": "org.tangled.repo.declaration", + "createdAt": "2025-09-22T11:14:35+01:00", + "repo": "did:plc:35vbwlqbtciobm5emdvgg7by" +} diff --git a/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.snippet.string/3lzg6psksq222/bafyreieohwiytxwzkuczdrm35hxpdm2h2jyuffg2fatpxpo7iyxfnuk5ka.json b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.snippet.string/3lzg6psksq222/bafyreieohwiytxwzkuczdrm35hxpdm2h2jyuffg2fatpxpo7iyxfnuk5ka.json new file mode 100644 index 0000000000000000000000000000000000000000..a184f63adb175cf51450c9fe11437b115a1693d9 --- /dev/null +++ b/localinfra/fixtures/records/did:plc:xbserj4jfuhzkbl3mszkvvrb/org.tangled.snippet.string/3lzg6psksq222/bafyreieohwiytxwzkuczdrm35hxpdm2h2jyuffg2fatpxpo7iyxfnuk5ka.json @@ -0,0 +1,16 @@ +{ + "$type": "org.tangled.snippet.string", + "createdAt": "2025-09-22T10:20:35Z", + "snippet": { + "$type": "org.tangled.snippet.string#textFile", + "content": { + "$type": "blob", + "mimeType": "text/plain", + "ref": { + "$link": "bafkreiaovmar5rtz7dw35cfylzmb3htqg7kpksf76pvascxlv3s56rb7o4" + }, + "size": 188 + }, + "name": "todo.txt" + } +} diff --git a/localinfra/github-stub/__init__.py b/localinfra/github-stub/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2cc75fcdf664603939a65e0323d638d5ef4effd0 --- /dev/null +++ b/localinfra/github-stub/__init__.py @@ -0,0 +1 @@ +"""Fake github.com, api.github.com and their git remotes, for local e2e runs.""" diff --git a/localinfra/github-stub/__main__.py b/localinfra/github-stub/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..2e660a80191daeeeac144157262db7612cec9e5f --- /dev/null +++ b/localinfra/github-stub/__main__.py @@ -0,0 +1,4 @@ +from .server import run + +if __name__ == "__main__": + run() diff --git a/localinfra/github-stub/server.py b/localinfra/github-stub/server.py new file mode 100644 index 0000000000000000000000000000000000000000..99038fd97598a5abbad1f1251e48b8668938e29b --- /dev/null +++ b/localinfra/github-stub/server.py @@ -0,0 +1,673 @@ +#!/usr/bin/env python3 +import json +import os +import random +import re +import struct +import subprocess +import sys +import tempfile +import urllib.parse +import zlib +from datetime import datetime, timedelta, timezone +from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler + +PORT = int(os.environ.get("PORT", "8000")) +CLIENT_ID = os.environ.get("GITHUB_CLIENT_ID", "ghstub-client-id") +CLIENT_SECRET = os.environ.get("GITHUB_CLIENT_SECRET", "ghstub-client-secret") +APP_SLUG = os.environ.get("GITHUB_APP_SLUG", "tangled-dev-stub") +OBJECT_FORMAT = os.environ.get("GITHUB_STUB_OBJECT_FORMAT", "sha1") +REPOS_DIR = os.environ.get("REPOS_DIR", os.path.join(tempfile.gettempdir(), "github-stub-repos")) + +PUBLIC_ORIGIN = os.environ.get("GITHUB_PUBLIC_ORIGIN", "https://github.com").rstrip("/") +API_ORIGIN = os.environ.get("GITHUB_API_ORIGIN", "https://api.github.com").rstrip("/") +AVATARS_ORIGIN = os.environ.get("GITHUB_AVATARS_ORIGIN", "https://avatars.githubusercontent.com").rstrip("/") + +OWNER = { + "login": "octostub", + "id": 10001, + "avatar_url": f"{AVATARS_ORIGIN}/u/10001", + "bio": "stub bio for the import flow", + "blog": "https://octostub.example.com", +} + +# sha256 exercises knot2 object-format negotiation locally +SEED = [ + ( + "hello", + "Hello world stub repository", + { + "README.md": "# Hello\n\nHello world stub repository for Tangled import testing.\n", + "hello.txt": "Hello from octostub/hello!\n", + }, + "sha1", + ), + ( + "world", + "World stub repository for bulk import", + {"README.md": "# World\n\nWorld stub repository for bulk import testing.\n"}, + "sha1", + ), + ( + "tangled-demo", + "Tangled demo repository", + {"README.md": "# Tangled Demo\n\nDemo repository for tangled import.\n"}, + "sha1", + ), + ( + "fresh", + "Fresh stub repository, never imported before", + { + "README.md": "# Fresh\n\nA repository nobody has imported yet.\n", + "src/lib.rs": "pub fn fresh() -> &'static str {\n \"fresh\"\n}\n", + }, + "sha256", + ), + ( + "second", + "Second fresh stub repository, for bulk import", + { + "README.md": "# Second\n\nThe second repository in a bulk import.\n", + "notes.md": "two\n", + }, + "sha1", + ), + ( + "toolkit", + "Small tools, for the bulk import", + { + "README.md": "# Toolkit\n\nTools nobody has imported yet.\n", + "tools.md": "hammer\n", + }, + "sha1", + ), + ( + "notes", + "Scratch notes, also unimported", + {"README.md": "# Notes\n\nNothing to see here yet.\n", "today.md": "one\n"}, + "sha1", + ), +] + + +REPO_METADATA = { + "hello": {"language": "TypeScript", "stars": 12, "issues": 3, "forks": 2, "size_kb": 21504, "days_ago": 2}, + "world": {"language": "Rust", "stars": 340, "issues": 7, "forks": 41, "size_kb": 512, "days_ago": 5}, + "tangled-demo": {"language": "Go", "stars": 88, "issues": 1, "forks": 9, "size_kb": 330, "hours_ago": 3}, + "fresh": {"language": "Rust", "stars": 0, "issues": 0, "forks": 1, "size_kb": 64, "days_ago": 1}, + "second": {"language": "Python", "stars": 2048, "issues": 22, "forks": 190, "size_kb": 20480, "days_ago": 200}, + "toolkit": {"language": "Shell", "stars": 5, "issues": 0, "forks": 0, "size_kb": 88, "days_ago": 3}, + "notes": {"language": None, "stars": 1, "issues": 0, "forks": 0, "size_kb": 16, "days_ago": 45}, +} + + +# filler so the import list has enough rows to page, scroll and truncate; seeded so it's stable +def _filler(count: int): + rng = random.Random(7) + adjectives = ["tiny", "rusty", "quiet", "brisk", "lazy", "shiny", "fuzzy", "bold", "spare", "odd", + "swift", "mellow", "plain", "wild", "sleepy", "eager", "hollow", "neon", "paper", "frozen"] + nouns = ["parser", "kettle", "badger", "lantern", "compiler", "garden", "relay", "sketch", "atlas", + "widget", "harbor", "beacon", "notebook", "cache", "spindle", "loom", "orbit", "pebble", + "toolbox", "ledger", "router", "canvas", "sampler", "daemon", "shelf"] + subjects = ["A small library", "Experimental tooling", "Personal dotfiles", "A toy interpreter", + "Benchmarks and notes", "A static site", "Bindings", "A CLI", "Config and scripts", + "Half-finished prototype"] + purposes = ["for parsing odd file formats", "for syncing things between machines", + "that nobody asked for", "for the homelab", "for learning the language", + "kept around for reference", "for a weekend hackathon", "extracted from a bigger project", + "for generating release notes", "for poking at network protocols"] + tails = ["", "", "", " Mostly works, occasionally surprises. Contributions welcome but expect slow reviews.", + " Includes a test suite, a fuzzing harness, a handful of examples and a README that is more" + " aspirational than accurate."] + languages = ["Rust", "Go", "TypeScript", "Python", "Shell", "C", "Haskell", "Nix", "Lua", "Zig", + "OCaml", "Elixir", "JavaScript", "Svelte", None] + + names = set(REPO_METADATA) + seeds = [] + while len(seeds) < count: + name = f"{rng.choice(adjectives)}-{rng.choice(nouns)}" + if name in names: + name = f"{name}-{rng.randint(2, 99)}" + if name in names: + continue + names.add(name) + description = "" if rng.random() < 0.1 else f"{rng.choice(subjects)} {rng.choice(purposes)}.{rng.choice(tails)}" + seeds.append((name, description, {"README.md": f"# {name}\n\n{description}\n"}, "sha1")) + REPO_METADATA[name] = { + "language": rng.choice(languages), + "stars": rng.choice([rng.randint(0, 40)] * 8 + [rng.randint(40, 2000)] * 3 + [rng.randint(2000, 25000)]), + "issues": rng.randint(0, 40), + "forks": rng.randint(0, 200), + "size_kb": rng.randint(8, 50000), + "days_ago": rng.randint(0, 1500), + } + return seeds + + +SEED += _filler(93) + + +def _iso_age(**kwargs) -> str: + return (datetime.now(timezone.utc) - timedelta(**kwargs)).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def repository(name: str, description: str, files: dict, object_format: str = "sha1") -> dict: + meta = REPO_METADATA[name] + pushed_at = _iso_age(hours=meta.get("hours_ago", 0), days=meta.get("days_ago", 0)) + return { + "id": 101 + [seed[0] for seed in SEED].index(name), + "name": name, + "full_name": f"octostub/{name}", + "owner": OWNER, + "html_url": f"{PUBLIC_ORIGIN}/octostub/{name}", + "description": description, + "clone_url": f"{PUBLIC_ORIGIN}/octostub/{name}.git", + "default_branch": "main", + "visibility": "public", + "private": False, + "language": meta["language"], + "stargazers_count": meta["stars"], + "open_issues_count": meta["issues"], + "forks_count": meta["forks"], + "size": meta["size_kb"], + "pushed_at": pushed_at, + "updated_at": pushed_at, + "files": files, + "object_format": object_format, + } + + +REPOSITORIES = [repository(*seed) for seed in SEED] + +# lists refs fine and then refuses the pack, so an import passes its ls-refs probe and fails later +UNFETCHABLE = {"unfetchable": "hello"} + +STUB_TOKEN_PREFIX = "ghstub-token-" + + +def email_for(token: str) -> str: + """The caller's address. Deliberi refuses one address for two accounts, so every + authorization gets its own: the token is derived from the code minted per connect.""" + match = re.fullmatch(STUB_TOKEN_PREFIX + r"([0-9a-f]{16})", token) + suffix = f"+{match.group(1)[:8]}" if match else "" + return f"octostub{suffix}@octostub.dev" + + +GITHUB_EMAILS = [ + {"email": "octostub@octostub.dev", "primary": True, "verified": True, "visibility": "public"}, + { + "email": "octostub-secondary@octostub.dev", + "primary": False, + "verified": False, + "visibility": "public", + }, + { + "email": "octostub-alt@octostub.dev", + "primary": False, + "verified": True, + "visibility": "public", + }, +] + + +def _clean_repo(r: dict) -> dict: + return {k: v for k, v in r.items() if k not in ("files", "object_format")} + + +def make_png(width=16, height=16, color=(120, 80, 200)) -> bytes: + png = b"\x89PNG\r\n\x1a\n" + ihdr_data = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + ihdr_crc = struct.pack(">I", zlib.crc32(b"IHDR" + ihdr_data) & 0xffffffff) + png += struct.pack(">I", len(ihdr_data)) + b"IHDR" + ihdr_data + ihdr_crc + + raw = bytearray() + for _ in range(height): + raw.append(0) + for _ in range(width): + raw.extend(color) + + compressed = zlib.compress(bytes(raw)) + idat_crc = struct.pack(">I", zlib.crc32(b"IDAT" + compressed) & 0xffffffff) + png += struct.pack(">I", len(compressed)) + b"IDAT" + compressed + idat_crc + + iend_crc = struct.pack(">I", zlib.crc32(b"IEND") & 0xffffffff) + png += struct.pack(">I", 0) + b"IEND" + iend_crc + return png + + +AVATAR_PNG = make_png() + + +def seed_repositories(): + os.makedirs(REPOS_DIR, exist_ok=True) + for repo_info in REPOSITORIES: + full_name = repo_info["full_name"] + bare_path = os.path.join(REPOS_DIR, f"{full_name}.git") + if os.path.exists(bare_path): + continue + os.makedirs(os.path.dirname(bare_path), exist_ok=True) + fmt = repo_info.get("object_format") or OBJECT_FORMAT + with tempfile.TemporaryDirectory() as tmp_dir: + subprocess.run( + ["git", "init", f"--object-format={fmt}", tmp_dir], + check=True, + capture_output=True, + ) + subprocess.run(["git", "-C", tmp_dir, "checkout", "-B", "main"], check=True, capture_output=True) + for fname, content in repo_info["files"].items(): + fpath = os.path.join(tmp_dir, fname) + os.makedirs(os.path.dirname(fpath), exist_ok=True) + with open(fpath, "w") as f: + f.write(content) + subprocess.run(["git", "-C", tmp_dir, "config", "user.name", "Octo Stub"], check=True, capture_output=True) + subprocess.run(["git", "-C", tmp_dir, "config", "user.email", "octostub@octostub.dev"], check=True, capture_output=True) + subprocess.run(["git", "-C", tmp_dir, "add", "."], check=True, capture_output=True) + subprocess.run(["git", "-C", tmp_dir, "commit", "-m", "Initial commit"], check=True, capture_output=True) + subprocess.run(["git", "clone", "--bare", tmp_dir, bare_path], check=True, capture_output=True) + subprocess.run(["git", "-C", bare_path, "symbolic-ref", "HEAD", "refs/heads/main"], check=True, capture_output=True) + subprocess.run(["git", "-C", bare_path, "config", "http.receivepack", "false"], check=True, capture_output=True) + desc_path = os.path.join(bare_path, "description") + with open(desc_path, "w") as f: + f.write(repo_info["description"] + "\n") + print(f"Seeded bare repository ({fmt}): {bare_path}", flush=True) + + +class GitHubStubHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, format, *args): + sys.stderr.write(f"[{self.log_date_time_string()}] {self.address_string()} {format % args}\n") + sys.stderr.flush() + + def read_body(self) -> bytes: + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + chunks = [] + while True: + line = self.rfile.readline().strip() + if not line: + break + chunk_len = int(line.split(b";")[0], 16) + if chunk_len == 0: + while self.rfile.readline().strip(): + pass + break + chunk = self.rfile.read(chunk_len) + chunks.append(chunk) + self.rfile.readline() + return b"".join(chunks) + length = int(self.headers.get("Content-Length", 0)) + return self.rfile.read(length) if length > 0 else b"" + + def send_json(self, status: int, data: any, link: str = ""): + body = json.dumps(data, indent=2).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + if link: + self.send_header("Link", link) + self.end_headers() + self.wfile.write(body) + + def page_link(self, path: str, query: dict, page: int, per_page: int, total: int) -> str: + if page * per_page >= total: + return "" + params = {k: v[0] for k, v in query.items()} + params.update({"page": str(page + 1), "per_page": str(per_page)}) + nxt = urllib.parse.urlencode(params) + return f'<{API_ORIGIN}{path}?{nxt}>; rel="next"' + + def send_avatar(self): + self.send_response(200) + self.send_header("Content-Type", "image/png") + self.send_header("Content-Length", str(len(AVATAR_PNG))) + self.send_header("Cache-Control", "public, max-age=86400") + self.end_headers() + self.wfile.write(AVATAR_PNG) + + def do_GET(self): + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") + if not path: + path = "/" + query = urllib.parse.parse_qs(parsed.query) + + if path.startswith("/u/") or path.startswith("/avatar") or path.startswith("/avatars/") or path == "/user/avatar": + self.send_avatar() + return + + git_refs_match = re.match(r"^/([^/]+)/([^/]+?)(?:\.git)?/info/refs$", parsed.path) + if git_refs_match: + owner, repo_name = git_refs_match.group(1), git_refs_match.group(2) + repo_name = UNFETCHABLE.get(repo_name, repo_name) + service = query.get("service", [""])[0] + if service != "git-upload-pack": + self.send_error(400, "Unsupported git service") + return + repo_path = os.path.join(REPOS_DIR, owner, f"{repo_name}.git") + if not os.path.exists(repo_path): + self.send_error(404, "Repository not found") + return + env = os.environ.copy() + proto = self.headers.get("Git-Protocol") + if proto: + env["GIT_PROTOCOL"] = proto + env["GIT_CONFIG_NOSYSTEM"] = "1" + proc = subprocess.run( + ["git", "upload-pack", "--stateless-rpc", "--advertise-refs", repo_path], + capture_output=True, + env=env + ) + if proc.returncode != 0: + self.send_error(500, f"git upload-pack failed: {proc.stderr.decode('utf-8', errors='replace')}") + return + body = b"001e# service=git-upload-pack\n0000" + proc.stdout + self.send_response(200) + self.send_header("Content-Type", "application/x-git-upload-pack-advertisement") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-cache, no-store, must-revalidate") + self.send_header("Pragma", "no-cache") + self.end_headers() + self.wfile.write(body) + return + + if path == "/login/oauth/authorize": + redirect_uri = query.get("redirect_uri", [""])[0] + state = query.get("state", [""])[0] + code = "ghstub-" + os.urandom(8).hex() + if not redirect_uri: + self.send_error(400, "Missing redirect_uri") + return + sep = "&" if "?" in redirect_uri else "?" + dest = f"{redirect_uri}{sep}code={code}&state={urllib.parse.quote(state)}" + self.send_response(302) + self.send_header("Location", dest) + self.send_header("Cache-Control", "no-store") + self.send_header("Content-Length", "0") + self.end_headers() + return + + if re.match(r"^/apps/[^/]+/installations/new$", path): + setup = os.environ.get( + "GITHUB_APP_SETUP_URL", + "http://127.0.0.1:5174/_internal/github/install/callback", + ) + sep = "&" if "?" in setup else "?" + state = query.get("state", [""])[0] + dest = f"{setup}{sep}installation_id=1&setup_action=install" + if state: + dest += f"&state={urllib.parse.quote(state)}" + self.send_response(302) + self.send_header("Location", dest) + self.send_header("Cache-Control", "no-store") + self.send_header("Content-Length", "0") + self.end_headers() + return + + if path == "/stub/installed": + html = ( + b"GitHub App Installed" + b"

    GitHub App Installed

    " + b"

    The Tangled GitHub App has been installed on your stub account.

    " + b"" + ) + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(html))) + self.end_headers() + self.wfile.write(html) + return + + if path == "/user": + token = self.headers.get("Authorization", "").removeprefix("Bearer ").strip() + user_data = { + "login": "octostub", + "id": 10001, + "node_id": "MDQ6VXNlcjEwMDAx", + "avatar_url": f"{AVATARS_ORIGIN}/u/10001", + "gravatar_id": "", + "url": f"{API_ORIGIN}/users/octostub", + "html_url": f"{PUBLIC_ORIGIN}/octostub", + "type": "User", + "site_admin": False, + "name": "Octo Stub", + "company": None, + "blog": "https://octostub.example.com", + "location": None, + "email": email_for(token), + "hireable": None, + "bio": "Stub GitHub user for local development", + "public_repos": len(REPOSITORIES), + "public_gists": 0, + "followers": 0, + "following": 0, + "created_at": "2024-01-01T00:00:00Z", + "updated_at": "2024-01-01T00:00:00Z" + } + self.send_json(200, user_data) + return + + if path == "/user/emails": + token = self.headers.get("Authorization", "").removeprefix("Bearer ").strip() + address = email_for(token) + local_suffix = address[len("octostub") : address.index("@")] + self.send_json( + 200, + [ + {**row, "email": row["email"].replace("@octostub.dev", "").replace("octostub", "octostub" + local_suffix) + "@octostub.dev"} + for row in GITHUB_EMAILS + ], + ) + return + + if path == "/user/keys": + keys = [ + { + "id": 1, + "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOr0s3gZf7W/8vV/EXAMPLEEXAMPLEEXAMPLEEXAMPLE octostub@octostub.dev", + "title": "Stub Key" + } + ] + self.send_json(200, keys) + return + + if path == "/user/installations": + installations_data = { + "total_count": 1, + "installations": [ + { + "id": 1, + "account": { + "login": "octostub", + "id": 10001, + "type": "User", + "avatar_url": f"{AVATARS_ORIGIN}/u/10001" + }, + "app_id": 1, + "app_slug": APP_SLUG, + "target_id": 10001, + "target_type": "User", + "permissions": { + "metadata": "read", + "contents": "read" + }, + "events": [], + "repository_selection": "all" + } + ] + } + self.send_json(200, installations_data) + return + + inst_repos_match = re.match(r"^/user/installations/(\d+)/repositories$", path) + if inst_repos_match: + clean_repos = [_clean_repo(r) for r in REPOSITORIES] + page = max(1, int(query.get("page", ["1"])[0])) + per_page = max(1, min(100, int(query.get("per_page", ["30"])[0]))) + self.send_json(200, { + "total_count": len(clean_repos), + "repositories": clean_repos[(page - 1) * per_page : page * per_page] + }, self.page_link(path, query, page, per_page, len(clean_repos))) + return + + if path == "/user/repos": + visibility = query.get("visibility", [""])[0] + affiliation = query.get("affiliation", ["owner,collaborator,organization_member"])[0] + page = max(1, int(query.get("page", ["1"])[0])) + per_page = max(1, min(100, int(query.get("per_page", ["30"])[0]))) + owned = "owner" in {a for a in affiliation.split(",") if a} + rows = [] if not owned else [ + _clean_repo(r) + for r in REPOSITORIES + if not visibility or r.get("visibility") == visibility + ] + self.send_json(200, rows[(page - 1) * per_page : page * per_page], + self.page_link(path, query, page, per_page, len(rows))) + return + + users_repos_match = re.match(r"^/users/([^/]+)/repos$", path) + if users_repos_match: + login = users_repos_match.group(1) + page = max(1, int(query.get("page", ["1"])[0])) + per_page = max(1, min(100, int(query.get("per_page", ["30"])[0]))) + rows = [ + _clean_repo(r) + for r in REPOSITORIES + if r.get("visibility") == "public" and r.get("full_name", "").startswith(f"{login}/") + ] + self.send_json(200, rows[(page - 1) * per_page : page * per_page], + self.page_link(path, query, page, per_page, len(rows))) + return + + repo_match = re.match(r"^/repos/([^/]+)/([^/]+)$", path) + if repo_match: + owner, name = repo_match.group(1), repo_match.group(2) + full_name = f"{owner}/{name}" + for r in REPOSITORIES: + if r["full_name"] == full_name: + self.send_json(200, _clean_repo(r)) + return + self.send_json(404, {"message": "Not Found"}) + return + + if path == "/": + self.send_json(200, { + "status": "ok", + "service": "github-stub", + "client_id": CLIENT_ID, + "app_slug": APP_SLUG, + "repositories": [r["full_name"] for r in REPOSITORIES] + }) + return + + self.send_json(404, {"message": f"Not Found: {path}"}) + + def do_POST(self): + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") + if not path: + path = "/" + + git_pack_match = re.match(r"^/([^/]+)/([^/]+?)(?:\.git)?/git-upload-pack$", parsed.path) + if git_pack_match: + owner, repo_name = git_pack_match.group(1), git_pack_match.group(2) + unfetchable = repo_name in UNFETCHABLE + repo_name = UNFETCHABLE.get(repo_name, repo_name) + repo_path = os.path.join(REPOS_DIR, owner, f"{repo_name}.git") + if not os.path.exists(repo_path): + self.send_error(404, "Repository not found") + return + + body = self.read_body() + content_encoding = self.headers.get("Content-Encoding", "").lower() + if content_encoding == "gzip": + body = zlib.decompress(body, 16 + zlib.MAX_WBITS) + elif content_encoding == "deflate": + body = zlib.decompress(body) + # ls-refs is a POST to this same url, so only the fetch command is refused + if unfetchable and b"command=fetch" in body: + self.send_error(500, "this stub repository never sends its pack") + return + + env = os.environ.copy() + proto = self.headers.get("Git-Protocol") + if proto: + env["GIT_PROTOCOL"] = proto + env["GIT_CONFIG_NOSYSTEM"] = "1" + + proc = subprocess.Popen( + ["git", "upload-pack", "--stateless-rpc", repo_path], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env + ) + stdout, stderr = proc.communicate(input=body) + if proc.returncode != 0: + self.send_error(500, f"git upload-pack failed: {stderr.decode('utf-8', errors='replace')}") + return + + self.send_response(200) + self.send_header("Content-Type", "application/x-git-upload-pack-result") + self.send_header("Content-Length", str(len(stdout))) + self.send_header("Cache-Control", "no-cache, no-store, must-revalidate") + self.send_header("Pragma", "no-cache") + self.end_headers() + self.wfile.write(stdout) + return + + if path == "/login/oauth/access_token": + raw_body = self.read_body() + content_type = self.headers.get("Content-Type", "").lower() + + params = {} + if "application/json" in content_type: + try: + params = json.loads(raw_body.decode("utf-8")) + except Exception: + pass + else: + try: + parsed_qs = urllib.parse.parse_qs(raw_body.decode("utf-8")) + params = {k: v[0] for k, v in parsed_qs.items()} + except Exception: + pass + + token_data = { + "access_token": STUB_TOKEN_PREFIX + + (str(params.get("code", "")).removeprefix("ghstub-") or "0123456789abcdef")[:16], + "token_type": "bearer", + "scope": "repo,read:user,user:email" + } + + accept = self.headers.get("Accept", "").lower() + if "application/json" in accept or "application/json" in content_type: + self.send_json(200, token_data) + else: + body = urllib.parse.urlencode(token_data).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/x-www-form-urlencoded") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + return + + self.send_json(404, {"message": f"Not Found: {path}"}) + + +def run(): + print(f"Starting GitHub Stub on port {PORT}...", flush=True) + seed_repositories() + server = ThreadingHTTPServer(("0.0.0.0", PORT), GitHubStubHandler) + print(f"GitHub Stub listening on 0.0.0.0:{PORT}", flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + + +if __name__ == "__main__": + run() diff --git a/localinfra/ncps-secret-key b/localinfra/ncps-secret-key new file mode 100644 index 0000000000000000000000000000000000000000..cad6b248103cba4c46d1eb535fa079e1058a3e7d --- /dev/null +++ b/localinfra/ncps-secret-key @@ -0,0 +1 @@ +cache.local:hay0+jvBNguou2tNt19FvrBCogHwHc+mqQe3bww5ZX4XtiqkzO4F0gth3+/7Axk3ZgrELOWJdDIWZ57M7HDutQ== diff --git a/localinfra/ncps.yaml b/localinfra/ncps.yaml new file mode 100644 index 0000000000000000000000000000000000000000..751e1c139c4e54566151482ca89aaeee20774fa5 --- /dev/null +++ b/localinfra/ncps.yaml @@ -0,0 +1,16 @@ +# Development only. Not for production use. +cache: + allow-put-verb: true + hostname: "cache.local" + database-url: "sqlite:/storage/db.sqlite" + secret-key-path: "/keys/ncps-secret-key" + sign-narinfo: true + storage: + local: "/storage" + upstream: + urls: + - https://cache.nixos.org + public-keys: + - cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= +server: + addr: "0.0.0.0:8501" diff --git a/localinfra/observability/grafana/provisioning/dashboards/bobbin.json b/localinfra/observability/grafana/provisioning/dashboards/bobbin.json new file mode 100644 index 0000000000000000000000000000000000000000..d0120730d80f46aba39b8211147c64928e488210 --- /dev/null +++ b/localinfra/observability/grafana/provisioning/dashboards/bobbin.json @@ -0,0 +1,1777 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "schemaVersion": 39, + "tags": [ + "bobbin" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(bobbin_ingest_ready, job)", + "hide": 0, + "includeAll": true, + "label": "Service", + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(bobbin_ingest_ready, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(bobbin_ingest_ready{job=~\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(bobbin_ingest_ready{job=~\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m" + ] + }, + "timezone": "browser", + "title": "Bobbin", + "uid": "bobbin-overview", + "version": 1, + "weekStart": "", + "refresh": "30s", + "panels": [ + { + "type": "row", + "title": "overview", + "collapsed": false, + "id": 1, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 2, + "title": "ingest", + "description": "whether ingest has caught up with the firehose", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "warming", + "color": "orange" + }, + "1": { + "text": "ready", + "color": "green" + } + } + } + ], + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "min(bobbin_ingest_ready{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "ingest", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 3, + "title": "hydrant stream", + "description": "whether the hydrant firehose is connected", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "disconnected", + "color": "red" + }, + "1": { + "text": "connected", + "color": "green" + } + } + } + ], + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "min(bobbin_ingest_connected{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "hydrant stream", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 4, + "title": "events", + "description": "hydrant events committed per second", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "mappings": [], + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(bobbin_ingest_events_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "events", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 5, + "title": "xrpc requests", + "description": "xrpc requests per second", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "mappings": [], + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(bobbin_xrpc_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "xrpc requests", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 6, + "title": "xrpc 5xx", + "description": "share of xrpc requests answered with a 5xx", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(sum(rate(bobbin_xrpc_requests_total{job=~\"$job\", instance=~\"$instance\", status=~\"5..\"}[$__rate_interval])) or vector(0)) / sum(rate(bobbin_xrpc_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "xrpc 5xx", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 7, + "title": "resident memory", + "description": "resident set size across instances", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "mappings": [], + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "resident memory", + "range": true, + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "ingest", + "collapsed": false, + "id": 8, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 9, + "title": "records by outcome", + "description": "records off the firehose per second, by what ingest did with them", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 40, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (outcome) (rate(bobbin_ingest_records_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 10, + "title": "rejections", + "description": "rejected records per second by reason and collection. unknown collections show up as other", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (reason, collection) (rate(bobbin_ingest_records_total{job=~\"$job\", instance=~\"$instance\", outcome=\"rejected\"}[$__rate_interval]))", + "legendFormat": "{{reason}} {{collection}}", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 11, + "title": "dropped before indexing", + "description": "record frames and edges dropped before they reached the index", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (reason) (rate(bobbin_ingest_frames_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "frame {{reason}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (reason) (rate(bobbin_ingest_edges_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "edge {{reason}}", + "range": true, + "refId": "B" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 12, + "title": "hydrant disconnects", + "description": "hydrant sessions that ended in an error, by kind", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (kind) (increase(bobbin_ingest_disconnects_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{kind}}", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 13, + "title": "parked upserts", + "description": "upserts waiting on repo did resolution, and what happens to them", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(bobbin_ingest_parked{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "parked now", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (outcome) (rate(bobbin_ingest_parked_outcomes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{outcome}} /s", + "range": true, + "refId": "B" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 14, + "title": "posthog events", + "description": "posthog events per second by whether they reached posthog", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 40, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (result) (rate(bobbin_ingest_analytics_events_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{result}}", + "range": true, + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "xrpc", + "collapsed": false, + "id": 15, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 16, + "title": "requests by route", + "description": "xrpc requests per second by route", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 40, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (route) (rate(bobbin_xrpc_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{route}}", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 17, + "title": "errors by status", + "description": "4xx and 5xx responses per second", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (status, route) (rate(bobbin_xrpc_requests_total{job=~\"$job\", instance=~\"$instance\", status=~\"[45]..\"}[$__rate_interval]))", + "legendFormat": "{{status}} {{route}}", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 18, + "title": "latency", + "description": "xrpc latency quantiles across routes", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(bobbin_xrpc_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(bobbin_xrpc_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(bobbin_xrpc_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 19, + "title": "p95 latency by route", + "description": "95th percentile xrpc latency per route", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, route) (rate(bobbin_xrpc_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "{{route}}", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 20, + "title": "in flight", + "description": "xrpc requests being handled right now", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 40, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (route) (bobbin_xrpc_in_flight{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "{{route}}", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 21, + "title": "heavy requests", + "description": "heavy requests in flight against the limit memory pressure sets, and requests shed as overloaded", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (kind) (bobbin_heavy_requests{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "{{kind}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(bobbin_heavy_rejected_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "rejected /s", + "range": true, + "refId": "B" + } + ] + }, + { + "type": "row", + "title": "resolvers and caches", + "collapsed": false, + "id": 22, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 55 + }, + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 23, + "title": "repo did lookups", + "description": "repo did lookups per second by result", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 40, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (result) (rate(bobbin_resolver_lookups_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{result}}", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 24, + "title": "repo did miss latency", + "description": "average time a repo did miss spent upstream", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(bobbin_resolver_miss_latency_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) / sum(rate(bobbin_resolver_lookups_total{job=~\"$job\", instance=~\"$instance\", result!~\"hit|no_client\"}[$__rate_interval]))", + "legendFormat": "avg", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 25, + "title": "identity cache", + "description": "identity cache hit ratio, and background warming by result", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(bobbin_identity_lookups_total{job=~\"$job\", instance=~\"$instance\", result=\"hit\"}[$__rate_interval])) / sum(rate(bobbin_identity_lookups_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "hit ratio", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (result) (rate(bobbin_identity_warm_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "warm {{result}} /s", + "range": true, + "refId": "B" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 26, + "title": "record cache", + "description": "bytes the record cache holds against its capacity", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (kind) (bobbin_record_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "{{kind}}", + "range": true, + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "process", + "collapsed": false, + "id": 27, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 72 + }, + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 28, + "title": "memory", + "description": "resident set, cgroup usage against the budget, and jemalloc's view", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 73 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "rss", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (kind) (bobbin_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "cgroup {{kind}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (kind) (bobbin_jemalloc_bytes{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "jemalloc {{kind}}", + "range": true, + "refId": "C" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 29, + "title": "cpu", + "description": "cpu cores in use", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 73 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (instance) (rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 30, + "title": "threads and file descriptors", + "description": "os threads and open file descriptors", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 81 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (instance) (process_threads{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "threads {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (instance) (process_open_fds{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "fds {{instance}}", + "range": true, + "refId": "B" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "id": 31, + "title": "uptime", + "description": "time since each instance started", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 81 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "time() - max by (instance) (process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "logs", + "collapsed": false, + "id": 32, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 89 + }, + "panels": [] + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "id": 33, + "title": "warnings and errors", + "type": "logs", + "description": "bobbin log lines at warn or above", + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 90 + }, + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{service_name=~\"$job\"} | severity_text=~\"WARN|ERROR\"", + "queryType": "range", + "refId": "A" + } + ] + } + ] +} diff --git a/localinfra/observability/grafana/provisioning/dashboards/spindle.json b/localinfra/observability/grafana/provisioning/dashboards/spindle.json new file mode 100644 index 0000000000000000000000000000000000000000..af6e3f1d08ef150acab85e15558d94e5f5b5c14c --- /dev/null +++ b/localinfra/observability/grafana/provisioning/dashboards/spindle.json @@ -0,0 +1,2547 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "System Status & Health", + "type": "row", + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 1, + "title": "Targets Up", + "description": "Indicates whether the Spindle instances are successfully scraped by Prometheus.", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "up{job=\"spindle\", instance=~\"$instance\"}", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "unit": "percentunit", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "title": "5xx Error Ratio", + "description": "Ratio of HTTP 5xx errors to total HTTP requests.", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(sum(rate(spindle_http_requests_total{job=\"spindle\", instance=~\"$instance\", route=~\"$route\", status_class=\"5xx\"}[$__rate_interval])) or (0 * sum(rate(spindle_http_requests_total{job=\"spindle\", instance=~\"$instance\", route=~\"$route\"}[$__rate_interval])))) / sum(rate(spindle_http_requests_total{job=\"spindle\", instance=~\"$instance\", route=~\"$route\"}[$__rate_interval]))", + "legendFormat": "5xx Error Ratio", + "range": true, + "refId": "A" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 200, + "title": "HTTP Server API Performance", + "type": "row", + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 10, + "title": "HTTP Request Rate & In-Flight", + "description": "Total count of HTTP requests per second split by method, route, and status class, and current in-flight requests.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_http_requests_total{job=\"spindle\", instance=~\"$instance\", route=~\"$route\"}[$__rate_interval])) by (route, method, status_class)", + "legendFormat": "{{method}} {{route}} ({{status_class}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_http_requests_in_flight{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "In-Flight Requests", + "range": true, + "refId": "B" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 11, + "title": "HTTP Request Latency Percentiles", + "description": "HTTP request latency percentiles (p50, p90, p99) across all endpoints.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(spindle_http_request_duration_seconds_bucket{job=\"spindle\", instance=~\"$instance\", route=~\"$route\"}[$__rate_interval])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "A", + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(spindle_http_request_duration_seconds_bucket{job=\"spindle\", instance=~\"$instance\", route=~\"$route\"}[$__rate_interval])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B", + "exemplar": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(spindle_http_request_duration_seconds_bucket{job=\"spindle\", instance=~\"$instance\", route=~\"$route\"}[$__rate_interval])) by (le))", + "legendFormat": "p50", + "range": true, + "refId": "C", + "exemplar": true + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 300, + "title": "Workflow & Step Execution", + "type": "row", + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 13, + "title": "Active Workflows & Steps", + "description": "Current active workflow and step counts by engine type.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_workflows_active{job=\"spindle\", instance=~\"$instance\"}) by (engine)", + "legendFormat": "{{engine}} Active Workflows", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_steps_active{job=\"spindle\", instance=~\"$instance\"}) by (engine)", + "legendFormat": "{{engine}} Active Steps", + "range": true, + "refId": "B" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 14, + "title": "Workflow & Step Completion Rates", + "description": "Rate of completed workflows and steps per second.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_workflows_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (engine, result)", + "legendFormat": "{{engine}} Completed: {{result}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_steps_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (engine, status)", + "legendFormat": "{{engine}} Step Completed: {{status}}", + "range": true, + "refId": "B" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 16, + "title": "Workflow & Step Latency & Durations", + "description": "Workflow and step execution duration percentiles and average durations.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(spindle_workflow_duration_seconds_bucket{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (le, engine))", + "legendFormat": "{{engine}} Workflow p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(spindle_workflow_duration_seconds_bucket{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (le, engine))", + "legendFormat": "{{engine}} Workflow p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(spindle_step_duration_seconds_bucket{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (le, engine))", + "legendFormat": "{{engine}} Step p99", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(spindle_step_duration_seconds_bucket{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (le, engine))", + "legendFormat": "{{engine}} Step p90", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_workflow_duration_seconds_sum{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) / sum(rate(spindle_workflow_duration_seconds_count{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Avg Workflow Duration", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_step_duration_seconds_sum{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) / sum(rate(spindle_step_duration_seconds_count{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Avg Step Duration", + "range": true, + "refId": "F" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 400, + "title": "Ingestion, Queues, & Placement", + "type": "row", + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 20, + "title": "Ingestion, Job Queues & Mill Placements", + "description": "Event ingestion rate, job queue activity rate, mill placement admission rate, and execution outcome rates.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_event_ingestion_outcomes_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (consumer, status)", + "legendFormat": "Consumer: {{consumer}} Status: {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_job_queue_activity_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (action)", + "legendFormat": "Action: {{action}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_mill_placement_admission_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (status, reason)", + "legendFormat": "Admission: {{status}} ({{reason}})", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_mill_placement_results_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (result)", + "legendFormat": "Result: {{result}}", + "range": true, + "refId": "D" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 500, + "title": "Admission Quotas & Decisions", + "type": "row", + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "suffix: MiB" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 24, + "title": "Engine Resource Pool", + "description": "Physical scheduler pool of the local engine (used, limit, max_request). This is host capacity, not per-user or per-repository quota.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (state) (spindle_engine_pool_memory_mib{job=\"spindle\", instance=~\"$instance\", state=~\"used|limit\"})", + "legendFormat": "Memory {{state}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "max by (state) (spindle_engine_pool_memory_mib{job=\"spindle\", instance=~\"$instance\", state=\"max_request\"})", + "legendFormat": "Memory {{state}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (state) (spindle_engine_pool_vcpus{job=\"spindle\", instance=~\"$instance\", state=~\"used|limit\"})", + "legendFormat": "vCPUs {{state}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "max by (state) (spindle_engine_pool_vcpus{job=\"spindle\", instance=~\"$instance\", state=\"max_request\"})", + "legendFormat": "vCPUs {{state}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by (state) (spindle_engine_pool_disk_mib{job=\"spindle\", instance=~\"$instance\", state=~\"used|limit\"})", + "legendFormat": "Disk {{state}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "max by (state) (spindle_engine_pool_disk_mib{job=\"spindle\", instance=~\"$instance\", state=\"max_request\"})", + "legendFormat": "Disk {{state}}", + "range": true, + "refId": "F" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 27, + "title": "Quota Decisions & Rejection Rate", + "description": "Rate of quota admission decisions by kind, resource, decision and reason, with the share of decisions that were not granted immediately.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_quota_decisions_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (kind, resource, decision, reason)", + "legendFormat": "{{kind}}/{{resource}} {{decision}} ({{reason}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(sum(rate(spindle_quota_decisions_total{job=\"spindle\", instance=~\"$instance\", decision!=\"allowed\"}[$__rate_interval])) by (kind) or (sum(rate(spindle_quota_decisions_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (kind) * 0)) / clamp_min(sum(rate(spindle_quota_decisions_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (kind), 0.000001)", + "legendFormat": "Rejection rate: {{kind}}", + "range": true, + "refId": "B" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 31 + }, + "id": 28, + "title": "Quota Waits & Engine Pool Pressure", + "description": "Requests queued by the fair quota manager, the p95 time they wait before a grant, and the local engine scheduler queue and admission rate for comparison. Engine pool series are host capacity, not tenant quota.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_quota_wait_depth{job=\"spindle\", instance=~\"$instance\"}) by (kind)", + "legendFormat": "Waiting: {{kind}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(spindle_quota_wait_duration_seconds_bucket{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (le, kind, resource))", + "legendFormat": "p95 wait: {{kind}}/{{resource}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_engine_pool_queue_depth{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Engine pool queue", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_engine_pool_admission_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (decision, reason)", + "legendFormat": "Engine pool: {{decision}} ({{reason}})", + "range": true, + "refId": "D" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 600, + "title": "Mill & Executor States", + "type": "row", + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "unit", + "value": "ops" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 29, + "title": "Mill States & Sessions", + "description": "Mill executor sessions, job queue, limits, and lease/reservation occupancy. No data is displayed if the selected instance does not run in the Mill role.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_mill_active_sessions{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Active Sessions", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_mill_disconnected_sessions{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Disconnected Sessions", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_mill_reconnects_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Reconnects Rate", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_mill_pending_jobs{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Pending Jobs", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_mill_max_pending_jobs{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Max Pending Jobs", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_mill_leases_active{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Active Leases", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_mill_reservations_active{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Active Seat Reservations", + "range": true, + "refId": "G" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 31, + "title": "Executor States & Buffers", + "description": "Executor active seat reservations, running jobs, and outbox buffer size in bytes. No data is displayed if the selected instance does not run in the Executor role.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_executor_reservations_active{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Reservations Active", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_executor_jobs_active{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Jobs Active", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_executor_outbox_bytes{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Outbox Bytes", + "range": true, + "refId": "C" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 1000, + "title": "Quota Usage & Cache Uploads", + "type": "row", + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 47 + }, + "id": 1001, + "title": "Cache Storage Quota Usage & Subjects", + "description": "Committed cache storage quota usage, configured defaults (zero means unlimited), and count of quota subjects by scope and status.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_quota_usage{job=\"spindle\", instance=~\"$instance\", resource=\"cache_storage_bytes\"}) by (scope)", + "legendFormat": "Usage: {{scope}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_quota_subjects{job=\"spindle\", instance=~\"$instance\", resource=\"cache_storage_bytes\"}) by (scope, status)", + "legendFormat": "Subjects: {{scope}} ({{status}})", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "spindle_quota_default_limit{job=\"spindle\", instance=~\"$instance\", resource=\"cache_storage_bytes\"}", + "legendFormat": "Default limit: {{scope}}", + "range": true, + "refId": "C" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 47 + }, + "id": 1002, + "title": "Workflow, vCPU, Memory & Disk Quota Usage", + "description": "Committed workflow, vCPU, memory (MiB) and disk (MiB) quota usage, configured defaults (zero means unlimited), and subject counts by scope and resource.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_quota_usage{job=\"spindle\", instance=~\"$instance\", resource=~\"workflows|vcpus|memory_mib|disk_mib\"}) by (scope, resource)", + "legendFormat": "Usage: {{scope}} {{resource}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_quota_subjects{job=\"spindle\", instance=~\"$instance\", resource=~\"workflows|vcpus|memory_mib|disk_mib\"}) by (scope, resource, status)", + "legendFormat": "Subjects: {{scope}} {{resource}} ({{status}})", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "spindle_quota_default_limit{job=\"spindle\", instance=~\"$instance\", resource=~\"workflows|vcpus|memory_mib|disk_mib\"}", + "legendFormat": "Default limit: {{scope}} {{resource}}", + "range": true, + "refId": "C" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 47 + }, + "id": 1003, + "title": "Cache Uploads (Count & Bytes)", + "description": "Rate of cache upload completions (outcome counts and throughput bytes/sec) by backend and result.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_cache_uploads_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (backend, result)", + "legendFormat": "Uploads: {{backend}} ({{result}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_cache_upload_bytes_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (backend, result)", + "legendFormat": "Bytes: {{backend}} ({{result}})", + "range": true, + "refId": "B" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 700, + "title": "Database Connection Pool & Performance", + "type": "row", + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 54 + }, + "id": 28, + "title": "Database Connection Counts", + "description": "Database connection pool status (limits, open, in-use, idle connections).", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_db_max_open_connections{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Max Open", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_db_open_connections{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Open", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_db_in_use_connections{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "In Use", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_db_idle_connections{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Idle", + "range": true, + "refId": "D" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 54 + }, + "id": 29, + "title": "Database Pool Rates & Failures", + "description": "Database pool operation rates, closed connection rates, and collection failures.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_db_wait_count{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Wait Count Rate", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_db_max_idle_closed{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Max Idle Closed Rate", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_db_max_lifetime_closed{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Max Lifetime Closed Rate", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_collection_failures_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Collection Failures Rate", + "range": true, + "refId": "D" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 54 + }, + "id": 30, + "title": "Average Database Wait Duration", + "description": "Average time spent blocked for each database connection wait.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_db_wait_duration_seconds{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) / sum(rate(spindle_db_wait_count{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Average Wait Duration", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 54 + }, + "id": 31, + "title": "Database Workflow & Queue Counts", + "description": "Current workflow counts by status and jobs waiting in the durable database queue.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_db_workflow_status_count{job=\"spindle\", instance=~\"$instance\"}) by (status)", + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_db_queued_jobs{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "queued jobs", + "range": true, + "refId": "B" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + } + ] + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 800, + "title": "Debug Jump System", + "type": "row", + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 32, + "title": "Debug Jump Connections", + "description": "Current active and maximum allowed debug jump connections.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_jump_active_connections{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Active Connections", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(spindle_jump_max_connections{job=\"spindle\", instance=~\"$instance\"})", + "legendFormat": "Max Limit", + "range": true, + "refId": "B" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 33, + "title": "Debug Jump Rejections Rate", + "description": "Rate of rejected debug jump connections by rejection reason.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(spindle_jump_rejections_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (reason)", + "legendFormat": "Rejection: {{reason}}", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + } + ] + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 900, + "title": "Go & Process Runtime", + "type": "row", + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 70 + }, + "id": 7, + "title": "Process CPU Usage", + "description": "CPU usage in terms of cores utilized per instance.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(process_cpu_seconds_total{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "legendFormat": "{{instance}} CPU Cores", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 70 + }, + "id": 8, + "title": "Process Memory Usage", + "description": "Resident memory size (RSS) per instance.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{job=\"spindle\", instance=~\"$instance\"}) by (instance)", + "legendFormat": "{{instance}} Memory", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 70 + }, + "id": 9, + "title": "Open File Descriptors", + "description": "Current and maximum allowed open file descriptors.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(process_open_fds{job=\"spindle\", instance=~\"$instance\"}) by (instance)", + "legendFormat": "{{instance}} Open FDs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(process_max_fds{job=\"spindle\", instance=~\"$instance\"}) by (instance)", + "legendFormat": "{{instance}} Max FDs", + "range": true, + "refId": "B" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 76 + }, + "id": 34, + "title": "Go Concurrency", + "description": "Total number of active Go goroutines and OS threads.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(go_goroutines{job=\"spindle\", instance=~\"$instance\"}) by (instance)", + "legendFormat": "{{instance}} Goroutines", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(go_threads{job=\"spindle\", instance=~\"$instance\"}) by (instance)", + "legendFormat": "{{instance}} OS Threads", + "range": true, + "refId": "B" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 76 + }, + "id": 35, + "title": "Go Heap Memory", + "description": "Allocated heap bytes and active heap bytes currently in-use.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(go_memstats_alloc_bytes{job=\"spindle\", instance=~\"$instance\"}) by (instance)", + "legendFormat": "{{instance}} Heap Alloc", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(go_memstats_heap_inuse_bytes{job=\"spindle\", instance=~\"$instance\"}) by (instance)", + "legendFormat": "{{instance}} Heap In-Use", + "range": true, + "refId": "B" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 76 + }, + "id": 36, + "title": "Go GC Rate", + "description": "Rate of garbage collection cycles per second.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(go_gc_duration_seconds_count{job=\"spindle\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "legendFormat": "{{instance}} GC Rate", + "range": true, + "refId": "A" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 1004, + "title": "Workflow Logs & Traces", + "type": "row", + "panels": [] + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 1005, + "title": "Recent Workflow Lifecycle Logs", + "description": "Loki log stream for recent spindle workflow execution events.", + "type": "logs", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{service_name=\"spindle\"} | workflow_id =~ \"$workflow_id\"", + "queryType": "range", + "refId": "A" + } + ], + "options": { + "showLabels": false, + "wrapLogMessage": true, + "enableLogDetails": true, + "showCommonLabels": false, + "showTime": true, + "sortOrder": "Descending" + } + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 1006, + "title": "Recent Workflow Failures", + "description": "Completed spindle workflows whose result was not success.", + "type": "logs", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{service_name=\"spindle\"} |= \"workflow finished\" | result != \"success\" | workflow_id =~ \"$workflow_id\"", + "queryType": "range", + "refId": "A" + } + ], + "options": { + "showLabels": false, + "wrapLogMessage": true, + "enableLogDetails": true, + "showCommonLabels": false, + "showTime": true, + "sortOrder": "Descending" + } + } + ], + "schemaVersion": 39, + "style": "dark", + "tags": [ + "spindle", + "production", + "operations" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "", + "current": { + "selected": true, + "value": [ + "$__all" + ], + "text": "All" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(up{job=\"spindle\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(up{job=\"spindle\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": true, + "value": [ + "$__all" + ], + "text": "All" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(spindle_http_requests_total{job=\"spindle\", instance=~\"$instance\"}, route)", + "hide": 0, + "includeAll": true, + "label": "Route", + "multi": true, + "name": "route", + "options": [], + "query": { + "query": "label_values(spindle_http_requests_total{job=\"spindle\", instance=~\"$instance\"}, route)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": ".*", + "value": ".*" + }, + "description": "regular expression matched against structured workflow_id metadata", + "hide": 0, + "label": "Workflow ID", + "name": "workflow_id", + "options": [], + "query": ".*", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m" + ] + }, + "timezone": "browser", + "title": "Spindle Overview", + "uid": "efv22ywqutn28f", + "version": 1, + "weekStart": "", + "refresh": "10s" +} \ No newline at end of file diff --git a/localinfra/observability/grafana/provisioning/dashboards/spindle.yml b/localinfra/observability/grafana/provisioning/dashboards/spindle.yml new file mode 100644 index 0000000000000000000000000000000000000000..d938f5792d4f2d566f143a2ff7b5cb82e1ab586e --- /dev/null +++ b/localinfra/observability/grafana/provisioning/dashboards/spindle.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'Spindle Dashboards' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/localinfra/observability/grafana/provisioning/datasources/prometheus.yml b/localinfra/observability/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000000000000000000000000000000000000..e86f7f44db20b988c3253792a68926c9aabf655e --- /dev/null +++ b/localinfra/observability/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,44 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + httpMethod: POST + exemplarTraceIdDestinations: + - name: traceID + datasourceUid: tempo + + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + uid: tempo + jsonData: + tracesToLogsV2: + datasourceUid: 'loki' + spanStartTimeShift: '-1h' + spanEndTimeShift: '1h' + tags: + - key: 'service.name' + filterByTraceID: true + filterBySpanID: false + customQuery: true + query: '{service_name="$${__span.tags["service.name"]}"} | trace_id="$${__trace.traceId}"' + + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + uid: loki + jsonData: + derivedFields: + - datasourceUid: tempo + matcherRegex: 'trace_id' + matcherType: label + name: trace_id + urlDisplayLabel: 'View trace' + url: '$${__value.raw}' diff --git a/localinfra/observability/loki/loki.yml b/localinfra/observability/loki/loki.yml new file mode 100644 index 0000000000000000000000000000000000000000..af41365d0fd7a0444dbad9cde316da0612b15e9c --- /dev/null +++ b/localinfra/observability/loki/loki.yml @@ -0,0 +1,32 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + reject_old_samples: true + reject_old_samples_max_age: 168h diff --git a/localinfra/observability/prometheus/prometheus.yml b/localinfra/observability/prometheus/prometheus.yml new file mode 100644 index 0000000000000000000000000000000000000000..37b32a93b13695363c72d6e0b040df19aafcf077 --- /dev/null +++ b/localinfra/observability/prometheus/prometheus.yml @@ -0,0 +1,9 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'spindle' + static_configs: + - targets: + - 'spindle:9091' diff --git a/localinfra/observability/tempo/tempo.yml b/localinfra/observability/tempo/tempo.yml new file mode 100644 index 0000000000000000000000000000000000000000..4c3fb5bc0534ca539e3057193539d49921471247 --- /dev/null +++ b/localinfra/observability/tempo/tempo.yml @@ -0,0 +1,23 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +usage_report: + reporting_enabled: false diff --git a/localinfra/ozone.env b/localinfra/ozone.env new file mode 100644 index 0000000000000000000000000000000000000000..9c19a5914407dd4aafb01ffc2ca534333cadf860 --- /dev/null +++ b/localinfra/ozone.env @@ -0,0 +1,14 @@ +# Development only. Not for production use. +OZONE_SERVER_DID=did:plc:zu6y5f5sbudyxhv5ytiirkjx +OZONE_PUBLIC_URL=https://ozone.tngl.boltless.dev +# alice +OZONE_ADMIN_DIDS=did:plc:igzuueb63dytyy3tpaweofuu +OZONE_ADMIN_PASSWORD=f9e80b32515443a6429d745c1e5eaa9e +# signs labels; init-data publishes its did:key as the labeler's #atproto_label +OZONE_SIGNING_KEY_HEX=790340973101cb4b24c15e8ef7667d4fc7a686100a69ef8514b819db5de167fa +OZONE_DB_POSTGRES_URL=postgresql://tnglr:tnglr@postgres:5432/ozone +OZONE_DB_MIGRATE=1 +OZONE_DID_PLC_URL=http://plc:8080 +OZONE_APPVIEW_URL=http://pds:3000 +OZONE_APPVIEW_DID=did:web:pds.tngl.boltless.dev +LOG_ENABLED=1 diff --git a/localinfra/pc/Caddyfile b/localinfra/pc/Caddyfile new file mode 100644 index 0000000000000000000000000000000000000000..9847d4479d2bb9e1366111afde51e4e84f4be701 --- /dev/null +++ b/localinfra/pc/Caddyfile @@ -0,0 +1,59 @@ +{ + admin off + auto_https disable_redirects + skip_install_trust + https_port {$TNGL_HTTPS_PORT} + storage file_system {$TNGL_STATE}/data/caddy + + # the root comes from $TNGL_CERTS so a rebuild keeps the ca people already trust + pki { + ca local { + name "Tangled Dev" + root_cn "Tangled Dev Root" + root { + format pem_file + cert {$TNGL_CERTS}/root.crt + key {$TNGL_CERTS}/root.key + } + } + } +} + +plc.tngl.boltless.dev { + tls internal + reverse_proxy 127.0.0.1:{$TNGL_PLC_PORT} +} + +*.pds.tngl.boltless.dev, pds.tngl.boltless.dev { + tls internal + reverse_proxy 127.0.0.1:{$TNGL_PDS_PORT} +} + +hydrant.tngl.boltless.dev { + tls internal + reverse_proxy 127.0.0.1:{$TNGL_HYDRANT_PORT} +} + +# the knot answers cors preflight itself, so this block skips the import +knot2.tngl.boltless.dev, knot2-alt.tngl.boltless.dev { + tls internal + reverse_proxy 127.0.0.1:{$TNGL_KNOT_PORT} +} + +# a doubled Access-Control-Allow-Origin gets the response rejected, so only import where the upstream has none +(cors) { + @cors_preflight method OPTIONS + handle @cors_preflight { + header Access-Control-Allow-Origin "*" + header Access-Control-Allow-Methods "GET, POST, OPTIONS" + header Access-Control-Allow-Headers "Content-Type, authorization, atproto-proxy" + header Access-Control-Max-Age "86400" + respond 204 + } + handle { + header Access-Control-Allow-Origin "*" + header Access-Control-Allow-Methods "GET, POST, OPTIONS" + header Access-Control-Allow-Headers "Content-Type, authorization, atproto-proxy" + reverse_proxy {args[0]} + } +} diff --git a/localinfra/pc/README.md b/localinfra/pc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..06e63f23762c340d0214859a2a992a4707b5718f --- /dev/null +++ b/localinfra/pc/README.md @@ -0,0 +1,114 @@ +# pc + +process-compose runs the localinfra stack as plain processes on your machine. each service +is wrapped in `offline`, so it only reaches loopback unless its yaml says otherwise. + +```sh +nix run .#localinfra # start the stack +bin/up # without nix, running doctor and setup first +bin/up --online # skip offline sandboxing +bin/up --profile spindle # add an optional profile +bin/pc process list # check running services +bin/pc process restart knot2 # restart one service +bin/pc down # stop the stack +``` + +`nix run .#localinfra` brings the monorepo toolchain, so it needs no `nix develop` and +runs fine from inside one. extra arguments go to `bin/up`. without nix, `bin/up` runs +`bin/doctor` for missing programs, then `bin/setup` to fetch and build. + +`bin/up --exec selftest` checks that the offline sandbox works. `TNGL_ONLINE=1` disables +it. + +## optional services + +some services only run under a profile: + +- `spindle`: spindle with the dummy engine, so no kvm +- `ncps`: ncps and its migration, the nix cache spindle uses +- `microvm`: linux only. spindle with the real microvm engine, implying `spindle` and + `ncps`. wants `/dev/kvm`, `/dev/vhost-vsock` and `/dev/net/tun` writable +- `observability`: prometheus, loki, tempo, otlp gateway and grafana +- `pdsls`: pds browser, needs bun and a cpu with avx2 +- `sites`: the sites worker under wrangler with its local r2 bridge, and sitesd + +pass `--profile ` to `bin/up`, or set `TNGL_PROFILES=a,b`. + +## swapping what a service runs + +`localinfra/pc/.env` overrides commands, ports and urls. `FOO_BIN` swaps a service's +program and keeps our flags, `FOO_CMD` replaces the whole command as a shell string that +runs after its env is set. names and defaults are in the `commands.env` files. + +services run what is already built, so build and restart to pick up edits. knot2, bobbin, +knotmirror and tranquil-gate build into this checkout's `target/`: + +```sh +cargo build -p knot-server && bin/pc process restart knot2 +``` + +to run your own checkout, put the build in the command: + +```sh +PDS_CMD='cd ~/proj/atproto/services/pds && exec node --enable-source-maps --import=@atproto/pds/telemetry index.ts' +PLC_CMD='cd ~/proj/did-method-plc/packages/server && DATABASE_URL=postgres://tnglr:tnglr@127.0.0.1:$TNGL_PG_PORT/plc PORT=$TNGL_PLC_PORT exec node dist/bin.js' +``` + +your plc checkout has to be built already for `PLC_CMD` (`node build.js`), and the atproto +one for `PDS_CMD` (`pnpm build`). + +on macos cargo builds here need `nix develop`, because `.cargo/config.toml` names a nix +linker. + +`process-compose.override.yaml` merges over the stack yaml: `is_disabled: "true"` drops a +service, a `watch` block restarts one when files change. + +## fixtures + +`init-data` runs `localinfra/scripts/init-data.sh` for the accounts and repos compose has. +later runs skip it while `.localinfra/data/init-data.stamp` matches, delete the stamp to +reseed. after a failed import delete `.localinfra/data/knot2`, since a rerun skips repos +the knot already has. + +## machine setup + +- trust `localinfra/certs/root.crt` in your browser. +- host dns: `*.tngl.boltless.dev` must resolve on the host, the pds checks handles over + dns. on macos: + + ```sh + sudo mkdir -p /etc/resolver + printf 'nameserver 127.0.0.1\nport 5354\n' | sudo tee /etc/resolver/tngl.boltless.dev + ``` +- linux ports: `sysctl net.ipv4.ip_unprivileged_port_start=443` lets the forwarder bind + 443 without root, and `doctor` says so when it can't. +- loopback alias: if using `TNGL_IP=11.0.0.253` (for a hydrant older than + `HYDRANT_ALLOW_PRIVATE_NETWORKS`), macos needs `sudo ifconfig lo0 alias 11.0.0.253/32` + (lost on reboot) and `--online`. +- caddy binds 443. if colima or podman holds it, set `TNGL_HTTPS_PORT` in + `localinfra/pc/.env`. + +## layout + +``` +process-compose.yaml dns, postgres, pg-init, plc, caddy, pds, init-data, knot2, hydrant +Caddyfile shared config, each service adds its vhosts +ports.env host ports, and TNGL_FORWARD +commands.env *_BIN defaults of the core programs +airgap.sb macos sandbox profile behind offline +bin/ + up parses its flags, runs setup, doctor, then process-compose + setup the only part that may use the internet + doctor checks the needed programs and the host + env.sh sets TNGL_* and PATH, sources every service's env files + services lists the active services + profile-on exit 0 when a profile is on + offline wraps a command so its only way out is loopback + linux-ns, ns-inner linux user and network namespace + pc process-compose against the stack's socket + tcp-up exit 0 once something accepts connections on a host and port + selftest checks the offline wrapper, run as `up --exec selftest` + patch-workerd points npm's workerd at the nix loader on nixos + svc/ launch scripts for the core services +services/ one directory per service, see services/README.md +``` diff --git a/localinfra/pc/airgap.sb b/localinfra/pc/airgap.sb new file mode 100644 index 0000000000000000000000000000000000000000..2cb9e42327c572d8859a8e5169bfb9fa3caa742a --- /dev/null +++ b/localinfra/pc/airgap.sb @@ -0,0 +1,5 @@ +(version 1) +(allow default) +(deny network-outbound (with message "tngl-offline")) +(allow network-outbound (remote ip "localhost:*")) +(allow network-outbound (remote unix-socket)) diff --git a/localinfra/pc/bin/doctor b/localinfra/pc/bin/doctor new file mode 100755 index 0000000000000000000000000000000000000000..f4af7b3fda5fcab42a2bc2dca5c4f979e28230ac --- /dev/null +++ b/localinfra/pc/bin/doctor @@ -0,0 +1,67 @@ +#!/bin/sh +here=$(cd "$(dirname "$0")/.." && pwd) +. "$here/bin/env.sh" + +missing="" +need() { + for c in "$@"; do + command -v "$c" >/dev/null 2>&1 || missing="$missing $c" + done +} + +need process-compose pg_isready psql dig curl jq git go openssl +[ -n "${PLC_CMD:-}" ] || need git node yarn +[ -n "${POSTGRES_CMD:-}" ] || need postgres initdb +[ -n "${CADDY_CMD:-}" ] || need "$CADDY_BIN" +[ -n "${DNS_CMD:-}" ] || need "$DNS_BIN" +[ -n "${PDS_CMD:-}" ] || need "$PDS_BIN" +[ -n "${HYDRANT_CMD:-}" ] || need "$HYDRANT_BIN" +if [ -z "${KNOT2_CMD:-}" ]; then + if [ "$KNOT2_BIN" = "$KNOT2_BUILT" ]; then need cargo; else need "$KNOT2_BIN"; fi +fi +for svc in $(services); do + [ -f "$here/services/$svc/doctor" ] && . "$here/services/$svc/doctor" +done +case "$(uname -s)" in +Darwin) + [ -x /usr/bin/sandbox-exec ] || missing="$missing sandbox-exec" + case "$TNGL_IP" in + 127.*) ;; + *) + [ "${TNGL_ONLINE:-0}" = 1 ] || { + echo "the macos sandbox only allows loopback, so offline services can't reach $TNGL_IP. use a hydrant with HYDRANT_ALLOW_PRIVATE_NETWORKS, or --online" >&2 + exit 1 + } + ifconfig lo0 | grep -q "inet $TNGL_IP " || { + echo "names resolve to $TNGL_IP and loopback has to answer for it: sudo ifconfig lo0 alias $TNGL_IP/32" >&2 + exit 1 + } + ;; + esac + ;; +Linux) + need unshare setpriv nsenter socat ip iptables ip6tables sysctl awk + command -v slirp4netns >/dev/null 2>&1 || echo "note: no slirp4netns, so nothing in the stack can reach the internet" >&2 + if ! grep -q "^$(id -un):" /etc/subgid 2>/dev/null || ! command -v newgidmap >/dev/null 2>&1; then + echo "need a /etc/subgid entry for $(id -un) and newgidmap, the offline marker is a mapped group" >&2 + exit 1 + fi + unshare --user true 2>/dev/null || { + echo "unprivileged user namespaces are off (ubuntu 24.04+ restricts them with apparmor)" >&2 + exit 1 + } + limit=$(sysctl -n net.ipv4.ip_unprivileged_port_start 2>/dev/null || echo 1024) + [ "$(id -u)" = 0 ] || for m in $(echo "${TNGL_FORWARD:-}" | tr , ' '); do + [ "${m%%:*}" -ge "$limit" ] || { + echo "TNGL_FORWARD binds ${m%%:*} on the host, which is below net.ipv4.ip_unprivileged_port_start=$limit. lower it (sudo sysctl -w net.ipv4.ip_unprivileged_port_start=${m%%:*}) or forward a higher port, like TNGL_FORWARD=8443:443" >&2 + exit 1 + } + done + ;; +esac + +if [ -n "$missing" ]; then + echo "missing:$missing" >&2 + echo "with nix, run this through: nix run .#localinfra (it brings go and cargo along)" >&2 + exit 1 +fi diff --git a/localinfra/pc/bin/env.sh b/localinfra/pc/bin/env.sh new file mode 100644 index 0000000000000000000000000000000000000000..98c3a9398108b55dddef2f6d490eb514d6f5c9f1 --- /dev/null +++ b/localinfra/pc/bin/env.sh @@ -0,0 +1,25 @@ +set -a +[ -f "$here/.env" ] && . "$here/.env" +set +a + +TNGL_PC=$here +TNGL_ROOT=$(cd "$here/../.." && pwd) +TNGL_STATE=${TNGL_STATE:-$TNGL_ROOT/.localinfra} +TNGL_CERTS=${TNGL_CERTS:-$TNGL_ROOT/localinfra/certs} +TNGL_OFFLINE_GID=${TNGL_OFFLINE_GID:-5000} + +set -a +. "$here/ports.env" +. "$here/commands.env" +for svc in $("$here/bin/services"); do + for f in ports.env commands.env; do + [ -f "$here/services/$svc/$f" ] && . "$here/services/$svc/$f" + done +done +set +a + +export TNGL_PC TNGL_ROOT TNGL_STATE TNGL_CERTS TNGL_OFFLINE_GID +# stack hosts resolve to loopback, which the did resolver refuses without this +export IDRESOLVER_DEV=1 +PATH=$TNGL_PC/bin:$PATH +export PATH diff --git a/localinfra/pc/bin/linux-ns b/localinfra/pc/bin/linux-ns new file mode 100755 index 0000000000000000000000000000000000000000..7a61fd751dcda481f8c8406f43501a028c2aaf21 --- /dev/null +++ b/localinfra/pc/bin/linux-ns @@ -0,0 +1,51 @@ +#!/bin/sh +set -eu +here=$(cd "$(dirname "$0")/.." && pwd) +. "$here/bin/env.sh" + +run=$TNGL_STATE/run +mkdir -p "$run/empty" +pidf=$run/ns.pid +rm -f "$pidf" + +# slirp4netns serves dns at 10.0.2.3, so that is the upstream +command -v slirp4netns >/dev/null 2>&1 && export TNGL_DNS_UPSTREAM=10.0.2.3 + +uid=$(id -u) +gid=$(id -g) +sub=$(awk -F: -v u="$(id -un)" '$1 == u { print $2; exit }' /etc/subgid) + +# postgres refuses root, so the uid maps to itself, and the offline gid goes to a subgid so files stay yours +unshare --user --keep-caps --map-users="$uid:$uid:1" \ + --map-groups="$gid:$gid:1" --map-groups="$TNGL_OFFLINE_GID:$sub:1" \ + --net --mount sh "$TNGL_PC/bin/ns-inner" "$pidf" "$@" & +ns=$! + +jobs_to_kill="$ns" +cleanup() { + # shellcheck disable=SC2086 + kill $jobs_to_kill 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +while [ ! -s "$pidf" ]; do + kill -0 "$ns" 2>/dev/null || exit 1 + sleep 0.1 +done + +if command -v slirp4netns >/dev/null 2>&1; then + slirp4netns --configure --mtu=65520 --disable-host-loopback "$ns" tap0 >"$run/slirp.log" 2>&1 & + jobs_to_kill="$jobs_to_kill $!" +fi + +for m in $(echo "${TNGL_FORWARD:-}" | tr , ' '); do + host=${m%%:*} + inner=${m##*:} + socat "TCP-LISTEN:$host,bind=127.0.0.1,fork,reuseaddr" \ + EXEC:"nsenter --user --preserve-credentials --net -t $ns socat STDIO TCP\\:127.0.0.1\\:$inner" & + jobs_to_kill="$jobs_to_kill $!" +done + +status=0 +wait "$ns" || status=$? +exit "$status" diff --git a/localinfra/pc/bin/ns-inner b/localinfra/pc/bin/ns-inner new file mode 100755 index 0000000000000000000000000000000000000000..61618bf3f1db2bd8294645bf98c0f9dcefd6ed91 --- /dev/null +++ b/localinfra/pc/bin/ns-inner @@ -0,0 +1,31 @@ +#!/bin/sh +# setgid has to survive the drop or the children can't take the offline gid +set -eu +echo $$ > "$1" +shift + +ip link set lo up +# names resolve to TNGL_IP, so lo has to hold it or local connects fail +case "$TNGL_IP" in +127.*) ;; +*) ip addr add "$TNGL_IP/32" dev lo ;; +esac +sysctl -qw net.ipv4.ip_unprivileged_port_start=0 +# stop outgoing connects from grabbing a service port before its service binds +sysctl -qw net.ipv4.ip_local_reserved_ports="$(env | sed -n 's/^TNGL_[A-Z0-9_]*PORT=\([0-9][0-9]*\)$/\1/p' | sort -un | paste -sd, -)" + +# nscd would answer from the host cache and skip the dnsmasq, so /run/nscd gets an empty dir too +printf 'nameserver 127.0.0.1\n' > "$TNGL_STATE/run/resolv.conf" +mount --bind "$TNGL_STATE/run/resolv.conf" /etc/resolv.conf +[ -d /run/nscd ] && mount --bind "$TNGL_STATE/run/empty" /run/nscd + +# with no caps the marked gid can't lift these rules, so it stays boxed in +case "$TNGL_IP" in +127.*) ;; +*) iptables -A OUTPUT -m owner --gid-owner "$TNGL_OFFLINE_GID" -d "$TNGL_IP" -j ACCEPT ;; +esac +iptables -A OUTPUT -m owner --gid-owner "$TNGL_OFFLINE_GID" ! -d 127.0.0.0/8 -j REJECT +ip6tables -A OUTPUT -m owner --gid-owner "$TNGL_OFFLINE_GID" ! -d ::1/128 -j REJECT + +export TNGL_DNS_PORT=53 +exec setpriv --inh-caps=-all,+setgid --ambient-caps=-all,+setgid --bounding-set=-all,+setgid "$@" diff --git a/localinfra/pc/bin/offline b/localinfra/pc/bin/offline new file mode 100755 index 0000000000000000000000000000000000000000..cc613dfe7512440761c52454aef7ad1badac4725 --- /dev/null +++ b/localinfra/pc/bin/offline @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +[ "${TNGL_ONLINE:-0}" = 1 ] && exec "$@" +case "$(uname -s)" in +Darwin) exec /usr/bin/sandbox-exec -f "$TNGL_PC/airgap.sb" "$@" ;; +Linux) exec setpriv --regid="$TNGL_OFFLINE_GID" --clear-groups --inh-caps=-all --ambient-caps=-all "$@" ;; +esac diff --git a/localinfra/pc/bin/patch-workerd b/localinfra/pc/bin/patch-workerd new file mode 100755 index 0000000000000000000000000000000000000000..04af491b98991e61b07776531d049614e4aa5493 --- /dev/null +++ b/localinfra/pc/bin/patch-workerd @@ -0,0 +1,9 @@ +#!/bin/sh +# the workerd npm downloads asks for a generic linux loader that nixos doesn't ship +# only the nix app sets TNGL_DYNAMIC_LINKER, so everywhere else this is a no-op +set -eu +[ -n "${TNGL_DYNAMIC_LINKER:-}" ] || exit 0 +for bin in "$1"/node_modules/.pnpm/@cloudflare+workerd-linux-*/node_modules/@cloudflare/workerd-linux-*/bin/workerd; do + [ -f "$bin" ] || continue + patchelf --set-interpreter "$TNGL_DYNAMIC_LINKER" --set-rpath "$TNGL_NIX_LIBS" "$bin" +done diff --git a/localinfra/pc/bin/pc b/localinfra/pc/bin/pc new file mode 100755 index 0000000000000000000000000000000000000000..00983509f77d23cf60f6735a273d4e0d2a5d2dfd --- /dev/null +++ b/localinfra/pc/bin/pc @@ -0,0 +1,4 @@ +#!/bin/sh +here=$(cd "$(dirname "$0")/.." && pwd) +. "$here/bin/env.sh" +exec process-compose -U -u "$TNGL_STATE/run/pc.sock" "$@" diff --git a/localinfra/pc/bin/profile-on b/localinfra/pc/bin/profile-on new file mode 100755 index 0000000000000000000000000000000000000000..0256ada505dbaae1f5f3800d8b28c33dfca488f4 --- /dev/null +++ b/localinfra/pc/bin/profile-on @@ -0,0 +1,6 @@ +#!/bin/sh +# services parses TNGL_PROFILES too, so change the two together +for p in $(echo "${TNGL_PROFILES:-}" | tr , ' '); do + [ "$p" = "$1" ] && exit 0 +done +exit 1 diff --git a/localinfra/pc/bin/selftest b/localinfra/pc/bin/selftest new file mode 100755 index 0000000000000000000000000000000000000000..50725e74860b9bfc6f8cd22c7969ec3dd806751c --- /dev/null +++ b/localinfra/pc/bin/selftest @@ -0,0 +1,31 @@ +#!/bin/sh +# run as `up --exec selftest` +here=$(cd "$(dirname "$0")/.." && pwd) +. "$here/bin/env.sh" + +fail=0 +check() { + want=$1 name=$2 + shift 2 + if "$@" >/dev/null 2>&1; then got=ok; else got=blocked; fi + if [ "$got" = "$want" ]; then echo "pass $name"; else echo "FAIL $name (wanted $want, got $got)"; fail=1; fi +} + +port=${TNGL_SELFTEST_PORT:-52999} +python3 -m http.server "$port" --bind 127.0.0.1 >/dev/null 2>&1 & +srv=$! +sleep 1 + +check ok "offline reaches loopback" offline curl -sS -m 3 "http://127.0.0.1:$port/" +check blocked "offline can't reach the internet" offline curl -sS -m 3 https://1.1.1.1 +check blocked "offline can't send udp out" offline python3 -c \ + "import socket; s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.connect(('1.1.1.1', 53)); s.send(b'x')" +if [ "$(uname -s)" = Linux ]; then + check blocked "offline can't unmark itself" offline setpriv --regid="$(id -g)" --clear-groups true +fi +if [ "${TNGL_SELFTEST_ONLINE:-0}" = 1 ]; then + check ok "an unwrapped process reaches the internet" curl -sS -m 5 https://1.1.1.1 +fi + +kill "$srv" 2>/dev/null +exit "$fail" diff --git a/localinfra/pc/bin/services b/localinfra/pc/bin/services new file mode 100755 index 0000000000000000000000000000000000000000..8e0adc3f00c600c0c18fdbcd26426081f2bdc652 --- /dev/null +++ b/localinfra/pc/bin/services @@ -0,0 +1,22 @@ +#!/bin/sh +here=$(cd "$(dirname "$0")/.." && pwd) + +wanted() { + [ -f "$1/profile" ] || return 0 + for p in $(echo "${TNGL_PROFILES:-}" | tr , ' '); do + grep -qx "$p" "$1/profile" && return 0 + done + return 1 +} + +if [ -n "${TNGL_SERVICES:-}" ]; then + for s in $(echo "$TNGL_SERVICES" | tr , ' '); do + [ -d "$here/services/$s" ] || { echo "no service $s in $here/services" >&2; exit 1; } + echo "$s" + done +else + for d in "$here"/services/*/; do + s=$(basename "$d") + [ -d "$d" ] && wanted "$d" && case "$s" in _*) ;; *) echo "$s" ;; esac + done +fi diff --git a/localinfra/pc/bin/setup b/localinfra/pc/bin/setup new file mode 100755 index 0000000000000000000000000000000000000000..578b70219e2f5753290289aa889ee5ce2bf592e3 --- /dev/null +++ b/localinfra/pc/bin/setup @@ -0,0 +1,40 @@ +#!/bin/sh +# internet access happens here, because the stack itself runs offline +set -eu +here=$(cd "$(dirname "$0")/.." && pwd) +. "$here/bin/env.sh" + +mkdir -p "$TNGL_STATE"/data "$TNGL_STATE"/run "$TNGL_STATE"/src "$TNGL_STATE"/bin "$TNGL_CERTS" + +# make-certs.sh makes this ca too, but it needs go and makes keys nothing here uses +if [ ! -s "$TNGL_CERTS/root.crt" ]; then + (umask 077 && openssl req -x509 -newkey rsa:2048 \ + -keyout "$TNGL_CERTS/root.key" -out "$TNGL_CERTS/root.crt" \ + -days 3650 -nodes -subj "/CN=Tangled Dev CA" \ + -addext "basicConstraints=critical,CA:TRUE,pathlen:1" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -addext "nameConstraints=critical,permitted;DNS:tngl.boltless.dev") + echo "made a dev ca in $TNGL_CERTS, trust root.crt to use the stack from a browser" >&2 +fi + +if [ -z "${PLC_CMD:-}" ]; then + plc=$TNGL_STATE/src/did-method-plc + if [ ! -d "$plc/.git" ]; then + mkdir -p "$plc" + git -C "$plc" init -q + git -C "$plc" fetch -q --depth 1 https://github.com/did-method-plc/did-method-plc f2ab7516bac5bc0f3f86842fa94e996bd1b3815b + git -C "$plc" checkout -q FETCH_HEAD + fi + [ -d "$plc/node_modules" ] || (cd "$plc" && yarn install --frozen-lockfile --silent) + [ -f "$plc/packages/server/dist/bin.js" ] || (cd "$plc/packages/server" && node build.js >/dev/null) +fi + +(cd "$TNGL_ROOT" && go build -o "$TNGL_STATE/bin/seed" ./localinfra/cmd/seed) + +if [ -z "${KNOT2_CMD:-}" ] && [ "$KNOT2_BIN" = "$KNOT2_BUILT" ]; then + (cd "$TNGL_ROOT" && cargo build -p knot-server) +fi + +for svc in $(services); do + if [ -f "$here/services/$svc/setup" ]; then sh "$here/services/$svc/setup"; fi +done diff --git a/localinfra/pc/bin/svc/caddy b/localinfra/pc/bin/svc/caddy new file mode 100755 index 0000000000000000000000000000000000000000..f5c6366119afe2852cf27c0129367f0f5f668408 --- /dev/null +++ b/localinfra/pc/bin/svc/caddy @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +conf=$TNGL_STATE/run/Caddyfile +cp "$TNGL_PC/Caddyfile" "$conf" +for svc in $(services); do + if [ -f "$TNGL_PC/services/$svc/Caddyfile" ]; then { echo; cat "$TNGL_PC/services/$svc/Caddyfile"; } >> "$conf"; fi +done +# the default runs under sh -c, which only sees exported vars +export conf +default='exec "$CADDY_BIN" run --config "$conf" --adapter caddyfile' +exec sh -c "${CADDY_CMD:-$default}" diff --git a/localinfra/pc/bin/svc/dns b/localinfra/pc/bin/svc/dns new file mode 100755 index 0000000000000000000000000000000000000000..00c808cc3d668f4d14d5114ff51bc1b68f715de0 --- /dev/null +++ b/localinfra/pc/bin/svc/dns @@ -0,0 +1,7 @@ +#!/bin/sh +# octostub.dev too, deliberi looks up committer email domains over dns +default='exec "$DNS_BIN" --keep-in-foreground --conf-file=/dev/null --pid-file= --no-hosts --no-resolv \ + --bind-interfaces --listen-address=127.0.0.1 --port="$TNGL_DNS_PORT" \ + --address=/tngl.boltless.dev/$TNGL_IP --address=/octostub.dev/$TNGL_IP ${TNGL_DNS_UPSTREAM:+--server=$TNGL_DNS_UPSTREAM}' +ulimit -n 4096 +exec sh -c "${DNS_CMD:-$default}" diff --git a/localinfra/pc/bin/svc/hydrant b/localinfra/pc/bin/svc/hydrant new file mode 100755 index 0000000000000000000000000000000000000000..487dca24abda3b07214e42ebf76a4b79df41fced --- /dev/null +++ b/localinfra/pc/bin/svc/hydrant @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +mkdir -p "$TNGL_STATE/data/hydrant" +export HYDRANT_API_BIND=127.0.0.1:$TNGL_HYDRANT_PORT +export HYDRANT_DATABASE_PATH=$TNGL_STATE/data/hydrant/hydrant.db +export HYDRANT_RELAY_HOSTS=pds::wss://pds.tngl.boltless.dev,pds::wss://knot2.tngl.boltless.dev +export HYDRANT_CRAWLER_URLS=list_repos::https://pds.tngl.boltless.dev,list_repos::https://knot2.tngl.boltless.dev +if services | grep -qx tranquil; then + HYDRANT_RELAY_HOSTS=$HYDRANT_RELAY_HOSTS,pds::wss://tranquil.tngl.boltless.dev + HYDRANT_CRAWLER_URLS=$HYDRANT_CRAWLER_URLS,list_repos::https://tranquil.tngl.boltless.dev +fi +export HYDRANT_PLC_URL=https://plc.tngl.boltless.dev +export HYDRANT_FILTER_COLLECTIONS='org.tangled.*' +export HYDRANT_FILTER_SIGNALS=org.tangled.actor.profile,org.tangled.repo.manifest,org.tangled.repo.declaration,org.tangled.git.refs +export HYDRANT_BACKFILL_STRATEGY=sparse-filter +export HYDRANT_VERIFY_SIGNATURES=none +# these hosts resolve to loopback, and hydrant refuses that unless this is set +export HYDRANT_ALLOW_PRIVATE_NETWORKS=true +export SSL_CERT_FILE=$TNGL_CERTS/root.crt +default='exec "$HYDRANT_BIN"' +exec sh -c "${HYDRANT_CMD:-$default}" diff --git a/localinfra/pc/bin/svc/init-data b/localinfra/pc/bin/svc/init-data new file mode 100755 index 0000000000000000000000000000000000000000..af2afd748c7f42c680f96fbb2c2188804ae3b7dd --- /dev/null +++ b/localinfra/pc/bin/svc/init-data @@ -0,0 +1,63 @@ +#!/bin/sh +# imports need the internet, so this runs its own knot2 and kills it once they are done +set -eu + +# the stamp lives in data/, so wiping data/ reseeds too +stamp=$TNGL_STATE/data/init-data.stamp +want=$({ + cat "$0" "$TNGL_ROOT/localinfra/scripts/init-data.sh" "$TNGL_ROOT/localinfra/scripts/lib.sh" "$TNGL_ROOT/localinfra/pds.env" + services + find "$TNGL_ROOT/localinfra/fixtures" "$TNGL_ROOT/localinfra/cmd/seed" -type f | sort | xargs cksum +} | cksum) +if [ -s "$TNGL_STATE/shared/accounts.tsv" ] && [ "$(cat "$stamp" 2>/dev/null || true)" = "$want" ]; then + echo "[init-data] already seeded" >&2 + exit 0 +fi + +set -a +. "$TNGL_ROOT/localinfra/pds.env" +set +a +export PDS_URL=http://127.0.0.1:$TNGL_PDS_PORT +export PLC_URL=http://127.0.0.1:$TNGL_PLC_PORT +export KNOT_URL=http://127.0.0.1:$TNGL_KNOT_PORT +export KNOT_HOSTNAME=knot2.tngl.boltless.dev +export SCRIPTS=$TNGL_ROOT/localinfra/scripts +export FIXTURES=$TNGL_ROOT/localinfra/fixtures +export SHARED_DIR=$TNGL_STATE/shared +export PATH=$TNGL_STATE/bin:$PATH + +if services | grep -qx tranquil; then + export TRANQUIL_URL=http://127.0.0.1:$TNGL_TRANQUIL_PORT + export TRANQUIL_HOSTNAME=tranquil.tngl.boltless.dev +fi + +if services | grep -qx ozone; then + set -a + . "$TNGL_ROOT/localinfra/ozone.env" + set +a +fi + +"$TNGL_PC/bin/svc/knot2" & +knot=$! +trap 'kill $knot 2>/dev/null || true' EXIT +knot_alive() { + kill -0 "$knot" 2>/dev/null || { echo "[init-data] the temporary knot exited" >&2; exit 1; } +} +until tcp-up 127.0.0.1 "$TNGL_KNOT_PORT"; do knot_alive; sleep 1; done + +sh "$SCRIPTS/init-data.sh" + +# a knot that dies mid fetch can leave its state stuck on fetching, so knot_alive is what gets this loop out +while find "$TNGL_STATE/data/knot2/repositories" -name tangled.import.json -exec grep -lE '"state":"(queued|fetching)"' {} + | grep -q .; do + knot_alive + echo "[init-data] waiting for the knot to finish its imports" >&2 + sleep 2 +done +# if find "$TNGL_STATE/data/knot2/repositories" -name tangled.import.json -exec grep -lE '"state":"(failed|partial)"' {} + | grep -q .; then +# echo "[init-data] an import failed, see the knot2 lines above. delete $TNGL_STATE/data/knot2 to retry" >&2 +# exit 1 +# fi + +kill "$knot" +wait "$knot" || true +echo "$want" > "$stamp" diff --git a/localinfra/pc/bin/svc/knot2 b/localinfra/pc/bin/svc/knot2 new file mode 100755 index 0000000000000000000000000000000000000000..35a22db2237d92059171d4d05b59b09949062fb3 --- /dev/null +++ b/localinfra/pc/bin/svc/knot2 @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu +data=$TNGL_STATE/data/knot2 +# the knot makes keys on demand but expects the scan dir to already exist +mkdir -p "$data/repositories" "$data/lfs" +export KNOT_HOSTNAME=knot2.tngl.boltless.dev +export KNOT_ADMINS=did:plc:igzuueb63dytyy3tpaweofuu +export KNOT_CONTACT_EMAIL=admin@knot2.tngl.boltless.dev +export KNOT_LISTEN_ADDR=127.0.0.1:$TNGL_KNOT_PORT +export KNOT_INTERNAL_LISTEN_ADDR=127.0.0.1:$TNGL_KNOT_INTERNAL_PORT +export KNOT_SSH_LISTEN_ADDR=127.0.0.1:$TNGL_KNOT_SSH_PORT +export KNOT_SCAN_PATH=$data/repositories +export KNOT_LFS_STORE_PATH=$data/lfs +export KNOT_SSH_HOST_KEY_FILE=$data/keys/host_key +export KNOT_SEALED_KEY_FILE=$data/keys/sealed.bin +export KNOT_MASTER_KEY_ENV=KNOT_MASTER_KEY +export KNOT_MASTER_KEY=9MfYcpOFaJYNzYJziRP3rrkM7DJdxQH/Hjey3LuwPF4= +export KNOT_LEGACY_ADMIN_SECRET_ENV=KNOT_LEGACY_ADMIN_SECRET +export KNOT_LEGACY_ADMIN_SECRET=super-strong-devonly-key +export KNOT_PLC_DIRECTORY=$TNGL_PLC_URL +export KNOT_APPVIEW_ENDPOINT=https://web.tngl.boltless.dev +export KNOT_CRAWLERS=https://hydrant.tngl.boltless.dev +export KNOT_EXTRA_CA_FILE=$TNGL_CERTS/root.crt +# plc and the other knots are on loopback, so the private address guard has to be off +export KNOT_HTTP_BLOCK_PRIVATE_ADDRESSES=false +# caddy is the only caller here, so the client ip comes from x-forwarded-for +export KNOT_XRPC_TRUSTED_PROXY_HEADER=x-forwarded-for +export KNOT_XRPC_TRUSTED_PROXIES=127.0.0.0/8 +export KNOT_XRPC_PREAUTH_BURST=500 +export KNOT_XRPC_PREAUTH_REFILL_MS=5 +export KNOT_XRPC_PER_PEER_INFLIGHT=64 +export KNOT_XRPC_EVENTS_MAX_PER_PEER=64 +export KNOT_LISTEN_REQUEST_TIMEOUT_MS=600000 +export KNOT_TLS_HTTP3=false +export KNOT_CONTRIBUTION_POLICY=anyone +export KNOT_ADMISSION=open +export RUST_LOG=${RUST_LOG:-info} + +default='exec "$KNOT2_BIN"' +exec sh -c "${KNOT2_CMD:-$default}" diff --git a/localinfra/pc/bin/svc/pds b/localinfra/pc/bin/svc/pds new file mode 100755 index 0000000000000000000000000000000000000000..3d354b4acba531e652568224c42ff38a824a82f3 --- /dev/null +++ b/localinfra/pc/bin/svc/pds @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +set -a +. "$TNGL_ROOT/localinfra/pds.env" +set +a +export PDS_PORT=$TNGL_PDS_PORT +export PDS_DATA_DIRECTORY=$TNGL_STATE/data/pds +export PDS_BLOBSTORE_DISK_LOCATION=$TNGL_STATE/data/pds/blocks +export PDS_DID_PLC_URL=$TNGL_PLC_URL +# plc and handles are on loopback, which the ssrf guard refuses +export PDS_DISABLE_SSRF_PROTECTION=true +export NODE_EXTRA_CA_CERTS=$TNGL_CERTS/root.crt +mkdir -p "$PDS_DATA_DIRECTORY" + +default='exec "$PDS_BIN"' +exec sh -c "${PDS_CMD:-$default}" diff --git a/localinfra/pc/bin/svc/pg-init b/localinfra/pc/bin/svc/pg-init new file mode 100755 index 0000000000000000000000000000000000000000..b9fbb4e7787b4ac28b2a93fcfbe1d8eee13dbb0b --- /dev/null +++ b/localinfra/pc/bin/svc/pg-init @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +psql="psql -h 127.0.0.1 -p $TNGL_PG_PORT -U tnglr -d postgres -tAq" +for db in $(sed -n 's/^CREATE DATABASE \(.*\);$/\1/p' "$TNGL_ROOT/localinfra/postgres-init.sql"); do + $psql -c "SELECT 1 FROM pg_database WHERE datname = '$db'" | grep -q 1 || $psql -c "CREATE DATABASE $db" +done diff --git a/localinfra/pc/bin/svc/plc b/localinfra/pc/bin/svc/plc new file mode 100755 index 0000000000000000000000000000000000000000..f848225b528d24d5ecc60f967e42f20453d46354 --- /dev/null +++ b/localinfra/pc/bin/svc/plc @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +default='cd "$TNGL_STATE/src/did-method-plc/packages/server" && DATABASE_URL="postgres://tnglr:tnglr@127.0.0.1:$TNGL_PG_PORT/plc" PORT=$TNGL_PLC_PORT exec node dist/bin.js' +exec sh -c "${PLC_CMD:-$default}" diff --git a/localinfra/pc/bin/svc/postgres b/localinfra/pc/bin/svc/postgres new file mode 100755 index 0000000000000000000000000000000000000000..4c7688fc977104046563addce0f6fb85c8917448 --- /dev/null +++ b/localinfra/pc/bin/svc/postgres @@ -0,0 +1,8 @@ +#!/bin/sh +# unix sockets are off, a long state dir path would overflow the socket path limit +set -eu +default='set -e +data=$TNGL_STATE/data/postgres +[ -f "$data/PG_VERSION" ] || { mkdir -p "$data" && initdb -D "$data" -U tnglr --auth=trust -E UTF8 --no-locale >/dev/null; } +exec postgres -D "$data" -h 127.0.0.1 -p "$TNGL_PG_PORT" -c unix_socket_directories=' +exec sh -c "${POSTGRES_CMD:-$default}" diff --git a/localinfra/pc/bin/tcp-up b/localinfra/pc/bin/tcp-up new file mode 100755 index 0000000000000000000000000000000000000000..ae3dc0a4063ef25c00e44a838b273cfd3df33fd0 --- /dev/null +++ b/localinfra/pc/bin/tcp-up @@ -0,0 +1,4 @@ +#!/bin/sh +# curl exit 7 (refused) is the only down, timeouts and http errors still count as up +curl -s -m 2 -o /dev/null "http://$1:$2/" +[ $? -ne 7 ] diff --git a/localinfra/pc/bin/up b/localinfra/pc/bin/up new file mode 100755 index 0000000000000000000000000000000000000000..cabe1e11ef2872f3a207034a9cdfe3d0d1fd5e81 --- /dev/null +++ b/localinfra/pc/bin/up @@ -0,0 +1,36 @@ +#!/bin/sh +# up [--online] [--profile NAME]... [process-compose args] +# up --exec CMD... runs CMD in the same environment instead, for poking around +set -eu +here=$(cd "$(dirname "$0")/.." && pwd) + +# env.sh reads TNGL_PROFILES, so the flags have to be parsed before it is sourced +while [ $# -gt 0 ]; do + case "$1" in + --online) TNGL_ONLINE=1; export TNGL_ONLINE; shift ;; + --profile) TNGL_PROFILES="${TNGL_PROFILES:+$TNGL_PROFILES,}${2:?--profile needs a name}"; export TNGL_PROFILES; shift 2 ;; + *) break ;; + esac +done + +. "$here/bin/env.sh" +cd "$here" + +doctor +if [ "${1:-}" = --exec ]; then + shift +else + setup + files="-f process-compose.yaml" + for svc in $(services); do + [ -f "services/$svc/process.yaml" ] && files="$files -f services/$svc/process.yaml" + done + [ -f process-compose.override.yaml ] && files="$files -f process-compose.override.yaml" + # shellcheck disable=SC2086 + set -- process-compose up -U -u "$TNGL_STATE/run/pc.sock" $files "$@" +fi + +case "$(uname -s)" in +Linux) exec "$here/bin/linux-ns" "$@" ;; +*) exec "$@" ;; +esac diff --git a/localinfra/pc/commands.env b/localinfra/pc/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..5ae4be7c17d0f71949ec9e39c19439bbbe81dd6f --- /dev/null +++ b/localinfra/pc/commands.env @@ -0,0 +1,7 @@ +# single quote the values in .env, or the $ references get expanded when it's sourced +DNS_BIN=${DNS_BIN:-dnsmasq} +CADDY_BIN=${CADDY_BIN:-caddy} +PDS_BIN=${PDS_BIN:-pds} +HYDRANT_BIN=${HYDRANT_BIN:-hydrant} +KNOT2_BUILT=${CARGO_TARGET_DIR:-$TNGL_ROOT/target}/debug/knot-server +KNOT2_BIN=${KNOT2_BIN:-$KNOT2_BUILT} diff --git a/localinfra/pc/ports.env b/localinfra/pc/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..62c07ead1a21b4ad333f557d3d658b63a0c04032 --- /dev/null +++ b/localinfra/pc/ports.env @@ -0,0 +1,16 @@ +TNGL_HTTPS_PORT=${TNGL_HTTPS_PORT:-443} +TNGL_DNS_PORT=${TNGL_DNS_PORT:-5354} +TNGL_PG_PORT=${TNGL_PG_PORT:-55432} +TNGL_PLC_PORT=${TNGL_PLC_PORT:-52582} +TNGL_PDS_PORT=${TNGL_PDS_PORT:-52583} +TNGL_HYDRANT_PORT=${TNGL_HYDRANT_PORT:-52584} +TNGL_KNOT_PORT=${TNGL_KNOT_PORT:-52585} +TNGL_KNOT_INTERNAL_PORT=${TNGL_KNOT_INTERNAL_PORT:-52586} +TNGL_KNOT_SSH_PORT=${TNGL_KNOT_SSH_PORT:-2223} +TNGL_PLC_URL=${TNGL_PLC_URL:-https://plc.tngl.boltless.dev} + +# services that refuse loopback need TNGL_IP=11.0.0.253, which macos only answers after `sudo ifconfig lo0 alias 11.0.0.253/32` +TNGL_IP=${TNGL_IP:-127.0.0.1} + +# only bin/linux-ns reads this +TNGL_FORWARD=${TNGL_FORWARD:-443:443,2223:2223} diff --git a/localinfra/pc/process-compose.yaml b/localinfra/pc/process-compose.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cbecc6f815e23698a796c1721eb2cd8ffc93732 --- /dev/null +++ b/localinfra/pc/process-compose.yaml @@ -0,0 +1,106 @@ +version: "0.5" +is_strict: true + +processes: + dns: + command: ${TNGL_PC}/bin/svc/dns + readiness_probe: + exec: + command: dig +short +time=1 +tries=1 @127.0.0.1 -p ${TNGL_DNS_PORT} plc.tngl.boltless.dev | grep -q ${TNGL_IP} + period_seconds: 1 + failure_threshold: 30 + + postgres: + command: offline ${TNGL_PC}/bin/svc/postgres + readiness_probe: + exec: + command: pg_isready -q -h 127.0.0.1 -p ${TNGL_PG_PORT} -U tnglr -d postgres + period_seconds: 1 + failure_threshold: 60 + + pg-init: + command: offline ${TNGL_PC}/bin/svc/pg-init + depends_on: + postgres: + condition: process_healthy + availability: + restart: "no" + + plc: + command: offline ${TNGL_PC}/bin/svc/plc + depends_on: + pg-init: + condition: process_completed_successfully + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: /_health + port: ${TNGL_PLC_PORT} + period_seconds: 1 + failure_threshold: 60 + + caddy: + command: offline ${TNGL_PC}/bin/svc/caddy + readiness_probe: + exec: + command: tcp-up 127.0.0.1 ${TNGL_HTTPS_PORT} + period_seconds: 1 + failure_threshold: 30 + + pds: + command: offline ${TNGL_PC}/bin/svc/pds + depends_on: + dns: + condition: process_healthy + caddy: + condition: process_healthy + plc: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: /xrpc/_health + port: ${TNGL_PDS_PORT} + period_seconds: 1 + failure_threshold: 180 + + init-data: + command: ${TNGL_PC}/bin/svc/init-data + depends_on: + dns: + condition: process_healthy + caddy: + condition: process_healthy + pds: + condition: process_healthy + availability: + restart: "no" + + knot2: + command: offline ${TNGL_PC}/bin/svc/knot2 + depends_on: + init-data: + condition: process_completed_successfully + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: / + port: ${TNGL_KNOT_PORT} + period_seconds: 1 + failure_threshold: 120 + + hydrant: + command: offline ${TNGL_PC}/bin/svc/hydrant + depends_on: + pds: + condition: process_healthy + knot2: + condition: process_healthy + readiness_probe: + exec: + command: tcp-up 127.0.0.1 ${TNGL_HYDRANT_PORT} + period_seconds: 1 + failure_threshold: 60 diff --git a/localinfra/pc/services/README.md b/localinfra/pc/services/README.md new file mode 100644 index 0000000000000000000000000000000000000000..22e5dee7149f91d54af2ceed664c5c5e6004fd13 --- /dev/null +++ b/localinfra/pc/services/README.md @@ -0,0 +1,28 @@ +# adding a service + +one directory per service, `services//`. nothing outside it needs editing, the +scripts discover what is there. every file is optional except `run` and `process.yaml`. +`services/redis/` is a small example. + +- `run`: exports its env and execs `${FOO_CMD:-$default}`. runs what is already built, + builds go in `setup` +- `process.yaml`: process-compose entry with a readiness probe and `depends_on`, wrapped + in `offline` unless it needs the internet while it runs +- `ports.env`: loopback ports, `TNGL_FOO_PORT=${TNGL_FOO_PORT:-52590}`. pick one + nobody has with + `grep -rh _PORT localinfra/pc/ports.env localinfra/pc/services/*/ports.env` +- `commands.env`: defaults for `FOO_BIN` and `FOO_CMD` +- `Caddyfile`: vhosts, upstream `127.0.0.1:{$TNGL_FOO_PORT}` +- `setup`: runs before the stack starts, may fetch and build +- `doctor`: checks required binaries with `need prog` +- `profile`: profile names, one per line. the service only runs when one is on, without + the file it always runs +- `packages`: nixpkgs attributes `nix run .#localinfra` should bring + +## rules + +- loopback only: bind `127.0.0.1`. +- call sibling services through `*.tngl.boltless.dev` because caddy terminates tls for all + of them. +- trust `$TNGL_CERTS/root.crt` for outgoing https calls. +- one service per directory, matching the compose service name. diff --git a/localinfra/pc/services/avatar/Caddyfile b/localinfra/pc/services/avatar/Caddyfile new file mode 100644 index 0000000000000000000000000000000000000000..419d99c3f7f80354eea1d9429514b41d512d1fad --- /dev/null +++ b/localinfra/pc/services/avatar/Caddyfile @@ -0,0 +1,4 @@ +avatar.tngl.boltless.dev { + tls internal + reverse_proxy 127.0.0.1:{$TNGL_AVATAR_PORT} +} diff --git a/localinfra/pc/services/avatar/commands.env b/localinfra/pc/services/avatar/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..966c9e4dd484ae8dac845b20e9b38f10d6b7a430 --- /dev/null +++ b/localinfra/pc/services/avatar/commands.env @@ -0,0 +1,2 @@ +AVATAR_BUILT=$TNGL_ROOT/avatar/node_modules/.bin/wrangler +AVATAR_BIN=${AVATAR_BIN:-$AVATAR_BUILT} diff --git a/localinfra/pc/services/avatar/doctor b/localinfra/pc/services/avatar/doctor new file mode 100644 index 0000000000000000000000000000000000000000..6c4bd2d68e59e68ec4000c43a6cdea8230149913 --- /dev/null +++ b/localinfra/pc/services/avatar/doctor @@ -0,0 +1,3 @@ +if [ -z "${AVATAR_CMD:-}" ]; then + if [ "$AVATAR_BIN" = "$AVATAR_BUILT" ]; then need pnpm node; else need "$AVATAR_BIN"; fi +fi diff --git a/localinfra/pc/services/avatar/packages b/localinfra/pc/services/avatar/packages new file mode 100644 index 0000000000000000000000000000000000000000..7ef5d368121780b15ba317f360bd8166d74b4861 --- /dev/null +++ b/localinfra/pc/services/avatar/packages @@ -0,0 +1,2 @@ +pnpm +nodejs_22 diff --git a/localinfra/pc/services/avatar/ports.env b/localinfra/pc/services/avatar/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..3f332925d6d4ec2a061a8c6c8ae6b080142f023c --- /dev/null +++ b/localinfra/pc/services/avatar/ports.env @@ -0,0 +1,2 @@ +TNGL_AVATAR_PORT=${TNGL_AVATAR_PORT:-52661} +TNGL_AVATAR_INSPECTOR_PORT=${TNGL_AVATAR_INSPECTOR_PORT:-52671} diff --git a/localinfra/pc/services/avatar/process.yaml b/localinfra/pc/services/avatar/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13c0d74b789c09a89c682785fcc79cac82698512 --- /dev/null +++ b/localinfra/pc/services/avatar/process.yaml @@ -0,0 +1,19 @@ +version: "0.5" +processes: + avatar: + command: offline ${TNGL_PC}/services/avatar/run + depends_on: + dns: + condition: process_healthy + caddy: + condition: process_healthy + bobbin: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: / + port: ${TNGL_AVATAR_PORT} + period_seconds: 1 + failure_threshold: 60 diff --git a/localinfra/pc/services/avatar/run b/localinfra/pc/services/avatar/run new file mode 100755 index 0000000000000000000000000000000000000000..47739bdd9d53a9544fa0fc24e22bedcc35ed2dfe --- /dev/null +++ b/localinfra/pc/services/avatar/run @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +export AVATAR_SHARED_SECRET=${AVATAR_SHARED_SECRET:-localinfra-avatar-secret} +export BOBBIN_URL=${BOBBIN_URL:-https://bobbin.tngl.boltless.dev} +export WRANGLER_SEND_METRICS=false +export SSL_CERT_FILE=$TNGL_CERTS/root.crt +export NODE_EXTRA_CA_CERTS=$TNGL_CERTS/root.crt + +# wrangler keeps its local state in the project dir, so dev has to cd there first +default='cd "$TNGL_ROOT/avatar" && exec "$AVATAR_BIN" dev --ip 127.0.0.1 --port "$TNGL_AVATAR_PORT" --inspector-port "$TNGL_AVATAR_INSPECTOR_PORT" --var "AVATAR_SHARED_SECRET:$AVATAR_SHARED_SECRET" --var "BOBBIN_URL:$BOBBIN_URL"' +exec sh -c "${AVATAR_CMD:-$default}" diff --git a/localinfra/pc/services/avatar/setup b/localinfra/pc/services/avatar/setup new file mode 100755 index 0000000000000000000000000000000000000000..77c6388484e9b4f4ad180fe48edf17be11b7455e --- /dev/null +++ b/localinfra/pc/services/avatar/setup @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +if [ -z "${AVATAR_CMD:-}" ] && [ "$AVATAR_BIN" = "$AVATAR_BUILT" ] && [ ! -e "$AVATAR_BUILT" ]; then + (cd "$TNGL_ROOT/avatar" && pnpm install --frozen-lockfile) +fi +patch-workerd "$TNGL_ROOT/avatar" diff --git a/localinfra/pc/services/bobbin/Caddyfile b/localinfra/pc/services/bobbin/Caddyfile new file mode 100644 index 0000000000000000000000000000000000000000..2ae7768415fda88bd2be945de0381dd721071271 --- /dev/null +++ b/localinfra/pc/services/bobbin/Caddyfile @@ -0,0 +1,8 @@ +bobbin.tngl.boltless.dev { + tls internal + import cors 127.0.0.1:{$TNGL_BOBBIN_PORT} +} + +:8090 { + import cors 127.0.0.1:{$TNGL_BOBBIN_PORT} +} diff --git a/localinfra/pc/services/bobbin/commands.env b/localinfra/pc/services/bobbin/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..08e59102ffad4f110414f6ce813c5713b596c014 --- /dev/null +++ b/localinfra/pc/services/bobbin/commands.env @@ -0,0 +1,2 @@ +BOBBIN_BUILT=${CARGO_TARGET_DIR:-$TNGL_ROOT/target}/debug/bobbin +BOBBIN_BIN=${BOBBIN_BIN:-$BOBBIN_BUILT} diff --git a/localinfra/pc/services/bobbin/doctor b/localinfra/pc/services/bobbin/doctor new file mode 100644 index 0000000000000000000000000000000000000000..97ef833e92e186c8e66fe8ba08e6cef5c24d1f06 --- /dev/null +++ b/localinfra/pc/services/bobbin/doctor @@ -0,0 +1,3 @@ +if [ -z "${BOBBIN_CMD:-}" ]; then + if [ "$BOBBIN_BIN" = "$BOBBIN_BUILT" ]; then need cargo; else need "$BOBBIN_BIN"; fi +fi diff --git a/localinfra/pc/services/bobbin/ports.env b/localinfra/pc/services/bobbin/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..53ffdb847a19a13adde226b0313f7c126b3635bc --- /dev/null +++ b/localinfra/pc/services/bobbin/ports.env @@ -0,0 +1 @@ +TNGL_BOBBIN_PORT=${TNGL_BOBBIN_PORT:-52681} diff --git a/localinfra/pc/services/bobbin/process.yaml b/localinfra/pc/services/bobbin/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5471d507f29abf9c6fa9c56c24dceddcd15939e --- /dev/null +++ b/localinfra/pc/services/bobbin/process.yaml @@ -0,0 +1,19 @@ +version: "0.5" +processes: + bobbin: + command: offline ${TNGL_PC}/services/bobbin/run + depends_on: + hydrant: + condition: process_healthy + caddy: + condition: process_healthy + pocket: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: /xrpc/org.tangled.bobbin.getCoverage + port: ${TNGL_BOBBIN_PORT} + period_seconds: 1 + failure_threshold: 60 diff --git a/localinfra/pc/services/bobbin/run b/localinfra/pc/services/bobbin/run new file mode 100755 index 0000000000000000000000000000000000000000..88858cf652815d7f43bcfdfbce5c43089cf5d561 --- /dev/null +++ b/localinfra/pc/services/bobbin/run @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu + +mkdir -p "$TNGL_STATE/data/bobbin" + +export BOBBIN_BIND=127.0.0.1:$TNGL_BOBBIN_PORT +export BOBBIN_HYDRANT_URL=https://hydrant.tngl.boltless.dev +export BOBBIN_SLINGSHOT_URL=https://hydrant.tngl.boltless.dev +export BOBBIN_SERVICE_DID=did:web:bobbin.tngl.boltless.dev +export BOBBIN_MIRROR_URL=https://mirror.tngl.boltless.dev +export BOBBIN_MIRROR_BREAKER_THRESHOLD=20 +export BOBBIN_MIRROR_BREAKER_COOLDOWN_SECS=3 +export BOBBIN_POCKET_URL=https://pocket.tngl.boltless.dev +export BOBBIN_CODESEARCH_ZOEKT_URL=https://zoekt.tngl.boltless.dev +export BOBBIN_LOG=${BOBBIN_LOG:-info} +export BOBBIN_INGEST_IDLE_PROMOTE_MIN_EVENTS=1 +export BOBBIN_MODERATION_LABELER_DID=did:plc:zu6y5f5sbudyxhv5ytiirkjx +export BOBBIN_MODERATION_LABELER_URL=https://ozone.tngl.boltless.dev +export BOBBIN_OTLP_ENDPOINT=${BOBBIN_OTLP_ENDPOINT:-} +export SSL_CERT_FILE=$TNGL_CERTS/root.crt + +default='exec "$BOBBIN_BIN"' +exec sh -c "${BOBBIN_CMD:-$default}" diff --git a/localinfra/pc/services/bobbin/setup b/localinfra/pc/services/bobbin/setup new file mode 100755 index 0000000000000000000000000000000000000000..316ad1d93659008fc2c02b13a19953c316df148e --- /dev/null +++ b/localinfra/pc/services/bobbin/setup @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +if [ -z "${BOBBIN_CMD:-}" ] && [ "$BOBBIN_BIN" = "$BOBBIN_BUILT" ]; then + (cd "$TNGL_ROOT" && cargo build -p bobbin) +fi diff --git a/localinfra/pc/services/camo/Caddyfile b/localinfra/pc/services/camo/Caddyfile new file mode 100644 index 0000000000000000000000000000000000000000..6c11a96e4b328360e9d1a90c16e78a0f66fefd64 --- /dev/null +++ b/localinfra/pc/services/camo/Caddyfile @@ -0,0 +1,4 @@ +camo.tngl.boltless.dev { + tls internal + reverse_proxy 127.0.0.1:{$TNGL_CAMO_PORT} +} diff --git a/localinfra/pc/services/camo/commands.env b/localinfra/pc/services/camo/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..edefb072696618fa84d5a2e89e4d68a3d75a1477 --- /dev/null +++ b/localinfra/pc/services/camo/commands.env @@ -0,0 +1,2 @@ +CAMO_BUILT=$TNGL_ROOT/camo/node_modules/.bin/wrangler +CAMO_BIN=${CAMO_BIN:-$CAMO_BUILT} diff --git a/localinfra/pc/services/camo/doctor b/localinfra/pc/services/camo/doctor new file mode 100644 index 0000000000000000000000000000000000000000..ee48c3e68af28669cdb652a45294dc2e17503574 --- /dev/null +++ b/localinfra/pc/services/camo/doctor @@ -0,0 +1,3 @@ +if [ -z "${CAMO_CMD:-}" ]; then + if [ "$CAMO_BIN" = "$CAMO_BUILT" ]; then need pnpm node; else need "$CAMO_BIN"; fi +fi diff --git a/localinfra/pc/services/camo/packages b/localinfra/pc/services/camo/packages new file mode 100644 index 0000000000000000000000000000000000000000..7ef5d368121780b15ba317f360bd8166d74b4861 --- /dev/null +++ b/localinfra/pc/services/camo/packages @@ -0,0 +1,2 @@ +pnpm +nodejs_22 diff --git a/localinfra/pc/services/camo/ports.env b/localinfra/pc/services/camo/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..56188d33f9572f393339cc0c8fb0b9d62152cfde --- /dev/null +++ b/localinfra/pc/services/camo/ports.env @@ -0,0 +1,2 @@ +TNGL_CAMO_PORT=${TNGL_CAMO_PORT:-52660} +TNGL_CAMO_INSPECTOR_PORT=${TNGL_CAMO_INSPECTOR_PORT:-52670} diff --git a/localinfra/pc/services/camo/process.yaml b/localinfra/pc/services/camo/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3b5f4c8da8314147002ee08dfb2b3d260c3da28 --- /dev/null +++ b/localinfra/pc/services/camo/process.yaml @@ -0,0 +1,17 @@ +version: "0.5" +processes: + camo: + command: offline ${TNGL_PC}/services/camo/run + depends_on: + dns: + condition: process_healthy + caddy: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: / + port: ${TNGL_CAMO_PORT} + period_seconds: 1 + failure_threshold: 60 diff --git a/localinfra/pc/services/camo/run b/localinfra/pc/services/camo/run new file mode 100755 index 0000000000000000000000000000000000000000..ba48b80db892bb6afc3d2d12847fef4fdceab4aa --- /dev/null +++ b/localinfra/pc/services/camo/run @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +export CAMO_SHARED_SECRET=${CAMO_SHARED_SECRET:-localinfra-camo-secret} +export WRANGLER_SEND_METRICS=false +export SSL_CERT_FILE=$TNGL_CERTS/root.crt +export NODE_EXTRA_CA_CERTS=$TNGL_CERTS/root.crt + +# wrangler keeps its state in the project dir, so the cd has to come first +default='cd "$TNGL_ROOT/camo" && exec "$CAMO_BIN" dev --ip 127.0.0.1 --port "$TNGL_CAMO_PORT" --inspector-port "$TNGL_CAMO_INSPECTOR_PORT" --var "CAMO_SHARED_SECRET:$CAMO_SHARED_SECRET"' +exec sh -c "${CAMO_CMD:-$default}" diff --git a/localinfra/pc/services/camo/setup b/localinfra/pc/services/camo/setup new file mode 100755 index 0000000000000000000000000000000000000000..74e3d1d6c43b621158e7de0d7677cf0a598f4188 --- /dev/null +++ b/localinfra/pc/services/camo/setup @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +if [ -z "${CAMO_CMD:-}" ] && [ "$CAMO_BIN" = "$CAMO_BUILT" ] && [ ! -e "$CAMO_BUILT" ]; then + (cd "$TNGL_ROOT/camo" && pnpm install --frozen-lockfile) +fi +patch-workerd "$TNGL_ROOT/camo" diff --git a/localinfra/pc/services/deliberi/Caddyfile b/localinfra/pc/services/deliberi/Caddyfile new file mode 100644 index 0000000000000000000000000000000000000000..d7821953126b0cb21e8c9aabdc7cb2514a24f744 --- /dev/null +++ b/localinfra/pc/services/deliberi/Caddyfile @@ -0,0 +1,10 @@ +deliberi.tngl.boltless.dev { + tls internal + reverse_proxy 127.0.0.1:{$TNGL_DELIBERI_PORT} +} + +turnstile.tngl.boltless.dev { + tls internal + header Content-Type application/json + respond /siteverify `{"success":true}` 200 +} diff --git a/localinfra/pc/services/deliberi/commands.env b/localinfra/pc/services/deliberi/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..90ba2c9edfb688b47f6495d0e36fe0683e0643c2 --- /dev/null +++ b/localinfra/pc/services/deliberi/commands.env @@ -0,0 +1,4 @@ +DELIBERI_BUILT=$TNGL_STATE/bin/deliberi +DELIBERI_BIN=${DELIBERI_BIN:-$DELIBERI_BUILT} +DELIBERI_BOOTSTRAP_BUILT=$TNGL_STATE/bin/deliberi-bootstrap +DELIBERI_BOOTSTRAP_BIN=${DELIBERI_BOOTSTRAP_BIN:-$DELIBERI_BOOTSTRAP_BUILT} diff --git a/localinfra/pc/services/deliberi/doctor b/localinfra/pc/services/deliberi/doctor new file mode 100644 index 0000000000000000000000000000000000000000..c12bbbbda46b7124834bdfd021208b2ee7b03c85 --- /dev/null +++ b/localinfra/pc/services/deliberi/doctor @@ -0,0 +1,4 @@ +if [ -z "${DELIBERI_CMD:-}" ]; then + if [ "$DELIBERI_BIN" = "$DELIBERI_BUILT" ]; then need go; else need "$DELIBERI_BIN"; fi +fi +[ "$DELIBERI_BOOTSTRAP_BIN" = "$DELIBERI_BOOTSTRAP_BUILT" ] || need "$DELIBERI_BOOTSTRAP_BIN" diff --git a/localinfra/pc/services/deliberi/packages b/localinfra/pc/services/deliberi/packages new file mode 100644 index 0000000000000000000000000000000000000000..f63a3046c08633a9ffc147e26c3f63790042329f --- /dev/null +++ b/localinfra/pc/services/deliberi/packages @@ -0,0 +1,2 @@ +sqlite +jq diff --git a/localinfra/pc/services/deliberi/ports.env b/localinfra/pc/services/deliberi/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..09811d6e41889211b0023bab61db436c306ceddd --- /dev/null +++ b/localinfra/pc/services/deliberi/ports.env @@ -0,0 +1 @@ +TNGL_DELIBERI_PORT=${TNGL_DELIBERI_PORT:-52622} diff --git a/localinfra/pc/services/deliberi/process.yaml b/localinfra/pc/services/deliberi/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ebb494d655e9b6d713ac1ac3e105cbb0f766388 --- /dev/null +++ b/localinfra/pc/services/deliberi/process.yaml @@ -0,0 +1,23 @@ +version: "0.5" +processes: + deliberi: + command: offline ${TNGL_PC}/services/deliberi/run + depends_on: + dns: + condition: process_healthy + caddy: + condition: process_healthy + pds: + condition: process_healthy + hydrant: + condition: process_healthy + init-data: + condition: process_completed_successfully + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: /xrpc/_health + port: ${TNGL_DELIBERI_PORT} + period_seconds: 1 + failure_threshold: 120 diff --git a/localinfra/pc/services/deliberi/run b/localinfra/pc/services/deliberi/run new file mode 100755 index 0000000000000000000000000000000000000000..f8d030f432a15ade2f8f5ffc45f6a850fc605f07 --- /dev/null +++ b/localinfra/pc/services/deliberi/run @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu +data=$TNGL_STATE/data/deliberi +mkdir -p "$data" + +[ -f "$TNGL_ROOT/localinfra/pds.env" ] && . "$TNGL_ROOT/localinfra/pds.env" + +export DELIBERI_DEV=true +export DELIBERI_HOSTNAME=deliberi.tngl.boltless.dev +export DELIBERI_LISTEN_ADDR=127.0.0.1:$TNGL_DELIBERI_PORT +export DELIBERI_DB_PATH=$data/deliberi.db +export DELIBERI_PLC_URL=$TNGL_PLC_URL +export DELIBERI_JETSTREAM_ENDPOINT=wss://hydrant.tngl.boltless.dev/subscribe +export DELIBERI_BOBBIN_API_URL=http://127.0.0.1:${TNGL_BOBBIN_PORT:-52681} +export DELIBERI_BASE_URL=https://web.tngl.boltless.dev +export DELIBERI_PDS_HOST=https://pds.tngl.boltless.dev +export DELIBERI_PDS_ADMIN_SECRET=${PDS_ADMIN_PASSWORD:-d6a902588cd93bee1af83f924f60cfd3} +export DELIBERI_TURNSTILE_SECRET_KEY=localinfra-turnstile-secret +export DELIBERI_TURNSTILE_VERIFY_URL=https://turnstile.tngl.boltless.dev/siteverify +export DELIBERI_TRUSTED_SERVICE_DIDS=did:web:mirror.tngl.boltless.dev,did:web:delegates.tngl.boltless.dev +export SSL_CERT_FILE=$TNGL_CERTS/root.crt + +if [ -f "$TNGL_STATE/shared/verified-emails.jsonl" ]; then + "$DELIBERI_BOOTSTRAP_BIN" --file "$TNGL_STATE/shared/verified-emails.jsonl" +fi + +default='exec "$DELIBERI_BIN" serve' +exec sh -c "${DELIBERI_CMD:-$default}" diff --git a/localinfra/pc/services/deliberi/setup b/localinfra/pc/services/deliberi/setup new file mode 100755 index 0000000000000000000000000000000000000000..a6089f423f63626967b850fc63de78954a1450cd --- /dev/null +++ b/localinfra/pc/services/deliberi/setup @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +if [ -z "${DELIBERI_CMD:-}" ] && [ "$DELIBERI_BIN" = "$DELIBERI_BUILT" ] && [ ! -x "$DELIBERI_BUILT" ]; then + (cd "$TNGL_ROOT" && go build -tags libsqlite3 -o "$TNGL_STATE/bin/deliberi" ./cmd/deliberi) +fi +if [ "$DELIBERI_BOOTSTRAP_BIN" = "$DELIBERI_BOOTSTRAP_BUILT" ] && [ ! -x "$DELIBERI_BOOTSTRAP_BUILT" ]; then + (cd "$TNGL_ROOT" && go build -tags libsqlite3 -o "$TNGL_STATE/bin/deliberi-bootstrap" ./localinfra/cmd/deliberi-bootstrap) +fi diff --git a/localinfra/pc/services/github-stub/Caddyfile b/localinfra/pc/services/github-stub/Caddyfile new file mode 100644 index 0000000000000000000000000000000000000000..b60ee7359ad1d9f3379f68a6d94fb9344e23fd9c --- /dev/null +++ b/localinfra/pc/services/github-stub/Caddyfile @@ -0,0 +1,4 @@ +github.tngl.boltless.dev, api.github.tngl.boltless.dev { + tls internal + reverse_proxy 127.0.0.1:{$TNGL_GITHUB_STUB_PORT} +} diff --git a/localinfra/pc/services/github-stub/commands.env b/localinfra/pc/services/github-stub/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..e30c063048b7fae3e6c9a46bd1e0c9199ec330f0 --- /dev/null +++ b/localinfra/pc/services/github-stub/commands.env @@ -0,0 +1 @@ +GITHUB_STUB_BIN=${GITHUB_STUB_BIN:-python3} diff --git a/localinfra/pc/services/github-stub/doctor b/localinfra/pc/services/github-stub/doctor new file mode 100644 index 0000000000000000000000000000000000000000..7765240cfc98bd8fe325cfe1bd62accbaa872f66 --- /dev/null +++ b/localinfra/pc/services/github-stub/doctor @@ -0,0 +1,3 @@ +if [ -z "${GITHUB_STUB_CMD:-}" ]; then + need "$GITHUB_STUB_BIN" git +fi diff --git a/localinfra/pc/services/github-stub/packages b/localinfra/pc/services/github-stub/packages new file mode 100644 index 0000000000000000000000000000000000000000..dae42454293f66cbc62e464bd476639989cc1275 --- /dev/null +++ b/localinfra/pc/services/github-stub/packages @@ -0,0 +1,2 @@ +python3 +git diff --git a/localinfra/pc/services/github-stub/ports.env b/localinfra/pc/services/github-stub/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..6b0204de4f8d93ddd6ddf957b37b5cdb4fc6f6d2 --- /dev/null +++ b/localinfra/pc/services/github-stub/ports.env @@ -0,0 +1 @@ +TNGL_GITHUB_STUB_PORT=${TNGL_GITHUB_STUB_PORT:-52662} diff --git a/localinfra/pc/services/github-stub/process.yaml b/localinfra/pc/services/github-stub/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb0c268b0628663e1ed6cfd0aafc66611f994a4a --- /dev/null +++ b/localinfra/pc/services/github-stub/process.yaml @@ -0,0 +1,15 @@ +version: "0.5" +processes: + github-stub: + command: offline ${TNGL_PC}/services/github-stub/run + depends_on: + caddy: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: /user + port: ${TNGL_GITHUB_STUB_PORT} + period_seconds: 1 + failure_threshold: 60 diff --git a/localinfra/pc/services/github-stub/run b/localinfra/pc/services/github-stub/run new file mode 100755 index 0000000000000000000000000000000000000000..4365e9195c9db944d3f4509dce87f37bf5d81a62 --- /dev/null +++ b/localinfra/pc/services/github-stub/run @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu + +data=$TNGL_STATE/data/github-stub +mkdir -p "$data/repos" + +export PORT=$TNGL_GITHUB_STUB_PORT +export REPOS_DIR=$data/repos +export GITHUB_PUBLIC_ORIGIN=https://github.tngl.boltless.dev +export GITHUB_API_ORIGIN=https://github.tngl.boltless.dev +export GITHUB_AVATARS_ORIGIN=https://github.tngl.boltless.dev +export GITHUB_APP_SETUP_URL=${GITHUB_APP_SETUP_URL:-https://web.tngl.boltless.dev/_internal/github/install/callback} +export PYTHONUNBUFFERED=1 + +default='exec "$GITHUB_STUB_BIN" "$TNGL_ROOT/localinfra/github-stub/server.py"' +exec sh -c "${GITHUB_STUB_CMD:-$default}" diff --git a/localinfra/pc/services/github-stub/setup b/localinfra/pc/services/github-stub/setup new file mode 100755 index 0000000000000000000000000000000000000000..a33c49b7568f08fdd86c6289bc372ba80dbddb92 --- /dev/null +++ b/localinfra/pc/services/github-stub/setup @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +data=$TNGL_STATE/data/github-stub +mkdir -p "$data/repos" + +if [ -z "${GITHUB_STUB_CMD:-}" ]; then + REPOS_DIR="$data/repos" "$GITHUB_STUB_BIN" -c 'import sys; sys.path.insert(0, "'"$TNGL_ROOT"'/localinfra/github-stub"); from server import seed_repositories; seed_repositories()' +fi diff --git a/localinfra/pc/services/grafana/commands.env b/localinfra/pc/services/grafana/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..a49ab89679ab6f16889af6223e1c8695ffc102fc --- /dev/null +++ b/localinfra/pc/services/grafana/commands.env @@ -0,0 +1 @@ +GRAFANA_BIN=${GRAFANA_BIN:-grafana} diff --git a/localinfra/pc/services/grafana/doctor b/localinfra/pc/services/grafana/doctor new file mode 100644 index 0000000000000000000000000000000000000000..e478a067f1f409b5cf8cd17a77aaef7081b120bd --- /dev/null +++ b/localinfra/pc/services/grafana/doctor @@ -0,0 +1 @@ +[ -n "${GRAFANA_CMD:-}" ] || need "$GRAFANA_BIN" diff --git a/localinfra/pc/services/grafana/packages b/localinfra/pc/services/grafana/packages new file mode 100644 index 0000000000000000000000000000000000000000..59081c90a063f2f9e678ad2b46dbd8c0f3548196 --- /dev/null +++ b/localinfra/pc/services/grafana/packages @@ -0,0 +1 @@ +grafana diff --git a/localinfra/pc/services/grafana/ports.env b/localinfra/pc/services/grafana/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..fc6a695586aa2ffef10dfa788a6249d5a5ee05c5 --- /dev/null +++ b/localinfra/pc/services/grafana/ports.env @@ -0,0 +1 @@ +TNGL_GRAFANA_PORT=${TNGL_GRAFANA_PORT:-52755} diff --git a/localinfra/pc/services/grafana/process.yaml b/localinfra/pc/services/grafana/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd3cbf400257fd74b4658ea4a161b6a9ccc94c5c --- /dev/null +++ b/localinfra/pc/services/grafana/process.yaml @@ -0,0 +1,19 @@ +version: "0.5" +processes: + grafana: + command: offline ${TNGL_PC}/services/grafana/run + depends_on: + prometheus: + condition: process_healthy + loki: + condition: process_healthy + tempo: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: /api/health + port: ${TNGL_GRAFANA_PORT} + period_seconds: 1 + failure_threshold: 60 diff --git a/localinfra/pc/services/grafana/profile b/localinfra/pc/services/grafana/profile new file mode 100644 index 0000000000000000000000000000000000000000..d5739294086c81f669e404c9c01aa577c7175d99 --- /dev/null +++ b/localinfra/pc/services/grafana/profile @@ -0,0 +1 @@ +observability diff --git a/localinfra/pc/services/grafana/run b/localinfra/pc/services/grafana/run new file mode 100755 index 0000000000000000000000000000000000000000..2e4dcfc12dfe1f5065db952d3e233a6390cf11d7 --- /dev/null +++ b/localinfra/pc/services/grafana/run @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu +export data=$TNGL_STATE/data/grafana +prov=$data/provisioning +mkdir -p "$data/data" "$data/logs" "$prov/datasources" "$prov/dashboards" + +# same source compose mounts, so the dashboards can't drift between the two +cp "$TNGL_ROOT"/localinfra/observability/grafana/provisioning/dashboards/*.json "$prov/dashboards/" +cat > "$prov/dashboards/spindle.yml" < "$prov/datasources/prometheus.yml" + +export GF_PATHS_DATA=$data/data +export GF_PATHS_LOGS=$data/logs +export GF_PATHS_PROVISIONING=$prov +export GF_SERVER_HTTP_ADDR=127.0.0.1 +export GF_SERVER_HTTP_PORT=$TNGL_GRAFANA_PORT +export GF_ANALYTICS_REPORTING_ENABLED=false +export GF_ANALYTICS_CHECK_FOR_UPDATES=false + +export home=$(dirname "$(readlink -f "$(command -v "$GRAFANA_BIN")")")/../share/grafana +default='exec "$GRAFANA_BIN" server --homepath "$home"' +exec sh -c "${GRAFANA_CMD:-$default}" diff --git a/localinfra/pc/services/knotmirror/Caddyfile b/localinfra/pc/services/knotmirror/Caddyfile new file mode 100644 index 0000000000000000000000000000000000000000..9fda97a10e5f34d215f6dc0dc7385de1ab3f050c --- /dev/null +++ b/localinfra/pc/services/knotmirror/Caddyfile @@ -0,0 +1,4 @@ +mirror.tngl.boltless.dev { + tls internal + import cors 127.0.0.1:{$TNGL_KNOTMIRROR_PORT} +} diff --git a/localinfra/pc/services/knotmirror/commands.env b/localinfra/pc/services/knotmirror/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..8831276929d53016dc7b2c4c7a73ef7c56ffdbad --- /dev/null +++ b/localinfra/pc/services/knotmirror/commands.env @@ -0,0 +1,2 @@ +KNOTMIRROR_BUILT=${CARGO_TARGET_DIR:-$TNGL_ROOT/target}/debug/knotmirror +KNOTMIRROR_BIN=${KNOTMIRROR_BIN:-$KNOTMIRROR_BUILT} diff --git a/localinfra/pc/services/knotmirror/doctor b/localinfra/pc/services/knotmirror/doctor new file mode 100644 index 0000000000000000000000000000000000000000..03b2f254d6eba9673c319e3ffcf770c337d72a39 --- /dev/null +++ b/localinfra/pc/services/knotmirror/doctor @@ -0,0 +1,3 @@ +if [ -z "${KNOTMIRROR_CMD:-}" ]; then + if [ "$KNOTMIRROR_BIN" = "$KNOTMIRROR_BUILT" ]; then need cargo git; else need "$KNOTMIRROR_BIN" git; fi +fi diff --git a/localinfra/pc/services/knotmirror/packages b/localinfra/pc/services/knotmirror/packages new file mode 100644 index 0000000000000000000000000000000000000000..5664e303b5dc2e9ef8e14a0845d9486ec1920afd --- /dev/null +++ b/localinfra/pc/services/knotmirror/packages @@ -0,0 +1 @@ +git diff --git a/localinfra/pc/services/knotmirror/ports.env b/localinfra/pc/services/knotmirror/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..31810124110f983a6168edfe67798cb29f8f37e8 --- /dev/null +++ b/localinfra/pc/services/knotmirror/ports.env @@ -0,0 +1,2 @@ +TNGL_KNOTMIRROR_PORT=${TNGL_KNOTMIRROR_PORT:-52600} +TNGL_KNOTMIRROR_METRICS_PORT=${TNGL_KNOTMIRROR_METRICS_PORT:-52601} diff --git a/localinfra/pc/services/knotmirror/process.yaml b/localinfra/pc/services/knotmirror/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84d8e9f1ad9803bb2958fca637b298e8a733f596 --- /dev/null +++ b/localinfra/pc/services/knotmirror/process.yaml @@ -0,0 +1,21 @@ +version: "0.5" +processes: + knotmirror: + command: offline ${TNGL_PC}/services/knotmirror/run + depends_on: + dns: + condition: process_healthy + pg-init: + condition: process_completed_successfully + redis: + condition: process_healthy + hydrant: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: /metrics + port: ${TNGL_KNOTMIRROR_METRICS_PORT} + period_seconds: 1 + failure_threshold: 60 diff --git a/localinfra/pc/services/knotmirror/run b/localinfra/pc/services/knotmirror/run new file mode 100755 index 0000000000000000000000000000000000000000..4506c57b8cf3791702d18d7ce62e15af9523b233 --- /dev/null +++ b/localinfra/pc/services/knotmirror/run @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu +data=$TNGL_STATE/data/gitmirror +mkdir -p "$data/repos" + +export KNOTMIRROR_BIND=127.0.0.1:$TNGL_KNOTMIRROR_PORT +export KNOTMIRROR_METRICS_BIND=127.0.0.1:$TNGL_KNOTMIRROR_METRICS_PORT +export KNOTMIRROR_SCAN_PATH=$data/repos +export KNOTMIRROR_SERVICE_DID=did:web:mirror.tngl.boltless.dev +export KNOTMIRROR_PLC_URL=$TNGL_PLC_URL +export KNOTMIRROR_REDIS_URL=redis://127.0.0.1:$TNGL_REDIS_PORT +export KNOTMIRROR_DATABASE_URL=postgres://tnglr:tnglr@127.0.0.1:$TNGL_PG_PORT/gitmirror +export KNOTMIRROR_HYDRANT_URL=ws://127.0.0.1:$TNGL_HYDRANT_PORT +# zoekt only learns about a push when knotmirror asks it to index +export KNOTMIRROR_ZOEKT_URL=http://127.0.0.1:$TNGL_ZOEKT_TNGL_INDEXSERVER_PORT +# knot2 and plc are on loopback, which the outbound checks refuse unless this is set +export KNOTMIRROR_KNOT_ALLOW_PRIVATE=true +export KNOTMIRROR_KNOT_REQUIRE_HTTPS=false +export SSL_CERT_FILE=$TNGL_CERTS/root.crt +export RUST_LOG=${RUST_LOG:-knotmirror=debug} +export KNOTMIRROR_LOG=${KNOTMIRROR_LOG:-knotmirror_ingest=debug,knotmirror_sync=debug} + +default='exec "$KNOTMIRROR_BIN" serve' +exec sh -c "${KNOTMIRROR_CMD:-$default}" diff --git a/localinfra/pc/services/knotmirror/setup b/localinfra/pc/services/knotmirror/setup new file mode 100755 index 0000000000000000000000000000000000000000..6b177ae8887521bade2bd9392d0a8a41df6b2290 --- /dev/null +++ b/localinfra/pc/services/knotmirror/setup @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +if [ -z "${KNOTMIRROR_CMD:-}" ] && [ "$KNOTMIRROR_BIN" = "$KNOTMIRROR_BUILT" ]; then + (cd "$TNGL_ROOT" && cargo build -p knotmirror) +fi diff --git a/localinfra/pc/services/loki/commands.env b/localinfra/pc/services/loki/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..a70b32462e5d8e6fbca244c9f8ccce5f532ffabf --- /dev/null +++ b/localinfra/pc/services/loki/commands.env @@ -0,0 +1 @@ +LOKI_BIN=${LOKI_BIN:-loki} diff --git a/localinfra/pc/services/loki/doctor b/localinfra/pc/services/loki/doctor new file mode 100644 index 0000000000000000000000000000000000000000..797a738df4527d1459271dd92b9db5454f20c30d --- /dev/null +++ b/localinfra/pc/services/loki/doctor @@ -0,0 +1 @@ +[ -n "${LOKI_CMD:-}" ] || need "$LOKI_BIN" diff --git a/localinfra/pc/services/loki/packages b/localinfra/pc/services/loki/packages new file mode 100644 index 0000000000000000000000000000000000000000..f601634893f43dc119b74d8b32c347ac634a53b0 --- /dev/null +++ b/localinfra/pc/services/loki/packages @@ -0,0 +1 @@ +grafana-loki diff --git a/localinfra/pc/services/loki/ports.env b/localinfra/pc/services/loki/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..a50648fe8597a6192b231ae87edb5535d9dfc818 --- /dev/null +++ b/localinfra/pc/services/loki/ports.env @@ -0,0 +1,2 @@ +TNGL_LOKI_PORT=${TNGL_LOKI_PORT:-52751} +TNGL_LOKI_GRPC_PORT=${TNGL_LOKI_GRPC_PORT:-52752} diff --git a/localinfra/pc/services/loki/process.yaml b/localinfra/pc/services/loki/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a68047def21f9918e5555f7f5bd1f9f28bde4b4c --- /dev/null +++ b/localinfra/pc/services/loki/process.yaml @@ -0,0 +1,12 @@ +version: "0.5" +processes: + loki: + command: offline ${TNGL_PC}/services/loki/run + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: /ready + port: ${TNGL_LOKI_PORT} + period_seconds: 1 + failure_threshold: 60 diff --git a/localinfra/pc/services/loki/profile b/localinfra/pc/services/loki/profile new file mode 100644 index 0000000000000000000000000000000000000000..d5739294086c81f669e404c9c01aa577c7175d99 --- /dev/null +++ b/localinfra/pc/services/loki/profile @@ -0,0 +1 @@ +observability diff --git a/localinfra/pc/services/loki/run b/localinfra/pc/services/loki/run new file mode 100755 index 0000000000000000000000000000000000000000..040ac204e08f7aed7d56842dda796be15aa13c58 --- /dev/null +++ b/localinfra/pc/services/loki/run @@ -0,0 +1,49 @@ +#!/bin/sh +set -eu +export data=$TNGL_STATE/data/loki +mkdir -p "$data" + +cat > "$data/loki.yml" <&2; exit 1; } + master=$(head -c 32 /dev/urandom | base64) + token=$(head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n') + cat > "$data/migrator.env" < "$data/ncps.yaml" < "$data/nginx.conf" < "$TNGL_STATE/data/pdsls/Caddyfile" < "$data/prometheus.yml" < "$SITES_STATE/wrangler.toml" + +# the r2 bridge config goes first so it owns the port, and both share one state dir +default='"$SITES_BIN" d1 migrations apply tangled-sites --local --config "$SITES_STATE/wrangler.toml" --persist-to "$SITES_STATE" \ + && exec "$SITES_BIN" dev --local \ + --config "$TNGL_ROOT/localinfra/sites-r2-bridge/wrangler.toml" --config "$SITES_STATE/wrangler.toml" \ + --persist-to "$SITES_STATE" --ip 127.0.0.1 --port "$TNGL_SITES_PORT" --inspector-port "$TNGL_SITES_INSPECTOR_PORT" \ + --var "SITES_LOCAL_R2_TOKEN:$SITES_LOCAL_R2_TOKEN"' +exec sh -c "${SITES_CMD:-$default}" diff --git a/localinfra/pc/services/sites/setup b/localinfra/pc/services/sites/setup new file mode 100755 index 0000000000000000000000000000000000000000..64389e166431578bcf4d316fa1ca91dcd2c0be58 --- /dev/null +++ b/localinfra/pc/services/sites/setup @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +[ -z "${SITES_CMD:-}" ] || exit 0 +# worker-build is the slow part, so only rerun it when the worker's sources moved +cd "$TNGL_ROOT/sites" +if [ ! -e build/index.js ] || [ -n "$(find src Cargo.toml Cargo.lock -newer build/index.js | head -n 1)" ]; then + worker-build --release +fi diff --git a/localinfra/pc/services/sitesd/Caddyfile b/localinfra/pc/services/sitesd/Caddyfile new file mode 100644 index 0000000000000000000000000000000000000000..d25920a45987a915bad670d8c31eeb58955a57d8 --- /dev/null +++ b/localinfra/pc/services/sitesd/Caddyfile @@ -0,0 +1,4 @@ +sitesd.tngl.boltless.dev { + tls internal + reverse_proxy 127.0.0.1:{$TNGL_SITESD_PORT} +} diff --git a/localinfra/pc/services/sitesd/commands.env b/localinfra/pc/services/sitesd/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..c2281cad229aef66e171bf8a66169f28f47c4c9a --- /dev/null +++ b/localinfra/pc/services/sitesd/commands.env @@ -0,0 +1,2 @@ +SITESD_BUILT=$TNGL_STATE/bin/sitesd +SITESD_BIN=${SITESD_BIN:-$SITESD_BUILT} diff --git a/localinfra/pc/services/sitesd/doctor b/localinfra/pc/services/sitesd/doctor new file mode 100644 index 0000000000000000000000000000000000000000..3edad24079be93f4ce256e9ce88ec548df3ee86f --- /dev/null +++ b/localinfra/pc/services/sitesd/doctor @@ -0,0 +1,3 @@ +if [ -z "${SITESD_CMD:-}" ]; then + if [ "$SITESD_BIN" = "$SITESD_BUILT" ]; then need go; else need "$SITESD_BIN"; fi +fi diff --git a/localinfra/pc/services/sitesd/ports.env b/localinfra/pc/services/sitesd/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..fa4b38c561d580e14f218c31738b14e0262ae5a2 --- /dev/null +++ b/localinfra/pc/services/sitesd/ports.env @@ -0,0 +1,2 @@ +TNGL_SITESD_PORT=${TNGL_SITESD_PORT:-52780} +TNGL_SITESD_INTERNAL_PORT=${TNGL_SITESD_INTERNAL_PORT:-52781} diff --git a/localinfra/pc/services/sitesd/process.yaml b/localinfra/pc/services/sitesd/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..133b8aa274e1649c4b09ffcb7ba7df071bcfc5cf --- /dev/null +++ b/localinfra/pc/services/sitesd/process.yaml @@ -0,0 +1,19 @@ +version: "0.5" +processes: + sitesd: + command: offline ${TNGL_PC}/services/sitesd/run + depends_on: + redis: + condition: process_healthy + knot2: + condition: process_healthy + sites: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: /healthz + port: ${TNGL_SITESD_PORT} + period_seconds: 1 + failure_threshold: 60 diff --git a/localinfra/pc/services/sitesd/profile b/localinfra/pc/services/sitesd/profile new file mode 100644 index 0000000000000000000000000000000000000000..debbde8822c035c09943d06642763f7be569aa40 --- /dev/null +++ b/localinfra/pc/services/sitesd/profile @@ -0,0 +1 @@ +sites diff --git a/localinfra/pc/services/sitesd/run b/localinfra/pc/services/sitesd/run new file mode 100755 index 0000000000000000000000000000000000000000..43f7058de2f4e44b1ac3681bbd071209f822fbac --- /dev/null +++ b/localinfra/pc/services/sitesd/run @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +export SITESD_DEV=true +export SITESD_LISTEN=127.0.0.1:$TNGL_SITESD_PORT +export SITESD_INTERNAL_ADDR=127.0.0.1:$TNGL_SITESD_INTERNAL_PORT +export SITESD_WORKER_URL=http://127.0.0.1:$TNGL_SITES_PORT +export SITESD_AUDIENCE=did:web:sites.tngl.boltless.dev +export SITESD_ISSUER=did:key:zDnaeg9UkMwsCRgMWXhSc6cn2VUfBLoypo3sRx4GovnUUA6e8 +export SITESD_PRIVATE_KEY=${SITESD_PRIVATE_KEY:-708539e1ad16e9f26329b470f51d328de0977982e8d63da6238c76f134224e68} +export SITESD_KNOTS=knot2.tngl.boltless.dev +export SITESD_JETSTREAM_ENDPOINT=wss://hydrant.tngl.boltless.dev/subscribe +export SITESD_REDIS=127.0.0.1:$TNGL_REDIS_PORT +export SITESD_R2_BRIDGE_URL=http://127.0.0.1:$TNGL_SITES_PORT/__tangled_r2/tangled-sites +export SITESD_R2_BRIDGE_TOKEN=$SITES_LOCAL_R2_TOKEN +export CLOUDFLARE_R2_BUCKET=tangled-sites +export SSL_CERT_FILE=$TNGL_CERTS/root.crt + +default='exec "$SITESD_BIN"' +exec sh -c "${SITESD_CMD:-$default}" diff --git a/localinfra/pc/services/sitesd/setup b/localinfra/pc/services/sitesd/setup new file mode 100755 index 0000000000000000000000000000000000000000..e51c3e9d44cc82834c860419a551ebf73a6b6e7d --- /dev/null +++ b/localinfra/pc/services/sitesd/setup @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +# go build is incremental, so it runs every start and a changed sitesd never runs stale +if [ -z "${SITESD_CMD:-}" ] && [ "$SITESD_BIN" = "$SITESD_BUILT" ]; then + (cd "$TNGL_ROOT" && go build -tags libsqlite3 -o "$TNGL_STATE/bin/sitesd" ./cmd/sitesd) +fi diff --git a/localinfra/pc/services/spindle/Caddyfile b/localinfra/pc/services/spindle/Caddyfile new file mode 100644 index 0000000000000000000000000000000000000000..d4189b0026199436d337927a84b3f55a2bd377af --- /dev/null +++ b/localinfra/pc/services/spindle/Caddyfile @@ -0,0 +1,5 @@ +# spindle sends its own cors headers, and a second set from caddy makes browsers drop the response +spindle.tngl.boltless.dev { + tls internal + reverse_proxy 127.0.0.1:{$TNGL_SPINDLE_PORT} +} diff --git a/localinfra/pc/services/spindle/commands.env b/localinfra/pc/services/spindle/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..227e55f39c11e07b97135e7bd64a4fd69f7658d5 --- /dev/null +++ b/localinfra/pc/services/spindle/commands.env @@ -0,0 +1,3 @@ +SPINDLE_BUILT=$TNGL_STATE/bin/spindle +SPINDLE_BIN=${SPINDLE_BIN:-$SPINDLE_BUILT} +SPINDLE_IMAGE_DIR=${SPINDLE_IMAGE_DIR:-$TNGL_STATE/data/spindle/images} diff --git a/localinfra/pc/services/spindle/doctor b/localinfra/pc/services/spindle/doctor new file mode 100644 index 0000000000000000000000000000000000000000..7132432f51f0e17d85c40c12c64507a9c14f8915 --- /dev/null +++ b/localinfra/pc/services/spindle/doctor @@ -0,0 +1,13 @@ +if [ -z "${SPINDLE_CMD:-}" ]; then + if [ "$SPINDLE_BIN" = "$SPINDLE_BUILT" ]; then need nix; else need "$SPINDLE_BIN"; fi +fi + +if profile-on microvm; then + [ "$(uname -s)" = Linux ] || { echo "the microvm profile needs linux, there is no kvm on macos" >&2; exit 1; } + need "qemu-system-$(uname -m)" mkfs.ext4 unshare ip slirp4netns + # opening the device is what loads vhost_vsock, nothing else triggers it + { : < /dev/vhost-vsock; } 2>/dev/null || { echo "microvm needs the vhost_vsock module: sudo modprobe vhost_vsock" >&2; exit 1; } + for dev in /dev/kvm /dev/vhost-vsock /dev/net/tun; do + [ -w "$dev" ] || { echo "microvm needs to write $dev" >&2; exit 1; } + done +fi diff --git a/localinfra/pc/services/spindle/ports.env b/localinfra/pc/services/spindle/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..9f6142be9d2fd9d545e7c6f2c7daee0d34054ab7 --- /dev/null +++ b/localinfra/pc/services/spindle/ports.env @@ -0,0 +1,2 @@ +TNGL_SPINDLE_PORT=${TNGL_SPINDLE_PORT:-52740} +TNGL_SPINDLE_METRICS_PORT=${TNGL_SPINDLE_METRICS_PORT:-52741} diff --git a/localinfra/pc/services/spindle/process.yaml b/localinfra/pc/services/spindle/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0a3a1de3cf92a2c584d041c3e88729bbc1abfc5 --- /dev/null +++ b/localinfra/pc/services/spindle/process.yaml @@ -0,0 +1,21 @@ +version: "0.5" +processes: + spindle: + command: offline ${TNGL_PC}/services/spindle/run + depends_on: + dns: + condition: process_healthy + plc: + condition: process_healthy + hydrant: + condition: process_healthy + caddy: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: / + port: ${TNGL_SPINDLE_PORT} + period_seconds: 1 + failure_threshold: 60 diff --git a/localinfra/pc/services/spindle/profile b/localinfra/pc/services/spindle/profile new file mode 100644 index 0000000000000000000000000000000000000000..a082ebf85afe9a9796b8ec6008c43108638078af --- /dev/null +++ b/localinfra/pc/services/spindle/profile @@ -0,0 +1,2 @@ +spindle +microvm diff --git a/localinfra/pc/services/spindle/run b/localinfra/pc/services/spindle/run new file mode 100755 index 0000000000000000000000000000000000000000..22107c4cb72a7eca5b527d5c57aa5a5a3ade313f --- /dev/null +++ b/localinfra/pc/services/spindle/run @@ -0,0 +1,54 @@ +#!/bin/sh +set -eu +data=$TNGL_STATE/data/spindle +mkdir -p "$data/hydrant" "$data/repos" "$data/logs" + +export SPINDLE_SERVER_HOSTNAME=spindle.tngl.boltless.dev +export SPINDLE_SERVER_OWNER=did:plc:igzuueb63dytyy3tpaweofuu +# e2e runs arbitrary test accounts on this spindle +export SPINDLE_SERVER_ADMISSION=open +export SPINDLE_SERVER_KEY_PATH=$data/key.pem +export SPINDLE_CONTACT_EMAIL=admin@spindle.tngl.boltless.dev +export SPINDLE_SERVER_LISTEN_ADDR=127.0.0.1:$TNGL_SPINDLE_PORT +export SPINDLE_SERVER_METRICS_LISTEN_ADDR=127.0.0.1:$TNGL_SPINDLE_METRICS_PORT +export SPINDLE_SERVER_DB_PATH=$data/spindle.db +export SPINDLE_SERVER_REPO_DIR=$data/repos +export SPINDLE_SERVER_LOG_DIR=$data/logs +export SPINDLE_SERVER_HYDRANT_DIR=$data/hydrant +export SPINDLE_SERVER_PLC_URL=$TNGL_PLC_URL +export SPINDLE_SERVER_JETSTREAM_ENDPOINT=wss://hydrant.tngl.boltless.dev/subscribe +export SPINDLE_SERVER_DEV=true +export SPINDLE_SERVER_DEV_EXTRA_HOSTS=knot2.tngl.boltless.dev,mirror.tngl.boltless.dev +export SPINDLE_SERVER_CORS_ORIGINS='*' +export SPINDLE_SERVER_LOGS_ADDR=spindle.tngl.boltless.dev:3333 +# schedules default to a 5m minimum, too slow for the cron e2e flow +export SPINDLE_SCHEDULE_MINIMUM_INTERVAL=1m +export SSL_CERT_FILE=$TNGL_CERTS/root.crt + +# 0 is the off switch for these quotas, compose does the same +for scope in USER REPO; do + for resource in CACHE_STORAGE_MIB WORKFLOWS VCPUS MEMORY_MIB DISK_MIB CI_MINUTES_MONTHLY; do + export "SPINDLE_QUOTA_${scope}_${resource}=0" + done +done + +if services | grep -qx ncps; then + export SPINDLE_CACHE_BACKEND=disk + export SPINDLE_NIX_CACHE_READ_URLS=http://127.0.0.1:$TNGL_NCPS_PORT + export SPINDLE_NIX_CACHE_UPLOAD_URL=http://127.0.0.1:$TNGL_NCPS_PORT/upload + export SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= +fi + +if profile-on microvm; then + mkdir -p "$data/overlays" + export SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=$SPINDLE_IMAGE_DIR + export SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=$data/overlays + export SPINDLE_MICROVM_PIPELINES_AGENT_PORT=11240 + # the default image is nixos-x86_64, which won't match what this machine builds + export SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=nixos + # a normal user has no delegated cgroup subtree, so cgroups stay off + export SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=false +fi + +default='exec "$SPINDLE_BIN" run' +exec sh -c "${SPINDLE_CMD:-$default}" diff --git a/localinfra/pc/services/spindle/setup b/localinfra/pc/services/spindle/setup new file mode 100644 index 0000000000000000000000000000000000000000..144a323663c805af6bd24ddc56a33a1c2dc95ea8 --- /dev/null +++ b/localinfra/pc/services/spindle/setup @@ -0,0 +1,36 @@ +#!/bin/sh +set -eu + +# plain go build skips the hydrant rust staticlib, so build the flake package +if [ -z "${SPINDLE_CMD:-}" ] && [ "$SPINDLE_BIN" = "$SPINDLE_BUILT" ]; then + out=$(cd "$TNGL_ROOT" && nix build .#spindle --no-link --print-out-paths) + mkdir -p "$TNGL_STATE/bin" + ln -sf "$out/bin/spindle" "$SPINDLE_BUILT" +fi + +# spindle never makes this key, that's the operator's job +key=$TNGL_STATE/data/spindle/key.pem +if [ ! -f "$key" ]; then + mkdir -p "$(dirname "$key")" + openssl ecparam -name prime256v1 -genkey -noout -out "$key" + chmod 600 "$key" +fi + +if profile-on microvm; then + arch=$(uname -m) + mkdir -p "$SPINDLE_IMAGE_DIR" + for name in ${SPINDLE_IMAGES:-nixos alpine}; do + tarball=$(cd "$TNGL_ROOT" && nix build ".#spindle-$name-image-tarball" --no-link --print-out-paths) + dest=$SPINDLE_IMAGE_DIR/$name-$arch + # outside dest, or the rm -rf on a new tarball takes the stamp with it + stamp=$SPINDLE_IMAGE_DIR/.$name-$arch.tarball + if [ "$(cat "$stamp" 2>/dev/null)" != "$tarball" ]; then + [ -d "$dest" ] && chmod -R +w "$dest" + rm -rf "$dest" + mkdir -p "$dest" + tar -C "$dest" -xzf "$tarball" + echo "$tarball" > "$stamp" + fi + ln -sfn "$name-$arch" "$SPINDLE_IMAGE_DIR/$name" + done +fi diff --git a/localinfra/pc/services/tempo/commands.env b/localinfra/pc/services/tempo/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..65c70082860b798e04bb218ec2bd6bf7c0aca12a --- /dev/null +++ b/localinfra/pc/services/tempo/commands.env @@ -0,0 +1 @@ +TEMPO_BIN=${TEMPO_BIN:-tempo} diff --git a/localinfra/pc/services/tempo/doctor b/localinfra/pc/services/tempo/doctor new file mode 100644 index 0000000000000000000000000000000000000000..317176c43afd48dd669a4db09903426c5ff2fb10 --- /dev/null +++ b/localinfra/pc/services/tempo/doctor @@ -0,0 +1 @@ +[ -n "${TEMPO_CMD:-}" ] || need "$TEMPO_BIN" diff --git a/localinfra/pc/services/tempo/packages b/localinfra/pc/services/tempo/packages new file mode 100644 index 0000000000000000000000000000000000000000..e764a30b350afab0ba74c5c7224feb43176c9fb7 --- /dev/null +++ b/localinfra/pc/services/tempo/packages @@ -0,0 +1 @@ +tempo diff --git a/localinfra/pc/services/tempo/ports.env b/localinfra/pc/services/tempo/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..7dff7798e075c8a41d2b8ce29ee23cf762146a7a --- /dev/null +++ b/localinfra/pc/services/tempo/ports.env @@ -0,0 +1,3 @@ +TNGL_TEMPO_PORT=${TNGL_TEMPO_PORT:-52753} +TNGL_TEMPO_GRPC_PORT=${TNGL_TEMPO_GRPC_PORT:-52754} +TNGL_TEMPO_OTLP_PORT=${TNGL_TEMPO_OTLP_PORT:-52758} diff --git a/localinfra/pc/services/tempo/process.yaml b/localinfra/pc/services/tempo/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90cbd8b10549bac07b2eb03f3134f7051108bba0 --- /dev/null +++ b/localinfra/pc/services/tempo/process.yaml @@ -0,0 +1,12 @@ +version: "0.5" +processes: + tempo: + command: offline ${TNGL_PC}/services/tempo/run + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: /ready + port: ${TNGL_TEMPO_PORT} + period_seconds: 1 + failure_threshold: 60 diff --git a/localinfra/pc/services/tempo/profile b/localinfra/pc/services/tempo/profile new file mode 100644 index 0000000000000000000000000000000000000000..d5739294086c81f669e404c9c01aa577c7175d99 --- /dev/null +++ b/localinfra/pc/services/tempo/profile @@ -0,0 +1 @@ +observability diff --git a/localinfra/pc/services/tempo/run b/localinfra/pc/services/tempo/run new file mode 100755 index 0000000000000000000000000000000000000000..fc57f8af5ee23a4a8da63b89218b8e92e019f135 --- /dev/null +++ b/localinfra/pc/services/tempo/run @@ -0,0 +1,33 @@ +#!/bin/sh +set -eu +export data=$TNGL_STATE/data/tempo +mkdir -p "$data" + +cat > "$data/tempo.yml" </dev/null) +fi +if [ -z "${TRANQUIL_CMD:-}" ] && [ "$TRANQUIL_BIN" = "$TRANQUIL_BUILT" ] && [ ! -x "$TRANQUIL_BUILT" ]; then + (cd "$src" && SQLX_OFFLINE=true cargo build --locked -p tranquil-server --features reqwest/rustls-tls-native-roots) +fi diff --git a/localinfra/pc/services/web/Caddyfile b/localinfra/pc/services/web/Caddyfile new file mode 100644 index 0000000000000000000000000000000000000000..4e3d31fa2aa0220b9a3c721358152f95bb849468 --- /dev/null +++ b/localinfra/pc/services/web/Caddyfile @@ -0,0 +1,9 @@ +web.tngl.boltless.dev { + tls internal + # macos drops past a backlog of 128 and a page asks for a hundred chunks, so caddy queues the rest + reverse_proxy 127.0.0.1:{$TNGL_WEB_PORT} { + transport http { + max_conns_per_host 32 + } + } +} diff --git a/localinfra/pc/services/web/commands.env b/localinfra/pc/services/web/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..5dfc93533183c102e427595a94b22a115574bcf5 --- /dev/null +++ b/localinfra/pc/services/web/commands.env @@ -0,0 +1,2 @@ +WEB_BUILT=$TNGL_ROOT/web/node_modules/.bin/vite +WEB_BIN=${WEB_BIN:-$WEB_BUILT} diff --git a/localinfra/pc/services/web/doctor b/localinfra/pc/services/web/doctor new file mode 100644 index 0000000000000000000000000000000000000000..ba2b841163863fbdb2d930c6df3e436bfb9702e9 --- /dev/null +++ b/localinfra/pc/services/web/doctor @@ -0,0 +1,3 @@ +if [ -z "${WEB_CMD:-}" ]; then + if [ "$WEB_BIN" = "$WEB_BUILT" ]; then need node corepack; else need "$WEB_BIN"; fi +fi diff --git a/localinfra/pc/services/web/packages b/localinfra/pc/services/web/packages new file mode 100644 index 0000000000000000000000000000000000000000..fc406b7e562565fb7753fe989c7d334972fd1407 --- /dev/null +++ b/localinfra/pc/services/web/packages @@ -0,0 +1 @@ +nodejs_22 diff --git a/localinfra/pc/services/web/ports.env b/localinfra/pc/services/web/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..15ffbd94810e8a23b963fe41594cf29991a7cd04 --- /dev/null +++ b/localinfra/pc/services/web/ports.env @@ -0,0 +1 @@ +TNGL_WEB_PORT=${TNGL_WEB_PORT:-52720} diff --git a/localinfra/pc/services/web/process.yaml b/localinfra/pc/services/web/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ce442312ecb520ae413d585c6b90cef293f123e --- /dev/null +++ b/localinfra/pc/services/web/process.yaml @@ -0,0 +1,19 @@ +version: "0.5" +processes: + web: + command: offline ${TNGL_PC}/services/web/run + depends_on: + dns: + condition: process_healthy + caddy: + condition: process_healthy + bobbin: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: / + port: ${TNGL_WEB_PORT} + period_seconds: 1 + failure_threshold: 120 diff --git a/localinfra/pc/services/web/run b/localinfra/pc/services/web/run new file mode 100755 index 0000000000000000000000000000000000000000..913ea5a0c9faf5edc1a22350c063728ebdaa0050 --- /dev/null +++ b/localinfra/pc/services/web/run @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +set -a +. "$TNGL_PC/services/web/web.env" +set +a + +default='cd "$TNGL_ROOT/web" && exec "$WEB_BIN" preview --host 127.0.0.1 --port "$TNGL_WEB_PORT" --strictPort' +exec sh -c "${WEB_CMD:-$default}" diff --git a/localinfra/pc/services/web/setup b/localinfra/pc/services/web/setup new file mode 100755 index 0000000000000000000000000000000000000000..27459c7b81e580e39caf6a80f8b4cb9798adda2e --- /dev/null +++ b/localinfra/pc/services/web/setup @@ -0,0 +1,33 @@ +#!/bin/sh +set -eu +[ -z "${WEB_CMD:-}" ] && [ "$WEB_BIN" = "$WEB_BUILT" ] || exit 0 +web=$TNGL_ROOT/web +data=$TNGL_STATE/data/web +mkdir -p "$data" + +# the ghstub client id and secret have to match make-certs.sh and github-stub +if [ ! -s "$TNGL_CERTS/github.env" ]; then + secret=$(head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n') + printf 'GITHUB_CLIENT_ID=ghstub-client-id\nGITHUB_CLIENT_SECRET=ghstub-client-secret\nGITHUB_SESSION_SECRET=%s\nGITHUB_APP_SLUG=tangled-dev-stub\n' "$secret" > "$TNGL_CERTS/github.env" +fi + +# fonts and logos only exist as a nix package, and web-hash covers web/static so a change rebuilds +static=$(cd "$TNGL_ROOT" && nix build .#web-static-files --no-link --print-out-paths) +if [ "$(cat "$data/static" 2>/dev/null || true)" != "$static" ] || [ ! -d "$web/static/fonts" ]; then + mkdir -p "$web/static" + rm -rf "$web/static/fonts" "$web/static/logos" + cp -fr --no-preserve=ownership,mode "$static"/* "$web/static" + echo "$static" > "$data/static" +fi + +want="$(node "$TNGL_ROOT/e2e/harness/web-hash.ts") $(cksum < "$TNGL_PC/services/web/web.env" | cut -d ' ' -f 1)" +[ "$(cat "$data/built" 2>/dev/null || true)" != "$want" ] || { patch-workerd "$web"; exit 0; } + +set -a +. "$TNGL_PC/services/web/web.env" +set +a +cd "$web" +corepack pnpm install --frozen-lockfile +patch-workerd "$web" +"$WEB_BIN" build +echo "$want" > "$data/built" diff --git a/localinfra/pc/services/web/web.env b/localinfra/pc/services/web/web.env new file mode 100644 index 0000000000000000000000000000000000000000..0c052d89708d91ac3ec517c469166b66a670debf --- /dev/null +++ b/localinfra/pc/services/web/web.env @@ -0,0 +1,34 @@ +# VITE_ vars are baked in at build time, so a change needs a web rebuild +. "$TNGL_CERTS/github.env" +SVELTE_ADAPTER=cloudflare +E2E_TESTIDS=true +BOBBIN_URL=https://bobbin.tngl.boltless.dev +TANGLED_API_URL=http://127.0.0.1:3000 +SITES_DOMAIN=sites.tngl.boltless.dev +TANGLED_SITES_URL=https://sites.tngl.boltless.dev +DELIBERI_URL=https://deliberi.tngl.boltless.dev +PRIMARY_KNOT=knot2.tngl.boltless.dev +PRIMARY_SPINDLE=spindle.tngl.boltless.dev +DELEGATION_URL=https://delegates.tngl.boltless.dev +DELEGATION_DID=did:web:tranquil.tngl.boltless.dev +DELEGATION_HANDLE_DOMAIN=tranquil.tngl.boltless.dev +CAMO_URL=https://camo.tngl.boltless.dev +CAMO_SHARED_SECRET=localinfra-camo-secret +AVATAR_URL=https://avatar.tngl.boltless.dev +AVATAR_SHARED_SECRET=localinfra-avatar-secret +OGRE_URL=https://ogre.tngl.boltless.dev +BLOB_URL=https://web.tngl.boltless.dev +MODERATION_SERVICE_DID=did:plc:zu6y5f5sbudyxhv5ytiirkjx +VITE_HANDLE_RESOLVER_URL=https://tranquil.tngl.boltless.dev,https://pds.tngl.boltless.dev +VITE_PLC_DIRECTORY_URL=https://plc.tngl.boltless.dev +MIGRATOR_URL=https://migrator.tngl.boltless.dev +GITHUB_PUBLIC_ORIGIN=https://github.tngl.boltless.dev +GITHUB_API_ORIGIN=https://github.tngl.boltless.dev +GITHUB_AVATARS_ORIGIN=https://github.tngl.boltless.dev +TURNSTILE_SITE_KEY=${TANGLED_CLOUDFLARE_TURNSTILE_SITE_KEY:-1x00000000000000000000AA} +VITE_DEV_PORT=$TNGL_WEB_PORT +VITE_ALLOWED_HOSTS=.tngl.boltless.dev +VITE_OAUTH_CLIENT_ID=https://web.tngl.boltless.dev/oauth-client-metadata.json +VITE_OAUTH_REDIRECT_URI=https://web.tngl.boltless.dev/oauth/callback +NODE_EXTRA_CA_CERTS=$TNGL_CERTS/root.crt +WRANGLER_SEND_METRICS=false diff --git a/localinfra/pc/services/zoekt-tngl-indexserver/commands.env b/localinfra/pc/services/zoekt-tngl-indexserver/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..5beb4d198439ce51a74fadbaa63fe14593beffb5 --- /dev/null +++ b/localinfra/pc/services/zoekt-tngl-indexserver/commands.env @@ -0,0 +1 @@ +ZOEKT_TNGL_INDEXSERVER_BIN=${ZOEKT_TNGL_INDEXSERVER_BIN:-$TNGL_STATE/bin/zoekt-tngl-indexserver} diff --git a/localinfra/pc/services/zoekt-tngl-indexserver/doctor b/localinfra/pc/services/zoekt-tngl-indexserver/doctor new file mode 100644 index 0000000000000000000000000000000000000000..d65fa44a9a8d76ab420e02f9d185f7dcc1a81acc --- /dev/null +++ b/localinfra/pc/services/zoekt-tngl-indexserver/doctor @@ -0,0 +1,4 @@ +case "${ZOEKT_TNGL_INDEXSERVER_BIN:-}" in +"$TNGL_STATE/bin/zoekt-tngl-indexserver") [ -n "${ZOEKT_TNGL_INDEXSERVER_CMD:-}" ] || need go git ;; +*) [ -n "${ZOEKT_TNGL_INDEXSERVER_CMD:-}" ] || need "$ZOEKT_TNGL_INDEXSERVER_BIN" ;; +esac diff --git a/localinfra/pc/services/zoekt-tngl-indexserver/packages b/localinfra/pc/services/zoekt-tngl-indexserver/packages new file mode 100644 index 0000000000000000000000000000000000000000..5664e303b5dc2e9ef8e14a0845d9486ec1920afd --- /dev/null +++ b/localinfra/pc/services/zoekt-tngl-indexserver/packages @@ -0,0 +1 @@ +git diff --git a/localinfra/pc/services/zoekt-tngl-indexserver/ports.env b/localinfra/pc/services/zoekt-tngl-indexserver/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..abaff7543e8c3e2749635f58e78836d1fbd34824 --- /dev/null +++ b/localinfra/pc/services/zoekt-tngl-indexserver/ports.env @@ -0,0 +1 @@ +TNGL_ZOEKT_TNGL_INDEXSERVER_PORT=${TNGL_ZOEKT_TNGL_INDEXSERVER_PORT:-52641} diff --git a/localinfra/pc/services/zoekt-tngl-indexserver/process.yaml b/localinfra/pc/services/zoekt-tngl-indexserver/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3b6e9b604b5ec17ba8b08de377140a04c00f256 --- /dev/null +++ b/localinfra/pc/services/zoekt-tngl-indexserver/process.yaml @@ -0,0 +1,23 @@ +version: "0.5" +processes: + zoekt-tngl-indexserver: + command: offline ${TNGL_PC}/services/zoekt-tngl-indexserver/run + depends_on: + init-data: + condition: process_completed_successfully + knot2: + condition: process_healthy + caddy: + condition: process_healthy + dns: + condition: process_healthy + plc: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: / + port: ${TNGL_ZOEKT_TNGL_INDEXSERVER_PORT} + period_seconds: 1 + failure_threshold: 30 diff --git a/localinfra/pc/services/zoekt-tngl-indexserver/run b/localinfra/pc/services/zoekt-tngl-indexserver/run new file mode 100755 index 0000000000000000000000000000000000000000..56018a2048a57ca6a63c045133529af49d6bc29c --- /dev/null +++ b/localinfra/pc/services/zoekt-tngl-indexserver/run @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +index_dir=$TNGL_STATE/data/zoekt-index +mkdir -p "$index_dir" "$TNGL_STATE/data/zoekt-tngl-indexserver" +export PATH=$TNGL_STATE/bin:$PATH +export TANGLED_ZOEKT_INDEX_DIR=$index_dir +export TANGLED_ZOEKT_PLC_URL=${TNGL_PLC_URL:-https://plc.tngl.boltless.dev} +export TANGLED_ZOEKT_ALLOW_HTTP=true +# result links use this as their base, so it has to be the web app +export TANGLED_ZOEKT_APPVIEW_URL=https://web.tngl.boltless.dev +export TANGLED_ZOEKT_SERVER_LISTEN=127.0.0.1:$TNGL_ZOEKT_TNGL_INDEXSERVER_PORT +# local knots and plc use the dev ca, which the system store doesn't trust +export SSL_CERT_FILE=$TNGL_CERTS/root.crt +export GIT_SSL_CAINFO=$TNGL_CERTS/root.crt +default='exec "$ZOEKT_TNGL_INDEXSERVER_BIN" serve' +exec sh -c "${ZOEKT_TNGL_INDEXSERVER_CMD:-$default}" diff --git a/localinfra/pc/services/zoekt-tngl-indexserver/setup b/localinfra/pc/services/zoekt-tngl-indexserver/setup new file mode 100755 index 0000000000000000000000000000000000000000..a94322da02b8dec1a1e11be47e0e506b5a7e6a72 --- /dev/null +++ b/localinfra/pc/services/zoekt-tngl-indexserver/setup @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +mkdir -p "$TNGL_STATE/bin" +if [ ! -f "$TNGL_STATE/bin/zoekt-tngl-indexserver" ]; then + (cd "$TNGL_ROOT" && go build -o "$TNGL_STATE/bin/zoekt-tngl-indexserver" ./cmd/zoekt-tngl-indexserver) +fi +if [ ! -f "$TNGL_STATE/bin/zoekt-git-index" ]; then + zoekt_src=$TNGL_STATE/src/zoekt + if [ ! -d "$zoekt_src/.git" ]; then + mkdir -p "$zoekt_src" + git -C "$zoekt_src" init -q + git -C "$zoekt_src" fetch -q --depth 1 https://tangled.org/boltless.me/zoekt.git cfc4aa0d9ea620143fef099bfed61a88e434c9cd + git -C "$zoekt_src" checkout -q FETCH_HEAD + fi + (cd "$zoekt_src" && go build -o "$TNGL_STATE/bin/zoekt-git-index" ./cmd/zoekt-git-index) +fi diff --git a/localinfra/pc/services/zoekt-webserver/Caddyfile b/localinfra/pc/services/zoekt-webserver/Caddyfile new file mode 100644 index 0000000000000000000000000000000000000000..1e014084b81658dc1c5957488dc26d39279f987a --- /dev/null +++ b/localinfra/pc/services/zoekt-webserver/Caddyfile @@ -0,0 +1,11 @@ +zoekt.tngl.boltless.dev { + tls internal + + handle_path /indexserver/* { + reverse_proxy 127.0.0.1:{$TNGL_ZOEKT_TNGL_INDEXSERVER_PORT:52641} + } + + handle { + reverse_proxy 127.0.0.1:{$TNGL_ZOEKT_WEBSERVER_PORT:52640} + } +} diff --git a/localinfra/pc/services/zoekt-webserver/commands.env b/localinfra/pc/services/zoekt-webserver/commands.env new file mode 100644 index 0000000000000000000000000000000000000000..56239d89ff3465067d941909040e5cd3e8046530 --- /dev/null +++ b/localinfra/pc/services/zoekt-webserver/commands.env @@ -0,0 +1 @@ +ZOEKT_WEBSERVER_BIN=${ZOEKT_WEBSERVER_BIN:-$TNGL_STATE/bin/zoekt-webserver} diff --git a/localinfra/pc/services/zoekt-webserver/doctor b/localinfra/pc/services/zoekt-webserver/doctor new file mode 100644 index 0000000000000000000000000000000000000000..67a5397cf01e7a545ae48a5c44ace8b6a74c462d --- /dev/null +++ b/localinfra/pc/services/zoekt-webserver/doctor @@ -0,0 +1,4 @@ +case "${ZOEKT_WEBSERVER_BIN:-}" in +"$TNGL_STATE/bin/zoekt-webserver") [ -n "${ZOEKT_WEBSERVER_CMD:-}" ] || need go git ;; +*) [ -n "${ZOEKT_WEBSERVER_CMD:-}" ] || need "$ZOEKT_WEBSERVER_BIN" ;; +esac diff --git a/localinfra/pc/services/zoekt-webserver/packages b/localinfra/pc/services/zoekt-webserver/packages new file mode 100644 index 0000000000000000000000000000000000000000..5664e303b5dc2e9ef8e14a0845d9486ec1920afd --- /dev/null +++ b/localinfra/pc/services/zoekt-webserver/packages @@ -0,0 +1 @@ +git diff --git a/localinfra/pc/services/zoekt-webserver/ports.env b/localinfra/pc/services/zoekt-webserver/ports.env new file mode 100644 index 0000000000000000000000000000000000000000..9fc638a7ff25a17a612383f343be3b011b349181 --- /dev/null +++ b/localinfra/pc/services/zoekt-webserver/ports.env @@ -0,0 +1 @@ +TNGL_ZOEKT_WEBSERVER_PORT=${TNGL_ZOEKT_WEBSERVER_PORT:-52640} diff --git a/localinfra/pc/services/zoekt-webserver/process.yaml b/localinfra/pc/services/zoekt-webserver/process.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de621e0040af88ad3a08b6a1633311a1e607772a --- /dev/null +++ b/localinfra/pc/services/zoekt-webserver/process.yaml @@ -0,0 +1,17 @@ +version: "0.5" +processes: + zoekt-webserver: + command: offline ${TNGL_PC}/services/zoekt-webserver/run + depends_on: + init-data: + condition: process_completed_successfully + caddy: + condition: process_healthy + readiness_probe: + http_get: + host: 127.0.0.1 + scheme: http + path: / + port: ${TNGL_ZOEKT_WEBSERVER_PORT} + period_seconds: 1 + failure_threshold: 30 diff --git a/localinfra/pc/services/zoekt-webserver/run b/localinfra/pc/services/zoekt-webserver/run new file mode 100755 index 0000000000000000000000000000000000000000..54524aba927c30eee1aa73dacc6c924d1130564e --- /dev/null +++ b/localinfra/pc/services/zoekt-webserver/run @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +export ZOEKT_INDEX_DIR=${ZOEKT_INDEX_DIR:-$TNGL_STATE/data/zoekt-index} +mkdir -p "$ZOEKT_INDEX_DIR" "$TNGL_STATE/data/zoekt-webserver" +default='exec "$ZOEKT_WEBSERVER_BIN" -listen 127.0.0.1:$TNGL_ZOEKT_WEBSERVER_PORT -index "'"$ZOEKT_INDEX_DIR"'" -pprof -rpc' +exec sh -c "${ZOEKT_WEBSERVER_CMD:-$default}" diff --git a/localinfra/pc/services/zoekt-webserver/setup b/localinfra/pc/services/zoekt-webserver/setup new file mode 100755 index 0000000000000000000000000000000000000000..5af91f0b7f2cedf1bedffa91c5264b1e85159224 --- /dev/null +++ b/localinfra/pc/services/zoekt-webserver/setup @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +mkdir -p "$TNGL_STATE/src" "$TNGL_STATE/bin" +zoekt_src=$TNGL_STATE/src/zoekt +if [ ! -d "$zoekt_src/.git" ]; then + mkdir -p "$zoekt_src" + git -C "$zoekt_src" init -q + git -C "$zoekt_src" fetch -q --depth 1 https://tangled.org/boltless.me/zoekt.git cfc4aa0d9ea620143fef099bfed61a88e434c9cd + git -C "$zoekt_src" checkout -q FETCH_HEAD +fi +if [ ! -f "$TNGL_STATE/bin/zoekt-webserver" ]; then + (cd "$zoekt_src" && go build -o "$TNGL_STATE/bin/zoekt-webserver" ./cmd/zoekt-webserver) +fi +if [ ! -f "$TNGL_STATE/bin/zoekt-git-index" ]; then + (cd "$zoekt_src" && go build -o "$TNGL_STATE/bin/zoekt-git-index" ./cmd/zoekt-git-index) +fi diff --git a/localinfra/pds-patch.mjs b/localinfra/pds-patch.mjs new file mode 100644 index 0000000000000000000000000000000000000000..b38d4c3e235a3b35cd26fedf197e7b046039627d --- /dev/null +++ b/localinfra/pds-patch.mjs @@ -0,0 +1,103 @@ +import { createHash } from "node:crypto"; +import { copyFileSync, existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { basename, dirname, join } from "node:path"; + +const [appDir, themePath, fontPath] = process.argv.slice(2); + +function packageDir(from, name) { + for (const dir of createRequire(join(from, "/")).resolve.paths(name)) { + const candidate = join(dir, name); + if (existsSync(candidate)) return realpathSync(candidate); + } + throw new Error(`${name} not resolvable from ${from}`); +} + +const pdsDir = packageDir(appDir, "@atproto/pds"); +const providerDir = join(packageDir(pdsDir, "@atproto/oauth-provider"), "dist"); +const uiManifestPath = createRequire(join(providerDir, "router/assets/assets.js")).resolve( + "@atproto/oauth-provider-ui/bundle-manifest.json", +); + +const sha256 = (data) => createHash("sha256").update(data).digest("base64"); + +function patch(path, replacements, { all = false, append = "" } = {}) { + let text = readFileSync(path, "utf8"); + for (const [from, to] of replacements) { + const count = text.split(from).length - 1; + if (all ? count === 0 : count !== 1) throw new Error(`${path}: expected ${all ? "some" : "one"} ${from}, found ${count}`); + text = text.replaceAll(from, () => to); + } + text += append; + writeFileSync(path, text); + return text; +} + +patch(join(pdsDir, "dist/mailer/index.js"), [ + ["logoUrl: branding.logo ?? DEFAULT_LOGO_URL,", "logoUrl: process.env.PDS_EMAIL_LOGO_URL ?? branding.logo ?? DEFAULT_LOGO_URL,"], + ["markUrl: branding.logo ?? DEFAULT_MARK_URL,", "markUrl: process.env.PDS_EMAIL_LOGO_URL ?? branding.logo ?? DEFAULT_MARK_URL,"], +]); + +patch(join(providerDir, "oauth-provider.js"), [["ttl: 600e3", "ttl: 1e3"]], { all: true }); + +patch(join(providerDir, "lib/csp/index.js"), [[" 'default-src',\n", " 'default-src',\n 'font-src',\n"]]); +patch(join(providerDir, "router/assets/assets.js"), [ + [`'connect-src': ["'self'"],`, `'connect-src': ["'self'"],\n 'font-src': ["'self'"],`], +]); +patch(join(providerDir, "router/assets/assets.js"), [["'img-src': ['https:'],", "'img-src': ['https:', 'data:'],"]]); +patch(join(providerDir, "router/assets/assets-manifest.js"), [ + ["[null, 'document', 'style', 'script']", "[null, 'document', 'style', 'script', 'font']"], + ["'public, max-age=31536000, immutable'", "'no-cache'"], +]); + +const uiDir = dirname(uiManifestPath); +const manifest = JSON.parse(readFileSync(uiManifestPath, "utf8")); +const styles = Object.entries(manifest).filter(([, asset]) => asset.mime === "text/css"); +if (styles.length === 0) throw new Error(`${uiManifestPath}: no stylesheet in manifest`); + +const theme = readFileSync(themePath, "utf8"); +for (const [filename, asset] of styles) { + if (asset.data) throw new Error(`${filename}: inlined in manifest`); + const css = patch( + join(uiDir, filename), + [ + ["--radius:.625rem", "--radius:.5rem"], + ["calc(var(--radius) + 4px)", "var(--radius)"], + ["calc(var(--radius) - 2px)", "var(--radius)"], + ["calc(var(--radius) - 4px)", "var(--radius)"], + ], + { all: true, append: theme }, + ); + asset.sha256 = sha256(css); +} + +const scripts = [ + [/flag:`[\u{1F1E6}-\u{1F1FF}]{2}`/gu, "flag:``"], + [ + /(side:\w+=`bottom`,sideOffset:\w+=)4(,align:\w+=)`center`(,alignOffset:\w+=0,alignItemWithTrigger:\w+=)!0/g, + "$18$2`start`$3!1", + ], +]; +const hits = scripts.map(() => 0); +for (const [filename, asset] of Object.entries(manifest)) { + if (asset.type !== "chunk") continue; + const path = join(uiDir, filename); + let js = readFileSync(path, "utf8"); + scripts.forEach(([pattern, replacement], i) => { + js = js.replace(pattern, (...match) => { + hits[i]++; + return replacement.replace(/\$(\d)/g, (_, n) => match[n]); + }); + }); + writeFileSync(path, js); + asset.sha256 = sha256(js); +} +scripts.forEach(([pattern], i) => { + if (hits[i] === 0) throw new Error(`${uiDir}: ${pattern} not found`); +}); + +const font = basename(fontPath); +copyFileSync(fontPath, join(uiDir, font)); +manifest[font] = { type: "asset", mime: "font/woff2", sha256: sha256(readFileSync(fontPath)) }; + +writeFileSync(uiManifestPath, JSON.stringify(manifest)); diff --git a/localinfra/pds-theme.css b/localinfra/pds-theme.css new file mode 100644 index 0000000000000000000000000000000000000000..2568fc42735e75d714844f487157c142cb2b73d8 --- /dev/null +++ b/localinfra/pds-theme.css @@ -0,0 +1,384 @@ +@font-face { + font-family: "TimelessSans"; + src: url("/@atproto/oauth-provider/~assets/TimelessSansVF.woff2") format("woff2-variations"); + font-weight: 300 800; + font-style: normal; + font-display: swap; +} + +:root { + --font-sans: "TimelessSans", "system-ui", sans-serif, ui-sans-serif; + --default-font-feature-settings: "kern" 1, "liga" 1, "tnum" 1; + --font-weight-medium: 510; + --font-weight-semibold: 590; + --radius-2xl: var(--radius); + --text-sm: 0.8125rem; + --text-sm--line-height: 1.25rem; + --text-2xl: 1.5rem; + --text-2xl--line-height: 1.875rem; + + --background: #f5f5f5; + --foreground: #0a0a0a; + --primary: #0a0a0a; + --primary-foreground: #ffffff; + --card: #ffffff; + --card-foreground: #0a0a0a; + --popover: #ffffff; + --popover-foreground: #0a0a0a; + --secondary: #f2f2f2; + --secondary-foreground: #0a0a0a; + --muted: #f2f2f2; + --muted-foreground: #525252; + --accent: #f2f2f2; + --accent-foreground: #0a0a0a; + --border: #e5e5e5; + --input: #e5e5e5; + --ring: #d4d4d4; + --sidebar: #fafafa; + --sidebar-foreground: #0a0a0a; + --sidebar-accent: #f2f2f2; + --sidebar-accent-foreground: #0a0a0a; + --sidebar-border: #e5e5e5; + + --page: #ffffff; + --subtle: #fafafa; + --subtle-foreground: #737373; + --placeholder: #a3a3a3; + --focus: #8b85f7; + --brand: #5750f1; + --selection: #e0e7ff; + --shadow-ring: color-mix(in srgb, var(--foreground) 10%, transparent); + --shadow-ambient-strong: rgb(0 0 0 / 0.08); + --shadow-ambient-subtle: rgb(0 0 0 / 0.04); + + color-scheme: light; +} + +@media (prefers-color-scheme: dark) { + :root { + --background: #0a0a0a; + --foreground: #ededed; + --primary: #f5f5f5; + --primary-foreground: #0a0a0a; + --card: #141414; + --card-foreground: #ededed; + --popover: #141414; + --popover-foreground: #ededed; + --secondary: #1f1f1f; + --secondary-foreground: #ededed; + --muted: #1f1f1f; + --muted-foreground: #a3a3a3; + --accent: #1f1f1f; + --accent-foreground: #ededed; + --border: #262626; + --input: #262626; + --ring: #404040; + --sidebar: #1a1a1a; + --sidebar-foreground: #ededed; + --sidebar-accent: #1f1f1f; + --sidebar-accent-foreground: #ededed; + --sidebar-border: #262626; + + --page: #0a0a0a; + --subtle: #1a1a1a; + --placeholder: #737373; + --selection: #27235f; + --brand: #6d67f5; + --shadow-ambient-strong: rgb(0 0 0 / 0.35); + --shadow-ambient-subtle: rgb(0 0 0 / 0.2); + + color-scheme: dark; + } + + img[src$="/logotype.svg"] { + filter: invert(1); + } +} + +html { + scrollbar-gutter: stable; +} + +::selection { + background-color: var(--selection); + color: var(--foreground); +} + +html.auth-background { + background-color: var(--page); +} + +#root > .min-h-svh { + position: relative; + padding: 1.75rem 1.75rem 4.5rem; +} + +[data-slot="card-footer"] > div:has(> a) { + position: absolute; + right: 0; + bottom: 1.75rem; + left: 0; +} + +.max-w-auth-card { + max-width: 28rem; +} + +[data-slot="card"] { + overflow: visible; + background-color: transparent; + box-shadow: none; + padding-block: 0; +} + +[data-slot="card"] > * { + padding-inline: 0; +} + +[data-slot="card"] > div:has(> img[src$="/logotype.svg"]) { + padding-top: 0; +} + +[data-slot="card-title"] { + font-size: 1.875rem; + line-height: 1.875rem; + font-weight: 600; +} + +img[src$="/logotype.svg"] { + width: 12rem; + height: 6rem; + max-width: none; + translate: 0.5rem 0; +} + +[data-slot="card-content"] div:has(> div.items-center > label) { + gap: 0; +} + +[data-slot="card-content"] div.items-center > label { + padding-block: 0.5rem; + font-size: 0.75rem; + line-height: 1rem; + font-weight: 400; +} + +[data-slot="card-content"] p, +[data-slot="card-footer"] a { + font-size: 0.75rem; + line-height: 1rem; + color: var(--subtle-foreground); +} + +input[data-slot="input"] { + height: 2rem; + padding: 0.25rem 0.5rem; + font-size: 0.8125rem; + line-height: 1.25rem; + background-color: var(--card); + border: 0; + box-shadow: inset 0 0 0 1px var(--shadow-ring); +} + +span.absolute + input[data-slot="input"] { + padding-left: 1.875rem; +} + +span.absolute:has(+ input[data-slot="input"]) { + left: 0.5625rem; + color: var(--subtle-foreground); +} + +span.absolute:has(+ input[data-slot="input"]) svg, +input[data-slot="input"] + span.absolute svg { + width: 0.875rem; + height: 0.875rem; +} + +input[data-slot="input"]::placeholder { + color: var(--placeholder); +} + +input[data-slot="input"]:focus-visible { + box-shadow: + inset 0 0 0 1px var(--ring), + 0 0 0 1px var(--ring); +} + +button[data-slot="button"] { + font-size: 0.8125rem; + line-height: 1.25rem; + font-weight: 400; + translate: none; +} + +button[data-slot="button"].bg-primary, +button[data-slot="button"].bg-secondary { + height: 2rem; + padding: 0.125rem 0.75rem; +} + +button[data-slot="button"].bg-primary { + --btn-from: color-mix(in oklch, var(--brand), white 15%); + --btn-highlight: color-mix(in oklch, var(--brand), white 30%); + color: #ffffff; + background-color: var(--btn-highlight); + background-image: linear-gradient(to bottom in oklab, var(--btn-from), var(--brand)); + border: 0; + box-shadow: + inset 0 0 0 1px color-mix(in oklch, var(--brand), black 10%), + inset 0 2px 0 0 var(--btn-highlight); +} + +button[data-slot="button"].bg-primary:hover { + --btn-from: var(--btn-highlight); +} + +button[data-slot="button"].bg-primary:focus-visible { + outline-color: var(--brand); +} + +button[data-slot="button"].bg-primary:disabled { + opacity: 0.5; +} + +button[data-slot="button"]:focus-visible { + outline: 2px solid var(--focus); + outline-offset: 2px; +} + +button[data-slot="button"][type="submit"].bg-primary { + gap: 0.375rem; + padding-right: 0.5625rem; +} + +button[data-slot="button"][type="submit"].bg-primary::after { + content: ""; + width: 0.875rem; + height: 0.875rem; + flex-shrink: 0; + background-color: currentColor; + mask: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M5 12h14'/%3E%3Cpath d='m12 5 7 7-7 7'/%3E%3C/svg%3E") center / contain no-repeat; +} + +form:has([data-slot="avatar"]) button[data-slot="button"][type="submit"].bg-primary { + padding-right: 0.75rem; + padding-left: 0.5625rem; +} + +form:has([data-slot="avatar"]) button[data-slot="button"][type="submit"].bg-primary::after { + order: -1; + mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M20 6 9 17l-5-5'/%3E%3C/svg%3E"); +} + +button[data-slot="button"].bg-secondary { + background-color: transparent; +} + +button[data-slot="button"].bg-secondary:hover { + background-color: var(--muted); +} + +[data-slot="select-trigger"] { + height: 2rem; + gap: 0.375rem; + padding: 0.125rem 0.75rem; + font-size: 0.8125rem; + line-height: 1.25rem; + color: var(--foreground); + background-color: var(--card); + border: 0; + box-shadow: inset 0 0 0 1px var(--shadow-ring); +} + +[data-slot="select-trigger"]:hover { + background-color: var(--muted); +} + +[data-slot="select-trigger"]:focus-visible, +[data-slot="select-trigger"][aria-expanded="true"] { + box-shadow: + inset 0 0 0 1px var(--ring), + 0 0 0 1px var(--ring); +} + +[data-slot="select-trigger"] svg { + width: 0.875rem; + height: 0.875rem; + color: var(--subtle-foreground); +} + +[data-slot="select-content"] { + width: 14rem; + padding: 0.25rem; + border: 0; + border-radius: var(--radius); + background-color: var(--card); + box-shadow: + 0 0 0 1px var(--shadow-ring), + 0 10px 15px -3px rgb(0 0 0 / 0.1), + 0 4px 6px -4px rgb(0 0 0 / 0.1); +} + +[data-slot="select-item"] { + min-height: 2rem; + border-radius: 0.25rem; + padding: 0.125rem 2rem 0.125rem 0.5rem; + font-size: 0.8125rem; + line-height: 1.25rem; +} + +[data-slot="select-item"]:hover, +[data-slot="select-item"]:focus { + background-color: var(--muted); +} + +[data-slot="select-item"] svg { + width: 0.875rem; + height: 0.875rem; +} + +div:has(> button[data-slot="item"]) { + gap: 0.5rem; +} + +button[data-slot="item"] { + min-height: 2.25rem; + gap: 0.375rem; + padding: 0.375rem 0.5625rem 0.375rem 0.5rem; + border: 0; + border-radius: var(--radius); + background-color: var(--card); + box-shadow: inset 0 0 0 1px var(--shadow-ring); +} + +button[data-slot="item"]:hover { + background-color: var(--muted); +} + +button[data-slot="item"]:focus-visible { + outline: 2px solid var(--focus); + outline-offset: 2px; + box-shadow: inset 0 0 0 1px var(--shadow-ring); +} + +button[data-slot="item"] [data-slot="avatar"], +button[data-slot="item"] [data-slot="item-media"].rounded-full { + width: 1.3125rem; + height: 1.3125rem; +} + +button[data-slot="item"] [data-slot="item-description"], +button[data-slot="item"] [data-slot="item-title"], +button[data-slot="item"] [data-slot="item-title"] > span { + font-size: 0.8125rem; + line-height: 1.25rem; + font-weight: 400; + color: var(--foreground); +} + +button[data-slot="item"] [data-slot="item-actions"] svg { + width: 0.875rem; + height: 0.875rem; + color: var(--subtle-foreground); +} diff --git a/localinfra/pds.env b/localinfra/pds.env new file mode 100644 index 0000000000000000000000000000000000000000..b919b724779024225d9237efa309df03a9679466 --- /dev/null +++ b/localinfra/pds.env @@ -0,0 +1,37 @@ +LOG_ENABLED=true + +PDS_JWT_SECRET=8cae8bffcc73d9932819650791e4e89a +PDS_ADMIN_PASSWORD=d6a902588cd93bee1af83f924f60cfd3 +PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=2e92e336a50a618458e1097d94a1db86ec3fd8829d7735020cbae80625c761d7 + +PDS_DATA_DIRECTORY=/pds +PDS_BLOBSTORE_DISK_LOCATION=/pds/blocks + +PDS_DID_PLC_URL=http://localhost:8080 +PDS_HOSTNAME=pds.tngl.boltless.dev +PDS_PORT=3000 + +# firehose backfill window. default is 24h (DAY), which makes cursor=0 emit an +# OutdatedCursor #info once history is >24h old; hydrant mishandles that info +# frame and loops. widen the window so cursor=0 streams from seq 1 cleanly. +PDS_REPO_BACKFILL_LIMIT_MS=31536000000 + +PDS_MOD_SERVICE_DID=did:plc:zu6y5f5sbudyxhv5ytiirkjx +PDS_MOD_SERVICE_URL=http://ozone:3000 +# PDS_CRAWLERS=https://relay.tngl.boltless.dev + +# client ids a new signup gets signed in to without the pds pages (pds/preauthorize.mjs) +TANGLED_PREAUTHORIZE_CLIENTS=https://web.tngl.boltless.dev/oauth-client-metadata.json + +PDS_SERVICE_NAME=Tangled +PDS_LOGO_URL=https://next.tangled.org/logos/logotype.svg +PDS_EMAIL_LOGO_URL=https://assets.tangled.network/email/dolly.png +PDS_EMAIL_DISABLE_CONFIRMATION_LINK=true +PDS_HOME_URL=https://tangled.org +PDS_TERMS_OF_SERVICE_URL=https://tangled.org/terms +PDS_PRIVACY_POLICY_URL=https://tangled.org/privacy +PDS_PRIMARY_COLOR=#0a0a0a +PDS_ERROR_COLOR=#e11d48 +PDS_WARNING_COLOR=#d97706 +PDS_INFO_COLOR=#4338ca +PDS_SUCCESS_COLOR=#16a34a diff --git a/localinfra/pdsls.patch b/localinfra/pdsls.patch new file mode 100644 index 0000000000000000000000000000000000000000..3ddcac75d8904cc0709c256c7713bb0a156e011a --- /dev/null +++ b/localinfra/pdsls.patch @@ -0,0 +1,219 @@ +diff --git a/bun.lock b/bun.lock +index 12bc3ba..3d6395a 100644 +--- a/bun.lock ++++ b/bun.lock +@@ -37,14 +37,14 @@ + "@takumi-rs/wasm": "^1.8.7", + "@tanstack/solid-virtual": "^3.13.31", + "codemirror": "^6.0.2", +- "solid-js": "^1.9.13", ++ "solid-js": "^1.9.14", + "valibot": "^1.4.2", + }, + "devDependencies": { + "@iconify-json/lucide": "^1.2.115", + "@iconify/tailwind4": "^1.2.3", + "@tailwindcss/vite": "^4.3.2", +- "@types/node": "^26.0.1", ++ "@types/node": "^26.1.0", + "esbuild": "^0.28.1", + "oxfmt": "^0.57.0", + "tailwindcss": "^4.3.2", +diff --git a/src/auth/account.tsx b/src/auth/account.tsx +index e660c1a..7ae64b1 100644 +--- a/src/auth/account.tsx ++++ b/src/auth/account.tsx +@@ -43,6 +43,7 @@ const AccountDropdown = (props: { did: Did; onEditPermissions: (did: Did) => voi + } catch { + deleteStoredSession(did); + } ++ localStorage.removeItem(`pass_session:${did}`); + setSessions( + produce((accs) => { + delete accs[did]; +diff --git a/src/auth/session-manager.ts b/src/auth/session-manager.ts +index 6ac8570..bca6de4 100644 +--- a/src/auth/session-manager.ts ++++ b/src/auth/session-manager.ts +@@ -42,8 +42,79 @@ export const loadHandleForSession = async (did: Did, storedSessions: Sessions) = + } + }; + ++export class PasswordUserAgent { ++ sub: Did; ++ service: string; ++ accessJwt: string; ++ constructor(sub: Did, service: string, accessJwt: string) { ++ this.sub = sub; ++ this.service = service; ++ this.accessJwt = accessJwt; ++ } ++ async signOut() {} ++ async handle(pathname: string, init?: RequestInit): Promise { ++ const url = new URL(pathname, this.service); ++ const headers = new Headers(init?.headers); ++ headers.set("Authorization", `Bearer ${this.accessJwt}`); ++ return await fetch(url.href, { ...init, headers }); ++ } ++} ++ ++export const autoLoginLocalinfra = async (): Promise => { ++ try { ++ const pdsUrl = "https://pds.tngl.boltless.dev"; ++ ++ const healthRes = await fetch(`${pdsUrl}/xrpc/_health`, { signal: AbortSignal.timeout(2000) }); ++ if (!healthRes.ok) return; ++ ++ const resolveAndLogin = async (username: string) => { ++ const handle = `${username}.pds.tngl.boltless.dev`; ++ const resolveRes = await fetch(`${pdsUrl}/xrpc/com.atproto.identity.resolveHandle?handle=${handle}`); ++ if (!resolveRes.ok) return null; ++ const { did } = await resolveRes.json(); ++ ++ const loginRes = await fetch(`${pdsUrl}/xrpc/com.atproto.server.createSession`, { ++ method: "POST", ++ headers: { "Content-Type": "application/json" }, ++ body: JSON.stringify({ identifier: handle, password: "password" }), ++ }); ++ if (!loginRes.ok) return null; ++ const { accessJwt } = await loginRes.json(); ++ return { did, handle, accessJwt }; ++ }; ++ ++ const alice = await resolveAndLogin("alice"); ++ const bob = await resolveAndLogin("bob"); ++ ++ const sessions = loadSessionsFromStorage() || {}; ++ let lastSignedIn = localStorage.getItem("lastSignedIn"); ++ ++ if (alice) { ++ sessions[alice.did] = { signedIn: true, handle: alice.handle, grantedScopes: "atproto,create,update,delete,blob" }; ++ localStorage.setItem(`pass_session:${alice.did}`, JSON.stringify({ service: pdsUrl, accessJwt: alice.accessJwt })); ++ if (!lastSignedIn) lastSignedIn = alice.did; ++ } ++ if (bob) { ++ sessions[bob.did] = { signedIn: true, handle: bob.handle, grantedScopes: "atproto,create,update,delete,blob" }; ++ localStorage.setItem(`pass_session:${bob.did}`, JSON.stringify({ service: pdsUrl, accessJwt: bob.accessJwt })); ++ if (!lastSignedIn) lastSignedIn = lastSignedIn || bob.did; ++ } ++ ++ if (alice || bob) { ++ saveSessionToStorage(sessions); ++ if (lastSignedIn) { ++ localStorage.setItem("lastSignedIn", lastSignedIn); ++ } ++ } ++ } catch (e) { ++ console.error("Auto login failed:", e); ++ } ++}; ++ + export const retrieveSession = async (): Promise => { +- const init = async (): Promise => { ++ await autoLoginLocalinfra(); ++ ++ const init = async (): Promise => { + const params = new URLSearchParams(decodeURIComponent(location.hash.slice(1))); + + if (params.has("state") && (params.has("code") || params.has("error"))) { +@@ -61,21 +133,34 @@ export const retrieveSession = async (): Promise => { + const newSessions: Sessions = sessions || {}; + newSessions[did] = { signedIn: true, grantedScopes }; + saveSessionToStorage(newSessions); +- return auth.session; ++ return new OAuthUserAgent(auth.session); + } else { + const lastSignedIn = localStorage.getItem("lastSignedIn"); + + if (lastSignedIn) { + const sessions = loadSessionsFromStorage(); + const newSessions: Sessions = sessions || {}; ++ ++ const passSession = localStorage.getItem(`pass_session:${lastSignedIn}`); ++ if (passSession) { ++ try { ++ const { service, accessJwt } = JSON.parse(passSession); ++ const agent = new PasswordUserAgent(lastSignedIn as Did, service, accessJwt); ++ newSessions[lastSignedIn].signedIn = true; ++ saveSessionToStorage(newSessions); ++ return agent; ++ } catch {} ++ } ++ + try { + const session = await getSession(lastSignedIn as Did); +- const rpc = new Client({ handler: new OAuthUserAgent(session) }); ++ const agent = new OAuthUserAgent(session); ++ const rpc = new Client({ handler: agent }); + const res = await rpc.get("com.atproto.server.getSession"); + newSessions[lastSignedIn].signedIn = true; + saveSessionToStorage(newSessions); + if (!res.ok) throw res.data.error; +- return session; ++ return agent; + } catch (err) { + newSessions[lastSignedIn].signedIn = false; + saveSessionToStorage(newSessions); +@@ -85,9 +170,9 @@ export const retrieveSession = async (): Promise => { + } + }; + +- const session = await init(); ++ const agentInstance = await init(); + +- if (session) setAgent(new OAuthUserAgent(session)); ++ if (agentInstance) setAgent(agentInstance); + }; + + export const resumeSession = async (did: Did): Promise => { +diff --git a/src/layout.tsx b/src/layout.tsx +index 51757fa..b76e556 100644 +--- a/src/layout.tsx ++++ b/src/layout.tsx +@@ -195,7 +195,7 @@ const Layout = (props: RouteSectionProps) => { + + + +- ++ +
    + + PLC directory: {plcDirectory()} +diff --git a/src/views/settings.tsx b/src/views/settings.tsx +index 5f16783..7621cf8 100644 +--- a/src/views/settings.tsx ++++ b/src/views/settings.tsx +@@ -5,7 +5,7 @@ import { ThemeSelection } from "../components/theme.jsx"; + + export const [hideMedia, setHideMedia] = createSignal(localStorage.hideMedia === "true"); + export const [plcDirectory, setPlcDirectory] = createSignal( +- localStorage.plcDirectory || "https://plc.directory", ++ localStorage.plcDirectory || "https://plc.tngl.boltless.dev", + ); + + const Settings = () => { +@@ -18,13 +18,13 @@ const Settings = () => { + +- {plcDirectory() !== "https://plc.directory" && ( ++ {plcDirectory() !== "https://plc.tngl.boltless.dev" && ( +
    + + `); + + // Services list + body.push(`
    +

    Services

    +
      `); + for (const m of services) { +body.push(`
    • +
      + ${escapeHtml(m.name)} + ${m.enabled === 1 ? 'enabled' : 'disabled'} +
      ${escapeHtml(m.url)}
      +
      +
      + + +
      +
      • `); + } + body.push('
    '); + + // New incident + body.push(`
    +

    New incident

    +
    + + + +
    +
    +
    `); + + // Active incidents + body.push(`
    +

    Active incidents

    `); + if (incidents.length === 0) { + body.push('

    None.

    '); + } + for (const inc of incidents) { + body.push(` +
    +
    + ${stateLabel(inc.state)} + ${escapeHtml(inc.title)} + ${fmt(inc.created_at)} +
    + ${inc.description ? `

    ${escapeHtml(inc.description)}

    ` : ''} +
    + +
    + + + +
    +
    +
    `); + } + body.push('
    '); + + const script = ` +`; + + return html(page('Tangled · Status admin', body.join('\n') + script, false)); +} + +// --------------------------------------------------------------------------- +// JSON API (Cloudflare Access-scoped) +// --------------------------------------------------------------------------- + +async function api( + request: Request, + env: Env, + path: string, + method: string, +): Promise { + if (method !== 'GET' && method !== 'HEAD' && !isSameOrigin(request)) { + return errorJson('cross-origin request rejected', 403); + } + try { + await ensureSchema(env); + if (path === '/api/services' && method === 'POST') { + return createService(env, request); + } + if (path === '/api/incidents' && method === 'POST') { + return createIncident(env, request); + } + + let m = path.match(/^\/api\/services\/(\d+)\/toggle$/); + if (m && method === 'POST') { + return toggleService(env, Number(m[1])); + } + m = path.match(/^\/api\/services\/(\d+)$/); + if (m && method === 'DELETE') { + return deleteService(env, Number(m[1])); + } + m = path.match(/^\/api\/incidents\/(\d+)\/updates$/); + if (m && method === 'POST') { + return addIncidentUpdate(env, request, Number(m[1])); + } + m = path.match(/^\/api\/incidents\/(\d+)\/resolve$/); + if (m && method === 'POST') { + return resolveIncident(env, Number(m[1])); + } + if (/^\/api\/(services|incidents)(\/|$)/.test(path)) { + // Known API path with an unsupported method. + return errorJson('method not allowed', 405); + } + return errorJson('not found', 404); + } catch (e) { + return errorJson('internal error', 500); + } +} + +async function createService(env: Env, request: Request): Promise { + const body = await readJsonObject(request); + if (!body) return errorJson('invalid JSON body', 400); + const name = typeof body.name === 'string' ? body.name.trim() : ''; + const url = typeof body.url === 'string' ? body.url.trim() : ''; + if (!name) return errorJson('name is required', 400); + if (!url) return errorJson('url is required', 400); + const method = methodOr(body.method); + const expectedStatus = intOr(body.expected_status, 200); + const interval = intOr(body.interval_seconds, 60); + const timeout = intOr(body.timeout_ms, 5000); + const res = await env.DB.prepare( + `INSERT INTO services (name, url, method, expected_status, interval_seconds, timeout_ms, enabled) + VALUES (?, ?, ?, ?, ?, ?, 1)`, + ).bind(name, url, method, expectedStatus, interval, timeout).run(); + const row = await env.DB.prepare('SELECT * FROM services WHERE id = ?') + .bind(res.meta.last_row_id).first(); + return json(row); +} + +async function toggleService(env: Env, id: number): Promise { + const res = await env.DB.prepare('UPDATE services SET enabled = NOT enabled WHERE id = ?') + .bind(id).run(); + if (res.meta.changes === 0) return errorJson('service not found', 404); + const row = await env.DB.prepare('SELECT id, enabled FROM services WHERE id = ?') + .bind(id).first<{ id: number; enabled: number }>(); + return json(row); +} + +async function deleteService(env: Env, id: number): Promise { + const res = await env.DB.prepare('DELETE FROM services WHERE id = ?').bind(id).run(); + if (res.meta.changes === 0) return errorJson('service not found', 404); + return json({ ok: true }); +} + +async function createIncident(env: Env, request: Request): Promise { + const body = await readJsonObject(request); + if (!body) return errorJson('invalid JSON body', 400); + const title = typeof body.title === 'string' ? body.title.trim() : ''; + if (!title) return errorJson('title is required', 400); + const description = typeof body.description === 'string' ? body.description : ''; + let state: IncidentState; + if (body.state === undefined) state = 'investigating'; + else if (isState(body.state)) state = body.state; + else return errorJson('invalid state', 400); + const res = await env.DB.prepare( + 'INSERT INTO incidents (title, description, state) VALUES (?, ?, ?)', + ).bind(title, description, state).run(); + const row = await env.DB.prepare('SELECT * FROM incidents WHERE id = ?') + .bind(res.meta.last_row_id).first(); + return json(row, 201); +} + +async function addIncidentUpdate( + env: Env, + request: Request, + id: number, +): Promise { + const body = await readJsonObject(request); + if (!body) return errorJson('invalid JSON body', 400); + const text = typeof body.body === 'string' ? body.body.trim() : ''; + if (!text) return errorJson('body is required', 400); + let state: IncidentState | null = null; + if (typeof body.state === 'string' && body.state !== '') { + if (!isState(body.state)) return errorJson('invalid state', 400); + state = body.state; + } + + const exists = await env.DB.prepare('SELECT id FROM incidents WHERE id = ?') + .bind(id).first<{ id: number }>(); + if (!exists) return errorJson('incident not found', 404); + + const insert = env.DB.prepare( + 'INSERT INTO incident_updates (incident_id, body, state) VALUES (?, ?, ?)', + ).bind(id, text, state); + let rowId = 0; + if (state) { + const update = env.DB.prepare( + `UPDATE incidents SET + state = ?, + resolved_at = CASE WHEN ? = 'resolved' THEN strftime('%Y-%m-%dT%H:%M:%fZ', 'now') ELSE resolved_at END + WHERE id = ?`, + ).bind(state, state, id); + const batch = await env.DB.batch([insert, update]); + rowId = Number(batch[0].meta.last_row_id); + } else { + const res = await insert.run(); + rowId = Number(res.meta.last_row_id); + } + const row = await env.DB.prepare('SELECT * FROM incident_updates WHERE id = ?') + .bind(rowId).first(); + return json(row); +} + +async function resolveIncident(env: Env, id: number): Promise { + const exists = await env.DB.prepare('SELECT id FROM incidents WHERE id = ?') + .bind(id).first<{ id: number }>(); + if (!exists) return errorJson('incident not found', 404); + const insert = env.DB.prepare( + 'INSERT INTO incident_updates (incident_id, body, state) VALUES (?, ?, ?)', + ).bind(id, 'Incident resolved', 'resolved'); + const update = env.DB.prepare( + `UPDATE incidents SET state = 'resolved', resolved_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = ?`, + ).bind(id); + await env.DB.batch([insert, update]); + const row = await env.DB.prepare('SELECT * FROM incidents WHERE id = ?') + .bind(id).first(); + return json(row); +} + +// --------------------------------------------------------------------------- +// Scheduled checks +// --------------------------------------------------------------------------- + +interface DueService { + id: number; + url: string; + method: string; + expected_status: number; + interval_seconds: number; + timeout_ms: number; +} + +async function runChecks(env: Env): Promise { + await ensureSchema(env); + const rows = ( + await env.DB.prepare( + `SELECT m.id, m.url, m.method, m.expected_status, m.interval_seconds, m.timeout_ms, + (SELECT MAX(checked_at) FROM checks c WHERE c.service_id = m.id) AS last + FROM services m WHERE m.enabled = 1`, + ).all() + ).results; + + const now = Date.now(); + const due = rows.filter((r) => { + if (!r.last) return true; + return now - Date.parse(r.last) >= r.interval_seconds * 1000; + }); + + await Promise.allSettled(due.map((m) => runCheck(env, m))); + + // Retention: keep 90 days of raw checks; idempotent, cheap. Runs even when + // individual checks failed (allSettled above). + await env.DB.prepare( + "DELETE FROM checks WHERE checked_at < strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-90 days')", + ).run(); +} + +async function runCheck(env: Env, m: DueService): Promise { + const start = Date.now(); + let ok = 0; + let httpStatus: number | null = null; + let latency: number | null = null; + let error: string | null = null; + try { + const res = await fetch(m.url, { + method: m.method, + headers: { 'user-agent': 'tangled-status/1.0' }, + signal: AbortSignal.timeout(m.timeout_ms), + }); + ok = res.status === m.expected_status ? 1 : 0; + httpStatus = res.status; + latency = Date.now() - start; + } catch (e) { + error = e instanceof Error ? e.message : String(e); + } + await env.DB.prepare( + 'INSERT INTO checks (service_id, ok, http_status, latency_ms, error) VALUES (?, ?, ?, ?, ?)', + ).bind(m.id, ok, httpStatus, latency, error).run(); +} + +// --------------------------------------------------------------------------- +// Handler +// --------------------------------------------------------------------------- + +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + const path = url.pathname; + const method = request.method; + + if (path === '/') { + if (method !== 'GET') return errorJson('method not allowed', 405); + return dashboard(env); + } + if (path === '/status.json') { + if (method !== 'GET') return errorJson('method not allowed', 405); + return statusJson(env); + } + if (path === '/rss.xml') { + if (method !== 'GET') return errorJson('method not allowed', 405); + return rssFeed(env); + } + if (path === '/admin' || path.startsWith('/api/')) { + // We assert access using team keys. + // If no team/audience configured, no admin! + if (env.ENVIRONMENT !== 'development') { + if (!(env.ACCESS_TEAM_DOMAIN && env.ACCESS_AUD)) { + return errorJson('access verification is not configured', 503); + } + if ( + (await VerifiedAccessIdentity.verify(request.headers.get('cf-access-jwt-assertion'), env)) === null + ) { + return errorJson('forbidden', 403); + } + } + } + if (path === '/admin') { + if (method !== 'GET') return errorJson('method not allowed', 405); + return adminPage(env); + } + if ((path.startsWith('/static/') || path === '/favicon.svg') && method !== 'GET' && method !== 'HEAD') { + return errorJson('method not allowed', 405); + } + if (path.startsWith('/static/')) { + // Fonts are bundled into the Worker (gitignored, not uploaded as + // assets); serve them from the bundle so `/static/fonts/*` matches the + // shared stylesheet's @font-face URLs. + if (path.startsWith('/static/fonts/')) { + const font = serveFont(path); + if (font) return font; + } + // The assets directory (`./static`) is served at the URL root, but the + // shared stylesheet's @font-face URLs are `/static/fonts/...`; translate + // `/static/` to the binding's asset path `/fonts/...` etc. + const assetUrl = new URL(request.url); + assetUrl.pathname = path.slice('/static'.length); + return env.ASSETS.fetch(assetUrl); + } + if (path === '/favicon.svg') { + return env.ASSETS.fetch(request); + } + if (path.startsWith('/api/')) { + return api(request, env, path, method); + } + // /static/* and /favicon.svg are served by the assets binding before the + // Worker runs; anything else is a miss. + return html('not found', 404); + }, + + async scheduled(_controller: ScheduledController, env: Env, ctx: ExecutionContext) { + const work = runChecks(env); + ctx.waitUntil(work); + await work; + }, +} satisfies ExportedHandler; diff --git a/status/src/logotype.ts b/status/src/logotype.ts new file mode 100644 index 0000000000000000000000000000000000000000..da9943d8f2a696ff9bf6cd5990d27fa2330f5402 --- /dev/null +++ b/status/src/logotype.ts @@ -0,0 +1,9 @@ +// the shared web logo, so the wordmark has one source +import { DOLLY, WORDMARK } from "../../web/src/lib/components/ui/logotype"; + +export const LOGOTYPE_SVG = `${[ + ...WORDMARK, + DOLLY, +] + .map((d) => ``) + .join("")}`; diff --git a/status/static/.assetsignore b/status/static/.assetsignore new file mode 100644 index 0000000000000000000000000000000000000000..440232316e3ab48ff7cd6f28306e3ae34a3d3c5b --- /dev/null +++ b/status/static/.assetsignore @@ -0,0 +1,5 @@ +# Assets that must never be published from ./static: fonts are bundled into +# the Worker via the Data rule, and tw.css is inlined via the Text rule. +# Wrangler consults this file (not .gitignore) for the assets upload. +fonts/ +tw.css \ No newline at end of file diff --git a/status/static/favicon.svg b/status/static/favicon.svg new file mode 100644 index 0000000000000000000000000000000000000000..d5013ddcfdcf3195e814021e31b7e785cea40d87 --- /dev/null +++ b/status/static/favicon.svg @@ -0,0 +1,21 @@ + + + + + + + diff --git a/status/static/logos/dolly.svg b/status/static/logos/dolly.svg new file mode 100644 index 0000000000000000000000000000000000000000..d5013ddcfdcf3195e814021e31b7e785cea40d87 --- /dev/null +++ b/status/static/logos/dolly.svg @@ -0,0 +1,21 @@ + + + + + + + diff --git a/status/tailwind.config.js b/status/tailwind.config.js new file mode 100644 index 0000000000000000000000000000000000000000..9fa34ba96d47d4b81693638e447e41582f3b4444 --- /dev/null +++ b/status/tailwind.config.js @@ -0,0 +1,3 @@ +const path = require('path'); + +module.exports = require(path.resolve(__dirname, '..', 'tailwind.config.js')); \ No newline at end of file diff --git a/status/tsconfig.json b/status/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..71b60b6f15b0682432df8374500812cfcf07aea0 --- /dev/null +++ b/status/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "ES2022", + "moduleResolution": "bundler", + "types": ["@cloudflare/workers-types"], + "strict": true, + "noEmit": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true + }, + "include": ["src/**/*.ts"], + "exclude": ["src/access.test.ts"] +} \ No newline at end of file diff --git a/status/tsconfig.test.json b/status/tsconfig.test.json new file mode 100644 index 0000000000000000000000000000000000000000..1267ba5129834c011cd78c8a81dfce79f6a866bf --- /dev/null +++ b/status/tsconfig.test.json @@ -0,0 +1,6 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { "types": ["node"], "allowImportingTsExtensions": true }, + "include": ["src/access.ts", "src/access.test.ts"], + "exclude": [] +} diff --git a/status/wrangler.jsonc b/status/wrangler.jsonc new file mode 100644 index 0000000000000000000000000000000000000000..3228449a723e4df5261c4a7eb9943300ec4a37b5 --- /dev/null +++ b/status/wrangler.jsonc @@ -0,0 +1,46 @@ +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "status", + "main": "src/index.ts", + "compatibility_date": "2026-08-01", + "observability": { + "enabled": true + }, + "workers_dev": false, + "vars": { + "ENVIRONMENT": "production" + }, + "d1_databases": [ + { + "binding": "DB", + "database_name": "tangled-status", + "database_id": "3568d9f0-7a40-4c7e-b8f2-a4c925c39509" + } + ], + "assets": { + "directory": "./static", + "binding": "ASSETS" + }, + "triggers": { + "crons": ["*/1 * * * *"] + }, + "routes": [ + { + "pattern": "status.tangled.org", + "zone_name": "tangled.org", + "custom_domain": true + } + ], + "rules": [ + { + "type": "Text", + "globs": ["**/*.css"], + "fallthrough": true + }, + { + "type": "Data", + "globs": ["**/*.woff2"], + "fallthrough": true + } + ] +} \ No newline at end of file diff --git a/systemd/knot.service b/systemd/knot.service new file mode 100644 index 0000000000000000000000000000000000000000..298bddc0c874ec24581781b986aef1247e90d744 --- /dev/null +++ b/systemd/knot.service @@ -0,0 +1,52 @@ +[Unit] +Description=tangled knot 2 server +After=network-online.target +Wants=network-online.target +StartLimitIntervalSec=60 +StartLimitBurst=5 + +[Service] +ExecStart=/usr/local/bin/knot-server --config /var/lib/knot/config.toml +User=git +Group=git +StateDirectory=knot +WorkingDirectory=/var/lib/knot +Environment=HOME=/var/lib/knot +EnvironmentFile=/etc/knot/knot.env +UMask=0077 +Restart=on-failure +RestartSec=5 +TimeoutStopSec=120 +LimitNOFILE=65536 +StandardOutput=journal +StandardError=journal + +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +NoNewPrivileges=true +ProtectProc=invisible +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/knot +PrivateTmp=true +PrivateDevices=true +ProtectHostname=true +ProtectClock=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_INET AF_INET6 AF_NETLINK AF_UNIX +RestrictNamespaces=true +LockPersonality=true +MemoryDenyWriteExecute=true +RestrictRealtime=true +RestrictSUIDSGID=true +RemoveIPC=true +PrivateMounts=true +SystemCallFilter=@system-service +SystemCallFilter=~@privileged @resources +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target diff --git a/tailwind.config.js b/tailwind.config.js new file mode 100644 index 0000000000000000000000000000000000000000..3072210e08b5071d4387e13afea2efa28f5cf0d7 --- /dev/null +++ b/tailwind.config.js @@ -0,0 +1,92 @@ +/** @type {import('tailwindcss').Config} */ +const colors = require("tailwindcss/colors"); + +module.exports = { + safelist: [ + { pattern: /^term-/ }, + ], + content: [ + "./docs/*.html", + "./blog/templates/**/*.html", + "./blog/posts/**/*.md", + "./status/src/**/*.ts", + ], + darkMode: ['selector', '[data-theme="dark"]'], + theme: { + container: { + padding: "2rem", + center: true, + screens: { + sm: "500px", + md: "600px", + lg: "800px", + xl: "1000px", + "2xl": "1200px", + }, + }, + extend: { + fontFamily: { + sans: ["InterVariable", "system-ui", "sans-serif", "ui-sans-serif"], + mono: [ + "IBMPlexMono", + "ui-monospace", + "SFMono-Regular", + "Menlo", + "Monaco", + "Consolas", + "Liberation Mono", + "Courier New", + "monospace", + ], + }, + typography: { + DEFAULT: { + css: { + maxWidth: "none", + pre: { + "@apply font-normal text-black bg-gray-50 dark:bg-gray-900 dark:text-gray-300 dark:border-gray-700 border": + {}, + }, + code: { + "@apply font-normal font-mono p-1 rounded text-black bg-gray-100 dark:bg-gray-900 dark:text-gray-300 dark:border-gray-700": + {}, + }, + "code::before": { + content: '""', + }, + "code::after": { + content: '""', + }, + blockquote: { + quotes: "none", + fontWeight: "400", + "@apply text-gray-500 dark:text-gray-400": {}, + }, + "h1, h2, h3, h4": { + "@apply mt-4 mb-2": {}, + }, + h1: { + "@apply mt-3 pb-3 border-b border-gray-300 dark:border-gray-600": + {}, + }, + h2: { + "@apply mt-3 pb-3 border-b border-gray-200 dark:border-gray-700": + {}, + }, + h3: { + "@apply mt-2": {}, + }, + "img, video": { + "@apply rounded border border-gray-200 dark:border-gray-700": {}, + }, + }, + }, + }, + gridTemplateColumns: { + 14: "repeat(14, minmax(0, 1fr))", + 28: "repeat(28, minmax(0, 1fr))", + }, + }, + }, + plugins: [require("@tailwindcss/typography"), require("tailwindcss-animated")], +}; diff --git a/tapc/readme.md b/tapc/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..423d4227d4adaa1a9d7b4a6e68641c5d2104a9a9 --- /dev/null +++ b/tapc/readme.md @@ -0,0 +1,3 @@ +basic tap client package + +Replace this to official indigo package when gets merged. diff --git a/tapc/simple_indexer.go b/tapc/simple_indexer.go new file mode 100644 index 0000000000000000000000000000000000000000..977fb23a853bca02a1147db75f52553b1e8579a5 --- /dev/null +++ b/tapc/simple_indexer.go @@ -0,0 +1,35 @@ +package tapc + +import "context" + +type SimpleIndexer struct { + EventHandler func(ctx context.Context, evt Event) error + ErrorHandler func(ctx context.Context, err error) + ConnectHandler func(ctx context.Context) +} + +var ( + _ Handler = (*SimpleIndexer)(nil) + _ ConnectHandler = (*SimpleIndexer)(nil) +) + +func (i *SimpleIndexer) OnEvent(ctx context.Context, evt Event) error { + if i.EventHandler == nil { + return nil + } + return i.EventHandler(ctx, evt) +} + +func (i *SimpleIndexer) OnError(ctx context.Context, err error) { + if i.ErrorHandler == nil { + return + } + i.ErrorHandler(ctx, err) +} + +func (i *SimpleIndexer) OnConnect(ctx context.Context) { + if i.ConnectHandler == nil { + return + } + i.ConnectHandler(ctx) +} diff --git a/tapc/tap.go b/tapc/tap.go new file mode 100644 index 0000000000000000000000000000000000000000..c0e155cbf938749101aa77258163dd0d3a35f11b --- /dev/null +++ b/tapc/tap.go @@ -0,0 +1,176 @@ +/// heavily inspired by + +package tapc + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/gorilla/websocket" + "tangled.org/core/log" +) + +type Handler interface { + OnEvent(ctx context.Context, evt Event) error + OnError(ctx context.Context, err error) +} + +type ConnectHandler interface { + OnConnect(ctx context.Context) +} + +type Client struct { + Url string + AdminPassword string + HTTPClient *http.Client +} + +func NewClient(url, adminPassword string) Client { + return Client{ + Url: url, + AdminPassword: adminPassword, + HTTPClient: &http.Client{}, + } +} + +// tap inserts one /repos/add body as a single sqlite statement at about 9 +// variables per did, so a spindle's whole member list blows past sqlite's +// 32766 cap and tap 500s +const addReposBatchSize = 1000 + +func (c *Client) AddRepos(ctx context.Context, dids []syntax.DID) error { + for len(dids) > 0 { + n := min(len(dids), addReposBatchSize) + if err := c.postDids(ctx, "/repos/add", dids[:n]); err != nil { + return err + } + dids = dids[n:] + } + return nil +} + +func (c *Client) RemoveRepos(ctx context.Context, dids []syntax.DID) error { + return c.postDids(ctx, "/repos/remove", dids) +} + +func (c *Client) postDids(ctx context.Context, path string, dids []syntax.DID) error { + body, err := json.Marshal(map[string][]syntax.DID{"dids": dids}) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, "POST", c.Url+path, bytes.NewReader(body)) + if err != nil { + return err + } + req.SetBasicAuth("admin", c.AdminPassword) + req.Header.Set("Content-Type", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("tap: %s failed with status %d", path, resp.StatusCode) + } + return nil +} + +func (c *Client) Connect(ctx context.Context, handler Handler) error { + l := log.FromContext(ctx) + + u, err := url.Parse(c.Url) + if err != nil { + return err + } + if u.Scheme == "https" { + u.Scheme = "wss" + } else { + u.Scheme = "ws" + } + u.Path = "/channel" + + url := u.String() + basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte("admin:"+c.AdminPassword)) + + var backoff int + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + header := http.Header{ + "Authorization": []string{basicAuth}, + } + conn, res, err := websocket.DefaultDialer.DialContext(ctx, url, header) + if err != nil { + if backoff < 12 { + backoff++ + } + l.Warn("dialing failed", "url", url, "err", err, "backoff", backoff) + time.Sleep(time.Duration(5*backoff) * time.Second) + + continue + } + backoff = 0 + l.Info("connected to tap service", "subscription_code", res.StatusCode) + + if ch, ok := handler.(ConnectHandler); ok { + ch.OnConnect(ctx) + } + + if err = c.handleConnection(ctx, conn, handler); err != nil { + l.Warn("tap connection failed", "err", err) + } + } +} + +func (c *Client) handleConnection(ctx context.Context, conn *websocket.Conn, handler Handler) error { + l := log.FromContext(ctx) + + defer func() { + conn.Close() + l.Warn("closed tap connection") + }() + l.Info("established tap connection") + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + _, message, err := conn.ReadMessage() + if err != nil { + return err + } + + var ev Event + if err := json.Unmarshal(message, &ev); err != nil { + handler.OnError(ctx, fmt.Errorf("failed to parse message: %w", err)) + continue + } + if err := handler.OnEvent(ctx, ev); err != nil { + handler.OnError(ctx, fmt.Errorf("failed to process event %d: %w", ev.ID, err)) + continue + } + + ack := map[string]any{ + "type": "ack", + "id": ev.ID, + } + if err := conn.WriteJSON(ack); err != nil { + l.Warn("failed to send ack", "err", err) + continue + } + } +} diff --git a/tapc/tap_test.go b/tapc/tap_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7f236d233c8ca70599c280b973f7b532173fb756 --- /dev/null +++ b/tapc/tap_test.go @@ -0,0 +1,65 @@ +package tapc + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +func TestAddReposBatchesLargeLists(t *testing.T) { + var batches [][]syntax.DID + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + DIDs []syntax.DID `json:"dids"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode body: %v", err) + } + // same failure real tap gives once sqlite runs out of variables + if len(body.DIDs) > addReposBatchSize { + w.WriteHeader(http.StatusInternalServerError) + return + } + batches = append(batches, body.DIDs) + })) + defer srv.Close() + + dids := make([]syntax.DID, 2*addReposBatchSize+500) + for i := range dids { + dids[i] = syntax.DID(fmt.Sprintf("did:plc:%024d", i)) + } + + c := NewClient(srv.URL, "pw") + if err := c.AddRepos(context.Background(), dids); err != nil { + t.Fatalf("AddRepos: %v", err) + } + + if len(batches) != 3 { + t.Fatalf("got %d batches, want 3", len(batches)) + } + var sent []syntax.DID + for _, b := range batches { + sent = append(sent, b...) + } + if !slices.Equal(sent, dids) { + t.Fatalf("sent %d dids, want all %d in order", len(sent), len(dids)) + } +} + +func TestAddReposEmptySendsNothing(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected request to %s", r.URL.Path) + })) + defer srv.Close() + + c := NewClient(srv.URL, "pw") + if err := c.AddRepos(context.Background(), nil); err != nil { + t.Fatalf("AddRepos: %v", err) + } +} diff --git a/tapc/types.go b/tapc/types.go new file mode 100644 index 0000000000000000000000000000000000000000..565649c0e3d013a42fb0b2e5bad1d501dfc06a6b --- /dev/null +++ b/tapc/types.go @@ -0,0 +1,62 @@ +package tapc + +import ( + "encoding/json" + "fmt" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +type EventType string + +const ( + EvtRecord EventType = "record" + EvtIdentity EventType = "identity" +) + +type Event struct { + ID int64 `json:"id"` + Type EventType `json:"type"` + Record *RecordEventData `json:"record,omitempty"` + Identity *IdentityEventData `json:"identity,omitempty"` +} + +type RecordEventData struct { + Live bool `json:"live"` + Did syntax.DID `json:"did"` + Rev string `json:"rev"` + Collection syntax.NSID `json:"collection"` + Rkey syntax.RecordKey `json:"rkey"` + Action RecordAction `json:"action"` + Record json.RawMessage `json:"record,omitempty"` + CID *syntax.CID `json:"cid,omitempty"` +} + +func (r *RecordEventData) AtUri() syntax.ATURI { + return syntax.ATURI(fmt.Sprintf("at://%s/%s/%s", r.Did, r.Collection, r.Rkey)) +} + +type RecordAction string + +const ( + RecordCreateAction RecordAction = "create" + RecordUpdateAction RecordAction = "update" + RecordDeleteAction RecordAction = "delete" +) + +type IdentityEventData struct { + DID syntax.DID `json:"did"` + Handle string `json:"handle"` + IsActive bool `json:"is_active"` + Status RepoStatus `json:"status"` +} + +type RepoStatus string + +const ( + RepoStatusActive RepoStatus = "active" + RepoStatusTakendown RepoStatus = "takendown" + RepoStatusSuspended RepoStatus = "suspended" + RepoStatusDeactivated RepoStatus = "deactivated" + RepoStatusDeleted RepoStatus = "deleted" +) diff --git a/tid/tid.go b/tid/tid.go new file mode 100644 index 0000000000000000000000000000000000000000..f2485eb947a7042d3aa828d80b3935befd076a8a --- /dev/null +++ b/tid/tid.go @@ -0,0 +1,9 @@ +package tid + +import "github.com/bluesky-social/indigo/atproto/syntax" + +var TIDClock = syntax.NewTIDClock(0) + +func TID() string { + return TIDClock.Next().String() +} diff --git a/web/.dockerignore b/web/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..117a21252ac0e888203cd0f226ef541c713ce604 --- /dev/null +++ b/web/.dockerignore @@ -0,0 +1,8 @@ +node_modules +build +.svelte-kit +.wrangler +test-results +storybook-static +.env +.env.* diff --git a/web/.gitignore b/web/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..511bc0a9f3aad2de9d5ce6ec3b47a640851b37c0 --- /dev/null +++ b/web/.gitignore @@ -0,0 +1,37 @@ +node_modules +.pnpm-store + +# Output +.output +.vercel +.netlify +.wrangler +/.svelte-kit +/build + +# OS +Thumbs.db + +# Env +.env +.env.* +!.env.example +!.env.test + +# Vite +vite.config.js.timestamp-* +vite.config.ts.timestamp-* +# Playwright +test-results +.vitest +.vscode/ + +*storybook.log +storybook-static + +.eslintcache + +wrangler.preview.jsonc + +# if anything imports this registry every route would ship all the bones +src/lib/bones/registry.ts diff --git a/web/.npmrc b/web/.npmrc new file mode 100644 index 0000000000000000000000000000000000000000..b6f27f135954640c8cc5bfd7b8c9922ca6eb2aad --- /dev/null +++ b/web/.npmrc @@ -0,0 +1 @@ +engine-strict=true diff --git a/web/.prettierignore b/web/.prettierignore new file mode 100644 index 0000000000000000000000000000000000000000..a10418ad57014e126af82d368a51ffcb4ca350f9 --- /dev/null +++ b/web/.prettierignore @@ -0,0 +1,9 @@ +package-lock.json +pnpm-lock.yaml +yarn.lock +bun.lock +bun.lockb + +/static/ + +/src/lib/api/lexicons/ diff --git a/web/.storybook/BoneCaptureDecorator.svelte b/web/.storybook/BoneCaptureDecorator.svelte new file mode 100644 index 0000000000000000000000000000000000000000..392537018429008c06ef157693dcc85db41294f1 --- /dev/null +++ b/web/.storybook/BoneCaptureDecorator.svelte @@ -0,0 +1,16 @@ + + + + {@render children()} + diff --git a/web/.storybook/DatasetToggle.svelte b/web/.storybook/DatasetToggle.svelte new file mode 100644 index 0000000000000000000000000000000000000000..b1f5241f57edc268d7009998c65df633b99f22c6 --- /dev/null +++ b/web/.storybook/DatasetToggle.svelte @@ -0,0 +1,68 @@ + + +
    + {#each options as option (option.value)} + + {/each} +
    + + diff --git a/web/.storybook/MockAuth.svelte b/web/.storybook/MockAuth.svelte new file mode 100644 index 0000000000000000000000000000000000000000..088f6f704a02abd2e64ffa91c6c3234706e13753 --- /dev/null +++ b/web/.storybook/MockAuth.svelte @@ -0,0 +1,111 @@ + + + + +{@render children()} diff --git a/web/.storybook/MockNotifications.svelte b/web/.storybook/MockNotifications.svelte new file mode 100644 index 0000000000000000000000000000000000000000..637a04625027a3d75db18725584243b5a095fb1b --- /dev/null +++ b/web/.storybook/MockNotifications.svelte @@ -0,0 +1,42 @@ + + + + +{@render children()} diff --git a/web/.storybook/SimulatedRepoSettings.svelte b/web/.storybook/SimulatedRepoSettings.svelte new file mode 100644 index 0000000000000000000000000000000000000000..cf920eb533fbf1926d1e5f533e4ee8ba97ec02ef --- /dev/null +++ b/web/.storybook/SimulatedRepoSettings.svelte @@ -0,0 +1,208 @@ + + +
    +
    + Request Delay: + + +
    + + +
    + {#each statusLog as entry, i (i)} +
    {entry}
    + {/each} +
    + + +
    + {#each items as item (item.id)} + + {/each} +
    + + + + {#key activeTab} + {#if activeTab === "general"} + + {#snippet skeleton()} +
    + [GENERAL SKELETON] + +
    + {/snippet} + {@const labels = await pageData.labels} +
    + + + + {#each labels as label (label)} + + {/each} + + +
    +     + {:else if activeTab === "access"} + + {#snippet skeleton()} +
    + [ACCESS SKELETON] + +
    + {/snippet} + {@const collaborators = await pageData.collaborators} +
    + + + + {#each collaborators as c (c.did)} + + {#snippet label()} + {c.handle} ({c.role}) + {/snippet} + + {/each} + + +
    +     + {:else if activeTab === "pipelines"} + + {#snippet skeleton()} +
    + [PIPELINES SKELETON] + +
    + {/snippet} + {@const spindles = await pageData.spindles} +
    + + + + {#each spindles as s (s)} + + {/each} + + +
    +     + {/if} + {/key} +     +
    diff --git a/web/.storybook/VouchArrivalReplay.svelte b/web/.storybook/VouchArrivalReplay.svelte new file mode 100644 index 0000000000000000000000000000000000000000..014f47ecb2c188fa3ba4fe678c895b874d462239 --- /dev/null +++ b/web/.storybook/VouchArrivalReplay.svelte @@ -0,0 +1,23 @@ + + +{@render children({ preview })} diff --git a/web/.storybook/WorkflowRunReplay.svelte b/web/.storybook/WorkflowRunReplay.svelte new file mode 100644 index 0000000000000000000000000000000000000000..5e9d793f93eb86dab9f04dddb3366da998541a53 --- /dev/null +++ b/web/.storybook/WorkflowRunReplay.svelte @@ -0,0 +1,60 @@ + + +
    + {@render children({ steps, status })} +
    + {status} + +
    +
    diff --git a/web/.storybook/arrival.ts b/web/.storybook/arrival.ts new file mode 100644 index 0000000000000000000000000000000000000000..dcfd6927db9e1e2e29f0105e251782d05d81c026 --- /dev/null +++ b/web/.storybook/arrival.ts @@ -0,0 +1,45 @@ +import { expect, within } from "storybook/test"; + +interface Arrival { + button: string; + item: string; + edge: string; +} + +export const frame = () => new Promise((resolve) => requestAnimationFrame(resolve)); + +export const expectSteady = (values: number[]) => { + const direction = Math.sign(values.at(-1)! - values[0]!); + const steps = values.slice(1).map((value, i) => value - values[i]!); + for (const step of steps) expect(step * direction).toBeGreaterThanOrEqual(-0.5); + const total = Math.abs(values.at(-1)! - values[0]!); + expect(Math.max(0, ...steps.map(Math.abs))).toBeLessThanOrEqual(Math.max(total * 0.35, 1)); +}; + +export const expectSmoothArrival = async ( + canvasElement: HTMLElement, + { button, item, edge }: Arrival +) => { + const canvas = within(canvasElement); + const neighbour = canvas.getAllByTestId(edge)[0]!; + await Promise.all([...document.fonts].map((font) => font.load())); + const frames: { height: number; top: number; edge: number }[] = []; + canvas.getByRole("button", { name: button }).click(); + const start = performance.now(); + while (performance.now() - start < 600) { + await frame(); + const node = canvas.queryByText(/fresh comment number 1,/i)?.closest(item); + if (!node) continue; + const rect = node.getBoundingClientRect(); + frames.push({ + height: rect.height, + top: rect.top, + edge: neighbour.getBoundingClientRect().top + }); + } + expect(frames.length).toBeGreaterThan(10); + expect(frames[0]!.height).toBeLessThan(frames.at(-1)!.height / 2); + for (const key of ["height", "top", "edge"] as const) { + expectSteady(frames.map((f) => f[key])); + } +}; diff --git a/web/.storybook/clipboard.ts b/web/.storybook/clipboard.ts new file mode 100644 index 0000000000000000000000000000000000000000..16b6158b0f3ae36fd793023e273fb09df87e2966 --- /dev/null +++ b/web/.storybook/clipboard.ts @@ -0,0 +1,6 @@ +export const stubClipboard = (writeText: (text: string) => Promise) => { + Object.defineProperty(navigator, "clipboard", { + value: { writeText }, + configurable: true + }); +}; diff --git a/web/.storybook/holdErrorReset.ts b/web/.storybook/holdErrorReset.ts new file mode 100644 index 0000000000000000000000000000000000000000..df76b6a961b7840934ead2e31a317f5c7d8e7a09 --- /dev/null +++ b/web/.storybook/holdErrorReset.ts @@ -0,0 +1,8 @@ +export const holdErrorReset = () => { + const original = window.setTimeout; + window.setTimeout = ((handler: TimerHandler, delay?: number, ...rest: unknown[]) => + delay === 2000 ? 0 : original(handler, delay, ...rest)) as typeof window.setTimeout; + return () => { + window.setTimeout = original; + }; +}; diff --git a/web/.storybook/installFetch.ts b/web/.storybook/installFetch.ts new file mode 100644 index 0000000000000000000000000000000000000000..9759bbce8dfd749dce713d62521d19763c905e23 --- /dev/null +++ b/web/.storybook/installFetch.ts @@ -0,0 +1,15 @@ +export type FetchResponder = ( + url: URL, + init?: RequestInit +) => Response | Promise | undefined; + +export const installFetch = (respond: FetchResponder) => { + const original = window.fetch; + window.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input), location.href); + return (await respond(url, init)) ?? original(input, init); + }) as typeof fetch; + return () => { + window.fetch = original; + }; +}; diff --git a/web/.storybook/main.ts b/web/.storybook/main.ts new file mode 100644 index 0000000000000000000000000000000000000000..2f3f99a0eb95ce9a719f3dd19d8a3d91346cab41 --- /dev/null +++ b/web/.storybook/main.ts @@ -0,0 +1,51 @@ +import path from "node:path"; +import type { StorybookConfig } from "@storybook/sveltekit"; +import type { PluginOption } from "vite"; + +const NAVIGATION_MOCK = "@storybook/sveltekit/internal/mocks/app/navigation"; +const navigationShim = path.join(import.meta.dirname, "mocks/navigation.ts"); + +async function withoutKitDevServer(option: PluginOption): Promise { + const plugin = await option; + if (Array.isArray(plugin)) return Promise.all(plugin.map(withoutKitDevServer)); + if (plugin && plugin.name === "vite-plugin-sveltekit-setup") + return { ...plugin, configureServer: undefined }; + return plugin; +} + +const config: StorybookConfig = { + stories: ["../src/**/*.stories.@(js|ts|svelte)"], + staticDirs: ["../static"], + addons: [ + "@storybook/addon-svelte-csf", + "@storybook/addon-vitest", + "@storybook/addon-a11y", + "@storybook/addon-themes", + "@storybook/addon-docs" + ], + framework: "@storybook/sveltekit", + // the framework aliases $app/navigation to its own mock; route that through + // mocks/navigation.ts, which fills in the kit 3 exports the mock lacks + viteFinal: async ({ base: _base, ...config }) => ({ + ...config, + server: { + ...config.server, + hmr: { + ...(typeof config.server?.hmr === "object" ? config.server.hmr : {}), + overlay: false + } + }, + plugins: [ + { + name: "tangled:kit3-navigation-mock", + enforce: "pre", + resolveId(id, importer) { + if (id === NAVIGATION_MOCK && importer !== navigationShim) + return navigationShim; + } + }, + await withoutKitDevServer(config.plugins ?? []) + ] + }) +}; +export default config; diff --git a/web/.storybook/mocks/navigation.ts b/web/.storybook/mocks/navigation.ts new file mode 100644 index 0000000000000000000000000000000000000000..2c82c002326d9dce05c51f9ec0f667d23cc45bf8 --- /dev/null +++ b/web/.storybook/mocks/navigation.ts @@ -0,0 +1,23 @@ +// storybook's sveltekit framework mocks $app/navigation as kit 2 shaped it. until it ships +// kit 3 mocks (storybook 11), this adds what kit 3 added over it: refreshAll, which +// replaces invalidateAll +export * from "@storybook/sveltekit/internal/mocks/app/navigation"; +import { + afterNavigate as mockAfterNavigate, + invalidateAll +} from "@storybook/sveltekit/internal/mocks/app/navigation"; +import type { AfterNavigate } from "@sveltejs/kit"; + +export const refreshAll = invalidateAll; + +const entered = (): AfterNavigate => + ({ + type: "enter", + from: null, + to: { url: new URL(location.href), params: {}, route: { id: null } }, + willUnload: false, + complete: Promise.resolve() + }) as unknown as AfterNavigate; + +export const afterNavigate = (callback: (navigation: AfterNavigate) => void): void => + mockAfterNavigate((navigation?: AfterNavigate) => callback(navigation ?? entered())); diff --git a/web/.storybook/moderation.ts b/web/.storybook/moderation.ts new file mode 100644 index 0000000000000000000000000000000000000000..09e80c6e0693dab36e169b6e9e8499cd72aba2b0 --- /dev/null +++ b/web/.storybook/moderation.ts @@ -0,0 +1,5 @@ +export const moderationPage = { + sveltekit_experimental: { + state: { page: { data: { publicConfig: { moderationServiceDid: "did:plc:labeler" } } } } + } +}; diff --git a/web/.storybook/pendingOnboarding.ts b/web/.storybook/pendingOnboarding.ts new file mode 100644 index 0000000000000000000000000000000000000000..c0a3673874ad70f2ea665594de7fbb7aa4496c3f --- /dev/null +++ b/web/.storybook/pendingOnboarding.ts @@ -0,0 +1,16 @@ +import { installFetch } from "./installFetch.ts"; + +export const installPendingOnboarding = () => + installFetch((url) => + url.pathname === "/xrpc/com.bad-example.pocket.getPreferences" + ? Response.json({ + preferences: [ + { + $type: "org.tangled.temp.actor.defs#onboardingPref", + startedAt: "2026-10-01T00:00:00.000Z" + } + ], + rev: "1" + }) + : undefined + ); diff --git a/web/.storybook/play.ts b/web/.storybook/play.ts new file mode 100644 index 0000000000000000000000000000000000000000..991866a8b2bc580b2d8e925f58472e37ac1f5df5 --- /dev/null +++ b/web/.storybook/play.ts @@ -0,0 +1,5 @@ +import type { StoryContext } from "@storybook/addon-svelte-csf"; + +type Context = StoryContext>; + +export type PlayContext = Pick; diff --git a/web/.storybook/preview.ts b/web/.storybook/preview.ts new file mode 100644 index 0000000000000000000000000000000000000000..751cb9dd6f935b6e2e9a50ffd068d3469cdf7ba6 --- /dev/null +++ b/web/.storybook/preview.ts @@ -0,0 +1,65 @@ +import "#lib/twMergeConfig.ts"; +import type { Preview } from "@storybook/sveltekit"; +import { withThemeByDataAttribute } from "@storybook/addon-themes"; +import MockAuth, { type MockAuthParam } from "./MockAuth.svelte"; +import MockNotifications, { type MockNotificationsParam } from "./MockNotifications.svelte"; +import BoneCaptureDecorator from "./BoneCaptureDecorator.svelte"; + +// Load the app's global styles (Tailwind + theme tokens) so components +// render with real styling inside Storybook. +import "../src/app.css"; + +const preview: Preview = { + decorators: [ + withThemeByDataAttribute({ + themes: { light: "light", dark: "dark" }, + defaultTheme: "light", + attributeName: "data-theme" + }), + (_story, context) => ({ + Component: MockAuth, + props: { auth: context.parameters.auth as MockAuthParam } + }), + (_story, context) => ({ + Component: MockNotifications, + props: { notifications: context.parameters.notifications as MockNotificationsParam } + }), + (Story, { id, parameters }) => { + void Story; + return { + Component: BoneCaptureDecorator, + props: { name: id, class: parameters.bones?.class } + }; + } + ], + parameters: { + // the two swatches are --color-background-canvas in each mode, so the + // backdrop matches the app instead of Storybook's grey + backgrounds: { + options: { + light: { name: "Light", value: "#f5f5f5" }, + dark: { name: "Dark", value: "#0a0a0a" } + } + }, + + controls: { + matchers: { + color: /(background|color)$/i, + date: /Date$/i + } + }, + + a11y: { + // 'todo' - show a11y violations in the test UI only + // 'error' - fail CI on a11y violations + // 'off' - skip a11y checks entirely + test: "todo" + } + }, + + initialGlobals: { + backgrounds: { value: "light" } + } +}; + +export default preview; diff --git a/web/.storybook/respond.ts b/web/.storybook/respond.ts new file mode 100644 index 0000000000000000000000000000000000000000..4b839aaf417f2dfe1818c611309ba7c54ed4c0ee --- /dev/null +++ b/web/.storybook/respond.ts @@ -0,0 +1,21 @@ +export const hang = () => new Promise(() => {}); + +export const serviceAuth = (pathname: string): Response => + pathname.includes("getServiceAuth") + ? Response.json({ token: "service-auth-jwt" }) + : new Response(null, { status: 404 }); + +export const recordWrite = + (collection: string) => + (pathname: string): Response => { + if (pathname.includes("createRecord")) { + return Response.json({ + uri: `at://did:plc:alice/${collection}/3lknew`, + cid: "bafyreibfko5xzzdptcimcaql4tmsaffgr6jhreyjz3tv6nlh34si6k4u7e" + }); + } + if (pathname.includes("deleteRecord")) { + return Response.json({}); + } + return new Response(null, { status: 404 }); + }; diff --git a/web/Containerfile b/web/Containerfile new file mode 100644 index 0000000000000000000000000000000000000000..512b761d5d7d7628e6d05086436b53606c1bfc24 --- /dev/null +++ b/web/Containerfile @@ -0,0 +1,41 @@ +FROM node:24-slim AS manifests +WORKDIR /app +RUN corepack enable +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY patches ./patches + +FROM manifests AS deps +RUN corepack pnpm install --frozen-lockfile + +# adapter-bun bundles devDependencies into build/, so the runtime only needs these. +# pnpm's own state files carry the install time, and nothing at runtime reads them +FROM manifests AS prod-deps +RUN corepack pnpm install --frozen-lockfile --prod \ + && rm node_modules/.modules.yaml node_modules/.pnpm-workspace-state-v1.json + +FROM node:24-slim AS build +WORKDIR /app +RUN corepack enable +COPY --from=oven/bun:1.4.4-slim /usr/local/bin/bun /usr/local/bin/bun +COPY --from=deps /app/node_modules ./node_modules +COPY . . +# vite.config.ts falls back to the prod oauth client when these are unset +ARG VITE_OAUTH_CLIENT_ID +ARG VITE_OAUTH_REDIRECT_URI +ARG APP_VERSION +RUN corepack pnpm run build + +FROM oven/bun:1.4.4-slim +WORKDIR /app +ENV NODE_ENV=production +ENV HOST=0.0.0.0 +ENV PORT=3000 +ENV BUN_FEATURE_FLAG_EXPERIMENTAL_HTTP2_CLIENT=1 +# the 2.0 default peaks above node's memory, 1.6 doesn't and still beats node on cpu +# vs node on prod pages: 1.41x req/s, 9.9 vs 14.0 ms cpu per page, 359 vs 379 MB peak +ENV BUN_JSC_smallHeapGrowthFactor=1.6 +COPY --from=build /app/build ./build +COPY --from=build /app/package.json ./package.json +COPY --from=prod-deps /app/node_modules ./node_modules +EXPOSE 3000 +CMD ["bun", "build/index.js"] diff --git a/web/README.md b/web/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1956820798ffc78502e909e0eb37bfe616f0be52 --- /dev/null +++ b/web/README.md @@ -0,0 +1,43 @@ +# web + +The Tangled web app at [tangled.org](https://tangled.org). It is built with SvelteKit (Svelte 5) +and Tailwind 4, and talks to the Tangled services and to users' PDSes over AT Protocol. + +## Requirements + +- Node 24 +- pnpm 11 (`corepack enable` picks up the pinned version) +- Bun 1.4+ (`pnpm build` runs vite under it, and the production server is `bun build/index.js`) + +## Getting started + +```sh +pnpm install +pnpm dev # http://127.0.0.1:5173 +``` + +By default the server expects bobbin on `127.0.0.1:8090` and the API on `127.0.0.1:8080`, +which is the [`localinfra`](../localinfra) stack. To work against deployed services instead, +put the URLs in `.env`, e.g. as in `wrangler.dev.jsonc`: + +```sh +BOBBIN_URL=https://next.tangled.org +TANGLED_API_URL=https://tangled.org +``` + +Every server variable is optional and listed in `src/env.ts`; the defaults are in +`src/lib/server/config.ts` (and `github/env.ts`, `indexnow.ts` beside it). + +## Commands + +| Command | What it does | +| ---------------- | ------------------------------- | +| `pnpm dev` | Dev server | +| `pnpm check` | Type-check | +| `pnpm lint` | Prettier, ESLint, test-id check | +| `pnpm test` | Unit tests (Vitest) | +| `pnpm storybook` | Component stories on :6006 | +| `pnpm build` | Production build into `build/` | +| `pnpm codegen` | Regenerate lexicon types | + +End-to-end tests live in [`../e2e`](../e2e/README.md). diff --git a/web/_headers b/web/_headers new file mode 100644 index 0000000000000000000000000000000000000000..657ccbf6b9bde38928f75f153bbceb6a9da386e9 --- /dev/null +++ b/web/_headers @@ -0,0 +1,5 @@ +# merged into the build's _headers by adapter-cloudflare (production's adapter-bun +# sets the same header via patches/@sveltejs__adapter-bun@1.0.0.patch). +# font filenames aren't hashed, so cache for a month rather than forever +/fonts/* + Cache-Control: public, max-age=2592000, stale-while-revalidate=604800 diff --git a/web/boneyard.config.json b/web/boneyard.config.json new file mode 100644 index 0000000000000000000000000000000000000000..05a5b53455dd18fe8cd1447b4122a1a5cf9b5d28 --- /dev/null +++ b/web/boneyard.config.json @@ -0,0 +1,152 @@ +{ + "breakpoints": [375, 768, 1024, 1440], + "out": "./src/lib/bones", + "wait": 100, + "animate": "pulse", + "skeletons": { + "bones-forklist--skeleton-fixture": { + "route": "/iframe.html?id=bones-forklist--skeleton-fixture&viewMode=story" + }, + "bones-loadingprimitives--count": { + "route": "/iframe.html?id=bones-loadingprimitives--count&viewMode=story" + }, + "bones-loadingprimitives--short-text": { + "route": "/iframe.html?id=bones-loadingprimitives--short-text&viewMode=story" + }, + "bones-radio--loading-row": { + "route": "/iframe.html?id=bones-radio--loading-row&viewMode=story" + }, + "bones-starredby--skeleton-fixture": { + "route": "/iframe.html?id=bones-starredby--skeleton-fixture&viewMode=story" + }, + "bones-treeview--skeleton-fixture": { + "route": "/iframe.html?id=bones-treeview--skeleton-fixture&viewMode=story" + }, + "profile-followcard--bordered": { + "route": "/iframe.html?id=profile-followcard--bordered&viewMode=story" + }, + "profile-networkvouches--many-vouches": { + "route": "/iframe.html?id=profile-networkvouches--many-vouches&viewMode=story" + }, + "profile-profilechrome--skeleton-fixture": { + "route": "/iframe.html?id=profile-profilechrome--skeleton-fixture&viewMode=story" + }, + "profile-overviewtab--skeleton-fixture": { + "route": "/iframe.html?id=profile-overviewtab--skeleton-fixture&viewMode=story" + }, + "profile-stringcard--default": { + "route": "/iframe.html?id=profile-stringcard--default&viewMode=story" + }, + "repo-blobview--code-file": { + "route": "/iframe.html?id=repo-blobview--code-file&viewMode=story" + }, + "repo-blobview--markdown-skeleton-fixture": { + "route": "/iframe.html?id=repo-blobview--markdown-skeleton-fixture&viewMode=story" + }, + "repo-branchtable--skeleton-fixture": { + "route": "/iframe.html?id=repo-branchtable--skeleton-fixture&viewMode=story" + }, + "repo-commitlogview--skeleton-fixture": { + "route": "/iframe.html?id=repo-commitlogview--skeleton-fixture&viewMode=story" + }, + "repo-commitview--full-commit": { + "route": "/iframe.html?id=repo-commitview--full-commit&viewMode=story" + }, + "repo-filehistoryview--skeleton-fixture": { + "route": "/iframe.html?id=repo-filehistoryview--skeleton-fixture&viewMode=story" + }, + "repo-repoforkform--skeleton-fixture": { + "route": "/iframe.html?id=repo-repoforkform--skeleton-fixture&viewMode=story" + }, + "repo-issues-issueform--create": { + "route": "/iframe.html?id=repo-issues-issueform--create&viewMode=story" + }, + "repo-issues-issuethreadview--skeleton-fixture": { + "route": "/iframe.html?id=repo-issues-issuethreadview--skeleton-fixture&viewMode=story" + }, + "repo-pipelines-pipelinelist--default": { + "route": "/iframe.html?id=repo-pipelines-pipelinelist--default&viewMode=story" + }, + "repo-pipelines-pipelinetoolbar--all": { + "route": "/iframe.html?id=repo-pipelines-pipelinetoolbar--all&viewMode=story" + }, + "repo-pipelines-workflowrunview--skeleton-fixture": { + "route": "/iframe.html?id=repo-pipelines-workflowrunview--skeleton-fixture&viewMode=story" + }, + "repo-pulls-pullcompose--default": { + "route": "/iframe.html?id=repo-pulls-pullcompose--default&viewMode=story" + }, + "repo-pulls-pullviewpage--skeleton-fixture": { + "route": "/iframe.html?id=repo-pulls-pullviewpage--skeleton-fixture&viewMode=story" + }, + "repo-readme--rendered-sanitized-html": { + "route": "/iframe.html?id=repo-readme--rendered-sanitized-html&viewMode=story" + }, + "repo-repocard--with-star-button": { + "route": "/iframe.html?id=repo-repocard--with-star-button&viewMode=story" + }, + "repo-repochrome--skeleton-fixture": { + "route": "/iframe.html?id=repo-repochrome--skeleton-fixture&viewMode=story" + }, + "repo-repoheader--full-metadata": { + "route": "/iframe.html?id=repo-repoheader--full-metadata&viewMode=story" + }, + "repo-repoindexview--panel-skeleton-fixture": { + "route": "/iframe.html?id=repo-repoindexview--panel-skeleton-fixture&viewMode=story" + }, + "repo-repoindexview--skeleton-fixture": { + "route": "/iframe.html?id=repo-repoindexview--skeleton-fixture&viewMode=story" + }, + "repo-tagcard--annotated": { + "route": "/iframe.html?id=repo-tagcard--annotated&viewMode=story" + }, + "repo-taglist--skeleton-fixture": { + "route": "/iframe.html?id=repo-taglist--skeleton-fixture&viewMode=story" + }, + "repo-tickets-ticketlist--issues": { + "route": "/iframe.html?id=repo-tickets-ticketlist--issues&viewMode=story" + }, + "repo-tickets-ticketlist--pulls": { + "route": "/iframe.html?id=repo-tickets-ticketlist--pulls&viewMode=story" + }, + "repo-tickets-tickettoolbar--issues-open": { + "route": "/iframe.html?id=repo-tickets-tickettoolbar--issues-open&viewMode=story" + }, + "repo-tickets-tickettoolbar--pulls-open": { + "route": "/iframe.html?id=repo-tickets-tickettoolbar--pulls-open&viewMode=story" + }, + "timeline-timeline--skeleton-fixture": { + "route": "/iframe.html?id=timeline-timeline--skeleton-fixture&viewMode=story" + }, + "settings-settingspanel--content-skeleton-fixture": { + "route": "/iframe.html?id=settings-settingspanel--content-skeleton-fixture&viewMode=story" + }, + "ui-toggle--checked": { + "route": "/iframe.html?id=ui-toggle--checked&viewMode=story" + }, + "notifications-notificationlist--skeleton-fixture": { + "route": "/iframe.html?id=notifications-notificationlist--skeleton-fixture&viewMode=story" + }, + "notifications-notificationlist--social-skeleton-fixture": { + "route": "/iframe.html?id=notifications-notificationlist--social-skeleton-fixture&viewMode=story" + }, + "timeline-recentspreview--skeleton-fixture": { + "route": "/iframe.html?id=timeline-recentspreview--skeleton-fixture&viewMode=story" + }, + "timeline-notificationspreview--skeleton-fixture": { + "route": "/iframe.html?id=timeline-notificationspreview--skeleton-fixture&viewMode=story" + }, + "repo-diffview--skeleton-fixture": { + "route": "/iframe.html?id=repo-diffview--skeleton-fixture&viewMode=story" + }, + "repo-diffview--skeleton-fixture-collapsed": { + "route": "/iframe.html?id=repo-diffview--skeleton-fixture-collapsed&viewMode=story" + }, + "strings-stringview--code-file": { + "route": "/iframe.html?id=strings-stringview--code-file&viewMode=story" + }, + "strings-stringview--markdown-skeleton-fixture": { + "route": "/iframe.html?id=strings-stringview--markdown-skeleton-fixture&viewMode=story" + } + } +} diff --git a/web/eslint.config.js b/web/eslint.config.js new file mode 100644 index 0000000000000000000000000000000000000000..68d8a0fb58636b20282036f5e265c70ed6007228 --- /dev/null +++ b/web/eslint.config.js @@ -0,0 +1,127 @@ +import storybook from "eslint-plugin-storybook"; +import betterTailwindcss from "eslint-plugin-better-tailwindcss"; +import prettier from "eslint-config-prettier"; +import path from "node:path"; +import js from "@eslint/js"; +import svelte from "eslint-plugin-svelte"; +import { defineConfig, includeIgnoreFile } from "eslint/config"; +import globals from "globals"; +import ts from "typescript-eslint"; +import { staleAwait } from "./scripts/staleAwait.ts"; + +const gitignorePath = path.resolve(import.meta.dirname, ".gitignore"); + +const fixtures = { + // the leading # is escaped: these are gitignore-style patterns, where # starts a comment + group: ["\\#fixtures/*", "**/src/fixtures", "**/src/fixtures/*"], + message: "fixtures are storybook and test input: import them from a story or a *.test.ts." +}; + +const relative = { + group: ["./*", "../*"], + message: "import through #lib, not a relative path." +}; + +const relativeRoute = { + group: ["./*", "../*", "!./$types", "!./+*", "!../app.css"], + message: "import through #lib or #routes, not a relative path." +}; + +export default defineConfig( + { ignores: ["src/lib/api/lexicons/"] }, + includeIgnoreFile(gitignorePath), + js.configs.recommended, + ts.configs.recommended, + svelte.configs.recommended, + storybook.configs["flat/recommended"], + prettier, + svelte.configs.prettier, + { + languageOptions: { globals: { ...globals.browser, ...globals.node } }, + rules: { + // typescript owns undefined-name checks. + "no-undef": "off", + "no-empty": ["error", { allowEmptyCatch: true }], + "@typescript-eslint/no-unused-vars": [ + "error", + { argsIgnorePattern: "^_", varsIgnorePattern: "^_", caughtErrorsIgnorePattern: "^_" } + ] + } + }, + { + files: ["**/*.svelte", "**/*.svelte.ts", "**/*.svelte.js"], + languageOptions: { + parserOptions: { + projectService: { + allowDefaultProject: [ + ".storybook/MockAuth.svelte", + ".storybook/MockNotifications.svelte", + ".storybook/BoneCaptureDecorator.svelte", + ".storybook/WorkflowRunReplay.svelte", + ".storybook/SimulatedRepoSettings.svelte", + ".storybook/DatasetToggle.svelte", + ".storybook/VouchArrivalReplay.svelte" + ] + }, + extraFileExtensions: [".svelte"], + parser: ts.parser + } + } + }, + { + files: ["**/*.svelte", "**/*.svelte.ts", "**/*.svelte.js", "**/*.ts", "**/*.js"], + plugins: { "better-tailwindcss": betterTailwindcss }, + settings: { + "better-tailwindcss": { + entryPoint: "src/app.css", + // without this the rule silently skips px -> spacing-scale rewrites. + rootFontSize: 16 + } + }, + rules: { + "better-tailwindcss/enforce-canonical-classes": "warn", + "better-tailwindcss/no-deprecated-classes": "warn" + } + }, + { + // fixtures feed stories and nothing else. without this the app can import + // them and ship sample data, which is how the notifications page got its + // mock feed + files: ["**/*.svelte", "**/*.svelte.ts", "**/*.svelte.js", "**/*.ts", "**/*.js"], + ignores: ["**/*.stories.svelte", "**/*.test.ts", "src/lib/**", "src/routes/**"], + rules: { + "no-restricted-imports": ["error", { patterns: [fixtures] }] + } + }, + { + files: ["src/routes/**/*.svelte"], + ignores: ["**/*.stories.svelte"], + rules: { "no-restricted-syntax": ["error", ...staleAwait] } + }, + { + files: ["src/routes/**"], + ignores: ["**/*.stories.svelte", "**/*.test.ts"], + rules: { + "no-restricted-imports": ["error", { patterns: [fixtures, relativeRoute] }] + } + }, + { + files: ["src/routes/**/*.stories.svelte", "src/routes/**/*.test.ts"], + rules: { + "no-restricted-imports": ["error", { patterns: [relativeRoute] }] + } + }, + { + files: ["src/lib/**"], + ignores: ["**/*.stories.svelte", "**/*.test.ts"], + rules: { + "no-restricted-imports": ["error", { patterns: [fixtures, relative] }] + } + }, + { + files: ["src/lib/**/*.stories.svelte", "src/lib/**/*.test.ts"], + rules: { + "no-restricted-imports": ["error", { patterns: [relative] }] + } + } +); diff --git a/web/knip.json b/web/knip.json new file mode 100644 index 0000000000000000000000000000000000000000..5a91dd5fa4378e4f85ae520eca6cb7850d33bdc8 --- /dev/null +++ b/web/knip.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://unpkg.com/knip@5/schema.json", + "entry": ["lex.config.ts", "scripts/**/*.{ts,js,sh}"], + "ignore": ["src/lib/api/lexicons/**"], + "ignoreUnresolved": ["^\\$icon/", "^\\./(server|router|logger)$"] +} diff --git a/web/lex.config.ts b/web/lex.config.ts new file mode 100644 index 0000000000000000000000000000000000000000..0ec345b4b76efd139fb8151c5c4f6f3e1d83852d --- /dev/null +++ b/web/lex.config.ts @@ -0,0 +1,66 @@ +import { defineLexiconConfig } from "@atcute/lex-cli"; + +// generate tangled types; resolve com.atproto refs from @atcute/atproto. +export default defineLexiconConfig({ + formatter: { type: "prettier" }, + generate: { + outdir: "src/lib/api/lexicons/", + clean: true, + imports: ["@atcute/atproto"], + files: [ + "../lexicons/*.json", + "../lexicons/com/bad-example/pocket/*.json", + "../lexicons/bobbin/**/*.json", + "../lexicons/actor/**/*.json", + "../lexicons/ci/**/*.json", + "../lexicons/embed/**/*.json", + "../lexicons/feed/**/*.json", + "../lexicons/git/**/*.json", + "../lexicons/graph/**/*.json", + "../lexicons/identity/**/*.json", + "../lexicons/issue/**/*.json", + "../lexicons/knot/**/*.json", + "../lexicons/label/**/*.json", + "../lexicons/markup/**/*.json", + "../lexicons/org/tangled/*.json", + "../lexicons/org/tangled/actor/*.json", + "../lexicons/org/tangled/bobbin/*.json", + "../lexicons/org/tangled/ci/*.json", + "../lexicons/org/tangled/ci/trigger/*.json", + "../lexicons/org/tangled/embed/*.json", + "../lexicons/org/tangled/event/*.json", + "../lexicons/org/tangled/feed/*.json", + "../lexicons/org/tangled/git/*.json", + "../lexicons/org/tangled/graph/*.json", + "../lexicons/org/tangled/key/*.json", + "../lexicons/org/tangled/knot/*.json", + "../lexicons/org/tangled/label/*.json", + "../lexicons/org/tangled/markup/*.json", + "../lexicons/org/tangled/markup/embed/*.json", + "../lexicons/org/tangled/publicKey/*.json", + "../lexicons/org/tangled/query/*.json", + "../lexicons/org/tangled/repo/*.json", + "../lexicons/org/tangled/review/*.json", + "../lexicons/org/tangled/search/*.json", + "../lexicons/org/tangled/secret/*.json", + "../lexicons/org/tangled/snippet/*.json", + "../lexicons/org/tangled/string/*.json", + "../lexicons/org/tangled/track/*.json", + "../lexicons/org/tangled/webhook/*.json", + "../lexicons/org/tangled/temp/actor/*.json", + "../lexicons/org/tangled/temp/git/*.json", + "../lexicons/org/tangled/temp/server/*.json", + "../lexicons/org/tangled/temp/webhook/*.json", + "../lexicons/pipeline/**/*.json", + "../lexicons/publicKey/**/*.json", + "../lexicons/pulls/**/*.json", + "../lexicons/query/**/*.json", + "../lexicons/repo/**/*.json", + "../lexicons/search/**/*.json", + "../lexicons/spindle/**/*.json", + "../lexicons/string/**/*.json", + "../lexicons/temp/**/*.json", + "../lexicons/sync/**/*.json" + ] + } +}); diff --git a/web/package.json b/web/package.json new file mode 100644 index 0000000000000000000000000000000000000000..841651d547c197b3b47f61c96389ffc5b5f2b4ef --- /dev/null +++ b/web/package.json @@ -0,0 +1,112 @@ +{ + "name": "web", + "private": true, + "version": "0.0.1", + "type": "module", + "packageManager": "pnpm@11.17.0", + "scripts": { + "dev": "vite dev", + "build": "bun --bun vite build", + "preview": "vite preview --host 0.0.0.0", + "deploy:cloudflare-preview": "./scripts/deploy-cloudflare-preview.sh", + "check": "svelte-kit sync && tsc --noEmit -p scripts/tsconfig.json && svelte-check --tsconfig ./tsconfig.json", + "lint": "prettier --check --cache --cache-strategy content . && eslint . --cache --cache-strategy content && node scripts/testids-check.ts", + "lint:cached": "prettier --check --cache . && eslint . --cache && node scripts/testids-check.ts", + "format": "prettier --write .", + "test": "vitest --run --project server", + "test:client": "vitest --run --project client", + "test:storybook": "vitest --run --project storybook", + "storybook": "storybook dev -p 6006", + "codegen": "lex-cli generate", + "bones": "node scripts/bones.js", + "prepare": "svelte-kit sync || echo ''" + }, + "devDependencies": { + "@atcute/car": "^6.0.2", + "@atcute/cid": "^2.4.2", + "@atcute/lex-cli": "3.2.1", + "@atcute/mst": "^1.0.2", + "@eslint/js": "^10.0.1", + "@iconify-json/lucide": "^1.2.134", + "@storybook/addon-a11y": "^10.6.5", + "@storybook/addon-docs": "^10.6.5", + "@storybook/addon-svelte-csf": "^5.1.6", + "@storybook/addon-themes": "^10.6.5", + "@storybook/addon-vitest": "^10.6.5", + "@storybook/sveltekit": "^10.6.5", + "@sveltejs/adapter-bun": "^1.0.0", + "@sveltejs/adapter-cloudflare": "^8.0.0", + "@sveltejs/kit": "^3.1.0", + "@sveltejs/vite-plugin-svelte": "^7.3.1", + "@tailwindcss/vite": "^4.3.3", + "@types/markdown-it": "^14.1.2", + "@types/markdown-it-emoji": "^3.0.1", + "@types/node": "^24.13.3", + "@types/sanitize-html": "^2.16.1", + "@types/textarea-caret": "^3.0.4", + "@vitest/browser-playwright": "5.0.4", + "@vitest/coverage-v8": "5.0.4", + "eslint": "^10.7.0", + "eslint-config-prettier": "^10.1.8", + "eslint-plugin-better-tailwindcss": "^4.7.0", + "eslint-plugin-storybook": "^10.6.5", + "eslint-plugin-svelte": "^3.23.1", + "globals": "^17.7.0", + "playwright": "^1.61.1", + "prettier": "^3.9.5", + "prettier-plugin-svelte": "^4.1.1", + "prettier-plugin-tailwindcss": "^0.8.1", + "react": "^19.2.8", + "react-dom": "^19.2.8", + "storybook": "^10.6.5", + "svelte": "^5.57.2", + "svelte-check": "^4.7.6", + "tailwindcss": "^4.3.3", + "tw-easing-gradients": "^1.1.2", + "typescript": "^6.0.3", + "typescript-eslint": "^8.64.0", + "unplugin-icons": "^24.0.0", + "vite": "^8.3.4", + "vitest": "^5.0.4", + "wrangler": "^4.125.0" + }, + "dependencies": { + "@atcute/atproto": "^4.0.3", + "@atcute/cbor": "^2.3.6", + "@atcute/client": "^5.1.1", + "@atcute/identity-resolver": "^2.0.1", + "@atcute/lexicons": "^2.0.2", + "@atcute/microcosm": "^2.0.2", + "@atcute/oauth-browser-client": "^4.0.1", + "@atcute/repo": "^1.0.2", + "@atcute/tid": "^1.1.4", + "@mdit/plugin-alert": "^1.0.1", + "@mdit/plugin-footnote": "^1.0.1", + "@mdit/plugin-tasklist": "^1.0.1", + "@pierre/diffs": "^1.6.1", + "@tanstack/svelte-virtual": "^3.13.35", + "ansi_up": "^6.0.6", + "boneyard-js": "1.9.0", + "cobe": "2.0.1", + "cuelume": "0.2.4", + "feed": "^6.0.0", + "fuse.js": "^7.5.0", + "markdown-it": "^14.3.1", + "markdown-it-anchor": "^9.2.1", + "markdown-it-emoji": "^3.1.0", + "pele": "^1.5.2", + "sanitize-html": "^2.17.7", + "svg-pathdata": "9.0.0", + "tailwind-merge": "^3.6.0", + "tailwind-variants": "^3.2.2", + "textarea-caret": "^3.1.0", + "undici": "^7.29.0" + }, + "imports": { + "#lib/*": "./src/lib/*", + "#fixtures/*": "./src/fixtures/*", + "#routes/*": "./src/routes/*", + "#static/*": "./static/*", + "#storybook/*": "./.storybook/*" + } +} diff --git a/web/patches/@pierre__diffs@1.6.1.patch b/web/patches/@pierre__diffs@1.6.1.patch new file mode 100644 index 0000000000000000000000000000000000000000..8bc61352b305f4b03a0f63eff16003e720e6b0b0 --- /dev/null +++ b/web/patches/@pierre__diffs@1.6.1.patch @@ -0,0 +1,81 @@ +diff --git a/dist/components/VirtualizedFile.js b/dist/components/VirtualizedFile.js +index 95d4ca8fdf9e176e23d354094a11022eefdb2321..94f5cbc14d1a0907a3468b21c9162035d88fda18 100644 +--- a/dist/components/VirtualizedFile.js ++++ b/dist/components/VirtualizedFile.js +@@ -568,12 +568,17 @@ var VirtualizedFile = class extends File { + const startingLine = Math.max(0, Math.min(idealStartHunk, totalHunksInFile)) * hunkLineCount; + const clampedTotalLines = idealStartHunk < 0 ? totalLines + idealStartHunk * hunkLineCount : totalLines; + const bufferBefore = startingLine === 0 ? 0 : fileAnnotationHeight + startingLine * lineHeight; +- const renderedLines = Math.min(clampedTotalLines, lineCount - startingLine); ++ // tangled patch: the hunk-centered window under-allocates at the very end of ++ // content — render every remaining line when the item ends at the window bottom ++ const atContentEnd = sourceRowsBottom <= bottom + lineHeight; ++ const renderedLines = atContentEnd ++ ? lineCount - startingLine ++ : Math.min(clampedTotalLines, lineCount - startingLine); + return { + startingLine, +- totalLines: clampedTotalLines, ++ totalLines: atContentEnd ? lineCount - startingLine : clampedTotalLines, + bufferBefore, +- bufferAfter: Math.max(0, (lineCount - startingLine - renderedLines) * lineHeight) ++ bufferAfter: atContentEnd ? 0 : Math.max(0, (lineCount - startingLine - renderedLines) * lineHeight) + }; + } + const overflowHunks = totalHunks; +diff --git a/dist/components/VirtualizedFileDiff.js b/dist/components/VirtualizedFileDiff.js +index eb34dd4a774cb06add3280fe6abd5f0dc292505e..2ac1cb03bf264bd97832c797a95e1d15b8e45454 100644 +--- a/dist/components/VirtualizedFileDiff.js ++++ b/dist/components/VirtualizedFileDiff.js +@@ -1049,13 +1049,18 @@ var VirtualizedFileDiff = class extends FileDiff { + const clampedTotalLines = idealStartHunk < 0 ? totalLines + idealStartHunk * hunkLineCount : totalLines; + const codeBufferBefore = hunkOffsets[startHunk] ?? 0; + const bufferBefore = startingLine === 0 ? 0 : fileAnnotationHeight + codeBufferBefore; ++ // tangled patch: at the very end of content the hunk-centered window ++ // under-allocates and leaves the final lines permanently unrendered at max ++ // scroll (12M+ px mega-diffs). when the item's rows end within a line-height ++ // of the window bottom, render every remaining line. ++ const atContentEnd = fileTop + codeRegionTop + codeHeight <= bottom + lineHeight; + const finalHunkIndex = startHunk + clampedTotalLines / hunkLineCount; + const bufferAfter = finalHunkIndex < hunkOffsets.length ? codeHeight - hunkOffsets[finalHunkIndex] : codeHeight - (absoluteLineTop - fileTop - codeRegionTop); + return { + startingLine, +- totalLines: clampedTotalLines, ++ totalLines: atContentEnd ? lineCount - startingLine : clampedTotalLines, + bufferBefore, +- bufferAfter: Math.max(0, bufferAfter) ++ bufferAfter: atContentEnd ? 0 : Math.max(0, bufferAfter) + }; + } + }; +diff --git a/dist/highlighter/shiki-editor.js b/dist/highlighter/shiki-editor.js +index c68b3c518ca88a589802b2d354ecb990577a1648..8a41fe9d11fb9955c79833b23b3ebe63f6a63f8e 100644 +--- a/dist/highlighter/shiki-editor.js ++++ b/dist/highlighter/shiki-editor.js +@@ -90,8 +90,8 @@ var ShikiEditorTokenizer = class { + #ensureActiveTheme() { + if (this.#themeName === "") return; + this.backend?.getTheme(this.#themeName); +- const { colorMap } = this.#highlighter.setTheme(this.#themeName); +- this.#colorMap = colorMap; ++ const { theme, colorMap } = this.#highlighter.setTheme(this.#themeName); ++ this.#colorMap = resolveColorMap(colorMap, theme); + } + dispose() { + this.#isCleanedUp = true; +@@ -441,6 +441,15 @@ var ShikiEditorTokenizer = class { + this.#postTokenizeMessage(jobId); + } + }; ++// tangled patch: shiki encodes a css-variable theme's colors as sentinel hexes ++// and keeps the reverse map on theme.colorReplacements; codeToHast applies it ++// through applyColorReplacements, while this tokenizer wrote the sentinels ++// straight into inline styles and painted rgba(0, 0, 0, 0.004) ++function resolveColorMap(colorMap, theme) { ++ const replacements = theme?.colorReplacements; ++ if (replacements == null) return colorMap; ++ return colorMap.map((color) => replacements[color?.toLowerCase()] ?? color); ++} + function tokenizeLine(grammar, colorMap, lineText, stateStack, timeLimit, collectBracketIgnoredRanges = true, resolveTokens = true) { + const result = grammar.tokenizeLine2(lineText, stateStack, timeLimit); + if (result.stoppedEarly) console.warn(`[diffs] Time limit reached when tokenizing line: ${lineText.substring(0, 100)}`); diff --git a/web/patches/@sveltejs__adapter-bun@1.0.0.patch b/web/patches/@sveltejs__adapter-bun@1.0.0.patch new file mode 100644 index 0000000000000000000000000000000000000000..0f87c45b9ebe06d9f3b233ad8f53e9acad23faca --- /dev/null +++ b/web/patches/@sveltejs__adapter-bun@1.0.0.patch @@ -0,0 +1,51 @@ +diff --git a/src/env.js b/src/env.js +index 2dc88c6529f0cbaf47a68e3fcb1ceda704665574..d27d7c227a0c7873aacc8e088935e7a713a6e8ee 100644 +--- a/src/env.js ++++ b/src/env.js +@@ -5,6 +5,8 @@ const expected = new Set([ + 'SOCKET_PATH', + 'HOST', + 'PORT', ++ // tangled: read at runtime by handler.js ++ 'ORIGIN', + 'REUSE_PORT', + 'IPV6_ONLY', + 'CONNECTION_IDLE_TIMEOUT', +diff --git a/src/handler.js b/src/handler.js +index a5cce744b7873975a161beb1deb02766e5635deb..ea0b9ee1f3f07cf3d7695ee5387426bfa2856b49 100644 +--- a/src/handler.js ++++ b/src/handler.js +@@ -1,5 +1,5 @@ + /** @import { Server as BunServer } from 'bun' */ +-import { env_prefix, origin, server } from '#@sveltejs/adapter-bun'; ++import { env_prefix, origin as build_origin, server } from '#@sveltejs/adapter-bun'; + import { server_assets } from './routes.js'; + import { env, number_env } from './env.js'; + +@@ -8,6 +8,8 @@ const protocol_header = env('PROTOCOL_HEADER', '').toLowerCase(); + const host_header = env('HOST_HEADER', '').toLowerCase(); + const port_header = env('PORT_HEADER', '').toLowerCase(); + const xff_depth = number_env('XFF_DEPTH', 1, { min: 1 }); ++// tangled: ORIGIN at runtime like adapter-node, then the build-time kit.paths.origin ++const origin = env('ORIGIN', build_origin); + + await server.init({ + env: Bun.env, +diff --git a/src/routes-util.js b/src/routes-util.js +index 6504ad01fbf2a26504c8e98d89c52dd7ebf9e0c0..82881af6427d8a1a04091dfd59ea94dcf434b402 100644 +--- a/src/routes-util.js ++++ b/src/routes-util.js +@@ -173,7 +173,12 @@ export function client_asset(url, filename = url, meta) { + const route = file_route( + resolve_file('client', filename), + meta, +- immutable ? { 'cache-control': 'public,max-age=31536000,immutable' } : {} ++ immutable ++ ? { 'cache-control': 'public,max-age=31536000,immutable' } ++ : url.startsWith('fonts/') ++ ? // tangled: self-hosted fonts aren't hashed, so a month rather than forever ++ { 'cache-control': 'public,max-age=2592000,stale-while-revalidate=604800' } ++ : {} + ); + + const paths = to_paths(url); diff --git a/web/patches/boneyard-js@1.9.0.patch b/web/patches/boneyard-js@1.9.0.patch new file mode 100644 index 0000000000000000000000000000000000000000..15286e82f84a87d035830ffec5a3cbc80958aa24 --- /dev/null +++ b/web/patches/boneyard-js@1.9.0.patch @@ -0,0 +1,457 @@ +diff --git a/bin/cli.js b/bin/cli.js +index c1ff2ea660e419d328e7147b0d95a07e156a36a4..ea8c4cce06a955dbd20b480583bb8e4e7942866d 100755 +--- a/bin/cli.js ++++ b/bin/cli.js +@@ -650,7 +650,7 @@ async function capturePage(pageUrl) { + if (result.bones) { + result.bones = result.bones.map(b => { + const arr = [b.x, b.y, b.w, b.h, b.r] +- if (b.c) arr.push(true) ++ if (b.c) arr.push(true, b.e ?? 0, b.f ?? false) + return arr + }) + } +diff --git a/dist/extract.js b/dist/extract.js +index 030ae33b9698b5a5f1b8bb16449154c4c3446dfd..1aaf1ab168bb62be87a1538d2e61ebef351410b0 100644 +--- a/dist/extract.js ++++ b/dist/extract.js +@@ -1,9 +1,9 @@ + const DEFAULT_LEAF_TAGS = new Set(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'td', 'th']); + /** +- * Snapshot the exact visual layout of a rendered DOM element as skeleton bones. +- * Walks the DOM, finds every visible leaf element, and captures its exact +- * bounding rect relative to the root. No layout engine, no heuristics — +- * just pixel-perfect positions read directly from the browser. ++ * Snapshot a rendered DOM element as skeleton geometry. ++ * Structural surfaces keep their boxes; textual leaves become one compact bone ++ * per visual line so the result reads as a loading state rather than a blurred ++ * photocopy of the content. + * + * const { bones, width, height } = snapshotBones(el, 'my-component', config) + */ +@@ -11,78 +11,257 @@ export function snapshotBones(el, name = 'component', config) { + const rootRect = el.getBoundingClientRect(); + const bones = []; + const leafTags = config?.leafTags ? new Set([...DEFAULT_LEAF_TAGS, ...config.leafTags]) : DEFAULT_LEAF_TAGS; +- const captureRoundedBorders = config?.captureRoundedBorders ?? true; ++ const captureBorders = config?.captureBorders ?? config?.captureRoundedBorders ?? true; + const excludeTags = config?.excludeTags ? new Set(config.excludeTags) : null; + const excludeSelectors = config?.excludeSelectors ?? null; ++ const renderedChildren = node => [...(node.shadowRoot?.children ?? node.children)]; ++ const composedParent = node => node.parentElement ?? node.getRootNode().host ?? null; ++ const isTransparent = color => color === 'rgba(0, 0, 0, 0)' || color === 'transparent'; ++ const cssPixels = (value, style) => { ++ const number = parseFloat(value); ++ if (!Number.isFinite(number)) ++ return 0; ++ const unit = value.trim().toLowerCase(); ++ if (unit.endsWith('rem')) ++ return number * (parseFloat(getComputedStyle(document.documentElement).fontSize) || 16); ++ if (unit.endsWith('em')) ++ return number * (parseFloat(style.fontSize) || 16); ++ return number; ++ }; ++ const backdropOf = node => { ++ for (let parent = composedParent(node); parent; parent = composedParent(parent)) { ++ const color = getComputedStyle(parent).backgroundColor; ++ if (!isTransparent(color)) ++ return color; ++ } ++ return 'transparent'; ++ }; ++ const ownsCenter = (node, rect) => { ++ if (getComputedStyle(node).pointerEvents === 'none') ++ return true; ++ if (rect.right <= 0 || rect.bottom <= 0 || rect.left >= innerWidth || rect.top >= innerHeight) ++ return true; ++ const x = Math.min(innerWidth - 1, Math.max(0, (rect.left + rect.right) / 2)); ++ const y = Math.min(innerHeight - 1, Math.max(0, (rect.top + rect.bottom) / 2)); ++ const root = node.getRootNode(); ++ const top = root.elementsFromPoint(x, y)[0]; ++ return top === node || (top ? node.contains(top) : false); ++ }; ++ const addBone = (node, rect, r, container = false, edges = 0, filled = false, kind = 'graphic') => { ++ if (!ownsCenter(node, rect)) ++ return; ++ const left = Math.max(rect.left, rootRect.left); ++ const right = Math.min(rect.right, rootRect.right); ++ const top = Math.max(rect.top, rootRect.top); ++ const bottom = Math.min(rect.bottom, rootRect.bottom); ++ if (right - left < 1 || bottom - top < 1) ++ return; ++ const rw = rootRect.width; ++ bones.push({ ++ x: rw > 0 ? +((left - rootRect.left) / rw * 100).toFixed(4) : 0, ++ y: Math.round(top - rootRect.top), ++ w: rw > 0 ? +((right - left) / rw * 100).toFixed(4) : 0, ++ h: Math.round(bottom - top), ++ r, ++ c: container || undefined, ++ e: edges || undefined, ++ f: filled || undefined, ++ _kind: container ? 'surface' : kind, ++ }); ++ }; ++ const addTextBones = (node, style) => { ++ const walker = document.createTreeWalker(node, NodeFilter.SHOW_TEXT); ++ const range = document.createRange(); ++ const fragments = []; ++ for (let text = walker.nextNode(); text; text = walker.nextNode()) { ++ if (!text.textContent?.trim()) ++ continue; ++ range.selectNodeContents(text); ++ fragments.push(...[...range.getClientRects()].filter(rect => rect.width >= 8 && rect.height >= 1 && ownsCenter(node, rect))); ++ } ++ range.detach(); ++ fragments.sort((a, b) => a.top - b.top || a.left - b.left); ++ const runs = []; ++ const maxGap = (parseFloat(style.fontSize) || 16) * 1.25; ++ for (const rect of fragments) { ++ const center = (rect.top + rect.bottom) / 2; ++ const run = runs.find(candidate => Math.abs(candidate.center - center) <= Math.max(2, Math.min(candidate.height, rect.height) * 0.35) && ++ rect.left - candidate.right <= maxGap); ++ if (run) { ++ run.right = Math.max(run.right, rect.right); ++ run.top = Math.min(run.top, rect.top); ++ run.bottom = Math.max(run.bottom, rect.bottom); ++ run.height = run.bottom - run.top; ++ run.center = (run.top + run.bottom) / 2; ++ } ++ else { ++ runs.push({ ++ left: rect.left, ++ right: rect.right, ++ top: rect.top, ++ bottom: rect.bottom, ++ height: rect.height, ++ center, ++ }); ++ } ++ } ++ const radius = cssPixels(style.getPropertyValue('--radius-sm'), style) || 4; ++ for (const run of runs) { ++ const height = Math.max(8, Math.round(run.height)); ++ const top = run.center - height / 2; ++ addBone(node, { ++ left: run.left, ++ right: run.right, ++ top, ++ bottom: top + height, ++ }, radius, false, 0, false, 'text'); ++ } ++ }; ++ const paintsBackground = node => { ++ const style = getComputedStyle(node); ++ return !isTransparent(style.backgroundColor) || style.backgroundImage !== 'none'; ++ }; ++ const isBackgroundOnlyLeaf = node => renderedChildren(node).length === 0 && ++ !node.textContent?.trim() && ++ paintsBackground(node); ++ const overlapRatio = (a, b) => { ++ const aLeft = a.x * rootRect.width / 100; ++ const aRight = (a.x + a.w) * rootRect.width / 100; ++ const bLeft = b.x * rootRect.width / 100; ++ const bRight = (b.x + b.w) * rootRect.width / 100; ++ const width = Math.max(0, Math.min(aRight, bRight) - Math.max(aLeft, bLeft)); ++ const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y)); ++ const smaller = Math.min((aRight - aLeft) * a.h, (bRight - bLeft) * b.h); ++ return smaller > 0 ? width * height / smaller : 0; ++ }; ++ const sameGeometry = (a, b) => Math.abs(a.x - b.x) < 0.1 && ++ Math.abs(a.y - b.y) < 1 && ++ Math.abs(a.w - b.w) < 0.1 && ++ Math.abs(a.h - b.h) < 1; ++ const cleanBones = values => values ++ .filter((bone, index) => { ++ const duplicate = bone._kind !== 'surface' && ++ values.some((other, otherIndex) => otherIndex > index && ++ other._kind !== 'surface' && ++ sameGeometry(bone, other)); ++ if (duplicate) ++ return false; ++ if (bone._kind !== 'media' && bone._kind !== 'graphic') ++ return true; ++ const coveredByControl = values.some((other, otherIndex) => otherIndex !== index && ++ other._kind === 'control' && ++ other.w >= bone.w * 0.9 && ++ overlapRatio(bone, other) >= 0.8); ++ if (coveredByControl) ++ return false; ++ return bone._kind !== 'media' || ++ !values.some((other, otherIndex) => otherIndex > index && ++ other._kind === 'text' && ++ other.w >= bone.w * 2 && ++ overlapRatio(bone, other) >= 0.8); ++ }) ++ .map(({ _kind, ...bone }) => bone); + function walk(node) { + const style = getComputedStyle(node); + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') + return; ++ const box = node.getBoundingClientRect(); ++ const clipped = style.clipPath !== 'none' || (style.clip !== 'auto' && style.clip !== ''); ++ if (clipped && box.width <= 1 && box.height <= 1) ++ return; + const tag = node.tagName.toLowerCase(); +- // Exclusions — skip element and all descendants + if (excludeTags?.has(tag)) + return; + if (excludeSelectors?.some(sel => node.matches(sel))) + return; +- const children = [...node.children].filter(child => { ++ const children = renderedChildren(node).filter(child => { + const cs = getComputedStyle(child); + return cs.display !== 'none' && cs.visibility !== 'hidden' && cs.opacity !== '0'; + }); +- const isMedia = tag === 'img' || tag === 'svg' || tag === 'video' || tag === 'canvas'; ++ const isMedia = tag === 'img' || tag === 'svg' || tag === 'video' || tag === 'canvas' || ++ node.matches('[role="img"]'); + const isFormEl = tag === 'input' || tag === 'button' || tag === 'textarea' || tag === 'select'; +- const isLeaf = children.length === 0 || isMedia || isFormEl || leafTags.has(tag); +- // Container emits a bone if it has any non-transparent background, a background image, +- // or (when captureRoundedBorders is true) a visible border on a rounded element. +- // Every background color counts — white cards are still cards. ++ const isControl = isFormEl || node.matches('[role="button"], [role="tab"], [data-variant], [aria-current="page"], [aria-selected="true"]'); ++ const atomicSelector = 'button, input, textarea, select, [role="button"], [role="tab"], [data-variant], [aria-current="page"], [aria-selected="true"], img, svg, video, canvas, [role="img"]'; ++ const hasAtomicDescendant = children.some(child => child.matches(atomicSelector) || child.querySelector(atomicSelector)); ++ const isLeaf = children.length === 0 || isMedia || isControl || node.hasAttribute('data-line') || (leafTags.has(tag) && !hasAtomicDescendant); + const bg = style.backgroundColor; +- const hasBg = bg !== 'rgba(0, 0, 0, 0)' && bg !== 'transparent'; ++ const hasDistinctBg = !isTransparent(bg) && bg !== backdropOf(node); + const hasBgImage = style.backgroundImage !== 'none'; +- const borderTopWidth = parseFloat(style.borderTopWidth) || 0; +- const hasBorder = captureRoundedBorders && borderTopWidth > 0 && style.borderTopColor !== 'rgba(0, 0, 0, 0)' && style.borderTopColor !== 'transparent'; +- const hasBorderRadius = (parseFloat(style.borderTopLeftRadius) || 0) > 0; +- const hasVisualSurface = hasBg || hasBgImage || (hasBorder && hasBorderRadius); ++ const visibleBorder = (width, color) => captureBorders && ++ (parseFloat(width) || 0) > 0 && ++ !isTransparent(color); ++ const borderEdges = ++ (visibleBorder(style.borderTopWidth, style.borderTopColor) ? 1 : 0) | ++ (visibleBorder(style.borderRightWidth, style.borderRightColor) ? 2 : 0) | ++ (visibleBorder(style.borderBottomWidth, style.borderBottomColor) ? 4 : 0) | ++ (visibleBorder(style.borderLeftWidth, style.borderLeftColor) ? 8 : 0); ++ const hasVisualSurface = hasDistinctBg || hasBgImage || borderEdges > 0; + const isTableNode = tag === 'tr' || tag === 'td' || tag === 'th' || tag === 'thead' || tag === 'tbody' || tag === 'table'; ++ const radius = isTableNode ? 0 : parseBorderRadius(style, node); ++ const clipsChildren = style.overflowX !== 'visible' || style.overflowY !== 'visible'; ++ const isGraphicGroup = children.length > 0 && ++ !node.textContent?.trim() && ++ radius !== 0 && ++ clipsChildren && ++ children.every(isBackgroundOnlyLeaf); ++ if (isGraphicGroup) { ++ addBone(node, node.getBoundingClientRect(), radius); ++ return; ++ } + if (isLeaf) { + const rect = node.getBoundingClientRect(); + if (rect.width < 1 || rect.height < 1) + return; +- // Squarish media elements (SVG icons, circular images) get 50% radius +- const isSquarish = isMedia && rect.width > 0 && rect.height > 0 && Math.abs(rect.width - rect.height) < 4; +- const br = isTableNode ? 0 : isSquarish ? '50%' : (parseBorderRadius(style, node) ?? 8); +- const rw = rootRect.width; +- bones.push({ +- x: rw > 0 ? +((rect.left - rootRect.left) / rw * 100).toFixed(4) : 0, +- y: Math.round(rect.top - rootRect.top), +- w: rw > 0 ? +((rect.width) / rw * 100).toFixed(4) : 0, +- h: Math.round(rect.height), +- r: br, +- }); ++ if (isMedia || isControl) { ++ addBone(node, rect, radius, false, 0, false, isMedia ? 'media' : 'control'); ++ return; ++ } ++ if (node.textContent?.trim()) { ++ addTextBones(node, style); ++ } ++ else if (hasDistinctBg || hasBgImage) { ++ addBone(node, rect, radius); ++ } ++ else if (borderEdges > 0) { ++ addBone(node, rect, radius, true, borderEdges); ++ } ++ return; ++ } ++ // compact one-line metadata reads as one placeholder; walking every ++ // decorative icon, separator and one-character count turns it into noise ++ const childRects = children ++ .map(child => child.getBoundingClientRect()) ++ .filter(rect => rect.width >= 1 && rect.height >= 1); ++ const compactInline = !isControl && ++ !hasVisualSurface && ++ (style.display === 'flex' || style.display === 'inline-flex') && ++ box.height <= 32 && ++ childRects.length >= 2 && ++ !!node.textContent?.trim() && ++ !node.querySelector('button, input, textarea, select, [role="button"], [role="tab"], [data-variant], [aria-current="page"], [aria-selected="true"]') && ++ Math.max(...childRects.map(rect => rect.bottom)) - ++ Math.min(...childRects.map(rect => rect.top)) <= 24; ++ if (compactInline) { ++ const left = Math.min(...childRects.map(rect => rect.left)); ++ const right = Math.max(...childRects.map(rect => rect.right)); ++ const top = Math.min(...childRects.map(rect => rect.top)); ++ const bottom = Math.max(...childRects.map(rect => rect.bottom)); ++ const textRadius = cssPixels(style.getPropertyValue('--radius-sm'), style) || 4; ++ addBone(node, { left, right, top, bottom }, textRadius, false, 0, false, 'text'); + return; + } +- // Container with visible background: emit a lighter bone (c: true) so that +- // child bones rendered on top stand out against it, then recurse. + if (hasVisualSurface) { + const rect = node.getBoundingClientRect(); + if (rect.width >= 1 && rect.height >= 1) { +- const br = isTableNode ? 0 : (parseBorderRadius(style, node) ?? 8); +- const rw = rootRect.width; +- bones.push({ +- x: rw > 0 ? +((rect.left - rootRect.left) / rw * 100).toFixed(4) : 0, +- y: Math.round(rect.top - rootRect.top), +- w: rw > 0 ? +((rect.width) / rw * 100).toFixed(4) : 0, +- h: Math.round(rect.height), +- r: br, +- c: true, +- }); ++ addBone(node, rect, radius, true, borderEdges, hasDistinctBg || hasBgImage); + } + } +- // Recurse into children regardless — their bones overlay the container bone + for (const child of children) { + walk(child); + } + } +- // Don't snapshot the root itself — walk its children +- for (const child of el.children) { ++ for (const child of renderedChildren(el)) { + walk(child); + } + return { +@@ -90,7 +269,7 @@ export function snapshotBones(el, name = 'component', config) { + viewportWidth: Math.round(rootRect.width), + width: Math.round(rootRect.width), + height: Math.round(rootRect.height), +- bones, ++ bones: cleanBones(bones), + }; + } + /** +@@ -321,9 +500,8 @@ function parseBorderRadius(style, el) { + const tr = parseFloat(style.borderTopRightRadius) || 0; + const br = parseFloat(style.borderBottomRightRadius) || 0; + const bl = parseFloat(style.borderBottomLeftRadius) || 0; +- // No radius + if (tl === 0 && tr === 0 && br === 0 && bl === 0) +- return undefined; ++ return 0; + // Detect if element is roughly square (circle candidate) + const isSquarish = el ? (() => { + const rect = el.getBoundingClientRect(); +@@ -338,9 +516,8 @@ function parseBorderRadius(style, el) { + if (maxCorner > 9998) { + return isSquarish ? '50%' : 9999; + } +- // All corners equal — return single number (skip if default 8) + if (tl === tr && tr === br && br === bl) { +- return tl !== 8 ? tl : undefined; ++ return tl; + } + // Asymmetric corners — return full CSS string + return `${tl}px ${tr}px ${br}px ${bl}px`; +diff --git a/dist/shared.js b/dist/shared.js +index 6f8a382b44b1484dc0989da866f7f44df3ab74ae..be354dce42e08ab13e6e481dec9ef98edc3c5e32 100644 +--- a/dist/shared.js ++++ b/dist/shared.js +@@ -1,4 +1,3 @@ +-import { snapshotBones } from './extract.js'; + const bonesRegistry = new Map(); + export function registerBones(map) { + for (const [name, bones] of Object.entries(map)) { +@@ -10,7 +9,9 @@ export function getRegisteredBones(name) { + } + export function ensureBuildSnapshotHook() { + if (typeof window !== 'undefined' && window.__BONEYARD_BUILD) { +- window.__BONEYARD_SNAPSHOT = snapshotBones; ++ void import('./extract.js').then(({ snapshotBones }) => { ++ window.__BONEYARD_SNAPSHOT = snapshotBones; ++ }); + } + } + export function isBuildMode() { +diff --git a/dist/types.d.ts b/dist/types.d.ts +index 2959d4273534869e38825946aa61f599b865cd60..e048ee8d72277a5133cf903b1b965f7e533ffbcb 100644 +--- a/dist/types.d.ts ++++ b/dist/types.d.ts +@@ -22,6 +22,11 @@ export interface SnapshotConfig { + * Default: true + */ + captureRoundedBorders?: boolean; ++ /** ++ * Capture visible container borders, including one-sided dividers. ++ * Defaults to captureRoundedBorders when omitted. ++ */ ++ captureBorders?: boolean; + /** + * HTML tags to skip entirely — no bone emitted, children not walked. + * Useful for decorative elements, icons, or custom components you don't +@@ -53,21 +58,24 @@ export interface Bone { + w: number; + h: number; + r: number | string; +- /** True if this bone is a background container — rendered lighter so children stand out */ ++ /** True if this bone is a structural container */ + c?: boolean; ++ /** Visible border edges: top=1, right=2, bottom=4, left=8 */ ++ e?: number; ++ /** True if this container has a visible background */ ++ f?: boolean; + } +-/** Compact bone format: [x, y, w, h, r, c?] — c is omitted if false */ +-export type CompactBone = [number, number, number, number, number | string] | [number, number, number, number, number | string, boolean]; ++/** Compact bone format: [x, y, w, h, r, c?, e?, f?] */ ++export type CompactBone = ++ | [number, number, number, number, number | string] ++ | [number, number, number, number, number | string, boolean, number, boolean]; + /** + * Any form a bone may take at the input boundary. + * + * - `Bone`: structured object form. +- * - `CompactBone`: hand-authored 5/6-element tuple. ++ * - `CompactBone`: hand-authored compact tuple. + * - `(number | string | boolean)[]`: what TypeScript infers for tuples loaded +- * from `*.bones.json` imports under `resolveJsonModule: true`. Tuple types +- * are narrowed to arrays on serialization; container bones emit a trailing +- * `true` so the union must cover booleans too. `normalizeBone` validates +- * the length at runtime. ++ * from `*.bones.json` imports under `resolveJsonModule: true`. + */ + export type AnyBone = Bone | CompactBone | (number | string | boolean)[]; + /** Normalize a bone from either format to the object format */ +diff --git a/dist/types.js b/dist/types.js +index 8dc71277e58b72de6cf964fb830cfa94fb82c0a3..0512718737d44449da7303c7378a0b3b29a9ea26 100644 +--- a/dist/types.js ++++ b/dist/types.js +@@ -1,12 +1,21 @@ + /** Normalize a bone from either format to the object format */ + export function normalizeBone(b) { + if (Array.isArray(b)) { +- if (b.length < 5 || b.length > 6) { +- throw new Error(`Invalid bone format: expected [x,y,w,h,r,c?] but got ${b.length} elements`); ++ if (b.length < 5 || b.length > 8) { ++ throw new Error(`Invalid bone format: expected [x,y,w,h,r,c?,e?,f?] but got ${b.length} elements`); + } + // Length is validated; narrow to the compact tuple shape. + const t = b; +- return { x: t[0], y: t[1], w: t[2], h: t[3], r: t[4], c: t[5] || undefined }; ++ return { ++ x: t[0], ++ y: t[1], ++ w: t[2], ++ h: t[3], ++ r: t[4], ++ c: t[5] || undefined, ++ e: t[6] || undefined, ++ f: t[7] || undefined, ++ }; + } + return b; + } diff --git a/web/patches/cobe@2.0.1.patch b/web/patches/cobe@2.0.1.patch new file mode 100644 index 0000000000000000000000000000000000000000..e613b75998feb45604a82c421a0e858d2f7b1622 --- /dev/null +++ b/web/patches/cobe@2.0.1.patch @@ -0,0 +1,8 @@ +diff --git a/dist/index.esm.js b/dist/index.esm.js +index fc189005cee9f824e49973c05d115c4e8ea35447..a8e4e81ee90aaed254e850998bce4387fec23e7c 100644 +--- a/dist/index.esm.js ++++ b/dist/index.esm.js +@@ -1 +1 @@ +-function Re(e,t,r){let a=e.createShader(t);return e.shaderSource(a,r),e.compileShader(a),e.getShaderParameter(a,e.COMPILE_STATUS)?a:(e.deleteShader(a),null)}function Y(e,t,r){let a=Re(e,e.VERTEX_SHADER,t),o=Re(e,e.FRAGMENT_SHADER,r);if(!a||!o)return null;let i=e.createProgram();return e.attachShader(i,a),e.attachShader(i,o),e.linkProgram(i),e.getProgramParameter(i,e.LINK_STATUS)?(e.deleteShader(a),e.deleteShader(o),i):(e.deleteProgram(i),null)}function H(e,t,r){let a={};for(let o of r)a[o]=e.getUniformLocation(t,o);return a}function Q(e,t,r){let a={};for(let o of r)a[o]=e.getAttribLocation(t,o);return a}function Ae(e){let t={},r={},a={},o=document.createElement("style");function i(t,r,a,o){let i=t[r];i||(i=document.createElement("div"),i.style.cssText="position:absolute;width:1px;height:1px;pointer-events:none;anchor-name:"+a,e.append(i),t[r]=i),i.style.left=100*o.x+"%",i.style.top=100*o.y+"%"}return document.head.append(o),{m:function(e,r){let o={};for(let n of e){let e=n.id;if(!e)continue;let f=r(n.location);o[e]=1,i(t,e,"--cobe-"+e,f),f.visible?a["--cobe-visible-"+e]="N":delete a["--cobe-visible-"+e]}for(let e in t)o[e]||(t[e].remove(),delete t[e],delete a["--cobe-visible-"+e])},a:function(e,t){let o={};for(let n of e){let e=n.id;if(!e)continue;let f=t(n);o[e]=1,i(r,e,"--cobe-arc-"+e,f),f.visible?a["--cobe-visible-arc-"+e]="N":delete a["--cobe-visible-arc-"+e]}for(let e in r)o[e]||(r[e].remove(),delete r[e],delete a["--cobe-visible-arc-"+e])},r:function(){for(let e in t)t[e].remove();for(let e in r)r[e].remove();o.remove()},s:function(){let e="";for(let t in a)e+=t+":"+a[t]+";";o.textContent=":root{"+e+"}"}}}var{PI:Z,sin:le,cos:_e}=Math,Be="attribute vec2 a;void main(){gl_Position=vec4(a,0,1);}",Ce="precision highp float;uniform vec2 t,v,s;uniform vec3 F,w;uniform vec4 n;uniform float k,x,y;uniform sampler2D z;float u;mat3 A(float a,float b){float c=cos(a),d=cos(b),e=sin(a),f=sin(b);return mat3(d,f*e,-f*c,0,c,e,f,d*-e,d*c);}vec3 B(vec3 c,out float G){c=c.xzy;float q=max(2.,floor(log2(2.236068*k*3.141593*(1.-c.z*c.z))*.72021));vec2 g=floor(pow(1.618034,q)/2.236068*vec2(1,1.618034)+.5),d=fract((g+1.)*.618034)*6.283185-3.883222,e=-2.*g,f=vec2(atan(c.y,c.x),c.z-1.),r=floor(vec2(e.y*f.x-d.y*(f.y*k+1.),-e.x*f.x+d.x*(f.y*k+1.))/(d.x*e.y-e.x*d.y));float o=3.141593;vec3 C;for(float h=0.;h<4.;h+=1.){vec2 D=vec2(mod(h,2.),floor(h*.5));float j=dot(g,r+D);if(j>k)continue;float a=j,b=0.;a>=16384.?(a-=16384.,b+=.868872):0.,a>=8192.?(a-=8192.,b+=.934436):0.,a>=4096.?(a-=4096.,b+=.467218):0.,a>=2048.?(a-=2048.,b+=.733609):0.,a>=1024.?(a-=1024.,b+=.866804):0.,a>=512.?(a-=512.,b+=.433402):0.,a>=256.?(a-=256.,b+=.216701):0.,a>=128.?(a-=128.,b+=.108351):0.,a>=64.?(a-=64.,b+=.554175):0.,a>=32.?(a-=32.,b+=.777088):0.,a>=16.?(a-=16.,b+=.888544):0.,a>=8.?(a-=8.,b+=.944272):0.,a>=4.?(a-=4.,b+=.472136):0.,a>=2.?(a-=2.,b+=.236068):0.,a>=1.?(a-=1.,b+=.618034):0.;float l=fract(b)*6.283185,i=1.-2.*j*u,m=sqrt(1.-i*i);vec3 p=vec3(cos(l)*m,sin(l)*m,i);float E=length(c-p);if(E.25)discard;vec3 a=h>.5?g:v;gl_FragColor=vec4(a,1);}",Fe="varying vec3 i;varying float j,s,t;attribute vec2 k;attribute vec3 l,m,N;attribute float v,w,O;uniform vec2 g,x;uniform float y,z,h,A;mat3 B(float a,float b){float c=cos(a),d=cos(b),e=sin(a),f=sin(b);return mat3(d,f*e,-f*c,0,c,e,f,d*-e,d*c);}vec3 C(vec3 c,vec3 d,vec3 e,float a){float b=1.-a;return b*b*c+2.*b*a*d+a*a*e;}vec3 D(vec3 c,vec3 b,vec3 d,float a){float e=1.-a;return 2.*e*(b-c)+2.*a*(d-b);}void main(){mat3 b=B(z,y);float c=.8+A;vec3 d=l*c,e=m*c,f=l+m;float n=length(f);vec3 E=n>1e-3?f/n:vec3(0,1,0),o=E*(.8+v);float p=k.x;vec3 F=C(d,o,e,p),q=b*F,G=D(d,o,e,p),H=b*G;vec2 a=H.xy;float r=length(a);vec2 I=r>1e-3?vec2(-a.y,a.x)/r:vec2(1,0);float J=g.x/g.y;vec2 K=q.xy*vec2(1./J,1)*h+x*vec2(1,-1)*h/g,P=K+I*w*k.y*h;gl_Position=vec4(P,0,1),i=N,j=O,s=q.z,t=length(q.xy);}",Le="precision highp float;varying vec3 i;varying float j,s,t;uniform vec3 M;void main(){if(s<0.&&t<.8)discard;vec3 a=j>.5?i:M;gl_FragColor=vec4(a,1);}",ee=.8;function U([e,t]){let r=e*Z/180,a=t*Z/180-Z,o=_e(r);return[-o*_e(a),le(r),o*le(a)]}var Pe=(e,t)=>{let r={alpha:!0,stencil:!1,antialias:!0,depth:!1,preserveDrawingBuffer:!1,...t.context},a=e.getContext("webgl2",r),o=!!a;if(a||(a=e.getContext("webgl",r)),!a)return{destroy:()=>{},update:()=>{}};let i=o?null:a.getExtension("ANGLE_instanced_arrays"),n=t.devicePixelRatio||1;e.width=t.width*n,e.height=t.height*n;let f=t.phi||0,l=t.theta||0,c=t.markers||[],s=t.arcs||[],A=t.mapSamples||1e4,v=t.mapBrightness||1,d=t.mapBaseBrightness||0,g=t.baseColor||[1,1,1],m=t.markerColor||[1,.5,0],u=t.glowColor||[1,1,1],h=t.arcColor||[.3,.6,1],E=t.arcWidth??1,R=t.arcHeight??.2,b=t.diffuse||1,x=t.dark||0,y=t.opacity??1,T=t.offset||[0,0],B=t.scale||1,p=t.markerElevation??.05,w=Y(a,Be,Ce),C=Y(a,be,ge),F=Y(a,Fe,Le);if(!w)return{destroy:()=>{},update:()=>{}};let D=a.createBuffer();a.bindBuffer(a.ARRAY_BUFFER,D),a.bufferData(a.ARRAY_BUFFER,new Float32Array([-1,-1,1,-1,-1,1,-1,1,1,-1,1,1]),a.STATIC_DRAW);let _=a.createBuffer();a.bindBuffer(a.ARRAY_BUFFER,_);let N=[];for(let e=0;e<=32;e++){let t=e/32;N.push(t,-1,t,1)}a.bufferData(a.ARRAY_BUFFER,new Float32Array(N),a.STATIC_DRAW);let k=a.createBuffer(),S=a.createBuffer(),I=H(a,w,["t","s","k","x","v","F","w","n","y","z"]),P=H(a,C,["i","j","b","k","r","v","s"]),L=Q(a,C,["n","p","q","w","x"]),z=H(a,F,["y","z","g","h","x","M","A"]),G=Q(a,F,["k","l","m","v","w","N","O"]),M=a.getAttribLocation(w,"a"),V=a.createTexture();a.bindTexture(a.TEXTURE_2D,V),a.texImage2D(a.TEXTURE_2D,0,a.RGB,1,1,0,a.RGB,a.UNSIGNED_BYTE,new Uint8Array([0,0,0])),a.texParameteri(a.TEXTURE_2D,a.TEXTURE_MIN_FILTER,a.NEAREST),a.texParameteri(a.TEXTURE_2D,a.TEXTURE_MAG_FILTER,a.NEAREST);let j=new Image;j.onload=()=>{a.bindTexture(a.TEXTURE_2D,V),a.texImage2D(a.TEXTURE_2D,0,a.RGB,a.RGB,a.UNSIGNED_BYTE,j),a.generateMipmap(a.TEXTURE_2D),a.texParameteri(a.TEXTURE_2D,a.TEXTURE_MIN_FILTER,a.NEAREST),a.texParameteri(a.TEXTURE_2D,a.TEXTURE_MAG_FILTER,a.NEAREST),a.activeTexture(a.TEXTURE0),a.bindTexture(a.TEXTURE_2D,V)},j.src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAQAAAACAAQAAAADMzoqnAAAECklEQVR42u3VsW4jRRzH8d94gzfF4Q0VQaC4vBLTRTp0mze4ggfAPAE5XQEFsGNAVIjwBrmW7h7gJE+giKjyABTZE4g06LKJETdRJvtD65kdz6yduKABiW+TVfzRf2bXYxtcE/59YJCz6YdbgQF6ACSRrwYKYImmh5PbwOewlV3wlQNbAN6SEExjUOO+BU0aCSnxReHABUlK4YFQeJeUT3da8IIkZ6NGoSnFY5KsMoVzMKfECUnqxgPYRArarmUCndHwzIEaQEpg5xVdBXROl8mpAQx5dUgPiHoYAAkg5w3JABR06byGAVgcRGAz5bznj6phBQNRFwyqgdxebH6gshJAesWoFhgYpApAFoG8BIZ/fEhSox5jDjQXmV0Ar5XJfAIrALi3URVs09gHIL4XJCkLC5LH9JWiArABFCSrQjdgkBzRJ0WJeUOSNyQAfJJwUSWUBRlJQ8oGHATACGlBynnzy2kEYLNjrxouigD8BZcgOeVPqh12RtufaCN5wCPVDpvQ9lsIrqndsJtDcWqBCpf4hWN7OdWHBw58FwIaNOU/n1TpMW2DFaD48cmr4185T8NHkpUFX749pQPVdgRKC/DGoQPVeAEKv+WHvY8OOWNTPRp5kHuwSf8wzXtVBKR7YwEH9H3lQUaypUfSATOALyVNu5vZJW31Bnx98nkLfDUWJaz6ixvm+RIQRdl3kmRxxiaDoGnZW4CpPfkaQadlcPim1xOSvETQo7Lv75enVAXJ3xGUlony4KQBBWUM1NiDc6qhyS8RgQs18OCMMtPDaAUIyg0PZkRWDqs+wnKJBTDI1Js6BolegOsKmUxNDBAAKqQyMQmidhegBlLZ+wwKYdv5M/8x1khkb1cgKqP2H+MKyV5vS+whrE8DQDgAlUAoRBX056EElJCjJVACeJBZgNfVp+iCCm4RBWCgKsRxASSA9KgDhDtCiTuMyfHsKXzhC6wNAIjjWb8LKAOA2ctk3FmCOlgKFy8f1N0JJtgsxinYnVAHt4t3gPzZXSCTyCWCQmBT91QE3B5yarSN40dNHYPka4TlDhTUI8zLvl0JSL3vZn6DsCFZOeB2yROEpR68sECQQA++xIGCR2X7DwlEoLRgUrZrqlUg50S1uy43YqDcN6UFBVkhAjWiCV2Q0jgQPdplMKxvBXodcOfAwJYvgdL+1etA1YJJfBcZlQV7sO1i2gHoNiyxtQ5sBsCgWyoxCHiFFd2L5nUTCqMAqGUgsQ9f5kCcCiZgRYkMgMTd5WsB1rTzj0Em14BE4r+QxN1lCEsVur2PoF5Wbg8RJXR4djgvBgauhLywoEZQrt1KKRdVS4CdlJ8qafyP+9KIj/nE/d7kKwH9jgS72e9DV+kvfTWgct4ZyP8Byb8BPG7MaaIIkAQAAAAASUVORK5CYII=";let J=0;function O(t){let r=Math.cos(l),a=Math.cos(f),o=Math.sin(l),i=Math.sin(f),c=a*t[0]+i*t[2],s=i*o*t[0]+r*t[1]-a*o*t[2];return[(c/(e.width/e.height)*B+T[0]*B*n/e.width+1)/2,(-s*B+T[1]*B*n/e.height+1)/2,-i*r*t[0]+o*t[1]+a*r*t[2]>=0||c*c+s*s>=.64]}function W(e){let t=U(e),r=ee+p,a=O([t[0]*r,t[1]*r,t[2]*r]);return{x:a[0],y:a[1],visible:a[2]}}function X(e){let t=U(e.from),r=U(e.to),a=[t[0]+r[0],t[1]+r[1],t[2]+r[2]],o=(a[0]**2+a[1]**2+a[2]**2)**.5;if(o<.001)return null;let i=.25*(ee+p)+.5*(ee+R+p)/o,n=O([a[0]*i,a[1]*i,a[2]*i]);return{x:n[0],y:n[1],visible:n[2]}}function K(e,t,r,n,f){e<0||(a.enableVertexAttribArray(e),a.vertexAttribPointer(e,t,a.FLOAT,!1,r,n),o?a.vertexAttribDivisor(e,f):i&&i.vertexAttribDivisorANGLE(e,f))}let q,Z=document.createElement("div");Z.style.cssText="position:relative;width:100%;height:100%",e.parentElement?.insertBefore(Z,e),Z.append(e);let $=Ae(Z);function te(t){if(t.phi!=q&&(f=t.phi),t.theta!=q&&(l=t.theta),t.markers&&function(e){c=e;let t=new Float32Array(8*c.length);c.forEach((e,r)=>{t.set([...U(e.location),e.size,...e.color||[0,0,0],e.color?1:0],8*r)}),a.bindBuffer(a.ARRAY_BUFFER,k),a.bufferData(a.ARRAY_BUFFER,t,a.DYNAMIC_DRAW)}(t.markers),t.arcs&&function(e){s=e,J=s.length;let t=new Float32Array(12*s.length);s.forEach((e,r)=>{t.set([...U(e.from),...U(e.to),R+p,.005*E,...e.color||[0,0,0],e.color?1:0],12*r)}),a.bindBuffer(a.ARRAY_BUFFER,S),a.bufferData(a.ARRAY_BUFFER,t,a.DYNAMIC_DRAW)}(t.arcs),t.width&&t.height&&(e.width=t.width*n,e.height=t.height*n),t.mapSamples!=q&&(A=t.mapSamples),t.mapBrightness!=q&&(v=t.mapBrightness),t.mapBaseBrightness!=q&&(d=t.mapBaseBrightness),t.baseColor!=q&&(g=t.baseColor),t.markerColor!=q&&(m=t.markerColor),t.glowColor!=q&&(u=t.glowColor),t.arcColor!=q&&(h=t.arcColor),t.arcWidth!=q&&(E=t.arcWidth),t.arcHeight!=q&&(R=t.arcHeight),t.diffuse!=q&&(b=t.diffuse),t.dark!=q&&(x=t.dark),t.opacity!=q&&(y=t.opacity),t.offset!=q&&(T=t.offset),t.scale!=q&&(B=t.scale),t.markerElevation!=q&&(p=t.markerElevation),$.m(c,W),$.a(s,X),$.s(),a.viewport(0,0,e.width,e.height),a.clearColor(0,0,0,0),a.clear(a.COLOR_BUFFER_BIT),a.enable(a.BLEND),a.blendFunc(a.SRC_ALPHA,a.ONE_MINUS_SRC_ALPHA),a.useProgram(w),a.bindBuffer(a.ARRAY_BUFFER,D),a.enableVertexAttribArray(M),a.vertexAttribPointer(M,2,a.FLOAT,!1,0,0),o?a.vertexAttribDivisor(M,0):i&&i.vertexAttribDivisorANGLE(M,0),a.uniform2f(I.t,e.width,e.height),a.uniform2f(I.s,f,l),a.uniform1f(I.k,A),a.uniform1f(I.x,B),a.uniform2f(I.v,T[0]*n,T[1]*n),a.uniform3fv(I.F,g),a.uniform3fv(I.w,u),a.uniform4f(I.n,v,b,x,y),a.uniform1f(I.y,d),a.uniform1i(I.z,0),a.activeTexture(a.TEXTURE0),a.bindTexture(a.TEXTURE_2D,V),a.drawArrays(a.TRIANGLES,0,6),F&&J>0){a.useProgram(F),a.bindBuffer(a.ARRAY_BUFFER,_),G.k>=0&&(a.enableVertexAttribArray(G.k),a.vertexAttribPointer(G.k,2,a.FLOAT,!1,0,0),o?a.vertexAttribDivisor(G.k,0):i&&i.vertexAttribDivisorANGLE(G.k,0)),a.bindBuffer(a.ARRAY_BUFFER,S);let t=48;if(K(G.l,3,t,0,1),K(G.m,3,t,12,1),K(G.v,1,t,24,1),K(G.w,1,t,28,1),K(G.N,3,t,32,1),K(G.O,1,t,44,1),a.uniform1f(z.y,f),a.uniform1f(z.z,l),a.uniform2f(z.g,e.width,e.height),a.uniform1f(z.h,B),a.uniform2f(z.x,T[0]*n,T[1]*n),a.uniform3fv(z.M,h),a.uniform1f(z.A,p),o)a.drawArraysInstanced(a.TRIANGLE_STRIP,0,66,J);else if(i)i.drawArraysInstancedANGLE(a.TRIANGLE_STRIP,0,66,J);else for(let e=0;e0){a.useProgram(C),a.bindBuffer(a.ARRAY_BUFFER,D),L.n>=0&&(a.enableVertexAttribArray(L.n),a.vertexAttribPointer(L.n,2,a.FLOAT,!1,0,0),o?a.vertexAttribDivisor(L.n,0):i&&i.vertexAttribDivisorANGLE(L.n,0)),a.bindBuffer(a.ARRAY_BUFFER,k);let t=32;K(L.p,3,t,0,1),K(L.q,1,t,12,1),K(L.w,3,t,16,1),K(L.x,1,t,28,1),a.uniform1f(P.i,f),a.uniform1f(P.j,l),a.uniform2f(P.b,e.width,e.height),a.uniform1f(P.k,B),a.uniform2f(P.r,T[0]*n,T[1]*n),a.uniform3fv(P.v,m),a.uniform1f(P.s,p),function(e){if(o)a.drawArraysInstanced(a.TRIANGLES,0,6,e);else if(i)i.drawArraysInstancedANGLE(a.TRIANGLES,0,6,e);else for(let t=0;t{a.deleteBuffer(D),a.deleteBuffer(_),a.deleteBuffer(k),a.deleteBuffer(S),a.deleteProgram(w),C&&a.deleteProgram(C),F&&a.deleteProgram(F),$.r()}}};export{Pe as default}; +\ No newline at end of file ++function Re(e,t,r){let a=e.createShader(t);return e.shaderSource(a,r),e.compileShader(a),e.getShaderParameter(a,e.COMPILE_STATUS)?a:(e.deleteShader(a),null)}function H(e,t,r){let a=Re(e,e.VERTEX_SHADER,t),o=Re(e,e.FRAGMENT_SHADER,r);if(!a||!o)return null;let i=e.createProgram();return e.attachShader(i,a),e.attachShader(i,o),e.linkProgram(i),e.getProgramParameter(i,e.LINK_STATUS)?(e.deleteShader(a),e.deleteShader(o),i):(e.deleteProgram(i),null)}function V(e,t,r){let a={};for(let o of r)a[o]=e.getUniformLocation(t,o);return a}function Z(e,t,r){let a={};for(let o of r)a[o]=e.getAttribLocation(t,o);return a}function le(e){let t={},r={},a={},o=document.createElement("style");function i(t,r,a,o){let i=t[r];i||(i=document.createElement("div"),i.style.cssText="position:absolute;width:1px;height:1px;pointer-events:none;anchor-name:"+a,e.append(i),t[r]=i),i.style.left=100*o.x+"%",i.style.top=100*o.y+"%"}return document.head.append(o),{m:function(e,r){let o={};for(let n of e){let e=n.id;if(!e)continue;let l=r(n.location);o[e]=1,i(t,e,"--cobe-"+e,l),l.visible?a["--cobe-visible-"+e]="N":delete a["--cobe-visible-"+e]}for(let e in t)o[e]||(t[e].remove(),delete t[e],delete a["--cobe-visible-"+e])},a:function(e,t){let o={};for(let n of e){let e=n.id;if(!e)continue;let l=t(n);o[e]=1,i(r,e,"--cobe-arc-"+e,l),l.visible?a["--cobe-visible-arc-"+e]="N":delete a["--cobe-visible-arc-"+e]}for(let e in r)o[e]||(r[e].remove(),delete r[e],delete a["--cobe-visible-arc-"+e])},r:function(){for(let e in t)t[e].remove();for(let e in r)r[e].remove();o.remove()},s:function(){let e="";for(let t in a)e+=t+":"+a[t]+";";let t=":root{"+e+"}";t!==o.textContent&&(o.textContent=t)}}}var{PI:ee,sin:Ae,cos:_e}=Math,Be="attribute vec2 a;void main(){gl_Position=vec4(a,0,1);}",Ce="precision highp float;uniform vec2 t,v,s;uniform vec3 F,w;uniform vec4 n;uniform float k,x,y;uniform sampler2D z;float u;mat3 A(float a,float b){float c=cos(a),d=cos(b),e=sin(a),f=sin(b);return mat3(d,f*e,-f*c,0,c,e,f,d*-e,d*c);}vec3 B(vec3 c,out float G){c=c.xzy;float q=max(2.,floor(log2(2.236068*k*3.141593*(1.-c.z*c.z))*.72021));vec2 g=floor(pow(1.618034,q)/2.236068*vec2(1,1.618034)+.5),d=fract((g+1.)*.618034)*6.283185-3.883222,e=-2.*g,f=vec2(atan(c.y,c.x),c.z-1.),r=floor(vec2(e.y*f.x-d.y*(f.y*k+1.),-e.x*f.x+d.x*(f.y*k+1.))/(d.x*e.y-e.x*d.y));float o=3.141593;vec3 C;for(float h=0.;h<4.;h+=1.){vec2 D=vec2(mod(h,2.),floor(h*.5));float j=dot(g,r+D);if(j>k)continue;float a=j,b=0.;a>=16384.?(a-=16384.,b+=.868872):0.,a>=8192.?(a-=8192.,b+=.934436):0.,a>=4096.?(a-=4096.,b+=.467218):0.,a>=2048.?(a-=2048.,b+=.733609):0.,a>=1024.?(a-=1024.,b+=.866804):0.,a>=512.?(a-=512.,b+=.433402):0.,a>=256.?(a-=256.,b+=.216701):0.,a>=128.?(a-=128.,b+=.108351):0.,a>=64.?(a-=64.,b+=.554175):0.,a>=32.?(a-=32.,b+=.777088):0.,a>=16.?(a-=16.,b+=.888544):0.,a>=8.?(a-=8.,b+=.944272):0.,a>=4.?(a-=4.,b+=.472136):0.,a>=2.?(a-=2.,b+=.236068):0.,a>=1.?(a-=1.,b+=.618034):0.;float l=fract(b)*6.283185,i=1.-2.*j*u,m=sqrt(1.-i*i);vec3 p=vec3(cos(l)*m,sin(l)*m,i);float E=length(c-p);if(E.25)discard;vec3 a=h>.5?g:v;gl_FragColor=vec4(a,1);}",Fe="varying vec3 i;varying float j,s,t;attribute vec2 k;attribute vec3 l,m,N;attribute float v,w,O;uniform vec2 g,x;uniform float y,z,h,A;mat3 B(float a,float b){float c=cos(a),d=cos(b),e=sin(a),f=sin(b);return mat3(d,f*e,-f*c,0,c,e,f,d*-e,d*c);}vec3 C(vec3 c,vec3 d,vec3 e,float a){float b=1.-a;return b*b*c+2.*b*a*d+a*a*e;}vec3 D(vec3 c,vec3 b,vec3 d,float a){float e=1.-a;return 2.*e*(b-c)+2.*a*(d-b);}void main(){mat3 b=B(z,y);float c=.8+A;vec3 d=l*c,e=m*c,f=l+m;float n=length(f);vec3 E=n>1e-3?f/n:vec3(0,1,0),o=E*(.8+v);float p=k.x;vec3 F=C(d,o,e,p),q=b*F,G=D(d,o,e,p),H=b*G;vec2 a=H.xy;float r=length(a);vec2 I=r>1e-3?vec2(-a.y,a.x)/r:vec2(1,0);float J=g.x/g.y;vec2 K=q.xy*vec2(1./J,1)*h+x*vec2(1,-1)*h/g,P=K+I*w*k.y*h;gl_Position=vec4(P,0,1),i=N,j=O,s=q.z,t=length(q.xy);}",Le="precision highp float;varying vec3 i;varying float j,s,t;uniform vec3 M;void main(){if(s<0.&&t<.8)discard;vec3 a=j>.5?i:M;gl_FragColor=vec4(a,1);}",re=.8;function U([e,t]){let r=e*ee/180,a=t*ee/180-ee,o=_e(r);return[-o*_e(a),Ae(r),o*Ae(a)]}var Pe=(e,t)=>{let r={alpha:!0,stencil:!1,antialias:!0,depth:!1,preserveDrawingBuffer:!1,...t.context},a=e.getContext("webgl2",r),o=!!a;if(a||(a=e.getContext("webgl",r)),!a)return{destroy:()=>{},update:()=>{}};let i=o?null:a.getExtension("ANGLE_instanced_arrays"),n=t.devicePixelRatio||1;e.width=t.width*n,e.height=t.height*n;let l=t.phi||0,f=t.theta||0,c=t.markers||[],s=t.arcs||[],A=t.mapSamples||1e4,v=t.mapBrightness||1,d=t.mapBaseBrightness||0,g=t.baseColor||[1,1,1],u=t.markerColor||[1,.5,0],m=t.glowColor||[1,1,1],h=t.arcColor||[.3,.6,1],E=t.arcWidth??1,R=t.arcHeight??.2,b=t.diffuse||1,x=t.dark||0,y=t.opacity??1,T=t.offset||[0,0],B=t.scale||1,p=t.markerElevation??.05,C=H(a,Be,Ce),w=H(a,be,ge),F=H(a,Fe,Le);if(!C)return{destroy:()=>{},update:()=>{}};let D=a.createBuffer();a.bindBuffer(a.ARRAY_BUFFER,D),a.bufferData(a.ARRAY_BUFFER,new Float32Array([-1,-1,1,-1,-1,1,-1,1,1,-1,1,1]),a.STATIC_DRAW);let _=a.createBuffer();a.bindBuffer(a.ARRAY_BUFFER,_);let N=[];for(let e=0;e<=32;e++){let t=e/32;N.push(t,-1,t,1)}a.bufferData(a.ARRAY_BUFFER,new Float32Array(N),a.STATIC_DRAW);let k=a.createBuffer(),S=a.createBuffer(),I=V(a,C,["t","s","k","x","v","F","w","n","y","z"]),P=V(a,w,["i","j","b","k","r","v","s"]),L=Z(a,w,["n","p","q","w","x"]),z=V(a,F,["y","z","g","h","x","M","A"]),G=Z(a,F,["k","l","m","v","w","N","O"]),Q=a.getAttribLocation(C,"a"),Y=a.createTexture();a.bindTexture(a.TEXTURE_2D,Y),a.texImage2D(a.TEXTURE_2D,0,a.RGB,1,1,0,a.RGB,a.UNSIGNED_BYTE,new Uint8Array([0,0,0])),a.texParameteri(a.TEXTURE_2D,a.TEXTURE_MIN_FILTER,a.NEAREST),a.texParameteri(a.TEXTURE_2D,a.TEXTURE_MAG_FILTER,a.NEAREST);let M=new Image;M.onload=()=>{a.bindTexture(a.TEXTURE_2D,Y),a.texImage2D(a.TEXTURE_2D,0,a.RGB,a.RGB,a.UNSIGNED_BYTE,M),a.generateMipmap(a.TEXTURE_2D),a.texParameteri(a.TEXTURE_2D,a.TEXTURE_MIN_FILTER,a.NEAREST),a.texParameteri(a.TEXTURE_2D,a.TEXTURE_MAG_FILTER,a.NEAREST),a.activeTexture(a.TEXTURE0),a.bindTexture(a.TEXTURE_2D,Y),ee({})},M.src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAQAAAACAAQAAAADMzoqnAAAECklEQVR42u3VsW4jRRzH8d94gzfF4Q0VQaC4vBLTRTp0mze4ggfAPAE5XQEFsGNAVIjwBrmW7h7gJE+giKjyABTZE4g06LKJETdRJvtD65kdz6yduKABiW+TVfzRf2bXYxtcE/59YJCz6YdbgQF6ACSRrwYKYImmh5PbwOewlV3wlQNbAN6SEExjUOO+BU0aCSnxReHABUlK4YFQeJeUT3da8IIkZ6NGoSnFY5KsMoVzMKfECUnqxgPYRArarmUCndHwzIEaQEpg5xVdBXROl8mpAQx5dUgPiHoYAAkg5w3JABR06byGAVgcRGAz5bznj6phBQNRFwyqgdxebH6gshJAesWoFhgYpApAFoG8BIZ/fEhSox5jDjQXmV0Ar5XJfAIrALi3URVs09gHIL4XJCkLC5LH9JWiArABFCSrQjdgkBzRJ0WJeUOSNyQAfJJwUSWUBRlJQ8oGHATACGlBynnzy2kEYLNjrxouigD8BZcgOeVPqh12RtufaCN5wCPVDpvQ9lsIrqndsJtDcWqBCpf4hWN7OdWHBw58FwIaNOU/n1TpMW2DFaD48cmr4185T8NHkpUFX749pQPVdgRKC/DGoQPVeAEKv+WHvY8OOWNTPRp5kHuwSf8wzXtVBKR7YwEH9H3lQUaypUfSATOALyVNu5vZJW31Bnx98nkLfDUWJaz6ixvm+RIQRdl3kmRxxiaDoGnZW4CpPfkaQadlcPim1xOSvETQo7Lv75enVAXJ3xGUlony4KQBBWUM1NiDc6qhyS8RgQs18OCMMtPDaAUIyg0PZkRWDqs+wnKJBTDI1Js6BolegOsKmUxNDBAAKqQyMQmidhegBlLZ+wwKYdv5M/8x1khkb1cgKqP2H+MKyV5vS+whrE8DQDgAlUAoRBX056EElJCjJVACeJBZgNfVp+iCCm4RBWCgKsRxASSA9KgDhDtCiTuMyfHsKXzhC6wNAIjjWb8LKAOA2ctk3FmCOlgKFy8f1N0JJtgsxinYnVAHt4t3gPzZXSCTyCWCQmBT91QE3B5yarSN40dNHYPka4TlDhTUI8zLvl0JSL3vZn6DsCFZOeB2yROEpR68sECQQA++xIGCR2X7DwlEoLRgUrZrqlUg50S1uy43YqDcN6UFBVkhAjWiCV2Q0jgQPdplMKxvBXodcOfAwJYvgdL+1etA1YJJfBcZlQV7sO1i2gHoNiyxtQ5sBsCgWyoxCHiFFd2L5nUTCqMAqGUgsQ9f5kCcCiZgRYkMgMTd5WsB1rTzj0Em14BE4r+QxN1lCEsVur2PoF5Wbg8RJXR4djgvBgauhLywoEZQrt1KKRdVS4CdlJ8qafyP+9KIj/nE/d7kKwH9jgS72e9DV+kvfTWgct4ZyP8Byb8BPG7MaaIIkAQAAAAASUVORK5CYII=";let j=0;function J(t){let r=Math.cos(f),a=Math.cos(l),o=Math.sin(f),i=Math.sin(l),c=a*t[0]+i*t[2],s=i*o*t[0]+r*t[1]-a*o*t[2];return[(c/(e.width/e.height)*B+T[0]*B*n/e.width+1)/2,(-s*B+T[1]*B*n/e.height+1)/2,-i*r*t[0]+o*t[1]+a*r*t[2]>=0||c*c+s*s>=.64]}function O(e){let t=U(e),r=re+p,a=J([t[0]*r,t[1]*r,t[2]*r]);return{x:a[0],y:a[1],visible:a[2]}}function W(e){let t=U(e.from),r=U(e.to),a=[t[0]+r[0],t[1]+r[1],t[2]+r[2]],o=(a[0]**2+a[1]**2+a[2]**2)**.5;if(o<.001)return null;let i=.25*(re+p)+.5*(re+R+p)/o,n=J([a[0]*i,a[1]*i,a[2]*i]);return{x:n[0],y:n[1],visible:n[2]}}function X(e,t,r,n,l){e<0||(a.enableVertexAttribArray(e),a.vertexAttribPointer(e,t,a.FLOAT,!1,r,n),o?a.vertexAttribDivisor(e,l):i&&i.vertexAttribDivisorANGLE(e,l))}let K,q=document.createElement("div");q.style.cssText="position:relative;width:100%;height:100%",e.parentElement?.insertBefore(q,e),q.append(e);let $=le(q);function ee(t){if(t.phi!=K&&(l=t.phi),t.theta!=K&&(f=t.theta),t.markers&&function(e){c=e;let t=new Float32Array(8*c.length);c.forEach((e,r)=>{t.set([...U(e.location),e.size,...e.color||[0,0,0],e.color?1:0],8*r)}),a.bindBuffer(a.ARRAY_BUFFER,k),a.bufferData(a.ARRAY_BUFFER,t,a.DYNAMIC_DRAW)}(t.markers),t.arcs&&function(e){s=e,j=s.length;let t=new Float32Array(12*s.length);s.forEach((e,r)=>{t.set([...U(e.from),...U(e.to),R+p,.005*E,...e.color||[0,0,0],e.color?1:0],12*r)}),a.bindBuffer(a.ARRAY_BUFFER,S),a.bufferData(a.ARRAY_BUFFER,t,a.DYNAMIC_DRAW)}(t.arcs),t.width&&t.height&&(e.width=t.width*n,e.height=t.height*n),t.mapSamples!=K&&(A=t.mapSamples),t.mapBrightness!=K&&(v=t.mapBrightness),t.mapBaseBrightness!=K&&(d=t.mapBaseBrightness),t.baseColor!=K&&(g=t.baseColor),t.markerColor!=K&&(u=t.markerColor),t.glowColor!=K&&(m=t.glowColor),t.arcColor!=K&&(h=t.arcColor),t.arcWidth!=K&&(E=t.arcWidth),t.arcHeight!=K&&(R=t.arcHeight),t.diffuse!=K&&(b=t.diffuse),t.dark!=K&&(x=t.dark),t.opacity!=K&&(y=t.opacity),t.offset!=K&&(T=t.offset),t.scale!=K&&(B=t.scale),t.markerElevation!=K&&(p=t.markerElevation),$.m(c,O),$.a(s,W),$.s(),a.viewport(0,0,e.width,e.height),a.clearColor(0,0,0,0),a.clear(a.COLOR_BUFFER_BIT),a.enable(a.BLEND),a.blendFunc(a.SRC_ALPHA,a.ONE_MINUS_SRC_ALPHA),a.useProgram(C),a.bindBuffer(a.ARRAY_BUFFER,D),a.enableVertexAttribArray(Q),a.vertexAttribPointer(Q,2,a.FLOAT,!1,0,0),o?a.vertexAttribDivisor(Q,0):i&&i.vertexAttribDivisorANGLE(Q,0),a.uniform2f(I.t,e.width,e.height),a.uniform2f(I.s,l,f),a.uniform1f(I.k,A),a.uniform1f(I.x,B),a.uniform2f(I.v,T[0]*n,T[1]*n),a.uniform3fv(I.F,g),a.uniform3fv(I.w,m),a.uniform4f(I.n,v,b,x,y),a.uniform1f(I.y,d),a.uniform1i(I.z,0),a.activeTexture(a.TEXTURE0),a.bindTexture(a.TEXTURE_2D,Y),a.drawArrays(a.TRIANGLES,0,6),F&&j>0){a.useProgram(F),a.bindBuffer(a.ARRAY_BUFFER,_),G.k>=0&&(a.enableVertexAttribArray(G.k),a.vertexAttribPointer(G.k,2,a.FLOAT,!1,0,0),o?a.vertexAttribDivisor(G.k,0):i&&i.vertexAttribDivisorANGLE(G.k,0)),a.bindBuffer(a.ARRAY_BUFFER,S);let t=48;if(X(G.l,3,t,0,1),X(G.m,3,t,12,1),X(G.v,1,t,24,1),X(G.w,1,t,28,1),X(G.N,3,t,32,1),X(G.O,1,t,44,1),a.uniform1f(z.y,l),a.uniform1f(z.z,f),a.uniform2f(z.g,e.width,e.height),a.uniform1f(z.h,B),a.uniform2f(z.x,T[0]*n,T[1]*n),a.uniform3fv(z.M,h),a.uniform1f(z.A,p),o)a.drawArraysInstanced(a.TRIANGLE_STRIP,0,66,j);else if(i)i.drawArraysInstancedANGLE(a.TRIANGLE_STRIP,0,66,j);else for(let e=0;e0){a.useProgram(w),a.bindBuffer(a.ARRAY_BUFFER,D),L.n>=0&&(a.enableVertexAttribArray(L.n),a.vertexAttribPointer(L.n,2,a.FLOAT,!1,0,0),o?a.vertexAttribDivisor(L.n,0):i&&i.vertexAttribDivisorANGLE(L.n,0)),a.bindBuffer(a.ARRAY_BUFFER,k);let t=32;X(L.p,3,t,0,1),X(L.q,1,t,12,1),X(L.w,3,t,16,1),X(L.x,1,t,28,1),a.uniform1f(P.i,l),a.uniform1f(P.j,f),a.uniform2f(P.b,e.width,e.height),a.uniform1f(P.k,B),a.uniform2f(P.r,T[0]*n,T[1]*n),a.uniform3fv(P.v,u),a.uniform1f(P.s,p),function(e){if(o)a.drawArraysInstanced(a.TRIANGLES,0,6,e);else if(i)i.drawArraysInstancedANGLE(a.TRIANGLES,0,6,e);else for(let t=0;t{M.onload=null,a.deleteTexture(Y),a.deleteBuffer(D),a.deleteBuffer(_),a.deleteBuffer(k),a.deleteBuffer(S),a.deleteProgram(C),w&&a.deleteProgram(w),F&&a.deleteProgram(F),$.r()}}};export{Pe as default}; diff --git a/web/patches/svelte@5.57.2.patch b/web/patches/svelte@5.57.2.patch new file mode 100644 index 0000000000000000000000000000000000000000..f888d8476c4c3460c38bfed7799c46393aeff10a --- /dev/null +++ b/web/patches/svelte@5.57.2.patch @@ -0,0 +1,38 @@ +diff --git a/src/internal/client/loop.js b/src/internal/client/loop.js +index 9c1e972800fdb2c5cce510a03df7c3373dc86541..e5bbe981cc4839cb47726ab0ed5ca46c5cbd4e15 100644 +--- a/src/internal/client/loop.js ++++ b/src/internal/client/loop.js +@@ -6,7 +6,12 @@ import { raf } from './timing.js'; + /** + * @returns {void} + */ ++/** Whether a `run_tasks` frame is already scheduled */ ++let scheduled = false; ++ + function run_tasks() { ++ scheduled = false; ++ + // use `raf.now()` instead of the `requestAnimationFrame` callback argument, because + // otherwise things can get wonky https://github.com/sveltejs/svelte/pull/14541 + const now = raf.now(); +@@ -19,6 +24,7 @@ function run_tasks() { + }); + + if (raf.tasks.size !== 0) { ++ scheduled = true; + raf.tick(run_tasks); + } + } +@@ -33,7 +39,11 @@ export function loop(callback) { + /** @type {TaskEntry} */ + let task; + +- if (raf.tasks.size === 0) { ++ // Only schedule a frame if one isn't already pending: an aborted task can empty the ++ // set while a frame is still scheduled, and scheduling another would run every task ++ // twice per frame from then on (and again for every abort after that) ++ if (!scheduled) { ++ scheduled = true; + raf.tick(run_tasks); + } + diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aeb2b995b81c627e52f98146b6616efe5a037d89 --- /dev/null +++ b/web/pnpm-lock.yaml @@ -0,0 +1,6753 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +overrides: + aria-query: 5.3.2 + '@storybook/svelte': 10.6.1 + +patchedDependencies: + '@pierre/diffs@1.6.1': 3e5f4b454dc1b27920ac06ec6b2a03eeac337333cebd93b896f01f7c3bcfcdf6 + '@sveltejs/adapter-bun@1.0.0': 7a4161ba8d719c64682bae85cfec228d726017b4217ca64b3758527625d972fb + boneyard-js@1.9.0: e9bda6b0720bed34eefee605bd041784ac2849ae4d27120d9815663d342f694f + cobe@2.0.1: 3b6a4b0984026f2dc3a5794e2542b83b483af86b03ee6531f396e9d6fd62e62d + svelte@5.57.2: ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117 + +importers: + + .: + dependencies: + '@atcute/atproto': + specifier: ^4.0.3 + version: 4.0.3(@atcute/lexicons@2.0.2) + '@atcute/cbor': + specifier: ^2.3.6 + version: 2.3.6(@atcute/cid@2.4.2) + '@atcute/client': + specifier: ^5.1.1 + version: 5.1.1(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/identity-resolver': + specifier: ^2.0.1 + version: 2.0.1(@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/lexicons': + specifier: ^2.0.2 + version: 2.0.2 + '@atcute/microcosm': + specifier: ^2.0.2 + version: 2.0.2(@atcute/lexicons@2.0.2) + '@atcute/oauth-browser-client': + specifier: ^4.0.1 + version: 4.0.1(@atcute/identity-resolver@2.0.1(@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/repo': + specifier: ^1.0.2 + version: 1.0.2(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2)(@atcute/lexicons@2.0.2) + '@atcute/tid': + specifier: ^1.1.4 + version: 1.1.4 + '@mdit/plugin-alert': + specifier: ^1.0.1 + version: 1.0.1(markdown-it@14.3.2) + '@mdit/plugin-footnote': + specifier: ^1.0.1 + version: 1.0.1(markdown-it@14.3.2) + '@mdit/plugin-tasklist': + specifier: ^1.0.1 + version: 1.0.1(markdown-it@14.3.2) + '@pierre/diffs': + specifier: ^1.6.1 + version: 1.6.1(patch_hash=3e5f4b454dc1b27920ac06ec6b2a03eeac337333cebd93b896f01f7c3bcfcdf6)(@shikijs/themes@4.3.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) + '@tanstack/svelte-virtual': + specifier: ^3.13.35 + version: 3.13.35(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)) + ansi_up: + specifier: ^6.0.6 + version: 6.0.6 + boneyard-js: + specifier: 1.9.0 + version: 1.9.0(patch_hash=e9bda6b0720bed34eefee605bd041784ac2849ae4d27120d9815663d342f694f)(react@19.2.8)(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + cobe: + specifier: 2.0.1 + version: 2.0.1(patch_hash=3b6a4b0984026f2dc3a5794e2542b83b483af86b03ee6531f396e9d6fd62e62d) + cuelume: + specifier: 0.2.4 + version: 0.2.4 + feed: + specifier: ^6.0.0 + version: 6.0.0 + fuse.js: + specifier: ^7.5.0 + version: 7.5.0 + markdown-it: + specifier: ^14.3.1 + version: 14.3.2 + markdown-it-anchor: + specifier: ^9.2.1 + version: 9.2.1(@types/markdown-it@14.1.2)(markdown-it@14.3.2) + markdown-it-emoji: + specifier: ^3.1.0 + version: 3.1.0 + pele: + specifier: ^1.5.2 + version: 1.5.2 + sanitize-html: + specifier: ^2.17.7 + version: 2.18.1 + svg-pathdata: + specifier: 9.0.0 + version: 9.0.0 + tailwind-merge: + specifier: ^3.6.0 + version: 3.6.0 + tailwind-variants: + specifier: ^3.2.2 + version: 3.2.2(tailwind-merge@3.6.0)(tailwindcss@4.3.3) + textarea-caret: + specifier: ^3.1.0 + version: 3.1.0 + undici: + specifier: ^7.29.0 + version: 7.29.0 + devDependencies: + '@atcute/car': + specifier: ^6.0.2 + version: 6.0.2(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2) + '@atcute/cid': + specifier: ^2.4.2 + version: 2.4.2 + '@atcute/lex-cli': + specifier: 3.2.1 + version: 3.2.1(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2)(prettier@3.9.5)(typescript@6.0.3) + '@atcute/mst': + specifier: ^1.0.2 + version: 1.0.2(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2) + '@eslint/js': + specifier: ^10.0.1 + version: 10.0.1(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0)) + '@iconify-json/lucide': + specifier: ^1.2.134 + version: 1.2.134 + '@storybook/addon-a11y': + specifier: ^10.6.5 + version: 10.6.5(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8)) + '@storybook/addon-docs': + specifier: ^10.6.5 + version: 10.6.5(@types/react-dom@19.2.4(@types/react@19.2.18))(@types/react@19.2.18)(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8)) + '@storybook/addon-svelte-csf': + specifier: ^5.1.6 + version: 5.1.6(@storybook/svelte@10.6.1(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)))(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + '@storybook/addon-themes': + specifier: ^10.6.5 + version: 10.6.5(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8)) + '@storybook/addon-vitest': + specifier: ^10.6.5 + version: 10.6.5(@vitest/browser-playwright@5.0.4)(@vitest/browser@5.0.4)(react@19.2.8)(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(vitest@5.0.4) + '@storybook/sveltekit': + specifier: ^10.6.5 + version: 10.6.5(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + '@sveltejs/adapter-bun': + specifier: ^1.0.0 + version: 1.0.0(patch_hash=7a4161ba8d719c64682bae85cfec228d726017b4217ca64b3758527625d972fb)(@sveltejs/kit@3.1.0(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@6.0.3)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))) + '@sveltejs/adapter-cloudflare': + specifier: ^8.0.0 + version: 8.0.0(@sveltejs/kit@3.1.0(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@6.0.3)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(wrangler@4.125.0(@cloudflare/workers-types@5.20261012.1)) + '@sveltejs/kit': + specifier: ^3.1.0 + version: 3.1.0(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@6.0.3)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + '@sveltejs/vite-plugin-svelte': + specifier: ^7.3.1 + version: 7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + '@tailwindcss/vite': + specifier: ^4.3.3 + version: 4.3.3(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + '@types/markdown-it': + specifier: ^14.1.2 + version: 14.1.2 + '@types/markdown-it-emoji': + specifier: ^3.0.1 + version: 3.0.1 + '@types/node': + specifier: ^24.13.3 + version: 24.13.3 + '@types/sanitize-html': + specifier: ^2.16.1 + version: 2.16.1 + '@types/textarea-caret': + specifier: ^3.0.4 + version: 3.0.4 + '@vitest/browser-playwright': + specifier: 5.0.4 + version: 5.0.4(playwright@1.61.1)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))(vitest@5.0.4) + '@vitest/coverage-v8': + specifier: 5.0.4 + version: 5.0.4(@vitest/browser@5.0.4)(vitest@5.0.4) + eslint: + specifier: ^10.7.0 + version: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + eslint-config-prettier: + specifier: ^10.1.8 + version: 10.1.8(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0)) + eslint-plugin-better-tailwindcss: + specifier: ^4.7.0 + version: 4.7.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(tailwindcss@4.3.3)(typescript@6.0.3) + eslint-plugin-storybook: + specifier: ^10.6.5 + version: 10.6.5(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3) + eslint-plugin-svelte: + specifier: ^3.23.1 + version: 3.23.1(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)) + globals: + specifier: ^17.7.0 + version: 17.7.0 + playwright: + specifier: ^1.61.1 + version: 1.61.1 + prettier: + specifier: ^3.9.5 + version: 3.9.5 + prettier-plugin-svelte: + specifier: ^4.1.1 + version: 4.1.1(prettier@3.9.5)(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)) + prettier-plugin-tailwindcss: + specifier: ^0.8.1 + version: 0.8.1(prettier-plugin-svelte@4.1.1(prettier@3.9.5)(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)))(prettier@3.9.5) + react: + specifier: ^19.2.8 + version: 19.2.8 + react-dom: + specifier: ^19.2.8 + version: 19.2.8(react@19.2.8) + storybook: + specifier: ^10.6.5 + version: 10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8) + svelte: + specifier: ^5.57.2 + version: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + svelte-check: + specifier: ^4.7.6 + version: 4.7.6(picomatch@4.0.7)(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@6.0.3) + tailwindcss: + specifier: ^4.3.3 + version: 4.3.3 + tw-easing-gradients: + specifier: ^1.1.2 + version: 1.1.2(tailwindcss@4.3.3) + typescript: + specifier: ^6.0.3 + version: 6.0.3 + typescript-eslint: + specifier: ^8.64.0 + version: 8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3) + unplugin-icons: + specifier: ^24.0.0 + version: 24.0.0(esbuild@0.28.1)(rolldown@1.2.13)(rollup@4.62.2)(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + vite: + specifier: ^8.3.4 + version: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^5.0.4 + version: 5.0.4(@types/node@24.13.3)(@vitest/browser-playwright@5.0.4)(@vitest/coverage-v8@5.0.4)(@vitest/ui@5.0.4)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + wrangler: + specifier: ^4.125.0 + version: 4.125.0(@cloudflare/workers-types@5.20261012.1) + +packages: + + '@adobe/css-tools@4.5.0': + resolution: {integrity: sha512-6OzddxPio9UiWTCemp4N8cYLV2ZN1ncRnV1cVGtve7dhPOtRkleRyx32GQCYSwDYgaHU3USMm84tNsvKzRCa1Q==} + + '@antfu/install-pkg@1.1.0': + resolution: {integrity: sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==} + + '@antfu/install-pkg@2.1.0': + resolution: {integrity: sha512-sdg9NxU3zR4Mnawfbc/x6GB5Wf17WYud5qOuEuxXjaKpYpMkISSJEjItGebXJ2bQ4DIcly4NYH23mtkGJjvKUw==} + + '@atcute/atproto@4.0.3': + resolution: {integrity: sha512-BNylfO7nK0yYBpSpnGhOYgrJTeZWrXHPrb6tOQmp9A3Am0epctIWm6/5lPC4ZNPHpUbwr5w/LzH/v7kjAoKEDg==} + peerDependencies: + '@atcute/lexicons': ^2.0.0 + + '@atcute/car@6.0.2': + resolution: {integrity: sha512-6AaLjO0zrFD8R/aK7jwrqHEmLzfVilu/5pv4LAcUjxIYfBrw+nThV2FVLPtA4Jt59tIhjp5tQDP+9sx1eqQp+A==} + peerDependencies: + '@atcute/cbor': ^2.0.0 + '@atcute/cid': ^2.0.0 + + '@atcute/cbor@2.3.6': + resolution: {integrity: sha512-Ml/G8i6KK5UWSx1kxgm41eTYm1EQ6O2fNWSXMJ5DxoQv/lmZ0Hmk7dmHda2seRE+Cj2X3HJKAuD9cQRh3m2P7A==} + peerDependencies: + '@atcute/cid': ^2.0.0 + + '@atcute/cid@2.4.2': + resolution: {integrity: sha512-Uy48yfyo/hPQXF+XMXWIGomF6v8IvX5ErjozXMV7rlfU3EV7PSVX3J7plJXV6MRC3iI1z3PgTZTS7V0drCQVVw==} + + '@atcute/client@5.1.1': + resolution: {integrity: sha512-cn5/Zi/qo37WtQG6gzIC7JPs0RDzX9Z4eaceX45SpKgLZoc3fCFDJcE7C8xsbxBNfjry2T6PmUxWA8obebZsEQ==} + peerDependencies: + '@atcute/lexicons': ^2.0.0 + + '@atcute/crypto@2.4.2': + resolution: {integrity: sha512-WKVVstCsgGZaaEJFemC6UPh5MtEB2SMU0tiOi744i9T2a6oInJg3VnbTzQsGkyQqqTNC30IHF7//Dwi4hYnu1g==} + + '@atcute/identity-resolver@2.0.1': + resolution: {integrity: sha512-0enA9w7XnbbqsZ5Rcl6jXLf7ZZuwFQ9dBmxFq3qOxPHLaCETsqsrQflXDPqiM27TnZwYq8sqCV5D1mFOksggDQ==} + peerDependencies: + '@atcute/identity': ^2.0.0 + '@atcute/lexicons': ^2.0.0 + + '@atcute/identity@2.0.1': + resolution: {integrity: sha512-FEURUvl30SyyWWikkvm+MLz0Snuf0OF10L/qxRhWjj6qDB5Ib+XWhiBuwidjvhCkrCepTUNLbj4TlUm/gHaUig==} + peerDependencies: + '@atcute/lexicons': ^2.0.0 + + '@atcute/lex-cli@3.2.1': + resolution: {integrity: sha512-uKoQfCdrsdRJQApfe4ENpMrWlRFwd6uTmI5+iJ8cIwv4pUQU13ILwwnFqJE4FqTHzCBGk9Dey4SCiGdQSBvRaw==} + hasBin: true + peerDependencies: + prettier: ^3.8.4 + peerDependenciesMeta: + prettier: + optional: true + + '@atcute/lexicon-doc@3.0.1': + resolution: {integrity: sha512-giKTzRo9AqoX8GQMN7vvcaWSC4iHoK9CwA5TZpGD7KQXxJNBrK270QrA6+Ad53yxaqmTjay+3qm+EOux2G7m0A==} + peerDependencies: + '@atcute/lexicons': ^2.0.0 + + '@atcute/lexicon-resolver@1.0.1': + resolution: {integrity: sha512-/IXEGEMB3uyYqWSNxsH1V2QIEO/d8TdLeSbdqTv75Bqd3aZqf2Q1Fx/asOE1P9bE0Xz0twnqxW3qQvy2SHGG8Q==} + peerDependencies: + '@atcute/identity': ^2.0.0 + '@atcute/identity-resolver': ^2.0.0 + '@atcute/lexicon-doc': ^3.0.0 + '@atcute/lexicons': ^2.0.0 + + '@atcute/lexicons@2.0.2': + resolution: {integrity: sha512-ATBADJAy4KQ76NB86BjgYKrRdbDRUo76Cbqna4WIfQAgN105Rcy972MiNKs+BSmcOOM3WakilgTm0CXD4RC0iA==} + + '@atcute/microcosm@2.0.2': + resolution: {integrity: sha512-cmE5UWM1sZKSQD5b0gBD3+9FF3HtWCp1aCpP1BCUACeHor2eUlkKW9/YDVzCSTK54wVt3hlRbihiHBIgn4g/Ig==} + peerDependencies: + '@atcute/lexicons': ^2.0.0 + + '@atcute/mst@1.0.2': + resolution: {integrity: sha512-fb98Ygs0PjVAOE4WmtMGqi+PS4Y0Jsip4CIkzWBxyIcbfix8kBGQUwZi54k4rKiGfT9qroCY2rlaPFzwkkAFzQ==} + peerDependencies: + '@atcute/cbor': ^2.0.0 + '@atcute/cid': ^2.0.0 + + '@atcute/multibase@1.2.4': + resolution: {integrity: sha512-WeX12hvFZEim6C+cyv7Eqd93w6DzubNWQGmTFBghjsEuXvMe4HbBCYvsti0OUnbA5qLBPlsTyssQUJeLlHCzIw==} + + '@atcute/multibase@1.2.5': + resolution: {integrity: sha512-cReTONgYpQo/VHD3ZmzPNoyBKJgSk1J4h//cvvdVVJBMar+SjlQ/sUXeTjQfuyfmDKv+TLKhmutLcvbMcQ9Rvw==} + + '@atcute/oauth-browser-client@4.0.1': + resolution: {integrity: sha512-k5GGDFWUqTifbi7bMk4LhwG12XWNJop9atZ3ocBpzHJvXboTqp7E2zaD/arhJWxHXp5+0A1R3EKXdHsVbWjHyA==} + peerDependencies: + '@atcute/identity-resolver': ^2.0.0 + '@atcute/lexicons': ^2.0.0 + + '@atcute/oauth-crypto@1.0.1': + resolution: {integrity: sha512-ghC8ceFx0r6pi5X3dNoET1K6PNg8afeAOZLD4PKLhrpVYdRtRrgY4uqAe3ojtSIjoS6GXmVm063CoJ2xtg/MoQ==} + + '@atcute/oauth-keyset@0.1.2': + resolution: {integrity: sha512-2CUazWBWRjSEoYZDaLS4zEPgCOcr9LgnMhPr1pzr9I+DytRnoXmrdbRs4xBVg4iUdXtdzU48DsMaGi6bHXaFcg==} + + '@atcute/oauth-types@1.0.1': + resolution: {integrity: sha512-9uFP/nK0PgPDsaNbLArzrCHvTZO1k04cSEX5jZ75W2Jh7Jxbh/CSTenSrad3CZXuX6y6hDquEtl6H/IqLtTmHw==} + + '@atcute/repo@1.0.2': + resolution: {integrity: sha512-fsAuGbagOW52nSFtf/TWBx25A6xpZvzxBk0+S14Bp5sGNq7JoQ0zyu40P7b6+L8b/buB+1ACHPntYsKqCI4Awg==} + peerDependencies: + '@atcute/cbor': ^2.0.0 + '@atcute/cid': ^2.0.0 + '@atcute/lexicons': ^2.0.0 + + '@atcute/tid@1.1.4': + resolution: {integrity: sha512-RVjGAxu+WgcQjs7AkicZKOqLOncnuXh7a0xUHTfXTYZjXSszB7jBLgrcLWjpU0mBTko9dT+GajsXAcBGYmzYKQ==} + + '@atcute/time-ms@1.3.3': + resolution: {integrity: sha512-FeK2Xsv4ymP+PoTWtmMFuJmpxDBQsj+jNMnd/YpIm3etcIPj1YFb7DeTAG1c/Xvp8AM38MLdqUCc35/mANnBtw==} + + '@atcute/uint8array@1.1.4': + resolution: {integrity: sha512-rSW5AFVCIN4ooH7vEZB+J60+uWjn5fRBQAQL58qHLiDm8+xDPmHfEU5GfYOJuuD+7UBj8KKiQugIzohFn8/xPw==} + + '@atcute/uint8array@1.1.5': + resolution: {integrity: sha512-1SFCXOtjE3ismP92CqzbOfYSwJmPPPFbj798wLXSpLv2hSd5OIaUrTh6EqRVR1VZ3ZhR+vaBmd3kxAN0wkw+gQ==} + + '@atcute/util-fetch@2.0.1': + resolution: {integrity: sha512-ugWTOLemA8OxSOj7c8q6ncRmBGFDHSwwE1YinO+PCtaw6WLQFGBfHn+yikQ0e3wTK2t4IPjQ5PxZcRXm961ZVA==} + + '@atcute/util-text@1.3.3': + resolution: {integrity: sha512-WhedTmg/msFhrdwXw9RjnNcDl8Vmisxl4+Vzyf5k3+8Gj5TKQg72dLSDtBNmNLd61RbHjgfQRBgE0ez6q/jciw==} + + '@atcute/varint@2.0.1': + resolution: {integrity: sha512-reTtgQ1VPGVRcaTohKVb1tUbpvE7MpguoNS0gjhsxDWFY3t1yqWWvwKK6cJodAmePML36JombvgFwfsblir9Jg==} + + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-string-parser@7.29.7': + resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.29.7': + resolution: {integrity: sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==} + engines: {node: '>=6.0.0'} + hasBin: true + + '@babel/runtime@7.29.7': + resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} + engines: {node: '>=6.9.0'} + + '@babel/types@7.29.7': + resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==} + engines: {node: '>=6.9.0'} + + '@bcoe/v8-coverage@1.0.2': + resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} + engines: {node: '>=18'} + + '@blazediff/core@1.10.0': + resolution: {integrity: sha512-AOQff0zgR7cGsZL+4E7hVkmujoPUpm0J9xzWGWZj5wCjd3gmxESXAPfKyuzs93VdpQNFhHlBhfOjrcZ+XTERtQ==} + + '@chenglou/pretext@0.0.5': + resolution: {integrity: sha512-A8GZN10REdFGsyuiUgLV8jjPDDFMg5GmgxGWV0I3igxBOnzj+jgz2VMmVD7g+SFyoctfeqHFxbNatKSzVRWtRg==} + + '@cloudflare/kv-asset-handler@0.5.0': + resolution: {integrity: sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==} + engines: {node: '>=22.0.0'} + + '@cloudflare/unenv-preset@2.16.1': + resolution: {integrity: sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==} + peerDependencies: + unenv: 2.0.0-rc.24 + workerd: '>1.20260305.0 <2.0.0-0' + peerDependenciesMeta: + workerd: + optional: true + + '@cloudflare/workerd-darwin-64@1.20260820.1': + resolution: {integrity: sha512-5F2/t7SVnugG3rscSe9da1LoHst+GiuVGPaE9BP6j5AonlFpiYi0eoJrl3wof9zDBIIgJZ87NjRj9TKjbbYgHg==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@cloudflare/workerd-darwin-arm64@1.20260820.1': + resolution: {integrity: sha512-jFnG7715+r9FXRZPpuWWe5Ayd9v/IJKDODARg56ffFJWWtte6bFNi5VY3GazBM04hEmxny6OjXQBh3j2E/wNZw==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@cloudflare/workerd-linux-64@1.20260820.1': + resolution: {integrity: sha512-TbTYaCBht0OaOWmnVpg43hXVYtIti/Kg6lvO819H2DwbxPpK1BH0z2C+Y7ySrVJLlxZQeic5eN7/noqbP80hJg==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@cloudflare/workerd-linux-arm64@1.20260820.1': + resolution: {integrity: sha512-FQmri1UF7hBnnpeyC5SZJCAnsSRs3+Ykn9wlO5zxmE2qIgKfbJZ+toJ6qrQyugWlxE65juT33HU6aYLtutLJHw==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@cloudflare/workerd-windows-64@1.20260820.1': + resolution: {integrity: sha512-BPvCuMxIQfA47wtYsGbBG6Bcar57Qs7yHqU2jWkT1+sRnL/741/ZbQGP9kVRMQ5fwBMuqNFmQRzAu8gSm7ri/w==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + + '@cloudflare/workers-types@5.20261012.1': + resolution: {integrity: sha512-5Y0NrKXG9CZaNUZ6Dt3TLRE4UQe2/DGuxIhTEVYHUl0jAlv1TpQtI59YBAszRpSaZAMzbM8nqGc1n9Z2q4TVyA==} + + '@cspotcode/source-map-support@0.8.1': + resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} + engines: {node: '>=12'} + + '@emnapi/core@1.11.0': + resolution: {integrity: sha512-l9Oo58x0HOP5znGzVhYW9U3e5wVuA4LAZU2AGezTmkhO1CgQRFDhDg4nneHsu/t3WniXg9QrG2nIXL/ZS8ln8Q==} + + '@emnapi/core@1.9.2': + resolution: {integrity: sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==} + + '@emnapi/runtime@1.11.0': + resolution: {integrity: sha512-55coeOFKHv1ywEcUXJtWU5f+Jr/W5tZDvZig8DLKSwUN1JpROQ4rk/SNOQiFWmaR/VKF4zuFyW1B8JduOSv6Pg==} + + '@emnapi/runtime@1.11.2': + resolution: {integrity: sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==} + + '@emnapi/runtime@1.9.2': + resolution: {integrity: sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==} + + '@emnapi/wasi-threads@1.2.1': + resolution: {integrity: sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==} + + '@emnapi/wasi-threads@1.2.2': + resolution: {integrity: sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==} + + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@eslint-community/eslint-utils@4.9.1': + resolution: {integrity: sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + + '@eslint-community/regexpp@4.12.2': + resolution: {integrity: sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + + '@eslint/config-array@0.23.5': + resolution: {integrity: sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/config-helpers@0.6.0': + resolution: {integrity: sha512-ii6Bw9jJ2zi2cWA2Z+9/QZ/+3DX6kwaV5Q986D/CdP3Lap3w/pgQZ373FV7byY/i7L4IRH/G43I5dz1ClsCbpA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/core@1.2.1': + resolution: {integrity: sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/css-tree@4.0.5': + resolution: {integrity: sha512-iPmijIAq4hlIJB86PYmY/fcZORHtjphSqICDbwuw32A/JmkhZQ/K/6TjHE03zqf3n5yABpVcbRAMG8Mi9ojy8g==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/js@10.0.1': + resolution: {integrity: sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + peerDependencies: + eslint: ^10.0.0 + peerDependenciesMeta: + eslint: + optional: true + + '@eslint/object-schema@3.0.5': + resolution: {integrity: sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/plugin-kit@0.7.2': + resolution: {integrity: sha512-+CNAzxglkrpNf/kKywqQfk74QjtceuOE7Qm+AF8miRvPF/wmmK5+OJOgVh3AVTT3RP2mH3+FOaxlE5v72owk0A==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@humanfs/core@0.19.2': + resolution: {integrity: sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==} + engines: {node: '>=18.18.0'} + + '@humanfs/node@0.16.8': + resolution: {integrity: sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==} + engines: {node: '>=18.18.0'} + + '@humanfs/types@0.15.0': + resolution: {integrity: sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==} + engines: {node: '>=18.18.0'} + + '@humanwhocodes/module-importer@1.0.1': + resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} + engines: {node: '>=12.22'} + + '@humanwhocodes/retry@0.4.3': + resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} + engines: {node: '>=18.18'} + + '@iconify-json/lucide@1.2.134': + resolution: {integrity: sha512-MXzwT7H8Ii/aGN6NLtMbMLDaxLYsOxCKEdW0F5h3MemNUaUBqAxr9nz0mPlXzozd/piQyrdSiUyk0t5b5ErADg==} + + '@iconify/types@2.0.0': + resolution: {integrity: sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==} + + '@iconify/utils@3.1.7': + resolution: {integrity: sha512-JZHlwdID+dy+lTgbYC8NEC4zeugqeYsc6jewvzb4c58kHauJn+X7rNwQjxz5p2qSjqaEeQoLkCIQ9v/H4PK0/w==} + + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.35.2': + resolution: {integrity: sha512-eEieHsMksAW4IiO5NzauESRl2D2qz3J/kwUxUrSfV06A93eEaRfMpHXyUb1mAqrR7i8U9A0GRqE9pjn6u1Jjpg==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.35.2': + resolution: {integrity: sha512-BaktuGPCeHJMARpodR8jK4uKiZrPAy9WrfQW0sdI37clracq8Bp01AYS3SZgi5FS/y5twa9t4+LIuuxQjqRrWw==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [darwin] + + '@img/sharp-freebsd-wasm32@0.35.2': + resolution: {integrity: sha512-YoAxdnd8hPUkvLHd3bWY+YA8nw3xM/RyRopYucNsWHVSan8NLVM3X2volsfoRDcXdUJPg6tXahSd7HXPK7lRnw==} + engines: {node: '>=20.9.0'} + os: [freebsd] + + '@img/sharp-libvips-darwin-arm64@1.3.1': + resolution: {integrity: sha512-4V/M3roRMTYjiwZY9IOVQOE8OyeCxFAkYmyZDrZl51uOKjibm3oeEJ4WAmLxutAfzFbC9jqUiPs2gbnGflH+7g==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.3.1': + resolution: {integrity: sha512-c0/DxItpJv2+dGhgycJBBgotdqruGYDvA79drdh0MD1dFpy7JzJ/PlXwi1H4rFf0eTy8tgbI91aHDnZIceY3jQ==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.3.1': + resolution: {integrity: sha512-JznefmcK9j1JKPz8AkQDh89kjojubyfOasWBPKfzMIhPwsgDy9evpE/naJTXXXmghS1iFwR8u/kTwh/I2/+GCw==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-arm@1.3.1': + resolution: {integrity: sha512-aGGy9aWzXgHBG7HNyQPWorZthlp7+x6fDRoPAQbGO3ThcttuTyKIx3NuSHb6zb4gBNq6/yNn9f1cy9nFKS/Vmg==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-ppc64@1.3.1': + resolution: {integrity: sha512-1EkwGNCZk6iWNCMWqrvdJ+r1j0PT1zIz60CNPhYnJlK/zyeWqlsPZIe+ocBVqPF8k/Ssee/NCk+tE9Ryrko6ng==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-riscv64@1.3.1': + resolution: {integrity: sha512-Ilays+w2bXdnxzxtQdmXR62u8o8GYa3eL4+Gr+1KiE4xperMZUslRaVPJwwPkzlHEjGfXAfRVAa/7CYCtSqsBw==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-s390x@1.3.1': + resolution: {integrity: sha512-VfBwVHQTbRoj4XlpA/KLZ7ltgMpz+4WSejFzQ+GnoImjo1PtEJ59QB2qR1xQEeRPYIkNrPIm2L4cICMvz4C2ew==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-x64@1.3.1': + resolution: {integrity: sha512-+c8ukgwU62DS54nCAjw7keOfHUkmr0B5QHEdcOqRnodF/MNXJbVI8Eopoj4B/0H8Asr65I+A4Amrn7a85/md6A==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linuxmusl-arm64@1.3.1': + resolution: {integrity: sha512-qlKb/pwbkAi1WMsJrYHk7CuDrd12s27U2QnRhFYUoJNrRCmkosMTttuRFat/DDB3IlDm5qE1TJgZ4JDnHX8Ldw==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-libvips-linuxmusl-x64@1.3.1': + resolution: {integrity: sha512-yO21HwoUVLN8Qa+/SBjQLMYwBWAVJjeGPNe+hc0OUeMeifEtJqu5a1c4HayE1nNpDih9y3/KkoltfkDodmKAlg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-linux-arm64@0.35.2': + resolution: {integrity: sha512-af12Pnd0ZGu2HfP8NayB0kk6eC/lrfbQE6HlR4jD+34wdJ1Vw9TF6TMn6ZvffT+WgqVsl0hRbmNvz2u/23VmwA==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-arm@0.35.2': + resolution: {integrity: sha512-SE4kzF2mepn6z+6E7L6lsV8FzuLL6IPQdyX8ZiwROAG/G8td+hP/m7FsFPwidtrF19gvajuC9l6TxAVcsA4S7A==} + engines: {node: '>=20.9.0'} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-ppc64@0.35.2': + resolution: {integrity: sha512-hYSBm7zcNtDCozCxQHYZJiu63b/bXsgRZuOxCIBZsStMM9Vap47iFHdbX4kCvQsblPB/k+clhELpdQJHQLSHvg==} + engines: {node: '>=20.9.0'} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-riscv64@0.35.2': + resolution: {integrity: sha512-qQt0Kc13+Hoan/Awq/qMSQw3L+RI1NCRPgD5cUJ/1WSSmIoysLOc72jlRM3E0OHN9Yr313jgeQ2T+zW+F03QFA==} + engines: {node: '>=20.9.0'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-s390x@0.35.2': + resolution: {integrity: sha512-E4fLLfRPzDLlEeDaTzI98OFLcv++WL5ChLLMwPoVd0CIoZQqupBSNbOisPL5am9XsbQ9T84+iiMpUvbFtkunbA==} + engines: {node: '>=20.9.0'} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-x64@0.35.2': + resolution: {integrity: sha512-gi0zFJJRLswfCZmHtJdikXPOc5u7qamSOS3NHedLqLd4W8Q0NqjdBr6TTRIgsfFjqfTsHFgdfvJ9LwqSgcHiAA==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-linuxmusl-arm64@0.35.2': + resolution: {integrity: sha512-siWbOW1u6HFnFLrp0waKyW7VEf7jYvcDWdrXEFa8AkdAQgEvuu5Fz8/Y70w9EeqAdwDtfU012BhEHHaDqvQNzg==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-linuxmusl-x64@0.35.2': + resolution: {integrity: sha512-YBqMMcjDi4QGYiSn4vNOYBhmlC4z5AXqkOUUqI2e0AFA4urNv4ESgOgwNl3K+4etQhha0twXlzeF20bbULm9Yg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-wasm32@0.35.2': + resolution: {integrity: sha512-Mrv4JQNYVQ94xH+jzZ9r+gowleN8mv2FTgKT+PI6bx5C0G8TdNYndu161pg2i7uoBwxy2ImPMHrJOM2LZef7Bw==} + engines: {node: '>=20.9.0'} + + '@img/sharp-webcontainers-wasm32@0.35.2': + resolution: {integrity: sha512-QNV27pxs9wpApEiCfvHM1RDoP1w1+2KrUWWDPEhEwg+latvOrfuhWrHWZKwdSFwU6jh3myjw/yOCRsUIuOft3g==} + engines: {node: '>=20.9.0'} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.35.2': + resolution: {integrity: sha512-BiVRYc/t6/Vl3e1hBx0hugG4oN9Pydf4fgMSpxTQJmwGUg/YoXTWHiFeRymHfCZzifxu4F4rpk/I67D0LQ20wQ==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.35.2': + resolution: {integrity: sha512-YYEhx9PImCC7T0tI8JDMi4DB9LwLCXCU5OWNYEXAxh5Q1ShKkyC6byxzoBJ3gEFDnH2lQckWuDe70G7mB2XJog==} + engines: {node: ^20.9.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.35.2': + resolution: {integrity: sha512-imoOyBcoM/iiUr4J6VPpCNjPnjvP/Gks95898yB8YqoGGYmHYbOyCuNv9FMhFgtaiHFGbHW8bxKqRV6VjtXThQ==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [win32] + + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/remapping@2.3.5': + resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@jridgewell/trace-mapping@0.3.9': + resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + + '@mdit/plugin-alert@1.0.1': + resolution: {integrity: sha512-gEZgbwlK2JSNIW3izqQyPoYJ8NXAEU0aON7XYpUHBzyD9km+ny5zFkAmoogMAzR8ird8iCQiK5pSv3nOtPBlLw==} + engines: {node: '>=22'} + peerDependencies: + markdown-it: ^14.2.0 + peerDependenciesMeta: + markdown-it: + optional: true + + '@mdit/plugin-footnote@1.0.1': + resolution: {integrity: sha512-PrH02dlVQT8/vPvfGrLHpcHq8N0+gOaAVNOfuhwlPqWCL4He3o+buUCxm1Uem02+UR8/TRBW50M/XD/dVSe2jw==} + engines: {node: '>=22'} + peerDependencies: + markdown-it: ^14.2.0 + + '@mdit/plugin-tasklist@1.0.1': + resolution: {integrity: sha512-Ks/Tihw5ibbkP2qOZzltkbiTk8RfpxDRvqdn4hl1wIs3VmeKITIQ36gN4DsuotSauA9SBe/mL0QJIfuSCrXqrw==} + engines: {node: '>=22'} + peerDependencies: + markdown-it: ^14.2.0 + peerDependenciesMeta: + markdown-it: + optional: true + + '@mdx-js/react@3.1.1': + resolution: {integrity: sha512-f++rKLQgUVYDAtECQ6fn/is15GkEH9+nZPM3MS0RcxVqoTfawHvDlSCH7JbMhAM6uJ32v3eXLvLmLvjGu7PTQw==} + peerDependencies: + '@types/react': '>=16' + react: '>=16' + + '@napi-rs/wasm-runtime@1.1.6': + resolution: {integrity: sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==} + peerDependencies: + '@emnapi/core': ^1.7.1 + '@emnapi/runtime': ^1.7.1 + + '@noble/secp256k1@3.1.0': + resolution: {integrity: sha512-+F7iS7tUMaNGXcc9X3PjmjvuQnXEuSjCRNzVVA2xAcKXgCaP0dHYz4SFyt4FKNHef7sOP//xihowcySSS7PK9g==} + + '@oomfware/kempt@0.1.4': + resolution: {integrity: sha512-7ik6b58VC17dMX2SZq0LUHaOYupavbtXFA3UxIsAPV2Gc8+HXv9BLJ9Di52zn/8RkDWhoLf0TAcfGYHlnHUiuw==} + + '@optique/core@1.1.1': + resolution: {integrity: sha512-tbZLdH5wEnh8IRG8KTmWPZCKHLmD4bXigoPdzoSUVi+V46qZTNEaTdqtR7EuEwhbPI0KQjLSYRZCxFTdr9OZkA==} + engines: {bun: '>=1.2.0', deno: '>=2.3.0', node: '>=20.0.0'} + + '@optique/run@1.1.1': + resolution: {integrity: sha512-jSZeuiljDEWYkQ67bVDucRknzFVwiSH1j4ho2jFWfcdMMa+WDRSfZA3GZoHA8zKj0Bf6UbJ+FMs/2IReQ0KYNQ==} + engines: {bun: '>=1.2.0', deno: '>=2.3.0', node: '>=20.0.0'} + + '@oxc-parser/binding-android-arm-eabi@0.127.0': + resolution: {integrity: sha512-0LC7ye4hvqbIKxAzThzvswgHLFu2AURKzYLeSVvLdu2TBOYWQDmHnTqPLeA597BcUCxiLqLsS4CJ5uoI5WYWCQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [android] + + '@oxc-parser/binding-android-arm64@0.127.0': + resolution: {integrity: sha512-b5jtVTH6AU5CJXHNdj7Jj9IEiR9yVjjnwHzPJhGyHGPdcsZSzBCkS9GBbV33niRMvKthDwQRFRJfI4a+k4PvYg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@oxc-parser/binding-darwin-arm64@0.127.0': + resolution: {integrity: sha512-obCE8B7ISKkJidjlhv9xRGJPOSDG2Yu6PRga9Ruaz35uintHxbp1Ki/Yc71wx4rj3Edrm0a1kzG1TAwit0wFpg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@oxc-parser/binding-darwin-x64@0.127.0': + resolution: {integrity: sha512-JL6Xb5IwPQT8rUzlpsX7E+AgfcdNklXNPFp8pjCQQ5MQOQo5rtEB2ui+3Hgg9Sn7Y9Egj6YOLLiHhLpdAe12Aw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@oxc-parser/binding-freebsd-x64@0.127.0': + resolution: {integrity: sha512-SDQ/3MQFw58fqQz3Z1PhSKFF3JoCF4gmlNjziDm8X02tTahCw0qJbd7FGPDKw1i4VTBZene9JPyC3mHtSvi+wA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@oxc-parser/binding-linux-arm-gnueabihf@0.127.0': + resolution: {integrity: sha512-Av+D1MIqzV0YMGPT9we2SIZaMKD7Cxs4CvXSx/yxaWHewZjYEjScpOf5igc8IILASViw4WTnjlwUdI1KzVtDHQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@oxc-parser/binding-linux-arm-musleabihf@0.127.0': + resolution: {integrity: sha512-Cs2fdJ8cPpFdeebj6p4dag8A4+56hPvZ0AhQQzlaLswGz1tz7bXt1nETLeorrM9+AMcWFFkqxcXwDGfTVidY8g==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@oxc-parser/binding-linux-arm64-gnu@0.127.0': + resolution: {integrity: sha512-qdOfTcT6SY8gsJrrV92uyEUyjqMGPpIB5JZUG6QN5dukYd+7/j0kX6MwK1DgQj39jtUYixxPiaRUiEN1+0CXgQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-arm64-musl@0.127.0': + resolution: {integrity: sha512-EoTCZneNFU/P2qrpEM+RHmQwt+CvDkyGESG6qhr7KaegXLZwePfbrkCDfAk8/rhxbDUVGsZILX+2tqPzFtoFWA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-linux-ppc64-gnu@0.127.0': + resolution: {integrity: sha512-zALjmZYgxFLHjXeudcDF0xFGNydTAtkAeXAr2EuC17ywCyFxcmQra4w0BMde0Yi/re4Bi4iwEoEXtYN7l6eBLQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-riscv64-gnu@0.127.0': + resolution: {integrity: sha512-fPP8M6zQLS7Jz7o9d5ArUSuAuSK3e+WCYVrCpdzeCOejidtZExJ9tjhDrAd3HEPqARBCPmdpqxESPFqy44vkBQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-riscv64-musl@0.127.0': + resolution: {integrity: sha512-7IcC4Ao02oGpfnjt+X/oF4U2mllo2qoSkw5xxiXNKL9MCTsTiAC6616beOuehdxGcnz1bRoPC1RQ2f1GQDdN+g==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-linux-s390x-gnu@0.127.0': + resolution: {integrity: sha512-pbXIhiNFHoqWeqDNLiJ9JkpHz1IM9k4DXa66x+1GTWMG7iLxtkXgE53iiuKSXwmk3zIYmaPVfBvgcAhS583K4Q==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-x64-gnu@0.127.0': + resolution: {integrity: sha512-MYCguB9RvBvlSd6gbuNI7QwiLoCCAlGnlRJFPrzLI6U1/9wkC/WK6LtBAUln55H1Ctqw45PWmqrobKoMhsYQzQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-x64-musl@0.127.0': + resolution: {integrity: sha512-5eY0B/bxf1xIUxb4NOTvOI3KWtBQfPWYyKAzgcrCt0mDibSZygVpO1Pz8bkeiSZ5Jj9+M09dkggG3H8I5d0Uyg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-openharmony-arm64@0.127.0': + resolution: {integrity: sha512-Gld0ajrFTUXNtdw20fVBuTQx66FA75nIVg+//pPfR3sXkuABB4mTBhl3r9JNzrJpgW//qiwxf0nWXUWGJSL3UQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@oxc-parser/binding-wasm32-wasi@0.127.0': + resolution: {integrity: sha512-T6KVD7rhLzFlwGRXMnxUFfkCZD8FHnb968wVXW1mXzgRFc5RNXOBY2mPPDZ77x5Ln76ltLMgtPg0cOkU1NSrEQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [wasm32] + + '@oxc-parser/binding-win32-arm64-msvc@0.127.0': + resolution: {integrity: sha512-Ujvw4X+LD1CCGULcsQcvb4YNVoBGqt+JHgNNzGGaCImELiZLk477ifUH53gIbE7EKd933NdTi25JWEr9K2HwXw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@oxc-parser/binding-win32-ia32-msvc@0.127.0': + resolution: {integrity: sha512-0cwxKO7KHQQQfo4Uf4B2SQrhgm+cJaP9OvFFhx52Tkg4bezsacu83GB2/In5bC415Ueeym+kXdnge/57rbSfTw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ia32] + os: [win32] + + '@oxc-parser/binding-win32-x64-msvc@0.127.0': + resolution: {integrity: sha512-rOrnSQSCbhI2kowr9XxE7m9a8oQXnBHjnS6j95LxxAnEZ0+Fz20WlRXG4ondQb+ejjt2KOsa65sE6++L6kUd+w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + + '@oxc-project/types@0.127.0': + resolution: {integrity: sha512-aIYXQBo4lCbO4z0R3FHeucQHpF46l2LbMdxRvqvuRuW2OxdnSkcng5B8+K12spgLDj93rtN3+J2Vac/TIO+ciQ==} + + '@oxc-project/types@0.153.0': + resolution: {integrity: sha512-jSNrRIyvPVLL5E1z/tS5zLAprosGz/FuI1PWwEOz59j3E4TTTzSpqKb/RDGqJhhW0QHYISGXxq582wGx6T11Tw==} + + '@oxc-resolver/binding-android-arm-eabi@11.21.2': + resolution: {integrity: sha512-xQoCRv+gKax9KTdwdaQNnAFOai8neay7g3jExDIORzhbrejwGSJaZNTdOJHR5ziLg2joMxOCMOFMo4zuxza2uQ==} + cpu: [arm] + os: [android] + + '@oxc-resolver/binding-android-arm64@11.21.2': + resolution: {integrity: sha512-HF5oiE2L05yInPYCFD/4uxSrEZW4SuIfn99Y6L1xnJnzl066JR+MJs2rIdstw8A2MPlAKH+13dpFPNycjqzvGg==} + cpu: [arm64] + os: [android] + + '@oxc-resolver/binding-darwin-arm64@11.21.2': + resolution: {integrity: sha512-UX4u49CVCAD8QZNELaW8eMGgMAGwFWYEPbvNsh+3r/gs4NX3KfpiACMVwRQT0EuH3uat9hM5Zl+Ppm9pJD8tgg==} + cpu: [arm64] + os: [darwin] + + '@oxc-resolver/binding-darwin-x64@11.21.2': + resolution: {integrity: sha512-J9xPx7YBkrRmJ+xl561ztnMWEc1aOyjEIxBiGX1dVb3u7bGSnfObfcZk+Pd+uM0HZAPNsQ1xvD8j52A/uOSqNQ==} + cpu: [x64] + os: [darwin] + + '@oxc-resolver/binding-freebsd-x64@11.21.2': + resolution: {integrity: sha512-fRlt7OvSaQkWj6+EDTVxawVxOlqJB2QSnBfkeCyK4RTvsGctbw3BiH2Tb7DzMs7bikc4BRBpvWP5zF9K8b54Zg==} + cpu: [x64] + os: [freebsd] + + '@oxc-resolver/binding-linux-arm-gnueabihf@11.21.2': + resolution: {integrity: sha512-gK+vPUcPQITkGwBKpZGrcDHSlU6eDGl7AQacxS2CEKAZIBHWkOVFeJwLZ4tYnA1acJqRM5lt7yYwPCVqGHIJ7A==} + cpu: [arm] + os: [linux] + + '@oxc-resolver/binding-linux-arm-musleabihf@11.21.2': + resolution: {integrity: sha512-L/Rgas7SrOKy/z7IH+HxSFRqVO4PuLDKLEGKvnhoCBBq3UJ0YzGBou3qMzaAGgkJwsIvX2pBF+7ojzfUhLiZxQ==} + cpu: [arm] + os: [linux] + + '@oxc-resolver/binding-linux-arm64-gnu@11.21.2': + resolution: {integrity: sha512-CUEYvlX1Fk7E9kUMzuswru1J9HLxMwnpDeQGjQuI4ZH+iNCoa2X9T+pvyzrbsgl7WnIeTFTlNlHsRfVdf5g9/g==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@oxc-resolver/binding-linux-arm64-musl@11.21.2': + resolution: {integrity: sha512-ViN1ZibQyxwC67GpoP33oo+S9UyUnkog13vzQb9+v9bCNvrVzJuk0MRdacjtv/9xfRMVF/eqHFyl8YOeykI10Q==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@oxc-resolver/binding-linux-ppc64-gnu@11.21.2': + resolution: {integrity: sha512-CU1sCqWnhGqYD5I1HedHk5pujrn7ssDkNB/AEQd/pd3D/EojVSgJUlpbafwIbyhia3PgIfkvdFpRPWSALzVumA==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@oxc-resolver/binding-linux-riscv64-gnu@11.21.2': + resolution: {integrity: sha512-jWVyZtIHca4Gb96x7dag+y69vlei7ffjrsveLkmf2ZhqEAz6ZSBnY1GWvgZUaZlwZP62A3xD092BW97Q5VGc+g==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@oxc-resolver/binding-linux-riscv64-musl@11.21.2': + resolution: {integrity: sha512-LF29obFqNgBUgDX7rmUK7M4D0JQG5LxhYzn3xXmECcHU9aQAdWG7NiY052qybtesEdwHQXKNTWYQ7mTsybNvWg==} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@oxc-resolver/binding-linux-s390x-gnu@11.21.2': + resolution: {integrity: sha512-+NYcm+cCHBbtdQQ3A4phQTSuVRYnNHz7wrl9XRAPEovcdoqi0mb1K5ZOl+jN54ZD+q1zz3V0vltbFJmzecJKmw==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@oxc-resolver/binding-linux-x64-gnu@11.21.2': + resolution: {integrity: sha512-UQqZDdG2r2HhAOsZEgufkIWHPQ886IUyuJQkoZByvzhW8j51R4UNzGBJFkTiTnLhQnggwJRdJgFWK4uY6ZVIMw==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@oxc-resolver/binding-linux-x64-musl@11.21.2': + resolution: {integrity: sha512-3Q9PMRjalWkT6NZ4jfujuqTCFwoWErg3y3BnOgb544B8IMw4PktiwWOigMfOHNRLMghZeJ7hpfpZf4CP7rV7Og==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@oxc-resolver/binding-openharmony-arm64@11.21.2': + resolution: {integrity: sha512-Eljeq3ndtyKhM+Es8LITi4Zl2htzuRZcrPMF3kMCsrILztvU6AjZ3FuEhHHKobPUB9rMpzLpJP5bDqXI7r+iog==} + cpu: [arm64] + os: [openharmony] + + '@oxc-resolver/binding-wasm32-wasi@11.21.2': + resolution: {integrity: sha512-HGDbNsIywqc4LxU38+CTJNnB/6BF7rheWOJ259b4eE0aEnelYblC8x+1tEd63bp31fYne63RQz9Jb4yVnC7Yig==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@oxc-resolver/binding-win32-arm64-msvc@11.21.2': + resolution: {integrity: sha512-iWx25CBEgH49iE9q5coEGI/jb1jl5kkCY9z6U5Og67xCkQ/WFMDc2J5U78+AE91SUxM2NqSLhJC8/PLfWnImww==} + cpu: [arm64] + os: [win32] + + '@oxc-resolver/binding-win32-x64-msvc@11.21.2': + resolution: {integrity: sha512-VPoCAhKvCQTlG7vxqaBXcmuvbh77BfnGXj8g0pbvVXpm1F/R8rDVwqIWcEbMzrI1JvJlm8v7T9uMVQb6UctMRg==} + cpu: [x64] + os: [win32] + + '@pierre/diffs@1.6.1': + resolution: {integrity: sha512-XTfLW4bFETyosZfFNUoyRh1zPE4tmAijOoxdHx4X805tlaq/7/fuQXIiJSm21+jLkDo8vn5elQ+2O4YtMTKM0Q==} + peerDependencies: + react: ^18.3.1 || ^19.0.0 + react-dom: ^18.3.1 || ^19.0.0 + peerDependenciesMeta: + react: + optional: true + react-dom: + optional: true + + '@pierre/highlights@0.2.0': + resolution: {integrity: sha512-hTpcOU1EzDAP4bYtEhxmMu2jwMvtkUydRUBUpWMSHTs3dw0IaC3Ld5ansMy18L0GE3b6Vfl2OGkLK+pWWpnBmw==} + + '@pierre/theme@2.0.0': + resolution: {integrity: sha512-yNDd9GYLQl1mEUJR8AneJ5e4ohLIHQd/wZLWr4fagt78vS2RwwZNW530vVgHqXFAyFVcFlRmGUD5ramXH46OXw==} + engines: {vscode: ^1.0.0} + + '@pierre/theming@1.0.2': + resolution: {integrity: sha512-RmXLLqtx31UGcR8kZyO5hGAQ1EXz6loT1Cu0BFAGsbnBboIxp2BlomkLfWyvdT4WZXnBGyu25onL1gKpFQuSUw==} + peerDependencies: + '@pierre/theme': ^1.1.0 || ^2.0.0 + '@shikijs/themes': ^3.0.0 || ^4.0.0 + react: ^18.3.1 || ^19.0.0 + react-dom: ^18.3.1 || ^19.0.0 + shiki: ^3.0.0 || ^4.0.0 + peerDependenciesMeta: + '@pierre/theme': + optional: true + '@shikijs/themes': + optional: true + react: + optional: true + react-dom: + optional: true + shiki: + optional: true + + '@pkgr/core@0.3.6': + resolution: {integrity: sha512-SEeaJLb3qBNF/OaXnaR1NmmBbFYk1zC0ZH/52fATcRPLFg/p791YrcyFFy44Bo9sLaGuSuLp5Q6axbb/O+v/RA==} + engines: {node: ^14.18.0 || >=16.0.0} + + '@polka/url@1.0.0-next.29': + resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==} + + '@poppinss/colors@4.1.6': + resolution: {integrity: sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==} + + '@poppinss/dumper@0.6.5': + resolution: {integrity: sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==} + + '@poppinss/exception@1.2.3': + resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} + + '@rolldown/binding-android-arm-eabi@1.2.13': + resolution: {integrity: sha512-Vu/NICrgdRvUOeaJZIyB2FP63s6fO/4ud5l8dXmxDx9yDyn+mN0y+gsvsVviSli+T361Aqo0h46EuMoobrwawA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [android] + + '@rolldown/binding-android-arm64@1.2.13': + resolution: {integrity: sha512-6LEoIrGy5vvCd2boNnv2soPSUZj/PRvhp8R/t9KlX+mIZxThIE/ZqxrGXxmuNX0FuvQFvLdZr7S4HrYpYBVhEQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@rolldown/binding-darwin-arm64@1.2.13': + resolution: {integrity: sha512-CaJSRzlQ6Q+zkDMVA9OkVxBjVoPRCDEE3v0FSJ9kIQBgRjFApA3Si4ijhbNVUji7ZJt+PfO0e46qy4nPInHUPA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@rolldown/binding-darwin-x64@1.2.13': + resolution: {integrity: sha512-e4r340I5hxSzJkD3YJBxgzq/rJ3wwxwdLLb2FPt1Zpyem20nrQPT11DDEEkazwxaHumH8wpglsp/Ozgj/wT6mw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@rolldown/binding-freebsd-x64@1.2.13': + resolution: {integrity: sha512-hoBWDZ4/K/mJKVMbsgUd1jfpA2DiOrJ54mpegObU9TBHBBrtX4FLIXJJQRDPZ/2Dn91GvPob+Z2hwKJzf4E5SQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@rolldown/binding-linux-arm-gnueabihf@1.2.13': + resolution: {integrity: sha512-oF9fZu3vTWnTiVAkDFxU9PJLt1nh20K9RRpuHsR7ZnE2L7XookBbDBVLojCVSLGbvtSlxeUNnjAj6DbNsKyzbg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@rolldown/binding-linux-arm64-gnu@1.2.13': + resolution: {integrity: sha512-DTpb/+jMiqWMPgphn4ksW1zW+XLJ6dJJ9FMvnIk9I291LhCctMPhMYJrP5t9p6C9qa3KXadDdUEZHBaPh3kNxQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-arm64-musl@1.2.13': + resolution: {integrity: sha512-xU+Kvv9N2A3SY35zFh1FWWNjxbadE/BKeqnkXe7vScU4GZsw757BlVmDLITZEq8DPpk6m7uoDYkK7P+x/uU5Ow==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rolldown/binding-linux-ppc64-gnu@1.2.13': + resolution: {integrity: sha512-XMk/hnhh25oQ756Lclb1Q11h4ll0xC0Esa5w4Vzp+mAE2S3AGJgeM0SonrIgY8fHhjarb6hcoTuwoFG0VVP2ig==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-s390x-gnu@1.2.13': + resolution: {integrity: sha512-U1BoTHYmlywE2n3HUWjzZ+8EQsKr+OwpxYuDWaubx42tUL8QSL7nOqRz/NVHanibbFP3M1Rxuprz/wNcbMKjqw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-gnu@1.2.13': + resolution: {integrity: sha512-J4cm80Pc6wEwV5AZ42o62sIQgDk+Y7R2s646jIwuvO+4V46QB58N4vVuMPWCBaNgursXLpmUWJV//WQAkPvHBQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-musl@1.2.13': + resolution: {integrity: sha512-W0euZS40U4V6JkmEnVCIX9mcsL6cbXQ4X8c2w+MMOIjFg68TXUW+u3RT+2e1t1pv9mg/EP3pDtK3POXdQ7Rxyw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rolldown/binding-openharmony-arm64@1.2.13': + resolution: {integrity: sha512-do/AF/4n9MRshia3a5aXoy8xM2c6mgfT6D7H/dY6my5z+hfCIR4ab7mk4IRj++nsEQpKy39kd0DQnDq+r692tw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@rolldown/binding-win32-arm64-msvc@1.2.13': + resolution: {integrity: sha512-6FtUf2cJZobutJdxnGYX8z0vtbgqU0b2v8vfBhkXb9wp2MppMFtkunE+IaOntyi01rkIQoPW2GHABtfzsC0BRQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@rolldown/binding-win32-x64-msvc@1.2.13': + resolution: {integrity: sha512-UKhFaZTeTcbKrb3iU82m17jVcPEAoqDkCN5MBqgPyZSwp/cQmib21crTS0DdH7T1B7wZRWEI89YQAOJMet5yhg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + + '@rolldown/pluginutils@1.0.1': + resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + + '@rollup/rollup-android-arm-eabi@4.62.2': + resolution: {integrity: sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.62.2': + resolution: {integrity: sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.62.2': + resolution: {integrity: sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.62.2': + resolution: {integrity: sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.62.2': + resolution: {integrity: sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.62.2': + resolution: {integrity: sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.62.2': + resolution: {integrity: sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm-musleabihf@4.62.2': + resolution: {integrity: sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==} + cpu: [arm] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-arm64-gnu@4.62.2': + resolution: {integrity: sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm64-musl@4.62.2': + resolution: {integrity: sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-loong64-gnu@4.62.2': + resolution: {integrity: sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==} + cpu: [loong64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-loong64-musl@4.62.2': + resolution: {integrity: sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==} + cpu: [loong64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-ppc64-gnu@4.62.2': + resolution: {integrity: sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-ppc64-musl@4.62.2': + resolution: {integrity: sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==} + cpu: [ppc64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-riscv64-gnu@4.62.2': + resolution: {integrity: sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-riscv64-musl@4.62.2': + resolution: {integrity: sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-s390x-gnu@4.62.2': + resolution: {integrity: sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-gnu@4.62.2': + resolution: {integrity: sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-musl@4.62.2': + resolution: {integrity: sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rollup/rollup-openbsd-x64@4.62.2': + resolution: {integrity: sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.62.2': + resolution: {integrity: sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.62.2': + resolution: {integrity: sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.62.2': + resolution: {integrity: sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.62.2': + resolution: {integrity: sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.62.2': + resolution: {integrity: sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==} + cpu: [x64] + os: [win32] + + '@shikijs/core@4.3.1': + resolution: {integrity: sha512-ANMDxuaPsNMdDC1m4vfvhlDmJweMwkE5XitTwrq2rWHx5jM+dlm4MmHt2PP6t0uejfR77SuhrhJ0zEijIF/uhA==} + engines: {node: '>=20'} + + '@shikijs/engine-javascript@4.3.1': + resolution: {integrity: sha512-JBItcnPuYq7jVJdZo/vMj94r+szT7XEjHFX+mvFDGSEIbVAXAGyHAHzhbWzpGOwYidCZrErJLLgn2PVeiokHnQ==} + engines: {node: '>=20'} + + '@shikijs/engine-oniguruma@4.3.1': + resolution: {integrity: sha512-OXyNMzg0pews+msMj4cHeqT4xiYKKvbnn6VbdAXxfoFl3SSx4fJTc8FadECuc5/H9p3BzhNAoAUXKwAu9rWYhg==} + engines: {node: '>=20'} + + '@shikijs/langs@4.3.1': + resolution: {integrity: sha512-m0l9nsDqgBHvbZbk7A0/kXz/impK3uB/c6rAn6Gpg/uPtdZRQ+alsN/17MU5thb68XTj/4DxkZAotrM0GGSpDQ==} + engines: {node: '>=20'} + + '@shikijs/primitive@4.3.1': + resolution: {integrity: sha512-CXQRQOYy1leqQ8ceTeJdmXv/bsUY++6QyLpXJ94LZAAYj5X2SKRdc5ipguv4NPyGVKItB2PPwUpRNe0Sjh5S1A==} + engines: {node: '>=20'} + + '@shikijs/themes@4.3.1': + resolution: {integrity: sha512-dgpoJ4WqNi2yTmizQHBJ5zcX6j2lE6icN/0yt4l1kkf16jrY/pwPLoTb1ETsWMz0OBLf9ZNvwmxft+cH+N9qSA==} + engines: {node: '>=20'} + + '@shikijs/types@4.3.1': + resolution: {integrity: sha512-CHFxE0jztBIZRHH6gxXE7DXUCFXjReEGxZ/j0rfSLGKZuwp2xBYycEP14875DSa9KLL/6700oxIq6oO6ef9K2g==} + engines: {node: '>=20'} + + '@shikijs/vscode-textmate@10.0.2': + resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==} + + '@sindresorhus/is@7.2.0': + resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} + engines: {node: '>=18'} + + '@speed-highlight/core@1.2.17': + resolution: {integrity: sha512-Z92FwKpCtfaW1V0jTU/fh3QzYEZN8wDwrzRIBoADCJfn4mJCNcJN/XegifX7BDrQ8/h9Xh/JnbyMchL0FqXrkg==} + + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + + '@storybook/addon-a11y@10.6.5': + resolution: {integrity: sha512-7dEH3OHKdE+0zakqEy50KtfhPORIm1xnhWeehyROO1Ct7Rx9s7cYcx6edUZvh0tCuX5VhDnDmkvQV+dbrPDmCA==} + peerDependencies: + storybook: ^10.6.5 + + '@storybook/addon-docs@10.6.5': + resolution: {integrity: sha512-Kt0pGLA+nwdu2g80p8jbdoiiKXxwQbhrY6uzMzq/0G1lytxpJUzrOQAM+GDcEbqyvpKr0u5d1q05/LPem/uooQ==} + peerDependencies: + '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + storybook: ^10.6.5 + peerDependenciesMeta: + '@types/react': + optional: true + + '@storybook/addon-svelte-csf@5.1.6': + resolution: {integrity: sha512-Pzzfr9Lqx0p6a2IBVRlR3zIhA8usl46ADcSSHoe2cU1hICB2HqpDyukqXFSK1NFuTP53yDH99LBbl7drii9l4Q==} + peerDependencies: + '@storybook/svelte': 10.6.1 + '@sveltejs/vite-plugin-svelte': ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 + storybook: ^0.0.0-0 || ^8.2.0 || ^9.0.0 || ^9.1.0-0 || ^10.0.0-0 || ^10.1.0-0 || ^10.2.0-0 || ^10.3.0-0 || ^10.4.0-0 || ^10.5.0-0 || ^10.6.0-0 || ^11.0.0-0 + svelte: ^5.0.0 + vite: ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 + + '@storybook/addon-themes@10.6.5': + resolution: {integrity: sha512-2x6M1vWafp/cDUJzGRkGpVpB6sF4VIuVgpPTyCfEOLtF3S7HaIBalnWqDJ0JvQ/t2e7qtUXKCHZ1C7gREdYOsw==} + peerDependencies: + storybook: ^10.6.5 + + '@storybook/addon-vitest@10.6.5': + resolution: {integrity: sha512-0N0E7TQgmdrPNCPqNiVdTWOEQ1Nes+YsUphYX/abzlp6ReOyYBoQJw6mZ0b7BN5ACgdXQsr9Tth161t72k+51Q==} + peerDependencies: + '@vitest/browser': ^4.0.0 || ^5.0.0 + '@vitest/browser-playwright': ^4.0.0 || ^5.0.0 + storybook: ^10.6.5 + vitest: ^4.0.0 || ^5.0.0 + peerDependenciesMeta: + '@vitest/browser': + optional: true + '@vitest/browser-playwright': + optional: true + vitest: + optional: true + + '@storybook/builder-vite@10.6.5': + resolution: {integrity: sha512-udrGjg3yt38pz4FxplypXxw1creIGS7N6QS8xfvTFM8MQNtPyojT/gZdd4fhX0dFrQx5MMuJEYJA5gUBiPsLtg==} + peerDependencies: + storybook: ^10.6.5 + vite: ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 + + '@storybook/csf@0.1.13': + resolution: {integrity: sha512-7xOOwCLGB3ebM87eemep89MYRFTko+D8qE7EdAAq74lgdqRR5cOUtYWJLjO2dLtP94nqoOdHJo6MdLLKzg412Q==} + + '@storybook/global@5.0.0': + resolution: {integrity: sha512-FcOqPAXACP0I3oJ/ws6/rrPT9WGhu915Cg8D02a9YxLo0DE9zI+a9A5gRGvmQ09fiWPukqI8ZAEoQEdWUKMQdQ==} + + '@storybook/icons@2.1.0': + resolution: {integrity: sha512-Fxh9vYpX9bQqFeHRiY8h2ApeRGDzRSMLwJwNZ/AIRqnyOKHxRKL+yFe+ctEkVJmuptRE9u1Hrn8ZZNHyfDKKNg==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + '@storybook/react-dom-shim@10.6.5': + resolution: {integrity: sha512-Qbbkp+5Otg32w8Ddav6rH6MGPiYVfEzrgn0HqbqlqS8kY+DRXo+NYXJwZTVSDSwrYPrfH+Ok2tV4uE8Zo0c3cg==} + peerDependencies: + '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + '@types/react-dom': ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + storybook: ^10.6.5 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@storybook/svelte-vite@10.6.5': + resolution: {integrity: sha512-0nxO+6VJvnHITU+9k0QF9nQDLoZYWPLj5k6Fp0T5FTpo2OEc5w2FdcVyrTDg3su/SqTjnc+DYoYSoj7j+Z1h2A==} + peerDependencies: + '@sveltejs/vite-plugin-svelte': ^2.0.0 || ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 + storybook: ^10.6.5 + svelte: ^5.0.0 + vite: ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 + + '@storybook/svelte@10.6.1': + resolution: {integrity: sha512-r4mH9gX2unus6IpBVAt5b9Q50SgoT2fMX+NmLoZ3WwTFwIfgWHV7aOYEZgJsk61KEFwUFtJp2il2nI67O3G8Xw==} + peerDependencies: + storybook: ^10.6.1 + svelte: ^5.0.0 + + '@storybook/sveltekit@10.6.5': + resolution: {integrity: sha512-RIxf9wuAYM2AmzOw1wqhKzZltCQbyzz+kDIvstLh54jEY9B/8FBewcWxailjV/gdDkg0Y7a1JJNvqqUPH4eZ1g==} + peerDependencies: + storybook: ^10.6.5 + svelte: ^5.0.0 + vite: ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 + + '@sveltejs/acorn-typescript@1.0.13': + resolution: {integrity: sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ==} + peerDependencies: + acorn: ^8.9.0 + + '@sveltejs/adapter-bun@1.0.0': + resolution: {integrity: sha512-CUfFNxrWrKk/0VqSM1DrXCAwpY3tWocaFwBohuBeyxKx07859shLqPdT9oo00lDti9PvVwhQMDS6bKQylTyv0g==} + engines: {bun: '>=1.4.0'} + peerDependencies: + '@sveltejs/kit': ^3.0.0-next.0 + + '@sveltejs/adapter-cloudflare@8.0.0': + resolution: {integrity: sha512-68jVq1x4nya+/Dpg1NrWlz6LFWAX/F3DrsR8KQGl1bOt4G8oUp0kLLeMx+LI4AZQccGEmE7Nt2tOZwsParE6FA==} + peerDependencies: + '@sveltejs/kit': ^3.0.0-next.0 + wrangler: ^4.118.0 + + '@sveltejs/kit@3.1.0': + resolution: {integrity: sha512-JTWlq2HQlkqx6vIoxhC0R6sc4WVqbQoRyEmr5YlqKmiYbzY1U3awtSWTmjMLlS056a49AmkQR6rnNANXIE8B0Q==} + engines: {node: '>=22.17'} + hasBin: true + peerDependencies: + '@opentelemetry/api': ^1.0.0 + '@sveltejs/vite-plugin-svelte': ^7.0.0 + svelte: ^5.57.1 + typescript: ^6.0.0 + vite: ^8.0.12 + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + typescript: + optional: true + + '@sveltejs/load-config@0.2.3': + resolution: {integrity: sha512-VT3qmUb8pRV2QrZjd8iAmtg8lf4W0TIjZbvXtz5MKei/q96teWZgGJyyidJzOjzZzvdq616eSRVeMYIQChUTAQ==} + engines: {node: '>= 18.0.0'} + + '@sveltejs/vite-plugin-svelte@7.3.1': + resolution: {integrity: sha512-ZPsLN8B1e/En+Ak5s4V7srFDT532oS0qieLsQwu63NGKsS+iAjoO2Js1BochlHlglcU+Pt7WAO3C5Ee+4f6gVA==} + engines: {node: ^20.19 || ^22.12 || >=24} + peerDependencies: + svelte: ^5.46.4 + vite: ^8.0.0-beta.7 || ^8.0.0 + + '@tailwindcss/node@4.3.3': + resolution: {integrity: sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==} + + '@tailwindcss/oxide-android-arm64@4.3.3': + resolution: {integrity: sha512-Y85A2gmPSkl5Ve5qR86GL4HT509cFqQh1aes9p3sSkyTPwt0Pppf3GkwGe4JPACcRYjgJIEhQgM6dBClnr0NYw==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [android] + + '@tailwindcss/oxide-darwin-arm64@4.3.3': + resolution: {integrity: sha512-BiaWatpBcERQFDlOjRDpIVXuFK5PJez5SA4JMg6VYZdBYU+qKfV/vqjcIs+IYmtitf1xYQZTwXvU/8y4lfZUGw==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [darwin] + + '@tailwindcss/oxide-darwin-x64@4.3.3': + resolution: {integrity: sha512-fAeUqfV5ndhxRwai8cXGzdLvul9utWOmeTkv69unv4ZXixjn61Z+p9lCWdwOwA3TYboG3BwdVuN/RDjhBRl0mw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [darwin] + + '@tailwindcss/oxide-freebsd-x64@4.3.3': + resolution: {integrity: sha512-iyf5bV6+wnAlflVeEy7R25dupxTNECZN5QMI0qNT6eT+EgaGdZcKhGkr5SdoaWiLJ3spLqIY9VCeSGrwmtg4kw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [freebsd] + + '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.3': + resolution: {integrity: sha512-aAYUprJAJQWWbRrPvtjdroZ56Md+JM8pMiopS6xGEwDfLhqj+2ver2p4nU4Mb3CRqcMmNBjo8KkUgcxhkzVQGQ==} + engines: {node: '>= 20'} + cpu: [arm] + os: [linux] + + '@tailwindcss/oxide-linux-arm64-gnu@4.3.3': + resolution: {integrity: sha512-nDxldcEENOxZRzC2uu9jrutZdAAQtb+8WWDCSnWL1zvBk1+FN+x6MtDViPB5AJMfttVCUhehGWus3XBPgatM/w==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@tailwindcss/oxide-linux-arm64-musl@4.3.3': + resolution: {integrity: sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@tailwindcss/oxide-linux-x64-gnu@4.3.3': + resolution: {integrity: sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==} + engines: {node: '>= 20'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@tailwindcss/oxide-linux-x64-musl@4.3.3': + resolution: {integrity: sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==} + engines: {node: '>= 20'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@tailwindcss/oxide-wasm32-wasi@4.3.3': + resolution: {integrity: sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + bundledDependencies: + - '@napi-rs/wasm-runtime' + - '@emnapi/core' + - '@emnapi/runtime' + - '@tybys/wasm-util' + - '@emnapi/wasi-threads' + - tslib + + '@tailwindcss/oxide-win32-arm64-msvc@4.3.3': + resolution: {integrity: sha512-3rc292Ca2ceK6Ulcc/bAVnTs/3nDtoPhyEKlgPv+yQJQi/JS/AMJlqzxvlDacL1nekbrcf6bTqp/jV4qgnPxNQ==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [win32] + + '@tailwindcss/oxide-win32-x64-msvc@4.3.3': + resolution: {integrity: sha512-yJ0pwIVc/nYeGoV02WtsN8KYyLQv7kyI2wDnkezyJlGGjkd4QLwDGAwl47YpPJeuI0M0ObaXGSPjvWDPeTPggw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [win32] + + '@tailwindcss/oxide@4.3.3': + resolution: {integrity: sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==} + engines: {node: '>= 20'} + + '@tailwindcss/vite@4.3.3': + resolution: {integrity: sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==} + peerDependencies: + vite: ^5.2.0 || ^6 || ^7 || ^8 + + '@tanstack/svelte-virtual@3.13.35': + resolution: {integrity: sha512-4ur3bPzSXnZZtQWwns/ziCR9pJ/4x/AlBIeDZu96MKrRMbZBL1TQkGiO+ZaX5zCgdpXUZwJnehKzBp32KOumCQ==} + peerDependencies: + svelte: ^3.48.0 || ^4.0.0 || ^5.0.0 + + '@tanstack/virtual-core@3.17.7': + resolution: {integrity: sha512-bp+v10y65sp2H7WpWfIMyxTNfl8ZVfxFTLRjPIFRryi6FV/J33z4IS53WO4pTk36KlvJ4iLiQz+oaydDC1xbcA==} + + '@testing-library/dom@10.4.1': + resolution: {integrity: sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==} + engines: {node: '>=18'} + + '@testing-library/jest-dom@6.9.1': + resolution: {integrity: sha512-zIcONa+hVtVSSep9UT3jZ5rizo2BsxgyDYU7WFD5eICBE7no3881HGeb/QkGfsJs6JTkY1aQhT7rIPC7e+0nnA==} + engines: {node: '>=14', npm: '>=6', yarn: '>=1'} + + '@testing-library/user-event@14.6.7': + resolution: {integrity: sha512-MPCpX8bxe8zS+JmmTwLp8jd0dy1rAm60Te/SL8JrQM3qvQJcBOs1d7IefJMyZzqM3EWBrDn/LWDt1BCGu4ASfg==} + engines: {node: '>=12', npm: '>=6'} + peerDependencies: + '@testing-library/dom': '>=7.21.4' + + '@tybys/wasm-util@0.10.3': + resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} + + '@types/aria-query@5.0.4': + resolution: {integrity: sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==} + + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + + '@types/esrecurse@4.3.1': + resolution: {integrity: sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/hast@3.0.5': + resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} + + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + + '@types/linkify-it@5.0.0': + resolution: {integrity: sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==} + + '@types/markdown-it-emoji@3.0.1': + resolution: {integrity: sha512-cz1j8R35XivBqq9mwnsrP2fsz2yicLhB8+PDtuVkKOExwEdsVBNI+ROL3sbhtR5occRZ66vT0QnwFZCqdjf3pA==} + + '@types/markdown-it@14.1.2': + resolution: {integrity: sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==} + + '@types/mdast@4.0.4': + resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} + + '@types/mdurl@2.0.0': + resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==} + + '@types/mdx@2.0.14': + resolution: {integrity: sha512-T48PeuJtvLosNTPVhfnIp3i/n3a4g4Bad7YCq5k64D4u7NwDrAotikQ+5+sjtUvBmxCMlbo3dVL+C2dP0rWHzg==} + + '@types/node@24.13.3': + resolution: {integrity: sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==} + + '@types/react-dom@19.2.4': + resolution: {integrity: sha512-Bsc+QHgp+P/F02XDzNCY9jnZNCUuLki36KT7VKrTXXLdHf+vHMNZnW1rVu5DNW/rCK+fya3DATySbLM4yhtKUw==} + peerDependencies: + '@types/react': ^19.2.0 + + '@types/react@19.2.18': + resolution: {integrity: sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==} + + '@types/sanitize-html@2.16.1': + resolution: {integrity: sha512-n9wjs8bCOTyN/ynwD8s/nTcTreIHB1vf31vhLMGqUPNHaweKC4/fAl4Dj+hUlCTKYgm4P3k83fmiFfzkZ6sgMA==} + + '@types/textarea-caret@3.0.4': + resolution: {integrity: sha512-epJGYB37/sNrTDbhfyRjHkXsQSAcO6zby0JBDS0QMt6HQ1f1W2E4YpSc7TQkNmWaWmYXv92zOIfN5PHA8CmThg==} + + '@types/unist@3.0.3': + resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} + + '@typescript-eslint/eslint-plugin@8.64.0': + resolution: {integrity: sha512-CGvQPBxN3wZLu6Rz2kFUpZeoCm78xUic92ck39KPePkO1NPOwjCqdQnm5Q87tpWw9vcBvW8XLrDXjH9PWYtJ3Q==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + '@typescript-eslint/parser': ^8.64.0 + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/parser@8.64.0': + resolution: {integrity: sha512-KA0OshtlcCCXmbfqyZkM5pV3/WNraJf7DkJRLpyrmwPtud57H5BDX7C3k0LPSPxpprfRL+cJDGabF10mvNCoCw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/project-service@8.64.0': + resolution: {integrity: sha512-tk4WpOJ6IEbGrVHaNmM0YRrwAD3exZlIK3iadQNAxh4YKk6jvUQ4ecq18n+v7+meh+cJ3j+D8nbk8sRKhlwLQg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/scope-manager@8.64.0': + resolution: {integrity: sha512-CXEaFdYXjSTgKhisNkwCcJwTP8Pl+fmRrEQrri4nm3vU743bALrxzLmq7fHG/7e6a5xO0lDYeURpZmBuhHk54w==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/tsconfig-utils@8.64.0': + resolution: {integrity: sha512-2yo8rRNKuzbVWQp5kslhANqZ2uDAeROQHBRZNPu8JDsHmeFNj/XJJhX/FhNUWmkHHvoNsKa6+tHJiig87EzsQw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/type-utils@8.64.0': + resolution: {integrity: sha512-XWG4Fmmv/6SvyS9nH8jWrKs6terwJvE8cyRt1CzYYqzp9OrPhCT4cMc/f7C6RZCwG+qMmiffJS1/qJP8G1URtg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/types@8.64.0': + resolution: {integrity: sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/typescript-estree@8.64.0': + resolution: {integrity: sha512-Pztpsn1aCE1oWDvDEfUk31nngvvF7vUB5SwHFEaZIFpvw7WJtqUHHL4plBZDA9HfWJJjL13BdG0YrJInTUvoVA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/utils@8.64.0': + resolution: {integrity: sha512-aJUGVB3+U0htrrCjoA8qukw8cm8fNCGAxK/tVoS70k8aeb7DETKeFozRiVFIwEeN9WJLsjaP3ph8I60tY2XZoQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/visitor-keys@8.64.0': + resolution: {integrity: sha512-mrtuL8Nsn6gi2H4mo5KMTp823M+3Q19Ew/i+Zlikq20tIMm99C3Ez0dCmkWWnxut20esQvTg8aUSEhMcAOXhEw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@ungap/structured-clone@1.3.3': + resolution: {integrity: sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg==} + + '@valibot/to-json-schema@1.7.1': + resolution: {integrity: sha512-3qkmU6KXWh8GIThEAW3kuRHPQBMjWkKy+Ppz3WkUucx53DTpOa6siMn4xDGSOhlVyMrDaJTCTMLYPZVAIk1P0A==} + peerDependencies: + valibot: ^1.4.0 + + '@vitest/browser-playwright@5.0.4': + resolution: {integrity: sha512-RYbUmphqONHDcpjMHLNQRy47XbQHHnZokBgtjM8MSPGSiqwBcqwYufdSYEceOwk6zZjKUr3ps5HgjbiXo7sOTg==} + peerDependencies: + playwright: '*' + vitest: 5.0.4 + + '@vitest/browser@5.0.4': + resolution: {integrity: sha512-pRRkI2am/lmhGiaeHkVVLQb7SZyCOe5ytjVNQ0zLNZjA1xb+7aNTRqaaPeMJzWM958/5IrxJ50yTUpKSsigHMw==} + peerDependencies: + vitest: 5.0.4 + + '@vitest/coverage-v8@5.0.4': + resolution: {integrity: sha512-P5YcQY8Y6RmoE9xRXI38rT8pssbcubEzqtYTM2XQoxOXMDn04nRaTI1SM8qCl9/NfW3ag9GlzdSXu7IwO5Ft8A==} + peerDependencies: + '@vitest/browser': 5.0.4 + vitest: 5.0.4 + peerDependenciesMeta: + '@vitest/browser': + optional: true + + '@vitest/expect@3.2.4': + resolution: {integrity: sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==} + + '@vitest/istanbul-lib-coverage@1.0.2': + resolution: {integrity: sha512-9J/JMwOf9AoJhAywhrn7ScKTL38hsWQP/qPG60OtaAFcQ5OXPwKsxZFlbnuCKmZ61m8/lGgHYnFpdyQZUvG/iA==} + engines: {node: '>=22'} + + '@vitest/istanbul-lib-report@1.0.2': + resolution: {integrity: sha512-gUsfXZJbzPamoIY5TvHFiMMoXESBrUMo+xqaj+rYrWI69+EnvRlYBlP96ZnHPY4vX8kyUpgAnFUCg5wZG/HkDQ==} + engines: {node: '>=22'} + + '@vitest/mocker@5.0.4': + resolution: {integrity: sha512-tqAlrYWce4iUxhr3ncNohejtCTOcHeE8XWrSNZY3OhPoDn01yeGv+gnaBy64ATp7o6pb03eIbBAjzmibpc9vIQ==} + peerDependencies: + msw: ^2.4.9 || ^3.0.0 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@3.2.4': + resolution: {integrity: sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==} + + '@vitest/pretty-format@5.0.4': + resolution: {integrity: sha512-kh/bp0oLr6w18kt4MFn5/zS+Hpc5k3bsf6S2m2Qksj7sKHrnc7Tu90UVYyiXnRvkYruN6kSCHQ3RR+SBE6FinA==} + + '@vitest/spy@3.2.4': + resolution: {integrity: sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==} + + '@vitest/spy@5.0.4': + resolution: {integrity: sha512-MZKk/gSkm2q+ICobMxCpH1mA+cCAWm8WV7UhxlqGjq/sJ3AGc1BqFOs5CRVXxH78wQtme9q/NAAgBquj4AcyMw==} + + '@vitest/ui@5.0.4': + resolution: {integrity: sha512-L40ToOGwFgeZ+JVPMKHb7T0b671mqcmCpMUh9P68xq9YZ+TXM72rkDf9ljuv02e5Y1zj8ww0CnWIUnqs3FEWKQ==} + peerDependencies: + vitest: 5.0.4 + + '@vitest/utils@3.2.4': + resolution: {integrity: sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==} + + '@vitest/utils@5.0.4': + resolution: {integrity: sha512-hgpnyMuMqCz2PARmk3v62FxUr8jHIkY13ygvUeh8s+dmQxfnvkB/sgtoZFzG4lMitnEdnrg/lCvDbtVzuAsGWQ==} + + '@webcontainer/env@1.1.1': + resolution: {integrity: sha512-6aN99yL695Hi9SuIk1oC88l9o0gmxL1nGWWQ/kNy81HigJ0FoaoTXpytCj6ItzgyCEwA9kF1wixsTuv5cjsgng==} + + acorn-jsx@5.3.2: + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + + acorn@8.18.0: + resolution: {integrity: sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==} + engines: {node: '>=0.4.0'} + hasBin: true + + ajv@6.15.0: + resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + ansi-styles@5.2.0: + resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==} + engines: {node: '>=10'} + + ansi_up@6.0.6: + resolution: {integrity: sha512-yIa1x3Ecf8jWP4UWEunNjqNX6gzE4vg2gGz+xqRGY+TBSucnYp6RRdPV4brmtg6bQ1ljD48mZ5iGSEj7QEpRKA==} + + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + + aria-query@5.3.2: + resolution: {integrity: sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==} + engines: {node: '>= 0.4'} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + ast-types@0.16.1: + resolution: {integrity: sha512-6t10qk83GOG8p0vKmaCr8eiilZwO171AvbROMtvvNiwrTly62t+7XkA8RdIIVbpMhCASAsxgAzdRSwh6nw/5Dg==} + engines: {node: '>=4'} + + ast-v8-to-istanbul@1.0.7: + resolution: {integrity: sha512-kFL68AG6ajd8fg248zwM9GQrUWEp79gsmjum34OEXjs4yHuUMZfYKwOLW9GMmB4oNvVrj+EAGxsP7ye2UR9UlA==} + + axe-core@4.12.1: + resolution: {integrity: sha512-s7iGf5GaVMxEG0ENN9x+xTr7GFZCb1ZP/1uATUpCEK2X78nDB3RwbtFCo9pGAf9ru+VwoQ464DkaLEeRM08wJA==} + engines: {node: '>=4'} + + axobject-query@4.1.0: + resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} + engines: {node: '>= 0.4'} + + balanced-match@4.0.4: + resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} + engines: {node: 18 || 20 || >=22} + + blake3-wasm@2.1.5: + resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} + + boneyard-js@1.9.0: + resolution: {integrity: sha512-FlIHm7YyQ6GMBPaNY6um5ESHNyZGONoSZ0IMhayuyemBsCUhhDl4nC/gMTOi/25ojtrlnRInTWKqFKb7hGK6Cg==} + hasBin: true + peerDependencies: + '@angular/core': '>=14' + preact: '>=10' + react: '>=18' + react-native: '>=0.71' + svelte: '>=5.29' + vite: '>=5' + vue: '>=3' + peerDependenciesMeta: + '@angular/core': + optional: true + preact: + optional: true + react: + optional: true + react-native: + optional: true + svelte: + optional: true + vite: + optional: true + vue: + optional: true + + brace-expansion@5.0.7: + resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==} + engines: {node: 18 || 20 || >=22} + + bun-types@1.4.4: + resolution: {integrity: sha512-yjrJh5oGX7mMPKGd19YXyIjmEuj4G4zNC30bs7PM9Az7NQ3ndF3j6hbeRdvJTRQEhJ/FYQwW0a2moweEi2hMxQ==} + + bundle-name@4.1.0: + resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} + engines: {node: '>=18'} + + ccount@2.0.1: + resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + + chai@5.3.3: + resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + engines: {node: '>=18'} + + chai@6.3.0: + resolution: {integrity: sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==} + engines: {node: '>=18'} + + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + character-entities-html4@2.1.0: + resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} + + character-entities-legacy@3.0.0: + resolution: {integrity: sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==} + + check-error@2.1.3: + resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} + engines: {node: '>= 16'} + + chokidar@4.0.3: + resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} + engines: {node: '>= 14.16.0'} + + clsx@2.1.1: + resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} + engines: {node: '>=6'} + + cobe@2.0.1: + resolution: {integrity: sha512-aaa6vcIlaC8C1SF50LDH0Anybo/EAXnrxqe+bwvr4+YUtZydqjeBjTTD7ziCCkbRrRGSns3I3F6cZsf3W+L+ag==} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + comma-separated-tokens@2.0.3: + resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + + confbox@0.1.8: + resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} + + confbox@0.3.1: + resolution: {integrity: sha512-cKUSoKa8YxFZZSmraVi7onONx3amu77ngK3kGpsYHDH7drPwCRkQE1RYMPlLRrMtnciRj274XNRxcHxnKmDSnA==} + + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + + cookie@2.0.1: + resolution: {integrity: sha512-yuToqVvRrj6pfDXREyQAAv8SkAEk/8GS3jQRTiUMm66TVtBYmqQeoEjL2Lmq8Rpo6271vH76InTChTitEAm65w==} + engines: {node: '>=22'} + + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + + css.escape@1.5.1: + resolution: {integrity: sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==} + + cssesc@3.0.0: + resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} + engines: {node: '>=4'} + hasBin: true + + csstype@3.2.3: + resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + + cuelume@0.2.4: + resolution: {integrity: sha512-w5Kj1h/MC8chBlwgfIdo/stB73m0RDa2KcIO93yI0mSq5qP6ERKOKTzv2JTj1KD9YUct4vZLAatohdeAe/mZJw==} + + dayjs@1.11.23: + resolution: {integrity: sha512-QDTCU0M0MxR3hQfnlDJfwekQiaanm1ubOD231u73WBckQ/fsamwRLiE2GBz6D3a/xF1NgfiDLJjXBa1hYOYTtQ==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + dedent-js@1.0.1: + resolution: {integrity: sha512-OUepMozQULMLUmhxS95Vudo0jb0UchLimi3+pQ2plj61Fcy8axbP9hbiD4Sz6DPqn6XG3kfmziVfQ1rSys5AJQ==} + + dedent@1.7.2: + resolution: {integrity: sha512-WzMx3mW98SN+zn3hgemf4OzdmyNhhhKz5Ay0pUfQiMQ3e1g+xmTJWp/pKdwKVXhdSkAEGIIzqeuWrL3mV/AXbA==} + peerDependencies: + babel-plugin-macros: ^3.1.0 + peerDependenciesMeta: + babel-plugin-macros: + optional: true + + deep-eql@5.0.2: + resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} + engines: {node: '>=6'} + + deep-is@0.1.4: + resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + + deepmerge@4.3.1: + resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} + engines: {node: '>=0.10.0'} + + default-browser-id@5.0.1: + resolution: {integrity: sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==} + engines: {node: '>=18'} + + default-browser@5.5.0: + resolution: {integrity: sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==} + engines: {node: '>=18'} + + define-lazy-prop@3.0.0: + resolution: {integrity: sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==} + engines: {node: '>=12'} + + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + devalue@5.9.4: + resolution: {integrity: sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==} + + devlop@1.1.0: + resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} + + diff@9.0.0: + resolution: {integrity: sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw==} + engines: {node: '>=0.3.1'} + + dom-accessibility-api@0.5.16: + resolution: {integrity: sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==} + + dom-accessibility-api@0.6.3: + resolution: {integrity: sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w==} + + dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + + dom-serializer@3.1.1: + resolution: {integrity: sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw==} + engines: {node: '>=20.19.0'} + + domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + + domelementtype@3.0.0: + resolution: {integrity: sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg==} + engines: {node: '>=20.19.0'} + + domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + + domhandler@6.0.1: + resolution: {integrity: sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg==} + engines: {node: '>=20.19.0'} + + domutils@3.2.2: + resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==} + + domutils@4.0.2: + resolution: {integrity: sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA==} + engines: {node: '>=20.19.0'} + + enhanced-resolve@5.21.6: + resolution: {integrity: sha512-aNnGCvbJ/RIyWo1IuhNdVjnNF+EjH9wpzpNHt+ci/m9He9LJvUN8wrCcXjp9cWsGNAuvSpVFTx/vraAFQ8qGjQ==} + engines: {node: '>=10.13.0'} + + enhanced-resolve@5.24.5: + resolution: {integrity: sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==} + engines: {node: '>=10.13.0'} + + entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + + entities@7.0.1: + resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==} + engines: {node: '>=0.12'} + + entities@8.0.0: + resolution: {integrity: sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==} + engines: {node: '>=20.19.0'} + + error-stack-parser-es@1.0.5: + resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} + + es-module-lexer@2.3.2: + resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==} + + es-toolkit@1.49.0: + resolution: {integrity: sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g==} + + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + eslint-config-prettier@10.1.8: + resolution: {integrity: sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==} + hasBin: true + peerDependencies: + eslint: '>=7.0.0' + + eslint-plugin-better-tailwindcss@4.7.0: + resolution: {integrity: sha512-lrdlVW4pzLPj/zX5HRqMhKPesGijDfRhnSRJMlNcsrCyJHsndmHCLKTiRNa1eREUZ6G3D3QjdLc+G4tlfGrmkw==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23.0.0} + peerDependencies: + eslint: ^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 + oxlint: ^1.35.0 + tailwindcss: ^3.3.0 || ^4.1.17 + peerDependenciesMeta: + eslint: + optional: true + oxlint: + optional: true + + eslint-plugin-storybook@10.6.5: + resolution: {integrity: sha512-sQSRwmbqcyGVtjVGUSOPTM5TFjopoit3blQxMcsHeqmmR1PJsciLH/WjgG7CT+u2/F0HMJFsQa+TqJSFDxXwWQ==} + peerDependencies: + eslint: '>=8' + + eslint-plugin-svelte@3.23.1: + resolution: {integrity: sha512-+w86Jlx4IRUuuvtqD6j4njql+akx+81JnpvbN5de3AV5iq2SexvST/Jr51CiTlKeY0i+vhoKjPO/ywFKIP75sA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.1 || ^9.0.0 || ^10.0.0 + svelte: ^3.37.0 || ^4.0.0 || ^5.0.0 + peerDependenciesMeta: + svelte: + optional: true + + eslint-scope@8.4.0: + resolution: {integrity: sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint-scope@9.1.2: + resolution: {integrity: sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + eslint-visitor-keys@3.4.3: + resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + eslint-visitor-keys@4.2.1: + resolution: {integrity: sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint-visitor-keys@5.0.1: + resolution: {integrity: sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + eslint@10.7.0: + resolution: {integrity: sha512-GVTD7s1vdIl6UYvAfriOPeY1Df8LIZjfofLvHwde+erDHGGuHyuM6xoxRxmHiebhYuD2p1vN4wWh0XzPARSGDQ==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + hasBin: true + peerDependencies: + jiti: '*' + peerDependenciesMeta: + jiti: + optional: true + + esm-env@1.2.2: + resolution: {integrity: sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==} + + espree@10.4.0: + resolution: {integrity: sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + espree@11.2.0: + resolution: {integrity: sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true + + esquery@1.7.0: + resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} + engines: {node: '>=0.10'} + + esrap@1.2.2: + resolution: {integrity: sha512-F2pSJklxx1BlQIQgooczXCPHmcWpn6EsP5oo73LQfonG9fIlIENQ8vMmfGXeojP9MrkzUNAfyU5vdFlR9shHAw==} + + esrap@1.4.9: + resolution: {integrity: sha512-3OMlcd0a03UGuZpPeUC1HxR3nA23l+HEyCiZw3b3FumJIN9KphoGzDJKMXI1S72jVS1dsenDyQC0kJlO1U9E1g==} + + esrap@2.4.0: + resolution: {integrity: sha512-CQPcneEHw/czj32uwQC52nYnax8H4SXt25FCM+LIwGZ49cMIxpOj3bP8wmzcI+Z0HIy5KkClaPhRLfCySzwHPA==} + peerDependencies: + '@typescript-eslint/types': ^8.2.0 + peerDependenciesMeta: + '@typescript-eslint/types': + optional: true + + esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + + exsolve@1.1.1: + resolution: {integrity: sha512-9U/jZUgjnSGyntRr6y5Muu1MJcwFl6kPu7k8qLF0IMNfLqvw0NZ4nnVDq0RVoZ0RvCyumib4Ez3KYrVfilrw+g==} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + + fast-levenshtein@2.0.6: + resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + feed@6.0.0: + resolution: {integrity: sha512-rvFOF1POce0ijAKy3tZN5T8PuprgIit/VbMyXJWcoYwcMOTSIJ7NqZb8DTLPJr77yiEO27kidm/fw0kcSIV+ww==} + engines: {node: '>=20', pnpm: '>=10'} + + fflate@0.8.3: + resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} + + file-entry-cache@8.0.0: + resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} + engines: {node: '>=16.0.0'} + + find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + + flat-cache@4.0.1: + resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} + engines: {node: '>=16'} + + flatted@3.4.2: + resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==} + + flatted@3.4.4: + resolution: {integrity: sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==} + + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + fuse.js@7.5.0: + resolution: {integrity: sha512-sQtrEfA+ez/3G0cCZecF70oqpCRttCexYUG4mUrtWL49ULUzUyxokt5kyqwtKzj1270RaKih+hcP3qLcumccow==} + engines: {node: '>=10'} + + glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + + globals@16.5.0: + resolution: {integrity: sha512-c/c15i26VrJ4IRt5Z89DnIzCGDn9EcebibhAOjw5ibqEHsE1wLUgkPn9RDmNcUKyU87GeaL633nyJ+pplFR2ZQ==} + engines: {node: '>=18'} + + globals@17.7.0: + resolution: {integrity: sha512-Czmyns5dUsq4seFBR/Kdydhmo8y9kC79hiSkPn0YcGtNnYWnrgt0vjrSjx9tspoDGWm2CMarffRuLjM4xUz8xg==} + engines: {node: '>=18'} + + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + hast-util-to-html@9.0.5: + resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==} + + hast-util-whitespace@3.0.0: + resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==} + + html-void-elements@3.0.0: + resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + + htmlparser2@10.1.0: + resolution: {integrity: sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==} + + htmlparser2@12.0.0: + resolution: {integrity: sha512-Tz7u1i95/g2x2jz81+x0FBVhBhY5aRTvD3tXXdFaljuNdzDLJ8UGNRrTcj2cgQvAg3iW/h77Fz15nLW0L0CrZw==} + engines: {node: '>=20.19.0'} + + ignore@5.3.2: + resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} + engines: {node: '>= 4'} + + ignore@7.0.6: + resolution: {integrity: sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==} + engines: {node: '>= 4'} + + import-meta-resolve@4.2.0: + resolution: {integrity: sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==} + + imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + + indent-string@4.0.0: + resolution: {integrity: sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==} + engines: {node: '>=8'} + + is-docker@3.0.0: + resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + hasBin: true + + is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + + is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + + is-inside-container@1.0.0: + resolution: {integrity: sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==} + engines: {node: '>=14.16'} + hasBin: true + + is-plain-object@5.1.0: + resolution: {integrity: sha512-bUi/yjmtKYcRVUtWRGr0UA6xEFh2I6zWUwMrUXB3s7bmYCaZ8a+0ZsTRkrawh/mzlSD1Y0Ph8bp/U+TvBpWDNw==} + engines: {node: '>=0.10.0'} + + is-reference@3.0.3: + resolution: {integrity: sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==} + + is-wsl@3.1.1: + resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} + engines: {node: '>=16'} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + jiti@2.7.0: + resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} + hasBin: true + + js-tokens@10.0.0: + resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==} + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + json-buffer@3.0.1: + resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + + json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + + json-stable-stringify-without-jsonify@1.0.1: + resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + + json5@2.2.3: + resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} + engines: {node: '>=6'} + hasBin: true + + jsonc-parser@3.3.1: + resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} + + keyv@4.5.4: + resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + + kleur@4.1.5: + resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} + engines: {node: '>=6'} + + known-css-properties@0.37.0: + resolution: {integrity: sha512-JCDrsP4Z1Sb9JwG0aJ8Eo2r7k4Ou5MwmThS/6lcIe1ICyb7UBJKGRIUUdqc2ASdE/42lgz6zFUnzAIhtXnBVrQ==} + + launder@1.7.2: + resolution: {integrity: sha512-DLg3HPnHUfBi5/MxMLmJD12dmlMpFEC2HgMW6vkZ/9JR0RU00kXoRGvDxAvFmdO0o610OA77i4FgNTLucmhDVg==} + + levn@0.4.1: + resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} + engines: {node: '>= 0.8.0'} + + lightningcss-android-arm64@1.32.0: + resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-android-arm64@1.33.0: + resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.32.0: + resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-arm64@1.33.0: + resolution: {integrity: sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.32.0: + resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-darwin-x64@1.33.0: + resolution: {integrity: sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.32.0: + resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-freebsd-x64@1.33.0: + resolution: {integrity: sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.32.0: + resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm-gnueabihf@1.33.0: + resolution: {integrity: sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.32.0: + resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + lightningcss-linux-arm64-gnu@1.33.0: + resolution: {integrity: sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + lightningcss-linux-arm64-musl@1.32.0: + resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + lightningcss-linux-arm64-musl@1.33.0: + resolution: {integrity: sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + lightningcss-linux-x64-gnu@1.32.0: + resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + lightningcss-linux-x64-gnu@1.33.0: + resolution: {integrity: sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + lightningcss-linux-x64-musl@1.32.0: + resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + lightningcss-linux-x64-musl@1.33.0: + resolution: {integrity: sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + lightningcss-win32-arm64-msvc@1.32.0: + resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-arm64-msvc@1.33.0: + resolution: {integrity: sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.32.0: + resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss-win32-x64-msvc@1.33.0: + resolution: {integrity: sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.32.0: + resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} + engines: {node: '>= 12.0.0'} + + lightningcss@1.33.0: + resolution: {integrity: sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==} + engines: {node: '>= 12.0.0'} + + lilconfig@2.1.0: + resolution: {integrity: sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==} + engines: {node: '>=10'} + + linkify-it@5.0.2: + resolution: {integrity: sha512-ONTm2jCMAVZjgQa/Fy1kScXsuOoF5NPTsoFBdE1KVIZ2vAh/r9+Bqo+0jINCBYnavTPQZz38QzFTme79ENoN3Q==} + + local-pkg@1.2.1: + resolution: {integrity: sha512-++gUqRDEvcnN6Zhqrr+y/CkVEHhlrR96vZn3nZZPYzMcBUyBtTKzB9NadClFIsIVSsu+3i9tfk/erqy9kAmt7Q==} + engines: {node: '>=14'} + + locate-character@3.0.0: + resolution: {integrity: sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==} + + locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + + loupe@3.2.1: + resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + + lru_map@0.4.1: + resolution: {integrity: sha512-I+lBvqMMFfqaV8CJCISjI3wbjmwVu/VyOoU7+qtu9d7ioW5klMgsTTiUOUp+DJvfTTzKXoPbyC6YfgkNcyPSOg==} + + lz-string@1.5.0: + resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} + hasBin: true + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + magic-string@1.4.3: + resolution: {integrity: sha512-z12OxmaPGE0F4xlpGdjuAUckipLpQlTAuAmyGhNoD7ODpYUzvDfvtxUbjM/jwznyP178oju/KDdyi220wNJ0RQ==} + + magicast@0.5.5: + resolution: {integrity: sha512-UicdXN8zQ3JHlxVq+28afMXPr1z7WNY6+7EJnzTdQWkTAlMLF5fNCCKxJHBQwGaNGR11581EiQmQzx73+MvszA==} + + markdown-it-anchor@9.2.1: + resolution: {integrity: sha512-p6APiLJDFAW2GEvaavDvhIBn7jrX2jLv77NkBGgNacFTurbORYc4pyYySg/mI6mpR6cHQuAtzKtmqgQr4K8dsQ==} + peerDependencies: + '@types/markdown-it': '*' + markdown-it: '*' + + markdown-it-emoji@3.1.0: + resolution: {integrity: sha512-NhmMEH2ywduD4Nty1E8uB5NqfLhAT1VR0dyvoJyStKOqCzbZmVdn/+8wj7zpDsb/fLBikpCPsWwxqKlvMmbz4g==} + + markdown-it@14.3.2: + resolution: {integrity: sha512-sHHjZ5fJKlgrG4qns2YwVcdNep35h5fERrfkD2YNsb9UFk0UIHarbiTaHKVMlPuWAoiilyK8Fv/jAm11slsY7Q==} + hasBin: true + + mdast-util-to-hast@13.2.1: + resolution: {integrity: sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==} + + mdn-data@2.29.0: + resolution: {integrity: sha512-pVxQFCcaYUEAH853+v7yoI/qzhxXSq1bTb9obMYGYAN1c3Hen+XDCEvr296XhstrwlSTNgOR7mCSD4JPjbJe5A==} + + mdurl@2.1.0: + resolution: {integrity: sha512-1+HBaOx0zi/dQWht8rNv9MYf9qqpqL/kxI0hXImU6Y547zM6Sni8BQibt7ifgMcYtQg41ao3Ivd6cnSM86inpg==} + + micromark-util-character@2.1.1: + resolution: {integrity: sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==} + + micromark-util-encode@2.0.1: + resolution: {integrity: sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==} + + micromark-util-sanitize-uri@2.0.1: + resolution: {integrity: sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==} + + micromark-util-symbol@2.0.1: + resolution: {integrity: sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==} + + micromark-util-types@2.0.2: + resolution: {integrity: sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==} + + min-indent@1.0.1: + resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} + engines: {node: '>=4'} + + miniflare@5.20260820.0-alpha: + resolution: {integrity: sha512-Bv1j2kcKKNwXLWuCx+j0xGt7z318mqQkJmEN6ellM9sCESbPBDTM9ofZMbKqx47jSnoGA3CiaUkAmzGVXUa/wQ==} + engines: {node: '>=22.0.0'} + + minimatch@10.2.5: + resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} + engines: {node: 18 || 20 || >=22} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + mlly@1.8.2: + resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + + mri@1.2.0: + resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} + engines: {node: '>=4'} + + mrmime@2.0.1: + resolution: {integrity: sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==} + engines: {node: '>=10'} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.16: + resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + nanoid@3.3.20: + resolution: {integrity: sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + nanoid@5.1.16: + resolution: {integrity: sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ==} + engines: {node: ^18 || >=20} + hasBin: true + + natural-compare@1.4.0: + resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + + obug@2.2.1: + resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} + engines: {node: '>=12.20.0'} + + obug@3.0.0: + resolution: {integrity: sha512-5vvB5+W7ePv+p3uqxi+RcW1XAzLW0/hxt3/4X4Lc4qHudzOhmBiBwOY6DRob4WnanAEGvNcLjF+KNOufrUoEQw==} + engines: {node: '>=12.20.0'} + + oniguruma-parser@0.12.2: + resolution: {integrity: sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==} + + oniguruma-to-es@4.3.6: + resolution: {integrity: sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==} + + open@10.2.0: + resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} + engines: {node: '>=18'} + + optionator@0.9.4: + resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} + engines: {node: '>= 0.8.0'} + + oxc-parser@0.127.0: + resolution: {integrity: sha512-bkgD4qHlN7WxLdX8bLXdaU54TtQtAIg/ZBAfm0aje/mo3MRDo3P0hZSgr4U7O3xfX+fQmR5AP04JS/TGcZLcFA==} + engines: {node: ^20.19.0 || >=22.12.0} + + oxc-resolver@11.21.2: + resolution: {integrity: sha512-w5tLwYN3Zo24w5EeWJjJWZOwhYqTtC8PS2B1tIt7BZUuqTIcU07sQValbDw+rq7+AuAGzOHklgK+ifsy4lpXfw==} + + p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + + p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + + package-manager-detector@1.8.0: + resolution: {integrity: sha512-yQA4H19AmPEoMUeavPMDIe1higySl/gH/yaQrkT/s07Qp+7pp2hYz30N3z2l5BkjVkF9Ow6o0wjJamm2y7Sn0A==} + + parse-srcset@1.0.2: + resolution: {integrity: sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==} + + path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-to-regexp@6.3.0: + resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + pathval@2.0.1: + resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} + engines: {node: '>= 14.16'} + + pele@1.5.2: + resolution: {integrity: sha512-t1XarodlFluWbSy8eXE4PnkFNXUaP1qLUUKJANHiZpTM0VgDoLCEv9CENj3iHM8s507QXyRdSJFMDiZCmkvtGg==} + engines: {node: '>=20'} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.7: + resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} + engines: {node: '>=12'} + + pkg-types@1.3.1: + resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + + pkg-types@2.3.3: + resolution: {integrity: sha512-j/lCFdcppV0JxWpCEITdbDltBxPP6cHT+yNJ6Go2OgoSA9518X847X9z0p6LtA4Nc16+eQzCZjRrWanTGvHJ5w==} + + playwright-core@1.61.1: + resolution: {integrity: sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.61.1: + resolution: {integrity: sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==} + engines: {node: '>=18'} + hasBin: true + + pngjs@7.0.0: + resolution: {integrity: sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==} + engines: {node: '>=14.19.0'} + + postcss-load-config@3.1.4: + resolution: {integrity: sha512-6DiM4E7v4coTE4uzA8U//WhtPwyhiim3eyjEMFCnUpzbrkK9wJHgKDT2mR+HbtSrd/NubVaYTOpSpjUl8NQeRg==} + engines: {node: '>= 10'} + peerDependencies: + postcss: '>=8.0.9' + ts-node: '>=9.0.0' + peerDependenciesMeta: + postcss: + optional: true + ts-node: + optional: true + + postcss-safe-parser@7.0.1: + resolution: {integrity: sha512-0AioNCJZ2DPYz5ABT6bddIqlhgwhpHZ/l65YAYo0BCIn0xiDpsnTHz0gnoTGk0OXZW0JRs+cDwL8u/teRdz+8A==} + engines: {node: '>=18.0'} + peerDependencies: + postcss: ^8.4.31 + + postcss-scss@4.0.9: + resolution: {integrity: sha512-AjKOeiwAitL/MXxQW2DliT28EKukvvbEWx3LBmJIRN8KfBGZbRTxNYW0kSqi1COiTZ57nZ9NW06S6ux//N1c9A==} + engines: {node: '>=12.0'} + peerDependencies: + postcss: ^8.4.29 + + postcss-selector-parser@7.1.4: + resolution: {integrity: sha512-HeP7D2wyhkR+XaK6v4W8oRF62Dsz4flyuczALJp61GckGm42u1saSSJ/0auvcBqxs3jMRFEcPK34At/0JBKdOg==} + engines: {node: '>=4'} + + postcss@8.5.19: + resolution: {integrity: sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==} + engines: {node: ^10 || ^12 || >=14} + + postcss@8.5.29: + resolution: {integrity: sha512-49cGhUbXj8Qenv0iTMxA1cFBzxXoctpC9Ujd77t1WcbJIr6nF/eI7g/8MgxrYldFRuAXvja7xQRwavoW7kgrxQ==} + engines: {node: ^10 || ^12 || >=14} + + prelude-ls@1.2.1: + resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} + engines: {node: '>= 0.8.0'} + + prettier-plugin-svelte@4.1.1: + resolution: {integrity: sha512-wXvbXMjSvb4C9ENWTHXyd+ihakKCsJ6rJhLP6/8HFNj4GkZr48jqL9PoKsl2sk7SyCZRTnJ7O2TTowUpOxP/KA==} + engines: {node: '>=20'} + peerDependencies: + prettier: ^3.0.0 + svelte: ^5.0.0 + + prettier-plugin-tailwindcss@0.8.1: + resolution: {integrity: sha512-iaFMYqDsE4ffdDkn5qup0j5f2aCEBFZrdrZnvu9QKTlWx/iGPeQ4HHu7b7fCPMxeo9nwQBiOAh2nSypdFYWJkw==} + engines: {node: '>=20.19'} + peerDependencies: + '@ianvs/prettier-plugin-sort-imports': '*' + '@prettier/plugin-hermes': '*' + '@prettier/plugin-oxc': '*' + '@prettier/plugin-pug': '*' + '@shopify/prettier-plugin-liquid': '*' + '@trivago/prettier-plugin-sort-imports': '*' + '@zackad/prettier-plugin-twig': '*' + prettier: ^3.0 + prettier-plugin-astro: '*' + prettier-plugin-css-order: '*' + prettier-plugin-jsdoc: '*' + prettier-plugin-marko: '*' + prettier-plugin-multiline-arrays: '*' + prettier-plugin-organize-attributes: '*' + prettier-plugin-organize-imports: '*' + prettier-plugin-sort-imports: '*' + prettier-plugin-svelte: '*' + peerDependenciesMeta: + '@ianvs/prettier-plugin-sort-imports': + optional: true + '@prettier/plugin-hermes': + optional: true + '@prettier/plugin-oxc': + optional: true + '@prettier/plugin-pug': + optional: true + '@shopify/prettier-plugin-liquid': + optional: true + '@trivago/prettier-plugin-sort-imports': + optional: true + '@zackad/prettier-plugin-twig': + optional: true + prettier-plugin-astro: + optional: true + prettier-plugin-css-order: + optional: true + prettier-plugin-jsdoc: + optional: true + prettier-plugin-marko: + optional: true + prettier-plugin-multiline-arrays: + optional: true + prettier-plugin-organize-attributes: + optional: true + prettier-plugin-organize-imports: + optional: true + prettier-plugin-sort-imports: + optional: true + prettier-plugin-svelte: + optional: true + + prettier@3.9.5: + resolution: {integrity: sha512-/FVl766LpUfB5vXgCYOYa0MeV/441Ia99AeICQIQFTY/Nw0roZwULcXpku5i1/m5kt/baz+s4Zogspd839HSMg==} + engines: {node: '>=14'} + hasBin: true + + pretty-format@27.5.1: + resolution: {integrity: sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==} + engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} + + property-information@7.2.0: + resolution: {integrity: sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==} + + punycode.js@2.3.1: + resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==} + engines: {node: '>=6'} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + quansync@0.2.11: + resolution: {integrity: sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==} + + react-dom@19.2.8: + resolution: {integrity: sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==} + peerDependencies: + react: ^19.2.8 + + react-is@17.0.2: + resolution: {integrity: sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==} + + react@19.2.8: + resolution: {integrity: sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==} + engines: {node: '>=0.10.0'} + + readdirp@4.1.2: + resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} + engines: {node: '>= 14.18.0'} + + recast@0.23.12: + resolution: {integrity: sha512-dEWRjcINDu/F4l2dYx57ugBtD7HV9KXESyxhzw/MqWLeglJrsjJKqACPyUPg+6AF8mIgm+Zi0dZ3ACoIg+QtpA==} + engines: {node: '>= 4'} + + redent@3.0.0: + resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==} + engines: {node: '>=8'} + + regex-recursion@6.0.2: + resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==} + + regex-utilities@2.3.0: + resolution: {integrity: sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==} + + regex@6.1.0: + resolution: {integrity: sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==} + + rolldown@1.2.13: + resolution: {integrity: sha512-/5apOkwWxtACcQtZ5WlrAsKS4JwXVThQPJZ3RqIruRY9UG9M8e0VlXN068xum/JRKA4aIUoJ13ghe/kGCvidiQ==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + + rollup@4.62.2: + resolution: {integrity: sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + run-applescript@7.1.0: + resolution: {integrity: sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==} + engines: {node: '>=18'} + + sade@1.8.1: + resolution: {integrity: sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==} + engines: {node: '>=6'} + + sanitize-html@2.18.1: + resolution: {integrity: sha512-2X5lC8fYla8YaT8g73Nj+HcLWXan5dLyCWLgVvaOmfw7JI78IRsyNtIDytTHdn7hBQbIkCah7Zbmd5jabzeDig==} + engines: {node: '>=22.12.0'} + + sax@1.6.1: + resolution: {integrity: sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q==} + engines: {node: '>=11.0.0'} + + scheduler@0.27.0: + resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} + + scule@1.3.0: + resolution: {integrity: sha512-6FtHJEvt+pVMIB9IBY+IcCJ6Z5f1iQnytgyfKMhDKgmzYG+TeH/wx1y3l27rshSbLiSanrR9ffZDrEsmjlQF2g==} + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + sharp@0.35.2: + resolution: {integrity: sha512-FVtFjtBCMiJS6yb5CX7Sop45WFMpeGw6oRKuJnXYgf/f1ms/D7LE/ZUSNxnW7rZ/dbslQWYkoqFHGPaDBtaK4w==} + engines: {node: '>=20.9.0'} + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + shiki@4.3.1: + resolution: {integrity: sha512-oR+qDVi2OjX1tmDpyv+3KviX01KzO6Af+0NNnKnsp9491UEGz2YpxTuJboS/6VhYpTdqzmuJBuiTlrAWWJAssw==} + engines: {node: '>=20'} + + sirv@3.0.2: + resolution: {integrity: sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==} + engines: {node: '>=18'} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + source-map-js@1.2.2: + resolution: {integrity: sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==} + engines: {node: '>=0.10.0'} + + source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + + space-separated-tokens@2.0.2: + resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + + std-env@4.3.0: + resolution: {integrity: sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==} + + storybook@10.6.5: + resolution: {integrity: sha512-qurC38Nx4PDF9YIq6GO95dpUx8a3B9AZwlcHBo4cOI9At09ifjQRsqcwcdOf2YX0cpVhFKmHkI20SWCrxqYQrg==} + hasBin: true + peerDependencies: + '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + prettier: ^2 || ^3 + vite-plus: ^0.1.15 || ^0.2.0 + peerDependenciesMeta: + '@types/react': + optional: true + prettier: + optional: true + vite-plus: + optional: true + + stringify-entities@4.0.4: + resolution: {integrity: sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==} + + strip-bom@3.0.0: + resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} + engines: {node: '>=4'} + + strip-indent@3.0.0: + resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==} + engines: {node: '>=8'} + + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + svelte-ast-print@0.4.2: + resolution: {integrity: sha512-hRHHufbJoArFmDYQKCpCvc0xUuIEfwYksvyLYEQyH+1xb5LD5sM/IthfooCdXZQtOIqXz6xm7NmaqdfwG4kh6w==} + engines: {node: '>=18'} + peerDependencies: + svelte: ^5.0.0 + + svelte-check@4.7.6: + resolution: {integrity: sha512-t2scM//ZuVbSY/T2w6FSBw1v9s2NEmh/g+sy1lqtosW5ylBV5AF4wFb1Ts9Kf3MbfPDUDJDZ9L436YT0SPTdvw==} + engines: {node: '>= 18.0.0'} + hasBin: true + peerDependencies: + svelte: ^4.0.0 || ^5.0.0-next.0 + typescript: ^5.0.0 || ^6.0.0 + + svelte-eslint-parser@1.8.0: + resolution: {integrity: sha512-mikR1qwIVy3t5WthUoAXkMwxkXvabZP9FJgdx35Ei7EbGWmctva1Pih16Koeor/bdNNq8NXHlwKGS6NkYTawLg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0, pnpm: 10.34.1} + peerDependencies: + svelte: ^3.37.0 || ^4.0.0 || ^5.0.0 + peerDependenciesMeta: + svelte: + optional: true + + svelte2tsx@0.7.57: + resolution: {integrity: sha512-nQo0xEfUpSVjfkxan2UmC6Wl9UZVe9+/pglUiyBNMJ7KDQ9DDooucmXdToBOvzSfgYjj/kMYwf6CTTDz/z048w==} + peerDependencies: + svelte: ^3.55 || ^4.0.0-next.0 || ^4.0 || ^5.0.0-next.0 + typescript: ^4.9.4 || ^5.0.0 || ^6.0.0 + + svelte@5.57.2: + resolution: {integrity: sha512-1YiPCAXT0Ct6g1vOMmSLa0Nsc3QRMvtDF32ihgglGVrugIuIK38RyDiFNXVDhtQ81TJjSO57NdMw6DyYLqSGDQ==} + engines: {node: '>=18'} + + svg-pathdata@9.0.0: + resolution: {integrity: sha512-h9FuqmNsgDKq2hQBqTMOWjZwqdZOnWijZmcg6FL4iSmH6VvRnobhHrBQSdOiP0W9nwgv0qiW5vQ3//Arrmp19g==} + engines: {node: '>=24.14.0'} + + synckit@0.11.13: + resolution: {integrity: sha512-eNRKgb3z66Yp3D2CixVujOUvXLFUTij/zVnV8KRyvFdQwpz7I5DS8UfRkTeLzb64u+dkzDSdelE24izu+zSSUg==} + engines: {node: ^14.18.0 || >=16.0.0} + + tagged-tag@1.0.0: + resolution: {integrity: sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==} + engines: {node: '>=20'} + + tailwind-csstree@0.3.3: + resolution: {integrity: sha512-je9J5UYRsTJqAjYrIBMMlge8T/rreRd44pJxgG5Zx/zeo4kAC/liUKqzztRZrGlYRJLvIf2Cb1DVJMTXSzEShA==} + engines: {node: '>=18.18'} + peerDependencies: + '@eslint/css': '>=1.0.0' + peerDependenciesMeta: + '@eslint/css': + optional: true + + tailwind-merge@3.6.0: + resolution: {integrity: sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w==} + + tailwind-variants@3.2.2: + resolution: {integrity: sha512-Mi4kHeMTLvKlM98XPnK+7HoBPmf4gygdFmqQPaDivc3DpYS6aIY6KiG/PgThrGvii5YZJqRsPz0aPyhoFzmZgg==} + engines: {node: '>=16.x', pnpm: '>=7.x'} + peerDependencies: + tailwind-merge: '>=3.0.0' + tailwindcss: '*' + peerDependenciesMeta: + tailwind-merge: + optional: true + + tailwindcss@4.3.3: + resolution: {integrity: sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==} + + tapable@2.3.3: + resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} + engines: {node: '>=6'} + + textarea-caret@3.1.0: + resolution: {integrity: sha512-cXAvzO9pP5CGa6NKx0WYHl+8CHKZs8byMkt3PCJBCmq2a34YA9pO1NrQET5pzeqnBjBdToF5No4rrmkDUgQC2Q==} + + tiny-invariant@1.3.3: + resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} + + tinybench@6.2.2: + resolution: {integrity: sha512-nIZ2N2dFpjBsg2hgitKu7mA+FvsIJuGJOwysRngmVRRmWgR/UyPjwJj8c/fT/dhm8EkHnF5EOogRf+0Z2qrimA==} + engines: {node: '>=20.0.0'} + + tinyexec@1.3.1: + resolution: {integrity: sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==} + engines: {node: '>=18'} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tinyrainbow@2.0.0: + resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} + engines: {node: '>=14.0.0'} + + tinyrainbow@3.2.0: + resolution: {integrity: sha512-LgO3D9yZJjApUiuUfl9iFAwrtaX4+lok3wJIqttGoKCHlWUqHqbQpnxCf82L8FjgKsh4iGo78hqJwgL8F6To2A==} + engines: {node: '>=14.0.0'} + + tinyspy@4.0.4: + resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} + engines: {node: '>=14.0.0'} + + totalist@3.0.1: + resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} + engines: {node: '>=6'} + + trim-lines@3.0.1: + resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} + + ts-api-utils@2.5.0: + resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} + engines: {node: '>=18.12'} + peerDependencies: + typescript: '>=4.8.4' + + ts-dedent@2.3.0: + resolution: {integrity: sha512-JfJeIHke7y2egdGGgRAvpCwYFUsHlM2gPcrVOxFkznt/4uzQ7HFmvE63iFHVLBJNDuyDOQgijDK/tXH/f6Msjg==} + engines: {node: '>=6.10'} + + tsconfig-paths-webpack-plugin@4.2.0: + resolution: {integrity: sha512-zbem3rfRS8BgeNK50Zz5SIQgXzLafiHjOwUAvk/38/o1jHn/V5QAgVUcz884or7WYcPaH3N2CIfUc2u0ul7UcA==} + engines: {node: '>=10.13.0'} + + tsconfig-paths@4.2.0: + resolution: {integrity: sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==} + engines: {node: '>=6'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + tw-easing-gradients@1.1.2: + resolution: {integrity: sha512-nNrvbzgIGE1eIGwmfzer97oOG6ettcrh3rTYfgB+o9q1hhXCF106VRRfulyAAgpLef7DUF+Ut9tpBMLt430JrQ==} + peerDependencies: + tailwindcss: '>=4.0.0' + + type-check@0.4.0: + resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} + engines: {node: '>= 0.8.0'} + + type-fest@2.19.0: + resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} + engines: {node: '>=12.20'} + + type-fest@5.8.0: + resolution: {integrity: sha512-YGYEVz3Fm5iy/AybuA0oyNFq7H4CgQNfRp/qfe8nurE1kuCeNm3/vfm9X4Mtl+qLyaKJUh5xrFZwogr41SMjYA==} + engines: {node: '>=20'} + + typescript-eslint@8.64.0: + resolution: {integrity: sha512-0qg+pDNMnqYzqH9AnNK+39tejHvsShUOUUoRUgtnTGE7QuMZhiFDnozq8nHJVq+Wae6NMLKNWLg5WmkcC/ndyQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + typescript@6.0.3: + resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} + engines: {node: '>=14.17'} + hasBin: true + + uc.micro@2.1.0: + resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==} + + ufo@1.6.4: + resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} + + undici-types@7.18.2: + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + + unenv@2.0.0-rc.24: + resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} + + unicode-segmenter@0.14.5: + resolution: {integrity: sha512-jHGmj2LUuqDcX3hqY12Ql+uhUTn8huuxNZGq7GvtF6bSybzH3aFgedYu/KTzQStEgt1Ra2F3HxadNXsNjb3m3g==} + + unist-util-is@6.0.1: + resolution: {integrity: sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==} + + unist-util-position@5.0.0: + resolution: {integrity: sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==} + + unist-util-stringify-position@4.0.0: + resolution: {integrity: sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==} + + unist-util-visit-parents@6.0.2: + resolution: {integrity: sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==} + + unist-util-visit@5.1.0: + resolution: {integrity: sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==} + + unplugin-icons@24.0.0: + resolution: {integrity: sha512-wV6c7I0PtiIP8KIU7Z1Ukyc2uzgY9ind3RXNu7j63L9qTWDn7CzLnW9FD57DKbfGpWKQv3qyV2PCtBG8Y5Llcg==} + peerDependencies: + '@svgr/core': '>=7.0.0' + '@svgx/core': ^1.0.1 + '@vue/compiler-sfc': ^3.0.2 + '@vue/compiler-vapor': ^3.6.0-rc.5 + svelte: ^3.0.0 || ^4.0.0 || ^5.0.0 + peerDependenciesMeta: + '@svgr/core': + optional: true + '@svgx/core': + optional: true + '@vue/compiler-sfc': + optional: true + '@vue/compiler-vapor': + optional: true + svelte: + optional: true + + unplugin@2.3.11: + resolution: {integrity: sha512-5uKD0nqiYVzlmCRs01Fhs2BdkEgBS3SAVP6ndrBsuK42iC2+JHyxM05Rm9G8+5mkmRtzMZGY8Ct5+mliZxU/Ww==} + engines: {node: '>=18.12.0'} + + unplugin@3.4.0: + resolution: {integrity: sha512-9skdIFlCsPdFV7wUfZxNsFInlW+7nJmGu2gkTu0OUhF56aXGsHab9x52/QhdJ4lC7ZDPWTxbiC4ANqVlsuaW3w==} + engines: {node: ^20.19.0 || >=22.12.0} + peerDependencies: + '@farmfe/core': '*' + '@rsbuild/core': '*' + '@rspack/core': '*' + bun-types-no-globals: '*' + esbuild: '*' + rolldown: '*' + rollup: '*' + unloader: '*' + vite: '*' + webpack: '*' + peerDependenciesMeta: + '@farmfe/core': + optional: true + '@rsbuild/core': + optional: true + '@rspack/core': + optional: true + bun-types-no-globals: + optional: true + esbuild: + optional: true + rolldown: + optional: true + rollup: + optional: true + unloader: + optional: true + vite: + optional: true + webpack: + optional: true + + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + + use-sync-external-store@1.6.0: + resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + valibot@1.4.2: + resolution: {integrity: sha512-gjdCvJ6d3RyHAneqxMYMW9QMCwYMb3jpOO0IyHZV1bnRHFBHrX3VkIILt5XYR0WhwHiH7Mty8ovuPZ/O3gamrg==} + peerDependencies: + typescript: '>=5' + peerDependenciesMeta: + typescript: + optional: true + + vfile-message@4.0.3: + resolution: {integrity: sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==} + + vfile@6.0.3: + resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} + + vite@8.3.4: + resolution: {integrity: sha512-IftLE8phJhdNRIGNEV4JCCvfNScoZQVfcXaN8TQropWGJtOHcu+iAB9pHcuGvmEb/kddw0LBe/jS8ztdC6xdiQ==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + '@vitejs/devtools': ^0.7.1 + esbuild: ^0.27.0 || ^0.28.0 + jiti: '>=1.21.0' + less: ^4.0.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + '@vitejs/devtools': + optional: true + esbuild: + optional: true + jiti: + optional: true + less: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitefu@1.1.3: + resolution: {integrity: sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg==} + peerDependencies: + vite: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + vite: + optional: true + + vitest@5.0.4: + resolution: {integrity: sha512-z0WxZRV2Sg48wF0FSgMq3TFG8JS38/Qr96vZVZdX6+Em9raQTvnXQvdTFJe2m3rfyJnEHeHQ/snceIdi8D1pVg==} + engines: {node: ^22.12.0 || ^24.0.0 || >=26.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 5.0.4 + '@vitest/browser-preview': 5.0.4 + '@vitest/browser-webdriverio': ^5.0.0-beta.5 || >=5.0.0 + '@vitest/coverage-istanbul': 5.0.4 + '@vitest/coverage-v8': 5.0.4 + '@vitest/ui': 5.0.4 + happy-dom: '*' + jsdom: '*' + vite: ^6.4.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + webpack-virtual-modules@0.6.2: + resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==} + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + why-is-node-running@3.2.1: + resolution: {integrity: sha512-Tb2FUhB4vUsGQlfSquQLYkApkuPAFQXGFzxWKHHumVz2dK+X1RUm/HnID4+TfIGYJ1kTcwOaCk/buYCEJr6YjQ==} + engines: {node: '>=20.11'} + hasBin: true + + word-wrap@1.2.5: + resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} + engines: {node: '>=0.10.0'} + + workerd@1.20260820.1: + resolution: {integrity: sha512-/wk4rFNHH6IVMXFe6aPEZsT5YWpWtfKTUMt7+rFkyeGbXcGCy4yxODmJatbqYj0jmqHETdz0+m2mT+vNjXnF7w==} + engines: {node: '>=16'} + hasBin: true + + wrangler@4.125.0: + resolution: {integrity: sha512-yFpvggu+xk1Hdm/Uxwaqa19bb7GArME4CrCS3Vov68a2TZq2MPO+wLocKbbnIC9K0oLowcdau7/ycxbbNHKCEg==} + engines: {node: '>=22.0.0'} + hasBin: true + peerDependencies: + '@cloudflare/workers-types': ^5.20260820.1 + peerDependenciesMeta: + '@cloudflare/workers-types': + optional: true + + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + ws@8.22.0: + resolution: {integrity: sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + wsl-utils@0.1.0: + resolution: {integrity: sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==} + engines: {node: '>=18'} + + xml-js@1.6.11: + resolution: {integrity: sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==} + hasBin: true + + yaml@1.10.3: + resolution: {integrity: sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==} + engines: {node: '>= 6'} + + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + + yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + + youch-core@0.3.3: + resolution: {integrity: sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==} + + youch@4.1.0-beta.10: + resolution: {integrity: sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==} + + zimmerframe@1.1.2: + resolution: {integrity: sha512-rAbqEGa8ovJy4pyBxZM70hg4pE6gDgaQ0Sl9M3enG3I0d6H4XSAM3GeNGLKnsBpuijUow064sf7ww1nutC5/3w==} + + zimmerframe@1.1.4: + resolution: {integrity: sha512-B58NGBEoc8Y9MWWCQGl/gq9xBCe4IiKM0a2x7GZdQKOW5Exr8S1W24J6OgM1njK8xCRGvAJIL/MxXHf6SkmQKQ==} + + zwitch@2.0.4: + resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} + +snapshots: + + '@adobe/css-tools@4.5.0': {} + + '@antfu/install-pkg@1.1.0': + dependencies: + package-manager-detector: 1.8.0 + tinyexec: 1.3.1 + + '@antfu/install-pkg@2.1.0': + dependencies: + package-manager-detector: 1.8.0 + tinyexec: 1.3.1 + + '@atcute/atproto@4.0.3(@atcute/lexicons@2.0.2)': + dependencies: + '@atcute/lexicons': 2.0.2 + + '@atcute/car@6.0.2(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2)': + dependencies: + '@atcute/cbor': 2.3.6(@atcute/cid@2.4.2) + '@atcute/cid': 2.4.2 + '@atcute/uint8array': 1.1.5 + '@atcute/varint': 2.0.1 + + '@atcute/cbor@2.3.6(@atcute/cid@2.4.2)': + dependencies: + '@atcute/cid': 2.4.2 + '@atcute/multibase': 1.2.5 + '@atcute/uint8array': 1.1.5 + + '@atcute/cid@2.4.2': + dependencies: + '@atcute/multibase': 1.2.5 + '@atcute/uint8array': 1.1.5 + + '@atcute/client@5.1.1(@atcute/lexicons@2.0.2)(typescript@6.0.3)': + dependencies: + '@atcute/identity': 2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/lexicons': 2.0.2 + transitivePeerDependencies: + - typescript + + '@atcute/crypto@2.4.2': + dependencies: + '@atcute/multibase': 1.2.5 + '@atcute/uint8array': 1.1.5 + '@noble/secp256k1': 3.1.0 + + '@atcute/identity-resolver@2.0.1(@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3)': + dependencies: + '@atcute/identity': 2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/lexicons': 2.0.2 + '@atcute/util-fetch': 2.0.1(typescript@6.0.3) + valibot: 1.4.2(typescript@6.0.3) + transitivePeerDependencies: + - typescript + + '@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3)': + dependencies: + '@atcute/lexicons': 2.0.2 + valibot: 1.4.2(typescript@6.0.3) + transitivePeerDependencies: + - typescript + + '@atcute/lex-cli@3.2.1(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2)(prettier@3.9.5)(typescript@6.0.3)': + dependencies: + '@atcute/identity': 2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/identity-resolver': 2.0.1(@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/lexicon-doc': 3.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/lexicon-resolver': 1.0.1(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2)(@atcute/identity-resolver@2.0.1(@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicon-doc@3.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/lexicons': 2.0.2 + '@oomfware/kempt': 0.1.4 + '@optique/core': 1.1.1 + '@optique/run': 1.1.1 + picocolors: 1.1.1 + valibot: 1.4.2(typescript@6.0.3) + optionalDependencies: + prettier: 3.9.5 + transitivePeerDependencies: + - '@atcute/cbor' + - '@atcute/cid' + - typescript + + '@atcute/lexicon-doc@3.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3)': + dependencies: + '@atcute/identity': 2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/lexicons': 2.0.2 + '@atcute/uint8array': 1.1.4 + '@atcute/util-text': 1.3.3 + valibot: 1.4.2(typescript@6.0.3) + transitivePeerDependencies: + - typescript + + '@atcute/lexicon-resolver@1.0.1(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2)(@atcute/identity-resolver@2.0.1(@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicon-doc@3.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3)': + dependencies: + '@atcute/crypto': 2.4.2 + '@atcute/identity': 2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/identity-resolver': 2.0.1(@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/lexicon-doc': 3.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/lexicons': 2.0.2 + '@atcute/repo': 1.0.2(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2)(@atcute/lexicons@2.0.2) + '@atcute/util-fetch': 2.0.1(typescript@6.0.3) + valibot: 1.4.2(typescript@6.0.3) + transitivePeerDependencies: + - '@atcute/cbor' + - '@atcute/cid' + - typescript + + '@atcute/lexicons@2.0.2': + dependencies: + '@atcute/uint8array': 1.1.4 + '@atcute/util-text': 1.3.3 + '@standard-schema/spec': 1.1.0 + esm-env: 1.2.2 + + '@atcute/microcosm@2.0.2(@atcute/lexicons@2.0.2)': + dependencies: + '@atcute/lexicons': 2.0.2 + + '@atcute/mst@1.0.2(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2)': + dependencies: + '@atcute/cbor': 2.3.6(@atcute/cid@2.4.2) + '@atcute/cid': 2.4.2 + '@atcute/uint8array': 1.1.5 + + '@atcute/multibase@1.2.4': + dependencies: + '@atcute/uint8array': 1.1.4 + + '@atcute/multibase@1.2.5': + dependencies: + '@atcute/uint8array': 1.1.5 + + '@atcute/oauth-browser-client@4.0.1(@atcute/identity-resolver@2.0.1(@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3)': + dependencies: + '@atcute/client': 5.1.1(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/identity-resolver': 2.0.1(@atcute/identity@2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3))(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/lexicons': 2.0.2 + '@atcute/multibase': 1.2.4 + '@atcute/oauth-crypto': 1.0.1(typescript@6.0.3) + '@atcute/oauth-types': 1.0.1(typescript@6.0.3) + nanoid: 5.1.16 + transitivePeerDependencies: + - typescript + + '@atcute/oauth-crypto@1.0.1(typescript@6.0.3)': + dependencies: + '@atcute/multibase': 1.2.4 + '@atcute/uint8array': 1.1.4 + nanoid: 5.1.16 + valibot: 1.4.2(typescript@6.0.3) + transitivePeerDependencies: + - typescript + + '@atcute/oauth-keyset@0.1.2(typescript@6.0.3)': + dependencies: + '@atcute/oauth-crypto': 1.0.1(typescript@6.0.3) + transitivePeerDependencies: + - typescript + + '@atcute/oauth-types@1.0.1(typescript@6.0.3)': + dependencies: + '@atcute/identity': 2.0.1(@atcute/lexicons@2.0.2)(typescript@6.0.3) + '@atcute/lexicons': 2.0.2 + '@atcute/oauth-keyset': 0.1.2(typescript@6.0.3) + valibot: 1.4.2(typescript@6.0.3) + transitivePeerDependencies: + - typescript + + '@atcute/repo@1.0.2(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2)(@atcute/lexicons@2.0.2)': + dependencies: + '@atcute/car': 6.0.2(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2) + '@atcute/cbor': 2.3.6(@atcute/cid@2.4.2) + '@atcute/cid': 2.4.2 + '@atcute/crypto': 2.4.2 + '@atcute/lexicons': 2.0.2 + '@atcute/mst': 1.0.2(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2) + '@atcute/uint8array': 1.1.5 + + '@atcute/tid@1.1.4': + dependencies: + '@atcute/time-ms': 1.3.3 + + '@atcute/time-ms@1.3.3': {} + + '@atcute/uint8array@1.1.4': {} + + '@atcute/uint8array@1.1.5': {} + + '@atcute/util-fetch@2.0.1(typescript@6.0.3)': + dependencies: + valibot: 1.4.2(typescript@6.0.3) + transitivePeerDependencies: + - typescript + + '@atcute/util-text@1.3.3': + dependencies: + unicode-segmenter: 0.14.5 + + '@atcute/varint@2.0.1': {} + + '@babel/code-frame@7.29.7': + dependencies: + '@babel/helper-validator-identifier': 7.29.7 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/helper-string-parser@7.29.7': {} + + '@babel/helper-validator-identifier@7.29.7': {} + + '@babel/parser@7.29.7': + dependencies: + '@babel/types': 7.29.7 + + '@babel/runtime@7.29.7': {} + + '@babel/types@7.29.7': + dependencies: + '@babel/helper-string-parser': 7.29.7 + '@babel/helper-validator-identifier': 7.29.7 + + '@bcoe/v8-coverage@1.0.2': {} + + '@blazediff/core@1.10.0': {} + + '@chenglou/pretext@0.0.5': + optional: true + + '@cloudflare/kv-asset-handler@0.5.0': {} + + '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260820.1)': + dependencies: + unenv: 2.0.0-rc.24 + optionalDependencies: + workerd: 1.20260820.1 + + '@cloudflare/workerd-darwin-64@1.20260820.1': + optional: true + + '@cloudflare/workerd-darwin-arm64@1.20260820.1': + optional: true + + '@cloudflare/workerd-linux-64@1.20260820.1': + optional: true + + '@cloudflare/workerd-linux-arm64@1.20260820.1': + optional: true + + '@cloudflare/workerd-windows-64@1.20260820.1': + optional: true + + '@cloudflare/workers-types@5.20261012.1': {} + + '@cspotcode/source-map-support@0.8.1': + dependencies: + '@jridgewell/trace-mapping': 0.3.9 + + '@emnapi/core@1.11.0': + dependencies: + '@emnapi/wasi-threads': 1.2.2 + tslib: 2.8.1 + optional: true + + '@emnapi/core@1.9.2': + dependencies: + '@emnapi/wasi-threads': 1.2.1 + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@1.11.0': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@1.11.2': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@1.9.2': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@1.2.1': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@1.2.2': + dependencies: + tslib: 2.8.1 + optional: true + + '@esbuild/aix-ppc64@0.28.1': + optional: true + + '@esbuild/android-arm64@0.28.1': + optional: true + + '@esbuild/android-arm@0.28.1': + optional: true + + '@esbuild/android-x64@0.28.1': + optional: true + + '@esbuild/darwin-arm64@0.28.1': + optional: true + + '@esbuild/darwin-x64@0.28.1': + optional: true + + '@esbuild/freebsd-arm64@0.28.1': + optional: true + + '@esbuild/freebsd-x64@0.28.1': + optional: true + + '@esbuild/linux-arm64@0.28.1': + optional: true + + '@esbuild/linux-arm@0.28.1': + optional: true + + '@esbuild/linux-ia32@0.28.1': + optional: true + + '@esbuild/linux-loong64@0.28.1': + optional: true + + '@esbuild/linux-mips64el@0.28.1': + optional: true + + '@esbuild/linux-ppc64@0.28.1': + optional: true + + '@esbuild/linux-riscv64@0.28.1': + optional: true + + '@esbuild/linux-s390x@0.28.1': + optional: true + + '@esbuild/linux-x64@0.28.1': + optional: true + + '@esbuild/netbsd-arm64@0.28.1': + optional: true + + '@esbuild/netbsd-x64@0.28.1': + optional: true + + '@esbuild/openbsd-arm64@0.28.1': + optional: true + + '@esbuild/openbsd-x64@0.28.1': + optional: true + + '@esbuild/openharmony-arm64@0.28.1': + optional: true + + '@esbuild/sunos-x64@0.28.1': + optional: true + + '@esbuild/win32-arm64@0.28.1': + optional: true + + '@esbuild/win32-ia32@0.28.1': + optional: true + + '@esbuild/win32-x64@0.28.1': + optional: true + + '@eslint-community/eslint-utils@4.9.1(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))': + dependencies: + eslint: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + eslint-visitor-keys: 3.4.3 + + '@eslint-community/regexpp@4.12.2': {} + + '@eslint/config-array@0.23.5(supports-color@7.2.0)': + dependencies: + '@eslint/object-schema': 3.0.5 + debug: 4.4.3(supports-color@7.2.0) + minimatch: 10.2.5 + transitivePeerDependencies: + - supports-color + + '@eslint/config-helpers@0.6.0': + dependencies: + '@eslint/core': 1.2.1 + + '@eslint/core@1.2.1': + dependencies: + '@types/json-schema': 7.0.15 + + '@eslint/css-tree@4.0.5': + dependencies: + mdn-data: 2.29.0 + source-map-js: 1.2.1 + + '@eslint/js@10.0.1(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))': + optionalDependencies: + eslint: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + + '@eslint/object-schema@3.0.5': {} + + '@eslint/plugin-kit@0.7.2': + dependencies: + '@eslint/core': 1.2.1 + levn: 0.4.1 + + '@humanfs/core@0.19.2': + dependencies: + '@humanfs/types': 0.15.0 + + '@humanfs/node@0.16.8': + dependencies: + '@humanfs/core': 0.19.2 + '@humanfs/types': 0.15.0 + '@humanwhocodes/retry': 0.4.3 + + '@humanfs/types@0.15.0': {} + + '@humanwhocodes/module-importer@1.0.1': {} + + '@humanwhocodes/retry@0.4.3': {} + + '@iconify-json/lucide@1.2.134': + dependencies: + '@iconify/types': 2.0.0 + + '@iconify/types@2.0.0': {} + + '@iconify/utils@3.1.7': + dependencies: + '@antfu/install-pkg': 2.1.0 + '@iconify/types': 2.0.0 + import-meta-resolve: 4.2.0 + + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.3.1 + optional: true + + '@img/sharp-darwin-x64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.3.1 + optional: true + + '@img/sharp-freebsd-wasm32@0.35.2': + dependencies: + '@img/sharp-wasm32': 0.35.2 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.3.1': + optional: true + + '@img/sharp-libvips-darwin-x64@1.3.1': + optional: true + + '@img/sharp-libvips-linux-arm64@1.3.1': + optional: true + + '@img/sharp-libvips-linux-arm@1.3.1': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.3.1': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.3.1': + optional: true + + '@img/sharp-libvips-linux-s390x@1.3.1': + optional: true + + '@img/sharp-libvips-linux-x64@1.3.1': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.3.1': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.3.1': + optional: true + + '@img/sharp-linux-arm64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.3.1 + optional: true + + '@img/sharp-linux-arm@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.3.1 + optional: true + + '@img/sharp-linux-ppc64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.3.1 + optional: true + + '@img/sharp-linux-riscv64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.3.1 + optional: true + + '@img/sharp-linux-s390x@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.3.1 + optional: true + + '@img/sharp-linux-x64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.3.1 + optional: true + + '@img/sharp-linuxmusl-arm64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.3.1 + optional: true + + '@img/sharp-linuxmusl-x64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.3.1 + optional: true + + '@img/sharp-wasm32@0.35.2': + dependencies: + '@emnapi/runtime': 1.11.2 + optional: true + + '@img/sharp-webcontainers-wasm32@0.35.2': + dependencies: + '@img/sharp-wasm32': 0.35.2 + optional: true + + '@img/sharp-win32-arm64@0.35.2': + optional: true + + '@img/sharp-win32-ia32@0.35.2': + optional: true + + '@img/sharp-win32-x64@0.35.2': + optional: true + + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/remapping@2.3.5': + dependencies: + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@jridgewell/sourcemap-codec@1.6.0': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@jridgewell/trace-mapping@0.3.9': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.6.0 + + '@mdit/plugin-alert@1.0.1(markdown-it@14.3.2)': + dependencies: + '@types/markdown-it': 14.1.2 + optionalDependencies: + markdown-it: 14.3.2 + + '@mdit/plugin-footnote@1.0.1(markdown-it@14.3.2)': + dependencies: + '@types/markdown-it': 14.1.2 + markdown-it: 14.3.2 + + '@mdit/plugin-tasklist@1.0.1(markdown-it@14.3.2)': + dependencies: + '@types/markdown-it': 14.1.2 + optionalDependencies: + markdown-it: 14.3.2 + + '@mdx-js/react@3.1.1(@types/react@19.2.18)(react@19.2.8)': + dependencies: + '@types/mdx': 2.0.14 + '@types/react': 19.2.18 + react: 19.2.8 + + '@napi-rs/wasm-runtime@1.1.6(@emnapi/core@1.11.0)(@emnapi/runtime@1.11.0)': + dependencies: + '@emnapi/core': 1.11.0 + '@emnapi/runtime': 1.11.0 + '@tybys/wasm-util': 0.10.3 + optional: true + + '@napi-rs/wasm-runtime@1.1.6(@emnapi/core@1.9.2)(@emnapi/runtime@1.9.2)': + dependencies: + '@emnapi/core': 1.9.2 + '@emnapi/runtime': 1.9.2 + '@tybys/wasm-util': 0.10.3 + optional: true + + '@noble/secp256k1@3.1.0': {} + + '@oomfware/kempt@0.1.4': {} + + '@optique/core@1.1.1': {} + + '@optique/run@1.1.1': + dependencies: + '@optique/core': 1.1.1 + + '@oxc-parser/binding-android-arm-eabi@0.127.0': + optional: true + + '@oxc-parser/binding-android-arm64@0.127.0': + optional: true + + '@oxc-parser/binding-darwin-arm64@0.127.0': + optional: true + + '@oxc-parser/binding-darwin-x64@0.127.0': + optional: true + + '@oxc-parser/binding-freebsd-x64@0.127.0': + optional: true + + '@oxc-parser/binding-linux-arm-gnueabihf@0.127.0': + optional: true + + '@oxc-parser/binding-linux-arm-musleabihf@0.127.0': + optional: true + + '@oxc-parser/binding-linux-arm64-gnu@0.127.0': + optional: true + + '@oxc-parser/binding-linux-arm64-musl@0.127.0': + optional: true + + '@oxc-parser/binding-linux-ppc64-gnu@0.127.0': + optional: true + + '@oxc-parser/binding-linux-riscv64-gnu@0.127.0': + optional: true + + '@oxc-parser/binding-linux-riscv64-musl@0.127.0': + optional: true + + '@oxc-parser/binding-linux-s390x-gnu@0.127.0': + optional: true + + '@oxc-parser/binding-linux-x64-gnu@0.127.0': + optional: true + + '@oxc-parser/binding-linux-x64-musl@0.127.0': + optional: true + + '@oxc-parser/binding-openharmony-arm64@0.127.0': + optional: true + + '@oxc-parser/binding-wasm32-wasi@0.127.0': + dependencies: + '@emnapi/core': 1.9.2 + '@emnapi/runtime': 1.9.2 + '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.9.2)(@emnapi/runtime@1.9.2) + optional: true + + '@oxc-parser/binding-win32-arm64-msvc@0.127.0': + optional: true + + '@oxc-parser/binding-win32-ia32-msvc@0.127.0': + optional: true + + '@oxc-parser/binding-win32-x64-msvc@0.127.0': + optional: true + + '@oxc-project/types@0.127.0': {} + + '@oxc-project/types@0.153.0': {} + + '@oxc-resolver/binding-android-arm-eabi@11.21.2': + optional: true + + '@oxc-resolver/binding-android-arm64@11.21.2': + optional: true + + '@oxc-resolver/binding-darwin-arm64@11.21.2': + optional: true + + '@oxc-resolver/binding-darwin-x64@11.21.2': + optional: true + + '@oxc-resolver/binding-freebsd-x64@11.21.2': + optional: true + + '@oxc-resolver/binding-linux-arm-gnueabihf@11.21.2': + optional: true + + '@oxc-resolver/binding-linux-arm-musleabihf@11.21.2': + optional: true + + '@oxc-resolver/binding-linux-arm64-gnu@11.21.2': + optional: true + + '@oxc-resolver/binding-linux-arm64-musl@11.21.2': + optional: true + + '@oxc-resolver/binding-linux-ppc64-gnu@11.21.2': + optional: true + + '@oxc-resolver/binding-linux-riscv64-gnu@11.21.2': + optional: true + + '@oxc-resolver/binding-linux-riscv64-musl@11.21.2': + optional: true + + '@oxc-resolver/binding-linux-s390x-gnu@11.21.2': + optional: true + + '@oxc-resolver/binding-linux-x64-gnu@11.21.2': + optional: true + + '@oxc-resolver/binding-linux-x64-musl@11.21.2': + optional: true + + '@oxc-resolver/binding-openharmony-arm64@11.21.2': + optional: true + + '@oxc-resolver/binding-wasm32-wasi@11.21.2': + dependencies: + '@emnapi/core': 1.11.0 + '@emnapi/runtime': 1.11.0 + '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.11.0)(@emnapi/runtime@1.11.0) + optional: true + + '@oxc-resolver/binding-win32-arm64-msvc@11.21.2': + optional: true + + '@oxc-resolver/binding-win32-x64-msvc@11.21.2': + optional: true + + '@pierre/diffs@1.6.1(patch_hash=3e5f4b454dc1b27920ac06ec6b2a03eeac337333cebd93b896f01f7c3bcfcdf6)(@shikijs/themes@4.3.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)': + dependencies: + '@pierre/highlights': 0.2.0 + '@pierre/theme': 2.0.0 + '@pierre/theming': 1.0.2(@pierre/theme@2.0.0)(@shikijs/themes@4.3.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(shiki@4.3.1) + diff: 9.0.0 + hast-util-to-html: 9.0.5 + lru_map: 0.4.1 + shiki: 4.3.1 + optionalDependencies: + react: 19.2.8 + react-dom: 19.2.8(react@19.2.8) + transitivePeerDependencies: + - '@shikijs/themes' + + '@pierre/highlights@0.2.0': {} + + '@pierre/theme@2.0.0': {} + + '@pierre/theming@1.0.2(@pierre/theme@2.0.0)(@shikijs/themes@4.3.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(shiki@4.3.1)': + optionalDependencies: + '@pierre/theme': 2.0.0 + '@shikijs/themes': 4.3.1 + react: 19.2.8 + react-dom: 19.2.8(react@19.2.8) + shiki: 4.3.1 + + '@pkgr/core@0.3.6': {} + + '@polka/url@1.0.0-next.29': {} + + '@poppinss/colors@4.1.6': + dependencies: + kleur: 4.1.5 + + '@poppinss/dumper@0.6.5': + dependencies: + '@poppinss/colors': 4.1.6 + '@sindresorhus/is': 7.2.0 + supports-color: 10.2.2 + + '@poppinss/exception@1.2.3': {} + + '@rolldown/binding-android-arm-eabi@1.2.13': + optional: true + + '@rolldown/binding-android-arm64@1.2.13': + optional: true + + '@rolldown/binding-darwin-arm64@1.2.13': + optional: true + + '@rolldown/binding-darwin-x64@1.2.13': + optional: true + + '@rolldown/binding-freebsd-x64@1.2.13': + optional: true + + '@rolldown/binding-linux-arm-gnueabihf@1.2.13': + optional: true + + '@rolldown/binding-linux-arm64-gnu@1.2.13': + optional: true + + '@rolldown/binding-linux-arm64-musl@1.2.13': + optional: true + + '@rolldown/binding-linux-ppc64-gnu@1.2.13': + optional: true + + '@rolldown/binding-linux-s390x-gnu@1.2.13': + optional: true + + '@rolldown/binding-linux-x64-gnu@1.2.13': + optional: true + + '@rolldown/binding-linux-x64-musl@1.2.13': + optional: true + + '@rolldown/binding-openharmony-arm64@1.2.13': + optional: true + + '@rolldown/binding-win32-arm64-msvc@1.2.13': + optional: true + + '@rolldown/binding-win32-x64-msvc@1.2.13': + optional: true + + '@rolldown/pluginutils@1.0.1': {} + + '@rollup/rollup-android-arm-eabi@4.62.2': + optional: true + + '@rollup/rollup-android-arm64@4.62.2': + optional: true + + '@rollup/rollup-darwin-arm64@4.62.2': + optional: true + + '@rollup/rollup-darwin-x64@4.62.2': + optional: true + + '@rollup/rollup-freebsd-arm64@4.62.2': + optional: true + + '@rollup/rollup-freebsd-x64@4.62.2': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.62.2': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.62.2': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-x64-musl@4.62.2': + optional: true + + '@rollup/rollup-openbsd-x64@4.62.2': + optional: true + + '@rollup/rollup-openharmony-arm64@4.62.2': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.62.2': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.62.2': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.62.2': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.62.2': + optional: true + + '@shikijs/core@4.3.1': + dependencies: + '@shikijs/primitive': 4.3.1 + '@shikijs/types': 4.3.1 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + hast-util-to-html: 9.0.5 + + '@shikijs/engine-javascript@4.3.1': + dependencies: + '@shikijs/types': 4.3.1 + '@shikijs/vscode-textmate': 10.0.2 + oniguruma-to-es: 4.3.6 + + '@shikijs/engine-oniguruma@4.3.1': + dependencies: + '@shikijs/types': 4.3.1 + '@shikijs/vscode-textmate': 10.0.2 + + '@shikijs/langs@4.3.1': + dependencies: + '@shikijs/types': 4.3.1 + + '@shikijs/primitive@4.3.1': + dependencies: + '@shikijs/types': 4.3.1 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + + '@shikijs/themes@4.3.1': + dependencies: + '@shikijs/types': 4.3.1 + + '@shikijs/types@4.3.1': + dependencies: + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + + '@shikijs/vscode-textmate@10.0.2': {} + + '@sindresorhus/is@7.2.0': {} + + '@speed-highlight/core@1.2.17': {} + + '@standard-schema/spec@1.1.0': {} + + '@storybook/addon-a11y@10.6.5(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))': + dependencies: + '@storybook/global': 5.0.0 + axe-core: 4.12.1 + storybook: 10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8) + + '@storybook/addon-docs@10.6.5(@types/react-dom@19.2.4(@types/react@19.2.18))(@types/react@19.2.18)(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))': + dependencies: + '@mdx-js/react': 3.1.1(@types/react@19.2.18)(react@19.2.8) + '@storybook/icons': 2.1.0(react@19.2.8) + '@storybook/react-dom-shim': 10.6.5(@types/react-dom@19.2.4(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8)) + react: 19.2.8 + react-dom: 19.2.8(react@19.2.8) + storybook: 10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8) + ts-dedent: 2.3.0 + unplugin: 2.3.11 + optionalDependencies: + '@types/react': 19.2.18 + transitivePeerDependencies: + - '@types/react-dom' + + '@storybook/addon-svelte-csf@5.1.6(@storybook/svelte@10.6.1(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)))(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': + dependencies: + '@storybook/csf': 0.1.13 + '@storybook/svelte': 10.6.1(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)) + '@sveltejs/vite-plugin-svelte': 7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + dedent: 1.7.2 + es-toolkit: 1.49.0 + esrap: 1.4.9 + magic-string: 0.30.21 + storybook: 10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8) + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + svelte-ast-print: 0.4.2(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)) + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + zimmerframe: 1.1.4 + transitivePeerDependencies: + - babel-plugin-macros + + '@storybook/addon-themes@10.6.5(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))': + dependencies: + storybook: 10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8) + ts-dedent: 2.3.0 + + '@storybook/addon-vitest@10.6.5(@vitest/browser-playwright@5.0.4)(@vitest/browser@5.0.4)(react@19.2.8)(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(vitest@5.0.4)': + dependencies: + '@storybook/global': 5.0.0 + '@storybook/icons': 2.1.0(react@19.2.8) + storybook: 10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8) + optionalDependencies: + '@vitest/browser': 5.0.4(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))(vitest@5.0.4) + '@vitest/browser-playwright': 5.0.4(playwright@1.61.1)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))(vitest@5.0.4) + vitest: 5.0.4(@types/node@24.13.3)(@vitest/browser-playwright@5.0.4)(@vitest/coverage-v8@5.0.4)(@vitest/ui@5.0.4)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + transitivePeerDependencies: + - react + + '@storybook/builder-vite@10.6.5(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': + dependencies: + storybook: 10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8) + ts-dedent: 2.3.0 + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + + '@storybook/csf@0.1.13': + dependencies: + type-fest: 2.19.0 + + '@storybook/global@5.0.0': {} + + '@storybook/icons@2.1.0(react@19.2.8)': + dependencies: + react: 19.2.8 + + '@storybook/react-dom-shim@10.6.5(@types/react-dom@19.2.4(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))': + dependencies: + react: 19.2.8 + react-dom: 19.2.8(react@19.2.8) + storybook: 10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8) + optionalDependencies: + '@types/react': 19.2.18 + '@types/react-dom': 19.2.4(@types/react@19.2.18) + + '@storybook/svelte-vite@10.6.5(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': + dependencies: + '@storybook/builder-vite': 10.6.5(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + '@storybook/svelte': 10.6.1(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)) + '@sveltejs/vite-plugin-svelte': 7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + magic-string: 1.4.3 + storybook: 10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8) + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + svelte2tsx: 0.7.57(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@5.9.3) + typescript: 5.9.3 + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + + '@storybook/svelte@10.6.1(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))': + dependencies: + storybook: 10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8) + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + ts-dedent: 2.3.0 + type-fest: 5.8.0 + + '@storybook/sveltekit@10.6.5(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': + dependencies: + '@storybook/builder-vite': 10.6.5(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + '@storybook/svelte': 10.6.1(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)) + '@storybook/svelte-vite': 10.6.5(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + storybook: 10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8) + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + transitivePeerDependencies: + - '@sveltejs/vite-plugin-svelte' + + '@sveltejs/acorn-typescript@1.0.13(acorn@8.18.0)': + dependencies: + acorn: 8.18.0 + + '@sveltejs/adapter-bun@1.0.0(patch_hash=7a4161ba8d719c64682bae85cfec228d726017b4217ca64b3758527625d972fb)(@sveltejs/kit@3.1.0(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@6.0.3)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))': + dependencies: + '@sveltejs/kit': 3.1.0(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@6.0.3)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + bun-types: 1.4.4 + + '@sveltejs/adapter-cloudflare@8.0.0(@sveltejs/kit@3.1.0(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@6.0.3)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(wrangler@4.125.0(@cloudflare/workers-types@5.20261012.1))': + dependencies: + '@cloudflare/workers-types': 5.20261012.1 + '@rolldown/pluginutils': 1.0.1 + '@sveltejs/kit': 3.1.0(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@6.0.3)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + wrangler: 4.125.0(@cloudflare/workers-types@5.20261012.1) + + '@sveltejs/kit@3.1.0(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@6.0.3)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': + dependencies: + '@rolldown/pluginutils': 1.0.1 + '@standard-schema/spec': 1.1.0 + '@sveltejs/vite-plugin-svelte': 7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + cookie: 2.0.1 + devalue: 5.9.4 + esm-env: 1.2.2 + magic-string: 1.4.3 + mrmime: 2.0.1 + sirv: 3.0.2 + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + optionalDependencies: + typescript: 6.0.3 + + '@sveltejs/load-config@0.2.3': {} + + '@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': + dependencies: + deepmerge: 4.3.1 + magic-string: 1.4.3 + obug: 2.2.1 + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + vitefu: 1.1.3(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + + '@tailwindcss/node@4.3.3': + dependencies: + '@jridgewell/remapping': 2.3.5 + enhanced-resolve: 5.24.5 + jiti: 2.7.0 + lightningcss: 1.32.0 + magic-string: 0.30.21 + source-map-js: 1.2.1 + tailwindcss: 4.3.3 + + '@tailwindcss/oxide-android-arm64@4.3.3': + optional: true + + '@tailwindcss/oxide-darwin-arm64@4.3.3': + optional: true + + '@tailwindcss/oxide-darwin-x64@4.3.3': + optional: true + + '@tailwindcss/oxide-freebsd-x64@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-arm64-gnu@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-arm64-musl@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-x64-gnu@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-x64-musl@4.3.3': + optional: true + + '@tailwindcss/oxide-wasm32-wasi@4.3.3': + optional: true + + '@tailwindcss/oxide-win32-arm64-msvc@4.3.3': + optional: true + + '@tailwindcss/oxide-win32-x64-msvc@4.3.3': + optional: true + + '@tailwindcss/oxide@4.3.3': + optionalDependencies: + '@tailwindcss/oxide-android-arm64': 4.3.3 + '@tailwindcss/oxide-darwin-arm64': 4.3.3 + '@tailwindcss/oxide-darwin-x64': 4.3.3 + '@tailwindcss/oxide-freebsd-x64': 4.3.3 + '@tailwindcss/oxide-linux-arm-gnueabihf': 4.3.3 + '@tailwindcss/oxide-linux-arm64-gnu': 4.3.3 + '@tailwindcss/oxide-linux-arm64-musl': 4.3.3 + '@tailwindcss/oxide-linux-x64-gnu': 4.3.3 + '@tailwindcss/oxide-linux-x64-musl': 4.3.3 + '@tailwindcss/oxide-wasm32-wasi': 4.3.3 + '@tailwindcss/oxide-win32-arm64-msvc': 4.3.3 + '@tailwindcss/oxide-win32-x64-msvc': 4.3.3 + + '@tailwindcss/vite@4.3.3(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': + dependencies: + '@tailwindcss/node': 4.3.3 + '@tailwindcss/oxide': 4.3.3 + tailwindcss: 4.3.3 + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + + '@tanstack/svelte-virtual@3.13.35(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))': + dependencies: + '@tanstack/virtual-core': 3.17.7 + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + + '@tanstack/virtual-core@3.17.7': {} + + '@testing-library/dom@10.4.1': + dependencies: + '@babel/code-frame': 7.29.7 + '@babel/runtime': 7.29.7 + '@types/aria-query': 5.0.4 + aria-query: 5.3.2 + dom-accessibility-api: 0.5.16 + lz-string: 1.5.0 + picocolors: 1.1.1 + pretty-format: 27.5.1 + + '@testing-library/jest-dom@6.9.1': + dependencies: + '@adobe/css-tools': 4.5.0 + aria-query: 5.3.2 + css.escape: 1.5.1 + dom-accessibility-api: 0.6.3 + picocolors: 1.1.1 + redent: 3.0.0 + + '@testing-library/user-event@14.6.7(@testing-library/dom@10.4.1)': + dependencies: + '@testing-library/dom': 10.4.1 + + '@tybys/wasm-util@0.10.3': + dependencies: + tslib: 2.8.1 + optional: true + + '@types/aria-query@5.0.4': {} + + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + + '@types/deep-eql@4.0.2': {} + + '@types/esrecurse@4.3.1': {} + + '@types/estree@1.0.9': {} + + '@types/hast@3.0.5': + dependencies: + '@types/unist': 3.0.3 + + '@types/json-schema@7.0.15': {} + + '@types/linkify-it@5.0.0': {} + + '@types/markdown-it-emoji@3.0.1': + dependencies: + '@types/markdown-it': 14.1.2 + + '@types/markdown-it@14.1.2': + dependencies: + '@types/linkify-it': 5.0.0 + '@types/mdurl': 2.0.0 + + '@types/mdast@4.0.4': + dependencies: + '@types/unist': 3.0.3 + + '@types/mdurl@2.0.0': {} + + '@types/mdx@2.0.14': {} + + '@types/node@24.13.3': + dependencies: + undici-types: 7.18.2 + + '@types/react-dom@19.2.4(@types/react@19.2.18)': + dependencies: + '@types/react': 19.2.18 + optional: true + + '@types/react@19.2.18': + dependencies: + csstype: 3.2.3 + + '@types/sanitize-html@2.16.1': + dependencies: + htmlparser2: 10.1.0 + + '@types/textarea-caret@3.0.4': {} + + '@types/unist@3.0.3': {} + + '@typescript-eslint/eslint-plugin@8.64.0(@typescript-eslint/parser@8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3))(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3)': + dependencies: + '@eslint-community/regexpp': 4.12.2 + '@typescript-eslint/parser': 8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3) + '@typescript-eslint/scope-manager': 8.64.0 + '@typescript-eslint/type-utils': 8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3) + '@typescript-eslint/utils': 8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3) + '@typescript-eslint/visitor-keys': 8.64.0 + eslint: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + ignore: 7.0.6 + natural-compare: 1.4.0 + ts-api-utils: 2.5.0(typescript@6.0.3) + typescript: 6.0.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/parser@8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3)': + dependencies: + '@typescript-eslint/scope-manager': 8.64.0 + '@typescript-eslint/types': 8.64.0 + '@typescript-eslint/typescript-estree': 8.64.0(supports-color@10.2.2)(typescript@6.0.3) + '@typescript-eslint/visitor-keys': 8.64.0 + debug: 4.4.3(supports-color@7.2.0) + eslint: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + typescript: 6.0.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/project-service@8.64.0(supports-color@10.2.2)(typescript@6.0.3)': + dependencies: + '@typescript-eslint/tsconfig-utils': 8.64.0(typescript@6.0.3) + '@typescript-eslint/types': 8.64.0 + debug: 4.4.3(supports-color@10.2.2) + typescript: 6.0.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/scope-manager@8.64.0': + dependencies: + '@typescript-eslint/types': 8.64.0 + '@typescript-eslint/visitor-keys': 8.64.0 + + '@typescript-eslint/tsconfig-utils@8.64.0(typescript@6.0.3)': + dependencies: + typescript: 6.0.3 + + '@typescript-eslint/type-utils@8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3)': + dependencies: + '@typescript-eslint/types': 8.64.0 + '@typescript-eslint/typescript-estree': 8.64.0(supports-color@10.2.2)(typescript@6.0.3) + '@typescript-eslint/utils': 8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3) + debug: 4.4.3(supports-color@7.2.0) + eslint: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + ts-api-utils: 2.5.0(typescript@6.0.3) + typescript: 6.0.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/types@8.64.0': {} + + '@typescript-eslint/typescript-estree@8.64.0(supports-color@10.2.2)(typescript@6.0.3)': + dependencies: + '@typescript-eslint/project-service': 8.64.0(supports-color@10.2.2)(typescript@6.0.3) + '@typescript-eslint/tsconfig-utils': 8.64.0(typescript@6.0.3) + '@typescript-eslint/types': 8.64.0 + '@typescript-eslint/visitor-keys': 8.64.0 + debug: 4.4.3(supports-color@10.2.2) + minimatch: 10.2.5 + semver: 7.8.5 + tinyglobby: 0.2.17 + ts-api-utils: 2.5.0(typescript@6.0.3) + typescript: 6.0.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/utils@8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3)': + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0)) + '@typescript-eslint/scope-manager': 8.64.0 + '@typescript-eslint/types': 8.64.0 + '@typescript-eslint/typescript-estree': 8.64.0(supports-color@10.2.2)(typescript@6.0.3) + eslint: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + typescript: 6.0.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/visitor-keys@8.64.0': + dependencies: + '@typescript-eslint/types': 8.64.0 + eslint-visitor-keys: 5.0.1 + + '@ungap/structured-clone@1.3.3': {} + + '@valibot/to-json-schema@1.7.1(valibot@1.4.2(typescript@6.0.3))': + dependencies: + valibot: 1.4.2(typescript@6.0.3) + + '@vitest/browser-playwright@5.0.4(playwright@1.61.1)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))(vitest@5.0.4)': + dependencies: + '@vitest/browser': 5.0.4(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))(vitest@5.0.4) + '@vitest/mocker': 5.0.4(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + playwright: 1.61.1 + tinyrainbow: 3.2.0 + vitest: 5.0.4(@types/node@24.13.3)(@vitest/browser-playwright@5.0.4)(@vitest/coverage-v8@5.0.4)(@vitest/ui@5.0.4)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + transitivePeerDependencies: + - bufferutil + - msw + - utf-8-validate + - vite + + '@vitest/browser@5.0.4(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))(vitest@5.0.4)': + dependencies: + '@blazediff/core': 1.10.0 + '@vitest/mocker': 5.0.4(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + '@vitest/ui': 5.0.4(vitest@5.0.4) + '@vitest/utils': 5.0.4 + magic-string: 1.4.3 + pngjs: 7.0.0 + sirv: 3.0.2 + tinyrainbow: 3.2.0 + vitest: 5.0.4(@types/node@24.13.3)(@vitest/browser-playwright@5.0.4)(@vitest/coverage-v8@5.0.4)(@vitest/ui@5.0.4)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + ws: 8.22.0 + transitivePeerDependencies: + - bufferutil + - msw + - utf-8-validate + - vite + + '@vitest/coverage-v8@5.0.4(@vitest/browser@5.0.4)(vitest@5.0.4)': + dependencies: + '@bcoe/v8-coverage': 1.0.2 + '@vitest/istanbul-lib-coverage': 1.0.2 + '@vitest/istanbul-lib-report': 1.0.2 + ast-v8-to-istanbul: 1.0.7 + magicast: 0.5.5 + obug: 3.0.0 + std-env: 4.3.0 + tinyrainbow: 3.2.0 + vitest: 5.0.4(@types/node@24.13.3)(@vitest/browser-playwright@5.0.4)(@vitest/coverage-v8@5.0.4)(@vitest/ui@5.0.4)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + optionalDependencies: + '@vitest/browser': 5.0.4(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))(vitest@5.0.4) + + '@vitest/expect@3.2.4': + dependencies: + '@types/chai': 5.2.3 + '@vitest/spy': 3.2.4 + '@vitest/utils': 3.2.4 + chai: 5.3.3 + tinyrainbow: 2.0.0 + + '@vitest/istanbul-lib-coverage@1.0.2': {} + + '@vitest/istanbul-lib-report@1.0.2': + dependencies: + '@vitest/istanbul-lib-coverage': 1.0.2 + + '@vitest/mocker@5.0.4(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + '@vitest/spy': 5.0.4 + estree-walker: 3.0.3 + magic-string: 1.4.3 + optionalDependencies: + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + + '@vitest/pretty-format@3.2.4': + dependencies: + tinyrainbow: 2.0.0 + + '@vitest/pretty-format@5.0.4': + dependencies: + tinyrainbow: 3.2.0 + + '@vitest/spy@3.2.4': + dependencies: + tinyspy: 4.0.4 + + '@vitest/spy@5.0.4': {} + + '@vitest/ui@5.0.4(vitest@5.0.4)': + dependencies: + '@vitest/utils': 5.0.4 + fflate: 0.8.3 + flatted: 3.4.4 + pathe: 2.0.3 + sirv: 3.0.2 + tinyrainbow: 3.2.0 + vitest: 5.0.4(@types/node@24.13.3)(@vitest/browser-playwright@5.0.4)(@vitest/coverage-v8@5.0.4)(@vitest/ui@5.0.4)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + + '@vitest/utils@3.2.4': + dependencies: + '@vitest/pretty-format': 3.2.4 + loupe: 3.2.1 + tinyrainbow: 2.0.0 + + '@vitest/utils@5.0.4': + dependencies: + '@vitest/pretty-format': 5.0.4 + convert-source-map: 2.0.0 + tinyrainbow: 3.2.0 + + '@webcontainer/env@1.1.1': {} + + acorn-jsx@5.3.2(acorn@8.18.0): + dependencies: + acorn: 8.18.0 + + acorn@8.18.0: {} + + ajv@6.15.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + + ansi-regex@5.0.1: {} + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + ansi-styles@5.2.0: {} + + ansi_up@6.0.6: {} + + argparse@2.0.1: {} + + aria-query@5.3.2: {} + + assertion-error@2.0.1: {} + + ast-types@0.16.1: + dependencies: + tslib: 2.8.1 + + ast-v8-to-istanbul@1.0.7: + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + estree-walker: 3.0.3 + js-tokens: 10.0.0 + + axe-core@4.12.1: {} + + axobject-query@4.1.0: {} + + balanced-match@4.0.4: {} + + blake3-wasm@2.1.5: {} + + boneyard-js@1.9.0(patch_hash=e9bda6b0720bed34eefee605bd041784ac2849ae4d27120d9815663d342f694f)(react@19.2.8)(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)): + dependencies: + playwright: 1.61.1 + optionalDependencies: + '@chenglou/pretext': 0.0.5 + react: 19.2.8 + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + + brace-expansion@5.0.7: + dependencies: + balanced-match: 4.0.4 + + bun-types@1.4.4: + dependencies: + '@types/node': 24.13.3 + undici-types: 7.18.2 + + bundle-name@4.1.0: + dependencies: + run-applescript: 7.1.0 + + ccount@2.0.1: {} + + chai@5.3.3: + dependencies: + assertion-error: 2.0.1 + check-error: 2.1.3 + deep-eql: 5.0.2 + loupe: 3.2.1 + pathval: 2.0.1 + + chai@6.3.0: {} + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + character-entities-html4@2.1.0: {} + + character-entities-legacy@3.0.0: {} + + check-error@2.1.3: {} + + chokidar@4.0.3: + dependencies: + readdirp: 4.1.2 + + clsx@2.1.1: {} + + cobe@2.0.1(patch_hash=3b6a4b0984026f2dc3a5794e2542b83b483af86b03ee6531f396e9d6fd62e62d): {} + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + comma-separated-tokens@2.0.3: {} + + confbox@0.1.8: {} + + confbox@0.3.1: {} + + convert-source-map@2.0.0: {} + + cookie@1.1.1: {} + + cookie@2.0.1: {} + + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + css.escape@1.5.1: {} + + cssesc@3.0.0: {} + + csstype@3.2.3: {} + + cuelume@0.2.4: {} + + dayjs@1.11.23: {} + + debug@4.4.3(supports-color@10.2.2): + dependencies: + ms: 2.1.3 + optionalDependencies: + supports-color: 10.2.2 + + debug@4.4.3(supports-color@7.2.0): + dependencies: + ms: 2.1.3 + optionalDependencies: + supports-color: 7.2.0 + + dedent-js@1.0.1: {} + + dedent@1.7.2: {} + + deep-eql@5.0.2: {} + + deep-is@0.1.4: {} + + deepmerge@4.3.1: {} + + default-browser-id@5.0.1: {} + + default-browser@5.5.0: + dependencies: + bundle-name: 4.1.0 + default-browser-id: 5.0.1 + + define-lazy-prop@3.0.0: {} + + dequal@2.0.3: {} + + detect-libc@2.1.2: {} + + devalue@5.9.4: {} + + devlop@1.1.0: + dependencies: + dequal: 2.0.3 + + diff@9.0.0: {} + + dom-accessibility-api@0.5.16: {} + + dom-accessibility-api@0.6.3: {} + + dom-serializer@2.0.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + + dom-serializer@3.1.1: + dependencies: + domelementtype: 3.0.0 + domhandler: 6.0.1 + entities: 8.0.0 + + domelementtype@2.3.0: {} + + domelementtype@3.0.0: {} + + domhandler@5.0.3: + dependencies: + domelementtype: 2.3.0 + + domhandler@6.0.1: + dependencies: + domelementtype: 3.0.0 + + domutils@3.2.2: + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + + domutils@4.0.2: + dependencies: + dom-serializer: 3.1.1 + domelementtype: 3.0.0 + domhandler: 6.0.1 + + enhanced-resolve@5.21.6: + dependencies: + graceful-fs: 4.2.11 + tapable: 2.3.3 + + enhanced-resolve@5.24.5: + dependencies: + graceful-fs: 4.2.11 + tapable: 2.3.3 + + entities@4.5.0: {} + + entities@7.0.1: {} + + entities@8.0.0: {} + + error-stack-parser-es@1.0.5: {} + + es-module-lexer@2.3.2: {} + + es-toolkit@1.49.0: {} + + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + + escape-string-regexp@4.0.0: {} + + eslint-config-prettier@10.1.8(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0)): + dependencies: + eslint: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + + eslint-plugin-better-tailwindcss@4.7.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(tailwindcss@4.3.3)(typescript@6.0.3): + dependencies: + '@eslint/css-tree': 4.0.5 + '@valibot/to-json-schema': 1.7.1(valibot@1.4.2(typescript@6.0.3)) + enhanced-resolve: 5.24.5 + jiti: 2.7.0 + synckit: 0.11.13 + tailwind-csstree: 0.3.3 + tailwindcss: 4.3.3 + tsconfig-paths-webpack-plugin: 4.2.0 + valibot: 1.4.2(typescript@6.0.3) + optionalDependencies: + eslint: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + transitivePeerDependencies: + - '@eslint/css' + - typescript + + eslint-plugin-storybook@10.6.5(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3): + dependencies: + '@typescript-eslint/types': 8.64.0 + '@typescript-eslint/utils': 8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3) + eslint: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + transitivePeerDependencies: + - supports-color + - typescript + + eslint-plugin-svelte@3.23.1(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)): + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0)) + '@jridgewell/sourcemap-codec': 1.5.5 + eslint: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + esutils: 2.0.3 + globals: 16.5.0 + known-css-properties: 0.37.0 + postcss: 8.5.19 + postcss-load-config: 3.1.4(postcss@8.5.19) + postcss-safe-parser: 7.0.1(postcss@8.5.19) + semver: 7.8.5 + svelte-eslint-parser: 1.8.0(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)) + optionalDependencies: + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + transitivePeerDependencies: + - ts-node + + eslint-scope@8.4.0: + dependencies: + esrecurse: 4.3.0 + estraverse: 5.3.0 + + eslint-scope@9.1.2: + dependencies: + '@types/esrecurse': 4.3.1 + '@types/estree': 1.0.9 + esrecurse: 4.3.0 + estraverse: 5.3.0 + + eslint-visitor-keys@3.4.3: {} + + eslint-visitor-keys@4.2.1: {} + + eslint-visitor-keys@5.0.1: {} + + eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0): + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0)) + '@eslint-community/regexpp': 4.12.2 + '@eslint/config-array': 0.23.5(supports-color@7.2.0) + '@eslint/config-helpers': 0.6.0 + '@eslint/core': 1.2.1 + '@eslint/plugin-kit': 0.7.2 + '@humanfs/node': 0.16.8 + '@humanwhocodes/module-importer': 1.0.1 + '@humanwhocodes/retry': 0.4.3 + '@types/estree': 1.0.9 + ajv: 6.15.0 + cross-spawn: 7.0.6 + debug: 4.4.3(supports-color@7.2.0) + escape-string-regexp: 4.0.0 + eslint-scope: 9.1.2 + eslint-visitor-keys: 5.0.1 + espree: 11.2.0 + esquery: 1.7.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 8.0.0 + find-up: 5.0.0 + glob-parent: 6.0.2 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + json-stable-stringify-without-jsonify: 1.0.1 + minimatch: 10.2.5 + natural-compare: 1.4.0 + optionator: 0.9.4 + optionalDependencies: + jiti: 2.7.0 + transitivePeerDependencies: + - supports-color + + esm-env@1.2.2: {} + + espree@10.4.0: + dependencies: + acorn: 8.18.0 + acorn-jsx: 5.3.2(acorn@8.18.0) + eslint-visitor-keys: 4.2.1 + + espree@11.2.0: + dependencies: + acorn: 8.18.0 + acorn-jsx: 5.3.2(acorn@8.18.0) + eslint-visitor-keys: 5.0.1 + + esprima@4.0.1: {} + + esquery@1.7.0: + dependencies: + estraverse: 5.3.0 + + esrap@1.2.2: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + '@types/estree': 1.0.9 + + esrap@1.4.9: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + + esrap@2.4.0(@typescript-eslint/types@8.64.0): + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + optionalDependencies: + '@typescript-eslint/types': 8.64.0 + + esrecurse@4.3.0: + dependencies: + estraverse: 5.3.0 + + estraverse@5.3.0: {} + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + esutils@2.0.3: {} + + expect-type@1.4.0: {} + + exsolve@1.1.1: {} + + fast-deep-equal@3.1.3: {} + + fast-json-stable-stringify@2.1.0: {} + + fast-levenshtein@2.0.6: {} + + fdir@6.5.0(picomatch@4.0.7): + optionalDependencies: + picomatch: 4.0.7 + + feed@6.0.0: + dependencies: + xml-js: 1.6.11 + + fflate@0.8.3: {} + + file-entry-cache@8.0.0: + dependencies: + flat-cache: 4.0.1 + + find-up@5.0.0: + dependencies: + locate-path: 6.0.0 + path-exists: 4.0.0 + + flat-cache@4.0.1: + dependencies: + flatted: 3.4.2 + keyv: 4.5.4 + + flatted@3.4.2: {} + + flatted@3.4.4: {} + + fsevents@2.3.2: + optional: true + + fsevents@2.3.3: + optional: true + + fuse.js@7.5.0: {} + + glob-parent@6.0.2: + dependencies: + is-glob: 4.0.3 + + globals@16.5.0: {} + + globals@17.7.0: {} + + graceful-fs@4.2.11: {} + + has-flag@4.0.0: {} + + hast-util-to-html@9.0.5: + dependencies: + '@types/hast': 3.0.5 + '@types/unist': 3.0.3 + ccount: 2.0.1 + comma-separated-tokens: 2.0.3 + hast-util-whitespace: 3.0.0 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.2.1 + property-information: 7.2.0 + space-separated-tokens: 2.0.2 + stringify-entities: 4.0.4 + zwitch: 2.0.4 + + hast-util-whitespace@3.0.0: + dependencies: + '@types/hast': 3.0.5 + + html-void-elements@3.0.0: {} + + htmlparser2@10.1.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.2.2 + entities: 7.0.1 + + htmlparser2@12.0.0: + dependencies: + domelementtype: 3.0.0 + domhandler: 6.0.1 + domutils: 4.0.2 + entities: 8.0.0 + + ignore@5.3.2: {} + + ignore@7.0.6: {} + + import-meta-resolve@4.2.0: {} + + imurmurhash@0.1.4: {} + + indent-string@4.0.0: {} + + is-docker@3.0.0: {} + + is-extglob@2.1.1: {} + + is-glob@4.0.3: + dependencies: + is-extglob: 2.1.1 + + is-inside-container@1.0.0: + dependencies: + is-docker: 3.0.0 + + is-plain-object@5.1.0: {} + + is-reference@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + is-wsl@3.1.1: + dependencies: + is-inside-container: 1.0.0 + + isexe@2.0.0: {} + + jiti@2.7.0: {} + + js-tokens@10.0.0: {} + + js-tokens@4.0.0: {} + + json-buffer@3.0.1: {} + + json-schema-traverse@0.4.1: {} + + json-stable-stringify-without-jsonify@1.0.1: {} + + json5@2.2.3: {} + + jsonc-parser@3.3.1: {} + + keyv@4.5.4: + dependencies: + json-buffer: 3.0.1 + + kleur@4.1.5: {} + + known-css-properties@0.37.0: {} + + launder@1.7.2: + dependencies: + dayjs: 1.11.23 + + levn@0.4.1: + dependencies: + prelude-ls: 1.2.1 + type-check: 0.4.0 + + lightningcss-android-arm64@1.32.0: + optional: true + + lightningcss-android-arm64@1.33.0: + optional: true + + lightningcss-darwin-arm64@1.32.0: + optional: true + + lightningcss-darwin-arm64@1.33.0: + optional: true + + lightningcss-darwin-x64@1.32.0: + optional: true + + lightningcss-darwin-x64@1.33.0: + optional: true + + lightningcss-freebsd-x64@1.32.0: + optional: true + + lightningcss-freebsd-x64@1.33.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.32.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.33.0: + optional: true + + lightningcss-linux-arm64-gnu@1.32.0: + optional: true + + lightningcss-linux-arm64-gnu@1.33.0: + optional: true + + lightningcss-linux-arm64-musl@1.32.0: + optional: true + + lightningcss-linux-arm64-musl@1.33.0: + optional: true + + lightningcss-linux-x64-gnu@1.32.0: + optional: true + + lightningcss-linux-x64-gnu@1.33.0: + optional: true + + lightningcss-linux-x64-musl@1.32.0: + optional: true + + lightningcss-linux-x64-musl@1.33.0: + optional: true + + lightningcss-win32-arm64-msvc@1.32.0: + optional: true + + lightningcss-win32-arm64-msvc@1.33.0: + optional: true + + lightningcss-win32-x64-msvc@1.32.0: + optional: true + + lightningcss-win32-x64-msvc@1.33.0: + optional: true + + lightningcss@1.32.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.32.0 + lightningcss-darwin-arm64: 1.32.0 + lightningcss-darwin-x64: 1.32.0 + lightningcss-freebsd-x64: 1.32.0 + lightningcss-linux-arm-gnueabihf: 1.32.0 + lightningcss-linux-arm64-gnu: 1.32.0 + lightningcss-linux-arm64-musl: 1.32.0 + lightningcss-linux-x64-gnu: 1.32.0 + lightningcss-linux-x64-musl: 1.32.0 + lightningcss-win32-arm64-msvc: 1.32.0 + lightningcss-win32-x64-msvc: 1.32.0 + + lightningcss@1.33.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.33.0 + lightningcss-darwin-arm64: 1.33.0 + lightningcss-darwin-x64: 1.33.0 + lightningcss-freebsd-x64: 1.33.0 + lightningcss-linux-arm-gnueabihf: 1.33.0 + lightningcss-linux-arm64-gnu: 1.33.0 + lightningcss-linux-arm64-musl: 1.33.0 + lightningcss-linux-x64-gnu: 1.33.0 + lightningcss-linux-x64-musl: 1.33.0 + lightningcss-win32-arm64-msvc: 1.33.0 + lightningcss-win32-x64-msvc: 1.33.0 + + lilconfig@2.1.0: {} + + linkify-it@5.0.2: + dependencies: + uc.micro: 2.1.0 + + local-pkg@1.2.1: + dependencies: + mlly: 1.8.2 + pkg-types: 2.3.3 + quansync: 0.2.11 + + locate-character@3.0.0: {} + + locate-path@6.0.0: + dependencies: + p-locate: 5.0.0 + + loupe@3.2.1: {} + + lru_map@0.4.1: {} + + lz-string@1.5.0: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + + magic-string@1.4.3: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + + magicast@0.5.5: + dependencies: + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 + source-map-js: 1.2.1 + + markdown-it-anchor@9.2.1(@types/markdown-it@14.1.2)(markdown-it@14.3.2): + dependencies: + '@types/markdown-it': 14.1.2 + markdown-it: 14.3.2 + + markdown-it-emoji@3.1.0: {} + + markdown-it@14.3.2: + dependencies: + argparse: 2.0.1 + entities: 4.5.0 + linkify-it: 5.0.2 + mdurl: 2.1.0 + punycode.js: 2.3.1 + uc.micro: 2.1.0 + + mdast-util-to-hast@13.2.1: + dependencies: + '@types/hast': 3.0.5 + '@types/mdast': 4.0.4 + '@ungap/structured-clone': 1.3.3 + devlop: 1.1.0 + micromark-util-sanitize-uri: 2.0.1 + trim-lines: 3.0.1 + unist-util-position: 5.0.0 + unist-util-visit: 5.1.0 + vfile: 6.0.3 + + mdn-data@2.29.0: {} + + mdurl@2.1.0: {} + + micromark-util-character@2.1.1: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-util-encode@2.0.1: {} + + micromark-util-sanitize-uri@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-encode: 2.0.1 + micromark-util-symbol: 2.0.1 + + micromark-util-symbol@2.0.1: {} + + micromark-util-types@2.0.2: {} + + min-indent@1.0.1: {} + + miniflare@5.20260820.0-alpha: + dependencies: + '@cspotcode/source-map-support': 0.8.1 + sharp: 0.35.2 + undici: 7.29.0 + workerd: 1.20260820.1 + ws: 8.21.0 + youch: 4.1.0-beta.10 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + minimatch@10.2.5: + dependencies: + brace-expansion: 5.0.7 + + minimist@1.2.8: {} + + mlly@1.8.2: + dependencies: + acorn: 8.18.0 + pathe: 2.0.3 + pkg-types: 1.3.1 + ufo: 1.6.4 + + mri@1.2.0: {} + + mrmime@2.0.1: {} + + ms@2.1.3: {} + + nanoid@3.3.16: {} + + nanoid@3.3.20: {} + + nanoid@5.1.16: {} + + natural-compare@1.4.0: {} + + obug@2.2.1: {} + + obug@3.0.0: {} + + oniguruma-parser@0.12.2: {} + + oniguruma-to-es@4.3.6: + dependencies: + oniguruma-parser: 0.12.2 + regex: 6.1.0 + regex-recursion: 6.0.2 + + open@10.2.0: + dependencies: + default-browser: 5.5.0 + define-lazy-prop: 3.0.0 + is-inside-container: 1.0.0 + wsl-utils: 0.1.0 + + optionator@0.9.4: + dependencies: + deep-is: 0.1.4 + fast-levenshtein: 2.0.6 + levn: 0.4.1 + prelude-ls: 1.2.1 + type-check: 0.4.0 + word-wrap: 1.2.5 + + oxc-parser@0.127.0: + dependencies: + '@oxc-project/types': 0.127.0 + optionalDependencies: + '@oxc-parser/binding-android-arm-eabi': 0.127.0 + '@oxc-parser/binding-android-arm64': 0.127.0 + '@oxc-parser/binding-darwin-arm64': 0.127.0 + '@oxc-parser/binding-darwin-x64': 0.127.0 + '@oxc-parser/binding-freebsd-x64': 0.127.0 + '@oxc-parser/binding-linux-arm-gnueabihf': 0.127.0 + '@oxc-parser/binding-linux-arm-musleabihf': 0.127.0 + '@oxc-parser/binding-linux-arm64-gnu': 0.127.0 + '@oxc-parser/binding-linux-arm64-musl': 0.127.0 + '@oxc-parser/binding-linux-ppc64-gnu': 0.127.0 + '@oxc-parser/binding-linux-riscv64-gnu': 0.127.0 + '@oxc-parser/binding-linux-riscv64-musl': 0.127.0 + '@oxc-parser/binding-linux-s390x-gnu': 0.127.0 + '@oxc-parser/binding-linux-x64-gnu': 0.127.0 + '@oxc-parser/binding-linux-x64-musl': 0.127.0 + '@oxc-parser/binding-openharmony-arm64': 0.127.0 + '@oxc-parser/binding-wasm32-wasi': 0.127.0 + '@oxc-parser/binding-win32-arm64-msvc': 0.127.0 + '@oxc-parser/binding-win32-ia32-msvc': 0.127.0 + '@oxc-parser/binding-win32-x64-msvc': 0.127.0 + + oxc-resolver@11.21.2: + optionalDependencies: + '@oxc-resolver/binding-android-arm-eabi': 11.21.2 + '@oxc-resolver/binding-android-arm64': 11.21.2 + '@oxc-resolver/binding-darwin-arm64': 11.21.2 + '@oxc-resolver/binding-darwin-x64': 11.21.2 + '@oxc-resolver/binding-freebsd-x64': 11.21.2 + '@oxc-resolver/binding-linux-arm-gnueabihf': 11.21.2 + '@oxc-resolver/binding-linux-arm-musleabihf': 11.21.2 + '@oxc-resolver/binding-linux-arm64-gnu': 11.21.2 + '@oxc-resolver/binding-linux-arm64-musl': 11.21.2 + '@oxc-resolver/binding-linux-ppc64-gnu': 11.21.2 + '@oxc-resolver/binding-linux-riscv64-gnu': 11.21.2 + '@oxc-resolver/binding-linux-riscv64-musl': 11.21.2 + '@oxc-resolver/binding-linux-s390x-gnu': 11.21.2 + '@oxc-resolver/binding-linux-x64-gnu': 11.21.2 + '@oxc-resolver/binding-linux-x64-musl': 11.21.2 + '@oxc-resolver/binding-openharmony-arm64': 11.21.2 + '@oxc-resolver/binding-wasm32-wasi': 11.21.2 + '@oxc-resolver/binding-win32-arm64-msvc': 11.21.2 + '@oxc-resolver/binding-win32-x64-msvc': 11.21.2 + + p-limit@3.1.0: + dependencies: + yocto-queue: 0.1.0 + + p-locate@5.0.0: + dependencies: + p-limit: 3.1.0 + + package-manager-detector@1.8.0: {} + + parse-srcset@1.0.2: {} + + path-exists@4.0.0: {} + + path-key@3.1.1: {} + + path-to-regexp@6.3.0: {} + + pathe@2.0.3: {} + + pathval@2.0.1: {} + + pele@1.5.2: {} + + picocolors@1.1.1: {} + + picomatch@4.0.7: {} + + pkg-types@1.3.1: + dependencies: + confbox: 0.1.8 + mlly: 1.8.2 + pathe: 2.0.3 + + pkg-types@2.3.3: + dependencies: + confbox: 0.3.1 + exsolve: 1.1.1 + pathe: 2.0.3 + + playwright-core@1.61.1: {} + + playwright@1.61.1: + dependencies: + playwright-core: 1.61.1 + optionalDependencies: + fsevents: 2.3.2 + + pngjs@7.0.0: {} + + postcss-load-config@3.1.4(postcss@8.5.19): + dependencies: + lilconfig: 2.1.0 + yaml: 1.10.3 + optionalDependencies: + postcss: 8.5.19 + + postcss-safe-parser@7.0.1(postcss@8.5.19): + dependencies: + postcss: 8.5.19 + + postcss-scss@4.0.9(postcss@8.5.19): + dependencies: + postcss: 8.5.19 + + postcss-selector-parser@7.1.4: + dependencies: + cssesc: 3.0.0 + util-deprecate: 1.0.2 + + postcss@8.5.19: + dependencies: + nanoid: 3.3.16 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + postcss@8.5.29: + dependencies: + nanoid: 3.3.20 + picocolors: 1.1.1 + source-map-js: 1.2.2 + + prelude-ls@1.2.1: {} + + prettier-plugin-svelte@4.1.1(prettier@3.9.5)(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)): + dependencies: + prettier: 3.9.5 + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + + prettier-plugin-tailwindcss@0.8.1(prettier-plugin-svelte@4.1.1(prettier@3.9.5)(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)))(prettier@3.9.5): + dependencies: + prettier: 3.9.5 + optionalDependencies: + prettier-plugin-svelte: 4.1.1(prettier@3.9.5)(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)) + + prettier@3.9.5: {} + + pretty-format@27.5.1: + dependencies: + ansi-regex: 5.0.1 + ansi-styles: 5.2.0 + react-is: 17.0.2 + + property-information@7.2.0: {} + + punycode.js@2.3.1: {} + + punycode@2.3.1: {} + + quansync@0.2.11: {} + + react-dom@19.2.8(react@19.2.8): + dependencies: + react: 19.2.8 + scheduler: 0.27.0 + + react-is@17.0.2: {} + + react@19.2.8: {} + + readdirp@4.1.2: {} + + recast@0.23.12: + dependencies: + ast-types: 0.16.1 + esprima: 4.0.1 + source-map: 0.6.1 + tiny-invariant: 1.3.3 + tslib: 2.8.1 + + redent@3.0.0: + dependencies: + indent-string: 4.0.0 + strip-indent: 3.0.0 + + regex-recursion@6.0.2: + dependencies: + regex-utilities: 2.3.0 + + regex-utilities@2.3.0: {} + + regex@6.1.0: + dependencies: + regex-utilities: 2.3.0 + + rolldown@1.2.13: + dependencies: + '@oxc-project/types': 0.153.0 + '@rolldown/pluginutils': 1.0.1 + optionalDependencies: + '@rolldown/binding-android-arm-eabi': 1.2.13 + '@rolldown/binding-android-arm64': 1.2.13 + '@rolldown/binding-darwin-arm64': 1.2.13 + '@rolldown/binding-darwin-x64': 1.2.13 + '@rolldown/binding-freebsd-x64': 1.2.13 + '@rolldown/binding-linux-arm-gnueabihf': 1.2.13 + '@rolldown/binding-linux-arm64-gnu': 1.2.13 + '@rolldown/binding-linux-arm64-musl': 1.2.13 + '@rolldown/binding-linux-ppc64-gnu': 1.2.13 + '@rolldown/binding-linux-s390x-gnu': 1.2.13 + '@rolldown/binding-linux-x64-gnu': 1.2.13 + '@rolldown/binding-linux-x64-musl': 1.2.13 + '@rolldown/binding-openharmony-arm64': 1.2.13 + '@rolldown/binding-win32-arm64-msvc': 1.2.13 + '@rolldown/binding-win32-x64-msvc': 1.2.13 + + rollup@4.62.2: + dependencies: + '@types/estree': 1.0.9 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.62.2 + '@rollup/rollup-android-arm64': 4.62.2 + '@rollup/rollup-darwin-arm64': 4.62.2 + '@rollup/rollup-darwin-x64': 4.62.2 + '@rollup/rollup-freebsd-arm64': 4.62.2 + '@rollup/rollup-freebsd-x64': 4.62.2 + '@rollup/rollup-linux-arm-gnueabihf': 4.62.2 + '@rollup/rollup-linux-arm-musleabihf': 4.62.2 + '@rollup/rollup-linux-arm64-gnu': 4.62.2 + '@rollup/rollup-linux-arm64-musl': 4.62.2 + '@rollup/rollup-linux-loong64-gnu': 4.62.2 + '@rollup/rollup-linux-loong64-musl': 4.62.2 + '@rollup/rollup-linux-ppc64-gnu': 4.62.2 + '@rollup/rollup-linux-ppc64-musl': 4.62.2 + '@rollup/rollup-linux-riscv64-gnu': 4.62.2 + '@rollup/rollup-linux-riscv64-musl': 4.62.2 + '@rollup/rollup-linux-s390x-gnu': 4.62.2 + '@rollup/rollup-linux-x64-gnu': 4.62.2 + '@rollup/rollup-linux-x64-musl': 4.62.2 + '@rollup/rollup-openbsd-x64': 4.62.2 + '@rollup/rollup-openharmony-arm64': 4.62.2 + '@rollup/rollup-win32-arm64-msvc': 4.62.2 + '@rollup/rollup-win32-ia32-msvc': 4.62.2 + '@rollup/rollup-win32-x64-gnu': 4.62.2 + '@rollup/rollup-win32-x64-msvc': 4.62.2 + fsevents: 2.3.3 + optional: true + + run-applescript@7.1.0: {} + + sade@1.8.1: + dependencies: + mri: 1.2.0 + + sanitize-html@2.18.1: + dependencies: + escape-string-regexp: 4.0.0 + htmlparser2: 12.0.0 + is-plain-object: 5.1.0 + launder: 1.7.2 + parse-srcset: 1.0.2 + postcss: 8.5.29 + + sax@1.6.1: {} + + scheduler@0.27.0: {} + + scule@1.3.0: {} + + semver@7.8.5: {} + + sharp@0.35.2: + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.8.5 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.35.2 + '@img/sharp-darwin-x64': 0.35.2 + '@img/sharp-freebsd-wasm32': 0.35.2 + '@img/sharp-libvips-darwin-arm64': 1.3.1 + '@img/sharp-libvips-darwin-x64': 1.3.1 + '@img/sharp-libvips-linux-arm': 1.3.1 + '@img/sharp-libvips-linux-arm64': 1.3.1 + '@img/sharp-libvips-linux-ppc64': 1.3.1 + '@img/sharp-libvips-linux-riscv64': 1.3.1 + '@img/sharp-libvips-linux-s390x': 1.3.1 + '@img/sharp-libvips-linux-x64': 1.3.1 + '@img/sharp-libvips-linuxmusl-arm64': 1.3.1 + '@img/sharp-libvips-linuxmusl-x64': 1.3.1 + '@img/sharp-linux-arm': 0.35.2 + '@img/sharp-linux-arm64': 0.35.2 + '@img/sharp-linux-ppc64': 0.35.2 + '@img/sharp-linux-riscv64': 0.35.2 + '@img/sharp-linux-s390x': 0.35.2 + '@img/sharp-linux-x64': 0.35.2 + '@img/sharp-linuxmusl-arm64': 0.35.2 + '@img/sharp-linuxmusl-x64': 0.35.2 + '@img/sharp-webcontainers-wasm32': 0.35.2 + '@img/sharp-win32-arm64': 0.35.2 + '@img/sharp-win32-ia32': 0.35.2 + '@img/sharp-win32-x64': 0.35.2 + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + shiki@4.3.1: + dependencies: + '@shikijs/core': 4.3.1 + '@shikijs/engine-javascript': 4.3.1 + '@shikijs/engine-oniguruma': 4.3.1 + '@shikijs/langs': 4.3.1 + '@shikijs/themes': 4.3.1 + '@shikijs/types': 4.3.1 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + + sirv@3.0.2: + dependencies: + '@polka/url': 1.0.0-next.29 + mrmime: 2.0.1 + totalist: 3.0.1 + + source-map-js@1.2.1: {} + + source-map-js@1.2.2: {} + + source-map@0.6.1: {} + + space-separated-tokens@2.0.2: {} + + std-env@4.3.0: {} + + storybook@10.6.5(@types/react@19.2.18)(prettier@3.9.5)(react@19.2.8): + dependencies: + '@storybook/global': 5.0.0 + '@storybook/icons': 2.1.0(react@19.2.8) + '@testing-library/dom': 10.4.1 + '@testing-library/jest-dom': 6.9.1 + '@testing-library/user-event': 14.6.7(@testing-library/dom@10.4.1) + '@vitest/expect': 3.2.4 + '@vitest/spy': 3.2.4 + '@webcontainer/env': 1.1.1 + esbuild: 0.28.1 + jsonc-parser: 3.3.1 + open: 10.2.0 + oxc-parser: 0.127.0 + oxc-resolver: 11.21.2 + recast: 0.23.12 + semver: 7.8.5 + use-sync-external-store: 1.6.0(react@19.2.8) + ws: 8.22.0 + optionalDependencies: + '@types/react': 19.2.18 + prettier: 3.9.5 + transitivePeerDependencies: + - bufferutil + - react + - utf-8-validate + + stringify-entities@4.0.4: + dependencies: + character-entities-html4: 2.1.0 + character-entities-legacy: 3.0.0 + + strip-bom@3.0.0: {} + + strip-indent@3.0.0: + dependencies: + min-indent: 1.0.1 + + supports-color@10.2.2: {} + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + svelte-ast-print@0.4.2(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)): + dependencies: + esrap: 1.2.2 + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + zimmerframe: 1.1.2 + + svelte-check@4.7.6(picomatch@4.0.7)(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@6.0.3): + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + '@sveltejs/load-config': 0.2.3 + chokidar: 4.0.3 + fdir: 6.5.0(picomatch@4.0.7) + picocolors: 1.1.1 + sade: 1.8.1 + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + typescript: 6.0.3 + transitivePeerDependencies: + - picomatch + + svelte-eslint-parser@1.8.0(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0)): + dependencies: + eslint-scope: 8.4.0 + eslint-visitor-keys: 4.2.1 + espree: 10.4.0 + postcss: 8.5.19 + postcss-scss: 4.0.9(postcss@8.5.19) + postcss-selector-parser: 7.1.4 + semver: 7.8.5 + optionalDependencies: + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + + svelte2tsx@0.7.57(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(typescript@5.9.3): + dependencies: + dedent-js: 1.0.1 + scule: 1.3.0 + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + typescript: 5.9.3 + + svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0): + dependencies: + '@jridgewell/remapping': 2.3.5 + '@jridgewell/sourcemap-codec': 1.6.0 + '@sveltejs/acorn-typescript': 1.0.13(acorn@8.18.0) + '@types/estree': 1.0.9 + acorn: 8.18.0 + aria-query: 5.3.2 + axobject-query: 4.1.0 + clsx: 2.1.1 + devalue: 5.9.4 + esm-env: 1.2.2 + esrap: 2.4.0(@typescript-eslint/types@8.64.0) + is-reference: 3.0.3 + locate-character: 3.0.0 + magic-string: 0.30.21 + zimmerframe: 1.1.4 + transitivePeerDependencies: + - '@typescript-eslint/types' + + svg-pathdata@9.0.0: {} + + synckit@0.11.13: + dependencies: + '@pkgr/core': 0.3.6 + + tagged-tag@1.0.0: {} + + tailwind-csstree@0.3.3: {} + + tailwind-merge@3.6.0: {} + + tailwind-variants@3.2.2(tailwind-merge@3.6.0)(tailwindcss@4.3.3): + dependencies: + tailwindcss: 4.3.3 + optionalDependencies: + tailwind-merge: 3.6.0 + + tailwindcss@4.3.3: {} + + tapable@2.3.3: {} + + textarea-caret@3.1.0: {} + + tiny-invariant@1.3.3: {} + + tinybench@6.2.2: {} + + tinyexec@1.3.1: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + + tinyrainbow@2.0.0: {} + + tinyrainbow@3.2.0: {} + + tinyspy@4.0.4: {} + + totalist@3.0.1: {} + + trim-lines@3.0.1: {} + + ts-api-utils@2.5.0(typescript@6.0.3): + dependencies: + typescript: 6.0.3 + + ts-dedent@2.3.0: {} + + tsconfig-paths-webpack-plugin@4.2.0: + dependencies: + chalk: 4.1.2 + enhanced-resolve: 5.21.6 + tapable: 2.3.3 + tsconfig-paths: 4.2.0 + + tsconfig-paths@4.2.0: + dependencies: + json5: 2.2.3 + minimist: 1.2.8 + strip-bom: 3.0.0 + + tslib@2.8.1: {} + + tw-easing-gradients@1.1.2(tailwindcss@4.3.3): + dependencies: + tailwindcss: 4.3.3 + + type-check@0.4.0: + dependencies: + prelude-ls: 1.2.1 + + type-fest@2.19.0: {} + + type-fest@5.8.0: + dependencies: + tagged-tag: 1.0.0 + + typescript-eslint@8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3): + dependencies: + '@typescript-eslint/eslint-plugin': 8.64.0(@typescript-eslint/parser@8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3))(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3) + '@typescript-eslint/parser': 8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3) + '@typescript-eslint/typescript-estree': 8.64.0(supports-color@10.2.2)(typescript@6.0.3) + '@typescript-eslint/utils': 8.64.0(eslint@10.7.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@10.2.2)(typescript@6.0.3) + eslint: 10.7.0(jiti@2.7.0)(supports-color@7.2.0) + typescript: 6.0.3 + transitivePeerDependencies: + - supports-color + + typescript@5.9.3: {} + + typescript@6.0.3: {} + + uc.micro@2.1.0: {} + + ufo@1.6.4: {} + + undici-types@7.18.2: {} + + undici@7.29.0: {} + + unenv@2.0.0-rc.24: + dependencies: + pathe: 2.0.3 + + unicode-segmenter@0.14.5: {} + + unist-util-is@6.0.1: + dependencies: + '@types/unist': 3.0.3 + + unist-util-position@5.0.0: + dependencies: + '@types/unist': 3.0.3 + + unist-util-stringify-position@4.0.0: + dependencies: + '@types/unist': 3.0.3 + + unist-util-visit-parents@6.0.2: + dependencies: + '@types/unist': 3.0.3 + unist-util-is: 6.0.1 + + unist-util-visit@5.1.0: + dependencies: + '@types/unist': 3.0.3 + unist-util-is: 6.0.1 + unist-util-visit-parents: 6.0.2 + + unplugin-icons@24.0.0(esbuild@0.28.1)(rolldown@1.2.13)(rollup@4.62.2)(svelte@5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0))(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)): + dependencies: + '@antfu/install-pkg': 1.1.0 + '@iconify/utils': 3.1.7 + local-pkg: 1.2.1 + obug: 2.2.1 + unplugin: 3.4.0(esbuild@0.28.1)(rolldown@1.2.13)(rollup@4.62.2)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + optionalDependencies: + svelte: 5.57.2(patch_hash=ea4cd117072ee1f738eef9403b73710aff64ede205564fb5b70173803dddb117)(@typescript-eslint/types@8.64.0) + transitivePeerDependencies: + - '@farmfe/core' + - '@rsbuild/core' + - '@rspack/core' + - bun-types-no-globals + - esbuild + - rolldown + - rollup + - unloader + - vite + - webpack + + unplugin@2.3.11: + dependencies: + '@jridgewell/remapping': 2.3.5 + acorn: 8.18.0 + picomatch: 4.0.7 + webpack-virtual-modules: 0.6.2 + + unplugin@3.4.0(esbuild@0.28.1)(rolldown@1.2.13)(rollup@4.62.2)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)): + dependencies: + '@jridgewell/remapping': 2.3.5 + picomatch: 4.0.7 + webpack-virtual-modules: 0.6.2 + optionalDependencies: + esbuild: 0.28.1 + rolldown: 1.2.13 + rollup: 4.62.2 + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + + uri-js@4.4.1: + dependencies: + punycode: 2.3.1 + + use-sync-external-store@1.6.0(react@19.2.8): + dependencies: + react: 19.2.8 + + util-deprecate@1.0.2: {} + + valibot@1.4.2(typescript@6.0.3): + optionalDependencies: + typescript: 6.0.3 + + vfile-message@4.0.3: + dependencies: + '@types/unist': 3.0.3 + unist-util-stringify-position: 4.0.0 + + vfile@6.0.3: + dependencies: + '@types/unist': 3.0.3 + vfile-message: 4.0.3 + + vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0): + dependencies: + lightningcss: 1.33.0 + picomatch: 4.0.7 + postcss: 8.5.29 + rolldown: 1.2.13 + tinyglobby: 0.2.17 + optionalDependencies: + '@types/node': 24.13.3 + esbuild: 0.28.1 + fsevents: 2.3.3 + jiti: 2.7.0 + yaml: 2.9.0 + + vitefu@1.1.3(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)): + optionalDependencies: + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + + vitest@5.0.4(@types/node@24.13.3)(@vitest/browser-playwright@5.0.4)(@vitest/coverage-v8@5.0.4)(@vitest/ui@5.0.4)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)): + dependencies: + '@types/chai': 5.2.3 + '@vitest/mocker': 5.0.4(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) + chai: 6.3.0 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 1.4.3 + obug: 3.0.0 + picomatch: 4.0.7 + std-env: 4.3.0 + tinybench: 6.2.2 + tinyexec: 1.3.1 + tinyglobby: 0.2.17 + vite: 8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + why-is-node-running: 3.2.1 + optionalDependencies: + '@types/node': 24.13.3 + '@vitest/browser-playwright': 5.0.4(playwright@1.61.1)(vite@8.3.4(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))(vitest@5.0.4) + '@vitest/coverage-v8': 5.0.4(@vitest/browser@5.0.4)(vitest@5.0.4) + '@vitest/ui': 5.0.4(vitest@5.0.4) + transitivePeerDependencies: + - msw + + webpack-virtual-modules@0.6.2: {} + + which@2.0.2: + dependencies: + isexe: 2.0.0 + + why-is-node-running@3.2.1: {} + + word-wrap@1.2.5: {} + + workerd@1.20260820.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20260820.1 + '@cloudflare/workerd-darwin-arm64': 1.20260820.1 + '@cloudflare/workerd-linux-64': 1.20260820.1 + '@cloudflare/workerd-linux-arm64': 1.20260820.1 + '@cloudflare/workerd-windows-64': 1.20260820.1 + + wrangler@4.125.0(@cloudflare/workers-types@5.20261012.1): + dependencies: + '@cloudflare/kv-asset-handler': 0.5.0 + '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260820.1) + blake3-wasm: 2.1.5 + esbuild: 0.28.1 + miniflare: 5.20260820.0-alpha + path-to-regexp: 6.3.0 + unenv: 2.0.0-rc.24 + workerd: 1.20260820.1 + optionalDependencies: + '@cloudflare/workers-types': 5.20261012.1 + fsevents: 2.3.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + ws@8.21.0: {} + + ws@8.22.0: {} + + wsl-utils@0.1.0: + dependencies: + is-wsl: 3.1.1 + + xml-js@1.6.11: + dependencies: + sax: 1.6.1 + + yaml@1.10.3: {} + + yaml@2.9.0: + optional: true + + yocto-queue@0.1.0: {} + + youch-core@0.3.3: + dependencies: + '@poppinss/exception': 1.2.3 + error-stack-parser-es: 1.0.5 + + youch@4.1.0-beta.10: + dependencies: + '@poppinss/colors': 4.1.6 + '@poppinss/dumper': 0.6.5 + '@speed-highlight/core': 1.2.17 + cookie: 1.1.1 + youch-core: 0.3.3 + + zimmerframe@1.1.2: {} + + zimmerframe@1.1.4: {} + + zwitch@2.0.4: {} diff --git a/web/pnpm-workspace.yaml b/web/pnpm-workspace.yaml new file mode 100644 index 0000000000000000000000000000000000000000..577c2717132ab4e6c6afb3da02d29bc17ac20bb0 --- /dev/null +++ b/web/pnpm-workspace.yaml @@ -0,0 +1,16 @@ +allowBuilds: + esbuild: true + workerd: true +overrides: + # @testing-library/dom pins 5.3.0, whose named exports the esm interop can't + # see; 5.3.2 assigns them where the cjs lexer finds them + aria-query: 5.3.2 + # held to the storybook release the rest of the addons are on (10.6.2 is under the + # minimum release age) + "@storybook/svelte": 10.6.1 +patchedDependencies: + "@pierre/diffs@1.6.1": patches/@pierre__diffs@1.6.1.patch + "@sveltejs/adapter-bun@1.0.0": patches/@sveltejs__adapter-bun@1.0.0.patch + boneyard-js@1.9.0: patches/boneyard-js@1.9.0.patch + cobe@2.0.1: patches/cobe@2.0.1.patch + svelte@5.57.2: patches/svelte@5.57.2.patch diff --git a/web/prettier.config.js b/web/prettier.config.js new file mode 100644 index 0000000000000000000000000000000000000000..104c4c36a2d2a898e4738d8d2777862a1a0ad494 --- /dev/null +++ b/web/prettier.config.js @@ -0,0 +1,13 @@ +/** @type {import("prettier").Config} */ +const config = { + useTabs: true, + singleQuote: false, + trailingComma: "none", + printWidth: 100, + plugins: ["prettier-plugin-svelte", "prettier-plugin-tailwindcss"], + overrides: [{ files: "*.svelte", options: { parser: "svelte" } }], + tailwindStylesheet: "./src/app.css", + tailwindFunctions: ["tv"] +}; + +export default config; diff --git a/web/scripts/bones.js b/web/scripts/bones.js new file mode 100755 index 0000000000000000000000000000000000000000..6bb3c48bbb23216f8558e9dc9e8f501915ce28a2 --- /dev/null +++ b/web/scripts/bones.js @@ -0,0 +1,111 @@ +#!/usr/bin/env node +import { readFileSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawn, spawnSync } from "node:child_process"; +import os from "node:os"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const root = resolve(__dirname, ".."); +const configPath = resolve(root, "boneyard.config.json"); +const config = JSON.parse(readFileSync(configPath, "utf-8")); + +const port = process.env.STORYBOOK_PORT || 6006; +const base = `http://127.0.0.1:${port}`; +const rawArgs = process.argv.slice(2); +const force = rawArgs.includes("--force"); +const args = rawArgs.filter((a) => a !== "--force"); + +let urls = []; +if (args.length > 0) { + const patterns = args.map((a) => a.toLowerCase()); + const matched = []; + for (const [name, entry] of Object.entries(config.skeletons || {})) { + if (patterns.some((p) => name.toLowerCase().includes(p))) { + matched.push(name); + urls.push(`${base}${entry.route}`); + } + } + if (urls.length === 0) { + console.error(`boneyard: no skeletons matched '${args.join(", ")}'`); + process.exit(1); + } + console.log(`boneyard: scoping to ${urls.length} skeleton(s): ${matched.join(", ")}\n`); +} else { + // For all skeletons, use explicit URLs so we can parallelize across workers + urls = Object.values(config.skeletons || {}).map((entry) => `${base}${entry.route}`); +} + +const flags = force ? ["--force"] : []; +const defaultConcurrency = Math.min(6, Math.max(1, os.cpus().length - 2)); +const concurrency = Math.min( + urls.length, + parseInt(process.env.BONES_CONCURRENCY || String(defaultConcurrency), 10) +); + +async function run() { + if (concurrency <= 1 || urls.length <= 1) { + const res = spawnSync("pnpm", ["exec", "boneyard-js", "build", ...urls, ...flags], { + cwd: root, + stdio: "inherit" + }); + if (res.status !== 0) process.exit(res.status ?? 1); + } else { + // Partition URLs across workers + const buckets = Array.from({ length: concurrency }, () => []); + urls.forEach((url, i) => { + buckets[i % concurrency].push(url); + }); + + console.log( + `boneyard: parallelizing ${urls.length} skeletons across ${concurrency} workers...\n` + ); + + const startTime = Date.now(); + await Promise.all( + buckets.map((bucket, i) => { + return new Promise((resolveWorker, rejectWorker) => { + const child = spawn("pnpm", ["exec", "boneyard-js", "build", ...bucket, ...flags], { + cwd: root, + stdio: ["ignore", "pipe", "pipe"] + }); + + child.stdout.on("data", (d) => { + // Stream lines prefixed by worker id for clarity + const text = d.toString(); + for (const line of text.split("\n")) { + if (line.trim()) console.log(`[w${i + 1}] ${line}`); + } + }); + + child.stderr.on("data", (d) => { + const text = d.toString(); + for (const line of text.split("\n")) { + if (line.trim()) console.error(`[w${i + 1}] ${line}`); + } + }); + + child.on("close", (code) => { + if (code === 0) resolveWorker(); + else rejectWorker(new Error(`Worker ${i + 1} exited with code ${code}`)); + }); + }); + }) + ); + + const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); + console.log(`\n💀 all workers completed in ${elapsed}s!`); + } + + // Clean up unused registry and format + spawnSync("rm", ["-f", "src/lib/bones/registry.ts"], { cwd: root }); + spawnSync("pnpm", ["exec", "prettier", "--write", "src/lib/bones"], { + cwd: root, + stdio: "inherit" + }); +} + +run().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/web/scripts/deploy-cloudflare-preview.sh b/web/scripts/deploy-cloudflare-preview.sh new file mode 100755 index 0000000000000000000000000000000000000000..e637b9ba4bec88a3a0272994bea9013bde96d1cc --- /dev/null +++ b/web/scripts/deploy-cloudflare-preview.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +name=$(sh scripts/preview-worker-name.sh) +worker="svfe-preview-$name" +host="$name.preview.tangled.org" +config="wrangler.preview.jsonc" + +jq --arg worker "$worker" --arg host "$host" ' + .name = $worker + | .workers_dev = false + | del(.routes) + | .routes = [{ pattern: $host, custom_domain: true }] +' wrangler.dev.jsonc >"$config" + +jq -e --arg worker "$worker" --arg host "$host" ' + .name == $worker + and .workers_dev == false + and .routes == [{ pattern: $host, custom_domain: true }] +' "$config" >/dev/null || { + echo "deploy-cloudflare-preview: refusing to deploy, $config did not come out clean" >&2 + cat "$config" >&2 + exit 1 +} + +export SVELTE_ADAPTER=cloudflare +export WRANGLER_CONFIG="$config" +export VITE_OAUTH_CLIENT_ID="https://$host/oauth-client-metadata.json" +export VITE_OAUTH_REDIRECT_URI="https://$host/oauth/callback" + +# not `pnpm run build`, which needs bun +pnpm exec vite build +pnpm exec wrangler deploy --config "$config" + +echo "preview deployed: https://$host" diff --git a/web/scripts/preview-worker-name.sh b/web/scripts/preview-worker-name.sh new file mode 100755 index 0000000000000000000000000000000000000000..533367de5295bcc31b5991f2fe512145036fda6d --- /dev/null +++ b/web/scripts/preview-worker-name.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu + +branch="${1:-${TANGLED_PR_SOURCE_BRANCH:-}}" +if [ -z "$branch" ]; then + echo "preview-worker-name: no branch given and TANGLED_PR_SOURCE_BRANCH is unset" >&2 + exit 1 +fi + +name=$(printf '%s' "$branch" | + tr '[:upper:]' '[:lower:]' | + tr '/_' '--' | + tr -cd 'a-z0-9-' | + cut -c1-40 | + sed 's/^-*//; s/-*$//') + +if [ -z "$name" ]; then + echo "preview-worker-name: branch '$branch' sanitizes to an empty name" >&2 + exit 1 +fi + +printf '%s\n' "$name" diff --git a/web/scripts/staleAwait.test.ts b/web/scripts/staleAwait.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..d30ab90f2c405495251cd49e505dff4e834a9912 --- /dev/null +++ b/web/scripts/staleAwait.test.ts @@ -0,0 +1,44 @@ +import { Linter } from "eslint"; +import svelte from "eslint-plugin-svelte"; +import { describe, expect, it } from "vitest"; +import { staleAwait } from "./staleAwait.ts"; + +const linter = new Linter(); +const config = [ + ...svelte.configs.base, + { files: ["**/*.svelte"], rules: { "no-restricted-syntax": ["error" as const, ...staleAwait] } } +]; + +const flagged = (markup: string) => + linter + .verify(`\n${markup}`, config, "+page.svelte") + .map((message) => message.line - 1); + +describe("stale await", () => { + it("flags the issue list before 32879b21f3", () => + expect( + flagged(`{#await data.issues} + +{:then issues} + +{/await}`) + ).toEqual([3])); + + it("flags a then branch that reads page", () => + expect(flagged("{#await data.x then x}{page.url.search}{/await}")).toEqual([1])); + it("flags a pending or catch branch that reads data", () => { + expect(flagged("{#await data.x}{data.q}{:then x}ok{/await}")).toEqual([1]); + expect(flagged("{#await data.x}wait{:then x}ok{:catch e}{data.q}{/await}")).toEqual([1]); + }); + it("flags one block once, however often it reads", () => + expect(flagged("{#await data.x then x}{data.a}{data.b}{/await}")).toHaveLength(1)); + + it("leaves a branch that only reads the settled value", () => + expect(flagged("{#await data.x}wait{:then x}{x.length}{/await}")).toEqual([])); + it("leaves the expression alone, it runs with the navigation", () => + expect(flagged("{#await data.x then x}{x.length}{/await}")).toEqual([])); + it("leaves a property or key that happens to be called data", () => { + expect(flagged("{#await load() then x}{x.data}{/await}")).toEqual([]); + expect(flagged("{#await load() then x}{JSON.stringify({ data: x })}{/await}")).toEqual([]); + }); +}); diff --git a/web/scripts/staleAwait.ts b/web/scripts/staleAwait.ts new file mode 100644 index 0000000000000000000000000000000000000000..25c7d62fcaa7457a047ecfb09c687950bed63660 --- /dev/null +++ b/web/scripts/staleAwait.ts @@ -0,0 +1,9 @@ +// a settled promise runs {:then} in its own batch, which sees the old data and page while a navigation lands +const readsRouteState = + "Identifier[name=/^(data|page)$/]:not(MemberExpression > Identifier.property, Property > Identifier.key)"; + +export const staleAwait = ["Pending", "Then", "Catch"].map((branch) => ({ + selector: `SvelteAwait${branch}Block:has(${readsRouteState})`, + message: + "{#await} branches can run before a navigation lands, seeing the old data and page. await in a instead: {@const value = await ...}." +})); diff --git a/web/scripts/testids-check.ts b/web/scripts/testids-check.ts new file mode 100644 index 0000000000000000000000000000000000000000..4c2ecfc6c068c2b76ac2062f358a3a5eb7923df9 --- /dev/null +++ b/web/scripts/testids-check.ts @@ -0,0 +1,46 @@ +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { componentName, transform } from "./testids.ts"; + +const web = path.resolve(import.meta.dirname, ".."); +const e2e = path.resolve(web, "../e2e"); + +const walk = (dir: string, keep: (file: string) => boolean): string[] => + readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const file = path.join(dir, entry.name); + if (entry.isDirectory()) return entry.name === "node_modules" ? [] : walk(file, keep); + return keep(file) ? [file] : []; + }); + +const ids = new Set(); +const components = new Set(); +for (const file of walk( + path.join(web, "src"), + (f) => f.endsWith(".svelte") && !f.endsWith(".stories.svelte") +)) { + const name = componentName(file); + const { seen, inserts } = transform(readFileSync(file, "utf8"), { component: name }); + if (inserts.some((insert) => insert.text.includes("data-component="))) components.add(name); + for (const item of seen) if (item.id) ids.add(item.id); +} + +const unknown: string[] = []; +const uses = + /(testid|component)\(\s*"([^"]+)"|data-(testid|component)=\\?"([A-Za-z0-9_:/.[\]()-]+)/g; +for (const file of walk(e2e, (f) => f.endsWith(".ts"))) { + readFileSync(file, "utf8") + .split("\n") + .forEach((line, index) => { + for (const match of line.matchAll(uses)) { + const kind = match[1] ?? match[3]; + const name = match[2] ?? match[4]; + if (kind === "component" ? !components.has(name) : !ids.has(name)) + unknown.push( + `${kind} ${name} ${path.relative(path.dirname(e2e), file)}:${index + 1}` + ); + } + }); +} + +if (unknown.length) console.error(`not in the web:\n ${unknown.join("\n ")}`); +process.exit(unknown.length ? 1 : 0); diff --git a/web/scripts/testids.test.ts b/web/scripts/testids.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..163b899327517019bf132d8bcd595d361116336f --- /dev/null +++ b/web/scripts/testids.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from "vitest"; +import { apply, transform } from "./testids.ts"; + +const run = (source: string, options?: Parameters[1]) => { + const { inserts, seen } = transform(source, options); + return { out: apply(source, inserts), seen }; +}; + +const idOf = (markup: string) => run(markup).seen[0]?.id ?? null; + +describe("handlers", () => { + it("names a plain handler", () => + expect(idOf("")).toBe("deleteIssue")); + it("names the call inside an arrow", () => { + expect(idOf("")).toBe("archive"); + expect(idOf("")).toBe("copy"); + expect(idOf("")).toBe("upload"); + }); + it("names an action() call after the action, not .run", () => + expect(idOf("")).toBe("save")); + it("reads assignments as what they do", () => { + expect(idOf("")).toBe("close"); + expect(idOf("")).toBe("openDrawer"); + expect(idOf("")).toBe("toggleMenu"); + }); + it("drops the on prefix a prop carries", () => { + expect(idOf("")).toBe("cancel"); + expect(idOf("")).toBe("confirm"); + }); + it("leaves a forwarded onclick for the call site", () => + expect(idOf("")).toBeNull()); + it("gives up on a handler with no single action", () => + expect(idOf("")).toBeNull()); +}); + +describe("links", () => { + it("names a literal href by its last word", () => { + expect(idOf('x')).toBe("keys"); + expect(idOf('x')).toBe("signup"); + }); + it("skips the parts only known at runtime", () => { + expect(idOf("x")).toBe("issues"); + expect(idOf("x")).toBeNull(); + }); + it("names a variable href after the variable", () => + expect(idOf("x")).toBe("newRepoHref")); + it("ignores external links and a forwarded href", () => { + expect(idOf('x')).toBeNull(); + expect(idOf("x")).toBeNull(); + }); +}); + +describe("fields", () => { + it("names an input after what it binds", () => + expect(idOf("")).toBe("formTitle")); + it("names a submit button submit", () => + expect(idOf('')).toBe("submit")); + it("prefers a handler over a name", () => + expect(idOf('')).toBe("go")); +}); + +describe("injection", () => { + it("puts the id on a component call site", () => + expect(run("").out).toBe( + '' + )); + it("leaves a hand-written id alone", () => { + const { out, seen } = run(''); + expect(out).toBe(''); + expect(seen.map((item) => [item.id, item.source])).toEqual([["mine", "hand"]]); + }); + it("does not touch elements that are not interactive", () => + expect(run("

    hi

    ").out).toBe("

    hi

    ")); + it("does not touch a component it has no rule for", () => + expect(run("y").out).toBe( + "y" + )); + it("marks the root elements with the component, through blocks too", () => { + const { out } = run("{#if a}
    x
    {/if}", { + component: "Panel" + }); + expect(out).toBe( + '{#if a}
    x
    {/if}' + ); + }); + it("does not mark elements nested in another", () => + expect(run("
    x
    ", { component: "Panel" }).out).toBe( + '
    x
    ' + )); + it("copes with typescript in the script", () => + expect( + run( + '' + ).out + ).toContain(" + + Attestations were added after you signed in. + + {:else} + + {/if} + + +{/if} diff --git a/web/src/lib/components/auth/AccountSelector.stories.svelte b/web/src/lib/components/auth/AccountSelector.stories.svelte new file mode 100644 index 0000000000000000000000000000000000000000..098f683d4a9dd822e8df9ed319aa2fe3bf76b251 --- /dev/null +++ b/web/src/lib/components/auth/AccountSelector.stories.svelte @@ -0,0 +1,48 @@ + + + + + + + + + + + + + { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: /comment as alice/i })); + await userEvent.click(await canvas.findByRole("menuitem", { name: /bob/i })); + await waitFor(() => + expect(canvas.getByRole("button", { name: /comment as bob/i })).toBeVisible() + ); + }} +> + + diff --git a/web/src/lib/components/auth/AccountSelector.svelte b/web/src/lib/components/auth/AccountSelector.svelte new file mode 100644 index 0000000000000000000000000000000000000000..1384b68afeb0fdd65defd7333cba8e8ed53369af --- /dev/null +++ b/web/src/lib/components/auth/AccountSelector.svelte @@ -0,0 +1,90 @@ + + + + +{#if selected && others.length > 0} + + {#snippet trigger()} + + {#if prefix !== undefined} + {prefix}{selected.handle} + {/if} + +{/if} diff --git a/web/src/lib/components/comment/Comment.stories.svelte b/web/src/lib/components/comment/Comment.stories.svelte new file mode 100644 index 0000000000000000000000000000000000000000..13bb7cd07b84706b51928750e22c350e4316041e --- /dev/null +++ b/web/src/lib/components/comment/Comment.stories.svelte @@ -0,0 +1,38 @@ + + + + diff --git a/web/src/lib/components/comment/Comment.svelte b/web/src/lib/components/comment/Comment.svelte new file mode 100644 index 0000000000000000000000000000000000000000..865b3f52b04c80908a4470d14575a09ea92d90f4 --- /dev/null +++ b/web/src/lib/components/comment/Comment.svelte @@ -0,0 +1,191 @@ + + + + +{#if deleted && unloaded} +
    + In reply to an earlier comment. Load older comments to see it. +
    +{:else if deleted} +
    + This comment was deleted. +
    +{:else if gate.masked} +
    + + + +
    +{:else} +
    +
    + + + +
    +
    + + + + {#if anchorId} + + {:else} + + {/if} + {#if actions} + +
    {@render actions()}
    + {/if} +
    +
    + {#if editor} +
    {@render editor()}
    + {:else} + + {/if} + {#if !editor} + + {/if} +
    +
    +{/if} diff --git a/web/src/lib/components/comment/CommentArrivalFixture.svelte b/web/src/lib/components/comment/CommentArrivalFixture.svelte new file mode 100644 index 0000000000000000000000000000000000000000..eb97eb6c573b049d8e25d98f2740c2e7a219b3c5 --- /dev/null +++ b/web/src/lib/components/comment/CommentArrivalFixture.svelte @@ -0,0 +1,46 @@ + + +
    +
    + + {#each threads as thread, i (thread.self.uri)} + + {/each} +
    +
    + +
    +
    +
    diff --git a/web/src/lib/components/comment/CommentBox.stories.svelte b/web/src/lib/components/comment/CommentBox.stories.svelte new file mode 100644 index 0000000000000000000000000000000000000000..929a9857dc32852f35f8259b5fbbcb0400eaa3c5 --- /dev/null +++ b/web/src/lib/components/comment/CommentBox.stories.svelte @@ -0,0 +1,42 @@ + + + + + + { + const canvas = within(canvasElement); + await userEvent.type( + canvas.getByPlaceholderText(/add to the discussion/i), + "This will fail{Control>}{Enter}{/Control}" + ); + await waitFor(() => + expect(canvas.getByRole("alert")).toHaveTextContent(/network request failed/i) + ); + }} +/> diff --git a/web/src/lib/components/comment/CommentBox.svelte b/web/src/lib/components/comment/CommentBox.svelte new file mode 100644 index 0000000000000000000000000000000000000000..097bb37c97a749554aa82dc5972eac3d68c4ffcc --- /dev/null +++ b/web/src/lib/components/comment/CommentBox.svelte @@ -0,0 +1,66 @@ + + +{#snippet editor()} + +{/snippet} + +{#if isThread} +
    + {@render editor()} +
    +{:else} + {@render editor()} +{/if} diff --git a/web/src/lib/components/comment/CommentCard.stories.svelte b/web/src/lib/components/comment/CommentCard.stories.svelte new file mode 100644 index 0000000000000000000000000000000000000000..3228c98b57a4c4723eb6d74e2cf050fb05494f7c --- /dev/null +++ b/web/src/lib/components/comment/CommentCard.stories.svelte @@ -0,0 +1,299 @@ + + + + + + + + + + { + const canvas = within(canvasElement); + const textarea = canvas.getByPlaceholderText("Leave a reply..."); + expect(canvas.queryByRole("button", { name: "Reply" })).toBeNull(); + expect(canvas.queryByRole("switch")).toBeNull(); + await userEvent.type(textarea, "first line{Shift>}{Enter}{/Shift}second line"); + await expect(canvas.getByRole("button", { name: "Reply" })).toBeInTheDocument(); + expect(canvas.queryByRole("button", { name: "Cancel" })).toBeNull(); + await userEvent.clear(textarea); + await waitFor(() => expect(canvas.queryByRole("button", { name: "Reply" })).toBeNull()); + }} +/> + + + { + const canvas = within(canvasElement); + await userEvent.click( + canvas.getByRole("button", { + name: /more actions for alice\.tngl\.sh's comment/i + }) + ); + await userEvent.click(await canvas.findByRole("menuitem", { name: "Edit comment" })); + const textarea = await waitFor(() => canvas.getByPlaceholderText(/edit your comment/i)); + await expect(textarea).toHaveValue(sampleComments.mine.body); + await expect(canvas.getByRole("button", { name: "Save" })).toBeInTheDocument(); + await expect(canvas.getByRole("button", { name: "Cancel" })).toBeInTheDocument(); + }} +/> + + + { + const canvas = within(canvasElement); + const card = canvas.getByTestId("commentCard"); + await waitFor(() => + expect(canvas.getByRole("img", { name: "screenshot" }).offsetHeight).toBeGreaterThan( + 100 + ) + ); + const resting = card.offsetHeight; + await userEvent.click( + canvas.getByRole("button", { + name: /more actions for alice\.tngl\.sh's comment/i + }) + ); + await userEvent.click(await canvas.findByRole("menuitem", { name: "Edit comment" })); + await waitFor(() => canvas.getByPlaceholderText(/edit your comment/i)); + await expect(card.offsetHeight).toBe(resting); + }} +/> + + { + const canvas = within(canvasElement); + await userEvent.click( + canvas.getByRole("button", { name: /more actions for frank\.tngl\.sh's reply/i }) + ); + // the actions row fades in on real hover, which a synthetic pointer cannot do, so + // assert the menu is in the tree rather than painted + await expect( + await canvas.findByRole("menuitem", { name: /report reply/i }) + ).toBeInTheDocument(); + await expect(canvas.getByRole("menuitem", { name: /report user/i })).toBeInTheDocument(); + expect(canvas.queryByRole("button", { name: "Edit comment" })).toBeNull(); + + await userEvent.click( + canvas.getByRole("button", { name: /more actions for erin\.tngl\.sh's comment/i }) + ); + await expect( + await canvas.findByRole("menuitem", { name: /report comment/i }) + ).toBeInTheDocument(); + }} +/> + + { + const canvas = within(canvasElement); + await userEvent.click( + canvas.getByRole("button", { + name: /more actions for alice\.tngl\.sh's comment/i + }) + ); + await userEvent.click(await canvas.findByRole("menuitem", { name: "Edit comment" })); + await waitFor(() => canvas.getByPlaceholderText(/edit your comment/i)); + await expect(canvas.getByRole("button", { name: "Save" })).toBeInTheDocument(); + expect(canvas.queryByRole("button", { name: /comment as/i })).toBeNull(); + }} +/> + + { + const canvas = within(canvasElement); + await expect(canvas.getByText("This account has been flagged.")).toBeInTheDocument(); + expect(canvas.queryByText(/on the cusp/)).toBeNull(); + expect(canvas.queryByPlaceholderText("Leave a reply...")).toBeNull(); + }} +/> + + + + { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Show anyway" })); + await expect(canvas.getByText(/on the cusp/)).toBeInTheDocument(); + await expect(canvas.getByPlaceholderText("Leave a reply...")).toBeInTheDocument(); + }} +/> + + { + const canvas = within(canvasElement); + await expect( + canvas.getByText("This account has been hidden by moderators.") + ).toBeInTheDocument(); + expect(canvas.queryByRole("button", { name: "Show anyway" })).toBeNull(); + expect(canvas.queryByText(/to leave a reply/)).toBeNull(); + }} +/> + + + + { + const canvas = within(canvasElement); + expect(canvas.queryByText("This account has been hidden by moderators.")).toBeNull(); + await expect(canvas.getByPlaceholderText("Leave a reply...")).toBeInTheDocument(); + }} +/> + + { + const canvas = within(canvasElement); + await expect(canvas.getByText("This comment has been flagged.")).toBeInTheDocument(); + }} +/> + + + + + + + + + + + diff --git a/web/src/lib/components/comment/CommentCard.svelte b/web/src/lib/components/comment/CommentCard.svelte new file mode 100644 index 0000000000000000000000000000000000000000..31cb66f8a0f447ee701e1722ad304399e27cd7ae --- /dev/null +++ b/web/src/lib/components/comment/CommentCard.svelte @@ -0,0 +1,297 @@ + + +{#snippet commentActions(comment: CommentView, noun: "comment" | "reply")} +
    + {#if currentUser} + held.set(comment.uri, next)} + dropdownOrigin="tr" + dropdownAlign="right" + triggerClass="hit-area-x-0.5 overflow-visible" + /> + {/if} + + {#if auth?.hasAccount(comment.authorDid)} + (editingUri = comment.uri)}> + Edit {noun} + + handleDelete(comment, noun)} + > + Delete {noun} + + + {/if} + navigator.clipboard.writeText(comment.uri)}> + Copy AT URI + + {#if reporting.canReport(comment.authorDid)} + + {/if} + + +
    +{/snippet} + +{#snippet commentEditor( + comment: CommentView, + replyUri: string | undefined, + replyCid: string | undefined +)} + { + editingUri = null; + onedited?.(edited); + }} + oncancel={() => (editingUri = null)} + /> +{/snippet} + +{#snippet selfEditor()} + {@render commentEditor(thread.self, undefined, undefined)} +{/snippet} + +{#snippet selfActions()} + {@render commentActions(thread.self, "comment")} +{/snippet} + + +
    + held.set(thread.self.uri, next)} + editor={editingUri === thread.self.uri ? selfEditor : undefined} + actions={thread.self.deleted ? undefined : selfActions} + /> +
    + + {#if thread.replies.length} +
    + {#each thread.replies as reply, i (reply.uri)} + {#snippet replyEditor()} + {@render commentEditor(reply, thread.self.uri, thread.self.cid)} + {/snippet} + {#snippet replyActions()} + {@render commentActions(reply, "reply")} + {/snippet} +
    + +
    + held.set(reply.uri, next)} + editor={editingUri === reply.uri ? replyEditor : undefined} + actions={replyActions} + /> +
    + {/each} +
    + {/if} + + {#if deleteSelected.error} +
    + +
    + {/if} + + {#if (thread.self.deleted && !thread.self.cid) || selfGate.masked} + + {:else if currentUser} +
    + +
    + {/if} +     + + diff --git a/web/src/lib/components/comment/CommentEditor.stories.svelte b/web/src/lib/components/comment/CommentEditor.stories.svelte new file mode 100644 index 0000000000000000000000000000000000000000..0a185fec891ff8d095468a0ab067e8f1eeecde38 --- /dev/null +++ b/web/src/lib/components/comment/CommentEditor.stories.svelte @@ -0,0 +1,109 @@ + + + + + + + + + + { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Save" })); + await waitFor(() => + expect(canvas.getByRole("alert")).toHaveTextContent(/network request failed/i) + ); + }} +/> + + { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: /comment as alice/i })); + await userEvent.click(await canvas.findByRole("menuitem", { name: /bob/i })); + await waitFor(() => + expect(canvas.getByRole("button", { name: /comment as bob/i })).toBeVisible() + ); + }} +/> + + { + const canvas = within(canvasElement); + expect(canvas.queryByRole("button", { name: /comment as/i })).toBeNull(); + await waitFor(() => expect(canvas.getByRole("button", { name: "Save" })).toBeVisible()); + }} +/> + + { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: /comment as alice/i })); + await userEvent.click(await canvas.findByRole("menuitem", { name: /bob/i })); + await userEvent.type(canvas.getByPlaceholderText(/add to the discussion/i), "as bob"); + await userEvent.click(canvas.getByRole("button", { name: "Comment" })); + await waitFor(() => + expect(canvas.getByRole("alert")).toHaveTextContent(/session expired/i) + ); + await waitFor(() => + expect(canvas.getByRole("button", { name: /comment as alice/i })).toBeVisible() + ); + await userEvent.click(canvas.getByRole("button", { name: /comment as alice/i })); + await waitFor(() => + expect(canvas.getByRole("menuitem", { name: /charlie/i })).toBeVisible() + ); + expect(canvas.queryByRole("menuitem", { name: /bob/i })).toBeNull(); + }} +/> diff --git a/web/src/lib/components/comment/CommentEditor.svelte b/web/src/lib/components/comment/CommentEditor.svelte new file mode 100644 index 0000000000000000000000000000000000000000..7644ce51b11bc244d49409f398f38707a3978330 --- /dev/null +++ b/web/src/lib/components/comment/CommentEditor.svelte @@ -0,0 +1,239 @@ + + +{#snippet submit()} + +{/snippet} + + + + +
    + + + + {#if !compact} +
    + {#if !rkey} + acting.did, (did) => (picked = did)} + label="Comment as" + disabled={publish.loading} + group="comment-editor" + /> + {/if} +
    + {#if oncancel} + + {/if} + {@render submit()} +
    +
    + {/if} +
    + diff --git a/web/src/lib/components/comment/CommentList.stories.svelte b/web/src/lib/components/comment/CommentList.stories.svelte new file mode 100644 index 0000000000000000000000000000000000000000..0b18595cc6ee4119a83e2e00718f42c422e85cae --- /dev/null +++ b/web/src/lib/components/comment/CommentList.stories.svelte @@ -0,0 +1,112 @@ + + + + + + + + { + const canvas = within(canvasElement); + // only alice's (middle) reply exposes an edit control + await userEvent.click( + canvas.getByRole("button", { + name: /more actions for alice\.tngl\.sh's reply/i + }) + ); + await userEvent.click(await canvas.findByRole("menuitem", { name: "Edit reply" })); + const textarea = await waitFor(() => canvas.getByPlaceholderText(/edit your comment/i)); + await expect(textarea).toHaveValue(sampleComments.mine.body); + await expect(canvas.getByRole("button", { name: "Save" })).toBeInTheDocument(); + await expect(canvas.getByRole("button", { name: "Cancel" })).toBeInTheDocument(); + }} +/> + + + expectSmoothArrival(canvasElement, { + button: "Add comment", + item: '[data-testid="arrival-list"] > div > div', + edge: "arrival-end" + })} +> + + + + + expectSmoothArrival(canvasElement, { + button: "Reply to thread 1", + item: '[data-testid="replies"] > div', + edge: "arrival-end" + })} +> + + + + + expectSmoothArrival(canvasElement, { + button: "Reply to thread 1", + item: '[data-testid="replies"] > div', + edge: "arrival-end" + })} +> + + diff --git a/web/src/lib/components/comment/CommentList.svelte b/web/src/lib/components/comment/CommentList.svelte new file mode 100644 index 0000000000000000000000000000000000000000..44321c8228d0595a1a50db4cb8d278aa65bcf5e7 --- /dev/null +++ b/web/src/lib/components/comment/CommentList.svelte @@ -0,0 +1,40 @@ + + +{#if threads.length} +
    + {#each threads as thread (thread.self.uri)} +
    + +
    + {/each} +
    +{/if} diff --git a/web/src/lib/components/comment/CommentSection.svelte b/web/src/lib/components/comment/CommentSection.svelte new file mode 100644 index 0000000000000000000000000000000000000000..8bdeb7c91652f12346d37005f31000d7c5e67a1a --- /dev/null +++ b/web/src/lib/components/comment/CommentSection.svelte @@ -0,0 +1,84 @@ + + +
    + (comments = withOlderPage(comments, page))} + /> + + + {#if auth.currentDid} + + {:else} + + {/if} +
    diff --git a/web/src/lib/components/comment/OlderComments.svelte b/web/src/lib/components/comment/OlderComments.svelte new file mode 100644 index 0000000000000000000000000000000000000000..0c27fa12b78c877913264b4495a821c7e8dc4b52 --- /dev/null +++ b/web/src/lib/components/comment/OlderComments.svelte @@ -0,0 +1,46 @@ + + +{#if cursor && bobbinUrl} + +{/if} +{#if load.error} + +{/if} diff --git a/web/src/lib/components/comment/comments.test.ts b/web/src/lib/components/comment/comments.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..ff470a4fbd2bdf7ca99ae60fecc7a857f7779a0b --- /dev/null +++ b/web/src/lib/components/comment/comments.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, it } from "vitest"; +import { + commentModerationOf, + countCommentsByCommit, + filterCommentsByCommit, + groupCommentsByVersion, + threadsOf, + withOlderPage, + withRemoved, + type CommentView +} from "#lib/components/comment/comments.ts"; +import type { CommitSummary } from "#lib/api/repo.ts"; +import type { Did, ResourceUri } from "@atcute/lexicons/syntax"; +import { COMMENT_COLLECTION } from "#lib/api/comment.ts"; +import { ACCOUNT_MODERATION, recordModeration } from "#lib/api/moderationLabels.ts"; +import type { LabelsDescriptor, Sidecar } from "#lib/api/enrich.ts"; + +const labeled = (ref: string, of: LabelsDescriptor, val: string): Sidecar => ({ + [ref]: { + [of.source]: { + [of.type]: { + labels: [{ src: "did:plc:labeler", uri: ref, val, cts: "2026-10-01T00:00:00.000Z" }] + } + } + } +}); + +const comment = (rkey: string, createdAt: string): CommentView => ({ + uri: `at://did:plc:alice/sh.tangled.feed.comment/${rkey}`, + rkey, + authorDid: "did:plc:alice", + authorHandle: "alice.test", + createdAt, + body: rkey +}); + +const versions = [{ createdAt: "2026-01-01T00:00:00Z" }, { createdAt: "2026-01-03T00:00:00Z" }]; + +const layout = (comments: CommentView[]) => + groupCommentsByVersion(comments, versions).map((group) => [ + group.version, + group.comments.map((c) => c.rkey) + ]); + +describe("groupCommentsByVersion", () => { + it("files a comment under the newest version at or before it", () => { + // on the boundary belongs to that version, not the one before + expect( + layout([ + comment("boundary", "2026-01-01T00:00:00Z"), + comment("during-v0", "2026-01-02T00:00:00Z"), + comment("after-v1", "2026-01-04T00:00:00Z") + ]) + ).toEqual([ + [0, ["boundary", "during-v0"]], + [1, ["after-v1"]] + ]); + }); + + it("keeps a version with no comments, so the timeline stays complete", () => { + expect(layout([comment("only", "2026-01-02T00:00:00Z")])).toEqual([ + [0, ["only"]], + [1, []] + ]); + }); + + it("puts anything older than v0 in a headerless leading group", () => { + expect(layout([comment("early", "2025-12-31T00:00:00Z")])).toEqual([ + [undefined, ["early"]], + [0, []], + [1, []] + ]); + }); + + it("orders comments itself rather than trusting the caller", () => { + expect( + layout([ + comment("late", "2026-01-04T00:00:00Z"), + comment("early", "2026-01-02T00:00:00Z") + ]) + ).toEqual([ + [0, ["early"]], + [1, ["late"]] + ]); + }); + + it("has nothing to group without versions", () => { + expect(groupCommentsByVersion([comment("a", "2026-01-02T00:00:00Z")], [])).toEqual([ + { version: undefined, comments: [expect.objectContaining({ rkey: "a" })] } + ]); + }); +}); + +const pinned = (rkey: string, oid: string): CommentView => ({ + ...comment(rkey, "2026-01-02T00:00:00Z"), + embed: { repo: "did:plc:alice", commit: oid } +}); + +describe("countCommentsByCommit", () => { + it("sums the comments pinned to one commit and skips the unpinned ones", () => { + const counts = countCommentsByCommit([ + pinned("a", "cafe1234"), + pinned("b", "cafe1234"), + pinned("c", "beef5678"), + comment("loose", "2026-01-02T00:00:00Z") + ]); + expect([...counts]).toEqual([ + ["cafe1234", 2], + ["beef5678", 1] + ]); + }); + + it("leaves an unpinned commit out rather than reporting 0", () => { + const counts = countCommentsByCommit([pinned("a", "cafe1234")]); + expect(counts.has("beef5678")).toBe(false); + }); +}); + +const commit = (hash: string): CommitSummary => ({ + hash, + shortHash: hash.slice(0, 8), + subject: hash, + body: "", + authorName: "alice", + authorEmail: "alice@test", + when: "2026-01-02T00:00:00Z" +}); + +describe("filterCommentsByCommit", () => { + const commits = [commit("cafe1234"), commit("beef5678")]; + const rkeys = (comments: CommentView[]) => comments.map((c) => c.rkey); + + it("drops an unpinned comment and a pin the filter rejects", () => { + expect( + rkeys( + filterCommentsByCommit( + [ + comment("loose", "2026-01-02T00:00:00Z"), + pinned("in", "cafe1234"), + pinned("out", "beef5678") + ], + commits, + (c) => c.hash === "cafe1234" + ) + ) + ).toEqual(["in"]); + }); + + it("drops a pin that names no known commit", () => { + // an older version's commit, or one a force-push dropped: never in range + expect( + rkeys(filterCommentsByCommit([pinned("gone", "d00d9999")], commits, () => true)) + ).toEqual([]); + }); + + it("filters nothing without a filter", () => { + const comments = [pinned("gone", "d00d9999"), comment("loose", "2026-01-02T00:00:00Z")]; + expect(filterCommentsByCommit(comments, commits)).toBe(comments); + }); +}); + +describe("commentModerationOf", () => { + const item: { uri: ResourceUri; editor: Did } = { + uri: "at://did:plc:repo/org.tangled.feed.comment/3c", + editor: "did:plc:bob" + }; + const commentLabels = recordModeration(COMMENT_COLLECTION); + + it("covers just the comment when only the comment is labeled", () => { + expect(commentModerationOf(labeled(item.uri, commentLabels, "!hide"), item)).toEqual({ + level: "hide", + target: item.uri + }); + }); + + it("reads the editor's account, not the repo the uri names", () => { + const docs = { + ...labeled("did:plc:repo", ACCOUNT_MODERATION, "!hide"), + ...labeled(item.editor, ACCOUNT_MODERATION, "!warn") + }; + expect(commentModerationOf(docs, item)).toEqual({ level: "warn", target: item.editor }); + }); + + it("lets the stronger decision win", () => { + const docs = { + ...labeled(item.uri, commentLabels, "!hide"), + ...labeled(item.editor, ACCOUNT_MODERATION, "!warn") + }; + expect(commentModerationOf(docs, item)).toEqual({ level: "hide", target: item.uri }); + }); + + it("prefers the author's account on a tie", () => { + const docs = { + ...labeled(item.uri, commentLabels, "!warn"), + ...labeled(item.editor, ACCOUNT_MODERATION, "!warn") + }; + expect(commentModerationOf(docs, item)).toEqual({ level: "warn", target: item.editor }); + }); +}); + +describe("withOlderPage", () => { + it("fills in what the newest page only stood in for once older pages load", () => { + const a = comment("a", "2026-01-01T00:00:00Z"); + const gone = "at://did:plc:alice/sh.tangled.feed.comment/gone"; + const newest = { + inputs: [ + { comment: comment("r2", "2026-01-05T00:00:00Z"), replyTo: a.uri }, + { comment: comment("rg", "2026-01-06T00:00:00Z"), replyTo: gone }, + { comment: comment("c", "2026-01-07T00:00:00Z"), replyTo: null } + ], + cursor: "older", + total: 6 + }; + expect(threadsOf(newest)[0].self).toMatchObject({ + uri: a.uri, + deleted: true, + unloaded: true + }); + + const merged = withOlderPage(newest, { + inputs: [ + { comment: a, replyTo: null }, + { comment: comment("r1", "2026-01-02T00:00:00Z"), replyTo: a.uri }, + { comment: comment("b", "2026-01-04T00:00:00Z"), replyTo: null }, + { comment: comment("c", "2026-01-07T00:00:00Z"), replyTo: null } + ], + cursor: null, + total: 6 + }); + expect(merged.inputs).toHaveLength(6); + const threads = threadsOf(merged); + expect( + threads.map((thread) => [thread.self.rkey, thread.replies.map((r) => r.rkey)]) + ).toEqual([ + ["a", ["r1", "r2"]], + ["b", []], + ["gone", ["rg"]], + ["c", []] + ]); + expect(threads[0].self).toEqual(a); + expect(threads[2].self).toMatchObject({ deleted: true }); + expect(threads[2].self.unloaded).toBeUndefined(); + }); +}); + +describe("withRemoved", () => { + it("keeps a removed parent as deleted, so its replies don't wait on an older page", () => { + const parent = comment("p", "2026-01-01T00:00:00Z"); + const page = { + inputs: [ + { comment: parent, replyTo: null }, + { comment: comment("r", "2026-01-02T00:00:00Z"), replyTo: parent.uri }, + { comment: comment("solo", "2026-01-03T00:00:00Z"), replyTo: null } + ], + cursor: "older", + total: 3 + }; + const removed = withRemoved(withRemoved(page, parent.uri), page.inputs[2].comment.uri); + expect(removed.total).toBe(1); + const threads = threadsOf(removed); + expect(threads.map((thread) => thread.self.rkey)).toEqual(["p"]); + expect(threads[0].self).toMatchObject({ deleted: true, body: "" }); + expect(threads[0].self.unloaded).toBeUndefined(); + }); +}); diff --git a/web/src/lib/components/comment/comments.ts b/web/src/lib/components/comment/comments.ts new file mode 100644 index 0000000000000000000000000000000000000000..9f01b23a679f1ed6eb69c7e4b92c2f70b5137198 --- /dev/null +++ b/web/src/lib/components/comment/comments.ts @@ -0,0 +1,274 @@ +import type { BobbinContext } from "#lib/api/client.ts"; +import type { CommentListPage } from "#lib/api/comment.ts"; +import type * as OrgTangledEmbedCommit from "#lib/api/lexicons/types/org/tangled/embed/commit.ts"; +import { COMMENT_AUTHOR_DOCS, REACTIONS } from "#lib/api/descriptors.ts"; +import { enrich, handleOf, target, type Sidecar } from "#lib/api/enrich.ts"; +import { INVALID_HANDLE } from "#lib/api/identity.ts"; +import { COMMENT_COLLECTION } from "#lib/api/comment.ts"; +import { + moderationTargets, + subjectModeration, + type Moderation +} from "#lib/api/moderationLabels.ts"; +import { didFromUri, rkeyFromUri } from "#lib/api/uri.ts"; +import type { CommitSummary } from "#lib/api/repo.ts"; +import { reactionGroupsOf, type ReactionGroup } from "#lib/components/reaction/reactions.ts"; + +export interface CommentView { + uri: string; + cid?: string; + rkey: string; + authorDid: string; + authorHandle: string; + moderation?: Moderation; + createdAt: string; + // the appview's comment header shows "Edited
    +
    + + A set of labels applied to an issue. + +
    A set of labels applied to an issue.
    +
    + +
    + + Create custom key-value type labels. + +
    Create custom key-value type labels.
    +
    +